diff --git a/.gitignore b/.gitignore index 02a2a4b1a78..2d5e0e2c4a9 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,6 @@ zz_generated_*_test.go /bazel-* *.pyc + +# generated by verify-godeps.sh +vendordiff.patch diff --git a/CHANGELOG-1.10.md b/CHANGELOG-1.10.md index eb7bb1823a0..07ceddde31c 100644 --- a/CHANGELOG-1.10.md +++ b/CHANGELOG-1.10.md @@ -1,75 +1,82 @@ -- [v1.10.9](#v1109) - - [Downloads for v1.10.9](#downloads-for-v1109) +- [v1.10.10](#v11010) + - [Downloads for v1.10.10](#downloads-for-v11010) - [Client Binaries](#client-binaries) - [Server Binaries](#server-binaries) - [Node Binaries](#node-binaries) - - [Changelog since v1.10.8](#changelog-since-v1108) + - [Changelog since v1.10.9](#changelog-since-v1109) - [Other notable changes](#other-notable-changes) -- [v1.10.8](#v1108) - - [Downloads for v1.10.8](#downloads-for-v1108) +- [v1.10.9](#v1109) + - [Downloads for v1.10.9](#downloads-for-v1109) - [Client Binaries](#client-binaries-1) - [Server Binaries](#server-binaries-1) - [Node Binaries](#node-binaries-1) - - [Changelog since v1.10.7](#changelog-since-v1107) + - [Changelog since v1.10.8](#changelog-since-v1108) - [Other notable changes](#other-notable-changes-1) -- [v1.10.7](#v1107) - - [Downloads for v1.10.7](#downloads-for-v1107) +- [v1.10.8](#v1108) + - [Downloads for v1.10.8](#downloads-for-v1108) - [Client Binaries](#client-binaries-2) - [Server Binaries](#server-binaries-2) - [Node Binaries](#node-binaries-2) - - [Changelog since v1.10.6](#changelog-since-v1106) - - [Action Required](#action-required) + - [Changelog since v1.10.7](#changelog-since-v1107) - [Other notable changes](#other-notable-changes-2) -- [v1.10.6](#v1106) - - [Downloads for v1.10.6](#downloads-for-v1106) +- [v1.10.7](#v1107) + - [Downloads for v1.10.7](#downloads-for-v1107) - [Client Binaries](#client-binaries-3) - [Server Binaries](#server-binaries-3) - [Node Binaries](#node-binaries-3) - - [Changelog since v1.10.5](#changelog-since-v1105) - - [Action Required](#action-required-1) + - [Changelog since v1.10.6](#changelog-since-v1106) + - [Action Required](#action-required) - [Other notable changes](#other-notable-changes-3) -- [v1.10.5](#v1105) - - [Downloads for v1.10.5](#downloads-for-v1105) +- [v1.10.6](#v1106) + - [Downloads for v1.10.6](#downloads-for-v1106) - [Client Binaries](#client-binaries-4) - [Server Binaries](#server-binaries-4) - [Node Binaries](#node-binaries-4) - - [Changelog since v1.10.4](#changelog-since-v1104) - - [Action Required](#action-required-2) + - [Changelog since v1.10.5](#changelog-since-v1105) + - [Action Required](#action-required-1) - [Other notable changes](#other-notable-changes-4) -- [v1.10.4](#v1104) - - [Downloads for v1.10.4](#downloads-for-v1104) +- [v1.10.5](#v1105) + - [Downloads for v1.10.5](#downloads-for-v1105) - [Client Binaries](#client-binaries-5) - [Server Binaries](#server-binaries-5) - [Node Binaries](#node-binaries-5) - - [Changelog since v1.10.3](#changelog-since-v1103) + - [Changelog since v1.10.4](#changelog-since-v1104) + - [Action Required](#action-required-2) - [Other notable changes](#other-notable-changes-5) -- [v1.10.3](#v1103) - - [Downloads for v1.10.3](#downloads-for-v1103) +- [v1.10.4](#v1104) + - [Downloads for v1.10.4](#downloads-for-v1104) - [Client Binaries](#client-binaries-6) - [Server Binaries](#server-binaries-6) - [Node Binaries](#node-binaries-6) - - [Changelog since v1.10.2](#changelog-since-v1102) + - [Changelog since v1.10.3](#changelog-since-v1103) - [Other notable changes](#other-notable-changes-6) -- [v1.10.2](#v1102) - - [Downloads for v1.10.2](#downloads-for-v1102) +- [v1.10.3](#v1103) + - [Downloads for v1.10.3](#downloads-for-v1103) - [Client Binaries](#client-binaries-7) - [Server Binaries](#server-binaries-7) - [Node Binaries](#node-binaries-7) - - [Changelog since v1.10.1](#changelog-since-v1101) + - [Changelog since v1.10.2](#changelog-since-v1102) - [Other notable changes](#other-notable-changes-7) -- [v1.10.1](#v1101) - - [Downloads for v1.10.1](#downloads-for-v1101) +- [v1.10.2](#v1102) + - [Downloads for v1.10.2](#downloads-for-v1102) - [Client Binaries](#client-binaries-8) - [Server Binaries](#server-binaries-8) - [Node Binaries](#node-binaries-8) - - [Changelog since v1.10.0](#changelog-since-v1100) + - [Changelog since v1.10.1](#changelog-since-v1101) - [Other notable changes](#other-notable-changes-8) -- [v1.10.0](#v1100) - - [Downloads for v1.10.0](#downloads-for-v1100) +- [v1.10.1](#v1101) + - [Downloads for v1.10.1](#downloads-for-v1101) - [Client Binaries](#client-binaries-9) - [Server Binaries](#server-binaries-9) - [Node Binaries](#node-binaries-9) + - [Changelog since v1.10.0](#changelog-since-v1100) + - [Other notable changes](#other-notable-changes-9) +- [v1.10.0](#v1100) + - [Downloads for v1.10.0](#downloads-for-v1100) + - [Client Binaries](#client-binaries-10) + - [Server Binaries](#server-binaries-10) + - [Node Binaries](#node-binaries-10) - [Major Themes](#major-themes) - [Node](#node) - [Storage](#storage) @@ -83,7 +90,7 @@ - [Before Upgrading](#before-upgrading) - [Known Issues](#known-issues) - [Deprecations](#deprecations) - - [Other Notable Changes](#other-notable-changes-9) + - [Other Notable Changes](#other-notable-changes-10) - [Apps](#apps) - [AWS](#aws) - [Auth](#auth-1) @@ -106,69 +113,137 @@ - [External Dependencies](#external-dependencies) - [v1.10.0-rc.1](#v1100-rc1) - [Downloads for v1.10.0-rc.1](#downloads-for-v1100-rc1) - - [Client Binaries](#client-binaries-10) - - [Server Binaries](#server-binaries-10) - - [Node Binaries](#node-binaries-10) - - [Changelog since v1.10.0-beta.4](#changelog-since-v1100-beta4) - - [Other notable changes](#other-notable-changes-10) -- [v1.10.0-beta.4](#v1100-beta4) - - [Downloads for v1.10.0-beta.4](#downloads-for-v1100-beta4) - [Client Binaries](#client-binaries-11) - [Server Binaries](#server-binaries-11) - [Node Binaries](#node-binaries-11) - - [Changelog since v1.10.0-beta.3](#changelog-since-v1100-beta3) + - [Changelog since v1.10.0-beta.4](#changelog-since-v1100-beta4) - [Other notable changes](#other-notable-changes-11) -- [v1.10.0-beta.3](#v1100-beta3) - - [Downloads for v1.10.0-beta.3](#downloads-for-v1100-beta3) +- [v1.10.0-beta.4](#v1100-beta4) + - [Downloads for v1.10.0-beta.4](#downloads-for-v1100-beta4) - [Client Binaries](#client-binaries-12) - [Server Binaries](#server-binaries-12) - [Node Binaries](#node-binaries-12) - - [Changelog since v1.10.0-beta.2](#changelog-since-v1100-beta2) + - [Changelog since v1.10.0-beta.3](#changelog-since-v1100-beta3) - [Other notable changes](#other-notable-changes-12) -- [v1.10.0-beta.2](#v1100-beta2) - - [Downloads for v1.10.0-beta.2](#downloads-for-v1100-beta2) +- [v1.10.0-beta.3](#v1100-beta3) + - [Downloads for v1.10.0-beta.3](#downloads-for-v1100-beta3) - [Client Binaries](#client-binaries-13) - [Server Binaries](#server-binaries-13) - [Node Binaries](#node-binaries-13) - - [Changelog since v1.10.0-beta.1](#changelog-since-v1100-beta1) - - [Action Required](#action-required-3) + - [Changelog since v1.10.0-beta.2](#changelog-since-v1100-beta2) - [Other notable changes](#other-notable-changes-13) -- [v1.10.0-beta.1](#v1100-beta1) - - [Downloads for v1.10.0-beta.1](#downloads-for-v1100-beta1) +- [v1.10.0-beta.2](#v1100-beta2) + - [Downloads for v1.10.0-beta.2](#downloads-for-v1100-beta2) - [Client Binaries](#client-binaries-14) - [Server Binaries](#server-binaries-14) - [Node Binaries](#node-binaries-14) - - [Changelog since v1.10.0-alpha.3](#changelog-since-v1100-alpha3) - - [Action Required](#action-required-4) + - [Changelog since v1.10.0-beta.1](#changelog-since-v1100-beta1) + - [Action Required](#action-required-3) - [Other notable changes](#other-notable-changes-14) -- [v1.10.0-alpha.3](#v1100-alpha3) - - [Downloads for v1.10.0-alpha.3](#downloads-for-v1100-alpha3) +- [v1.10.0-beta.1](#v1100-beta1) + - [Downloads for v1.10.0-beta.1](#downloads-for-v1100-beta1) - [Client Binaries](#client-binaries-15) - [Server Binaries](#server-binaries-15) - [Node Binaries](#node-binaries-15) - - [Changelog since v1.10.0-alpha.2](#changelog-since-v1100-alpha2) + - [Changelog since v1.10.0-alpha.3](#changelog-since-v1100-alpha3) + - [Action Required](#action-required-4) - [Other notable changes](#other-notable-changes-15) -- [v1.10.0-alpha.2](#v1100-alpha2) - - [Downloads for v1.10.0-alpha.2](#downloads-for-v1100-alpha2) +- [v1.10.0-alpha.3](#v1100-alpha3) + - [Downloads for v1.10.0-alpha.3](#downloads-for-v1100-alpha3) - [Client Binaries](#client-binaries-16) - [Server Binaries](#server-binaries-16) - [Node Binaries](#node-binaries-16) - - [Changelog since v1.10.0-alpha.1](#changelog-since-v1100-alpha1) - - [Action Required](#action-required-5) + - [Changelog since v1.10.0-alpha.2](#changelog-since-v1100-alpha2) - [Other notable changes](#other-notable-changes-16) -- [v1.10.0-alpha.1](#v1100-alpha1) - - [Downloads for v1.10.0-alpha.1](#downloads-for-v1100-alpha1) +- [v1.10.0-alpha.2](#v1100-alpha2) + - [Downloads for v1.10.0-alpha.2](#downloads-for-v1100-alpha2) - [Client Binaries](#client-binaries-17) - [Server Binaries](#server-binaries-17) - [Node Binaries](#node-binaries-17) + - [Changelog since v1.10.0-alpha.1](#changelog-since-v1100-alpha1) + - [Action Required](#action-required-5) + - [Other notable changes](#other-notable-changes-17) +- [v1.10.0-alpha.1](#v1100-alpha1) + - [Downloads for v1.10.0-alpha.1](#downloads-for-v1100-alpha1) + - [Client Binaries](#client-binaries-18) + - [Server Binaries](#server-binaries-18) + - [Node Binaries](#node-binaries-18) - [Changelog since v1.9.0](#changelog-since-v190) - [Action Required](#action-required-6) - - [Other notable changes](#other-notable-changes-17) + - [Other notable changes](#other-notable-changes-18) +# v1.10.10 + +[Documentation](https://docs.k8s.io) + +## Downloads for v1.10.10 + + +filename | sha512 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes.tar.gz) | `605609b9b20b1c17f2accc251bb4e5dfb878fce351c7efadd8b6684323bdb8f87f31fbb27cf8a92c5d52c719f92f754ffe0fd065bbec43cb03ec91a6680ee327` +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-src.tar.gz) | `0f01069a5198016b616c3e71fd280c5396fdc4c679d2f37fc2db3463f21a9d947a18f5ce69314e71c2d1ad63725807ab1639dad89362b6100ddfdf9cd57d06c5` + +### Client Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-client-darwin-386.tar.gz) | `ec93db4049f700d3f912ff3b3f463abb809938ac78510a48da58d13644c14a881c5d676d014cdf216d0d30ac8a107f0de6f0fdb48f337d35cf2bf5ab32d686e5` +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-client-darwin-amd64.tar.gz) | `1a13d3c60a7c8f1556e8690cb90a418fa20cd2dda35dafaddc9f17330d2142f25b0c22fc629e92f828af4cf761d190647a49f5df474719c828d381243dbf00be` +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-client-linux-386.tar.gz) | `d291f6472f35e3dbd869865a35f16036f80e79ae5c21ba4c4e5dabc028279ea068924a916a95807fe2659b972c3618094d307e5a98246d83f304aa552aaaefb0` +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-client-linux-amd64.tar.gz) | `101acba3a200fc33026c78adaffc7ca0f6e8dc8be860f06724a01129e6cc29286c6bd735c7166a938f10cb19aa1185f5bbbdcc8c97f6b5750c7417cbf4b36484` +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-client-linux-arm.tar.gz) | `12cd6c4ada86dd9febf15e018e1215c6fc3900e51873f7efef4cf10e5770e1ea3bb2657d89a3a14c334149f3726e98ec7d6b56bef7558c8d53ab0a747b21847a` +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-client-linux-arm64.tar.gz) | `aefa008d4a71f78ea1d9152de754c61d3103b7aaceb7fe6cf47caa1e3cf2c1f0dc203f7253df8f167f37fd3209cf885903d11a2f34fbfd5118c2db7d3edf93bc` +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-client-linux-ppc64le.tar.gz) | `bedbed22c4265aae574240a01ec350cf9961a74977648df4102aa5fc3c827903114be97b057dac0b7a19e92d4cadba4322d8a918083deac63816250c520a9651` +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-client-linux-s390x.tar.gz) | `a7f9c70fb3be422f37dbe87ddc1b3237788121463d8bd1fec668f280488ee034009cdbd3011386a9a894a9a1313f34ce851afb495b62f425698e8e2423078216` +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-client-windows-386.tar.gz) | `872ea8da0478fd80231066afcb7a79a1b1007b727cefa67561210f1d279798dec16c3e32e7a9cc515621faf56294741ecbcdd0044e570d9d28f1af377c10e20d` +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-client-windows-amd64.tar.gz) | `ab296181bb60294a08d5b849ea1bc73d800bdc32a14c5a9e31d152224bc51240cb82d1765387997d991cccab30137a798360d8ab787fde171aace69bc8db7f31` + +### Server Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-server-linux-amd64.tar.gz) | `d27e0eecb362f0ff02ae96093749b8b2e44695cb8ed36fe00dc75095812396588a575a994314d78f79d26601ddbef69734944e202991e4b0a44e3ea768c515e3` +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-server-linux-arm.tar.gz) | `b0d91a4d6e0700d098ca7c93c8c99c23e23cec61b3d5229d62b42d0f0769be73948832e49c073616da2edc72054017e8081315ea562eaca0da1db32475594b3d` +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-server-linux-arm64.tar.gz) | `7cbdfb5e19a499848b9df23a8e68fe827fe913a8b8596d88fb565303dee3a49b9850ab9bc07137f2453a6336e374aaee0cdda4fdd43cb8877308165f295ec976` +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-server-linux-ppc64le.tar.gz) | `4e71ab08406ae4e27ddb105901d4f8544e6e61652734539a939ef112672a10a3f89a86cb7eecb0421d1db70123f5d994be2545f8714305ce74c073f83ef8d3fa` +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-server-linux-s390x.tar.gz) | `fb45be7bd8d2bba7d8105ea904377007cc697770fe0d1b81f0780ce56d442f20a26df87871594bc3692b59ca6f0f15bae58a473b1d31ebe3e9fbaadeb651408b` + +### Node Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-node-linux-amd64.tar.gz) | `4f97b6f2282af262bf53995e39c2e8eeeda8492cdad34e3616c1aaa9b8bc8c04a6e0ac9ef79ba6290aa53d2ee3bfb65d6cd69b1c3fd1c31dd851bce4420bcfc0` +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-node-linux-arm.tar.gz) | `27ba1d0c5b53a26d066ac80b59f997e748b022666f7374a471c4822a551aa6e5396634f0c77666004a0c8aa90d7e4bbb3f4badd4847545e32da49c535d32736d` +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-node-linux-arm64.tar.gz) | `b2f361621f24d3c6872f277f790e19be51a4466ca576725554f61ff60539740984d720c75d9266311fd0d2cf18c2eab8aa8454c41ba66deb8685844c263ce53e` +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-node-linux-ppc64le.tar.gz) | `19e13f95b8372221d08eb26c694fa533f24889e052c1a3635df6630c2ca3f768bf34b59e6948b05c058c901794905448d648129070c14ec48a64262af1a059f2` +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-node-linux-s390x.tar.gz) | `1631f0e2f2c9a7674ec73f2c27cd808f94ef48a782042bfaa7c007caac5fa4d5016863b8fca7c3c2dd92c606b0d079e6cb0038d874e9c1c521cdc73a723a346d` +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.10.10/kubernetes-node-windows-amd64.tar.gz) | `7d0d709cf8cbfd076c921ea9cb19b185db1ac04e9915c92bd02be4fc79b3971997b1dcd2c75efeb985e0ff42e85e96c1ac09d9e4a82d03f95460e057d26c6a0f` + +## Changelog since v1.10.9 + +### Other notable changes + +* Do not remove shutdown nodes from api-server for vSphere. ([#70291](https://github.com/kubernetes/kubernetes/pull/70291), [@gnufied](https://github.com/gnufied)) + * Fixes volume detach from shutdown nodes. +* remove retry operation on attach/detach azure disk ([#70568](https://github.com/kubernetes/kubernetes/pull/70568), [@andyzhangx](https://github.com/andyzhangx)) +* fix azure disk attachment error on Linux ([#70002](https://github.com/kubernetes/kubernetes/pull/70002), [@andyzhangx](https://github.com/andyzhangx)) +* Ensure orphan public IPs on Azure deleted when service recreated with the same name. ([#70463](https://github.com/kubernetes/kubernetes/pull/70463), [@feiskyer](https://github.com/feiskyer)) +* Improve Azure instance metadata handling by adding caches. ([#70353](https://github.com/kubernetes/kubernetes/pull/70353), [@feiskyer](https://github.com/feiskyer)) +* Fix cloud-controller-manager crash when using OpenStack provider and PersistentVolume initializing controller ([#70459](https://github.com/kubernetes/kubernetes/pull/70459), [@mvladev](https://github.com/mvladev)) +* GCE/GKE load balancer health check default interval changes from 2 seconds to 8 seconds, unhealthyThreshold to 3. ([#70099](https://github.com/kubernetes/kubernetes/pull/70099), [@grayluck](https://github.com/grayluck)) + * Health check parameters are configurable to be bigger than default values. +* Scheduling conformance tests related to daemonsets should set the annotation that relaxes node selection restrictions, if any are set. This ensures conformance tests can run on a wider array of clusters. ([#68793](https://github.com/kubernetes/kubernetes/pull/68793), [@aveshagarwal](https://github.com/aveshagarwal)) +* Fix cluster autoscaler addon permissions so it can access batch/job. ([#69858](https://github.com/kubernetes/kubernetes/pull/69858), [@losipiuk](https://github.com/losipiuk)) +* change default azure file mount permission to 0777 ([#69854](https://github.com/kubernetes/kubernetes/pull/69854), [@andyzhangx](https://github.com/andyzhangx)) +* Verify invalid secret/configmap/projected volumes before calling setup ([#68691](https://github.com/kubernetes/kubernetes/pull/68691), [@gnufied](https://github.com/gnufied)) + + + # v1.10.9 [Documentation](https://docs.k8s.io) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 5507aa07655..7683bf490cc 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,11 +1,10 @@ { "ImportPath": "k8s.io/kubernetes", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "github.com/onsi/ginkgo/ginkgo", "github.com/jteeuwen/go-bindata/go-bindata", - "github.com/tools/godep", "github.com/client9/misspell/cmd/misspell", "github.com/cloudflare/cfssl/cmd/cfssl", "github.com/cloudflare/cfssl/cmd/cfssljson", @@ -26,12 +25,12 @@ }, { "ImportPath": "cloud.google.com/go/compute/metadata", - "Comment": "v0.1.0-115-g3b1ae45", + "Comment": "v0.1.0-115-g3b1ae45394a234", "Rev": "3b1ae45394a234c385be014e9a488f2bb6eef821" }, { "ImportPath": "cloud.google.com/go/internal", - "Comment": "v0.1.0-115-g3b1ae45", + "Comment": "v0.1.0-115-g3b1ae45394a234", "Rev": "3b1ae45394a234c385be014e9a488f2bb6eef821" }, { @@ -157,22 +156,22 @@ }, { "ImportPath": "github.com/Rican7/retry", - "Comment": "v0.1.0-9-g272ad12", + "Comment": "v0.1.0-9-g272ad122d6e5ce", "Rev": "272ad122d6e5ce1be757544007cf8bcd1c9c9ab0" }, { "ImportPath": "github.com/Rican7/retry/backoff", - "Comment": "v0.1.0-9-g272ad12", + "Comment": "v0.1.0-9-g272ad122d6e5ce", "Rev": "272ad122d6e5ce1be757544007cf8bcd1c9c9ab0" }, { "ImportPath": "github.com/Rican7/retry/jitter", - "Comment": "v0.1.0-9-g272ad12", + "Comment": "v0.1.0-9-g272ad122d6e5ce", "Rev": "272ad122d6e5ce1be757544007cf8bcd1c9c9ab0" }, { "ImportPath": "github.com/Rican7/retry/strategy", - "Comment": "v0.1.0-9-g272ad12", + "Comment": "v0.1.0-9-g272ad122d6e5ce", "Rev": "272ad122d6e5ce1be757544007cf8bcd1c9c9ab0" }, { @@ -181,7 +180,7 @@ }, { "ImportPath": "github.com/asaskevich/govalidator", - "Comment": "v9-26-gf9ffefc", + "Comment": "v9-26-gf9ffefc3facfbe", "Rev": "f9ffefc3facfbe0caee3fea233cbb6e8208f4541" }, { @@ -361,87 +360,92 @@ }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/cmd/gazelle", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/config", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/flag", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/label", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/language", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/language/go", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/language/proto", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/merger", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/pathtools", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/repos", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/resolve", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/rule", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" + }, + { + "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/testtools", + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/version", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/walk", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/bazel-gazelle/internal/wspace", - "Comment": "0.14.0", - "Rev": "6a1b93cc9b1c7e55e7d05a6d324bcf9d87ea3ab1" + "Comment": "0.15.0", + "Rev": "c728ce9f663e2bff26361ba5978ec5c9e6816a3c" }, { "ImportPath": "github.com/bazelbuild/buildtools/build", - "Comment": "0.6.0-60-g1a9c38e", + "Comment": "0.6.0-60-g1a9c38e0df9397", "Rev": "1a9c38e0df9397d033a1ca535596de5a7c1cf18f" }, { "ImportPath": "github.com/bazelbuild/buildtools/tables", - "Comment": "0.6.0-60-g1a9c38e", + "Comment": "0.6.0-60-g1a9c38e0df9397", "Rev": "1a9c38e0df9397d033a1ca535596de5a7c1cf18f" }, { @@ -471,327 +475,327 @@ }, { "ImportPath": "github.com/client9/misspell", - "Comment": "v0.3.0-7-g9ce5d97", + "Comment": "v0.3.0-7-g9ce5d979ffdaca", "Rev": "9ce5d979ffdaca6385988d7ad1079a33ec942d20" }, { "ImportPath": "github.com/client9/misspell/cmd/misspell", - "Comment": "v0.3.0-7-g9ce5d97", + "Comment": "v0.3.0-7-g9ce5d979ffdaca", "Rev": "9ce5d979ffdaca6385988d7ad1079a33ec942d20" }, { "ImportPath": "github.com/cloudflare/cfssl/api", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/api/bundle", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/api/certinfo", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/api/client", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/api/crl", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/api/gencrl", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/api/generator", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/api/health", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/api/info", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/api/initca", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/api/ocsp", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/api/revoke", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/api/scan", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/api/signhandler", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/auth", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/bundler", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/certdb", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/certdb/dbconf", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/certdb/sql", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/certinfo", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/bundle", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/certinfo", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/crl", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/gencert", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/gencrl", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/gencsr", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/genkey", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/info", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/ocspdump", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/ocsprefresh", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/ocspserve", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/ocspsign", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/printdefault", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/revoke", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/scan", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/selfsign", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/serve", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/sign", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cli/version", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cmd/cfssl", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/cmd/cfssljson", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/config", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/crl", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/crypto/pkcs7", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/csr", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/errors", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/helpers", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/helpers/derhelpers", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/info", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/initca", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/log", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/ocsp", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/ocsp/config", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/revoke", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/scan", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/scan/crypto/tls", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/selfsign", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/signer", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/signer/local", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/signer/remote", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/signer/universal", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { "ImportPath": "github.com/cloudflare/cfssl/ubiquity", - "Comment": "1.3.2-21-g56268a6", + "Comment": "1.3.2-21-g56268a613adfed", "Rev": "56268a613adfed278936377c18b1152d2c4ad5da" }, { @@ -807,9 +811,9 @@ "Rev": "20e2ce2cf8852dc78bd42b76698dcd8dcd77b7b1" }, { - "ImportPath": "github.com/container-storage-interface/spec/lib/go/csi/v0", - "Comment": "v0.3.0", - "Rev": "2178fdeea87f1150a17a63252eee28d4d8141f72" + "ImportPath": "github.com/container-storage-interface/spec/lib/go/csi", + "Comment": "v1.0.0", + "Rev": "ed0bb0e1557548aa028307f48728767cfe8f6345" }, { "ImportPath": "github.com/containerd/console", @@ -1266,7 +1270,7 @@ }, { "ImportPath": "github.com/coreos/go-semver/semver", - "Comment": "v0.2.0-9-ge214231", + "Comment": "v0.2.0-9-ge214231b295a8e", "Rev": "e214231b295a8ea9479f11b70b35d5acf3556d9b" }, { @@ -1311,7 +1315,7 @@ }, { "ImportPath": "github.com/cyphar/filepath-securejoin", - "Comment": "v0.2.1-1-gae69057", + "Comment": "v0.2.1-1-gae69057f2299fb", "Rev": "ae69057f2299fb9e5ba2df738607e6a505b74ab6" }, { @@ -1328,7 +1332,7 @@ }, { "ImportPath": "github.com/davecgh/go-spew/spew", - "Comment": "v1.1.0-1-g782f496", + "Comment": "v1.1.0-1-g782f4967f2dc45", "Rev": "782f4967f2dc4564575ca782fe2d04090b5faca8" }, { @@ -1337,172 +1341,172 @@ }, { "ImportPath": "github.com/dgrijalva/jwt-go", - "Comment": "v3.0.0-4-g01aeca5", + "Comment": "v3.0.0-4-g01aeca54ebda6e", "Rev": "01aeca54ebda6e0fbfafd0a524d234159c05ec20" }, { "ImportPath": "github.com/docker/distribution/digestset", - "Comment": "v2.6.0-rc.1-209-gedc3ab29", + "Comment": "v2.6.0-rc.1-209-gedc3ab29cdff86", "Rev": "edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c" }, { "ImportPath": "github.com/docker/distribution/reference", - "Comment": "v2.6.0-rc.1-209-gedc3ab29", + "Comment": "v2.6.0-rc.1-209-gedc3ab29cdff86", "Rev": "edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c" }, { "ImportPath": "github.com/docker/docker/api", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/blkiodev", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/container", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/events", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/filters", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/image", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/mount", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/network", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/registry", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/strslice", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/swarm", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/swarm/runtime", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/time", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/versions", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/api/types/volume", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/client", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/pkg/jsonmessage", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/pkg/mount", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/pkg/parsers", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/pkg/parsers/operatingsystem", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/pkg/stdcopy", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/pkg/sysinfo", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/pkg/term", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/docker/pkg/term/windows", - "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd", + "Comment": "docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794", "Rev": "a9fbbdc8dd8794b20af358382ab780559bca589d" }, { "ImportPath": "github.com/docker/go-connections/nat", - "Comment": "v0.2.1-30-g3ede32e", + "Comment": "v0.3.0", "Rev": "3ede32e2033de7505e6500d6c868c2b9ed9f169d" }, { "ImportPath": "github.com/docker/go-connections/sockets", - "Comment": "v0.2.1-30-g3ede32e", + "Comment": "v0.3.0", "Rev": "3ede32e2033de7505e6500d6c868c2b9ed9f169d" }, { "ImportPath": "github.com/docker/go-connections/tlsconfig", - "Comment": "v0.2.1-30-g3ede32e", + "Comment": "v0.3.0", "Rev": "3ede32e2033de7505e6500d6c868c2b9ed9f169d" }, { "ImportPath": "github.com/docker/go-units", - "Comment": "v0.3.1-11-g9e638d3", + "Comment": "v0.3.1-11-g9e638d38cf6977", "Rev": "9e638d38cf6977a37a8ea0078f3ee75a7cdb2dd1" }, { "ImportPath": "github.com/docker/libnetwork/ipvs", - "Comment": "v0.8.0-dev.2-1265-ga9cd636e", + "Comment": "v0.8.0-dev.2-1265-ga9cd636e378982", "Rev": "a9cd636e37898226332c439363e2ed0ea185ae92" }, { @@ -1519,12 +1523,12 @@ }, { "ImportPath": "github.com/elazarl/goproxy", - "Comment": "v1.0-104-gc4fc265", + "Comment": "v1.0-104-gc4fc26588b6ef8", "Rev": "c4fc26588b6ef8af07a191fcb6476387bdd46711" }, { "ImportPath": "github.com/emicklei/go-restful", - "Comment": "2.2.0-4-gff4f55a", + "Comment": "2.2.0-4-gff4f55a206334e", "Rev": "ff4f55a206334ef123e4f79bbf348980da81ca46" }, { @@ -1534,7 +1538,7 @@ }, { "ImportPath": "github.com/emicklei/go-restful/log", - "Comment": "2.2.0-4-gff4f55a", + "Comment": "2.2.0-4-gff4f55a206334e", "Rev": "ff4f55a206334ef123e4f79bbf348980da81ca46" }, { @@ -1544,7 +1548,7 @@ }, { "ImportPath": "github.com/evanphx/json-patch", - "Comment": "v4.0.0-3-g36442db", + "Comment": "v4.0.0-3-g36442dbdb58521", "Rev": "36442dbdb585210f8d5a1b45e67aa323c197d5c4" }, { @@ -1557,22 +1561,22 @@ }, { "ImportPath": "github.com/fsnotify/fsnotify", - "Comment": "v1.3.1-1-gf12c623", + "Comment": "v1.3.1-1-gf12c6236fe7b5c", "Rev": "f12c6236fe7b5cf6bcf30e5935d08cb079d78334" }, { "ImportPath": "github.com/ghodss/yaml", - "Comment": "v1.0.0-4-gc7ce166", + "Comment": "v1.0.0-4-gc7ce16629ff4cd", "Rev": "c7ce16629ff4cd059ed96ed06419dd3856fd3577" }, { "ImportPath": "github.com/globalsign/mgo/bson", - "Comment": "r2018.06.15-9-geeefdec", + "Comment": "r2018.06.15-9-geeefdecb41b842", "Rev": "eeefdecb41b842af6dc652aaea4026e8403e62df" }, { "ImportPath": "github.com/globalsign/mgo/internal/json", - "Comment": "r2018.06.15-9-geeefdec", + "Comment": "r2018.06.15-9-geeefdecb41b842", "Rev": "eeefdecb41b842af6dc652aaea4026e8403e62df" }, { @@ -1617,7 +1621,7 @@ }, { "ImportPath": "github.com/go-openapi/spec", - "Comment": "v0.17.1", + "Comment": "v0.17.2", "Rev": "5bae59e25b21498baea7f9d46e9c147ec106a42e" }, { @@ -1632,9 +1636,19 @@ }, { "ImportPath": "github.com/go-openapi/validate", - "Comment": "v0.17.1", + "Comment": "v0.17.2", "Rev": "d2eab7d93009e9215fc85b2faa2c2f2a98c2af48" }, + { + "ImportPath": "github.com/go-ozzo/ozzo-validation", + "Comment": "v3.5.0", + "Rev": "106681dbb37bfa3e7683c4c8129cb7f5925ea3e9" + }, + { + "ImportPath": "github.com/go-ozzo/ozzo-validation/is", + "Comment": "v3.5.0", + "Rev": "106681dbb37bfa3e7683c4c8129cb7f5925ea3e9" + }, { "ImportPath": "github.com/go-sql-driver/mysql", "Comment": "v1.3.0", @@ -1775,10 +1789,6 @@ "Comment": "v0.5", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/golang/groupcache/lru", "Rev": "02826c3e79038b59d737d3b1c0a1d937f71a4433" @@ -1797,6 +1807,11 @@ "Comment": "v1.1.0", "Rev": "b4deda0973fb4c70b50d226b1af49f3da59f5265" }, + { + "ImportPath": "github.com/golang/protobuf/protoc-gen-go/descriptor", + "Comment": "v1.1.0", + "Rev": "b4deda0973fb4c70b50d226b1af49f3da59f5265" + }, { "ImportPath": "github.com/golang/protobuf/ptypes", "Comment": "v1.1.0", @@ -1833,188 +1848,188 @@ }, { "ImportPath": "github.com/google/cadvisor/accelerators", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/cache/memory", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/client/v2", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/collector", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/container", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/container/common", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/container/containerd", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/container/crio", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/container/docker", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/container/libcontainer", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/container/mesos", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/container/raw", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/container/rkt", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/container/systemd", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/devicemapper", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/events", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/fs", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/info/v1", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/info/v2", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/machine", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/manager", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/manager/watcher", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/manager/watcher/raw", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/manager/watcher/rkt", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/metrics", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/storage", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/summary", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/utils", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/utils/cloudinfo", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/utils/cpuload", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/utils/cpuload/netlink", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/utils/docker", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/utils/oomparser", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/utils/sysfs", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/utils/sysinfo", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/version", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/cadvisor/zfs", - "Comment": "v0.31.0", - "Rev": "fc17731afdcf184832482e324913c8f1a91b54ee" + "Comment": "v0.32.0", + "Rev": "8949c822ea91fa6b4996614a5ad6ade840be24ee" }, { "ImportPath": "github.com/google/certificate-transparency-go", @@ -2062,7 +2077,7 @@ }, { "ImportPath": "github.com/google/uuid", - "Comment": "0.2-15-g8c31c18", + "Comment": "0.2-15-g8c31c18f31ede9", "Rev": "8c31c18f31ede9fc8eae72290a7e7a8064e9b3e3" }, { @@ -2199,7 +2214,7 @@ }, { "ImportPath": "github.com/gorilla/websocket", - "Comment": "v1.2.0-9-g4201258", + "Comment": "v1.2.0-9-g4201258b820c74", "Rev": "4201258b820c74ac8e6922fc9e6b52f71fe46f8d" }, { @@ -2212,12 +2227,12 @@ }, { "ImportPath": "github.com/grpc-ecosystem/go-grpc-middleware", - "Comment": "v1.0.0-4-g498ae20", + "Comment": "v1.0.0-4-g498ae206fc3cfe", "Rev": "498ae206fc3cfe81cd82e48c1d4354026fa5f9ec" }, { "ImportPath": "github.com/grpc-ecosystem/go-grpc-prometheus", - "Comment": "v1.1-4-g2500245", + "Comment": "v1.1-4-g2500245aa6110c", "Rev": "2500245aa6110c562d17020fb31a2c133d737799" }, { @@ -2281,18 +2296,18 @@ }, { "ImportPath": "github.com/heketi/heketi/client/api/go-client", - "Comment": "v4.0.0-95-gaaf4061", - "Rev": "aaf40619d85fda757e7a1c1ea1b5118cea65594b" + "Comment": "v8.0.0-49-g558b29266ce0a8", + "Rev": "558b29266ce0a873991ecfb3edc41a668a998514" }, { "ImportPath": "github.com/heketi/heketi/pkg/glusterfs/api", - "Comment": "v4.0.0-95-gaaf4061", - "Rev": "aaf40619d85fda757e7a1c1ea1b5118cea65594b" + "Comment": "v8.0.0-49-g558b29266ce0a8", + "Rev": "558b29266ce0a873991ecfb3edc41a668a998514" }, { "ImportPath": "github.com/heketi/heketi/pkg/utils", - "Comment": "v4.0.0-95-gaaf4061", - "Rev": "aaf40619d85fda757e7a1c1ea1b5118cea65594b" + "Comment": "v8.0.0-49-g558b29266ce0a8", + "Rev": "558b29266ce0a873991ecfb3edc41a668a998514" }, { "ImportPath": "github.com/imdario/mergo", @@ -2301,11 +2316,12 @@ }, { "ImportPath": "github.com/inconshreveable/mousetrap", + "Comment": "v1.0", "Rev": "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" }, { "ImportPath": "github.com/jmespath/go-jmespath", - "Comment": "0.2.2-12-g0b12d6b", + "Comment": "0.2.2-12-g0b12d6b521d83f", "Rev": "0b12d6b521d83fc7f755e7cfc1b1fbdd35a01a74" }, { @@ -2315,12 +2331,12 @@ }, { "ImportPath": "github.com/jmoiron/sqlx", - "Comment": "sqlx-v1.1-131-g05cef07", + "Comment": "sqlx-v1.1-131-g05cef0741ade10", "Rev": "05cef0741ade10ca668982355b3f3f0bcf0ff0a8" }, { "ImportPath": "github.com/jmoiron/sqlx/reflectx", - "Comment": "sqlx-v1.1-131-g05cef07", + "Comment": "sqlx-v1.1-131-g05cef0741ade10", "Rev": "05cef0741ade10ca668982355b3f3f0bcf0ff0a8" }, { @@ -2329,23 +2345,28 @@ }, { "ImportPath": "github.com/json-iterator/go", - "Comment": "1.1.3-22-gf2b4162", + "Comment": "1.1.3-22-gf2b4162afba355", "Rev": "f2b4162afba35581b6d4a50d3b8f34e33c144682" }, { "ImportPath": "github.com/jteeuwen/go-bindata", - "Comment": "v3.0.7-72-ga0ff256", + "Comment": "v3.0.7-72-ga0ff2567cfb709", "Rev": "a0ff2567cfb70903282db057e799fd826784d41d" }, { "ImportPath": "github.com/jteeuwen/go-bindata/go-bindata", - "Comment": "v3.0.7-72-ga0ff256", + "Comment": "v3.0.7-72-ga0ff2567cfb709", "Rev": "a0ff2567cfb70903282db057e799fd826784d41d" }, { "ImportPath": "github.com/kardianos/osext", "Rev": "8fef92e41e22a70e700a96b29f066cda30ea24ef" }, + { + "ImportPath": "github.com/karrick/godirwalk", + "Comment": "v1.7.5", + "Rev": "2de2192f9e35ce981c152a873ed943b93b79ced4" + }, { "ImportPath": "github.com/kisielk/sqlstruct", "Rev": "648daed35d49dac24a4bff253b190a80da3ab6a5" @@ -2356,7 +2377,7 @@ }, { "ImportPath": "github.com/kr/pretty", - "Comment": "go.weekly.2011-12-22-24-gf31442d", + "Comment": "go.weekly.2011-12-22-24-gf31442d60e5146", "Rev": "f31442d60e51465c69811e2107ae978868dbea5c" }, { @@ -2365,7 +2386,7 @@ }, { "ImportPath": "github.com/kubernetes/repo-infra/kazel", - "Rev": "d9bb9fdc907665c61c228baa64fed9b91c7dc1b0" + "Rev": "f2459dc75fc429b813d92c0622b408fd7f0d4cac" }, { "ImportPath": "github.com/lib/pq", @@ -2403,14 +2424,9 @@ "ImportPath": "github.com/libopenstorage/openstorage/volume", "Rev": "093a0c3888753c2056e7373183693d670c6bba01" }, - { - "ImportPath": "github.com/lpabon/godbc", - "Comment": "v1.0-1-g9577782", - "Rev": "9577782540c1398b710ddae1b86268ba03a19b0c" - }, { "ImportPath": "github.com/magiconair/properties", - "Comment": "v1.7.0-4-g61b492c", + "Comment": "v1.7.0-4-g61b492c03cf472", "Rev": "61b492c03cf472e0c6419be5899b8e0dc28b1b88" }, { @@ -2427,12 +2443,12 @@ }, { "ImportPath": "github.com/marstr/guid", - "Comment": "v1.1.0-2-g8bdf7d1", + "Comment": "v1.1.0-2-g8bdf7d1a087ccc", "Rev": "8bdf7d1a087ccc975cf37dd6507da50698fd19ca" }, { "ImportPath": "github.com/mattn/go-shellwords", - "Comment": "v1.0.3-20-gf8471b0", + "Comment": "v1.0.3-20-gf8471b0a71ded0", "Rev": "f8471b0a71ded0ab910825ee2cf230f25de000f1" }, { @@ -2442,82 +2458,82 @@ }, { "ImportPath": "github.com/matttproud/golang_protobuf_extensions/pbutil", - "Comment": "v1.0.0-2-gc12348c", + "Comment": "v1.0.1", "Rev": "c12348ce28de40eed0136aa2b644d0ee0650e56c" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib/agent", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib/agent/calls", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib/client", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib/debug", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib/encoding", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib/encoding/codecs", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib/encoding/framing", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib/encoding/json", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib/encoding/proto", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib/httpcli", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib/httpcli/apierrors", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib/recordio", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mesos/mesos-go/api/v1/lib/roles", - "Comment": "mesos-1.6.x-12-gff8175b", + "Comment": "mesos-1.6.x-12-gff8175bfda54b1", "Rev": "ff8175bfda54b1eb1f1954c8102a9dc612aa2312" }, { "ImportPath": "github.com/mholt/caddy/caddyfile", - "Comment": "v0.10.10-57-g2de4950", + "Comment": "v0.10.10-57-g2de495001514ed", "Rev": "2de495001514ed50eac2e09f474c32d417100c5c" }, { @@ -2530,7 +2546,7 @@ }, { "ImportPath": "github.com/mistifyio/go-zfs", - "Comment": "v2.1.1-5-g1b4ae6f", + "Comment": "v2.1.1-5-g1b4ae6fb4e77b0", "Rev": "1b4ae6fb4e77b095934d4430860ff202060169f8" }, { @@ -2561,7 +2577,7 @@ }, { "ImportPath": "github.com/mvdan/xurls", - "Comment": "v0.8.0-14-g1b768d7", + "Comment": "v0.8.0-14-g1b768d7c393abd", "Rev": "1b768d7c393abd8e8dda1458385a57becd4b2d4e" }, { @@ -2570,197 +2586,197 @@ }, { "ImportPath": "github.com/onsi/ginkgo", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/config", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/ginkgo", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/ginkgo/convert", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/ginkgo/interrupthandler", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/ginkgo/nodot", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/ginkgo/testrunner", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/ginkgo/testsuite", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/ginkgo/watch", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/internal/codelocation", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/internal/containernode", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/internal/failer", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/internal/leafnodes", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/internal/remote", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/internal/spec", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/internal/spec_iterator", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/internal/specrunner", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/internal/suite", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/internal/testingtproxy", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/internal/writer", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/reporters", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/reporters/stenographer", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/ginkgo/types", - "Comment": "v1.2.0-95-g67b9df7", + "Comment": "v1.2.0-95-g67b9df7f55fe11", "Rev": "67b9df7f55fe1165fd9ad49aca7754cce01a42b8" }, { "ImportPath": "github.com/onsi/gomega", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { "ImportPath": "github.com/onsi/gomega/format", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { "ImportPath": "github.com/onsi/gomega/gstruct", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { "ImportPath": "github.com/onsi/gomega/gstruct/errors", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { "ImportPath": "github.com/onsi/gomega/internal/assertion", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { "ImportPath": "github.com/onsi/gomega/internal/asyncassertion", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { "ImportPath": "github.com/onsi/gomega/internal/oraclematcher", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { "ImportPath": "github.com/onsi/gomega/internal/testingtsupport", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { "ImportPath": "github.com/onsi/gomega/matchers", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { "ImportPath": "github.com/onsi/gomega/matchers/support/goraph/bipartitegraph", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { "ImportPath": "github.com/onsi/gomega/matchers/support/goraph/edge", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { "ImportPath": "github.com/onsi/gomega/matchers/support/goraph/node", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { "ImportPath": "github.com/onsi/gomega/matchers/support/goraph/util", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { "ImportPath": "github.com/onsi/gomega/types", - "Comment": "v1.0-122-gd59fa0a", + "Comment": "v1.0-122-gd59fa0ac68bb5d", "Rev": "d59fa0ac68bb5dd932ee8d24eed631cdd519efc3" }, { @@ -2769,92 +2785,92 @@ }, { "ImportPath": "github.com/opencontainers/image-spec/specs-go", - "Comment": "v1.0.0-rc6-12-g372ad78", + "Comment": "v1.0.0-rc6-12-g372ad780f63454", "Rev": "372ad780f63454fbbbbcc7cf80e5b90245c13e13" }, { "ImportPath": "github.com/opencontainers/image-spec/specs-go/v1", - "Comment": "v1.0.0-rc6-12-g372ad78", + "Comment": "v1.0.0-rc6-12-g372ad780f63454", "Rev": "372ad780f63454fbbbbcc7cf80e5b90245c13e13" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/apparmor", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/cgroups", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/cgroups/fs", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/cgroups/systemd", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/configs", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/configs/validate", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/criurpc", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/intelrdt", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/keys", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/mount", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/seccomp", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/stacktrace", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/system", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/user", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/utils", - "Comment": "v1.0.0-rc5-46-g871ba2e5", + "Comment": "v1.0.0-rc5-46-g871ba2e58e2431", "Rev": "871ba2e58e24314d1fab4517a80410191ba5ad01" }, { @@ -2864,12 +2880,12 @@ }, { "ImportPath": "github.com/opencontainers/selinux/go-selinux", - "Comment": "v1.0.0-rc1-5-g4a2974b", + "Comment": "v1.0.0-rc1-5-g4a2974bf1ee960", "Rev": "4a2974bf1ee960774ffd517717f1f45325af0206" }, { "ImportPath": "github.com/opencontainers/selinux/go-selinux/label", - "Comment": "v1.0.0-rc1-5-g4a2974b", + "Comment": "v1.0.0-rc1-5-g4a2974bf1ee960", "Rev": "4a2974bf1ee960774ffd517717f1f45325af0206" }, { @@ -2917,17 +2933,17 @@ }, { "ImportPath": "github.com/prometheus/client_golang/prometheus", - "Comment": "v0.8.0-83-ge7e9030", + "Comment": "v0.8.0-83-ge7e903064f5e9e", "Rev": "e7e903064f5e9eb5da98208bae10b475d4db0f8c" }, { "ImportPath": "github.com/prometheus/client_golang/prometheus/promhttp", - "Comment": "v0.8.0-83-ge7e9030", + "Comment": "v0.8.0-83-ge7e903064f5e9e", "Rev": "e7e903064f5e9eb5da98208bae10b475d4db0f8c" }, { "ImportPath": "github.com/prometheus/client_model/go", - "Comment": "model-0.0.2-12-gfa8ad6f", + "Comment": "model-0.0.2-12-gfa8ad6fec33561", "Rev": "fa8ad6fec33561be4280a8f0514318c79d7f6cb6" }, { @@ -2952,22 +2968,22 @@ }, { "ImportPath": "github.com/quobyte/api", - "Comment": "v0.1.1-4-g206ef83", + "Comment": "v0.1.1-4-g206ef832283c1a", "Rev": "206ef832283c1a0144bbc762be2634d49987b5ff" }, { "ImportPath": "github.com/rancher/go-rancher/client", - "Comment": "v0.1.0-196-g09693a8", + "Comment": "v0.1.0-196-g09693a8743ba5e", "Rev": "09693a8743ba5ee58c58c1b1e8a4abd17af00d45" }, { "ImportPath": "github.com/renstrom/dedent", - "Comment": "v1.0.0-3-g020d11c", + "Comment": "v1.0.0-3-g020d11c3b9c0c7", "Rev": "020d11c3b9c0c7a3c2efcc8e5cf5b9ef7bcea21f" }, { "ImportPath": "github.com/robfig/cron", - "Comment": "v1-53-gdf38d32", + "Comment": "v1-53-gdf38d32658d878", "Rev": "df38d32658d8788cd446ba74db4bb5375c4b0cb3" }, { @@ -2976,7 +2992,7 @@ }, { "ImportPath": "github.com/russross/blackfriday", - "Comment": "v1.4-2-g300106c", + "Comment": "v1.4-2-g300106c228d52c", "Rev": "300106c228d52c8941d4b3de6054a6062a86dda3" }, { @@ -2992,9 +3008,13 @@ "ImportPath": "github.com/shurcooL/sanitized_anchor_name", "Rev": "10ef21a441db47d8b13ebcc5fd2310f636973c77" }, + { + "ImportPath": "github.com/sigma/go-inotify", + "Rev": "c87b6cf5033d2c6486046f045eeebdc3d910fd38" + }, { "ImportPath": "github.com/sirupsen/logrus", - "Comment": "v1.0.3-11-g89742ae", + "Comment": "v1.0.3-11-g89742aefa4b206", "Rev": "89742aefa4b206dcf400792f3bd35b542998eb3b" }, { @@ -3020,12 +3040,12 @@ }, { "ImportPath": "github.com/spf13/cobra", - "Comment": "v0.0.1-34-gc439c4f", + "Comment": "v0.0.1-34-gc439c4fa093711", "Rev": "c439c4fa093711d42e1b01acb1235b52004753c1" }, { "ImportPath": "github.com/spf13/cobra/doc", - "Comment": "v0.0.1-34-gc439c4f", + "Comment": "v0.0.1-34-gc439c4fa093711", "Rev": "c439c4fa093711d42e1b01acb1235b52004753c1" }, { @@ -3067,17 +3087,17 @@ }, { "ImportPath": "github.com/stretchr/testify/assert", - "Comment": "v1.2.1-14-gc679ae2", + "Comment": "v1.2.1-14-gc679ae2cc0cb27", "Rev": "c679ae2cc0cb27ec3293fea7e254e47386f05d69" }, { "ImportPath": "github.com/stretchr/testify/mock", - "Comment": "v1.2.1-14-gc679ae2", + "Comment": "v1.2.1-14-gc679ae2cc0cb27", "Rev": "c679ae2cc0cb27ec3293fea7e254e47386f05d69" }, { "ImportPath": "github.com/stretchr/testify/require", - "Comment": "v1.2.1-14-gc679ae2", + "Comment": "v1.2.1-14-gc679ae2cc0cb27", "Rev": "c679ae2cc0cb27ec3293fea7e254e47386f05d69" }, { @@ -3088,11 +3108,6 @@ "ImportPath": "github.com/tmc/grpc-websocket-proxy/wsproxy", "Rev": "89b8d40f7ca833297db804fcb3be53a76d01c238" }, - { - "ImportPath": "github.com/tools/godep", - "Comment": "v80", - "Rev": "ce0bfadeb516ab23c845a85c4b0eae421ec9614b" - }, { "ImportPath": "github.com/ugorji/go/codec", "Rev": "bdcc60b419d136a85cdf2e7cbcac34b3f1cd6e57" @@ -3111,167 +3126,167 @@ }, { "ImportPath": "github.com/vmware/govmomi", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/find", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/list", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/lookup", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/lookup/methods", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/lookup/simulator", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/lookup/types", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/nfc", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/object", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/pbm", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/pbm/methods", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/pbm/types", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/property", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/session", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/simulator", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/simulator/esx", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/simulator/vpx", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/sts", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/sts/internal", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/sts/simulator", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/task", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/vapi/internal", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/vapi/rest", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/vapi/simulator", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/vapi/tags", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/vim25", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/vim25/debug", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/vim25/methods", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/vim25/mo", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/vim25/progress", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/vim25/soap", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/vim25/types", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { "ImportPath": "github.com/vmware/govmomi/vim25/xml", - "Comment": "v0.18.0-48-g22f7465", + "Comment": "v0.18.0-48-g22f74650cf39ba", "Rev": "22f74650cf39ba4649fba45e770df0f44df6f758" }, { @@ -3291,7 +3306,7 @@ }, { "ImportPath": "github.com/xanzy/go-cloudstack/cloudstack", - "Comment": "v2.1.1-1-g1e2cbf6", + "Comment": "v2.1.1-1-g1e2cbf647e57fa", "Rev": "1e2cbf647e57fa90353612074fdfc42faf5073bf" }, { @@ -3301,42 +3316,42 @@ }, { "ImportPath": "go.uber.org/atomic", - "Comment": "v1.3.2-3-g8dc6146", + "Comment": "v1.3.2-3-g8dc6146f756937", "Rev": "8dc6146f7569370a472715e178d8ae31172ee6da" }, { "ImportPath": "go.uber.org/multierr", - "Comment": "v1.1.0-2-gddea229", + "Comment": "v1.1.0-2-gddea229ff1dff9", "Rev": "ddea229ff1dff9e6fe8a6c0344ac73b09e81fce5" }, { "ImportPath": "go.uber.org/zap", - "Comment": "v1.9.1-1-g67bc79d", + "Comment": "v1.9.1-1-g67bc79d13d155c", "Rev": "67bc79d13d155c02fd008f721863ff8cc5f30659" }, { "ImportPath": "go.uber.org/zap/buffer", - "Comment": "v1.9.1-1-g67bc79d", + "Comment": "v1.9.1-1-g67bc79d13d155c", "Rev": "67bc79d13d155c02fd008f721863ff8cc5f30659" }, { "ImportPath": "go.uber.org/zap/internal/bufferpool", - "Comment": "v1.9.1-1-g67bc79d", + "Comment": "v1.9.1-1-g67bc79d13d155c", "Rev": "67bc79d13d155c02fd008f721863ff8cc5f30659" }, { "ImportPath": "go.uber.org/zap/internal/color", - "Comment": "v1.9.1-1-g67bc79d", + "Comment": "v1.9.1-1-g67bc79d13d155c", "Rev": "67bc79d13d155c02fd008f721863ff8cc5f30659" }, { "ImportPath": "go.uber.org/zap/internal/exit", - "Comment": "v1.9.1-1-g67bc79d", + "Comment": "v1.9.1-1-g67bc79d13d155c", "Rev": "67bc79d13d155c02fd008f721863ff8cc5f30659" }, { "ImportPath": "go.uber.org/zap/zapcore", - "Comment": "v1.9.1-1-g67bc79d", + "Comment": "v1.9.1-1-g67bc79d13d155c", "Rev": "67bc79d13d155c02fd008f721863ff8cc5f30659" }, { @@ -3413,51 +3428,51 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/context/ctxhttp", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/html", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/html/atom", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/internal/timeseries", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/proxy", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/trace", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/websocket", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/oauth2", @@ -3737,98 +3752,143 @@ }, { "ImportPath": "google.golang.org/grpc", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/balancer", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/balancer/base", + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/balancer/roundrobin", + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/codes", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/connectivity", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/credentials", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/encoding", + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/encoding/proto", + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/grpclb/grpc_lb_v1/messages", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/grpclog", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/health", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/health/grpc_health_v1", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/internal", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/backoff", + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/channelz", + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/grpcrand", + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/keepalive", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/metadata", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/naming", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/peer", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/resolver", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/resolver/dns", + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/resolver/passthrough", + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/stats", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/status", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/tap", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/transport", - "Comment": "v1.7.5", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Comment": "v1.13.0", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "gopkg.in/gcfg.v1", @@ -3857,27 +3917,27 @@ }, { "ImportPath": "gopkg.in/natefinch/lumberjack.v2", - "Comment": "v1.0-16-g20b71e5", + "Comment": "v1.0-16-g20b71e5b60d756", "Rev": "20b71e5b60d756d3d2f80def009790325acc2b23" }, { "ImportPath": "gopkg.in/square/go-jose.v2", - "Comment": "v2.1.6-4-g89060de", + "Comment": "v2.1.6-4-g89060dee6a84df", "Rev": "89060dee6a84df9a4dae49f676f0c755037834f1" }, { "ImportPath": "gopkg.in/square/go-jose.v2/cipher", - "Comment": "v2.1.6-4-g89060de", + "Comment": "v2.1.6-4-g89060dee6a84df", "Rev": "89060dee6a84df9a4dae49f676f0c755037834f1" }, { "ImportPath": "gopkg.in/square/go-jose.v2/json", - "Comment": "v2.1.6-4-g89060de", + "Comment": "v2.1.6-4-g89060dee6a84df", "Rev": "89060dee6a84df9a4dae49f676f0c755037834f1" }, { "ImportPath": "gopkg.in/square/go-jose.v2/jwt", - "Comment": "v2.1.6-4-g89060de", + "Comment": "v2.1.6-4-g89060dee6a84df", "Rev": "89060dee6a84df9a4dae49f676f0c755037834f1" }, { @@ -3892,100 +3952,104 @@ }, { "ImportPath": "k8s.io/gengo/args", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/examples/deepcopy-gen/generators", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/examples/defaulter-gen/generators", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/examples/import-boss/generators", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/examples/set-gen/generators", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/examples/set-gen/sets", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/generator", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/namer", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/parser", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/types", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/heapster/metrics/api/v1/types", "Comment": "v1.2.0-beta.1", "Rev": "c2ac40f1adf8c42a79badddb2a2acd673cae3bcb" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/kube-openapi/cmd/openapi-gen", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/cmd/openapi-gen/args", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/aggregator", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/builder", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/common", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/generators", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/generators/rules", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/handler", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto/testing", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto/validation", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/sets", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/utils/clock", @@ -4003,6 +4067,11 @@ "ImportPath": "k8s.io/utils/pointer", "Rev": "66066c83e385e385ccc3c964b44fd7dcd413d0ed" }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Comment": "v1.1.0", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" + }, { "ImportPath": "vbom.ml/util/sortorder", "Rev": "db5cfe13f5cc80a4990d98e2e1b0707a4d1a5394" diff --git a/Godeps/LICENSES b/Godeps/LICENSES index 93bb89075be..a46ce6e3f45 100644 --- a/Godeps/LICENSES +++ b/Godeps/LICENSES @@ -13566,6 +13566,216 @@ THE SOFTWARE. ================================================================================ +================================================================================ += vendor/github.com/bazelbuild/bazel-gazelle/internal/testtools licensed under: = + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += vendor/github.com/bazelbuild/bazel-gazelle/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 +================================================================================ + + ================================================================================ = vendor/github.com/bazelbuild/bazel-gazelle/internal/version licensed under: = @@ -17131,7 +17341,7 @@ Apache License ================================================================================ -= vendor/github.com/container-storage-interface/spec/lib/go/csi/v0 licensed under: = += vendor/github.com/container-storage-interface/spec/lib/go/csi licensed under: = Apache License Version 2.0, January 2004 @@ -48570,6 +48780,56 @@ third-party archives. ================================================================================ +================================================================================ += vendor/github.com/go-ozzo/ozzo-validation licensed under: = + +The MIT License (MIT) +Copyright (c) 2016, Qiang Xue + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software +and associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + += vendor/github.com/go-ozzo/ozzo-validation/LICENSE da12d993f2ce14947ad6eec35520b081 +================================================================================ + + +================================================================================ += vendor/github.com/go-ozzo/ozzo-validation/is licensed under: = + +The MIT License (MIT) +Copyright (c) 2016, Qiang Xue + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software +and associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + += vendor/github.com/go-ozzo/ozzo-validation/LICENSE da12d993f2ce14947ad6eec35520b081 +================================================================================ + + ================================================================================ = vendor/github.com/go-sql-driver/mysql licensed under: = @@ -50128,205 +50388,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -================================================================================ -= vendor/github.com/golang/glog licensed under: = - -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of -this License; and -You must cause any modified files to carry prominent notices stating that You -changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, -all copyright, patent, trademark, and attribution notices from the Source form -of the Work, excluding those notices that do not pertain to any part of the -Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any -Derivative Works that You distribute must include a readable copy of the -attribution notices contained within such NOTICE file, excluding those notices -that do not pertain to any part of the Derivative Works, in at least one of the -following places: within a NOTICE text file distributed as part of the -Derivative Works; within the Source form or documentation, if provided along -with the Derivative Works; or, within a display generated by the Derivative -Works, if and wherever such third-party notices normally appear. The contents of -the NOTICE file are for informational purposes only and do not modify the -License. You may add Your own attribution notices within Derivative Works that -You distribute, alongside or as an addendum to the NOTICE text from the Work, -provided that such additional attribution notices cannot be construed as -modifying the License. -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, -NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate -notice, with the fields enclosed by brackets "[]" replaced with your own -identifying information. (Don't include the brackets!) The text should be -enclosed in the appropriate comment syntax for the file format. We also -recommend that a file or class name and description of purpose be included on -the same "printed page" as the copyright notice for easier identification within -third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -= vendor/github.com/golang/glog/LICENSE 19cbd64715b51267a47bf3750cc6a8a5 -================================================================================ - - ================================================================================ = vendor/github.com/golang/groupcache/lru licensed under: = @@ -50814,6 +50875,45 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ +================================================================================ += vendor/github.com/golang/protobuf/protoc-gen-go/descriptor licensed under: = + +Go support for Protocol Buffers - Google's data interchange format + +Copyright 2010 The Go Authors. All rights reserved. +https://github.com/golang/protobuf + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + += vendor/github.com/golang/protobuf/LICENSE 14db3a56c3796a940ba32948a15f97d0 +================================================================================ + + ================================================================================ = vendor/github.com/golang/protobuf/ptypes licensed under: = @@ -72227,6 +72327,39 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ +================================================================================ += vendor/github.com/karrick/godirwalk licensed under: = + +BSD 2-Clause License + +Copyright (c) 2017, Karrick McDermott +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/github.com/karrick/godirwalk/LICENSE 7bea66fc0a31c6329f9392034bee75d2 +================================================================================ + + ================================================================================ = vendor/github.com/kisielk/sqlstruct licensed under: = @@ -73980,214 +74113,6 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI ================================================================================ -================================================================================ -= vendor/github.com/lpabon/godbc licensed under: = - -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -= vendor/github.com/lpabon/godbc/LICENSE 6c4db32a2fa8717faffa1d4f10136f47 -================================================================================ - - ================================================================================ = vendor/github.com/magiconair/properties licensed under: = @@ -87828,6 +87753,41 @@ THE SOFTWARE. ================================================================================ +================================================================================ += vendor/github.com/sigma/go-inotify licensed under: = + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/github.com/sigma/go-inotify/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 +================================================================================ + + ================================================================================ = vendor/github.com/sirupsen/logrus licensed under: = @@ -89477,42 +89437,6 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI ================================================================================ -================================================================================ -= vendor/github.com/tools/godep licensed under: = - -Copyright © 2013 Keith Rarick. -Portions Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -= vendor/github.com/tools/godep/License 71eb66e9b353dd06ca5a81ce0f469e1a -================================================================================ - - ================================================================================ = vendor/github.com/ugorji/go/codec licensed under: = @@ -101704,6 +101628,426 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ +================================================================================ += vendor/google.golang.org/grpc/balancer/base licensed under: = + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += vendor/google.golang.org/grpc/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 +================================================================================ + + +================================================================================ += vendor/google.golang.org/grpc/balancer/roundrobin licensed under: = + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += vendor/google.golang.org/grpc/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 +================================================================================ + + ================================================================================ = vendor/google.golang.org/grpc/codes licensed under: = @@ -102334,6 +102678,426 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ +================================================================================ += vendor/google.golang.org/grpc/encoding licensed under: = + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += vendor/google.golang.org/grpc/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 +================================================================================ + + +================================================================================ += vendor/google.golang.org/grpc/encoding/proto licensed under: = + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += vendor/google.golang.org/grpc/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 +================================================================================ + + ================================================================================ = vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages licensed under: = @@ -103384,6 +104148,636 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ +================================================================================ += vendor/google.golang.org/grpc/internal/backoff licensed under: = + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += vendor/google.golang.org/grpc/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 +================================================================================ + + +================================================================================ += vendor/google.golang.org/grpc/internal/channelz licensed under: = + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += vendor/google.golang.org/grpc/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 +================================================================================ + + +================================================================================ += vendor/google.golang.org/grpc/internal/grpcrand licensed under: = + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += vendor/google.golang.org/grpc/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 +================================================================================ + + ================================================================================ = vendor/google.golang.org/grpc/keepalive licensed under: = @@ -104434,6 +105828,426 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ +================================================================================ += vendor/google.golang.org/grpc/resolver/dns licensed under: = + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += vendor/google.golang.org/grpc/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 +================================================================================ + + +================================================================================ += vendor/google.golang.org/grpc/resolver/passthrough licensed under: = + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += vendor/google.golang.org/grpc/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 +================================================================================ + + ================================================================================ = vendor/google.golang.org/grpc/stats licensed under: = @@ -108872,6 +110686,205 @@ Apache License ================================================================================ +================================================================================ += vendor/k8s.io/klog licensed under: = + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += vendor/k8s.io/klog/LICENSE 19cbd64715b51267a47bf3750cc6a8a5 +================================================================================ + + ================================================================================ = vendor/k8s.io/kube-openapi/cmd/openapi-gen licensed under: = @@ -112442,6 +114455,64 @@ Apache License ================================================================================ +================================================================================ += vendor/sigs.k8s.io/yaml licensed under: = + +The MIT License (MIT) + +Copyright (c) 2014 Sam Ghods + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/sigs.k8s.io/yaml/LICENSE 0ceb9ff3b27d3a8cf451ca3785d73c71 +================================================================================ + + ================================================================================ = vendor/vbom.ml/util/sortorder licensed under: = diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 1e671c2c464..88959953bde 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -1,4 +1,125 @@ aliases: + # sig-auth subproject aliases + sig-auth-audit-approvers: + - sttts + - tallclair + sig-auth-audit-reviewers: + - CaoShuFeng + - hzxuzhonghu + - lavalamp + - sttts + - tallclair + + sig-auth-authenticators-approvers: + - deads2k + - liggitt + - mikedanese + sig-auth-authenticators-reviewers: + - deads2k + - enj + - jianhuiz + - lavalamp + - liggitt + - mbohlool + - mikedanese + - sttts + - wojtek-t + + sig-auth-authorizers-approvers: + - deads2k + - liggitt + - mikedanese + sig-auth-authorizers-reviewers: + - david-mcmahon + - deads2k + - dims + - enj + - erictune + - jianhuiz + - krousey + - lavalamp + - liggitt + - mbohlool + - mikedanese + - mml + - ncdc + - nikhiljindal + - smarterclayton + - sttts + - thockin + - wojtek-t + + sig-auth-certificates-approvers: + - liggitt + - mikedanese + - smarterclayton + sig-auth-certificates-reviewers: + - awly + - caesarxuchao + - david-mcmahon + - deads2k + - dims + - enj + - errordeveloper + - hongchaodeng + - jianhuiz + - lavalamp + - liggitt + - mbohlool + - mikedanese + - smarterclayton + - sttts + - thockin + - timothysc + - wojtek-t + + sig-auth-encryption-at-rest-approvers: + - immutableT + - smarterclayton + sig-auth-encryption-at-rest-reviewers: + - enj + - immutableT + - lavalamp + - liggitt + - sakshamsharma + - smarterclayton + - wojtek-t + + sig-auth-node-isolation-approvers: + - deads2k + - liggitt + - mikedanese + - tallclair + sig-auth-node-isolation-reviewers: + - deads2k + - liggitt + - mikedanese + - tallclair + + sig-auth-policy-approvers: + - deads2k + - liggitt + - tallclair + sig-auth-policy-reviewers: + - deads2k + - hongchaodeng + - jianhuiz + - liggitt + - mbohlool + - pweil- + - tallclair + + sig-auth-serviceaccounts-approvers: + - deads2k + - liggitt + - mikedanese + sig-auth-serviceaccounts-reviewers: + - awly + - deads2k + - enj + - liggitt + - mikedanese + sig-storage-reviewers: - saad-ali - childsb diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index 6e1f831554f..4fae72c7a94 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -4,6 +4,7 @@ API rule violation: names_match,k8s.io/api/core/v1,ContainerStatus,LastTerminati API rule violation: names_match,k8s.io/api/core/v1,DaemonEndpoint,Port API rule violation: names_match,k8s.io/api/core/v1,Event,ReportingController API rule violation: names_match,k8s.io/api/core/v1,FCVolumeSource,WWIDs +API rule violation: names_match,k8s.io/api/core/v1,GlusterfsPersistentVolumeSource,EndpointsName API rule violation: names_match,k8s.io/api/core/v1,GlusterfsVolumeSource,EndpointsName API rule violation: names_match,k8s.io/api/core/v1,ISCSIPersistentVolumeSource,DiscoveryCHAPAuth API rule violation: names_match,k8s.io/api/core/v1,ISCSIPersistentVolumeSource,SessionCHAPAuth diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 8d24ae47f30..ed880538214 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -2,7 +2,7 @@ "swagger": "2.0", "info": { "title": "Kubernetes", - "version": "v1.13.0" + "version": "v1.14.0" }, "paths": { "/api/": { @@ -74409,6 +74409,688 @@ } ] }, + "/apis/storage.k8s.io/v1/volumeattachments": { + "get": { + "description": "list or watch objects of kind VolumeAttachment", + "consumes": [ + "*/*" + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf", + "application/json;stream=watch", + "application/vnd.kubernetes.protobuf;stream=watch" + ], + "schemes": [ + "https" + ], + "tags": [ + "storage_v1" + ], + "operationId": "listStorageV1VolumeAttachment", + "parameters": [ + { + "uniqueItems": true, + "type": "string", + "description": "The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the \"next key\".\n\nThis field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.", + "name": "continue", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "A selector to restrict the list of returned objects by their fields. Defaults to everything.", + "name": "fieldSelector", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "A selector to restrict the list of returned objects by their labels. Defaults to everything.", + "name": "labelSelector", + "in": "query" + }, + { + "uniqueItems": true, + "type": "integer", + "description": "limit is a maximum number of responses to return for a list call. If more items exist, the server will set the `continue` field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true.\n\nThe server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.", + "name": "limit", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "When specified with a watch call, shows changes that occur after that particular version of a resource. Defaults to changes from the beginning of history. When specified for list: - if unset, then the result is returned from remote storage based on quorum-read flag; - if it's 0, then we simply return what we currently have in cache, no guarantee; - if set to non zero, then the result is at least as fresh as given rv.", + "name": "resourceVersion", + "in": "query" + }, + { + "uniqueItems": true, + "type": "integer", + "description": "Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.", + "name": "timeoutSeconds", + "in": "query" + }, + { + "uniqueItems": true, + "type": "boolean", + "description": "Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.", + "name": "watch", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachmentList" + } + }, + "401": { + "description": "Unauthorized" + } + }, + "x-kubernetes-action": "list", + "x-kubernetes-group-version-kind": { + "group": "storage.k8s.io", + "kind": "VolumeAttachment", + "version": "v1" + } + }, + "post": { + "description": "create a VolumeAttachment", + "consumes": [ + "*/*" + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "schemes": [ + "https" + ], + "tags": [ + "storage_v1" + ], + "operationId": "createStorageV1VolumeAttachment", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + { + "uniqueItems": true, + "type": "string", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "name": "dryRun", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + "202": { + "description": "Accepted", + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + "401": { + "description": "Unauthorized" + } + }, + "x-kubernetes-action": "post", + "x-kubernetes-group-version-kind": { + "group": "storage.k8s.io", + "kind": "VolumeAttachment", + "version": "v1" + } + }, + "delete": { + "description": "delete collection of VolumeAttachment", + "consumes": [ + "*/*" + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "schemes": [ + "https" + ], + "tags": [ + "storage_v1" + ], + "operationId": "deleteStorageV1CollectionVolumeAttachment", + "parameters": [ + { + "uniqueItems": true, + "type": "string", + "description": "The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the \"next key\".\n\nThis field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.", + "name": "continue", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "A selector to restrict the list of returned objects by their fields. Defaults to everything.", + "name": "fieldSelector", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "A selector to restrict the list of returned objects by their labels. Defaults to everything.", + "name": "labelSelector", + "in": "query" + }, + { + "uniqueItems": true, + "type": "integer", + "description": "limit is a maximum number of responses to return for a list call. If more items exist, the server will set the `continue` field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true.\n\nThe server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.", + "name": "limit", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "When specified with a watch call, shows changes that occur after that particular version of a resource. Defaults to changes from the beginning of history. When specified for list: - if unset, then the result is returned from remote storage based on quorum-read flag; - if it's 0, then we simply return what we currently have in cache, no guarantee; - if set to non zero, then the result is at least as fresh as given rv.", + "name": "resourceVersion", + "in": "query" + }, + { + "uniqueItems": true, + "type": "integer", + "description": "Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.", + "name": "timeoutSeconds", + "in": "query" + }, + { + "uniqueItems": true, + "type": "boolean", + "description": "Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.", + "name": "watch", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Status" + } + }, + "401": { + "description": "Unauthorized" + } + }, + "x-kubernetes-action": "deletecollection", + "x-kubernetes-group-version-kind": { + "group": "storage.k8s.io", + "kind": "VolumeAttachment", + "version": "v1" + } + }, + "parameters": [ + { + "uniqueItems": true, + "type": "boolean", + "description": "If true, partially initialized resources are included in the response.", + "name": "includeUninitialized", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "If 'true', then the output is pretty printed.", + "name": "pretty", + "in": "query" + } + ] + }, + "/apis/storage.k8s.io/v1/volumeattachments/{name}": { + "get": { + "description": "read the specified VolumeAttachment", + "consumes": [ + "*/*" + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "schemes": [ + "https" + ], + "tags": [ + "storage_v1" + ], + "operationId": "readStorageV1VolumeAttachment", + "parameters": [ + { + "uniqueItems": true, + "type": "boolean", + "description": "Should the export be exact. Exact export maintains cluster-specific fields like 'Namespace'.", + "name": "exact", + "in": "query" + }, + { + "uniqueItems": true, + "type": "boolean", + "description": "Should this value be exported. Export strips fields that a user can not specify.", + "name": "export", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + "401": { + "description": "Unauthorized" + } + }, + "x-kubernetes-action": "get", + "x-kubernetes-group-version-kind": { + "group": "storage.k8s.io", + "kind": "VolumeAttachment", + "version": "v1" + } + }, + "put": { + "description": "replace the specified VolumeAttachment", + "consumes": [ + "*/*" + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "schemes": [ + "https" + ], + "tags": [ + "storage_v1" + ], + "operationId": "replaceStorageV1VolumeAttachment", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + { + "uniqueItems": true, + "type": "string", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "name": "dryRun", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + "401": { + "description": "Unauthorized" + } + }, + "x-kubernetes-action": "put", + "x-kubernetes-group-version-kind": { + "group": "storage.k8s.io", + "kind": "VolumeAttachment", + "version": "v1" + } + }, + "delete": { + "description": "delete a VolumeAttachment", + "consumes": [ + "*/*" + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "schemes": [ + "https" + ], + "tags": [ + "storage_v1" + ], + "operationId": "deleteStorageV1VolumeAttachment", + "parameters": [ + { + "name": "body", + "in": "body", + "schema": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.DeleteOptions" + } + }, + { + "uniqueItems": true, + "type": "string", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "name": "dryRun", + "in": "query" + }, + { + "uniqueItems": true, + "type": "integer", + "description": "The duration in seconds before the object should be deleted. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period for the specified type will be used. Defaults to a per object value if not specified. zero means delete immediately.", + "name": "gracePeriodSeconds", + "in": "query" + }, + { + "uniqueItems": true, + "type": "boolean", + "description": "Deprecated: please use the PropagationPolicy, this field will be deprecated in 1.7. Should the dependent objects be orphaned. If true/false, the \"orphan\" finalizer will be added to/removed from the object's finalizers list. Either this field or PropagationPolicy may be set, but not both.", + "name": "orphanDependents", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "Whether and how garbage collection will be performed. Either this field or OrphanDependents may be set, but not both. The default policy is decided by the existing finalizer set in the metadata.finalizers and the resource-specific default policy. Acceptable values are: 'Orphan' - orphan the dependents; 'Background' - allow the garbage collector to delete the dependents in the background; 'Foreground' - a cascading policy that deletes all dependents in the foreground.", + "name": "propagationPolicy", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Status" + } + }, + "202": { + "description": "Accepted", + "schema": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Status" + } + }, + "401": { + "description": "Unauthorized" + } + }, + "x-kubernetes-action": "delete", + "x-kubernetes-group-version-kind": { + "group": "storage.k8s.io", + "kind": "VolumeAttachment", + "version": "v1" + } + }, + "patch": { + "description": "partially update the specified VolumeAttachment", + "consumes": [ + "application/json-patch+json", + "application/merge-patch+json", + "application/strategic-merge-patch+json" + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "schemes": [ + "https" + ], + "tags": [ + "storage_v1" + ], + "operationId": "patchStorageV1VolumeAttachment", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Patch" + } + }, + { + "uniqueItems": true, + "type": "string", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "name": "dryRun", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + "401": { + "description": "Unauthorized" + } + }, + "x-kubernetes-action": "patch", + "x-kubernetes-group-version-kind": { + "group": "storage.k8s.io", + "kind": "VolumeAttachment", + "version": "v1" + } + }, + "parameters": [ + { + "uniqueItems": true, + "type": "string", + "description": "name of the VolumeAttachment", + "name": "name", + "in": "path", + "required": true + }, + { + "uniqueItems": true, + "type": "string", + "description": "If 'true', then the output is pretty printed.", + "name": "pretty", + "in": "query" + } + ] + }, + "/apis/storage.k8s.io/v1/volumeattachments/{name}/status": { + "get": { + "description": "read status of the specified VolumeAttachment", + "consumes": [ + "*/*" + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "schemes": [ + "https" + ], + "tags": [ + "storage_v1" + ], + "operationId": "readStorageV1VolumeAttachmentStatus", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + "401": { + "description": "Unauthorized" + } + }, + "x-kubernetes-action": "get", + "x-kubernetes-group-version-kind": { + "group": "storage.k8s.io", + "kind": "VolumeAttachment", + "version": "v1" + } + }, + "put": { + "description": "replace status of the specified VolumeAttachment", + "consumes": [ + "*/*" + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "schemes": [ + "https" + ], + "tags": [ + "storage_v1" + ], + "operationId": "replaceStorageV1VolumeAttachmentStatus", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + { + "uniqueItems": true, + "type": "string", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "name": "dryRun", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + "401": { + "description": "Unauthorized" + } + }, + "x-kubernetes-action": "put", + "x-kubernetes-group-version-kind": { + "group": "storage.k8s.io", + "kind": "VolumeAttachment", + "version": "v1" + } + }, + "patch": { + "description": "partially update status of the specified VolumeAttachment", + "consumes": [ + "application/json-patch+json", + "application/merge-patch+json", + "application/strategic-merge-patch+json" + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "schemes": [ + "https" + ], + "tags": [ + "storage_v1" + ], + "operationId": "patchStorageV1VolumeAttachmentStatus", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Patch" + } + }, + { + "uniqueItems": true, + "type": "string", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "name": "dryRun", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + "401": { + "description": "Unauthorized" + } + }, + "x-kubernetes-action": "patch", + "x-kubernetes-group-version-kind": { + "group": "storage.k8s.io", + "kind": "VolumeAttachment", + "version": "v1" + } + }, + "parameters": [ + { + "uniqueItems": true, + "type": "string", + "description": "name of the VolumeAttachment", + "name": "name", + "in": "path", + "required": true + }, + { + "uniqueItems": true, + "type": "string", + "description": "If 'true', then the output is pretty printed.", + "name": "pretty", + "in": "query" + } + ] + }, "/apis/storage.k8s.io/v1/watch/storageclasses": { "get": { "description": "watch individual changes to a list of StorageClass. deprecated: use the 'watch' parameter with a list operation instead.", @@ -74625,6 +75307,222 @@ } ] }, + "/apis/storage.k8s.io/v1/watch/volumeattachments": { + "get": { + "description": "watch individual changes to a list of VolumeAttachment. deprecated: use the 'watch' parameter with a list operation instead.", + "consumes": [ + "*/*" + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf", + "application/json;stream=watch", + "application/vnd.kubernetes.protobuf;stream=watch" + ], + "schemes": [ + "https" + ], + "tags": [ + "storage_v1" + ], + "operationId": "watchStorageV1VolumeAttachmentList", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.WatchEvent" + } + }, + "401": { + "description": "Unauthorized" + } + }, + "x-kubernetes-action": "watchlist", + "x-kubernetes-group-version-kind": { + "group": "storage.k8s.io", + "kind": "VolumeAttachment", + "version": "v1" + } + }, + "parameters": [ + { + "uniqueItems": true, + "type": "string", + "description": "The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the \"next key\".\n\nThis field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.", + "name": "continue", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "A selector to restrict the list of returned objects by their fields. Defaults to everything.", + "name": "fieldSelector", + "in": "query" + }, + { + "uniqueItems": true, + "type": "boolean", + "description": "If true, partially initialized resources are included in the response.", + "name": "includeUninitialized", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "A selector to restrict the list of returned objects by their labels. Defaults to everything.", + "name": "labelSelector", + "in": "query" + }, + { + "uniqueItems": true, + "type": "integer", + "description": "limit is a maximum number of responses to return for a list call. If more items exist, the server will set the `continue` field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true.\n\nThe server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.", + "name": "limit", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "If 'true', then the output is pretty printed.", + "name": "pretty", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "When specified with a watch call, shows changes that occur after that particular version of a resource. Defaults to changes from the beginning of history. When specified for list: - if unset, then the result is returned from remote storage based on quorum-read flag; - if it's 0, then we simply return what we currently have in cache, no guarantee; - if set to non zero, then the result is at least as fresh as given rv.", + "name": "resourceVersion", + "in": "query" + }, + { + "uniqueItems": true, + "type": "integer", + "description": "Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.", + "name": "timeoutSeconds", + "in": "query" + }, + { + "uniqueItems": true, + "type": "boolean", + "description": "Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.", + "name": "watch", + "in": "query" + } + ] + }, + "/apis/storage.k8s.io/v1/watch/volumeattachments/{name}": { + "get": { + "description": "watch changes to an object of kind VolumeAttachment. deprecated: use the 'watch' parameter with a list operation instead, filtered to a single item with the 'fieldSelector' parameter.", + "consumes": [ + "*/*" + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf", + "application/json;stream=watch", + "application/vnd.kubernetes.protobuf;stream=watch" + ], + "schemes": [ + "https" + ], + "tags": [ + "storage_v1" + ], + "operationId": "watchStorageV1VolumeAttachment", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.WatchEvent" + } + }, + "401": { + "description": "Unauthorized" + } + }, + "x-kubernetes-action": "watch", + "x-kubernetes-group-version-kind": { + "group": "storage.k8s.io", + "kind": "VolumeAttachment", + "version": "v1" + } + }, + "parameters": [ + { + "uniqueItems": true, + "type": "string", + "description": "The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the \"next key\".\n\nThis field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.", + "name": "continue", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "A selector to restrict the list of returned objects by their fields. Defaults to everything.", + "name": "fieldSelector", + "in": "query" + }, + { + "uniqueItems": true, + "type": "boolean", + "description": "If true, partially initialized resources are included in the response.", + "name": "includeUninitialized", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "A selector to restrict the list of returned objects by their labels. Defaults to everything.", + "name": "labelSelector", + "in": "query" + }, + { + "uniqueItems": true, + "type": "integer", + "description": "limit is a maximum number of responses to return for a list call. If more items exist, the server will set the `continue` field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true.\n\nThe server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.", + "name": "limit", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "name of the VolumeAttachment", + "name": "name", + "in": "path", + "required": true + }, + { + "uniqueItems": true, + "type": "string", + "description": "If 'true', then the output is pretty printed.", + "name": "pretty", + "in": "query" + }, + { + "uniqueItems": true, + "type": "string", + "description": "When specified with a watch call, shows changes that occur after that particular version of a resource. Defaults to changes from the beginning of history. When specified for list: - if unset, then the result is returned from remote storage based on quorum-read flag; - if it's 0, then we simply return what we currently have in cache, no guarantee; - if set to non zero, then the result is at least as fresh as given rv.", + "name": "resourceVersion", + "in": "query" + }, + { + "uniqueItems": true, + "type": "integer", + "description": "Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.", + "name": "timeoutSeconds", + "in": "query" + }, + { + "uniqueItems": true, + "type": "boolean", + "description": "Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.", + "name": "watch", + "in": "query" + } + ] + }, "/apis/storage.k8s.io/v1alpha1/": { "get": { "description": "get available resources", @@ -79973,6 +80871,13 @@ "io.k8s.api.authentication.v1.TokenReviewSpec": { "description": "TokenReviewSpec is a description of the token authentication request.", "properties": { + "audiences": { + "description": "Audiences is a list of the identifiers that the resource server presented with the token identifies as. Audience-aware token authenticators will verify that the token was intended for at least one of the audiences in this list. If no audiences are provided, the audience will default to the audience of the Kubernetes apiserver.", + "type": "array", + "items": { + "type": "string" + } + }, "token": { "description": "Token is the opaque bearer token.", "type": "string" @@ -79982,6 +80887,13 @@ "io.k8s.api.authentication.v1.TokenReviewStatus": { "description": "TokenReviewStatus is the result of the token authentication request.", "properties": { + "audiences": { + "description": "Audiences are audience identifiers chosen by the authenticator that are compatible with both the TokenReview and token. An identifier is any identifier in the intersection of the TokenReviewSpec audiences and the token's audiences. A client of the TokenReview API that sets the spec.audiences field should validate that a compatible audience identifier is returned in the status.audiences field to ensure that the TokenReview server is audience aware. If a TokenReview returns an empty status.audience field where status.authenticated is \"true\", the token is valid against the audience of the Kubernetes API server.", + "type": "array", + "items": { + "type": "string" + } + }, "authenticated": { "description": "Authenticated indicates that the token was associated with a known user.", "type": "boolean" @@ -80063,6 +80975,13 @@ "io.k8s.api.authentication.v1beta1.TokenReviewSpec": { "description": "TokenReviewSpec is a description of the token authentication request.", "properties": { + "audiences": { + "description": "Audiences is a list of the identifiers that the resource server presented with the token identifies as. Audience-aware token authenticators will verify that the token was intended for at least one of the audiences in this list. If no audiences are provided, the audience will default to the audience of the Kubernetes apiserver.", + "type": "array", + "items": { + "type": "string" + } + }, "token": { "description": "Token is the opaque bearer token.", "type": "string" @@ -80072,6 +80991,13 @@ "io.k8s.api.authentication.v1beta1.TokenReviewStatus": { "description": "TokenReviewStatus is the result of the token authentication request.", "properties": { + "audiences": { + "description": "Audiences are audience identifiers chosen by the authenticator that are compatible with both the TokenReview and token. An identifier is any identifier in the intersection of the TokenReviewSpec audiences and the token's audiences. A client of the TokenReview API that sets the spec.audiences field should validate that a compatible audience identifier is returned in the status.audiences field to ensure that the TokenReview server is audience aware. If a TokenReview returns an empty status.audience field where status.authenticated is \"true\", the token is valid against the audience of the Kubernetes API server.", + "type": "array", + "items": { + "type": "string" + } + }, "authenticated": { "description": "Authenticated indicates that the token was associated with a known user.", "type": "boolean" @@ -83342,7 +84268,7 @@ "type": "boolean" }, "volumeDevices": { - "description": "volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future.", + "description": "volumeDevices is the list of block devices to be used by the container. This is a beta feature.", "type": "array", "items": { "$ref": "#/definitions/io.k8s.api.core.v1.VolumeDevice" @@ -84125,6 +85051,31 @@ } } }, + "io.k8s.api.core.v1.GlusterfsPersistentVolumeSource": { + "description": "Represents a Glusterfs mount that lasts the lifetime of a pod. Glusterfs volumes do not support ownership management or SELinux relabeling.", + "required": [ + "endpoints", + "path" + ], + "properties": { + "endpoints": { + "description": "EndpointsName is the endpoint name that details Glusterfs topology. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod", + "type": "string" + }, + "endpointsNamespace": { + "description": "EndpointsNamespace is the namespace that contains Glusterfs endpoint. If this field is empty, the EndpointNamespace defaults to the same namespace as the bound PVC. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod", + "type": "string" + }, + "path": { + "description": "Path is the Glusterfs volume path. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod", + "type": "string" + }, + "readOnly": { + "description": "ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod", + "type": "boolean" + } + } + }, "io.k8s.api.core.v1.GlusterfsVolumeSource": { "description": "Represents a Glusterfs mount that lasts the lifetime of a pod. Glusterfs volumes do not support ownership management or SELinux relabeling.", "required": [ @@ -85270,7 +86221,7 @@ "type": "string" }, "volumeMode": { - "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is an alpha feature and may change in the future.", + "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.", "type": "string" }, "volumeName": { @@ -85424,7 +86375,7 @@ }, "glusterfs": { "description": "Glusterfs represents a Glusterfs volume that is attached to a host and exposed to the pod. Provisioned by an admin. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md", - "$ref": "#/definitions/io.k8s.api.core.v1.GlusterfsVolumeSource" + "$ref": "#/definitions/io.k8s.api.core.v1.GlusterfsPersistentVolumeSource" }, "hostPath": { "description": "HostPath represents a directory on the host. Provisioned by a developer or tester. This is useful for single-node development and testing only! On-host storage is not supported in any way and WILL NOT WORK in a multi-node cluster. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", @@ -85486,7 +86437,7 @@ "$ref": "#/definitions/io.k8s.api.core.v1.StorageOSPersistentVolumeSource" }, "volumeMode": { - "description": "volumeMode defines if a volume is intended to be used with a formatted filesystem or to remain in raw block state. Value of Filesystem is implied when not included in spec. This is an alpha feature and may change in the future.", + "description": "volumeMode defines if a volume is intended to be used with a formatted filesystem or to remain in raw block state. Value of Filesystem is implied when not included in spec. This is a beta feature.", "type": "string" }, "vsphereVolume": { @@ -91410,6 +92361,146 @@ } ] }, + "io.k8s.api.storage.v1.VolumeAttachment": { + "description": "VolumeAttachment captures the intent to attach or detach the specified volume to/from the specified node.\n\nVolumeAttachment objects are non-namespaced.", + "required": [ + "spec" + ], + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", + "type": "string" + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", + "type": "string" + }, + "metadata": { + "description": "Standard object metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta" + }, + "spec": { + "description": "Specification of the desired attach/detach volume behavior. Populated by the Kubernetes system.", + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachmentSpec" + }, + "status": { + "description": "Status of the VolumeAttachment request. Populated by the entity completing the attach or detach operation, i.e. the external-attacher.", + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachmentStatus" + } + }, + "x-kubernetes-group-version-kind": [ + { + "group": "storage.k8s.io", + "kind": "VolumeAttachment", + "version": "v1" + } + ] + }, + "io.k8s.api.storage.v1.VolumeAttachmentList": { + "description": "VolumeAttachmentList is a collection of VolumeAttachment objects.", + "required": [ + "items" + ], + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", + "type": "string" + }, + "items": { + "description": "Items is the list of VolumeAttachments", + "type": "array", + "items": { + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachment" + } + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", + "type": "string" + }, + "metadata": { + "description": "Standard list metadata More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ListMeta" + } + }, + "x-kubernetes-group-version-kind": [ + { + "group": "storage.k8s.io", + "kind": "VolumeAttachmentList", + "version": "v1" + } + ] + }, + "io.k8s.api.storage.v1.VolumeAttachmentSource": { + "description": "VolumeAttachmentSource represents a volume that should be attached. Right now only PersistenVolumes can be attached via external attacher, in future we may allow also inline volumes in pods. Exactly one member can be set.", + "properties": { + "persistentVolumeName": { + "description": "Name of the persistent volume to attach.", + "type": "string" + } + } + }, + "io.k8s.api.storage.v1.VolumeAttachmentSpec": { + "description": "VolumeAttachmentSpec is the specification of a VolumeAttachment request.", + "required": [ + "attacher", + "source", + "nodeName" + ], + "properties": { + "attacher": { + "description": "Attacher indicates the name of the volume driver that MUST handle this request. This is the name returned by GetPluginName().", + "type": "string" + }, + "nodeName": { + "description": "The node that the volume should be attached to.", + "type": "string" + }, + "source": { + "description": "Source represents the volume that should be attached.", + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeAttachmentSource" + } + } + }, + "io.k8s.api.storage.v1.VolumeAttachmentStatus": { + "description": "VolumeAttachmentStatus is the status of a VolumeAttachment request.", + "required": [ + "attached" + ], + "properties": { + "attachError": { + "description": "The last error encountered during attach operation, if any. This field must only be set by the entity completing the attach operation, i.e. the external-attacher.", + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeError" + }, + "attached": { + "description": "Indicates the volume is successfully attached. This field must only be set by the entity completing the attach operation, i.e. the external-attacher.", + "type": "boolean" + }, + "attachmentMetadata": { + "description": "Upon successful attach, this field is populated with any information returned by the attach operation that must be passed into subsequent WaitForAttach or Mount calls. This field must only be set by the entity completing the attach operation, i.e. the external-attacher.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "detachError": { + "description": "The last error encountered during detach operation, if any. This field must only be set by the entity completing the detach operation, i.e. the external-attacher.", + "$ref": "#/definitions/io.k8s.api.storage.v1.VolumeError" + } + } + }, + "io.k8s.api.storage.v1.VolumeError": { + "description": "VolumeError captures an error encountered during a volume operation.", + "properties": { + "message": { + "description": "String detailing the error encountered during Attach or Detach operation. This string maybe logged, so it should not contain sensitive information.", + "type": "string" + }, + "time": { + "description": "Time the error was encountered.", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Time" + } + } + }, "io.k8s.api.storage.v1alpha1.VolumeAttachment": { "description": "VolumeAttachment captures the intent to attach or detach the specified volume to/from the specified node.\n\nVolumeAttachment objects are non-namespaced.", "required": [ @@ -91983,7 +93074,7 @@ ], "properties": { "additionalPrinterColumns": { - "description": "AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column.", + "description": "AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column. Optional, the global columns for all versions. Top-level and per-version columns are mutually exclusive.", "type": "array", "items": { "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceColumnDefinition" @@ -92006,11 +93097,11 @@ "type": "string" }, "subresources": { - "description": "Subresources describes the subresources for CustomResources", + "description": "Subresources describes the subresources for CustomResource Optional, the global subresources for all versions. Top-level and per-version subresources are mutually exclusive.", "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceSubresources" }, "validation": { - "description": "Validation describes the validation methods for CustomResources", + "description": "Validation describes the validation methods for CustomResources Optional, the global validation schema for all versions. Top-level and per-version schemas are mutually exclusive.", "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceValidation" }, "version": { @@ -92062,10 +93153,21 @@ "storage" ], "properties": { + "additionalPrinterColumns": { + "description": "AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column. Top-level and per-version columns are mutually exclusive. Per-version columns must not all be set to identical values (top-level columns should be used instead) This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature. NOTE: CRDs created prior to 1.13 populated the top-level additionalPrinterColumns field by default. To apply an update that changes to per-version additionalPrinterColumns, the top-level additionalPrinterColumns field must be explicitly set to null", + "type": "array", + "items": { + "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceColumnDefinition" + } + }, "name": { "description": "Name is the version name, e.g. “v1”, “v2beta1”, etc.", "type": "string" }, + "schema": { + "description": "Schema describes the schema for CustomResource used in validation, pruning, and defaulting. Top-level and per-version schemas are mutually exclusive. Per-version schemas must not all be set to identical values (top-level validation schema should be used instead) This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature.", + "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceValidation" + }, "served": { "description": "Served is a flag enabling/disabling this version from being served via REST APIs", "type": "boolean" @@ -92073,6 +93175,10 @@ "storage": { "description": "Storage flags the version as storage version. There must be exactly one flagged as storage version.", "type": "boolean" + }, + "subresources": { + "description": "Subresources describes the subresources for CustomResource Top-level and per-version subresources are mutually exclusive. Per-version subresources must not all be set to identical values (top-level subresources should be used instead) This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature.", + "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceSubresources" } } }, diff --git a/api/swagger-spec/apps_v1.json b/api/swagger-spec/apps_v1.json index e1f00418afa..50ddd86ef75 100644 --- a/api/swagger-spec/apps_v1.json +++ b/api/swagger-spec/apps_v1.json @@ -8227,7 +8227,7 @@ "items": { "$ref": "v1.VolumeDevice" }, - "description": "volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future." + "description": "volumeDevices is the list of block devices to be used by the container. This is a beta feature." }, "livenessProbe": { "$ref": "v1.Probe", @@ -9796,7 +9796,7 @@ }, "volumeMode": { "$ref": "v1.PersistentVolumeMode", - "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is an alpha feature and may change in the future." + "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature." }, "dataSource": { "$ref": "v1.TypedLocalObjectReference", diff --git a/api/swagger-spec/apps_v1beta1.json b/api/swagger-spec/apps_v1beta1.json index 78bc79c312e..d829551f17f 100644 --- a/api/swagger-spec/apps_v1beta1.json +++ b/api/swagger-spec/apps_v1beta1.json @@ -5739,7 +5739,7 @@ "items": { "$ref": "v1.VolumeDevice" }, - "description": "volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future." + "description": "volumeDevices is the list of block devices to be used by the container. This is a beta feature." }, "livenessProbe": { "$ref": "v1.Probe", @@ -6982,7 +6982,7 @@ }, "volumeMode": { "$ref": "v1.PersistentVolumeMode", - "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is an alpha feature and may change in the future." + "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature." }, "dataSource": { "$ref": "v1.TypedLocalObjectReference", diff --git a/api/swagger-spec/apps_v1beta2.json b/api/swagger-spec/apps_v1beta2.json index c120736f433..c20e4236d6e 100644 --- a/api/swagger-spec/apps_v1beta2.json +++ b/api/swagger-spec/apps_v1beta2.json @@ -8227,7 +8227,7 @@ "items": { "$ref": "v1.VolumeDevice" }, - "description": "volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future." + "description": "volumeDevices is the list of block devices to be used by the container. This is a beta feature." }, "livenessProbe": { "$ref": "v1.Probe", @@ -9800,7 +9800,7 @@ }, "volumeMode": { "$ref": "v1.PersistentVolumeMode", - "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is an alpha feature and may change in the future." + "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature." }, "dataSource": { "$ref": "v1.TypedLocalObjectReference", diff --git a/api/swagger-spec/authentication.k8s.io_v1.json b/api/swagger-spec/authentication.k8s.io_v1.json index 74e689e0cdc..efdcb2f2263 100644 --- a/api/swagger-spec/authentication.k8s.io_v1.json +++ b/api/swagger-spec/authentication.k8s.io_v1.json @@ -395,6 +395,13 @@ "token": { "type": "string", "description": "Token is the opaque bearer token." + }, + "audiences": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Audiences is a list of the identifiers that the resource server presented with the token identifies as. Audience-aware token authenticators will verify that the token was intended for at least one of the audiences in this list. If no audiences are provided, the audience will default to the audience of the Kubernetes apiserver." } } }, @@ -410,6 +417,13 @@ "$ref": "v1.UserInfo", "description": "User is the UserInfo associated with the provided token." }, + "audiences": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Audiences are audience identifiers chosen by the authenticator that are compatible with both the TokenReview and token. An identifier is any identifier in the intersection of the TokenReviewSpec audiences and the token's audiences. A client of the TokenReview API that sets the spec.audiences field should validate that a compatible audience identifier is returned in the status.audiences field to ensure that the TokenReview server is audience aware. If a TokenReview returns an empty status.audience field where status.authenticated is \"true\", the token is valid against the audience of the Kubernetes API server." + }, "error": { "type": "string", "description": "Error indicates that the token couldn't be checked" diff --git a/api/swagger-spec/authentication.k8s.io_v1beta1.json b/api/swagger-spec/authentication.k8s.io_v1beta1.json index ab5cf7822dc..a1d29689f5a 100644 --- a/api/swagger-spec/authentication.k8s.io_v1beta1.json +++ b/api/swagger-spec/authentication.k8s.io_v1beta1.json @@ -395,6 +395,13 @@ "token": { "type": "string", "description": "Token is the opaque bearer token." + }, + "audiences": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Audiences is a list of the identifiers that the resource server presented with the token identifies as. Audience-aware token authenticators will verify that the token was intended for at least one of the audiences in this list. If no audiences are provided, the audience will default to the audience of the Kubernetes apiserver." } } }, @@ -410,6 +417,13 @@ "$ref": "v1beta1.UserInfo", "description": "User is the UserInfo associated with the provided token." }, + "audiences": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Audiences are audience identifiers chosen by the authenticator that are compatible with both the TokenReview and token. An identifier is any identifier in the intersection of the TokenReviewSpec audiences and the token's audiences. A client of the TokenReview API that sets the spec.audiences field should validate that a compatible audience identifier is returned in the status.audiences field to ensure that the TokenReview server is audience aware. If a TokenReview returns an empty status.audience field where status.authenticated is \"true\", the token is valid against the audience of the Kubernetes API server." + }, "error": { "type": "string", "description": "Error indicates that the token couldn't be checked" diff --git a/api/swagger-spec/batch_v1.json b/api/swagger-spec/batch_v1.json index d42ab3d3d81..b28f65f494b 100644 --- a/api/swagger-spec/batch_v1.json +++ b/api/swagger-spec/batch_v1.json @@ -2923,7 +2923,7 @@ "items": { "$ref": "v1.VolumeDevice" }, - "description": "volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future." + "description": "volumeDevices is the list of block devices to be used by the container. This is a beta feature." }, "livenessProbe": { "$ref": "v1.Probe", diff --git a/api/swagger-spec/batch_v1beta1.json b/api/swagger-spec/batch_v1beta1.json index dcf1a670d31..e40a742cda3 100644 --- a/api/swagger-spec/batch_v1beta1.json +++ b/api/swagger-spec/batch_v1beta1.json @@ -2978,7 +2978,7 @@ "items": { "$ref": "v1.VolumeDevice" }, - "description": "volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future." + "description": "volumeDevices is the list of block devices to be used by the container. This is a beta feature." }, "livenessProbe": { "$ref": "v1.Probe", diff --git a/api/swagger-spec/batch_v2alpha1.json b/api/swagger-spec/batch_v2alpha1.json index ec644ae9495..3e3f32a5c54 100644 --- a/api/swagger-spec/batch_v2alpha1.json +++ b/api/swagger-spec/batch_v2alpha1.json @@ -2978,7 +2978,7 @@ "items": { "$ref": "v1.VolumeDevice" }, - "description": "volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future." + "description": "volumeDevices is the list of block devices to be used by the container. This is a beta feature." }, "livenessProbe": { "$ref": "v1.Probe", diff --git a/api/swagger-spec/extensions_v1beta1.json b/api/swagger-spec/extensions_v1beta1.json index 00e23d5fb12..9b42ba5de56 100644 --- a/api/swagger-spec/extensions_v1beta1.json +++ b/api/swagger-spec/extensions_v1beta1.json @@ -8923,7 +8923,7 @@ "items": { "$ref": "v1.VolumeDevice" }, - "description": "volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future." + "description": "volumeDevices is the list of block devices to be used by the container. This is a beta feature." }, "livenessProbe": { "$ref": "v1.Probe", diff --git a/api/swagger-spec/storage.k8s.io_v1.json b/api/swagger-spec/storage.k8s.io_v1.json index 265b173984f..23c9f051d75 100644 --- a/api/swagger-spec/storage.k8s.io_v1.json +++ b/api/swagger-spec/storage.k8s.io_v1.json @@ -747,6 +747,907 @@ } ] }, + { + "path": "/apis/storage.k8s.io/v1/volumeattachments", + "description": "API at /apis/storage.k8s.io/v1", + "operations": [ + { + "type": "v1.VolumeAttachmentList", + "method": "GET", + "summary": "list or watch objects of kind VolumeAttachment", + "nickname": "listVolumeAttachment", + "parameters": [ + { + "type": "string", + "paramType": "query", + "name": "pretty", + "description": "If 'true', then the output is pretty printed.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "labelSelector", + "description": "A selector to restrict the list of returned objects by their labels. Defaults to everything.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "fieldSelector", + "description": "A selector to restrict the list of returned objects by their fields. Defaults to everything.", + "required": false, + "allowMultiple": false + }, + { + "type": "boolean", + "paramType": "query", + "name": "includeUninitialized", + "description": "If true, partially initialized resources are included in the response.", + "required": false, + "allowMultiple": false + }, + { + "type": "boolean", + "paramType": "query", + "name": "watch", + "description": "Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "resourceVersion", + "description": "When specified with a watch call, shows changes that occur after that particular version of a resource. Defaults to changes from the beginning of history. When specified for list: - if unset, then the result is returned from remote storage based on quorum-read flag; - if it's 0, then we simply return what we currently have in cache, no guarantee; - if set to non zero, then the result is at least as fresh as given rv.", + "required": false, + "allowMultiple": false + }, + { + "type": "integer", + "paramType": "query", + "name": "timeoutSeconds", + "description": "Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.", + "required": false, + "allowMultiple": false + }, + { + "type": "integer", + "paramType": "query", + "name": "limit", + "description": "limit is a maximum number of responses to return for a list call. If more items exist, the server will set the `continue` field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true.\n\nThe server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "continue", + "description": "The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the \"next key\".\n\nThis field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.", + "required": false, + "allowMultiple": false + } + ], + "responseMessages": [ + { + "code": 200, + "message": "OK", + "responseModel": "v1.VolumeAttachmentList" + } + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf", + "application/json;stream=watch", + "application/vnd.kubernetes.protobuf;stream=watch" + ], + "consumes": [ + "*/*" + ] + }, + { + "type": "v1.VolumeAttachment", + "method": "POST", + "summary": "create a VolumeAttachment", + "nickname": "createVolumeAttachment", + "parameters": [ + { + "type": "string", + "paramType": "query", + "name": "pretty", + "description": "If 'true', then the output is pretty printed.", + "required": false, + "allowMultiple": false + }, + { + "type": "v1.VolumeAttachment", + "paramType": "body", + "name": "body", + "description": "", + "required": true, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "dryRun", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "required": false, + "allowMultiple": false + }, + { + "type": "boolean", + "paramType": "query", + "name": "includeUninitialized", + "description": "If IncludeUninitialized is specified, the object may be returned without completing initialization.", + "required": false, + "allowMultiple": false + } + ], + "responseMessages": [ + { + "code": 200, + "message": "OK", + "responseModel": "v1.VolumeAttachment" + }, + { + "code": 201, + "message": "Created", + "responseModel": "v1.VolumeAttachment" + }, + { + "code": 202, + "message": "Accepted", + "responseModel": "v1.VolumeAttachment" + } + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "consumes": [ + "*/*" + ] + }, + { + "type": "v1.Status", + "method": "DELETE", + "summary": "delete collection of VolumeAttachment", + "nickname": "deletecollectionVolumeAttachment", + "parameters": [ + { + "type": "string", + "paramType": "query", + "name": "pretty", + "description": "If 'true', then the output is pretty printed.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "labelSelector", + "description": "A selector to restrict the list of returned objects by their labels. Defaults to everything.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "fieldSelector", + "description": "A selector to restrict the list of returned objects by their fields. Defaults to everything.", + "required": false, + "allowMultiple": false + }, + { + "type": "boolean", + "paramType": "query", + "name": "includeUninitialized", + "description": "If true, partially initialized resources are included in the response.", + "required": false, + "allowMultiple": false + }, + { + "type": "boolean", + "paramType": "query", + "name": "watch", + "description": "Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "resourceVersion", + "description": "When specified with a watch call, shows changes that occur after that particular version of a resource. Defaults to changes from the beginning of history. When specified for list: - if unset, then the result is returned from remote storage based on quorum-read flag; - if it's 0, then we simply return what we currently have in cache, no guarantee; - if set to non zero, then the result is at least as fresh as given rv.", + "required": false, + "allowMultiple": false + }, + { + "type": "integer", + "paramType": "query", + "name": "timeoutSeconds", + "description": "Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.", + "required": false, + "allowMultiple": false + }, + { + "type": "integer", + "paramType": "query", + "name": "limit", + "description": "limit is a maximum number of responses to return for a list call. If more items exist, the server will set the `continue` field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true.\n\nThe server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "continue", + "description": "The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the \"next key\".\n\nThis field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.", + "required": false, + "allowMultiple": false + } + ], + "responseMessages": [ + { + "code": 200, + "message": "OK", + "responseModel": "v1.Status" + } + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "consumes": [ + "*/*" + ] + } + ] + }, + { + "path": "/apis/storage.k8s.io/v1/watch/volumeattachments", + "description": "API at /apis/storage.k8s.io/v1", + "operations": [ + { + "type": "v1.WatchEvent", + "method": "GET", + "summary": "watch individual changes to a list of VolumeAttachment. deprecated: use the 'watch' parameter with a list operation instead.", + "nickname": "watchVolumeAttachmentList", + "parameters": [ + { + "type": "string", + "paramType": "query", + "name": "pretty", + "description": "If 'true', then the output is pretty printed.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "labelSelector", + "description": "A selector to restrict the list of returned objects by their labels. Defaults to everything.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "fieldSelector", + "description": "A selector to restrict the list of returned objects by their fields. Defaults to everything.", + "required": false, + "allowMultiple": false + }, + { + "type": "boolean", + "paramType": "query", + "name": "includeUninitialized", + "description": "If true, partially initialized resources are included in the response.", + "required": false, + "allowMultiple": false + }, + { + "type": "boolean", + "paramType": "query", + "name": "watch", + "description": "Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "resourceVersion", + "description": "When specified with a watch call, shows changes that occur after that particular version of a resource. Defaults to changes from the beginning of history. When specified for list: - if unset, then the result is returned from remote storage based on quorum-read flag; - if it's 0, then we simply return what we currently have in cache, no guarantee; - if set to non zero, then the result is at least as fresh as given rv.", + "required": false, + "allowMultiple": false + }, + { + "type": "integer", + "paramType": "query", + "name": "timeoutSeconds", + "description": "Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.", + "required": false, + "allowMultiple": false + }, + { + "type": "integer", + "paramType": "query", + "name": "limit", + "description": "limit is a maximum number of responses to return for a list call. If more items exist, the server will set the `continue` field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true.\n\nThe server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "continue", + "description": "The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the \"next key\".\n\nThis field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.", + "required": false, + "allowMultiple": false + } + ], + "responseMessages": [ + { + "code": 200, + "message": "OK", + "responseModel": "v1.WatchEvent" + } + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf", + "application/json;stream=watch", + "application/vnd.kubernetes.protobuf;stream=watch" + ], + "consumes": [ + "*/*" + ] + } + ] + }, + { + "path": "/apis/storage.k8s.io/v1/volumeattachments/{name}", + "description": "API at /apis/storage.k8s.io/v1", + "operations": [ + { + "type": "v1.VolumeAttachment", + "method": "GET", + "summary": "read the specified VolumeAttachment", + "nickname": "readVolumeAttachment", + "parameters": [ + { + "type": "string", + "paramType": "query", + "name": "pretty", + "description": "If 'true', then the output is pretty printed.", + "required": false, + "allowMultiple": false + }, + { + "type": "boolean", + "paramType": "query", + "name": "export", + "description": "Should this value be exported. Export strips fields that a user can not specify.", + "required": false, + "allowMultiple": false + }, + { + "type": "boolean", + "paramType": "query", + "name": "exact", + "description": "Should the export be exact. Exact export maintains cluster-specific fields like 'Namespace'.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "path", + "name": "name", + "description": "name of the VolumeAttachment", + "required": true, + "allowMultiple": false + } + ], + "responseMessages": [ + { + "code": 200, + "message": "OK", + "responseModel": "v1.VolumeAttachment" + } + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "consumes": [ + "*/*" + ] + }, + { + "type": "v1.VolumeAttachment", + "method": "PUT", + "summary": "replace the specified VolumeAttachment", + "nickname": "replaceVolumeAttachment", + "parameters": [ + { + "type": "string", + "paramType": "query", + "name": "pretty", + "description": "If 'true', then the output is pretty printed.", + "required": false, + "allowMultiple": false + }, + { + "type": "v1.VolumeAttachment", + "paramType": "body", + "name": "body", + "description": "", + "required": true, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "dryRun", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "path", + "name": "name", + "description": "name of the VolumeAttachment", + "required": true, + "allowMultiple": false + } + ], + "responseMessages": [ + { + "code": 200, + "message": "OK", + "responseModel": "v1.VolumeAttachment" + }, + { + "code": 201, + "message": "Created", + "responseModel": "v1.VolumeAttachment" + } + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "consumes": [ + "*/*" + ] + }, + { + "type": "v1.VolumeAttachment", + "method": "PATCH", + "summary": "partially update the specified VolumeAttachment", + "nickname": "patchVolumeAttachment", + "parameters": [ + { + "type": "string", + "paramType": "query", + "name": "pretty", + "description": "If 'true', then the output is pretty printed.", + "required": false, + "allowMultiple": false + }, + { + "type": "v1.Patch", + "paramType": "body", + "name": "body", + "description": "", + "required": true, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "dryRun", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "path", + "name": "name", + "description": "name of the VolumeAttachment", + "required": true, + "allowMultiple": false + } + ], + "responseMessages": [ + { + "code": 200, + "message": "OK", + "responseModel": "v1.VolumeAttachment" + } + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "consumes": [ + "application/json-patch+json", + "application/merge-patch+json", + "application/strategic-merge-patch+json" + ] + }, + { + "type": "v1.Status", + "method": "DELETE", + "summary": "delete a VolumeAttachment", + "nickname": "deleteVolumeAttachment", + "parameters": [ + { + "type": "string", + "paramType": "query", + "name": "pretty", + "description": "If 'true', then the output is pretty printed.", + "required": false, + "allowMultiple": false + }, + { + "type": "v1.DeleteOptions", + "paramType": "body", + "name": "body", + "description": "", + "required": false, + "allowMultiple": false + }, + { + "type": "integer", + "paramType": "query", + "name": "gracePeriodSeconds", + "description": "The duration in seconds before the object should be deleted. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period for the specified type will be used. Defaults to a per object value if not specified. zero means delete immediately.", + "required": false, + "allowMultiple": false + }, + { + "type": "boolean", + "paramType": "query", + "name": "orphanDependents", + "description": "Deprecated: please use the PropagationPolicy, this field will be deprecated in 1.7. Should the dependent objects be orphaned. If true/false, the \"orphan\" finalizer will be added to/removed from the object's finalizers list. Either this field or PropagationPolicy may be set, but not both.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "propagationPolicy", + "description": "Whether and how garbage collection will be performed. Either this field or OrphanDependents may be set, but not both. The default policy is decided by the existing finalizer set in the metadata.finalizers and the resource-specific default policy. Acceptable values are: 'Orphan' - orphan the dependents; 'Background' - allow the garbage collector to delete the dependents in the background; 'Foreground' - a cascading policy that deletes all dependents in the foreground.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "dryRun", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "path", + "name": "name", + "description": "name of the VolumeAttachment", + "required": true, + "allowMultiple": false + } + ], + "responseMessages": [ + { + "code": 200, + "message": "OK", + "responseModel": "v1.Status" + }, + { + "code": 202, + "message": "Accepted", + "responseModel": "v1.Status" + } + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "consumes": [ + "*/*" + ] + } + ] + }, + { + "path": "/apis/storage.k8s.io/v1/watch/volumeattachments/{name}", + "description": "API at /apis/storage.k8s.io/v1", + "operations": [ + { + "type": "v1.WatchEvent", + "method": "GET", + "summary": "watch changes to an object of kind VolumeAttachment. deprecated: use the 'watch' parameter with a list operation instead, filtered to a single item with the 'fieldSelector' parameter.", + "nickname": "watchVolumeAttachment", + "parameters": [ + { + "type": "string", + "paramType": "query", + "name": "pretty", + "description": "If 'true', then the output is pretty printed.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "labelSelector", + "description": "A selector to restrict the list of returned objects by their labels. Defaults to everything.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "fieldSelector", + "description": "A selector to restrict the list of returned objects by their fields. Defaults to everything.", + "required": false, + "allowMultiple": false + }, + { + "type": "boolean", + "paramType": "query", + "name": "includeUninitialized", + "description": "If true, partially initialized resources are included in the response.", + "required": false, + "allowMultiple": false + }, + { + "type": "boolean", + "paramType": "query", + "name": "watch", + "description": "Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "resourceVersion", + "description": "When specified with a watch call, shows changes that occur after that particular version of a resource. Defaults to changes from the beginning of history. When specified for list: - if unset, then the result is returned from remote storage based on quorum-read flag; - if it's 0, then we simply return what we currently have in cache, no guarantee; - if set to non zero, then the result is at least as fresh as given rv.", + "required": false, + "allowMultiple": false + }, + { + "type": "integer", + "paramType": "query", + "name": "timeoutSeconds", + "description": "Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.", + "required": false, + "allowMultiple": false + }, + { + "type": "integer", + "paramType": "query", + "name": "limit", + "description": "limit is a maximum number of responses to return for a list call. If more items exist, the server will set the `continue` field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true.\n\nThe server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "continue", + "description": "The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the \"next key\".\n\nThis field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "path", + "name": "name", + "description": "name of the VolumeAttachment", + "required": true, + "allowMultiple": false + } + ], + "responseMessages": [ + { + "code": 200, + "message": "OK", + "responseModel": "v1.WatchEvent" + } + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf", + "application/json;stream=watch", + "application/vnd.kubernetes.protobuf;stream=watch" + ], + "consumes": [ + "*/*" + ] + } + ] + }, + { + "path": "/apis/storage.k8s.io/v1/volumeattachments/{name}/status", + "description": "API at /apis/storage.k8s.io/v1", + "operations": [ + { + "type": "v1.VolumeAttachment", + "method": "GET", + "summary": "read status of the specified VolumeAttachment", + "nickname": "readVolumeAttachmentStatus", + "parameters": [ + { + "type": "string", + "paramType": "query", + "name": "pretty", + "description": "If 'true', then the output is pretty printed.", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "path", + "name": "name", + "description": "name of the VolumeAttachment", + "required": true, + "allowMultiple": false + } + ], + "responseMessages": [ + { + "code": 200, + "message": "OK", + "responseModel": "v1.VolumeAttachment" + } + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "consumes": [ + "*/*" + ] + }, + { + "type": "v1.VolumeAttachment", + "method": "PUT", + "summary": "replace status of the specified VolumeAttachment", + "nickname": "replaceVolumeAttachmentStatus", + "parameters": [ + { + "type": "string", + "paramType": "query", + "name": "pretty", + "description": "If 'true', then the output is pretty printed.", + "required": false, + "allowMultiple": false + }, + { + "type": "v1.VolumeAttachment", + "paramType": "body", + "name": "body", + "description": "", + "required": true, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "dryRun", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "path", + "name": "name", + "description": "name of the VolumeAttachment", + "required": true, + "allowMultiple": false + } + ], + "responseMessages": [ + { + "code": 200, + "message": "OK", + "responseModel": "v1.VolumeAttachment" + }, + { + "code": 201, + "message": "Created", + "responseModel": "v1.VolumeAttachment" + } + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "consumes": [ + "*/*" + ] + }, + { + "type": "v1.VolumeAttachment", + "method": "PATCH", + "summary": "partially update status of the specified VolumeAttachment", + "nickname": "patchVolumeAttachmentStatus", + "parameters": [ + { + "type": "string", + "paramType": "query", + "name": "pretty", + "description": "If 'true', then the output is pretty printed.", + "required": false, + "allowMultiple": false + }, + { + "type": "v1.Patch", + "paramType": "body", + "name": "body", + "description": "", + "required": true, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "query", + "name": "dryRun", + "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + "required": false, + "allowMultiple": false + }, + { + "type": "string", + "paramType": "path", + "name": "name", + "description": "name of the VolumeAttachment", + "required": true, + "allowMultiple": false + } + ], + "responseMessages": [ + { + "code": 200, + "message": "OK", + "responseModel": "v1.VolumeAttachment" + } + ], + "produces": [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf" + ], + "consumes": [ + "application/json-patch+json", + "application/merge-patch+json", + "application/strategic-merge-patch+json" + ] + } + ] + }, { "path": "/apis/storage.k8s.io/v1", "description": "API at /apis/storage.k8s.io/v1", @@ -1229,6 +2130,135 @@ "id": "v1.DeletionPropagation", "properties": {} }, + "v1.VolumeAttachmentList": { + "id": "v1.VolumeAttachmentList", + "description": "VolumeAttachmentList is a collection of VolumeAttachment objects.", + "required": [ + "items" + ], + "properties": { + "kind": { + "type": "string", + "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds" + }, + "apiVersion": { + "type": "string", + "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources" + }, + "metadata": { + "$ref": "v1.ListMeta", + "description": "Standard list metadata More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata" + }, + "items": { + "type": "array", + "items": { + "$ref": "v1.VolumeAttachment" + }, + "description": "Items is the list of VolumeAttachments" + } + } + }, + "v1.VolumeAttachment": { + "id": "v1.VolumeAttachment", + "description": "VolumeAttachment captures the intent to attach or detach the specified volume to/from the specified node.\n\nVolumeAttachment objects are non-namespaced.", + "required": [ + "spec" + ], + "properties": { + "kind": { + "type": "string", + "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds" + }, + "apiVersion": { + "type": "string", + "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources" + }, + "metadata": { + "$ref": "v1.ObjectMeta", + "description": "Standard object metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata" + }, + "spec": { + "$ref": "v1.VolumeAttachmentSpec", + "description": "Specification of the desired attach/detach volume behavior. Populated by the Kubernetes system." + }, + "status": { + "$ref": "v1.VolumeAttachmentStatus", + "description": "Status of the VolumeAttachment request. Populated by the entity completing the attach or detach operation, i.e. the external-attacher." + } + } + }, + "v1.VolumeAttachmentSpec": { + "id": "v1.VolumeAttachmentSpec", + "description": "VolumeAttachmentSpec is the specification of a VolumeAttachment request.", + "required": [ + "attacher", + "source", + "nodeName" + ], + "properties": { + "attacher": { + "type": "string", + "description": "Attacher indicates the name of the volume driver that MUST handle this request. This is the name returned by GetPluginName()." + }, + "source": { + "$ref": "v1.VolumeAttachmentSource", + "description": "Source represents the volume that should be attached." + }, + "nodeName": { + "type": "string", + "description": "The node that the volume should be attached to." + } + } + }, + "v1.VolumeAttachmentSource": { + "id": "v1.VolumeAttachmentSource", + "description": "VolumeAttachmentSource represents a volume that should be attached. Right now only PersistenVolumes can be attached via external attacher, in future we may allow also inline volumes in pods. Exactly one member can be set.", + "properties": { + "persistentVolumeName": { + "type": "string", + "description": "Name of the persistent volume to attach." + } + } + }, + "v1.VolumeAttachmentStatus": { + "id": "v1.VolumeAttachmentStatus", + "description": "VolumeAttachmentStatus is the status of a VolumeAttachment request.", + "required": [ + "attached" + ], + "properties": { + "attached": { + "type": "boolean", + "description": "Indicates the volume is successfully attached. This field must only be set by the entity completing the attach operation, i.e. the external-attacher." + }, + "attachmentMetadata": { + "type": "object", + "description": "Upon successful attach, this field is populated with any information returned by the attach operation that must be passed into subsequent WaitForAttach or Mount calls. This field must only be set by the entity completing the attach operation, i.e. the external-attacher." + }, + "attachError": { + "$ref": "v1.VolumeError", + "description": "The last error encountered during attach operation, if any. This field must only be set by the entity completing the attach operation, i.e. the external-attacher." + }, + "detachError": { + "$ref": "v1.VolumeError", + "description": "The last error encountered during detach operation, if any. This field must only be set by the entity completing the detach operation, i.e. the external-attacher." + } + } + }, + "v1.VolumeError": { + "id": "v1.VolumeError", + "description": "VolumeError captures an error encountered during a volume operation.", + "properties": { + "time": { + "type": "string", + "description": "Time the error was encountered." + }, + "message": { + "type": "string", + "description": "String detailing the error encountered during Attach or Detach operation. This string maybe logged, so it should not contain sensitive information." + } + } + }, "v1.APIResourceList": { "id": "v1.APIResourceList", "description": "APIResourceList is a list of APIResource, it is used to expose the name of the resources supported in a specific group and version, and if the resource is namespaced.", diff --git a/api/swagger-spec/v1.json b/api/swagger-spec/v1.json index d0aa3e77b22..21d43eb62ff 100644 --- a/api/swagger-spec/v1.json +++ b/api/swagger-spec/v1.json @@ -19987,7 +19987,7 @@ }, "volumeMode": { "$ref": "v1.PersistentVolumeMode", - "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is an alpha feature and may change in the future." + "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature." }, "dataSource": { "$ref": "v1.TypedLocalObjectReference", @@ -20217,7 +20217,7 @@ "description": "HostPath represents a directory on the host. Provisioned by a developer or tester. This is useful for single-node development and testing only! On-host storage is not supported in any way and WILL NOT WORK in a multi-node cluster. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" }, "glusterfs": { - "$ref": "v1.GlusterfsVolumeSource", + "$ref": "v1.GlusterfsPersistentVolumeSource", "description": "Glusterfs represents a Glusterfs volume that is attached to a host and exposed to the pod. Provisioned by an admin. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md" }, "nfs": { @@ -20320,7 +20320,7 @@ }, "volumeMode": { "$ref": "v1.PersistentVolumeMode", - "description": "volumeMode defines if a volume is intended to be used with a formatted filesystem or to remain in raw block state. Value of Filesystem is implied when not included in spec. This is an alpha feature and may change in the future." + "description": "volumeMode defines if a volume is intended to be used with a formatted filesystem or to remain in raw block state. Value of Filesystem is implied when not included in spec. This is a beta feature." }, "nodeAffinity": { "$ref": "v1.VolumeNodeAffinity", @@ -20401,8 +20401,8 @@ "id": "v1.HostPathType", "properties": {} }, - "v1.GlusterfsVolumeSource": { - "id": "v1.GlusterfsVolumeSource", + "v1.GlusterfsPersistentVolumeSource": { + "id": "v1.GlusterfsPersistentVolumeSource", "description": "Represents a Glusterfs mount that lasts the lifetime of a pod. Glusterfs volumes do not support ownership management or SELinux relabeling.", "required": [ "endpoints", @@ -20420,6 +20420,10 @@ "readOnly": { "type": "boolean", "description": "ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod" + }, + "endpointsNamespace": { + "type": "string", + "description": "EndpointsNamespace is the namespace that contains Glusterfs endpoint. If this field is empty, the EndpointNamespace defaults to the same namespace as the bound PVC. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod" } } }, @@ -21568,6 +21572,28 @@ } } }, + "v1.GlusterfsVolumeSource": { + "id": "v1.GlusterfsVolumeSource", + "description": "Represents a Glusterfs mount that lasts the lifetime of a pod. Glusterfs volumes do not support ownership management or SELinux relabeling.", + "required": [ + "endpoints", + "path" + ], + "properties": { + "endpoints": { + "type": "string", + "description": "EndpointsName is the endpoint name that details Glusterfs topology. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod" + }, + "path": { + "type": "string", + "description": "Path is the Glusterfs volume path. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod" + }, + "readOnly": { + "type": "boolean", + "description": "ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod" + } + } + }, "v1.PersistentVolumeClaimVolumeSource": { "id": "v1.PersistentVolumeClaimVolumeSource", "description": "PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace. This volume finds the bound PV and mounts that volume for the pod. A PersistentVolumeClaimVolumeSource is, essentially, a wrapper around another type of volume that is owned by someone else (the system).", @@ -22117,7 +22143,7 @@ "items": { "$ref": "v1.VolumeDevice" }, - "description": "volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future." + "description": "volumeDevices is the list of block devices to be used by the container. This is a beta feature." }, "livenessProbe": { "$ref": "v1.Probe", diff --git a/build/build-image/cross/Dockerfile b/build/build-image/cross/Dockerfile index c9d058d2602..8e4bc0f8843 100644 --- a/build/build-image/cross/Dockerfile +++ b/build/build-image/cross/Dockerfile @@ -15,7 +15,7 @@ # This file creates a standard build environment for building cross # platform go binary for the architecture kubernetes cares about. -FROM golang:1.11.1 +FROM golang:1.11.2 ENV GOARM 7 ENV KUBE_DYNAMIC_CROSSPLATFORMS \ diff --git a/build/build-image/cross/VERSION b/build/build-image/cross/VERSION index 7cd7543d299..19abf8ebbbf 100644 --- a/build/build-image/cross/VERSION +++ b/build/build-image/cross/VERSION @@ -1 +1 @@ -v1.11.1-2 +v1.11.2-1 diff --git a/build/debian-base/Makefile b/build/debian-base/Makefile index 47569c75862..d91a6857cee 100755 --- a/build/debian-base/Makefile +++ b/build/debian-base/Makefile @@ -27,6 +27,8 @@ ALL_ARCH = amd64 arm arm64 ppc64le s390x TEMP_DIR:=$(shell mktemp -d) QEMUVERSION=v2.9.1 +SUDO=$(if $(filter 0,$(shell id -u)),,sudo) + # This option is for running docker manifest command export DOCKER_CLI_EXPERIMENTAL := enabled @@ -80,7 +82,7 @@ ifeq ($(ARCH),amd64) else # When cross-building, only the placeholder "CROSS_BUILD_" should be removed # Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel - docker run --rm --privileged multiarch/qemu-user-static:register --reset + $(SUDO) ../../third_party/multiarch/qemu-user-static/register/register.sh --reset curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/$(QEMUVERSION)/x86_64_qemu-$(QEMUARCH)-static.tar.gz | tar -xz -C $(TEMP_DIR) # Ensure we don't get surprised by umask settings chmod 0755 $(TEMP_DIR)/qemu-$(QEMUARCH)-static diff --git a/build/debian-hyperkube-base/Makefile b/build/debian-hyperkube-base/Makefile index 9a74426def7..39e6e4887df 100644 --- a/build/debian-hyperkube-base/Makefile +++ b/build/debian-hyperkube-base/Makefile @@ -33,6 +33,8 @@ CNI_TARBALL=cni-plugins-$(ARCH)-$(CNI_VERSION).tgz # This option is for running docker manifest command export DOCKER_CLI_EXPERIMENTAL := enabled +SUDO=$(if $(filter 0,$(shell id -u)),,sudo) + .PHONY: all build push clean all-build all-push-images all-push push-manifest all: all-push @@ -74,7 +76,7 @@ endif ifneq ($(ARCH),amd64) # Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel - docker run --rm --privileged multiarch/qemu-user-static:register --reset + $(SUDO) ../../third_party/multiarch/qemu-user-static/register/register.sh --reset endif docker build --pull -t $(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR) rm -rf $(TEMP_DIR) diff --git a/build/debian-iptables/Makefile b/build/debian-iptables/Makefile index bef45123dfb..2061fe683f0 100644 --- a/build/debian-iptables/Makefile +++ b/build/debian-iptables/Makefile @@ -26,13 +26,15 @@ BASEIMAGE?=k8s.gcr.io/debian-base-$(ARCH):0.4.0 # This option is for running docker manifest command export DOCKER_CLI_EXPERIMENTAL := enabled +SUDO=$(if $(filter 0,$(shell id -u)),,sudo) + build: cp ./* $(TEMP_DIR) cd $(TEMP_DIR) && sed -i "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile ifneq ($(ARCH),amd64) # Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel - docker run --rm --privileged multiarch/qemu-user-static:register --reset + $(SUDO) ../../third_party/multiarch/qemu-user-static/register/register.sh --reset endif docker build --pull -t $(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR) diff --git a/build/root/WORKSPACE b/build/root/WORKSPACE index 33bd8c98ebd..cee89622008 100644 --- a/build/root/WORKSPACE +++ b/build/root/WORKSPACE @@ -4,8 +4,8 @@ load("//build:workspace.bzl", "CRI_TOOLS_VERSION") http_archive( name = "io_bazel_rules_go", - sha256 = "7519e9e1c716ae3c05bd2d984a42c3b02e690c5df728dc0a84b23f90c355c5a1", - urls = mirror("https://github.com/bazelbuild/rules_go/releases/download/0.15.4/rules_go-0.15.4.tar.gz"), + sha256 = "f87fa87475ea107b3c69196f39c82b7bbf58fe27c62a338684c20ca17d1d8613", + urls = mirror("https://github.com/bazelbuild/rules_go/releases/download/0.16.2/rules_go-0.16.2.tar.gz"), ) http_archive( @@ -41,7 +41,7 @@ http_archive( load("@bazel_skylib//:lib.bzl", "versions") -versions.check(minimum_bazel_version = "0.16.0") +versions.check(minimum_bazel_version = "0.17.2") load("@io_bazel_rules_go//go:def.bzl", "go_download_sdk", "go_register_toolchains", "go_rules_dependencies") load("@io_bazel_rules_docker//docker:docker.bzl", "docker_pull", "docker_repositories") @@ -49,7 +49,7 @@ load("@io_bazel_rules_docker//docker:docker.bzl", "docker_pull", "docker_reposit go_rules_dependencies() go_register_toolchains( - go_version = "1.11.1", + go_version = "1.11.2", ) docker_repositories() diff --git a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml index fe6b23c1f44..167e887cd04 100644 --- a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml @@ -79,7 +79,7 @@ spec: - /eventer - --source=kubernetes:'' - --sink=gcl - - image: k8s.gcr.io/addon-resizer:1.8.3 + - image: k8s.gcr.io/addon-resizer:1.8.4 name: heapster-nanny resources: limits: @@ -115,7 +115,7 @@ spec: # Specifies the smallest cluster (defined in number of nodes) # resources will be scaled to. - --minClusterSize={{ heapster_min_cluster_size }} - - image: k8s.gcr.io/addon-resizer:1.8.3 + - image: k8s.gcr.io/addon-resizer:1.8.4 name: eventer-nanny resources: limits: diff --git a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml index 02e739e3764..606b302152f 100644 --- a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml +++ b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml @@ -80,7 +80,7 @@ spec: - /eventer - --source=kubernetes:'' - --sink=gcl - - image: k8s.gcr.io/addon-resizer:1.8.3 + - image: k8s.gcr.io/addon-resizer:1.8.4 name: heapster-nanny resources: limits: @@ -116,7 +116,7 @@ spec: # Specifies the smallest cluster (defined in number of nodes) # resources will be scaled to. - --minClusterSize={{ heapster_min_cluster_size }} - - image: k8s.gcr.io/addon-resizer:1.8.3 + - image: k8s.gcr.io/addon-resizer:1.8.4 name: eventer-nanny resources: limits: diff --git a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml index 51d1d622f09..d3a29c5daa7 100644 --- a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml @@ -79,7 +79,7 @@ spec: - /eventer - --source=kubernetes:'' - --sink=influxdb:http://monitoring-influxdb:8086 - - image: k8s.gcr.io/addon-resizer:1.8.3 + - image: k8s.gcr.io/addon-resizer:1.8.4 name: heapster-nanny resources: limits: @@ -115,7 +115,7 @@ spec: # Specifies the smallest cluster (defined in number of nodes) # resources will be scaled to. - --minClusterSize={{ heapster_min_cluster_size }} - - image: k8s.gcr.io/addon-resizer:1.8.3 + - image: k8s.gcr.io/addon-resizer:1.8.4 name: eventer-nanny resources: limits: diff --git a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml index e890b1fbced..bc92cb9cb12 100644 --- a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml @@ -81,7 +81,7 @@ spec: fieldRef: fieldPath: metadata.namespace # END_PROMETHEUS_TO_SD - - image: k8s.gcr.io/addon-resizer:1.8.3 + - image: k8s.gcr.io/addon-resizer:1.8.4 name: heapster-nanny resources: limits: diff --git a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml index a6ee6a8ca61..d55c70cda69 100644 --- a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml @@ -59,7 +59,7 @@ spec: command: - /heapster - --source=kubernetes.summary_api:'' - - image: k8s.gcr.io/addon-resizer:1.8.3 + - image: k8s.gcr.io/addon-resizer:1.8.4 name: heapster-nanny resources: limits: diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base index 35e14918a1b..fd9025ae090 100644 --- a/cluster/addons/dns/coredns/coredns.yaml.base +++ b/cluster/addons/dns/coredns/coredns.yaml.base @@ -108,13 +108,11 @@ spec: spec: serviceAccountName: coredns tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - key: "CriticalAddonsOnly" operator: "Exists" containers: - name: coredns - image: k8s.gcr.io/coredns:1.2.4 + image: k8s.gcr.io/coredns:1.2.6 imagePullPolicy: IfNotPresent resources: limits: diff --git a/cluster/addons/dns/coredns/coredns.yaml.in b/cluster/addons/dns/coredns/coredns.yaml.in index 245a00d4f39..05db14e0d2a 100644 --- a/cluster/addons/dns/coredns/coredns.yaml.in +++ b/cluster/addons/dns/coredns/coredns.yaml.in @@ -108,13 +108,11 @@ spec: spec: serviceAccountName: coredns tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - key: "CriticalAddonsOnly" operator: "Exists" containers: - name: coredns - image: k8s.gcr.io/coredns:1.2.4 + image: k8s.gcr.io/coredns:1.2.6 imagePullPolicy: IfNotPresent resources: limits: diff --git a/cluster/addons/dns/coredns/coredns.yaml.sed b/cluster/addons/dns/coredns/coredns.yaml.sed index 00f18803093..169ddf266aa 100644 --- a/cluster/addons/dns/coredns/coredns.yaml.sed +++ b/cluster/addons/dns/coredns/coredns.yaml.sed @@ -108,13 +108,11 @@ spec: spec: serviceAccountName: coredns tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - key: "CriticalAddonsOnly" operator: "Exists" containers: - name: coredns - image: k8s.gcr.io/coredns:1.2.4 + image: k8s.gcr.io/coredns:1.2.6 imagePullPolicy: IfNotPresent resources: limits: diff --git a/cluster/addons/dns/nodelocaldns/README.md b/cluster/addons/dns/nodelocaldns/README.md new file mode 100644 index 00000000000..976e440f768 --- /dev/null +++ b/cluster/addons/dns/nodelocaldns/README.md @@ -0,0 +1,35 @@ +# Nodelocal DNS Cache + +This addon runs a node-local-dns pod on all cluster nodes. The pod runs CoreDNS as the dns cache. It runs with `hostNetwork:True` and creates a dedicated dummy interface with a link local ip(169.254.20.10/32 by default) to listen for DNS queries. The cache instances connect to clusterDNS in case of cache misses. + +Design details [here](https://github.com/kubernetes/community/blob/master/keps/sig-network/0030-nodelocal-dns-cache.md) + +## nodelocaldns addon template + +This directory contains the addon config yaml - `nodelocaldns.yaml` +The variables will be substituted by the configure scripts when the yaml is copied into master. + +### Network policy and DNS connectivity + +When running nodelocaldns addon on clusters using network policy, additional rules might be required to enable dns connectivity. +Using a namespace selector for dns egress traffic as shown [here](https://docs.projectcalico.org/v2.6/getting-started/kubernetes/tutorials/advanced-policy) +might not be enough since the node-local-dns pods run with `hostNetwork: True` + +One way to enable connectivity from node-local-dns pods to clusterDNS ip is to use an ipBlock rule instead: + +``` +spec: + egress: + - ports: + - port: 53 + protocol: TCP + - port: 53 + protocol: UDP + to: + - ipBlock: + cidr: /32 + podSelector: {} + policyTypes: + - Ingress + - Egress +``` \ No newline at end of file diff --git a/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml b/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml new file mode 100644 index 00000000000..fdf0f421eac --- /dev/null +++ b/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml @@ -0,0 +1,144 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-local-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-local-dns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +data: + Corefile: | + __PILLAR__DNS__DOMAIN__:53 { + errors + cache 30 + reload + loop + bind __PILLAR__LOCAL__DNS__ + forward . __PILLAR__DNS__SERVER__ { + force_tcp + } + prometheus :9253 + health __PILLAR__LOCAL__DNS__:8080 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind __PILLAR__LOCAL__DNS__ + forward . __PILLAR__DNS__SERVER__ { + force_tcp + } + prometheus :9253 + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind __PILLAR__LOCAL__DNS__ + forward . __PILLAR__DNS__SERVER__ { + force_tcp + } + prometheus :9253 + } + .:53 { + errors + cache 30 + reload + loop + bind __PILLAR__LOCAL__DNS__ + forward . /etc/resolv.conf { + force_tcp + } + prometheus :9253 + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-local-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + selector: + matchLabels: + k8s-app: node-local-dns + template: + metadata: + labels: + k8s-app: node-local-dns + spec: + priorityClassName: system-node-critical + serviceAccountName: node-local-dns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - name: node-cache + image: k8s.gcr.io/k8s-dns-node-cache:1.15.0 + resources: + limits: + memory: 30Mi + requests: + cpu: 25m + memory: 5Mi + args: [ "-localip", "__PILLAR__LOCAL__DNS__", "-conf", "/etc/coredns/Corefile" ] + securityContext: + privileged: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + host: __PILLAR__LOCAL__DNS__ + path: /health + port: 8080 + initialDelaySeconds: 60 + timeoutSeconds: 5 + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + volumes: + - name: config-volume + configMap: + name: node-local-dns + items: + - key: Corefile + path: Corefile diff --git a/cluster/addons/fluentd-elasticsearch/es-image/BUILD b/cluster/addons/fluentd-elasticsearch/es-image/BUILD index 95b1699e345..2d97357375a 100644 --- a/cluster/addons/fluentd-elasticsearch/es-image/BUILD +++ b/cluster/addons/fluentd-elasticsearch/es-image/BUILD @@ -22,7 +22,7 @@ go_library( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cluster/addons/fluentd-elasticsearch/es-image/elasticsearch_logging_discovery.go b/cluster/addons/fluentd-elasticsearch/es-image/elasticsearch_logging_discovery.go index cf283dbf9b0..a9158128bb6 100644 --- a/cluster/addons/fluentd-elasticsearch/es-image/elasticsearch_logging_discovery.go +++ b/cluster/addons/fluentd-elasticsearch/es-image/elasticsearch_logging_discovery.go @@ -24,11 +24,11 @@ import ( "strings" "time" - "github.com/golang/glog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/klog" api "k8s.io/kubernetes/pkg/apis/core" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" ) @@ -61,22 +61,22 @@ func flattenSubsets(subsets []api.EndpointSubset) []string { func main() { flag.Parse() - glog.Info("Kubernetes Elasticsearch logging discovery") + klog.Info("Kubernetes Elasticsearch logging discovery") cc, err := buildConfigFromEnvs(os.Getenv("APISERVER_HOST"), os.Getenv("KUBE_CONFIG_FILE")) if err != nil { - glog.Fatalf("Failed to make client: %v", err) + klog.Fatalf("Failed to make client: %v", err) } client, err := clientset.NewForConfig(cc) if err != nil { - glog.Fatalf("Failed to make client: %v", err) + klog.Fatalf("Failed to make client: %v", err) } namespace := metav1.NamespaceSystem envNamespace := os.Getenv("NAMESPACE") if envNamespace != "" { if _, err := client.Core().Namespaces().Get(envNamespace, metav1.GetOptions{}); err != nil { - glog.Fatalf("%s namespace doesn't exist: %v", envNamespace, err) + klog.Fatalf("%s namespace doesn't exist: %v", envNamespace, err) } namespace = envNamespace } @@ -98,7 +98,7 @@ func main() { // If we did not find an elasticsearch logging service then log a warning // and return without adding any unicast hosts. if elasticsearch == nil { - glog.Warningf("Failed to find the elasticsearch-logging service: %v", err) + klog.Warningf("Failed to find the elasticsearch-logging service: %v", err) return } @@ -112,17 +112,17 @@ func main() { continue } addrs = flattenSubsets(endpoints.Subsets) - glog.Infof("Found %s", addrs) - if len(addrs) > 0 && len(addrs) == count { + klog.Infof("Found %s", addrs) + if len(addrs) > 0 && len(addrs) >= count { break } } // If there was an error finding endpoints then log a warning and quit. if err != nil { - glog.Warningf("Error finding endpoints: %v", err) + klog.Warningf("Error finding endpoints: %v", err) return } - glog.Infof("Endpoints = %s", addrs) + klog.Infof("Endpoints = %s", addrs) fmt.Printf("discovery.zen.ping.unicast.hosts: [%s]\n", strings.Join(addrs, ", ")) } diff --git a/cluster/addons/metrics-server/metrics-server-deployment.yaml b/cluster/addons/metrics-server/metrics-server-deployment.yaml index bd412047b64..8fc49327d1f 100644 --- a/cluster/addons/metrics-server/metrics-server-deployment.yaml +++ b/cluster/addons/metrics-server/metrics-server-deployment.yaml @@ -62,7 +62,7 @@ spec: name: https protocol: TCP - name: metrics-server-nanny - image: k8s.gcr.io/addon-resizer:1.8.3 + image: k8s.gcr.io/addon-resizer:1.8.4 resources: limits: cpu: 100m diff --git a/cluster/addons/prometheus/kube-state-metrics-deployment.yaml b/cluster/addons/prometheus/kube-state-metrics-deployment.yaml index 61996fae570..1a673db0b6d 100644 --- a/cluster/addons/prometheus/kube-state-metrics-deployment.yaml +++ b/cluster/addons/prometheus/kube-state-metrics-deployment.yaml @@ -39,7 +39,7 @@ spec: initialDelaySeconds: 5 timeoutSeconds: 5 - name: addon-resizer - image: k8s.gcr.io/addon-resizer:1.8.3 + image: k8s.gcr.io/addon-resizer:1.8.4 resources: limits: cpu: 100m diff --git a/cluster/addons/python-image/README.md b/cluster/addons/python-image/README.md deleted file mode 100644 index 06c82b25c6e..00000000000 --- a/cluster/addons/python-image/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Python image - -The python image here is used by OS distros that don't have python installed to -run python scripts to parse the yaml files in the addon updater script. - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/python-image/README.md?pixel)]() diff --git a/cluster/addons/storage-crds/OWNERS b/cluster/addons/storage-crds/OWNERS new file mode 100644 index 00000000000..852f35cac61 --- /dev/null +++ b/cluster/addons/storage-crds/OWNERS @@ -0,0 +1,6 @@ +approvers: +- saad-ali +- jsafrane +- msau42 +reviewers: +- davidz627 \ No newline at end of file diff --git a/cluster/addons/storage-crds/README.md b/cluster/addons/storage-crds/README.md new file mode 100644 index 00000000000..0ac4681c13e --- /dev/null +++ b/cluster/addons/storage-crds/README.md @@ -0,0 +1,13 @@ +# Kubernetes CSI CRDs + +The Kubernetes Container Storage Interface implementation defines some API objects as CRDs that Kubernetes components +including the Attach/Detach controller depend on. + +If you are using CSI, it is recommended that you enable the relevant feature gates (e.g. `CSIDriverRegistry`, `CSINodeInfo`, etc.), and ensure the CRDs in this directory are installed. + +These objects and their CRDs are defined in `staging/src/k8s.io/csi-api/pkg/crd/manifests`, the source of truth. +They are copied from that CRD manifest directory to this addon directory. +A unit test in `staging/src/k8s.io/csi-api/pkg/crd` verifies that this (and any other) copies of the manifest outside of `staging/src/k8s.io/csi-api/pkg/crd/manifests` do not drift from that source of truth. +If you need to make changes please make changes in the `staging/src/k8s.io/csi-api/pkg/crd/manifests` directory and then update this copy. + +For more information, see: https://kubernetes-csi.github.io/docs/ diff --git a/staging/src/k8s.io/csi-api/pkg/crd/testdata/csidriver.yaml b/cluster/addons/storage-crds/csidriver.yaml similarity index 88% rename from staging/src/k8s.io/csi-api/pkg/crd/testdata/csidriver.yaml rename to cluster/addons/storage-crds/csidriver.yaml index d950cbf494a..54416b24f93 100644 --- a/staging/src/k8s.io/csi-api/pkg/crd/testdata/csidriver.yaml +++ b/cluster/addons/storage-crds/csidriver.yaml @@ -1,8 +1,9 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - creationTimestamp: null name: csidrivers.csi.storage.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile spec: group: csi.storage.k8s.io names: @@ -25,9 +26,3 @@ spec: information (like podName, podUID, etc.) during mount operations. type: string version: v1alpha1 -status: - acceptedNames: - kind: "" - plural: "" - conditions: null - storedVersions: null diff --git a/cluster/addons/storage-crds/csinodeinfo.yaml b/cluster/addons/storage-crds/csinodeinfo.yaml new file mode 100644 index 00000000000..7b12ba57666 --- /dev/null +++ b/cluster/addons/storage-crds/csinodeinfo.yaml @@ -0,0 +1,54 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: csinodeinfos.csi.storage.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + group: csi.storage.k8s.io + names: + kind: CSINodeInfo + plural: csinodeinfos + scope: Cluster + validation: + openAPIV3Schema: + properties: + spec: + description: Specification of CSINodeInfo + properties: + drivers: + description: List of CSI drivers running on the node and their specs. + type: array + items: + properties: + name: + description: The CSI driver that this object refers to. + type: string + nodeID: + description: The node from the driver point of view. + type: string + topologyKeys: + description: List of keys supported by the driver. + items: + type: string + type: array + status: + description: Status of CSINodeInfo + properties: + drivers: + description: List of CSI drivers running on the node and their statuses. + type: array + items: + properties: + name: + description: The CSI driver that this object refers to. + type: string + available: + description: Whether the CSI driver is installed. + type: boolean + volumePluginMechanism: + description: Indicates to external components the required mechanism + to use for any in-tree plugins replaced by this driver. + pattern: in-tree|csi + type: string + version: v1alpha1 diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 671c1a5db7b..4fd5fec4412 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -198,6 +198,9 @@ if [[ ${ENABLE_NETD:-} == "true" ]]; then NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}cloud.google.com/gke-netd-ready=true" fi +ENABLE_NODELOCAL_DNS="${KUBE_ENABLE_NODELOCAL_DNS:-false}" +LOCAL_DNS_IP="${KUBE_LOCAL_DNS_IP:-169.254.20.10}" + # Enable metadata concealment by firewalling pod traffic to the metadata server # and run a proxy daemonset on nodes. # @@ -216,8 +219,8 @@ fi ENCRYPTION_PROVIDER_CONFIG="${ENCRYPTION_PROVIDER_CONFIG:-}" if [[ -z "${ENCRYPTION_PROVIDER_CONFIG}" ]]; then ENCRYPTION_PROVIDER_CONFIG=$(cat << EOM | base64 | tr -d '\r\n' -kind: EncryptionConfig -apiVersion: v1 +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 resources: - resources: - secrets @@ -455,11 +458,7 @@ ROTATE_CERTIFICATES="${ROTATE_CERTIFICATES:-}" # into kube-controller-manager via `--concurrent-service-syncs` CONCURRENT_SERVICE_SYNCS="${CONCURRENT_SERVICE_SYNCS:-}" -if [[ "${ENABLE_TOKENREQUEST:-}" == "true" ]]; then - FEATURE_GATES="${FEATURE_GATES},TokenRequest=true" - SERVICEACCOUNT_ISSUER="https://kubernetes.io/${CLUSTER_NAME}" - SERVICEACCOUNT_API_AUDIENCES="https://kubernetes.default.svc" -fi +SERVICEACCOUNT_ISSUER="https://kubernetes.io/${CLUSTER_NAME}" # Optional: Enable Node termination Handler for Preemptible and GPU VMs. # https://github.com/GoogleCloudPlatform/k8s-node-termination-handler diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index c9ce3694c5c..ad350582a16 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -53,6 +53,10 @@ NODE_LOCAL_SSDS_EXT=${NODE_LOCAL_SSDS_EXT:-} NODE_ACCELERATORS=${NODE_ACCELERATORS:-""} REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-true} KUBE_APISERVER_REQUEST_TIMEOUT=300 +# Increase initial delay for the apiserver liveness probe, to avoid prematurely tearing it down +KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC=${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-45} +# Also increase the initial delay for etcd just to be safe +ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC=${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC:-45} PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false} PREEMPTIBLE_MASTER=${PREEMPTIBLE_MASTER:-false} KUBE_DELETE_NODES=${KUBE_DELETE_NODES:-true} @@ -230,6 +234,8 @@ if [[ ${ENABLE_NETD:-} == "true" ]]; then NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}cloud.google.com/gke-netd-ready=true" fi +ENABLE_NODELOCAL_DNS="${KUBE_ENABLE_NODELOCAL_DNS:-false}" + # To avoid running Calico on a node that is not configured appropriately, # label each Node so that the DaemonSet can run the Pods only on ready Nodes. if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then @@ -272,6 +278,7 @@ fi CLUSTER_DNS_CORE_DNS="${CLUSTER_DNS_CORE_DNS:-true}" ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}" DNS_SERVER_IP="10.0.0.10" +LOCAL_DNS_IP="${KUBE_LOCAL_DNS_IP:-169.254.20.10}" DNS_DOMAIN="cluster.local" # Optional: Enable DNS horizontal autoscaler @@ -470,11 +477,7 @@ ROTATE_CERTIFICATES="${ROTATE_CERTIFICATES:-}" # into kube-controller-manager via `--concurrent-service-syncs` CONCURRENT_SERVICE_SYNCS="${CONCURRENT_SERVICE_SYNCS:-}" -if [[ "${ENABLE_TOKENREQUEST:-}" == "true" ]]; then - FEATURE_GATES="${FEATURE_GATES},TokenRequest=true" - SERVICEACCOUNT_ISSUER="https://kubernetes.io/${CLUSTER_NAME}" - SERVICEACCOUNT_API_AUDIENCES="https://kubernetes.default.svc" -fi +SERVICEACCOUNT_ISSUER="https://kubernetes.io/${CLUSTER_NAME}" # Optional: Enable Node termination Handler for Preemptible and GPU VMs. # https://github.com/GoogleCloudPlatform/k8s-node-termination-handler diff --git a/cluster/gce/gci/apiserver_manifest_test.go b/cluster/gce/gci/apiserver_manifest_test.go index 2b430bd4295..db327771b34 100644 --- a/cluster/gce/gci/apiserver_manifest_test.go +++ b/cluster/gce/gci/apiserver_manifest_test.go @@ -55,6 +55,8 @@ readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty readonly LOG_OWNER_USER=$(id -un) readonly LOG_OWNER_GROUP=$(id -gn) +readonly SERVICEACCOUNT_ISSUER=https://foo.bar.baz +readonly SERVICEACCOUNT_KEY_PATH=/foo/bar/baz.key {{if .EncryptionProviderConfig}} ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}} {{end}} diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 411b529b287..e95f31598cb 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -828,6 +828,13 @@ rules: resources: - group: "" # core resources: ["namespaces", "namespaces/status", "namespaces/finalize"] + - level: None + users: ["cluster-autoscaler"] + verbs: ["get", "update"] + namespaces: ["kube-system"] + resources: + - group: "" # core + resources: ["configmaps", "endpoints"] # Don't log HPA fetching metrics. - level: None users: @@ -1563,11 +1570,9 @@ function start-kube-apiserver { if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" fi - if [[ -n "${SERVICEACCOUNT_ISSUER:-}" ]]; then - params+=" --service-account-issuer=${SERVICEACCOUNT_ISSUER}" - params+=" --service-account-signing-key-file=${SERVICEACCOUNT_KEY_PATH}" - params+=" --service-account-api-audiences=${SERVICEACCOUNT_API_AUDIENCES}" - fi + params+=" --service-account-issuer=${SERVICEACCOUNT_ISSUER}" + params+=" --service-account-api-audiences=${SERVICEACCOUNT_ISSUER}" + params+=" --service-account-signing-key-file=${SERVICEACCOUNT_KEY_PATH}" local audit_policy_config_mount="" local audit_policy_config_volume="" @@ -2349,6 +2354,16 @@ EOF fi } +# Sets up the manifests of local dns cache agent for k8s addons. +function setup-nodelocaldns-manifest { + setup-addon-manifests "addons" "dns/nodelocaldns" + local -r localdns_file="${dst_dir}/dns/nodelocaldns/nodelocaldns.yaml" + # Replace the sed configurations with variable values. + sed -i -e "s/__PILLAR__DNS__DOMAIN__/${DNS_DOMAIN}/g" "${localdns_file}" + sed -i -e "s/__PILLAR__DNS__SERVER__/${DNS_SERVER_IP}/g" "${localdns_file}" + sed -i -e "s/__PILLAR__LOCAL__DNS__/${LOCAL_DNS_IP}/g" "${localdns_file}" +} + # Sets up the manifests of netd for k8s addons. function setup-netd-manifest { local -r netd_file="${dst_dir}/netd/netd.yaml" @@ -2520,6 +2535,9 @@ EOF setup-addon-manifests "addons" "dns/kube-dns" setup-kube-dns-manifest fi + if [[ "${ENABLE_NODELOCAL_DNS:-}" == "true" ]]; then + setup-nodelocaldns-manifest + fi fi if [[ "${ENABLE_NETD:-}" == "true" ]]; then setup-netd-manifest @@ -2567,6 +2585,9 @@ EOF if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce" fi + if [[ "${FEATURE_GATES:-}" =~ "AllAlpha=true" || "${FEATURE_GATES:-}" =~ "CSIDriverRegistry=true" || "${FEATURE_GATES:-}" =~ "CSINodeInfo=true" ]]; then + setup-addon-manifests "addons" "storage-crds" + fi if [[ "${ENABLE_IP_MASQ_AGENT:-}" == "true" ]]; then setup-addon-manifests "addons" "ip-masq-agent" fi diff --git a/cluster/gce/manifests/kube-apiserver.manifest b/cluster/gce/manifests/kube-apiserver.manifest index a8a74722ebb..4be27c81b20 100644 --- a/cluster/gce/manifests/kube-apiserver.manifest +++ b/cluster/gce/manifests/kube-apiserver.manifest @@ -34,11 +34,19 @@ "httpGet": { "host": "127.0.0.1", "port": 8080, - "path": "/healthz" + "path": "/healthz?exclude=etcd" }, "initialDelaySeconds": {{liveness_probe_initial_delay}}, "timeoutSeconds": 15 }, + "readinessProbe": { + "httpGet": { + "host": "127.0.0.1", + "port": 8080, + "path": "/healthz" + }, + "timeoutSeconds": 15 + }, "ports":[ { "name": "https", "containerPort": {{secure_port}}, diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 0f6af3a2c76..79ba5f7db85 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -708,6 +708,9 @@ function build-kubelet-config { declare quoted_dns_server_ip declare quoted_dns_domain quoted_dns_server_ip=$(yaml-quote "${DNS_SERVER_IP}") + if [[ "${ENABLE_NODELOCAL_DNS:-}" == "true" ]]; then + quoted_dns_server_ip=$(yaml-quote "${LOCAL_DNS_IP}") + fi quoted_dns_domain=$(yaml-quote "${DNS_DOMAIN}") cat <>$file <' and 'etcdctl-' binaries exist in --bin-dir '%s' for all --bundled-verions '%s': %v", + klog.Errorf("Failed to validate that 'etcd-' and 'etcdctl-' binaries exist in --bin-dir '%s' for all --bundled-verions '%s': %v", opts.binDir, opts.bundledVersionString, err) os.Exit(1) } @@ -139,7 +139,7 @@ func migrate(name string, port uint64, peerListenUrls string, peerAdvertiseUrls dataDir, err := OpenOrCreateDataDirectory(dataDirPath) if err != nil { - glog.Errorf("Error opening or creating data directory %s: %v", dataDirPath, err) + klog.Errorf("Error opening or creating data directory %s: %v", dataDirPath, err) os.Exit(1) } @@ -158,7 +158,7 @@ func migrate(name string, port uint64, peerListenUrls string, peerAdvertiseUrls } client, err := NewEtcdMigrateClient(cfg) if err != nil { - glog.Errorf("Migration failed: %v", err) + klog.Errorf("Migration failed: %v", err) os.Exit(1) } defer client.Close() @@ -167,7 +167,7 @@ func migrate(name string, port uint64, peerListenUrls string, peerAdvertiseUrls err = migrator.MigrateIfNeeded(target) if err != nil { - glog.Errorf("Migration failed: %v", err) + klog.Errorf("Migration failed: %v", err) os.Exit(1) } } diff --git a/cluster/images/etcd/migrate/migrate_client.go b/cluster/images/etcd/migrate/migrate_client.go index 5bb183cdaf9..b9c9cfb62bb 100644 --- a/cluster/images/etcd/migrate/migrate_client.go +++ b/cluster/images/etcd/migrate/migrate_client.go @@ -29,7 +29,7 @@ import ( clientv2 "github.com/coreos/etcd/client" "github.com/coreos/etcd/clientv3" - "github.com/golang/glog" + "k8s.io/klog" ) // CombinedEtcdClient provides an implementation of EtcdMigrateClient using a combination of the etcd v2 client, v3 client @@ -202,13 +202,13 @@ func (e *CombinedEtcdClient) AttachLease(leaseDuration time.Duration) error { if err != nil { return fmt.Errorf("Error while creating lease: %v", err) } - glog.Infof("Lease with TTL: %v created", lease.TTL) + klog.Infof("Lease with TTL: %v created", lease.TTL) - glog.Infof("Attaching lease to %d entries", len(objectsResp.Kvs)) + klog.Infof("Attaching lease to %d entries", len(objectsResp.Kvs)) for _, kv := range objectsResp.Kvs { putResp, err := v3client.KV.Put(ctx, string(kv.Key), string(kv.Value), clientv3.WithLease(lease.ID), clientv3.WithPrevKV()) if err != nil { - glog.Errorf("Error while attaching lease to: %s", string(kv.Key)) + klog.Errorf("Error while attaching lease to: %s", string(kv.Key)) } if bytes.Compare(putResp.PrevKv.Value, kv.Value) != 0 { return fmt.Errorf("concurrent access to key detected when setting lease on %s, expected previous value of %s but got %s", diff --git a/cluster/images/etcd/migrate/migrate_server.go b/cluster/images/etcd/migrate/migrate_server.go index a1dd1a732f5..ea630ff8b4a 100644 --- a/cluster/images/etcd/migrate/migrate_server.go +++ b/cluster/images/etcd/migrate/migrate_server.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" ) // EtcdMigrateServer manages starting and stopping a versioned etcd server binary. @@ -75,10 +75,10 @@ func (r *EtcdMigrateServer) Start(version *EtcdVersion) error { case <-interval.C: err := r.client.SetEtcdVersionKeyValue(version) if err != nil { - glog.Infof("Still waiting for etcd to start, current error: %v", err) + klog.Infof("Still waiting for etcd to start, current error: %v", err) // keep waiting } else { - glog.Infof("Etcd on port %d is up.", r.cfg.port) + klog.Infof("Etcd on port %d is up.", r.cfg.port) r.cmd = etcdCmd return nil } @@ -114,7 +114,7 @@ func (r *EtcdMigrateServer) Stop() error { case <-stopped: return case <-timedout: - glog.Infof("etcd server has not terminated gracefully after %s, killing it.", gracefulWait) + klog.Infof("etcd server has not terminated gracefully after %s, killing it.", gracefulWait) r.cmd.Process.Kill() return } @@ -122,11 +122,11 @@ func (r *EtcdMigrateServer) Stop() error { err = r.cmd.Wait() stopped <- true if exiterr, ok := err.(*exec.ExitError); ok { - glog.Infof("etcd server stopped (signal: %s)", exiterr.Error()) + klog.Infof("etcd server stopped (signal: %s)", exiterr.Error()) // stopped } else if err != nil { return fmt.Errorf("error waiting for etcd to stop: %v", err) } - glog.Infof("Stopped etcd server %s", r.cfg.name) + klog.Infof("Stopped etcd server %s", r.cfg.name) return nil } diff --git a/cluster/images/etcd/migrate/migrator.go b/cluster/images/etcd/migrate/migrator.go index e1e64f18238..6235096612d 100644 --- a/cluster/images/etcd/migrate/migrator.go +++ b/cluster/images/etcd/migrate/migrator.go @@ -23,7 +23,7 @@ import ( "time" "github.com/blang/semver" - "github.com/golang/glog" + "k8s.io/klog" ) // EtcdMigrateCfg provides all configuration required to perform etcd data upgrade/downgrade migrations. @@ -63,7 +63,7 @@ type Migrator struct { // MigrateIfNeeded upgrades or downgrades the etcd data directory to the given target version. func (m *Migrator) MigrateIfNeeded(target *EtcdVersionPair) error { - glog.Infof("Starting migration to %s", target) + klog.Infof("Starting migration to %s", target) err := m.dataDirectory.Initialize(target) if err != nil { return fmt.Errorf("failed to initialize data directory %s: %v", m.dataDirectory.path, err) @@ -84,28 +84,28 @@ func (m *Migrator) MigrateIfNeeded(target *EtcdVersionPair) error { } for { - glog.Infof("Converging current version '%s' to target version '%s'", current, target) + klog.Infof("Converging current version '%s' to target version '%s'", current, target) currentNextMinorVersion := &EtcdVersion{Version: semver.Version{Major: current.version.Major, Minor: current.version.Minor + 1}} switch { case current.version.MajorMinorEquals(target.version) || currentNextMinorVersion.MajorMinorEquals(target.version): - glog.Infof("current version '%s' equals or is one minor version previous of target version '%s' - migration complete", current, target) + klog.Infof("current version '%s' equals or is one minor version previous of target version '%s' - migration complete", current, target) err = m.dataDirectory.versionFile.Write(target) if err != nil { return fmt.Errorf("failed to write version.txt to '%s': %v", m.dataDirectory.path, err) } return nil case current.storageVersion == storageEtcd2 && target.storageVersion == storageEtcd3: - glog.Infof("upgrading from etcd2 storage to etcd3 storage") + klog.Infof("upgrading from etcd2 storage to etcd3 storage") current, err = m.etcd2ToEtcd3Upgrade(current, target) case current.version.Major == 3 && target.version.Major == 2: - glog.Infof("downgrading from etcd 3.x to 2.x") + klog.Infof("downgrading from etcd 3.x to 2.x") current, err = m.rollbackToEtcd2(current, target) case current.version.Major == target.version.Major && current.version.Minor < target.version.Minor: stepVersion := m.cfg.supportedVersions.NextVersionPair(current) - glog.Infof("upgrading etcd from %s to %s", current, stepVersion) + klog.Infof("upgrading etcd from %s to %s", current, stepVersion) current, err = m.minorVersionUpgrade(current, stepVersion) case current.version.Major == 3 && target.version.Major == 3 && current.version.Minor > target.version.Minor: - glog.Infof("rolling etcd back from %s to %s", current, target) + klog.Infof("rolling etcd back from %s to %s", current, target) current, err = m.rollbackEtcd3MinorVersion(current, target) } if err != nil { @@ -116,13 +116,13 @@ func (m *Migrator) MigrateIfNeeded(target *EtcdVersionPair) error { func (m *Migrator) backupEtcd2(current *EtcdVersion) error { backupDir := fmt.Sprintf("%s/%s", m.dataDirectory, "migration-backup") - glog.Infof("Backup etcd before starting migration") + klog.Infof("Backup etcd before starting migration") err := os.Mkdir(backupDir, 0666) if err != nil { return fmt.Errorf("failed to create backup directory before starting migration: %v", err) } m.client.Backup(current, backupDir) - glog.Infof("Backup done in %s", backupDir) + klog.Infof("Backup done in %s", backupDir) return nil } @@ -131,7 +131,7 @@ func (m *Migrator) rollbackEtcd3MinorVersion(current *EtcdVersionPair, target *E return nil, fmt.Errorf("rollback from %s to %s not supported, only rollbacks to the previous minor version are supported", current.version, target.version) } - glog.Infof("Performing etcd %s -> %s rollback", current.version, target.version) + klog.Infof("Performing etcd %s -> %s rollback", current.version, target.version) err := m.dataDirectory.Backup() if err != nil { return nil, err @@ -145,14 +145,14 @@ func (m *Migrator) rollbackEtcd3MinorVersion(current *EtcdVersionPair, target *E // Start current version of etcd. runner := m.newServer() - glog.Infof("Starting etcd version %s to capture rollback snapshot.", current.version) + klog.Infof("Starting etcd version %s to capture rollback snapshot.", current.version) err = runner.Start(current.version) if err != nil { - glog.Fatalf("Unable to automatically downgrade etcd: starting etcd version %s to capture rollback snapshot failed: %v", current.version, err) + klog.Fatalf("Unable to automatically downgrade etcd: starting etcd version %s to capture rollback snapshot failed: %v", current.version, err) return nil, err } - glog.Infof("Snapshotting etcd %s to %s", current.version, snapshotFilename) + klog.Infof("Snapshotting etcd %s to %s", current.version, snapshotFilename) err = m.client.Snapshot(current.version, snapshotFilename) if err != nil { return nil, err @@ -163,7 +163,7 @@ func (m *Migrator) rollbackEtcd3MinorVersion(current *EtcdVersionPair, target *E return nil, err } - glog.Infof("Backing up data before rolling back") + klog.Infof("Backing up data before rolling back") backupDir := fmt.Sprintf("%s.bak", m.dataDirectory) err = os.RemoveAll(backupDir) if err != nil { @@ -178,7 +178,7 @@ func (m *Migrator) rollbackEtcd3MinorVersion(current *EtcdVersionPair, target *E return nil, err } - glog.Infof("Restoring etcd %s from %s", target.version, snapshotFilename) + klog.Infof("Restoring etcd %s from %s", target.version, snapshotFilename) err = m.client.Restore(target.version, snapshotFilename) if err != nil { return nil, err @@ -195,7 +195,7 @@ func (m *Migrator) rollbackToEtcd2(current *EtcdVersionPair, target *EtcdVersion if !(current.version.Major == 3 && current.version.Minor == 0 && target.version.Major == 2 && target.version.Minor == 2) { return nil, fmt.Errorf("etcd3 -> etcd2 downgrade is supported only between 3.0.x and 2.2.x, got current %s target %s", current, target) } - glog.Infof("Backup and remove all existing v2 data") + klog.Infof("Backup and remove all existing v2 data") err := m.dataDirectory.Backup() if err != nil { return nil, err @@ -214,12 +214,12 @@ func (m *Migrator) etcd2ToEtcd3Upgrade(current *EtcdVersionPair, target *EtcdVer } runner := m.newServer() - glog.Infof("Performing etcd2 -> etcd3 migration") + klog.Infof("Performing etcd2 -> etcd3 migration") err := m.client.Migrate(target.version) if err != nil { return nil, err } - glog.Infof("Attaching leases to TTL entries") + klog.Infof("Attaching leases to TTL entries") // Now attach lease to all keys. // To do it, we temporarily start etcd on a random port (so that diff --git a/cluster/images/etcd/migrate/rollback_v2.go b/cluster/images/etcd/migrate/rollback_v2.go index 0d3b3ce2b01..1b4655770fc 100644 --- a/cluster/images/etcd/migrate/rollback_v2.go +++ b/cluster/images/etcd/migrate/rollback_v2.go @@ -42,7 +42,7 @@ import ( "github.com/coreos/etcd/wal" "github.com/coreos/etcd/wal/walpb" "github.com/coreos/go-semver/semver" - "github.com/golang/glog" + "k8s.io/klog" ) const rollbackVersion = "2.2.0" @@ -50,7 +50,7 @@ const rollbackVersion = "2.2.0" // RollbackV3ToV2 rolls back an etcd 3.0.x data directory to the 2.x.x version specified by rollbackVersion. func RollbackV3ToV2(migrateDatadir string, ttl time.Duration) error { dbpath := path.Join(migrateDatadir, "member", "snap", "db") - glog.Infof("Rolling db file %s back to etcd 2.x", dbpath) + klog.Infof("Rolling db file %s back to etcd 2.x", dbpath) // etcd3 store backend. We will use it to parse v3 data files and extract information. be := backend.NewDefaultBackend(dbpath) @@ -139,7 +139,7 @@ func RollbackV3ToV2(migrateDatadir string, ttl time.Duration) error { v = rollbackVersion } if _, err := st.Set(n.Key, n.Dir, v, store.TTLOptionSet{}); err != nil { - glog.Error(err) + klog.Error(err) } // update nodes @@ -147,7 +147,7 @@ func RollbackV3ToV2(migrateDatadir string, ttl time.Duration) error { if len(fields) == 4 && fields[2] == "members" { nodeID, err := strconv.ParseUint(fields[3], 16, 64) if err != nil { - glog.Fatalf("failed to parse member ID (%s): %v", fields[3], err) + klog.Fatalf("failed to parse member ID (%s): %v", fields[3], err) } nodes = append(nodes, nodeID) } @@ -172,7 +172,7 @@ func RollbackV3ToV2(migrateDatadir string, ttl time.Duration) error { if err := snapshotter.SaveSnap(raftSnap); err != nil { return err } - glog.Infof("Finished successfully") + klog.Infof("Finished successfully") return nil } @@ -214,7 +214,7 @@ func traverseAndDeleteEmptyDir(st store.Store, dir string) error { } for _, node := range e.Node.Nodes { if !node.Dir { - glog.V(2).Infof("key: %s", node.Key[len(etcdserver.StoreKeysPrefix):]) + klog.V(2).Infof("key: %s", node.Key[len(etcdserver.StoreKeysPrefix):]) } else { err := traverseAndDeleteEmptyDir(st, node.Key) if err != nil { @@ -344,6 +344,6 @@ func applyRequest(r *pb.Request, applyV2 etcdserver.ApplierV2) { case "POST", "QGET", "SYNC": return default: - glog.Fatal("unknown command") + klog.Fatal("unknown command") } } diff --git a/cmd/cloud-controller-manager/app/BUILD b/cmd/cloud-controller-manager/app/BUILD index e0fe5bd0703..78e517a9b30 100644 --- a/cmd/cloud-controller-manager/app/BUILD +++ b/cmd/cloud-controller-manager/app/BUILD @@ -19,13 +19,14 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library", "//staging/src/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/cloud-controller-manager/app/controllermanager.go b/cmd/cloud-controller-manager/app/controllermanager.go index 4fa93a65b29..99cf6741926 100644 --- a/cmd/cloud-controller-manager/app/controllermanager.go +++ b/cmd/cloud-controller-manager/app/controllermanager.go @@ -24,12 +24,13 @@ import ( "strings" "time" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/healthz" apiserverflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/leaderelection" @@ -50,13 +51,15 @@ import ( const ( // ControllerStartJitter is the jitter value used when starting controller managers. ControllerStartJitter = 1.0 + // ConfigzName is the name used for register cloud-controller manager /configz, same with GroupName. + ConfigzName = "cloudcontrollermanager.config.k8s.io" ) // NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters func NewCloudControllerManagerCommand() *cobra.Command { s, err := options.NewCloudControllerManagerOptions() if err != nil { - glog.Fatalf("unable to initialize command options: %v", err) + klog.Fatalf("unable to initialize command options: %v", err) } cmd := &cobra.Command{ @@ -104,41 +107,49 @@ the cloud specific control loops shipped with Kubernetes.`, // Run runs the ExternalCMServer. This should never exit. func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error { // To help debugging, immediately log version - glog.Infof("Version: %+v", version.Get()) + klog.Infof("Version: %+v", version.Get()) cloud, err := cloudprovider.InitCloudProvider(c.ComponentConfig.KubeCloudShared.CloudProvider.Name, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile) if err != nil { - glog.Fatalf("Cloud provider could not be initialized: %v", err) + klog.Fatalf("Cloud provider could not be initialized: %v", err) } if cloud == nil { - glog.Fatalf("cloud provider is nil") + klog.Fatalf("cloud provider is nil") } if cloud.HasClusterID() == false { if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud == true { - glog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") + klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") } else { - glog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") + klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") } } // setup /configz endpoint - if cz, err := configz.New("componentconfig"); err == nil { + if cz, err := configz.New(ConfigzName); err == nil { cz.Set(c.ComponentConfig) } else { - glog.Errorf("unable to register configz: %c", err) + klog.Errorf("unable to register configz: %c", err) + } + + // Setup any healthz checks we will want to use. + var checks []healthz.HealthzChecker + var electionChecker *leaderelection.HealthzAdaptor + if c.ComponentConfig.Generic.LeaderElection.LeaderElect { + electionChecker = leaderelection.NewLeaderHealthzAdaptor(time.Second * 20) + checks = append(checks, electionChecker) } // Start the controller manager HTTP server if c.SecureServing != nil { - unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging) + unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...) handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication) if err := c.SecureServing.Serve(handler, 0, stopCh); err != nil { return err } } if c.InsecureServing != nil { - unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging) + unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...) insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}} handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn) if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil { @@ -148,7 +159,7 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error run := func(ctx context.Context) { if err := startControllers(c, ctx.Done(), cloud); err != nil { - glog.Fatalf("error running controllers: %v", err) + klog.Fatalf("error running controllers: %v", err) } } @@ -175,7 +186,7 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error EventRecorder: c.EventRecorder, }) if err != nil { - glog.Fatalf("error creating lock: %v", err) + klog.Fatalf("error creating lock: %v", err) } // Try and become the leader and start cloud controller manager loops @@ -187,9 +198,11 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: run, OnStoppedLeading: func() { - glog.Fatalf("leaderelection lost") + klog.Fatalf("leaderelection lost") }, }, + WatchDog: electionChecker, + Name: "cloud-controller-manager", }) panic("unreachable") } @@ -228,7 +241,7 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, stop <-chan stru c.ComponentConfig.KubeCloudShared.ClusterName, ) if err != nil { - glog.Errorf("Failed to start service controller: %v", err) + klog.Errorf("Failed to start service controller: %v", err) } else { go serviceController.Run(stop, int(c.ComponentConfig.ServiceController.ConcurrentServiceSyncs)) time.Sleep(wait.Jitter(c.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter)) @@ -237,13 +250,13 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, stop <-chan stru // If CIDRs should be allocated for pods and set on the CloudProvider, then start the route controller if c.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs && c.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes { if routes, ok := cloud.Routes(); !ok { - glog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.") + klog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.") } else { var clusterCIDR *net.IPNet if len(strings.TrimSpace(c.ComponentConfig.KubeCloudShared.ClusterCIDR)) != 0 { _, clusterCIDR, err = net.ParseCIDR(c.ComponentConfig.KubeCloudShared.ClusterCIDR) if err != nil { - glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", c.ComponentConfig.KubeCloudShared.ClusterCIDR, err) + klog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", c.ComponentConfig.KubeCloudShared.ClusterCIDR, err) } } @@ -252,14 +265,14 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, stop <-chan stru time.Sleep(wait.Jitter(c.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter)) } } else { - glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", c.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs, c.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes) + klog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", c.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs, c.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes) } // If apiserver is not running we should wait for some time and fail only then. This is particularly // important when we start apiserver and controller manager at the same time. err = genericcontrollermanager.WaitForAPIServer(c.VersionedClient, 10*time.Second) if err != nil { - glog.Fatalf("Failed to wait for apiserver being healthy: %v", err) + klog.Fatalf("Failed to wait for apiserver being healthy: %v", err) } c.SharedInformers.Start(stop) diff --git a/cmd/cloud-controller-manager/app/options/BUILD b/cmd/cloud-controller-manager/app/options/BUILD index 07f82ee0096..c2707eb973c 100644 --- a/cmd/cloud-controller-manager/app/options/BUILD +++ b/cmd/cloud-controller-manager/app/options/BUILD @@ -32,7 +32,7 @@ go_library( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/cloud-controller-manager/app/options/options.go b/cmd/cloud-controller-manager/app/options/options.go index e15a659f012..27c60157a29 100644 --- a/cmd/cloud-controller-manager/app/options/options.go +++ b/cmd/cloud-controller-manager/app/options/options.go @@ -22,7 +22,7 @@ import ( "net" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -265,7 +265,7 @@ func (o *CloudControllerManagerOptions) Config() (*cloudcontrollerconfig.Config, func createRecorder(kubeClient clientset.Interface, userAgent string) record.EventRecorder { eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) // TODO: remove dependence on the legacyscheme return eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: userAgent}) diff --git a/cmd/controller-manager/app/BUILD b/cmd/controller-manager/app/BUILD index 1ba2af44255..4f75ca22f44 100644 --- a/cmd/controller-manager/app/BUILD +++ b/cmd/controller-manager/app/BUILD @@ -21,8 +21,8 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/server/mux:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/routes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/controller-manager/app/helper.go b/cmd/controller-manager/app/helper.go index 38ec45417d4..8bf1edee52a 100644 --- a/cmd/controller-manager/app/helper.go +++ b/cmd/controller-manager/app/helper.go @@ -21,9 +21,9 @@ import ( "net/http" "time" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" ) // WaitForAPIServer waits for the API Server's /healthz endpoint to report "ok" with timeout. @@ -40,7 +40,7 @@ func WaitForAPIServer(client clientset.Interface, timeout time.Duration) error { if healthStatus != http.StatusOK { content, _ := result.Raw() lastErr = fmt.Errorf("APIServer isn't healthy: %v", string(content)) - glog.Warningf("APIServer isn't healthy yet: %v. Waiting a little while.", string(content)) + klog.Warningf("APIServer isn't healthy yet: %v. Waiting a little while.", string(content)) return false, nil } diff --git a/cmd/controller-manager/app/serve.go b/cmd/controller-manager/app/serve.go index 9c958128b41..37fd22199d0 100644 --- a/cmd/controller-manager/app/serve.go +++ b/cmd/controller-manager/app/serve.go @@ -17,11 +17,10 @@ limitations under the License. package app import ( + "github.com/prometheus/client_golang/prometheus" "net/http" goruntime "runtime" - "github.com/prometheus/client_golang/prometheus" - apiserverconfig "k8s.io/apiserver/pkg/apis/config" genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" apirequest "k8s.io/apiserver/pkg/endpoints/request" @@ -53,9 +52,9 @@ func BuildHandlerChain(apiHandler http.Handler, authorizationInfo *apiserver.Aut } // NewBaseHandler takes in CompletedConfig and returns a handler. -func NewBaseHandler(c *apiserverconfig.DebuggingConfiguration) *mux.PathRecorderMux { +func NewBaseHandler(c *apiserverconfig.DebuggingConfiguration, checks ...healthz.HealthzChecker) *mux.PathRecorderMux { mux := mux.NewPathRecorderMux("controller-manager") - healthz.InstallHandler(mux) + healthz.InstallHandler(mux, checks...) if c.EnableProfiling { routes.Profiling{}.Install(mux) if c.EnableContentionProfiling { diff --git a/cmd/genswaggertypedocs/BUILD b/cmd/genswaggertypedocs/BUILD index be4c59a8976..5fabe41559d 100644 --- a/cmd/genswaggertypedocs/BUILD +++ b/cmd/genswaggertypedocs/BUILD @@ -17,8 +17,8 @@ go_library( importpath = "k8s.io/kubernetes/cmd/genswaggertypedocs", deps = [ "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/genswaggertypedocs/swagger_type_docs.go b/cmd/genswaggertypedocs/swagger_type_docs.go index 8916d42ae81..03621f44ed2 100644 --- a/cmd/genswaggertypedocs/swagger_type_docs.go +++ b/cmd/genswaggertypedocs/swagger_type_docs.go @@ -23,8 +23,8 @@ import ( kruntime "k8s.io/apimachinery/pkg/runtime" - "github.com/golang/glog" flag "github.com/spf13/pflag" + "k8s.io/klog" ) var ( @@ -37,7 +37,7 @@ func main() { flag.Parse() if *typeSrc == "" { - glog.Fatalf("Please define -s flag as it is the source file") + klog.Fatalf("Please define -s flag as it is the source file") } var funcOut io.Writer @@ -46,7 +46,7 @@ func main() { } else { file, err := os.Create(*functionDest) if err != nil { - glog.Fatalf("Couldn't open %v: %v", *functionDest, err) + klog.Fatalf("Couldn't open %v: %v", *functionDest, err) } defer file.Close() funcOut = file diff --git a/cmd/kube-apiserver/app/BUILD b/cmd/kube-apiserver/app/BUILD index eb338464cbb..8ec3a3c952d 100644 --- a/cmd/kube-apiserver/app/BUILD +++ b/cmd/kube-apiserver/app/BUILD @@ -14,7 +14,6 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/capabilities:go_default_library", "//pkg/controller/serviceaccount:go_default_library", - "//pkg/features:go_default_library", "//pkg/generated/openapi:go_default_library", "//pkg/kubeapiserver:go_default_library", "//pkg/kubeapiserver/admission:go_default_library", @@ -73,8 +72,8 @@ go_library( "//staging/src/k8s.io/kube-aggregator/pkg/client/informers/internalversion/apiregistration/internalversion:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", ], ) diff --git a/cmd/kube-apiserver/app/aggregator.go b/cmd/kube-apiserver/app/aggregator.go index 4ff4726a4a3..00b97e32af6 100644 --- a/cmd/kube-apiserver/app/aggregator.go +++ b/cmd/kube-apiserver/app/aggregator.go @@ -26,7 +26,7 @@ import ( "strings" "sync" - "github.com/golang/glog" + "k8s.io/klog" apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -167,7 +167,7 @@ func makeAPIService(gv schema.GroupVersion) *apiregistration.APIService { if !ok { // if we aren't found, then we shouldn't register ourselves because it could result in a CRD group version // being permanently stuck in the APIServices list. - glog.Infof("Skipping APIService creation for %v", gv) + klog.Infof("Skipping APIService creation for %v", gv) return nil } return &apiregistration.APIService{ @@ -270,6 +270,7 @@ var apiVersionPriorities = map[schema.GroupVersion]priority{ {Group: "scheduling.k8s.io", Version: "v1beta1"}: {group: 16600, version: 12}, {Group: "scheduling.k8s.io", Version: "v1alpha1"}: {group: 16600, version: 9}, {Group: "coordination.k8s.io", Version: "v1beta1"}: {group: 16500, version: 9}, + {Group: "auditregistration.k8s.io", Version: "v1alpha1"}: {group: 16400, version: 1}, // Append a new group to the end of the list if unsure. // You can use min(existing group)-100 as the initial value for a group. // Version can be set to 9 (to have space around) for a new group. diff --git a/cmd/kube-apiserver/app/apiextensions.go b/cmd/kube-apiserver/app/apiextensions.go index ee5f743daad..d179644a250 100644 --- a/cmd/kube-apiserver/app/apiextensions.go +++ b/cmd/kube-apiserver/app/apiextensions.go @@ -28,6 +28,7 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/apiserver/pkg/util/webhook" kubeexternalinformers "k8s.io/client-go/informers" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" ) @@ -38,6 +39,8 @@ func createAPIExtensionsConfig( pluginInitializers []admission.PluginInitializer, commandOptions *options.ServerRunOptions, masterCount int, + serviceResolver webhook.ServiceResolver, + authResolverWrapper webhook.AuthenticationInfoResolverWrapper, ) (*apiextensionsapiserver.Config, error) { // make a shallow copy to let us twiddle a few things // most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the apiextensions @@ -74,6 +77,8 @@ func createAPIExtensionsConfig( ExtraConfig: apiextensionsapiserver.ExtraConfig{ CRDRESTOptionsGetter: apiextensionsoptions.NewCRDRESTOptionsGetter(etcdOptions), MasterCount: masterCount, + AuthResolverWrapper: authResolverWrapper, + ServiceResolver: serviceResolver, }, } diff --git a/cmd/kube-apiserver/app/options/BUILD b/cmd/kube-apiserver/app/options/BUILD index 857701f7903..770d36dbb8e 100644 --- a/cmd/kube-apiserver/app/options/BUILD +++ b/cmd/kube-apiserver/app/options/BUILD @@ -21,10 +21,12 @@ go_library( "//pkg/kubelet/client:go_default_library", "//pkg/master/ports:go_default_library", "//pkg/master/reconcilers:go_default_library", + "//pkg/serviceaccount:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme:go_default_library", ], diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index 7416c214fbb..23146dd913a 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -27,13 +27,12 @@ import ( "k8s.io/apiserver/pkg/storage/storagebackend" apiserverflag "k8s.io/apiserver/pkg/util/flag" api "k8s.io/kubernetes/pkg/apis/core" + _ "k8s.io/kubernetes/pkg/features" // add the kubernetes feature gates kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" "k8s.io/kubernetes/pkg/master/ports" "k8s.io/kubernetes/pkg/master/reconcilers" - - // add the kubernetes feature gates - _ "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/serviceaccount" ) // ServerRunOptions runs a kubernetes api server. @@ -70,7 +69,9 @@ type ServerRunOptions struct { MasterCount int EndpointReconcilerType string - ServiceAccountSigningKeyFile string + ServiceAccountSigningKeyFile string + ServiceAccountIssuer serviceaccount.TokenGenerator + ServiceAccountTokenMaxExpiration time.Duration } // NewServerRunOptions creates a new ServerRunOptions object with default parameters diff --git a/cmd/kube-apiserver/app/options/validation.go b/cmd/kube-apiserver/app/options/validation.go index f1295756621..d0b1bd0ada9 100644 --- a/cmd/kube-apiserver/app/options/validation.go +++ b/cmd/kube-apiserver/app/options/validation.go @@ -17,74 +17,89 @@ limitations under the License. package options import ( + "errors" "fmt" apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" + utilfeature "k8s.io/apiserver/pkg/util/feature" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/kubernetes/pkg/features" ) // TODO: Longer term we should read this from some config store, rather than a flag. func validateClusterIPFlags(options *ServerRunOptions) []error { - errors := []error{} + var errs []error + if options.ServiceClusterIPRange.IP == nil { - errors = append(errors, fmt.Errorf("no --service-cluster-ip-range specified")) + errs = append(errs, errors.New("no --service-cluster-ip-range specified")) } var ones, bits = options.ServiceClusterIPRange.Mask.Size() if bits-ones > 20 { - errors = append(errors, fmt.Errorf("specified --service-cluster-ip-range is too large")) + errs = append(errs, errors.New("specified --service-cluster-ip-range is too large")) } - return errors + + return errs } func validateServiceNodePort(options *ServerRunOptions) []error { - errors := []error{} + var errs []error + if options.KubernetesServiceNodePort < 0 || options.KubernetesServiceNodePort > 65535 { - errors = append(errors, fmt.Errorf("--kubernetes-service-node-port %v must be between 0 and 65535, inclusive. If 0, the Kubernetes master service will be of type ClusterIP", options.KubernetesServiceNodePort)) + errs = append(errs, fmt.Errorf("--kubernetes-service-node-port %v must be between 0 and 65535, inclusive. If 0, the Kubernetes master service will be of type ClusterIP", options.KubernetesServiceNodePort)) } if options.KubernetesServiceNodePort > 0 && !options.ServiceNodePortRange.Contains(options.KubernetesServiceNodePort) { - errors = append(errors, fmt.Errorf("kubernetes service port range %v doesn't contain %v", options.ServiceNodePortRange, (options.KubernetesServiceNodePort))) + errs = append(errs, fmt.Errorf("kubernetes service port range %v doesn't contain %v", options.ServiceNodePortRange, (options.KubernetesServiceNodePort))) } - return errors + return errs } -// Validate checks ServerRunOptions and return a slice of found errors. +func validateTokenRequest(options *ServerRunOptions) []error { + var errs []error + + enableAttempted := options.ServiceAccountSigningKeyFile != "" || + options.Authentication.ServiceAccounts.Issuer != "" || + len(options.Authentication.APIAudiences) != 0 + + enableSucceeded := options.ServiceAccountIssuer != nil + + if enableAttempted && !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) { + errs = append(errs, errors.New("the TokenRequest feature is not enabled but --service-account-signing-key-file, --service-account-issuer and/or --api-audiences flags were passed")) + } + + if utilfeature.DefaultFeatureGate.Enabled(features.BoundServiceAccountTokenVolume) && !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) { + errs = append(errs, errors.New("the BoundServiceAccountTokenVolume feature depends on the TokenRequest feature, but the TokenRequest features is not enabled")) + } + + if !enableAttempted && utilfeature.DefaultFeatureGate.Enabled(features.BoundServiceAccountTokenVolume) { + errs = append(errs, errors.New("--service-account-signing-key-file and --service-account-issuer are required flags")) + } + + if enableAttempted && !enableSucceeded { + errs = append(errs, errors.New("--service-account-signing-key-file, --service-account-issuer, and --api-audiences should be specified together")) + } + + return errs +} + +// Validate checks ServerRunOptions and return a slice of found errs. func (s *ServerRunOptions) Validate() []error { - var errors []error - if errs := s.Etcd.Validate(); len(errs) > 0 { - errors = append(errors, errs...) - } - if errs := validateClusterIPFlags(s); len(errs) > 0 { - errors = append(errors, errs...) - } - if errs := validateServiceNodePort(s); len(errs) > 0 { - errors = append(errors, errs...) - } - if errs := s.SecureServing.Validate(); len(errs) > 0 { - errors = append(errors, errs...) - } - if errs := s.Authentication.Validate(); len(errs) > 0 { - errors = append(errors, errs...) - } - if errs := s.Authorization.Validate(); len(errs) > 0 { - errors = append(errors, errs...) - } - if errs := s.Audit.Validate(); len(errs) > 0 { - errors = append(errors, errs...) - } - if errs := s.Admission.Validate(); len(errs) > 0 { - errors = append(errors, errs...) - } - if errs := s.InsecureServing.Validate(); len(errs) > 0 { - errors = append(errors, errs...) - } + var errs []error if s.MasterCount <= 0 { - errors = append(errors, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount)) - } - if errs := s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme); len(errs) > 0 { - errors = append(errors, errs...) + errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount)) } + errs = append(errs, s.Etcd.Validate()...) + errs = append(errs, validateClusterIPFlags(s)...) + errs = append(errs, validateServiceNodePort(s)...) + errs = append(errs, s.SecureServing.Validate()...) + errs = append(errs, s.Authentication.Validate()...) + errs = append(errs, s.Authorization.Validate()...) + errs = append(errs, s.Audit.Validate()...) + errs = append(errs, s.Admission.Validate()...) + errs = append(errs, s.InsecureServing.Validate()...) + errs = append(errs, s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme)...) + errs = append(errs, validateTokenRequest(s)...) - return errors + return errs } diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 6b70661d47d..e913571f9b5 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -32,7 +32,6 @@ import ( "time" "github.com/go-openapi/spec" - "github.com/golang/glog" "github.com/spf13/cobra" extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" @@ -50,13 +49,13 @@ import ( serveroptions "k8s.io/apiserver/pkg/server/options" serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/apiserver/pkg/storage/etcd3/preflight" - utilfeature "k8s.io/apiserver/pkg/util/feature" apiserverflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/apiserver/pkg/util/webhook" clientgoinformers "k8s.io/client-go/informers" clientgoclientset "k8s.io/client-go/kubernetes" certutil "k8s.io/client-go/util/cert" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" openapi "k8s.io/kube-openapi/pkg/common" @@ -64,7 +63,6 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/capabilities" serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" - "k8s.io/kubernetes/pkg/features" generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" "k8s.io/kubernetes/pkg/kubeapiserver" kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" @@ -78,13 +76,12 @@ import ( "k8s.io/kubernetes/pkg/registry/cachesize" rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" "k8s.io/kubernetes/pkg/serviceaccount" - "k8s.io/kubernetes/pkg/version" - "k8s.io/kubernetes/pkg/version/verflag" - "k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap" - utilflag "k8s.io/kubernetes/pkg/util/flag" _ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration + "k8s.io/kubernetes/pkg/version" + "k8s.io/kubernetes/pkg/version/verflag" + "k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap" ) const etcdRetryLimit = 60 @@ -142,7 +139,7 @@ cluster's shared state through which all other components interact.`, // Run runs the specified APIServer. This should never exit. func Run(completeOptions completedServerRunOptions, stopCh <-chan struct{}) error { // To help debugging, immediately log version - glog.Infof("Version: %+v", version.Get()) + klog.Infof("Version: %+v", version.Get()) server, err := CreateServerChain(completeOptions, stopCh) if err != nil { @@ -165,7 +162,8 @@ func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan } // If additional API servers are added, they should be gated. - apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount) + apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount, + serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(proxyTransport, kubeAPIServerConfig.GenericConfig.LoopbackClientConfig)) if err != nil { return nil, err } @@ -317,50 +315,6 @@ func CreateKubeAPIServerConfig( return } - var ( - issuer serviceaccount.TokenGenerator - apiAudiences []string - maxExpiration time.Duration - ) - - if s.ServiceAccountSigningKeyFile != "" || - s.Authentication.ServiceAccounts.Issuer != "" || - len(s.Authentication.APIAudiences) > 0 { - if !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) { - lastErr = fmt.Errorf("the TokenRequest feature is not enabled but --service-account-signing-key-file, --service-account-issuer and/or --service-account-api-audiences flags were passed") - return - } - if s.ServiceAccountSigningKeyFile == "" || - s.Authentication.ServiceAccounts.Issuer == "" || - len(s.Authentication.APIAudiences) == 0 || - len(s.Authentication.ServiceAccounts.KeyFiles) == 0 { - lastErr = fmt.Errorf("service-account-signing-key-file, service-account-issuer, service-account-api-audiences and service-account-key-file should be specified together") - return - } - sk, err := certutil.PrivateKeyFromFile(s.ServiceAccountSigningKeyFile) - if err != nil { - lastErr = fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err) - return - } - if s.Authentication.ServiceAccounts.MaxExpiration != 0 { - lowBound := time.Hour - upBound := time.Duration(1<<32) * time.Second - if s.Authentication.ServiceAccounts.MaxExpiration < lowBound || - s.Authentication.ServiceAccounts.MaxExpiration > upBound { - lastErr = fmt.Errorf("the serviceaccount max expiration is out of range, must be between 1 hour to 2^32 seconds") - return - } - } - - issuer, err = serviceaccount.JWTTokenGenerator(s.Authentication.ServiceAccounts.Issuer, sk) - if err != nil { - lastErr = fmt.Errorf("failed to build token generator: %v", err) - return - } - apiAudiences = s.Authentication.APIAudiences - maxExpiration = s.Authentication.ServiceAccounts.MaxExpiration - } - config = &master.Config{ GenericConfig: genericConfig, ExtraConfig: master.ExtraConfig{ @@ -392,9 +346,8 @@ func CreateKubeAPIServerConfig( EndpointReconcilerType: reconcilers.Type(s.EndpointReconcilerType), MasterCount: s.MasterCount, - ServiceAccountIssuer: issuer, - APIAudiences: apiAudiences, - ServiceAccountMaxExpiration: maxExpiration, + ServiceAccountIssuer: s.ServiceAccountIssuer, + ServiceAccountMaxExpiration: s.ServiceAccountTokenMaxExpiration, VersionedInformers: versionedInformers, }, @@ -438,9 +391,6 @@ func buildGenericConfig( if lastErr = s.Authentication.ApplyTo(genericConfig); lastErr != nil { return } - if lastErr = s.Audit.ApplyTo(genericConfig); lastErr != nil { - return - } if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil { return } @@ -504,13 +454,29 @@ func buildGenericConfig( genericConfig.DisabledPostStartHooks.Insert(rbacrest.PostStartHookName) } - admissionConfig := &kubeapiserveradmission.AdmissionConfig{ + admissionConfig := &kubeapiserveradmission.Config{ ExternalInformers: versionedInformers, LoopbackClientConfig: genericConfig.LoopbackClientConfig, CloudConfigFile: s.CloudProvider.CloudConfigFile, } serviceResolver = buildServiceResolver(s.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers) + authInfoResolverWrapper := webhook.NewDefaultAuthenticationInfoResolverWrapper(proxyTransport, genericConfig.LoopbackClientConfig) + + lastErr = s.Audit.ApplyTo( + genericConfig, + genericConfig.LoopbackClientConfig, + versionedInformers, + serveroptions.NewProcessInfo("kube-apiserver", "kube-system"), + &serveroptions.WebhookOptions{ + AuthInfoResolverWrapper: authInfoResolverWrapper, + ServiceResolver: serviceResolver, + }, + ) + if lastErr != nil { + return + } + pluginInitializers, admissionPostStartHook, err = admissionConfig.New(proxyTransport, serviceResolver) if err != nil { lastErr = fmt.Errorf("failed to create admission plugin initializer: %v", err) @@ -584,7 +550,7 @@ func Complete(s *options.ServerRunOptions) (completedServerRunOptions, error) { return options, fmt.Errorf("error finding host name: %v", err) } } - glog.Infof("external host was not specified, using %v", s.GenericServerRunOptions.ExternalHost) + klog.Infof("external host was not specified, using %v", s.GenericServerRunOptions.ExternalHost) } s.Authentication.ApplyAuthorization(s.Authorization) @@ -600,13 +566,34 @@ func Complete(s *options.ServerRunOptions) (completedServerRunOptions, error) { if kubeauthenticator.IsValidServiceAccountKeyFile(s.SecureServing.ServerCert.CertKey.KeyFile) { s.Authentication.ServiceAccounts.KeyFiles = []string{s.SecureServing.ServerCert.CertKey.KeyFile} } else { - glog.Warning("No TLS key provided, service account token authentication disabled") + klog.Warning("No TLS key provided, service account token authentication disabled") } } } + if s.ServiceAccountSigningKeyFile != "" && s.Authentication.ServiceAccounts.Issuer != "" { + sk, err := certutil.PrivateKeyFromFile(s.ServiceAccountSigningKeyFile) + if err != nil { + return options, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err) + } + if s.Authentication.ServiceAccounts.MaxExpiration != 0 { + lowBound := time.Hour + upBound := time.Duration(1<<32) * time.Second + if s.Authentication.ServiceAccounts.MaxExpiration < lowBound || + s.Authentication.ServiceAccounts.MaxExpiration > upBound { + return options, fmt.Errorf("the serviceaccount max expiration must be between 1 hour to 2^32 seconds") + } + } + + s.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(s.Authentication.ServiceAccounts.Issuer, sk) + if err != nil { + return options, fmt.Errorf("failed to build token generator: %v", err) + } + s.ServiceAccountTokenMaxExpiration = s.Authentication.ServiceAccounts.MaxExpiration + } + if s.Etcd.EnableWatchCache { - glog.V(2).Infof("Initializing cache sizes based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB) + klog.V(2).Infof("Initializing cache sizes based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB) sizes := cachesize.NewHeuristicWatchCacheSizes(s.GenericServerRunOptions.TargetRAMMB) if userSpecified, err := serveroptions.ParseWatchCacheSizes(s.Etcd.WatchCacheSizes); err == nil { for resource, size := range userSpecified { diff --git a/cmd/kube-controller-manager/OWNERS b/cmd/kube-controller-manager/OWNERS index dc963a9e2fe..e8e10481cd4 100644 --- a/cmd/kube-controller-manager/OWNERS +++ b/cmd/kube-controller-manager/OWNERS @@ -2,6 +2,7 @@ approvers: - deads2k - lavalamp - mikedanese +- sttts reviewers: - '249043822' - a-robinson diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index 41024caab96..2fa0f43dc78 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -43,6 +43,7 @@ go_library( "//pkg/controller/bootstrap:go_default_library", "//pkg/controller/certificates/approver:go_default_library", "//pkg/controller/certificates/cleaner:go_default_library", + "//pkg/controller/certificates/rootcacertpublisher:go_default_library", "//pkg/controller/certificates/signer:go_default_library", "//pkg/controller/clusterroleaggregation:go_default_library", "//pkg/controller/cronjob:go_default_library", @@ -112,6 +113,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/mux:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", @@ -131,8 +133,8 @@ go_library( "//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/custom_metrics:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/external_metrics:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/cmd/kube-controller-manager/app/certificates.go b/cmd/kube-controller-manager/app/certificates.go index 078a5f8452c..752e2c5a1bb 100644 --- a/cmd/kube-controller-manager/app/certificates.go +++ b/cmd/kube-controller-manager/app/certificates.go @@ -24,15 +24,18 @@ import ( "fmt" "os" - "github.com/golang/glog" + "k8s.io/klog" "net/http" "k8s.io/apimachinery/pkg/runtime/schema" + utilfeature "k8s.io/apiserver/pkg/util/feature" kubeoptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" "k8s.io/kubernetes/pkg/controller/certificates/approver" "k8s.io/kubernetes/pkg/controller/certificates/cleaner" + "k8s.io/kubernetes/pkg/controller/certificates/rootcacertpublisher" "k8s.io/kubernetes/pkg/controller/certificates/signer" + "k8s.io/kubernetes/pkg/features" ) func startCSRSigningController(ctx ControllerContext) (http.Handler, bool, error) { @@ -66,7 +69,7 @@ func startCSRSigningController(ctx ControllerContext) (http.Handler, bool, error switch { case (keyFileExists && keyUsesDefault) || (certFileExists && certUsesDefault): - glog.Warningf("You might be using flag defaulting for --cluster-signing-cert-file and" + + klog.Warningf("You might be using flag defaulting for --cluster-signing-cert-file and" + " --cluster-signing-key-file. These defaults are deprecated and will be removed" + " in a subsequent release. Please pass these options explicitly.") case (!keyFileExists && keyUsesDefault) && (!certFileExists && certUsesDefault): @@ -120,3 +123,33 @@ func startCSRCleanerController(ctx ControllerContext) (http.Handler, bool, error go cleaner.Run(1, ctx.Stop) return nil, true, nil } + +func startRootCACertPublisher(ctx ControllerContext) (http.Handler, bool, error) { + if !utilfeature.DefaultFeatureGate.Enabled(features.BoundServiceAccountTokenVolume) { + return nil, false, nil + } + + var ( + rootCA []byte + err error + ) + if ctx.ComponentConfig.SAController.RootCAFile != "" { + if rootCA, err = readCA(ctx.ComponentConfig.SAController.RootCAFile); err != nil { + return nil, true, fmt.Errorf("error parsing root-ca-file at %s: %v", ctx.ComponentConfig.SAController.RootCAFile, err) + } + } else { + rootCA = ctx.ClientBuilder.ConfigOrDie("root-ca-cert-publisher").CAData + } + + sac, err := rootcacertpublisher.NewPublisher( + ctx.InformerFactory.Core().V1().ConfigMaps(), + ctx.InformerFactory.Core().V1().Namespaces(), + ctx.ClientBuilder.ClientOrDie("root-ca-cert-publisher"), + rootCA, + ) + if err != nil { + return nil, true, fmt.Errorf("error creating root CA certificate publisher: %v", err) + } + go sac.Run(1, ctx.Stop) + return nil, true, nil +} diff --git a/cmd/kube-controller-manager/app/cloudproviders.go b/cmd/kube-controller-manager/app/cloudproviders.go index 687d71cb8b4..762f5828b4d 100644 --- a/cmd/kube-controller-manager/app/cloudproviders.go +++ b/cmd/kube-controller-manager/app/cloudproviders.go @@ -19,7 +19,7 @@ package app import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/client-go/informers" cloudprovider "k8s.io/cloud-provider" @@ -50,7 +50,7 @@ func createCloudProvider(cloudProvider string, externalCloudVolumePlugin string, if cloud != nil && cloud.HasClusterID() == false { if allowUntaggedCloud == true { - glog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") + klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") } else { return nil, loopMode, fmt.Errorf("no ClusterID Found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") } diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index b417071c5fd..5e686092194 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -29,8 +29,8 @@ import ( "os" "time" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -38,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/mux" apiserverflag "k8s.io/apiserver/pkg/util/flag" cacheddiscovery "k8s.io/client-go/discovery/cached" @@ -64,6 +65,8 @@ import ( const ( // Jitter used when starting controller managers ControllerStartJitter = 1.0 + // ConfigzName is the name used for register kube-controller manager /configz, same with GroupName. + ConfigzName = "kubecontrollermanager.config.k8s.io" ) type ControllerLoopMode int @@ -77,7 +80,7 @@ const ( func NewControllerManagerCommand() *cobra.Command { s, err := options.NewKubeControllerManagerOptions() if err != nil { - glog.Fatalf("unable to initialize command options: %v", err) + klog.Fatalf("unable to initialize command options: %v", err) } cmd := &cobra.Command{ @@ -140,26 +143,34 @@ func ResyncPeriod(c *config.CompletedConfig) func() time.Duration { // Run runs the KubeControllerManagerOptions. This should never exit. func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { // To help debugging, immediately log version - glog.Infof("Version: %+v", version.Get()) + klog.Infof("Version: %+v", version.Get()) - if cfgz, err := configz.New("componentconfig"); err == nil { + if cfgz, err := configz.New(ConfigzName); err == nil { cfgz.Set(c.ComponentConfig) } else { - glog.Errorf("unable to register configz: %c", err) + klog.Errorf("unable to register configz: %c", err) + } + + // Setup any healthz checks we will want to use. + var checks []healthz.HealthzChecker + var electionChecker *leaderelection.HealthzAdaptor + if c.ComponentConfig.Generic.LeaderElection.LeaderElect { + electionChecker = leaderelection.NewLeaderHealthzAdaptor(time.Second * 20) + checks = append(checks, electionChecker) } // Start the controller manager HTTP server // unsecuredMux is the handler for these controller *after* authn/authz filters have been applied var unsecuredMux *mux.PathRecorderMux if c.SecureServing != nil { - unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging) + unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...) handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication) if err := c.SecureServing.Serve(handler, 0, stopCh); err != nil { return err } } if c.InsecureServing != nil { - unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging) + unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...) insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}} handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn) if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil { @@ -176,7 +187,7 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { if len(c.ComponentConfig.SAController.ServiceAccountKeyFile) == 0 { // It'c possible another controller process is creating the tokens for us. // If one isn't, we'll timeout and exit when our client builder is unable to create the tokens. - glog.Warningf("--use-service-account-credentials was specified without providing a --service-account-private-key-file") + klog.Warningf("--use-service-account-credentials was specified without providing a --service-account-private-key-file") } clientBuilder = controller.SAControllerClientBuilder{ ClientConfig: restclient.AnonymousClientConfig(c.Kubeconfig), @@ -189,12 +200,12 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { } controllerContext, err := CreateControllerContext(c, rootClientBuilder, clientBuilder, ctx.Done()) if err != nil { - glog.Fatalf("error building controller context: %v", err) + klog.Fatalf("error building controller context: %v", err) } saTokenControllerInitFunc := serviceAccountTokenControllerStarter{rootClientBuilder: rootClientBuilder}.startServiceAccountTokenController if err := StartControllers(controllerContext, saTokenControllerInitFunc, NewControllerInitializers(controllerContext.LoopMode), unsecuredMux); err != nil { - glog.Fatalf("error starting controllers: %v", err) + klog.Fatalf("error starting controllers: %v", err) } controllerContext.InformerFactory.Start(controllerContext.Stop) @@ -224,7 +235,7 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { EventRecorder: c.EventRecorder, }) if err != nil { - glog.Fatalf("error creating lock: %v", err) + klog.Fatalf("error creating lock: %v", err) } leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{ @@ -235,9 +246,11 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: run, OnStoppedLeading: func() { - glog.Fatalf("leaderelection lost") + klog.Fatalf("leaderelection lost") }, }, + WatchDog: electionChecker, + Name: "kube-controller-manager", }) panic("unreachable") } @@ -379,6 +392,7 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc controllers["pvc-protection"] = startPVCProtectionController controllers["pv-protection"] = startPVProtectionController controllers["ttl-after-finished"] = startTTLAfterFinishedController + controllers["root-ca-cert-publisher"] = startRootCACertPublisher return controllers } @@ -473,20 +487,20 @@ func StartControllers(ctx ControllerContext, startSATokenController InitFunc, co for controllerName, initFn := range controllers { if !ctx.IsControllerEnabled(controllerName) { - glog.Warningf("%q is disabled", controllerName) + klog.Warningf("%q is disabled", controllerName) continue } time.Sleep(wait.Jitter(ctx.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter)) - glog.V(1).Infof("Starting %q", controllerName) + klog.V(1).Infof("Starting %q", controllerName) debugHandler, started, err := initFn(ctx) if err != nil { - glog.Errorf("Error starting %q", controllerName) + klog.Errorf("Error starting %q", controllerName) return err } if !started { - glog.Warningf("Skipping %q", controllerName) + klog.Warningf("Skipping %q", controllerName) continue } if debugHandler != nil && unsecuredMux != nil { @@ -494,7 +508,7 @@ func StartControllers(ctx ControllerContext, startSATokenController InitFunc, co unsecuredMux.UnlistedHandle(basePath, http.StripPrefix(basePath, debugHandler)) unsecuredMux.UnlistedHandlePrefix(basePath+"/", http.StripPrefix(basePath, debugHandler)) } - glog.Infof("Started %q", controllerName) + klog.Infof("Started %q", controllerName) } return nil @@ -509,12 +523,12 @@ type serviceAccountTokenControllerStarter struct { func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController(ctx ControllerContext) (http.Handler, bool, error) { if !ctx.IsControllerEnabled(saTokenControllerName) { - glog.Warningf("%q is disabled", saTokenControllerName) + klog.Warningf("%q is disabled", saTokenControllerName) return nil, false, nil } if len(ctx.ComponentConfig.SAController.ServiceAccountKeyFile) == 0 { - glog.Warningf("%q is disabled because there is no private key", saTokenControllerName) + klog.Warningf("%q is disabled because there is no private key", saTokenControllerName) return nil, false, nil } privateKey, err := certutil.PrivateKeyFromFile(ctx.ComponentConfig.SAController.ServiceAccountKeyFile) @@ -524,11 +538,7 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController var rootCA []byte if ctx.ComponentConfig.SAController.RootCAFile != "" { - rootCA, err = ioutil.ReadFile(ctx.ComponentConfig.SAController.RootCAFile) - if err != nil { - return nil, true, fmt.Errorf("error reading root-ca-file at %s: %v", ctx.ComponentConfig.SAController.RootCAFile, err) - } - if _, err := certutil.ParseCertsPEM(rootCA); err != nil { + if rootCA, err = readCA(ctx.ComponentConfig.SAController.RootCAFile); err != nil { return nil, true, fmt.Errorf("error parsing root-ca-file at %s: %v", ctx.ComponentConfig.SAController.RootCAFile, err) } } else { @@ -558,3 +568,15 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController return nil, true, nil } + +func readCA(file string) ([]byte, error) { + rootCA, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + if _, err := certutil.ParseCertsPEM(rootCA); err != nil { + return nil, err + } + + return rootCA, err +} diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index ace08a9818e..f485ff74191 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -26,7 +26,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "net/http" @@ -73,7 +73,7 @@ func startServiceController(ctx ControllerContext) (http.Handler, bool, error) { ) if err != nil { // This error shouldn't fail. It lives like this as a legacy. - glog.Errorf("Failed to start service controller: %v", err) + klog.Errorf("Failed to start service controller: %v", err) return nil, false, nil } go serviceController.Run(ctx.Stop, int(ctx.ComponentConfig.ServiceController.ConcurrentServiceSyncs)) @@ -92,14 +92,14 @@ func startNodeIpamController(ctx ControllerContext) (http.Handler, bool, error) if len(strings.TrimSpace(ctx.ComponentConfig.KubeCloudShared.ClusterCIDR)) != 0 { _, clusterCIDR, err = net.ParseCIDR(ctx.ComponentConfig.KubeCloudShared.ClusterCIDR) if err != nil { - glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", ctx.ComponentConfig.KubeCloudShared.ClusterCIDR, err) + klog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", ctx.ComponentConfig.KubeCloudShared.ClusterCIDR, err) } } if len(strings.TrimSpace(ctx.ComponentConfig.NodeIPAMController.ServiceCIDR)) != 0 { _, serviceCIDR, err = net.ParseCIDR(ctx.ComponentConfig.NodeIPAMController.ServiceCIDR) if err != nil { - glog.Warningf("Unsuccessful parsing of service CIDR %v: %v", ctx.ComponentConfig.NodeIPAMController.ServiceCIDR, err) + klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", ctx.ComponentConfig.NodeIPAMController.ServiceCIDR, err) } } @@ -148,21 +148,21 @@ func startNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, er func startRouteController(ctx ControllerContext) (http.Handler, bool, error) { if !ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs || !ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes { - glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs, ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes) + klog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs, ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes) return nil, false, nil } if ctx.Cloud == nil { - glog.Warning("configure-cloud-routes is set, but no cloud provider specified. Will not configure cloud provider routes.") + klog.Warning("configure-cloud-routes is set, but no cloud provider specified. Will not configure cloud provider routes.") return nil, false, nil } routes, ok := ctx.Cloud.Routes() if !ok { - glog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.") + klog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.") return nil, false, nil } _, clusterCIDR, err := net.ParseCIDR(ctx.ComponentConfig.KubeCloudShared.ClusterCIDR) if err != nil { - glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", ctx.ComponentConfig.KubeCloudShared.ClusterCIDR, err) + klog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", ctx.ComponentConfig.KubeCloudShared.ClusterCIDR, err) } routeController := routecontroller.New(routes, ctx.ClientBuilder.ClientOrDie("route-controller"), ctx.InformerFactory.Core().V1().Nodes(), ctx.ComponentConfig.KubeCloudShared.ClusterName, clusterCIDR) go routeController.Run(ctx.Stop, ctx.ComponentConfig.KubeCloudShared.RouteReconciliationPeriod.Duration) diff --git a/cmd/kube-controller-manager/app/options/BUILD b/cmd/kube-controller-manager/app/options/BUILD index fcae9358cf7..9d14caa78f7 100644 --- a/cmd/kube-controller-manager/app/options/BUILD +++ b/cmd/kube-controller-manager/app/options/BUILD @@ -53,8 +53,8 @@ go_library( "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/kube-controller-manager/config/v1alpha1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 377a6ee3de9..a93d3fd8fae 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -45,7 +45,7 @@ import ( // add the kubernetes feature gates _ "k8s.io/kubernetes/pkg/features" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -438,7 +438,7 @@ func (s KubeControllerManagerOptions) Config(allControllers []string, disabledBy func createRecorder(kubeClient clientset.Interface, userAgent string) record.EventRecorder { eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) // TODO: remove dependency on the legacyscheme return eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: userAgent}) diff --git a/cmd/kube-controller-manager/app/plugins.go b/cmd/kube-controller-manager/app/plugins.go index de70fc98447..e56752b22ad 100644 --- a/cmd/kube-controller-manager/app/plugins.go +++ b/cmd/kube-controller-manager/app/plugins.go @@ -23,7 +23,7 @@ import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" // Cloud providers cloudprovider "k8s.io/cloud-provider" @@ -133,7 +133,7 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config kubectrl ProvisioningEnabled: config.EnableHostPathProvisioning, } if err := AttemptToLoadRecycler(config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, &hostPathConfig); err != nil { - glog.Fatalf("Could not create hostpath recycler pod from file %s: %+v", config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, err) + klog.Fatalf("Could not create hostpath recycler pod from file %s: %+v", config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, err) } allPlugins = append(allPlugins, host_path.ProbeVolumePlugins(hostPathConfig)...) @@ -143,7 +143,7 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config kubectrl RecyclerPodTemplate: volume.NewPersistentVolumeRecyclerPodTemplate(), } if err := AttemptToLoadRecycler(config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, &nfsConfig); err != nil { - glog.Fatalf("Could not create NFS recycler pod from file %s: %+v", config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, err) + klog.Fatalf("Could not create NFS recycler pod from file %s: %+v", config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, err) } allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(nfsConfig)...) allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...) diff --git a/cmd/kube-controller-manager/app/policy.go b/cmd/kube-controller-manager/app/policy.go index 3dc430300cb..d841d8dd144 100644 --- a/cmd/kube-controller-manager/app/policy.go +++ b/cmd/kube-controller-manager/app/policy.go @@ -26,7 +26,7 @@ import ( "net/http" - "github.com/golang/glog" + "k8s.io/klog" ) func startDisruptionController(ctx ControllerContext) (http.Handler, bool, error) { @@ -35,7 +35,7 @@ func startDisruptionController(ctx ControllerContext) (http.Handler, bool, error var resource = "poddisruptionbudgets" if !ctx.AvailableResources[schema.GroupVersionResource{Group: group, Version: version, Resource: resource}] { - glog.Infof( + klog.Infof( "Refusing to start disruption because resource %q in group %q is not available.", resource, group+"/"+version) return nil, false, nil diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index 70a79a35978..b99cf2ca155 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -61,10 +61,10 @@ go_library( "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/kube-proxy/config/v1alpha1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", ] + select({ diff --git a/cmd/kube-proxy/app/conntrack.go b/cmd/kube-proxy/app/conntrack.go index 5e858663fb1..f3b7776e361 100644 --- a/cmd/kube-proxy/app/conntrack.go +++ b/cmd/kube-proxy/app/conntrack.go @@ -22,7 +22,7 @@ import ( "strconv" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/sysctl" @@ -49,7 +49,7 @@ func (rct realConntracker) SetMax(max int) error { if err := rct.setIntSysCtl("nf_conntrack_max", max); err != nil { return err } - glog.Infof("Setting nf_conntrack_max to %d", max) + klog.Infof("Setting nf_conntrack_max to %d", max) // Linux does not support writing to /sys/module/nf_conntrack/parameters/hashsize // when the writer process is not in the initial network namespace @@ -80,7 +80,7 @@ func (rct realConntracker) SetMax(max int) error { return readOnlySysFSError } // TODO: generify this and sysctl to a new sysfs.WriteInt() - glog.Infof("Setting conntrack hashsize to %d", max/4) + klog.Infof("Setting conntrack hashsize to %d", max/4) return writeIntStringFile("/sys/module/nf_conntrack/parameters/hashsize", max/4) } @@ -97,7 +97,7 @@ func (realConntracker) setIntSysCtl(name string, value int) error { sys := sysctl.New() if val, _ := sys.GetSysctl(entry); val != value { - glog.Infof("Set sysctl '%v' to %v", entry, value) + klog.Infof("Set sysctl '%v' to %v", entry, value) if err := sys.SetSysctl(entry, value); err != nil { return err } @@ -112,7 +112,7 @@ func isSysFSWritable() (bool, error) { m := mount.New("" /* default mount path */) mountPoints, err := m.List() if err != nil { - glog.Errorf("failed to list mount points: %v", err) + klog.Errorf("failed to list mount points: %v", err) return false, err } @@ -124,7 +124,7 @@ func isSysFSWritable() (bool, error) { if len(mountPoint.Opts) > 0 && mountPoint.Opts[0] == permWritable { return true, nil } - glog.Errorf("sysfs is not writable: %+v (mount options are %v)", + klog.Errorf("sysfs is not writable: %+v (mount options are %v)", mountPoint, mountPoint.Opts) return false, readOnlySysFSError } diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 793004f6fa1..04189b98d30 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -72,10 +72,10 @@ import ( "k8s.io/utils/exec" utilpointer "k8s.io/utils/pointer" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" "github.com/spf13/cobra" "github.com/spf13/pflag" + "k8s.io/klog" ) const ( @@ -191,7 +191,7 @@ func NewOptions() *Options { // Complete completes all the required options. func (o *Options) Complete() error { if len(o.ConfigFile) == 0 && len(o.WriteConfigTo) == 0 { - glog.Warning("WARNING: all flags other than --config, --write-config-to, and --cleanup are deprecated. Please begin using a config file ASAP.") + klog.Warning("WARNING: all flags other than --config, --write-config-to, and --cleanup are deprecated. Please begin using a config file ASAP.") o.applyDeprecatedHealthzPortToConfig() } @@ -280,7 +280,7 @@ func (o *Options) writeConfigFile() error { return err } - glog.Infof("Wrote configuration to: %s\n", o.WriteConfigTo) + klog.Infof("Wrote configuration to: %s\n", o.WriteConfigTo) return nil } @@ -365,23 +365,23 @@ with the apiserver API to configure the proxy.`, utilflag.PrintFlags(cmd.Flags()) if err := initForOS(opts.WindowsService); err != nil { - glog.Fatalf("failed OS init: %v", err) + klog.Fatalf("failed OS init: %v", err) } if err := opts.Complete(); err != nil { - glog.Fatalf("failed complete: %v", err) + klog.Fatalf("failed complete: %v", err) } if err := opts.Validate(args); err != nil { - glog.Fatalf("failed validate: %v", err) + klog.Fatalf("failed validate: %v", err) } - glog.Fatal(opts.Run()) + klog.Fatal(opts.Run()) }, } var err error opts.config, err = opts.ApplyDefaults(opts.config) if err != nil { - glog.Fatalf("unable to create flag defaults: %v", err) + klog.Fatalf("unable to create flag defaults: %v", err) } opts.AddFlags(cmd.Flags()) @@ -426,7 +426,7 @@ func createClients(config apimachineryconfig.ClientConnectionConfiguration, mast var err error if len(config.Kubeconfig) == 0 && len(masterOverride) == 0 { - glog.Info("Neither kubeconfig file nor master URL was specified. Falling back to in-cluster config.") + klog.Info("Neither kubeconfig file nor master URL was specified. Falling back to in-cluster config.") kubeConfig, err = rest.InClusterConfig() } else { // This creates a client, first loading any specified kubeconfig @@ -461,7 +461,7 @@ func createClients(config apimachineryconfig.ClientConnectionConfiguration, mast // Run runs the specified ProxyServer. This should never exit (unless CleanupAndExit is set). func (s *ProxyServer) Run() error { // To help debugging, immediately log version - glog.Infof("Version: %+v", version.Get()) + klog.Infof("Version: %+v", version.Get()) // remove iptables rules and exit if s.CleanupAndExit { encounteredError := userspace.CleanupLeftovers(s.IptInterface) @@ -478,16 +478,16 @@ func (s *ProxyServer) Run() error { if s.OOMScoreAdj != nil { oomAdjuster = oom.NewOOMAdjuster() if err := oomAdjuster.ApplyOOMScoreAdj(0, int(*s.OOMScoreAdj)); err != nil { - glog.V(2).Info(err) + klog.V(2).Info(err) } } if len(s.ResourceContainer) != 0 { // Run in its own container. if err := resourcecontainer.RunInResourceContainer(s.ResourceContainer); err != nil { - glog.Warningf("Failed to start in resource-only container %q: %v", s.ResourceContainer, err) + klog.Warningf("Failed to start in resource-only container %q: %v", s.ResourceContainer, err) } else { - glog.V(2).Infof("Running in resource-only container %q", s.ResourceContainer) + klog.V(2).Infof("Running in resource-only container %q", s.ResourceContainer) } } @@ -595,7 +595,7 @@ func getConntrackMax(config kubeproxyconfig.KubeProxyConntrackConfiguration) (in if config.MaxPerCore != nil && *config.MaxPerCore > 0 { return -1, fmt.Errorf("invalid config: Conntrack Max and Conntrack MaxPerCore are mutually exclusive") } - glog.V(3).Infof("getConntrackMax: using absolute conntrack-max (deprecated)") + klog.V(3).Infof("getConntrackMax: using absolute conntrack-max (deprecated)") return int(*config.Max), nil } if config.MaxPerCore != nil && *config.MaxPerCore > 0 { @@ -605,10 +605,10 @@ func getConntrackMax(config kubeproxyconfig.KubeProxyConntrackConfiguration) (in } scaled := int(*config.MaxPerCore) * goruntime.NumCPU() if scaled > floor { - glog.V(3).Infof("getConntrackMax: using scaled conntrack-max-per-core") + klog.V(3).Infof("getConntrackMax: using scaled conntrack-max-per-core") return scaled, nil } - glog.V(3).Infof("getConntrackMax: using conntrack-min") + klog.V(3).Infof("getConntrackMax: using conntrack-min") return floor, nil } return 0, nil diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 83e359cf457..a92c1492f5d 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -48,7 +48,7 @@ import ( utilsysctl "k8s.io/kubernetes/pkg/util/sysctl" "k8s.io/utils/exec" - "github.com/golang/glog" + "k8s.io/klog" ) // NewProxyServer returns a new ProxyServer. @@ -75,7 +75,7 @@ func newProxyServer( protocol := utiliptables.ProtocolIpv4 if net.ParseIP(config.BindAddress).To4() == nil { - glog.V(0).Infof("IPv6 bind address (%s), assume IPv6 operation", config.BindAddress) + klog.V(0).Infof("IPv6 bind address (%s), assume IPv6 operation", config.BindAddress) protocol = utiliptables.ProtocolIpv6 } @@ -145,7 +145,7 @@ func newProxyServer( nodeIP = utilnode.GetNodeIP(client, hostname) } if proxyMode == proxyModeIPTables { - glog.V(0).Info("Using iptables Proxier.") + klog.V(0).Info("Using iptables Proxier.") if config.IPTables.MasqueradeBit == nil { // MasqueradeBit must be specified or defaulted. return nil, fmt.Errorf("unable to read IPTables MasqueradeBit from config") @@ -175,7 +175,7 @@ func newProxyServer( serviceEventHandler = proxierIPTables endpointsEventHandler = proxierIPTables // No turning back. Remove artifacts that might still exist from the userspace Proxier. - glog.V(0).Info("Tearing down inactive rules.") + klog.V(0).Info("Tearing down inactive rules.") // TODO this has side effects that should only happen when Run() is invoked. userspace.CleanupLeftovers(iptInterface) // IPVS Proxier will generate some iptables rules, need to clean them before switching to other proxy mode. @@ -186,7 +186,7 @@ func newProxyServer( ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface, cleanupIPVS) } } else if proxyMode == proxyModeIPVS { - glog.V(0).Info("Using ipvs Proxier.") + klog.V(0).Info("Using ipvs Proxier.") proxierIPVS, err := ipvs.NewProxier( iptInterface, ipvsInterface, @@ -213,12 +213,12 @@ func newProxyServer( proxier = proxierIPVS serviceEventHandler = proxierIPVS endpointsEventHandler = proxierIPVS - glog.V(0).Info("Tearing down inactive rules.") + klog.V(0).Info("Tearing down inactive rules.") // TODO this has side effects that should only happen when Run() is invoked. userspace.CleanupLeftovers(iptInterface) iptables.CleanupLeftovers(iptInterface) } else { - glog.V(0).Info("Using userspace Proxier.") + klog.V(0).Info("Using userspace Proxier.") // This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for // our config.EndpointsConfigHandler. loadBalancer := userspace.NewLoadBalancerRR() @@ -244,7 +244,7 @@ func newProxyServer( proxier = proxierUserspace // Remove artifacts from the iptables and ipvs Proxier, if not on Windows. - glog.V(0).Info("Tearing down inactive rules.") + klog.V(0).Info("Tearing down inactive rules.") // TODO this has side effects that should only happen when Run() is invoked. iptables.CleanupLeftovers(iptInterface) // IPVS Proxier will generate some iptables rules, need to clean them before switching to other proxy mode. @@ -292,7 +292,7 @@ func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, khandle i case proxyModeIPVS: return tryIPVSProxy(iptver, khandle, ipsetver, kcompat) } - glog.Warningf("Flag proxy-mode=%q unknown, assuming iptables proxy", proxyMode) + klog.Warningf("Flag proxy-mode=%q unknown, assuming iptables proxy", proxyMode) return tryIPTablesProxy(iptver, kcompat) } @@ -309,7 +309,7 @@ func tryIPVSProxy(iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, } // Try to fallback to iptables before falling back to userspace - glog.V(1).Infof("Can't use ipvs proxier, trying iptables proxier") + klog.V(1).Infof("Can't use ipvs proxier, trying iptables proxier") return tryIPTablesProxy(iptver, kcompat) } @@ -324,6 +324,6 @@ func tryIPTablesProxy(iptver iptables.IPTablesVersioner, kcompat iptables.Kernel return proxyModeIPTables } // Fallback. - glog.V(1).Infof("Can't use iptables proxy, using userspace proxier") + klog.V(1).Infof("Can't use iptables proxy, using userspace proxier") return proxyModeUserspace } diff --git a/cmd/kube-proxy/app/server_windows.go b/cmd/kube-proxy/app/server_windows.go index 725b0df7185..5ef2ce16182 100644 --- a/cmd/kube-proxy/app/server_windows.go +++ b/cmd/kube-proxy/app/server_windows.go @@ -42,7 +42,7 @@ import ( utilnode "k8s.io/kubernetes/pkg/util/node" "k8s.io/utils/exec" - "github.com/golang/glog" + "k8s.io/klog" ) // NewProxyServer returns a new ProxyServer. @@ -99,7 +99,7 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi proxyMode := getProxyMode(string(config.Mode), winkernel.WindowsKernelCompatTester{}) if proxyMode == proxyModeKernelspace { - glog.V(0).Info("Using Kernelspace Proxier.") + klog.V(0).Info("Using Kernelspace Proxier.") proxierKernelspace, err := winkernel.NewProxier( config.IPTables.SyncPeriod.Duration, config.IPTables.MinSyncPeriod.Duration, @@ -118,7 +118,7 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi endpointsEventHandler = proxierKernelspace serviceEventHandler = proxierKernelspace } else { - glog.V(0).Info("Using userspace Proxier.") + klog.V(0).Info("Using userspace Proxier.") execer := exec.New() var netshInterface utilnetsh.Interface netshInterface = utilnetsh.New(execer) @@ -143,7 +143,7 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi } proxier = proxierUserspace serviceEventHandler = proxierUserspace - glog.V(0).Info("Tearing down pure-winkernel proxy rules.") + klog.V(0).Info("Tearing down pure-winkernel proxy rules.") winkernel.CleanupLeftovers() } @@ -182,13 +182,13 @@ func tryWinKernelSpaceProxy(kcompat winkernel.KernelCompatTester) string { // guaranteed false on error, error only necessary for debugging useWinKerelProxy, err := winkernel.CanUseWinKernelProxier(kcompat) if err != nil { - glog.Errorf("Can't determine whether to use windows kernel proxy, using userspace proxier: %v", err) + klog.Errorf("Can't determine whether to use windows kernel proxy, using userspace proxier: %v", err) return proxyModeUserspace } if useWinKerelProxy { return proxyModeKernelspace } // Fallback. - glog.V(1).Infof("Can't use winkernel proxy, using userspace proxier") + klog.V(1).Infof("Can't use winkernel proxy, using userspace proxier") return proxyModeUserspace } diff --git a/cmd/kube-scheduler/app/BUILD b/cmd/kube-scheduler/app/BUILD index 3e3d92597d8..a3243f96ab4 100644 --- a/cmd/kube-scheduler/app/BUILD +++ b/cmd/kube-scheduler/app/BUILD @@ -39,12 +39,14 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/server/mux:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/routes:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/globalflag:go_default_library", "//staging/src/k8s.io/client-go/informers/storage/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -61,6 +63,7 @@ filegroup( ":package-srcs", "//cmd/kube-scheduler/app/config:all-srcs", "//cmd/kube-scheduler/app/options:all-srcs", + "//cmd/kube-scheduler/app/testing:all-srcs", ], tags = ["automanaged"], ) diff --git a/cmd/kube-scheduler/app/config/BUILD b/cmd/kube-scheduler/app/config/BUILD index 6545f5ff84a..877ceef1e05 100644 --- a/cmd/kube-scheduler/app/config/BUILD +++ b/cmd/kube-scheduler/app/config/BUILD @@ -12,6 +12,7 @@ go_library( "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", ], diff --git a/cmd/kube-scheduler/app/config/config.go b/cmd/kube-scheduler/app/config/config.go index 930759d28c0..2ad5ae9bf34 100644 --- a/cmd/kube-scheduler/app/config/config.go +++ b/cmd/kube-scheduler/app/config/config.go @@ -22,6 +22,7 @@ import ( coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" v1core "k8s.io/client-go/kubernetes/typed/core/v1" + restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/record" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -32,6 +33,9 @@ type Config struct { // config is the scheduler server's configuration object. ComponentConfig kubeschedulerconfig.KubeSchedulerConfiguration + // LoopbackClientConfig is a config for a privileged loopback connection + LoopbackClientConfig *restclient.Config + InsecureServing *apiserver.DeprecatedInsecureServingInfo // nil will disable serving on an insecure port InsecureMetricsServing *apiserver.DeprecatedInsecureServingInfo // non-nil if metrics should be served independently Authentication apiserver.AuthenticationInfo @@ -70,5 +74,7 @@ func (c *Config) Complete() CompletedConfig { c.InsecureMetricsServing.Name = "metrics" } + apiserver.AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization) + return CompletedConfig{&cc} } diff --git a/cmd/kube-scheduler/app/options/BUILD b/cmd/kube-scheduler/app/options/BUILD index ebe68f08340..65050784ee0 100644 --- a/cmd/kube-scheduler/app/options/BUILD +++ b/cmd/kube-scheduler/app/options/BUILD @@ -14,6 +14,7 @@ go_library( "//cmd/kube-scheduler/app/config:go_default_library", "//pkg/api/legacyscheme:go_default_library", "//pkg/client/leaderelectionconfig:go_default_library", + "//pkg/master/ports:go_default_library", "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/apis/config/scheme:go_default_library", "//pkg/scheduler/apis/config/v1alpha1:go_default_library", @@ -28,6 +29,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", @@ -38,8 +40,8 @@ go_library( "//staging/src/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/kube-scheduler/config/v1alpha1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/kube-scheduler/app/options/insecure_serving.go b/cmd/kube-scheduler/app/options/insecure_serving.go index 78662f69eb3..db3c7351100 100644 --- a/cmd/kube-scheduler/app/options/insecure_serving.go +++ b/cmd/kube-scheduler/app/options/insecure_serving.go @@ -31,8 +31,8 @@ import ( // CombinedInsecureServingOptions sets up to two insecure listeners for healthz and metrics. The flags // override the ComponentConfig and DeprecatedInsecureServingOptions values for both. type CombinedInsecureServingOptions struct { - Healthz *apiserveroptions.DeprecatedInsecureServingOptions - Metrics *apiserveroptions.DeprecatedInsecureServingOptions + Healthz *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback + Metrics *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback BindPort int // overrides the structs above on ApplyTo, ignored on ApplyToFromLoadedConfig BindAddress string // overrides the structs above on ApplyTo, ignored on ApplyToFromLoadedConfig @@ -60,11 +60,11 @@ func (o *CombinedInsecureServingOptions) applyTo(c *schedulerappconfig.Config, c return err } - if err := o.Healthz.ApplyTo(&c.InsecureServing); err != nil { + if err := o.Healthz.ApplyTo(&c.InsecureServing, &c.LoopbackClientConfig); err != nil { return err } if o.Metrics != nil && (c.ComponentConfig.MetricsBindAddress != c.ComponentConfig.HealthzBindAddress || o.Healthz == nil) { - if err := o.Metrics.ApplyTo(&c.InsecureMetricsServing); err != nil { + if err := o.Metrics.ApplyTo(&c.InsecureMetricsServing, &c.LoopbackClientConfig); err != nil { return err } } @@ -108,7 +108,7 @@ func (o *CombinedInsecureServingOptions) ApplyToFromLoadedConfig(c *schedulerapp return o.applyTo(c, componentConfig) } -func updateAddressFromDeprecatedInsecureServingOptions(addr *string, is *apiserveroptions.DeprecatedInsecureServingOptions) error { +func updateAddressFromDeprecatedInsecureServingOptions(addr *string, is *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback) error { if is == nil { *addr = "" } else { @@ -124,7 +124,7 @@ func updateAddressFromDeprecatedInsecureServingOptions(addr *string, is *apiserv return nil } -func updateDeprecatedInsecureServingOptionsFromAddress(is *apiserveroptions.DeprecatedInsecureServingOptions, addr string) error { +func updateDeprecatedInsecureServingOptionsFromAddress(is *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback, addr string) error { if is == nil { return nil } diff --git a/cmd/kube-scheduler/app/options/insecure_serving_test.go b/cmd/kube-scheduler/app/options/insecure_serving_test.go index 09677b6aa06..28bcb600e60 100644 --- a/cmd/kube-scheduler/app/options/insecure_serving_test.go +++ b/cmd/kube-scheduler/app/options/insecure_serving_test.go @@ -46,8 +46,8 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindPort: 0, }, }, @@ -61,7 +61,7 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindPort: 0, }, }, @@ -79,7 +79,7 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindPort: 0, }, }, @@ -97,8 +97,8 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindPort: 0, }, }, @@ -118,8 +118,8 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1235", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindPort: 0, }, }, @@ -141,8 +141,8 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindPort: 1236, BindAddress: "1.2.3.4", }, @@ -163,8 +163,8 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindAddress: "2.3.4.5", BindPort: 1234, }, @@ -185,8 +185,8 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindAddress: "2.3.4.5", BindPort: 0, }, diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index f35c173687a..15c79acd7c5 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -23,15 +23,13 @@ import ( "strconv" "time" - "github.com/golang/glog" - "github.com/spf13/pflag" - corev1 "k8s.io/api/core/v1" apimachineryconfig "k8s.io/apimachinery/pkg/apis/config" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" apiserveroptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + apiserverflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" v1core "k8s.io/client-go/kubernetes/typed/core/v1" @@ -41,10 +39,12 @@ import ( "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/tools/record" + "k8s.io/klog" kubeschedulerconfigv1alpha1 "k8s.io/kube-scheduler/config/v1alpha1" schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/client/leaderelectionconfig" + "k8s.io/kubernetes/pkg/master/ports" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" @@ -56,7 +56,7 @@ type Options struct { // The default values. These are overridden if ConfigFile is set or by values in InsecureServing. ComponentConfig kubeschedulerconfig.KubeSchedulerConfiguration - SecureServing *apiserveroptions.SecureServingOptions + SecureServing *apiserveroptions.SecureServingOptionsWithLoopback CombinedInsecureServing *CombinedInsecureServingOptions Authentication *apiserveroptions.DelegatingAuthenticationOptions Authorization *apiserveroptions.DelegatingAuthorizationOptions @@ -85,25 +85,34 @@ func NewOptions() (*Options, error) { o := &Options{ ComponentConfig: *cfg, - SecureServing: nil, // TODO: enable with apiserveroptions.NewSecureServingOptions() + SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{ + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{ BindNetwork: "tcp", - }, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{ + }).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{ BindNetwork: "tcp", - }, + }).WithLoopback(), BindPort: hport, BindAddress: hhost, }, - Authentication: nil, // TODO: enable with apiserveroptions.NewDelegatingAuthenticationOptions() - Authorization: nil, // TODO: enable with apiserveroptions.NewDelegatingAuthorizationOptions() + Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), + Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(), Deprecated: &DeprecatedOptions{ UseLegacyPolicyConfig: false, PolicyConfigMapNamespace: metav1.NamespaceSystem, }, } + o.Authentication.RemoteKubeConfigFileOptional = true + o.Authorization.RemoteKubeConfigFileOptional = true + o.Authorization.AlwaysAllowPaths = []string{"/healthz"} + + // Set the PairName but leave certificate directory blank to generate in-memory by default + o.SecureServing.ServerCert.CertDirectory = "" + o.SecureServing.ServerCert.PairName = "kube-scheduler" + o.SecureServing.BindPort = ports.KubeSchedulerPort + return o, nil } @@ -129,20 +138,23 @@ func newDefaultComponentConfig() (*kubeschedulerconfig.KubeSchedulerConfiguratio return &cfg, nil } -// AddFlags adds flags for the scheduler options. -func (o *Options) AddFlags(fs *pflag.FlagSet) { +// Flags returns flags for a specific scheduler by section name +func (o *Options) Flags() (nfs apiserverflag.NamedFlagSets) { + fs := nfs.FlagSet("misc") fs.StringVar(&o.ConfigFile, "config", o.ConfigFile, "The path to the configuration file. Flags override values in this file.") fs.StringVar(&o.WriteConfigTo, "write-config-to", o.WriteConfigTo, "If set, write the configuration values to this file and exit.") fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") - o.SecureServing.AddFlags(fs) - o.CombinedInsecureServing.AddFlags(fs) - o.Authentication.AddFlags(fs) - o.Authorization.AddFlags(fs) - o.Deprecated.AddFlags(fs, &o.ComponentConfig) + o.SecureServing.AddFlags(nfs.FlagSet("secure serving")) + o.CombinedInsecureServing.AddFlags(nfs.FlagSet("insecure serving")) + o.Authentication.AddFlags(nfs.FlagSet("authentication")) + o.Authorization.AddFlags(nfs.FlagSet("authorization")) + o.Deprecated.AddFlags(nfs.FlagSet("deprecated"), &o.ComponentConfig) - leaderelectionconfig.BindFlags(&o.ComponentConfig.LeaderElection.LeaderElectionConfiguration, fs) - utilfeature.DefaultFeatureGate.AddFlag(fs) + leaderelectionconfig.BindFlags(&o.ComponentConfig.LeaderElection.LeaderElectionConfiguration, nfs.FlagSet("leader election")) + utilfeature.DefaultFeatureGate.AddFlag(nfs.FlagSet("feature gate")) + + return nfs } // ApplyTo applies the scheduler options to the given scheduler app configuration. @@ -173,13 +185,19 @@ func (o *Options) ApplyTo(c *schedulerappconfig.Config) error { } } - if err := o.SecureServing.ApplyTo(&c.SecureServing); err != nil { + if err := o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil { return err } - if err := o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil { - return err + if o.SecureServing != nil && (o.SecureServing.BindPort != 0 || o.SecureServing.Listener != nil) { + if err := o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil { + return err + } + if err := o.Authorization.ApplyTo(&c.Authorization); err != nil { + return err + } } - return o.Authorization.ApplyTo(&c.Authorization) + + return nil } // Validate validates all the required options. @@ -200,6 +218,12 @@ func (o *Options) Validate() []error { // Config return a scheduler config object func (o *Options) Config() (*schedulerappconfig.Config, error) { + if o.SecureServing != nil { + if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil { + return nil, fmt.Errorf("error creating self-signed certificates: %v", err) + } + } + c := &schedulerappconfig.Config{} if err := o.ApplyTo(c); err != nil { return nil, err @@ -262,6 +286,8 @@ func makeLeaderElectionConfig(config kubeschedulerconfig.KubeSchedulerLeaderElec LeaseDuration: config.LeaseDuration.Duration, RenewDeadline: config.RenewDeadline.Duration, RetryPeriod: config.RetryPeriod.Duration, + WatchDog: leaderelection.NewLeaderHealthzAdaptor(time.Second * 20), + Name: "kube-scheduler", }, nil } @@ -269,7 +295,7 @@ func makeLeaderElectionConfig(config kubeschedulerconfig.KubeSchedulerLeaderElec // TODO remove masterOverride when CLI flags are removed. func createClients(config apimachineryconfig.ClientConnectionConfiguration, masterOverride string, timeout time.Duration) (clientset.Interface, clientset.Interface, v1core.EventsGetter, error) { if len(config.Kubeconfig) == 0 && len(masterOverride) == 0 { - glog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.") + klog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.") } // This creates a client, first loading any specified kubeconfig diff --git a/cmd/kube-scheduler/app/options/options_test.go b/cmd/kube-scheduler/app/options/options_test.go index b63fac090c4..34cd2404231 100644 --- a/cmd/kube-scheduler/app/options/options_test.go +++ b/cmd/kube-scheduler/app/options/options_test.go @@ -32,6 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" apiserverconfig "k8s.io/apiserver/pkg/apis/config" + apiserveroptions "k8s.io/apiserver/pkg/server/options" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" ) @@ -175,6 +176,29 @@ users: } return *cfg }(), + SecureServing: (&apiserveroptions.SecureServingOptions{ + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "/a/b/c", + PairName: "kube-scheduler", + }, + HTTP2MaxStreamsPerConnection: 47, + }).WithLoopback(), + Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ + CacheTTL: 10 * time.Second, + ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, + RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ + UsernameHeaders: []string{"x-remote-user"}, + GroupHeaders: []string{"x-remote-group"}, + ExtraHeaderPrefixes: []string{"x-remote-extra-"}, + }, + RemoteKubeConfigFileOptional: true, + }, + Authorization: &apiserveroptions.DelegatingAuthorizationOptions{ + AllowCacheTTL: 10 * time.Second, + DenyCacheTTL: 10 * time.Second, + RemoteKubeConfigFileOptional: true, + AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or /healthz/* + }, }, expectedUsername: "config", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ @@ -233,6 +257,29 @@ users: cfg.ClientConnection.Kubeconfig = flagKubeconfig return *cfg }(), + SecureServing: (&apiserveroptions.SecureServingOptions{ + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "/a/b/c", + PairName: "kube-scheduler", + }, + HTTP2MaxStreamsPerConnection: 47, + }).WithLoopback(), + Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ + CacheTTL: 10 * time.Second, + ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, + RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ + UsernameHeaders: []string{"x-remote-user"}, + GroupHeaders: []string{"x-remote-group"}, + ExtraHeaderPrefixes: []string{"x-remote-extra-"}, + }, + RemoteKubeConfigFileOptional: true, + }, + Authorization: &apiserveroptions.DelegatingAuthorizationOptions{ + AllowCacheTTL: 10 * time.Second, + DenyCacheTTL: 10 * time.Second, + RemoteKubeConfigFileOptional: true, + AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or /healthz/* + }, }, expectedUsername: "flag", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ @@ -264,8 +311,32 @@ users: }, }, { - name: "overridden master", - options: &Options{Master: insecureserver.URL}, + name: "overridden master", + options: &Options{ + Master: insecureserver.URL, + SecureServing: (&apiserveroptions.SecureServingOptions{ + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "/a/b/c", + PairName: "kube-scheduler", + }, + HTTP2MaxStreamsPerConnection: 47, + }).WithLoopback(), + Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ + CacheTTL: 10 * time.Second, + RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ + UsernameHeaders: []string{"x-remote-user"}, + GroupHeaders: []string{"x-remote-group"}, + ExtraHeaderPrefixes: []string{"x-remote-extra-"}, + }, + RemoteKubeConfigFileOptional: true, + }, + Authorization: &apiserveroptions.DelegatingAuthorizationOptions{ + AllowCacheTTL: 10 * time.Second, + DenyCacheTTL: 10 * time.Second, + RemoteKubeConfigFileOptional: true, + AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or /healthz/* + }, + }, expectedUsername: "none, http", }, { diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index c88d3ed60bf..615f9e8f18f 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -39,6 +39,8 @@ import ( "k8s.io/apiserver/pkg/server/mux" "k8s.io/apiserver/pkg/server/routes" utilfeature "k8s.io/apiserver/pkg/util/feature" + apiserverflag "k8s.io/apiserver/pkg/util/flag" + "k8s.io/apiserver/pkg/util/globalflag" storageinformers "k8s.io/client-go/informers/storage/v1" v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/leaderelection" @@ -59,16 +61,16 @@ import ( "k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/pkg/version/verflag" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" "github.com/spf13/cobra" + "k8s.io/klog" ) // NewSchedulerCommand creates a *cobra.Command object with default parameters func NewSchedulerCommand() *cobra.Command { opts, err := options.NewOptions() if err != nil { - glog.Fatalf("unable to initialize command options: %v", err) + klog.Fatalf("unable to initialize command options: %v", err) } cmd := &cobra.Command{ @@ -81,21 +83,38 @@ constraints, affinity and anti-affinity specifications, data locality, inter-wor interference, deadlines, and so on. Workload-specific requirements will be exposed through the API as necessary.`, Run: func(cmd *cobra.Command, args []string) { - if err := run(cmd, args, opts); err != nil { + if err := runCommand(cmd, args, opts); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } }, } + fs := cmd.Flags() + namedFlagSets := opts.Flags() + verflag.AddFlags(namedFlagSets.FlagSet("global")) + globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name()) + for _, f := range namedFlagSets.FlagSets { + fs.AddFlagSet(f) + } - opts.AddFlags(cmd.Flags()) + usageFmt := "Usage:\n %s\n" + cols, _, _ := apiserverflag.TerminalSize(cmd.OutOrStdout()) + cmd.SetUsageFunc(func(cmd *cobra.Command) error { + fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine()) + apiserverflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols) + return nil + }) + cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { + fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine()) + apiserverflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols) + }) cmd.MarkFlagFilename("config", "yaml", "yml", "json") return cmd } -// run runs the scheduler. -func run(cmd *cobra.Command, args []string, opts *options.Options) error { +// runCommand runs the scheduler. +func runCommand(cmd *cobra.Command, args []string, opts *options.Options) error { verflag.PrintAndExitIfRequested() utilflag.PrintFlags(cmd.Flags()) @@ -113,7 +132,7 @@ func run(cmd *cobra.Command, args []string, opts *options.Options) error { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } - glog.Infof("Wrote configuration to: %s\n", opts.WriteConfigTo) + klog.Infof("Wrote configuration to: %s\n", opts.WriteConfigTo) } c, err := opts.Config() @@ -128,7 +147,7 @@ func run(cmd *cobra.Command, args []string, opts *options.Options) error { cc := c.Complete() // To help debugging, immediately log version - glog.Infof("Version: %+v", version.Get()) + klog.Infof("Version: %+v", version.Get()) // Apply algorithms based on feature gates. // TODO: make configurable? @@ -136,36 +155,42 @@ func run(cmd *cobra.Command, args []string, opts *options.Options) error { // Configz registration. if cz, err := configz.New("componentconfig"); err == nil { - cz.Set(c.ComponentConfig) + cz.Set(cc.ComponentConfig) } else { return fmt.Errorf("unable to register configz: %s", err) } + return Run(cc, stopCh) +} + +// Run executes the scheduler based on the given configuration. It only return on error or when stopCh is closed. +func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error { var storageClassInformer storageinformers.StorageClassInformer if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) { - storageClassInformer = c.InformerFactory.Storage().V1().StorageClasses() + storageClassInformer = cc.InformerFactory.Storage().V1().StorageClasses() } // Create the scheduler. - sched, err := scheduler.New(c.Client, - c.InformerFactory.Core().V1().Nodes(), - c.PodInformer, - c.InformerFactory.Core().V1().PersistentVolumes(), - c.InformerFactory.Core().V1().PersistentVolumeClaims(), - c.InformerFactory.Core().V1().ReplicationControllers(), - c.InformerFactory.Apps().V1().ReplicaSets(), - c.InformerFactory.Apps().V1().StatefulSets(), - c.InformerFactory.Core().V1().Services(), - c.InformerFactory.Policy().V1beta1().PodDisruptionBudgets(), + sched, err := scheduler.New(cc.Client, + cc.InformerFactory.Core().V1().Nodes(), + cc.PodInformer, + cc.InformerFactory.Core().V1().PersistentVolumes(), + cc.InformerFactory.Core().V1().PersistentVolumeClaims(), + cc.InformerFactory.Core().V1().ReplicationControllers(), + cc.InformerFactory.Apps().V1().ReplicaSets(), + cc.InformerFactory.Apps().V1().StatefulSets(), + cc.InformerFactory.Core().V1().Services(), + cc.InformerFactory.Policy().V1beta1().PodDisruptionBudgets(), storageClassInformer, - c.Recorder, - c.ComponentConfig.AlgorithmSource, - scheduler.WithName(c.ComponentConfig.SchedulerName), - scheduler.WithHardPodAffinitySymmetricWeight(c.ComponentConfig.HardPodAffinitySymmetricWeight), - scheduler.WithEquivalenceClassCacheEnabled(c.ComponentConfig.EnableContentionProfiling), - scheduler.WithPreemptionDisabled(c.ComponentConfig.DisablePreemption), - scheduler.WithPercentageOfNodesToScore(c.ComponentConfig.PercentageOfNodesToScore), - scheduler.WithBindTimeoutSeconds(*c.ComponentConfig.BindTimeoutSeconds)) + cc.Recorder, + cc.ComponentConfig.AlgorithmSource, + stopCh, + scheduler.WithName(cc.ComponentConfig.SchedulerName), + scheduler.WithHardPodAffinitySymmetricWeight(cc.ComponentConfig.HardPodAffinitySymmetricWeight), + scheduler.WithEquivalenceClassCacheEnabled(cc.ComponentConfig.EnableContentionProfiling), + scheduler.WithPreemptionDisabled(cc.ComponentConfig.DisablePreemption), + scheduler.WithPercentageOfNodesToScore(cc.ComponentConfig.PercentageOfNodesToScore), + scheduler.WithBindTimeoutSeconds(*cc.ComponentConfig.BindTimeoutSeconds)) if err != nil { return err } @@ -175,10 +200,16 @@ func run(cmd *cobra.Command, args []string, opts *options.Options) error { cc.Broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: cc.EventClient.Events("")}) } + // Setup healthz checks. + var checks []healthz.HealthzChecker + if cc.ComponentConfig.LeaderElection.LeaderElect { + checks = append(checks, cc.LeaderElection.WatchDog) + } + // Start up the healthz server. if cc.InsecureServing != nil { separateMetrics := cc.InsecureMetricsServing != nil - handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, separateMetrics), nil, nil) + handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, separateMetrics, checks...), nil, nil) if err := cc.InsecureServing.Serve(handler, 0, stopCh); err != nil { return fmt.Errorf("failed to start healthz server: %v", err) } @@ -190,7 +221,7 @@ func run(cmd *cobra.Command, args []string, opts *options.Options) error { } } if cc.SecureServing != nil { - handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, false), cc.Authentication.Authenticator, cc.Authorization.Authorizer) + handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, false, checks...), cc.Authentication.Authenticator, cc.Authorization.Authorizer) if err := cc.SecureServing.Serve(handler, 0, stopCh); err != nil { // fail early for secure handlers, removing the old error loop from above return fmt.Errorf("failed to start healthz server: %v", err) @@ -205,7 +236,7 @@ func run(cmd *cobra.Command, args []string, opts *options.Options) error { cc.InformerFactory.WaitForCacheSync(stopCh) controller.WaitForCacheSync("scheduler", stopCh, cc.PodInformer.Informer().HasSynced) - // Prepare a reusable run function. + // Prepare a reusable runCommand function. run := func(ctx context.Context) { sched.Run() <-ctx.Done() @@ -222,7 +253,7 @@ func run(cmd *cobra.Command, args []string, opts *options.Options) error { } }() - // If leader election is enabled, run via LeaderElector until done and exit. + // If leader election is enabled, runCommand via LeaderElector until done and exit. if cc.LeaderElection != nil { cc.LeaderElection.Callbacks = leaderelection.LeaderCallbacks{ OnStartedLeading: run, @@ -240,7 +271,7 @@ func run(cmd *cobra.Command, args []string, opts *options.Options) error { return fmt.Errorf("lost lease") } - // Leader election is disabled, so run inline until done. + // Leader election is disabled, so runCommand inline until done. run(ctx) return fmt.Errorf("finished without leader elect") } @@ -288,9 +319,9 @@ func newMetricsHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration) h // newHealthzServer creates a healthz server from the config, and will also // embed the metrics handler if the healthz and metrics address configurations // are the same. -func newHealthzHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, separateMetrics bool) http.Handler { +func newHealthzHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, separateMetrics bool, checks ...healthz.HealthzChecker) http.Handler { pathRecorderMux := mux.NewPathRecorderMux("kube-scheduler") - healthz.InstallHandler(pathRecorderMux) + healthz.InstallHandler(pathRecorderMux, checks...) if !separateMetrics { installMetricHandler(pathRecorderMux) } diff --git a/cmd/kube-scheduler/app/testing/BUILD b/cmd/kube-scheduler/app/testing/BUILD new file mode 100644 index 00000000000..5e4a38bd2ea --- /dev/null +++ b/cmd/kube-scheduler/app/testing/BUILD @@ -0,0 +1,32 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["testserver.go"], + importpath = "k8s.io/kubernetes/cmd/kube-scheduler/app/testing", + visibility = ["//visibility:public"], + deps = [ + "//cmd/kube-scheduler/app:go_default_library", + "//cmd/kube-scheduler/app/config:go_default_library", + "//cmd/kube-scheduler/app/options:go_default_library", + "//pkg/scheduler/algorithmprovider/defaults:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/kube-scheduler/app/testing/testserver.go b/cmd/kube-scheduler/app/testing/testserver.go new file mode 100644 index 00000000000..9ce7ec3dbe5 --- /dev/null +++ b/cmd/kube-scheduler/app/testing/testserver.go @@ -0,0 +1,186 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "io/ioutil" + "net" + "os" + "time" + + "github.com/spf13/pflag" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/cmd/kube-scheduler/app" + kubeschedulerconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" + "k8s.io/kubernetes/cmd/kube-scheduler/app/options" + + // import DefaultProvider + _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider/defaults" +) + +// TearDownFunc is to be called to tear down a test server. +type TearDownFunc func() + +// TestServer return values supplied by kube-test-ApiServer +type TestServer struct { + LoopbackClientConfig *restclient.Config // Rest client config using the magic token + Options *options.Options + Config *kubeschedulerconfig.Config + TearDownFn TearDownFunc // TearDown function + TmpDir string // Temp Dir used, by the apiserver +} + +// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie +type Logger interface { + Errorf(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) + Logf(format string, args ...interface{}) +} + +// StartTestServer starts a kube-scheduler. A rest client config and a tear-down func, +// and location of the tmpdir are returned. +// +// Note: we return a tear-down func instead of a stop channel because the later will leak temporary +// files that because Golang testing's call to os.Exit will not give a stop channel go routine +// enough time to remove temporary files. +func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { + stopCh := make(chan struct{}) + tearDown := func() { + close(stopCh) + if len(result.TmpDir) != 0 { + os.RemoveAll(result.TmpDir) + } + } + defer func() { + if result.TearDownFn == nil { + tearDown() + } + }() + + result.TmpDir, err = ioutil.TempDir("", "kube-scheduler") + if err != nil { + return result, fmt.Errorf("failed to create temp dir: %v", err) + } + + fs := pflag.NewFlagSet("test", pflag.PanicOnError) + + s, err := options.NewOptions() + if err != nil { + return TestServer{}, err + } + namedFlagSets := s.Flags() + for _, f := range namedFlagSets.FlagSets { + fs.AddFlagSet(f) + } + + fs.Parse(customFlags) + + if s.SecureServing.BindPort != 0 { + s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %v", err) + } + s.SecureServing.ServerCert.CertDirectory = result.TmpDir + + t.Logf("kube-scheduler will listen securely on port %d...", s.SecureServing.BindPort) + } + + if s.CombinedInsecureServing.BindPort != 0 { + listener, port, err := createListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %v", err) + } + s.CombinedInsecureServing.BindPort = port + s.CombinedInsecureServing.Healthz.Listener = listener + s.CombinedInsecureServing.Metrics.Listener = listener + t.Logf("kube-scheduler will listen insecurely on port %d...", s.CombinedInsecureServing.BindPort) + } + config, err := s.Config() + if err != nil { + return result, fmt.Errorf("failed to create config from options: %v", err) + } + + errCh := make(chan error) + go func(stopCh <-chan struct{}) { + if err := app.Run(config.Complete(), stopCh); err != nil { + errCh <- err + } + }(stopCh) + + t.Logf("Waiting for /healthz to be ok...") + client, err := kubernetes.NewForConfig(config.LoopbackClientConfig) + if err != nil { + return result, fmt.Errorf("failed to create a client: %v", err) + } + err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) { + select { + case err := <-errCh: + return false, err + default: + } + + result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do() + status := 0 + result.StatusCode(&status) + if status == 200 { + return true, nil + } + return false, nil + }) + if err != nil { + return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err) + } + + // from here the caller must call tearDown + result.LoopbackClientConfig = config.LoopbackClientConfig + result.Options = s + result.Config = config + result.TearDownFn = tearDown + + return result, nil +} + +// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. +func StartTestServerOrDie(t Logger, flags []string) *TestServer { + result, err := StartTestServer(t, flags) + if err == nil { + return &result + } + + t.Fatalf("failed to launch server: %v", err) + return nil +} + +func createListenerOnFreePort() (net.Listener, int, error) { + ln, err := net.Listen("tcp", ":0") + if err != nil { + return nil, 0, err + } + + // get port + tcpAddr, ok := ln.Addr().(*net.TCPAddr) + if !ok { + ln.Close() + return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String()) + } + + return ln, tcpAddr.Port, nil +} diff --git a/cmd/kube-scheduler/scheduler.go b/cmd/kube-scheduler/scheduler.go index 45e1b2617f7..5d44ed612cf 100644 --- a/cmd/kube-scheduler/scheduler.go +++ b/cmd/kube-scheduler/scheduler.go @@ -17,13 +17,13 @@ limitations under the License. package main import ( - goflag "flag" "fmt" "math/rand" "os" "time" "github.com/spf13/pflag" + utilflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/apiserver/pkg/util/logs" "k8s.io/kubernetes/cmd/kube-scheduler/app" @@ -40,7 +40,6 @@ func main() { // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the // normalize func and add the go flag set by hand. pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc) - pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) // utilflag.InitFlags() logs.InitLogs() defer logs.FlushLogs() diff --git a/cmd/kubeadm/.import-restrictions b/cmd/kubeadm/.import-restrictions index 0149df47262..988c92e551e 100644 --- a/cmd/kubeadm/.import-restrictions +++ b/cmd/kubeadm/.import-restrictions @@ -132,7 +132,6 @@ "github.com/ghodss/yaml", "github.com/gogo/protobuf/proto", "github.com/gogo/protobuf/sortkeys", - "github.com/golang/glog", "github.com/golang/groupcache/lru", "github.com/golang/protobuf/proto", "github.com/golang/protobuf/protoc-gen-go/descriptor", diff --git a/cmd/kubeadm/BUILD b/cmd/kubeadm/BUILD index 8e35ea32b0d..8e45f031df0 100644 --- a/cmd/kubeadm/BUILD +++ b/cmd/kubeadm/BUILD @@ -18,7 +18,10 @@ go_library( name = "go_default_library", srcs = ["kubeadm.go"], importpath = "k8s.io/kubernetes/cmd/kubeadm", - deps = ["//cmd/kubeadm/app:go_default_library"], + deps = [ + "//cmd/kubeadm/app:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], ) filegroup( diff --git a/cmd/kubeadm/app/BUILD b/cmd/kubeadm/app/BUILD index b3411caef40..f7fad736978 100644 --- a/cmd/kubeadm/app/BUILD +++ b/cmd/kubeadm/app/BUILD @@ -12,8 +12,8 @@ go_library( deps = [ "//cmd/kubeadm/app/cmd:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -44,7 +44,7 @@ filegroup( "//cmd/kubeadm/app/phases/etcd:all-srcs", "//cmd/kubeadm/app/phases/kubeconfig:all-srcs", "//cmd/kubeadm/app/phases/kubelet:all-srcs", - "//cmd/kubeadm/app/phases/markmaster:all-srcs", + "//cmd/kubeadm/app/phases/markcontrolplane:all-srcs", "//cmd/kubeadm/app/phases/patchnode:all-srcs", "//cmd/kubeadm/app/phases/selfhosting:all-srcs", "//cmd/kubeadm/app/phases/upgrade:all-srcs", diff --git a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go index a1768876783..414ece80b93 100644 --- a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go +++ b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go @@ -18,7 +18,6 @@ package fuzzer import ( fuzz "github.com/google/gofuzz" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -31,9 +30,9 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { return []interface{}{ fuzzInitConfiguration, fuzzClusterConfiguration, - fuzzAuditPolicyConfiguration, fuzzComponentConfigs, fuzzNodeRegistration, + fuzzDNS, fuzzLocalEtcd, fuzzNetworking, fuzzJoinConfiguration, @@ -50,9 +49,13 @@ func fuzzInitConfiguration(obj *kubeadm.InitConfiguration, c fuzz.Continue) { // More specifically: // internal with manually applied defaults -> external object : loosing ClusterConfiguration) -> internal object with automatically applied defaults obj.ClusterConfiguration = kubeadm.ClusterConfiguration{ - AuditPolicyConfiguration: kubeadm.AuditPolicyConfiguration{ - LogDir: constants.StaticPodAuditPolicyLogDir, - LogMaxAge: &v1beta1.DefaultAuditPolicyLogMaxAge, + APIServer: kubeadm.APIServer{ + TimeoutForControlPlane: &metav1.Duration{ + Duration: constants.DefaultControlPlaneTimeout, + }, + }, + DNS: kubeadm.DNS{ + Type: kubeadm.CoreDNS, }, CertificatesDir: v1beta1.DefaultCertificatesDir, ClusterName: v1beta1.DefaultClusterName, @@ -97,14 +100,17 @@ func fuzzClusterConfiguration(obj *kubeadm.ClusterConfiguration, c fuzz.Continue obj.ClusterName = "bar" obj.ImageRepository = "baz" obj.KubernetesVersion = "qux" + obj.APIServer.TimeoutForControlPlane = &metav1.Duration{ + Duration: constants.DefaultControlPlaneTimeout, + } } -func fuzzAuditPolicyConfiguration(obj *kubeadm.AuditPolicyConfiguration, c fuzz.Continue) { - c.FuzzNoCustom(obj) +func fuzzDNS(obj *kubeadm.DNS, c fuzz.Continue) { + // This is intentionally not calling c.FuzzNoCustom because DNS struct does not exists in v1alpha3 api + // (so no random value will be applied, and this is necessary for getting roundtrip passing) - // Pinning values for fields that get defaults if fuzz value is empty string or nil (thus making the round trip test fail) - obj.LogDir = "foo" - obj.LogMaxAge = new(int32) + // Pinning values for fields that get defaults if fuzz value is empty string or nil + obj.Type = kubeadm.CoreDNS } func fuzzComponentConfigs(obj *kubeadm.ComponentConfigs, c fuzz.Continue) { @@ -117,6 +123,10 @@ func fuzzLocalEtcd(obj *kubeadm.LocalEtcd, c fuzz.Continue) { // Pinning values for fields that get defaults if fuzz value is empty string or nil (thus making the round trip test fail) obj.DataDir = "foo" + + // Pinning values for fields that does not exists in v1alpha3 api + obj.ImageRepository = "" + obj.ImageTag = "" } func fuzzNetworking(obj *kubeadm.Networking, c fuzz.Continue) { @@ -132,7 +142,6 @@ func fuzzJoinConfiguration(obj *kubeadm.JoinConfiguration, c fuzz.Continue) { // Pinning values for fields that get defaults if fuzz value is empty string or nil (thus making the round trip test fail) obj.CACertPath = "foo" - obj.ClusterName = "bar" obj.Discovery = kubeadm.Discovery{ BootstrapToken: &kubeadm.BootstrapTokenDiscovery{Token: "baz"}, TLSBootstrapToken: "qux", diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 3351c919511..ebbd20d025e 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -46,8 +46,13 @@ type InitConfiguration struct { // NodeRegistration holds fields that relate to registering the new master node to the cluster NodeRegistration NodeRegistrationOptions - // APIEndpoint represents the endpoint of the instance of the API server to be deployed on this node. - APIEndpoint APIEndpoint + // LocalAPIEndpoint represents the endpoint of the API server instance that's deployed on this control plane node + // In HA setups, this differs from ClusterConfiguration.ControlPlaneEndpoint in the sense that ControlPlaneEndpoint + // is the global endpoint for the cluster, which then loadbalances the requests to each individual API server. This + // configuration object lets you customize what IP/DNS name and port the local API server advertises it's accessible + // on. By default, kubeadm tries to auto-detect the IP of the default interface and use that, but in case that process + // fails you may set the desired value here. + LocalAPIEndpoint APIEndpoint } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -90,10 +95,16 @@ type ClusterConfiguration struct { // Scheduler contains extra settings for the scheduler control plane component Scheduler ControlPlaneComponent + // DNS defines the options for the DNS add-on installed in the cluster. + DNS DNS + // CertificatesDir specifies where to store or look for all required certificates. CertificatesDir string - // ImageRepository is the container registry to pull control plane images from. + // ImageRepository sets the container registry to pull images from. + // If empty, `k8s.gcr.io` will be used by default; in case of kubernetes version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`) + // `gcr.io/kubernetes-ci-images` will be used as a default for control plane components and for kube-proxy, while `k8s.gcr.io` + // will be used for all the other images. ImageRepository string // CIImageRepository is the container registry for core images generated by CI. @@ -101,12 +112,8 @@ type ClusterConfiguration struct { // +k8s:conversion-gen=false CIImageRepository string - // UnifiedControlPlaneImage specifies if a specific container image should be - // used for all control plane components. - UnifiedControlPlaneImage string - - // AuditPolicyConfiguration defines the options for the api server audit system. - AuditPolicyConfiguration AuditPolicyConfiguration + // UseHyperKubeImage controls if hyperkube should be used for Kubernetes components instead of their respective separate images + UseHyperKubeImage bool // FeatureGates enabled by the user. FeatureGates map[string]bool @@ -118,6 +125,8 @@ type ClusterConfiguration struct { // ControlPlaneComponent holds settings common to control plane component of the cluster type ControlPlaneComponent struct { // ExtraArgs is an extra set of flags to pass to the control plane component. + // TODO: This is temporary and ideally we would like to switch all components to + // use ComponentConfig + ConfigMaps. ExtraArgs map[string]string // ExtraVolumes is an extra set of host volumes, mounted to the control plane component. @@ -130,6 +139,43 @@ type APIServer struct { // CertSANs sets extra Subject Alternative Names for the API Server signing cert. CertSANs []string + + // TimeoutForControlPlane controls the timeout that we use for API server to appear + TimeoutForControlPlane *metav1.Duration +} + +// DNSAddOnType defines string identifying DNS add-on types +type DNSAddOnType string + +const ( + // CoreDNS add-on type + CoreDNS DNSAddOnType = "CoreDNS" + + // KubeDNS add-on type + KubeDNS DNSAddOnType = "kube-dns" +) + +// DNS defines the DNS addon that should be used in the cluster +type DNS struct { + // Type defines the DNS add-on to be used + Type DNSAddOnType + + // ImageMeta allows to customize the image used for the DNS component + ImageMeta `json:",inline"` +} + +// ImageMeta allows to customize the image used for components that are not +// originated from the Kubernetes/Kubernetes release process +type ImageMeta struct { + // ImageRepository sets the container registry to pull images from. + // if not set, the ImageRepository defined in ClusterConfiguration will be used instead. + ImageRepository string + + // ImageTag allows to specify a tag for the image. + // In case this value is set, kubeadm does not change automatically the version of the above components during upgrades. + ImageTag string + + //TODO: evaluate if we need also a ImageName based on user feedbacks } // ComponentConfigs holds known internal ComponentConfig types for other components @@ -232,11 +278,8 @@ type Etcd struct { // LocalEtcd describes that kubeadm should run an etcd cluster locally type LocalEtcd struct { - - // Image specifies which container image to use for running etcd. - // If empty, automatically populated by kubeadm using the image - // repository and default etcd version. - Image string + // ImageMeta allows to customize the container used for etcd + ImageMeta `json:",inline"` // DataDir is the directory etcd will place its data. // Defaults to "/var/lib/etcd". @@ -283,18 +326,15 @@ type JoinConfiguration struct { // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process Discovery Discovery - // The cluster name - ClusterName string + // ControlPlane defines the additional control plane instance to be deployed on the joining node. + // If nil, no additional control plane instance will be deployed. + ControlPlane *JoinControlPlane +} - // ControlPlane flag specifies that the joining node should host an additional - // control plane instance. - ControlPlane bool - - // APIEndpoint represents the endpoint of the instance of the API server eventually to be deployed on this node. - APIEndpoint APIEndpoint - - // FeatureGates enabled by the user. - FeatureGates map[string]bool +// JoinControlPlane contains elements describing an additional control plane instance to be deployed on the joining node. +type JoinControlPlane struct { + // LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node. + LocalAPIEndpoint APIEndpoint } // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process @@ -369,23 +409,12 @@ type HostPathMount struct { HostPath string // MountPath is the path inside the pod where hostPath will be mounted. MountPath string - // Writable controls write access to the volume - Writable bool + // ReadOnly controls write access to the volume + ReadOnly bool // PathType is the type of the HostPath. PathType v1.HostPathType } -// AuditPolicyConfiguration holds the options for configuring the api server audit policy. -type AuditPolicyConfiguration struct { - // Path is the local path to an audit policy. - Path string - // LogDir is the local path to the directory where logs should be stored. - LogDir string - // LogMaxAge is the number of days logs will be stored for. 0 indicates forever. - LogMaxAge *int32 - //TODO(chuckha) add other options for audit policy. -} - // CommonConfiguration defines the list of common configuration elements and the getter // methods that must exist for both the InitConfiguration and JoinConfiguration objects. // This is used internally to deduplicate the kubeadm preflight checks. diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/BUILD b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/BUILD index 873fd4bbe03..61e591e9b67 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/BUILD +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/BUILD @@ -20,6 +20,9 @@ go_library( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", + "//cmd/kubeadm/app/images:go_default_library", + "//cmd/kubeadm/app/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library", @@ -47,7 +50,13 @@ filegroup( go_test( name = "go_default_test", - srcs = ["bootstraptokenstring_test.go"], + srcs = [ + "bootstraptokenstring_test.go", + "conversion_test.go", + ], embed = [":go_default_library"], - deps = ["//vendor/github.com/pkg/errors:go_default_library"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", + ], ) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion.go index 4ff1cdebfb7..d2ed1c15d5c 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion.go @@ -17,17 +17,48 @@ limitations under the License. package v1alpha3 import ( - "unsafe" + "regexp" + "strings" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/conversion" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" + "k8s.io/kubernetes/cmd/kubeadm/app/images" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) +var imageRegEx = regexp.MustCompile(`(?P.+/)(?P[^:]+)(?P:.+)`) + +func Convert_v1alpha3_InitConfiguration_To_kubeadm_InitConfiguration(in *InitConfiguration, out *kubeadm.InitConfiguration, s conversion.Scope) error { + if err := autoConvert_v1alpha3_InitConfiguration_To_kubeadm_InitConfiguration(in, out, s); err != nil { + return err + } + return Convert_v1alpha3_APIEndpoint_To_kubeadm_APIEndpoint(&in.APIEndpoint, &out.LocalAPIEndpoint, s) +} + +func Convert_kubeadm_InitConfiguration_To_v1alpha3_InitConfiguration(in *kubeadm.InitConfiguration, out *InitConfiguration, s conversion.Scope) error { + if err := autoConvert_kubeadm_InitConfiguration_To_v1alpha3_InitConfiguration(in, out, s); err != nil { + return err + } + return Convert_kubeadm_APIEndpoint_To_v1alpha3_APIEndpoint(&in.LocalAPIEndpoint, &out.APIEndpoint, s) +} + func Convert_v1alpha3_JoinConfiguration_To_kubeadm_JoinConfiguration(in *JoinConfiguration, out *kubeadm.JoinConfiguration, s conversion.Scope) error { if err := autoConvert_v1alpha3_JoinConfiguration_To_kubeadm_JoinConfiguration(in, out, s); err != nil { return err } + if len(in.ClusterName) != 0 { + return errors.New("clusterName has been removed from JoinConfiguration and clusterName from ClusterConfiguration will be used instead. Please cleanup JoinConfiguration.ClusterName fields") + } + + if len(in.FeatureGates) != 0 { + return errors.New("featureGates has been removed from JoinConfiguration and featureGates from ClusterConfiguration will be used instead. Please cleanup JoinConfiguration.FeatureGates fields") + } + out.Discovery.Timeout = in.DiscoveryTimeout if len(in.TLSBootstrapToken) != 0 { @@ -55,6 +86,13 @@ func Convert_v1alpha3_JoinConfiguration_To_kubeadm_JoinConfiguration(in *JoinCon } } + if in.ControlPlane == true { + out.ControlPlane = &kubeadm.JoinControlPlane{} + if err := autoConvert_v1alpha3_APIEndpoint_To_kubeadm_APIEndpoint(&in.APIEndpoint, &out.ControlPlane.LocalAPIEndpoint, s); err != nil { + return err + } + } + return nil } @@ -76,6 +114,13 @@ func Convert_kubeadm_JoinConfiguration_To_v1alpha3_JoinConfiguration(in *kubeadm out.DiscoveryFile = in.Discovery.File.KubeConfigPath } + if in.ControlPlane != nil { + out.ControlPlane = true + if err := autoConvert_kubeadm_APIEndpoint_To_v1alpha3_APIEndpoint(&in.ControlPlane.LocalAPIEndpoint, &out.APIEndpoint, s); err != nil { + return err + } + } + return nil } @@ -84,33 +129,187 @@ func Convert_v1alpha3_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in *C return err } + if len(in.AuditPolicyConfiguration.Path) > 0 { + return errors.New("AuditPolicyConfiguration has been removed from ClusterConfiguration. Please cleanup ClusterConfiguration.AuditPolicyConfiguration fields") + } + out.APIServer.ExtraArgs = in.APIServerExtraArgs - out.APIServer.ExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.APIServerExtraVolumes)) out.APIServer.CertSANs = in.APIServerCertSANs + out.APIServer.TimeoutForControlPlane = &metav1.Duration{ + Duration: constants.DefaultControlPlaneTimeout, + } + if err := convertSlice_v1alpha3_HostPathMount_To_kubeadm_HostPathMount(&in.APIServerExtraVolumes, &out.APIServer.ExtraVolumes, s); err != nil { + return err + } out.ControllerManager.ExtraArgs = in.ControllerManagerExtraArgs - out.ControllerManager.ExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.ControllerManagerExtraVolumes)) + if err := convertSlice_v1alpha3_HostPathMount_To_kubeadm_HostPathMount(&in.ControllerManagerExtraVolumes, &out.ControllerManager.ExtraVolumes, s); err != nil { + return err + } out.Scheduler.ExtraArgs = in.SchedulerExtraArgs - out.Scheduler.ExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.SchedulerExtraVolumes)) + if err := convertSlice_v1alpha3_HostPathMount_To_kubeadm_HostPathMount(&in.SchedulerExtraVolumes, &out.Scheduler.ExtraVolumes, s); err != nil { + return err + } + + if err := Convert_v1alpha3_UnifiedControlPlaneImage_To_kubeadm_UseHyperKubeImage(in, out); err != nil { + return err + } + + // converting v1alpha3 featureGate CoreDNS to internal DNS.Type + if features.Enabled(in.FeatureGates, features.CoreDNS) { + out.DNS.Type = kubeadm.CoreDNS + } else { + out.DNS.Type = kubeadm.KubeDNS + } + delete(out.FeatureGates, features.CoreDNS) return nil } +func Convert_v1alpha3_UnifiedControlPlaneImage_To_kubeadm_UseHyperKubeImage(in *ClusterConfiguration, out *kubeadm.ClusterConfiguration) error { + if len(in.UnifiedControlPlaneImage) == 0 { + out.UseHyperKubeImage = false + return nil + } + + k8sImageTag := kubeadmutil.KubernetesVersionToImageTag(in.KubernetesVersion) + expectedImage := images.GetGenericImage(in.ImageRepository, constants.HyperKube, k8sImageTag) + if expectedImage == in.UnifiedControlPlaneImage { + out.UseHyperKubeImage = true + return nil + } + + return errors.Errorf("cannot convert unifiedControlPlaneImage=%q to useHyperKubeImage", in.UnifiedControlPlaneImage) +} + func Convert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(in *kubeadm.ClusterConfiguration, out *ClusterConfiguration, s conversion.Scope) error { if err := autoConvert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(in, out, s); err != nil { return err } out.APIServerExtraArgs = in.APIServer.ExtraArgs - out.APIServerExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.APIServer.ExtraVolumes)) out.APIServerCertSANs = in.APIServer.CertSANs + if err := convertSlice_kubeadm_HostPathMount_To_v1alpha3_HostPathMount(&in.APIServer.ExtraVolumes, &out.APIServerExtraVolumes, s); err != nil { + return err + } out.ControllerManagerExtraArgs = in.ControllerManager.ExtraArgs - out.ControllerManagerExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.ControllerManager.ExtraVolumes)) + if err := convertSlice_kubeadm_HostPathMount_To_v1alpha3_HostPathMount(&in.ControllerManager.ExtraVolumes, &out.ControllerManagerExtraVolumes, s); err != nil { + return err + } out.SchedulerExtraArgs = in.Scheduler.ExtraArgs - out.SchedulerExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.Scheduler.ExtraVolumes)) + if err := convertSlice_kubeadm_HostPathMount_To_v1alpha3_HostPathMount(&in.Scheduler.ExtraVolumes, &out.SchedulerExtraVolumes, s); err != nil { + return err + } + + if in.UseHyperKubeImage { + out.UnifiedControlPlaneImage = images.GetKubernetesImage("", in) + } else { + out.UnifiedControlPlaneImage = "" + } + + // converting internal DNS.Type to v1alpha3 featureGate CoreDNS (this is only for getting roundtrip passing, but it is never used in reality) + if out.FeatureGates == nil { + out.FeatureGates = map[string]bool{} + } + if in.DNS.Type == kubeadm.KubeDNS { + out.FeatureGates[features.CoreDNS] = false + } + + return nil +} + +func Convert_v1alpha3_HostPathMount_To_kubeadm_HostPathMount(in *HostPathMount, out *kubeadm.HostPathMount, s conversion.Scope) error { + if err := autoConvert_v1alpha3_HostPathMount_To_kubeadm_HostPathMount(in, out, s); err != nil { + return err + } + + out.ReadOnly = !in.Writable + return nil +} + +func Convert_kubeadm_HostPathMount_To_v1alpha3_HostPathMount(in *kubeadm.HostPathMount, out *HostPathMount, s conversion.Scope) error { + if err := autoConvert_kubeadm_HostPathMount_To_v1alpha3_HostPathMount(in, out, s); err != nil { + return err + } + + out.Writable = !in.ReadOnly + return nil +} + +func convertSlice_v1alpha3_HostPathMount_To_kubeadm_HostPathMount(in *[]HostPathMount, out *[]kubeadm.HostPathMount, s conversion.Scope) error { + if *in != nil { + *out = make([]kubeadm.HostPathMount, len(*in)) + for i := range *in { + if err := Convert_v1alpha3_HostPathMount_To_kubeadm_HostPathMount(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + *out = nil + } + return nil +} + +func convertSlice_kubeadm_HostPathMount_To_v1alpha3_HostPathMount(in *[]kubeadm.HostPathMount, out *[]HostPathMount, s conversion.Scope) error { + if *in != nil { + *out = make([]HostPathMount, len(*in)) + for i := range *in { + if err := Convert_kubeadm_HostPathMount_To_v1alpha3_HostPathMount(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + *out = nil + } + + return nil +} + +func Convert_v1alpha3_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm.LocalEtcd, s conversion.Scope) error { + if err := autoConvert_v1alpha3_LocalEtcd_To_kubeadm_LocalEtcd(in, out, s); err != nil { + return err + } + + var err error + out.ImageMeta, err = etcdImageToImageMeta(in.Image) + return err +} + +func etcdImageToImageMeta(image string) (kubeadm.ImageMeta, error) { + // empty image -> empty image meta + if image == "" { + return kubeadm.ImageMeta{}, nil + } + + matches := imageRegEx.FindStringSubmatch(image) + if len(matches) != 4 { + return kubeadm.ImageMeta{}, errors.New("Conversion Error: kubeadm does not support converting v1alpha3 configurations with etcd image without explicit repository or tag definition. Please fix the image name") + } + + imageRepository := strings.TrimSuffix(matches[1], "/") + imageName := matches[2] + imageTag := strings.TrimPrefix(matches[3], ":") + + if imageName != constants.Etcd { + return kubeadm.ImageMeta{}, errors.New("Conversion Error: kubeadm does not support converting v1alpha3 configurations with etcd imageName different than etcd. Please fix the image name") + } + + return kubeadm.ImageMeta{ + ImageRepository: imageRepository, + ImageTag: imageTag, + }, nil +} + +func Convert_kubeadm_LocalEtcd_To_v1alpha3_LocalEtcd(in *kubeadm.LocalEtcd, out *LocalEtcd, s conversion.Scope) error { + if err := autoConvert_kubeadm_LocalEtcd_To_v1alpha3_LocalEtcd(in, out, s); err != nil { + return err + } + + // converting internal LocalEtcd.ImageMeta to v1alpha3 LocalEtcd.Image (this is only for getting roundtrip passing, but it is + // never used in reality) return nil } diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion_test.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion_test.go new file mode 100644 index 00000000000..1891e506b76 --- /dev/null +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion_test.go @@ -0,0 +1,214 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha3 + +import ( + "testing" + + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" +) + +func TestJoinConfigurationConversion(t *testing.T) { + testcases := map[string]struct { + old *JoinConfiguration + expectedError bool + }{ + "conversion succeeds": { + old: &JoinConfiguration{}, + expectedError: false, + }, + "cluster name fails to be converted": { + old: &JoinConfiguration{ + ClusterName: "kubernetes", + }, + expectedError: true, + }, + "feature gates fails to be converted": { + old: &JoinConfiguration{ + FeatureGates: map[string]bool{ + "someGate": true, + }, + }, + expectedError: true, + }, + } + for _, tc := range testcases { + internal := &kubeadm.JoinConfiguration{} + err := Convert_v1alpha3_JoinConfiguration_To_kubeadm_JoinConfiguration(tc.old, internal, nil) + if (err != nil) != tc.expectedError { + t.Errorf("ImageToImageMeta returned unexpected error: %v, saw: %v", tc.expectedError, (err != nil)) + return + } + } +} + +func TestInitConfigurationConversion(t *testing.T) { + testcases := map[string]struct { + old *InitConfiguration + expectedErr bool + }{ + "conversion succeeds": { + old: &InitConfiguration{}, + expectedErr: false, + }, + "feature gates fails to be converted": { + old: &InitConfiguration{ + ClusterConfiguration: ClusterConfiguration{ + AuditPolicyConfiguration: AuditPolicyConfiguration{ + Path: "test", + }, + }, + }, + expectedErr: true, + }, + } + for _, tc := range testcases { + internal := &kubeadm.InitConfiguration{} + err := Convert_v1alpha3_InitConfiguration_To_kubeadm_InitConfiguration(tc.old, internal, nil) + if (err != nil) != tc.expectedErr { + t.Errorf("no error was expected but '%s' was found", err) + } + } +} + +func TestConvertToUseHyperKubeImage(t *testing.T) { + tests := []struct { + desc string + in *ClusterConfiguration + useHyperKubeImage bool + expectedErr bool + }{ + { + desc: "unset UnifiedControlPlaneImage sets UseHyperKubeImage to false", + in: &ClusterConfiguration{}, + useHyperKubeImage: false, + expectedErr: false, + }, + { + desc: "matching UnifiedControlPlaneImage sets UseHyperKubeImage to true", + in: &ClusterConfiguration{ + ImageRepository: "k8s.gcr.io", + KubernetesVersion: "v1.12.2", + UnifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.12.2", + }, + useHyperKubeImage: true, + expectedErr: false, + }, + { + desc: "mismatching UnifiedControlPlaneImage tag causes an error", + in: &ClusterConfiguration{ + ImageRepository: "k8s.gcr.io", + KubernetesVersion: "v1.12.0", + UnifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.12.2", + }, + expectedErr: true, + }, + { + desc: "mismatching UnifiedControlPlaneImage repo causes an error", + in: &ClusterConfiguration{ + ImageRepository: "my.repo", + KubernetesVersion: "v1.12.2", + UnifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.12.2", + }, + expectedErr: true, + }, + { + desc: "mismatching UnifiedControlPlaneImage image name causes an error", + in: &ClusterConfiguration{ + ImageRepository: "k8s.gcr.io", + KubernetesVersion: "v1.12.2", + UnifiedControlPlaneImage: "k8s.gcr.io/otherimage:v1.12.2", + }, + expectedErr: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + out := &kubeadm.ClusterConfiguration{} + err := Convert_v1alpha3_UnifiedControlPlaneImage_To_kubeadm_UseHyperKubeImage(test.in, out) + if test.expectedErr { + if err == nil { + t.Fatalf("unexpected success, UseHyperKubeImage: %t", out.UseHyperKubeImage) + } + } else { + if err != nil { + t.Fatalf("unexpected failure: %v", err) + } + if out.UseHyperKubeImage != test.useHyperKubeImage { + t.Fatalf("mismatching result from conversion:\n\tExpected: %t\n\tReceived: %t", test.useHyperKubeImage, out.UseHyperKubeImage) + } + } + }) + } +} + +func TestEtcdImageToImageMeta(t *testing.T) { + tests := []struct { + name string + image string + expectedImageMeta kubeadm.ImageMeta + expectedError bool + }{ + { + name: "Empty image -> Empty image meta", + image: "", + expectedImageMeta: kubeadm.ImageMeta{ + ImageRepository: "", + ImageTag: "", + }, + }, + { + name: "image with tag and repository", + image: "custom.repo/etcd:custom.tag", + expectedImageMeta: kubeadm.ImageMeta{ + ImageRepository: "custom.repo", + ImageTag: "custom.tag", + }, + }, + { + name: "image with custom imageName", + image: "real.repo/custom-image-name-for-etcd:real.tag", + expectedError: true, + }, + { + name: "image without repository", + image: "etcd:real.tag", + expectedError: true, + }, + { + name: "image without tag", + image: "real.repo/etcd", + expectedError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ret, err := etcdImageToImageMeta(test.image) + + if (err != nil) != test.expectedError { + t.Errorf("etcdImageToImageMeta returned unexpected error: %v, saw: %v", test.expectedError, (err != nil)) + return + } + + if ret != test.expectedImageMeta { + t.Errorf("etcdImageToImageMeta returned unexpected ImageMeta: %v, saw: %v", test.expectedImageMeta, ret) + } + }) + } +} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/defaults.go index c833b82deeb..087b5b01b53 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/defaults.go @@ -138,9 +138,6 @@ func SetDefaults_JoinConfiguration(obj *JoinConfiguration) { Duration: DefaultDiscoveryTimeout, } } - if obj.ClusterName == "" { - obj.ClusterName = DefaultClusterName - } SetDefaults_NodeRegistrationOptions(&obj.NodeRegistration) SetDefaults_APIEndpoint(&obj.APIEndpoint) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/defaults_unix.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/defaults_unix.go index 8a8962872fd..7e60d20b080 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/defaults_unix.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/defaults_unix.go @@ -21,7 +21,7 @@ package v1alpha3 const ( // DefaultCACertPath defines default location of CA certificate on Linux DefaultCACertPath = "/etc/kubernetes/pki/ca.crt" - // DefaultSocketUrlScheme defines default socket url prefix + // DefaultUrlScheme defines default socket url prefix DefaultUrlScheme = "unix" // DefaultCRISocket defines the default cri socket DefaultCRISocket = "/var/run/dockershim.sock" diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/zz_generated.conversion.go index 1cc66d9f3bf..5ecc75eaf67 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/zz_generated.conversion.go @@ -47,16 +47,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*AuditPolicyConfiguration)(nil), (*kubeadm.AuditPolicyConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha3_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(a.(*AuditPolicyConfiguration), b.(*kubeadm.AuditPolicyConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.AuditPolicyConfiguration)(nil), (*AuditPolicyConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_AuditPolicyConfiguration_To_v1alpha3_AuditPolicyConfiguration(a.(*kubeadm.AuditPolicyConfiguration), b.(*AuditPolicyConfiguration), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*BootstrapToken)(nil), (*kubeadm.BootstrapToken)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_BootstrapToken_To_kubeadm_BootstrapToken(a.(*BootstrapToken), b.(*kubeadm.BootstrapToken), scope) }); err != nil { @@ -182,21 +172,51 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*kubeadm.HostPathMount)(nil), (*HostPathMount)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kubeadm_HostPathMount_To_v1alpha3_HostPathMount(a.(*kubeadm.HostPathMount), b.(*HostPathMount), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*kubeadm.InitConfiguration)(nil), (*InitConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kubeadm_InitConfiguration_To_v1alpha3_InitConfiguration(a.(*kubeadm.InitConfiguration), b.(*InitConfiguration), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*kubeadm.JoinConfiguration)(nil), (*JoinConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_kubeadm_JoinConfiguration_To_v1alpha3_JoinConfiguration(a.(*kubeadm.JoinConfiguration), b.(*JoinConfiguration), scope) }); err != nil { return err } + if err := s.AddConversionFunc((*kubeadm.LocalEtcd)(nil), (*LocalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kubeadm_LocalEtcd_To_v1alpha3_LocalEtcd(a.(*kubeadm.LocalEtcd), b.(*LocalEtcd), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*ClusterConfiguration)(nil), (*kubeadm.ClusterConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_ClusterConfiguration_To_kubeadm_ClusterConfiguration(a.(*ClusterConfiguration), b.(*kubeadm.ClusterConfiguration), scope) }); err != nil { return err } + if err := s.AddConversionFunc((*HostPathMount)(nil), (*kubeadm.HostPathMount)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_HostPathMount_To_kubeadm_HostPathMount(a.(*HostPathMount), b.(*kubeadm.HostPathMount), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*InitConfiguration)(nil), (*kubeadm.InitConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_InitConfiguration_To_kubeadm_InitConfiguration(a.(*InitConfiguration), b.(*kubeadm.InitConfiguration), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*JoinConfiguration)(nil), (*kubeadm.JoinConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_JoinConfiguration_To_kubeadm_JoinConfiguration(a.(*JoinConfiguration), b.(*kubeadm.JoinConfiguration), scope) }); err != nil { return err } + if err := s.AddConversionFunc((*LocalEtcd)(nil), (*kubeadm.LocalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_LocalEtcd_To_kubeadm_LocalEtcd(a.(*LocalEtcd), b.(*kubeadm.LocalEtcd), scope) + }); err != nil { + return err + } return nil } @@ -222,30 +242,6 @@ func Convert_kubeadm_APIEndpoint_To_v1alpha3_APIEndpoint(in *kubeadm.APIEndpoint return autoConvert_kubeadm_APIEndpoint_To_v1alpha3_APIEndpoint(in, out, s) } -func autoConvert_v1alpha3_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(in *AuditPolicyConfiguration, out *kubeadm.AuditPolicyConfiguration, s conversion.Scope) error { - out.Path = in.Path - out.LogDir = in.LogDir - out.LogMaxAge = (*int32)(unsafe.Pointer(in.LogMaxAge)) - return nil -} - -// Convert_v1alpha3_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration is an autogenerated conversion function. -func Convert_v1alpha3_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(in *AuditPolicyConfiguration, out *kubeadm.AuditPolicyConfiguration, s conversion.Scope) error { - return autoConvert_v1alpha3_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(in, out, s) -} - -func autoConvert_kubeadm_AuditPolicyConfiguration_To_v1alpha3_AuditPolicyConfiguration(in *kubeadm.AuditPolicyConfiguration, out *AuditPolicyConfiguration, s conversion.Scope) error { - out.Path = in.Path - out.LogDir = in.LogDir - out.LogMaxAge = (*int32)(unsafe.Pointer(in.LogMaxAge)) - return nil -} - -// Convert_kubeadm_AuditPolicyConfiguration_To_v1alpha3_AuditPolicyConfiguration is an autogenerated conversion function. -func Convert_kubeadm_AuditPolicyConfiguration_To_v1alpha3_AuditPolicyConfiguration(in *kubeadm.AuditPolicyConfiguration, out *AuditPolicyConfiguration, s conversion.Scope) error { - return autoConvert_kubeadm_AuditPolicyConfiguration_To_v1alpha3_AuditPolicyConfiguration(in, out, s) -} - func autoConvert_v1alpha3_BootstrapToken_To_kubeadm_BootstrapToken(in *BootstrapToken, out *kubeadm.BootstrapToken, s conversion.Scope) error { out.Token = (*kubeadm.BootstrapTokenString)(unsafe.Pointer(in.Token)) out.Description = in.Description @@ -316,10 +312,8 @@ func autoConvert_v1alpha3_ClusterConfiguration_To_kubeadm_ClusterConfiguration(i // WARNING: in.APIServerCertSANs requires manual conversion: does not exist in peer-type out.CertificatesDir = in.CertificatesDir out.ImageRepository = in.ImageRepository - out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage - if err := Convert_v1alpha3_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil { - return err - } + // WARNING: in.UnifiedControlPlaneImage requires manual conversion: does not exist in peer-type + // WARNING: in.AuditPolicyConfiguration requires manual conversion: does not exist in peer-type out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) out.ClusterName = in.ClusterName return nil @@ -338,13 +332,11 @@ func autoConvert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(i // WARNING: in.APIServer requires manual conversion: does not exist in peer-type // WARNING: in.ControllerManager requires manual conversion: does not exist in peer-type // WARNING: in.Scheduler requires manual conversion: does not exist in peer-type + // WARNING: in.DNS requires manual conversion: does not exist in peer-type out.CertificatesDir = in.CertificatesDir out.ImageRepository = in.ImageRepository // INFO: in.CIImageRepository opted out of conversion generation - out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage - if err := Convert_kubeadm_AuditPolicyConfiguration_To_v1alpha3_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil { - return err - } + // WARNING: in.UseHyperKubeImage requires manual conversion: does not exist in peer-type out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) out.ClusterName = in.ClusterName return nil @@ -371,7 +363,15 @@ func Convert_kubeadm_ClusterStatus_To_v1alpha3_ClusterStatus(in *kubeadm.Cluster } func autoConvert_v1alpha3_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s conversion.Scope) error { - out.Local = (*kubeadm.LocalEtcd)(unsafe.Pointer(in.Local)) + if in.Local != nil { + in, out := &in.Local, &out.Local + *out = new(kubeadm.LocalEtcd) + if err := Convert_v1alpha3_LocalEtcd_To_kubeadm_LocalEtcd(*in, *out, s); err != nil { + return err + } + } else { + out.Local = nil + } out.External = (*kubeadm.ExternalEtcd)(unsafe.Pointer(in.External)) return nil } @@ -382,7 +382,15 @@ func Convert_v1alpha3_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s conver } func autoConvert_kubeadm_Etcd_To_v1alpha3_Etcd(in *kubeadm.Etcd, out *Etcd, s conversion.Scope) error { - out.Local = (*LocalEtcd)(unsafe.Pointer(in.Local)) + if in.Local != nil { + in, out := &in.Local, &out.Local + *out = new(LocalEtcd) + if err := Convert_kubeadm_LocalEtcd_To_v1alpha3_LocalEtcd(*in, *out, s); err != nil { + return err + } + } else { + out.Local = nil + } out.External = (*ExternalEtcd)(unsafe.Pointer(in.External)) return nil } @@ -422,30 +430,20 @@ func autoConvert_v1alpha3_HostPathMount_To_kubeadm_HostPathMount(in *HostPathMou out.Name = in.Name out.HostPath = in.HostPath out.MountPath = in.MountPath - out.Writable = in.Writable + // WARNING: in.Writable requires manual conversion: does not exist in peer-type out.PathType = corev1.HostPathType(in.PathType) return nil } -// Convert_v1alpha3_HostPathMount_To_kubeadm_HostPathMount is an autogenerated conversion function. -func Convert_v1alpha3_HostPathMount_To_kubeadm_HostPathMount(in *HostPathMount, out *kubeadm.HostPathMount, s conversion.Scope) error { - return autoConvert_v1alpha3_HostPathMount_To_kubeadm_HostPathMount(in, out, s) -} - func autoConvert_kubeadm_HostPathMount_To_v1alpha3_HostPathMount(in *kubeadm.HostPathMount, out *HostPathMount, s conversion.Scope) error { out.Name = in.Name out.HostPath = in.HostPath out.MountPath = in.MountPath - out.Writable = in.Writable + // WARNING: in.ReadOnly requires manual conversion: does not exist in peer-type out.PathType = corev1.HostPathType(in.PathType) return nil } -// Convert_kubeadm_HostPathMount_To_v1alpha3_HostPathMount is an autogenerated conversion function. -func Convert_kubeadm_HostPathMount_To_v1alpha3_HostPathMount(in *kubeadm.HostPathMount, out *HostPathMount, s conversion.Scope) error { - return autoConvert_kubeadm_HostPathMount_To_v1alpha3_HostPathMount(in, out, s) -} - func autoConvert_v1alpha3_InitConfiguration_To_kubeadm_InitConfiguration(in *InitConfiguration, out *kubeadm.InitConfiguration, s conversion.Scope) error { if err := Convert_v1alpha3_ClusterConfiguration_To_kubeadm_ClusterConfiguration(&in.ClusterConfiguration, &out.ClusterConfiguration, s); err != nil { return err @@ -454,17 +452,10 @@ func autoConvert_v1alpha3_InitConfiguration_To_kubeadm_InitConfiguration(in *Ini if err := Convert_v1alpha3_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { return err } - if err := Convert_v1alpha3_APIEndpoint_To_kubeadm_APIEndpoint(&in.APIEndpoint, &out.APIEndpoint, s); err != nil { - return err - } + // WARNING: in.APIEndpoint requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha3_InitConfiguration_To_kubeadm_InitConfiguration is an autogenerated conversion function. -func Convert_v1alpha3_InitConfiguration_To_kubeadm_InitConfiguration(in *InitConfiguration, out *kubeadm.InitConfiguration, s conversion.Scope) error { - return autoConvert_v1alpha3_InitConfiguration_To_kubeadm_InitConfiguration(in, out, s) -} - func autoConvert_kubeadm_InitConfiguration_To_v1alpha3_InitConfiguration(in *kubeadm.InitConfiguration, out *InitConfiguration, s conversion.Scope) error { if err := Convert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(&in.ClusterConfiguration, &out.ClusterConfiguration, s); err != nil { return err @@ -473,17 +464,10 @@ func autoConvert_kubeadm_InitConfiguration_To_v1alpha3_InitConfiguration(in *kub if err := Convert_kubeadm_NodeRegistrationOptions_To_v1alpha3_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { return err } - if err := Convert_kubeadm_APIEndpoint_To_v1alpha3_APIEndpoint(&in.APIEndpoint, &out.APIEndpoint, s); err != nil { - return err - } + // WARNING: in.LocalAPIEndpoint requires manual conversion: does not exist in peer-type return nil } -// Convert_kubeadm_InitConfiguration_To_v1alpha3_InitConfiguration is an autogenerated conversion function. -func Convert_kubeadm_InitConfiguration_To_v1alpha3_InitConfiguration(in *kubeadm.InitConfiguration, out *InitConfiguration, s conversion.Scope) error { - return autoConvert_kubeadm_InitConfiguration_To_v1alpha3_InitConfiguration(in, out, s) -} - func autoConvert_v1alpha3_JoinConfiguration_To_kubeadm_JoinConfiguration(in *JoinConfiguration, out *kubeadm.JoinConfiguration, s conversion.Scope) error { if err := Convert_v1alpha3_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { return err @@ -495,14 +479,12 @@ func autoConvert_v1alpha3_JoinConfiguration_To_kubeadm_JoinConfiguration(in *Joi // WARNING: in.DiscoveryTimeout requires manual conversion: does not exist in peer-type // WARNING: in.TLSBootstrapToken requires manual conversion: does not exist in peer-type // WARNING: in.Token requires manual conversion: does not exist in peer-type - out.ClusterName = in.ClusterName + // WARNING: in.ClusterName requires manual conversion: does not exist in peer-type // WARNING: in.DiscoveryTokenCACertHashes requires manual conversion: does not exist in peer-type // WARNING: in.DiscoveryTokenUnsafeSkipCAVerification requires manual conversion: does not exist in peer-type - out.ControlPlane = in.ControlPlane - if err := Convert_v1alpha3_APIEndpoint_To_kubeadm_APIEndpoint(&in.APIEndpoint, &out.APIEndpoint, s); err != nil { - return err - } - out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) + // WARNING: in.ControlPlane requires manual conversion: inconvertible types (bool vs *k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm.JoinControlPlane) + // WARNING: in.APIEndpoint requires manual conversion: does not exist in peer-type + // WARNING: in.FeatureGates requires manual conversion: does not exist in peer-type return nil } @@ -512,17 +494,12 @@ func autoConvert_kubeadm_JoinConfiguration_To_v1alpha3_JoinConfiguration(in *kub } out.CACertPath = in.CACertPath // WARNING: in.Discovery requires manual conversion: does not exist in peer-type - out.ClusterName = in.ClusterName - out.ControlPlane = in.ControlPlane - if err := Convert_kubeadm_APIEndpoint_To_v1alpha3_APIEndpoint(&in.APIEndpoint, &out.APIEndpoint, s); err != nil { - return err - } - out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) + // WARNING: in.ControlPlane requires manual conversion: inconvertible types (*k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm.JoinControlPlane vs bool) return nil } func autoConvert_v1alpha3_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm.LocalEtcd, s conversion.Scope) error { - out.Image = in.Image + // WARNING: in.Image requires manual conversion: does not exist in peer-type out.DataDir = in.DataDir out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) @@ -530,13 +507,8 @@ func autoConvert_v1alpha3_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kub return nil } -// Convert_v1alpha3_LocalEtcd_To_kubeadm_LocalEtcd is an autogenerated conversion function. -func Convert_v1alpha3_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm.LocalEtcd, s conversion.Scope) error { - return autoConvert_v1alpha3_LocalEtcd_To_kubeadm_LocalEtcd(in, out, s) -} - func autoConvert_kubeadm_LocalEtcd_To_v1alpha3_LocalEtcd(in *kubeadm.LocalEtcd, out *LocalEtcd, s conversion.Scope) error { - out.Image = in.Image + // WARNING: in.ImageMeta requires manual conversion: does not exist in peer-type out.DataDir = in.DataDir out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) @@ -544,11 +516,6 @@ func autoConvert_kubeadm_LocalEtcd_To_v1alpha3_LocalEtcd(in *kubeadm.LocalEtcd, return nil } -// Convert_kubeadm_LocalEtcd_To_v1alpha3_LocalEtcd is an autogenerated conversion function. -func Convert_kubeadm_LocalEtcd_To_v1alpha3_LocalEtcd(in *kubeadm.LocalEtcd, out *LocalEtcd, s conversion.Scope) error { - return autoConvert_kubeadm_LocalEtcd_To_v1alpha3_LocalEtcd(in, out, s) -} - func autoConvert_v1alpha3_Networking_To_kubeadm_Networking(in *Networking, out *kubeadm.Networking, s conversion.Scope) error { out.ServiceSubnet = in.ServiceSubnet out.PodSubnet = in.PodSubnet diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults.go index 68564fa7720..60c72a73d36 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults.go @@ -70,7 +70,7 @@ func SetDefaults_InitConfiguration(obj *InitConfiguration) { SetDefaults_ClusterConfiguration(&obj.ClusterConfiguration) SetDefaults_NodeRegistrationOptions(&obj.NodeRegistration) SetDefaults_BootstrapTokens(obj) - SetDefaults_APIEndpoint(&obj.APIEndpoint) + SetDefaults_APIEndpoint(&obj.LocalAPIEndpoint) } // SetDefaults_ClusterConfiguration assigns default values for the ClusterConfiguration @@ -99,11 +99,28 @@ func SetDefaults_ClusterConfiguration(obj *ClusterConfiguration) { obj.ClusterName = DefaultClusterName } + SetDefaults_DNS(obj) SetDefaults_Etcd(obj) - SetDefaults_AuditPolicyConfiguration(obj) + SetDefaults_APIServer(&obj.APIServer) } -// SetDefaults_Etcd assigns default values for the Proxy +// SetDefaults_APIServer assigns default values for the API Server +func SetDefaults_APIServer(obj *APIServer) { + if obj.TimeoutForControlPlane == nil { + obj.TimeoutForControlPlane = &metav1.Duration{ + Duration: constants.DefaultControlPlaneTimeout, + } + } +} + +// SetDefaults_DNS assigns default values for the DNS component +func SetDefaults_DNS(obj *ClusterConfiguration) { + if obj.DNS.Type == "" { + obj.DNS.Type = CoreDNS + } +} + +// SetDefaults_Etcd assigns default values for the proxy func SetDefaults_Etcd(obj *ClusterConfiguration) { if obj.Etcd.External == nil && obj.Etcd.Local == nil { obj.Etcd.Local = &LocalEtcd{} @@ -121,12 +138,8 @@ func SetDefaults_JoinConfiguration(obj *JoinConfiguration) { obj.CACertPath = DefaultCACertPath } - if obj.ClusterName == "" { - obj.ClusterName = DefaultClusterName - } - SetDefaults_NodeRegistrationOptions(&obj.NodeRegistration) - SetDefaults_APIEndpoint(&obj.APIEndpoint) + SetDefaults_JoinControlPlane(obj.ControlPlane) SetDefaults_Discovery(&obj.Discovery) } @@ -136,6 +149,12 @@ func SetDefaults_NodeRegistrationOptions(obj *NodeRegistrationOptions) { } } +func SetDefaults_JoinControlPlane(obj *JoinControlPlane) { + if obj != nil { + SetDefaults_APIEndpoint(&obj.LocalAPIEndpoint) + } +} + // SetDefaults_Discovery assigns default values for the discovery process func SetDefaults_Discovery(obj *Discovery) { if len(obj.TLSBootstrapToken) == 0 && obj.BootstrapToken != nil { @@ -164,16 +183,6 @@ func SetDefaults_FileDiscovery(obj *FileDiscovery) { } } -// SetDefaults_AuditPolicyConfiguration sets default values for the AuditPolicyConfiguration -func SetDefaults_AuditPolicyConfiguration(obj *ClusterConfiguration) { - if obj.AuditPolicyConfiguration.LogDir == "" { - obj.AuditPolicyConfiguration.LogDir = constants.StaticPodAuditPolicyLogDir - } - if obj.AuditPolicyConfiguration.LogMaxAge == nil { - obj.AuditPolicyConfiguration.LogMaxAge = &DefaultAuditPolicyLogMaxAge - } -} - // SetDefaults_BootstrapTokens sets the defaults for the .BootstrapTokens field // If the slice is empty, it's defaulted with one token. Otherwise it just loops // through the slice and sets the defaults for the omitempty fields that are TTL, diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_unix.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_unix.go index 1c0161bffc6..f4a50f99723 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_unix.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_unix.go @@ -21,7 +21,7 @@ package v1beta1 const ( // DefaultCACertPath defines default location of CA certificate on Linux DefaultCACertPath = "/etc/kubernetes/pki/ca.crt" - // DefaultSocketUrlScheme defines default socket url prefix + // DefaultUrlScheme defines default socket url prefix DefaultUrlScheme = "unix" // DefaultCRISocket defines the default cri socket DefaultCRISocket = "/var/run/dockershim.sock" diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go index ebba802fbd9..892c12bfbcd 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go @@ -19,14 +19,26 @@ limitations under the License. // +k8s:deepcopy-gen=package // +k8s:conversion-gen=k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm -// Package v1beta1 defines the v1beta1 version of the kubeadm config file format, that is a big step -// forward the objective of graduate kubeadm config to beta. +// Package v1beta1 defines the v1beta1 version of the kubeadm config file format. +// This version graduates the kubeadm config to BETA and is a big step towards GA. // -// //TODO add notes about big changes introduced by this release +//A list of changes since v1alpha3: +// - "apiServerEndpoint" in InitConfiguration was renamed to "localAPIServerEndpoint" for better clarity of what the field +// represents. +// - Common fields in ClusterConfiguration such as "*extraArgs" and "*extraVolumes" for control plane components are now moved +// under component structs - i.e. "apiServer", "controllerManager", "scheduler". +// - "auditPolicy" was removed from ClusterConfiguration. Please use "extraArgs" in "apiServer" to configure this feature instead. +// - "unifiedControlPlaneImage" in ClusterConfiguration was changed to a boolean field called "useHyperKubeImage". +// - ClusterConfiguration now has a "dns" field which can be used to select and configure the cluster DNS addon. +// - "featureGates" still exists under ClusterConfiguration, but there are no supported feature gates in 1.13. +// See the Kubernetes 1.13 changelog for further details. +// - Both "localEtcd" and "dns" configurations now support custom image repositories. +// - the "controlPlane*" related fields in JoinConfiguration were refactored into a sub structure. +// - "clusterName" was removed from JoinConfiguration and the name is now fetched from the existing cluster. // // Migration from old kubeadm config versions // -// Please convert your v1alpha3 configuration files to v1beta1 using the kubeadm config migrate command of kubeadm v1.13.x +// Please convert your v1alpha3 configuration files to v1beta1 using the "kubeadm config migrate" command of kubeadm v1.13.x // (conversion from older releases of kubeadm config files requires older release of kubeadm as well e.g. // kubeadm v1.11 should be used to migrate v1alpha1 to v1alpha2; kubeadm v1.12 should be used to translate v1alpha2 to v1alpha3) // @@ -88,7 +100,7 @@ limitations under the License. // ... // nodeRegistration: // ... -// apiEndpoint: +// localApiEndpoint: // ... // // The InitConfiguration type should be used to configure runtime settings, that in case of kubeadm init @@ -99,7 +111,7 @@ limitations under the License. // use it to customize the node name, the CRI socket to use or any other settings that should apply to this // node only (e.g. the node ip). // -// - APIEndpoint, that represents the endpoint of the instance of the API server to be deployed on this node; +// - LocalAPIEndpoint, that represents the endpoint of the instance of the API server to be deployed on this node; // use it e.g. to customize the API server advertise address. // // apiVersion: kubeadm.k8s.io/v1beta1 @@ -108,9 +120,10 @@ limitations under the License. // ... // etcd: // ... -// apiServerExtraArgs: +// apiServer: +// extraArgs: // ... -// APIServerExtraVolumes: +// extraVolumes: // ... // ... // @@ -170,7 +183,7 @@ limitations under the License. // effect: "NoSchedule" // kubeletExtraArgs: // cgroupDriver: "cgroupfs" -// apiEndpoint: +// localApiEndpoint: // advertiseAddress: "10.100.0.1" // bindPort: 6443 // --- @@ -207,11 +220,12 @@ limitations under the License. // - name: "some-volume" // hostPath: "/etc/some-path" // mountPath: "/etc/some-pod-path" -// writable: true +// readOnly: false // pathType: File // certSANs: // - "10.100.1.1" // - "ec2-10-100-0-1.compute-1.amazonaws.com" +// timeoutForControlPlane: 4m0s // controllerManager: // extraArgs: // node-cidr-mask-size: 20 @@ -219,7 +233,7 @@ limitations under the License. // - name: "some-volume" // hostPath: "/etc/some-path" // mountPath: "/etc/some-pod-path" -// writable: true +// readOnly: false // pathType: File // scheduler: // extraArgs: @@ -228,18 +242,11 @@ limitations under the License. // - name: "some-volume" // hostPath: "/etc/some-path" // mountPath: "/etc/some-pod-path" -// writable: true +// readOnly: false // pathType: File // certificatesDir: "/etc/kubernetes/pki" // imageRepository: "k8s.gcr.io" -// unifiedControlPlaneImage: "k8s.gcr.io/controlplane:v1.12.0" -// auditPolicy: -// # https://kubernetes.io/docs/tasks/debug-application-cluster/audit/#audit-policy -// path: "/var/log/audit/audit.json" -// logDir: "/var/log/audit" -// logMaxAge: 7 # in days -// featureGates: -// selfhosting: false +// useHyperKubeImage: false // clusterName: "example-cluster" // // Kubeadm join configuration types diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go index 9378243e0a5..9daf711450a 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go @@ -45,8 +45,13 @@ type InitConfiguration struct { // NodeRegistration holds fields that relate to registering the new master node to the cluster NodeRegistration NodeRegistrationOptions `json:"nodeRegistration,omitempty"` - // APIEndpoint represents the endpoint of the instance of the API server to be deployed on this node. - APIEndpoint APIEndpoint `json:"apiEndpoint,omitempty"` + // LocalAPIEndpoint represents the endpoint of the API server instance that's deployed on this control plane node + // In HA setups, this differs from ClusterConfiguration.ControlPlaneEndpoint in the sense that ControlPlaneEndpoint + // is the global endpoint for the cluster, which then loadbalances the requests to each individual API server. This + // configuration object lets you customize what IP/DNS name and port the local API server advertises it's accessible + // on. By default, kubeadm tries to auto-detect the IP of the default interface and use that, but in case that process + // fails you may set the desired value here. + LocalAPIEndpoint APIEndpoint `json:"localAPIEndpoint,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -86,17 +91,20 @@ type ClusterConfiguration struct { // Scheduler contains extra settings for the scheduler control plane component Scheduler ControlPlaneComponent `json:"scheduler,omitempty"` + // DNS defines the options for the DNS add-on installed in the cluster. + DNS DNS `json:"dns"` + // CertificatesDir specifies where to store or look for all required certificates. CertificatesDir string `json:"certificatesDir"` - // ImageRepository what container registry to pull control plane images from + // ImageRepository sets the container registry to pull images from. + // If empty, `k8s.gcr.io` will be used by default; in case of kubernetes version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`) + // `gcr.io/kubernetes-ci-images` will be used as a default for control plane components and for kube-proxy, while `k8s.gcr.io` + // will be used for all the other images. ImageRepository string `json:"imageRepository"` - // UnifiedControlPlaneImage specifies if a specific container image should - // be used for all control plane components. - UnifiedControlPlaneImage string `json:"unifiedControlPlaneImage"` - // AuditPolicyConfiguration defines the options for the api server audit system - AuditPolicyConfiguration AuditPolicyConfiguration `json:"auditPolicy"` + // UseHyperKubeImage controls if hyperkube should be used for Kubernetes components instead of their respective separate images + UseHyperKubeImage bool `json:"useHyperKubeImage,omitempty"` // FeatureGates enabled by the user. FeatureGates map[string]bool `json:"featureGates,omitempty"` @@ -108,6 +116,8 @@ type ClusterConfiguration struct { // ControlPlaneComponent holds settings common to control plane component of the cluster type ControlPlaneComponent struct { // ExtraArgs is an extra set of flags to pass to the control plane component. + // TODO: This is temporary and ideally we would like to switch all components to + // use ComponentConfig + ConfigMaps. ExtraArgs map[string]string `json:"extraArgs,omitempty"` // ExtraVolumes is an extra set of host volumes, mounted to the control plane component. @@ -120,6 +130,43 @@ type APIServer struct { // CertSANs sets extra Subject Alternative Names for the API Server signing cert. CertSANs []string `json:"certSANs,omitempty"` + + // TimeoutForControlPlane controls the timeout that we use for API server to appear + TimeoutForControlPlane *metav1.Duration `json:"timeoutForControlPlane,omitempty"` +} + +// DNSAddOnType defines string identifying DNS add-on types +type DNSAddOnType string + +const ( + // CoreDNS add-on type + CoreDNS DNSAddOnType = "CoreDNS" + + // KubeDNS add-on type + KubeDNS DNSAddOnType = "kube-dns" +) + +// DNS defines the DNS addon that should be used in the cluster +type DNS struct { + // Type defines the DNS add-on to be used + Type DNSAddOnType `json:"type"` + + // ImageMeta allows to customize the image used for the DNS component + ImageMeta `json:",inline"` +} + +// ImageMeta allows to customize the image used for components that are not +// originated from the Kubernetes/Kubernetes release process +type ImageMeta struct { + // ImageRepository sets the container registry to pull images from. + // if not set, the ImageRepository defined in ClusterConfiguration will be used instead. + ImageRepository string `json:"imageRepository,omitempty"` + + // ImageTag allows to specify a tag for the image. + // In case this value is set, kubeadm does not change automatically the version of the above components during upgrades. + ImageTag string `json:"imageTag,omitempty"` + + //TODO: evaluate if we need also a ImageName based on user feedbacks } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -212,11 +259,8 @@ type Etcd struct { // LocalEtcd describes that kubeadm should run an etcd cluster locally type LocalEtcd struct { - - // Image specifies which container image to use for running etcd. - // If empty, automatically populated by kubeadm using the image - // repository and default etcd version. - Image string `json:"image"` + // ImageMeta allows to customize the container used for etcd + ImageMeta `json:",inline"` // DataDir is the directory etcd will place its data. // Defaults to "/var/lib/etcd". @@ -232,22 +276,28 @@ type LocalEtcd struct { PeerCertSANs []string `json:"peerCertSANs,omitempty"` } -// ExternalEtcd describes an external etcd cluster +// ExternalEtcd describes an external etcd cluster. +// Kubeadm has no knowledge of where certificate files live and they must be supplied. type ExternalEtcd struct { // Endpoints of etcd members. Required for ExternalEtcd. Endpoints []string `json:"endpoints"` + // CAFile is an SSL Certificate Authority file used to secure etcd communication. + // Required if using a TLS connection. CAFile string `json:"caFile"` + // CertFile is an SSL certification file used to secure etcd communication. + // Required if using a TLS connection. CertFile string `json:"certFile"` + // KeyFile is an SSL key file used to secure etcd communication. + // Required if using a TLS connection. KeyFile string `json:"keyFile"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // JoinConfiguration contains elements describing a particular node. -// TODO: This struct should be replaced by dynamic kubelet configuration. type JoinConfiguration struct { metav1.TypeMeta `json:",inline"` @@ -262,18 +312,15 @@ type JoinConfiguration struct { // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process Discovery Discovery `json:"discovery"` - // ClusterName is the name for the cluster in kubeconfig. - ClusterName string `json:"clusterName,omitempty"` + // ControlPlane defines the additional control plane instance to be deployed on the joining node. + // If nil, no additional control plane instance will be deployed. + ControlPlane *JoinControlPlane `json:"controlPlane,omitempty"` +} - // ControlPlane flag specifies that the joining node should host an additional - // control plane instance. - ControlPlane bool `json:"controlPlane,omitempty"` - - // APIEndpoint represents the endpoint of the instance of the API server eventually to be deployed on this node. - APIEndpoint APIEndpoint `json:"apiEndpoint,omitempty"` - - // FeatureGates enabled by the user. - FeatureGates map[string]bool `json:"featureGates,omitempty"` +// JoinControlPlane contains elements describing an additional control plane instance to be deployed on the joining node. +type JoinControlPlane struct { + // LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node. + LocalAPIEndpoint APIEndpoint `json:"localAPIEndpoint,omitempty"` } // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process @@ -336,19 +383,8 @@ type HostPathMount struct { HostPath string `json:"hostPath"` // MountPath is the path inside the pod where hostPath will be mounted. MountPath string `json:"mountPath"` - // Writable controls write access to the volume - Writable bool `json:"writable,omitempty"` + // ReadOnly controls write access to the volume + ReadOnly bool `json:"readOnly,omitempty"` // PathType is the type of the HostPath. PathType v1.HostPathType `json:"pathType,omitempty"` } - -// AuditPolicyConfiguration holds the options for configuring the api server audit policy. -type AuditPolicyConfiguration struct { - // Path is the local path to an audit policy. - Path string `json:"path"` - // LogDir is the local path to the directory where logs should be stored. - LogDir string `json:"logDir"` - // LogMaxAge is the number of days logs will be stored for. 0 indicates forever. - LogMaxAge *int32 `json:"logMaxAge,omitempty"` - //TODO(chuckha) add other options for audit policy. -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go index fda28f950d3..4863c317fb3 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go @@ -57,16 +57,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*AuditPolicyConfiguration)(nil), (*kubeadm.AuditPolicyConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(a.(*AuditPolicyConfiguration), b.(*kubeadm.AuditPolicyConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.AuditPolicyConfiguration)(nil), (*AuditPolicyConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_AuditPolicyConfiguration_To_v1beta1_AuditPolicyConfiguration(a.(*kubeadm.AuditPolicyConfiguration), b.(*AuditPolicyConfiguration), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*BootstrapToken)(nil), (*kubeadm.BootstrapToken)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_BootstrapToken_To_kubeadm_BootstrapToken(a.(*BootstrapToken), b.(*kubeadm.BootstrapToken), scope) }); err != nil { @@ -127,6 +117,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*DNS)(nil), (*kubeadm.DNS)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_DNS_To_kubeadm_DNS(a.(*DNS), b.(*kubeadm.DNS), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kubeadm.DNS)(nil), (*DNS)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kubeadm_DNS_To_v1beta1_DNS(a.(*kubeadm.DNS), b.(*DNS), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*Discovery)(nil), (*kubeadm.Discovery)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_Discovery_To_kubeadm_Discovery(a.(*Discovery), b.(*kubeadm.Discovery), scope) }); err != nil { @@ -177,6 +177,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*ImageMeta)(nil), (*kubeadm.ImageMeta)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(a.(*ImageMeta), b.(*kubeadm.ImageMeta), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kubeadm.ImageMeta)(nil), (*ImageMeta)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kubeadm_ImageMeta_To_v1beta1_ImageMeta(a.(*kubeadm.ImageMeta), b.(*ImageMeta), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*InitConfiguration)(nil), (*kubeadm.InitConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_InitConfiguration_To_kubeadm_InitConfiguration(a.(*InitConfiguration), b.(*kubeadm.InitConfiguration), scope) }); err != nil { @@ -197,6 +207,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*JoinControlPlane)(nil), (*kubeadm.JoinControlPlane)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_JoinControlPlane_To_kubeadm_JoinControlPlane(a.(*JoinControlPlane), b.(*kubeadm.JoinControlPlane), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kubeadm.JoinControlPlane)(nil), (*JoinControlPlane)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(a.(*kubeadm.JoinControlPlane), b.(*JoinControlPlane), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*LocalEtcd)(nil), (*kubeadm.LocalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_LocalEtcd_To_kubeadm_LocalEtcd(a.(*LocalEtcd), b.(*kubeadm.LocalEtcd), scope) }); err != nil { @@ -257,6 +277,7 @@ func autoConvert_v1beta1_APIServer_To_kubeadm_APIServer(in *APIServer, out *kube return err } out.CertSANs = *(*[]string)(unsafe.Pointer(&in.CertSANs)) + out.TimeoutForControlPlane = (*v1.Duration)(unsafe.Pointer(in.TimeoutForControlPlane)) return nil } @@ -270,6 +291,7 @@ func autoConvert_kubeadm_APIServer_To_v1beta1_APIServer(in *kubeadm.APIServer, o return err } out.CertSANs = *(*[]string)(unsafe.Pointer(&in.CertSANs)) + out.TimeoutForControlPlane = (*v1.Duration)(unsafe.Pointer(in.TimeoutForControlPlane)) return nil } @@ -278,30 +300,6 @@ func Convert_kubeadm_APIServer_To_v1beta1_APIServer(in *kubeadm.APIServer, out * return autoConvert_kubeadm_APIServer_To_v1beta1_APIServer(in, out, s) } -func autoConvert_v1beta1_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(in *AuditPolicyConfiguration, out *kubeadm.AuditPolicyConfiguration, s conversion.Scope) error { - out.Path = in.Path - out.LogDir = in.LogDir - out.LogMaxAge = (*int32)(unsafe.Pointer(in.LogMaxAge)) - return nil -} - -// Convert_v1beta1_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration is an autogenerated conversion function. -func Convert_v1beta1_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(in *AuditPolicyConfiguration, out *kubeadm.AuditPolicyConfiguration, s conversion.Scope) error { - return autoConvert_v1beta1_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(in, out, s) -} - -func autoConvert_kubeadm_AuditPolicyConfiguration_To_v1beta1_AuditPolicyConfiguration(in *kubeadm.AuditPolicyConfiguration, out *AuditPolicyConfiguration, s conversion.Scope) error { - out.Path = in.Path - out.LogDir = in.LogDir - out.LogMaxAge = (*int32)(unsafe.Pointer(in.LogMaxAge)) - return nil -} - -// Convert_kubeadm_AuditPolicyConfiguration_To_v1beta1_AuditPolicyConfiguration is an autogenerated conversion function. -func Convert_kubeadm_AuditPolicyConfiguration_To_v1beta1_AuditPolicyConfiguration(in *kubeadm.AuditPolicyConfiguration, out *AuditPolicyConfiguration, s conversion.Scope) error { - return autoConvert_kubeadm_AuditPolicyConfiguration_To_v1beta1_AuditPolicyConfiguration(in, out, s) -} - func autoConvert_v1beta1_BootstrapToken_To_kubeadm_BootstrapToken(in *BootstrapToken, out *kubeadm.BootstrapToken, s conversion.Scope) error { out.Token = (*kubeadm.BootstrapTokenString)(unsafe.Pointer(in.Token)) out.Description = in.Description @@ -398,12 +396,12 @@ func autoConvert_v1beta1_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in if err := Convert_v1beta1_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(&in.Scheduler, &out.Scheduler, s); err != nil { return err } - out.CertificatesDir = in.CertificatesDir - out.ImageRepository = in.ImageRepository - out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage - if err := Convert_v1beta1_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil { + if err := Convert_v1beta1_DNS_To_kubeadm_DNS(&in.DNS, &out.DNS, s); err != nil { return err } + out.CertificatesDir = in.CertificatesDir + out.ImageRepository = in.ImageRepository + out.UseHyperKubeImage = in.UseHyperKubeImage out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) out.ClusterName = in.ClusterName return nil @@ -433,13 +431,13 @@ func autoConvert_kubeadm_ClusterConfiguration_To_v1beta1_ClusterConfiguration(in if err := Convert_kubeadm_ControlPlaneComponent_To_v1beta1_ControlPlaneComponent(&in.Scheduler, &out.Scheduler, s); err != nil { return err } + if err := Convert_kubeadm_DNS_To_v1beta1_DNS(&in.DNS, &out.DNS, s); err != nil { + return err + } out.CertificatesDir = in.CertificatesDir out.ImageRepository = in.ImageRepository // INFO: in.CIImageRepository opted out of conversion generation - out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage - if err := Convert_kubeadm_AuditPolicyConfiguration_To_v1beta1_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil { - return err - } + out.UseHyperKubeImage = in.UseHyperKubeImage out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) out.ClusterName = in.ClusterName return nil @@ -492,6 +490,32 @@ func Convert_kubeadm_ControlPlaneComponent_To_v1beta1_ControlPlaneComponent(in * return autoConvert_kubeadm_ControlPlaneComponent_To_v1beta1_ControlPlaneComponent(in, out, s) } +func autoConvert_v1beta1_DNS_To_kubeadm_DNS(in *DNS, out *kubeadm.DNS, s conversion.Scope) error { + out.Type = kubeadm.DNSAddOnType(in.Type) + if err := Convert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_DNS_To_kubeadm_DNS is an autogenerated conversion function. +func Convert_v1beta1_DNS_To_kubeadm_DNS(in *DNS, out *kubeadm.DNS, s conversion.Scope) error { + return autoConvert_v1beta1_DNS_To_kubeadm_DNS(in, out, s) +} + +func autoConvert_kubeadm_DNS_To_v1beta1_DNS(in *kubeadm.DNS, out *DNS, s conversion.Scope) error { + out.Type = DNSAddOnType(in.Type) + if err := Convert_kubeadm_ImageMeta_To_v1beta1_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { + return err + } + return nil +} + +// Convert_kubeadm_DNS_To_v1beta1_DNS is an autogenerated conversion function. +func Convert_kubeadm_DNS_To_v1beta1_DNS(in *kubeadm.DNS, out *DNS, s conversion.Scope) error { + return autoConvert_kubeadm_DNS_To_v1beta1_DNS(in, out, s) +} + func autoConvert_v1beta1_Discovery_To_kubeadm_Discovery(in *Discovery, out *kubeadm.Discovery, s conversion.Scope) error { out.BootstrapToken = (*kubeadm.BootstrapTokenDiscovery)(unsafe.Pointer(in.BootstrapToken)) out.File = (*kubeadm.FileDiscovery)(unsafe.Pointer(in.File)) @@ -590,7 +614,7 @@ func autoConvert_v1beta1_HostPathMount_To_kubeadm_HostPathMount(in *HostPathMoun out.Name = in.Name out.HostPath = in.HostPath out.MountPath = in.MountPath - out.Writable = in.Writable + out.ReadOnly = in.ReadOnly out.PathType = corev1.HostPathType(in.PathType) return nil } @@ -604,7 +628,7 @@ func autoConvert_kubeadm_HostPathMount_To_v1beta1_HostPathMount(in *kubeadm.Host out.Name = in.Name out.HostPath = in.HostPath out.MountPath = in.MountPath - out.Writable = in.Writable + out.ReadOnly = in.ReadOnly out.PathType = corev1.HostPathType(in.PathType) return nil } @@ -614,6 +638,28 @@ func Convert_kubeadm_HostPathMount_To_v1beta1_HostPathMount(in *kubeadm.HostPath return autoConvert_kubeadm_HostPathMount_To_v1beta1_HostPathMount(in, out, s) } +func autoConvert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(in *ImageMeta, out *kubeadm.ImageMeta, s conversion.Scope) error { + out.ImageRepository = in.ImageRepository + out.ImageTag = in.ImageTag + return nil +} + +// Convert_v1beta1_ImageMeta_To_kubeadm_ImageMeta is an autogenerated conversion function. +func Convert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(in *ImageMeta, out *kubeadm.ImageMeta, s conversion.Scope) error { + return autoConvert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(in, out, s) +} + +func autoConvert_kubeadm_ImageMeta_To_v1beta1_ImageMeta(in *kubeadm.ImageMeta, out *ImageMeta, s conversion.Scope) error { + out.ImageRepository = in.ImageRepository + out.ImageTag = in.ImageTag + return nil +} + +// Convert_kubeadm_ImageMeta_To_v1beta1_ImageMeta is an autogenerated conversion function. +func Convert_kubeadm_ImageMeta_To_v1beta1_ImageMeta(in *kubeadm.ImageMeta, out *ImageMeta, s conversion.Scope) error { + return autoConvert_kubeadm_ImageMeta_To_v1beta1_ImageMeta(in, out, s) +} + func autoConvert_v1beta1_InitConfiguration_To_kubeadm_InitConfiguration(in *InitConfiguration, out *kubeadm.InitConfiguration, s conversion.Scope) error { if err := Convert_v1beta1_ClusterConfiguration_To_kubeadm_ClusterConfiguration(&in.ClusterConfiguration, &out.ClusterConfiguration, s); err != nil { return err @@ -622,7 +668,7 @@ func autoConvert_v1beta1_InitConfiguration_To_kubeadm_InitConfiguration(in *Init if err := Convert_v1beta1_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { return err } - if err := Convert_v1beta1_APIEndpoint_To_kubeadm_APIEndpoint(&in.APIEndpoint, &out.APIEndpoint, s); err != nil { + if err := Convert_v1beta1_APIEndpoint_To_kubeadm_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil { return err } return nil @@ -641,7 +687,7 @@ func autoConvert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in *kube if err := Convert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { return err } - if err := Convert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(&in.APIEndpoint, &out.APIEndpoint, s); err != nil { + if err := Convert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil { return err } return nil @@ -660,12 +706,7 @@ func autoConvert_v1beta1_JoinConfiguration_To_kubeadm_JoinConfiguration(in *Join if err := Convert_v1beta1_Discovery_To_kubeadm_Discovery(&in.Discovery, &out.Discovery, s); err != nil { return err } - out.ClusterName = in.ClusterName - out.ControlPlane = in.ControlPlane - if err := Convert_v1beta1_APIEndpoint_To_kubeadm_APIEndpoint(&in.APIEndpoint, &out.APIEndpoint, s); err != nil { - return err - } - out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) + out.ControlPlane = (*kubeadm.JoinControlPlane)(unsafe.Pointer(in.ControlPlane)) return nil } @@ -682,12 +723,7 @@ func autoConvert_kubeadm_JoinConfiguration_To_v1beta1_JoinConfiguration(in *kube if err := Convert_kubeadm_Discovery_To_v1beta1_Discovery(&in.Discovery, &out.Discovery, s); err != nil { return err } - out.ClusterName = in.ClusterName - out.ControlPlane = in.ControlPlane - if err := Convert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(&in.APIEndpoint, &out.APIEndpoint, s); err != nil { - return err - } - out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) + out.ControlPlane = (*JoinControlPlane)(unsafe.Pointer(in.ControlPlane)) return nil } @@ -696,8 +732,34 @@ func Convert_kubeadm_JoinConfiguration_To_v1beta1_JoinConfiguration(in *kubeadm. return autoConvert_kubeadm_JoinConfiguration_To_v1beta1_JoinConfiguration(in, out, s) } +func autoConvert_v1beta1_JoinControlPlane_To_kubeadm_JoinControlPlane(in *JoinControlPlane, out *kubeadm.JoinControlPlane, s conversion.Scope) error { + if err := Convert_v1beta1_APIEndpoint_To_kubeadm_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_JoinControlPlane_To_kubeadm_JoinControlPlane is an autogenerated conversion function. +func Convert_v1beta1_JoinControlPlane_To_kubeadm_JoinControlPlane(in *JoinControlPlane, out *kubeadm.JoinControlPlane, s conversion.Scope) error { + return autoConvert_v1beta1_JoinControlPlane_To_kubeadm_JoinControlPlane(in, out, s) +} + +func autoConvert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in *kubeadm.JoinControlPlane, out *JoinControlPlane, s conversion.Scope) error { + if err := Convert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil { + return err + } + return nil +} + +// Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane is an autogenerated conversion function. +func Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in *kubeadm.JoinControlPlane, out *JoinControlPlane, s conversion.Scope) error { + return autoConvert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in, out, s) +} + func autoConvert_v1beta1_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm.LocalEtcd, s conversion.Scope) error { - out.Image = in.Image + if err := Convert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { + return err + } out.DataDir = in.DataDir out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) @@ -711,7 +773,9 @@ func Convert_v1beta1_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm. } func autoConvert_kubeadm_LocalEtcd_To_v1beta1_LocalEtcd(in *kubeadm.LocalEtcd, out *LocalEtcd, s conversion.Scope) error { - out.Image = in.Image + if err := Convert_kubeadm_ImageMeta_To_v1beta1_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { + return err + } out.DataDir = in.DataDir out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.deepcopy.go index cc2e1ad7125..d6cdfda335a 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.deepcopy.go @@ -51,6 +51,11 @@ func (in *APIServer) DeepCopyInto(out *APIServer) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.TimeoutForControlPlane != nil { + in, out := &in.TimeoutForControlPlane, &out.TimeoutForControlPlane + *out = new(v1.Duration) + **out = **in + } return } @@ -64,27 +69,6 @@ func (in *APIServer) DeepCopy() *APIServer { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AuditPolicyConfiguration) DeepCopyInto(out *AuditPolicyConfiguration) { - *out = *in - if in.LogMaxAge != nil { - in, out := &in.LogMaxAge, &out.LogMaxAge - *out = new(int32) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuditPolicyConfiguration. -func (in *AuditPolicyConfiguration) DeepCopy() *AuditPolicyConfiguration { - if in == nil { - return nil - } - out := new(AuditPolicyConfiguration) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BootstrapToken) DeepCopyInto(out *BootstrapToken) { *out = *in @@ -171,7 +155,7 @@ func (in *ClusterConfiguration) DeepCopyInto(out *ClusterConfiguration) { in.APIServer.DeepCopyInto(&out.APIServer) in.ControllerManager.DeepCopyInto(&out.ControllerManager) in.Scheduler.DeepCopyInto(&out.Scheduler) - in.AuditPolicyConfiguration.DeepCopyInto(&out.AuditPolicyConfiguration) + out.DNS = in.DNS if in.FeatureGates != nil { in, out := &in.FeatureGates, &out.FeatureGates *out = make(map[string]bool, len(*in)) @@ -260,6 +244,23 @@ func (in *ControlPlaneComponent) DeepCopy() *ControlPlaneComponent { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DNS) DeepCopyInto(out *DNS) { + *out = *in + out.ImageMeta = in.ImageMeta + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNS. +func (in *DNS) DeepCopy() *DNS { + if in == nil { + return nil + } + out := new(DNS) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Discovery) DeepCopyInto(out *Discovery) { *out = *in @@ -370,6 +371,22 @@ func (in *HostPathMount) DeepCopy() *HostPathMount { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImageMeta) DeepCopyInto(out *ImageMeta) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageMeta. +func (in *ImageMeta) DeepCopy() *ImageMeta { + if in == nil { + return nil + } + out := new(ImageMeta) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InitConfiguration) DeepCopyInto(out *InitConfiguration) { *out = *in @@ -383,7 +400,7 @@ func (in *InitConfiguration) DeepCopyInto(out *InitConfiguration) { } } in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) - out.APIEndpoint = in.APIEndpoint + out.LocalAPIEndpoint = in.LocalAPIEndpoint return } @@ -411,13 +428,10 @@ func (in *JoinConfiguration) DeepCopyInto(out *JoinConfiguration) { out.TypeMeta = in.TypeMeta in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) in.Discovery.DeepCopyInto(&out.Discovery) - out.APIEndpoint = in.APIEndpoint - if in.FeatureGates != nil { - in, out := &in.FeatureGates, &out.FeatureGates - *out = make(map[string]bool, len(*in)) - for key, val := range *in { - (*out)[key] = val - } + if in.ControlPlane != nil { + in, out := &in.ControlPlane, &out.ControlPlane + *out = new(JoinControlPlane) + **out = **in } return } @@ -440,9 +454,27 @@ func (in *JoinConfiguration) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JoinControlPlane) DeepCopyInto(out *JoinControlPlane) { + *out = *in + out.LocalAPIEndpoint = in.LocalAPIEndpoint + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoinControlPlane. +func (in *JoinControlPlane) DeepCopy() *JoinControlPlane { + if in == nil { + return nil + } + out := new(JoinControlPlane) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LocalEtcd) DeepCopyInto(out *LocalEtcd) { *out = *in + out.ImageMeta = in.ImageMeta if in.ExtraArgs != nil { in, out := &in.ExtraArgs, &out.ExtraArgs *out = make(map[string]string, len(*in)) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.defaults.go index a27612cce03..4860aec6a27 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.defaults.go @@ -37,6 +37,7 @@ func RegisterDefaults(scheme *runtime.Scheme) error { func SetObjectDefaults_ClusterConfiguration(in *ClusterConfiguration) { SetDefaults_ClusterConfiguration(in) + SetDefaults_APIServer(&in.APIServer) } func SetObjectDefaults_ClusterStatus(in *ClusterStatus) { @@ -50,7 +51,7 @@ func SetObjectDefaults_InitConfiguration(in *InitConfiguration) { SetDefaults_BootstrapToken(a) } SetDefaults_NodeRegistrationOptions(&in.NodeRegistration) - SetDefaults_APIEndpoint(&in.APIEndpoint) + SetDefaults_APIEndpoint(&in.LocalAPIEndpoint) } func SetObjectDefaults_JoinConfiguration(in *JoinConfiguration) { @@ -60,5 +61,8 @@ func SetObjectDefaults_JoinConfiguration(in *JoinConfiguration) { if in.Discovery.File != nil { SetDefaults_FileDiscovery(in.Discovery.File) } - SetDefaults_APIEndpoint(&in.APIEndpoint) + if in.ControlPlane != nil { + SetDefaults_JoinControlPlane(in.ControlPlane) + SetDefaults_APIEndpoint(&in.ControlPlane.LocalAPIEndpoint) + } } diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go index 30bdc776127..c97566b6ec8 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go @@ -27,7 +27,6 @@ import ( "github.com/pkg/errors" "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" @@ -49,7 +48,7 @@ func ValidateInitConfiguration(c *kubeadm.InitConfiguration) field.ErrorList { allErrs = append(allErrs, ValidateNodeRegistrationOptions(&c.NodeRegistration, field.NewPath("nodeRegistration"))...) allErrs = append(allErrs, ValidateBootstrapTokens(c.BootstrapTokens, field.NewPath("bootstrapTokens"))...) allErrs = append(allErrs, ValidateClusterConfiguration(&c.ClusterConfiguration)...) - allErrs = append(allErrs, ValidateAPIEndpoint(&c.APIEndpoint, field.NewPath("apiEndpoint"))...) + allErrs = append(allErrs, ValidateAPIEndpoint(&c.LocalAPIEndpoint, field.NewPath("localAPIEndpoint"))...) return allErrs } @@ -78,7 +77,7 @@ func ValidateJoinConfiguration(c *kubeadm.JoinConfiguration) field.ErrorList { allErrs := field.ErrorList{} allErrs = append(allErrs, ValidateDiscovery(&c.Discovery, field.NewPath("discovery"))...) allErrs = append(allErrs, ValidateNodeRegistrationOptions(&c.NodeRegistration, field.NewPath("nodeRegistration"))...) - allErrs = append(allErrs, ValidateAPIEndpoint(&c.APIEndpoint, field.NewPath("apiEndpoint"))...) + allErrs = append(allErrs, ValidateJoinControlPlane(c.ControlPlane, field.NewPath("controlPlane"))...) if !filepath.IsAbs(c.CACertPath) || !strings.HasSuffix(c.CACertPath, ".crt") { allErrs = append(allErrs, field.Invalid(field.NewPath("caCertPath"), c.CACertPath, "the ca certificate path must be an absolute path")) @@ -86,6 +85,15 @@ func ValidateJoinConfiguration(c *kubeadm.JoinConfiguration) field.ErrorList { return allErrs } +// ValidateJoinControlPlane validates joining control plane configuration and collects all encountered errors +func ValidateJoinControlPlane(c *kubeadm.JoinControlPlane, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if c != nil { + allErrs = append(allErrs, ValidateAPIEndpoint(&c.LocalAPIEndpoint, fldPath.Child("localAPIEndpoint"))...) + } + return allErrs +} + // ValidateNodeRegistrationOptions validates the NodeRegistrationOptions object func ValidateNodeRegistrationOptions(nro *kubeadm.NodeRegistrationOptions, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} @@ -407,13 +415,12 @@ func ValidateMixedArguments(flag *pflag.FlagSet) error { // ValidateFeatureGates validates provided feature gates func ValidateFeatureGates(featureGates map[string]bool, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} - validFeatures := features.Keys(features.InitFeatureGates) // check valid feature names are provided for k := range featureGates { if !features.Supports(features.InitFeatureGates, k) { allErrs = append(allErrs, field.Invalid(fldPath, featureGates, - fmt.Sprintf("%s is not a valid feature name. Valid features are: %s", k, validFeatures))) + fmt.Sprintf("%s is not a valid feature name.", k))) } } diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go index 14d98eb42ab..5e7c058681a 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go @@ -23,7 +23,6 @@ import ( "time" "github.com/spf13/pflag" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -359,7 +358,7 @@ func TestValidateInitConfiguration(t *testing.T) { &kubeadm.InitConfiguration{}, false}, {"invalid missing token with IPv4 service subnet", &kubeadm.InitConfiguration{ - APIEndpoint: kubeadm.APIEndpoint{ + LocalAPIEndpoint: kubeadm.APIEndpoint{ AdvertiseAddress: "1.2.3.4", BindPort: 6443, }, @@ -374,7 +373,7 @@ func TestValidateInitConfiguration(t *testing.T) { }, false}, {"invalid missing token with IPv6 service subnet", &kubeadm.InitConfiguration{ - APIEndpoint: kubeadm.APIEndpoint{ + LocalAPIEndpoint: kubeadm.APIEndpoint{ AdvertiseAddress: "1.2.3.4", BindPort: 6443, }, @@ -389,7 +388,7 @@ func TestValidateInitConfiguration(t *testing.T) { }, false}, {"invalid missing node name", &kubeadm.InitConfiguration{ - APIEndpoint: kubeadm.APIEndpoint{ + LocalAPIEndpoint: kubeadm.APIEndpoint{ AdvertiseAddress: "1.2.3.4", BindPort: 6443, }, @@ -403,7 +402,7 @@ func TestValidateInitConfiguration(t *testing.T) { }, false}, {"valid master configuration with incorrect IPv4 pod subnet", &kubeadm.InitConfiguration{ - APIEndpoint: kubeadm.APIEndpoint{ + LocalAPIEndpoint: kubeadm.APIEndpoint{ AdvertiseAddress: "1.2.3.4", BindPort: 6443, }, @@ -419,7 +418,7 @@ func TestValidateInitConfiguration(t *testing.T) { }, false}, {"valid master configuration with IPv4 service subnet", &kubeadm.InitConfiguration{ - APIEndpoint: kubeadm.APIEndpoint{ + LocalAPIEndpoint: kubeadm.APIEndpoint{ AdvertiseAddress: "1.2.3.4", BindPort: 6443, }, @@ -466,7 +465,7 @@ func TestValidateInitConfiguration(t *testing.T) { }, true}, {"valid master configuration using IPv6 service subnet", &kubeadm.InitConfiguration{ - APIEndpoint: kubeadm.APIEndpoint{ + LocalAPIEndpoint: kubeadm.APIEndpoint{ AdvertiseAddress: "1:2:3::4", BindPort: 3446, }, @@ -541,6 +540,84 @@ func TestValidateJoinConfiguration(t *testing.T) { }, }, }, false}, + {&kubeadm.JoinConfiguration{ // Pass without JoinControlPlane + CACertPath: "/some/cert.crt", + Discovery: kubeadm.Discovery{ + BootstrapToken: &kubeadm.BootstrapTokenDiscovery{ + Token: "abcdef.1234567890123456", + APIServerEndpoint: "1.2.3.4:6443", + CACertHashes: []string{"aaaa"}, + }, + TLSBootstrapToken: "abcdef.1234567890123456", + }, + NodeRegistration: kubeadm.NodeRegistrationOptions{ + Name: "aaa", + CRISocket: "/var/run/dockershim.sock", + }, + }, true}, + {&kubeadm.JoinConfiguration{ // Pass with JoinControlPlane + CACertPath: "/some/cert.crt", + Discovery: kubeadm.Discovery{ + BootstrapToken: &kubeadm.BootstrapTokenDiscovery{ + Token: "abcdef.1234567890123456", + APIServerEndpoint: "1.2.3.4:6443", + CACertHashes: []string{"aaaa"}, + }, + TLSBootstrapToken: "abcdef.1234567890123456", + }, + NodeRegistration: kubeadm.NodeRegistrationOptions{ + Name: "aaa", + CRISocket: "/var/run/dockershim.sock", + }, + ControlPlane: &kubeadm.JoinControlPlane{ + LocalAPIEndpoint: kubeadm.APIEndpoint{ + AdvertiseAddress: "1.2.3.4", + BindPort: 1234, + }, + }, + }, true}, + {&kubeadm.JoinConfiguration{ // Fail JoinControlPlane.AdvertiseAddress validation + CACertPath: "/some/cert.crt", + Discovery: kubeadm.Discovery{ + BootstrapToken: &kubeadm.BootstrapTokenDiscovery{ + Token: "abcdef.1234567890123456", + APIServerEndpoint: "1.2.3.4:6443", + CACertHashes: []string{"aaaa"}, + }, + TLSBootstrapToken: "abcdef.1234567890123456", + }, + NodeRegistration: kubeadm.NodeRegistrationOptions{ + Name: "aaa", + CRISocket: "/var/run/dockershim.sock", + }, + ControlPlane: &kubeadm.JoinControlPlane{ + LocalAPIEndpoint: kubeadm.APIEndpoint{ + AdvertiseAddress: "aaa", + BindPort: 1234, + }, + }, + }, false}, + {&kubeadm.JoinConfiguration{ // Fail JoinControlPlane.BindPort validation + CACertPath: "/some/cert.crt", + Discovery: kubeadm.Discovery{ + BootstrapToken: &kubeadm.BootstrapTokenDiscovery{ + Token: "abcdef.1234567890123456", + APIServerEndpoint: "1.2.3.4:6443", + CACertHashes: []string{"aaaa"}, + }, + TLSBootstrapToken: "abcdef.1234567890123456", + }, + NodeRegistration: kubeadm.NodeRegistrationOptions{ + Name: "aaa", + CRISocket: "/var/run/dockershim.sock", + }, + ControlPlane: &kubeadm.JoinControlPlane{ + LocalAPIEndpoint: kubeadm.APIEndpoint{ + AdvertiseAddress: "1.2.3.4", + BindPort: -1, + }, + }, + }, false}, } for _, rt := range tests { actual := ValidateJoinConfiguration(rt.s) @@ -603,17 +680,15 @@ func TestValidateFeatureGates(t *testing.T) { featureGates featureFlag expected bool }{ - {featureFlag{"SelfHosting": true}, true}, - {featureFlag{"SelfHosting": false}, true}, - {featureFlag{"StoreCertsInSecrets": true}, true}, - {featureFlag{"StoreCertsInSecrets": false}, true}, - {featureFlag{"Foo": true}, false}, + {featureFlag{"Unknown": true}, false}, + {featureFlag{"Unknown": false}, false}, } for _, rt := range tests { actual := ValidateFeatureGates(rt.featureGates, nil) if (len(actual) == 0) != rt.expected { t.Errorf( - "failed featureGates:\n\texpected: %t\n\t actual: %t", + "failed featureGates %v:\n\texpected: %t\n\t actual: %t", + rt.featureGates, rt.expected, (len(actual) == 0), ) diff --git a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go index ec241bdf08d..ebdf10bb2e2 100644 --- a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go @@ -53,6 +53,11 @@ func (in *APIServer) DeepCopyInto(out *APIServer) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.TimeoutForControlPlane != nil { + in, out := &in.TimeoutForControlPlane, &out.TimeoutForControlPlane + *out = new(v1.Duration) + **out = **in + } return } @@ -66,27 +71,6 @@ func (in *APIServer) DeepCopy() *APIServer { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AuditPolicyConfiguration) DeepCopyInto(out *AuditPolicyConfiguration) { - *out = *in - if in.LogMaxAge != nil { - in, out := &in.LogMaxAge, &out.LogMaxAge - *out = new(int32) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuditPolicyConfiguration. -func (in *AuditPolicyConfiguration) DeepCopy() *AuditPolicyConfiguration { - if in == nil { - return nil - } - out := new(AuditPolicyConfiguration) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BootstrapToken) DeepCopyInto(out *BootstrapToken) { *out = *in @@ -174,7 +158,7 @@ func (in *ClusterConfiguration) DeepCopyInto(out *ClusterConfiguration) { in.APIServer.DeepCopyInto(&out.APIServer) in.ControllerManager.DeepCopyInto(&out.ControllerManager) in.Scheduler.DeepCopyInto(&out.Scheduler) - in.AuditPolicyConfiguration.DeepCopyInto(&out.AuditPolicyConfiguration) + out.DNS = in.DNS if in.FeatureGates != nil { in, out := &in.FeatureGates, &out.FeatureGates *out = make(map[string]bool, len(*in)) @@ -289,6 +273,23 @@ func (in *ControlPlaneComponent) DeepCopy() *ControlPlaneComponent { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DNS) DeepCopyInto(out *DNS) { + *out = *in + out.ImageMeta = in.ImageMeta + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNS. +func (in *DNS) DeepCopy() *DNS { + if in == nil { + return nil + } + out := new(DNS) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Discovery) DeepCopyInto(out *Discovery) { *out = *in @@ -399,6 +400,22 @@ func (in *HostPathMount) DeepCopy() *HostPathMount { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImageMeta) DeepCopyInto(out *ImageMeta) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageMeta. +func (in *ImageMeta) DeepCopy() *ImageMeta { + if in == nil { + return nil + } + out := new(ImageMeta) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InitConfiguration) DeepCopyInto(out *InitConfiguration) { *out = *in @@ -412,7 +429,7 @@ func (in *InitConfiguration) DeepCopyInto(out *InitConfiguration) { } } in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) - out.APIEndpoint = in.APIEndpoint + out.LocalAPIEndpoint = in.LocalAPIEndpoint return } @@ -440,13 +457,10 @@ func (in *JoinConfiguration) DeepCopyInto(out *JoinConfiguration) { out.TypeMeta = in.TypeMeta in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) in.Discovery.DeepCopyInto(&out.Discovery) - out.APIEndpoint = in.APIEndpoint - if in.FeatureGates != nil { - in, out := &in.FeatureGates, &out.FeatureGates - *out = make(map[string]bool, len(*in)) - for key, val := range *in { - (*out)[key] = val - } + if in.ControlPlane != nil { + in, out := &in.ControlPlane, &out.ControlPlane + *out = new(JoinControlPlane) + **out = **in } return } @@ -469,9 +483,27 @@ func (in *JoinConfiguration) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JoinControlPlane) DeepCopyInto(out *JoinControlPlane) { + *out = *in + out.LocalAPIEndpoint = in.LocalAPIEndpoint + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoinControlPlane. +func (in *JoinControlPlane) DeepCopy() *JoinControlPlane { + if in == nil { + return nil + } + out := new(JoinControlPlane) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LocalEtcd) DeepCopyInto(out *LocalEtcd) { *out = *in + out.ImageMeta = in.ImageMeta if in.ExtraArgs != nil { in, out := &in.ExtraArgs, &out.ExtraArgs *out = make(map[string]string, len(*in)) diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index 7d881c39d42..e542aa60f27 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -36,18 +36,14 @@ go_library( "//cmd/kubeadm/app/discovery:go_default_library", "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/images:go_default_library", - "//cmd/kubeadm/app/phases/addons/dns:go_default_library", - "//cmd/kubeadm/app/phases/addons/proxy:go_default_library", - "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/app/phases/controlplane:go_default_library", "//cmd/kubeadm/app/phases/etcd:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", "//cmd/kubeadm/app/phases/kubelet:go_default_library", - "//cmd/kubeadm/app/phases/markmaster:go_default_library", + "//cmd/kubeadm/app/phases/markcontrolplane:go_default_library", "//cmd/kubeadm/app/phases/patchnode:go_default_library", - "//cmd/kubeadm/app/phases/selfhosting:go_default_library", "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", @@ -69,13 +65,13 @@ go_library( "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", "//vendor/github.com/renstrom/dedent:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) @@ -97,7 +93,6 @@ go_test( "//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library", "//cmd/kubeadm/app/componentconfigs:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", @@ -110,12 +105,12 @@ go_test( "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", "//vendor/github.com/renstrom/dedent:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec/testing:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/cmd/kubeadm/app/cmd/alpha/BUILD b/cmd/kubeadm/app/cmd/alpha/BUILD index 49f1f7d7d48..1e2b35048c7 100644 --- a/cmd/kubeadm/app/cmd/alpha/BUILD +++ b/cmd/kubeadm/app/cmd/alpha/BUILD @@ -8,6 +8,7 @@ go_library( "kubeconfig.go", "kubelet.go", "preflight.go", + "selfhosting.go", ], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/alpha", visibility = ["//visibility:public"], @@ -19,12 +20,15 @@ go_library( "//cmd/kubeadm/app/cmd/phases:go_default_library", "//cmd/kubeadm/app/cmd/util:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/app/phases/certs/renewal:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", "//cmd/kubeadm/app/phases/kubelet:go_default_library", + "//cmd/kubeadm/app/phases/selfhosting:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", + "//cmd/kubeadm/app/util/apiclient:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//pkg/util/normalizer:go_default_library", @@ -58,6 +62,7 @@ go_test( embed = [":go_default_library"], deps = [ "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/app/util/pkiutil:go_default_library", "//cmd/kubeadm/test:go_default_library", "//cmd/kubeadm/test/certs:go_default_library", diff --git a/cmd/kubeadm/app/cmd/alpha/alpha.go b/cmd/kubeadm/app/cmd/alpha/alpha.go index dbcc722ed21..b5dd4334214 100644 --- a/cmd/kubeadm/app/cmd/alpha/alpha.go +++ b/cmd/kubeadm/app/cmd/alpha/alpha.go @@ -20,12 +20,11 @@ import ( "io" "github.com/spf13/cobra" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" ) // NewCmdAlpha returns "kubeadm alpha" command. -func NewCmdAlpha(out io.Writer) *cobra.Command { +func NewCmdAlpha(in io.Reader, out io.Writer) *cobra.Command { cmd := &cobra.Command{ Use: "alpha", Short: "Kubeadm experimental sub-commands", @@ -35,6 +34,7 @@ func NewCmdAlpha(out io.Writer) *cobra.Command { cmd.AddCommand(newCmdKubeletUtility()) cmd.AddCommand(newCmdKubeConfigUtility(out)) cmd.AddCommand(newCmdPreFlightUtility()) + cmd.AddCommand(NewCmdSelfhosting(in)) // TODO: This command should be removed as soon as the kubeadm init phase refactoring is completed. // current phases implemented as cobra.Commands should become workflow.Phases, while other utilities @@ -51,11 +51,5 @@ func newCmdPhase(out io.Writer) *cobra.Command { Long: cmdutil.MacroCommandLongDescription, } - cmd.AddCommand(phases.NewCmdAddon()) - cmd.AddCommand(phases.NewCmdBootstrapToken()) - cmd.AddCommand(phases.NewCmdMarkMaster()) - cmd.AddCommand(phases.NewCmdSelfhosting()) - cmd.AddCommand(phases.NewCmdUploadConfig()) - return cmd } diff --git a/cmd/kubeadm/app/cmd/alpha/certs.go b/cmd/kubeadm/app/cmd/alpha/certs.go index 320a21e9933..08ee5294d2f 100644 --- a/cmd/kubeadm/app/cmd/alpha/certs.go +++ b/cmd/kubeadm/app/cmd/alpha/certs.go @@ -76,6 +76,8 @@ type renewConfig struct { kubeconfigPath string cfg kubeadmapiv1beta1.InitConfiguration useAPI bool + useCSR bool + csrPath string } func getRenewSubCommands() []*cobra.Command { @@ -126,6 +128,8 @@ func addFlags(cmd *cobra.Command, cfg *renewConfig) { options.AddConfigFlag(cmd.Flags(), &cfg.cfgPath) options.AddCertificateDirFlag(cmd.Flags(), &cfg.cfg.CertificatesDir) options.AddKubeConfigFlag(cmd.Flags(), &cfg.kubeconfigPath) + options.AddCSRFlag(cmd.Flags(), &cfg.useCSR) + options.AddCSRDirFlag(cmd.Flags(), &cfg.csrPath) cmd.Flags().BoolVar(&cfg.useAPI, "use-api", cfg.useAPI, "Use the Kubernetes certificate API to renew certificates") } @@ -133,6 +137,17 @@ func generateRenewalFunction(cert *certsphase.KubeadmCert, caCert *certsphase.Ku return func() { internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfg.cfgPath, &cfg.cfg) kubeadmutil.CheckErr(err) + + if cfg.useCSR { + path := cfg.csrPath + if path == "" { + path = cfg.cfg.CertificatesDir + } + err := certsphase.CreateCSR(cert, internalcfg, path) + kubeadmutil.CheckErr(err) + return + } + renewer, err := getRenewer(cfg, caCert.BaseName) kubeadmutil.CheckErr(err) diff --git a/cmd/kubeadm/app/cmd/alpha/certs_test.go b/cmd/kubeadm/app/cmd/alpha/certs_test.go index be73d693de3..47798fd76cf 100644 --- a/cmd/kubeadm/app/cmd/alpha/certs_test.go +++ b/cmd/kubeadm/app/cmd/alpha/certs_test.go @@ -30,6 +30,7 @@ import ( "github.com/spf13/cobra" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" testutil "k8s.io/kubernetes/cmd/kubeadm/test" certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs" @@ -219,3 +220,16 @@ func TestRunRenewCommands(t *testing.T) { }) } } + +func TestRenewUsingCSR(t *testing.T) { + tmpDir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpDir) + cert := &certs.KubeadmCertEtcdServer + + renewCmds := getRenewSubCommands() + cmdtestutil.RunSubCommand(t, renewCmds, cert.Name, "--csr-only", "--csr-dir="+tmpDir) + + if _, _, err := pkiutil.TryLoadCSRAndKeyFromDisk(tmpDir, cert.BaseName); err != nil { + t.Fatalf("couldn't load certificate %q: %v", cert.BaseName, err) + } +} diff --git a/cmd/kubeadm/app/cmd/alpha/kubeconfig.go b/cmd/kubeadm/app/cmd/alpha/kubeconfig.go index 6b1178ee628..0c25c368a23 100644 --- a/cmd/kubeadm/app/cmd/alpha/kubeconfig.go +++ b/cmd/kubeadm/app/cmd/alpha/kubeconfig.go @@ -96,8 +96,8 @@ func newCmdUserKubeConfig(out io.Writer) *cobra.Command { // Add flags to the command cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, "The path where certificates are stored") - cmd.Flags().StringVar(&cfg.APIEndpoint.AdvertiseAddress, "apiserver-advertise-address", cfg.APIEndpoint.AdvertiseAddress, "The IP address the API server is accessible on") - cmd.Flags().Int32Var(&cfg.APIEndpoint.BindPort, "apiserver-bind-port", cfg.APIEndpoint.BindPort, "The port the API server is accessible on") + cmd.Flags().StringVar(&cfg.LocalAPIEndpoint.AdvertiseAddress, "apiserver-advertise-address", cfg.LocalAPIEndpoint.AdvertiseAddress, "The IP address the API server is accessible on") + cmd.Flags().Int32Var(&cfg.LocalAPIEndpoint.BindPort, "apiserver-bind-port", cfg.LocalAPIEndpoint.BindPort, "The port the API server is accessible on") cmd.Flags().StringVar(&token, "token", token, "The token that should be used as the authentication mechanism for this kubeconfig, instead of client certificates") cmd.Flags().StringVar(&clientName, "client-name", clientName, "The name of user. It will be used as the CN if client certificates are created") cmd.Flags().StringSliceVar(&organizations, "org", organizations, "The orgnizations of the client certificate. It will be used as the O if client certificates are created") diff --git a/cmd/kubeadm/app/cmd/alpha/preflight.go b/cmd/kubeadm/app/cmd/alpha/preflight.go index 40ff684c4bf..0b12d2cac5a 100644 --- a/cmd/kubeadm/app/cmd/alpha/preflight.go +++ b/cmd/kubeadm/app/cmd/alpha/preflight.go @@ -81,8 +81,11 @@ func newCmdPreFlightNode() *cobra.Command { internalcfg, err := configutil.JoinConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) kubeadmutil.CheckErr(err) - err = configutil.VerifyAPIServerBindAddress(internalcfg.APIEndpoint.AdvertiseAddress) - kubeadmutil.CheckErr(err) + + if internalcfg.ControlPlane != nil { + err = configutil.VerifyAPIServerBindAddress(internalcfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress) + kubeadmutil.CheckErr(err) + } fmt.Println("[preflight] running pre-flight checks") diff --git a/cmd/kubeadm/app/cmd/phases/selfhosting.go b/cmd/kubeadm/app/cmd/alpha/selfhosting.go similarity index 77% rename from cmd/kubeadm/app/cmd/phases/selfhosting.go rename to cmd/kubeadm/app/cmd/alpha/selfhosting.go index 6008181a391..b45b6b54d55 100644 --- a/cmd/kubeadm/app/cmd/phases/selfhosting.go +++ b/cmd/kubeadm/app/cmd/alpha/selfhosting.go @@ -14,12 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package phases +package alpha import ( - "os" + "bufio" + "errors" + "fmt" + "io" "strings" - "time" "github.com/spf13/cobra" @@ -27,6 +29,7 @@ import ( kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" @@ -36,6 +39,9 @@ import ( configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/pkg/util/normalizer" + + "os" + "time" ) var ( @@ -47,16 +53,14 @@ var ( ` + cmdutil.AlphaDisclaimer) selfhostingExample = normalizer.Examples(` - # Converts a static Pod-hosted control plane into a self-hosted one, - # functionally equivalent to what generated by kubeadm init executed - # with --feature-gates=SelfHosting=true. + # Converts a static Pod-hosted control plane into a self-hosted one. - kubeadm alpha phase selfhosting convert-from-staticpods + kubeadm alpha phase self-hosting convert-from-staticpods `) ) // NewCmdSelfhosting returns the self-hosting Cobra command -func NewCmdSelfhosting() *cobra.Command { +func NewCmdSelfhosting(in io.Reader) *cobra.Command { cmd := &cobra.Command{ Use: "selfhosting", Aliases: []string{"selfhosted", "self-hosting"}, @@ -64,29 +68,48 @@ func NewCmdSelfhosting() *cobra.Command { Long: cmdutil.MacroCommandLongDescription, } - cmd.AddCommand(getSelfhostingSubCommand()) + cmd.AddCommand(getSelfhostingSubCommand(in)) return cmd } -// getSelfhostingSubCommand returns sub commands for Selfhosting phase -func getSelfhostingSubCommand() *cobra.Command { +// getSelfhostingSubCommand returns sub commands for Self-hosting phase +func getSelfhostingSubCommand(in io.Reader) *cobra.Command { cfg := &kubeadmapiv1beta1.InitConfiguration{} // Default values for the cobra help text kubeadmscheme.Scheme.Default(cfg) var cfgPath, featureGatesString string + forcePivot, certsInSecrets := false, false kubeConfigFile := constants.GetAdminKubeConfigPath() // Creates the UX Command cmd := &cobra.Command{ - Use: "convert-from-staticpods", + Use: "pivot", Aliases: []string{"from-staticpods"}, Short: "Converts a static Pod-hosted control plane into a self-hosted one", Long: selfhostingLongDesc, Example: selfhostingExample, Run: func(cmd *cobra.Command, args []string) { + var err error + + if !forcePivot { + fmt.Println("WARNING: self-hosted clusters are not supported by kubeadm upgrade and by other kubeadm commands!") + fmt.Print("[pivot] are you sure you want to proceed? [y/n]: ") + s := bufio.NewScanner(in) + s.Scan() + + err = s.Err() + kubeadmutil.CheckErr(err) + + if strings.ToLower(s.Text()) != "y" { + kubeadmutil.CheckErr(errors.New("aborted pivot operation")) + } + } + + fmt.Println("[pivot] pivoting cluster to self-hosted") + if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { kubeadmutil.CheckErr(err) } @@ -102,7 +125,7 @@ func getSelfhostingSubCommand() *cobra.Command { // KubernetesVersion is not used, but we set it explicitly to avoid the lookup // of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig - SetKubernetesVersion(cfg) + phases.SetKubernetesVersion(cfg) // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) @@ -110,7 +133,7 @@ func getSelfhostingSubCommand() *cobra.Command { // Converts the Static Pod-hosted control plane into a self-hosted one waiter := apiclient.NewKubeWaiter(client, 2*time.Minute, os.Stdout) - err = selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter, false) + err = selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter, false, certsInSecrets) kubeadmutil.CheckErr(err) }, } @@ -119,8 +142,15 @@ func getSelfhostingSubCommand() *cobra.Command { // flags bound to the configuration object cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, `The path where certificates are stored`) cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to a kubeadm config file. WARNING: Usage of a configuration file is experimental") - cmd.Flags().StringVar(&featureGatesString, "feature-gates", featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ - "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) + + cmd.Flags().BoolVarP( + &certsInSecrets, "store-certs-in-secrets", "s", + false, "Enable storing certs in secrets") + + cmd.Flags().BoolVarP( + &forcePivot, "force", "f", false, + "Pivot the cluster without prompting for confirmation", + ) // flags that are not bound to the configuration object // Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go diff --git a/cmd/kubeadm/app/cmd/cmd.go b/cmd/kubeadm/app/cmd/cmd.go index 74656288eeb..17b827f4056 100644 --- a/cmd/kubeadm/app/cmd/cmd.go +++ b/cmd/kubeadm/app/cmd/cmd.go @@ -37,32 +37,31 @@ func NewKubeadmCommand(in io.Reader, out, err io.Writer) *cobra.Command { Use: "kubeadm", Short: "kubeadm: easily bootstrap a secure Kubernetes cluster", Long: dedent.Dedent(` - kubeadm: easily bootstrap a secure Kubernetes cluster. ┌──────────────────────────────────────────────────────────┐ - │ KUBEADM IS CURRENTLY IN BETA │ + │ KUBEADM │ + │ Easily bootstrap a secure Kubernetes cluster │ │ │ - │ But please, try it out and give us feedback at: │ + │ Please give us feedback at: │ │ https://github.com/kubernetes/kubeadm/issues │ - │ and at-mention @kubernetes/sig-cluster-lifecycle-bugs │ - │ or @kubernetes/sig-cluster-lifecycle-feature-requests │ └──────────────────────────────────────────────────────────┘ Example usage: - Create a two-machine cluster with one master (which controls the cluster), - and one node (where your workloads, like Pods and Deployments run). + Create a two-machine cluster with one control-plane node + (which controls the cluster), and one worker node + (where your workloads, like Pods and Deployments run). ┌──────────────────────────────────────────────────────────┐ │ On the first machine: │ ├──────────────────────────────────────────────────────────┤ - │ master# kubeadm init │ + │ control-plane# kubeadm init │ └──────────────────────────────────────────────────────────┘ ┌──────────────────────────────────────────────────────────┐ │ On the second machine: │ ├──────────────────────────────────────────────────────────┤ - │ node# kubeadm join │ + │ worker# kubeadm join │ └──────────────────────────────────────────────────────────┘ You can then repeat the second step on as many other machines as you like. @@ -89,7 +88,7 @@ func NewKubeadmCommand(in io.Reader, out, err io.Writer) *cobra.Command { cmds.AddCommand(NewCmdVersion(out)) cmds.AddCommand(NewCmdToken(out, err)) cmds.AddCommand(upgrade.NewCmdUpgrade(out)) - cmds.AddCommand(alpha.NewCmdAlpha(out)) + cmds.AddCommand(alpha.NewCmdAlpha(in, out)) AddKubeadmOtherFlags(cmds.PersistentFlags(), &rootfsPath) diff --git a/cmd/kubeadm/app/cmd/completion.go b/cmd/kubeadm/app/cmd/completion.go index 12da3e2fe09..2fae2b34326 100644 --- a/cmd/kubeadm/app/cmd/completion.go +++ b/cmd/kubeadm/app/cmd/completion.go @@ -20,10 +20,10 @@ import ( "bytes" "io" - "github.com/golang/glog" "github.com/pkg/errors" "github.com/renstrom/dedent" "github.com/spf13/cobra" + "k8s.io/klog" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) @@ -138,7 +138,7 @@ func RunCompletion(out io.Writer, boilerPlate string, cmd *cobra.Command, args [ } func runCompletionBash(out io.Writer, kubeadm *cobra.Command) error { - glog.V(1).Infoln("[completion] writing completion code for Bash") + klog.V(1).Infoln("[completion] writing completion code for Bash") return kubeadm.GenBashCompletion(out) } @@ -284,12 +284,12 @@ __kubeadm_convert_bash_to_zsh() { -e "s/\\\$(type${RWORD}/\$(__kubeadm_type/g" \ <<'BASH_COMPLETION_EOF' ` - glog.V(1).Infoln("[completion] writing completion code for Zsh") + klog.V(1).Infoln("[completion] writing completion code for Zsh") out.Write([]byte(zshInitialization)) buf := new(bytes.Buffer) kubeadm.GenBashCompletion(buf) - glog.V(1).Infoln("[completion] writing completion code for Bash") + klog.V(1).Infoln("[completion] writing completion code for Bash") out.Write(buf.Bytes()) zshTail := ` diff --git a/cmd/kubeadm/app/cmd/config.go b/cmd/kubeadm/app/cmd/config.go index ef8ff29f7b4..5f4ff01b87d 100644 --- a/cmd/kubeadm/app/cmd/config.go +++ b/cmd/kubeadm/app/cmd/config.go @@ -23,11 +23,11 @@ import ( "io/ioutil" "strings" - "github.com/golang/glog" "github.com/pkg/errors" "github.com/renstrom/dedent" "github.com/spf13/cobra" flag "github.com/spf13/pflag" + "k8s.io/klog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" @@ -243,7 +243,7 @@ func getAllAPIObjectNames() []string { func getDefaultedInitConfig() (*kubeadmapi.InitConfiguration, error) { return configutil.ConfigFileAndDefaultsToInternalConfig("", &kubeadmapiv1beta1.InitConfiguration{ // TODO: Probably move to getDefaultedClusterConfig? - APIEndpoint: kubeadmapiv1beta1.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapiv1beta1.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapiv1beta1.ClusterConfiguration{ KubernetesVersion: fmt.Sprintf("v1.%d.0", constants.MinimumControlPlaneVersion.Minor()+1), }, @@ -373,7 +373,7 @@ func NewCmdConfigView(out io.Writer, kubeConfigFile *string) *cobra.Command { The configuration is located in the %q namespace in the %q ConfigMap. `), metav1.NamespaceSystem, constants.KubeadmConfigConfigMap), Run: func(cmd *cobra.Command, args []string) { - glog.V(1).Infoln("[config] retrieving ClientSet from file") + klog.V(1).Infoln("[config] retrieving ClientSet from file") client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) kubeadmutil.CheckErr(err) @@ -402,15 +402,15 @@ func NewCmdConfigUploadFromFile(out io.Writer, kubeConfigFile *string) *cobra.Co kubeadmutil.CheckErr(errors.New("The --config flag is mandatory")) } - glog.V(1).Infoln("[config] retrieving ClientSet from file") + klog.V(1).Infoln("[config] retrieving ClientSet from file") client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) kubeadmutil.CheckErr(err) // The default configuration is empty; everything should come from the file on disk - glog.V(1).Infoln("[config] creating empty default configuration") + klog.V(1).Infoln("[config] creating empty default configuration") defaultcfg := &kubeadmapiv1beta1.InitConfiguration{} // Upload the configuration using the file; don't care about the defaultcfg really - glog.V(1).Infof("[config] uploading configuration") + klog.V(1).Infof("[config] uploading configuration") err = uploadConfiguration(client, cfgPath, defaultcfg) kubeadmutil.CheckErr(err) }, @@ -438,17 +438,17 @@ func NewCmdConfigUploadFromFlags(out io.Writer, kubeConfigFile *string) *cobra.C `), metav1.NamespaceSystem, constants.KubeadmConfigConfigMap), Run: func(cmd *cobra.Command, args []string) { var err error - glog.V(1).Infoln("[config] creating new FeatureGates") + klog.V(1).Infoln("[config] creating new FeatureGates") if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { kubeadmutil.CheckErr(err) } - glog.V(1).Infoln("[config] retrieving ClientSet from file") + klog.V(1).Infoln("[config] retrieving ClientSet from file") client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) kubeadmutil.CheckErr(err) // Default both statically and dynamically, convert to internal API type, and validate everything // The cfgPath argument is unset here as we shouldn't load a config file from disk, just go with cfg - glog.V(1).Infof("[config] uploading configuration") + klog.V(1).Infof("[config] uploading configuration") err = uploadConfiguration(client, "", cfg) kubeadmutil.CheckErr(err) }, @@ -460,7 +460,7 @@ func NewCmdConfigUploadFromFlags(out io.Writer, kubeConfigFile *string) *cobra.C // RunConfigView gets the configuration persisted in the cluster func RunConfigView(out io.Writer, client clientset.Interface) error { - glog.V(1).Infoln("[config] getting the cluster configuration") + klog.V(1).Infoln("[config] getting the cluster configuration") cfgConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(constants.KubeadmConfigConfigMap, metav1.GetOptions{}) if err != nil { return err @@ -478,7 +478,7 @@ func uploadConfiguration(client clientset.Interface, cfgPath string, defaultcfg // Default both statically and dynamically, convert to internal API type, and validate everything // First argument is unset here as we shouldn't load a config file from disk - glog.V(1).Infoln("[config] converting to internal API type") + klog.V(1).Infoln("[config] converting to internal API type") internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, defaultcfg) if err != nil { return err diff --git a/cmd/kubeadm/app/cmd/config_test.go b/cmd/kubeadm/app/cmd/config_test.go index 697f1291d69..3c0cdf38aca 100644 --- a/cmd/kubeadm/app/cmd/config_test.go +++ b/cmd/kubeadm/app/cmd/config_test.go @@ -29,11 +29,9 @@ import ( "github.com/renstrom/dedent" "github.com/spf13/cobra" - kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" @@ -89,8 +87,6 @@ func TestImagesListRunWithCustomConfigPath(t *testing.T) { apiVersion: kubeadm.k8s.io/v1beta1 kind: ClusterConfiguration kubernetesVersion: v1.11.0 - featureGates: - CoreDNS: true `)), }, } @@ -167,14 +163,23 @@ func TestConfigImagesListRunWithoutPath(t *testing.T) { name: "coredns enabled", cfg: kubeadmapiv1beta1.InitConfiguration{ ClusterConfiguration: kubeadmapiv1beta1.ClusterConfiguration{ - FeatureGates: map[string]bool{ - features.CoreDNS: true, - }, KubernetesVersion: dummyKubernetesVersion, }, }, expectedImages: defaultNumberOfImages, }, + { + name: "kube-dns enabled", + cfg: kubeadmapiv1beta1.InitConfiguration{ + ClusterConfiguration: kubeadmapiv1beta1.ClusterConfiguration{ + KubernetesVersion: dummyKubernetesVersion, + DNS: kubeadmapiv1beta1.DNS{ + Type: kubeadmapiv1beta1.KubeDNS, + }, + }, + }, + expectedImages: defaultNumberOfImages + 2, + }, } for _, tc := range testcases { @@ -242,7 +247,6 @@ func TestMigrate(t *testing.T) { # This is intentionally testing an old API version and the old kind naming and making sure the output is correct apiVersion: kubeadm.k8s.io/v1alpha3 kind: InitConfiguration - kubernetesVersion: v1.12.0 `)) configFile, cleanup := tempConfig(t, cfg) defer cleanup() diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 4d82e9d3c96..4e779f2f2b2 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -25,7 +25,6 @@ import ( "strings" "text/template" - "github.com/golang/glog" "github.com/pkg/errors" "github.com/renstrom/dedent" "github.com/spf13/cobra" @@ -42,22 +41,12 @@ import ( cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" - dnsaddonphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" - proxyaddonphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" - clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" - nodebootstraptokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" - markmasterphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster" - patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" - selfhostingphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" - uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" - "k8s.io/kubernetes/cmd/kubeadm/app/preflight" + kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - utilsexec "k8s.io/utils/exec" ) var ( @@ -109,6 +98,8 @@ type initOptions struct { cfgPath string skipTokenPrint bool dryRun bool + kubeconfigDir string + kubeconfigPath string featureGatesString string ignorePreflightErrors []string bto *options.BootstrapTokenOptions @@ -121,6 +112,8 @@ type initData struct { cfg *kubeadmapi.InitConfiguration skipTokenPrint bool dryRun bool + kubeconfigDir string + kubeconfigPath string ignorePreflightErrors sets.String certificatesDir string dryRunDir string @@ -132,7 +125,7 @@ type initData struct { // NewCmdInit returns "kubeadm init" command. func NewCmdInit(out io.Writer) *cobra.Command { - options := newInitOptions() + initOptions := newInitOptions() initRunner := workflow.NewRunner() cmd := &cobra.Command{ @@ -148,18 +141,26 @@ func NewCmdInit(out io.Writer) *cobra.Command { err = initRunner.Run() kubeadmutil.CheckErr(err) - // TODO: the code in runInit should be progressively converted in phases; each phase will be exposed - // via the subcommands automatically created by initRunner.BindToCommand - err = runInit(&data, out) + err = showJoinCommand(&data, out) kubeadmutil.CheckErr(err) }, } - // adds command flags - AddInitConfigFlags(cmd.PersistentFlags(), options.externalcfg, &options.featureGatesString) - AddInitOtherFlags(cmd.PersistentFlags(), &options.cfgPath, &options.skipTokenPrint, &options.dryRun, &options.ignorePreflightErrors) - options.bto.AddTokenFlag(cmd.PersistentFlags()) - options.bto.AddTTLFlag(cmd.PersistentFlags()) + // adds flags to the init command + // init command local flags could be eventually inherited by the sub-commands automatically generated for phases + AddInitConfigFlags(cmd.Flags(), initOptions.externalcfg, &initOptions.featureGatesString) + AddInitOtherFlags(cmd.Flags(), &initOptions.cfgPath, &initOptions.skipTokenPrint, &initOptions.dryRun, &initOptions.ignorePreflightErrors) + initOptions.bto.AddTokenFlag(cmd.Flags()) + initOptions.bto.AddTTLFlag(cmd.Flags()) + options.AddImageMetaFlags(cmd.Flags(), &initOptions.externalcfg.ImageRepository) + + // defines additional flag that are not used by the init command but that could be eventually used + // by the sub-commands automatically generated for phases + initRunner.SetAdditionalFlags(func(flags *flag.FlagSet) { + options.AddKubeConfigFlag(flags, &initOptions.kubeconfigPath) + options.AddKubeConfigDirFlag(flags, &initOptions.kubeconfigDir) + options.AddControlPlanExtraArgsFlags(flags, &initOptions.externalcfg.APIServer.ExtraArgs, &initOptions.externalcfg.ControllerManager.ExtraArgs, &initOptions.externalcfg.Scheduler.ExtraArgs) + }) // initialize the workflow runner with the list of phases initRunner.AppendPhase(phases.NewPreflightMasterPhase()) @@ -169,12 +170,15 @@ func NewCmdInit(out io.Writer) *cobra.Command { initRunner.AppendPhase(phases.NewControlPlanePhase()) initRunner.AppendPhase(phases.NewEtcdPhase()) initRunner.AppendPhase(phases.NewWaitControlPlanePhase()) - // TODO: add other phases to the runner. + initRunner.AppendPhase(phases.NewUploadConfigPhase()) + initRunner.AppendPhase(phases.NewMarkControlPlanePhase()) + initRunner.AppendPhase(phases.NewBootstrapTokenPhase()) + initRunner.AppendPhase(phases.NewAddonPhase()) // sets the data builder function, that will be used by the runner // both when running the entire workflow or single phases - initRunner.SetDataInitializer(func() (workflow.RunData, error) { - return newInitData(cmd, options, out) + initRunner.SetDataInitializer(func(cmd *cobra.Command) (workflow.RunData, error) { + return newInitData(cmd, initOptions, out) }) // binds the Runner to kubeadm init command by altering @@ -187,67 +191,67 @@ func NewCmdInit(out io.Writer) *cobra.Command { // AddInitConfigFlags adds init flags bound to the config to the specified flagset func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta1.InitConfiguration, featureGatesString *string) { flagSet.StringVar( - &cfg.APIEndpoint.AdvertiseAddress, "apiserver-advertise-address", cfg.APIEndpoint.AdvertiseAddress, + &cfg.LocalAPIEndpoint.AdvertiseAddress, options.APIServerAdvertiseAddress, cfg.LocalAPIEndpoint.AdvertiseAddress, "The IP address the API Server will advertise it's listening on. Specify '0.0.0.0' to use the address of the default network interface.", ) flagSet.Int32Var( - &cfg.APIEndpoint.BindPort, "apiserver-bind-port", cfg.APIEndpoint.BindPort, + &cfg.LocalAPIEndpoint.BindPort, options.APIServerBindPort, cfg.LocalAPIEndpoint.BindPort, "Port for the API Server to bind to.", ) flagSet.StringVar( - &cfg.Networking.ServiceSubnet, "service-cidr", cfg.Networking.ServiceSubnet, + &cfg.Networking.ServiceSubnet, options.NetworkingServiceSubnet, cfg.Networking.ServiceSubnet, "Use alternative range of IP address for service VIPs.", ) flagSet.StringVar( - &cfg.Networking.PodSubnet, "pod-network-cidr", cfg.Networking.PodSubnet, + &cfg.Networking.PodSubnet, options.NetworkingPodSubnet, cfg.Networking.PodSubnet, "Specify range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node.", ) flagSet.StringVar( - &cfg.Networking.DNSDomain, "service-dns-domain", cfg.Networking.DNSDomain, + &cfg.Networking.DNSDomain, options.NetworkingDNSDomain, cfg.Networking.DNSDomain, `Use alternative domain for services, e.g. "myorg.internal".`, ) flagSet.StringVar( - &cfg.KubernetesVersion, "kubernetes-version", cfg.KubernetesVersion, + &cfg.KubernetesVersion, options.KubernetesVersion, cfg.KubernetesVersion, `Choose a specific Kubernetes version for the control plane.`, ) flagSet.StringVar( - &cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, + &cfg.CertificatesDir, options.CertificatesDir, cfg.CertificatesDir, `The path where to save and store the certificates.`, ) flagSet.StringSliceVar( - &cfg.APIServer.CertSANs, "apiserver-cert-extra-sans", cfg.APIServer.CertSANs, + &cfg.APIServer.CertSANs, options.APIServerCertSANs, cfg.APIServer.CertSANs, `Optional extra Subject Alternative Names (SANs) to use for the API Server serving certificate. Can be both IP addresses and DNS names.`, ) flagSet.StringVar( - &cfg.NodeRegistration.Name, "node-name", cfg.NodeRegistration.Name, + &cfg.NodeRegistration.Name, options.NodeName, cfg.NodeRegistration.Name, `Specify the node name.`, ) flagSet.StringVar( - &cfg.NodeRegistration.CRISocket, "cri-socket", cfg.NodeRegistration.CRISocket, + &cfg.NodeRegistration.CRISocket, options.NodeCRISocket, cfg.NodeRegistration.CRISocket, `Specify the CRI socket to connect to.`, ) - flagSet.StringVar(featureGatesString, "feature-gates", *featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ + flagSet.StringVar(featureGatesString, options.FeatureGatesString, *featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) } // AddInitOtherFlags adds init flags that are not bound to a configuration file to the given flagset func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipTokenPrint, dryRun *bool, ignorePreflightErrors *[]string) { flagSet.StringVar( - cfgPath, "config", *cfgPath, + cfgPath, options.CfgPath, *cfgPath, "Path to kubeadm config file. WARNING: Usage of a configuration file is experimental.", ) flagSet.StringSliceVar( - ignorePreflightErrors, "ignore-preflight-errors", *ignorePreflightErrors, + ignorePreflightErrors, options.IgnorePreflightErrors, *ignorePreflightErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", ) // Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go flagSet.BoolVar( - skipTokenPrint, "skip-token-print", *skipTokenPrint, + skipTokenPrint, options.SkipTokenPrint, *skipTokenPrint, "Skip printing of the default bootstrap token generated by 'kubeadm init'.", ) // Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go flagSet.BoolVar( - dryRun, "dry-run", *dryRun, + dryRun, options.DryRun, *dryRun, "Don't apply any changes; just output what would be done.", ) } @@ -263,8 +267,10 @@ func newInitOptions() *initOptions { bto.Description = "The default bootstrap token generated by 'kubeadm init'." return &initOptions{ - externalcfg: externalcfg, - bto: bto, + externalcfg: externalcfg, + bto: bto, + kubeconfigDir: kubeadmconstants.KubernetesDir, + kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(), } } @@ -299,7 +305,7 @@ func newInitData(cmd *cobra.Command, options *initOptions, out io.Writer) (initD if err != nil { return initData{}, err } - if err := configutil.VerifyAPIServerBindAddress(cfg.APIEndpoint.AdvertiseAddress); err != nil { + if err := configutil.VerifyAPIServerBindAddress(cfg.LocalAPIEndpoint.AdvertiseAddress); err != nil { return initData{}, err } if err := features.ValidateVersion(features.InitFeatureGates, cfg.FeatureGates, cfg.KubernetesVersion); err != nil { @@ -316,6 +322,15 @@ func newInitData(cmd *cobra.Command, options *initOptions, out io.Writer) (initD // Checks if an external CA is provided by the user. externalCA, _ := certsphase.UsingExternalCA(cfg) + if externalCA { + kubeconfigDir := kubeadmconstants.KubernetesDir + if options.dryRun { + kubeconfigDir = dryRunDir + } + if err := kubeconfigphase.ValidateKubeconfigsForExternalCA(kubeconfigDir, cfg); err != nil { + return initData{}, err + } + } return initData{ cfg: cfg, @@ -323,6 +338,8 @@ func newInitData(cmd *cobra.Command, options *initOptions, out io.Writer) (initD skipTokenPrint: options.skipTokenPrint, dryRun: options.dryRun, dryRunDir: dryRunDir, + kubeconfigDir: options.kubeconfigDir, + kubeconfigPath: options.kubeconfigPath, ignorePreflightErrors: ignorePreflightErrorsSet, externalCA: externalCA, outputWriter: out, @@ -367,7 +384,15 @@ func (d initData) KubeConfigDir() string { if d.dryRun { return d.dryRunDir } - return kubeadmconstants.KubernetesDir + return d.kubeconfigDir +} + +// KubeConfigPath returns the path to the kubeconfig file to use for connecting to Kubernetes +func (d initData) KubeConfigPath() string { + if d.dryRun { + d.kubeconfigPath = filepath.Join(d.dryRunDir, kubeadmconstants.AdminKubeConfigFileName) + } + return d.kubeconfigPath } // ManifestDir returns the path where manifest should be stored or the temporary folder path in case of DryRun. @@ -408,7 +433,7 @@ func (d initData) Client() (clientset.Interface, error) { } else { // If we're acting for real, we should create a connection to the API server and wait for it to come up var err error - d.client, err = kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath()) + d.client, err = kubeconfigutil.ClientSetFromFile(d.KubeConfigPath()) if err != nil { return nil, err } @@ -426,151 +451,6 @@ func (d initData) Tokens() []string { return tokens } -// runInit executes master node provisioning -func runInit(i *initData, out io.Writer) error { - - // Get directories to write files to; can be faked if we're dry-running - glog.V(1).Infof("[init] Getting certificates directory from configuration") - certsDirToWriteTo, kubeConfigDir, manifestDir, _, err := getDirectoriesToUse(i.dryRun, i.dryRunDir, i.cfg.CertificatesDir) - if err != nil { - return errors.Wrap(err, "error getting directories to use") - } - - // certsDirToWriteTo is gonna equal cfg.CertificatesDir in the normal case, but gonna be a temp directory if dryrunning - i.cfg.CertificatesDir = certsDirToWriteTo - - adminKubeConfigPath := filepath.Join(kubeConfigDir, kubeadmconstants.AdminKubeConfigFileName) - - // TODO: client and waiter are temporary until the rest of the phases that use them - // are removed from this function. - client, err := i.Client() - if err != nil { - return errors.Wrap(err, "failed to create client") - } - // TODO: NewControlPlaneWaiter should be converted to private after the self-hosting phase is removed. - waiter, err := phases.NewControlPlaneWaiter(i.dryRun, client, i.outputWriter) - if err != nil { - return errors.Wrap(err, "failed to create waiter") - } - - // Upload currently used configuration to the cluster - // Note: This is done right in the beginning of cluster initialization; as we might want to make other phases - // depend on centralized information from this source in the future - glog.V(1).Infof("[init] uploading currently used configuration to the cluster") - if err := uploadconfigphase.UploadConfiguration(i.cfg, client); err != nil { - return errors.Wrap(err, "error uploading configuration") - } - - glog.V(1).Infof("[init] creating kubelet configuration configmap") - if err := kubeletphase.CreateConfigMap(i.cfg, client); err != nil { - return errors.Wrap(err, "error creating kubelet configuration ConfigMap") - } - - // PHASE 4: Mark the master with the right label/taint - glog.V(1).Infof("[init] marking the master with right label") - if err := markmasterphase.MarkMaster(client, i.cfg.NodeRegistration.Name, i.cfg.NodeRegistration.Taints); err != nil { - return errors.Wrap(err, "error marking master") - } - - glog.V(1).Infof("[init] preserving the crisocket information for the master") - if err := patchnodephase.AnnotateCRISocket(client, i.cfg.NodeRegistration.Name, i.cfg.NodeRegistration.CRISocket); err != nil { - return errors.Wrap(err, "error uploading crisocket") - } - - // This feature is disabled by default - if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { - kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New()) - if err != nil { - return err - } - - // Enable dynamic kubelet configuration for the node. - if err := kubeletphase.EnableDynamicConfigForNode(client, i.cfg.NodeRegistration.Name, kubeletVersion); err != nil { - return errors.Wrap(err, "error enabling dynamic kubelet configuration") - } - } - - // PHASE 5: Set up the node bootstrap tokens - tokens := []string{} - for _, bt := range i.cfg.BootstrapTokens { - tokens = append(tokens, bt.Token.String()) - } - if !i.skipTokenPrint { - if len(tokens) == 1 { - fmt.Printf("[bootstraptoken] using token: %s\n", tokens[0]) - } else if len(tokens) > 1 { - fmt.Printf("[bootstraptoken] using tokens: %v\n", tokens) - } - } - - // Create the default node bootstrap token - glog.V(1).Infof("[init] creating RBAC rules to generate default bootstrap token") - if err := nodebootstraptokenphase.UpdateOrCreateTokens(client, false, i.cfg.BootstrapTokens); err != nil { - return errors.Wrap(err, "error updating or creating token") - } - // Create RBAC rules that makes the bootstrap tokens able to post CSRs - glog.V(1).Infof("[init] creating RBAC rules to allow bootstrap tokens to post CSR") - if err := nodebootstraptokenphase.AllowBootstrapTokensToPostCSRs(client); err != nil { - return errors.Wrap(err, "error allowing bootstrap tokens to post CSRs") - } - // Create RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically - glog.V(1).Infof("[init] creating RBAC rules to automatic approval of CSRs automatically") - if err := nodebootstraptokenphase.AutoApproveNodeBootstrapTokens(client); err != nil { - return errors.Wrap(err, "error auto-approving node bootstrap tokens") - } - - // Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically - glog.V(1).Infof("[init] creating/updating RBAC rules for rotating certificate") - if err := nodebootstraptokenphase.AutoApproveNodeCertificateRotation(client); err != nil { - return err - } - - // Create the cluster-info ConfigMap with the associated RBAC rules - glog.V(1).Infof("[init] creating bootstrap configmap") - if err := clusterinfophase.CreateBootstrapConfigMapIfNotExists(client, adminKubeConfigPath); err != nil { - return errors.Wrap(err, "error creating bootstrap configmap") - } - glog.V(1).Infof("[init] creating ClusterInfo RBAC rules") - if err := clusterinfophase.CreateClusterInfoRBACRules(client); err != nil { - return errors.Wrap(err, "error creating clusterinfo RBAC rules") - } - - glog.V(1).Infof("[init] ensuring DNS addon") - if err := dnsaddonphase.EnsureDNSAddon(i.cfg, client); err != nil { - return errors.Wrap(err, "error ensuring dns addon") - } - - glog.V(1).Infof("[init] ensuring proxy addon") - if err := proxyaddonphase.EnsureProxyAddon(i.cfg, client); err != nil { - return errors.Wrap(err, "error ensuring proxy addon") - } - - // PHASE 7: Make the control plane self-hosted if feature gate is enabled - if features.Enabled(i.cfg.FeatureGates, features.SelfHosting) { - glog.V(1).Infof("[init] feature gate is enabled. Making control plane self-hosted") - // Temporary control plane is up, now we create our self hosted control - // plane components and remove the static manifests: - fmt.Println("[self-hosted] creating self-hosted control plane") - if err := selfhostingphase.CreateSelfHostedControlPlane(manifestDir, kubeConfigDir, i.cfg, client, waiter, i.dryRun); err != nil { - return errors.Wrap(err, "error creating self hosted control plane") - } - } - - // Exit earlier if we're dryrunning - if i.dryRun { - fmt.Println("[dryrun] finished dry-running successfully. Above are the resources that would be created") - return nil - } - - // Prints the join command, multiple times in case the user has multiple tokens - for _, token := range tokens { - if err := printJoinCommand(out, adminKubeConfigPath, token, i.skipTokenPrint); err != nil { - return errors.Wrap(err, "failed to print join command") - } - } - return nil -} - func printJoinCommand(out io.Writer, adminKubeConfigPath, token string, skipTokenPrint bool) error { joinCommand, err := cmdutil.GetJoinCommand(adminKubeConfigPath, token, skipTokenPrint) if err != nil { @@ -595,3 +475,17 @@ func getDirectoriesToUse(dryRun bool, dryRunDir string, defaultPkiDir string) (s return defaultPkiDir, kubeadmconstants.KubernetesDir, kubeadmconstants.GetStaticPodDirectory(), kubeadmconstants.KubeletRunDirectory, nil } + +// showJoinCommand prints the join command after all the phases in init have finished +func showJoinCommand(i *initData, out io.Writer) error { + adminKubeConfigPath := i.KubeConfigPath() + + // Prints the join command, multiple times in case the user has multiple tokens + for _, token := range i.Tokens() { + if err := printJoinCommand(out, adminKubeConfigPath, token, i.skipTokenPrint); err != nil { + return errors.Wrap(err, "failed to print join command") + } + } + + return nil +} diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 40c6abceeaf..cce58a1f337 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -22,10 +22,8 @@ import ( "io" "os" "path/filepath" - "strings" "text/template" - "github.com/golang/glog" "github.com/pkg/errors" "github.com/renstrom/dedent" "github.com/spf13/cobra" @@ -34,19 +32,19 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" certutil "k8s.io/client-go/util/cert" + "k8s.io/klog" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/discovery" - "k8s.io/kubernetes/cmd/kubeadm/app/features" certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" - markmasterphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster" + markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane" patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" @@ -164,8 +162,10 @@ func NewCmdJoin(out io.Writer) *cobra.Command { var token string var cfgPath string - var featureGatesString string var ignorePreflightErrors []string + var controlPlane bool + var advertiseAddress string + var bindPort int32 = kubeadmapiv1beta1.DefaultAPIBindPort cmd := &cobra.Command{ Use: "join", @@ -182,7 +182,7 @@ func NewCmdJoin(out io.Writer) *cobra.Command { } if len(args) > 0 { if len(cfgPath) == 0 && len(args) > 1 { - glog.Warningf("[join] WARNING: More than one API server endpoint supplied on command line %v. Using the first one.", args) + klog.Warningf("[join] WARNING: More than one API server endpoint supplied on command line %v. Using the first one.", args) } cfg.Discovery.BootstrapToken.APIServerEndpoint = args[0] } @@ -192,28 +192,34 @@ func NewCmdJoin(out io.Writer) *cobra.Command { cfg.Discovery.TLSBootstrapToken = token } - j, err := NewValidJoin(cmd.PersistentFlags(), cfg, cfgPath, featureGatesString, ignorePreflightErrors) + if controlPlane { + cfg.ControlPlane = &kubeadmapiv1beta1.JoinControlPlane{ + LocalAPIEndpoint: kubeadmapiv1beta1.APIEndpoint{ + AdvertiseAddress: advertiseAddress, + BindPort: bindPort, + }, + } + } + + j, err := NewValidJoin(cmd.PersistentFlags(), cfg, cfgPath, ignorePreflightErrors) kubeadmutil.CheckErr(err) kubeadmutil.CheckErr(j.Run(out)) }, } - AddJoinConfigFlags(cmd.PersistentFlags(), cfg, &featureGatesString, &token) + AddJoinConfigFlags(cmd.PersistentFlags(), cfg, &token) AddJoinBootstrapTokenDiscoveryFlags(cmd.PersistentFlags(), btd) AddJoinFileDiscoveryFlags(cmd.PersistentFlags(), fd) - AddJoinOtherFlags(cmd.PersistentFlags(), &cfgPath, &ignorePreflightErrors) + AddJoinOtherFlags(cmd.PersistentFlags(), &cfgPath, &ignorePreflightErrors, &controlPlane, &advertiseAddress, &bindPort) return cmd } // NewValidJoin validates the command line that are passed to the cobra command -func NewValidJoin(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta1.JoinConfiguration, cfgPath, featureGatesString string, ignorePreflightErrors []string) (*Join, error) { +func NewValidJoin(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta1.JoinConfiguration, cfgPath string, ignorePreflightErrors []string) (*Join, error) { var err error - if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { - return nil, err - } - if err := validation.ValidateMixedArguments(flagSet); err != nil { + if err = validation.ValidateMixedArguments(flagSet); err != nil { return nil, err } @@ -226,32 +232,17 @@ func NewValidJoin(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta1.JoinConfiguratio } // AddJoinConfigFlags adds join flags bound to the config to the specified flagset -func AddJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta1.JoinConfiguration, featureGatesString *string, token *string) { +func AddJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta1.JoinConfiguration, token *string) { flagSet.StringVar( &cfg.NodeRegistration.Name, "node-name", cfg.NodeRegistration.Name, "Specify the node name.") flagSet.StringVar( token, "token", "", "Use this token for both discovery-token and tls-bootstrap-token when those values are not provided.") - flagSet.StringVar( - featureGatesString, "feature-gates", *featureGatesString, - "A set of key=value pairs that describe feature gates for various features. "+ - "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) flagSet.StringVar( &cfg.NodeRegistration.CRISocket, "cri-socket", cfg.NodeRegistration.CRISocket, `Specify the CRI socket to connect to.`, ) - flagSet.BoolVar( - &cfg.ControlPlane, "experimental-control-plane", cfg.ControlPlane, - "Create a new control plane instance on this node") - flagSet.StringVar( - &cfg.APIEndpoint.AdvertiseAddress, "apiserver-advertise-address", cfg.APIEndpoint.AdvertiseAddress, - "If the node should host a new control plane instance, the IP address the API Server will advertise it's listening on.", - ) - flagSet.Int32Var( - &cfg.APIEndpoint.BindPort, "apiserver-bind-port", cfg.APIEndpoint.BindPort, - "If the node should host a new control plane instance, the port for the API Server to bind to.", - ) } // AddJoinBootstrapTokenDiscoveryFlags adds bootstrap token specific discovery flags to the specified flagset @@ -275,20 +266,29 @@ func AddJoinFileDiscoveryFlags(flagSet *flag.FlagSet, fd *kubeadmapiv1beta1.File } // AddJoinOtherFlags adds join flags that are not bound to a configuration file to the given flagset -func AddJoinOtherFlags(flagSet *flag.FlagSet, cfgPath *string, ignorePreflightErrors *[]string) { +func AddJoinOtherFlags(flagSet *flag.FlagSet, cfgPath *string, ignorePreflightErrors *[]string, controlPlane *bool, advertiseAddress *string, bindPort *int32) { flagSet.StringVar( cfgPath, "config", *cfgPath, "Path to kubeadm config file.") - flagSet.StringSliceVar( ignorePreflightErrors, "ignore-preflight-errors", *ignorePreflightErrors, - "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", - ) + "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.") + flagSet.BoolVar( + controlPlane, "experimental-control-plane", *controlPlane, + "Create a new control plane instance on this node") + flagSet.StringVar( + advertiseAddress, "apiserver-advertise-address", *advertiseAddress, + "If the node should host a new control plane instance, the IP address the API Server will advertise it's listening on.") + flagSet.Int32Var( + bindPort, "apiserver-bind-port", *bindPort, + "If the node should host a new control plane instance, the port for the API Server to bind to.") } // Join defines struct used by kubeadm join command type Join struct { cfg *kubeadmapi.JoinConfiguration + initCfg *kubeadmapi.InitConfiguration + tlsBootstrapCfg *clientcmdapi.Config ignorePreflightErrors sets.String } @@ -296,59 +296,53 @@ type Join struct { func NewJoin(cfgPath string, defaultcfg *kubeadmapiv1beta1.JoinConfiguration, ignorePreflightErrors sets.String) (*Join, error) { if defaultcfg.NodeRegistration.Name == "" { - glog.V(1).Infoln("[join] found NodeName empty; using OS hostname as NodeName") + klog.V(1).Infoln("[join] found NodeName empty; using OS hostname as NodeName") } - if defaultcfg.APIEndpoint.AdvertiseAddress == "" { - glog.V(1).Infoln("[join] found advertiseAddress empty; using default interface's IP address as advertiseAddress") + if defaultcfg.ControlPlane != nil && defaultcfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress == "" { + klog.V(1).Infoln("[join] found advertiseAddress empty; using default interface's IP address as advertiseAddress") } - internalcfg, err := configutil.JoinConfigFileAndDefaultsToInternalConfig(cfgPath, defaultcfg) + internalCfg, err := configutil.JoinConfigFileAndDefaultsToInternalConfig(cfgPath, defaultcfg) if err != nil { return nil, err } - if err := configutil.VerifyAPIServerBindAddress(internalcfg.APIEndpoint.AdvertiseAddress); err != nil { + if defaultcfg.ControlPlane != nil { + if err := configutil.VerifyAPIServerBindAddress(internalCfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress); err != nil { + return nil, err + } + } + + fmt.Println("[preflight] Running pre-flight checks") + + // Start with general checks + klog.V(1).Infoln("[preflight] Running general checks") + if err := preflight.RunJoinNodeChecks(utilsexec.New(), internalCfg, ignorePreflightErrors); err != nil { return nil, err } - fmt.Println("[preflight] running pre-flight checks") - - // Then continue with the others... - glog.V(1).Infoln("[preflight] running various checks on all nodes") - if err := preflight.RunJoinNodeChecks(utilsexec.New(), internalcfg, ignorePreflightErrors); err != nil { + // Fetch the init configuration based on the join configuration + klog.V(1).Infoln("[preflight] Fetching init configuration") + initCfg, tlsBootstrapCfg, err := fetchInitConfigurationFromJoinConfiguration(internalCfg) + if err != nil { return nil, err } - return &Join{cfg: internalcfg, ignorePreflightErrors: ignorePreflightErrors}, nil + // Continue with more specific checks based on the init configuration + klog.V(1).Infoln("[preflight] Running configuration dependant checks") + if err := preflight.RunOptionalJoinNodeChecks(utilsexec.New(), initCfg, ignorePreflightErrors); err != nil { + return nil, err + } + + return &Join{cfg: internalCfg, initCfg: initCfg, tlsBootstrapCfg: tlsBootstrapCfg, ignorePreflightErrors: ignorePreflightErrors}, nil } // Run executes worker node provisioning and tries to join an existing cluster. func (j *Join) Run(out io.Writer) error { - // Perform the Discovery, which turns a Bootstrap Token and optionally (and preferably) a CA cert hash into a kubeconfig - // file that may be used for the TLS Bootstrapping process the kubelet performs using the Certificates API. - glog.V(1).Infoln("[join] discovering cluster-info") - tlsBootstrapCfg, err := discovery.For(j.cfg) - if err != nil { - return err - } - - // If the node should host a new control plane instance - var initConfiguration *kubeadmapi.InitConfiguration - if j.cfg.ControlPlane == true { - // Retrives the kubeadm configuration used during kubeadm init - glog.V(1).Infoln("[join] retrieving kubeconfig objects") - initConfiguration, err = j.FetchInitConfiguration(tlsBootstrapCfg) - if err != nil { - return err - } - - // injects into the kubeadm configuration the information about the joining node - initConfiguration.NodeRegistration = j.cfg.NodeRegistration - initConfiguration.APIEndpoint = j.cfg.APIEndpoint - + if j.cfg.ControlPlane != nil { // Checks if the cluster configuration supports // joining a new control plane instance and if all the necessary certificates are provided - if err = j.CheckIfReadyForAdditionalControlPlane(initConfiguration); err != nil { + if err := j.CheckIfReadyForAdditionalControlPlane(j.initCfg); err != nil { // outputs the not ready for hosting a new control plane instance message ctx := map[string]string{ "Error": err.Error(), @@ -360,12 +354,12 @@ func (j *Join) Run(out io.Writer) error { } // run kubeadm init preflight checks for checking all the prequisites - fmt.Printf("[join] running pre-flight checks before initializing the new control plane instance\n") - preflight.RunInitMasterChecks(utilsexec.New(), initConfiguration, j.ignorePreflightErrors) + fmt.Printf("[join] Running pre-flight checks before initializing the new control plane instance\n") + preflight.RunInitMasterChecks(utilsexec.New(), j.initCfg, j.ignorePreflightErrors) // Prepares the node for hosting a new control plane instance by writing necessary // kubeconfig files, and static pod manifests - if err = j.PrepareForHostingControlPlane(initConfiguration); err != nil { + if err := j.PrepareForHostingControlPlane(j.initCfg); err != nil { return err } } @@ -376,21 +370,21 @@ func (j *Join) Run(out io.Writer) error { // if the node is hosting a new control plane instance, since it uses static pods for the control plane, // as soon as the kubelet starts it will take charge of creating control plane // components on the node. - if err = j.BootstrapKubelet(tlsBootstrapCfg); err != nil { + if err := j.BootstrapKubelet(); err != nil { return err } // if the node is hosting a new control plane instance - if j.cfg.ControlPlane == true { + if j.cfg.ControlPlane != nil { // Completes the control plane setup - if err := j.PostInstallControlPlane(initConfiguration); err != nil { + if err := j.PostInstallControlPlane(j.initCfg); err != nil { return err } // outputs the join control plane done template and exits etcdMessage := "" // in case of local etcd - if initConfiguration.Etcd.External == nil { + if j.initCfg.Etcd.External == nil { etcdMessage = "* A new etcd member was added to the local/stacked etcd cluster." } @@ -408,23 +402,6 @@ func (j *Join) Run(out io.Writer) error { return nil } -// FetchInitConfiguration reads the cluster configuration from the kubeadm-admin configMap, -func (j *Join) FetchInitConfiguration(tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) { - // creates a client to access the cluster using the bootstrap token identity - tlsClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg) - if err != nil { - return nil, errors.Wrap(err, "Unable to access the cluster") - } - - // Fetches the init configuration - initConfiguration, err := configutil.FetchConfigFromFileOrCluster(tlsClient, os.Stdout, "join", "", true) - if err != nil { - return nil, errors.Wrap(err, "Unable to fetch the kubeadm-config ConfigMap") - } - - return initConfiguration, nil -} - // CheckIfReadyForAdditionalControlPlane ensures that the cluster is in a state that supports // joining an additional control plane instance and if the node is ready to join func (j *Join) CheckIfReadyForAdditionalControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error { @@ -433,16 +410,6 @@ func (j *Join) CheckIfReadyForAdditionalControlPlane(initConfiguration *kubeadma return errors.New("unable to add a new control plane instance a cluster that doesn't have a stable controlPlaneEndpoint address") } - // blocks if control plane is self-hosted - if features.Enabled(initConfiguration.FeatureGates, features.SelfHosting) { - return errors.New("self-hosted clusters are deprecated and won't be supported by `kubeadm join --experimental-control-plane`") - } - - // blocks if the certificates for the control plane are stored in secrets (instead of the local pki folder) - if features.Enabled(initConfiguration.FeatureGates, features.StoreCertsInSecrets) { - return errors.New("certificates stored in secrets, as well as self-hosted clusters are deprecated and won't be supported by `kubeadm join --experimental-control-plane`") - } - // checks if the certificates that must be equal across contolplane instances are provided if ret, err := certsphase.SharedCertificateExists(initConfiguration); !ret { return err @@ -495,19 +462,19 @@ func (j *Join) PrepareForHostingControlPlane(initConfiguration *kubeadmapi.InitC // BootstrapKubelet executes the kubelet TLS bootstrap process. // This process is executed by the kubelet and completes with the node joining the cluster // with a dedicates set of credentials as required by the node authorizer -func (j *Join) BootstrapKubelet(tlsBootstrapCfg *clientcmdapi.Config) error { +func (j *Join) BootstrapKubelet() error { bootstrapKubeConfigFile := kubeadmconstants.GetBootstrapKubeletKubeConfigPath() // Write the bootstrap kubelet config file or the TLS-Boostrapped kubelet config file down to disk - glog.V(1).Infoln("[join] writing bootstrap kubelet config file at", bootstrapKubeConfigFile) - if err := kubeconfigutil.WriteToDisk(bootstrapKubeConfigFile, tlsBootstrapCfg); err != nil { + klog.V(1).Infoln("[join] writing bootstrap kubelet config file at", bootstrapKubeConfigFile) + if err := kubeconfigutil.WriteToDisk(bootstrapKubeConfigFile, j.tlsBootstrapCfg); err != nil { return errors.Wrap(err, "couldn't save bootstrap-kubelet.conf to disk") } // Write the ca certificate to disk so kubelet can use it for authentication - cluster := tlsBootstrapCfg.Contexts[tlsBootstrapCfg.CurrentContext].Cluster + cluster := j.tlsBootstrapCfg.Contexts[j.tlsBootstrapCfg.CurrentContext].Cluster if _, err := os.Stat(j.cfg.CACertPath); os.IsNotExist(err) { - if err := certutil.WriteCert(j.cfg.CACertPath, tlsBootstrapCfg.Clusters[cluster].CertificateAuthorityData); err != nil { + if err := certutil.WriteCert(j.cfg.CACertPath, j.tlsBootstrapCfg.Clusters[cluster].CertificateAuthorityData); err != nil { return errors.Wrap(err, "couldn't save the CA certificate to disk") } } @@ -524,7 +491,7 @@ func (j *Join) BootstrapKubelet(tlsBootstrapCfg *clientcmdapi.Config) error { // Configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet // Try to stop the kubelet service so no race conditions occur when configuring it - glog.V(1).Infof("Stopping the kubelet") + klog.V(1).Infof("Stopping the kubelet") kubeletphase.TryStopKubelet() // Write the configuration for the kubelet (using the bootstrap token credentials) to disk so the kubelet can start @@ -535,13 +502,13 @@ func (j *Join) BootstrapKubelet(tlsBootstrapCfg *clientcmdapi.Config) error { // Write env file with flags for the kubelet to use. We only want to // register the joining node with the specified taints if the node // is not a master. The markmaster phase will register the taints otherwise. - registerTaintsUsingFlags := !j.cfg.ControlPlane - if err := kubeletphase.WriteKubeletDynamicEnvFile(&j.cfg.NodeRegistration, j.cfg.FeatureGates, registerTaintsUsingFlags, kubeadmconstants.KubeletRunDirectory); err != nil { + registerTaintsUsingFlags := j.cfg.ControlPlane == nil + if err := kubeletphase.WriteKubeletDynamicEnvFile(j.initCfg, registerTaintsUsingFlags, kubeadmconstants.KubeletRunDirectory); err != nil { return err } // Try to start the kubelet service in case it's inactive - glog.V(1).Infof("Starting the kubelet") + klog.V(1).Infof("Starting the kubelet") kubeletphase.TryStartKubelet() // Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf @@ -559,18 +526,11 @@ func (j *Join) BootstrapKubelet(tlsBootstrapCfg *clientcmdapi.Config) error { return err } - glog.V(1).Infof("[join] preserving the crisocket information for the node") + klog.V(1).Infof("[join] preserving the crisocket information for the node") if err := patchnodephase.AnnotateCRISocket(client, j.cfg.NodeRegistration.Name, j.cfg.NodeRegistration.CRISocket); err != nil { return errors.Wrap(err, "error uploading crisocket") } - // This feature is disabled by default in kubeadm - if features.Enabled(j.cfg.FeatureGates, features.DynamicKubeletConfig) { - if err := kubeletphase.EnableDynamicConfigForNode(client, j.cfg.NodeRegistration.Name, kubeletVersion); err != nil { - return errors.Wrap(err, "error consuming base kubelet configuration") - } - } - return nil } @@ -594,20 +554,20 @@ func (j *Join) PostInstallControlPlane(initConfiguration *kubeadmapi.InitConfigu // "If you add a new member to a 1-node cluster, the cluster cannot make progress before the new member starts // because it needs two members as majority to agree on the consensus. You will only see this behavior between the time // etcdctl member add informs the cluster about the new member and the new member successfully establishing a connection to the existing one." - glog.V(1).Info("[join] adding etcd") + klog.V(1).Info("[join] adding etcd") if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), initConfiguration); err != nil { return errors.Wrap(err, "error creating local etcd static pod manifest file") } } - glog.V(1).Info("[join] uploading currently used configuration to the cluster") + klog.V(1).Info("[join] uploading currently used configuration to the cluster") if err := uploadconfigphase.UploadConfiguration(initConfiguration, client); err != nil { return errors.Wrap(err, "error uploading configuration") } - glog.V(1).Info("[join] marking the master with right label") - if err = markmasterphase.MarkMaster(client, initConfiguration.NodeRegistration.Name, initConfiguration.NodeRegistration.Taints); err != nil { - return errors.Wrap(err, "error applying master label and taints") + klog.V(1).Info("[join] marking the control-plane with right label") + if err = markcontrolplanephase.MarkControlPlane(client, initConfiguration.NodeRegistration.Name, initConfiguration.NodeRegistration.Taints); err != nil { + return errors.Wrap(err, "error applying control-plane label and taints") } return nil @@ -624,3 +584,53 @@ func waitForTLSBootstrappedClient() error { return (err == nil), nil }) } + +// fetchInitConfigurationFromJoinConfiguration retrieves the init configuration from a join configuration, performing the discovery +func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfiguration) (*kubeadmapi.InitConfiguration, *clientcmdapi.Config, error) { + // Perform the Discovery, which turns a Bootstrap Token and optionally (and preferably) a CA cert hash into a KubeConfig + // file that may be used for the TLS Bootstrapping process the kubelet performs using the Certificates API. + klog.V(1).Infoln("[join] Discovering cluster-info") + tlsBootstrapCfg, err := discovery.For(cfg) + if err != nil { + return nil, nil, err + } + + // Retrieves the kubeadm configuration + klog.V(1).Infoln("[join] Retrieving KubeConfig objects") + initConfiguration, err := fetchInitConfiguration(tlsBootstrapCfg) + if err != nil { + return nil, nil, err + } + + // Create the final KubeConfig file with the cluster name discovered after fetching the cluster configuration + clusterinfo := kubeconfigutil.GetClusterFromKubeConfig(tlsBootstrapCfg) + tlsBootstrapCfg.Clusters = map[string]*clientcmdapi.Cluster{ + initConfiguration.ClusterName: clusterinfo, + } + tlsBootstrapCfg.Contexts[tlsBootstrapCfg.CurrentContext].Cluster = initConfiguration.ClusterName + + // injects into the kubeadm configuration the information about the joining node + initConfiguration.NodeRegistration = cfg.NodeRegistration + if cfg.ControlPlane != nil { + initConfiguration.LocalAPIEndpoint = cfg.ControlPlane.LocalAPIEndpoint + } + + return initConfiguration, tlsBootstrapCfg, nil +} + +// fetchInitConfiguration reads the cluster configuration from the kubeadm-admin configMap +func fetchInitConfiguration(tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) { + // creates a client to access the cluster using the bootstrap token identity + tlsClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg) + if err != nil { + return nil, errors.Wrap(err, "unable to access the cluster") + } + + // Fetches the init configuration + initConfiguration, err := configutil.FetchConfigFromFileOrCluster(tlsClient, os.Stdout, "join", "", true) + if err != nil { + return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") + } + + return initConfiguration, nil +} diff --git a/cmd/kubeadm/app/cmd/join_test.go b/cmd/kubeadm/app/cmd/join_test.go index 4a83ec5149f..1ea7ba9462d 100644 --- a/cmd/kubeadm/app/cmd/join_test.go +++ b/cmd/kubeadm/app/cmd/join_test.go @@ -73,7 +73,6 @@ func TestNewValidJoin(t *testing.T) { skipPreFlight bool cfgPath string configToWrite string - featureGatesString string ignorePreflightErrors []string testJoinValidate bool testJoinRun bool @@ -109,11 +108,6 @@ func TestNewValidJoin(t *testing.T) { ignorePreflightErrors: []string{"some-unsupported-preflight-arg"}, expectedError: true, }, - { - name: "invalid: incorrect featureGatesString", - featureGatesString: "bad-feature-gate-string", - expectedError: true, - }, { name: "invalid: fail Join.Validate() with wrong flags", skipPreFlight: true, @@ -162,7 +156,7 @@ func TestNewValidJoin(t *testing.T) { } } - join, err := NewValidJoin(cmd.PersistentFlags(), cfg, tc.cfgPath, tc.featureGatesString, tc.ignorePreflightErrors) + join, err := NewValidJoin(cmd.PersistentFlags(), cfg, tc.cfgPath, tc.ignorePreflightErrors) if tc.nodeConfig != nil { join.cfg = tc.nodeConfig diff --git a/cmd/kubeadm/app/cmd/options/BUILD b/cmd/kubeadm/app/cmd/options/BUILD index 0f69807e35d..8f8248b32cc 100644 --- a/cmd/kubeadm/app/cmd/options/BUILD +++ b/cmd/kubeadm/app/cmd/options/BUILD @@ -4,6 +4,8 @@ go_library( name = "go_default_library", srcs = [ "certs.go", + "constant.go", + "doc.go", "generic.go", "token.go", ], @@ -12,6 +14,7 @@ go_library( deps = [ "//cmd/kubeadm/app/apis/kubeadm/v1beta1:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", ], diff --git a/cmd/kubeadm/app/cmd/options/certs.go b/cmd/kubeadm/app/cmd/options/certs.go index 54adbf7fb9f..9d826dc8448 100644 --- a/cmd/kubeadm/app/cmd/options/certs.go +++ b/cmd/kubeadm/app/cmd/options/certs.go @@ -20,5 +20,15 @@ import "github.com/spf13/pflag" // AddCertificateDirFlag adds the --certs-dir flag to the given flagset func AddCertificateDirFlag(fs *pflag.FlagSet, certsDir *string) { - fs.StringVar(certsDir, "cert-dir", *certsDir, "The path where to save the certificates") + fs.StringVar(certsDir, CertificatesDir, *certsDir, "The path where to save the certificates") +} + +// AddCSRFlag adds the --csr-only flag to the given flagset +func AddCSRFlag(fs *pflag.FlagSet, csr *bool) { + fs.BoolVar(csr, CSROnly, *csr, "Create CSRs instead of generating certificates") +} + +// AddCSRDirFlag adds the --csr-dir flag to the given flagset +func AddCSRDirFlag(fs *pflag.FlagSet, csrDir *string) { + fs.StringVar(csrDir, CSRDir, *csrDir, "The path to output the CSRs and private keys to") } diff --git a/cmd/kubeadm/app/cmd/options/constant.go b/cmd/kubeadm/app/cmd/options/constant.go new file mode 100644 index 00000000000..acb51ecd554 --- /dev/null +++ b/cmd/kubeadm/app/cmd/options/constant.go @@ -0,0 +1,86 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +// APIServerAdvertiseAddress flag sets the IP address the API Server will advertise it's listening on. Specify '0.0.0.0' to use the address of the default network interface. +const APIServerAdvertiseAddress = "apiserver-advertise-address" + +// APIServerBindPort flag sets the port for the API Server to bind to. +const APIServerBindPort = "apiserver-bind-port" + +// APIServerCertSANs flag sets extra Subject Alternative Names (SANs) to use for the API Server serving certificate. Can be both IP addresses and DNS names. +const APIServerCertSANs = "apiserver-cert-extra-sans" + +// APIServerExtraArgs flag sets a extra flags to pass to the API Server or override default ones in form of =. +const APIServerExtraArgs = "apiserver-extra-args" + +// CertificatesDir flag sets the path where to save and read the certificates. +const CertificatesDir = "cert-dir" + +// CfgPath flag sets the path to kubeadm config file. WARNING: Usage of a configuration file is experimental. +const CfgPath = "config" + +// ControllerManagerExtraArgs flag sets extra flags to pass to the Controller Manager or override default ones in form of =. +const ControllerManagerExtraArgs = "controller-manager-extra-args" + +// DryRun flag instruct kubeadm to don't apply any changes; just output what would be done. +const DryRun = "dry-run" + +// FeatureGatesString flag sets key=value pairs that describe feature gates for various features. +const FeatureGatesString = "feature-gates" + +// IgnorePreflightErrors sets the path a list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks. +const IgnorePreflightErrors = "ignore-preflight-errors" + +// ImageRepository sets the container registry to pull control plane images from. +const ImageRepository = "image-repository" + +// KubeconfigDir flag sets the path where to save the kubeconfig file. +const KubeconfigDir = "kubeconfig-dir" + +// KubeconfigPath flag sets the kubeconfig file to use when talking to the cluster. If the flag is not set, a set of standard locations are searched for an existing KubeConfig file. +const KubeconfigPath = "kubeconfig" + +// KubernetesVersion flag sets the Kubernetes version for the control plane. +const KubernetesVersion = "kubernetes-version" + +// NetworkingDNSDomain flag sets the domain for services, e.g. "myorg.internal". +const NetworkingDNSDomain = "service-dns-domain" + +// NetworkingServiceSubnet flag sets the range of IP address for service VIPs. +const NetworkingServiceSubnet = "service-cidr" + +// NetworkingPodSubnet flag sets the range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node. +const NetworkingPodSubnet = "pod-network-cidr" + +// NodeCRISocket flag sets the CRI socket to connect to. +const NodeCRISocket = "cri-socket" + +// NodeName flag sets the node name. +const NodeName = "node-name" + +// SchedulerExtraArgs flag sets extra flags to pass to the Scheduler or override default ones in form of =". +const SchedulerExtraArgs = "scheduler-extra-args" + +// SkipTokenPrint flag instruct kubeadm to skip printing of the default bootstrap token generated by 'kubeadm init'. +const SkipTokenPrint = "skip-token-print" + +// CSROnly flag instructs kubeadm to create CSRs instead of automatically creating or renewing certs +const CSROnly = "csr-only" + +// CSRDir flag sets the location for CSRs and flags to be output +const CSRDir = "csr-dir" diff --git a/cmd/kubeadm/app/cmd/options/doc.go b/cmd/kubeadm/app/cmd/options/doc.go new file mode 100644 index 00000000000..8e04d08917e --- /dev/null +++ b/cmd/kubeadm/app/cmd/options/doc.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package options provide a central point for defining flags for kubeadm cobra commands, +no matter if hard coded commands or autogenerated command for phases. + +New kubeadm flags should always be defined in this package as a constant before their usage, +in order to enforce naming consistency across different commands and to control flag proliferation. + +In addition to defining the flags, the package also contains set of utilities for flag management. + +For additional details about how flags are managed in phases, please refer to the +"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" package. +*/ +package options diff --git a/cmd/kubeadm/app/cmd/options/generic.go b/cmd/kubeadm/app/cmd/options/generic.go index 532cd0d9080..3a08838b099 100644 --- a/cmd/kubeadm/app/cmd/options/generic.go +++ b/cmd/kubeadm/app/cmd/options/generic.go @@ -16,22 +16,42 @@ limitations under the License. package options -import "github.com/spf13/pflag" +import ( + "github.com/spf13/pflag" + utilflag "k8s.io/apiserver/pkg/util/flag" +) // AddKubeConfigFlag adds the --kubeconfig flag to the given flagset func AddKubeConfigFlag(fs *pflag.FlagSet, kubeConfigFile *string) { - fs.StringVar(kubeConfigFile, "kubeconfig", *kubeConfigFile, "The kubeconfig file to use when talking to the cluster. If the flag is not set, a set of standard locations are searched for an existing KubeConfig file.") + fs.StringVar(kubeConfigFile, KubeconfigPath, *kubeConfigFile, "The kubeconfig file to use when talking to the cluster. If the flag is not set, a set of standard locations are searched for an existing KubeConfig file.") +} + +// AddKubeConfigDirFlag adds the --kubeconfig-dir flag to the given flagset +func AddKubeConfigDirFlag(fs *pflag.FlagSet, kubeConfigDir *string) { + fs.StringVar(kubeConfigDir, KubeconfigDir, *kubeConfigDir, "The path where to save the kubeconfig file.") } // AddConfigFlag adds the --config flag to the given flagset func AddConfigFlag(fs *pflag.FlagSet, cfgPath *string) { - fs.StringVar(cfgPath, "config", *cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") + fs.StringVar(cfgPath, CfgPath, *cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental).") } // AddIgnorePreflightErrorsFlag adds the --ignore-preflight-errors flag to the given flagset func AddIgnorePreflightErrorsFlag(fs *pflag.FlagSet, ignorePreflightErrors *[]string) { fs.StringSliceVar( - ignorePreflightErrors, "ignore-preflight-errors", *ignorePreflightErrors, + ignorePreflightErrors, IgnorePreflightErrors, *ignorePreflightErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", ) } + +// AddControlPlanExtraArgsFlags adds the ExtraArgs flags for control plane components +func AddControlPlanExtraArgsFlags(fs *pflag.FlagSet, apiServerExtraArgs, controllerManagerExtraArgs, schedulerExtraArgs *map[string]string) { + fs.Var(utilflag.NewMapStringString(apiServerExtraArgs), APIServerExtraArgs, "A set of extra flags to pass to the API Server or override default ones in form of =") + fs.Var(utilflag.NewMapStringString(controllerManagerExtraArgs), ControllerManagerExtraArgs, "A set of extra flags to pass to the Controller Manager or override default ones in form of =") + fs.Var(utilflag.NewMapStringString(schedulerExtraArgs), SchedulerExtraArgs, "A set of extra flags to pass to the Scheduler or override default ones in form of =") +} + +// AddImageMetaFlags adds the --image-repository flag to the given flagset +func AddImageMetaFlags(fs *pflag.FlagSet, imageRepository *string) { + fs.StringVar(imageRepository, ImageRepository, *imageRepository, "Choose a container registry to pull control plane images from") +} diff --git a/cmd/kubeadm/app/cmd/phases/BUILD b/cmd/kubeadm/app/cmd/phases/BUILD index d2199ceeaee..932f6b732f7 100644 --- a/cmd/kubeadm/app/cmd/phases/BUILD +++ b/cmd/kubeadm/app/cmd/phases/BUILD @@ -10,9 +10,8 @@ go_library( "etcd.go", "kubeconfig.go", "kubelet.go", - "markmaster.go", + "markcontrolplane.go", "preflight.go", - "selfhosting.go", "uploadconfig.go", "util.go", "waitcontrolplane.go", @@ -28,7 +27,6 @@ go_library( "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library", "//cmd/kubeadm/app/cmd/util:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/phases/addons/dns:go_default_library", "//cmd/kubeadm/app/phases/addons/proxy:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library", @@ -38,28 +36,24 @@ go_library( "//cmd/kubeadm/app/phases/etcd:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", "//cmd/kubeadm/app/phases/kubelet:go_default_library", - "//cmd/kubeadm/app/phases/markmaster:go_default_library", + "//cmd/kubeadm/app/phases/markcontrolplane:go_default_library", "//cmd/kubeadm/app/phases/patchnode:go_default_library", - "//cmd/kubeadm/app/phases/selfhosting:go_default_library", "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/audit:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", "//cmd/kubeadm/app/util/dryrun:go_default_library", - "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//pkg/util/normalizer:go_default_library", "//pkg/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", "//vendor/github.com/renstrom/dedent:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) @@ -67,13 +61,17 @@ go_library( go_test( name = "go_default_test", srcs = [ - "addons_test.go", + "certs_test.go", "util_test.go", ], embed = [":go_default_library"], deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/v1beta1:go_default_library", - "//cmd/kubeadm/test/cmd:go_default_library", + "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library", + "//cmd/kubeadm/app/phases/certs:go_default_library", + "//cmd/kubeadm/app/util/pkiutil:go_default_library", + "//cmd/kubeadm/test:go_default_library", "//pkg/version:go_default_library", ], ) diff --git a/cmd/kubeadm/app/cmd/phases/addons.go b/cmd/kubeadm/app/cmd/phases/addons.go index 8b22ec026cd..298ac80c406 100644 --- a/cmd/kubeadm/app/cmd/phases/addons.go +++ b/cmd/kubeadm/app/cmd/phases/addons.go @@ -1,5 +1,5 @@ /* -Copyright 2017 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,186 +17,116 @@ limitations under the License. package phases import ( - "strings" - - "github.com/golang/glog" - "github.com/spf13/cobra" + "github.com/pkg/errors" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" dnsaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" proxyaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/pkg/util/normalizer" ) var ( - allAddonsLongDesc = normalizer.LongDesc(` - Installs the CoreDNS and the kube-proxy addons components via the API server. - Please note that although the DNS server is deployed, it will not be scheduled until CNI is installed. - ` + cmdutil.AlphaDisclaimer) - - allAddonsExample = normalizer.Examples(` - # Installs the CoreDNS and the kube-proxy addons components via the API server, - # functionally equivalent to what installed by kubeadm init. - - kubeadm alpha phase selfhosting from-staticpods - `) - - corednsAddonsLongDesc = normalizer.LongDesc(` + coreDNSAddonLongDesc = normalizer.LongDesc(` Installs the CoreDNS addon components via the API server. Please note that although the DNS server is deployed, it will not be scheduled until CNI is installed. - ` + cmdutil.AlphaDisclaimer) + `) - kubeproxyAddonsLongDesc = normalizer.LongDesc(` + kubeProxyAddonLongDesc = normalizer.LongDesc(` Installs the kube-proxy addon components via the API server. - ` + cmdutil.AlphaDisclaimer) + `) ) -// NewCmdAddon returns the addon Cobra command -func NewCmdAddon() *cobra.Command { - cmd := &cobra.Command{ - Use: "addon", - Aliases: []string{"addons"}, - Short: "Installs required addons for passing Conformance tests", - Long: cmdutil.MacroCommandLongDescription, - } - - cmd.AddCommand(getAddonsSubCommands()...) - return cmd +type addonData interface { + Cfg() *kubeadmapi.InitConfiguration + Client() (clientset.Interface, error) } -// EnsureAllAddons installs all addons to a Kubernetes cluster -func EnsureAllAddons(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error { - - addonActions := []func(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error{ - dnsaddon.EnsureDNSAddon, - proxyaddon.EnsureProxyAddon, - } - - glog.V(1).Infoln("[addon] installing all addons") - for _, action := range addonActions { - err := action(cfg, client) - if err != nil { - return err - } - } - - return nil -} - -// getAddonsSubCommands returns sub commands for addons phase -func getAddonsSubCommands() []*cobra.Command { - cfg := &kubeadmapiv1beta1.InitConfiguration{} - // Default values for the cobra help text - kubeadmscheme.Scheme.Default(cfg) - - var cfgPath, featureGatesString string - var subCmds []*cobra.Command - kubeConfigFile := kubeadmconstants.GetAdminKubeConfigPath() - - subCmdProperties := []struct { - use string - short string - long string - examples string - cmdFunc func(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error - }{ - { - use: "all", - short: "Installs all addons to a Kubernetes cluster", - long: allAddonsLongDesc, - examples: allAddonsExample, - cmdFunc: EnsureAllAddons, - }, - { - use: "coredns", - short: "Installs the CoreDNS addon to a Kubernetes cluster", - long: corednsAddonsLongDesc, - cmdFunc: dnsaddon.EnsureDNSAddon, - }, - { - use: "kube-proxy", - short: "Installs the kube-proxy addon to a Kubernetes cluster", - long: kubeproxyAddonsLongDesc, - cmdFunc: proxyaddon.EnsureProxyAddon, +// NewAddonPhase returns the addon Cobra command +func NewAddonPhase() workflow.Phase { + return workflow.Phase{ + Name: "addon", + Short: "Installs required addons for passing Conformance tests", + Long: cmdutil.MacroCommandLongDescription, + Phases: []workflow.Phase{ + { + Name: "all", + Short: "Installs all the addons", + InheritFlags: getAddonPhaseFlags("all"), + RunAllSiblings: true, + }, + { + Name: "coredns", + Short: "Installs the CoreDNS addon to a Kubernetes cluster", + Long: coreDNSAddonLongDesc, + InheritFlags: getAddonPhaseFlags("coredns"), + Run: runCoreDNSAddon, + }, + { + Name: "kube-proxy", + Short: "Installs the kube-proxy addon to a Kubernetes cluster", + Long: kubeProxyAddonLongDesc, + InheritFlags: getAddonPhaseFlags("kube-proxy"), + Run: runKubeProxyAddon, + }, }, } - - for _, properties := range subCmdProperties { - // Creates the UX Command - cmd := &cobra.Command{ - Use: properties.use, - Short: properties.short, - Long: properties.long, - Example: properties.examples, - Run: runAddonsCmdFunc(properties.cmdFunc, cfg, kubeConfigFile, &cfgPath, &featureGatesString), - } - - // Add flags to the command - options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile) - cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to a kubeadm config file. WARNING: Usage of a configuration file is experimental") - cmd.Flags().StringVar(&cfg.KubernetesVersion, "kubernetes-version", cfg.KubernetesVersion, `Choose a specific Kubernetes version for the control plane`) - cmd.Flags().StringVar(&cfg.ImageRepository, "image-repository", cfg.ImageRepository, `Choose a container registry to pull control plane images from`) - - if properties.use == "all" || properties.use == "kube-proxy" { - cmd.Flags().StringVar(&cfg.APIEndpoint.AdvertiseAddress, "apiserver-advertise-address", cfg.APIEndpoint.AdvertiseAddress, `The IP address the API server is accessible on`) - cmd.Flags().Int32Var(&cfg.APIEndpoint.BindPort, "apiserver-bind-port", cfg.APIEndpoint.BindPort, `The port the API server is accessible on`) - cmd.Flags().StringVar(&cfg.Networking.PodSubnet, "pod-network-cidr", cfg.Networking.PodSubnet, `The range of IP addresses used for the Pod network`) - } - - if properties.use == "all" || properties.use == "coredns" { - cmd.Flags().StringVar(&cfg.Networking.DNSDomain, "service-dns-domain", cfg.Networking.DNSDomain, `Alternative domain for services`) - cmd.Flags().StringVar(&cfg.Networking.ServiceSubnet, "service-cidr", cfg.Networking.ServiceSubnet, `The range of IP address used for service VIPs`) - cmd.Flags().StringVar(&featureGatesString, "feature-gates", featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ - "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) - } - subCmds = append(subCmds, cmd) - } - - return subCmds } -// runAddonsCmdFunc creates a cobra.Command Run function, by composing the call to the given cmdFunc with necessary additional steps (e.g preparation of input parameters) -func runAddonsCmdFunc(cmdFunc func(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error, cfg *kubeadmapiv1beta1.InitConfiguration, kubeConfigFile string, cfgPath *string, featureGatesString *string) func(cmd *cobra.Command, args []string) { - - // the following statement build a clousure that wraps a call to a cmdFunc, binding - // the function itself with the specific parameters of each sub command. - // Please note that specific parameter should be passed as value, while other parameters - passed as reference - - // are shared between sub commands and gets access to current value e.g. flags value. - - return func(cmd *cobra.Command, args []string) { - var err error - if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil { - kubeadmutil.CheckErr(err) - } - - if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, *featureGatesString); err != nil { - kubeadmutil.CheckErr(err) - } - - internalcfg := &kubeadmapi.InitConfiguration{} - kubeadmscheme.Scheme.Convert(cfg, internalcfg, nil) - kubeConfigFile = cmdutil.FindExistingKubeConfig(kubeConfigFile) - client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) - kubeadmutil.CheckErr(err) - internalcfg, err = configutil.ConfigFileAndDefaultsToInternalConfig(*cfgPath, cfg) - kubeadmutil.CheckErr(err) - if err := features.ValidateVersion(features.InitFeatureGates, internalcfg.FeatureGates, internalcfg.KubernetesVersion); err != nil { - kubeadmutil.CheckErr(err) - } - - // Execute the cmdFunc - err = cmdFunc(internalcfg, client) - kubeadmutil.CheckErr(err) +func getAddonData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, error) { + data, ok := c.(addonData) + if !ok { + return nil, nil, errors.New("addon phase invoked with an invalid data struct") } + cfg := data.Cfg() + client, err := data.Client() + if err != nil { + return nil, nil, err + } + return cfg, client, err +} + +// runCoreDNSAddon installs CoreDNS addon to a Kubernetes cluster +func runCoreDNSAddon(c workflow.RunData) error { + cfg, client, err := getAddonData(c) + if err != nil { + return err + } + return dnsaddon.EnsureDNSAddon(cfg, client) +} + +// runKubeProxyAddon installs KubeProxy addon to a Kubernetes cluster +func runKubeProxyAddon(c workflow.RunData) error { + cfg, client, err := getAddonData(c) + if err != nil { + return err + } + return proxyaddon.EnsureProxyAddon(cfg, client) +} + +func getAddonPhaseFlags(name string) []string { + flags := []string{ + options.CfgPath, + options.KubeconfigPath, + options.KubernetesVersion, + options.ImageRepository, + } + if name == "all" || name == "kube-proxy" { + flags = append(flags, + options.APIServerAdvertiseAddress, + options.APIServerBindPort, + options.NetworkingPodSubnet, + ) + } + if name == "all" || name == "coredns" { + flags = append(flags, + options.FeatureGatesString, + options.NetworkingDNSDomain, + options.NetworkingServiceSubnet, + ) + } + return flags } diff --git a/cmd/kubeadm/app/cmd/phases/addons_test.go b/cmd/kubeadm/app/cmd/phases/addons_test.go deleted file mode 100644 index 713e7f8dbda..00000000000 --- a/cmd/kubeadm/app/cmd/phases/addons_test.go +++ /dev/null @@ -1,71 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "testing" - - cmdtestutil "k8s.io/kubernetes/cmd/kubeadm/test/cmd" -) - -func TestAddonsSubCommandsHasFlags(t *testing.T) { - - subCmds := getAddonsSubCommands() - - commonFlags := []string{ - "kubeconfig", - "config", - "kubernetes-version", - "image-repository", - } - - var tests = []struct { - command string - additionalFlags []string - }{ - { - command: "all", - additionalFlags: []string{ - "apiserver-advertise-address", - "apiserver-bind-port", - "pod-network-cidr", - "service-dns-domain", - "service-cidr", - }, - }, - { - command: "kube-proxy", - additionalFlags: []string{ - "apiserver-advertise-address", - "apiserver-bind-port", - "pod-network-cidr", - }, - }, - { - command: "coredns", - additionalFlags: []string{ - "service-dns-domain", - "service-cidr", - }, - }, - } - - for _, test := range tests { - expectedFlags := append(commonFlags, test.additionalFlags...) - cmdtestutil.AssertSubCommandHasFlags(t, subCmds, test.command, expectedFlags...) - } -} diff --git a/cmd/kubeadm/app/cmd/phases/bootstraptoken.go b/cmd/kubeadm/app/cmd/phases/bootstraptoken.go index dd4617ec9cb..b6492960f8b 100644 --- a/cmd/kubeadm/app/cmd/phases/bootstraptoken.go +++ b/cmd/kubeadm/app/cmd/phases/bootstraptoken.go @@ -1,5 +1,5 @@ /* -Copyright 2017 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,316 +18,107 @@ package phases import ( "fmt" + "path/filepath" - "github.com/golang/glog" "github.com/pkg/errors" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" + nodebootstraptokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" "k8s.io/kubernetes/pkg/util/normalizer" ) var ( - allTokenLongDesc = normalizer.LongDesc(` + bootstrapTokenLongDesc = normalizer.LongDesc(` Bootstrap tokens are used for establishing bidirectional trust between a node joining - the cluster and a the master node. + the cluster and a the control-plane node. This command makes all the configurations required to make bootstrap tokens works and then creates an initial token. - ` + cmdutil.AlphaDisclaimer) - - allTokenExamples = normalizer.Examples(` - # Makes all the bootstrap token configurations and creates an initial token, functionally - # equivalent to what generated by kubeadm init. - kubeadm alpha phase bootstrap-token all `) - createTokenLongDesc = normalizer.LongDesc(` - Creates a bootstrap token. If no token value is given, kubeadm will generate a random token instead. - - Alternatively, you can use kubeadm token. - ` + cmdutil.AlphaDisclaimer) - - clusterInfoLongDesc = fmt.Sprintf(normalizer.LongDesc(` - Uploads the %q ConfigMap in the %q namespace, populating it with cluster information extracted from the - given kubeconfig file. The ConfigMap is used for the node bootstrap process in its initial phases, - before the client trusts the API server. - - See online documentation about Authenticating with Bootstrap Tokens for more details. - `+cmdutil.AlphaDisclaimer), bootstrapapi.ConfigMapClusterInfo, metav1.NamespacePublic) - - nodePostCSRsLongDesc = normalizer.LongDesc(` - Configures RBAC rules to allow node bootstrap tokens to post a certificate signing request, - thus enabling nodes joining the cluster to request long term certificate credentials. - - See online documentation about TLS bootstrapping for more details. - ` + cmdutil.AlphaDisclaimer) - - nodeAutoApproveLongDesc = normalizer.LongDesc(` - Configures RBAC rules to allow the csrapprover controller to automatically approve - certificate signing requests generated by nodes joining the cluster. - It configures also RBAC rules for certificates rotation (with auto approval of new certificates). - - See online documentation about TLS bootstrapping for more details. - ` + cmdutil.AlphaDisclaimer) + bootstrapTokenExamples = normalizer.Examples(` + # Makes all the bootstrap token configurations and creates an initial token, functionally + # equivalent to what generated by kubeadm init. + kubeadm init phase bootstrap-token + `) ) -// NewCmdBootstrapToken returns the Cobra command for running the mark-master phase -func NewCmdBootstrapToken() *cobra.Command { - kubeConfigFile := kubeadmconstants.GetAdminKubeConfigPath() +type bootstrapTokenData interface { + Cfg() *kubeadmapi.InitConfiguration + Client() (clientset.Interface, error) + KubeConfigDir() string + SkipTokenPrint() bool + Tokens() []string +} - cmd := &cobra.Command{ - Use: "bootstrap-token", - Short: "Manage kubeadm-specific bootstrap token functions", - Long: cmdutil.MacroCommandLongDescription, +// NewBootstrapTokenPhase returns the phase to boostrapToken +func NewBootstrapTokenPhase() workflow.Phase { + return workflow.Phase{ + Name: "bootstrap-token", Aliases: []string{"bootstraptoken"}, - } - - options.AddKubeConfigFlag(cmd.PersistentFlags(), &kubeConfigFile) - - // Add subcommands - kubeConfigFile = cmdutil.FindExistingKubeConfig(kubeConfigFile) - cmd.AddCommand(NewSubCmdBootstrapTokenAll(&kubeConfigFile)) - cmd.AddCommand(NewSubCmdBootstrapToken(&kubeConfigFile)) - cmd.AddCommand(NewSubCmdClusterInfo(&kubeConfigFile)) - cmd.AddCommand(NewSubCmdNodeBootstrapToken(&kubeConfigFile)) - - return cmd -} - -// NewSubCmdBootstrapTokenAll returns the Cobra command for running the token all sub-phase -func NewSubCmdBootstrapTokenAll(kubeConfigFile *string) *cobra.Command { - cfg := &kubeadmapiv1beta1.InitConfiguration{} - - // Default values for the cobra help text - kubeadmscheme.Scheme.Default(cfg) - - var cfgPath string - var skipTokenPrint bool - bto := options.NewBootstrapTokenOptions() - - cmd := &cobra.Command{ - Use: "all", - Short: "Makes all the bootstrap token configurations and creates an initial token", - Long: allTokenLongDesc, - Example: allTokenExamples, - Run: func(cmd *cobra.Command, args []string) { - err := validation.ValidateMixedArguments(cmd.Flags()) - kubeadmutil.CheckErr(err) - - err = bto.ApplyTo(cfg) - kubeadmutil.CheckErr(err) - - client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) - kubeadmutil.CheckErr(err) - - // Creates the bootstap token - err = createBootstrapToken(*kubeConfigFile, client, cfgPath, cfg, skipTokenPrint) - kubeadmutil.CheckErr(err) - - // Create the cluster-info ConfigMap or update if it already exists - err = clusterinfo.CreateBootstrapConfigMapIfNotExists(client, *kubeConfigFile) - kubeadmutil.CheckErr(err) - - // Create the RBAC rules that expose the cluster-info ConfigMap properly - err = clusterinfo.CreateClusterInfoRBACRules(client) - kubeadmutil.CheckErr(err) - - // Create RBAC rules that makes the bootstrap tokens able to post CSRs - err = node.AllowBootstrapTokensToPostCSRs(client) - kubeadmutil.CheckErr(err) - - // Create RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically - err = node.AutoApproveNodeBootstrapTokens(client) - kubeadmutil.CheckErr(err) - - // Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically - err = node.AutoApproveNodeCertificateRotation(client) - kubeadmutil.CheckErr(err) + Short: "Generates bootstrap tokens used to join a node to a cluster", + Example: bootstrapTokenExamples, + Long: bootstrapTokenLongDesc, + InheritFlags: []string{ + options.CfgPath, + options.KubeconfigDir, + options.SkipTokenPrint, }, + Run: runBoostrapToken, + } +} + +func runBoostrapToken(c workflow.RunData) error { + data, ok := c.(bootstrapTokenData) + if !ok { + return errors.New("bootstrap-token phase invoked with an invalid data struct") } - // Adds flags to the command - addGenericFlags(cmd.Flags(), &cfgPath, &skipTokenPrint) - bto.AddTokenFlag(cmd.Flags()) - bto.AddTTLFlag(cmd.Flags()) - bto.AddUsagesFlag(cmd.Flags()) - bto.AddGroupsFlag(cmd.Flags()) - bto.AddDescriptionFlag(cmd.Flags()) - - return cmd -} - -// NewSubCmdBootstrapToken returns the Cobra command for running the create token phase -func NewSubCmdBootstrapToken(kubeConfigFile *string) *cobra.Command { - cfg := &kubeadmapiv1beta1.InitConfiguration{} - - // Default values for the cobra help text - kubeadmscheme.Scheme.Default(cfg) - - var cfgPath string - var skipTokenPrint bool - bto := options.NewBootstrapTokenOptions() - - cmd := &cobra.Command{ - Use: "create", - Short: "Creates a bootstrap token to be used for node joining", - Long: createTokenLongDesc, - Run: func(cmd *cobra.Command, args []string) { - err := validation.ValidateMixedArguments(cmd.Flags()) - kubeadmutil.CheckErr(err) - - err = bto.ApplyTo(cfg) - kubeadmutil.CheckErr(err) - - client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) - kubeadmutil.CheckErr(err) - - err = createBootstrapToken(*kubeConfigFile, client, cfgPath, cfg, skipTokenPrint) - kubeadmutil.CheckErr(err) - }, - } - - // Adds flags to the command - addGenericFlags(cmd.Flags(), &cfgPath, &skipTokenPrint) - bto.AddTokenFlag(cmd.Flags()) - bto.AddTTLFlag(cmd.Flags()) - bto.AddUsagesFlag(cmd.Flags()) - bto.AddGroupsFlag(cmd.Flags()) - bto.AddDescriptionFlag(cmd.Flags()) - - return cmd -} - -// NewSubCmdClusterInfo returns the Cobra command for running the cluster-info sub-phase -func NewSubCmdClusterInfo(kubeConfigFile *string) *cobra.Command { - cmd := &cobra.Command{ - Use: "cluster-info", - Short: "Uploads the cluster-info ConfigMap from the given kubeconfig file", - Long: clusterInfoLongDesc, - Aliases: []string{"clusterinfo"}, - Run: func(cmd *cobra.Command, args []string) { - client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) - kubeadmutil.CheckErr(err) - - // Create the cluster-info ConfigMap or update if it already exists - err = clusterinfo.CreateBootstrapConfigMapIfNotExists(client, *kubeConfigFile) - kubeadmutil.CheckErr(err) - - // Create the RBAC rules that expose the cluster-info ConfigMap properly - err = clusterinfo.CreateClusterInfoRBACRules(client) - kubeadmutil.CheckErr(err) - }, - } - return cmd -} - -// NewSubCmdNodeBootstrapToken returns the Cobra command for running the node sub-phase -func NewSubCmdNodeBootstrapToken(kubeConfigFile *string) *cobra.Command { - cmd := &cobra.Command{ - Use: "node", - Short: "Configures the node bootstrap process", - Aliases: []string{"clusterinfo"}, - Long: cmdutil.MacroCommandLongDescription, - } - - cmd.AddCommand(NewSubCmdNodeBootstrapTokenPostCSRs(kubeConfigFile)) - cmd.AddCommand(NewSubCmdNodeBootstrapTokenAutoApprove(kubeConfigFile)) - - return cmd -} - -// NewSubCmdNodeBootstrapTokenPostCSRs returns the Cobra command for running the allow-post-csrs sub-phase -func NewSubCmdNodeBootstrapTokenPostCSRs(kubeConfigFile *string) *cobra.Command { - cmd := &cobra.Command{ - Use: "allow-post-csrs", - Short: "Configures RBAC to allow node bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials", - Long: nodePostCSRsLongDesc, - Run: func(cmd *cobra.Command, args []string) { - client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) - kubeadmutil.CheckErr(err) - - // Create RBAC rules that makes the bootstrap tokens able to post CSRs - err = node.AllowBootstrapTokensToPostCSRs(client) - kubeadmutil.CheckErr(err) - }, - } - return cmd -} - -// NewSubCmdNodeBootstrapTokenAutoApprove returns the Cobra command for running the allow-auto-approve sub-phase -func NewSubCmdNodeBootstrapTokenAutoApprove(kubeConfigFile *string) *cobra.Command { - cmd := &cobra.Command{ - Use: "allow-auto-approve", - Short: "Configures RBAC rules to allow the csrapprover controller automatically approve CSRs from a node bootstrap token", - Long: nodeAutoApproveLongDesc, - Run: func(cmd *cobra.Command, args []string) { - client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) - kubeadmutil.CheckErr(err) - - // Create RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically - err = node.AutoApproveNodeBootstrapTokens(client) - kubeadmutil.CheckErr(err) - - // Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically - err = node.AutoApproveNodeCertificateRotation(client) - kubeadmutil.CheckErr(err) - }, - } - return cmd -} - -func addGenericFlags(flagSet *pflag.FlagSet, cfgPath *string, skipTokenPrint *bool) { - flagSet.StringVar( - cfgPath, "config", *cfgPath, - "Path to kubeadm config file. WARNING: Usage of a configuration file is experimental", - ) - flagSet.BoolVar( - skipTokenPrint, "skip-token-print", *skipTokenPrint, - "Skip printing of the bootstrap token", - ) -} - -func createBootstrapToken(kubeConfigFile string, client clientset.Interface, cfgPath string, cfg *kubeadmapiv1beta1.InitConfiguration, skipTokenPrint bool) error { - // KubernetesVersion is not used, but we set it explicitly to avoid the lookup - // of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig - SetKubernetesVersion(cfg) - - // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags - internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) + client, err := data.Client() if err != nil { return err } - glog.V(1).Infoln("[bootstraptoken] creating/updating token") - // Creates or updates the token - if err := node.UpdateOrCreateTokens(client, false, internalcfg.BootstrapTokens); err != nil { + if !data.SkipTokenPrint() { + tokens := data.Tokens() + if len(tokens) == 1 { + fmt.Printf("[bootstrap-token] Using token: %s\n", tokens[0]) + } else if len(tokens) > 1 { + fmt.Printf("[bootstrap-token] Using tokens: %v\n", tokens) + } + } + + fmt.Println("[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles") + // Create the default node bootstrap token + if err := nodebootstraptokenphase.UpdateOrCreateTokens(client, false, data.Cfg().BootstrapTokens); err != nil { + return errors.Wrap(err, "error updating or creating token") + } + // Create RBAC rules that makes the bootstrap tokens able to post CSRs + if err := nodebootstraptokenphase.AllowBootstrapTokensToPostCSRs(client); err != nil { + return errors.Wrap(err, "error allowing bootstrap tokens to post CSRs") + } + // Create RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically + if err := nodebootstraptokenphase.AutoApproveNodeBootstrapTokens(client); err != nil { + return errors.Wrap(err, "error auto-approving node bootstrap tokens") + } + + // Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically + if err := nodebootstraptokenphase.AutoApproveNodeCertificateRotation(client); err != nil { return err } - fmt.Println("[bootstraptoken] bootstrap token created") - fmt.Println("[bootstraptoken] you can now join any number of machines by running:") - - if len(internalcfg.BootstrapTokens) > 0 { - joinCommand, err := cmdutil.GetJoinCommand(kubeConfigFile, internalcfg.BootstrapTokens[0].Token.String(), skipTokenPrint) - if err != nil { - return errors.Wrap(err, "failed to get join command") - } - fmt.Println(joinCommand) + // Create the cluster-info ConfigMap with the associated RBAC rules + adminKubeConfigPath := filepath.Join(data.KubeConfigDir(), kubeadmconstants.AdminKubeConfigFileName) + if err := clusterinfophase.CreateBootstrapConfigMapIfNotExists(client, adminKubeConfigPath); err != nil { + return errors.Wrap(err, "error creating bootstrap ConfigMap") + } + if err := clusterinfophase.CreateClusterInfoRBACRules(client); err != nil { + return errors.Wrap(err, "error creating clusterinfo RBAC rules") } return nil } diff --git a/cmd/kubeadm/app/cmd/phases/certs.go b/cmd/kubeadm/app/cmd/phases/certs.go index 20582e11137..778cd6c3689 100644 --- a/cmd/kubeadm/app/cmd/phases/certs.go +++ b/cmd/kubeadm/app/cmd/phases/certs.go @@ -21,9 +21,11 @@ import ( "strings" "github.com/pkg/errors" + "github.com/spf13/pflag" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" @@ -46,6 +48,11 @@ var ( ` + cmdutil.AlphaDisclaimer) ) +var ( + csrOnly bool + csrDir string +) + // certsData defines the behavior that a runtime data struct passed to the certs phase should // have. Please note that we are using an interface in order to make this phase reusable in different workflows // (and thus with different runtime data struct, all of them requested to be compliant to this interface) @@ -63,13 +70,31 @@ func NewCertsPhase() workflow.Phase { Short: "Certificate generation", Phases: newCertSubPhases(), Run: runCerts, + Long: cmdutil.MacroCommandLongDescription, } } +func localFlags() *pflag.FlagSet { + set := pflag.NewFlagSet("csr", pflag.ExitOnError) + options.AddCSRFlag(set, &csrOnly) + options.AddCSRDirFlag(set, &csrDir) + return set +} + // newCertSubPhases returns sub phases for certs phase func newCertSubPhases() []workflow.Phase { subPhases := []workflow.Phase{} + // All subphase + allPhase := workflow.Phase{ + Name: "all", + Short: "Generates all certificates", + InheritFlags: getCertPhaseFlags("all"), + RunAllSiblings: true, + } + + subPhases = append(subPhases, allPhase) + certTree, _ := certsphase.GetDefaultCertList().AsMap().CertTree() for ca, certList := range certTree { @@ -78,6 +103,7 @@ func newCertSubPhases() []workflow.Phase { for _, cert := range certList { certPhase := newCertSubPhase(cert, runCertPhase(cert, ca)) + certPhase.LocalFlags = localFlags() subPhases = append(subPhases, certPhase) } } @@ -105,15 +131,34 @@ func newCertSubPhase(certSpec *certsphase.KubeadmCert, run func(c workflow.RunDa certSpec.BaseName, getSANDescription(certSpec), ), - Run: run, + Run: run, + InheritFlags: getCertPhaseFlags(certSpec.Name), } return phase } +func getCertPhaseFlags(name string) []string { + flags := []string{ + options.CertificatesDir, + options.CfgPath, + options.CSROnly, + options.CSRDir, + } + if name == "all" || name == "apiserver" { + flags = append(flags, + options.APIServerAdvertiseAddress, + options.APIServerCertSANs, + options.NetworkingDNSDomain, + options.NetworkingServiceSubnet, + ) + } + return flags +} + func getSANDescription(certSpec *certsphase.KubeadmCert) string { //Defaulted config we will use to get SAN certs defaultConfig := &kubeadmapiv1beta1.InitConfiguration{ - APIEndpoint: kubeadmapiv1beta1.APIEndpoint{ + LocalAPIEndpoint: kubeadmapiv1beta1.APIEndpoint{ // GetAPIServerAltNames errors without an AdvertiseAddress; this is as good as any. AdvertiseAddress: "127.0.0.1", }, @@ -121,7 +166,8 @@ func getSANDescription(certSpec *certsphase.KubeadmCert) string { defaultInternalConfig := &kubeadmapi.InitConfiguration{} kubeadmscheme.Scheme.Default(defaultConfig) - kubeadmscheme.Scheme.Convert(defaultConfig, defaultInternalConfig, nil) + err := kubeadmscheme.Scheme.Convert(defaultConfig, defaultInternalConfig, nil) + kubeadmutil.CheckErr(err) certConfig, err := certSpec.GetConfig(defaultInternalConfig) kubeadmutil.CheckErr(err) @@ -217,6 +263,15 @@ func runCertPhase(cert *certsphase.KubeadmCert, caCert *certsphase.KubeadmCert) return nil } + if csrOnly { + fmt.Printf("[certs] Generating CSR for %s instead of certificate\n", cert.BaseName) + if csrDir == "" { + csrDir = data.CertificateWriteDir() + } + + return certsphase.CreateCSR(cert, data.Cfg(), csrDir) + } + // if using external etcd, skips etcd certificates generation if data.Cfg().Etcd.External != nil && cert.CAName == "etcd-ca" { fmt.Printf("[certs] External etcd mode: Skipping %s certificate authority generation\n", cert.BaseName) diff --git a/cmd/kubeadm/app/cmd/phases/certs_test.go b/cmd/kubeadm/app/cmd/phases/certs_test.go new file mode 100644 index 00000000000..06fc61ce63f --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/certs_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "os" + "testing" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" + testutil "k8s.io/kubernetes/cmd/kubeadm/test" +) + +type testCertsData struct { + cfg *kubeadmapi.InitConfiguration +} + +func (t *testCertsData) Cfg() *kubeadmapi.InitConfiguration { return t.cfg } +func (t *testCertsData) ExternalCA() bool { return false } +func (t *testCertsData) CertificateDir() string { return t.cfg.CertificatesDir } +func (t *testCertsData) CertificateWriteDir() string { return t.cfg.CertificatesDir } + +func TestCertsWithCSRs(t *testing.T) { + csrDir := testutil.SetupTempDir(t) + defer os.RemoveAll(csrDir) + certDir := testutil.SetupTempDir(t) + defer os.RemoveAll(certDir) + cert := &certs.KubeadmCertAPIServer + + certsData := &testCertsData{ + cfg: testutil.GetDefaultInternalConfig(t), + } + certsData.cfg.CertificatesDir = certDir + + // global vars + csrOnly = true + csrDir = certDir + + phase := NewCertsPhase() + // find the api cert phase + var apiServerPhase *workflow.Phase + for _, phase := range phase.Phases { + if phase.Name == cert.Name { + apiServerPhase = &phase + break + } + } + + if apiServerPhase == nil { + t.Fatalf("couldn't find apiserver phase") + } + + err := apiServerPhase.Run(certsData) + if err != nil { + t.Fatalf("couldn't run API server phase: %v", err) + } + + if _, _, err := pkiutil.TryLoadCSRAndKeyFromDisk(csrDir, cert.BaseName); err != nil { + t.Fatalf("couldn't load certificate %q: %v", cert.BaseName, err) + } +} diff --git a/cmd/kubeadm/app/cmd/phases/controlplane.go b/cmd/kubeadm/app/cmd/phases/controlplane.go index cfab182694a..6a191b06044 100644 --- a/cmd/kubeadm/app/cmd/phases/controlplane.go +++ b/cmd/kubeadm/app/cmd/phases/controlplane.go @@ -19,15 +19,13 @@ package phases import ( "errors" "fmt" - "os" - "path/filepath" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" - auditutil "k8s.io/kubernetes/cmd/kubeadm/app/util/audit" "k8s.io/kubernetes/pkg/util/normalizer" ) @@ -73,10 +71,16 @@ func getPhaseDescription(component string) string { // NewControlPlanePhase creates a kubeadm workflow phase that implements bootstrapping the control plane. func NewControlPlanePhase() workflow.Phase { phase := workflow.Phase{ - Name: "control-plane", - Short: "Generates all static Pod manifest files necessary to establish the control plane", - Example: controlPlaneExample, + Name: "control-plane", + Short: "Generates all static Pod manifest files necessary to establish the control plane", + Long: cmdutil.MacroCommandLongDescription, Phases: []workflow.Phase{ + { + Name: "all", + Short: "Generates all static Pod manifest files", + InheritFlags: getControlPlanePhaseFlags("all"), + RunAllSiblings: true, + }, newControlPlaneSubPhase(kubeadmconstants.KubeAPIServer), newControlPlaneSubPhase(kubeadmconstants.KubeControllerManager), newControlPlaneSubPhase(kubeadmconstants.KubeScheduler), @@ -88,13 +92,44 @@ func NewControlPlanePhase() workflow.Phase { func newControlPlaneSubPhase(component string) workflow.Phase { phase := workflow.Phase{ - Name: controlPlanePhaseProperties[component].name, - Short: controlPlanePhaseProperties[component].short, - Run: runControlPlaneSubPhase(component), + Name: controlPlanePhaseProperties[component].name, + Short: controlPlanePhaseProperties[component].short, + Run: runControlPlaneSubPhase(component), + InheritFlags: getControlPlanePhaseFlags(component), } return phase } +func getControlPlanePhaseFlags(name string) []string { + flags := []string{ + options.CfgPath, + options.CertificatesDir, + options.KubernetesVersion, + options.ImageRepository, + } + if name == "all" || name == kubeadmconstants.KubeAPIServer { + flags = append(flags, + options.APIServerAdvertiseAddress, + options.APIServerBindPort, + options.APIServerExtraArgs, + options.FeatureGatesString, + options.NetworkingServiceSubnet, + ) + } + if name == "all" || name == kubeadmconstants.KubeControllerManager { + flags = append(flags, + options.ControllerManagerExtraArgs, + options.NetworkingPodSubnet, + ) + } + if name == "all" || name == kubeadmconstants.KubeScheduler { + flags = append(flags, + options.SchedulerExtraArgs, + ) + } + return flags +} + func runControlPlanePhase(c workflow.RunData) error { data, ok := c.(controlPlaneData) if !ok { @@ -113,22 +148,6 @@ func runControlPlaneSubPhase(component string) func(c workflow.RunData) error { } cfg := data.Cfg() - // special case to handle audit policy for the API server - if component == kubeadmconstants.KubeAPIServer && features.Enabled(cfg.FeatureGates, features.Auditing) { - // Setup the AuditPolicy (either it was passed in and exists or it wasn't passed in and generate a default policy) - if cfg.AuditPolicyConfiguration.Path != "" { - // TODO(chuckha) ensure passed in audit policy is valid so users don't have to find the error in the api server log. - if _, err := os.Stat(cfg.AuditPolicyConfiguration.Path); err != nil { - return fmt.Errorf("error getting file info for audit policy file %q [%v]", cfg.AuditPolicyConfiguration.Path, err) - } - } else { - cfg.AuditPolicyConfiguration.Path = filepath.Join(data.KubeConfigDir(), kubeadmconstants.AuditPolicyDir, kubeadmconstants.AuditPolicyFile) - if err := auditutil.CreateDefaultAuditLogPolicy(cfg.AuditPolicyConfiguration.Path); err != nil { - return fmt.Errorf("error creating default audit policy %q [%v]", cfg.AuditPolicyConfiguration.Path, err) - } - } - } - fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component) if err := controlplane.CreateStaticPodFiles(data.ManifestDir(), cfg, component); err != nil { return err diff --git a/cmd/kubeadm/app/cmd/phases/etcd.go b/cmd/kubeadm/app/cmd/phases/etcd.go index bd6a2f33c36..1bae0067241 100644 --- a/cmd/kubeadm/app/cmd/phases/etcd.go +++ b/cmd/kubeadm/app/cmd/phases/etcd.go @@ -19,10 +19,12 @@ package phases import ( "fmt" - "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/klog" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" "k8s.io/kubernetes/pkg/util/normalizer" ) @@ -47,9 +49,9 @@ type etcdData interface { // NewEtcdPhase creates a kubeadm workflow phase that implements handling of etcd. func NewEtcdPhase() workflow.Phase { phase := workflow.Phase{ - Name: "etcd", - Short: "Generates static Pod manifest file for local etcd.", - Example: etcdLocalExample, + Name: "etcd", + Short: "Generates static Pod manifest file for local etcd.", + Long: cmdutil.MacroCommandLongDescription, Phases: []workflow.Phase{ newEtcdLocalSubPhase(), }, @@ -59,14 +61,24 @@ func NewEtcdPhase() workflow.Phase { func newEtcdLocalSubPhase() workflow.Phase { phase := workflow.Phase{ - Name: "local", - Short: "Generates the static Pod manifest file for a local, single-node local etcd instance.", - Example: etcdLocalExample, - Run: runEtcdPhaseLocal(), + Name: "local", + Short: "Generates the static Pod manifest file for a local, single-node local etcd instance.", + Example: etcdLocalExample, + Run: runEtcdPhaseLocal(), + InheritFlags: getEtcdPhaseFlags(), } return phase } +func getEtcdPhaseFlags() []string { + flags := []string{ + options.CertificatesDir, + options.CfgPath, + options.ImageRepository, + } + return flags +} + func runEtcdPhaseLocal() func(c workflow.RunData) error { return func(c workflow.RunData) error { data, ok := c.(etcdData) @@ -82,7 +94,7 @@ func runEtcdPhaseLocal() func(c workflow.RunData) error { return errors.Wrap(err, "error creating local etcd static pod manifest file") } } else { - glog.V(1).Infof("[etcd] External etcd mode. Skipping the creation of a manifest for local etcd") + klog.V(1).Infof("[etcd] External etcd mode. Skipping the creation of a manifest for local etcd") } return nil } diff --git a/cmd/kubeadm/app/cmd/phases/kubeconfig.go b/cmd/kubeadm/app/cmd/phases/kubeconfig.go index a905d3967f8..64a3698ce58 100644 --- a/cmd/kubeadm/app/cmd/phases/kubeconfig.go +++ b/cmd/kubeadm/app/cmd/phases/kubeconfig.go @@ -1,5 +1,5 @@ /* -Copyright 2017 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,7 +21,9 @@ import ( "github.com/pkg/errors" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" "k8s.io/kubernetes/pkg/util/normalizer" @@ -76,7 +78,14 @@ func NewKubeConfigPhase() workflow.Phase { return workflow.Phase{ Name: "kubeconfig", Short: "Generates all kubeconfig files necessary to establish the control plane and the admin kubeconfig file", + Long: cmdutil.MacroCommandLongDescription, Phases: []workflow.Phase{ + { + Name: "all", + Short: "Generates all kubeconfig files", + InheritFlags: getKubeConfigPhaseFlags("all"), + RunAllSiblings: true, + }, NewKubeConfigFilePhase(kubeadmconstants.AdminKubeConfigFileName), NewKubeConfigFilePhase(kubeadmconstants.KubeletKubeConfigFileName), NewKubeConfigFilePhase(kubeadmconstants.ControllerManagerKubeConfigFileName), @@ -89,13 +98,30 @@ func NewKubeConfigPhase() workflow.Phase { // NewKubeConfigFilePhase creates a kubeadm workflow phase that creates a kubeconfig file. func NewKubeConfigFilePhase(kubeConfigFileName string) workflow.Phase { return workflow.Phase{ - Name: kubeconfigFilePhaseProperties[kubeConfigFileName].name, - Short: kubeconfigFilePhaseProperties[kubeConfigFileName].short, - Long: fmt.Sprintf(kubeconfigFilePhaseProperties[kubeConfigFileName].long, kubeConfigFileName), - Run: runKubeConfigFile(kubeConfigFileName), + Name: kubeconfigFilePhaseProperties[kubeConfigFileName].name, + Short: kubeconfigFilePhaseProperties[kubeConfigFileName].short, + Long: fmt.Sprintf(kubeconfigFilePhaseProperties[kubeConfigFileName].long, kubeConfigFileName), + Run: runKubeConfigFile(kubeConfigFileName), + InheritFlags: getKubeConfigPhaseFlags(kubeConfigFileName), } } +func getKubeConfigPhaseFlags(name string) []string { + flags := []string{ + options.APIServerAdvertiseAddress, + options.APIServerBindPort, + options.CertificatesDir, + options.CfgPath, + options.KubeconfigDir, + } + if name == "all" || name == kubeadmconstants.KubeletKubeConfigFileName { + flags = append(flags, + options.NodeName, + ) + } + return flags +} + func runKubeConfig(c workflow.RunData) error { data, ok := c.(kubeConfigData) if !ok { @@ -116,7 +142,6 @@ func runKubeConfigFile(kubeConfigFileName string) func(workflow.RunData) error { // if external CA mode, skip certificate authority generation if data.ExternalCA() { - //TODO: implement validation of existing kubeconfig files fmt.Printf("[kubeconfig] External CA mode: Using user provided %s\n", kubeConfigFileName) return nil } diff --git a/cmd/kubeadm/app/cmd/phases/kubelet.go b/cmd/kubeadm/app/cmd/phases/kubelet.go index 18b8486b916..2d37fda4a03 100644 --- a/cmd/kubeadm/app/cmd/phases/kubelet.go +++ b/cmd/kubeadm/app/cmd/phases/kubelet.go @@ -17,20 +17,12 @@ limitations under the License. package phases import ( - "github.com/golang/glog" "github.com/pkg/errors" - "github.com/spf13/cobra" + "k8s.io/klog" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" - patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/pkg/util/normalizer" ) @@ -39,24 +31,6 @@ var ( # Writes a dynamic environment file with kubelet flags from a InitConfiguration file. kubeadm init phase kubelet-start --config masterconfig.yaml `) - - kubeletConfigUploadLongDesc = normalizer.LongDesc(` - Uploads kubelet configuration extracted from the kubeadm InitConfiguration object to a ConfigMap - of the form kubelet-config-1.X in the cluster, where X is the minor version of the current (API Server) Kubernetes version. - ` + cmdutil.AlphaDisclaimer) - - kubeletConfigUploadExample = normalizer.Examples(` - # Uploads the kubelet configuration from the kubeadm Config file to a ConfigMap in the cluster. - kubeadm alpha phase kubelet config upload --config kubeadm.yaml - `) - - kubeletConfigAnnotateCRILongDesc = normalizer.LongDesc(` - Adds an annotation to the current node with the CRI socket specified in the kubeadm InitConfiguration object. - ` + cmdutil.AlphaDisclaimer) - - kubeletConfigAnnotateCRIExample = normalizer.Examples(` - kubeadm alpha phase kubelet config annotate-cri --config kubeadm.yaml - `) ) // kubeletStartData defines the behavior that a runtime data struct passed to the kubelet start phase @@ -76,6 +50,11 @@ func NewKubeletStartPhase() workflow.Phase { Long: "Writes a file with KubeletConfiguration and an environment file with node specific kubelet settings, and then (re)starts kubelet.", Example: kubeletStartPhaseExample, Run: runKubeletStart, + InheritFlags: []string{ + options.CfgPath, + options.NodeCRISocket, + options.NodeName, + }, } } @@ -89,14 +68,14 @@ func runKubeletStart(c workflow.RunData) error { // First off, configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet // Try to stop the kubelet service so no race conditions occur when configuring it if !data.DryRun() { - glog.V(1).Infof("Stopping the kubelet") + klog.V(1).Infof("Stopping the kubelet") kubeletphase.TryStopKubelet() } // Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master, // as we handle that ourselves in the markmaster phase // TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase? - if err := kubeletphase.WriteKubeletDynamicEnvFile(&data.Cfg().NodeRegistration, data.Cfg().FeatureGates, false, data.KubeletDir()); err != nil { + if err := kubeletphase.WriteKubeletDynamicEnvFile(data.Cfg(), false, data.KubeletDir()); err != nil { return errors.Wrap(err, "error writing a dynamic environment file for the kubelet") } @@ -107,110 +86,9 @@ func runKubeletStart(c workflow.RunData) error { // Try to start the kubelet service in case it's inactive if !data.DryRun() { - glog.V(1).Infof("Starting the kubelet") + klog.V(1).Infof("Starting the kubelet") kubeletphase.TryStartKubelet() } return nil } - -// NewCmdKubelet returns command for `kubeadm phase kubelet` -func NewCmdKubelet() *cobra.Command { - cmd := &cobra.Command{ - Use: "kubelet", - Short: "Commands related to handling the kubelet.", - Long: cmdutil.MacroCommandLongDescription, - } - - cmd.AddCommand(NewCmdKubeletConfig()) - return cmd -} - -// NewCmdKubeletConfig returns command for `kubeadm phase kubelet config` -func NewCmdKubeletConfig() *cobra.Command { - cmd := &cobra.Command{ - Use: "config", - Short: "Handles kubelet configuration.", - Long: cmdutil.MacroCommandLongDescription, - } - - cmd.AddCommand(NewCmdKubeletConfigUpload()) - cmd.AddCommand(NewCmdKubeletAnnotateCRI()) - return cmd -} - -// NewCmdKubeletConfigUpload calls cobra.Command for uploading dynamic kubelet configuration -func NewCmdKubeletConfigUpload() *cobra.Command { - cfg := &kubeadmapiv1beta1.InitConfiguration{} - var cfgPath string - kubeConfigFile := constants.GetAdminKubeConfigPath() - - cmd := &cobra.Command{ - Use: "upload", - Short: "Uploads kubelet configuration to a ConfigMap based on a kubeadm InitConfiguration file.", - Long: kubeletConfigUploadLongDesc, - Example: kubeletConfigUploadExample, - Run: func(cmd *cobra.Command, args []string) { - if len(cfgPath) == 0 { - kubeadmutil.CheckErr(errors.New("The --config argument is required")) - } - - // KubernetesVersion is not used, but we set it explicitly to avoid the lookup - // of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig - SetKubernetesVersion(cfg) - - // This call returns the ready-to-use configuration based on the configuration file - internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) - kubeadmutil.CheckErr(err) - - kubeConfigFile = cmdutil.FindExistingKubeConfig(kubeConfigFile) - client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) - kubeadmutil.CheckErr(err) - - err = kubeletphase.CreateConfigMap(internalcfg, client) - kubeadmutil.CheckErr(err) - }, - } - - options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile) - options.AddConfigFlag(cmd.Flags(), &cfgPath) - return cmd -} - -// NewCmdKubeletAnnotateCRI calls cobra.Command for annotating the node with the given crisocket -func NewCmdKubeletAnnotateCRI() *cobra.Command { - cfg := &kubeadmapiv1beta1.InitConfiguration{} - var cfgPath string - kubeConfigFile := constants.GetAdminKubeConfigPath() - - cmd := &cobra.Command{ - Use: "annotate-cri", - Short: "annotates the node with the given crisocket", - Long: kubeletConfigAnnotateCRILongDesc, - Example: kubeletConfigAnnotateCRIExample, - Run: func(cmd *cobra.Command, args []string) { - if len(cfgPath) == 0 { - kubeadmutil.CheckErr(errors.New("The --config argument is required")) - } - - // KubernetesVersion is not used, but we set it explicitly to avoid the lookup - // of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig - SetKubernetesVersion(cfg) - - // This call returns the ready-to-use configuration based on the configuration file - internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) - kubeadmutil.CheckErr(err) - - kubeConfigFile = cmdutil.FindExistingKubeConfig(kubeConfigFile) - client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) - kubeadmutil.CheckErr(err) - - err = patchnodephase.AnnotateCRISocket(client, internalcfg.NodeRegistration.Name, internalcfg.NodeRegistration.CRISocket) - kubeadmutil.CheckErr(err) - }, - } - - options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile) - options.AddConfigFlag(cmd.Flags(), &cfgPath) - return cmd -} diff --git a/cmd/kubeadm/app/cmd/phases/markcontrolplane.go b/cmd/kubeadm/app/cmd/phases/markcontrolplane.go new file mode 100644 index 00000000000..eac8a1ed4f9 --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/markcontrolplane.go @@ -0,0 +1,76 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "github.com/pkg/errors" + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane" + "k8s.io/kubernetes/pkg/util/normalizer" +) + +var ( + markControlPlaneExample = normalizer.Examples(` + # Applies control-plane label and taint to the current node, functionally equivalent to what executed by kubeadm init. + kubeadm init phase mark-control-plane --config config.yml + + # Applies control-plane label and taint to a specific node + kubeadm init phase mark-control-plane --node-name myNode + `) +) + +type markControlPlaneData interface { + Cfg() *kubeadmapi.InitConfiguration + Client() (clientset.Interface, error) + DryRun() bool +} + +// NewMarkControlPlanePhase creates a kubeadm workflow phase that implements mark-controlplane checks. +func NewMarkControlPlanePhase() workflow.Phase { + return workflow.Phase{ + Name: "mark-control-plane", + Short: "Mark a node as a control-plane", + Example: markControlPlaneExample, + InheritFlags: []string{ + options.NodeName, + }, + Run: runMarkControlPlane, + } +} + +// runMarkControlPlane executes markcontrolplane checks logic. +func runMarkControlPlane(c workflow.RunData) error { + data, ok := c.(markControlPlaneData) + if !ok { + return errors.New("mark-control-plane phase invoked with an invalid data struct") + } + + client, err := data.Client() + if err != nil { + return err + } + + nodeRegistration := data.Cfg().NodeRegistration + if err := markcontrolplanephase.MarkControlPlane(client, nodeRegistration.Name, nodeRegistration.Taints); err != nil { + return err + } + + return nil +} diff --git a/cmd/kubeadm/app/cmd/phases/markmaster.go b/cmd/kubeadm/app/cmd/phases/markmaster.go deleted file mode 100644 index e13c557701d..00000000000 --- a/cmd/kubeadm/app/cmd/phases/markmaster.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "github.com/spf13/cobra" - - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - markmasterphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - "k8s.io/kubernetes/pkg/util/normalizer" -) - -var ( - markMasterLongDesc = normalizer.LongDesc(` - Applies a label that specifies that a node is a master and a taint that forces workloads to be deployed accordingly. - ` + cmdutil.AlphaDisclaimer) - - markMasterExample = normalizer.Examples(` - # Applies master label and taint to the current node, functionally equivalent to what executed by kubeadm init. - kubeadm alpha phase mark-master - - # Applies master label and taint to a specific node - kubeadm alpha phase mark-master --node-name myNode - `) -) - -// NewCmdMarkMaster returns the Cobra command for running the mark-master phase -func NewCmdMarkMaster() *cobra.Command { - cfg := &kubeadmapiv1beta1.InitConfiguration{} - - // Default values for the cobra help text - kubeadmscheme.Scheme.Default(cfg) - - var cfgPath string - kubeConfigFile := kubeadmconstants.GetAdminKubeConfigPath() - - cmd := &cobra.Command{ - Use: "mark-master", - Short: "Mark a node as master", - Long: markMasterLongDesc, - Example: markMasterExample, - Aliases: []string{"markmaster"}, - Run: func(cmd *cobra.Command, args []string) { - if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil { - kubeadmutil.CheckErr(err) - } - - kubeConfigFile = cmdutil.FindExistingKubeConfig(kubeConfigFile) - client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) - kubeadmutil.CheckErr(err) - - // KubernetesVersion is not used, but we set it explicitly to avoid the lookup - // of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig - SetKubernetesVersion(cfg) - - // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags - internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) - kubeadmutil.CheckErr(err) - - err = markmasterphase.MarkMaster(client, internalcfg.NodeRegistration.Name, internalcfg.NodeRegistration.Taints) - kubeadmutil.CheckErr(err) - }, - } - - options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile) - cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file. WARNING: Usage of a configuration file is experimental") - cmd.Flags().StringVar(&cfg.NodeRegistration.Name, "node-name", cfg.NodeRegistration.Name, `The node name to which label and taints should apply`) - - return cmd -} diff --git a/cmd/kubeadm/app/cmd/phases/preflight.go b/cmd/kubeadm/app/cmd/phases/preflight.go index de9fca2569f..8884a9fe864 100644 --- a/cmd/kubeadm/app/cmd/phases/preflight.go +++ b/cmd/kubeadm/app/cmd/phases/preflight.go @@ -22,6 +22,7 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/sets" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" "k8s.io/kubernetes/pkg/util/normalizer" @@ -52,6 +53,10 @@ func NewPreflightMasterPhase() workflow.Phase { Long: "Run master pre-flight checks, functionally equivalent to what implemented by kubeadm init.", Example: masterPreflightExample, Run: runPreflightMaster, + InheritFlags: []string{ + options.CfgPath, + options.IgnorePreflightErrors, + }, } } diff --git a/cmd/kubeadm/app/cmd/phases/uploadconfig.go b/cmd/kubeadm/app/cmd/phases/uploadconfig.go index 5eaa8c0cd73..c11844ad81a 100644 --- a/cmd/kubeadm/app/cmd/phases/uploadconfig.go +++ b/cmd/kubeadm/app/cmd/phases/uploadconfig.go @@ -20,69 +20,131 @@ import ( "fmt" "github.com/pkg/errors" - "github.com/spf13/cobra" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" + patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/pkg/util/normalizer" ) var ( - uploadConfigLongDesc = fmt.Sprintf(normalizer.LongDesc(` - Uploads the kubeadm init configuration of your cluster to a ConfigMap called %s in the %s namespace. + uploadKubeadmConfigLongDesc = fmt.Sprintf(normalizer.LongDesc(` + Uploads the kubeadm ClusterConfiguration to a ConfigMap called %s in the %s namespace. This enables correct configuration of system components and a seamless user experience when upgrading. Alternatively, you can use kubeadm config. - `+cmdutil.AlphaDisclaimer), kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem) + `), kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem) - uploadConfigExample = normalizer.Examples(` + uploadKubeadmConfigExample = normalizer.Examples(` # uploads the configuration of your cluster - kubeadm alpha phase upload-config --config=myConfig.yaml + kubeadm init phase upload-config --config=myConfig.yaml + `) + + uploadKubeletConfigLongDesc = normalizer.LongDesc(` + Uploads kubelet configuration extracted from the kubeadm InitConfiguration object to a ConfigMap + of the form kubelet-config-1.X in the cluster, where X is the minor version of the current (API Server) Kubernetes version. + `) + + uploadKubeletConfigExample = normalizer.Examples(` + # Uploads the kubelet configuration from the kubeadm Config file to a ConfigMap in the cluster. + kubeadm init phase upload-config kubelet --config kubeadm.yaml `) ) -// NewCmdUploadConfig returns the Cobra command for running the uploadconfig phase -func NewCmdUploadConfig() *cobra.Command { - cfg := &kubeadmapiv1beta1.InitConfiguration{} - kubeConfigFile := kubeadmconstants.GetAdminKubeConfigPath() - var cfgPath string +type uploadConfigData interface { + Cfg() *kubeadmapi.InitConfiguration + Client() (clientset.Interface, error) +} - cmd := &cobra.Command{ - Use: "upload-config", - Short: "Uploads the currently used configuration for kubeadm to a ConfigMap", - Long: uploadConfigLongDesc, - Example: uploadConfigExample, +// NewUploadConfigPhase returns the phase to uploadConfig +func NewUploadConfigPhase() workflow.Phase { + return workflow.Phase{ + Name: "upload-config", Aliases: []string{"uploadconfig"}, - Run: func(_ *cobra.Command, args []string) { - if len(cfgPath) == 0 { - kubeadmutil.CheckErr(errors.New("the --config flag is mandatory")) - } - - kubeConfigFile = cmdutil.FindExistingKubeConfig(kubeConfigFile) - client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) - kubeadmutil.CheckErr(err) - - // KubernetesVersion is not used, but we set it explicitly to avoid the lookup - // of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig - SetKubernetesVersion(cfg) - - internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) - kubeadmutil.CheckErr(err) - - err = uploadconfig.UploadConfiguration(internalcfg, client) - kubeadmutil.CheckErr(err) + Short: "Uploads the kubeadm and kubelet configuration to a ConfigMap", + Long: cmdutil.MacroCommandLongDescription, + Phases: []workflow.Phase{ + { + Name: "all", + Short: "Uploads all configuration to a config map", + RunAllSiblings: true, + }, + { + Name: "kubeadm", + Short: "Uploads the kubeadm ClusterConfiguration to a ConfigMap", + Long: uploadKubeadmConfigLongDesc, + Example: uploadKubeadmConfigExample, + Run: runUploadKubeadmConfig, + InheritFlags: getUploadConfigPhaseFlags(), + }, + { + Name: "kubelet", + Short: "Uploads the kubelet component config to a ConfigMap", + Long: uploadKubeletConfigLongDesc, + Example: uploadKubeletConfigExample, + Run: runUploadKubeletConfig, + InheritFlags: getUploadConfigPhaseFlags(), + }, }, } - - options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile) - cmd.Flags().StringVar(&cfgPath, "config", "", "Path to a kubeadm config file. WARNING: Usage of a configuration file is experimental") - - return cmd +} + +func getUploadConfigPhaseFlags() []string { + return []string{ + options.CfgPath, + options.KubeconfigPath, + } +} + +// runUploadKubeadmConfig uploads the kubeadm configuration to a ConfigMap +func runUploadKubeadmConfig(c workflow.RunData) error { + cfg, client, err := getUploadConfigData(c) + if err != nil { + return err + } + + klog.V(1).Infof("[upload-config] Uploading the kubeadm ClusterConfiguration to a ConfigMap") + if err := uploadconfig.UploadConfiguration(cfg, client); err != nil { + return errors.Wrap(err, "error uploading the kubeadm ClusterConfiguration") + } + return nil +} + +// runUploadKubeletConfig uploads the kubelet configuration to a ConfigMap +func runUploadKubeletConfig(c workflow.RunData) error { + cfg, client, err := getUploadConfigData(c) + if err != nil { + return err + } + + klog.V(1).Infof("[upload-config] Uploading the kubelet component config to a ConfigMap") + if err = kubeletphase.CreateConfigMap(cfg, client); err != nil { + return errors.Wrap(err, "error creating kubelet configuration ConfigMap") + } + + klog.V(1).Infof("[upload-config] Preserving the CRISocket information for the control-plane node") + if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { + return errors.Wrap(err, "Error writing Crisocket information for the control-plane node") + } + return nil +} + +func getUploadConfigData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, error) { + data, ok := c.(uploadConfigData) + if !ok { + return nil, nil, errors.New("upload-config phase invoked with an invalid data struct") + } + cfg := data.Cfg() + client, err := data.Client() + if err != nil { + return nil, nil, err + } + return cfg, client, err } diff --git a/cmd/kubeadm/app/cmd/phases/waitcontrolplane.go b/cmd/kubeadm/app/cmd/phases/waitcontrolplane.go index c54cbf0a035..b2983e17d84 100644 --- a/cmd/kubeadm/app/cmd/phases/waitcontrolplane.go +++ b/cmd/kubeadm/app/cmd/phases/waitcontrolplane.go @@ -23,10 +23,10 @@ import ( "text/template" "time" - "github.com/golang/glog" "github.com/pkg/errors" "github.com/renstrom/dedent" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" @@ -86,19 +86,20 @@ func runWaitControlPlanePhase(c workflow.RunData) error { } // waiter holds the apiclient.Waiter implementation of choice, responsible for querying the API server in various ways and waiting for conditions to be fulfilled - glog.V(1).Infof("[wait-control-plane] Waiting for the API server to be healthy") + klog.V(1).Infof("[wait-control-plane] Waiting for the API server to be healthy") client, err := data.Client() if err != nil { return errors.Wrap(err, "cannot obtain client") } - waiter, err := NewControlPlaneWaiter(data.DryRun(), client, data.OutputWriter()) + timeout := data.Cfg().ClusterConfiguration.APIServer.TimeoutForControlPlane.Duration + waiter, err := newControlPlaneWaiter(data.DryRun(), timeout, client, data.OutputWriter()) if err != nil { return errors.Wrap(err, "error creating waiter") } - fmt.Printf("[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory %q\n", data.ManifestDir()) + fmt.Printf("[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory %q. This can take up to %v\n", data.ManifestDir(), timeout) if err := waiter.WaitForKubeletAndFunc(waiter.WaitForAPI); err != nil { ctx := map[string]string{ @@ -143,14 +144,10 @@ func printFilesIfDryRunning(data waitControlPlaneData) error { // NewControlPlaneWaiter returns a new waiter that is used to wait on the control plane to boot up. // TODO: make private (lowercase) after self-hosting phase is removed. -func NewControlPlaneWaiter(dryRun bool, client clientset.Interface, out io.Writer) (apiclient.Waiter, error) { +func newControlPlaneWaiter(dryRun bool, timeout time.Duration, client clientset.Interface, out io.Writer) (apiclient.Waiter, error) { if dryRun { return dryrunutil.NewWaiter(), nil } - // We know that the images should be cached locally already as we have pulled them using - // crictl in the preflight checks. Hence we can have a pretty short timeout for the kubelet - // to start creating Static Pods. - timeout := 4 * time.Minute return apiclient.NewKubeWaiter(client, timeout, out), nil } diff --git a/cmd/kubeadm/app/cmd/phases/workflow/BUILD b/cmd/kubeadm/app/cmd/phases/workflow/BUILD index 2b8cbfcb5e8..01571c97a71 100644 --- a/cmd/kubeadm/app/cmd/phases/workflow/BUILD +++ b/cmd/kubeadm/app/cmd/phases/workflow/BUILD @@ -23,6 +23,11 @@ go_test( "runner_test.go", ], embed = [":go_default_library"], + deps = [ + "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", + ], ) filegroup( diff --git a/cmd/kubeadm/app/cmd/phases/workflow/doc.go b/cmd/kubeadm/app/cmd/phases/workflow/doc.go index defc5da3ac8..1dd69e315e6 100644 --- a/cmd/kubeadm/app/cmd/phases/workflow/doc.go +++ b/cmd/kubeadm/app/cmd/phases/workflow/doc.go @@ -40,8 +40,21 @@ Each workflow can be defined and managed using a Runner, that will run all the phases according to the given order; nested phases will be executed immediately after their parent phase. -The Runner behavior can be changed by setting the RunnerOptions, typically -exposed as kubeadm command line flags, thus allowing to filter the list of phases -to be executed. +The phase runner can be bound to a cobra command; this operation sets the command description +giving evidence of the list of phases, and automatically creates sub commands +for invoking phases atomically. + +Autogenerated sub commands get flags according to the following rule: + +- global flags will be always inherited by autogenerated commands (this is managed by cobra) + +- local flags defined in the parent command might be inherited by autogenerated commands, +but this requires explicit opt-in so each phase can select the subset of relevant flags + +- it is possible to define additional flags that might be inherited by autogenerated commands +via explicit opt-in, but are not applied to the parent command + +In order to keep flags definition under control, please refer to the +"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" package. */ package workflow diff --git a/cmd/kubeadm/app/cmd/phases/workflow/doc_test.go b/cmd/kubeadm/app/cmd/phases/workflow/doc_test.go index b97abe61232..0211f42364d 100644 --- a/cmd/kubeadm/app/cmd/phases/workflow/doc_test.go +++ b/cmd/kubeadm/app/cmd/phases/workflow/doc_test.go @@ -19,6 +19,8 @@ package workflow import ( "errors" "fmt" + + "github.com/spf13/cobra" ) var myWorkflowRunner = NewRunner() @@ -100,7 +102,7 @@ func ExampleRunner_Run() { // Defines the method that creates the runtime data shared // among all the phases included in the workflow - myWorkflowRunner.SetDataInitializer(func() (RunData, error) { + myWorkflowRunner.SetDataInitializer(func(cmd *cobra.Command) (RunData, error) { return myWorkflowData{data: "some data"}, nil }) diff --git a/cmd/kubeadm/app/cmd/phases/workflow/phase.go b/cmd/kubeadm/app/cmd/phases/workflow/phase.go index be4bf56c661..42ac7f337e7 100644 --- a/cmd/kubeadm/app/cmd/phases/workflow/phase.go +++ b/cmd/kubeadm/app/cmd/phases/workflow/phase.go @@ -16,6 +16,8 @@ limitations under the License. package workflow +import "github.com/spf13/pflag" + // Phase provides an implementation of a workflow phase that allows // creation of new phases by simply instantiating a variable of this type. type Phase struct { @@ -24,6 +26,9 @@ type Phase struct { // the same workflow or phases belonging to the same parent phase). Name string + // Aliases returns the aliases for the phase. + Aliases []string + // Short description of the phase. Short string @@ -40,6 +45,11 @@ type Phase struct { // Phases defines a nested, ordered sequence of phases. Phases []Phase + // RunAllSiblings allows to assign to a phase the responsibility to + // run all the sibling phases + // Nb. phase marked as RunAllSiblings can not have Run functions + RunAllSiblings bool + // Run defines a function implementing the phase action. // It is recommended to implent type assertion, e.g. using golang type switch, // for validating the RunData type. @@ -49,6 +59,18 @@ type Phase struct { // before executing the phase action. // If this function return nil, the phase action is always executed. RunIf func(data RunData) (bool, error) + + // InheritFlags defines the list of flags that the cobra command generated for this phase should Inherit + // from local flags defined in the parent command / or additional flags defined in the phase runner. + // If the values is not set or empty, no flags will be assigned to the command + // Nb. global flags are automatically inherited by nested cobra command + InheritFlags []string + + // LocalFlags defines the list of flags that should be assigned to the cobra command generated + // for this phase. + // Nb. if two or phases have the same local flags, please consider using local flags in the parent command + // or additional flags defined in the phase runner. + LocalFlags *pflag.FlagSet } // AppendPhase adds the given phase to the nested, ordered sequence of phases. diff --git a/cmd/kubeadm/app/cmd/phases/workflow/runner.go b/cmd/kubeadm/app/cmd/phases/workflow/runner.go index e25088a87f1..e9fa49b5413 100644 --- a/cmd/kubeadm/app/cmd/phases/workflow/runner.go +++ b/cmd/kubeadm/app/cmd/phases/workflow/runner.go @@ -53,13 +53,21 @@ type Runner struct { // runDataInitializer defines a function that creates the runtime data shared // among all the phases included in the workflow - runDataInitializer func() (RunData, error) + runDataInitializer func(*cobra.Command) (RunData, error) // runData is part of the internal state of the runner and it is used for implementing // a singleton in the InitData methods (thus avoiding to initialize data // more than one time) runData RunData + // runCmd is part of the internal state of the runner and it is used to track the + // command that will trigger the runner (only if the runner is BindToCommand). + runCmd *cobra.Command + + // cmdAdditionalFlags holds additional, shared flags that could be added to the subcommands generated + // for phases. Flags could be inherited from the parent command too or added directly to each phase + cmdAdditionalFlags *pflag.FlagSet + // phaseRunners is part of the internal state of the runner and provides // a list of wrappers to phases composing the workflow with contextual // information supporting phase execution. @@ -162,7 +170,8 @@ func (e *Runner) computePhaseRunFlags() (map[string]bool, error) { // SetDataInitializer allows to setup a function that initialize the runtime data shared // among all the phases included in the workflow. -func (e *Runner) SetDataInitializer(builder func() (RunData, error)) { +// The method will receive in input the cmd that triggers the Runner (only if the runner is BindToCommand) +func (e *Runner) SetDataInitializer(builder func(cmd *cobra.Command) (RunData, error)) { e.runDataInitializer = builder } @@ -172,7 +181,7 @@ func (e *Runner) SetDataInitializer(builder func() (RunData, error)) { func (e *Runner) InitData() (RunData, error) { if e.runData == nil && e.runDataInitializer != nil { var err error - if e.runData, err = e.runDataInitializer(); err != nil { + if e.runData, err = e.runDataInitializer(e.runCmd); err != nil { return nil, err } } @@ -202,6 +211,12 @@ func (e *Runner) Run() error { return nil } + // Errors if phases that are meant to create special subcommands only + // are wrongly assigned Run Methods + if p.RunAllSiblings && (p.RunIf != nil || p.Run != nil) { + return errors.Wrapf(err, "phase marked as RunAllSiblings can not have Run functions %s", p.generatedName) + } + // If the phase defines a condition to be checked before executing the phase action. if p.RunIf != nil { // Check the condition and returns if the condition isn't satisfied (or fails) @@ -235,7 +250,7 @@ func (e *Runner) Help(cmdUse string) string { // computes the max length of for each phase use line maxLength := 0 e.visitAll(func(p *phaseRunner) error { - if !p.Hidden { + if !p.Hidden && !p.RunAllSiblings { length := len(p.use) if maxLength < length { maxLength = length @@ -246,11 +261,11 @@ func (e *Runner) Help(cmdUse string) string { // prints the list of phases indented by level and formatted using the maxlength // the list is enclosed in a mardown code block for ensuring better readability in the public web site - line := fmt.Sprintf("The %q command executes the following internal workflow:\n", cmdUse) + line := fmt.Sprintf("The %q command executes the following phases:\n", cmdUse) line += "```\n" offset := 2 e.visitAll(func(p *phaseRunner) error { - if !p.Hidden { + if !p.Hidden && !p.RunAllSiblings { padding := maxLength - len(p.use) + offset line += strings.Repeat(" ", offset*p.level) // indentation line += p.use // name + aliases @@ -265,6 +280,17 @@ func (e *Runner) Help(cmdUse string) string { return line } +// SetAdditionalFlags allows to define flags to be added +// to the subcommands generated for each phase (but not existing in the parent command). +// Please note that this command needs to be done before BindToCommand. +// Nb. if a flag is used only by one phase, please consider using phase LocalFlags. +func (e *Runner) SetAdditionalFlags(fn func(*pflag.FlagSet)) { + // creates a new NewFlagSet + e.cmdAdditionalFlags = pflag.NewFlagSet("phaseAdditionalFlags", pflag.ContinueOnError) + // invokes the function that sets additional flags + fn(e.cmdAdditionalFlags) +} + // BindToCommand bind the Runner to a cobra command by altering // command help, adding phase related flags and by adding phases subcommands // Please note that this command needs to be done once all the phases are added to the Runner. @@ -273,15 +299,7 @@ func (e *Runner) BindToCommand(cmd *cobra.Command) { return } - // alters the command description to show available phases - if cmd.Long != "" { - cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Long, e.Help(cmd.Use)) - } else { - cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Short, e.Help(cmd.Use)) - } - - // adds phase related flags - cmd.Flags().StringSliceVar(&e.Options.SkipPhases, "skip-phases", nil, "List of phases to be skipped") + e.prepareForExecution() // adds the phases subcommand phaseCommand := &cobra.Command{ @@ -295,14 +313,36 @@ func (e *Runner) BindToCommand(cmd *cobra.Command) { // generate all the nested subcommands for invoking single phases subcommands := map[string]*cobra.Command{} e.visitAll(func(p *phaseRunner) error { - // creates nested phase subcommand - var phaseCmd = &cobra.Command{ + // skip hidden phases + if p.Hidden { + return nil + } + + // initialize phase selector + phaseSelector := p.generatedName + + // if requested, set the phase to run all the sibling phases + if p.RunAllSiblings { + phaseSelector = p.parent.generatedName + } + + // creates phase subcommand + phaseCmd := &cobra.Command{ Use: strings.ToLower(p.Name), Short: p.Short, Long: p.Long, Example: p.Example, + Aliases: p.Aliases, Run: func(cmd *cobra.Command, args []string) { - e.Options.FilterPhases = []string{p.generatedName} + // if the phase has subphases, print the help and exits + if len(p.Phases) > 0 { + cmd.Help() + return + } + + // overrides the command triggering the Runner using the phaseCmd + e.runCmd = cmd + e.Options.FilterPhases = []string{phaseSelector} if err := e.Run(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -311,10 +351,21 @@ func (e *Runner) BindToCommand(cmd *cobra.Command) { Args: cobra.NoArgs, // this forces cobra to fail if a wrong phase name is passed } - // makes the new command inherits flags from the main command - cmd.LocalNonPersistentFlags().VisitAll(func(f *pflag.Flag) { - phaseCmd.Flags().AddFlag(f) - }) + // makes the new command inherits local flags from the parent command + // Nb. global flags will be inherited automatically + inheritsFlags(cmd.Flags(), phaseCmd.Flags(), p.InheritFlags) + + // makes the new command inherits additional flags for phases + if e.cmdAdditionalFlags != nil { + inheritsFlags(e.cmdAdditionalFlags, phaseCmd.Flags(), p.InheritFlags) + } + + // If defined, added phase local flags + if p.LocalFlags != nil { + p.LocalFlags.VisitAll(func(f *pflag.Flag) { + phaseCmd.Flags().AddFlag(f) + }) + } // adds the command to parent if p.level == 0 { @@ -326,6 +377,35 @@ func (e *Runner) BindToCommand(cmd *cobra.Command) { subcommands[p.generatedName] = phaseCmd return nil }) + + // alters the command description to show available phases + if cmd.Long != "" { + cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Long, e.Help(cmd.Use)) + } else { + cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Short, e.Help(cmd.Use)) + } + + // adds phase related flags to the main command + cmd.Flags().StringSliceVar(&e.Options.SkipPhases, "skip-phases", nil, "List of phases to be skipped") + + // keep tracks of the command triggering the runner + e.runCmd = cmd +} + +func inheritsFlags(sourceFlags, targetFlags *pflag.FlagSet, cmdFlags []string) { + // If the list of flag to be inherited from the parent command is not defined, no flag is added + if cmdFlags == nil { + return + } + + // add all the flags to be inherited to the target flagSet + sourceFlags.VisitAll(func(f *pflag.Flag) { + for _, c := range cmdFlags { + if f.Name == c { + targetFlags.AddFlag(f) + } + } + }) } // visitAll provides a utility method for visiting all the phases in the workflow @@ -345,6 +425,12 @@ func (e *Runner) prepareForExecution() { e.phaseRunners = []*phaseRunner{} var parentRunner *phaseRunner for _, phase := range e.Phases { + // skips phases that are meant to create special subcommands only + if phase.RunAllSiblings { + continue + } + + // add phases to the execution list addPhaseRunner(e, parentRunner, phase) } } diff --git a/cmd/kubeadm/app/cmd/phases/workflow/runner_test.go b/cmd/kubeadm/app/cmd/phases/workflow/runner_test.go index 3aedf6acb7c..15ad369d073 100644 --- a/cmd/kubeadm/app/cmd/phases/workflow/runner_test.go +++ b/cmd/kubeadm/app/cmd/phases/workflow/runner_test.go @@ -17,10 +17,14 @@ limitations under the License. package workflow import ( - "errors" "fmt" "reflect" + "strings" "testing" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" ) func phaseBuilder(name string, phases ...Phase) Phase { @@ -265,7 +269,7 @@ func TestHelp(t *testing.T) { }, } - expected := "The \"myCommand\" command executes the following internal workflow:\n" + + expected := "The \"myCommand\" command executes the following phases:\n" + "```\n" + "foo long description for foo ...\n" + " /bar long description for bar ...\n" + @@ -277,3 +281,217 @@ func TestHelp(t *testing.T) { t.Errorf("\nactual:\n\t%v\nexpected:\n\t%v\n", actual, expected) } } + +func phaseBuilder4(name string, cmdFlags []string, phases ...Phase) Phase { + return Phase{ + Name: name, + Phases: phases, + InheritFlags: cmdFlags, + } +} + +func phaseBuilder5(name string, flags *pflag.FlagSet) Phase { + return Phase{ + Name: name, + LocalFlags: flags, + } +} + +func TestBindToCommand(t *testing.T) { + + var dummy string + localFlags := pflag.NewFlagSet("dummy", pflag.ContinueOnError) + localFlags.StringVarP(&dummy, "flag4", "d", "d", "d") + + var usecases = []struct { + name string + runner Runner + expectedCmdAndFlags map[string][]string + setAdditionalFlags func(*pflag.FlagSet) + }{ + { + name: "when there are no phases, cmd should be left untouched", + runner: Runner{}, + }, + { + name: "phases should not inherits any parent flags by default", + runner: Runner{ + Phases: []Phase{phaseBuilder4("foo", nil)}, + }, + expectedCmdAndFlags: map[string][]string{ + "phase foo": {}, + }, + }, + { + name: "phases should be allowed to select parent flags to inherits", + runner: Runner{ + Phases: []Phase{phaseBuilder4("foo", []string{"flag1"})}, + }, + expectedCmdAndFlags: map[string][]string{ + "phase foo": {"flag1"}, //not "flag2" + }, + }, + { + name: "it should be possible to apply additional flags to all phases", + runner: Runner{ + Phases: []Phase{ + phaseBuilder4("foo", []string{"flag3"}), + phaseBuilder4("bar", []string{"flag1", "flag2", "flag3"}), + phaseBuilder4("baz", []string{"flag1"}), //test if additional flags are filtered too + }, + }, + setAdditionalFlags: func(flags *pflag.FlagSet) { + var dummy3 string + flags.StringVarP(&dummy3, "flag3", "c", "c", "c") + }, + expectedCmdAndFlags: map[string][]string{ + "phase foo": {"flag3"}, + "phase bar": {"flag1", "flag2", "flag3"}, + "phase baz": {"flag1"}, + }, + }, + { + name: "it should be possible to apply custom flags to single phases", + runner: Runner{ + Phases: []Phase{phaseBuilder5("foo", localFlags)}, + }, + expectedCmdAndFlags: map[string][]string{ + "phase foo": {"flag4"}, + }, + }, + { + name: "all the above applies to nested phases too", + runner: Runner{ + Phases: []Phase{ + phaseBuilder4("foo", []string{"flag3"}, + phaseBuilder4("bar", []string{"flag1", "flag2", "flag3"}), + phaseBuilder4("baz", []string{"flag1"}), //test if additional flags are filtered too + phaseBuilder5("qux", localFlags), + ), + }, + }, + setAdditionalFlags: func(flags *pflag.FlagSet) { + var dummy3 string + flags.StringVarP(&dummy3, "flag3", "c", "c", "c") + }, + expectedCmdAndFlags: map[string][]string{ + "phase foo": {"flag3"}, + "phase foo bar": {"flag1", "flag2", "flag3"}, + "phase foo baz": {"flag1"}, + "phase foo qux": {"flag4"}, + }, + }, + } + for _, rt := range usecases { + t.Run(rt.name, func(t *testing.T) { + + var dummy1, dummy2 string + cmd := &cobra.Command{ + Use: "init", + } + + cmd.Flags().StringVarP(&dummy1, "flag1", "a", "a", "a") + cmd.Flags().StringVarP(&dummy2, "flag2", "b", "b", "b") + + if rt.setAdditionalFlags != nil { + rt.runner.SetAdditionalFlags(rt.setAdditionalFlags) + } + + rt.runner.BindToCommand(cmd) + + // in case of no phases, checks that cmd is untouched + if len(rt.runner.Phases) == 0 { + if cmd.Long != "" { + t.Error("cmd.Long is set while it should be leaved untouched\n") + } + + if cmd.Flags().Lookup("skip-phases") != nil { + t.Error("cmd has skip-phases flag while it should not\n") + } + + if getCmd(cmd, "phase") != nil { + t.Error("cmd has phase subcommand while it should not\n") + } + + return + } + + // Otherwise, if there are phases + + // Checks that cmd get the description set and the skip-phases flags + if cmd.Long == "" { + t.Error("cmd.Long not set\n") + } + + if cmd.Flags().Lookup("skip-phases") == nil { + t.Error("cmd didn't have skip-phases flag\n") + } + + // Checks that cmd gets a new phase subcommand (without local flags) + phaseCmd := getCmd(cmd, "phase") + if phaseCmd == nil { + t.Error("cmd didn't have phase subcommand\n") + return + } + if err := cmdHasFlags(phaseCmd); err != nil { + t.Errorf("command phase didn't have expected flags: %v\n", err) + } + + // Checks that cmd subcommand gets subcommand for phases (without flags properly sets) + for c, flags := range rt.expectedCmdAndFlags { + + cCmd := getCmd(cmd, c) + if cCmd == nil { + t.Errorf("cmd didn't have %s subcommand\n", c) + continue + } + + if err := cmdHasFlags(cCmd, flags...); err != nil { + t.Errorf("command %s didn't have expected flags: %v\n", c, err) + } + } + + }) + } +} + +func getCmd(parent *cobra.Command, nestedName string) *cobra.Command { + names := strings.Split(nestedName, " ") + for i, n := range names { + for _, c := range parent.Commands() { + if c.Name() == n { + if i == len(names)-1 { + return c + } + parent = c + } + } + } + + return nil +} + +func cmdHasFlags(cmd *cobra.Command, expectedFlags ...string) error { + flags := []string{} + cmd.Flags().VisitAll(func(f *pflag.Flag) { + flags = append(flags, f.Name) + }) + + for _, e := range expectedFlags { + found := false + for _, f := range flags { + if f == e { + found = true + } + } + if !found { + return errors.Errorf("flag %q does not exists in %s", e, flags) + } + } + + if len(flags) != len(expectedFlags) { + return errors.Errorf("expected flags %s, got %s", expectedFlags, flags) + } + + return nil +} diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go index ce2e7883cc7..26599c8c385 100644 --- a/cmd/kubeadm/app/cmd/reset.go +++ b/cmd/kubeadm/app/cmd/reset.go @@ -18,7 +18,6 @@ package cmd import ( "bufio" - "errors" "fmt" "io" "os" @@ -26,11 +25,11 @@ import ( "path/filepath" "strings" - "github.com/golang/glog" + "github.com/pkg/errors" "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" @@ -51,6 +50,7 @@ func NewCmdReset(in io.Reader, out io.Writer) *cobra.Command { var criSocketPath string var ignorePreflightErrors []string var forceReset bool + var client clientset.Interface kubeConfigFile := kubeadmconstants.GetAdminKubeConfigPath() cmd := &cobra.Command{ @@ -61,8 +61,10 @@ func NewCmdReset(in io.Reader, out io.Writer) *cobra.Command { kubeadmutil.CheckErr(err) kubeConfigFile = cmdutil.FindExistingKubeConfig(kubeConfigFile) - client, err := getClientset(kubeConfigFile, false) - kubeadmutil.CheckErr(err) + if _, err := os.Stat(kubeConfigFile); !os.IsNotExist(err) { + client, err = getClientset(kubeConfigFile, false) + kubeadmutil.CheckErr(err) + } r, err := NewReset(in, ignorePreflightErrorsSet, forceReset, certsDir, criSocketPath) kubeadmutil.CheckErr(err) @@ -129,7 +131,7 @@ func (r *Reset) Run(out io.Writer, client clientset.Interface) error { // Only clear etcd data when using local etcd. etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml") - glog.V(1).Infof("[reset] checking for etcd config") + klog.V(1).Infof("[reset] checking for etcd config") etcdDataDir, err := getEtcdDataDir(etcdManifestPath, client) if err == nil { dirsToClean = append(dirsToClean, etcdDataDir) @@ -139,16 +141,16 @@ func (r *Reset) Run(out io.Writer, client clientset.Interface) error { } // Try to stop the kubelet service - glog.V(1).Infof("[reset] getting init system") + klog.V(1).Infof("[reset] getting init system") initSystem, err := initsystem.GetInitSystem() if err != nil { - glog.Warningln("[reset] the kubelet service could not be stopped by kubeadm. Unable to detect a supported init system!") - glog.Warningln("[reset] please ensure kubelet is stopped manually") + klog.Warningln("[reset] the kubelet service could not be stopped by kubeadm. Unable to detect a supported init system!") + klog.Warningln("[reset] please ensure kubelet is stopped manually") } else { fmt.Println("[reset] stopping the kubelet service") if err := initSystem.ServiceStop("kubelet"); err != nil { - glog.Warningf("[reset] the kubelet service could not be stopped by kubeadm: [%v]\n", err) - glog.Warningln("[reset] please ensure kubelet is stopped manually") + klog.Warningf("[reset] the kubelet service could not be stopped by kubeadm: [%v]\n", err) + klog.Warningln("[reset] please ensure kubelet is stopped manually") } } @@ -156,32 +158,46 @@ func (r *Reset) Run(out io.Writer, client clientset.Interface) error { fmt.Printf("[reset] unmounting mounted directories in %q\n", kubeadmconstants.KubeletRunDirectory) umountDirsCmd := fmt.Sprintf("awk '$2 ~ path {print $2}' path=%s /proc/mounts | xargs -r umount", kubeadmconstants.KubeletRunDirectory) - glog.V(1).Infof("[reset] executing command %q", umountDirsCmd) + klog.V(1).Infof("[reset] executing command %q", umountDirsCmd) umountOutputBytes, err := exec.Command("sh", "-c", umountDirsCmd).Output() if err != nil { - glog.Errorf("[reset] failed to unmount mounted directories in %s: %s\n", kubeadmconstants.KubeletRunDirectory, string(umountOutputBytes)) + klog.Errorf("[reset] failed to unmount mounted directories in %s: %s\n", kubeadmconstants.KubeletRunDirectory, string(umountOutputBytes)) } - glog.V(1).Info("[reset] removing Kubernetes-managed containers") + klog.V(1).Info("[reset] removing Kubernetes-managed containers") if err := removeContainers(utilsexec.New(), r.criSocketPath); err != nil { - glog.Errorf("[reset] failed to remove containers: %+v", err) + klog.Errorf("[reset] failed to remove containers: %+v", err) } + dirsToClean = append(dirsToClean, []string{kubeadmconstants.KubeletRunDirectory, "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"}...) // Then clean contents from the stateful kubelet, etcd and cni directories fmt.Printf("[reset] deleting contents of stateful directories: %v\n", dirsToClean) for _, dir := range dirsToClean { - glog.V(1).Infof("[reset] deleting content of %s", dir) + klog.V(1).Infof("[reset] deleting content of %s", dir) cleanDir(dir) } // Remove contents from the config and pki directories - glog.V(1).Infoln("[reset] removing contents from the config and pki directories") + klog.V(1).Infoln("[reset] removing contents from the config and pki directories") if r.certsDir != kubeadmapiv1beta1.DefaultCertificatesDir { - glog.Warningf("[reset] WARNING: cleaning a non-default certificates directory: %q\n", r.certsDir) + klog.Warningf("[reset] WARNING: cleaning a non-default certificates directory: %q\n", r.certsDir) } resetConfigDir(kubeadmconstants.KubernetesDir, r.certsDir) + // Output help text instructing user how to remove iptables rules + msg := ` +The reset process does not reset or clean up iptables rules or IPVS tables. +If you wish to reset iptables, you must do so manually. +For example: +iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X + +If your cluster was setup to utilize IPVS, run ipvsadm --clear (or similar) +to reset your system's IPVS tables. + +` + fmt.Print(msg) + return nil } @@ -189,11 +205,13 @@ func getEtcdDataDir(manifestPath string, client clientset.Interface) (string, er const etcdVolumeName = "etcd-data" var dataDir string - cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "reset", "", false) - if err == nil { - return cfg.Etcd.Local.DataDir, nil + if client != nil { + cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "reset", "", false) + if err == nil { + return cfg.Etcd.Local.DataDir, nil + } + klog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap, using etcd pod spec as fallback: %v", err) } - glog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap, using etcd pod spec as fallback: %v", err) etcdPod, err := utilstaticpod.ReadStaticPodFromDisk(manifestPath) if err != nil { @@ -261,7 +279,7 @@ func resetConfigDir(configPathDir, pkiPathDir string) { fmt.Printf("[reset] deleting contents of config directories: %v\n", dirsToClean) for _, dir := range dirsToClean { if err := cleanDir(dir); err != nil { - glog.Errorf("[reset] failed to remove directory: %q [%v]\n", dir, err) + klog.Errorf("[reset] failed to remove directory: %q [%v]\n", dir, err) } } @@ -275,7 +293,7 @@ func resetConfigDir(configPathDir, pkiPathDir string) { fmt.Printf("[reset] deleting files: %v\n", filesToClean) for _, path := range filesToClean { if err := os.RemoveAll(path); err != nil { - glog.Errorf("[reset] failed to remove file: %q [%v]\n", path, err) + klog.Errorf("[reset] failed to remove file: %q [%v]\n", path, err) } } } diff --git a/cmd/kubeadm/app/cmd/reset_test.go b/cmd/kubeadm/app/cmd/reset_test.go index f15dec50392..063d59b6cbe 100644 --- a/cmd/kubeadm/app/cmd/reset_test.go +++ b/cmd/kubeadm/app/cmd/reset_test.go @@ -273,30 +273,38 @@ func TestGetEtcdDataDir(t *testing.T) { podYaml string expectErr bool writeManifest bool + validClient bool }{ "non-existent file returns error": { - dataDir: "", - podYaml: "", expectErr: true, writeManifest: false, + validClient: true, }, "return etcd data dir": { dataDir: "/path/to/etcd", podYaml: etcdPod, expectErr: false, writeManifest: true, + validClient: true, }, "invalid etcd pod": { - dataDir: "", podYaml: etcdPodInvalid, expectErr: true, writeManifest: true, + validClient: true, }, "etcd pod spec without data volume": { - dataDir: "", podYaml: etcdPodWithoutDataVolume, expectErr: true, writeManifest: true, + validClient: true, + }, + "kubeconfig file doesn't exist": { + dataDir: "/path/to/etcd", + podYaml: etcdPod, + expectErr: false, + writeManifest: true, + validClient: false, }, } @@ -312,8 +320,15 @@ func TestGetEtcdDataDir(t *testing.T) { } } - client := clientsetfake.NewSimpleClientset() - dataDir, err := getEtcdDataDir(manifestPath, client) + var dataDir string + var err error + if test.validClient { + client := clientsetfake.NewSimpleClientset() + dataDir, err = getEtcdDataDir(manifestPath, client) + } else { + dataDir, err = getEtcdDataDir(manifestPath, nil) + } + if (err != nil) != test.expectErr { t.Fatalf(dedent.Dedent( "getEtcdDataDir failed\n%s\nexpected error: %t\n\tgot: %t\nerror: %v"), diff --git a/cmd/kubeadm/app/cmd/token.go b/cmd/kubeadm/app/cmd/token.go index b5fc0334e48..9e28e9000d6 100644 --- a/cmd/kubeadm/app/cmd/token.go +++ b/cmd/kubeadm/app/cmd/token.go @@ -24,10 +24,10 @@ import ( "text/tabwriter" "time" - "github.com/golang/glog" "github.com/pkg/errors" "github.com/renstrom/dedent" "github.com/spf13/cobra" + "k8s.io/klog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -113,14 +113,14 @@ func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command { if len(args) > 0 { bto.TokenStr = args[0] } - glog.V(1).Infoln("[token] validating mixed arguments") + klog.V(1).Infoln("[token] validating mixed arguments") err := validation.ValidateMixedArguments(tokenCmd.Flags()) kubeadmutil.CheckErr(err) err = bto.ApplyTo(cfg) kubeadmutil.CheckErr(err) - glog.V(1).Infoln("[token] getting Clientsets from kubeconfig file") + klog.V(1).Infoln("[token] getting Clientsets from kubeconfig file") kubeConfigFile = cmdutil.FindExistingKubeConfig(kubeConfigFile) client, err := getClientset(kubeConfigFile, dryRun) kubeadmutil.CheckErr(err) @@ -215,13 +215,13 @@ func RunCreateToken(out io.Writer, client clientset.Interface, cfgPath string, c phaseutil.SetKubernetesVersion(cfg) // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags - glog.V(1).Infoln("[token] loading configurations") + klog.V(1).Infoln("[token] loading configurations") internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) if err != nil { return err } - glog.V(1).Infoln("[token] creating token") + klog.V(1).Infoln("[token] creating token") if err := tokenphase.CreateNewTokens(client, internalcfg.BootstrapTokens); err != nil { return err } @@ -243,7 +243,7 @@ func RunCreateToken(out io.Writer, client clientset.Interface, cfgPath string, c // RunGenerateToken just generates a random token for the user func RunGenerateToken(out io.Writer) error { - glog.V(1).Infoln("[token] generating random token") + klog.V(1).Infoln("[token] generating random token") token, err := bootstraputil.GenerateBootstrapToken() if err != nil { return err @@ -256,7 +256,7 @@ func RunGenerateToken(out io.Writer) error { // RunListTokens lists details on all existing bootstrap tokens on the server. func RunListTokens(out io.Writer, errW io.Writer, client clientset.Interface) error { // First, build our selector for bootstrap tokens only - glog.V(1).Infoln("[token] preparing selector for bootstrap token") + klog.V(1).Infoln("[token] preparing selector for bootstrap token") tokenSelector := fields.SelectorFromSet( map[string]string{ // TODO: We hard-code "type" here until `field_constants.go` that is @@ -269,7 +269,7 @@ func RunListTokens(out io.Writer, errW io.Writer, client clientset.Interface) er FieldSelector: tokenSelector.String(), } - glog.V(1).Infoln("[token] retrieving list of bootstrap tokens") + klog.V(1).Infoln("[token] retrieving list of bootstrap tokens") secrets, err := client.CoreV1().Secrets(metav1.NamespaceSystem).List(listOptions) if err != nil { return errors.Wrap(err, "failed to list bootstrap tokens") @@ -298,7 +298,7 @@ func RunListTokens(out io.Writer, errW io.Writer, client clientset.Interface) er func RunDeleteToken(out io.Writer, client clientset.Interface, tokenIDOrToken string) error { // Assume the given first argument is a token id and try to parse it tokenID := tokenIDOrToken - glog.V(1).Infoln("[token] parsing token ID") + klog.V(1).Infoln("[token] parsing token ID") if !bootstraputil.IsValidBootstrapTokenID(tokenIDOrToken) { // Okay, the full token with both id and secret was probably passed. Parse it and extract the ID only bts, err := kubeadmapiv1beta1.NewBootstrapTokenString(tokenIDOrToken) @@ -310,7 +310,7 @@ func RunDeleteToken(out io.Writer, client clientset.Interface, tokenIDOrToken st } tokenSecretName := bootstraputil.BootstrapTokenSecretName(tokenID) - glog.V(1).Infoln("[token] deleting token") + klog.V(1).Infoln("[token] deleting token") if err := client.CoreV1().Secrets(metav1.NamespaceSystem).Delete(tokenSecretName, nil); err != nil { return errors.Wrap(err, "failed to delete bootstrap token") } diff --git a/cmd/kubeadm/app/cmd/upgrade/BUILD b/cmd/kubeadm/app/cmd/upgrade/BUILD index 87a41556375..fa0e144a6f8 100644 --- a/cmd/kubeadm/app/cmd/upgrade/BUILD +++ b/cmd/kubeadm/app/cmd/upgrade/BUILD @@ -39,11 +39,11 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/client-go/discovery/fake:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", "//vendor/github.com/pmezard/go-difflib/difflib:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index af03ad615a6..248f7db564e 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -21,11 +21,11 @@ import ( "os" "time" - "github.com/golang/glog" "github.com/pkg/errors" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/util/version" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" @@ -83,13 +83,13 @@ func NewCmdApply(apf *applyPlanFlags) *cobra.Command { kubeadmutil.CheckErr(err) // Ensure the user is root - glog.V(1).Infof("running preflight checks") + klog.V(1).Infof("running preflight checks") err = runPreflightChecks(flags.ignorePreflightErrorsSet) kubeadmutil.CheckErr(err) // If the version is specified in config file, pick up that value. if flags.cfgPath != "" { - glog.V(1).Infof("fetching configuration from file %s", flags.cfgPath) + klog.V(1).Infof("fetching configuration from file %s", flags.cfgPath) // Note that cfg isn't preserved here, it's just an one-off to populate flags.newK8sVersionStr based on --config cfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(flags.cfgPath, &kubeadmapiv1beta1.InitConfiguration{}) kubeadmutil.CheckErr(err) @@ -147,8 +147,8 @@ func NewCmdApply(apf *applyPlanFlags) *cobra.Command { func RunApply(flags *applyFlags) error { // Start with the basics, verify that the cluster is healthy and get the configuration from the cluster (using the ConfigMap) - glog.V(1).Infof("[upgrade/apply] verifying health of cluster") - glog.V(1).Infof("[upgrade/apply] retrieving configuration from cluster") + klog.V(1).Infof("[upgrade/apply] verifying health of cluster") + klog.V(1).Infof("[upgrade/apply] retrieving configuration from cluster") upgradeVars, err := enforceRequirements(flags.applyPlanFlags, flags.dryRun, flags.newK8sVersionStr) if err != nil { return err @@ -160,7 +160,7 @@ func RunApply(flags *applyFlags) error { } // Validate requested and validate actual version - glog.V(1).Infof("[upgrade/apply] validating requested and actual version") + klog.V(1).Infof("[upgrade/apply] validating requested and actual version") if err := configutil.NormalizeKubernetesVersion(&upgradeVars.cfg.ClusterConfiguration); err != nil { return err } @@ -178,7 +178,7 @@ func RunApply(flags *applyFlags) error { } // Enforce the version skew policies - glog.V(1).Infof("[upgrade/version] enforcing version skew policies") + klog.V(1).Infof("[upgrade/version] enforcing version skew policies") if err := EnforceVersionPolicies(flags, upgradeVars.versionGetter); err != nil { return errors.Wrap(err, "[upgrade/version] FATAL") } @@ -192,10 +192,10 @@ func RunApply(flags *applyFlags) error { // Use a prepuller implementation based on creating DaemonSets // and block until all DaemonSets are ready; then we know for sure that all control plane images are cached locally - glog.V(1).Infof("[upgrade/apply] creating prepuller") + klog.V(1).Infof("[upgrade/apply] creating prepuller") prepuller := upgrade.NewDaemonSetPrepuller(upgradeVars.client, upgradeVars.waiter, &upgradeVars.cfg.ClusterConfiguration) componentsToPrepull := constants.MasterComponents - if upgradeVars.cfg.Etcd.External != nil { + if upgradeVars.cfg.Etcd.External == nil && flags.etcdUpgrade { componentsToPrepull = append(componentsToPrepull, constants.Etcd) } if err := upgrade.PrepullImagesInParallel(prepuller, flags.imagePullTimeout, componentsToPrepull); err != nil { @@ -203,13 +203,13 @@ func RunApply(flags *applyFlags) error { } // Now; perform the upgrade procedure - glog.V(1).Infof("[upgrade/apply] performing upgrade") + klog.V(1).Infof("[upgrade/apply] performing upgrade") if err := PerformControlPlaneUpgrade(flags, upgradeVars.client, upgradeVars.waiter, upgradeVars.cfg); err != nil { return errors.Wrap(err, "[upgrade/apply] FATAL") } // Upgrade RBAC rules and addons. - glog.V(1).Infof("[upgrade/postupgrade] upgrading RBAC rules and addons") + klog.V(1).Infof("[upgrade/postupgrade] upgrading RBAC rules and addons") if err := upgrade.PerformPostUpgradeTasks(upgradeVars.client, upgradeVars.cfg, flags.newK8sVersion, flags.dryRun); err != nil { return errors.Wrap(err, "[upgrade/postupgrade] FATAL post-upgrade error") } @@ -270,16 +270,6 @@ func EnforceVersionPolicies(flags *applyFlags, versionGetter upgrade.VersionGett // PerformControlPlaneUpgrade actually performs the upgrade procedure for the cluster of your type (self-hosted or static-pod-hosted) func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration) error { - // Check if the cluster is self-hosted and act accordingly - glog.V(1).Infoln("checking if cluster is self-hosted") - if upgrade.IsControlPlaneSelfHosted(client) { - fmt.Printf("[upgrade/apply] Upgrading your Self-Hosted control plane to version %q...\n", flags.newK8sVersionStr) - - // Upgrade the self-hosted cluster - glog.V(1).Infoln("[upgrade/apply] upgrading self-hosted cluster") - return upgrade.SelfHostedControlPlane(client, waiter, internalcfg, flags.newK8sVersion) - } - // OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q...\n", flags.newK8sVersionStr) diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 97d2bda4ccd..28158e9b142 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -58,6 +58,11 @@ func enforceRequirements(flags *applyPlanFlags, dryRun bool, newK8sVersion strin return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath) } + // Check if the cluster is self-hosted + if upgrade.IsControlPlaneSelfHosted(client) { + return nil, errors.Errorf("cannot upgrade a self-hosted control plane") + } + // Run healthchecks against the cluster if err := upgrade.CheckClusterHealth(client, flags.ignorePreflightErrorsSet); err != nil { return nil, errors.Wrap(err, "[upgrade/health] FATAL") diff --git a/cmd/kubeadm/app/cmd/upgrade/common_test.go b/cmd/kubeadm/app/cmd/upgrade/common_test.go index 5547b71cd2a..513ff84edeb 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/common_test.go @@ -41,20 +41,21 @@ func TestPrintConfiguration(t *testing.T) { DataDir: "/some/path", }, }, + DNS: kubeadmapi.DNS{ + Type: kubeadmapi.CoreDNS, + }, }, expectedBytes: []byte(`[upgrade/config] Configuration used: apiServer: {} apiVersion: kubeadm.k8s.io/v1beta1 - auditPolicy: - logDir: "" - path: "" certificatesDir: "" controlPlaneEndpoint: "" controllerManager: {} + dns: + type: CoreDNS etcd: local: dataDir: /some/path - image: "" imageRepository: "" kind: ClusterConfiguration kubernetesVersion: v1.7.1 @@ -63,7 +64,6 @@ func TestPrintConfiguration(t *testing.T) { podSubnet: "" serviceSubnet: "" scheduler: {} - unifiedControlPlaneImage: "" `), }, { @@ -77,16 +77,18 @@ func TestPrintConfiguration(t *testing.T) { Endpoints: []string{"https://one-etcd-instance:2379"}, }, }, + DNS: kubeadmapi.DNS{ + Type: kubeadmapi.CoreDNS, + }, }, expectedBytes: []byte(`[upgrade/config] Configuration used: apiServer: {} apiVersion: kubeadm.k8s.io/v1beta1 - auditPolicy: - logDir: "" - path: "" certificatesDir: "" controlPlaneEndpoint: "" controllerManager: {} + dns: + type: CoreDNS etcd: external: caFile: "" @@ -102,7 +104,6 @@ func TestPrintConfiguration(t *testing.T) { podSubnet: "" serviceSubnet: 10.96.0.1/12 scheduler: {} - unifiedControlPlaneImage: "" `), }, } diff --git a/cmd/kubeadm/app/cmd/upgrade/diff.go b/cmd/kubeadm/app/cmd/upgrade/diff.go index 5edd203d034..90c8855d44a 100644 --- a/cmd/kubeadm/app/cmd/upgrade/diff.go +++ b/cmd/kubeadm/app/cmd/upgrade/diff.go @@ -20,12 +20,12 @@ import ( "io" "io/ioutil" - "github.com/golang/glog" "github.com/pkg/errors" "github.com/pmezard/go-difflib/difflib" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/version" + "k8s.io/klog" kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" @@ -78,7 +78,7 @@ func NewCmdDiff(out io.Writer) *cobra.Command { func runDiff(flags *diffFlags, args []string) error { // If the version is specified in config file, pick up that value. - glog.V(1).Infof("fetching configuration from file %s", flags.cfgPath) + klog.V(1).Infof("fetching configuration from file %s", flags.cfgPath) cfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(flags.cfgPath, &kubeadmapiv1beta1.InitConfiguration{}) if err != nil { return err @@ -116,7 +116,7 @@ func runDiff(flags *diffFlags, args []string) error { case constants.KubeScheduler: path = flags.schedulerManifestPath default: - glog.Errorf("[diff] unknown spec %v", spec) + klog.Errorf("[diff] unknown spec %v", spec) continue } diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go index dbe3c9d9c5a..9e3c99a097c 100644 --- a/cmd/kubeadm/app/cmd/upgrade/node.go +++ b/cmd/kubeadm/app/cmd/upgrade/node.go @@ -22,10 +22,10 @@ import ( "os" "path/filepath" - "github.com/golang/glog" "github.com/pkg/errors" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/util/version" + "k8s.io/klog" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" "k8s.io/kubernetes/cmd/kubeadm/app/constants" @@ -126,7 +126,7 @@ func NewCmdUpgradeControlPlane() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { if flags.nodeName == "" { - glog.V(1).Infoln("[upgrade] found NodeName empty; considered OS hostname as NodeName") + klog.V(1).Infoln("[upgrade] found NodeName empty; considered OS hostname as NodeName") } nodeName, err := node.GetHostname(flags.nodeName) if err != nil { diff --git a/cmd/kubeadm/app/cmd/upgrade/plan.go b/cmd/kubeadm/app/cmd/upgrade/plan.go index b71719c1ada..482acdd4429 100644 --- a/cmd/kubeadm/app/cmd/upgrade/plan.go +++ b/cmd/kubeadm/app/cmd/upgrade/plan.go @@ -24,13 +24,13 @@ import ( "strings" "text/tabwriter" - "github.com/golang/glog" "github.com/pkg/errors" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/util/version" + "k8s.io/klog" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" @@ -62,7 +62,7 @@ func NewCmdPlan(apf *applyPlanFlags) *cobra.Command { // If the version is specified in config file, pick up that value. if flags.cfgPath != "" { - glog.V(1).Infof("fetching configuration from file %s", flags.cfgPath) + klog.V(1).Infof("fetching configuration from file %s", flags.cfgPath) cfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(flags.cfgPath, &kubeadmapiv1beta1.InitConfiguration{}) kubeadmutil.CheckErr(err) @@ -88,8 +88,8 @@ func NewCmdPlan(apf *applyPlanFlags) *cobra.Command { // RunPlan takes care of outputting available versions to upgrade to for the user func RunPlan(flags *planFlags) error { // Start with the basics, verify that the cluster is healthy, build a client and a versionGetter. Never dry-run when planning. - glog.V(1).Infof("[upgrade/plan] verifying health of cluster") - glog.V(1).Infof("[upgrade/plan] retrieving configuration from cluster") + klog.V(1).Infof("[upgrade/plan] verifying health of cluster") + klog.V(1).Infof("[upgrade/plan] retrieving configuration from cluster") upgradeVars, err := enforceRequirements(flags.applyPlanFlags, false, flags.newK8sVersionStr) if err != nil { return err @@ -120,8 +120,8 @@ func RunPlan(flags *planFlags) error { } // Compute which upgrade possibilities there are - glog.V(1).Infof("[upgrade/plan] computing upgrade possibilities") - availUpgrades, err := upgrade.GetAvailableUpgrades(upgradeVars.versionGetter, flags.allowExperimentalUpgrades, flags.allowRCUpgrades, etcdClient, upgradeVars.cfg.FeatureGates, upgradeVars.client) + klog.V(1).Infof("[upgrade/plan] computing upgrade possibilities") + availUpgrades, err := upgrade.GetAvailableUpgrades(upgradeVars.versionGetter, flags.allowExperimentalUpgrades, flags.allowRCUpgrades, etcdClient, upgradeVars.cfg.DNS.Type, upgradeVars.client) if err != nil { return errors.Wrap(err, "[upgrade/versions] FATAL") } @@ -207,19 +207,19 @@ func printAvailableUpgrades(upgrades []upgrade.Upgrade, w io.Writer, isExternalE coreDNSBeforeVersion, coreDNSAfterVersion, kubeDNSBeforeVersion, kubeDNSAfterVersion := "", "", "", "" switch upgrade.Before.DNSType { - case constants.CoreDNS: + case kubeadmapi.CoreDNS: printCoreDNS = true coreDNSBeforeVersion = upgrade.Before.DNSVersion - case constants.KubeDNS: + case kubeadmapi.KubeDNS: printKubeDNS = true kubeDNSBeforeVersion = upgrade.Before.DNSVersion } switch upgrade.After.DNSType { - case constants.CoreDNS: + case kubeadmapi.CoreDNS: printCoreDNS = true coreDNSAfterVersion = upgrade.After.DNSVersion - case constants.KubeDNS: + case kubeadmapi.KubeDNS: printKubeDNS = true kubeDNSAfterVersion = upgrade.After.DNSVersion } diff --git a/cmd/kubeadm/app/cmd/upgrade/plan_test.go b/cmd/kubeadm/app/cmd/upgrade/plan_test.go index acd30985e2c..5ea882565a9 100644 --- a/cmd/kubeadm/app/cmd/upgrade/plan_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/plan_test.go @@ -21,6 +21,7 @@ import ( "reflect" "testing" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" ) @@ -91,14 +92,14 @@ func TestPrintAvailableUpgrades(t *testing.T) { "v1.8.1": 1, }, KubeadmVersion: "v1.8.2", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.5", EtcdVersion: "3.0.17", }, After: upgrade.ClusterState{ KubeVersion: "v1.8.3", KubeadmVersion: "v1.8.3", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.5", EtcdVersion: "3.0.17", }, @@ -139,14 +140,14 @@ _____________________________________________________________________ "v1.8.3": 1, }, KubeadmVersion: "v1.9.0", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.5", EtcdVersion: "3.0.17", }, After: upgrade.ClusterState{ KubeVersion: "v1.9.0", KubeadmVersion: "v1.9.0", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.13", EtcdVersion: "3.1.12", }, @@ -185,14 +186,14 @@ _____________________________________________________________________ "v1.8.3": 1, }, KubeadmVersion: "v1.8.3", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.5", EtcdVersion: "3.0.17", }, After: upgrade.ClusterState{ KubeVersion: "v1.8.5", KubeadmVersion: "v1.8.3", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.5", EtcdVersion: "3.0.17", }, @@ -205,14 +206,14 @@ _____________________________________________________________________ "v1.8.3": 1, }, KubeadmVersion: "v1.8.3", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.5", EtcdVersion: "3.0.17", }, After: upgrade.ClusterState{ KubeVersion: "v1.9.0", KubeadmVersion: "v1.9.0", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.13", EtcdVersion: "3.1.12", }, @@ -273,14 +274,14 @@ _____________________________________________________________________ "v1.8.5": 1, }, KubeadmVersion: "v1.8.5", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.5", EtcdVersion: "3.0.17", }, After: upgrade.ClusterState{ KubeVersion: "v1.9.0-beta.1", KubeadmVersion: "v1.9.0-beta.1", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.13", EtcdVersion: "3.1.12", }, @@ -321,14 +322,14 @@ _____________________________________________________________________ "v1.8.5": 1, }, KubeadmVersion: "v1.8.5", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.5", EtcdVersion: "3.0.17", }, After: upgrade.ClusterState{ KubeVersion: "v1.9.0-rc.1", KubeadmVersion: "v1.9.0-rc.1", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.13", EtcdVersion: "3.1.12", }, @@ -370,14 +371,14 @@ _____________________________________________________________________ "v1.9.3": 2, }, KubeadmVersion: "v1.9.2", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.5", EtcdVersion: "3.0.17", }, After: upgrade.ClusterState{ KubeVersion: "v1.9.3", KubeadmVersion: "v1.9.3", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.8", EtcdVersion: "3.1.12", }, @@ -420,14 +421,14 @@ _____________________________________________________________________ "v1.9.2": 1, }, KubeadmVersion: "v1.9.2", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.5", EtcdVersion: "3.0.17", }, After: upgrade.ClusterState{ KubeVersion: "v1.9.3", KubeadmVersion: "v1.9.3", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.8", EtcdVersion: "3.1.12", }, @@ -472,14 +473,14 @@ _____________________________________________________________________ "v1.10.2": 1, }, KubeadmVersion: "v1.11.0", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.7", EtcdVersion: "3.1.11", }, After: upgrade.ClusterState{ KubeVersion: "v1.11.0", KubeadmVersion: "v1.11.0", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.2.18", }, @@ -519,14 +520,14 @@ _____________________________________________________________________ "v1.10.2": 1, }, KubeadmVersion: "v1.11.0", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.5", EtcdVersion: "3.1.11", }, After: upgrade.ClusterState{ KubeVersion: "v1.11.0", KubeadmVersion: "v1.11.0", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.2.18", }, @@ -565,14 +566,14 @@ _____________________________________________________________________ "v1.10.2": 1, }, KubeadmVersion: "v1.11.0", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.11", }, After: upgrade.ClusterState{ KubeVersion: "v1.11.0", KubeadmVersion: "v1.11.0", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.9", EtcdVersion: "3.2.18", }, diff --git a/cmd/kubeadm/app/cmd/version.go b/cmd/kubeadm/app/cmd/version.go index eb0628c6f1b..2d237d50b20 100644 --- a/cmd/kubeadm/app/cmd/version.go +++ b/cmd/kubeadm/app/cmd/version.go @@ -21,10 +21,10 @@ import ( "fmt" "io" - "github.com/ghodss/yaml" - "github.com/golang/glog" "github.com/pkg/errors" "github.com/spf13/cobra" + "k8s.io/klog" + "sigs.k8s.io/yaml" apimachineryversion "k8s.io/apimachinery/pkg/version" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" @@ -53,7 +53,7 @@ func NewCmdVersion(out io.Writer) *cobra.Command { // RunVersion provides the version information of kubeadm in format depending on arguments // specified in cobra.Command. func RunVersion(out io.Writer, cmd *cobra.Command) error { - glog.V(1).Infoln("[version] retrieving version info") + klog.V(1).Infoln("[version] retrieving version info") clientVersion := version.Get() v := Version{ ClientVersion: &clientVersion, @@ -62,7 +62,7 @@ func RunVersion(out io.Writer, cmd *cobra.Command) error { const flag = "output" of, err := cmd.Flags().GetString(flag) if err != nil { - glog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) + klog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) } switch of { diff --git a/cmd/kubeadm/app/cmd/version_test.go b/cmd/kubeadm/app/cmd/version_test.go index e22036b7f87..b72977e5806 100644 --- a/cmd/kubeadm/app/cmd/version_test.go +++ b/cmd/kubeadm/app/cmd/version_test.go @@ -19,8 +19,8 @@ package cmd import ( "bytes" "encoding/json" - "github.com/ghodss/yaml" "github.com/pkg/errors" + "sigs.k8s.io/yaml" "testing" ) diff --git a/cmd/kubeadm/app/constants/BUILD b/cmd/kubeadm/app/constants/BUILD index ba72217505c..7c1d85a6117 100644 --- a/cmd/kubeadm/app/constants/BUILD +++ b/cmd/kubeadm/app/constants/BUILD @@ -11,10 +11,12 @@ go_library( srcs = ["constants.go"], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/constants", deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//pkg/registry/core/service/ipallocator:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", ], ) @@ -35,5 +37,9 @@ go_test( name = "go_default_test", srcs = ["constants_test.go"], embed = [":go_default_library"], - deps = ["//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", + ], ) diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 61374d5806e..89f07d26df7 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -25,9 +25,12 @@ import ( "path/filepath" "time" + "github.com/pkg/errors" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/version" bootstrapapi "k8s.io/cluster-bootstrap/token/api" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" ) @@ -178,6 +181,9 @@ const ( // TLSBootstrapTimeout specifies how long kubeadm should wait for the kubelet to perform the TLS Bootstrap TLSBootstrapTimeout = 2 * time.Minute + // DefaultControlPlaneTimeout specifies the default control plane (actually API Server) timeout for use by kubeadm + DefaultControlPlaneTimeout = 4 * time.Minute + // MinimumAddressesInServiceSubnet defines minimum amount of nodes the Service subnet should allow. // We need at least ten, because the DNS service is always at the tenth cluster clusterIP MinimumAddressesInServiceSubnet = 10 @@ -260,6 +266,8 @@ const ( KubeScheduler = "kube-scheduler" // KubeProxy defines variable used internally when referring to kube-proxy component KubeProxy = "kube-proxy" + // HyperKube defines variable used internally when referring to the hyperkube image + HyperKube = "hyperkube" // SelfHostingPrefix describes the prefix workloads that are self-hosted by kubeadm has SelfHostingPrefix = "self-hosted-" @@ -276,10 +284,29 @@ const ( // DefaultCIImageRepository points to image registry where CI uploads images from ci-cross build job DefaultCIImageRepository = "gcr.io/kubernetes-ci-images" - // CoreDNS defines a variable used internally when referring to the CoreDNS addon for a cluster - CoreDNS = "coredns" - // KubeDNS defines a variable used internally when referring to the kube-dns addon for a cluster - KubeDNS = "kube-dns" + // CoreDNSConfigMap specifies in what ConfigMap in the kube-system namespace the CoreDNS config should be stored + CoreDNSConfigMap = "coredns" + + // CoreDNSDeploymentName specifies the name of the Deployment for CoreDNS add-on + CoreDNSDeploymentName = "coredns" + + // CoreDNSImageName specifies the name of the image for CoreDNS add-on + CoreDNSImageName = "coredns" + + // KubeDNSConfigMap specifies in what ConfigMap in the kube-system namespace the kube-dns config should be stored + KubeDNSConfigMap = "kube-dns" + + // KubeDNSDeploymentName specifies the name of the Deployment for kube-dns add-on + KubeDNSDeploymentName = "kube-dns" + + // KubeDNSKubeDNSImageName specifies the name of the image for the kubedns container in the kube-dns add-on + KubeDNSKubeDNSImageName = "k8s-dns-kube-dns" + + // KubeDNSSidecarImageName specifies the name of the image for the sidecar container in the kube-dns add-on + KubeDNSSidecarImageName = "k8s-dns-sidecar" + + // KubeDNSDnsMasqNannyImageName specifies the name of the image for the dnsmasq container in the kube-dns add-on + KubeDNSDnsMasqNannyImageName = "k8s-dns-dnsmasq-nanny" // CRICtlPackage defines the go package that installs crictl CRICtlPackage = "github.com/kubernetes-incubator/cri-tools/cmd/crictl" @@ -306,7 +333,7 @@ const ( KubeDNSVersion = "1.14.13" // CoreDNSVersion is the version of CoreDNS to be deployed if it is used - CoreDNSVersion = "1.2.4" + CoreDNSVersion = "1.2.6" // ClusterConfigurationKind is the string kind value for the ClusterConfiguration struct ClusterConfigurationKind = "ClusterConfiguration" @@ -379,7 +406,7 @@ func EtcdSupportedVersion(versionString string) (*version.Version, error) { } return etcdVersion, nil } - return nil, fmt.Errorf("Unsupported or unknown Kubernetes version(%v)", kubernetesVersion) + return nil, errors.Errorf("Unsupported or unknown Kubernetes version(%v)", kubernetesVersion) } // GetStaticPodDirectory returns the location on the disk where the Static Pod should be present @@ -417,12 +444,12 @@ func CreateTempDirForKubeadm(dirName string) (string, error) { tempDir := path.Join(KubernetesDir, TempDirForKubeadm) // creates target folder if not already exists if err := os.MkdirAll(tempDir, 0700); err != nil { - return "", fmt.Errorf("failed to create directory %q: %v", tempDir, err) + return "", errors.Wrapf(err, "failed to create directory %q", tempDir) } tempDir, err := ioutil.TempDir(tempDir, dirName) if err != nil { - return "", fmt.Errorf("couldn't create a temporary directory: %v", err) + return "", errors.Wrap(err, "couldn't create a temporary directory") } return tempDir, nil } @@ -432,13 +459,13 @@ func CreateTimestampDirForKubeadm(dirName string) (string, error) { tempDir := path.Join(KubernetesDir, TempDirForKubeadm) // creates target folder if not already exists if err := os.MkdirAll(tempDir, 0700); err != nil { - return "", fmt.Errorf("failed to create directory %q: %v", tempDir, err) + return "", errors.Wrapf(err, "failed to create directory %q", tempDir) } timestampDirName := fmt.Sprintf("%s-%s", dirName, time.Now().Format("2006-01-02-15-04-05")) timestampDir := path.Join(tempDir, timestampDirName) if err := os.Mkdir(timestampDir, 0700); err != nil { - return "", fmt.Errorf("could not create timestamp directory: %v", err) + return "", errors.Wrap(err, "could not create timestamp directory") } return timestampDir, nil @@ -449,13 +476,13 @@ func GetDNSIP(svcSubnet string) (net.IP, error) { // Get the service subnet CIDR _, svcSubnetCIDR, err := net.ParseCIDR(svcSubnet) if err != nil { - return nil, fmt.Errorf("couldn't parse service subnet CIDR %q: %v", svcSubnet, err) + return nil, errors.Wrapf(err, "couldn't parse service subnet CIDR %q", svcSubnet) } // Selects the 10th IP in service subnet CIDR range as dnsIP dnsIP, err := ipallocator.GetIndexedIP(svcSubnetCIDR, 10) if err != nil { - return nil, fmt.Errorf("unable to get tenth IP address from service subnet CIDR %s: %v", svcSubnetCIDR.String(), err) + return nil, errors.Wrapf(err, "unable to get tenth IP address from service subnet CIDR %s", svcSubnetCIDR.String()) } return dnsIP, nil @@ -467,12 +494,12 @@ func GetStaticPodAuditPolicyFile() string { } // GetDNSVersion is a handy function that returns the DNS version by DNS type -func GetDNSVersion(dnsType string) string { +func GetDNSVersion(dnsType kubeadmapi.DNSAddOnType) string { switch dnsType { - case CoreDNS: - return CoreDNSVersion - default: + case kubeadmapi.KubeDNS: return KubeDNSVersion + default: + return CoreDNSVersion } } diff --git a/cmd/kubeadm/app/constants/constants_test.go b/cmd/kubeadm/app/constants/constants_test.go index 50420c567d1..bbcab17240c 100644 --- a/cmd/kubeadm/app/constants/constants_test.go +++ b/cmd/kubeadm/app/constants/constants_test.go @@ -17,12 +17,14 @@ limitations under the License. package constants import ( - "fmt" "path/filepath" "strings" "testing" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/version" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" ) func TestGetStaticPodDirectory(t *testing.T) { @@ -151,7 +153,7 @@ func TestEtcdSupportedVersion(t *testing.T) { { kubernetesVersion: "1.99.0", expectedVersion: nil, - expectedError: fmt.Errorf("Unsupported or unknown Kubernetes version(1.99.0)"), + expectedError: errors.New("Unsupported or unknown Kubernetes version(1.99.0)"), }, { kubernetesVersion: "1.10.0", @@ -202,15 +204,15 @@ func TestEtcdSupportedVersion(t *testing.T) { func TestGetKubeDNSVersion(t *testing.T) { var tests = []struct { - dns string + dns kubeadmapi.DNSAddOnType expected string }{ { - dns: KubeDNS, + dns: kubeadmapi.KubeDNS, expected: KubeDNSVersion, }, { - dns: CoreDNS, + dns: kubeadmapi.CoreDNS, expected: CoreDNSVersion, }, } diff --git a/cmd/kubeadm/app/discovery/BUILD b/cmd/kubeadm/app/discovery/BUILD index f6f53e40f8c..076af51aa1c 100644 --- a/cmd/kubeadm/app/discovery/BUILD +++ b/cmd/kubeadm/app/discovery/BUILD @@ -12,6 +12,7 @@ go_library( importpath = "k8s.io/kubernetes/cmd/kubeadm/app/discovery", deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/apis/kubeadm/v1beta1:go_default_library", "//cmd/kubeadm/app/discovery/file:go_default_library", "//cmd/kubeadm/app/discovery/https:go_default_library", "//cmd/kubeadm/app/discovery/token:go_default_library", diff --git a/cmd/kubeadm/app/discovery/discovery.go b/cmd/kubeadm/app/discovery/discovery.go index 7003d7a8bab..fb08e556a24 100644 --- a/cmd/kubeadm/app/discovery/discovery.go +++ b/cmd/kubeadm/app/discovery/discovery.go @@ -23,6 +23,7 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/discovery/file" "k8s.io/kubernetes/cmd/kubeadm/app/discovery/https" "k8s.io/kubernetes/cmd/kubeadm/app/discovery/token" @@ -48,7 +49,7 @@ func For(cfg *kubeadmapi.JoinConfiguration) (*clientcmdapi.Config, error) { clusterinfo := kubeconfigutil.GetClusterFromKubeConfig(config) return kubeconfigutil.CreateWithToken( clusterinfo.Server, - cfg.ClusterName, + kubeadmapiv1beta1.DefaultClusterName, TokenUser, clusterinfo.CertificateAuthorityData, cfg.Discovery.TLSBootstrapToken, @@ -61,9 +62,9 @@ func DiscoverValidatedKubeConfig(cfg *kubeadmapi.JoinConfiguration) (*clientcmda case cfg.Discovery.File != nil: kubeConfigPath := cfg.Discovery.File.KubeConfigPath if isHTTPSURL(kubeConfigPath) { - return https.RetrieveValidatedConfigInfo(kubeConfigPath, cfg.ClusterName) + return https.RetrieveValidatedConfigInfo(kubeConfigPath, kubeadmapiv1beta1.DefaultClusterName) } - return file.RetrieveValidatedConfigInfo(kubeConfigPath, cfg.ClusterName) + return file.RetrieveValidatedConfigInfo(kubeConfigPath, kubeadmapiv1beta1.DefaultClusterName) case cfg.Discovery.BootstrapToken != nil: return token.RetrieveValidatedConfigInfo(cfg) default: diff --git a/cmd/kubeadm/app/discovery/token/BUILD b/cmd/kubeadm/app/discovery/token/BUILD index 5b6008286cb..17fa88a3b93 100644 --- a/cmd/kubeadm/app/discovery/token/BUILD +++ b/cmd/kubeadm/app/discovery/token/BUILD @@ -12,6 +12,7 @@ go_library( importpath = "k8s.io/kubernetes/cmd/kubeadm/app/discovery/token", deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/apis/kubeadm/v1beta1:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//cmd/kubeadm/app/util/pubkeypin:go_default_library", diff --git a/cmd/kubeadm/app/discovery/token/token.go b/cmd/kubeadm/app/discovery/token/token.go index 9e874815b05..f3ca46830e2 100644 --- a/cmd/kubeadm/app/discovery/token/token.go +++ b/cmd/kubeadm/app/discovery/token/token.go @@ -33,6 +33,7 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" bootstrapapi "k8s.io/cluster-bootstrap/token/api" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin" @@ -62,7 +63,7 @@ func RetrieveValidatedConfigInfo(cfg *kubeadmapi.JoinConfiguration) (*clientcmda // The endpoint that wins the race and completes the task first gets its kubeconfig returned below baseKubeConfig, err := fetchKubeConfigWithTimeout(cfg.Discovery.BootstrapToken.APIServerEndpoint, cfg.Discovery.Timeout.Duration, func(endpoint string) (*clientcmdapi.Config, error) { - insecureBootstrapConfig := buildInsecureBootstrapKubeConfig(endpoint, cfg.ClusterName) + insecureBootstrapConfig := buildInsecureBootstrapKubeConfig(endpoint, kubeadmapiv1beta1.DefaultClusterName) clusterName := insecureBootstrapConfig.Contexts[insecureBootstrapConfig.CurrentContext].Cluster insecureClient, err := kubeconfigutil.ToClientSet(insecureBootstrapConfig) diff --git a/cmd/kubeadm/app/features/BUILD b/cmd/kubeadm/app/features/BUILD index 75a5e600954..e02ece2dd8b 100644 --- a/cmd/kubeadm/app/features/BUILD +++ b/cmd/kubeadm/app/features/BUILD @@ -13,6 +13,7 @@ go_library( deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", ], ) diff --git a/cmd/kubeadm/app/features/features.go b/cmd/kubeadm/app/features/features.go index 3cc810beb03..5f0eb350dd7 100644 --- a/cmd/kubeadm/app/features/features.go +++ b/cmd/kubeadm/app/features/features.go @@ -22,45 +22,26 @@ import ( "strconv" "strings" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" ) const ( - // HighAvailability is alpha in v1.9 - deprecated in v1.12 (TODO remove in v1.13) - HighAvailability = "HighAvailability" // CoreDNS is GA in v1.11 CoreDNS = "CoreDNS" - // SelfHosting is alpha in v1.8 and v1.9 - deprecated in v1.12 (TODO remove in v1.13) - SelfHosting = "SelfHosting" - - // StoreCertsInSecrets is alpha in v1.8 and v1.9 - deprecated in v1.12 (TODO remove in v1.13) - StoreCertsInSecrets = "StoreCertsInSecrets" - // DynamicKubeletConfig is beta in v1.11 DynamicKubeletConfig = "DynamicKubeletConfig" - - // Auditing is beta in 1.8 - Auditing = "Auditing" ) -var selfHostingDeprecationMessage = "featureGates:SelfHosting has been removed in v1.12" - -var storeCertsInSecretsDeprecationMessage = "featureGates:StoreCertsInSecrets has been removed in v1.12" - -var highAvailabilityMessage = "featureGates:HighAvailability has been removed in v1.12\n" + - "\tThis feature has been replaced by the kubeadm join --control-plane workflow." +var coreDNSMessage = "featureGates:CoreDNS has been removed in v1.13\n" + + "\tUse kubeadm-config to select which DNS addon to install." // InitFeatureGates are the default feature gates for the init command var InitFeatureGates = FeatureList{ - SelfHosting: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Deprecated}, HiddenInHelpText: true, DeprecationMessage: selfHostingDeprecationMessage}, - StoreCertsInSecrets: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Deprecated}, HiddenInHelpText: true, DeprecationMessage: storeCertsInSecretsDeprecationMessage}, - HighAvailability: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Deprecated}, HiddenInHelpText: true, DeprecationMessage: highAvailabilityMessage}, - CoreDNS: {FeatureSpec: utilfeature.FeatureSpec{Default: true, PreRelease: utilfeature.GA}}, - DynamicKubeletConfig: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}}, - Auditing: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}}, + CoreDNS: {FeatureSpec: utilfeature.FeatureSpec{Default: true, PreRelease: utilfeature.Deprecated}, HiddenInHelpText: true, DeprecationMessage: coreDNSMessage}, } // Feature represents a feature being gated @@ -81,12 +62,12 @@ func ValidateVersion(allFeatures FeatureList, requestedFeatures map[string]bool, } parsedExpVersion, err := version.ParseSemantic(requestedVersion) if err != nil { - return fmt.Errorf("Error parsing version %s: %v", requestedVersion, err) + return errors.Wrapf(err, "error parsing version %s", requestedVersion) } for k := range requestedFeatures { if minVersion := allFeatures[k].MinimumVersion; minVersion != nil { if !parsedExpVersion.AtLeast(minVersion) { - return fmt.Errorf( + return errors.Errorf( "the requested Kubernetes version (%s) is incompatible with the %s feature gate, which needs %s as a minimum", requestedVersion, k, minVersion) } @@ -106,9 +87,9 @@ func Enabled(featureList map[string]bool, featureName string) bool { // Supports indicates whether a feature name is supported on the given // feature set func Supports(featureList FeatureList, featureName string) bool { - for k := range featureList { + for k, v := range featureList { if featureName == string(k) { - return true + return v.PreRelease != utilfeature.Deprecated } } return false @@ -152,7 +133,7 @@ func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) { arr := strings.SplitN(s, "=", 2) if len(arr) != 2 { - return nil, fmt.Errorf("missing bool value for feature-gate key:%s", s) + return nil, errors.Errorf("missing bool value for feature-gate key:%s", s) } k := strings.TrimSpace(arr[0]) @@ -160,22 +141,20 @@ func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) { featureSpec, ok := (*f)[k] if !ok { - return nil, fmt.Errorf("unrecognized feature-gate key: %s", k) + return nil, errors.Errorf("unrecognized feature-gate key: %s", k) } if featureSpec.PreRelease == utilfeature.Deprecated { - return nil, fmt.Errorf("feature-gate key is deprecated: %s", k) + return nil, errors.Errorf("feature-gate key is deprecated: %s", k) } boolValue, err := strconv.ParseBool(v) if err != nil { - return nil, fmt.Errorf("invalid value %v for feature-gate key: %s, use true|false instead", v, k) + return nil, errors.Errorf("invalid value %v for feature-gate key: %s, use true|false instead", v, k) } featureGate[k] = boolValue } - ResolveFeatureGateDependencies(featureGate) - return featureGate, nil } @@ -201,18 +180,3 @@ func CheckDeprecatedFlags(f *FeatureList, features map[string]bool) map[string]s return deprecatedMsg } - -// ResolveFeatureGateDependencies resolve dependencies between feature gates -func ResolveFeatureGateDependencies(featureGate map[string]bool) { - - // if HighAvailability enabled and StoreCertsInSecrets disabled, both StoreCertsInSecrets - // and SelfHosting should enabled - if Enabled(featureGate, HighAvailability) && !Enabled(featureGate, StoreCertsInSecrets) { - featureGate[StoreCertsInSecrets] = true - } - - // if StoreCertsInSecrets enabled, SelfHosting should enabled - if Enabled(featureGate, StoreCertsInSecrets) { - featureGate[SelfHosting] = true - } -} diff --git a/cmd/kubeadm/app/features/features_test.go b/cmd/kubeadm/app/features/features_test.go index a1b24dd23a3..c8af0e6d05f 100644 --- a/cmd/kubeadm/app/features/features_test.go +++ b/cmd/kubeadm/app/features/features_test.go @@ -166,39 +166,6 @@ func TestValidateVersion(t *testing.T) { } } -func TestResolveFeatureGateDependencies(t *testing.T) { - - var tests = []struct { - inputFeatures map[string]bool - expectedFeatures map[string]bool - }{ - { // no flags - inputFeatures: map[string]bool{}, - expectedFeatures: map[string]bool{}, - }, - { // others flags - inputFeatures: map[string]bool{CoreDNS: false}, - expectedFeatures: map[string]bool{CoreDNS: false}, - }, - { // just StoreCertsInSecrets flags - inputFeatures: map[string]bool{StoreCertsInSecrets: true}, - expectedFeatures: map[string]bool{StoreCertsInSecrets: true, SelfHosting: true}, - }, - { // just HighAvailability flags - inputFeatures: map[string]bool{HighAvailability: true}, - expectedFeatures: map[string]bool{HighAvailability: true, StoreCertsInSecrets: true, SelfHosting: true}, - }, - } - - for _, test := range tests { - ResolveFeatureGateDependencies(test.inputFeatures) - if !reflect.DeepEqual(test.inputFeatures, test.expectedFeatures) { - t.Errorf("ResolveFeatureGateDependencies failed, expected: %v, got: %v", test.inputFeatures, test.expectedFeatures) - - } - } -} - // TestEnabledDefaults tests that Enabled returns the default values for // each feature gate when no feature gates are specified. func TestEnabledDefaults(t *testing.T) { diff --git a/cmd/kubeadm/app/images/BUILD b/cmd/kubeadm/app/images/BUILD index a7ef57fb287..cc9aeb381ce 100644 --- a/cmd/kubeadm/app/images/BUILD +++ b/cmd/kubeadm/app/images/BUILD @@ -13,7 +13,6 @@ go_library( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/util:go_default_library", ], ) diff --git a/cmd/kubeadm/app/images/images.go b/cmd/kubeadm/app/images/images.go index 3d520cb3cee..c988fe0f280 100644 --- a/cmd/kubeadm/app/images/images.go +++ b/cmd/kubeadm/app/images/images.go @@ -21,7 +21,6 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) @@ -30,39 +29,78 @@ func GetGenericImage(prefix, image, tag string) string { return fmt.Sprintf("%s/%s:%s", prefix, image, tag) } -// GetKubeControlPlaneImage generates and returns the image for the core Kubernetes components or returns the unified control plane image if specified -func GetKubeControlPlaneImage(image string, cfg *kubeadmapi.ClusterConfiguration) string { - if cfg.UnifiedControlPlaneImage != "" { - return cfg.UnifiedControlPlaneImage +// GetKubernetesImage generates and returns the image for the components managed in the Kubernetes main repository, +// including the control-plane components ad kube-proxy. If specified, the HyperKube image will be used. +func GetKubernetesImage(image string, cfg *kubeadmapi.ClusterConfiguration) string { + if cfg.UseHyperKubeImage { + image = constants.HyperKube } repoPrefix := cfg.GetControlPlaneImageRepository() kubernetesImageTag := kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion) return GetGenericImage(repoPrefix, image, kubernetesImageTag) } -// GetEtcdImage generates and returns the image for etcd or returns cfg.Etcd.Local.Image if specified +// GetDNSImage generates and returns the image for the DNS, that can be CoreDNS or kube-dns. +// Given that kube-dns uses 3 containers, an additional imageName parameter was added +func GetDNSImage(cfg *kubeadmapi.ClusterConfiguration, imageName string) string { + // DNS uses default image repository by default + dnsImageRepository := cfg.ImageRepository + // unless an override is specified + if cfg.DNS.ImageRepository != "" { + dnsImageRepository = cfg.DNS.ImageRepository + } + // DNS uses an imageTag that corresponds to the DNS version matching the Kubernetes version + dnsImageTag := constants.GetDNSVersion(cfg.DNS.Type) + + // unless an override is specified + if cfg.DNS.ImageTag != "" { + dnsImageTag = cfg.DNS.ImageTag + } + return GetGenericImage(dnsImageRepository, imageName, dnsImageTag) +} + +// GetEtcdImage generates and returns the image for etcd func GetEtcdImage(cfg *kubeadmapi.ClusterConfiguration) string { - if cfg.Etcd.Local != nil && cfg.Etcd.Local.Image != "" { - return cfg.Etcd.Local.Image + // Etcd uses default image repository by default + etcdImageRepository := cfg.ImageRepository + // unless an override is specified + if cfg.Etcd.Local != nil && cfg.Etcd.Local.ImageRepository != "" { + etcdImageRepository = cfg.Etcd.Local.ImageRepository } + // Etcd uses an imageTag that corresponds to the etcd version matching the Kubernetes version etcdImageTag := constants.DefaultEtcdVersion - etcdImageVersion, err := constants.EtcdSupportedVersion(cfg.KubernetesVersion) + etcdVersion, err := constants.EtcdSupportedVersion(cfg.KubernetesVersion) if err == nil { - etcdImageTag = etcdImageVersion.String() + etcdImageTag = etcdVersion.String() } - return GetGenericImage(cfg.ImageRepository, constants.Etcd, etcdImageTag) + // unless an override is specified + if cfg.Etcd.Local != nil && cfg.Etcd.Local.ImageTag != "" { + etcdImageTag = cfg.Etcd.Local.ImageTag + } + return GetGenericImage(etcdImageRepository, constants.Etcd, etcdImageTag) +} + +// GetPauseImage returns the image for the "pause" container +func GetPauseImage(cfg *kubeadmapi.ClusterConfiguration) string { + return GetGenericImage(cfg.ImageRepository, "pause", constants.PauseVersion) } // GetAllImages returns a list of container images kubeadm expects to use on a control plane node func GetAllImages(cfg *kubeadmapi.ClusterConfiguration) []string { imgs := []string{} - imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeAPIServer, cfg)) - imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeControllerManager, cfg)) - imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeScheduler, cfg)) - imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeProxy, cfg)) - // pause, etcd and kube-dns are not available on the ci image repository so use the default image repository. - imgs = append(imgs, GetGenericImage(cfg.ImageRepository, "pause", constants.PauseVersion)) + // start with core kubernetes images + if cfg.UseHyperKubeImage { + imgs = append(imgs, GetKubernetesImage(constants.HyperKube, cfg)) + } else { + imgs = append(imgs, GetKubernetesImage(constants.KubeAPIServer, cfg)) + imgs = append(imgs, GetKubernetesImage(constants.KubeControllerManager, cfg)) + imgs = append(imgs, GetKubernetesImage(constants.KubeScheduler, cfg)) + imgs = append(imgs, GetKubernetesImage(constants.KubeProxy, cfg)) + } + + // pause is not available on the ci image repository so use the default image repository. + imgs = append(imgs, GetPauseImage(cfg)) // if etcd is not external then add the image as it will be required if cfg.Etcd.Local != nil { @@ -70,12 +108,12 @@ func GetAllImages(cfg *kubeadmapi.ClusterConfiguration) []string { } // Append the appropriate DNS images - if features.Enabled(cfg.FeatureGates, features.CoreDNS) { - imgs = append(imgs, GetGenericImage(cfg.ImageRepository, constants.CoreDNS, constants.CoreDNSVersion)) + if cfg.DNS.Type == kubeadmapi.CoreDNS { + imgs = append(imgs, GetDNSImage(cfg, constants.CoreDNSImageName)) } else { - imgs = append(imgs, GetGenericImage(cfg.ImageRepository, "k8s-dns-kube-dns", constants.KubeDNSVersion)) - imgs = append(imgs, GetGenericImage(cfg.ImageRepository, "k8s-dns-sidecar", constants.KubeDNSVersion)) - imgs = append(imgs, GetGenericImage(cfg.ImageRepository, "k8s-dns-dnsmasq-nanny", constants.KubeDNSVersion)) + imgs = append(imgs, GetDNSImage(cfg, constants.KubeDNSKubeDNSImageName)) + imgs = append(imgs, GetDNSImage(cfg, constants.KubeDNSSidecarImageName)) + imgs = append(imgs, GetDNSImage(cfg, constants.KubeDNSDnsMasqNannyImageName)) } return imgs diff --git a/cmd/kubeadm/app/images/images_test.go b/cmd/kubeadm/app/images/images_test.go index a7bba7477f5..d72ff458543 100644 --- a/cmd/kubeadm/app/images/images_test.go +++ b/cmd/kubeadm/app/images/images_test.go @@ -44,16 +44,18 @@ func TestGetGenericImage(t *testing.T) { } } -func TestGetKubeControlPlaneImage(t *testing.T) { +func TestGetKubernetesImage(t *testing.T) { var tests = []struct { image string expected string cfg *kubeadmapi.ClusterConfiguration }{ { - expected: "override", + expected: GetGenericImage(gcrPrefix, constants.HyperKube, expected), cfg: &kubeadmapi.ClusterConfiguration{ - UnifiedControlPlaneImage: "override", + ImageRepository: gcrPrefix, + KubernetesVersion: testversion, + UseHyperKubeImage: true, }, }, { @@ -82,10 +84,10 @@ func TestGetKubeControlPlaneImage(t *testing.T) { }, } for _, rt := range tests { - actual := GetKubeControlPlaneImage(rt.image, rt.cfg) + actual := GetKubernetesImage(rt.image, rt.cfg) if actual != rt.expected { t.Errorf( - "failed GetKubeControlPlaneImage:\n\texpected: %s\n\t actual: %s", + "failed GetKubernetesImage:\n\texpected: %s\n\t actual: %s", rt.expected, actual, ) @@ -99,14 +101,42 @@ func TestGetEtcdImage(t *testing.T) { cfg *kubeadmapi.ClusterConfiguration }{ { - expected: "override", cfg: &kubeadmapi.ClusterConfiguration{ + ImageRepository: "real.repo", + KubernetesVersion: "1.12.0", + Etcd: kubeadmapi.Etcd{ + Local: &kubeadmapi.LocalEtcd{}, + }, + }, + expected: "real.repo/etcd:3.2.24", + }, + { + cfg: &kubeadmapi.ClusterConfiguration{ + ImageRepository: "real.repo", + KubernetesVersion: "1.12.0", Etcd: kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{ - Image: "override", + ImageMeta: kubeadmapi.ImageMeta{ + ImageTag: "override", + }, }, }, }, + expected: "real.repo/etcd:override", + }, + { + cfg: &kubeadmapi.ClusterConfiguration{ + ImageRepository: "real.repo", + KubernetesVersion: "1.12.0", + Etcd: kubeadmapi.Etcd{ + Local: &kubeadmapi.LocalEtcd{ + ImageMeta: kubeadmapi.ImageMeta{ + ImageRepository: "override", + }, + }, + }, + }, + expected: "override/etcd:3.2.24", }, { expected: GetGenericImage(gcrPrefix, "etcd", constants.DefaultEtcdVersion), @@ -128,6 +158,34 @@ func TestGetEtcdImage(t *testing.T) { } } +func TestGetPauseImage(t *testing.T) { + testcases := []struct { + name string + cfg *kubeadmapi.ClusterConfiguration + expected string + }{ + { + name: "pause image defined", + cfg: &kubeadmapi.ClusterConfiguration{ + ImageRepository: "test.repo", + }, + expected: "test.repo/pause:" + constants.PauseVersion, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + actual := GetPauseImage(tc.cfg) + if actual != tc.expected { + t.Fatalf( + "failed GetPauseImage:\n\texpected: %s\n\t actual: %s", + tc.expected, + actual, + ) + } + }) + } +} + func TestGetAllImages(t *testing.T) { testcases := []struct { name string @@ -160,38 +218,38 @@ func TestGetAllImages(t *testing.T) { { name: "CoreDNS image is returned", cfg: &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{ - "CoreDNS": true, + DNS: kubeadmapi.DNS{ + Type: kubeadmapi.CoreDNS, }, }, - expect: constants.CoreDNS, + expect: constants.CoreDNSImageName, }, { name: "main kube-dns image is returned", cfg: &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{ - "CoreDNS": false, + DNS: kubeadmapi.DNS{ + Type: kubeadmapi.KubeDNS, }, }, - expect: "k8s-dns-kube-dns", + expect: constants.KubeDNSKubeDNSImageName, }, { name: "kube-dns sidecar image is returned", cfg: &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{ - "CoreDNS": false, + DNS: kubeadmapi.DNS{ + Type: kubeadmapi.KubeDNS, }, }, - expect: "k8s-dns-sidecar", + expect: constants.KubeDNSSidecarImageName, }, { name: "kube-dns dnsmasq-nanny image is returned", cfg: &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{ - "CoreDNS": false, + DNS: kubeadmapi.DNS{ + Type: kubeadmapi.KubeDNS, }, }, - expect: "k8s-dns-dnsmasq-nanny", + expect: constants.KubeDNSDnsMasqNannyImageName, }, } for _, tc := range testcases { diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index 01cd3a968a7..030abca5d94 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -20,8 +20,8 @@ import ( "flag" "os" - _ "github.com/golang/glog" "github.com/spf13/pflag" + _ "k8s.io/klog" utilflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/kubernetes/cmd/kubeadm/app/cmd" diff --git a/cmd/kubeadm/app/phases/addons/dns/BUILD b/cmd/kubeadm/app/phases/addons/dns/BUILD index adfebc4886b..aea83df8f4b 100644 --- a/cmd/kubeadm/app/phases/addons/dns/BUILD +++ b/cmd/kubeadm/app/phases/addons/dns/BUILD @@ -33,7 +33,7 @@ go_library( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", + "//cmd/kubeadm/app/images:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", diff --git a/cmd/kubeadm/app/phases/addons/dns/dns.go b/cmd/kubeadm/app/phases/addons/dns/dns.go index 6d238709c2c..dc9e604ba79 100644 --- a/cmd/kubeadm/app/phases/addons/dns/dns.go +++ b/cmd/kubeadm/app/phases/addons/dns/dns.go @@ -23,7 +23,6 @@ import ( "github.com/mholt/caddy/caddyfile" "github.com/pkg/errors" - apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" @@ -34,7 +33,7 @@ import ( clientsetscheme "k8s.io/client-go/kubernetes/scheme" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" + "k8s.io/kubernetes/cmd/kubeadm/app/images" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" ) @@ -48,7 +47,7 @@ const ( ) // DeployedDNSAddon returns the type of DNS addon currently deployed -func DeployedDNSAddon(client clientset.Interface) (string, string, error) { +func DeployedDNSAddon(client clientset.Interface) (kubeadmapi.DNSAddOnType, string, error) { deploymentsClient := client.AppsV1().Deployments(metav1.NamespaceSystem) deployments, err := deploymentsClient.List(metav1.ListOptions{LabelSelector: "k8s-app=kube-dns"}) if err != nil { @@ -60,10 +59,14 @@ func DeployedDNSAddon(client clientset.Interface) (string, string, error) { return "", "", nil case 1: addonName := deployments.Items[0].Name + addonType := kubeadmapi.CoreDNS + if addonName == kubeadmconstants.KubeDNSDeploymentName { + addonType = kubeadmapi.KubeDNS + } addonImage := deployments.Items[0].Spec.Template.Spec.Containers[0].Image addonImageParts := strings.Split(addonImage, ":") addonVersion := addonImageParts[len(addonImageParts)-1] - return addonName, addonVersion, nil + return addonType, addonVersion, nil default: return "", "", errors.Errorf("multiple DNS addon deployments found: %v", deployments.Items) } @@ -71,7 +74,7 @@ func DeployedDNSAddon(client clientset.Interface) (string, string, error) { // EnsureDNSAddon creates the kube-dns or CoreDNS addon func EnsureDNSAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error { - if features.Enabled(cfg.FeatureGates, features.CoreDNS) { + if cfg.DNS.Type == kubeadmapi.CoreDNS { return coreDNSAddon(cfg, client) } return kubeDNSAddon(cfg, client) @@ -97,13 +100,15 @@ func kubeDNSAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) } dnsDeploymentBytes, err := kubeadmutil.ParseTemplate(KubeDNSDeployment, - struct{ ImageRepository, Version, DNSBindAddr, DNSProbeAddr, DNSDomain, MasterTaintKey string }{ - ImageRepository: cfg.ImageRepository, - Version: kubeadmconstants.KubeDNSVersion, - DNSBindAddr: dnsBindAddr, - DNSProbeAddr: dnsProbeAddr, - DNSDomain: cfg.Networking.DNSDomain, - MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster, + struct{ DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, MasterTaintKey string }{ + DeploymentName: kubeadmconstants.KubeDNSDeploymentName, + KubeDNSImage: images.GetDNSImage(&cfg.ClusterConfiguration, kubeadmconstants.KubeDNSKubeDNSImageName), + DNSMasqImage: images.GetDNSImage(&cfg.ClusterConfiguration, kubeadmconstants.KubeDNSDnsMasqNannyImageName), + SidecarImage: images.GetDNSImage(&cfg.ClusterConfiguration, kubeadmconstants.KubeDNSSidecarImageName), + DNSBindAddr: dnsBindAddr, + DNSProbeAddr: dnsProbeAddr, + DNSDomain: cfg.Networking.DNSDomain, + MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster, }) if err != nil { return errors.Wrap(err, "error when parsing kube-dns deployment template") @@ -151,17 +156,17 @@ func createKubeDNSAddon(deploymentBytes, serviceBytes []byte, client clientset.I func coreDNSAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error { // Get the YAML manifest - coreDNSDeploymentBytes, err := kubeadmutil.ParseTemplate(CoreDNSDeployment, struct{ ImageRepository, MasterTaintKey, Version string }{ - ImageRepository: cfg.ImageRepository, - MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster, - Version: kubeadmconstants.CoreDNSVersion, + coreDNSDeploymentBytes, err := kubeadmutil.ParseTemplate(CoreDNSDeployment, struct{ DeploymentName, Image, MasterTaintKey string }{ + DeploymentName: kubeadmconstants.CoreDNSDeploymentName, + Image: images.GetDNSImage(&cfg.ClusterConfiguration, kubeadmconstants.CoreDNSImageName), + MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster, }) if err != nil { return errors.Wrap(err, "error when parsing CoreDNS deployment template") } // Get the kube-dns ConfigMap for translation to equivalent CoreDNS Config. - kubeDNSConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeDNS, metav1.GetOptions{}) + kubeDNSConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeDNSConfigMap, metav1.GetOptions{}) if err != nil && !apierrors.IsNotFound(err) { return err } @@ -219,7 +224,7 @@ func createCoreDNSAddon(deploymentBytes, serviceBytes, configBytes []byte, clien } // Create the ConfigMap for CoreDNS or retain it in case it already exists - if err := apiclient.CreateOrRetainConfigMap(client, coreDNSConfigMap, kubeadmconstants.CoreDNS); err != nil { + if err := apiclient.CreateOrRetainConfigMap(client, coreDNSConfigMap, kubeadmconstants.CoreDNSConfigMap); err != nil { return err } diff --git a/cmd/kubeadm/app/phases/addons/dns/dns_test.go b/cmd/kubeadm/app/phases/addons/dns/dns_test.go index bcc637f0ee3..907f6594f7f 100644 --- a/cmd/kubeadm/app/phases/addons/dns/dns_test.go +++ b/cmd/kubeadm/app/phases/addons/dns/dns_test.go @@ -95,13 +95,15 @@ func TestCompileManifests(t *testing.T) { }{ { manifest: KubeDNSDeployment, - data: struct{ ImageRepository, Version, DNSBindAddr, DNSProbeAddr, DNSDomain, MasterTaintKey string }{ - ImageRepository: "foo", - Version: "foo", - DNSBindAddr: "foo", - DNSProbeAddr: "foo", - DNSDomain: "foo", - MasterTaintKey: "foo", + data: struct{ DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, MasterTaintKey string }{ + DeploymentName: "foo", + KubeDNSImage: "foo", + DNSMasqImage: "foo", + SidecarImage: "foo", + DNSBindAddr: "foo", + DNSProbeAddr: "foo", + DNSDomain: "foo", + MasterTaintKey: "foo", }, expected: true, }, @@ -114,10 +116,10 @@ func TestCompileManifests(t *testing.T) { }, { manifest: CoreDNSDeployment, - data: struct{ ImageRepository, MasterTaintKey, Version string }{ - ImageRepository: "foo", - MasterTaintKey: "foo", - Version: "foo", + data: struct{ DeploymentName, Image, MasterTaintKey string }{ + DeploymentName: "foo", + Image: "foo", + MasterTaintKey: "foo", }, expected: true, }, diff --git a/cmd/kubeadm/app/phases/addons/dns/manifests.go b/cmd/kubeadm/app/phases/addons/dns/manifests.go index cdf84c2e713..1016061cb4c 100644 --- a/cmd/kubeadm/app/phases/addons/dns/manifests.go +++ b/cmd/kubeadm/app/phases/addons/dns/manifests.go @@ -22,7 +22,7 @@ const ( apiVersion: apps/v1 kind: Deployment metadata: - name: kube-dns + name: {{ .DeploymentName }} namespace: kube-system labels: k8s-app: kube-dns @@ -50,7 +50,7 @@ spec: optional: true containers: - name: kubedns - image: {{ .ImageRepository }}/k8s-dns-kube-dns:{{ .Version }} + image: {{ .KubeDNSImage }} imagePullPolicy: IfNotPresent resources: # TODO: Set memory limits when we've profiled the container for large @@ -102,7 +102,7 @@ spec: - name: kube-dns-config mountPath: /kube-dns-config - name: dnsmasq - image: {{ .ImageRepository }}/k8s-dns-dnsmasq-nanny:{{ .Version }} + image: {{ .DNSMasqImage }} imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -143,7 +143,7 @@ spec: - name: kube-dns-config mountPath: /etc/k8s/dns/dnsmasq-nanny - name: sidecar - image: {{ .ImageRepository }}/k8s-dns-sidecar:{{ .Version }} + image: {{ .SidecarImage }} imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -213,7 +213,7 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: coredns + name: {{ .DeploymentName }} namespace: kube-system labels: k8s-app: kube-dns @@ -239,7 +239,7 @@ spec: effect: NoSchedule containers: - name: coredns - image: {{ .ImageRepository }}/coredns:{{ .Version }} + image: {{ .Image }} imagePullPolicy: IfNotPresent resources: limits: diff --git a/cmd/kubeadm/app/phases/addons/proxy/proxy.go b/cmd/kubeadm/app/phases/addons/proxy/proxy.go index 3d30f0cb0f9..17b92acedbc 100644 --- a/cmd/kubeadm/app/phases/addons/proxy/proxy.go +++ b/cmd/kubeadm/app/phases/addons/proxy/proxy.go @@ -21,7 +21,6 @@ import ( "fmt" "github.com/pkg/errors" - apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" @@ -82,7 +81,7 @@ func EnsureProxyAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interf return errors.Wrap(err, "error when parsing kube-proxy configmap template") } proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{ - Image: images.GetKubeControlPlaneImage(constants.KubeProxy, &cfg.ClusterConfiguration), + Image: images.GetKubernetesImage(constants.KubeProxy, &cfg.ClusterConfiguration), ProxyConfigMap: constants.KubeProxyConfigMap, ProxyConfigMapKey: constants.KubeProxyConfigMapKey, }) diff --git a/cmd/kubeadm/app/phases/addons/proxy/proxy_test.go b/cmd/kubeadm/app/phases/addons/proxy/proxy_test.go index 9d7ab37bcba..9bfe210d1a5 100644 --- a/cmd/kubeadm/app/phases/addons/proxy/proxy_test.go +++ b/cmd/kubeadm/app/phases/addons/proxy/proxy_test.go @@ -174,7 +174,7 @@ func TestEnsureProxyAddon(t *testing.T) { client := clientsetfake.NewSimpleClientset() // TODO: Consider using a YAML file instead for this that makes it possible to specify YAML documents for the ComponentConfigs masterConfig := &kubeadmapiv1beta1.InitConfiguration{ - APIEndpoint: kubeadmapiv1beta1.APIEndpoint{ + LocalAPIEndpoint: kubeadmapiv1beta1.APIEndpoint{ AdvertiseAddress: "1.2.3.4", BindPort: 1234, }, @@ -194,9 +194,9 @@ func TestEnsureProxyAddon(t *testing.T) { return true, nil, apierrors.NewUnauthorized("") }) case InvalidMasterEndpoint: - masterConfig.APIEndpoint.AdvertiseAddress = "1.2.3" + masterConfig.LocalAPIEndpoint.AdvertiseAddress = "1.2.3" case IPv6SetBindAddress: - masterConfig.APIEndpoint.AdvertiseAddress = "1:2::3:4" + masterConfig.LocalAPIEndpoint.AdvertiseAddress = "1:2::3:4" masterConfig.Networking.PodSubnet = "2001:101::/96" } diff --git a/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/BUILD b/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/BUILD index 8bd587953cc..41d801a6980 100644 --- a/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/BUILD +++ b/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/BUILD @@ -34,8 +34,8 @@ go_library( "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go b/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go index 5e46d346e2a..f421251960c 100644 --- a/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go +++ b/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go @@ -19,9 +19,7 @@ package clusterinfo import ( "fmt" - "github.com/golang/glog" "github.com/pkg/errors" - "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,6 +28,7 @@ import ( "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" bootstrapapi "k8s.io/cluster-bootstrap/token/api" + "k8s.io/klog" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" ) @@ -44,7 +43,7 @@ func CreateBootstrapConfigMapIfNotExists(client clientset.Interface, file string fmt.Printf("[bootstraptoken] creating the %q ConfigMap in the %q namespace\n", bootstrapapi.ConfigMapClusterInfo, metav1.NamespacePublic) - glog.V(1).Infoln("[bootstraptoken] loading admin kubeconfig") + klog.V(1).Infoln("[bootstraptoken] loading admin kubeconfig") adminConfig, err := clientcmd.LoadFromFile(file) if err != nil { return errors.Wrap(err, "failed to load admin kubeconfig") @@ -52,7 +51,7 @@ func CreateBootstrapConfigMapIfNotExists(client clientset.Interface, file string adminCluster := adminConfig.Contexts[adminConfig.CurrentContext].Cluster // Copy the cluster from admin.conf to the bootstrap kubeconfig, contains the CA cert and the server URL - glog.V(1).Infoln("[bootstraptoken] copying the cluster from admin.conf to the bootstrap kubeconfig") + klog.V(1).Infoln("[bootstraptoken] copying the cluster from admin.conf to the bootstrap kubeconfig") bootstrapConfig := &clientcmdapi.Config{ Clusters: map[string]*clientcmdapi.Cluster{ "": adminConfig.Clusters[adminCluster], @@ -64,7 +63,7 @@ func CreateBootstrapConfigMapIfNotExists(client clientset.Interface, file string } // Create or update the ConfigMap in the kube-public namespace - glog.V(1).Infoln("[bootstraptoken] creating/updating ConfigMap in kube-public namespace") + klog.V(1).Infoln("[bootstraptoken] creating/updating ConfigMap in kube-public namespace") return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: bootstrapapi.ConfigMapClusterInfo, @@ -78,7 +77,7 @@ func CreateBootstrapConfigMapIfNotExists(client clientset.Interface, file string // CreateClusterInfoRBACRules creates the RBAC rules for exposing the cluster-info ConfigMap in the kube-public namespace to unauthenticated users func CreateClusterInfoRBACRules(client clientset.Interface) error { - glog.V(1).Infoln("creating the RBAC rules for exposing the cluster-info ConfigMap in the kube-public namespace") + klog.V(1).Infoln("creating the RBAC rules for exposing the cluster-info ConfigMap in the kube-public namespace") err := apiclient.CreateOrUpdateRole(client, &rbac.Role{ ObjectMeta: metav1.ObjectMeta{ Name: BootstrapSignerClusterRoleName, diff --git a/cmd/kubeadm/app/phases/certs/BUILD b/cmd/kubeadm/app/phases/certs/BUILD index ee39c18a4e6..a138ced2e8a 100644 --- a/cmd/kubeadm/app/phases/certs/BUILD +++ b/cmd/kubeadm/app/phases/certs/BUILD @@ -21,6 +21,7 @@ go_test( "//cmd/kubeadm/test/certs:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", ], ) @@ -37,8 +38,8 @@ go_library( "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util/pkiutil:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/certs/certlist.go b/cmd/kubeadm/app/phases/certs/certlist.go index c6d3eb22a07..24b2342c9d2 100644 --- a/cmd/kubeadm/app/phases/certs/certlist.go +++ b/cmd/kubeadm/app/phases/certs/certlist.go @@ -236,7 +236,7 @@ var ( // KubeadmCertRootCA is the definition of the Kubernetes Root CA for the API Server and kubelet. KubeadmCertRootCA = KubeadmCert{ Name: "ca", - LongName: "self-signed Kubernetes CA to provision identities for other Kubernets components", + LongName: "self-signed Kubernetes CA to provision identities for other Kubernetes components", BaseName: kubeadmconstants.CACertAndKeyBaseName, config: certutil.Config{ CommonName: "kubernetes", diff --git a/cmd/kubeadm/app/phases/certs/certs.go b/cmd/kubeadm/app/phases/certs/certs.go index b25fea917ea..e89176686e7 100644 --- a/cmd/kubeadm/app/phases/certs/certs.go +++ b/cmd/kubeadm/app/phases/certs/certs.go @@ -23,9 +23,9 @@ import ( "os" "path/filepath" - "github.com/golang/glog" "github.com/pkg/errors" certutil "k8s.io/client-go/util/cert" + "k8s.io/klog" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" @@ -34,7 +34,7 @@ import ( // CreatePKIAssets will create and write to disk all PKI assets necessary to establish the control plane. // If the PKI assets already exists in the target folder, they are used only if evaluated equal; otherwise an error is returned. func CreatePKIAssets(cfg *kubeadmapi.InitConfiguration) error { - glog.V(1).Infoln("creating PKI assets") + klog.V(1).Infoln("creating PKI assets") // This structure cannot handle multilevel CA hierarchies. // This isn't a problem right now, but may become one in the future. @@ -69,7 +69,7 @@ func CreatePKIAssets(cfg *kubeadmapi.InitConfiguration) error { // CreateServiceAccountKeyAndPublicKeyFiles create a new public/private key files for signing service account users. // If the sa public/private key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned. func CreateServiceAccountKeyAndPublicKeyFiles(cfg *kubeadmapi.InitConfiguration) error { - glog.V(1).Infoln("creating a new public/private key files for signing service account users") + klog.V(1).Infoln("creating a new public/private key files for signing service account users") saSigningKey, err := NewServiceAccountSigningKey() if err != nil { return err @@ -110,7 +110,7 @@ func CreateCACertAndKeyFiles(certSpec *KubeadmCert, cfg *kubeadmapi.InitConfigur if certSpec.CAName != "" { return errors.Errorf("this function should only be used for CAs, but cert %s has CA %s", certSpec.Name, certSpec.CAName) } - glog.V(1).Infof("creating a new certificate authority for %s", certSpec.Name) + klog.V(1).Infof("creating a new certificate authority for %s", certSpec.Name) certConfig, err := certSpec.GetConfig(cfg) if err != nil { @@ -130,6 +130,25 @@ func CreateCACertAndKeyFiles(certSpec *KubeadmCert, cfg *kubeadmapi.InitConfigur ) } +// NewCSR will generate a new CSR and accompanying key +func NewCSR(certSpec *KubeadmCert, cfg *kubeadmapi.InitConfiguration) (*x509.CertificateRequest, *rsa.PrivateKey, error) { + certConfig, err := certSpec.GetConfig(cfg) + if err != nil { + return nil, nil, fmt.Errorf("failed to retrieve cert configuration: %v", err) + } + + return pkiutil.NewCSRAndKey(certConfig) +} + +// CreateCSR creates a certificate signing request +func CreateCSR(certSpec *KubeadmCert, cfg *kubeadmapi.InitConfiguration, path string) error { + csr, key, err := NewCSR(certSpec, cfg) + if err != nil { + return err + } + return writeCSRFilesIfNotExist(path, certSpec.BaseName, csr, key) +} + // CreateCertAndKeyFilesWithCA loads the given certificate authority from disk, then generates and writes out the given certificate and key. // The certSpec and caCertSpec should both be one of the variables from this package. func CreateCertAndKeyFilesWithCA(certSpec *KubeadmCert, caCertSpec *KubeadmCert, cfg *kubeadmapi.InitConfiguration) error { @@ -276,6 +295,33 @@ func writeKeyFilesIfNotExist(pkiDir string, baseName string, key *rsa.PrivateKey return nil } +// writeCertificateAuthorithyFilesIfNotExist write a new CSR to the given path. +// If there already is a CSR file at the given path; kubeadm tries to load it and check if it's a valid certificate. +// otherwise this function returns an error. +func writeCSRFilesIfNotExist(csrDir string, baseName string, csr *x509.CertificateRequest, key *rsa.PrivateKey) error { + if pkiutil.CSROrKeyExist(csrDir, baseName) { + _, _, err := pkiutil.TryLoadCSRAndKeyFromDisk(csrDir, baseName) + if err != nil { + return errors.Wrapf(err, "%s CSR existed but it could not be loaded properly", baseName) + } + + fmt.Printf("[certs] Using the existing %q CSR\n", baseName) + } else { + // Write .key and .csr files to disk + fmt.Printf("[certs] Generating %q key and CSR\n", baseName) + + if err := pkiutil.WriteKey(csrDir, baseName, key); err != nil { + return errors.Wrapf(err, "failure while saving %s key", baseName) + } + + if err := pkiutil.WriteCSR(csrDir, baseName, csr); err != nil { + return errors.Wrapf(err, "failure while saving %s CSR", baseName) + } + } + + return nil +} + type certKeyLocation struct { pkiDir string caBaseName string diff --git a/cmd/kubeadm/app/phases/certs/certs_test.go b/cmd/kubeadm/app/phases/certs/certs_test.go index 5412303202b..cd5561b0efe 100644 --- a/cmd/kubeadm/app/phases/certs/certs_test.go +++ b/cmd/kubeadm/app/phases/certs/certs_test.go @@ -18,13 +18,16 @@ package certs import ( "crypto/rsa" + "crypto/sha256" "crypto/x509" + "io/ioutil" "os" "path" "path/filepath" "testing" "github.com/pkg/errors" + "github.com/stretchr/testify/assert" certutil "k8s.io/client-go/util/cert" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -55,6 +58,18 @@ func createTestCert(t *testing.T, caCert *x509.Certificate, caKey *rsa.PrivateKe return cert, key } +func createTestCSR(t *testing.T) (*x509.CertificateRequest, *rsa.PrivateKey) { + csr, key, err := pkiutil.NewCSRAndKey( + &certutil.Config{ + CommonName: "testCert", + }) + if err != nil { + t.Fatalf("couldn't create test cert: %v", err) + } + + return csr, key +} + func TestWriteCertificateAuthorithyFilesIfNotExist(t *testing.T) { setupCert, setupKey := createCACert(t) caCert, caKey := createCACert(t) @@ -209,6 +224,75 @@ func TestWriteCertificateFilesIfNotExist(t *testing.T) { } } +func TestWriteCSRFilesIfNotExist(t *testing.T) { + csr, key := createTestCSR(t) + csr2, key2 := createTestCSR(t) + + var tests = []struct { + name string + setupFunc func(csrPath string) error + expectedError bool + expectedCSR *x509.CertificateRequest + }{ + { + name: "no files exist", + expectedCSR: csr, + }, + { + name: "other key exists", + setupFunc: func(csrPath string) error { + if err := pkiutil.WriteCSR(csrPath, "dummy", csr2); err != nil { + return err + } + return pkiutil.WriteKey(csrPath, "dummy", key2) + }, + expectedCSR: csr2, + }, + { + name: "existing CSR is garbage", + setupFunc: func(csrPath string) error { + return ioutil.WriteFile(path.Join(csrPath, "dummy.csr"), []byte("a--bunch--of-garbage"), os.ModePerm) + }, + expectedError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + if test.setupFunc != nil { + if err := test.setupFunc(tmpdir); err != nil { + t.Fatalf("couldn't set up test: %v", err) + } + } + + if err := writeCSRFilesIfNotExist(tmpdir, "dummy", csr, key); err != nil { + if test.expectedError { + return + } + t.Fatalf("unexpected error %v: ", err) + } + + if test.expectedError { + t.Fatal("Expected error, but got none") + } + + parsedCSR, _, err := pkiutil.TryLoadCSRAndKeyFromDisk(tmpdir, "dummy") + if err != nil { + t.Fatalf("couldn't load csr and key: %v", err) + } + + if sha256.Sum256(test.expectedCSR.Raw) != sha256.Sum256(parsedCSR.Raw) { + t.Error("expected csr's fingerprint does not match ") + } + + }) + } + +} + func TestWriteKeyFilesIfNotExist(t *testing.T) { setupKey, _ := NewServiceAccountSigningKey() @@ -506,7 +590,7 @@ func TestUsingExternalCA(t *testing.T) { defer os.RemoveAll(dir) cfg := &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, CertificatesDir: dir, @@ -606,6 +690,38 @@ func TestValidateMethods(t *testing.T) { } } +func TestNewCSR(t *testing.T) { + kubeadmCert := KubeadmCertAPIServer + cfg := testutil.GetDefaultInternalConfig(t) + + certConfig, err := kubeadmCert.GetConfig(cfg) + if err != nil { + t.Fatalf("couldn't get cert config: %v", err) + } + + csr, _, err := NewCSR(&kubeadmCert, cfg) + + if err != nil { + t.Errorf("invalid signature on CSR: %v", err) + } + + assert.ElementsMatch(t, certConfig.Organization, csr.Subject.Organization, "organizations not equal") + + if csr.Subject.CommonName != certConfig.CommonName { + t.Errorf("expected common name %q, got %q", certConfig.CommonName, csr.Subject.CommonName) + } + + assert.ElementsMatch(t, certConfig.AltNames.DNSNames, csr.DNSNames, "dns names not equal") + + assert.Len(t, csr.IPAddresses, len(certConfig.AltNames.IPs)) + + for i, ip := range csr.IPAddresses { + if !ip.Equal(certConfig.AltNames.IPs[i]) { + t.Errorf("[%d]: %v != %v", i, ip, certConfig.AltNames.IPs[i]) + } + } +} + type pkiFiles map[string]interface{} func writePKIFiles(t *testing.T, dir string, files pkiFiles) { @@ -675,7 +791,7 @@ func TestCreateCertificateFilesMethods(t *testing.T) { defer os.RemoveAll(tmpdir) cfg := &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Etcd: kubeadmapi.Etcd{Local: &kubeadmapi.LocalEtcd{}}, Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, diff --git a/cmd/kubeadm/app/phases/certs/renewal/BUILD b/cmd/kubeadm/app/phases/certs/renewal/BUILD index 623ad5aa52e..6172ca48370 100644 --- a/cmd/kubeadm/app/phases/certs/renewal/BUILD +++ b/cmd/kubeadm/app/phases/certs/renewal/BUILD @@ -14,11 +14,10 @@ go_library( "//cmd/kubeadm/app/util/pkiutil:go_default_library", "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", + "//staging/src/k8s.io/client-go/util/certificate/csr:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/certs/renewal/certsapi.go b/cmd/kubeadm/app/phases/certs/renewal/certsapi.go index 949e17f3175..6326ac5f0a9 100644 --- a/cmd/kubeadm/app/phases/certs/renewal/certsapi.go +++ b/cmd/kubeadm/app/phases/certs/renewal/certsapi.go @@ -17,30 +17,25 @@ limitations under the License. package renewal import ( - "crypto/rand" "crypto/rsa" "crypto/x509" "crypto/x509/pkix" + "fmt" "time" "github.com/pkg/errors" certsapi "k8s.io/api/certificates/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" certstype "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" certutil "k8s.io/client-go/util/cert" + csrutil "k8s.io/client-go/util/certificate/csr" ) -const ( - certAPIPrefixName = "kubeadm-cert" -) +const certAPIPrefixName = "kubeadm-cert" -var ( - watchTimeout = 5 * time.Minute -) +var watchTimeout = 5 * time.Minute // CertsAPIRenewal creates new certificates using the certs API type CertsAPIRenewal struct { @@ -70,7 +65,7 @@ func (r *CertsAPIRenewal) Renew(cfg *certutil.Config) (*x509.Certificate, *rsa.P return nil, nil, errors.Wrap(err, "couldn't create new private key") } - csr, err := x509.CreateCertificateRequest(rand.Reader, reqTmp, key) + csr, err := certutil.MakeCSRFromTemplate(key, reqTmp) if err != nil { return nil, nil, errors.Wrap(err, "couldn't create certificate signing request") } @@ -86,7 +81,7 @@ func (r *CertsAPIRenewal) Renew(cfg *certutil.Config) (*x509.Certificate, *rsa.P k8sCSR := &certsapi.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: certAPIPrefixName, + GenerateName: fmt.Sprintf("%s-%s-", certAPIPrefixName, cfg.CommonName), }, Spec: certsapi.CertificateSigningRequestSpec{ Request: csr, @@ -99,43 +94,23 @@ func (r *CertsAPIRenewal) Renew(cfg *certutil.Config) (*x509.Certificate, *rsa.P return nil, nil, errors.Wrap(err, "couldn't create certificate signing request") } - watcher, err := r.client.CertificateSigningRequests().Watch(metav1.ListOptions{ - Watch: true, - FieldSelector: fields.Set{"metadata.name": req.Name}.String(), - }) + fmt.Printf("[certs] certificate request %q created\n", req.Name) + + certData, err := csrutil.WaitForCertificate(r.client.CertificateSigningRequests(), req, watchTimeout) if err != nil { - return nil, nil, errors.Wrap(err, "couldn't watch for certificate creation") - } - defer watcher.Stop() - - select { - case ev := <-watcher.ResultChan(): - if ev.Type != watch.Modified { - return nil, nil, errors.Errorf("unexpected event received: %q", ev.Type) - } - case <-time.After(watchTimeout): - return nil, nil, errors.New("timeout trying to sign certificate") + return nil, nil, errors.Wrap(err, "certificate failed to appear") } - req, err = r.client.CertificateSigningRequests().Get(req.Name, metav1.GetOptions{}) - if err != nil { - return nil, nil, errors.Wrap(err, "couldn't get certificate signing request") - } - if len(req.Status.Conditions) < 1 { - return nil, nil, errors.New("certificate signing request has no statuses") - } - - // TODO: under what circumstances are there more than one? - if status := req.Status.Conditions[0].Type; status != certsapi.CertificateApproved { - return nil, nil, errors.Errorf("unexpected certificate status: %v", status) - } - - cert, err := x509.ParseCertificate(req.Status.Certificate) + cert, err := certutil.ParseCertsPEM(certData) if err != nil { return nil, nil, errors.Wrap(err, "couldn't parse issued certificate") } - return cert, key, nil + if len(cert) != 1 { + return nil, nil, errors.Errorf("certificate request %q has %d certificates, wanted exactly 1", req.Name, len(cert)) + } + + return cert[0], key, nil } var usageMap = map[x509.ExtKeyUsage]certsapi.KeyUsage{ diff --git a/cmd/kubeadm/app/phases/certs/renewal/renewal_test.go b/cmd/kubeadm/app/phases/certs/renewal/renewal_test.go index 327542a4ab9..6b24c6f0606 100644 --- a/cmd/kubeadm/app/phases/certs/renewal/renewal_test.go +++ b/cmd/kubeadm/app/phases/certs/renewal/renewal_test.go @@ -49,8 +49,12 @@ func TestRenewImplementations(t *testing.T) { Fake: &k8stesting.Fake{}, } certReq := getCertReq(t, caCert, caKey) + certReqNoCert := certReq.DeepCopy() + certReqNoCert.Status.Certificate = nil client.AddReactor("get", "certificatesigningrequests", defaultReactionFunc(certReq)) - watcher := watch.NewFakeWithChanSize(1, false) + watcher := watch.NewFakeWithChanSize(3, false) + watcher.Add(certReqNoCert) + watcher.Modify(certReqNoCert) watcher.Modify(certReq) client.AddWatchReactor("certificatesigningrequests", k8stesting.DefaultWatchReactor(watcher, nil)) @@ -132,7 +136,7 @@ func getCertReq(t *testing.T, caCert *x509.Certificate, caKey *rsa.PrivateKey) * Type: certsapi.CertificateApproved, }, }, - Certificate: cert.Raw, + Certificate: certutil.EncodeCertPEM(cert), }, } } diff --git a/cmd/kubeadm/app/phases/controlplane/BUILD b/cmd/kubeadm/app/phases/controlplane/BUILD index faa2809ce90..0833d489e45 100644 --- a/cmd/kubeadm/app/phases/controlplane/BUILD +++ b/cmd/kubeadm/app/phases/controlplane/BUILD @@ -16,14 +16,12 @@ go_test( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/test:go_default_library", "//pkg/kubeapiserver/authorizer/modes:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", ], ) @@ -36,9 +34,7 @@ go_library( importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane", deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta1:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/images:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/app/util:go_default_library", @@ -47,8 +43,8 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 8376fdcc5ad..0f618a8471f 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -24,15 +24,12 @@ import ( "strconv" "strings" - "github.com/golang/glog" "github.com/pkg/errors" - "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/version" + "k8s.io/klog" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/images" certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" @@ -42,7 +39,7 @@ import ( // CreateInitStaticPodManifestFiles will write all static pod manifest files needed to bring up the control plane. func CreateInitStaticPodManifestFiles(manifestDir string, cfg *kubeadmapi.InitConfiguration) error { - glog.V(1).Infoln("[control-plane] creating static Pod files") + klog.V(1).Infoln("[control-plane] creating static Pod files") return CreateStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler) } @@ -56,17 +53,17 @@ func GetStaticPodSpecs(cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Ve staticPodSpecs := map[string]v1.Pod{ kubeadmconstants.KubeAPIServer: staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.KubeAPIServer, - Image: images.GetKubeControlPlaneImage(kubeadmconstants.KubeAPIServer, &cfg.ClusterConfiguration), + Image: images.GetKubernetesImage(kubeadmconstants.KubeAPIServer, &cfg.ClusterConfiguration), ImagePullPolicy: v1.PullIfNotPresent, Command: getAPIServerCommand(cfg), VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)), - LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeAPIServer, int(cfg.APIEndpoint.BindPort), "/healthz", v1.URISchemeHTTPS), + LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeAPIServer, int(cfg.LocalAPIEndpoint.BindPort), "/healthz", v1.URISchemeHTTPS), Resources: staticpodutil.ComponentResources("250m"), Env: getProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer)), kubeadmconstants.KubeControllerManager: staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.KubeControllerManager, - Image: images.GetKubeControlPlaneImage(kubeadmconstants.KubeControllerManager, &cfg.ClusterConfiguration), + Image: images.GetKubernetesImage(kubeadmconstants.KubeControllerManager, &cfg.ClusterConfiguration), ImagePullPolicy: v1.PullIfNotPresent, Command: getControllerManagerCommand(cfg, k8sVersion), VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), @@ -76,7 +73,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Ve }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager)), kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.KubeScheduler, - Image: images.GetKubeControlPlaneImage(kubeadmconstants.KubeScheduler, &cfg.ClusterConfiguration), + Image: images.GetKubernetesImage(kubeadmconstants.KubeScheduler, &cfg.ClusterConfiguration), ImagePullPolicy: v1.PullIfNotPresent, Command: getSchedulerCommand(cfg), VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), @@ -97,7 +94,7 @@ func CreateStaticPodFiles(manifestDir string, cfg *kubeadmapi.InitConfiguration, } // gets the StaticPodSpecs, actualized for the current InitConfiguration - glog.V(1).Infoln("[control-plane] getting StaticPodSpecs") + klog.V(1).Infoln("[control-plane] getting StaticPodSpecs") specs := GetStaticPodSpecs(cfg, k8sVersion) // creates required static pod specs @@ -113,7 +110,7 @@ func CreateStaticPodFiles(manifestDir string, cfg *kubeadmapi.InitConfiguration, return errors.Wrapf(err, "failed to create static pod manifest file for %q", componentName) } - glog.V(1).Infof("[control-plane] wrote static Pod manifest for component %q to %q\n", componentName, kubeadmconstants.GetStaticPodFilepath(componentName, manifestDir)) + klog.V(1).Infof("[control-plane] wrote static Pod manifest for component %q to %q\n", componentName, kubeadmconstants.GetStaticPodFilepath(componentName, manifestDir)) } return nil @@ -122,7 +119,7 @@ func CreateStaticPodFiles(manifestDir string, cfg *kubeadmapi.InitConfiguration, // getAPIServerCommand builds the right API server command from the given config object and version func getAPIServerCommand(cfg *kubeadmapi.InitConfiguration) []string { defaultArguments := map[string]string{ - "advertise-address": cfg.APIEndpoint.AdvertiseAddress, + "advertise-address": cfg.LocalAPIEndpoint.AdvertiseAddress, "insecure-port": "0", "enable-admission-plugins": "NodeRestriction", "service-cluster-ip-range": cfg.Networking.ServiceSubnet, @@ -133,7 +130,7 @@ func getAPIServerCommand(cfg *kubeadmapi.InitConfiguration) []string { "kubelet-client-certificate": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientCertName), "kubelet-client-key": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName), "enable-bootstrap-token-auth": "true", - "secure-port": fmt.Sprintf("%d", cfg.APIEndpoint.BindPort), + "secure-port": fmt.Sprintf("%d", cfg.LocalAPIEndpoint.BindPort), "allow-privileged": "true", "kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", // add options to configure the front proxy. Without the generated client cert, this will never be useable @@ -170,29 +167,12 @@ func getAPIServerCommand(cfg *kubeadmapi.InitConfiguration) []string { // Apply user configurations for local etcd if cfg.Etcd.Local != nil { - if value, ok := cfg.Etcd.Local.ExtraArgs["listen-client-urls"]; ok { + if value, ok := cfg.Etcd.Local.ExtraArgs["advertise-client-urls"]; ok { defaultArguments["etcd-servers"] = value } } } - if features.Enabled(cfg.FeatureGates, features.HighAvailability) { - defaultArguments["endpoint-reconciler-type"] = kubeadmconstants.LeaseEndpointReconcilerType - } - - if features.Enabled(cfg.FeatureGates, features.DynamicKubeletConfig) { - defaultArguments["feature-gates"] = "DynamicKubeletConfig=true" - } - - if features.Enabled(cfg.FeatureGates, features.Auditing) { - defaultArguments["audit-policy-file"] = kubeadmconstants.GetStaticPodAuditPolicyFile() - defaultArguments["audit-log-path"] = filepath.Join(kubeadmconstants.StaticPodAuditPolicyLogDir, kubeadmconstants.AuditPolicyLogFile) - if cfg.AuditPolicyConfiguration.LogMaxAge == nil { - defaultArguments["audit-log-maxage"] = fmt.Sprintf("%d", kubeadmapiv1beta1.DefaultAuditPolicyLogMaxAge) - } else { - defaultArguments["audit-log-maxage"] = fmt.Sprintf("%d", *cfg.AuditPolicyConfiguration.LogMaxAge) - } - } if cfg.APIServer.ExtraArgs == nil { cfg.APIServer.ExtraArgs = map[string]string{} } diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index df4044a1628..87f2847d252 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -29,12 +29,10 @@ import ( "k8s.io/apimachinery/pkg/util/version" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" testutil "k8s.io/kubernetes/cmd/kubeadm/test" - utilpointer "k8s.io/utils/pointer" ) const ( @@ -146,7 +144,7 @@ func TestGetAPIServerCommand(t *testing.T) { { name: "testing defaults", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, CertificatesDir: testCertsDir, @@ -185,15 +183,10 @@ func TestGetAPIServerCommand(t *testing.T) { { name: "ignores the audit policy if the feature gate is not enabled", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, CertificatesDir: testCertsDir, - AuditPolicyConfiguration: kubeadmapi.AuditPolicyConfiguration{ - Path: "/foo/bar", - LogDir: "/foo/baz", - LogMaxAge: utilpointer.Int32Ptr(10), - }, }, }, expected: []string{ @@ -229,7 +222,7 @@ func TestGetAPIServerCommand(t *testing.T) { { name: "ipv6 advertise address", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, CertificatesDir: testCertsDir, @@ -268,10 +261,9 @@ func TestGetAPIServerCommand(t *testing.T) { { name: "an external etcd with custom ca, certs and keys", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, - FeatureGates: map[string]bool{features.HighAvailability: true}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, Etcd: kubeadmapi.Etcd{ External: &kubeadmapi.ExternalEtcd{ Endpoints: []string{"https://8.6.4.1:2379", "https://8.6.4.2:2379"}, @@ -311,13 +303,12 @@ func TestGetAPIServerCommand(t *testing.T) { "--etcd-cafile=fuz", "--etcd-certfile=fiz", "--etcd-keyfile=faz", - fmt.Sprintf("--endpoint-reconciler-type=%s", kubeadmconstants.LeaseEndpointReconcilerType), }, }, { name: "an insecure etcd", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, Etcd: kubeadmapi.Etcd{ @@ -355,102 +346,13 @@ func TestGetAPIServerCommand(t *testing.T) { "--etcd-servers=http://127.0.0.1:2379,http://127.0.0.1:2380", }, }, - { - name: "auditing and HA are enabled with a custom log max age of 0", - cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, - FeatureGates: map[string]bool{features.HighAvailability: true, features.Auditing: true}, - CertificatesDir: testCertsDir, - AuditPolicyConfiguration: kubeadmapi.AuditPolicyConfiguration{ - LogMaxAge: utilpointer.Int32Ptr(0), - }, - }, - }, - expected: []string{ - "kube-apiserver", - "--insecure-port=0", - "--enable-admission-plugins=NodeRestriction", - "--service-cluster-ip-range=bar", - "--service-account-key-file=" + testCertsDir + "/sa.pub", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--tls-cert-file=" + testCertsDir + "/apiserver.crt", - "--tls-private-key-file=" + testCertsDir + "/apiserver.key", - "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", - "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", - fmt.Sprintf("--secure-port=%d", 123), - "--allow-privileged=true", - "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", - "--enable-bootstrap-token-auth=true", - "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", - "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", - "--requestheader-username-headers=X-Remote-User", - "--requestheader-group-headers=X-Remote-Group", - "--requestheader-extra-headers-prefix=X-Remote-Extra-", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--requestheader-allowed-names=front-proxy-client", - "--authorization-mode=Node,RBAC", - "--advertise-address=2001:db8::1", - fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort), - "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt", - "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", - "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", - fmt.Sprintf("--endpoint-reconciler-type=%s", kubeadmconstants.LeaseEndpointReconcilerType), - "--audit-policy-file=/etc/kubernetes/audit/audit.yaml", - "--audit-log-path=/var/log/kubernetes/audit/audit.log", - "--audit-log-maxage=0", - }, - }, - { - name: "ensure the DynamicKubelet flag gets passed through", - cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, - CertificatesDir: testCertsDir, - FeatureGates: map[string]bool{features.DynamicKubeletConfig: true}, - }, - }, - expected: []string{ - "kube-apiserver", - "--insecure-port=0", - "--enable-admission-plugins=NodeRestriction", - "--service-cluster-ip-range=bar", - "--service-account-key-file=" + testCertsDir + "/sa.pub", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--tls-cert-file=" + testCertsDir + "/apiserver.crt", - "--tls-private-key-file=" + testCertsDir + "/apiserver.key", - "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", - "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", - "--enable-bootstrap-token-auth=true", - "--secure-port=123", - "--allow-privileged=true", - "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", - "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", - "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", - "--requestheader-username-headers=X-Remote-User", - "--requestheader-group-headers=X-Remote-Group", - "--requestheader-extra-headers-prefix=X-Remote-Extra-", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--requestheader-allowed-names=front-proxy-client", - "--authorization-mode=Node,RBAC", - "--advertise-address=1.2.3.4", - fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort), - "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt", - "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", - "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", - "--feature-gates=DynamicKubeletConfig=true", - }, - }, { name: "test APIServer.ExtraArgs works as expected", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, CertificatesDir: testCertsDir, - FeatureGates: map[string]bool{features.DynamicKubeletConfig: true, features.Auditing: true}, APIServer: kubeadmapi.APIServer{ ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{ ExtraArgs: map[string]string{ @@ -491,16 +393,14 @@ func TestGetAPIServerCommand(t *testing.T) { "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", - "--feature-gates=DynamicKubeletConfig=true", "--audit-policy-file=/etc/config/audit.yaml", "--audit-log-path=/var/log/kubernetes", - "--audit-log-maxage=2", }, }, { name: "authorization-mode extra-args ABAC", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, CertificatesDir: testCertsDir, @@ -546,7 +446,7 @@ func TestGetAPIServerCommand(t *testing.T) { { name: "insecure-port extra-args", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, CertificatesDir: testCertsDir, @@ -592,7 +492,7 @@ func TestGetAPIServerCommand(t *testing.T) { { name: "authorization-mode extra-args Webhook", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, CertificatesDir: testCertsDir, @@ -974,7 +874,7 @@ func TestGetControllerManagerCommandExternalCA(t *testing.T) { { name: "caKeyPresent-false for v1.12.0-beta.2", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ KubernetesVersion: "v1.12.0-beta.2", Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, @@ -1003,7 +903,7 @@ func TestGetControllerManagerCommandExternalCA(t *testing.T) { { name: "caKeyPresent true for v1.12.0-beta.2", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ KubernetesVersion: "v1.12.0-beta.2", Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, @@ -1032,7 +932,7 @@ func TestGetControllerManagerCommandExternalCA(t *testing.T) { { name: "caKeyPresent-false for v1.11.3", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ KubernetesVersion: "v1.11.3", Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, @@ -1057,7 +957,7 @@ func TestGetControllerManagerCommandExternalCA(t *testing.T) { { name: "caKeyPresent true for v1.11.3", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ KubernetesVersion: "v1.11.3", Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, diff --git a/cmd/kubeadm/app/phases/controlplane/volumes.go b/cmd/kubeadm/app/phases/controlplane/volumes.go index ea96a971f0b..7892be889b6 100644 --- a/cmd/kubeadm/app/phases/controlplane/volumes.go +++ b/cmd/kubeadm/app/phases/controlplane/volumes.go @@ -26,7 +26,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" ) @@ -46,7 +45,6 @@ var caCertsExtraVolumePaths = []string{"/etc/pki", "/usr/share/ca-certificates", func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.InitConfiguration) controlPlaneHostPathMounts { hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate hostPathFileOrCreate := v1.HostPathFileOrCreate - hostPathFile := v1.HostPathFile mounts := newControlPlaneHostPathMounts() // HostPath volumes for the API Server @@ -55,12 +53,7 @@ func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.InitConfiguration) con mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeCertificatesVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true, &hostPathDirectoryOrCreate) // Read-only mount for the ca certs (/etc/ssl/certs) directory mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true, &hostPathDirectoryOrCreate) - if features.Enabled(cfg.FeatureGates, features.Auditing) { - // Read-only mount for the audit policy file. - mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeAuditPolicyVolumeName, cfg.AuditPolicyConfiguration.Path, kubeadmconstants.GetStaticPodAuditPolicyFile(), true, &hostPathFile) - // Write mount for the audit logs. - mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeAuditPolicyLogVolumeName, cfg.AuditPolicyConfiguration.LogDir, kubeadmconstants.StaticPodAuditPolicyLogDir, false, &hostPathDirectoryOrCreate) - } + // If external etcd is specified, mount the directories needed for accessing the CA/serving certs and the private key if cfg.Etcd.External != nil { etcdVols, etcdVolMounts := getEtcdCertVolumes(cfg.Etcd.External, cfg.CertificatesDir) @@ -151,7 +144,7 @@ func (c *controlPlaneHostPathMounts) AddExtraHostPathMounts(component string, ex for _, extraVol := range extraVols { fmt.Printf("[controlplane] Adding extra host path mount %q to %q\n", extraVol.Name, component) hostPathType := extraVol.PathType - c.NewHostPathMount(component, extraVol.Name, extraVol.HostPath, extraVol.MountPath, !extraVol.Writable, &hostPathType) + c.NewHostPathMount(component, extraVol.Name, extraVol.HostPath, extraVol.MountPath, extraVol.ReadOnly, &hostPathType) } } diff --git a/cmd/kubeadm/app/phases/controlplane/volumes_test.go b/cmd/kubeadm/app/phases/controlplane/volumes_test.go index 2f3b7649dd9..341e479c971 100644 --- a/cmd/kubeadm/app/phases/controlplane/volumes_test.go +++ b/cmd/kubeadm/app/phases/controlplane/volumes_test.go @@ -26,7 +26,6 @@ import ( "k8s.io/api/core/v1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" ) func TestGetEtcdCertVolumes(t *testing.T) { @@ -259,7 +258,6 @@ func TestGetEtcdCertVolumes(t *testing.T) { func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate hostPathFileOrCreate := v1.HostPathFileOrCreate - hostPathFile := v1.HostPathFile volMap := make(map[string]map[string]v1.Volume) volMap[kubeadmconstants.KubeAPIServer] = map[string]v1.Volume{} volMap[kubeadmconstants.KubeAPIServer]["k8s-certs"] = v1.Volume{ @@ -280,24 +278,6 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { }, }, } - volMap[kubeadmconstants.KubeAPIServer]["audit"] = v1.Volume{ - Name: "audit", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/foo/bar/baz.yaml", - Type: &hostPathFile, - }, - }, - } - volMap[kubeadmconstants.KubeAPIServer]["audit-log"] = v1.Volume{ - Name: "audit-log", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/bar/foo", - Type: &hostPathDirectoryOrCreate, - }, - }, - } volMap[kubeadmconstants.KubeControllerManager] = map[string]v1.Volume{} volMap[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.Volume{ Name: "k8s-certs", @@ -348,16 +328,6 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { MountPath: "/etc/ssl/certs", ReadOnly: true, } - volMountMap[kubeadmconstants.KubeAPIServer]["audit"] = v1.VolumeMount{ - Name: "audit", - MountPath: "/etc/kubernetes/audit/audit.yaml", - ReadOnly: true, - } - volMountMap[kubeadmconstants.KubeAPIServer]["audit-log"] = v1.VolumeMount{ - Name: "audit-log", - MountPath: "/var/log/kubernetes/audit", - ReadOnly: false, - } volMountMap[kubeadmconstants.KubeControllerManager] = map[string]v1.VolumeMount{} volMountMap[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.VolumeMount{ Name: "k8s-certs", @@ -511,11 +481,6 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { cfg: &kubeadmapi.ClusterConfiguration{ CertificatesDir: testCertsDir, Etcd: kubeadmapi.Etcd{}, - FeatureGates: map[string]bool{features.Auditing: true}, - AuditPolicyConfiguration: kubeadmapi.AuditPolicyConfiguration{ - Path: "/foo/bar/baz.yaml", - LogDir: "/bar/foo", - }, }, vol: volMap, volMount: volMountMap, @@ -621,28 +586,28 @@ func TestAddExtraHostPathMounts(t *testing.T) { Name: "foo-0", HostPath: "/tmp/qux-0", MountPath: "/tmp/qux-0", - Writable: false, + ReadOnly: true, PathType: v1.HostPathFile, }, { Name: "bar-0", HostPath: "/tmp/asd-0", MountPath: "/tmp/asd-0", - Writable: true, + ReadOnly: false, PathType: v1.HostPathDirectory, }, { Name: "foo-1", HostPath: "/tmp/qux-1", MountPath: "/tmp/qux-1", - Writable: false, + ReadOnly: true, PathType: v1.HostPathFileOrCreate, }, { Name: "bar-1", HostPath: "/tmp/asd-1", MountPath: "/tmp/asd-1", - Writable: true, + ReadOnly: false, PathType: v1.HostPathDirectoryOrCreate, }, } @@ -672,8 +637,8 @@ func TestAddExtraHostPathMounts(t *testing.T) { if volMount.MountPath != hostMount.MountPath { t.Errorf("Expected container path %q", hostMount.MountPath) } - if volMount.ReadOnly != !hostMount.Writable { - t.Errorf("Expected volume writable setting %t", hostMount.Writable) + if volMount.ReadOnly != hostMount.ReadOnly { + t.Errorf("Expected volume readOnly setting %t", hostMount.ReadOnly) } } } diff --git a/cmd/kubeadm/app/phases/etcd/BUILD b/cmd/kubeadm/app/phases/etcd/BUILD index d3268aa6e1a..c8182e8cb74 100644 --- a/cmd/kubeadm/app/phases/etcd/BUILD +++ b/cmd/kubeadm/app/phases/etcd/BUILD @@ -31,8 +31,8 @@ go_library( "//cmd/kubeadm/app/util/staticpod:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index 79e66190230..afe36b53702 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -21,8 +21,8 @@ import ( "path/filepath" "strings" - "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/klog" "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" @@ -54,7 +54,7 @@ func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.In return err } - glog.V(1).Infof("[etcd] wrote Static Pod manifest for a local etcd instance to %q\n", kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestDir)) + klog.V(1).Infof("[etcd] wrote Static Pod manifest for a local etcd instance to %q\n", kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestDir)) return nil } @@ -63,7 +63,7 @@ func CheckLocalEtcdClusterStatus(client clientset.Interface, cfg *kubeadmapi.Ini fmt.Println("[etcd] Checking Etcd cluster health") // creates an etcd client that connects to all the local/stacked etcd members - glog.V(1).Info("creating etcd client that connects to etcd pods") + klog.V(1).Info("creating etcd client that connects to etcd pods") etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir) if err != nil { return err @@ -83,24 +83,24 @@ func CheckLocalEtcdClusterStatus(client clientset.Interface, cfg *kubeadmapi.Ini // Other members of the etcd cluster will be notified of the joining node in beforehand as well. func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir string, cfg *kubeadmapi.InitConfiguration) error { // creates an etcd client that connects to all the local/stacked etcd members - glog.V(1).Info("creating etcd client that connects to etcd pods") + klog.V(1).Info("creating etcd client that connects to etcd pods") etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir) if err != nil { return err } // notifies the other members of the etcd cluster about the joining member - etcdPeerAddress := fmt.Sprintf("https://%s:%d", cfg.APIEndpoint.AdvertiseAddress, kubeadmconstants.EtcdListenPeerPort) + etcdPeerAddress := fmt.Sprintf("https://%s:%d", cfg.LocalAPIEndpoint.AdvertiseAddress, kubeadmconstants.EtcdListenPeerPort) - glog.V(1).Infof("Adding etcd member: %s", etcdPeerAddress) + klog.V(1).Infof("Adding etcd member: %s", etcdPeerAddress) initialCluster, err := etcdClient.AddMember(cfg.NodeRegistration.Name, etcdPeerAddress) if err != nil { return err } fmt.Println("[etcd] Announced new etcd member joining to the existing etcd cluster") - glog.V(1).Infof("Updated etcd member list: %v", initialCluster) + klog.V(1).Infof("Updated etcd member list: %v", initialCluster) - glog.V(1).Info("Creating local etcd static pod manifest file") + klog.V(1).Info("Creating local etcd static pod manifest file") // gets etcd StaticPodSpec, actualized for the current InitConfiguration and the new list of etcd members spec := GetEtcdPodSpec(cfg, initialCluster) // writes etcd StaticPod to disk @@ -141,10 +141,10 @@ func GetEtcdPodSpec(cfg *kubeadmapi.InitConfiguration, initialCluster []etcdutil func getEtcdCommand(cfg *kubeadmapi.InitConfiguration, initialCluster []etcdutil.Member) []string { defaultArguments := map[string]string{ "name": cfg.GetNodeName(), - "listen-client-urls": fmt.Sprintf("https://127.0.0.1:%d,https://%s:%d", kubeadmconstants.EtcdListenClientPort, cfg.APIEndpoint.AdvertiseAddress, kubeadmconstants.EtcdListenClientPort), - "advertise-client-urls": fmt.Sprintf("https://%s:%d", cfg.APIEndpoint.AdvertiseAddress, kubeadmconstants.EtcdListenClientPort), - "listen-peer-urls": fmt.Sprintf("https://%s:%d", cfg.APIEndpoint.AdvertiseAddress, kubeadmconstants.EtcdListenPeerPort), - "initial-advertise-peer-urls": fmt.Sprintf("https://%s:%d", cfg.APIEndpoint.AdvertiseAddress, kubeadmconstants.EtcdListenPeerPort), + "listen-client-urls": fmt.Sprintf("https://127.0.0.1:%d,https://%s:%d", kubeadmconstants.EtcdListenClientPort, cfg.LocalAPIEndpoint.AdvertiseAddress, kubeadmconstants.EtcdListenClientPort), + "advertise-client-urls": fmt.Sprintf("https://%s:%d", cfg.LocalAPIEndpoint.AdvertiseAddress, kubeadmconstants.EtcdListenClientPort), + "listen-peer-urls": fmt.Sprintf("https://%s:%d", cfg.LocalAPIEndpoint.AdvertiseAddress, kubeadmconstants.EtcdListenPeerPort), + "initial-advertise-peer-urls": fmt.Sprintf("https://%s:%d", cfg.LocalAPIEndpoint.AdvertiseAddress, kubeadmconstants.EtcdListenPeerPort), "data-dir": cfg.Etcd.Local.DataDir, "cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName), "key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName), @@ -158,7 +158,7 @@ func getEtcdCommand(cfg *kubeadmapi.InitConfiguration, initialCluster []etcdutil } if len(initialCluster) == 0 { - defaultArguments["initial-cluster"] = fmt.Sprintf("%s=https://%s:%d", cfg.GetNodeName(), cfg.APIEndpoint.AdvertiseAddress, kubeadmconstants.EtcdListenPeerPort) + defaultArguments["initial-cluster"] = fmt.Sprintf("%s=https://%s:%d", cfg.GetNodeName(), cfg.LocalAPIEndpoint.AdvertiseAddress, kubeadmconstants.EtcdListenPeerPort) } else { // NB. the joining etcd instance should be part of the initialCluster list endpoints := []string{} diff --git a/cmd/kubeadm/app/phases/etcd/local_test.go b/cmd/kubeadm/app/phases/etcd/local_test.go index 53c791a2706..e08c4d583ab 100644 --- a/cmd/kubeadm/app/phases/etcd/local_test.go +++ b/cmd/kubeadm/app/phases/etcd/local_test.go @@ -38,7 +38,6 @@ func TestGetEtcdPodSpec(t *testing.T) { Etcd: kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{ DataDir: "/var/lib/etcd", - Image: "", }, }, }, @@ -69,7 +68,6 @@ func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) { Etcd: kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{ DataDir: "/var/lib/etcd", - Image: "k8s.gcr.io/etcd", }, }, }, @@ -124,7 +122,7 @@ func TestGetEtcdCommand(t *testing.T) { { name: "Default args - with empty etcd initial cluster", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ AdvertiseAddress: "1.2.3.4", }, NodeRegistration: kubeadmapi.NodeRegistrationOptions{ @@ -161,7 +159,7 @@ func TestGetEtcdCommand(t *testing.T) { { name: "Default args - With an existing etcd cluster", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ AdvertiseAddress: "1.2.3.4", }, NodeRegistration: kubeadmapi.NodeRegistrationOptions{ @@ -203,7 +201,7 @@ func TestGetEtcdCommand(t *testing.T) { { name: "Extra args", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ AdvertiseAddress: "1.2.3.4", }, NodeRegistration: kubeadmapi.NodeRegistrationOptions{ diff --git a/cmd/kubeadm/app/phases/kubeconfig/BUILD b/cmd/kubeadm/app/phases/kubeconfig/BUILD index 59cb56519ec..c23e735b972 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/BUILD +++ b/cmd/kubeadm/app/phases/kubeconfig/BUILD @@ -22,8 +22,8 @@ go_library( "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -54,5 +54,6 @@ go_test( "//cmd/kubeadm/test/kubeconfig:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", + "//vendor/github.com/renstrom/dedent:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go index 865d9d2d7a5..805d3a42155 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -25,11 +25,11 @@ import ( "os" "path/filepath" - "github.com/golang/glog" "github.com/pkg/errors" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" certutil "k8s.io/client-go/util/cert" + "k8s.io/klog" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" @@ -61,7 +61,7 @@ type kubeConfigSpec struct { // to establish the control plane, including also the admin kubeconfig file. // If kubeconfig files already exists, they are used only if evaluated equal; otherwise an error is returned. func CreateInitKubeConfigFiles(outDir string, cfg *kubeadmapi.InitConfiguration) error { - glog.V(1).Infoln("creating all kubeconfig files") + klog.V(1).Infoln("creating all kubeconfig files") return createKubeConfigFiles( outDir, cfg, @@ -89,7 +89,7 @@ func CreateJoinControlPlaneKubeConfigFiles(outDir string, cfg *kubeadmapi.InitCo // CreateKubeConfigFile creates a kubeconfig file. // If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned. func CreateKubeConfigFile(kubeConfigFileName string, outDir string, cfg *kubeadmapi.InitConfiguration) error { - glog.V(1).Infof("creating kubeconfig file for %s", kubeConfigFileName) + klog.V(1).Infof("creating kubeconfig file for %s", kubeConfigFileName) return createKubeConfigFiles(outDir, cfg, kubeConfigFileName) } @@ -216,22 +216,12 @@ func buildKubeConfigFromSpec(spec *kubeConfigSpec, clustername string) (*clientc ), nil } -// createKubeConfigFileIfNotExists saves the KubeConfig object into a file if there isn't any file at the given path. -// If there already is a kubeconfig file at the given path; kubeadm tries to load it and check if the values in the -// existing and the expected config equals. If they do; kubeadm will just skip writing the file as it's up-to-date, -// but if a file exists but has old content or isn't a kubeconfig file, this function returns an error. -func createKubeConfigFileIfNotExists(outDir, filename string, config *clientcmdapi.Config) error { +// validateKubeConfig check if the kubeconfig file exist and has the expected CA and server URL +func validateKubeConfig(outDir, filename string, config *clientcmdapi.Config) error { kubeConfigFilePath := filepath.Join(outDir, filename) - // Check if the file exist, and if it doesn't, just write it to disk - if _, err := os.Stat(kubeConfigFilePath); os.IsNotExist(err) { - fmt.Printf("[kubeconfig] Writing %q kubeconfig file\n", filename) - - err = kubeconfigutil.WriteToDisk(kubeConfigFilePath, config) - if err != nil { - return errors.Wrapf(err, "failed to save kubeconfig file %s on disk", kubeConfigFilePath) - } - return nil + if _, err := os.Stat(kubeConfigFilePath); err != nil { + return err } // The kubeconfig already exists, let's check if it has got the same CA and server URL @@ -254,6 +244,29 @@ func createKubeConfigFileIfNotExists(outDir, filename string, config *clientcmda return errors.Errorf("a kubeconfig file %q exists already but has got the wrong API Server URL", kubeConfigFilePath) } + return nil +} + +// createKubeConfigFileIfNotExists saves the KubeConfig object into a file if there isn't any file at the given path. +// If there already is a kubeconfig file at the given path; kubeadm tries to load it and check if the values in the +// existing and the expected config equals. If they do; kubeadm will just skip writing the file as it's up-to-date, +// but if a file exists but has old content or isn't a kubeconfig file, this function returns an error. +func createKubeConfigFileIfNotExists(outDir, filename string, config *clientcmdapi.Config) error { + kubeConfigFilePath := filepath.Join(outDir, filename) + + err := validateKubeConfig(outDir, filename, config) + if err != nil { + // Check if the file exist, and if it doesn't, just write it to disk + if !os.IsNotExist(err) { + return err + } + fmt.Printf("[kubeconfig] Writing %q kubeconfig file\n", filename) + err = kubeconfigutil.WriteToDisk(kubeConfigFilePath, config) + if err != nil { + return errors.Wrapf(err, "failed to save kubeconfig file %q on disk", kubeConfigFilePath) + } + return nil + } // kubeadm doesn't validate the existing kubeconfig file more than this (kubeadm trusts the client certs to be valid) // Basically, if we find a kubeconfig file with the same path; the same CA cert and the same server URL; // kubeadm thinks those files are equal and doesn't bother writing a new file @@ -333,3 +346,35 @@ func writeKubeConfigFromSpec(out io.Writer, spec *kubeConfigSpec, clustername st fmt.Fprintln(out, string(configBytes)) return nil } + +// ValidateKubeconfigsForExternalCA check if the kubeconfig file exist and has the expected CA and server URL using kubeadmapi.InitConfiguration. +func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfiguration) error { + kubeConfigFileNames := []string{ + kubeadmconstants.AdminKubeConfigFileName, + kubeadmconstants.KubeletKubeConfigFileName, + kubeadmconstants.ControllerManagerKubeConfigFileName, + kubeadmconstants.SchedulerKubeConfigFileName, + } + + specs, err := getKubeConfigSpecs(cfg) + if err != nil { + return err + } + + for _, kubeConfigFileName := range kubeConfigFileNames { + spec, exists := specs[kubeConfigFileName] + if !exists { + return errors.Errorf("couldn't retrive KubeConfigSpec for %s", kubeConfigFileName) + } + + kubeconfig, err := buildKubeConfigFromSpec(spec, cfg.ClusterName) + if err != nil { + return err + } + + if err = validateKubeConfig(outDir, kubeConfigFileName, kubeconfig); err != nil { + return err + } + } + return nil +} diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go index 8ef9b2f7b88..8416c672fc0 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go @@ -1,5 +1,5 @@ /* -Copyright 2017 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,9 +23,12 @@ import ( "fmt" "io" "os" + "path/filepath" "reflect" "testing" + "github.com/renstrom/dedent" + "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -66,14 +69,14 @@ func TestGetKubeConfigSpecs(t *testing.T) { // Creates Master Configurations pointing to the pkidir folder cfgs := []*kubeadmapi.InitConfiguration{ { - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ CertificatesDir: pkidir, }, NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"}, }, { - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ ControlPlaneEndpoint: "api.k8s.io", CertificatesDir: pkidir, @@ -81,7 +84,7 @@ func TestGetKubeConfigSpecs(t *testing.T) { NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"}, }, { - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ ControlPlaneEndpoint: "api.k8s.io:4321", CertificatesDir: pkidir, @@ -89,7 +92,7 @@ func TestGetKubeConfigSpecs(t *testing.T) { NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"}, }, { - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ ControlPlaneEndpoint: "api.k8s.io", CertificatesDir: pkidir, @@ -97,7 +100,7 @@ func TestGetKubeConfigSpecs(t *testing.T) { NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"}, }, { - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ ControlPlaneEndpoint: "api.k8s.io:4321", CertificatesDir: pkidir, @@ -304,7 +307,7 @@ func TestCreateKubeconfigFilesAndWrappers(t *testing.T) { // Creates a Master Configuration pointing to the pkidir folder cfg := &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ CertificatesDir: pkidir, }, @@ -381,7 +384,7 @@ func TestWriteKubeConfig(t *testing.T) { // Creates a Master Configuration pointing to the pkidir folder cfg := &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ CertificatesDir: pkidir, }, @@ -437,6 +440,150 @@ func TestWriteKubeConfig(t *testing.T) { } } +func TestValidateKubeConfig(t *testing.T) { + caCert, caKey := certstestutil.SetupCertificateAuthorithy(t) + anotherCaCert, anotherCaKey := certstestutil.SetupCertificateAuthorithy(t) + + config := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://1.2.3.4:1234", "test-cluster", "myOrg1") + configWithAnotherClusterCa := setupdKubeConfigWithClientAuth(t, anotherCaCert, anotherCaKey, "https://1.2.3.4:1234", "test-cluster", "myOrg1") + configWithAnotherServerURL := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://4.3.2.1:4321", "test-cluster", "myOrg1") + + tests := map[string]struct { + existingKubeConfig *clientcmdapi.Config + kubeConfig *clientcmdapi.Config + expectedError bool + }{ + "kubeconfig don't exist": { + kubeConfig: config, + expectedError: true, + }, + "kubeconfig exist and has invalid ca": { + existingKubeConfig: configWithAnotherClusterCa, + kubeConfig: config, + expectedError: true, + }, + "kubeconfig exist and has invalid server url": { + existingKubeConfig: configWithAnotherServerURL, + kubeConfig: config, + expectedError: true, + }, + "kubeconfig exist and is valid": { + existingKubeConfig: config, + kubeConfig: config, + expectedError: false, + }, + } + + for name, test := range tests { + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + if test.existingKubeConfig != nil { + if err := createKubeConfigFileIfNotExists(tmpdir, "test.conf", test.existingKubeConfig); err != nil { + t.Errorf("createKubeConfigFileIfNotExists failed") + } + } + + err := validateKubeConfig(tmpdir, "test.conf", test.kubeConfig) + if (err != nil) != test.expectedError { + t.Fatalf(dedent.Dedent( + "validateKubeConfig failed\n%s\nexpected error: %t\n\tgot: %t\nerror: %v"), + name, + test.expectedError, + (err != nil), + err, + ) + } + } +} + +func TestValidateKubeconfigsForExternalCA(t *testing.T) { + tmpDir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpDir) + pkiDir := filepath.Join(tmpDir, "pki") + + initConfig := &kubeadmapi.InitConfiguration{ + ClusterConfiguration: kubeadmapi.ClusterConfiguration{ + CertificatesDir: pkiDir, + }, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ + BindPort: 1234, + AdvertiseAddress: "1.2.3.4", + }, + } + + caCert, caKey := certstestutil.SetupCertificateAuthorithy(t) + anotherCaCert, anotherCaKey := certstestutil.SetupCertificateAuthorithy(t) + if err := pkiutil.WriteCertAndKey(pkiDir, kubeadmconstants.CACertAndKeyBaseName, caCert, caKey); err != nil { + t.Fatalf("failure while saving CA certificate and key: %v", err) + } + + config := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://1.2.3.4:1234", "test-cluster", "myOrg1") + configWithAnotherClusterCa := setupdKubeConfigWithClientAuth(t, anotherCaCert, anotherCaKey, "https://1.2.3.4:1234", "test-cluster", "myOrg1") + configWithAnotherServerURL := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://4.3.2.1:4321", "test-cluster", "myOrg1") + + tests := map[string]struct { + filesToWrite map[string]*clientcmdapi.Config + initConfig *kubeadmapi.InitConfiguration + expectedError bool + }{ + "files don't exist": { + initConfig: initConfig, + expectedError: true, + }, + "some files don't exist": { + filesToWrite: map[string]*clientcmdapi.Config{ + kubeadmconstants.AdminKubeConfigFileName: config, + kubeadmconstants.KubeletKubeConfigFileName: config, + }, + initConfig: initConfig, + expectedError: true, + }, + "some files are invalid": { + filesToWrite: map[string]*clientcmdapi.Config{ + kubeadmconstants.AdminKubeConfigFileName: config, + kubeadmconstants.KubeletKubeConfigFileName: config, + kubeadmconstants.ControllerManagerKubeConfigFileName: configWithAnotherClusterCa, + kubeadmconstants.SchedulerKubeConfigFileName: configWithAnotherServerURL, + }, + initConfig: initConfig, + expectedError: true, + }, + "all files are valid": { + filesToWrite: map[string]*clientcmdapi.Config{ + kubeadmconstants.AdminKubeConfigFileName: config, + kubeadmconstants.KubeletKubeConfigFileName: config, + kubeadmconstants.ControllerManagerKubeConfigFileName: config, + kubeadmconstants.SchedulerKubeConfigFileName: config, + }, + initConfig: initConfig, + expectedError: false, + }, + } + + for name, test := range tests { + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + for name, config := range test.filesToWrite { + if err := createKubeConfigFileIfNotExists(tmpdir, name, config); err != nil { + t.Errorf("createKubeConfigFileIfNotExists failed") + } + } + + err := ValidateKubeconfigsForExternalCA(tmpdir, test.initConfig) + if (err != nil) != test.expectedError { + t.Fatalf(dedent.Dedent( + "ValidateKubeconfigsForExternalCA failed\n%s\nexpected error: %t\n\tgot: %t\nerror: %v"), + name, + test.expectedError, + (err != nil), + err, + ) + } + } +} + // setupdKubeConfigWithClientAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With ClientAuth func setupdKubeConfigWithClientAuth(t *testing.T, caCert *x509.Certificate, caKey *rsa.PrivateKey, APIServer, clientName, clustername string, organizations ...string) *clientcmdapi.Config { spec := &kubeConfigSpec{ diff --git a/cmd/kubeadm/app/phases/kubelet/BUILD b/cmd/kubeadm/app/phases/kubelet/BUILD index 4e440c47721..537c9134b59 100644 --- a/cmd/kubeadm/app/phases/kubelet/BUILD +++ b/cmd/kubeadm/app/phases/kubelet/BUILD @@ -15,7 +15,7 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm/v1beta1:go_default_library", "//cmd/kubeadm/app/componentconfigs:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", + "//cmd/kubeadm/app/images:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//pkg/apis/rbac/v1:go_default_library", @@ -29,8 +29,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) @@ -45,7 +45,6 @@ go_test( embed = [":go_default_library"], deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", "//pkg/kubelet/apis:go_default_library", "//pkg/kubelet/apis/config:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/cmd/kubeadm/app/phases/kubelet/config.go b/cmd/kubeadm/app/phases/kubelet/config.go index 4797404220b..8a88661c635 100644 --- a/cmd/kubeadm/app/phases/kubelet/config.go +++ b/cmd/kubeadm/app/phases/kubelet/config.go @@ -23,7 +23,6 @@ import ( "path/filepath" "github.com/pkg/errors" - "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/cmd/kubeadm/app/phases/kubelet/dynamic.go b/cmd/kubeadm/app/phases/kubelet/dynamic.go index d601b6d53c9..51992204fc6 100644 --- a/cmd/kubeadm/app/phases/kubelet/dynamic.go +++ b/cmd/kubeadm/app/phases/kubelet/dynamic.go @@ -31,7 +31,6 @@ import ( // EnableDynamicConfigForNode updates the Node's ConfigSource to enable Dynamic Kubelet Configuration, depending on what version the kubelet is // Used at "kubeadm init", "kubeadm join" and "kubeadm upgrade" time -// This func is ONLY run if the user enables the `DynamicKubeletConfig` feature gate, which is by default off func EnableDynamicConfigForNode(client clientset.Interface, nodeName string, kubeletVersion *version.Version) error { configMapName := kubeadmconstants.GetKubeletConfigMapName(kubeletVersion) diff --git a/cmd/kubeadm/app/phases/kubelet/flags.go b/cmd/kubeadm/app/phases/kubelet/flags.go index 6037e20292a..a1d6ba9bd96 100644 --- a/cmd/kubeadm/app/phases/kubelet/flags.go +++ b/cmd/kubeadm/app/phases/kubelet/flags.go @@ -23,13 +23,12 @@ import ( "path/filepath" "strings" - "github.com/golang/glog" "github.com/pkg/errors" - + "k8s.io/klog" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" + "k8s.io/kubernetes/cmd/kubeadm/app/images" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" nodeutil "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/util/procfs" @@ -39,6 +38,7 @@ import ( type kubeletFlagsOpts struct { nodeRegOpts *kubeadmapi.NodeRegistrationOptions featureGates map[string]bool + pauseImage string registerTaintsUsingFlags bool execer utilsexec.Interface pidOfFunc func(string) ([]int, error) @@ -47,22 +47,23 @@ type kubeletFlagsOpts struct { // WriteKubeletDynamicEnvFile writes an environment file with dynamic flags to the kubelet. // Used at "kubeadm init" and "kubeadm join" time. -func WriteKubeletDynamicEnvFile(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, featureGates map[string]bool, registerTaintsUsingFlags bool, kubeletDir string) error { +func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.InitConfiguration, registerTaintsUsingFlags bool, kubeletDir string) error { hostName, err := nodeutil.GetHostname("") if err != nil { return err } flagOpts := kubeletFlagsOpts{ - nodeRegOpts: nodeRegOpts, - featureGates: featureGates, + nodeRegOpts: &cfg.NodeRegistration, + featureGates: cfg.FeatureGates, + pauseImage: images.GetPauseImage(&cfg.ClusterConfiguration), registerTaintsUsingFlags: registerTaintsUsingFlags, execer: utilsexec.New(), pidOfFunc: procfs.PidOf, defaultHostname: hostName, } stringMap := buildKubeletArgMap(flagOpts) - argList := kubeadmutil.BuildArgumentListFromMap(stringMap, nodeRegOpts.KubeletExtraArgs) + argList := kubeadmutil.BuildArgumentListFromMap(stringMap, cfg.NodeRegistration.KubeletExtraArgs) envFileContent := fmt.Sprintf("%s=%s\n", constants.KubeletEnvFileVariableName, strings.Join(argList, " ")) return writeKubeletFlagBytesToDisk([]byte(envFileContent), kubeletDir) @@ -78,10 +79,13 @@ func buildKubeletArgMap(opts kubeletFlagsOpts) map[string]string { kubeletFlags["network-plugin"] = "cni" driver, err := kubeadmutil.GetCgroupDriverDocker(opts.execer) if err != nil { - glog.Warningf("cannot automatically assign a '--cgroup-driver' value when starting the Kubelet: %v\n", err) + klog.Warningf("cannot automatically assign a '--cgroup-driver' value when starting the Kubelet: %v\n", err) } else { kubeletFlags["cgroup-driver"] = driver } + if opts.pauseImage != "" { + kubeletFlags["pod-infra-container-image"] = opts.pauseImage + } } else { kubeletFlags["container-runtime"] = "remote" kubeletFlags["container-runtime-endpoint"] = opts.nodeRegOpts.CRISocket @@ -103,16 +107,10 @@ func buildKubeletArgMap(opts kubeletFlagsOpts) map[string]string { // Make sure the node name we're passed will work with Kubelet if opts.nodeRegOpts.Name != "" && opts.nodeRegOpts.Name != opts.defaultHostname { - glog.V(1).Infof("setting kubelet hostname-override to %q", opts.nodeRegOpts.Name) + klog.V(1).Infof("setting kubelet hostname-override to %q", opts.nodeRegOpts.Name) kubeletFlags["hostname-override"] = opts.nodeRegOpts.Name } - // If the user enabled Dynamic Kubelet Configuration (which is disabled by default), set the directory - // in the CLI flags so that the feature actually gets enabled - if features.Enabled(opts.featureGates, features.DynamicKubeletConfig) { - kubeletFlags["dynamic-config-dir"] = filepath.Join(constants.KubeletRunDirectory, constants.DynamicKubeletConfigurationDirectoryName) - } - // TODO: Conditionally set `--cgroup-driver` to either `systemd` or `cgroupfs` for CRI other than Docker return kubeletFlags diff --git a/cmd/kubeadm/app/phases/kubelet/flags_test.go b/cmd/kubeadm/app/phases/kubelet/flags_test.go index b0464dacbb0..26d4774c39e 100644 --- a/cmd/kubeadm/app/phases/kubelet/flags_test.go +++ b/cmd/kubeadm/app/phases/kubelet/flags_test.go @@ -19,7 +19,6 @@ package kubelet import ( "context" "errors" - "fmt" "io" "reflect" "strings" @@ -27,7 +26,6 @@ import ( "k8s.io/api/core/v1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/utils/exec" ) @@ -234,23 +232,21 @@ func TestBuildKubeletArgMap(t *testing.T) { }, }, { - name: "dynamic kubelet config enabled", + name: "pause image is set", opts: kubeletFlagsOpts{ nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ - CRISocket: "/var/run/containerd.sock", + CRISocket: "/var/run/dockershim.sock", Name: "foo", }, - featureGates: map[string]bool{ - "DynamicKubeletConfig": true, - }, + pauseImage: "gcr.io/pause:3.1", execer: cgroupfsCgroupExecer, pidOfFunc: binaryNotRunningPidOfFunc, defaultHostname: "foo", }, expected: map[string]string{ - "container-runtime": "remote", - "container-runtime-endpoint": "/var/run/containerd.sock", - "dynamic-config-dir": fmt.Sprintf("%s/dynamic-config", kubeadmconstants.KubeletRunDirectory), + "network-plugin": "cni", + "cgroup-driver": "cgroupfs", + "pod-infra-container-image": "gcr.io/pause:3.1", }, }, } diff --git a/cmd/kubeadm/app/phases/markmaster/BUILD b/cmd/kubeadm/app/phases/markcontrolplane/BUILD similarity index 88% rename from cmd/kubeadm/app/phases/markmaster/BUILD rename to cmd/kubeadm/app/phases/markcontrolplane/BUILD index 110b66ba9cd..039975f049d 100644 --- a/cmd/kubeadm/app/phases/markmaster/BUILD +++ b/cmd/kubeadm/app/phases/markcontrolplane/BUILD @@ -8,7 +8,7 @@ load( go_test( name = "go_default_test", - srcs = ["markmaster_test.go"], + srcs = ["markcontrolplane_test.go"], embed = [":go_default_library"], deps = [ "//cmd/kubeadm/app/constants:go_default_library", @@ -23,8 +23,8 @@ go_test( go_library( name = "go_default_library", - srcs = ["markmaster.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster", + srcs = ["markcontrolplane.go"], + importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane", deps = [ "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", diff --git a/cmd/kubeadm/app/phases/markmaster/markmaster.go b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go similarity index 70% rename from cmd/kubeadm/app/phases/markmaster/markmaster.go rename to cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go index 1119708403f..6b5fb9104c8 100644 --- a/cmd/kubeadm/app/phases/markmaster/markmaster.go +++ b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package markmaster +package markcontrolplane import ( "fmt" @@ -25,20 +25,20 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" ) -// MarkMaster taints the master and sets the master label -func MarkMaster(client clientset.Interface, masterName string, taints []v1.Taint) error { +// MarkControlPlane taints the control-plane and sets the control-plane label +func MarkControlPlane(client clientset.Interface, controlPlaneName string, taints []v1.Taint) error { - fmt.Printf("[markmaster] Marking the node %s as master by adding the label \"%s=''\"\n", masterName, constants.LabelNodeRoleMaster) + fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the label \"%s=''\"\n", controlPlaneName, constants.LabelNodeRoleMaster) if taints != nil && len(taints) > 0 { taintStrs := []string{} for _, taint := range taints { taintStrs = append(taintStrs, taint.ToString()) } - fmt.Printf("[markmaster] Marking the node %s as master by adding the taints %v\n", masterName, taintStrs) + fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the taints %v\n", controlPlaneName, taintStrs) } - return apiclient.PatchNode(client, masterName, func(n *v1.Node) { + return apiclient.PatchNode(client, controlPlaneName, func(n *v1.Node) { markMasterNode(n, taints) }) } diff --git a/cmd/kubeadm/app/phases/markmaster/markmaster_test.go b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go similarity index 78% rename from cmd/kubeadm/app/phases/markmaster/markmaster_test.go rename to cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go index 45bc0c63659..ab26bddd651 100644 --- a/cmd/kubeadm/app/phases/markmaster/markmaster_test.go +++ b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package markmaster +package markcontrolplane import ( "bytes" @@ -33,7 +33,7 @@ import ( "k8s.io/kubernetes/pkg/util/node" ) -func TestMarkMaster(t *testing.T) { +func TestMarkControlPlane(t *testing.T) { // Note: this test takes advantage of the deterministic marshalling of // JSON provided by strategicpatch so that "expectedPatch" can use a // string equality test instead of a logical JSON equality test. That @@ -47,28 +47,28 @@ func TestMarkMaster(t *testing.T) { expectedPatch string }{ { - "master label and taint missing", + "control-plane label and taint missing", "", nil, []v1.Taint{kubeadmconstants.MasterTaint}, "{\"metadata\":{\"labels\":{\"node-role.kubernetes.io/master\":\"\"}},\"spec\":{\"taints\":[{\"effect\":\"NoSchedule\",\"key\":\"node-role.kubernetes.io/master\"}]}}", }, { - "master label and taint missing but taint not wanted", + "control-plane label and taint missing but taint not wanted", "", nil, nil, "{\"metadata\":{\"labels\":{\"node-role.kubernetes.io/master\":\"\"}}}", }, { - "master label missing", + "control-plane label missing", "", []v1.Taint{kubeadmconstants.MasterTaint}, []v1.Taint{kubeadmconstants.MasterTaint}, "{\"metadata\":{\"labels\":{\"node-role.kubernetes.io/master\":\"\"}}}", }, { - "master taint missing", + "control-plane taint missing", kubeadmconstants.LabelNodeRoleMaster, nil, []v1.Taint{kubeadmconstants.MasterTaint}, @@ -110,9 +110,9 @@ func TestMarkMaster(t *testing.T) { for _, tc := range tests { hostname, err := node.GetHostname("") if err != nil { - t.Fatalf("MarkMaster(%s): unexpected error: %v", tc.name, err) + t.Fatalf("MarkControlPlane(%s): unexpected error: %v", tc.name, err) } - masterNode := &v1.Node{ + controlPlaneNode := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: hostname, Labels: map[string]string{ @@ -122,16 +122,16 @@ func TestMarkMaster(t *testing.T) { } if tc.existingLabel != "" { - masterNode.ObjectMeta.Labels[tc.existingLabel] = "" + controlPlaneNode.ObjectMeta.Labels[tc.existingLabel] = "" } if tc.existingTaints != nil { - masterNode.Spec.Taints = tc.existingTaints + controlPlaneNode.Spec.Taints = tc.existingTaints } - jsonNode, err := json.Marshal(masterNode) + jsonNode, err := json.Marshal(controlPlaneNode) if err != nil { - t.Fatalf("MarkMaster(%s): unexpected encoding error: %v", tc.name, err) + t.Fatalf("MarkControlPlane(%s): unexpected encoding error: %v", tc.name, err) } var patchRequest string @@ -139,7 +139,7 @@ func TestMarkMaster(t *testing.T) { w.Header().Set("Content-Type", "application/json") if req.URL.Path != "/api/v1/nodes/"+hostname { - t.Errorf("MarkMaster(%s): request for unexpected HTTP resource: %v", tc.name, req.URL.Path) + t.Errorf("MarkControlPlane(%s): request for unexpected HTTP resource: %v", tc.name, req.URL.Path) http.Error(w, "", http.StatusNotFound) return } @@ -149,7 +149,7 @@ func TestMarkMaster(t *testing.T) { case "PATCH": patchRequest = toString(req.Body) default: - t.Errorf("MarkMaster(%s): request for unexpected HTTP verb: %v", tc.name, req.Method) + t.Errorf("MarkControlPlane(%s): request for unexpected HTTP verb: %v", tc.name, req.Method) http.Error(w, "", http.StatusNotFound) return } @@ -161,15 +161,15 @@ func TestMarkMaster(t *testing.T) { cs, err := clientset.NewForConfig(&restclient.Config{Host: s.URL}) if err != nil { - t.Fatalf("MarkMaster(%s): unexpected error building clientset: %v", tc.name, err) + t.Fatalf("MarkControlPlane(%s): unexpected error building clientset: %v", tc.name, err) } - if err := MarkMaster(cs, hostname, tc.newTaints); err != nil { - t.Errorf("MarkMaster(%s) returned unexpected error: %v", tc.name, err) + if err := MarkControlPlane(cs, hostname, tc.newTaints); err != nil { + t.Errorf("MarkControlPlane(%s) returned unexpected error: %v", tc.name, err) } if tc.expectedPatch != patchRequest { - t.Errorf("MarkMaster(%s) wanted patch %v, got %v", tc.name, tc.expectedPatch, patchRequest) + t.Errorf("MarkControlPlane(%s) wanted patch %v, got %v", tc.name, tc.expectedPatch, patchRequest) } } } diff --git a/cmd/kubeadm/app/phases/selfhosting/BUILD b/cmd/kubeadm/app/phases/selfhosting/BUILD index af669d900e4..6a94bb81da6 100644 --- a/cmd/kubeadm/app/phases/selfhosting/BUILD +++ b/cmd/kubeadm/app/phases/selfhosting/BUILD @@ -34,7 +34,6 @@ go_library( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", @@ -43,8 +42,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go index c2faa3646d0..25142b1ad5b 100644 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go +++ b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go @@ -22,7 +22,6 @@ import ( "k8s.io/api/core/v1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) @@ -59,18 +58,18 @@ func GetDefaultMutators() map[string][]PodSpecMutatorFunc { } // GetMutatorsFromFeatureGates returns all mutators needed based on the feature gates passed -func GetMutatorsFromFeatureGates(featureGates map[string]bool) map[string][]PodSpecMutatorFunc { +func GetMutatorsFromFeatureGates(certsInSecrets bool) map[string][]PodSpecMutatorFunc { // Here the map of different mutators to use for the control plane's podspec is stored mutators := GetDefaultMutators() - // Some extra work to be done if we should store the control plane certificates in Secrets - if features.Enabled(featureGates, features.StoreCertsInSecrets) { - + if certsInSecrets { + // Some extra work to be done if we should store the control plane certificates in Secrets // Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them mutators[kubeadmconstants.KubeAPIServer] = append(mutators[kubeadmconstants.KubeAPIServer], setSelfHostedVolumesForAPIServer) mutators[kubeadmconstants.KubeControllerManager] = append(mutators[kubeadmconstants.KubeControllerManager], setSelfHostedVolumesForControllerManager) mutators[kubeadmconstants.KubeScheduler] = append(mutators[kubeadmconstants.KubeScheduler], setSelfHostedVolumesForScheduler) } + return mutators } diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go index 2048963369d..eb4d1dcdb29 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go @@ -22,7 +22,7 @@ import ( "os" "time" - "github.com/golang/glog" + "k8s.io/klog" "github.com/pkg/errors" apps "k8s.io/api/apps/v1" @@ -33,7 +33,6 @@ import ( clientscheme "k8s.io/client-go/kubernetes/scheme" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" ) @@ -57,18 +56,16 @@ const ( // 8. In order to avoid race conditions, we have to make sure that static pod is deleted correctly before we continue // Otherwise, there is a race condition when we proceed without kubelet having restarted the API server correctly and the next .Create call flakes // 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop -func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.InitConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool) error { - glog.V(1).Infoln("creating self hosted control plane") +func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.InitConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool, certsInSecrets bool) error { + klog.V(1).Infoln("creating self hosted control plane") // Adjust the timeout slightly to something self-hosting specific waiter.SetTimeout(selfHostingWaitTimeout) // Here the map of different mutators to use for the control plane's PodSpec is stored - glog.V(1).Infoln("getting mutators") - mutators := GetMutatorsFromFeatureGates(cfg.FeatureGates) - - // Some extra work to be done if we should store the control plane certificates in Secrets - if features.Enabled(cfg.FeatureGates, features.StoreCertsInSecrets) { + klog.V(1).Infoln("getting mutators") + mutators := GetMutatorsFromFeatureGates(certsInSecrets) + if certsInSecrets { // Upload the certificates and kubeconfig files from disk to the cluster as Secrets if err := uploadTLSSecrets(client, cfg.CertificatesDir); err != nil { return err diff --git a/cmd/kubeadm/app/phases/upgrade/BUILD b/cmd/kubeadm/app/phases/upgrade/BUILD index ad2dab44982..d804a9e2e70 100644 --- a/cmd/kubeadm/app/phases/upgrade/BUILD +++ b/cmd/kubeadm/app/phases/upgrade/BUILD @@ -8,7 +8,6 @@ go_library( "policy.go", "postupgrade.go", "prepull.go", - "selfhosted.go", "staticpods.go", "versiongetter.go", ], @@ -18,7 +17,6 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/v1beta1:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/images:go_default_library", "//cmd/kubeadm/app/phases/addons/dns:go_default_library", "//cmd/kubeadm/app/phases/addons/proxy:go_default_library", @@ -30,7 +28,6 @@ go_library( "//cmd/kubeadm/app/phases/etcd:go_default_library", "//cmd/kubeadm/app/phases/kubelet:go_default_library", "//cmd/kubeadm/app/phases/patchnode:go_default_library", - "//cmd/kubeadm/app/phases/selfhosting:go_default_library", "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", diff --git a/cmd/kubeadm/app/phases/upgrade/compute.go b/cmd/kubeadm/app/phases/upgrade/compute.go index 92cf121ae97..983815251bb 100644 --- a/cmd/kubeadm/app/phases/upgrade/compute.go +++ b/cmd/kubeadm/app/phases/upgrade/compute.go @@ -22,8 +22,8 @@ import ( versionutil "k8s.io/apimachinery/pkg/util/version" clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" ) @@ -56,20 +56,12 @@ func (u *Upgrade) CanUpgradeEtcd() bool { return u.Before.EtcdVersion != u.After.EtcdVersion } -// ActiveDNSAddon returns the version of CoreDNS or kube-dns -func ActiveDNSAddon(featureGates map[string]bool) string { - if features.Enabled(featureGates, features.CoreDNS) { - return kubeadmconstants.CoreDNS - } - return kubeadmconstants.KubeDNS -} - // ClusterState describes the state of certain versions for a cluster type ClusterState struct { // KubeVersion describes the version of the Kubernetes API Server, Controller Manager, Scheduler and Proxy. KubeVersion string // DNSType - DNSType string + DNSType kubeadmapi.DNSAddOnType // DNSVersion describes the version of the kube-dns images used and manifest version DNSVersion string // KubeadmVersion describes the version of the kubeadm CLI @@ -82,7 +74,7 @@ type ClusterState struct { // GetAvailableUpgrades fetches all versions from the specified VersionGetter and computes which // kinds of upgrades can be performed -func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesAllowed, rcUpgradesAllowed bool, etcdClient etcdutil.ClusterInterrogator, featureGates map[string]bool, client clientset.Interface) ([]Upgrade, error) { +func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesAllowed, rcUpgradesAllowed bool, etcdClient etcdutil.ClusterInterrogator, dnsType kubeadmapi.DNSAddOnType, client clientset.Interface) ([]Upgrade, error) { fmt.Println("[upgrade] Fetching available versions to upgrade to") // Collect the upgrades kubeadm can do in this list @@ -120,7 +112,7 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA return upgrades, err } - dnsType, dnsVersion, err := dns.DeployedDNSAddon(client) + currentDNSType, dnsVersion, err := dns.DeployedDNSAddon(client) if err != nil { return nil, err } @@ -128,7 +120,7 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA // Construct a descriptor for the current state of the world beforeState := ClusterState{ KubeVersion: clusterVersionStr, - DNSType: dnsType, + DNSType: currentDNSType, DNSVersion: dnsVersion, KubeadmVersion: kubeadmVersionStr, KubeletVersions: kubeletVersions, @@ -172,8 +164,8 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA Before: beforeState, After: ClusterState{ KubeVersion: patchVersionStr, - DNSType: ActiveDNSAddon(featureGates), - DNSVersion: kubeadmconstants.GetDNSVersion(ActiveDNSAddon(featureGates)), + DNSType: dnsType, + DNSVersion: kubeadmconstants.GetDNSVersion(dnsType), KubeadmVersion: newKubeadmVer, EtcdVersion: getSuggestedEtcdVersion(patchVersionStr), // KubeletVersions is unset here as it is not used anywhere in .After @@ -189,8 +181,8 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA Before: beforeState, After: ClusterState{ KubeVersion: stableVersionStr, - DNSType: ActiveDNSAddon(featureGates), - DNSVersion: kubeadmconstants.GetDNSVersion(ActiveDNSAddon(featureGates)), + DNSType: dnsType, + DNSVersion: kubeadmconstants.GetDNSVersion(dnsType), KubeadmVersion: stableVersionStr, EtcdVersion: getSuggestedEtcdVersion(stableVersionStr), // KubeletVersions is unset here as it is not used anywhere in .After @@ -235,8 +227,8 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA Before: beforeState, After: ClusterState{ KubeVersion: previousBranchLatestVersionStr, - DNSType: ActiveDNSAddon(featureGates), - DNSVersion: kubeadmconstants.GetDNSVersion(ActiveDNSAddon(featureGates)), + DNSType: dnsType, + DNSVersion: kubeadmconstants.GetDNSVersion(dnsType), KubeadmVersion: previousBranchLatestVersionStr, EtcdVersion: getSuggestedEtcdVersion(previousBranchLatestVersionStr), // KubeletVersions is unset here as it is not used anywhere in .After @@ -249,12 +241,12 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA // Default to assume that the experimental version to show is the unstable one unstableKubeVersion := latestVersionStr - unstableKubeDNSVersion := kubeadmconstants.GetDNSVersion(ActiveDNSAddon(featureGates)) + unstableKubeDNSVersion := kubeadmconstants.GetDNSVersion(dnsType) // Ẃe should not display alpha.0. The previous branch's beta/rc versions are more relevant due how the kube branching process works. if latestVersion.PreRelease() == "alpha.0" { unstableKubeVersion = previousBranchLatestVersionStr - unstableKubeDNSVersion = kubeadmconstants.GetDNSVersion(ActiveDNSAddon(featureGates)) + unstableKubeDNSVersion = kubeadmconstants.GetDNSVersion(dnsType) } upgrades = append(upgrades, Upgrade{ @@ -262,7 +254,7 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA Before: beforeState, After: ClusterState{ KubeVersion: unstableKubeVersion, - DNSType: ActiveDNSAddon(featureGates), + DNSType: dnsType, DNSVersion: unstableKubeDNSVersion, KubeadmVersion: unstableKubeVersion, EtcdVersion: getSuggestedEtcdVersion(unstableKubeVersion), diff --git a/cmd/kubeadm/app/phases/upgrade/compute_test.go b/cmd/kubeadm/app/phases/upgrade/compute_test.go index b0cbcdcd5ba..fd99f637e4c 100644 --- a/cmd/kubeadm/app/phases/upgrade/compute_test.go +++ b/cmd/kubeadm/app/phases/upgrade/compute_test.go @@ -24,12 +24,12 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/pkg/errors" - apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" versionutil "k8s.io/apimachinery/pkg/util/version" clientsetfake "k8s.io/client-go/kubernetes/fake" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/constants" etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" ) @@ -124,9 +124,9 @@ func TestGetAvailableUpgrades(t *testing.T) { allowExperimental, allowRCs bool errExpected bool etcdClient etcdutil.ClusterInterrogator - beforeDNSType string + beforeDNSType kubeadmapi.DNSAddOnType beforeDNSVersion string - featureGates map[string]bool + dnsType kubeadmapi.DNSAddOnType }{ { name: "no action needed, already up-to-date", @@ -138,9 +138,9 @@ func TestGetAvailableUpgrades(t *testing.T) { stablePatchVersion: "v1.10.3", stableVersion: "v1.10.3", }, - beforeDNSType: constants.CoreDNS, + beforeDNSType: kubeadmapi.CoreDNS, beforeDNSVersion: "v1.0.6", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{}, allowExperimental: false, errExpected: false, @@ -156,9 +156,9 @@ func TestGetAvailableUpgrades(t *testing.T) { stablePatchVersion: "v1.10.3", stableVersion: "v1.10.3", }, - beforeDNSType: constants.CoreDNS, + beforeDNSType: kubeadmapi.CoreDNS, beforeDNSVersion: "1.0.6", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{ { Description: "version in the v1.10 series", @@ -168,15 +168,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.10.1": 1, }, KubeadmVersion: "v1.10.2", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.10.3", KubeadmVersion: "v1.10.3", - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.1.12", }, }, @@ -195,9 +195,9 @@ func TestGetAvailableUpgrades(t *testing.T) { stablePatchVersion: "v1.10.3", stableVersion: "v1.10.3", }, ""), - beforeDNSType: constants.CoreDNS, + beforeDNSType: kubeadmapi.CoreDNS, beforeDNSVersion: "1.0.6", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{ { Description: "version in the v1.10 series", @@ -207,15 +207,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.10.1": 1, }, KubeadmVersion: "v1.10.2", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.10.3", KubeadmVersion: "v1.10.3", - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.1.12", }, }, @@ -234,9 +234,9 @@ func TestGetAvailableUpgrades(t *testing.T) { stablePatchVersion: "v1.10.1", stableVersion: "v1.11.0", }, - beforeDNSType: constants.CoreDNS, + beforeDNSType: kubeadmapi.CoreDNS, beforeDNSVersion: "1.0.6", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{ { Description: "stable version", @@ -246,15 +246,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.10.1": 1, }, KubeadmVersion: "v1.11.0", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.11.0", KubeadmVersion: "v1.11.0", - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.2.18", }, }, @@ -273,9 +273,9 @@ func TestGetAvailableUpgrades(t *testing.T) { stablePatchVersion: "v1.10.5", stableVersion: "v1.11.1", }, - beforeDNSType: constants.CoreDNS, + beforeDNSType: kubeadmapi.CoreDNS, beforeDNSVersion: "1.0.6", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{ { Description: "version in the v1.10 series", @@ -285,15 +285,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.10.3": 1, }, KubeadmVersion: "v1.10.5", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.10.5", KubeadmVersion: "v1.10.5", // Note: The kubeadm version mustn't be "downgraded" here - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.1.12", }, }, @@ -305,15 +305,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.10.3": 1, }, KubeadmVersion: "v1.10.5", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.11.1", KubeadmVersion: "v1.11.1", - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.2.18", }, }, @@ -333,9 +333,9 @@ func TestGetAvailableUpgrades(t *testing.T) { stableVersion: "v1.10.5", latestVersion: "v1.11.0-alpha.2", }, - beforeDNSType: constants.CoreDNS, + beforeDNSType: kubeadmapi.CoreDNS, beforeDNSVersion: "v1.0.6", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{}, allowExperimental: true, errExpected: false, @@ -352,9 +352,9 @@ func TestGetAvailableUpgrades(t *testing.T) { stableVersion: "v1.10.5", latestVersion: "v1.11.0-alpha.2", }, - beforeDNSType: constants.CoreDNS, + beforeDNSType: kubeadmapi.CoreDNS, beforeDNSVersion: "1.0.6", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{ { Description: "experimental version", @@ -364,15 +364,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.10.5": 1, }, KubeadmVersion: "v1.10.5", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.11.0-alpha.2", KubeadmVersion: "v1.11.0-alpha.2", - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.2.18", }, }, @@ -392,9 +392,9 @@ func TestGetAvailableUpgrades(t *testing.T) { stableVersion: "v1.10.5", latestVersion: "v1.11.0-alpha.2", }, - beforeDNSType: constants.CoreDNS, + beforeDNSType: kubeadmapi.CoreDNS, beforeDNSVersion: "1.0.6", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{ { Description: "experimental version", @@ -404,15 +404,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.10.5": 1, }, KubeadmVersion: "v1.10.5", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.11.0-alpha.2", KubeadmVersion: "v1.11.0-alpha.2", - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.2.18", }, }, @@ -433,9 +433,9 @@ func TestGetAvailableUpgrades(t *testing.T) { latestDevBranchVersion: "v1.11.0-beta.1", latestVersion: "v1.12.0-alpha.0", }, - beforeDNSType: constants.CoreDNS, + beforeDNSType: kubeadmapi.CoreDNS, beforeDNSVersion: "1.0.6", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{ { Description: "experimental version", @@ -445,15 +445,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.10.5": 1, }, KubeadmVersion: "v1.10.5", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.11.0-beta.1", KubeadmVersion: "v1.11.0-beta.1", - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.2.18", }, }, @@ -474,9 +474,9 @@ func TestGetAvailableUpgrades(t *testing.T) { latestDevBranchVersion: "v1.11.0-rc.1", latestVersion: "v1.12.0-alpha.1", }, - beforeDNSType: constants.CoreDNS, + beforeDNSType: kubeadmapi.CoreDNS, beforeDNSVersion: "1.0.6", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{ { Description: "release candidate version", @@ -486,15 +486,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.10.5": 1, }, KubeadmVersion: "v1.10.5", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.11.0-rc.1", KubeadmVersion: "v1.11.0-rc.1", - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.2.18", }, }, @@ -515,9 +515,9 @@ func TestGetAvailableUpgrades(t *testing.T) { latestDevBranchVersion: "v1.11.6-rc.1", latestVersion: "v1.12.1-alpha.0", }, - beforeDNSType: constants.CoreDNS, + beforeDNSType: kubeadmapi.CoreDNS, beforeDNSVersion: "1.0.6", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{ { Description: "experimental version", // Note that this is considered an experimental version in this uncommon scenario @@ -527,15 +527,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.10.5": 1, }, KubeadmVersion: "v1.10.5", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.11.6-rc.1", KubeadmVersion: "v1.11.6-rc.1", - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.2.18", }, }, @@ -556,9 +556,9 @@ func TestGetAvailableUpgrades(t *testing.T) { latestDevBranchVersion: "v1.11.0-rc.1", latestVersion: "v1.12.0-alpha.2", }, - beforeDNSType: constants.CoreDNS, + beforeDNSType: kubeadmapi.CoreDNS, beforeDNSVersion: "1.0.6", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{ { Description: "release candidate version", @@ -568,15 +568,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.10.5": 1, }, KubeadmVersion: "v1.10.5", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.11.0-rc.1", KubeadmVersion: "v1.11.0-rc.1", - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.2.18", }, }, @@ -588,15 +588,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.10.5": 1, }, KubeadmVersion: "v1.10.5", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.12.0-alpha.2", KubeadmVersion: "v1.12.0-alpha.2", - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.2.24", }, }, @@ -629,9 +629,9 @@ func TestGetAvailableUpgrades(t *testing.T) { kubeadmVersion: "v1.11.1", }, "v1.12.1"), etcdClient: etcdClient, - beforeDNSType: constants.CoreDNS, + beforeDNSType: kubeadmapi.CoreDNS, beforeDNSVersion: "1.0.6", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{ { Description: "version in the v1.11 series", @@ -641,15 +641,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.11.0": 1, }, KubeadmVersion: "v1.11.1", - DNSType: "coredns", + DNSType: kubeadmapi.CoreDNS, DNSVersion: "1.0.6", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.12.1", KubeadmVersion: "v1.12.1", - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.2.24", }, }, @@ -666,9 +666,9 @@ func TestGetAvailableUpgrades(t *testing.T) { stableVersion: "v1.12.0", }, etcdClient: etcdClient, - beforeDNSType: constants.KubeDNS, + beforeDNSType: kubeadmapi.KubeDNS, beforeDNSVersion: "1.14.7", - featureGates: make(map[string]bool), + dnsType: kubeadmapi.CoreDNS, expectedUpgrades: []Upgrade{ { Description: "version in the v1.11 series", @@ -678,15 +678,15 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.11.2": 1, }, KubeadmVersion: "v1.12.0", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.7", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.12.0", KubeadmVersion: "v1.12.0", - DNSType: "coredns", - DNSVersion: "1.2.4", + DNSType: kubeadmapi.CoreDNS, + DNSVersion: "1.2.6", EtcdVersion: "3.2.24", }, }, @@ -703,9 +703,9 @@ func TestGetAvailableUpgrades(t *testing.T) { stableVersion: "v1.12.0", }, etcdClient: etcdClient, - beforeDNSType: constants.KubeDNS, + beforeDNSType: kubeadmapi.KubeDNS, beforeDNSVersion: "1.14.7", - featureGates: map[string]bool{"CoreDNS": false}, + dnsType: kubeadmapi.KubeDNS, expectedUpgrades: []Upgrade{ { Description: "version in the v1.11 series", @@ -715,14 +715,14 @@ func TestGetAvailableUpgrades(t *testing.T) { "v1.11.2": 1, }, KubeadmVersion: "v1.12.0", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.7", EtcdVersion: "3.1.12", }, After: ClusterState{ KubeVersion: "v1.12.0", KubeadmVersion: "v1.12.0", - DNSType: "kube-dns", + DNSType: kubeadmapi.KubeDNS, DNSVersion: "1.14.13", EtcdVersion: "3.2.24", }, @@ -736,13 +736,18 @@ func TestGetAvailableUpgrades(t *testing.T) { for _, rt := range tests { t.Run(rt.name, func(t *testing.T) { + dnsName := constants.CoreDNSDeploymentName + if rt.beforeDNSType == kubeadmapi.KubeDNS { + dnsName = constants.KubeDNSDeploymentName + } + client := clientsetfake.NewSimpleClientset(&apps.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", APIVersion: "apps/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: rt.beforeDNSType, + Name: dnsName, Namespace: "kube-system", Labels: map[string]string{ "k8s-app": "kube-dns", @@ -761,7 +766,7 @@ func TestGetAvailableUpgrades(t *testing.T) { }, }) - actualUpgrades, actualErr := GetAvailableUpgrades(rt.vg, rt.allowExperimental, rt.allowRCs, rt.etcdClient, rt.featureGates, client) + actualUpgrades, actualErr := GetAvailableUpgrades(rt.vg, rt.allowExperimental, rt.allowRCs, rt.etcdClient, rt.dnsType, client) if !reflect.DeepEqual(actualUpgrades, rt.expectedUpgrades) { t.Errorf("failed TestGetAvailableUpgrades\n\texpected upgrades: %v\n\tgot: %v", rt.expectedUpgrades, actualUpgrades) } diff --git a/cmd/kubeadm/app/phases/upgrade/health.go b/cmd/kubeadm/app/phases/upgrade/health.go index 2c4eac7fa91..30169e8146f 100644 --- a/cmd/kubeadm/app/phases/upgrade/health.go +++ b/cmd/kubeadm/app/phases/upgrade/health.go @@ -76,20 +76,11 @@ func CheckClusterHealth(client clientset.Interface, ignoreChecksErrors sets.Stri // TODO: Add a check for ComponentStatuses here? } - // Run slightly different health checks depending on control plane hosting type - if IsControlPlaneSelfHosted(client) { - healthChecks = append(healthChecks, &healthCheck{ - name: "ControlPlaneHealth", - client: client, - f: controlPlaneHealth, - }) - } else { - healthChecks = append(healthChecks, &healthCheck{ - name: "StaticPodManifest", - client: client, - f: staticPodManifestHealth, - }) - } + healthChecks = append(healthChecks, &healthCheck{ + name: "StaticPodManifest", + client: client, + f: staticPodManifestHealth, + }) return preflight.RunChecks(healthChecks, os.Stderr, ignoreChecksErrors) } @@ -132,19 +123,6 @@ func masterNodesReady(client clientset.Interface) error { return nil } -// controlPlaneHealth ensures all control plane DaemonSets are healthy -func controlPlaneHealth(client clientset.Interface) error { - notReadyDaemonSets, err := getNotReadyDaemonSets(client) - if err != nil { - return err - } - - if len(notReadyDaemonSets) != 0 { - return errors.Errorf("there are control plane DaemonSets in the cluster that are not ready: %v", notReadyDaemonSets) - } - return nil -} - // staticPodManifestHealth makes sure the required static pods are presents func staticPodManifestHealth(_ clientset.Interface) error { nonExistentManifests := []string{} @@ -167,7 +145,7 @@ func IsControlPlaneSelfHosted(client clientset.Interface) bool { return false } - // If there are no NotReady DaemonSets, we are using self-hosting + // If there are no NotReady DaemonSets, we are using selfhosting return len(notReadyDaemonSets) == 0 } diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index 7d19b889a48..1e100eab8b2 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -24,7 +24,6 @@ import ( "time" pkgerrors "github.com/pkg/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/errors" @@ -34,7 +33,6 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" @@ -42,7 +40,6 @@ import ( certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" @@ -94,11 +91,6 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon errs = append(errs, err) } - // Upgrade to a self-hosted control plane if possible - if err := upgradeToSelfHosting(client, cfg, dryRun); err != nil { - errs = append(errs, err) - } - // TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade // Create the cluster-info ConfigMap with the associated RBAC rules // if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil { @@ -131,12 +123,12 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.InitConfiguration, client clientset.Interface, dryRun bool) error { return apiclient.TryRunCommand(func() error { - installedDeploymentName := kubeadmconstants.KubeDNS - deploymentToDelete := kubeadmconstants.CoreDNS + installedDeploymentName := kubeadmconstants.KubeDNSDeploymentName + deploymentToDelete := kubeadmconstants.CoreDNSDeploymentName - if features.Enabled(cfg.FeatureGates, features.CoreDNS) { - installedDeploymentName = kubeadmconstants.CoreDNS - deploymentToDelete = kubeadmconstants.KubeDNS + if cfg.DNS.Type == kubeadmapi.CoreDNS { + installedDeploymentName = kubeadmconstants.CoreDNSDeploymentName + deploymentToDelete = kubeadmconstants.KubeDNSDeploymentName } // If we're dry-running, we don't need to wait for the new DNS addon to become ready @@ -160,20 +152,6 @@ func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.InitConfiguration, }, 10) } -func upgradeToSelfHosting(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, dryRun bool) error { - if features.Enabled(cfg.FeatureGates, features.SelfHosting) && !IsControlPlaneSelfHosted(client) { - - waiter := getWaiter(dryRun, client) - - // kubeadm will now convert the static Pod-hosted control plane into a self-hosted one - fmt.Println("[self-hosted] Creating self-hosted control plane.") - if err := selfhosting.CreateSelfHostedControlPlane(kubeadmconstants.GetStaticPodDirectory(), kubeadmconstants.KubernetesDir, cfg, client, waiter, dryRun); err != nil { - return pkgerrors.Wrap(err, "error creating self hosted control plane") - } - } - return nil -} - // BackupAPIServerCertIfNeeded rotates the kube-apiserver certificate if older than 180 days func BackupAPIServerCertIfNeeded(cfg *kubeadmapi.InitConfiguration, dryRun bool) error { certAndKeyDir := kubeadmapiv1beta1.DefaultCertificatesDir @@ -231,7 +209,7 @@ func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.InitCon // Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master, // as we handle that ourselves in the markmaster phase // TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase? - if err := kubeletphase.WriteKubeletDynamicEnvFile(&cfg.NodeRegistration, cfg.FeatureGates, false, kubeletDir); err != nil { + if err := kubeletphase.WriteKubeletDynamicEnvFile(cfg, false, kubeletDir); err != nil { errs = append(errs, pkgerrors.Wrap(err, "error writing a dynamic environment file for the kubelet")) } @@ -242,15 +220,6 @@ func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.InitCon return errors.NewAggregate(errs) } -// getWaiter gets the right waiter implementation for the right occasion -// TODO: Consolidate this with what's in init.go? -func getWaiter(dryRun bool, client clientset.Interface) apiclient.Waiter { - if dryRun { - return dryrunutil.NewWaiter() - } - return apiclient.NewKubeWaiter(client, 30*time.Minute, os.Stdout) -} - // getKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not. // TODO: Consolidate this with similar funcs? func getKubeletDir(dryRun bool) (string, error) { diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade_test.go b/cmd/kubeadm/app/phases/upgrade/postupgrade_test.go index d35d0c19c92..6b2aebded55 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade_test.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade_test.go @@ -130,7 +130,7 @@ func TestRollbackFiles(t *testing.T) { func TestShouldBackupAPIServerCertAndKey(t *testing.T) { cfg := &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, }, diff --git a/cmd/kubeadm/app/phases/upgrade/prepull.go b/cmd/kubeadm/app/phases/upgrade/prepull.go index 9ee08a33a00..f96693333bd 100644 --- a/cmd/kubeadm/app/phases/upgrade/prepull.go +++ b/cmd/kubeadm/app/phases/upgrade/prepull.go @@ -21,7 +21,6 @@ import ( "time" "github.com/pkg/errors" - apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -65,7 +64,7 @@ func (d *DaemonSetPrepuller) CreateFunc(component string) error { if component == constants.Etcd { image = images.GetEtcdImage(d.cfg) } else { - image = images.GetKubeControlPlaneImage(component, d.cfg) + image = images.GetKubernetesImage(component, d.cfg) } ds := buildPrePullDaemonSet(component, image) diff --git a/cmd/kubeadm/app/phases/upgrade/selfhosted.go b/cmd/kubeadm/app/phases/upgrade/selfhosted.go deleted file mode 100644 index 21cddf8dd57..00000000000 --- a/cmd/kubeadm/app/phases/upgrade/selfhosted.go +++ /dev/null @@ -1,274 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "fmt" - "time" - - "github.com/pkg/errors" - - apps "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/version" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -const ( - // upgradeTempDSPrefix is the prefix added to the temporary DaemonSet's name used during the upgrade - upgradeTempDSPrefix = "temp-upgrade-" - - // upgradeTempLabel is the label key used for identifying the temporary component's DaemonSet - upgradeTempLabel = "temp-upgrade-component" - - // selfHostingWaitTimeout describes the maximum amount of time a self-hosting wait process should wait before timing out - selfHostingWaitTimeout = 2 * time.Minute - - // selfHostingFailureThreshold describes how many times kubeadm will retry creating the DaemonSets - selfHostingFailureThreshold int = 10 -) - -// controlPlaneComponentResources holds the relevant Pod and DaemonSet associated with a control plane component -type controlPlaneComponentResources struct { - pod *v1.Pod - daemonSet *apps.DaemonSet -} - -// SelfHostedControlPlane upgrades a self-hosted control plane -// It works as follows: -// - The client gets the currently running DaemonSets and their associated Pods used for self-hosting the control plane -// - A temporary DaemonSet for the component in question is created; but nearly identical to the DaemonSet for the self-hosted component running right now -// - Why use this temporary DaemonSet? Because, the RollingUpdate strategy for upgrading DaemonSets first kills the old Pod, and then adds the new one -// - This doesn't work for self-hosted upgrades, as if you remove the only API server for instance you have in the cluster, the cluster essentially goes down -// - So instead, a nearly identical copy of the pre-upgrade DaemonSet is created and applied to the cluster. In the beginning, this duplicate DS is just idle -// - kubeadm waits for the temporary DaemonSet's Pod to become Running -// - kubeadm updates the real, self-hosted component. This will result in the pre-upgrade component Pod being removed from the cluster -// - Luckily, the temporary, backup DaemonSet now kicks in and takes over and acts as the control plane. It recognizes that a new Pod should be created, -// - as the "real" DaemonSet is being updated. -// - kubeadm waits for the pre-upgrade Pod to become deleted. It now takes advantage of the backup/temporary component -// - kubeadm waits for the new, upgraded DaemonSet to become Running. -// - Now that the new, upgraded DaemonSet is Running, we can delete the backup/temporary DaemonSet -// - Lastly, make sure the API /healthz endpoint still is reachable -// -// TL;DR; This is what the flow looks like in pseudo-code: -// for [kube-apiserver, kube-controller-manager, kube-scheduler], do: -// 1. Self-Hosted component v1 Running -// -> Duplicate the DaemonSet manifest -// 2. Self-Hosted component v1 Running (active). Backup component v1 Running (passive) -// -> Upgrade the Self-Hosted component v1 to v2. -// -> Self-Hosted component v1 is Deleted from the cluster -// 3. Backup component v1 Running becomes active and completes the upgrade by creating the Self-Hosted component v2 Pod (passive) -// -> Wait for Self-Hosted component v2 to become Running -// 4. Backup component v1 Running (active). Self-Hosted component v2 Running (passive) -// -> Backup component v1 is Deleted -// 5. Wait for Self-Hosted component v2 Running to become active -// 6. Repeat for all control plane components -func SelfHostedControlPlane(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) error { - - // Adjust the timeout slightly to something self-hosting specific - waiter.SetTimeout(selfHostingWaitTimeout) - - // This function returns a map of DaemonSet objects ready to post to the API server - newControlPlaneDaemonSets := BuildUpgradedDaemonSetsFromConfig(cfg, k8sVersion) - - controlPlaneResources, err := getCurrentControlPlaneComponentResources(client) - if err != nil { - return err - } - - for _, component := range constants.MasterComponents { - // Make a shallow copy of the current DaemonSet in order to create a new, temporary one - tempDS := *controlPlaneResources[component].daemonSet - - // Mutate the temp daemonset a little to be suitable for this usage (change label selectors, etc) - mutateTempDaemonSet(&tempDS, component) - - // Create or update the DaemonSet in the API Server, and retry selfHostingFailureThreshold times if it errors out - if err := apiclient.TryRunCommand(func() error { - return apiclient.CreateOrUpdateDaemonSet(client, &tempDS) - }, selfHostingFailureThreshold); err != nil { - return err - } - - // Wait for the temporary/backup self-hosted component to come up - if err := waiter.WaitForPodsWithLabel(buildTempUpgradeDSLabelQuery(component)); err != nil { - return err - } - - newDS := newControlPlaneDaemonSets[component] - - // Upgrade the component's self-hosted resource - // During this upgrade; the temporary/backup component will take over - if err := apiclient.TryRunCommand(func() error { - - if _, err := client.AppsV1().DaemonSets(newDS.ObjectMeta.Namespace).Update(newDS); err != nil { - return errors.Wrapf(err, "couldn't update self-hosted component's DaemonSet") - } - return nil - }, selfHostingFailureThreshold); err != nil { - return err - } - - // Wait for the component's old Pod to disappear - oldPod := controlPlaneResources[component].pod - if err := waiter.WaitForPodToDisappear(oldPod.ObjectMeta.Name); err != nil { - return err - } - - // Wait for the main, upgraded self-hosted component to come up - // Here we're talking to the temporary/backup component; the upgraded component is in the process of starting up - if err := waiter.WaitForPodsWithLabel(selfhosting.BuildSelfHostedComponentLabelQuery(component)); err != nil { - return err - } - - // Delete the temporary DaemonSet, and retry selfHostingFailureThreshold times if it errors out - // In order to pivot back to the upgraded API server, we kill the temporary/backup component - if err := apiclient.TryRunCommand(func() error { - return apiclient.DeleteDaemonSetForeground(client, tempDS.ObjectMeta.Namespace, tempDS.ObjectMeta.Name) - }, selfHostingFailureThreshold); err != nil { - return err - } - - // Just as an extra safety check; make sure the API server is returning ok at the /healthz endpoint - if err := waiter.WaitForAPI(); err != nil { - return err - } - - fmt.Printf("[upgrade/apply] Self-hosted component %q upgraded successfully!\n", component) - } - return nil -} - -// BuildUpgradedDaemonSetsFromConfig takes a config object and the current version and returns the DaemonSet objects to post to the master -func BuildUpgradedDaemonSetsFromConfig(cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) map[string]*apps.DaemonSet { - // Here the map of different mutators to use for the control plane's podspec is stored - mutators := selfhosting.GetMutatorsFromFeatureGates(cfg.FeatureGates) - // Get the new PodSpecs to use - controlPlanePods := controlplane.GetStaticPodSpecs(cfg, k8sVersion) - // Store the created DaemonSets in this map - controlPlaneDaemonSets := map[string]*apps.DaemonSet{} - - for _, component := range constants.MasterComponents { - podSpec := controlPlanePods[component].Spec - - // Build the full DaemonSet object from the PodSpec generated from the control plane phase and - // using the self-hosting mutators available from the selfhosting phase - ds := selfhosting.BuildDaemonSet(component, &podSpec, mutators) - controlPlaneDaemonSets[component] = ds - } - return controlPlaneDaemonSets -} - -// addTempUpgradeDSPrefix adds the upgradeTempDSPrefix to the specified DaemonSet name -func addTempUpgradeDSPrefix(currentName string) string { - return fmt.Sprintf("%s%s", upgradeTempDSPrefix, currentName) -} - -// buildTempUpgradeLabels returns the label string-string map for identifying the temporary -func buildTempUpgradeLabels(component string) map[string]string { - return map[string]string{ - upgradeTempLabel: component, - } -} - -// buildTempUpgradeDSLabelQuery creates the right query for matching -func buildTempUpgradeDSLabelQuery(component string) string { - return fmt.Sprintf("%s=%s", upgradeTempLabel, component) -} - -// mutateTempDaemonSet mutates the specified self-hosted DaemonSet for the specified component -// in a way that makes it possible to post a nearly identical, temporary DaemonSet as a backup -func mutateTempDaemonSet(tempDS *apps.DaemonSet, component string) { - // Prefix the name of the temporary DaemonSet with upgradeTempDSPrefix - tempDS.ObjectMeta.Name = addTempUpgradeDSPrefix(tempDS.ObjectMeta.Name) - // Set .Labels to something else than the "real" self-hosted components have - tempDS.ObjectMeta.Labels = buildTempUpgradeLabels(component) - tempDS.Spec.Selector.MatchLabels = buildTempUpgradeLabels(component) - tempDS.Spec.Template.ObjectMeta.Labels = buildTempUpgradeLabels(component) - // Clean all unnecessary ObjectMeta fields - tempDS.ObjectMeta = extractRelevantObjectMeta(tempDS.ObjectMeta) - // Reset .Status as we're posting a new object - tempDS.Status = apps.DaemonSetStatus{} -} - -// extractRelevantObjectMeta returns only the relevant parts of ObjectMeta required when creating -// a new, identical resource. We should not POST ResourceVersion, UUIDs, etc., only the name, labels, -// namespace and annotations should be preserved. -func extractRelevantObjectMeta(ob metav1.ObjectMeta) metav1.ObjectMeta { - return metav1.ObjectMeta{ - Name: ob.Name, - Namespace: ob.Namespace, - Labels: ob.Labels, - Annotations: ob.Annotations, - } -} - -// listPodsWithLabelSelector returns the relevant Pods for the given LabelSelector -func listPodsWithLabelSelector(client clientset.Interface, kvLabel string) (*v1.PodList, error) { - return client.CoreV1().Pods(metav1.NamespaceSystem).List(metav1.ListOptions{ - LabelSelector: kvLabel, - }) -} - -// getCurrentControlPlaneComponentResources returns a string-(Pod|DaemonSet) map for later use -func getCurrentControlPlaneComponentResources(client clientset.Interface) (map[string]controlPlaneComponentResources, error) { - controlPlaneResources := map[string]controlPlaneComponentResources{} - - for _, component := range constants.MasterComponents { - var podList *v1.PodList - var currentDS *apps.DaemonSet - - // Get the self-hosted pod associated with the component - podLabelSelector := selfhosting.BuildSelfHostedComponentLabelQuery(component) - if err := apiclient.TryRunCommand(func() error { - var tryrunerr error - podList, tryrunerr = listPodsWithLabelSelector(client, podLabelSelector) - return tryrunerr // note that tryrunerr is most likely nil here (in successful cases) - }, selfHostingFailureThreshold); err != nil { - return nil, err - } - - // Make sure that there are only one Pod with this label selector; otherwise unexpected things can happen - if len(podList.Items) > 1 { - return nil, errors.Errorf("too many pods with label selector %q found in the %s namespace", podLabelSelector, metav1.NamespaceSystem) - } - - // Get the component's DaemonSet object - dsName := constants.AddSelfHostedPrefix(component) - if err := apiclient.TryRunCommand(func() error { - var tryrunerr error - // Try to get the current self-hosted component - currentDS, tryrunerr = client.AppsV1().DaemonSets(metav1.NamespaceSystem).Get(dsName, metav1.GetOptions{}) - return tryrunerr // note that tryrunerr is most likely nil here (in successful cases) - }, selfHostingFailureThreshold); err != nil { - return nil, err - } - - // Add the associated resources to the map to return later - controlPlaneResources[component] = controlPlaneComponentResources{ - pod: &podList.Items[0], - daemonSet: currentDS, - } - } - return controlPlaneResources, nil -} diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go index b5497bb496f..a64c2163c1e 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -282,7 +282,7 @@ func performEtcdStaticPodUpgrade(client clientset.Interface, waiter apiclient.Wa if err != nil { return true, errors.Wrap(err, "failed to retrieve the current etcd version") } - currentEtcdVersionStr, ok := currentEtcdVersions[fmt.Sprintf("https://%s:%d", cfg.APIEndpoint.AdvertiseAddress, constants.EtcdListenClientPort)] + currentEtcdVersionStr, ok := currentEtcdVersions[fmt.Sprintf("https://%s:%d", cfg.LocalAPIEndpoint.AdvertiseAddress, constants.EtcdListenClientPort)] if !ok { fmt.Println(currentEtcdVersions) return true, errors.Wrap(err, "failed to retrieve the current etcd version") diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go index 96ed5ff2621..74c5623a8d2 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go @@ -56,9 +56,12 @@ kind: InitConfiguration nodeRegistration: name: foo criSocket: "" -apiEndpoint: - advertiseAddress: 1.2.3.4 +localAPIEndpoint: + advertiseAddress: 192.168.2.2 bindPort: 6443 +bootstrapTokens: +- token: ce3aa5.5ec8455bb76b379f + ttl: 24h --- apiVersion: kubeadm.k8s.io/v1beta1 kind: ClusterConfiguration @@ -67,22 +70,17 @@ apiServer: certSANs: null extraArgs: null certificatesDir: %s -controllerManagerExtraArgs: null etcd: local: dataDir: %s image: "" -featureFlags: null imageRepository: k8s.gcr.io kubernetesVersion: %s networking: dnsDomain: cluster.local podSubnet: "" serviceSubnet: 10.96.0.0/12 -schedulerExtraArgs: null -token: ce3aa5.5ec8455bb76b379f -tokenTTL: 24h -unifiedControlPlaneImage: "" +useHyperKubeImage: false ` ) diff --git a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go index 963119b4481..a419eb0c637 100644 --- a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go +++ b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go @@ -65,7 +65,7 @@ func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Int if clusterStatus.APIEndpoints == nil { clusterStatus.APIEndpoints = map[string]kubeadmapi.APIEndpoint{} } - clusterStatus.APIEndpoints[cfg.NodeRegistration.Name] = cfg.APIEndpoint + clusterStatus.APIEndpoints[cfg.NodeRegistration.Name] = cfg.LocalAPIEndpoint // Marshal the ClusterStatus back into YAML clusterStatusYaml, err := configutil.MarshalKubeadmConfigObject(clusterStatus) diff --git a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig_test.go b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig_test.go index 7ef7191760a..1b69f49569e 100644 --- a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig_test.go +++ b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig_test.go @@ -65,7 +65,7 @@ func TestUploadConfiguration(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t2 *testing.T) { initialcfg := &kubeadmapiv1beta1.InitConfiguration{ - APIEndpoint: kubeadmapiv1beta1.APIEndpoint{ + LocalAPIEndpoint: kubeadmapiv1beta1.APIEndpoint{ AdvertiseAddress: "1.2.3.4", }, ClusterConfiguration: kubeadmapiv1beta1.ClusterConfiguration{ @@ -95,7 +95,7 @@ func TestUploadConfiguration(t *testing.T) { status := &kubeadmapi.ClusterStatus{ APIEndpoints: map[string]kubeadmapi.APIEndpoint{ - "node-foo": cfg.APIEndpoint, + "node-foo": cfg.LocalAPIEndpoint, }, } diff --git a/cmd/kubeadm/app/preflight/BUILD b/cmd/kubeadm/app/preflight/BUILD index df5bbf93de0..0c248791a57 100644 --- a/cmd/kubeadm/app/preflight/BUILD +++ b/cmd/kubeadm/app/preflight/BUILD @@ -30,7 +30,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//vendor/github.com/PuerkitoBio/purell:go_default_library", "//vendor/github.com/blang/semver:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) @@ -47,6 +48,7 @@ go_test( "//cmd/kubeadm/app/apis/kubeadm/v1beta1:go_default_library", "//cmd/kubeadm/app/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", "//vendor/github.com/renstrom/dedent:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec/testing:go_default_library", diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index fcc101426d2..a077722c70b 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -22,7 +22,6 @@ import ( "crypto/tls" "crypto/x509" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -37,11 +36,11 @@ import ( "github.com/PuerkitoBio/purell" "github.com/blang/semver" - "github.com/golang/glog" - + "github.com/pkg/errors" netutil "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" versionutil "k8s.io/apimachinery/pkg/util/version" + "k8s.io/klog" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/images" @@ -86,7 +85,7 @@ func (e *Error) Preflight() bool { // Checker validates the state of the system to ensure kubeadm will be // successful as often as possible. type Checker interface { - Check() (warnings, errors []error) + Check() (warnings, errorList []error) Name() string } @@ -101,12 +100,12 @@ func (ContainerRuntimeCheck) Name() string { } // Check validates the container runtime -func (crc ContainerRuntimeCheck) Check() (warnings, errors []error) { - glog.V(1).Infoln("validating the container runtime") +func (crc ContainerRuntimeCheck) Check() (warnings, errorList []error) { + klog.V(1).Infoln("validating the container runtime") if err := crc.runtime.IsRunning(); err != nil { - errors = append(errors, err) + errorList = append(errorList, err) } - return warnings, errors + return warnings, errorList } // ServiceCheck verifies that the given service is enabled and active. If we do not @@ -127,8 +126,8 @@ func (sc ServiceCheck) Name() string { } // Check validates if the service is enabled and active. -func (sc ServiceCheck) Check() (warnings, errors []error) { - glog.V(1).Infoln("validating if the service is enabled and active") +func (sc ServiceCheck) Check() (warnings, errorList []error) { + klog.V(1).Infoln("validating if the service is enabled and active") initSystem, err := initsystem.GetInitSystem() if err != nil { return []error{err}, nil @@ -137,23 +136,23 @@ func (sc ServiceCheck) Check() (warnings, errors []error) { warnings = []error{} if !initSystem.ServiceExists(sc.Service) { - warnings = append(warnings, fmt.Errorf("%s service does not exist", sc.Service)) + warnings = append(warnings, errors.Errorf("%s service does not exist", sc.Service)) return warnings, nil } if !initSystem.ServiceIsEnabled(sc.Service) { warnings = append(warnings, - fmt.Errorf("%s service is not enabled, please run 'systemctl enable %s.service'", + errors.Errorf("%s service is not enabled, please run 'systemctl enable %s.service'", sc.Service, sc.Service)) } if sc.CheckIfActive && !initSystem.ServiceIsActive(sc.Service) { - errors = append(errors, - fmt.Errorf("%s service is not active, please run 'systemctl start %s.service'", + errorList = append(errorList, + errors.Errorf("%s service is not active, please run 'systemctl start %s.service'", sc.Service, sc.Service)) } - return warnings, errors + return warnings, errorList } // FirewalldCheck checks if firewalld is enabled or active. If it is, warn the user that there may be problems @@ -168,8 +167,8 @@ func (FirewalldCheck) Name() string { } // Check validates if the firewall is enabled and active. -func (fc FirewalldCheck) Check() (warnings, errors []error) { - glog.V(1).Infoln("validating if the firewall is enabled and active") +func (fc FirewalldCheck) Check() (warnings, errorList []error) { + klog.V(1).Infoln("validating if the firewall is enabled and active") initSystem, err := initsystem.GetInitSystem() if err != nil { return []error{err}, nil @@ -183,11 +182,11 @@ func (fc FirewalldCheck) Check() (warnings, errors []error) { if initSystem.ServiceIsActive("firewalld") { warnings = append(warnings, - fmt.Errorf("firewalld is active, please ensure ports %v are open or your cluster may not function correctly", + errors.Errorf("firewalld is active, please ensure ports %v are open or your cluster may not function correctly", fc.ports)) } - return warnings, errors + return warnings, errorList } // PortOpenCheck ensures the given port is available for use. @@ -205,18 +204,18 @@ func (poc PortOpenCheck) Name() string { } // Check validates if the particular port is available. -func (poc PortOpenCheck) Check() (warnings, errors []error) { - glog.V(1).Infof("validating availability of port %d", poc.port) - errors = []error{} +func (poc PortOpenCheck) Check() (warnings, errorList []error) { + klog.V(1).Infof("validating availability of port %d", poc.port) + errorList = []error{} ln, err := net.Listen("tcp", fmt.Sprintf(":%d", poc.port)) if err != nil { - errors = append(errors, fmt.Errorf("Port %d is in use", poc.port)) + errorList = append(errorList, errors.Errorf("Port %d is in use", poc.port)) } if ln != nil { ln.Close() } - return nil, errors + return nil, errorList } // IsPrivilegedUserCheck verifies user is privileged (linux - root, windows - Administrator) @@ -242,9 +241,9 @@ func (dac DirAvailableCheck) Name() string { } // Check validates if a directory does not exist or empty. -func (dac DirAvailableCheck) Check() (warnings, errors []error) { - glog.V(1).Infof("validating the existence and emptiness of directory %s", dac.Path) - errors = []error{} +func (dac DirAvailableCheck) Check() (warnings, errorList []error) { + klog.V(1).Infof("validating the existence and emptiness of directory %s", dac.Path) + errorList = []error{} // If it doesn't exist we are good: if _, err := os.Stat(dac.Path); os.IsNotExist(err) { return nil, nil @@ -252,17 +251,17 @@ func (dac DirAvailableCheck) Check() (warnings, errors []error) { f, err := os.Open(dac.Path) if err != nil { - errors = append(errors, fmt.Errorf("unable to check if %s is empty: %s", dac.Path, err)) - return nil, errors + errorList = append(errorList, errors.Wrapf(err, "unable to check if %s is empty", dac.Path)) + return nil, errorList } defer f.Close() _, err = f.Readdirnames(1) if err != io.EOF { - errors = append(errors, fmt.Errorf("%s is not empty", dac.Path)) + errorList = append(errorList, errors.Errorf("%s is not empty", dac.Path)) } - return nil, errors + return nil, errorList } // FileAvailableCheck checks that the given file does not already exist. @@ -280,13 +279,13 @@ func (fac FileAvailableCheck) Name() string { } // Check validates if the given file does not already exist. -func (fac FileAvailableCheck) Check() (warnings, errors []error) { - glog.V(1).Infof("validating the existence of file %s", fac.Path) - errors = []error{} +func (fac FileAvailableCheck) Check() (warnings, errorList []error) { + klog.V(1).Infof("validating the existence of file %s", fac.Path) + errorList = []error{} if _, err := os.Stat(fac.Path); err == nil { - errors = append(errors, fmt.Errorf("%s already exists", fac.Path)) + errorList = append(errorList, errors.Errorf("%s already exists", fac.Path)) } - return nil, errors + return nil, errorList } // FileExistingCheck checks that the given file does not already exist. @@ -304,13 +303,13 @@ func (fac FileExistingCheck) Name() string { } // Check validates if the given file already exists. -func (fac FileExistingCheck) Check() (warnings, errors []error) { - glog.V(1).Infof("validating the existence of file %s", fac.Path) - errors = []error{} +func (fac FileExistingCheck) Check() (warnings, errorList []error) { + klog.V(1).Infof("validating the existence of file %s", fac.Path) + errorList = []error{} if _, err := os.Stat(fac.Path); err != nil { - errors = append(errors, fmt.Errorf("%s doesn't exist", fac.Path)) + errorList = append(errorList, errors.Errorf("%s doesn't exist", fac.Path)) } - return nil, errors + return nil, errorList } // FileContentCheck checks that the given file contains the string Content. @@ -329,11 +328,11 @@ func (fcc FileContentCheck) Name() string { } // Check validates if the given file contains the given content. -func (fcc FileContentCheck) Check() (warnings, errors []error) { - glog.V(1).Infof("validating the contents of file %s", fcc.Path) +func (fcc FileContentCheck) Check() (warnings, errorList []error) { + klog.V(1).Infof("validating the contents of file %s", fcc.Path) f, err := os.Open(fcc.Path) if err != nil { - return nil, []error{fmt.Errorf("%s does not exist", fcc.Path)} + return nil, []error{errors.Errorf("%s does not exist", fcc.Path)} } lr := io.LimitReader(f, int64(len(fcc.Content))) @@ -342,11 +341,11 @@ func (fcc FileContentCheck) Check() (warnings, errors []error) { buf := &bytes.Buffer{} _, err = io.Copy(buf, lr) if err != nil { - return nil, []error{fmt.Errorf("%s could not be read", fcc.Path)} + return nil, []error{errors.Errorf("%s could not be read", fcc.Path)} } if !bytes.Equal(buf.Bytes(), fcc.Content) { - return nil, []error{fmt.Errorf("%s contents are not set to %s", fcc.Path, fcc.Content)} + return nil, []error{errors.Errorf("%s contents are not set to %s", fcc.Path, fcc.Content)} } return nil, []error{} @@ -371,12 +370,12 @@ func (ipc InPathCheck) Name() string { // Check validates if the given executable is present in the path. func (ipc InPathCheck) Check() (warnings, errs []error) { - glog.V(1).Infof("validating the presence of executable %s", ipc.executable) + klog.V(1).Infof("validating the presence of executable %s", ipc.executable) _, err := ipc.exec.LookPath(ipc.executable) if err != nil { if ipc.mandatory { // Return as an error: - return nil, []error{fmt.Errorf("%s not found in system path", ipc.executable)} + return nil, []error{errors.Errorf("%s not found in system path", ipc.executable)} } // Return as a warning: warningMessage := fmt.Sprintf("%s not found in system path", ipc.executable) @@ -400,18 +399,18 @@ func (HostnameCheck) Name() string { } // Check validates if hostname match dns sub domain regex. -func (hc HostnameCheck) Check() (warnings, errors []error) { - glog.V(1).Infof("checking whether the given node name is reachable using net.LookupHost") - errors = []error{} +func (hc HostnameCheck) Check() (warnings, errorList []error) { + klog.V(1).Infof("checking whether the given node name is reachable using net.LookupHost") + errorList = []error{} warnings = []error{} addr, err := net.LookupHost(hc.nodeName) if addr == nil { - warnings = append(warnings, fmt.Errorf("hostname \"%s\" could not be reached", hc.nodeName)) + warnings = append(warnings, errors.Errorf("hostname \"%s\" could not be reached", hc.nodeName)) } if err != nil { - warnings = append(warnings, fmt.Errorf("hostname \"%s\" %s", hc.nodeName, err)) + warnings = append(warnings, errors.Wrapf(err, "hostname \"%s\"", hc.nodeName)) } - return warnings, errors + return warnings, errorList } // HTTPProxyCheck checks if https connection to specific host is going @@ -427,8 +426,8 @@ func (hst HTTPProxyCheck) Name() string { } // Check validates http connectivity type, direct or via proxy. -func (hst HTTPProxyCheck) Check() (warnings, errors []error) { - glog.V(1).Infof("validating if the connectivity type is via proxy or direct") +func (hst HTTPProxyCheck) Check() (warnings, errorList []error) { + klog.V(1).Infof("validating if the connectivity type is via proxy or direct") u := (&url.URL{Scheme: hst.Proto, Host: hst.Host}).String() req, err := http.NewRequest("GET", u, nil) @@ -441,7 +440,7 @@ func (hst HTTPProxyCheck) Check() (warnings, errors []error) { return nil, []error{err} } if proxy != nil { - return []error{fmt.Errorf("Connection to %q uses proxy %q. If that is not intended, adjust your proxy settings", u, proxy)}, nil + return []error{errors.Errorf("Connection to %q uses proxy %q. If that is not intended, adjust your proxy settings", u, proxy)}, nil } return nil, nil } @@ -463,20 +462,20 @@ func (HTTPProxyCIDRCheck) Name() string { // Check validates http connectivity to first IP address in the CIDR. // If it is not directly connected and goes via proxy it will produce warning. -func (subnet HTTPProxyCIDRCheck) Check() (warnings, errors []error) { - glog.V(1).Infoln("validating http connectivity to first IP address in the CIDR") +func (subnet HTTPProxyCIDRCheck) Check() (warnings, errorList []error) { + klog.V(1).Infoln("validating http connectivity to first IP address in the CIDR") if len(subnet.CIDR) == 0 { return nil, nil } _, cidr, err := net.ParseCIDR(subnet.CIDR) if err != nil { - return nil, []error{fmt.Errorf("error parsing CIDR %q: %v", subnet.CIDR, err)} + return nil, []error{errors.Wrapf(err, "error parsing CIDR %q", subnet.CIDR)} } testIP, err := ipallocator.GetIndexedIP(cidr, 1) if err != nil { - return nil, []error{fmt.Errorf("unable to get first IP address from the given CIDR (%s): %v", cidr.String(), err)} + return nil, []error{errors.Wrapf(err, "unable to get first IP address from the given CIDR (%s)", cidr.String())} } testIPstring := testIP.String() @@ -496,7 +495,7 @@ func (subnet HTTPProxyCIDRCheck) Check() (warnings, errors []error) { return nil, []error{err} } if proxy != nil { - return []error{fmt.Errorf("connection to %q uses proxy %q. This may lead to malfunctional cluster setup. Make sure that Pod and Services IP ranges specified correctly as exceptions in proxy configuration", subnet.CIDR, proxy)}, nil + return []error{errors.Errorf("connection to %q uses proxy %q. This may lead to malfunctional cluster setup. Make sure that Pod and Services IP ranges specified correctly as exceptions in proxy configuration", subnet.CIDR, proxy)}, nil } return nil, nil } @@ -512,8 +511,8 @@ func (SystemVerificationCheck) Name() string { } // Check runs all individual checks -func (sysver SystemVerificationCheck) Check() (warnings, errors []error) { - glog.V(1).Infoln("running all checks") +func (sysver SystemVerificationCheck) Check() (warnings, errorList []error) { + klog.V(1).Infoln("running all checks") // Create a buffered writer and choose a quite large value (1M) and suppose the output from the system verification test won't exceed the limit // Run the system verification check, but write to out buffered writer instead of stdout bufw := bufio.NewWriterSize(os.Stdout, 1*1024*1024) @@ -569,8 +568,8 @@ func (KubernetesVersionCheck) Name() string { } // Check validates Kubernetes and kubeadm versions -func (kubever KubernetesVersionCheck) Check() (warnings, errors []error) { - glog.V(1).Infoln("validating Kubernetes and kubeadm version") +func (kubever KubernetesVersionCheck) Check() (warnings, errorList []error) { + klog.V(1).Infoln("validating Kubernetes and kubeadm version") // Skip this check for "super-custom builds", where apimachinery/the overall codebase version is not set. if strings.HasPrefix(kubever.KubeadmVersion, "v0.0.0") { return nil, nil @@ -578,12 +577,12 @@ func (kubever KubernetesVersionCheck) Check() (warnings, errors []error) { kadmVersion, err := versionutil.ParseSemantic(kubever.KubeadmVersion) if err != nil { - return nil, []error{fmt.Errorf("couldn't parse kubeadm version %q: %v", kubever.KubeadmVersion, err)} + return nil, []error{errors.Wrapf(err, "couldn't parse kubeadm version %q", kubever.KubeadmVersion)} } k8sVersion, err := versionutil.ParseSemantic(kubever.KubernetesVersion) if err != nil { - return nil, []error{fmt.Errorf("couldn't parse Kubernetes version %q: %v", kubever.KubernetesVersion, err)} + return nil, []error{errors.Wrapf(err, "couldn't parse Kubernetes version %q", kubever.KubernetesVersion)} } // Checks if k8sVersion greater or equal than the first unsupported versions by current version of kubeadm, @@ -592,7 +591,7 @@ func (kubever KubernetesVersionCheck) Check() (warnings, errors []error) { // thus setting the value to x.y.0-0 we are defining the very first patch - prereleases within x.y minor release. firstUnsupportedVersion := versionutil.MustParseSemantic(fmt.Sprintf("%d.%d.%s", kadmVersion.Major(), kadmVersion.Minor()+1, "0-0")) if k8sVersion.AtLeast(firstUnsupportedVersion) { - return []error{fmt.Errorf("Kubernetes version is greater than kubeadm version. Please consider to upgrade kubeadm. Kubernetes version: %s. Kubeadm version: %d.%d.x", k8sVersion, kadmVersion.Components()[0], kadmVersion.Components()[1])}, nil + return []error{errors.Errorf("Kubernetes version is greater than kubeadm version. Please consider to upgrade kubeadm. Kubernetes version: %s. Kubeadm version: %d.%d.x", k8sVersion, kadmVersion.Components()[0], kadmVersion.Components()[1])}, nil } return nil, nil @@ -610,23 +609,23 @@ func (KubeletVersionCheck) Name() string { } // Check validates kubelet version. It should be not less than minimal supported version -func (kubever KubeletVersionCheck) Check() (warnings, errors []error) { - glog.V(1).Infoln("validating kubelet version") +func (kubever KubeletVersionCheck) Check() (warnings, errorList []error) { + klog.V(1).Infoln("validating kubelet version") kubeletVersion, err := GetKubeletVersion(kubever.exec) if err != nil { - return nil, []error{fmt.Errorf("couldn't get kubelet version: %v", err)} + return nil, []error{errors.Wrap(err, "couldn't get kubelet version")} } if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletVersion) { - return nil, []error{fmt.Errorf("Kubelet version %q is lower than kubadm can support. Please upgrade kubelet", kubeletVersion)} + return nil, []error{errors.Errorf("Kubelet version %q is lower than kubadm can support. Please upgrade kubelet", kubeletVersion)} } if kubever.KubernetesVersion != "" { k8sVersion, err := versionutil.ParseSemantic(kubever.KubernetesVersion) if err != nil { - return nil, []error{fmt.Errorf("couldn't parse Kubernetes version %q: %v", kubever.KubernetesVersion, err)} + return nil, []error{errors.Wrapf(err, "couldn't parse Kubernetes version %q", kubever.KubernetesVersion)} } if kubeletVersion.Major() > k8sVersion.Major() || kubeletVersion.Minor() > k8sVersion.Minor() { - return nil, []error{fmt.Errorf("the kubelet version is higher than the control plane version. This is not a supported version skew and may lead to a malfunctional cluster. Kubelet version: %q Control plane version: %q", kubeletVersion, k8sVersion)} + return nil, []error{errors.Errorf("the kubelet version is higher than the control plane version. This is not a supported version skew and may lead to a malfunctional cluster. Kubelet version: %q Control plane version: %q", kubeletVersion, k8sVersion)} } } return nil, nil @@ -641,8 +640,8 @@ func (SwapCheck) Name() string { } // Check validates whether swap is enabled or not -func (swc SwapCheck) Check() (warnings, errors []error) { - glog.V(1).Infoln("validating whether swap is enabled or not") +func (swc SwapCheck) Check() (warnings, errorList []error) { + klog.V(1).Infoln("validating whether swap is enabled or not") f, err := os.Open("/proc/swaps") if err != nil { // /proc/swaps not available, thus no reasons to warn @@ -655,11 +654,11 @@ func (swc SwapCheck) Check() (warnings, errors []error) { buf = append(buf, scanner.Text()) } if err := scanner.Err(); err != nil { - return nil, []error{fmt.Errorf("error parsing /proc/swaps: %v", err)} + return nil, []error{errors.Wrap(err, "error parsing /proc/swaps")} } if len(buf) > 1 { - return nil, []error{fmt.Errorf("running with swap on is not supported. Please disable swap")} + return nil, []error{errors.New("running with swap on is not supported. Please disable swap")} } return nil, nil @@ -682,8 +681,8 @@ func (ExternalEtcdVersionCheck) Name() string { // Check validates external etcd version // TODO: Use the official etcd Golang client for this instead? -func (evc ExternalEtcdVersionCheck) Check() (warnings, errors []error) { - glog.V(1).Infoln("validating the external etcd version") +func (evc ExternalEtcdVersionCheck) Check() (warnings, errorList []error) { + klog.V(1).Infoln("validating the external etcd version") // Return quickly if the user isn't using external etcd if evc.Etcd.External.Endpoints == nil { @@ -693,46 +692,46 @@ func (evc ExternalEtcdVersionCheck) Check() (warnings, errors []error) { var config *tls.Config var err error if config, err = evc.configRootCAs(config); err != nil { - errors = append(errors, err) - return nil, errors + errorList = append(errorList, err) + return nil, errorList } if config, err = evc.configCertAndKey(config); err != nil { - errors = append(errors, err) - return nil, errors + errorList = append(errorList, err) + return nil, errorList } client := evc.getHTTPClient(config) for _, endpoint := range evc.Etcd.External.Endpoints { if _, err := url.Parse(endpoint); err != nil { - errors = append(errors, fmt.Errorf("failed to parse external etcd endpoint %s : %v", endpoint, err)) + errorList = append(errorList, errors.Wrapf(err, "failed to parse external etcd endpoint %s", endpoint)) continue } resp := etcdVersionResponse{} var err error versionURL := fmt.Sprintf("%s/%s", endpoint, "version") if tmpVersionURL, err := purell.NormalizeURLString(versionURL, purell.FlagRemoveDuplicateSlashes); err != nil { - errors = append(errors, fmt.Errorf("failed to normalize external etcd version url %s : %v", versionURL, err)) + errorList = append(errorList, errors.Wrapf(err, "failed to normalize external etcd version url %s", versionURL)) continue } else { versionURL = tmpVersionURL } if err = getEtcdVersionResponse(client, versionURL, &resp); err != nil { - errors = append(errors, err) + errorList = append(errorList, err) continue } etcdVersion, err := semver.Parse(resp.Etcdserver) if err != nil { - errors = append(errors, fmt.Errorf("couldn't parse external etcd version %q: %v", resp.Etcdserver, err)) + errorList = append(errorList, errors.Wrapf(err, "couldn't parse external etcd version %q", resp.Etcdserver)) continue } if etcdVersion.LT(minExternalEtcdVersion) { - errors = append(errors, fmt.Errorf("this version of kubeadm only supports external etcd version >= %s. Current version: %s", kubeadmconstants.MinExternalEtcdVersion, resp.Etcdserver)) + errorList = append(errorList, errors.Errorf("this version of kubeadm only supports external etcd version >= %s. Current version: %s", kubeadmconstants.MinExternalEtcdVersion, resp.Etcdserver)) continue } } - return nil, errors + return nil, errorList } // configRootCAs configures and returns a reference to tls.Config instance if CAFile is provided @@ -741,7 +740,7 @@ func (evc ExternalEtcdVersionCheck) configRootCAs(config *tls.Config) (*tls.Conf if evc.Etcd.External.CAFile != "" { CACert, err := ioutil.ReadFile(evc.Etcd.External.CAFile) if err != nil { - return nil, fmt.Errorf("couldn't load external etcd's server certificate %s: %v", evc.Etcd.External.CAFile, err) + return nil, errors.Wrapf(err, "couldn't load external etcd's server certificate %s", evc.Etcd.External.CAFile) } CACertPool = x509.NewCertPool() CACertPool.AppendCertsFromPEM(CACert) @@ -762,7 +761,7 @@ func (evc ExternalEtcdVersionCheck) configCertAndKey(config *tls.Config) (*tls.C var err error cert, err = tls.LoadX509KeyPair(evc.Etcd.External.CertFile, evc.Etcd.External.KeyFile) if err != nil { - return nil, fmt.Errorf("couldn't load external etcd's certificate and key pair %s, %s: %v", evc.Etcd.External.CertFile, evc.Etcd.External.KeyFile, err) + return nil, errors.Wrapf(err, "couldn't load external etcd's certificate and key pair %s, %s", evc.Etcd.External.CertFile, evc.Etcd.External.KeyFile) } if config == nil { config = &tls.Config{} @@ -803,7 +802,7 @@ func getEtcdVersionResponse(client *http.Client, url string, target interface{}) if r != nil && r.StatusCode >= 500 && r.StatusCode <= 599 { loopCount-- - return false, fmt.Errorf("server responded with non-successful status: %s", r.Status) + return false, errors.Errorf("server responded with non-successful status: %s", r.Status) } return true, json.NewDecoder(r.Body).Decode(target) @@ -827,22 +826,22 @@ func (ImagePullCheck) Name() string { } // Check pulls images required by kubeadm. This is a mutating check -func (ipc ImagePullCheck) Check() (warnings, errors []error) { +func (ipc ImagePullCheck) Check() (warnings, errorList []error) { for _, image := range ipc.imageList { ret, err := ipc.runtime.ImageExists(image) if ret && err == nil { - glog.V(1).Infof("image exists: %s", image) + klog.V(1).Infof("image exists: %s", image) continue } if err != nil { - errors = append(errors, fmt.Errorf("failed to check if image %s exists: %v", image, err)) + errorList = append(errorList, errors.Wrapf(err, "failed to check if image %s exists", image)) } - glog.V(1).Infof("pulling %s", image) + klog.V(1).Infof("pulling %s", image) if err := ipc.runtime.PullImage(image); err != nil { - errors = append(errors, fmt.Errorf("failed to pull image %s: %v", image, err)) + errorList = append(errorList, errors.Wrapf(err, "failed to pull image %s", image)) } } - return warnings, errors + return warnings, errorList } // NumCPUCheck checks if current number of CPUs is not less than required @@ -856,12 +855,12 @@ func (NumCPUCheck) Name() string { } // Check number of CPUs required by kubeadm -func (ncc NumCPUCheck) Check() (warnings, errors []error) { +func (ncc NumCPUCheck) Check() (warnings, errorList []error) { numCPU := runtime.NumCPU() if numCPU < ncc.NumCPU { - errors = append(errors, fmt.Errorf("the number of available CPUs %d is less than the required %d", numCPU, ncc.NumCPU)) + errorList = append(errorList, errors.Errorf("the number of available CPUs %d is less than the required %d", numCPU, ncc.NumCPU)) } - return warnings, errors + return warnings, errorList } // RunInitMasterChecks executes all individual, applicable to Master node checks. @@ -875,15 +874,15 @@ func RunInitMasterChecks(execer utilsexec.Interface, cfg *kubeadmapi.InitConfigu checks := []Checker{ NumCPUCheck{NumCPU: kubeadmconstants.MasterNumCPU}, KubernetesVersionCheck{KubernetesVersion: cfg.KubernetesVersion, KubeadmVersion: kubeadmversion.Get().GitVersion}, - FirewalldCheck{ports: []int{int(cfg.APIEndpoint.BindPort), 10250}}, - PortOpenCheck{port: int(cfg.APIEndpoint.BindPort)}, + FirewalldCheck{ports: []int{int(cfg.LocalAPIEndpoint.BindPort), 10250}}, + PortOpenCheck{port: int(cfg.LocalAPIEndpoint.BindPort)}, PortOpenCheck{port: 10251}, PortOpenCheck{port: 10252}, FileAvailableCheck{Path: kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.KubeAPIServer, manifestsDir)}, FileAvailableCheck{Path: kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.KubeControllerManager, manifestsDir)}, FileAvailableCheck{Path: kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.KubeScheduler, manifestsDir)}, FileAvailableCheck{Path: kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestsDir)}, - HTTPProxyCheck{Proto: "https", Host: cfg.APIEndpoint.AdvertiseAddress}, + HTTPProxyCheck{Proto: "https", Host: cfg.LocalAPIEndpoint.AdvertiseAddress}, HTTPProxyCIDRCheck{Proto: "https", CIDR: cfg.Networking.ServiceSubnet}, HTTPProxyCIDRCheck{Proto: "https", CIDR: cfg.Networking.PodSubnet}, } @@ -919,7 +918,7 @@ func RunInitMasterChecks(execer utilsexec.Interface, cfg *kubeadmapi.InitConfigu checks = append(checks, ExternalEtcdVersionCheck{Etcd: cfg.Etcd}) } - if ip := net.ParseIP(cfg.APIEndpoint.AdvertiseAddress); ip != nil { + if ip := net.ParseIP(cfg.LocalAPIEndpoint.AdvertiseAddress); ip != nil { if ip.To4() == nil && ip.To16() != nil { checks = append(checks, FileContentCheck{Path: bridgenf6, Content: []byte{'1'}}, @@ -941,10 +940,9 @@ func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.JoinConfigura DirAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName)}, FileAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)}, FileAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName)}, - ipvsutil.RequiredIPVSKernelModulesAvailableCheck{Executor: execer}, } checks = addCommonChecks(execer, cfg, checks) - if !cfg.ControlPlane { + if cfg.ControlPlane == nil { checks = append(checks, FileAvailableCheck{Path: cfg.CACertPath}) } @@ -974,6 +972,20 @@ func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.JoinConfigura return RunChecks(checks, os.Stderr, ignorePreflightErrors) } +// RunOptionalJoinNodeChecks executes all individual, applicable to node configuration dependant checks +func RunOptionalJoinNodeChecks(execer utilsexec.Interface, initCfg *kubeadmapi.InitConfiguration, ignorePreflightErrors sets.String) error { + checks := []Checker{} + + // Check ipvs required kernel module if we use ipvs kube-proxy mode + if initCfg.ComponentConfigs.KubeProxy != nil && initCfg.ComponentConfigs.KubeProxy.Mode == ipvsutil.IPVSProxyMode { + checks = append(checks, + ipvsutil.RequiredIPVSKernelModulesAvailableCheck{Executor: execer}, + ) + } + + return RunChecks(checks, os.Stderr, ignorePreflightErrors) +} + // addCommonChecks is a helper function to deplicate checks that are common between both the // kubeadm init and join commands func addCommonChecks(execer utilsexec.Interface, cfg kubeadmapi.CommonConfiguration, checks []Checker) []Checker { diff --git a/cmd/kubeadm/app/preflight/checks_test.go b/cmd/kubeadm/app/preflight/checks_test.go index 3f1986d4734..1b7c01295ca 100644 --- a/cmd/kubeadm/app/preflight/checks_test.go +++ b/cmd/kubeadm/app/preflight/checks_test.go @@ -18,11 +18,11 @@ package preflight import ( "bytes" - "fmt" "io/ioutil" "strings" "testing" + "github.com/pkg/errors" "github.com/renstrom/dedent" "net/http" @@ -173,12 +173,12 @@ func (pfct preflightCheckTest) Name() string { return "preflightCheckTest" } -func (pfct preflightCheckTest) Check() (warning, errors []error) { +func (pfct preflightCheckTest) Check() (warning, errorList []error) { if pfct.msg == "warning" { - return []error{fmt.Errorf("warning")}, nil + return []error{errors.New("warning")}, nil } if pfct.msg != "" { - return nil, []error{fmt.Errorf("fake error")} + return nil, []error{errors.New("fake error")} } return } @@ -191,7 +191,7 @@ func TestRunInitMasterChecks(t *testing.T) { }{ {name: "Test valid advertised address", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "foo"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "foo"}, }, expected: false, }, @@ -224,7 +224,7 @@ func TestRunInitMasterChecks(t *testing.T) { }, { cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "2001:1234::1:15"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "2001:1234::1:15"}, }, expected: false, }, diff --git a/cmd/kubeadm/app/preflight/checks_unix.go b/cmd/kubeadm/app/preflight/checks_unix.go index 1f2fb754e01..b04edaf12ce 100644 --- a/cmd/kubeadm/app/preflight/checks_unix.go +++ b/cmd/kubeadm/app/preflight/checks_unix.go @@ -19,16 +19,17 @@ limitations under the License. package preflight import ( - "fmt" "os" + + "github.com/pkg/errors" ) // Check validates if an user has elevated (root) privileges. -func (ipuc IsPrivilegedUserCheck) Check() (warnings, errors []error) { - errors = []error{} +func (ipuc IsPrivilegedUserCheck) Check() (warnings, errorList []error) { + errorList = []error{} if os.Getuid() != 0 { - errors = append(errors, fmt.Errorf("user is not running as root")) + errorList = append(errorList, errors.New("user is not running as root")) } - return nil, errors + return nil, errorList } diff --git a/cmd/kubeadm/app/preflight/checks_windows.go b/cmd/kubeadm/app/preflight/checks_windows.go index 1cc9bb3e59e..1477204248d 100644 --- a/cmd/kubeadm/app/preflight/checks_windows.go +++ b/cmd/kubeadm/app/preflight/checks_windows.go @@ -19,14 +19,15 @@ limitations under the License. package preflight import ( - "fmt" "os/exec" "strings" + + "github.com/pkg/errors" ) // Check validates if an user has elevated (administrator) privileges. -func (ipuc IsPrivilegedUserCheck) Check() (warnings, errors []error) { - errors = []error{} +func (ipuc IsPrivilegedUserCheck) Check() (warnings, errorList []error) { + errorList = []error{} // The "Well-known SID" of Administrator group is S-1-5-32-544 // The following powershell will return "True" if run as an administrator, "False" otherwise @@ -35,10 +36,10 @@ func (ipuc IsPrivilegedUserCheck) Check() (warnings, errors []error) { isAdmin, err := exec.Command("powershell", args...).Output() if err != nil { - errors = append(errors, fmt.Errorf("unable to determine if user is running as administrator: %s", err)) + errorList = append(errorList, errors.Wrap(err, "unable to determine if user is running as administrator")) } else if strings.EqualFold(strings.TrimSpace(string(isAdmin)), "false") { - errors = append(errors, fmt.Errorf("user is not running as administrator")) + errorList = append(errorList, errors.New("user is not running as administrator")) } - return nil, errors + return nil, errorList } diff --git a/cmd/kubeadm/app/preflight/utils.go b/cmd/kubeadm/app/preflight/utils.go index 3f94fd76af3..a7831de2d30 100644 --- a/cmd/kubeadm/app/preflight/utils.go +++ b/cmd/kubeadm/app/preflight/utils.go @@ -17,10 +17,11 @@ limitations under the License. package preflight import ( - "fmt" "regexp" "strings" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/version" utilsexec "k8s.io/utils/exec" ) @@ -38,7 +39,7 @@ func GetKubeletVersion(execer utilsexec.Interface) (*version.Version, error) { cleanOutput := strings.TrimSpace(string(out)) subs := kubeletVersionRegex.FindAllStringSubmatch(cleanOutput, -1) if len(subs) != 1 || len(subs[0]) < 2 { - return nil, fmt.Errorf("Unable to parse output from Kubelet: %q", cleanOutput) + return nil, errors.Errorf("Unable to parse output from Kubelet: %q", cleanOutput) } return version.ParseSemantic(subs[0][1]) } diff --git a/cmd/kubeadm/app/preflight/utils_test.go b/cmd/kubeadm/app/preflight/utils_test.go index 5708f073bd1..608f841c567 100644 --- a/cmd/kubeadm/app/preflight/utils_test.go +++ b/cmd/kubeadm/app/preflight/utils_test.go @@ -17,9 +17,10 @@ limitations under the License. package preflight import ( - "fmt" "testing" + "github.com/pkg/errors" + utilsexec "k8s.io/utils/exec" fakeexec "k8s.io/utils/exec/testing" ) @@ -34,7 +35,7 @@ func TestGetKubeletVersion(t *testing.T) { {"Kubernetes v1.7.0", "1.7.0", nil, true}, {"Kubernetes v1.8.0-alpha.2.1231+afabd012389d53a", "1.8.0-alpha.2.1231+afabd012389d53a", nil, true}, {"something-invalid", "", nil, false}, - {"command not found", "", fmt.Errorf("kubelet not found"), false}, + {"command not found", "", errors.New("kubelet not found"), false}, {"", "", nil, false}, } diff --git a/cmd/kubeadm/app/util/BUILD b/cmd/kubeadm/app/util/BUILD index 80aa1a8081d..becafa093b2 100644 --- a/cmd/kubeadm/app/util/BUILD +++ b/cmd/kubeadm/app/util/BUILD @@ -34,10 +34,10 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/cmd/kubeadm/app/util/apiclient/BUILD b/cmd/kubeadm/app/util/apiclient/BUILD index 17342168e13..666c2bada99 100644 --- a/cmd/kubeadm/app/util/apiclient/BUILD +++ b/cmd/kubeadm/app/util/apiclient/BUILD @@ -62,14 +62,17 @@ go_test( name = "go_default_test", srcs = [ "dryrunclient_test.go", + "idempotency_test.go", "init_dryrun_test.go", ], embed = [":go_default_library"], deps = [ + "//pkg/kubelet/apis:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", ], ) diff --git a/cmd/kubeadm/app/util/apiclient/idempotency.go b/cmd/kubeadm/app/util/apiclient/idempotency.go index b5475080f15..450c4bd426d 100644 --- a/cmd/kubeadm/app/util/apiclient/idempotency.go +++ b/cmd/kubeadm/app/util/apiclient/idempotency.go @@ -194,13 +194,16 @@ func CreateOrUpdateClusterRoleBinding(client clientset.Interface, clusterRoleBin return nil } -// PatchNode tries to patch a node using the following client, executing patchFn for the actual mutating logic -func PatchNode(client clientset.Interface, nodeName string, patchFn func(*v1.Node)) error { - // Loop on every false return. Return with an error if raised. Exit successfully if true is returned. - return wait.Poll(constants.APICallRetryInterval, constants.PatchNodeTimeout, func() (bool, error) { +// PatchNodeOnce executes patchFn on the node object found by the node name. +// This is a condition function meant to be used with wait.Poll. false, nil +// implies it is safe to try again, an error indicates no more tries should be +// made and true indicates success. +func PatchNodeOnce(client clientset.Interface, nodeName string, patchFn func(*v1.Node)) func() (bool, error) { + return func() (bool, error) { // First get the node object n, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) if err != nil { + // TODO this should only be for timeouts return false, nil } @@ -212,7 +215,7 @@ func PatchNode(client clientset.Interface, nodeName string, patchFn func(*v1.Nod oldData, err := json.Marshal(n) if err != nil { - return false, err + return false, errors.Wrapf(err, "failed to marshal unmodified node %q into JSON", n.Name) } // Execute the mutating function @@ -220,22 +223,32 @@ func PatchNode(client clientset.Interface, nodeName string, patchFn func(*v1.Nod newData, err := json.Marshal(n) if err != nil { - return false, err + return false, errors.Wrapf(err, "failed to marshal modified node %q into JSON", n.Name) } patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) if err != nil { - return false, err + return false, errors.Wrap(err, "failed to create two way merge patch") } if _, err := client.CoreV1().Nodes().Patch(n.Name, types.StrategicMergePatchType, patchBytes); err != nil { + // TODO also check for timeouts if apierrors.IsConflict(err) { fmt.Println("[patchnode] Temporarily unable to update node metadata due to conflict (will retry)") return false, nil } - return false, err + return false, errors.Wrapf(err, "error patching node %q through apiserver", n.Name) } return true, nil - }) + } +} + +// PatchNode tries to patch a node using patchFn for the actual mutating logic. +// Retries are provided by the wait package. +func PatchNode(client clientset.Interface, nodeName string, patchFn func(*v1.Node)) error { + // wait.Poll will rerun the condition function every interval function if + // the function returns false. If the condition function returns an error + // then the retries end and the error is returned. + return wait.Poll(constants.APICallRetryInterval, constants.PatchNodeTimeout, PatchNodeOnce(client, nodeName, patchFn)) } diff --git a/cmd/kubeadm/app/util/apiclient/idempotency_test.go b/cmd/kubeadm/app/util/apiclient/idempotency_test.go new file mode 100644 index 00000000000..986eabdad4d --- /dev/null +++ b/cmd/kubeadm/app/util/apiclient/idempotency_test.go @@ -0,0 +1,83 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiclient_test + +import ( + "testing" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" +) + +func TestPatchNodeNonErrorCases(t *testing.T) { + testcases := []struct { + name string + lookupName string + node v1.Node + success bool + }{ + { + name: "simple update", + lookupName: "testnode", + node: v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testnode", + Labels: map[string]string{kubeletapis.LabelHostname: ""}, + }, + }, + success: true, + }, + { + name: "node does not exist", + lookupName: "whale", + success: false, + }, + { + name: "node not labelled yet", + lookupName: "robin", + node: v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "robin", + }, + }, + success: false, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + client := fake.NewSimpleClientset() + _, err := client.Core().Nodes().Create(&tc.node) + if err != nil { + t.Fatalf("failed to create node to fake client: %v", err) + } + conditionFunction := apiclient.PatchNodeOnce(client, tc.lookupName, func(node *v1.Node) { + node.Name = "testNewNode" + }) + success, err := conditionFunction() + if err != nil { + t.Fatalf("did not expect error: %v", err) + } + if success != tc.success { + t.Fatalf("expected %v got %v", tc.success, success) + } + }) + } +} diff --git a/cmd/kubeadm/app/util/config/BUILD b/cmd/kubeadm/app/util/config/BUILD index 531318c4835..012f544e1ac 100644 --- a/cmd/kubeadm/app/util/config/BUILD +++ b/cmd/kubeadm/app/util/config/BUILD @@ -23,6 +23,7 @@ go_library( "//cmd/kubeadm/app/componentconfigs:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util:go_default_library", + "//cmd/kubeadm/app/util/config/strict:go_default_library", "//pkg/util/node:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", @@ -35,8 +36,8 @@ go_library( "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -76,6 +77,9 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//cmd/kubeadm/app/util/config/strict:all-srcs", + ], tags = ["automanaged"], ) diff --git a/cmd/kubeadm/app/util/config/cluster.go b/cmd/kubeadm/app/util/config/cluster.go index e1b95f5e0b2..d20c046a5b1 100644 --- a/cmd/kubeadm/app/util/config/cluster.go +++ b/cmd/kubeadm/app/util/config/cluster.go @@ -25,9 +25,7 @@ import ( "strings" "github.com/pkg/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/version" @@ -93,7 +91,7 @@ func getInitConfigurationFromCluster(kubeconfigDir string, client clientset.Inte // Also, the config map really should be KubeadmConfigConfigMap... configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(constants.KubeadmConfigConfigMap, metav1.GetOptions{}) if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to get config map") } // InitConfiguration is composed with data from different places @@ -105,12 +103,12 @@ func getInitConfigurationFromCluster(kubeconfigDir string, client clientset.Inte return nil, errors.Errorf("unexpected error when reading kubeadm-config ConfigMap: %s key value pair missing", constants.ClusterConfigurationConfigMapKey) } if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigurationData), &initcfg.ClusterConfiguration); err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to decode cluster configuration data") } // gets the component configs from the corresponding config maps if err := getComponentConfigs(client, &initcfg.ClusterConfiguration); err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to get component configs") } // if this isn't a new controlplane instance (e.g. in case of kubeadm upgrades) @@ -118,11 +116,11 @@ func getInitConfigurationFromCluster(kubeconfigDir string, client clientset.Inte if !newControlPlane { // gets the nodeRegistration for the current from the node object if err := getNodeRegistration(kubeconfigDir, client, &initcfg.NodeRegistration); err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to get node registration") } // gets the APIEndpoint for the current node from then ClusterStatus in the kubeadm-config ConfigMap - if err := getAPIEndpoint(configMap.Data, initcfg.NodeRegistration.Name, &initcfg.APIEndpoint); err != nil { - return nil, err + if err := getAPIEndpoint(configMap.Data, initcfg.NodeRegistration.Name, &initcfg.LocalAPIEndpoint); err != nil { + return nil, errors.Wrap(err, "failed to getAPIEndpoint") } } @@ -134,13 +132,13 @@ func getNodeRegistration(kubeconfigDir string, client clientset.Interface, nodeR // gets the name of the current node nodeName, err := getNodeNameFromKubeletConfig(kubeconfigDir) if err != nil { - return err + return errors.Wrap(err, "failed to get node name from kubelet config") } // gets the corresponding node and retrives attributes stored there. node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) if err != nil { - return err + return errors.Wrap(err, "faild to get corresponding node") } criSocket, ok := node.ObjectMeta.Annotations[constants.AnnotationKubeadmCRISocket] diff --git a/cmd/kubeadm/app/util/config/cluster_test.go b/cmd/kubeadm/app/util/config/cluster_test.go index ecc6bae7661..5605ca6046a 100644 --- a/cmd/kubeadm/app/util/config/cluster_test.go +++ b/cmd/kubeadm/app/util/config/cluster_test.go @@ -459,7 +459,7 @@ func TestGetAPIEndpoint(t *testing.T) { for _, rt := range tests { t.Run(rt.name, func(t *testing.T) { cfg := &kubeadmapi.InitConfiguration{} - err := getAPIEndpoint(rt.configMap.data, nodeName, &cfg.APIEndpoint) + err := getAPIEndpoint(rt.configMap.data, nodeName, &cfg.LocalAPIEndpoint) if rt.expectedError != (err != nil) { t.Errorf("unexpected return err from getInitConfigurationFromCluster: %v", err) return @@ -468,7 +468,7 @@ func TestGetAPIEndpoint(t *testing.T) { return } - if cfg.APIEndpoint.AdvertiseAddress != "1.2.3.4" || cfg.APIEndpoint.BindPort != 1234 { + if cfg.LocalAPIEndpoint.AdvertiseAddress != "1.2.3.4" || cfg.LocalAPIEndpoint.BindPort != 1234 { t.Errorf("invalid cfg.APIEndpoint") } }) @@ -757,8 +757,8 @@ func TestGetInitConfigurationFromCluster(t *testing.T) { if cfg.ClusterConfiguration.KubernetesVersion != k8sVersionString { t.Errorf("invalid ClusterConfiguration.KubernetesVersion") } - if !rt.newControlPlane && (cfg.APIEndpoint.AdvertiseAddress != "1.2.3.4" || cfg.APIEndpoint.BindPort != 1234) { - t.Errorf("invalid cfg.APIEndpoint") + if !rt.newControlPlane && (cfg.LocalAPIEndpoint.AdvertiseAddress != "1.2.3.4" || cfg.LocalAPIEndpoint.BindPort != 1234) { + t.Errorf("invalid cfg.LocalAPIEndpoint") } if cfg.ComponentConfigs.Kubelet == nil { t.Errorf("invalid cfg.ComponentConfigs.Kubelet") diff --git a/cmd/kubeadm/app/util/config/common.go b/cmd/kubeadm/app/util/config/common.go index 3dc712f7b06..660210706ab 100644 --- a/cmd/kubeadm/app/util/config/common.go +++ b/cmd/kubeadm/app/util/config/common.go @@ -19,10 +19,11 @@ package config import ( "io/ioutil" "net" + "reflect" "strings" - "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/klog" "k8s.io/apimachinery/pkg/runtime" netutil "k8s.io/apimachinery/pkg/util/net" @@ -103,7 +104,7 @@ func DetectUnsupportedVersion(b []byte) error { } } if mutuallyExclusiveCount > 1 { - glog.Warningf("WARNING: Detected resource kinds that may not apply: %v", mutuallyExclusive) + klog.Warningf("WARNING: Detected resource kinds that may not apply: %v", mutuallyExclusive) } return nil @@ -141,7 +142,7 @@ func LowercaseSANs(sans []string) { for i, san := range sans { lowercase := strings.ToLower(san) if lowercase != san { - glog.V(1).Infof("lowercasing SAN %q to %q", san, lowercase) + klog.V(1).Infof("lowercasing SAN %q to %q", san, lowercase) sans[i] = lowercase } } @@ -166,7 +167,7 @@ func ChooseAPIServerBindAddress(bindAddress net.IP) (net.IP, error) { ip, err := netutil.ChooseBindAddress(bindAddress) if err != nil { if netutil.IsNoRoutesError(err) { - glog.Warningf("WARNING: could not obtain a bind address for the API Server: %v; using: %s", err, constants.DefaultAPIServerBindAddress) + klog.Warningf("WARNING: could not obtain a bind address for the API Server: %v; using: %s", err, constants.DefaultAPIServerBindAddress) defaultIP := net.ParseIP(constants.DefaultAPIServerBindAddress) if defaultIP == nil { return nil, errors.Errorf("cannot parse default IP address: %s", constants.DefaultAPIServerBindAddress) @@ -175,5 +176,8 @@ func ChooseAPIServerBindAddress(bindAddress net.IP) (net.IP, error) { } return nil, err } + if bindAddress != nil && !bindAddress.IsUnspecified() && !reflect.DeepEqual(ip, bindAddress) { + klog.Warningf("WARNING: overriding requested API server bind address: requested %q, actual %q", bindAddress, ip) + } return ip, nil } diff --git a/cmd/kubeadm/app/util/config/initconfiguration.go b/cmd/kubeadm/app/util/config/initconfiguration.go index 4592a32c041..e425e0cdb63 100644 --- a/cmd/kubeadm/app/util/config/initconfiguration.go +++ b/cmd/kubeadm/app/util/config/initconfiguration.go @@ -25,8 +25,8 @@ import ( "sort" "strconv" - "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -39,6 +39,7 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/cmd/kubeadm/app/util/config/strict" nodeutil "k8s.io/kubernetes/pkg/util/node" ) @@ -50,10 +51,10 @@ func SetInitDynamicDefaults(cfg *kubeadmapi.InitConfiguration) error { if err := SetNodeRegistrationDynamicDefaults(&cfg.NodeRegistration, true); err != nil { return err } - if err := SetAPIEndpointDynamicDefaults(&cfg.APIEndpoint); err != nil { + if err := SetAPIEndpointDynamicDefaults(&cfg.LocalAPIEndpoint); err != nil { return err } - if err := SetClusterDynamicDefaults(&cfg.ClusterConfiguration, cfg.APIEndpoint.AdvertiseAddress, cfg.APIEndpoint.BindPort); err != nil { + if err := SetClusterDynamicDefaults(&cfg.ClusterConfiguration, cfg.LocalAPIEndpoint.AdvertiseAddress, cfg.LocalAPIEndpoint.BindPort); err != nil { return err } return nil @@ -165,7 +166,7 @@ func ConfigFileAndDefaultsToInternalConfig(cfgPath string, defaultversionedcfg * if cfgPath != "" { // Loads configuration from config file, if provided // Nb. --config overrides command line flags - glog.V(1).Infoln("loading configuration from the given file") + klog.V(1).Infoln("loading configuration from the given file") b, err := ioutil.ReadFile(cfgPath) if err != nil { @@ -211,6 +212,9 @@ func BytesToInternalConfig(b []byte) (*kubeadmapi.InitConfiguration, error) { } for gvk, fileContent := range gvkmap { + // verify the validity of the YAML + strict.VerifyUnmarshalStrict(fileContent, gvk) + // Try to get the registration for the ComponentConfig based on the kind regKind := componentconfigs.RegistrationKind(gvk.Kind) registration, found := componentconfigs.Known[regKind] diff --git a/cmd/kubeadm/app/util/config/joinconfiguration.go b/cmd/kubeadm/app/util/config/joinconfiguration.go index 278fa81c87e..168de0b509f 100644 --- a/cmd/kubeadm/app/util/config/joinconfiguration.go +++ b/cmd/kubeadm/app/util/config/joinconfiguration.go @@ -19,32 +19,45 @@ package config import ( "io/ioutil" - "github.com/golang/glog" "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/cmd/kubeadm/app/util/config/strict" ) // SetJoinDynamicDefaults checks and sets configuration values for the JoinConfiguration object func SetJoinDynamicDefaults(cfg *kubeadmapi.JoinConfiguration) error { - - if err := SetNodeRegistrationDynamicDefaults(&cfg.NodeRegistration, cfg.ControlPlane); err != nil { + addMasterTaint := false + if cfg.ControlPlane != nil { + addMasterTaint = true + } + if err := SetNodeRegistrationDynamicDefaults(&cfg.NodeRegistration, addMasterTaint); err != nil { return err } - if err := SetAPIEndpointDynamicDefaults(&cfg.APIEndpoint); err != nil { + if err := SetJoinControlPlaneDefaults(cfg.ControlPlane); err != nil { return err } return nil } +// SetJoinControlPlaneDefaults checks and sets configuration values for the JoinControlPlane object +func SetJoinControlPlaneDefaults(cfg *kubeadmapi.JoinControlPlane) error { + if cfg != nil { + if err := SetAPIEndpointDynamicDefaults(&cfg.LocalAPIEndpoint); err != nil { + return err + } + } + return nil +} + // JoinConfigFileAndDefaultsToInternalConfig takes a path to a config file and a versioned configuration that can serve as the default config // If cfgPath is specified, defaultversionedcfg will always get overridden. Otherwise, the default config (often populated by flags) will be used. // Then the external, versioned configuration is defaulted and converted to the internal type. @@ -56,7 +69,7 @@ func JoinConfigFileAndDefaultsToInternalConfig(cfgPath string, defaultversionedc if cfgPath != "" { // Loads configuration from config file, if provided // Nb. --config overrides command line flags, TODO: fix this - glog.V(1).Infoln("loading configuration from the given file") + klog.V(1).Infoln("loading configuration from the given file") b, err := ioutil.ReadFile(cfgPath) if err != nil { @@ -76,6 +89,8 @@ func JoinConfigFileAndDefaultsToInternalConfig(cfgPath string, defaultversionedc for gvk, bytes := range gvkmap { if gvk.Kind == constants.JoinConfigurationKind { joinBytes = bytes + // verify the validity of the YAML + strict.VerifyUnmarshalStrict(bytes, gvk) } } diff --git a/cmd/kubeadm/app/util/config/strict/BUILD b/cmd/kubeadm/app/util/config/strict/BUILD new file mode 100644 index 00000000000..3390d15e8a6 --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/BUILD @@ -0,0 +1,46 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["strict.go"], + importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/config/strict", + visibility = ["//visibility:public"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library", + "//cmd/kubeadm/app/componentconfigs:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["strict_test.go"], + data = glob(["testdata/**"]), + embed = [":go_default_library"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library", + "//cmd/kubeadm/app/apis/kubeadm/v1beta1:go_default_library", + "//cmd/kubeadm/app/componentconfigs:go_default_library", + "//cmd/kubeadm/app/constants:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/kube-proxy/config/v1alpha1:go_default_library", + "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/kubeadm/app/util/config/strict/strict.go b/cmd/kubeadm/app/util/config/strict/strict.go new file mode 100644 index 00000000000..e80799e2837 --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/strict.go @@ -0,0 +1,58 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package strict + +import ( + "github.com/pkg/errors" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" + "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" + "sigs.k8s.io/yaml" +) + +// VerifyUnmarshalStrict takes a YAML byte slice and a GroupVersionKind and verifies if the YAML +// schema is known and if it unmarshals with strict mode. +// +// TODO(neolit123): The returned error here is currently ignored everywhere and a klog warning is thrown instead. +// We don't want to turn this into an actual error yet. Eventually this can be controlled with an optional CLI flag. +func VerifyUnmarshalStrict(bytes []byte, gvk schema.GroupVersionKind) error { + + var ( + iface interface{} + err error + ) + + iface, err = scheme.Scheme.New(gvk) + if err != nil { + iface, err = componentconfigs.Scheme.New(gvk) + if err != nil { + err := errors.Errorf("unknown configuration %#v for scheme definitions in %q and %q", + gvk, scheme.Scheme.Name(), componentconfigs.Scheme.Name()) + klog.Warning(err.Error()) + return err + } + } + + if err := yaml.UnmarshalStrict(bytes, iface); err != nil { + err := errors.Wrapf(err, "error unmarshaling configuration %#v", gvk) + klog.Warning(err.Error()) + return err + } + return nil +} diff --git a/cmd/kubeadm/app/util/config/strict/strict_test.go b/cmd/kubeadm/app/util/config/strict/strict_test.go new file mode 100644 index 00000000000..9b905e77ec7 --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/strict_test.go @@ -0,0 +1,168 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package strict + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "k8s.io/apimachinery/pkg/runtime/schema" + kubeproxyconfigv1alpha1 "k8s.io/kube-proxy/config/v1alpha1" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" + kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" + kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" + "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" +) + +func TestVerifyUnmarshalStrict(t *testing.T) { + const ( + pathTestData = "testdata/" + ) + + var testFiles = []struct { + fileName string + kind string + groupVersion schema.GroupVersion + expectedError bool + }{ + // tests with file errors + { + fileName: "invalid_duplicate_field_clustercfg.yaml", + kind: constants.InitConfigurationKind, + groupVersion: kubeadmapiv1beta1.SchemeGroupVersion, + expectedError: true, + }, + { + fileName: "invalid_duplicate_field_initcfg.yaml", + kind: constants.InitConfigurationKind, + groupVersion: kubeadmapiv1alpha3.SchemeGroupVersion, + expectedError: true, + }, + { + fileName: "invalid_duplicate_field_joincfg.yaml", + kind: constants.JoinConfigurationKind, + groupVersion: kubeadmapiv1beta1.SchemeGroupVersion, + expectedError: true, + }, + { + fileName: "invalid_duplicate_field_kubeletcfg.yaml", + kind: string(componentconfigs.KubeletConfigurationKind), + groupVersion: kubeletconfigv1beta1.SchemeGroupVersion, + expectedError: true, + }, + { + fileName: "invalid_duplicate_field_kubeproxycfg.yaml", + kind: string(componentconfigs.KubeProxyConfigurationKind), + groupVersion: kubeproxyconfigv1alpha1.SchemeGroupVersion, + expectedError: true, + }, + { + fileName: "invalid_unknown_field_clustercfg.yaml", + kind: constants.ClusterConfigurationKind, + groupVersion: kubeadmapiv1beta1.SchemeGroupVersion, + expectedError: true, + }, + { + fileName: "invalid_unknown_field_initcfg.yaml", + kind: constants.InitConfigurationKind, + groupVersion: kubeadmapiv1beta1.SchemeGroupVersion, + expectedError: true, + }, + { + fileName: "invalid_unknown_field_joincfg.yaml", + kind: constants.JoinConfigurationKind, + groupVersion: kubeadmapiv1beta1.SchemeGroupVersion, + expectedError: true, + }, + { + fileName: "invalid_unknown_field_kubeletcfg.yaml", + kind: string(componentconfigs.KubeletConfigurationKind), + groupVersion: kubeletconfigv1beta1.SchemeGroupVersion, + expectedError: true, + }, + { + fileName: "invalid_unknown_field_kubeproxycfg.yaml", + kind: string(componentconfigs.KubeProxyConfigurationKind), + groupVersion: kubeproxyconfigv1alpha1.SchemeGroupVersion, + expectedError: true, + }, + // test unknown groupVersion and kind + { + fileName: "valid_clustercfg.yaml", + kind: constants.ClusterConfigurationKind, + groupVersion: schema.GroupVersion{Group: "someGroup", Version: "v1"}, + expectedError: true, + }, + { + fileName: "valid_clustercfg.yaml", + kind: "SomeUnknownKind", + groupVersion: kubeadmapiv1beta1.SchemeGroupVersion, + expectedError: true, + }, + // valid tests + { + fileName: "valid_clustercfg.yaml", + kind: constants.ClusterConfigurationKind, + groupVersion: kubeadmapiv1beta1.SchemeGroupVersion, + expectedError: false, + }, + { + fileName: "valid_initcfg.yaml", + kind: constants.InitConfigurationKind, + groupVersion: kubeadmapiv1beta1.SchemeGroupVersion, + expectedError: false, + }, + { + fileName: "valid_joincfg.yaml", + kind: constants.JoinConfigurationKind, + groupVersion: kubeadmapiv1beta1.SchemeGroupVersion, + expectedError: false, + }, + { + fileName: "valid_kubeletcfg.yaml", + kind: string(componentconfigs.KubeletConfigurationKind), + groupVersion: kubeletconfigv1beta1.SchemeGroupVersion, + expectedError: false, + }, + { + fileName: "valid_kubeproxycfg.yaml", + kind: string(componentconfigs.KubeProxyConfigurationKind), + groupVersion: kubeproxyconfigv1alpha1.SchemeGroupVersion, + expectedError: false, + }, + } + + for _, test := range testFiles { + t.Run(test.fileName, func(t *testing.T) { + bytes, err := ioutil.ReadFile(filepath.Join(pathTestData, test.fileName)) + if err != nil { + t.Fatalf("couldn't read test data: %v", err) + } + gvk := schema.GroupVersionKind{ + Group: test.groupVersion.Group, + Version: test.groupVersion.Version, + Kind: test.kind, + } + err = VerifyUnmarshalStrict(bytes, gvk) + if (err != nil) != test.expectedError { + t.Errorf("expected error %v, got %v, error: %v", err != nil, test.expectedError, err) + } + }) + } +} diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_clustercfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_clustercfg.yaml new file mode 100644 index 00000000000..71d8496e85f --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_clustercfg.yaml @@ -0,0 +1,4 @@ +apiVersion: kubeadm.k8s.io/v1beta1 +kind: ClusterConfiguration +controlPlaneEndpoint: test1 +controlPlaneEndpoint: test2 diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_initcfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_initcfg.yaml new file mode 100644 index 00000000000..b8313789319 --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_initcfg.yaml @@ -0,0 +1,6 @@ +apiVersion: kubeadm.k8s.io/v1beta1 +kind: InitConfiguration +bootstrapTokens: +- token: "9a08jv.c0izixklcxtmnze7" +bootstrapTokens: +- token: "9a08jv.c0izixklcxtmnze7" diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_joincfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_joincfg.yaml new file mode 100644 index 00000000000..f22fa6eb3a0 --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_joincfg.yaml @@ -0,0 +1,4 @@ +apiVersion: kubeadm.k8s.io/v1beta1 +kind: JoinConfiguration +caCertPath: relativepath +caCertPath: relativepath diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_kubeletcfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_kubeletcfg.yaml new file mode 100644 index 00000000000..2578fb75dfa --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_kubeletcfg.yaml @@ -0,0 +1,4 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +address: 1.2.3.4 +address: 1.2.3.4 diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_kubeproxycfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_kubeproxycfg.yaml new file mode 100644 index 00000000000..7e232dcc823 --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_kubeproxycfg.yaml @@ -0,0 +1,4 @@ +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +portRange: "" +portRange: "" diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_clustercfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_clustercfg.yaml new file mode 100644 index 00000000000..2ba38b5355f --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_clustercfg.yaml @@ -0,0 +1,3 @@ +apiVersion: kubeadm.k8s.io/v1beta1 +kind: ClusterConfiguration +unknownField: test diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_initcfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_initcfg.yaml new file mode 100644 index 00000000000..476507e2a0e --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_initcfg.yaml @@ -0,0 +1,3 @@ +apiVersion: kubeadm.k8s.io/v1beta1 +kind: InitConfiguration +unknownField: test diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_joincfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_joincfg.yaml new file mode 100644 index 00000000000..93d0660e85e --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_joincfg.yaml @@ -0,0 +1,3 @@ +apiVersion: kubeadm.k8s.io/v1beta1 +kind: JoinConfiguration +unknownField: test diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_kubeletcfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_kubeletcfg.yaml new file mode 100644 index 00000000000..75e0e97b88b --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_kubeletcfg.yaml @@ -0,0 +1,3 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +unknownField: unknown diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_kubeproxycfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_kubeproxycfg.yaml new file mode 100644 index 00000000000..7529f30507f --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_kubeproxycfg.yaml @@ -0,0 +1,3 @@ +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +unknownField: unknown diff --git a/cmd/kubeadm/app/util/config/strict/testdata/valid_clustercfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/valid_clustercfg.yaml new file mode 100644 index 00000000000..5c5ad371841 --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/valid_clustercfg.yaml @@ -0,0 +1,3 @@ +apiVersion: kubeadm.k8s.io/v1beta1 +kind: ClusterConfiguration +controlPlaneEndpoint: 202.0.100.1 diff --git a/cmd/kubeadm/app/util/config/strict/testdata/valid_initcfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/valid_initcfg.yaml new file mode 100644 index 00000000000..aeb36bb7849 --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/valid_initcfg.yaml @@ -0,0 +1,4 @@ +apiVersion: kubeadm.k8s.io/v1beta1 +kind: InitConfiguration +bootstrapTokens: +- token: "9a08jv.c0izixklcxtmnze7" diff --git a/cmd/kubeadm/app/util/config/strict/testdata/valid_joincfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/valid_joincfg.yaml new file mode 100644 index 00000000000..65ec2261aa8 --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/valid_joincfg.yaml @@ -0,0 +1,3 @@ +apiVersion: kubeadm.k8s.io/v1beta1 +kind: JoinConfiguration +caCertPath: relativepath diff --git a/cmd/kubeadm/app/util/config/strict/testdata/valid_kubeletcfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/valid_kubeletcfg.yaml new file mode 100644 index 00000000000..f85f7d52c09 --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/valid_kubeletcfg.yaml @@ -0,0 +1,3 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +address: 1.2.3.4 diff --git a/cmd/kubeadm/app/util/config/strict/testdata/valid_kubeproxycfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/valid_kubeproxycfg.yaml new file mode 100644 index 00000000000..955b5dd78e8 --- /dev/null +++ b/cmd/kubeadm/app/util/config/strict/testdata/valid_kubeproxycfg.yaml @@ -0,0 +1,3 @@ +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +portRange: "" diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml index d1938ed2ec7..b6c3c66a889 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml @@ -1,15 +1,19 @@ -APIEndpoint: - AdvertiseAddress: 192.168.2.2 - BindPort: 6443 APIServer: CertSANs: null ExtraArgs: authorization-mode: Node,RBAC,Webhook - ExtraVolumes: null -AuditPolicyConfiguration: - LogDir: /var/log/kubernetes/audit - LogMaxAge: 2 - Path: "" + ExtraVolumes: + - HostPath: /host/read-only + MountPath: /mount/read-only + Name: ReadOnlyVolume + PathType: "" + ReadOnly: true + - HostPath: /host/writable + MountPath: /mount/writable + Name: WritableVolume + PathType: "" + ReadOnly: false + TimeoutForControlPlane: 4m0s BootstrapTokens: - Description: "" Expires: null @@ -130,6 +134,7 @@ ComponentConfigs: MaxOpenFiles: 1000000 MaxPods: 110 NodeLeaseDurationSeconds: 40 + NodeStatusReportFrequency: 1m0s NodeStatusUpdateFrequency: 10s OOMScoreAdj: -999 PodCIDR: "" @@ -163,17 +168,25 @@ ControlPlaneEndpoint: "" ControllerManager: ExtraArgs: null ExtraVolumes: null +DNS: + ImageRepository: "" + ImageTag: "" + Type: CoreDNS Etcd: External: null Local: DataDir: /var/lib/etcd ExtraArgs: null - Image: "" + ImageRepository: "" + ImageTag: "" PeerCertSANs: null ServerCertSANs: null FeatureGates: null ImageRepository: k8s.gcr.io KubernetesVersion: v1.11.2 +LocalAPIEndpoint: + AdvertiseAddress: 192.168.2.2 + BindPort: 6443 Networking: DNSDomain: cluster.local PodSubnet: "" @@ -188,4 +201,4 @@ NodeRegistration: Scheduler: ExtraArgs: null ExtraVolumes: null -UnifiedControlPlaneImage: "" +UseHyperKubeImage: true diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml index 4c9aadbd336..339e4f07957 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml @@ -20,6 +20,14 @@ nodeRegistration: --- apiServerExtraArgs: authorization-mode: Node,RBAC,Webhook +apiServerExtraVolumes: +- hostPath: /host/read-only + mountPath: /mount/read-only + name: ReadOnlyVolume +- hostPath: /host/writable + mountPath: /mount/writable + name: WritableVolume + writable: true apiVersion: kubeadm.k8s.io/v1alpha3 auditPolicy: logDir: /var/log/kubernetes/audit @@ -39,7 +47,7 @@ networking: dnsDomain: cluster.local podSubnet: "" serviceSubnet: 10.96.0.0/12 -unifiedControlPlaneImage: "" +unifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.11.2" --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 @@ -140,6 +148,7 @@ makeIPTablesUtilChains: true maxOpenFiles: 1000000 maxPods: 110 nodeLeaseDurationSeconds: 40 +nodeStatusReportFrequency: 1m0s nodeStatusUpdateFrequency: 10s oomScoreAdj: -999 podPidsLimit: -1 diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1beta1.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1beta1.yaml index 8d2fa38d448..bc8abfa27ba 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1beta1.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1beta1.yaml @@ -1,6 +1,3 @@ -apiEndpoint: - advertiseAddress: 192.168.2.2 - bindPort: 6443 apiVersion: kubeadm.k8s.io/v1beta1 bootstrapTokens: - groups: @@ -11,6 +8,9 @@ bootstrapTokens: - signing - authentication kind: InitConfiguration +localAPIEndpoint: + advertiseAddress: 192.168.2.2 + bindPort: 6443 nodeRegistration: criSocket: /var/run/dockershim.sock name: master-1 @@ -21,19 +21,25 @@ nodeRegistration: apiServer: extraArgs: authorization-mode: Node,RBAC,Webhook + extraVolumes: + - hostPath: /host/read-only + mountPath: /mount/read-only + name: ReadOnlyVolume + readOnly: true + - hostPath: /host/writable + mountPath: /mount/writable + name: WritableVolume + timeoutForControlPlane: 4m0s apiVersion: kubeadm.k8s.io/v1beta1 -auditPolicy: - logDir: /var/log/kubernetes/audit - logMaxAge: 2 - path: "" certificatesDir: /etc/kubernetes/pki clusterName: kubernetes controlPlaneEndpoint: "" controllerManager: {} +dns: + type: CoreDNS etcd: local: dataDir: /var/lib/etcd - image: "" imageRepository: k8s.gcr.io kind: ClusterConfiguration kubernetesVersion: v1.11.2 @@ -42,7 +48,7 @@ networking: podSubnet: "" serviceSubnet: 10.96.0.0/12 scheduler: {} -unifiedControlPlaneImage: "" +useHyperKubeImage: true --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 @@ -143,6 +149,7 @@ makeIPTablesUtilChains: true maxOpenFiles: 1000000 maxPods: 110 nodeLeaseDurationSeconds: 40 +nodeStatusReportFrequency: 1m0s nodeStatusUpdateFrequency: 10s oomScoreAdj: -999 podPidsLimit: -1 diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml index a1c6d4c537c..66219eaabc4 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml @@ -1,9 +1,8 @@ -APIEndpoint: - AdvertiseAddress: 192.168.2.2 - BindPort: 6443 CACertPath: /etc/kubernetes/pki/ca.crt -ClusterName: kubernetes -ControlPlane: false +ControlPlane: + LocalAPIEndpoint: + AdvertiseAddress: 192.168.2.2 + BindPort: 6443 Discovery: BootstrapToken: APIServerEndpoint: kube-apiserver:6443 @@ -13,9 +12,10 @@ Discovery: File: null TLSBootstrapToken: abcdef.0123456789abcdef Timeout: 5m0s -FeatureGates: null NodeRegistration: CRISocket: /var/run/dockershim.sock KubeletExtraArgs: null Name: master-1 - Taints: null + Taints: + - effect: NoSchedule + key: node-role.kubernetes.io/master diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/node/v1alpha3.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/node/v1alpha3.yaml index de9e2412fbc..a2aef712140 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/node/v1alpha3.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/node/v1alpha3.yaml @@ -3,7 +3,7 @@ apiEndpoint: bindPort: 6443 apiVersion: kubeadm.k8s.io/v1alpha3 caCertPath: /etc/kubernetes/pki/ca.crt -clusterName: kubernetes +controlPlane: true discoveryFile: "" discoveryTimeout: 5m0s discoveryToken: abcdef.0123456789abcdef @@ -14,5 +14,8 @@ kind: JoinConfiguration nodeRegistration: criSocket: /var/run/dockershim.sock name: master-1 + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/master tlsBootstrapToken: abcdef.0123456789abcdef token: abcdef.0123456789abcdef diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/node/v1beta1.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/node/v1beta1.yaml index a6b1b45a8f6..05ce7a0083e 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/node/v1beta1.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/node/v1beta1.yaml @@ -1,9 +1,9 @@ -apiEndpoint: - advertiseAddress: 192.168.2.2 - bindPort: 6443 apiVersion: kubeadm.k8s.io/v1beta1 caCertPath: /etc/kubernetes/pki/ca.crt -clusterName: kubernetes +controlPlane: + localAPIEndpoint: + advertiseAddress: 192.168.2.2 + bindPort: 6443 discovery: bootstrapToken: apiServerEndpoint: kube-apiserver:6443 @@ -15,3 +15,6 @@ kind: JoinConfiguration nodeRegistration: criSocket: /var/run/dockershim.sock name: master-1 + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/master diff --git a/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml b/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml index e8346d4f679..4456eb175ce 100644 --- a/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml +++ b/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml @@ -1,6 +1,3 @@ -apiEndpoint: - advertiseAddress: 192.168.2.2 - bindPort: 6443 apiVersion: kubeadm.k8s.io/v1beta1 bootstrapTokens: - groups: @@ -11,6 +8,9 @@ bootstrapTokens: - signing - authentication kind: InitConfiguration +localAPIEndpoint: + advertiseAddress: 192.168.2.2 + bindPort: 6443 nodeRegistration: criSocket: /var/run/criruntime.sock name: master-1 @@ -18,20 +18,18 @@ nodeRegistration: - effect: NoSchedule key: node-role.kubernetes.io/master --- -apiServer: {} +apiServer: + timeoutForControlPlane: 4m0s apiVersion: kubeadm.k8s.io/v1beta1 -auditPolicy: - logDir: /var/log/kubernetes/audit - logMaxAge: 2 - path: "" certificatesDir: /var/lib/kubernetes/pki clusterName: kubernetes controlPlaneEndpoint: "" controllerManager: {} +dns: + type: CoreDNS etcd: local: dataDir: /var/lib/etcd - image: "" imageRepository: my-company.com kind: ClusterConfiguration kubernetesVersion: v1.12.0 @@ -40,7 +38,6 @@ networking: podSubnet: 10.148.0.0/16 serviceSubnet: 10.196.0.0/12 scheduler: {} -unifiedControlPlaneImage: "" --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 @@ -138,6 +135,7 @@ makeIPTablesUtilChains: true maxOpenFiles: 1000000 maxPods: 110 nodeLeaseDurationSeconds: 40 +nodeStatusReportFrequency: 1m0s nodeStatusUpdateFrequency: 10s oomScoreAdj: -999 podPidsLimit: -1 diff --git a/cmd/kubeadm/app/util/config/testdata/defaulting/master/incomplete.yaml b/cmd/kubeadm/app/util/config/testdata/defaulting/master/incomplete.yaml index d402a1bfe15..362e8e31357 100644 --- a/cmd/kubeadm/app/util/config/testdata/defaulting/master/incomplete.yaml +++ b/cmd/kubeadm/app/util/config/testdata/defaulting/master/incomplete.yaml @@ -1,9 +1,9 @@ apiVersion: kubeadm.k8s.io/v1beta1 kind: InitConfiguration -apiEndpoint: - advertiseAddress: 192.168.2.2 bootstrapTokens: - token: s73ybu.6tw6wnqgp5z0wb77 +localAPIEndpoint: + advertiseAddress: 192.168.2.2 nodeRegistration: criSocket: /var/run/criruntime.sock name: master-1 diff --git a/cmd/kubeadm/app/util/config/testdata/defaulting/node/defaulted.yaml b/cmd/kubeadm/app/util/config/testdata/defaulting/node/defaulted.yaml index 4de98d37a3c..e8227edcae5 100644 --- a/cmd/kubeadm/app/util/config/testdata/defaulting/node/defaulted.yaml +++ b/cmd/kubeadm/app/util/config/testdata/defaulting/node/defaulted.yaml @@ -1,9 +1,5 @@ -apiEndpoint: - advertiseAddress: 192.168.2.2 - bindPort: 6443 apiVersion: kubeadm.k8s.io/v1beta1 caCertPath: /etc/kubernetes/pki/ca.crt -clusterName: kubernetes discovery: bootstrapToken: apiServerEndpoint: kube-apiserver:6443 diff --git a/cmd/kubeadm/app/util/config/testdata/defaulting/node/incomplete.yaml b/cmd/kubeadm/app/util/config/testdata/defaulting/node/incomplete.yaml index e335a18a522..ff990c1a70c 100644 --- a/cmd/kubeadm/app/util/config/testdata/defaulting/node/incomplete.yaml +++ b/cmd/kubeadm/app/util/config/testdata/defaulting/node/incomplete.yaml @@ -1,5 +1,3 @@ -apiEndpoint: - advertiseAddress: 192.168.2.2 apiVersion: kubeadm.k8s.io/v1alpha3 discoveryToken: abcdef.0123456789abcdef discoveryTokenAPIServers: diff --git a/cmd/kubeadm/app/util/config/testdata/validation/invalid_mastercfg.yaml b/cmd/kubeadm/app/util/config/testdata/validation/invalid_mastercfg.yaml index 34924a75acc..55c7fb1b5ad 100644 --- a/cmd/kubeadm/app/util/config/testdata/validation/invalid_mastercfg.yaml +++ b/cmd/kubeadm/app/util/config/testdata/validation/invalid_mastercfg.yaml @@ -4,4 +4,4 @@ networking: dnsDomain: INVALID-DOMAIN-!!!! podSubnet: "" serviceSubnet: 10.96.0.0/12 -unifiedControlPlaneImage: "" \ No newline at end of file +useHyperKubeImage: false \ No newline at end of file diff --git a/cmd/kubeadm/app/util/endpoint.go b/cmd/kubeadm/app/util/endpoint.go index 0f52a0b782a..8760aec6b0b 100644 --- a/cmd/kubeadm/app/util/endpoint.go +++ b/cmd/kubeadm/app/util/endpoint.go @@ -34,15 +34,15 @@ import ( // - Otherwise, in case the ControlPlaneEndpoint is not defined, use the api.AdvertiseAddress + the api.BindPort. func GetMasterEndpoint(cfg *kubeadmapi.InitConfiguration) (string, error) { // parse the bind port - bindPortString := strconv.Itoa(int(cfg.APIEndpoint.BindPort)) + bindPortString := strconv.Itoa(int(cfg.LocalAPIEndpoint.BindPort)) if _, err := ParsePort(bindPortString); err != nil { - return "", errors.Wrapf(err, "invalid value %q given for api.bindPort", cfg.APIEndpoint.BindPort) + return "", errors.Wrapf(err, "invalid value %q given for api.bindPort", cfg.LocalAPIEndpoint.BindPort) } // parse the AdvertiseAddress - var ip = net.ParseIP(cfg.APIEndpoint.AdvertiseAddress) + var ip = net.ParseIP(cfg.LocalAPIEndpoint.AdvertiseAddress) if ip == nil { - return "", errors.Errorf("invalid value `%s` given for api.advertiseAddress", cfg.APIEndpoint.AdvertiseAddress) + return "", errors.Errorf("invalid value `%s` given for api.advertiseAddress", cfg.LocalAPIEndpoint.AdvertiseAddress) } // set the master url using cfg.API.AdvertiseAddress + the cfg.API.BindPort diff --git a/cmd/kubeadm/app/util/endpoint_test.go b/cmd/kubeadm/app/util/endpoint_test.go index cdc877fe333..21b5931ba77 100644 --- a/cmd/kubeadm/app/util/endpoint_test.go +++ b/cmd/kubeadm/app/util/endpoint_test.go @@ -32,7 +32,7 @@ func TestGetMasterEndpoint(t *testing.T) { { name: "use ControlPlaneEndpoint (dns) if fully defined", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ BindPort: 4567, AdvertiseAddress: "4.5.6.7", }, @@ -45,7 +45,7 @@ func TestGetMasterEndpoint(t *testing.T) { { name: "use ControlPlaneEndpoint (ipv4) if fully defined", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ BindPort: 4567, AdvertiseAddress: "4.5.6.7", }, @@ -58,7 +58,7 @@ func TestGetMasterEndpoint(t *testing.T) { { name: "use ControlPlaneEndpoint (ipv6) if fully defined", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ BindPort: 4567, AdvertiseAddress: "4.5.6.7", }, @@ -71,7 +71,7 @@ func TestGetMasterEndpoint(t *testing.T) { { name: "use ControlPlaneEndpoint (dns) + BindPort if ControlPlaneEndpoint defined without port", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ BindPort: 4567, AdvertiseAddress: "4.5.6.7", }, @@ -85,7 +85,7 @@ func TestGetMasterEndpoint(t *testing.T) { { name: "use ControlPlaneEndpoint (ipv4) + BindPort if ControlPlaneEndpoint defined without port", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ BindPort: 4567, AdvertiseAddress: "4.5.6.7", }, @@ -98,7 +98,7 @@ func TestGetMasterEndpoint(t *testing.T) { { name: "use ControlPlaneEndpoint (ipv6) + BindPort if ControlPlaneEndpoint defined without port", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ BindPort: 4567, AdvertiseAddress: "4.5.6.7", }, @@ -112,7 +112,7 @@ func TestGetMasterEndpoint(t *testing.T) { { name: "use AdvertiseAddress (ipv4) + BindPort if ControlPlaneEndpoint is not defined", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ BindPort: 4567, AdvertiseAddress: "4.5.6.7", }, @@ -122,7 +122,7 @@ func TestGetMasterEndpoint(t *testing.T) { { name: "use AdvertiseAddress (ipv6) + BindPort if ControlPlaneEndpoint is not defined", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ BindPort: 4567, AdvertiseAddress: "2001:db8::1", }, @@ -132,7 +132,7 @@ func TestGetMasterEndpoint(t *testing.T) { { name: "fail if invalid BindPort", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ BindPort: 0, }, }, @@ -177,7 +177,7 @@ func TestGetMasterEndpoint(t *testing.T) { { name: "fail if invalid AdvertiseAddress (ip4)", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ AdvertiseAddress: "1..0", BindPort: 4567, }, @@ -187,7 +187,7 @@ func TestGetMasterEndpoint(t *testing.T) { { name: "fail if invalid AdvertiseAddress (ip6)", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ AdvertiseAddress: "1200::AB00:1234::2552:7777:1313", BindPort: 4567, }, diff --git a/cmd/kubeadm/app/util/etcd/BUILD b/cmd/kubeadm/app/util/etcd/BUILD index b3792bb4aef..bb666f7cabb 100644 --- a/cmd/kubeadm/app/util/etcd/BUILD +++ b/cmd/kubeadm/app/util/etcd/BUILD @@ -13,8 +13,8 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//vendor/github.com/coreos/etcd/clientv3:go_default_library", "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/kubeadm/app/util/etcd/etcd.go b/cmd/kubeadm/app/util/etcd/etcd.go index a11bba3cfe0..c41b0ccdadd 100644 --- a/cmd/kubeadm/app/util/etcd/etcd.go +++ b/cmd/kubeadm/app/util/etcd/etcd.go @@ -26,9 +26,9 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/pkg/transport" - "github.com/golang/glog" "github.com/pkg/errors" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util/config" @@ -113,27 +113,55 @@ func New(endpoints []string, ca, cert, key string) (*Client, error) { return &client, nil } -// NewFromStaticPod creates a GenericClient from the given endpoints, manifestDir, and certificatesDir -func NewFromStaticPod(endpoints []string, manifestDir string, certificatesDir string) (*Client, error) { - hasTLS, err := PodManifestsHaveTLS(manifestDir) +// NewFromCluster creates an etcd client for the the etcd endpoints defined in the ClusterStatus value stored in +// the kubeadm-config ConfigMap in kube-system namespace. +// Once created, the client synchronizes client's endpoints with the known endpoints from the etcd membership API (reality check). +func NewFromCluster(client clientset.Interface, certificatesDir string) (*Client, error) { + + // Kubeadm v1.13 should manage v1.12 clusters and v1.13 clusters + // v1.12 clusters can be have etcd listening on localhost only (if the cluster was created with kubeadm v1.12) + // or etcd listening on localhost and API server advertise address (if the cluster was created with kubeadm v1.13). + // The first case should be dropped in v1.14 when support for v1.12 clusters can be removed from the codebase. + + // Detect which type of etcd we are dealing with + oldManifest := false + klog.V(1).Infoln("checking etcd manifest") + + etcdManifestFile := constants.GetStaticPodFilepath(constants.Etcd, constants.GetStaticPodDirectory()) + etcdPod, err := staticpod.ReadStaticPodFromDisk(etcdManifestFile) if err != nil { - return nil, errors.Wrapf(err, "could not read manifests from: %s, error", manifestDir) + return nil, errors.Wrap(err, "error reading etcd manifest file") } - if hasTLS { - return New( + etcdContainer := etcdPod.Spec.Containers[0] + for _, arg := range etcdContainer.Command { + if arg == "--listen-client-urls=https://127.0.0.1:2379" { + klog.V(1).Infoln("etcd manifest created by kubeadm v1.12") + oldManifest = true + } + } + + // if etcd is listening on localhost only + if oldManifest == true { + // etcd cluster has a single member "by design" + endpoints := []string{fmt.Sprintf("localhost:%d", constants.EtcdListenClientPort)} + + etcdClient, err := New( endpoints, filepath.Join(certificatesDir, constants.EtcdCACertName), filepath.Join(certificatesDir, constants.EtcdHealthcheckClientCertName), filepath.Join(certificatesDir, constants.EtcdHealthcheckClientKeyName), ) - } - return New(endpoints, "", "", "") -} + if err != nil { + return nil, errors.Wrapf(err, "error creating etcd client for %v endpoint", endpoints) + } + + return etcdClient, nil + } + + // etcd is listening on localhost and API server advertise address, and + // the etcd cluster can have more than one etcd members, so it is necessary to get the + // list of endpoints from kubeadm cluster status before connecting -// NewFromCluster creates an etcd client for the the etcd endpoints defined in the ClusterStatus value stored in -// the kubeadm-config ConfigMap in kube-system namespace. -// Once created, the client synchronizes client's endpoints with the known endpoints from the etcd membership API (reality check). -func NewFromCluster(client clientset.Interface, certificatesDir string) (*Client, error) { // Gets the cluster status clusterStatus, err := config.GetClusterStatus(client) if err != nil { @@ -145,7 +173,7 @@ func NewFromCluster(client clientset.Interface, certificatesDir string) (*Client for _, e := range clusterStatus.APIEndpoints { endpoints = append(endpoints, fmt.Sprintf("https://%s:%d", e.AdvertiseAddress, constants.EtcdListenClientPort)) } - glog.V(1).Infof("etcd endpoints read from pods: %s", strings.Join(endpoints, ",")) + klog.V(1).Infof("etcd endpoints read from pods: %s", strings.Join(endpoints, ",")) // Creates an etcd client etcdClient, err := New( @@ -155,13 +183,13 @@ func NewFromCluster(client clientset.Interface, certificatesDir string) (*Client filepath.Join(certificatesDir, constants.EtcdHealthcheckClientKeyName), ) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "error creating etcd client for %v endpoints", endpoints) } // synchronizes client's endpoints with the known endpoints from the etcd membership. err = etcdClient.Sync() if err != nil { - return nil, err + return nil, errors.Wrap(err, "error syncing endpoints with etc") } return etcdClient, nil @@ -185,7 +213,7 @@ func (c Client) Sync() error { if err != nil { return err } - glog.V(1).Infof("etcd endpoints read from etcd: %s", strings.Join(cli.Endpoints(), ",")) + klog.V(1).Infof("etcd endpoints read from etcd: %s", strings.Join(cli.Endpoints(), ",")) c.Endpoints = cli.Endpoints() return nil diff --git a/cmd/kubeadm/app/util/marshal.go b/cmd/kubeadm/app/util/marshal.go index 950855f9459..d38dce75f2f 100644 --- a/cmd/kubeadm/app/util/marshal.go +++ b/cmd/kubeadm/app/util/marshal.go @@ -21,8 +21,8 @@ import ( "bytes" "io" - "github.com/ghodss/yaml" pkgerrors "github.com/pkg/errors" + "sigs.k8s.io/yaml" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/cmd/kubeadm/app/util/pkiutil/pki_helpers.go b/cmd/kubeadm/app/util/pkiutil/pki_helpers.go index c6dd0bff8bd..20cc1ce309b 100644 --- a/cmd/kubeadm/app/util/pkiutil/pki_helpers.go +++ b/cmd/kubeadm/app/util/pkiutil/pki_helpers.go @@ -17,9 +17,14 @@ limitations under the License. package pkiutil import ( + "crypto" + cryptorand "crypto/rand" "crypto/rsa" "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "fmt" + "io/ioutil" "net" "os" "path/filepath" @@ -65,6 +70,21 @@ func NewCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey, config *cert return cert, key, nil } +// NewCSRAndKey generates a new key and CSR and that could be signed to create the given certificate +func NewCSRAndKey(config *certutil.Config) (*x509.CertificateRequest, *rsa.PrivateKey, error) { + key, err := certutil.NewPrivateKey() + if err != nil { + return nil, nil, errors.Wrap(err, "unable to create private key") + } + + csr, err := NewCSR(*config, key) + if err != nil { + return nil, nil, errors.Wrap(err, "unable to generate CSR") + } + + return csr, key, nil +} + // HasServerAuth returns true if the given certificate is a ServerAuth func HasServerAuth(cert *x509.Certificate) bool { for i := range cert.ExtKeyUsage { @@ -78,7 +98,7 @@ func HasServerAuth(cert *x509.Certificate) bool { // WriteCertAndKey stores certificate and key at the specified location func WriteCertAndKey(pkiPath string, name string, cert *x509.Certificate, key *rsa.PrivateKey) error { if err := WriteKey(pkiPath, name, key); err != nil { - return err + return errors.Wrap(err, "couldn't write key") } return WriteCert(pkiPath, name, cert) @@ -112,6 +132,27 @@ func WriteKey(pkiPath, name string, key *rsa.PrivateKey) error { return nil } +// WriteCSR writes the pem-encoded CSR data to csrPath. +// The CSR file will be created with file mode 0644. +// If the CSR file already exists, it will be overwritten. +// The parent directory of the csrPath will be created as needed with file mode 0755. +func WriteCSR(csrDir, name string, csr *x509.CertificateRequest) error { + if csr == nil { + return errors.New("certificate request cannot be nil when writing to file") + } + + csrPath := pathForCSR(csrDir, name) + if err := os.MkdirAll(filepath.Dir(csrPath), os.FileMode(0755)); err != nil { + return errors.Wrapf(err, "failed to make directory %s", filepath.Dir(csrPath)) + } + + if err := ioutil.WriteFile(csrPath, EncodeCSRPEM(csr), os.FileMode(0644)); err != nil { + return errors.Wrapf(err, "unable to write CSR to file %s", csrPath) + } + + return nil +} + // WritePublicKey stores the given public key at the given location func WritePublicKey(pkiPath, name string, key *rsa.PublicKey) error { if key == nil { @@ -145,16 +186,27 @@ func CertOrKeyExist(pkiPath, name string) bool { return true } +// CSROrKeyExist returns true if one of the CSR or key exists +func CSROrKeyExist(csrDir, name string) bool { + csrPath := pathForCSR(csrDir, name) + keyPath := pathForKey(csrDir, name) + + _, csrErr := os.Stat(csrPath) + _, keyErr := os.Stat(keyPath) + + return !(os.IsNotExist(csrErr) && os.IsNotExist(keyErr)) +} + // TryLoadCertAndKeyFromDisk tries to load a cert and a key from the disk and validates that they are valid func TryLoadCertAndKeyFromDisk(pkiPath, name string) (*x509.Certificate, *rsa.PrivateKey, error) { cert, err := TryLoadCertFromDisk(pkiPath, name) if err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "failed to load certificate") } key, err := TryLoadKeyFromDisk(pkiPath, name) if err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "failed to load key") } return cert, key, nil @@ -201,12 +253,29 @@ func TryLoadKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, error) { case *rsa.PrivateKey: key = k default: - return nil, errors.Wrapf(err, "the private key file %s isn't in RSA format", privateKeyPath) + return nil, errors.Errorf("the private key file %s isn't in RSA format", privateKeyPath) } return key, nil } +// TryLoadCSRAndKeyFromDisk tries to load the CSR and key from the disk +func TryLoadCSRAndKeyFromDisk(pkiPath, name string) (*x509.CertificateRequest, *rsa.PrivateKey, error) { + csrPath := pathForCSR(pkiPath, name) + + csr, err := CertificateRequestFromFile(csrPath) + if err != nil { + return nil, nil, errors.Wrapf(err, "couldn't load the certificate request %s", csrPath) + } + + key, err := TryLoadKeyFromDisk(pkiPath, name) + if err != nil { + return nil, nil, errors.Wrap(err, "couldn't load key file") + } + + return csr, key, nil +} + // TryLoadPrivatePublicKeyFromDisk tries to load the key from the disk and validates that it is valid func TryLoadPrivatePublicKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, *rsa.PublicKey, error) { privateKeyPath := pathForKey(pkiPath, name) @@ -228,7 +297,7 @@ func TryLoadPrivatePublicKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, *rs // Allow RSA format only k, ok := privKey.(*rsa.PrivateKey) if !ok { - return nil, nil, errors.Wrapf(err, "the private key file %s isn't in RSA format", privateKeyPath) + return nil, nil, errors.Errorf("the private key file %s isn't in RSA format", privateKeyPath) } p := pubKeys[0].(*rsa.PublicKey) @@ -253,13 +322,17 @@ func pathForPublicKey(pkiPath, name string) string { return filepath.Join(pkiPath, fmt.Sprintf("%s.pub", name)) } +func pathForCSR(pkiPath, name string) string { + return filepath.Join(pkiPath, fmt.Sprintf("%s.csr", name)) +} + // GetAPIServerAltNames builds an AltNames object for to be used when generating apiserver certificate func GetAPIServerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) { // advertise address - advertiseAddress := net.ParseIP(cfg.APIEndpoint.AdvertiseAddress) + advertiseAddress := net.ParseIP(cfg.LocalAPIEndpoint.AdvertiseAddress) if advertiseAddress == nil { - return nil, errors.Errorf("error parsing APIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address", - cfg.APIEndpoint.AdvertiseAddress) + return nil, errors.Errorf("error parsing LocalAPIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address", + cfg.LocalAPIEndpoint.AdvertiseAddress) } // internal IP address for the API server @@ -311,9 +384,9 @@ func GetAPIServerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames // The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.ServerCertSANs`. func GetEtcdAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) { // advertise address - advertiseAddress := net.ParseIP(cfg.APIEndpoint.AdvertiseAddress) + advertiseAddress := net.ParseIP(cfg.LocalAPIEndpoint.AdvertiseAddress) if advertiseAddress == nil { - return nil, errors.Errorf("error parsing APIEndpoint AdvertiseAddress %q: is not a valid textual representation of an IP address", cfg.APIEndpoint.AdvertiseAddress) + return nil, errors.Errorf("error parsing LocalAPIEndpoint AdvertiseAddress %q: is not a valid textual representation of an IP address", cfg.LocalAPIEndpoint.AdvertiseAddress) } // create AltNames with defaults DNSNames/IPs @@ -334,10 +407,10 @@ func GetEtcdAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, err // The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.PeerCertSANs`. func GetEtcdPeerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) { // advertise address - advertiseAddress := net.ParseIP(cfg.APIEndpoint.AdvertiseAddress) + advertiseAddress := net.ParseIP(cfg.LocalAPIEndpoint.AdvertiseAddress) if advertiseAddress == nil { - return nil, errors.Errorf("error parsing APIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address", - cfg.APIEndpoint.AdvertiseAddress) + return nil, errors.Errorf("error parsing LocalAPIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address", + cfg.LocalAPIEndpoint.AdvertiseAddress) } // create AltNames with defaults DNSNames/IPs @@ -373,3 +446,61 @@ func appendSANsToAltNames(altNames *certutil.AltNames, SANs []string, certName s } } } + +// EncodeCSRPEM returns PEM-encoded CSR data +func EncodeCSRPEM(csr *x509.CertificateRequest) []byte { + block := pem.Block{ + Type: certutil.CertificateRequestBlockType, + Bytes: csr.Raw, + } + return pem.EncodeToMemory(&block) +} + +func parseCSRPEM(pemCSR []byte) (*x509.CertificateRequest, error) { + block, _ := pem.Decode(pemCSR) + if block == nil { + return nil, fmt.Errorf("data doesn't contain a valid certificate request") + } + + if block.Type != certutil.CertificateRequestBlockType { + var block *pem.Block + return nil, fmt.Errorf("expected block type %q, but PEM had type %v", certutil.CertificateRequestBlockType, block.Type) + } + + return x509.ParseCertificateRequest(block.Bytes) +} + +// CertificateRequestFromFile returns the CertificateRequest from a given PEM-encoded file. +// Returns an error if the file could not be read or if the CSR could not be parsed. +func CertificateRequestFromFile(file string) (*x509.CertificateRequest, error) { + pemBlock, err := ioutil.ReadFile(file) + if err != nil { + return nil, errors.Wrap(err, "failed to read file") + } + + csr, err := parseCSRPEM(pemBlock) + if err != nil { + return nil, fmt.Errorf("error reading certificate request file %s: %v", file, err) + } + return csr, nil +} + +// NewCSR creates a new CSR +func NewCSR(cfg certutil.Config, key crypto.Signer) (*x509.CertificateRequest, error) { + template := &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: cfg.CommonName, + Organization: cfg.Organization, + }, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + } + + csrBytes, err := x509.CreateCertificateRequest(cryptorand.Reader, template, key) + + if err != nil { + return nil, errors.Wrap(err, "failed to create a CSR") + } + + return x509.ParseCertificateRequest(csrBytes) +} diff --git a/cmd/kubeadm/app/util/pkiutil/pki_helpers_test.go b/cmd/kubeadm/app/util/pkiutil/pki_helpers_test.go index 5905796b3ca..ae8f224bf9e 100644 --- a/cmd/kubeadm/app/util/pkiutil/pki_helpers_test.go +++ b/cmd/kubeadm/app/util/pkiutil/pki_helpers_test.go @@ -29,6 +29,24 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" ) +// certificateRequest is an x509 certificate request in PEM encoded format +// openssl req -new -key rsa2048.pem -sha256 -nodes -out x509certrequest.pem -subj "/C=US/CN=not-valid" +const certificateRequest = `-----BEGIN CERTIFICATE REQUEST----- +MIICZjCCAU4CAQAwITELMAkGA1UEBhMCVVMxEjAQBgNVBAMMCW5vdC12YWxpZDCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMdoBxV0SbSS+7XrgVDF/P4x +tqyun+DLxeRF5265ZOFRJDXCJgYH7wKlxlkEaHZQhnNmnqFiy96MHSKaiQmlkEm4 +EhlqTf38yEWx+t98A0CDbHsIPZ0/+MPCjb2kf+OfBXJJl908io0grs02jxN9lceL +RFrKT6vaB+6i7LxbPQcOmjF7OUqWS6S2qSpShw2GY+mJz4HM7OFb9RcN4izh+GF6 +7hajYgt7pAFyWF1ua/H98Ysn4FVgIYk30rHCNBkQpJnna7EyGYuj08VuFa088W9g +c/DCpL+VgBDwTel9tfeMxRAoLIPF9iJ8Ftr7dsRZ/Y/SnxfUJo2ed8y7dgIiLuEC +AwEAAaAAMA0GCSqGSIb3DQEBCwUAA4IBAQCOjPB/4LKa2G7LarMMLAeNqvWF9SIG +y2VGQoTn9D5blXMvnfzWSYgU6nBzf/E/32q26OwiCriuOPXfxM/cxEMOJ62u7b50 +OR52JFvQdONsCZaLgylGWppl0YeqylbTosHjsWJNlp+zjXcQHjCQ9OoLgfmrwYyD +2MsYJR4p7JZ2ZN8FF1hgMUrDzypZ0NSBKAiQMU9TFhxgyk75RNDtmX+2K35zqLyr +0otimyYwPCGPD2GHwNfvu1oP0A+/cT+rCPz6AlXhWEbz2JkLo6/muRfRl0QSRgHE +Q3+eWlA1YdqEBwvp3NEQI9BtMnzxJVWA5dvYluMNllsV/q8s2IEEAFG9 +-----END CERTIFICATE REQUEST-----` + func TestNewCertificateAuthority(t *testing.T) { cert, key, err := NewCertificateAuthority(&certutil.Config{CommonName: "kubernetes"}) @@ -435,6 +453,13 @@ func TestPathForPublicKey(t *testing.T) { } } +func TestPathForCSR(t *testing.T) { + csrPath := pathForCSR("/foo", "bar") + if csrPath != "/foo/bar.csr" { + t.Errorf("unexpected certificate path: %s", csrPath) + } +} + func TestGetAPIServerAltNames(t *testing.T) { var tests = []struct { @@ -444,9 +469,9 @@ func TestGetAPIServerAltNames(t *testing.T) { expectedIPAddresses []string }{ { - name: "ControlPlaneEndpoint DNS", + name: "", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ ControlPlaneEndpoint: "api.k8s.io:6443", Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, @@ -462,7 +487,7 @@ func TestGetAPIServerAltNames(t *testing.T) { { name: "ControlPlaneEndpoint IP", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ ControlPlaneEndpoint: "4.5.6.7:6443", Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, @@ -517,7 +542,7 @@ func TestGetEtcdAltNames(t *testing.T) { proxy := "user-etcd-proxy" proxyIP := "10.10.10.100" cfg := &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ AdvertiseAddress: "1.2.3.4", }, NodeRegistration: kubeadmapi.NodeRegistrationOptions{ @@ -579,7 +604,7 @@ func TestGetEtcdPeerAltNames(t *testing.T) { proxyIP := "10.10.10.100" advertiseIP := "1.2.3.4" cfg := &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: advertiseIP}, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: advertiseIP}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Etcd: kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{ diff --git a/cmd/kubeadm/app/util/staticpod/BUILD b/cmd/kubeadm/app/util/staticpod/BUILD index 7ffdb4b1b75..72605aa487c 100644 --- a/cmd/kubeadm/app/util/staticpod/BUILD +++ b/cmd/kubeadm/app/util/staticpod/BUILD @@ -13,7 +13,6 @@ go_test( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/test:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -28,7 +27,6 @@ go_library( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//pkg/kubelet/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index 0c23d40c245..cdaeef5f33b 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -32,8 +32,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util" @@ -70,7 +68,6 @@ func ComponentPod(container v1.Container, volumes map[string]v1.Volume) v1.Pod { Containers: []v1.Container{container}, PriorityClassName: "system-cluster-critical", HostNetwork: true, - DNSPolicy: v1.DNSClusterFirstWithHostNet, Volumes: VolumeMapToSlice(volumes), }, } @@ -240,10 +237,8 @@ func GetProbeAddress(cfg *kubeadmapi.InitConfiguration, componentName string) st // future hosts that do not have the same address. Furthermore, since liveness and readiness // probes do not support the Downward API we cannot dynamically set the advertise address to // the node's IP. The only option then is to use localhost. - if features.Enabled(cfg.FeatureGates, features.SelfHosting) { - return "127.0.0.1" - } else if cfg.APIEndpoint.AdvertiseAddress != "" { - return cfg.APIEndpoint.AdvertiseAddress + if cfg.LocalAPIEndpoint.AdvertiseAddress != "" { + return cfg.LocalAPIEndpoint.AdvertiseAddress } case componentName == kubeadmconstants.KubeControllerManager: if addr, exists := cfg.ControllerManager.ExtraArgs[kubeControllerManagerAddressArg]; exists { diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go index cd2a3d05707..3aa91bc558d 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_test.go @@ -28,8 +28,6 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" testutil "k8s.io/kubernetes/cmd/kubeadm/test" @@ -57,7 +55,7 @@ func TestComponentProbe(t *testing.T) { { name: "default apiserver advertise address with http", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ AdvertiseAddress: "", }, }, @@ -67,28 +65,10 @@ func TestComponentProbe(t *testing.T) { scheme: v1.URISchemeHTTP, expected: "127.0.0.1", }, - { - name: "default apiserver advertise address with http", - cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{ - features.SelfHosting: true, - }, - }, - }, - component: kubeadmconstants.KubeAPIServer, - port: 1, - path: "foo", - scheme: v1.URISchemeHTTP, - expected: "127.0.0.1", - }, { name: "default apiserver advertise address with https", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ AdvertiseAddress: "", }, }, @@ -101,7 +81,7 @@ func TestComponentProbe(t *testing.T) { { name: "valid ipv4 apiserver advertise address with http", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ AdvertiseAddress: "1.2.3.4", }, }, @@ -114,7 +94,7 @@ func TestComponentProbe(t *testing.T) { { name: "valid ipv6 apiserver advertise address with http", cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ AdvertiseAddress: "2001:db8::1", }, }, @@ -389,7 +369,6 @@ func TestComponentPod(t *testing.T) { }, PriorityClassName: "system-cluster-critical", HostNetwork: true, - DNSPolicy: v1.DNSClusterFirstWithHostNet, Volumes: []v1.Volume{}, }, }, diff --git a/cmd/kubeadm/app/util/system/BUILD b/cmd/kubeadm/app/util/system/BUILD index 4435ddfbe86..eb403878543 100644 --- a/cmd/kubeadm/app/util/system/BUILD +++ b/cmd/kubeadm/app/util/system/BUILD @@ -27,8 +27,8 @@ go_library( "//vendor/github.com/blang/semver:go_default_library", "//vendor/github.com/docker/docker/api/types:go_default_library", "//vendor/github.com/docker/docker/client:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/kubeadm/app/util/system/docker_validator.go b/cmd/kubeadm/app/util/system/docker_validator.go index 75ebf8b4330..48977766f3b 100644 --- a/cmd/kubeadm/app/util/system/docker_validator.go +++ b/cmd/kubeadm/app/util/system/docker_validator.go @@ -73,7 +73,7 @@ func (d *DockerValidator) validateDockerInfo(spec *DockerSpec, info types.Info) if !matched { // If it's of the new Docker version scheme but didn't match above, it // must be a newer version than the most recently validated one. - ver := `\d{2}\.\d+\.\d+-[a-z]{2}` + ver := `\d{2}\.\d+\.\d+(?:-[a-z]{2})?` r := regexp.MustCompile(ver) if r.MatchString(info.ServerVersion) { d.Reporter.Report(dockerConfigPrefix+"VERSION", info.ServerVersion, good) diff --git a/cmd/kubeadm/app/util/system/docker_validator_test.go b/cmd/kubeadm/app/util/system/docker_validator_test.go index 058143e5674..19b7e9adf37 100644 --- a/cmd/kubeadm/app/util/system/docker_validator_test.go +++ b/cmd/kubeadm/app/util/system/docker_validator_test.go @@ -81,6 +81,11 @@ func TestValidateDockerInfo(t *testing.T) { err: false, warn: false, }, + { + info: types.Info{Driver: "driver_2", ServerVersion: "18.09.0"}, + err: false, + warn: true, + }, } { warn, err := v.validateDockerInfo(spec, test.info) if !test.err { diff --git a/cmd/kubeadm/app/util/system/kernel_validator.go b/cmd/kubeadm/app/util/system/kernel_validator.go index 5add3190f18..36ecd20dbdb 100644 --- a/cmd/kubeadm/app/util/system/kernel_validator.go +++ b/cmd/kubeadm/app/util/system/kernel_validator.go @@ -29,8 +29,8 @@ import ( "regexp" "strings" - "github.com/golang/glog" pkgerrors "github.com/pkg/errors" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/errors" ) @@ -254,7 +254,7 @@ func (k *KernelValidator) parseKernelConfig(r io.Reader) (map[string]kConfigOpti } fields := strings.Split(line, "=") if len(fields) != 2 { - glog.Errorf("Unexpected fields number in config %q", line) + klog.Errorf("Unexpected fields number in config %q", line) continue } config[fields[0]] = kConfigOption(fields[1]) diff --git a/cmd/kubeadm/app/util/system/package_validator.go b/cmd/kubeadm/app/util/system/package_validator.go index 3c9fa9c3192..820c68e6c2b 100644 --- a/cmd/kubeadm/app/util/system/package_validator.go +++ b/cmd/kubeadm/app/util/system/package_validator.go @@ -25,8 +25,8 @@ import ( "k8s.io/apimachinery/pkg/util/errors" "github.com/blang/semver" - "github.com/golang/glog" pkgerrors "github.com/pkg/errors" + "k8s.io/klog" ) // semVerDotsCount is the number of dots in a valid semantic version. @@ -127,7 +127,7 @@ func (self *packageValidator) validate(packageSpecs []PackageSpec, manager packa // Get the version of the package on the running machine. version, err := manager.getPackageVersion(packageName) if err != nil { - glog.V(1).Infof("Failed to get the version for the package %q: %s\n", packageName, err) + klog.V(1).Infof("Failed to get the version for the package %q: %s\n", packageName, err) errs = append(errs, err) self.reporter.Report(nameWithVerRange, "not installed", bad) continue @@ -145,7 +145,7 @@ func (self *packageValidator) validate(packageSpecs []PackageSpec, manager packa // the version is in the range. sv, err := semver.Make(toSemVer(version)) if err != nil { - glog.Errorf("Failed to convert %q to semantic version: %s\n", version, err) + klog.Errorf("Failed to convert %q to semantic version: %s\n", version, err) errs = append(errs, err) self.reporter.Report(nameWithVerRange, "internal error", bad) continue diff --git a/cmd/kubeadm/app/util/version.go b/cmd/kubeadm/app/util/version.go index 3830ed1a08a..4da6ce48aef 100644 --- a/cmd/kubeadm/app/util/version.go +++ b/cmd/kubeadm/app/util/version.go @@ -25,10 +25,10 @@ import ( "strings" "time" - "github.com/golang/glog" pkgerrors "github.com/pkg/errors" netutil "k8s.io/apimachinery/pkg/util/net" versionutil "k8s.io/apimachinery/pkg/util/version" + "k8s.io/klog" pkgversion "k8s.io/kubernetes/pkg/version" ) @@ -91,8 +91,8 @@ func KubernetesReleaseVersion(version string) (string, error) { return "", err } // Handle air-gapped environments by falling back to the client version. - glog.Infof("could not fetch a Kubernetes version from the internet: %v", err) - glog.Infof("falling back to the local client version: %s", clientVersion) + klog.Infof("could not fetch a Kubernetes version from the internet: %v", err) + klog.Infof("falling back to the local client version: %s", clientVersion) return KubernetesReleaseVersion(clientVersion) } // both the client and the remote version are obtained; validate them and pick a stable version @@ -160,7 +160,7 @@ func splitVersion(version string) (string, string, error) { // Internal helper: return content of URL func fetchFromURL(url string, timeout time.Duration) (string, error) { - glog.V(2).Infof("fetching Kubernetes version from URL: %s", url) + klog.V(2).Infof("fetching Kubernetes version from URL: %s", url) client := &http.Client{Timeout: timeout, Transport: netutil.SetOldTransportDefaults(&http.Transport{})} resp, err := client.Get(url) if err != nil { @@ -217,7 +217,7 @@ func kubeadmVersion(info string) (string, error) { // the same Patch level release. func validateStableVersion(remoteVersion, clientVersion string) (string, error) { if clientVersion == "" { - glog.Infof("could not obtain client version; using remote version: %s", remoteVersion) + klog.Infof("could not obtain client version; using remote version: %s", remoteVersion) return remoteVersion, nil } @@ -234,7 +234,7 @@ func validateStableVersion(remoteVersion, clientVersion string) (string, error) if verClient.Major() < verRemote.Major() || (verClient.Major() == verRemote.Major()) && verClient.Minor() < verRemote.Minor() { estimatedRelease := fmt.Sprintf("stable-%d.%d", verClient.Major(), verClient.Minor()) - glog.Infof("remote version is much newer: %s; falling back to: %s", remoteVersion, estimatedRelease) + klog.Infof("remote version is much newer: %s; falling back to: %s", remoteVersion, estimatedRelease) return estimatedRelease, nil } return remoteVersion, nil diff --git a/cmd/kubeadm/kubeadm.go b/cmd/kubeadm/kubeadm.go index 233b13f5cbc..1b3d58fa171 100644 --- a/cmd/kubeadm/kubeadm.go +++ b/cmd/kubeadm/kubeadm.go @@ -20,10 +20,12 @@ import ( "fmt" "os" + "k8s.io/klog" "k8s.io/kubernetes/cmd/kubeadm/app" ) func main() { + klog.InitFlags(nil) if err := app.Run(); err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) diff --git a/cmd/kubeadm/test/cmd/BUILD b/cmd/kubeadm/test/cmd/BUILD index 94771cffe00..fa944de0efd 100644 --- a/cmd/kubeadm/test/cmd/BUILD +++ b/cmd/kubeadm/test/cmd/BUILD @@ -33,8 +33,12 @@ go_test( "skip", ], deps = [ - "//vendor/github.com/ghodss/yaml:go_default_library", + "//cmd/kubeadm/app/phases/certs:go_default_library", + "//cmd/kubeadm/app/util/pkiutil:go_default_library", + "//cmd/kubeadm/test:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", "//vendor/github.com/renstrom/dedent:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/cmd/kubeadm/test/cmd/init_test.go b/cmd/kubeadm/test/cmd/init_test.go index a4dfe55c438..e986047ec68 100644 --- a/cmd/kubeadm/test/cmd/init_test.go +++ b/cmd/kubeadm/test/cmd/init_test.go @@ -17,9 +17,15 @@ limitations under the License. package kubeadm import ( + "os/exec" + "strings" "testing" + "github.com/pkg/errors" "github.com/renstrom/dedent" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" + testutil "k8s.io/kubernetes/cmd/kubeadm/test" ) func runKubeadmInit(args ...string) (string, string, error) { @@ -191,6 +197,71 @@ func TestCmdInitConfig(t *testing.T) { } } +func TestCmdInitCertPhaseCSR(t *testing.T) { + if *kubeadmCmdSkip { + t.Log("kubeadm cmd tests being skipped") + t.Skip() + } + + tests := []struct { + name string + baseName string + expectedError string + }{ + { + name: "generate CSR", + baseName: certs.KubeadmCertKubeletClient.BaseName, + }, + { + name: "fails on CSR", + baseName: certs.KubeadmCertRootCA.BaseName, + expectedError: "unknown flag: --csr-only", + }, + { + name: "fails on all", + baseName: "all", + expectedError: "unknown flag: --csr-only", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + csrDir := testutil.SetupTempDir(t) + cert := &certs.KubeadmCertKubeletClient + kubeadmPath := getKubeadmPath() + _, stderr, err := RunCmd(kubeadmPath, + "init", + "phase", + "certs", + test.baseName, + "--csr-only", + "--csr-dir="+csrDir, + ) + + if test.expectedError != "" { + cause := errors.Cause(err) + _, ok := cause.(*exec.ExitError) + if !ok { + t.Fatalf("expected exitErr: got %T (%v)", cause, err) + } + + if !strings.Contains(stderr, test.expectedError) { + t.Errorf("expected %q to contain %q", stderr, test.expectedError) + } + return + } + + if err != nil { + t.Fatalf("couldn't run kubeadm: %v", err) + } + + if _, _, err := pkiutil.TryLoadCSRAndKeyFromDisk(csrDir, cert.BaseName); err != nil { + t.Fatalf("couldn't load certificate %q: %v", cert.BaseName, err) + } + }) + } +} + func TestCmdInitAPIPort(t *testing.T) { if *kubeadmCmdSkip { t.Log("kubeadm cmd tests being skipped") diff --git a/cmd/kubeadm/test/cmd/util.go b/cmd/kubeadm/test/cmd/util.go index 5bed6d35782..ecc1951b5dd 100644 --- a/cmd/kubeadm/test/cmd/util.go +++ b/cmd/kubeadm/test/cmd/util.go @@ -18,26 +18,32 @@ package kubeadm import ( "bytes" - "github.com/pkg/errors" "os/exec" "testing" + "github.com/pkg/errors" + "github.com/spf13/cobra" ) // Forked from test/e2e/framework because the e2e framework is quite bloated // for our purposes here, and modified to remove undesired logging. -// RunCmd is a utility function for kubeadm testing that executes a specified command -func RunCmd(command string, args ...string) (string, string, error) { +func runCmdNoWrap(command string, args ...string) (string, string, error) { var bout, berr bytes.Buffer cmd := exec.Command(command, args...) cmd.Stdout = &bout cmd.Stderr = &berr err := cmd.Run() stdout, stderr := bout.String(), berr.String() + return stdout, stderr, err +} + +// RunCmd is a utility function for kubeadm testing that executes a specified command +func RunCmd(command string, args ...string) (string, string, error) { + stdout, stderr, err := runCmdNoWrap(command, args...) if err != nil { - return "", "", errors.Wrapf(err, "error running %s %v; \nstdout %q, \nstderr %q, \ngot error", + return stdout, stderr, errors.Wrapf(err, "error running %s %v; \nstdout %q, \nstderr %q, \ngot error", command, args, stdout, stderr) } return stdout, stderr, nil diff --git a/cmd/kubeadm/test/cmd/version_test.go b/cmd/kubeadm/test/cmd/version_test.go index 85fb1170615..345c0edabfc 100644 --- a/cmd/kubeadm/test/cmd/version_test.go +++ b/cmd/kubeadm/test/cmd/version_test.go @@ -21,7 +21,7 @@ import ( "regexp" "testing" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) const ( diff --git a/cmd/kubeadm/test/util.go b/cmd/kubeadm/test/util.go index 0346a451f26..eceea5712c0 100644 --- a/cmd/kubeadm/test/util.go +++ b/cmd/kubeadm/test/util.go @@ -58,8 +58,8 @@ func SetupInitConfigurationFile(t *testing.T, tmpdir string, cfg *kubeadmapi.Ini apiVersion: kubeadm.k8s.io/v1beta1 kind: InitConfiguration apiEndpoint: - advertiseAddress: {{.APIEndpoint.AdvertiseAddress}} - bindPort: {{.APIEndpoint.BindPort}} + advertiseAddress: {{.LocalAPIEndpoint.AdvertiseAddress}} + bindPort: {{.LocalAPIEndpoint.BindPort}} nodeRegistration: name: {{.NodeRegistration.Name}} --- diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index f79a45ca308..4dc0bbcc4ee 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -123,9 +123,9 @@ go_library( "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", "//vendor/github.com/coreos/go-systemd/daemon:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ diff --git a/cmd/kubelet/app/options/BUILD b/cmd/kubelet/app/options/BUILD index 42ba81e5283..d6e0d70f4a0 100644 --- a/cmd/kubelet/app/options/BUILD +++ b/cmd/kubelet/app/options/BUILD @@ -23,6 +23,7 @@ go_library( "//pkg/credentialprovider/azure:go_default_library", "//pkg/credentialprovider/gcp:go_default_library", "//pkg/features:go_default_library", + "//pkg/kubelet/apis:go_default_library", "//pkg/kubelet/apis/config:go_default_library", "//pkg/kubelet/apis/config/scheme:go_default_library", "//pkg/kubelet/apis/config/validation:go_default_library", @@ -33,12 +34,13 @@ go_library( "//pkg/util/taints:go_default_library", "//pkg/version/verflag:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library", "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//vendor/github.com/google/cadvisor/container/common:go_default_library", diff --git a/cmd/kubelet/app/options/globalflags.go b/cmd/kubelet/app/options/globalflags.go index dafd5447989..144837731e1 100644 --- a/cmd/kubelet/app/options/globalflags.go +++ b/cmd/kubelet/app/options/globalflags.go @@ -29,7 +29,7 @@ import ( "k8s.io/kubernetes/pkg/version/verflag" // ensure libs have a chance to globally register their flags - _ "github.com/golang/glog" + _ "k8s.io/klog" _ "k8s.io/kubernetes/pkg/credentialprovider/azure" _ "k8s.io/kubernetes/pkg/credentialprovider/gcp" ) @@ -91,7 +91,7 @@ func addCredentialProviderFlags(fs *pflag.FlagSet) { fs.AddFlagSet(local) } -// addGlogFlags adds flags from github.com/golang/glog +// addGlogFlags adds flags from k8s.io/klog func addGlogFlags(fs *pflag.FlagSet) { // lookup flags in global flag set and re-register the values with our flagset global := flag.CommandLine @@ -104,6 +104,7 @@ func addGlogFlags(fs *pflag.FlagSet) { register(global, local, "vmodule") register(global, local, "log_backtrace_at") register(global, local, "log_dir") + register(global, local, "log_file") fs.AddFlagSet(local) } diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 9dca28b9548..47f9eda3db8 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -27,11 +27,14 @@ import ( "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/flag" + "k8s.io/klog" "k8s.io/kubelet/config/v1beta1" "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/features" + kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/config/scheme" kubeletconfigvalidation "k8s.io/kubernetes/pkg/kubelet/apis/config/validation" @@ -250,9 +253,40 @@ func ValidateKubeletFlags(f *KubeletFlags) error { if f.NodeStatusMaxImages < -1 { return fmt.Errorf("invalid configuration: NodeStatusMaxImages (--node-status-max-images) must be -1 or greater") } + + unknownLabels := sets.NewString() + for k := range f.NodeLabels { + if isKubernetesLabel(k) && !kubeletapis.IsKubeletLabel(k) { + unknownLabels.Insert(k) + } + } + if len(unknownLabels) > 0 { + // TODO(liggitt): in 1.15, return an error + klog.Warningf("unknown 'kubernetes.io' or 'k8s.io' labels specified with --node-labels: %v", unknownLabels.List()) + klog.Warningf("in 1.15, --node-labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", ")) + } + return nil } +func isKubernetesLabel(key string) bool { + namespace := getLabelNamespace(key) + if namespace == "kubernetes.io" || strings.HasSuffix(namespace, ".kubernetes.io") { + return true + } + if namespace == "k8s.io" || strings.HasSuffix(namespace, ".k8s.io") { + return true + } + return false +} + +func getLabelNamespace(key string) string { + if parts := strings.SplitN(key, "/", 2); len(parts) == 2 { + return parts[0] + } + return "" +} + // NewKubeletConfiguration will create a new KubeletConfiguration with default values func NewKubeletConfiguration() (*kubeletconfig.KubeletConfiguration, error) { scheme, _, err := kubeletscheme.NewSchemeAndCodecs() @@ -368,7 +402,7 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) { fs.Var(&f.DynamicConfigDir, "dynamic-config-dir", "The Kubelet will use this directory for checkpointing downloaded configurations and tracking configuration health. The Kubelet will create this directory if it does not already exist. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Providing this flag enables dynamic Kubelet configuration. The DynamicKubeletConfig feature gate must be enabled to pass this flag; this gate currently defaults to true because the feature is beta.") - fs.BoolVar(&f.RegisterNode, "register-node", f.RegisterNode, "Register the node with the apiserver. If --kubeconfig is not provided, this flag is irrelevant, as the Kubelet won't have an apiserver to register with. Default=true.") + fs.BoolVar(&f.RegisterNode, "register-node", f.RegisterNode, "Register the node with the apiserver. If --kubeconfig is not provided, this flag is irrelevant, as the Kubelet won't have an apiserver to register with.") fs.Var(utiltaints.NewTaintsVar(&f.RegisterWithTaints), "register-with-taints", "Register the node with the given list of taints (comma separated \"=:\"). No-op if register-node is false.") fs.BoolVar(&f.Containerized, "containerized", f.Containerized, "Running kubelet in a container.") @@ -381,13 +415,13 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) { fs.BoolVar(&f.ExperimentalCheckNodeCapabilitiesBeforeMount, "experimental-check-node-capabilities-before-mount", f.ExperimentalCheckNodeCapabilitiesBeforeMount, "[Experimental] if set true, the kubelet will check the underlying node for required components (binaries, etc.) before performing the mount") fs.BoolVar(&f.ExperimentalNodeAllocatableIgnoreEvictionThreshold, "experimental-allocatable-ignore-eviction", f.ExperimentalNodeAllocatableIgnoreEvictionThreshold, "When set to 'true', Hard Eviction Thresholds will be ignored while calculating Node Allocatable. See https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ for more details. [default=false]") bindableNodeLabels := flag.ConfigurationMap(f.NodeLabels) - fs.Var(&bindableNodeLabels, "node-labels", " Labels to add when registering the node in the cluster. Labels must be key=value pairs separated by ','.") + fs.Var(&bindableNodeLabels, "node-labels", fmt.Sprintf(" Labels to add when registering the node in the cluster. Labels must be key=value pairs separated by ','. Labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", "))) fs.StringVar(&f.VolumePluginDir, "volume-plugin-dir", f.VolumePluginDir, "The full path of the directory in which to search for additional third party volume plugins") fs.StringVar(&f.LockFilePath, "lock-file", f.LockFilePath, " The path to file for kubelet to use as a lock file.") fs.BoolVar(&f.ExitOnLockContention, "exit-on-lock-contention", f.ExitOnLockContention, "Whether kubelet should exit upon lock-file contention.") fs.StringVar(&f.SeccompProfileRoot, "seccomp-profile-root", f.SeccompProfileRoot, " Directory path for seccomp profiles.") fs.StringVar(&f.BootstrapCheckpointPath, "bootstrap-checkpoint-path", f.BootstrapCheckpointPath, " Path to the directory where the checkpoints are stored") - fs.Int32Var(&f.NodeStatusMaxImages, "node-status-max-images", f.NodeStatusMaxImages, " The maximum number of images to report in Node.Status.Images. If -1 is specified, no cap will be applied. Default: 50") + fs.Int32Var(&f.NodeStatusMaxImages, "node-status-max-images", f.NodeStatusMaxImages, " The maximum number of images to report in Node.Status.Images. If -1 is specified, no cap will be applied.") // DEPRECATED FLAGS fs.StringVar(&f.BootstrapKubeconfig, "experimental-bootstrap-kubeconfig", f.BootstrapKubeconfig, "") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 606d60c295d..04422bb8990 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -34,9 +34,9 @@ import ( "time" "github.com/coreos/go-systemd/daemon" - "github.com/golang/glog" "github.com/spf13/cobra" "github.com/spf13/pflag" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -111,7 +111,7 @@ func NewKubeletCommand(stopCh <-chan struct{}) *cobra.Command { kubeletConfig, err := options.NewKubeletConfiguration() // programmer error if err != nil { - glog.Fatal(err) + klog.Fatal(err) } cmd := &cobra.Command{ @@ -144,20 +144,20 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API // initial flag parse, since we disable cobra's flag parsing if err := cleanFlagSet.Parse(args); err != nil { cmd.Usage() - glog.Fatal(err) + klog.Fatal(err) } // check if there are non-flag arguments in the command line cmds := cleanFlagSet.Args() if len(cmds) > 0 { cmd.Usage() - glog.Fatalf("unknown command: %s", cmds[0]) + klog.Fatalf("unknown command: %s", cmds[0]) } // short-circuit on help help, err := cleanFlagSet.GetBool("help") if err != nil { - glog.Fatal(`"help" flag is non-bool, programmer error, please correct`) + klog.Fatal(`"help" flag is non-bool, programmer error, please correct`) } if help { cmd.Help() @@ -170,40 +170,40 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API // set feature gates from initial flags-based config if err := utilfeature.DefaultFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil { - glog.Fatal(err) + klog.Fatal(err) } // validate the initial KubeletFlags if err := options.ValidateKubeletFlags(kubeletFlags); err != nil { - glog.Fatal(err) + klog.Fatal(err) } if kubeletFlags.ContainerRuntime == "remote" && cleanFlagSet.Changed("pod-infra-container-image") { - glog.Warning("Warning: For remote container runtime, --pod-infra-container-image is ignored in kubelet, which should be set in that remote runtime instead") + klog.Warning("Warning: For remote container runtime, --pod-infra-container-image is ignored in kubelet, which should be set in that remote runtime instead") } // load kubelet config file, if provided if configFile := kubeletFlags.KubeletConfigFile; len(configFile) > 0 { kubeletConfig, err = loadConfigFile(configFile) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } // We must enforce flag precedence by re-parsing the command line into the new object. // This is necessary to preserve backwards-compatibility across binary upgrades. // See issue #56171 for more details. if err := kubeletConfigFlagPrecedence(kubeletConfig, args); err != nil { - glog.Fatal(err) + klog.Fatal(err) } // update feature gates based on new config if err := utilfeature.DefaultFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil { - glog.Fatal(err) + klog.Fatal(err) } } // We always validate the local configuration (command line + config file). // This is the default "last-known-good" config for dynamic config, and must always remain valid. if err := kubeletconfigvalidation.ValidateKubeletConfiguration(kubeletConfig); err != nil { - glog.Fatal(err) + klog.Fatal(err) } // use dynamic kubelet config, if enabled @@ -219,7 +219,7 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API return kubeletConfigFlagPrecedence(kc, args) }) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } // If we should just use our existing, local config, the controller will return a nil config if dynamicKubeletConfig != nil { @@ -227,7 +227,7 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API // Note: flag precedence was already enforced in the controller, prior to validation, // by our above transform function. Now we simply update feature gates from the new config. if err := utilfeature.DefaultFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil { - glog.Fatal(err) + klog.Fatal(err) } } } @@ -241,7 +241,7 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API // use kubeletServer to construct the default KubeletDeps kubeletDeps, err := UnsecuredDependencies(kubeletServer) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } // add the kubelet config controller to kubeletDeps @@ -250,15 +250,15 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API // start the experimental docker shim, if enabled if kubeletServer.KubeletFlags.ExperimentalDockershim { if err := RunDockershim(&kubeletServer.KubeletFlags, kubeletConfig, stopCh); err != nil { - glog.Fatal(err) + klog.Fatal(err) } return } // run the kubelet - glog.V(5).Infof("KubeletConfiguration: %#v", kubeletServer.KubeletConfiguration) + klog.V(5).Infof("KubeletConfiguration: %#v", kubeletServer.KubeletConfiguration) if err := Run(kubeletServer, kubeletDeps, stopCh); err != nil { - glog.Fatal(err) + klog.Fatal(err) } }, } @@ -361,7 +361,7 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err mounter := mount.New(s.ExperimentalMounterPath) var pluginRunner = exec.New() if s.Containerized { - glog.V(2).Info("Running kubelet in containerized mode") + klog.V(2).Info("Running kubelet in containerized mode") ne, err := nsenter.NewNsenter(nsenter.DefaultHostRootFsPath, exec.New()) if err != nil { return nil, err @@ -404,7 +404,7 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err // not be generated. func Run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan struct{}) error { // To help debugging, immediately log version - glog.Infof("Version: %+v", version.Get()) + klog.Infof("Version: %+v", version.Get()) if err := initForOS(s.KubeletFlags.WindowsService); err != nil { return fmt.Errorf("failed OS init: %v", err) } @@ -439,11 +439,11 @@ func setConfigz(cz *configz.Config, kc *kubeletconfiginternal.KubeletConfigurati func initConfigz(kc *kubeletconfiginternal.KubeletConfiguration) error { cz, err := configz.New("kubeletconfig") if err != nil { - glog.Errorf("unable to register configz: %s", err) + klog.Errorf("unable to register configz: %s", err) return err } if err := setConfigz(cz, kc); err != nil { - glog.Errorf("unable to register config: %s", err) + klog.Errorf("unable to register config: %s", err) return err } return nil @@ -456,12 +456,12 @@ func makeEventRecorder(kubeDeps *kubelet.Dependencies, nodeName types.NodeName) } eventBroadcaster := record.NewBroadcaster() kubeDeps.Recorder = eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: componentKubelet, Host: string(nodeName)}) - eventBroadcaster.StartLogging(glog.V(3).Infof) + eventBroadcaster.StartLogging(klog.V(3).Infof) if kubeDeps.EventClient != nil { - glog.V(4).Infof("Sending events to api server.") + klog.V(4).Infof("Sending events to api server.") eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeDeps.EventClient.Events("")}) } else { - glog.Warning("No api server defined - no events will be sent to API server.") + klog.Warning("No api server defined - no events will be sent to API server.") } } @@ -482,12 +482,12 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan } done := make(chan struct{}) if s.LockFilePath != "" { - glog.Infof("acquiring file lock on %q", s.LockFilePath) + klog.Infof("acquiring file lock on %q", s.LockFilePath) if err := flock.Acquire(s.LockFilePath); err != nil { return fmt.Errorf("unable to acquire file lock on %q: %v", s.LockFilePath, err) } if s.ExitOnLockContention { - glog.Infof("watching for inotify events for: %v", s.LockFilePath) + klog.Infof("watching for inotify events for: %v", s.LockFilePath) if err := watchForLockfileContention(s.LockFilePath, done); err != nil { return err } @@ -497,7 +497,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan // Register current configuration with /configz endpoint err = initConfigz(&s.KubeletConfiguration) if err != nil { - glog.Errorf("unable to register KubeletConfiguration with configz, error: %v", err) + klog.Errorf("unable to register KubeletConfiguration with configz, error: %v", err) } // About to get clients and such, detect standaloneMode @@ -520,9 +520,9 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan return err } if cloud == nil { - glog.V(2).Infof("No cloud provider specified: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) + klog.V(2).Infof("No cloud provider specified: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) } else { - glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) + klog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) } kubeDeps.Cloud = cloud } @@ -549,7 +549,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan kubeDeps.DynamicKubeClient = nil kubeDeps.EventClient = nil kubeDeps.HeartbeatClient = nil - glog.Warningf("standalone mode, no API client") + klog.Warningf("standalone mode, no API client") } else if kubeDeps.KubeClient == nil || kubeDeps.EventClient == nil || kubeDeps.HeartbeatClient == nil || kubeDeps.DynamicKubeClient == nil { // initialize clients if not standalone mode and any of the clients are not provided var kubeClient clientset.Interface @@ -579,15 +579,15 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan kubeClient, err = clientset.NewForConfig(clientConfig) if err != nil { - glog.Warningf("New kubeClient from clientConfig error: %v", err) + klog.Warningf("New kubeClient from clientConfig error: %v", err) } else if kubeClient.CertificatesV1beta1() != nil && clientCertificateManager != nil { - glog.V(2).Info("Starting client certificate rotation.") + klog.V(2).Info("Starting client certificate rotation.") clientCertificateManager.SetCertificateSigningRequestClient(kubeClient.CertificatesV1beta1().CertificateSigningRequests()) clientCertificateManager.Start() } dynamicKubeClient, err = dynamic.NewForConfig(clientConfig) if err != nil { - glog.Warningf("Failed to initialize dynamic KubeClient: %v", err) + klog.Warningf("Failed to initialize dynamic KubeClient: %v", err) } // make a separate client for events @@ -596,7 +596,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan eventClientConfig.Burst = int(s.EventBurst) eventClient, err = v1core.NewForConfig(&eventClientConfig) if err != nil { - glog.Warningf("Failed to create API Server client for Events: %v", err) + klog.Warningf("Failed to create API Server client for Events: %v", err) } // make a separate client for heartbeat with throttling disabled and a timeout attached @@ -612,14 +612,14 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan heartbeatClientConfig.QPS = float32(-1) heartbeatClient, err = clientset.NewForConfig(&heartbeatClientConfig) if err != nil { - glog.Warningf("Failed to create API Server client for heartbeat: %v", err) + klog.Warningf("Failed to create API Server client for heartbeat: %v", err) } // csiClient works with CRDs that support json only clientConfig.ContentType = "application/json" csiClient, err := csiclientset.NewForConfig(clientConfig) if err != nil { - glog.Warningf("Failed to create CSI API client: %v", err) + klog.Warningf("Failed to create CSI API client: %v", err) } kubeDeps.KubeClient = kubeClient @@ -663,7 +663,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan if kubeDeps.ContainerManager == nil { if s.CgroupsPerQOS && s.CgroupRoot == "" { - glog.Infof("--cgroups-per-qos enabled, but --cgroup-root was not specified. defaulting to /") + klog.Infof("--cgroups-per-qos enabled, but --cgroup-root was not specified. defaulting to /") s.CgroupRoot = "/" } kubeReserved, err := parseResourceList(s.KubeReserved) @@ -727,7 +727,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan } if err := checkPermissions(); err != nil { - glog.Error(err) + klog.Error(err) } utilruntime.ReallyCrash = s.ReallyCrashForTesting @@ -737,7 +737,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan // TODO(vmarmol): Do this through container config. oomAdjuster := kubeDeps.OOMAdjuster if err := oomAdjuster.ApplyOOMScoreAdj(0, int(s.OOMScoreAdj)); err != nil { - glog.Warning(err) + klog.Warning(err) } if err := RunKubelet(s, kubeDeps, s.RunOnce); err != nil { @@ -749,7 +749,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan go wait.Until(func() { err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress, strconv.Itoa(int(s.HealthzPort))), nil) if err != nil { - glog.Errorf("Starting health server failed: %v", err) + klog.Errorf("Starting health server failed: %v", err) } }, 5*time.Second, wait.NeverStop) } @@ -788,7 +788,7 @@ func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName return "", fmt.Errorf("error fetching current node name from cloud provider: %v", err) } - glog.V(2).Infof("cloud provider determined current node name to be %s", nodeName) + klog.V(2).Infof("cloud provider determined current node name to be %s", nodeName) return nodeName, nil } @@ -822,7 +822,7 @@ func InitializeTLS(kf *options.KubeletFlags, kc *kubeletconfiginternal.KubeletCo return nil, err } - glog.V(4).Infof("Using self-signed cert (%s, %s)", kc.TLSCertFile, kc.TLSPrivateKeyFile) + klog.V(4).Infof("Using self-signed cert (%s, %s)", kc.TLSCertFile, kc.TLSPrivateKeyFile) } } @@ -938,7 +938,7 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie capabilities.Setup(kubeServer.AllowPrivileged, privilegedSources, 0) credentialprovider.SetPreferredDockercfgPath(kubeServer.RootDirectory) - glog.V(2).Infof("Using root directory: %v", kubeServer.RootDirectory) + klog.V(2).Infof("Using root directory: %v", kubeServer.RootDirectory) if kubeDeps.OSInterface == nil { kubeDeps.OSInterface = kubecontainer.RealOS{} @@ -993,10 +993,10 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie if _, err := k.RunOnce(podCfg.Updates()); err != nil { return fmt.Errorf("runonce failed: %v", err) } - glog.Infof("Started kubelet as runonce") + klog.Infof("Started kubelet as runonce") } else { startKubelet(k, podCfg, &kubeServer.KubeletConfiguration, kubeDeps, kubeServer.EnableServer) - glog.Infof("Started kubelet") + klog.Infof("Started kubelet") } return nil } @@ -1015,6 +1015,9 @@ func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubele if kubeCfg.ReadOnlyPort > 0 { go k.ListenAndServeReadOnly(net.ParseIP(kubeCfg.Address), uint(kubeCfg.ReadOnlyPort)) } + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPodResources) { + go k.ListenAndServePodResources() + } } func CreateAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, @@ -1180,7 +1183,7 @@ func RunDockershim(f *options.KubeletFlags, c *kubeletconfiginternal.KubeletConf if err != nil { return err } - glog.V(2).Infof("Starting the GRPC server for the docker CRI shim.") + klog.V(2).Infof("Starting the GRPC server for the docker CRI shim.") server := dockerremote.NewDockerServer(f.RemoteRuntimeEndpoint, ds) if err := server.Start(); err != nil { return err diff --git a/cmd/kubelet/app/server_linux.go b/cmd/kubelet/app/server_linux.go index ceab334c0ba..f7c39d4cb7d 100644 --- a/cmd/kubelet/app/server_linux.go +++ b/cmd/kubelet/app/server_linux.go @@ -17,26 +17,26 @@ limitations under the License. package app import ( - "github.com/golang/glog" "golang.org/x/exp/inotify" + "k8s.io/klog" ) func watchForLockfileContention(path string, done chan struct{}) error { watcher, err := inotify.NewWatcher() if err != nil { - glog.Errorf("unable to create watcher for lockfile: %v", err) + klog.Errorf("unable to create watcher for lockfile: %v", err) return err } if err = watcher.AddWatch(path, inotify.IN_OPEN|inotify.IN_DELETE_SELF); err != nil { - glog.Errorf("unable to watch lockfile: %v", err) + klog.Errorf("unable to watch lockfile: %v", err) return err } go func() { select { case ev := <-watcher.Event: - glog.Infof("inotify event: %v", ev) + klog.Infof("inotify event: %v", ev) case err = <-watcher.Error: - glog.Errorf("inotify watcher error: %v", err) + klog.Errorf("inotify watcher error: %v", err) } close(done) }() diff --git a/cmd/kubemark/BUILD b/cmd/kubemark/BUILD index f1a9e2fc346..489d3d69615 100644 --- a/cmd/kubemark/BUILD +++ b/cmd/kubemark/BUILD @@ -35,9 +35,9 @@ go_library( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec/testing:go_default_library", ], ) diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go index c63d6ebd0f1..2653e863a9d 100644 --- a/cmd/kubemark/hollow-node.go +++ b/cmd/kubemark/hollow-node.go @@ -23,9 +23,9 @@ import ( "os" "time" - "github.com/golang/glog" "github.com/spf13/cobra" "github.com/spf13/pflag" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -138,18 +138,18 @@ func newHollowNodeCommand() *cobra.Command { func run(config *HollowNodeConfig) { if !knownMorphs.Has(config.Morph) { - glog.Fatalf("Unknown morph: %v. Allowed values: %v", config.Morph, knownMorphs.List()) + klog.Fatalf("Unknown morph: %v. Allowed values: %v", config.Morph, knownMorphs.List()) } // create a client to communicate with API server. clientConfig, err := config.createClientConfigFromFile() if err != nil { - glog.Fatalf("Failed to create a ClientConfig: %v. Exiting.", err) + klog.Fatalf("Failed to create a ClientConfig: %v. Exiting.", err) } client, err := clientset.NewForConfig(clientConfig) if err != nil { - glog.Fatalf("Failed to create a ClientSet: %v. Exiting.", err) + klog.Fatalf("Failed to create a ClientSet: %v. Exiting.", err) } if config.Morph == "kubelet" { @@ -181,7 +181,7 @@ func run(config *HollowNodeConfig) { if config.Morph == "proxy" { client, err := clientset.NewForConfig(clientConfig) if err != nil { - glog.Fatalf("Failed to create API Server client: %v", err) + klog.Fatalf("Failed to create API Server client: %v", err) } iptInterface := fakeiptables.NewFake() sysctl := fakesysctl.NewFake() @@ -203,7 +203,7 @@ func run(config *HollowNodeConfig) { config.ProxierMinSyncPeriod, ) if err != nil { - glog.Fatalf("Failed to create hollowProxy instance: %v", err) + klog.Fatalf("Failed to create hollowProxy instance: %v", err) } hollowProxy.Run() } diff --git a/docs/.generated_docs b/docs/.generated_docs index 5709ff5bf54..89a63ed631d 100644 --- a/docs/.generated_docs +++ b/docs/.generated_docs @@ -22,24 +22,10 @@ docs/admin/kubeadm_alpha_kubelet.md docs/admin/kubeadm_alpha_kubelet_config.md docs/admin/kubeadm_alpha_kubelet_config_download.md docs/admin/kubeadm_alpha_kubelet_config_enable-dynamic.md -docs/admin/kubeadm_alpha_phase.md -docs/admin/kubeadm_alpha_phase_addon.md -docs/admin/kubeadm_alpha_phase_addon_all.md -docs/admin/kubeadm_alpha_phase_addon_coredns.md -docs/admin/kubeadm_alpha_phase_addon_kube-proxy.md -docs/admin/kubeadm_alpha_phase_bootstrap-token.md -docs/admin/kubeadm_alpha_phase_bootstrap-token_all.md -docs/admin/kubeadm_alpha_phase_bootstrap-token_cluster-info.md -docs/admin/kubeadm_alpha_phase_bootstrap-token_create.md -docs/admin/kubeadm_alpha_phase_bootstrap-token_node.md -docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-auto-approve.md -docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-post-csrs.md -docs/admin/kubeadm_alpha_phase_mark-master.md -docs/admin/kubeadm_alpha_phase_selfhosting.md -docs/admin/kubeadm_alpha_phase_selfhosting_convert-from-staticpods.md -docs/admin/kubeadm_alpha_phase_upload-config.md docs/admin/kubeadm_alpha_preflight.md docs/admin/kubeadm_alpha_preflight_node.md +docs/admin/kubeadm_alpha_selfhosting.md +docs/admin/kubeadm_alpha_selfhosting_pivot.md docs/admin/kubeadm_completion.md docs/admin/kubeadm_config.md docs/admin/kubeadm_config_images.md @@ -55,7 +41,13 @@ docs/admin/kubeadm_config_upload_from-flags.md docs/admin/kubeadm_config_view.md docs/admin/kubeadm_init.md docs/admin/kubeadm_init_phase.md +docs/admin/kubeadm_init_phase_addon.md +docs/admin/kubeadm_init_phase_addon_all.md +docs/admin/kubeadm_init_phase_addon_coredns.md +docs/admin/kubeadm_init_phase_addon_kube-proxy.md +docs/admin/kubeadm_init_phase_bootstrap-token.md docs/admin/kubeadm_init_phase_certs.md +docs/admin/kubeadm_init_phase_certs_all.md docs/admin/kubeadm_init_phase_certs_apiserver-etcd-client.md docs/admin/kubeadm_init_phase_certs_apiserver-kubelet-client.md docs/admin/kubeadm_init_phase_certs_apiserver.md @@ -68,6 +60,7 @@ docs/admin/kubeadm_init_phase_certs_front-proxy-ca.md docs/admin/kubeadm_init_phase_certs_front-proxy-client.md docs/admin/kubeadm_init_phase_certs_sa.md docs/admin/kubeadm_init_phase_control-plane.md +docs/admin/kubeadm_init_phase_control-plane_all.md docs/admin/kubeadm_init_phase_control-plane_apiserver.md docs/admin/kubeadm_init_phase_control-plane_controller-manager.md docs/admin/kubeadm_init_phase_control-plane_scheduler.md @@ -75,12 +68,17 @@ docs/admin/kubeadm_init_phase_etcd.md docs/admin/kubeadm_init_phase_etcd_local.md docs/admin/kubeadm_init_phase_kubeconfig.md docs/admin/kubeadm_init_phase_kubeconfig_admin.md +docs/admin/kubeadm_init_phase_kubeconfig_all.md docs/admin/kubeadm_init_phase_kubeconfig_controller-manager.md docs/admin/kubeadm_init_phase_kubeconfig_kubelet.md docs/admin/kubeadm_init_phase_kubeconfig_scheduler.md docs/admin/kubeadm_init_phase_kubelet-start.md +docs/admin/kubeadm_init_phase_mark-control-plane.md docs/admin/kubeadm_init_phase_preflight.md -docs/admin/kubeadm_init_phase_wait-control-plane.md +docs/admin/kubeadm_init_phase_upload-config.md +docs/admin/kubeadm_init_phase_upload-config_all.md +docs/admin/kubeadm_init_phase_upload-config_kubeadm.md +docs/admin/kubeadm_init_phase_upload-config_kubelet.md docs/admin/kubeadm_join.md docs/admin/kubeadm_reset.md docs/admin/kubeadm_token.md @@ -118,24 +116,11 @@ docs/man/man1/kubeadm-alpha-kubelet-config-download.1 docs/man/man1/kubeadm-alpha-kubelet-config-enable-dynamic.1 docs/man/man1/kubeadm-alpha-kubelet-config.1 docs/man/man1/kubeadm-alpha-kubelet.1 -docs/man/man1/kubeadm-alpha-phase-addon-all.1 -docs/man/man1/kubeadm-alpha-phase-addon-coredns.1 -docs/man/man1/kubeadm-alpha-phase-addon-kube-proxy.1 -docs/man/man1/kubeadm-alpha-phase-addon.1 -docs/man/man1/kubeadm-alpha-phase-bootstrap-token-all.1 -docs/man/man1/kubeadm-alpha-phase-bootstrap-token-cluster-info.1 -docs/man/man1/kubeadm-alpha-phase-bootstrap-token-create.1 -docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node-allow-auto-approve.1 -docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node-allow-post-csrs.1 -docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node.1 -docs/man/man1/kubeadm-alpha-phase-bootstrap-token.1 -docs/man/man1/kubeadm-alpha-phase-mark-master.1 -docs/man/man1/kubeadm-alpha-phase-selfhosting-convert-from-staticpods.1 -docs/man/man1/kubeadm-alpha-phase-selfhosting.1 -docs/man/man1/kubeadm-alpha-phase-upload-config.1 docs/man/man1/kubeadm-alpha-phase.1 docs/man/man1/kubeadm-alpha-preflight-node.1 docs/man/man1/kubeadm-alpha-preflight.1 +docs/man/man1/kubeadm-alpha-selfhosting-pivot.1 +docs/man/man1/kubeadm-alpha-selfhosting.1 docs/man/man1/kubeadm-alpha.1 docs/man/man1/kubeadm-completion.1 docs/man/man1/kubeadm-config-images-list.1 @@ -151,6 +136,12 @@ docs/man/man1/kubeadm-config-upload-from-flags.1 docs/man/man1/kubeadm-config-upload.1 docs/man/man1/kubeadm-config-view.1 docs/man/man1/kubeadm-config.1 +docs/man/man1/kubeadm-init-phase-addon-all.1 +docs/man/man1/kubeadm-init-phase-addon-coredns.1 +docs/man/man1/kubeadm-init-phase-addon-kube-proxy.1 +docs/man/man1/kubeadm-init-phase-addon.1 +docs/man/man1/kubeadm-init-phase-bootstrap-token.1 +docs/man/man1/kubeadm-init-phase-certs-all.1 docs/man/man1/kubeadm-init-phase-certs-apiserver-etcd-client.1 docs/man/man1/kubeadm-init-phase-certs-apiserver-kubelet-client.1 docs/man/man1/kubeadm-init-phase-certs-apiserver.1 @@ -163,6 +154,7 @@ docs/man/man1/kubeadm-init-phase-certs-front-proxy-ca.1 docs/man/man1/kubeadm-init-phase-certs-front-proxy-client.1 docs/man/man1/kubeadm-init-phase-certs-sa.1 docs/man/man1/kubeadm-init-phase-certs.1 +docs/man/man1/kubeadm-init-phase-control-plane-all.1 docs/man/man1/kubeadm-init-phase-control-plane-apiserver.1 docs/man/man1/kubeadm-init-phase-control-plane-controller-manager.1 docs/man/man1/kubeadm-init-phase-control-plane-scheduler.1 @@ -170,13 +162,18 @@ docs/man/man1/kubeadm-init-phase-control-plane.1 docs/man/man1/kubeadm-init-phase-etcd-local.1 docs/man/man1/kubeadm-init-phase-etcd.1 docs/man/man1/kubeadm-init-phase-kubeconfig-admin.1 +docs/man/man1/kubeadm-init-phase-kubeconfig-all.1 docs/man/man1/kubeadm-init-phase-kubeconfig-controller-manager.1 docs/man/man1/kubeadm-init-phase-kubeconfig-kubelet.1 docs/man/man1/kubeadm-init-phase-kubeconfig-scheduler.1 docs/man/man1/kubeadm-init-phase-kubeconfig.1 docs/man/man1/kubeadm-init-phase-kubelet-start.1 +docs/man/man1/kubeadm-init-phase-mark-control-plane.1 docs/man/man1/kubeadm-init-phase-preflight.1 -docs/man/man1/kubeadm-init-phase-wait-control-plane.1 +docs/man/man1/kubeadm-init-phase-upload-config-all.1 +docs/man/man1/kubeadm-init-phase-upload-config-kubeadm.1 +docs/man/man1/kubeadm-init-phase-upload-config-kubelet.1 +docs/man/man1/kubeadm-init-phase-upload-config.1 docs/man/man1/kubeadm-init-phase.1 docs/man/man1/kubeadm-init.1 docs/man/man1/kubeadm-join.1 @@ -278,7 +275,6 @@ docs/man/man1/kubectl-rollout-resume.1 docs/man/man1/kubectl-rollout-status.1 docs/man/man1/kubectl-rollout-undo.1 docs/man/man1/kubectl-rollout.1 -docs/man/man1/kubectl-run-container.1 docs/man/man1/kubectl-run.1 docs/man/man1/kubectl-scale.1 docs/man/man1/kubectl-set-env.1 @@ -431,7 +427,6 @@ docs/yaml/kubectl/kubectl_proxy.yaml docs/yaml/kubectl/kubectl_replace.yaml docs/yaml/kubectl/kubectl_rolling-update.yaml docs/yaml/kubectl/kubectl_rollout.yaml -docs/yaml/kubectl/kubectl_run-container.yaml docs/yaml/kubectl/kubectl_run.yaml docs/yaml/kubectl/kubectl_scale.yaml docs/yaml/kubectl/kubectl_set.yaml diff --git a/docs/admin/kubeadm_alpha_phase.md b/docs/admin/kubeadm_alpha_selfhosting.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase.md rename to docs/admin/kubeadm_alpha_selfhosting.md diff --git a/docs/admin/kubeadm_alpha_phase_addon.md b/docs/admin/kubeadm_alpha_selfhosting_pivot.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_addon.md rename to docs/admin/kubeadm_alpha_selfhosting_pivot.md diff --git a/docs/admin/kubeadm_alpha_phase_addon_all.md b/docs/admin/kubeadm_init_phase_addon.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_addon_all.md rename to docs/admin/kubeadm_init_phase_addon.md diff --git a/docs/admin/kubeadm_alpha_phase_addon_coredns.md b/docs/admin/kubeadm_init_phase_addon_all.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_addon_coredns.md rename to docs/admin/kubeadm_init_phase_addon_all.md diff --git a/docs/admin/kubeadm_alpha_phase_addon_kube-proxy.md b/docs/admin/kubeadm_init_phase_addon_coredns.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_addon_kube-proxy.md rename to docs/admin/kubeadm_init_phase_addon_coredns.md diff --git a/docs/admin/kubeadm_alpha_phase_bootstrap-token.md b/docs/admin/kubeadm_init_phase_addon_kube-proxy.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_bootstrap-token.md rename to docs/admin/kubeadm_init_phase_addon_kube-proxy.md diff --git a/docs/admin/kubeadm_alpha_phase_bootstrap-token_all.md b/docs/admin/kubeadm_init_phase_bootstrap-token.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_bootstrap-token_all.md rename to docs/admin/kubeadm_init_phase_bootstrap-token.md diff --git a/docs/admin/kubeadm_alpha_phase_bootstrap-token_cluster-info.md b/docs/admin/kubeadm_init_phase_certs_all.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_bootstrap-token_cluster-info.md rename to docs/admin/kubeadm_init_phase_certs_all.md diff --git a/docs/admin/kubeadm_alpha_phase_bootstrap-token_create.md b/docs/admin/kubeadm_init_phase_control-plane_all.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_bootstrap-token_create.md rename to docs/admin/kubeadm_init_phase_control-plane_all.md diff --git a/docs/admin/kubeadm_alpha_phase_bootstrap-token_node.md b/docs/admin/kubeadm_init_phase_kubeconfig_all.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_bootstrap-token_node.md rename to docs/admin/kubeadm_init_phase_kubeconfig_all.md diff --git a/docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-auto-approve.md b/docs/admin/kubeadm_init_phase_mark-control-plane.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-auto-approve.md rename to docs/admin/kubeadm_init_phase_mark-control-plane.md diff --git a/docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-post-csrs.md b/docs/admin/kubeadm_init_phase_upload-config.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-post-csrs.md rename to docs/admin/kubeadm_init_phase_upload-config.md diff --git a/docs/admin/kubeadm_alpha_phase_mark-master.md b/docs/admin/kubeadm_init_phase_upload-config_all.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_mark-master.md rename to docs/admin/kubeadm_init_phase_upload-config_all.md diff --git a/docs/admin/kubeadm_alpha_phase_selfhosting.md b/docs/admin/kubeadm_init_phase_upload-config_kubeadm.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_selfhosting.md rename to docs/admin/kubeadm_init_phase_upload-config_kubeadm.md diff --git a/docs/admin/kubeadm_alpha_phase_selfhosting_convert-from-staticpods.md b/docs/admin/kubeadm_init_phase_upload-config_kubelet.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_selfhosting_convert-from-staticpods.md rename to docs/admin/kubeadm_init_phase_upload-config_kubelet.md diff --git a/docs/api-reference/apps/v1/definitions.html b/docs/api-reference/apps/v1/definitions.html index 3488071d266..cb6bedc8f21 100755 --- a/docs/api-reference/apps/v1/definitions.html +++ b/docs/api-reference/apps/v1/definitions.html @@ -1012,7 +1012,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }

volumeMode

-

volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is an alpha feature and may change in the future.

+

volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.

false

v1.PersistentVolumeMode

@@ -5718,7 +5718,7 @@ Examples:

volumeDevices

-

volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future.

+

volumeDevices is the list of block devices to be used by the container. This is a beta feature.

false

v1.VolumeDevice array

diff --git a/docs/api-reference/apps/v1beta1/definitions.html b/docs/api-reference/apps/v1beta1/definitions.html index cd1c7582017..40de1ba05a8 100755 --- a/docs/api-reference/apps/v1beta1/definitions.html +++ b/docs/api-reference/apps/v1beta1/definitions.html @@ -1040,7 +1040,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }

volumeMode

-

volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is an alpha feature and may change in the future.

+

volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.

false

v1.PersistentVolumeMode

@@ -5853,7 +5853,7 @@ Examples:

volumeDevices

-

volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future.

+

volumeDevices is the list of block devices to be used by the container. This is a beta feature.

false

v1.VolumeDevice array

diff --git a/docs/api-reference/apps/v1beta2/definitions.html b/docs/api-reference/apps/v1beta2/definitions.html index b092861a423..4cc6df83ba9 100755 --- a/docs/api-reference/apps/v1beta2/definitions.html +++ b/docs/api-reference/apps/v1beta2/definitions.html @@ -987,7 +987,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }

volumeMode

-

volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is an alpha feature and may change in the future.

+

volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.

false

v1.PersistentVolumeMode

@@ -6187,7 +6187,7 @@ Examples:

volumeDevices

-

volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future.

+

volumeDevices is the list of block devices to be used by the container. This is a beta feature.

false

v1.VolumeDevice array

diff --git a/docs/api-reference/authentication.k8s.io/v1/definitions.html b/docs/api-reference/authentication.k8s.io/v1/definitions.html index f7cf09b9e69..f2964687302 100755 --- a/docs/api-reference/authentication.k8s.io/v1/definitions.html +++ b/docs/api-reference/authentication.k8s.io/v1/definitions.html @@ -1053,6 +1053,13 @@ When an object is created, the system will populate this list with the current s

string

+ +

audiences

+

Audiences is a list of the identifiers that the resource server presented with the token identifies as. Audience-aware token authenticators will verify that the token was intended for at least one of the audiences in this list. If no audiences are provided, the audience will default to the audience of the Kubernetes apiserver.

+

false

+

string array

+ + @@ -1095,6 +1102,13 @@ When an object is created, the system will populate this list with the current s +

audiences

+

Audiences are audience identifiers chosen by the authenticator that are compatible with both the TokenReview and token. An identifier is any identifier in the intersection of the TokenReviewSpec audiences and the token’s audiences. A client of the TokenReview API that sets the spec.audiences field should validate that a compatible audience identifier is returned in the status.audiences field to ensure that the TokenReview server is audience aware. If a TokenReview returns an empty status.audience field where status.authenticated is "true", the token is valid against the audience of the Kubernetes API server.

+

false

+

string array

+ + +

error

Error indicates that the token couldn’t be checked

false

diff --git a/docs/api-reference/authentication.k8s.io/v1beta1/definitions.html b/docs/api-reference/authentication.k8s.io/v1beta1/definitions.html index 4ab7aadb51c..4b383c273c5 100755 --- a/docs/api-reference/authentication.k8s.io/v1beta1/definitions.html +++ b/docs/api-reference/authentication.k8s.io/v1beta1/definitions.html @@ -567,6 +567,13 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }

string

+ +

audiences

+

Audiences is a list of the identifiers that the resource server presented with the token identifies as. Audience-aware token authenticators will verify that the token was intended for at least one of the audiences in this list. If no audiences are provided, the audience will default to the audience of the Kubernetes apiserver.

+

false

+

string array

+ + @@ -1060,6 +1067,13 @@ When an object is created, the system will populate this list with the current s +

audiences

+

Audiences are audience identifiers chosen by the authenticator that are compatible with both the TokenReview and token. An identifier is any identifier in the intersection of the TokenReviewSpec audiences and the token’s audiences. A client of the TokenReview API that sets the spec.audiences field should validate that a compatible audience identifier is returned in the status.audiences field to ensure that the TokenReview server is audience aware. If a TokenReview returns an empty status.audience field where status.authenticated is "true", the token is valid against the audience of the Kubernetes API server.

+

false

+

string array

+ + +

error

Error indicates that the token couldn’t be checked

false

diff --git a/docs/api-reference/batch/v1/definitions.html b/docs/api-reference/batch/v1/definitions.html index 86722b4d38d..8c96b4ccbfa 100755 --- a/docs/api-reference/batch/v1/definitions.html +++ b/docs/api-reference/batch/v1/definitions.html @@ -4610,7 +4610,7 @@ Examples:

volumeDevices

-

volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future.

+

volumeDevices is the list of block devices to be used by the container. This is a beta feature.

false

v1.VolumeDevice array

diff --git a/docs/api-reference/batch/v1beta1/definitions.html b/docs/api-reference/batch/v1beta1/definitions.html index edf9e64a1ac..793e1768cf6 100755 --- a/docs/api-reference/batch/v1beta1/definitions.html +++ b/docs/api-reference/batch/v1beta1/definitions.html @@ -4761,7 +4761,7 @@ Examples:

volumeDevices

-

volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future.

+

volumeDevices is the list of block devices to be used by the container. This is a beta feature.

false

v1.VolumeDevice array

diff --git a/docs/api-reference/batch/v2alpha1/definitions.html b/docs/api-reference/batch/v2alpha1/definitions.html index ae28550531e..5b032c12058 100755 --- a/docs/api-reference/batch/v2alpha1/definitions.html +++ b/docs/api-reference/batch/v2alpha1/definitions.html @@ -4617,7 +4617,7 @@ Examples:

volumeDevices

-

volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future.

+

volumeDevices is the list of block devices to be used by the container. This is a beta feature.

false

v1.VolumeDevice array

diff --git a/docs/api-reference/extensions/v1beta1/definitions.html b/docs/api-reference/extensions/v1beta1/definitions.html index 8b9523a3cfa..43d8eb01566 100755 --- a/docs/api-reference/extensions/v1beta1/definitions.html +++ b/docs/api-reference/extensions/v1beta1/definitions.html @@ -6565,7 +6565,7 @@ If PodSelector is also set, then the NetworkPolicyPeer as a whole selects the Po

volumeDevices

-

volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future.

+

volumeDevices is the list of block devices to be used by the container. This is a beta feature.

false

v1.VolumeDevice array

diff --git a/docs/api-reference/storage.k8s.io/v1/definitions.html b/docs/api-reference/storage.k8s.io/v1/definitions.html index 8b8becbae02..b0371e3bafa 100755 --- a/docs/api-reference/storage.k8s.io/v1/definitions.html +++ b/docs/api-reference/storage.k8s.io/v1/definitions.html @@ -375,6 +375,12 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
  • v1.StorageClassList

  • +
  • +

    v1.VolumeAttachment

    +
  • +
  • +

    v1.VolumeAttachmentList

    +
  • @@ -438,15 +444,9 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
    -

    v1.Patch

    +

    v1.VolumeAttachmentList

    -

    Patch is provided to give a concrete name and type to the Kubernetes PATCH request body.

    -
    -
    -
    -

    v1.DeleteOptions

    -
    -

    DeleteOptions may be provided when deleting an API object.

    +

    VolumeAttachmentList is a collection of VolumeAttachment objects.

    @@ -481,38 +481,17 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; } - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + @@ -568,9 +547,9 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
    -

    v1.TopologySelectorLabelRequirement

    +

    v1.VolumeAttachmentSource

    -

    A topology selector requirement is a selector that matches given label. This is an alpha feature and may change in the future.

    +

    VolumeAttachmentSource represents a volume that should be attached. Right now only PersistenVolumes can be attached via external attacher, in future we may allow also inline volumes in pods. Exactly one member can be set.

    gracePeriodSeconds

    The duration in seconds before the object should be deleted. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period for the specified type will be used. Defaults to a per object value if not specified. zero means delete immediately.

    metadata

    Standard list metadata More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata

    false

    integer (int64)

    v1.ListMeta

    preconditions

    Must be fulfilled before a deletion is carried out. If not possible, a 409 Conflict status will be returned.

    false

    v1.Preconditions

    orphanDependents

    Deprecated: please use the PropagationPolicy, this field will be deprecated in 1.7. Should the dependent objects be orphaned. If true/false, the "orphan" finalizer will be added to/removed from the object’s finalizers list. Either this field or PropagationPolicy may be set, but not both.

    false

    boolean

    false

    propagationPolicy

    Whether and how garbage collection will be performed. Either this field or OrphanDependents may be set, but not both. The default policy is decided by the existing finalizer set in the metadata.finalizers and the resource-specific default policy. Acceptable values are: Orphan - orphan the dependents; Background - allow the garbage collector to delete the dependents in the background; Foreground - a cascading policy that deletes all dependents in the foreground.

    false

    v1.DeletionPropagation

    dryRun

    When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed

    false

    string array

    items

    Items is the list of VolumeAttachments

    true

    v1.VolumeAttachment array

    @@ -591,122 +570,12 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; } - - - - - - - - - - - - - - -

    key

    The label key that the selector applies to.

    true

    string

    values

    An array of string values. One value must match the label to be selected. Each entry in Values is ORed.

    true

    string array

    - -
    -
    -

    v1.StatusDetails

    -
    -

    StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.

    -
    - ------- - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    NameDescriptionRequiredSchemaDefault

    name

    The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).

    persistentVolumeName

    Name of the persistent volume to attach.

    false

    string

    group

    The group attribute of the resource associated with the status StatusReason.

    false

    string

    kind

    The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds

    false

    string

    uid

    UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids

    false

    string

    causes

    The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.

    false

    v1.StatusCause array

    retryAfterSeconds

    If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.

    false

    integer (int32)

    - -
    -
    -

    v1.Preconditions

    -
    -

    Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.

    -
    - ------- - - - - - - - - - - - - - - - - -
    NameDescriptionRequiredSchemaDefault

    uid

    Specifies the target UID.

    false

    types.UID

    @@ -753,9 +622,9 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
    -

    v1.Initializer

    +

    v1.Preconditions

    -

    Initializer is information about an initializer that has not yet completed.

    +

    Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.

    @@ -776,10 +645,10 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; } - - - - + + + + @@ -868,6 +737,47 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }

    name

    name of the process that is responsible for initializing this object.

    true

    string

    uid

    Specifies the target UID.

    false

    types.UID

    +
    +
    +

    v1.VolumeError

    +
    +

    VolumeError captures an error encountered during a volume operation.

    +
    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameDescriptionRequiredSchemaDefault

    time

    Time the error was encountered.

    false

    string

    message

    String detailing the error encountered during Attach or Detach operation. This string maybe logged, so it should not contain sensitive information.

    false

    string

    +

    v1.WatchEvent

    @@ -906,6 +816,387 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; } +
    +
    +

    v1.VolumeAttachment

    +
    +

    VolumeAttachment captures the intent to attach or detach the specified volume to/from the specified node.

    +
    +
    +

    VolumeAttachment objects are non-namespaced.

    +
    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameDescriptionRequiredSchemaDefault

    kind

    Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds

    false

    string

    apiVersion

    APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources

    false

    string

    metadata

    Standard object metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata

    false

    v1.ObjectMeta

    spec

    Specification of the desired attach/detach volume behavior. Populated by the Kubernetes system.

    true

    v1.VolumeAttachmentSpec

    status

    Status of the VolumeAttachment request. Populated by the entity completing the attach or detach operation, i.e. the external-attacher.

    false

    v1.VolumeAttachmentStatus

    + +
    +
    +

    v1.PersistentVolumeReclaimPolicy

    + +
    +
    +

    v1.TopologySelectorTerm

    +
    +

    A topology selector term represents the result of label queries. A null or empty topology selector term matches no objects. The requirements of them are ANDed. It provides a subset of functionality as NodeSelectorTerm. This is an alpha feature and may change in the future.

    +
    + +++++++ + + + + + + + + + + + + + + + + + + +
    NameDescriptionRequiredSchemaDefault

    matchLabelExpressions

    A list of topology selector requirements by labels.

    false

    v1.TopologySelectorLabelRequirement array

    + +
    +
    +

    v1.VolumeAttachmentSpec

    +
    +

    VolumeAttachmentSpec is the specification of a VolumeAttachment request.

    +
    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameDescriptionRequiredSchemaDefault

    attacher

    Attacher indicates the name of the volume driver that MUST handle this request. This is the name returned by GetPluginName().

    true

    string

    source

    Source represents the volume that should be attached.

    true

    v1.VolumeAttachmentSource

    nodeName

    The node that the volume should be attached to.

    true

    string

    + +
    +
    +

    v1.DeletionPropagation

    + +
    +
    +

    v1.Patch

    +
    +

    Patch is provided to give a concrete name and type to the Kubernetes PATCH request body.

    +
    +
    +
    +

    v1.DeleteOptions

    +
    +

    DeleteOptions may be provided when deleting an API object.

    +
    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameDescriptionRequiredSchemaDefault

    kind

    Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds

    false

    string

    apiVersion

    APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources

    false

    string

    gracePeriodSeconds

    The duration in seconds before the object should be deleted. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period for the specified type will be used. Defaults to a per object value if not specified. zero means delete immediately.

    false

    integer (int64)

    preconditions

    Must be fulfilled before a deletion is carried out. If not possible, a 409 Conflict status will be returned.

    false

    v1.Preconditions

    orphanDependents

    Deprecated: please use the PropagationPolicy, this field will be deprecated in 1.7. Should the dependent objects be orphaned. If true/false, the "orphan" finalizer will be added to/removed from the object’s finalizers list. Either this field or PropagationPolicy may be set, but not both.

    false

    boolean

    false

    propagationPolicy

    Whether and how garbage collection will be performed. Either this field or OrphanDependents may be set, but not both. The default policy is decided by the existing finalizer set in the metadata.finalizers and the resource-specific default policy. Acceptable values are: Orphan - orphan the dependents; Background - allow the garbage collector to delete the dependents in the background; Foreground - a cascading policy that deletes all dependents in the foreground.

    false

    v1.DeletionPropagation

    dryRun

    When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed

    false

    string array

    + +
    +
    +

    v1.StatusDetails

    +
    +

    StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.

    +
    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameDescriptionRequiredSchemaDefault

    name

    The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).

    false

    string

    group

    The group attribute of the resource associated with the status StatusReason.

    false

    string

    kind

    The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds

    false

    string

    uid

    UID of the resource. (when there is a single resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids

    false

    string

    causes

    The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.

    false

    v1.StatusCause array

    retryAfterSeconds

    If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.

    false

    integer (int32)

    + +
    +
    +

    v1.TopologySelectorLabelRequirement

    +
    +

    A topology selector requirement is a selector that matches given label. This is an alpha feature and may change in the future.

    +
    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameDescriptionRequiredSchemaDefault

    key

    The label key that the selector applies to.

    true

    string

    values

    An array of string values. One value must match the label to be selected. Each entry in Values is ORed.

    true

    string array

    + +
    +
    +

    v1.Initializer

    +
    +

    Initializer is information about an initializer that has not yet completed.

    +
    + +++++++ + + + + + + + + + + + + + + + + + + +
    NameDescriptionRequiredSchemaDefault

    name

    name of the process that is responsible for initializing this object.

    true

    string

    +

    v1.StorageClassList

    @@ -963,12 +1254,9 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
    -

    v1.StorageClass

    +

    v1.OwnerReference

    -

    StorageClass describes the parameters for a class of storage for which PersistentVolumes can be dynamically provisioned.

    -
    -
    -

    StorageClasses are non-namespaced; the name of the storage class according to etcd is in ObjectMeta.Name.

    +

    OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.

    @@ -989,74 +1277,46 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; } - - - - - - - - - - - - - - - - - - - - - - + - - - - + + + + - - - - + + + + - - - - + + + + - - + + - - + + - - - - - - + - -

    kind

    Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds

    false

    string

    apiVersion

    APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources

    false

    string

    metadata

    Standard object’s metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata

    false

    v1.ObjectMeta

    provisioner

    Provisioner indicates the type of the provisioner.

    API version of the referent.

    true

    string

    parameters

    Parameters holds the parameters for the provisioner that should create volumes of this storage class.

    false

    object

    kind

    Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds

    true

    string

    reclaimPolicy

    Dynamically provisioned PersistentVolumes of this storage class are created with this reclaimPolicy. Defaults to Delete.

    false

    v1.PersistentVolumeReclaimPolicy

    name

    Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names

    true

    string

    mountOptions

    Dynamically provisioned PersistentVolumes of this storage class are created with these mountOptions, e.g. ["ro", "soft"]. Not validated - mount of the PVs will simply fail if one is invalid.

    false

    string array

    uid

    UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids

    true

    string

    allowVolumeExpansion

    AllowVolumeExpansion shows whether the storage class allow volume expand

    controller

    If true, this reference points to the managing controller.

    false

    boolean

    false

    volumeBindingMode

    VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is only honored by servers that enable the VolumeScheduling feature.

    blockOwnerDeletion

    If true, AND if the owner has the "foregroundDeletion" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs "delete" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.

    false

    v1.VolumeBindingMode

    allowedTopologies

    Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is only honored by servers that enable the VolumeScheduling feature.

    boolean

    false

    v1.TopologySelectorTerm array

    @@ -1218,9 +1478,12 @@ When an object is created, the system will populate this list with the current s
    -

    v1.OwnerReference

    +

    v1.StorageClass

    -

    OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.

    +

    StorageClass describes the parameters for a class of storage for which PersistentVolumes can be dynamically provisioned.

    +
    +
    +

    StorageClasses are non-namespaced; the name of the storage class according to etcd is in ObjectMeta.Name.

    @@ -1241,88 +1504,78 @@ When an object is created, the system will populate this list with the current s - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + - - - - + + + + - - + + + + + + + + + - - - - + + + + - -

    apiVersion

    API version of the referent.

    true

    string

    kind

    Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds

    Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds

    false

    string

    apiVersion

    APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources

    false

    string

    metadata

    Standard object’s metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata

    false

    v1.ObjectMeta

    provisioner

    Provisioner indicates the type of the provisioner.

    true

    string

    name

    Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names

    true

    string

    parameters

    Parameters holds the parameters for the provisioner that should create volumes of this storage class.

    false

    object

    uid

    UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids

    true

    string

    reclaimPolicy

    Dynamically provisioned PersistentVolumes of this storage class are created with this reclaimPolicy. Defaults to Delete.

    false

    v1.PersistentVolumeReclaimPolicy

    controller

    If true, this reference points to the managing controller.

    mountOptions

    Dynamically provisioned PersistentVolumes of this storage class are created with these mountOptions, e.g. ["ro", "soft"]. Not validated - mount of the PVs will simply fail if one is invalid.

    false

    string array

    allowVolumeExpansion

    AllowVolumeExpansion shows whether the storage class allow volume expand

    false

    boolean

    false

    blockOwnerDeletion

    If true, AND if the owner has the "foregroundDeletion" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs "delete" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.

    false

    boolean

    volumeBindingMode

    VolumeBindingMode indicates how PersistentVolumeClaims should be provisioned and bound. When unset, VolumeBindingImmediate is used. This field is only honored by servers that enable the VolumeScheduling feature.

    false

    v1.VolumeBindingMode

    - -
    -
    -

    v1.TopologySelectorTerm

    -
    -

    A topology selector term represents the result of label queries. A null or empty topology selector term matches no objects. The requirements of them are ANDed. It provides a subset of functionality as NodeSelectorTerm. This is an alpha feature and may change in the future.

    -
    - ------- - - - - - - - - - - - - + + - +
    NameDescriptionRequiredSchemaDefault

    matchLabelExpressions

    A list of topology selector requirements by labels.

    allowedTopologies

    Restrict the node topologies where volumes can be dynamically provisioned. Each volume plugin defines its own supported topology specifications. An empty TopologySelectorTerm list means there is no topology restriction. This field is only honored by servers that enable the VolumeScheduling feature.

    false

    v1.TopologySelectorLabelRequirement array

    v1.TopologySelectorTerm array

    -
    -
    -

    v1.PersistentVolumeReclaimPolicy

    -

    v1.APIResource

    @@ -1413,6 +1666,61 @@ When an object is created, the system will populate this list with the current s +
    +
    +

    v1.VolumeAttachmentStatus

    +
    +

    VolumeAttachmentStatus is the status of a VolumeAttachment request.

    +
    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameDescriptionRequiredSchemaDefault

    attached

    Indicates the volume is successfully attached. This field must only be set by the entity completing the attach operation, i.e. the external-attacher.

    true

    boolean

    false

    attachmentMetadata

    Upon successful attach, this field is populated with any information returned by the attach operation that must be passed into subsequent WaitForAttach or Mount calls. This field must only be set by the entity completing the attach operation, i.e. the external-attacher.

    false

    object

    attachError

    The last error encountered during attach operation, if any. This field must only be set by the entity completing the attach operation, i.e. the external-attacher.

    false

    v1.VolumeError

    detachError

    The last error encountered during detach operation, if any. This field must only be set by the entity completing the detach operation, i.e. the external-attacher.

    false

    v1.VolumeError

    +

    types.UID

    @@ -1469,10 +1777,6 @@ Examples:
    -
    -
    -

    v1.DeletionPropagation

    -

    v1.VolumeBindingMode

    diff --git a/docs/api-reference/storage.k8s.io/v1/operations.html b/docs/api-reference/storage.k8s.io/v1/operations.html index 15ad8bb7119..02a6cb2f3a9 100755 --- a/docs/api-reference/storage.k8s.io/v1/operations.html +++ b/docs/api-reference/storage.k8s.io/v1/operations.html @@ -1472,10 +1472,10 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
    -

    watch individual changes to a list of StorageClass. deprecated: use the watch parameter with a list operation instead.

    +

    list or watch objects of kind VolumeAttachment

    -
    GET /apis/storage.k8s.io/v1/watch/storageclasses
    +
    GET /apis/storage.k8s.io/v1/volumeattachments
    @@ -1597,7 +1597,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }

    200

    success

    -

    v1.WatchEvent

    +

    v1.VolumeAttachmentList

    @@ -1647,6 +1647,1411 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
    +

    delete collection of VolumeAttachment

    +
    +
    +
    DELETE /apis/storage.k8s.io/v1/volumeattachments
    +
    +
    +
    +

    Parameters

    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TypeNameDescriptionRequiredSchemaDefault

    QueryParameter

    pretty

    If true, then the output is pretty printed.

    false

    string

    QueryParameter

    labelSelector

    A selector to restrict the list of returned objects by their labels. Defaults to everything.

    false

    string

    QueryParameter

    fieldSelector

    A selector to restrict the list of returned objects by their fields. Defaults to everything.

    false

    string

    QueryParameter

    includeUninitialized

    If true, partially initialized resources are included in the response.

    false

    boolean

    QueryParameter

    watch

    Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.

    false

    boolean

    QueryParameter

    resourceVersion

    When specified with a watch call, shows changes that occur after that particular version of a resource. Defaults to changes from the beginning of history. When specified for list: - if unset, then the result is returned from remote storage based on quorum-read flag; - if it’s 0, then we simply return what we currently have in cache, no guarantee; - if set to non zero, then the result is at least as fresh as given rv.

    false

    string

    QueryParameter

    timeoutSeconds

    Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.

    false

    integer (int32)

    QueryParameter

    limit

    limit is a maximum number of responses to return for a list call. If more items exist, the server will set the continue field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true. +

    The server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.

    false

    integer (int32)

    QueryParameter

    continue

    The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the "next key". +

    This field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.

    false

    string

    + +
    +
    +

    Responses

    + +++++ + + + + + + + + + + + + + + +
    HTTP CodeDescriptionSchema

    200

    success

    v1.Status

    + +
    +
    +

    Consumes

    +
    +
      +
    • +

      /

      +
    • +
    +
    +
    +
    +

    Produces

    +
    +
      +
    • +

      application/json

      +
    • +
    • +

      application/yaml

      +
    • +
    • +

      application/vnd.kubernetes.protobuf

      +
    • +
    +
    +
    +
    +

    Tags

    +
    +
      +
    • +

      apisstorage.k8s.iov1

      +
    • +
    +
    +
    +
    +
    +

    create a VolumeAttachment

    +
    +
    +
    POST /apis/storage.k8s.io/v1/volumeattachments
    +
    +
    +
    +

    Parameters

    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TypeNameDescriptionRequiredSchemaDefault

    QueryParameter

    pretty

    If true, then the output is pretty printed.

    false

    string

    BodyParameter

    body

    true

    v1.VolumeAttachment

    QueryParameter

    dryRun

    When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed

    false

    string

    QueryParameter

    includeUninitialized

    If IncludeUninitialized is specified, the object may be returned without completing initialization.

    false

    boolean

    + +
    +
    +

    Responses

    + +++++ + + + + + + + + + + + + + + + + + + + + + + + + +
    HTTP CodeDescriptionSchema

    202

    Accepted

    v1.VolumeAttachment

    200

    success

    v1.VolumeAttachment

    201

    Created

    v1.VolumeAttachment

    + +
    +
    +

    Consumes

    +
    +
      +
    • +

      /

      +
    • +
    +
    +
    +
    +

    Produces

    +
    +
      +
    • +

      application/json

      +
    • +
    • +

      application/yaml

      +
    • +
    • +

      application/vnd.kubernetes.protobuf

      +
    • +
    +
    +
    +
    +

    Tags

    +
    +
      +
    • +

      apisstorage.k8s.iov1

      +
    • +
    +
    +
    +
    +
    +

    read the specified VolumeAttachment

    +
    +
    +
    GET /apis/storage.k8s.io/v1/volumeattachments/{name}
    +
    +
    +
    +

    Parameters

    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TypeNameDescriptionRequiredSchemaDefault

    QueryParameter

    pretty

    If true, then the output is pretty printed.

    false

    string

    QueryParameter

    export

    Should this value be exported. Export strips fields that a user can not specify.

    false

    boolean

    QueryParameter

    exact

    Should the export be exact. Exact export maintains cluster-specific fields like Namespace.

    false

    boolean

    PathParameter

    name

    name of the VolumeAttachment

    true

    string

    + +
    +
    +

    Responses

    + +++++ + + + + + + + + + + + + + + +
    HTTP CodeDescriptionSchema

    200

    success

    v1.VolumeAttachment

    + +
    +
    +

    Consumes

    +
    +
      +
    • +

      /

      +
    • +
    +
    +
    +
    +

    Produces

    +
    +
      +
    • +

      application/json

      +
    • +
    • +

      application/yaml

      +
    • +
    • +

      application/vnd.kubernetes.protobuf

      +
    • +
    +
    +
    +
    +

    Tags

    +
    +
      +
    • +

      apisstorage.k8s.iov1

      +
    • +
    +
    +
    +
    +
    +

    replace the specified VolumeAttachment

    +
    +
    +
    PUT /apis/storage.k8s.io/v1/volumeattachments/{name}
    +
    +
    +
    +

    Parameters

    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TypeNameDescriptionRequiredSchemaDefault

    QueryParameter

    pretty

    If true, then the output is pretty printed.

    false

    string

    BodyParameter

    body

    true

    v1.VolumeAttachment

    QueryParameter

    dryRun

    When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed

    false

    string

    PathParameter

    name

    name of the VolumeAttachment

    true

    string

    + +
    +
    +

    Responses

    + +++++ + + + + + + + + + + + + + + + + + + + +
    HTTP CodeDescriptionSchema

    200

    success

    v1.VolumeAttachment

    201

    Created

    v1.VolumeAttachment

    + +
    +
    +

    Consumes

    +
    +
      +
    • +

      /

      +
    • +
    +
    +
    +
    +

    Produces

    +
    +
      +
    • +

      application/json

      +
    • +
    • +

      application/yaml

      +
    • +
    • +

      application/vnd.kubernetes.protobuf

      +
    • +
    +
    +
    +
    +

    Tags

    +
    +
      +
    • +

      apisstorage.k8s.iov1

      +
    • +
    +
    +
    +
    +
    +

    delete a VolumeAttachment

    +
    +
    +
    DELETE /apis/storage.k8s.io/v1/volumeattachments/{name}
    +
    +
    +
    +

    Parameters

    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TypeNameDescriptionRequiredSchemaDefault

    QueryParameter

    pretty

    If true, then the output is pretty printed.

    false

    string

    BodyParameter

    body

    false

    v1.DeleteOptions

    QueryParameter

    gracePeriodSeconds

    The duration in seconds before the object should be deleted. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period for the specified type will be used. Defaults to a per object value if not specified. zero means delete immediately.

    false

    integer (int32)

    QueryParameter

    orphanDependents

    Deprecated: please use the PropagationPolicy, this field will be deprecated in 1.7. Should the dependent objects be orphaned. If true/false, the "orphan" finalizer will be added to/removed from the object’s finalizers list. Either this field or PropagationPolicy may be set, but not both.

    false

    boolean

    QueryParameter

    propagationPolicy

    Whether and how garbage collection will be performed. Either this field or OrphanDependents may be set, but not both. The default policy is decided by the existing finalizer set in the metadata.finalizers and the resource-specific default policy. Acceptable values are: Orphan - orphan the dependents; Background - allow the garbage collector to delete the dependents in the background; Foreground - a cascading policy that deletes all dependents in the foreground.

    false

    string

    QueryParameter

    dryRun

    When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed

    false

    string

    PathParameter

    name

    name of the VolumeAttachment

    true

    string

    + +
    +
    +

    Responses

    + +++++ + + + + + + + + + + + + + + + + + + + +
    HTTP CodeDescriptionSchema

    202

    Accepted

    v1.Status

    200

    success

    v1.Status

    + +
    +
    +

    Consumes

    +
    +
      +
    • +

      /

      +
    • +
    +
    +
    +
    +

    Produces

    +
    +
      +
    • +

      application/json

      +
    • +
    • +

      application/yaml

      +
    • +
    • +

      application/vnd.kubernetes.protobuf

      +
    • +
    +
    +
    +
    +

    Tags

    +
    +
      +
    • +

      apisstorage.k8s.iov1

      +
    • +
    +
    +
    +
    +
    +

    partially update the specified VolumeAttachment

    +
    +
    +
    PATCH /apis/storage.k8s.io/v1/volumeattachments/{name}
    +
    +
    +
    +

    Parameters

    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TypeNameDescriptionRequiredSchemaDefault

    QueryParameter

    pretty

    If true, then the output is pretty printed.

    false

    string

    BodyParameter

    body

    true

    v1.Patch

    QueryParameter

    dryRun

    When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed

    false

    string

    PathParameter

    name

    name of the VolumeAttachment

    true

    string

    + +
    +
    +

    Responses

    + +++++ + + + + + + + + + + + + + + +
    HTTP CodeDescriptionSchema

    200

    success

    v1.VolumeAttachment

    + +
    +
    +

    Consumes

    +
    +
      +
    • +

      application/json-patch+json

      +
    • +
    • +

      application/merge-patch+json

      +
    • +
    • +

      application/strategic-merge-patch+json

      +
    • +
    +
    +
    +
    +

    Produces

    +
    +
      +
    • +

      application/json

      +
    • +
    • +

      application/yaml

      +
    • +
    • +

      application/vnd.kubernetes.protobuf

      +
    • +
    +
    +
    +
    +

    Tags

    +
    +
      +
    • +

      apisstorage.k8s.iov1

      +
    • +
    +
    +
    +
    +
    +

    read status of the specified VolumeAttachment

    +
    +
    +
    GET /apis/storage.k8s.io/v1/volumeattachments/{name}/status
    +
    +
    +
    +

    Parameters

    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TypeNameDescriptionRequiredSchemaDefault

    QueryParameter

    pretty

    If true, then the output is pretty printed.

    false

    string

    PathParameter

    name

    name of the VolumeAttachment

    true

    string

    + +
    +
    +

    Responses

    + +++++ + + + + + + + + + + + + + + +
    HTTP CodeDescriptionSchema

    200

    success

    v1.VolumeAttachment

    + +
    +
    +

    Consumes

    +
    +
      +
    • +

      /

      +
    • +
    +
    +
    +
    +

    Produces

    +
    +
      +
    • +

      application/json

      +
    • +
    • +

      application/yaml

      +
    • +
    • +

      application/vnd.kubernetes.protobuf

      +
    • +
    +
    +
    +
    +

    Tags

    +
    +
      +
    • +

      apisstorage.k8s.iov1

      +
    • +
    +
    +
    +
    +
    +

    replace status of the specified VolumeAttachment

    +
    +
    +
    PUT /apis/storage.k8s.io/v1/volumeattachments/{name}/status
    +
    +
    +
    +

    Parameters

    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TypeNameDescriptionRequiredSchemaDefault

    QueryParameter

    pretty

    If true, then the output is pretty printed.

    false

    string

    BodyParameter

    body

    true

    v1.VolumeAttachment

    QueryParameter

    dryRun

    When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed

    false

    string

    PathParameter

    name

    name of the VolumeAttachment

    true

    string

    + +
    +
    +

    Responses

    + +++++ + + + + + + + + + + + + + + + + + + + +
    HTTP CodeDescriptionSchema

    200

    success

    v1.VolumeAttachment

    201

    Created

    v1.VolumeAttachment

    + +
    +
    +

    Consumes

    +
    +
      +
    • +

      /

      +
    • +
    +
    +
    +
    +

    Produces

    +
    +
      +
    • +

      application/json

      +
    • +
    • +

      application/yaml

      +
    • +
    • +

      application/vnd.kubernetes.protobuf

      +
    • +
    +
    +
    +
    +

    Tags

    +
    +
      +
    • +

      apisstorage.k8s.iov1

      +
    • +
    +
    +
    +
    +
    +

    partially update status of the specified VolumeAttachment

    +
    +
    +
    PATCH /apis/storage.k8s.io/v1/volumeattachments/{name}/status
    +
    +
    +
    +

    Parameters

    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TypeNameDescriptionRequiredSchemaDefault

    QueryParameter

    pretty

    If true, then the output is pretty printed.

    false

    string

    BodyParameter

    body

    true

    v1.Patch

    QueryParameter

    dryRun

    When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed

    false

    string

    PathParameter

    name

    name of the VolumeAttachment

    true

    string

    + +
    +
    +

    Responses

    + +++++ + + + + + + + + + + + + + + +
    HTTP CodeDescriptionSchema

    200

    success

    v1.VolumeAttachment

    + +
    +
    +

    Consumes

    +
    +
      +
    • +

      application/json-patch+json

      +
    • +
    • +

      application/merge-patch+json

      +
    • +
    • +

      application/strategic-merge-patch+json

      +
    • +
    +
    +
    +
    +

    Produces

    +
    +
      +
    • +

      application/json

      +
    • +
    • +

      application/yaml

      +
    • +
    • +

      application/vnd.kubernetes.protobuf

      +
    • +
    +
    +
    +
    +

    Tags

    +
    +
      +
    • +

      apisstorage.k8s.iov1

      +
    • +
    +
    +
    +
    +
    +

    watch individual changes to a list of StorageClass. deprecated: use the watch parameter with a list operation instead.

    +
    +
    +
    GET /apis/storage.k8s.io/v1/watch/storageclasses
    +
    +
    +
    +

    Parameters

    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TypeNameDescriptionRequiredSchemaDefault

    QueryParameter

    pretty

    If true, then the output is pretty printed.

    false

    string

    QueryParameter

    labelSelector

    A selector to restrict the list of returned objects by their labels. Defaults to everything.

    false

    string

    QueryParameter

    fieldSelector

    A selector to restrict the list of returned objects by their fields. Defaults to everything.

    false

    string

    QueryParameter

    includeUninitialized

    If true, partially initialized resources are included in the response.

    false

    boolean

    QueryParameter

    watch

    Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.

    false

    boolean

    QueryParameter

    resourceVersion

    When specified with a watch call, shows changes that occur after that particular version of a resource. Defaults to changes from the beginning of history. When specified for list: - if unset, then the result is returned from remote storage based on quorum-read flag; - if it’s 0, then we simply return what we currently have in cache, no guarantee; - if set to non zero, then the result is at least as fresh as given rv.

    false

    string

    QueryParameter

    timeoutSeconds

    Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.

    false

    integer (int32)

    QueryParameter

    limit

    limit is a maximum number of responses to return for a list call. If more items exist, the server will set the continue field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true. +

    The server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.

    false

    integer (int32)

    QueryParameter

    continue

    The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the "next key". +

    This field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.

    false

    string

    + +
    +
    +

    Responses

    + +++++ + + + + + + + + + + + + + + +
    HTTP CodeDescriptionSchema

    200

    success

    v1.WatchEvent

    + +
    +
    +

    Consumes

    +
    +
      +
    • +

      /

      +
    • +
    +
    +
    +
    +

    Produces

    +
    +
      +
    • +

      application/json

      +
    • +
    • +

      application/yaml

      +
    • +
    • +

      application/vnd.kubernetes.protobuf

      +
    • +
    • +

      application/json;stream=watch

      +
    • +
    • +

      application/vnd.kubernetes.protobuf;stream=watch

      +
    • +
    +
    +
    +
    +

    Tags

    +
    +
      +
    • +

      apisstorage.k8s.iov1

      +
    • +
    +
    +
    +
    +

    watch changes to an object of kind StorageClass. deprecated: use the watch parameter with a list operation instead, filtered to a single item with the fieldSelector parameter.

    @@ -1654,7 +3059,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
    -

    Parameters

    +

    Parameters

    @@ -1762,7 +3167,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
    -

    Responses

    +

    Responses

    @@ -1787,7 +3192,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
    -

    Consumes

    +

    Consumes

    • @@ -1797,7 +3202,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
    -

    Produces

    +

    Produces

    • @@ -1819,7 +3224,365 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
    -

    Tags

    +

    Tags

    +
    +
      +
    • +

      apisstorage.k8s.iov1

      +
    • +
    +
    +
    + +
    +

    watch individual changes to a list of VolumeAttachment. deprecated: use the watch parameter with a list operation instead.

    +
    +
    +
    GET /apis/storage.k8s.io/v1/watch/volumeattachments
    +
    +
    +
    +

    Parameters

    +
    ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TypeNameDescriptionRequiredSchemaDefault

    QueryParameter

    pretty

    If true, then the output is pretty printed.

    false

    string

    QueryParameter

    labelSelector

    A selector to restrict the list of returned objects by their labels. Defaults to everything.

    false

    string

    QueryParameter

    fieldSelector

    A selector to restrict the list of returned objects by their fields. Defaults to everything.

    false

    string

    QueryParameter

    includeUninitialized

    If true, partially initialized resources are included in the response.

    false

    boolean

    QueryParameter

    watch

    Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.

    false

    boolean

    QueryParameter

    resourceVersion

    When specified with a watch call, shows changes that occur after that particular version of a resource. Defaults to changes from the beginning of history. When specified for list: - if unset, then the result is returned from remote storage based on quorum-read flag; - if it’s 0, then we simply return what we currently have in cache, no guarantee; - if set to non zero, then the result is at least as fresh as given rv.

    false

    string

    QueryParameter

    timeoutSeconds

    Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.

    false

    integer (int32)

    QueryParameter

    limit

    limit is a maximum number of responses to return for a list call. If more items exist, the server will set the continue field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true. +

    The server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.

    false

    integer (int32)

    QueryParameter

    continue

    The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the "next key". +

    This field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.

    false

    string

    + +
    +
    +

    Responses

    + +++++ + + + + + + + + + + + + + + +
    HTTP CodeDescriptionSchema

    200

    success

    v1.WatchEvent

    + +
    +
    +

    Consumes

    +
    +
      +
    • +

      /

      +
    • +
    +
    +
    +
    +

    Produces

    +
    +
      +
    • +

      application/json

      +
    • +
    • +

      application/yaml

      +
    • +
    • +

      application/vnd.kubernetes.protobuf

      +
    • +
    • +

      application/json;stream=watch

      +
    • +
    • +

      application/vnd.kubernetes.protobuf;stream=watch

      +
    • +
    +
    +
    +
    +

    Tags

    +
    +
      +
    • +

      apisstorage.k8s.iov1

      +
    • +
    +
    +
    +
    +
    +

    watch changes to an object of kind VolumeAttachment. deprecated: use the watch parameter with a list operation instead, filtered to a single item with the fieldSelector parameter.

    +
    +
    +
    GET /apis/storage.k8s.io/v1/watch/volumeattachments/{name}
    +
    +
    +
    +

    Parameters

    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TypeNameDescriptionRequiredSchemaDefault

    QueryParameter

    pretty

    If true, then the output is pretty printed.

    false

    string

    QueryParameter

    labelSelector

    A selector to restrict the list of returned objects by their labels. Defaults to everything.

    false

    string

    QueryParameter

    fieldSelector

    A selector to restrict the list of returned objects by their fields. Defaults to everything.

    false

    string

    QueryParameter

    includeUninitialized

    If true, partially initialized resources are included in the response.

    false

    boolean

    QueryParameter

    watch

    Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.

    false

    boolean

    QueryParameter

    resourceVersion

    When specified with a watch call, shows changes that occur after that particular version of a resource. Defaults to changes from the beginning of history. When specified for list: - if unset, then the result is returned from remote storage based on quorum-read flag; - if it’s 0, then we simply return what we currently have in cache, no guarantee; - if set to non zero, then the result is at least as fresh as given rv.

    false

    string

    QueryParameter

    timeoutSeconds

    Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.

    false

    integer (int32)

    QueryParameter

    limit

    limit is a maximum number of responses to return for a list call. If more items exist, the server will set the continue field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true. +

    The server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.

    false

    integer (int32)

    QueryParameter

    continue

    The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the "next key". +

    This field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.

    false

    string

    PathParameter

    name

    name of the VolumeAttachment

    true

    string

    + +
    +
    +

    Responses

    + +++++ + + + + + + + + + + + + + + +
    HTTP CodeDescriptionSchema

    200

    success

    v1.WatchEvent

    + +
    +
    +

    Consumes

    +
    +
      +
    • +

      /

      +
    • +
    +
    +
    +
    +

    Produces

    +
    +
      +
    • +

      application/json

      +
    • +
    • +

      application/yaml

      +
    • +
    • +

      application/vnd.kubernetes.protobuf

      +
    • +
    • +

      application/json;stream=watch

      +
    • +
    • +

      application/vnd.kubernetes.protobuf;stream=watch

      +
    • +
    +
    +
    +
    +

    Tags

    • diff --git a/docs/api-reference/v1/definitions.html b/docs/api-reference/v1/definitions.html index 92d31c760e2..ef399f1ebcc 100755 --- a/docs/api-reference/v1/definitions.html +++ b/docs/api-reference/v1/definitions.html @@ -689,7 +689,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }

      volumeMode

      -

      volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is an alpha feature and may change in the future.

      +

      volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.

      false

      v1.PersistentVolumeMode

      @@ -2864,54 +2864,6 @@ The resulting set of endpoints can be viewed as:
      -
    -
    -

    v1.GlusterfsVolumeSource

    -
    -

    Represents a Glusterfs mount that lasts the lifetime of a pod. Glusterfs volumes do not support ownership management or SELinux relabeling.

    -
    - ------- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    NameDescriptionRequiredSchemaDefault

    endpoints

    EndpointsName is the endpoint name that details Glusterfs topology. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod

    true

    string

    path

    Path is the Glusterfs volume path. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod

    true

    string

    readOnly

    ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod

    false

    boolean

    false

    -

    v1.NodeStatus

    @@ -3016,6 +2968,54 @@ The resulting set of endpoints can be viewed as:
    +
    +
    +

    v1.GlusterfsVolumeSource

    +
    +

    Represents a Glusterfs mount that lasts the lifetime of a pod. Glusterfs volumes do not support ownership management or SELinux relabeling.

    +
    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameDescriptionRequiredSchemaDefault

    endpoints

    EndpointsName is the endpoint name that details Glusterfs topology. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod

    true

    string

    path

    Path is the Glusterfs volume path. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod

    true

    string

    readOnly

    ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod

    false

    boolean

    false

    +

    v1.Handler

    @@ -4116,7 +4116,7 @@ Examples:

    glusterfs

    Glusterfs represents a Glusterfs volume that is attached to a host and exposed to the pod. Provisioned by an admin. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md

    false

    -

    v1.GlusterfsVolumeSource

    +

    v1.GlusterfsPersistentVolumeSource

    @@ -4282,7 +4282,7 @@ Examples:

    volumeMode

    -

    volumeMode defines if a volume is intended to be used with a formatted filesystem or to remain in raw block state. Value of Filesystem is implied when not included in spec. This is an alpha feature and may change in the future.

    +

    volumeMode defines if a volume is intended to be used with a formatted filesystem or to remain in raw block state. Value of Filesystem is implied when not included in spec. This is a beta feature.

    false

    v1.PersistentVolumeMode

    @@ -8229,6 +8229,61 @@ Examples:
    +
    +
    +

    v1.GlusterfsPersistentVolumeSource

    +
    +

    Represents a Glusterfs mount that lasts the lifetime of a pod. Glusterfs volumes do not support ownership management or SELinux relabeling.

    +
    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameDescriptionRequiredSchemaDefault

    endpoints

    EndpointsName is the endpoint name that details Glusterfs topology. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod

    true

    string

    path

    Path is the Glusterfs volume path. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod

    true

    string

    readOnly

    ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod

    false

    boolean

    false

    endpointsNamespace

    EndpointsNamespace is the namespace that contains Glusterfs endpoint. If this field is empty, the EndpointNamespace defaults to the same namespace as the bound PVC. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod

    false

    string

    +

    v1.EventList

    @@ -10540,7 +10595,7 @@ More info:

    volumeDevices

    -

    volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future.

    +

    volumeDevices is the list of block devices to be used by the container. This is a beta feature.

    false

    v1.VolumeDevice array

    diff --git a/docs/man/man1/kubeadm-alpha-phase-selfhosting.1 b/docs/man/man1/kubeadm-alpha-phase-selfhosting.1 deleted file mode 100644 index b6fd7a0f989..00000000000 --- a/docs/man/man1/kubeadm-alpha-phase-selfhosting.1 +++ /dev/null @@ -1,3 +0,0 @@ -This file is autogenerated, but we've stopped checking such files into the -repository to reduce the need for rebases. Please run hack/generate-docs.sh to -populate this file. diff --git a/docs/man/man1/kubeadm-alpha-phase-upload-config.1 b/docs/man/man1/kubeadm-alpha-phase-upload-config.1 deleted file mode 100644 index b6fd7a0f989..00000000000 --- a/docs/man/man1/kubeadm-alpha-phase-upload-config.1 +++ /dev/null @@ -1,3 +0,0 @@ -This file is autogenerated, but we've stopped checking such files into the -repository to reduce the need for rebases. Please run hack/generate-docs.sh to -populate this file. diff --git a/docs/admin/kubeadm_alpha_phase_upload-config.md b/docs/man/man1/kubeadm-alpha-selfhosting-pivot.1 similarity index 100% rename from docs/admin/kubeadm_alpha_phase_upload-config.md rename to docs/man/man1/kubeadm-alpha-selfhosting-pivot.1 diff --git a/docs/admin/kubeadm_init_phase_wait-control-plane.md b/docs/man/man1/kubeadm-alpha-selfhosting.1 similarity index 100% rename from docs/admin/kubeadm_init_phase_wait-control-plane.md rename to docs/man/man1/kubeadm-alpha-selfhosting.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-addon-all.1 b/docs/man/man1/kubeadm-init-phase-addon-all.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-addon-all.1 rename to docs/man/man1/kubeadm-init-phase-addon-all.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-addon-coredns.1 b/docs/man/man1/kubeadm-init-phase-addon-coredns.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-addon-coredns.1 rename to docs/man/man1/kubeadm-init-phase-addon-coredns.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-addon-kube-proxy.1 b/docs/man/man1/kubeadm-init-phase-addon-kube-proxy.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-addon-kube-proxy.1 rename to docs/man/man1/kubeadm-init-phase-addon-kube-proxy.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-addon.1 b/docs/man/man1/kubeadm-init-phase-addon.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-addon.1 rename to docs/man/man1/kubeadm-init-phase-addon.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-bootstrap-token-all.1 b/docs/man/man1/kubeadm-init-phase-bootstrap-token.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-bootstrap-token-all.1 rename to docs/man/man1/kubeadm-init-phase-bootstrap-token.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-bootstrap-token-cluster-info.1 b/docs/man/man1/kubeadm-init-phase-certs-all.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-bootstrap-token-cluster-info.1 rename to docs/man/man1/kubeadm-init-phase-certs-all.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-bootstrap-token-create.1 b/docs/man/man1/kubeadm-init-phase-control-plane-all.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-bootstrap-token-create.1 rename to docs/man/man1/kubeadm-init-phase-control-plane-all.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node-allow-auto-approve.1 b/docs/man/man1/kubeadm-init-phase-kubeconfig-all.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node-allow-auto-approve.1 rename to docs/man/man1/kubeadm-init-phase-kubeconfig-all.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node-allow-post-csrs.1 b/docs/man/man1/kubeadm-init-phase-mark-control-plane.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node-allow-post-csrs.1 rename to docs/man/man1/kubeadm-init-phase-mark-control-plane.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node.1 b/docs/man/man1/kubeadm-init-phase-upload-config-all.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node.1 rename to docs/man/man1/kubeadm-init-phase-upload-config-all.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-bootstrap-token.1 b/docs/man/man1/kubeadm-init-phase-upload-config-kubeadm.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-bootstrap-token.1 rename to docs/man/man1/kubeadm-init-phase-upload-config-kubeadm.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-mark-master.1 b/docs/man/man1/kubeadm-init-phase-upload-config-kubelet.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-mark-master.1 rename to docs/man/man1/kubeadm-init-phase-upload-config-kubelet.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-selfhosting-convert-from-staticpods.1 b/docs/man/man1/kubeadm-init-phase-upload-config.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-selfhosting-convert-from-staticpods.1 rename to docs/man/man1/kubeadm-init-phase-upload-config.1 diff --git a/docs/man/man1/kubeadm-init-phase-wait-control-plane.1 b/docs/man/man1/kubeadm-init-phase-wait-control-plane.1 deleted file mode 100644 index b6fd7a0f989..00000000000 --- a/docs/man/man1/kubeadm-init-phase-wait-control-plane.1 +++ /dev/null @@ -1,3 +0,0 @@ -This file is autogenerated, but we've stopped checking such files into the -repository to reduce the need for rebases. Please run hack/generate-docs.sh to -populate this file. diff --git a/docs/man/man1/kubectl-run-container.1 b/docs/man/man1/kubectl-run-container.1 deleted file mode 100644 index b6fd7a0f989..00000000000 --- a/docs/man/man1/kubectl-run-container.1 +++ /dev/null @@ -1,3 +0,0 @@ -This file is autogenerated, but we've stopped checking such files into the -repository to reduce the need for rebases. Please run hack/generate-docs.sh to -populate this file. diff --git a/docs/yaml/kubectl/kubectl_run-container.yaml b/docs/yaml/kubectl/kubectl_run-container.yaml deleted file mode 100644 index b6fd7a0f989..00000000000 --- a/docs/yaml/kubectl/kubectl_run-container.yaml +++ /dev/null @@ -1,3 +0,0 @@ -This file is autogenerated, but we've stopped checking such files into the -repository to reduce the need for rebases. Please run hack/generate-docs.sh to -populate this file. diff --git a/hack/.golint_failures b/hack/.golint_failures index bee5a5a403d..f3879831d2f 100644 --- a/hack/.golint_failures +++ b/hack/.golint_failures @@ -134,14 +134,9 @@ pkg/credentialprovider/gcp pkg/credentialprovider/rancher pkg/features pkg/kubeapiserver -pkg/kubeapiserver/admission -pkg/kubeapiserver/authenticator -pkg/kubeapiserver/authorizer -pkg/kubeapiserver/authorizer/modes pkg/kubeapiserver/options pkg/kubectl pkg/kubectl/apps -pkg/kubectl/cmd pkg/kubectl/cmd/annotate pkg/kubectl/cmd/apiresources pkg/kubectl/cmd/apply @@ -186,11 +181,10 @@ pkg/kubectl/cmd/util/openapi pkg/kubectl/cmd/util/sanity pkg/kubectl/cmd/version pkg/kubectl/cmd/wait +pkg/kubectl/describe/versioned pkg/kubectl/generate pkg/kubectl/generate/versioned pkg/kubectl/metricsutil -pkg/kubectl/util -pkg/kubectl/util/slice pkg/kubectl/util/templates pkg/kubelet pkg/kubelet/apis @@ -400,8 +394,6 @@ pkg/version/verflag pkg/volume pkg/volume/azure_dd pkg/volume/azure_file -pkg/volume/cephfs -pkg/volume/configmap pkg/volume/csi/fake pkg/volume/git_repo pkg/volume/host_path @@ -412,7 +404,6 @@ pkg/volume/photon_pd pkg/volume/portworx pkg/volume/rbd pkg/volume/scaleio -pkg/volume/secret pkg/volume/storageos pkg/volume/testing pkg/volume/util/fs @@ -663,7 +654,6 @@ staging/src/k8s.io/client-go/tools/portforward staging/src/k8s.io/client-go/tools/record staging/src/k8s.io/client-go/tools/reference staging/src/k8s.io/client-go/transport -staging/src/k8s.io/client-go/util/cert/triple staging/src/k8s.io/client-go/util/exec staging/src/k8s.io/client-go/util/flowcontrol staging/src/k8s.io/client-go/util/integer diff --git a/hack/godep-restore.sh b/hack/godep-restore.sh index 02b7dbb55e0..b427cdc3b00 100755 --- a/hack/godep-restore.sh +++ b/hack/godep-restore.sh @@ -31,5 +31,5 @@ fi kube::util::ensure_godep_version kube::log::status "Downloading dependencies - this might take a while" -GOPATH="${GOPATH}:${KUBE_ROOT}/staging" godep restore "$@" +GOPATH="${GOPATH}:${KUBE_ROOT}/staging" ${KUBE_GODEP:?} restore "$@" kube::log::status "Done" diff --git a/hack/godep-save.sh b/hack/godep-save.sh index 414929750f1..4b30e5d0bec 100755 --- a/hack/godep-save.sh +++ b/hack/godep-save.sh @@ -57,7 +57,6 @@ fi REQUIRED_BINS=( "github.com/onsi/ginkgo/ginkgo" "github.com/jteeuwen/go-bindata/go-bindata" - "github.com/tools/godep" "github.com/client9/misspell/cmd/misspell" "github.com/cloudflare/cfssl/cmd/cfssl" "github.com/cloudflare/cfssl/cmd/cfssljson" @@ -71,7 +70,7 @@ kube::log::status "Running godep save - this might take a while" # This uses $(pwd) rather than ${KUBE_ROOT} because KUBE_ROOT will be # realpath'ed, and godep barfs ("... is not using a known version control # system") on our staging dirs. -GOPATH="${GOPATH}:$(pwd)/staging" godep save "${REQUIRED_BINS[@]}" +GOPATH="${GOPATH}:$(pwd)/staging" ${KUBE_GODEP:?} save "${REQUIRED_BINS[@]}" # create a symlink in vendor directory pointing to the staging client. This # let other packages use the staging client as if it were vendored. diff --git a/hack/import-restrictions.yaml b/hack/import-restrictions.yaml index 12f53d659d3..3ae27381c33 100644 --- a/hack/import-restrictions.yaml +++ b/hack/import-restrictions.yaml @@ -8,6 +8,7 @@ - k8s.io/kubernetes/pkg/util - k8s.io/api/core/v1 - k8s.io/utils/pointer + - k8s.io/klog # the following are temporary and should go away. Think twice (or more) before adding anything here. # Main goal: pkg/apis should be as self-contained as possible. @@ -31,11 +32,13 @@ allowedImports: - k8s.io/apimachinery - k8s.io/kube-openapi + - k8s.io/klog - baseImportPath: "./vendor/k8s.io/api/" allowedImports: - k8s.io/api - k8s.io/apimachinery + - k8s.io/klog - baseImportPath: "./vendor/k8s.io/code-generator/" ignoredSubTrees: @@ -44,12 +47,14 @@ - k8s.io/gengo - k8s.io/code-generator - k8s.io/kube-openapi + - k8s.io/klog - baseImportPath: "./vendor/k8s.io/client-go/" allowedImports: - k8s.io/api - k8s.io/apimachinery - k8s.io/client-go + - k8s.io/klog # prevent core machinery from taking explicit v1 references unless # necessary @@ -58,6 +63,7 @@ allowedImports: - k8s.io/apimachinery - k8s.io/client-go + - k8s.io/klog - baseImportPath: "./vendor/k8s.io/client-go/tools/" excludeTests: true ignoredSubTrees: @@ -70,7 +76,7 @@ allowedImports: - k8s.io/apimachinery - k8s.io/client-go - + - k8s.io/klog - baseImportPath: "./vendor/k8s.io/apiserver/" allowedImports: @@ -80,6 +86,7 @@ - k8s.io/client-go - k8s.io/kube-openapi - k8s.io/utils + - k8s.io/klog - baseImportPath: "./vendor/k8s.io/metrics/" allowedImports: @@ -87,6 +94,7 @@ - k8s.io/apimachinery - k8s.io/client-go - k8s.io/metrics + - k8s.io/klog - baseImportPath: "./vendor/k8s.io/kube-aggregator/" allowedImports: @@ -96,6 +104,7 @@ - k8s.io/client-go - k8s.io/kube-aggregator - k8s.io/kube-openapi + - k8s.io/klog - baseImportPath: "./vendor/k8s.io/sample-apiserver/" allowedImports: @@ -104,6 +113,7 @@ - k8s.io/apiserver - k8s.io/client-go - k8s.io/sample-apiserver + - k8s.io/klog - baseImportPath: "./vendor/k8s.io/apiextensions-apiserver/" allowedImports: @@ -112,12 +122,14 @@ - k8s.io/apimachinery - k8s.io/apiserver - k8s.io/client-go + - k8s.io/klog - baseImportPath: "./vendor/k8s.io/kube-openapi/" allowedImports: - k8s.io/apimachinery - k8s.io/kube-openapi - k8s.io/gengo + - k8s.io/klog - baseImportPath: "./vendor/k8s.io/sample-cli-plugin/" allowedImports: @@ -158,3 +170,4 @@ - k8s.io/api - k8s.io/apimachinery - k8s.io/client-go + - k8s.io/klog diff --git a/hack/lib/util.sh b/hack/lib/util.sh index 099abc55028..ccecdcfa880 100755 --- a/hack/lib/util.sh +++ b/hack/lib/util.sh @@ -441,28 +441,30 @@ kube::util::ensure_clean_working_dir() { # Ensure that the given godep version is installed and in the path. Almost # nobody should use any version but the default. +# +# Sets: +# KUBE_GODEP: The path to the godep binary +# kube::util::ensure_godep_version() { - GODEP_VERSION=${1:-"v80"} # this version is known to work + local godep_target_version=${1:-"v80-k8s-r1"} # this version is known to work - if [[ "$(godep version 2>/dev/null)" == *"godep ${GODEP_VERSION}"* ]]; then + # If KUBE_GODEP is already set, and it's the right version, then use it. + if [[ -n "${KUBE_GODEP:-}" && "$(${KUBE_GODEP:?} version 2>/dev/null)" == *"godep ${godep_target_version}"* ]]; then + kube::log::status "Using ${KUBE_GODEP}" return fi - kube::log::status "Installing godep version ${GODEP_VERSION}" - go install k8s.io/kubernetes/vendor/github.com/tools/godep/ - if ! which godep >/dev/null 2>&1; then - kube::log::error "Can't find godep - is your GOPATH 'bin' in your PATH?" - kube::log::error " GOPATH: ${GOPATH}" - kube::log::error " PATH: ${PATH}" - return 1 - fi + # Otherwise, install forked godep + kube::log::status "Installing godep version ${godep_target_version}" + # Run in hermetic GOPATH + kube::golang::setup_env + go install k8s.io/kubernetes/third_party/forked/godep + export KUBE_GODEP="${KUBE_GOPATH}/bin/godep" + kube::log::status "Installed ${KUBE_GODEP}" - if [[ "$(godep version 2>/dev/null)" != *"godep ${GODEP_VERSION}"* ]]; then - kube::log::error "Wrong godep version - is your GOPATH 'bin' in your PATH?" - kube::log::error " expected: godep ${GODEP_VERSION}" - kube::log::error " got: $(godep version)" - kube::log::error " GOPATH: ${GOPATH}" - kube::log::error " PATH: ${PATH}" + # Verify that the installed godep from fork is what we expect + if [[ "$(${KUBE_GODEP:?} version 2>/dev/null)" != *"godep ${godep_target_version}"* ]]; then + kube::log::error "Expected godep ${godep_target_version} from ${KUBE_GODEP}, got $(${KUBE_GODEP:?} version)" return 1 fi } diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index ba4f5eba59e..b137b776f9d 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -63,7 +63,9 @@ EVICTION_PRESSURE_TRANSITION_PERIOD=${EVICTION_PRESSURE_TRANSITION_PERIOD:-"1m"} # Note also that you need API_HOST (defined above) for correct DNS. KUBE_PROXY_MODE=${KUBE_PROXY_MODE:-""} ENABLE_CLUSTER_DNS=${KUBE_ENABLE_CLUSTER_DNS:-true} +ENABLE_NODELOCAL_DNS=${KUBE_ENABLE_NODELOCAL_DNS:-false} DNS_SERVER_IP=${KUBE_DNS_SERVER_IP:-10.0.0.10} +LOCAL_DNS_IP=${KUBE_LOCAL_DNS_IP:-169.254.20.10} DNS_DOMAIN=${KUBE_DNS_NAME:-"cluster.local"} KUBECTL=${KUBECTL:-"${KUBE_ROOT}/cluster/kubectl.sh"} WAIT_FOR_URL_API_SERVER=${WAIT_FOR_URL_API_SERVER:-60} @@ -704,7 +706,11 @@ function start_kubelet { mkdir -p "/var/lib/kubelet" &>/dev/null || sudo mkdir -p "/var/lib/kubelet" # Enable dns if [[ "${ENABLE_CLUSTER_DNS}" = true ]]; then - dns_args="--cluster-dns=${DNS_SERVER_IP} --cluster-domain=${DNS_DOMAIN}" + if [[ "${ENABLE_NODELOCAL_DNS:-}" == "true" ]]; then + dns_args="--cluster-dns=${LOCAL_DNS_IP} --cluster-domain=${DNS_DOMAIN}" + else + dns_args="--cluster-dns=${DNS_SERVER_IP} --cluster-domain=${DNS_DOMAIN}" + fi else # To start a private DNS server set ENABLE_CLUSTER_DNS and # DNS_SERVER_IP/DOMAIN. This will at least provide a working @@ -908,6 +914,17 @@ function start_kubedns { fi } +function start_nodelocaldns { + cp "${KUBE_ROOT}/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml" nodelocaldns.yaml + sed -i -e "s/__PILLAR__DNS__DOMAIN__/${DNS_DOMAIN}/g" nodelocaldns.yaml + sed -i -e "s/__PILLAR__DNS__SERVER__/${DNS_SERVER_IP}/g" nodelocaldns.yaml + sed -i -e "s/__PILLAR__LOCAL__DNS__/${LOCAL_DNS_IP}/g" nodelocaldns.yaml + # use kubectl to create nodelocaldns addon + ${KUBECTL} --kubeconfig="${CERT_DIR}/admin.kubeconfig" --namespace=kube-system create -f nodelocaldns.yaml + echo "NodeLocalDNS addon successfully deployed." + rm nodelocaldns.yaml +} + function start_kubedashboard { if [[ "${ENABLE_CLUSTER_DASHBOARD}" = true ]]; then echo "Creating kubernetes-dashboard" @@ -1056,6 +1073,9 @@ if [[ "${START_MODE}" != "kubeletonly" ]]; then fi start_kubeproxy start_kubedns + if [[ "${ENABLE_NODELOCAL_DNS:-}" == "true" ]]; then + start_nodelocaldns + fi start_kubedashboard fi diff --git a/hack/update-generated-pod-resources-dockerized.sh b/hack/update-generated-pod-resources-dockerized.sh new file mode 100755 index 00000000000..d11b136b59f --- /dev/null +++ b/hack/update-generated-pod-resources-dockerized.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE}")/../" && pwd -P)" +POD_RESOURCES_ALPHA="${KUBE_ROOT}/pkg/kubelet/apis/podresources/v1alpha1/" + +source "${KUBE_ROOT}/hack/lib/protoc.sh" +kube::protoc::generate_proto ${POD_RESOURCES_ALPHA} diff --git a/hack/update-generated-pod-resources.sh b/hack/update-generated-pod-resources.sh new file mode 100755 index 00000000000..bdf8c584c0e --- /dev/null +++ b/hack/update-generated-pod-resources.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +# NOTE: All output from this script needs to be copied back to the calling +# source tree. This is managed in kube::build::copy_output in build/common.sh. +# If the output set is changed update that function. + +${KUBE_ROOT}/build/run.sh hack/update-generated-pod-resources-dockerized.sh "$@" diff --git a/hack/update-staging-godeps-dockerized.sh b/hack/update-staging-godeps-dockerized.sh index a0e0e50c37c..6268465ebff 100755 --- a/hack/update-staging-godeps-dockerized.sh +++ b/hack/update-staging-godeps-dockerized.sh @@ -70,7 +70,7 @@ function updateGodepManifest() { pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null kube::log::status "Updating godeps for k8s.io/${repo}" rm -rf Godeps # remove the current Godeps.json so we always rebuild it - GOPATH="${TMP_GOPATH}:${GOPATH}:${GOPATH}/src/k8s.io/kubernetes/staging" godep save ${GODEP_OPTS} ./... 2>&1 | sed 's/^/ /' + GOPATH="${TMP_GOPATH}:${GOPATH}:${GOPATH}/src/k8s.io/kubernetes/staging" ${KUBE_GODEP:?} save ${GODEP_OPTS} ./... 2>&1 | sed 's/^/ /' # Rewriting Godeps.json to cross-out commits that don't really exist because we haven't pushed the prereqs yet local repo diff --git a/hack/verify-generated-pod-resources.sh b/hack/verify-generated-pod-resources.sh new file mode 100755 index 00000000000..84213cc9698 --- /dev/null +++ b/hack/verify-generated-pod-resources.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +KUBE_REMOTE_RUNTIME_ROOT="${KUBE_ROOT}/pkg/kubelet/apis/podresources/v1alpha1" +source "${KUBE_ROOT}/hack/lib/init.sh" + +kube::golang::setup_env + +function cleanup { + rm -rf ${KUBE_REMOTE_RUNTIME_ROOT}/_tmp/ +} + +trap cleanup EXIT + +mkdir -p ${KUBE_REMOTE_RUNTIME_ROOT}/_tmp +cp ${KUBE_REMOTE_RUNTIME_ROOT}/api.pb.go ${KUBE_REMOTE_RUNTIME_ROOT}/_tmp/ + +ret=0 +KUBE_VERBOSE=3 "${KUBE_ROOT}/hack/update-generated-pod-resources.sh" +diff -I "gzipped FileDescriptorProto" -I "0x" -Naupr ${KUBE_REMOTE_RUNTIME_ROOT}/_tmp/api.pb.go ${KUBE_REMOTE_RUNTIME_ROOT}/api.pb.go || ret=$? +if [[ $ret -eq 0 ]]; then + echo "Generated pod resources api is up to date." + cp ${KUBE_REMOTE_RUNTIME_ROOT}/_tmp/api.pb.go ${KUBE_REMOTE_RUNTIME_ROOT}/ +else + echo "Generated pod resources api is out of date. Please run hack/update-generated-pod-resources.sh" + exit 1 +fi diff --git a/hack/verify-godeps.sh b/hack/verify-godeps.sh index a7f46f99a7b..2de2e6022c9 100755 --- a/hack/verify-godeps.sh +++ b/hack/verify-godeps.sh @@ -51,7 +51,6 @@ function cleanup { echo "Removing ${_tmpdir}" rm -rf "${_tmpdir}" fi - export GODEP="" } trap cleanup EXIT @@ -85,7 +84,7 @@ ret=0 pushd "${KUBE_ROOT}" > /dev/null 2>&1 # Test for diffs - if ! _out="$(diff -Naupr --ignore-matching-lines='^\s*\"GoVersion\":' --ignore-matching-line='^\s*\"GodepVersion\":' --ignore-matching-lines='^\s*\"Comment\":' Godeps/Godeps.json ${_kubetmp}/Godeps/Godeps.json)"; then + if ! _out="$(diff -Naupr --ignore-matching-lines='^\s*\"GoVersion\":' Godeps/Godeps.json ${_kubetmp}/Godeps/Godeps.json)"; then echo "Your Godeps.json is different:" >&2 echo "${_out}" >&2 echo "Godeps Verify failed." >&2 diff --git a/pkg/BUILD b/pkg/BUILD index 84b7252c892..6f74b6995c9 100644 --- a/pkg/BUILD +++ b/pkg/BUILD @@ -12,7 +12,6 @@ filegroup( srcs = [ ":package-srcs", "//pkg/api/endpoints:all-srcs", - "//pkg/api/events:all-srcs", "//pkg/api/legacyscheme:all-srcs", "//pkg/api/persistentvolume:all-srcs", "//pkg/api/persistentvolumeclaim:all-srcs", diff --git a/pkg/api/events/OWNERS b/pkg/api/events/OWNERS deleted file mode 100755 index cbc4e8d9d72..00000000000 --- a/pkg/api/events/OWNERS +++ /dev/null @@ -1,2 +0,0 @@ -reviewers: -- gmarek diff --git a/pkg/api/persistentvolume/BUILD b/pkg/api/persistentvolume/BUILD index c4743ccc70f..7983e320316 100644 --- a/pkg/api/persistentvolume/BUILD +++ b/pkg/api/persistentvolume/BUILD @@ -1,9 +1,6 @@ package(default_visibility = ["//visibility:public"]) -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -28,3 +25,16 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) + +go_test( + name = "go_default_test", + srcs = ["util_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/apis/core:go_default_library", + "//pkg/features:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", + ], +) diff --git a/pkg/api/persistentvolume/util.go b/pkg/api/persistentvolume/util.go index 3b28f207a10..298a38250f7 100644 --- a/pkg/api/persistentvolume/util.go +++ b/pkg/api/persistentvolume/util.go @@ -22,10 +22,22 @@ import ( "k8s.io/kubernetes/pkg/features" ) -// DropDisabledAlphaFields removes disabled fields from the pv spec. +// DropDisabledFields removes disabled fields from the pv spec. // This should be called from PrepareForCreate/PrepareForUpdate for all resources containing a pv spec. -func DropDisabledAlphaFields(pvSpec *api.PersistentVolumeSpec) { +func DropDisabledFields(pvSpec *api.PersistentVolumeSpec, oldPVSpec *api.PersistentVolumeSpec) { if !utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { + // TODO(liggitt): change this to only drop pvSpec.VolumeMode if (oldPVSpec == nil || oldPVSpec.VolumeMode == nil) + // Requires more coordinated changes to validation pvSpec.VolumeMode = nil + if oldPVSpec != nil { + oldPVSpec.VolumeMode = nil + } + } + + if !utilfeature.DefaultFeatureGate.Enabled(features.CSIPersistentVolume) { + // if this is a new PV, or the old PV didn't already have the CSI field, clear it + if oldPVSpec == nil || oldPVSpec.PersistentVolumeSource.CSI == nil { + pvSpec.PersistentVolumeSource.CSI = nil + } } } diff --git a/pkg/api/persistentvolume/util_test.go b/pkg/api/persistentvolume/util_test.go new file mode 100644 index 00000000000..6ea33b5a38c --- /dev/null +++ b/pkg/api/persistentvolume/util_test.go @@ -0,0 +1,155 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package persistentvolume + +import ( + "reflect" + "testing" + + "k8s.io/apimachinery/pkg/util/diff" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" +) + +func TestDropDisabledFields(t *testing.T) { + specWithCSI := func() *api.PersistentVolumeSpec { + return &api.PersistentVolumeSpec{PersistentVolumeSource: api.PersistentVolumeSource{CSI: &api.CSIPersistentVolumeSource{}}} + } + specWithoutCSI := func() *api.PersistentVolumeSpec { + return &api.PersistentVolumeSpec{PersistentVolumeSource: api.PersistentVolumeSource{CSI: nil}} + } + specWithMode := func(mode *api.PersistentVolumeMode) *api.PersistentVolumeSpec { + return &api.PersistentVolumeSpec{VolumeMode: mode} + } + + modeBlock := api.PersistentVolumeBlock + + tests := map[string]struct { + oldSpec *api.PersistentVolumeSpec + newSpec *api.PersistentVolumeSpec + expectOldSpec *api.PersistentVolumeSpec + expectNewSpec *api.PersistentVolumeSpec + csiEnabled bool + blockEnabled bool + }{ + "disabled csi clears new": { + csiEnabled: false, + newSpec: specWithCSI(), + expectNewSpec: specWithoutCSI(), + oldSpec: nil, + expectOldSpec: nil, + }, + "disabled csi clears update when old pv did not use csi": { + csiEnabled: false, + newSpec: specWithCSI(), + expectNewSpec: specWithoutCSI(), + oldSpec: specWithoutCSI(), + expectOldSpec: specWithoutCSI(), + }, + "disabled csi preserves update when old pv did use csi": { + csiEnabled: false, + newSpec: specWithCSI(), + expectNewSpec: specWithCSI(), + oldSpec: specWithCSI(), + expectOldSpec: specWithCSI(), + }, + + "enabled csi preserves new": { + csiEnabled: true, + newSpec: specWithCSI(), + expectNewSpec: specWithCSI(), + oldSpec: nil, + expectOldSpec: nil, + }, + "enabled csi preserves update when old pv did not use csi": { + csiEnabled: true, + newSpec: specWithCSI(), + expectNewSpec: specWithCSI(), + oldSpec: specWithoutCSI(), + expectOldSpec: specWithoutCSI(), + }, + "enabled csi preserves update when old pv did use csi": { + csiEnabled: true, + newSpec: specWithCSI(), + expectNewSpec: specWithCSI(), + oldSpec: specWithCSI(), + expectOldSpec: specWithCSI(), + }, + + "disabled block clears new": { + blockEnabled: false, + newSpec: specWithMode(&modeBlock), + expectNewSpec: specWithMode(nil), + oldSpec: nil, + expectOldSpec: nil, + }, + "disabled block clears update when old pv did not use block": { + blockEnabled: false, + newSpec: specWithMode(&modeBlock), + expectNewSpec: specWithMode(nil), + oldSpec: specWithMode(nil), + expectOldSpec: specWithMode(nil), + }, + // TODO: consider changing this case to preserve + "disabled block clears old and new on update when old pv did use block": { + blockEnabled: false, + newSpec: specWithMode(&modeBlock), + expectNewSpec: specWithMode(nil), + oldSpec: specWithMode(&modeBlock), + expectOldSpec: specWithMode(nil), + }, + + "enabled block preserves new": { + blockEnabled: true, + newSpec: specWithMode(&modeBlock), + expectNewSpec: specWithMode(&modeBlock), + oldSpec: nil, + expectOldSpec: nil, + }, + "enabled block preserves update when old pv did not use block": { + blockEnabled: true, + newSpec: specWithMode(&modeBlock), + expectNewSpec: specWithMode(&modeBlock), + oldSpec: specWithMode(nil), + expectOldSpec: specWithMode(nil), + }, + "enabled block preserves update when old pv did use block": { + blockEnabled: true, + newSpec: specWithMode(&modeBlock), + expectNewSpec: specWithMode(&modeBlock), + oldSpec: specWithMode(&modeBlock), + expectOldSpec: specWithMode(&modeBlock), + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIPersistentVolume, tc.csiEnabled)() + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, tc.blockEnabled)() + + DropDisabledFields(tc.newSpec, tc.oldSpec) + if !reflect.DeepEqual(tc.newSpec, tc.expectNewSpec) { + t.Error(diff.ObjectReflectDiff(tc.newSpec, tc.expectNewSpec)) + } + if !reflect.DeepEqual(tc.oldSpec, tc.expectOldSpec) { + t.Error(diff.ObjectReflectDiff(tc.oldSpec, tc.expectOldSpec)) + } + }) + } +} diff --git a/pkg/api/persistentvolumeclaim/BUILD b/pkg/api/persistentvolumeclaim/BUILD index 1ff7b1dc95c..fba0dfd6598 100644 --- a/pkg/api/persistentvolumeclaim/BUILD +++ b/pkg/api/persistentvolumeclaim/BUILD @@ -32,11 +32,15 @@ filegroup( go_test( name = "go_default_test", - srcs = ["util_test.go"], + srcs = [ + "main_test.go", + "util_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", "//pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/pkg/kubectl/describe/describe.go b/pkg/api/persistentvolumeclaim/main_test.go similarity index 63% rename from pkg/kubectl/describe/describe.go rename to pkg/api/persistentvolumeclaim/main_test.go index aa6f37a071e..3ce0a93c8c1 100644 --- a/pkg/kubectl/describe/describe.go +++ b/pkg/api/persistentvolumeclaim/main_test.go @@ -14,13 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -package describe +package persistentvolumeclaim import ( - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/kubernetes/pkg/printers" + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" ) -// DescriberFunc gives a way to display the specified RESTMapping type -type DescriberFunc func(restClientGetter genericclioptions.RESTClientGetter, mapping *meta.RESTMapping) (printers.Describer, error) +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/api/persistentvolumeclaim/util_test.go b/pkg/api/persistentvolumeclaim/util_test.go index e12acb3a9e7..7bb61edd1dd 100644 --- a/pkg/api/persistentvolumeclaim/util_test.go +++ b/pkg/api/persistentvolumeclaim/util_test.go @@ -20,6 +20,7 @@ import ( "testing" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/features" ) @@ -36,36 +37,24 @@ func TestDropAlphaPVCVolumeMode(t *testing.T) { } // Enable alpha feature BlockVolume - err1 := utilfeature.DefaultFeatureGate.Set("BlockVolume=true") - if err1 != nil { - t.Fatalf("Failed to enable feature gate for BlockVolume: %v", err1) - } - + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() // now test dropping the fields - should not be dropped DropDisabledAlphaFields(&pvc.Spec) // check to make sure VolumeDevices is still present // if featureset is set to true - if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { - if pvc.Spec.VolumeMode == nil { - t.Error("VolumeMode in pvc.Spec should not have been dropped based on feature-gate") - } + if pvc.Spec.VolumeMode == nil { + t.Error("VolumeMode in pvc.Spec should not have been dropped based on feature-gate") } // Disable alpha feature BlockVolume - err := utilfeature.DefaultFeatureGate.Set("BlockVolume=false") - if err != nil { - t.Fatalf("Failed to disable feature gate for BlockVolume: %v", err) - } - + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, false)() // now test dropping the fields DropDisabledAlphaFields(&pvc.Spec) // check to make sure VolumeDevices is nil // if featureset is set to false - if !utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { - if pvc.Spec.VolumeMode != nil { - t.Error("DropDisabledAlphaFields VolumeMode for pvc.Spec failed") - } + if pvc.Spec.VolumeMode != nil { + t.Error("DropDisabledAlphaFields VolumeMode for pvc.Spec failed") } } diff --git a/pkg/api/pod/BUILD b/pkg/api/pod/BUILD index 77b08bc48bc..5721392c09a 100644 --- a/pkg/api/pod/BUILD +++ b/pkg/api/pod/BUILD @@ -33,7 +33,10 @@ filegroup( go_test( name = "go_default_test", - srcs = ["util_test.go"], + srcs = [ + "main_test.go", + "util_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", @@ -41,5 +44,6 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/pkg/api/pod/main_test.go b/pkg/api/pod/main_test.go new file mode 100644 index 00000000000..4fb45213978 --- /dev/null +++ b/pkg/api/pod/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pod + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/api/pod/util_test.go b/pkg/api/pod/util_test.go index e27ac9c4e70..afbaa8fd698 100644 --- a/pkg/api/pod/util_test.go +++ b/pkg/api/pod/util_test.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/features" ) @@ -303,42 +304,32 @@ func TestDropAlphaVolumeDevices(t *testing.T) { } // Enable alpha feature BlockVolume - err1 := utilfeature.DefaultFeatureGate.Set("BlockVolume=true") - if err1 != nil { - t.Fatalf("Failed to enable feature gate for BlockVolume: %v", err1) - } + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() // now test dropping the fields - should not be dropped DropDisabledAlphaFields(&testPod.Spec) // check to make sure VolumeDevices is still present // if featureset is set to true - if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { - if testPod.Spec.Containers[0].VolumeDevices == nil { - t.Error("VolumeDevices in Container should not have been dropped based on feature-gate") - } - if testPod.Spec.InitContainers[0].VolumeDevices == nil { - t.Error("VolumeDevices in InitContainers should not have been dropped based on feature-gate") - } + if testPod.Spec.Containers[0].VolumeDevices == nil { + t.Error("VolumeDevices in Container should not have been dropped based on feature-gate") + } + if testPod.Spec.InitContainers[0].VolumeDevices == nil { + t.Error("VolumeDevices in InitContainers should not have been dropped based on feature-gate") } // Disable alpha feature BlockVolume - err := utilfeature.DefaultFeatureGate.Set("BlockVolume=false") - if err != nil { - t.Fatalf("Failed to disable feature gate for BlockVolume: %v", err) - } + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, false)() // now test dropping the fields DropDisabledAlphaFields(&testPod.Spec) // check to make sure VolumeDevices is nil // if featureset is set to false - if !utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { - if testPod.Spec.Containers[0].VolumeDevices != nil { - t.Error("DropDisabledAlphaFields for Containers failed") - } - if testPod.Spec.InitContainers[0].VolumeDevices != nil { - t.Error("DropDisabledAlphaFields for InitContainers failed") - } + if testPod.Spec.Containers[0].VolumeDevices != nil { + t.Error("DropDisabledAlphaFields for Containers failed") + } + if testPod.Spec.InitContainers[0].VolumeDevices != nil { + t.Error("DropDisabledAlphaFields for InitContainers failed") } } diff --git a/pkg/api/podsecuritypolicy/BUILD b/pkg/api/podsecuritypolicy/BUILD index 4c1f9db836f..4d566715fb7 100644 --- a/pkg/api/podsecuritypolicy/BUILD +++ b/pkg/api/podsecuritypolicy/BUILD @@ -32,12 +32,16 @@ filegroup( go_test( name = "go_default_test", - srcs = ["util_test.go"], + srcs = [ + "main_test.go", + "util_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", "//pkg/apis/policy:go_default_library", "//pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/pkg/api/podsecuritypolicy/main_test.go b/pkg/api/podsecuritypolicy/main_test.go new file mode 100644 index 00000000000..f379a10cfbe --- /dev/null +++ b/pkg/api/podsecuritypolicy/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podsecuritypolicy + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/api/v1/persistentvolume/BUILD b/pkg/api/v1/persistentvolume/BUILD index 0a5cbd81f52..06cd0069710 100644 --- a/pkg/api/v1/persistentvolume/BUILD +++ b/pkg/api/v1/persistentvolume/BUILD @@ -5,11 +5,7 @@ go_library( srcs = ["util.go"], importpath = "k8s.io/kubernetes/pkg/api/v1/persistentvolume", visibility = ["//visibility:public"], - deps = [ - "//pkg/features:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - ], + deps = ["//staging/src/k8s.io/api/core/v1:go_default_library"], ) go_test( @@ -18,11 +14,9 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", - "//pkg/features:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", ], ) diff --git a/pkg/api/v1/persistentvolume/util.go b/pkg/api/v1/persistentvolume/util.go index 063c3ff8e74..e43ae58d2ee 100644 --- a/pkg/api/v1/persistentvolume/util.go +++ b/pkg/api/v1/persistentvolume/util.go @@ -18,8 +18,6 @@ package persistentvolume import ( corev1 "k8s.io/api/core/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/kubernetes/pkg/features" ) func getClaimRefNamespace(pv *corev1.PersistentVolume) string { @@ -134,11 +132,3 @@ func VisitPVSecretNames(pv *corev1.PersistentVolume, visitor Visitor) bool { } return true } - -// DropDisabledAlphaFields removes disabled fields from the pv spec. -// This should be called from PrepareForCreate/PrepareForUpdate for all resources containing a pv spec. -func DropDisabledAlphaFields(pvSpec *corev1.PersistentVolumeSpec) { - if !utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { - pvSpec.VolumeMode = nil - } -} diff --git a/pkg/api/v1/persistentvolume/util_test.go b/pkg/api/v1/persistentvolume/util_test.go index 56473d37bcd..62e60cc6e9e 100644 --- a/pkg/api/v1/persistentvolume/util_test.go +++ b/pkg/api/v1/persistentvolume/util_test.go @@ -25,9 +25,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" - utilfeature "k8s.io/apiserver/pkg/util/feature" api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/features" ) func TestPVSecrets(t *testing.T) { @@ -272,56 +270,3 @@ func newHostPathType(pathType string) *corev1.HostPathType { *hostPathType = corev1.HostPathType(pathType) return hostPathType } - -func TestDropAlphaPVVolumeMode(t *testing.T) { - vmode := corev1.PersistentVolumeFilesystem - - // PersistentVolume with VolumeMode set - pv := corev1.PersistentVolume{ - Spec: corev1.PersistentVolumeSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - PersistentVolumeSource: corev1.PersistentVolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/foo", - Type: newHostPathType(string(corev1.HostPathDirectory)), - }, - }, - StorageClassName: "test-storage-class", - VolumeMode: &vmode, - }, - } - - // Enable alpha feature BlockVolume - err1 := utilfeature.DefaultFeatureGate.Set("BlockVolume=true") - if err1 != nil { - t.Fatalf("Failed to enable feature gate for BlockVolume: %v", err1) - } - - // now test dropping the fields - should not be dropped - DropDisabledAlphaFields(&pv.Spec) - - // check to make sure VolumeDevices is still present - // if featureset is set to true - if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { - if pv.Spec.VolumeMode == nil { - t.Error("VolumeMode in pv.Spec should not have been dropped based on feature-gate") - } - } - - // Disable alpha feature BlockVolume - err := utilfeature.DefaultFeatureGate.Set("BlockVolume=false") - if err != nil { - t.Fatalf("Failed to disable feature gate for BlockVolume: %v", err) - } - - // now test dropping the fields - DropDisabledAlphaFields(&pv.Spec) - - // check to make sure VolumeDevices is nil - // if featureset is set to false - if !utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { - if pv.Spec.VolumeMode != nil { - t.Error("DropDisabledAlphaFields VolumeMode for pv.Spec failed") - } - } -} diff --git a/pkg/apis/auditregistration/validation/validation.go b/pkg/apis/auditregistration/validation/validation.go index 101b1fd6aa9..849d0b111a1 100644 --- a/pkg/apis/auditregistration/validation/validation.go +++ b/pkg/apis/auditregistration/validation/validation.go @@ -36,8 +36,8 @@ func ValidateAuditSink(as *auditregistration.AuditSink) field.ErrorList { // ValidateAuditSinkSpec validates the sink spec for audit func ValidateAuditSinkSpec(s auditregistration.AuditSinkSpec, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList - allErrs = append(allErrs, ValidatePolicy(s.Policy, field.NewPath("policy"))...) - allErrs = append(allErrs, ValidateWebhook(s.Webhook, field.NewPath("webhook"))...) + allErrs = append(allErrs, ValidatePolicy(s.Policy, fldPath.Child("policy"))...) + allErrs = append(allErrs, ValidateWebhook(s.Webhook, fldPath.Child("webhook"))...) return allErrs } diff --git a/pkg/apis/authentication/OWNERS b/pkg/apis/authentication/OWNERS index 2bdfd0ce5bc..3b7ea1b131f 100755 --- a/pkg/apis/authentication/OWNERS +++ b/pkg/apis/authentication/OWNERS @@ -1,9 +1,7 @@ +# approval on api packages bubbles to api-approvers reviewers: -- liggitt -- lavalamp -- wojtek-t -- deads2k -- sttts -- mbohlool -- jianhuiz -- enj +- sig-auth-authenticators-approvers +- sig-auth-authenticators-reviewers +labels: +- sig/auth + diff --git a/pkg/apis/authentication/types.go b/pkg/apis/authentication/types.go index deb33cfc198..203bf22bb34 100644 --- a/pkg/apis/authentication/types.go +++ b/pkg/apis/authentication/types.go @@ -59,6 +59,12 @@ type TokenReview struct { type TokenReviewSpec struct { // Token is the opaque bearer token. Token string + // Audiences is a list of the identifiers that the resource server presented + // with the token identifies as. Audience-aware token authenticators will + // verify that the token was intended for at least one of the audiences in + // this list. If no audiences are provided, the audience will default to the + // audience of the Kubernetes apiserver. + Audiences []string } // TokenReviewStatus is the result of the token authentication request. @@ -68,6 +74,16 @@ type TokenReviewStatus struct { Authenticated bool // User is the UserInfo associated with the provided token. User UserInfo + // Audiences are audience identifiers chosen by the authenticator that are + // compatible with both the TokenReview and token. An identifier is any + // identifier in the intersection of the TokenReviewSpec audiences and the + // token's audiences. A client of the TokenReview API that sets the + // spec.audiences field should validate that a compatible audience identifier + // is returned in the status.audiences field to ensure that the TokenReview + // server is audience aware. If a TokenReview returns an empty + // status.audience field where status.authenticated is "true", the token is + // valid against the audience of the Kubernetes API server. + Audiences []string // Error indicates that the token couldn't be checked Error string } diff --git a/pkg/apis/authentication/v1/zz_generated.conversion.go b/pkg/apis/authentication/v1/zz_generated.conversion.go index afa00b1594f..8e95da3274b 100644 --- a/pkg/apis/authentication/v1/zz_generated.conversion.go +++ b/pkg/apis/authentication/v1/zz_generated.conversion.go @@ -263,6 +263,7 @@ func Convert_authentication_TokenReview_To_v1_TokenReview(in *authentication.Tok func autoConvert_v1_TokenReviewSpec_To_authentication_TokenReviewSpec(in *v1.TokenReviewSpec, out *authentication.TokenReviewSpec, s conversion.Scope) error { out.Token = in.Token + out.Audiences = *(*[]string)(unsafe.Pointer(&in.Audiences)) return nil } @@ -273,6 +274,7 @@ func Convert_v1_TokenReviewSpec_To_authentication_TokenReviewSpec(in *v1.TokenRe func autoConvert_authentication_TokenReviewSpec_To_v1_TokenReviewSpec(in *authentication.TokenReviewSpec, out *v1.TokenReviewSpec, s conversion.Scope) error { out.Token = in.Token + out.Audiences = *(*[]string)(unsafe.Pointer(&in.Audiences)) return nil } @@ -286,6 +288,7 @@ func autoConvert_v1_TokenReviewStatus_To_authentication_TokenReviewStatus(in *v1 if err := Convert_v1_UserInfo_To_authentication_UserInfo(&in.User, &out.User, s); err != nil { return err } + out.Audiences = *(*[]string)(unsafe.Pointer(&in.Audiences)) out.Error = in.Error return nil } @@ -300,6 +303,7 @@ func autoConvert_authentication_TokenReviewStatus_To_v1_TokenReviewStatus(in *au if err := Convert_authentication_UserInfo_To_v1_UserInfo(&in.User, &out.User, s); err != nil { return err } + out.Audiences = *(*[]string)(unsafe.Pointer(&in.Audiences)) out.Error = in.Error return nil } diff --git a/pkg/apis/authentication/v1beta1/zz_generated.conversion.go b/pkg/apis/authentication/v1beta1/zz_generated.conversion.go index 97437b18f1a..95d65c5dab3 100644 --- a/pkg/apis/authentication/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/authentication/v1beta1/zz_generated.conversion.go @@ -113,6 +113,7 @@ func Convert_authentication_TokenReview_To_v1beta1_TokenReview(in *authenticatio func autoConvert_v1beta1_TokenReviewSpec_To_authentication_TokenReviewSpec(in *v1beta1.TokenReviewSpec, out *authentication.TokenReviewSpec, s conversion.Scope) error { out.Token = in.Token + out.Audiences = *(*[]string)(unsafe.Pointer(&in.Audiences)) return nil } @@ -123,6 +124,7 @@ func Convert_v1beta1_TokenReviewSpec_To_authentication_TokenReviewSpec(in *v1bet func autoConvert_authentication_TokenReviewSpec_To_v1beta1_TokenReviewSpec(in *authentication.TokenReviewSpec, out *v1beta1.TokenReviewSpec, s conversion.Scope) error { out.Token = in.Token + out.Audiences = *(*[]string)(unsafe.Pointer(&in.Audiences)) return nil } @@ -136,6 +138,7 @@ func autoConvert_v1beta1_TokenReviewStatus_To_authentication_TokenReviewStatus(i if err := Convert_v1beta1_UserInfo_To_authentication_UserInfo(&in.User, &out.User, s); err != nil { return err } + out.Audiences = *(*[]string)(unsafe.Pointer(&in.Audiences)) out.Error = in.Error return nil } @@ -150,6 +153,7 @@ func autoConvert_authentication_TokenReviewStatus_To_v1beta1_TokenReviewStatus(i if err := Convert_authentication_UserInfo_To_v1beta1_UserInfo(&in.User, &out.User, s); err != nil { return err } + out.Audiences = *(*[]string)(unsafe.Pointer(&in.Audiences)) out.Error = in.Error return nil } diff --git a/pkg/apis/authentication/zz_generated.deepcopy.go b/pkg/apis/authentication/zz_generated.deepcopy.go index 2a4cdd07abf..1f485500439 100644 --- a/pkg/apis/authentication/zz_generated.deepcopy.go +++ b/pkg/apis/authentication/zz_generated.deepcopy.go @@ -136,7 +136,7 @@ func (in *TokenReview) DeepCopyInto(out *TokenReview) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) return } @@ -162,6 +162,11 @@ func (in *TokenReview) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TokenReviewSpec) DeepCopyInto(out *TokenReviewSpec) { *out = *in + if in.Audiences != nil { + in, out := &in.Audiences, &out.Audiences + *out = make([]string, len(*in)) + copy(*out, *in) + } return } @@ -179,6 +184,11 @@ func (in *TokenReviewSpec) DeepCopy() *TokenReviewSpec { func (in *TokenReviewStatus) DeepCopyInto(out *TokenReviewStatus) { *out = *in in.User.DeepCopyInto(&out.User) + if in.Audiences != nil { + in, out := &in.Audiences, &out.Audiences + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/authorization/OWNERS b/pkg/apis/authorization/OWNERS index c1613fc2e02..ff4a7f4bf9a 100755 --- a/pkg/apis/authorization/OWNERS +++ b/pkg/apis/authorization/OWNERS @@ -1,17 +1,7 @@ +# approval on api packages bubbles to api-approvers reviewers: -- thockin -- lavalamp -- smarterclayton -- wojtek-t -- deads2k -- liggitt -- nikhiljindal -- erictune -- sttts -- ncdc -- dims -- mml -- mbohlool -- david-mcmahon -- jianhuiz -- enj +- sig-auth-authorizers-approvers +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/pkg/apis/batch/validation/BUILD b/pkg/apis/batch/validation/BUILD index d371966a354..4a29be29ae2 100644 --- a/pkg/apis/batch/validation/BUILD +++ b/pkg/apis/batch/validation/BUILD @@ -27,7 +27,10 @@ go_library( go_test( name = "go_default_test", - srcs = ["validation_test.go"], + srcs = [ + "main_test.go", + "validation_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/apis/batch:go_default_library", @@ -36,6 +39,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/pkg/apis/batch/validation/main_test.go b/pkg/apis/batch/validation/main_test.go new file mode 100644 index 00000000000..ad488a1caaa --- /dev/null +++ b/pkg/apis/batch/validation/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validation + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/apis/certificates/OWNERS b/pkg/apis/certificates/OWNERS index 1d1ab36e756..796d862bd9c 100755 --- a/pkg/apis/certificates/OWNERS +++ b/pkg/apis/certificates/OWNERS @@ -1,14 +1,7 @@ +# approval on api packages bubbles to api-approvers reviewers: -- thockin -- lavalamp -- smarterclayton -- deads2k -- caesarxuchao -- liggitt -- sttts -- dims -- errordeveloper -- mbohlool -- david-mcmahon -- jianhuiz -- enj +- sig-auth-certificates-approvers +- sig-auth-certificates-reviewers +labels: +- sig/auth + diff --git a/pkg/apis/core/fuzzer/fuzzer.go b/pkg/apis/core/fuzzer/fuzzer.go index cc18ed395d8..391226ba007 100644 --- a/pkg/apis/core/fuzzer/fuzzer.go +++ b/pkg/apis/core/fuzzer/fuzzer.go @@ -403,11 +403,15 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} { pv.Status.Message = c.RandString() reclamationPolicies := []core.PersistentVolumeReclaimPolicy{core.PersistentVolumeReclaimRecycle, core.PersistentVolumeReclaimRetain} pv.Spec.PersistentVolumeReclaimPolicy = reclamationPolicies[c.Rand.Intn(len(reclamationPolicies))] + volumeModes := []core.PersistentVolumeMode{core.PersistentVolumeFilesystem, core.PersistentVolumeBlock} + pv.Spec.VolumeMode = &volumeModes[c.Rand.Intn(len(volumeModes))] }, func(pvc *core.PersistentVolumeClaim, c fuzz.Continue) { c.FuzzNoCustom(pvc) // fuzz self without calling this function again types := []core.PersistentVolumeClaimPhase{core.ClaimBound, core.ClaimPending, core.ClaimLost} pvc.Status.Phase = types[c.Rand.Intn(len(types))] + volumeModes := []core.PersistentVolumeMode{core.PersistentVolumeFilesystem, core.PersistentVolumeBlock} + pvc.Spec.VolumeMode = &volumeModes[c.Rand.Intn(len(volumeModes))] }, func(obj *core.AzureDiskVolumeSource, c fuzz.Continue) { if obj.CachingMode == nil { diff --git a/pkg/apis/core/types.go b/pkg/apis/core/types.go index b6ea32da4ad..251547f601b 100644 --- a/pkg/apis/core/types.go +++ b/pkg/apis/core/types.go @@ -26,19 +26,19 @@ import ( const ( // NamespaceDefault means the object is in the default namespace which is applied when not specified by clients - NamespaceDefault string = "default" + NamespaceDefault = "default" // NamespaceAll is the default argument to specify on a context when you want to list or filter resources across all namespaces - NamespaceAll string = "" + NamespaceAll = "" // NamespaceNone is the argument for a context when there is no namespace. - NamespaceNone string = "" + NamespaceNone = "" // NamespaceSystem is the system namespace where we place system components. - NamespaceSystem string = "kube-system" + NamespaceSystem = "kube-system" // NamespacePublic is the namespace where we place public info (ConfigMaps) - NamespacePublic string = "kube-public" + NamespacePublic = "kube-public" // NamespaceNodeLease is the namespace where we place node lease objects (used for node heartbeats) - NamespaceNodeLease string = "kube-node-lease" + NamespaceNodeLease = "kube-node-lease" // TerminationMessagePathDefault means the default path to capture the application termination message running in a container - TerminationMessagePathDefault string = "/dev/termination-log" + TerminationMessagePathDefault = "/dev/termination-log" ) // Volume represents a named volume in a pod that may be accessed by any containers in the pod. @@ -175,7 +175,7 @@ type PersistentVolumeSource struct { HostPath *HostPathVolumeSource // Glusterfs represents a Glusterfs volume that is attached to a host and exposed to the pod // +optional - Glusterfs *GlusterfsVolumeSource + Glusterfs *GlusterfsPersistentVolumeSource // NFS represents an NFS mount on the host that shares a pod's lifetime // +optional NFS *NFSVolumeSource @@ -229,7 +229,7 @@ type PersistentVolumeSource struct { // More info: https://releases.k8s.io/HEAD/examples/volumes/storageos/README.md // +optional StorageOS *StorageOSPersistentVolumeSource - // CSI (Container Storage Interface) represents storage that handled by an external CSI driver (Beta feature). + // CSI (Container Storage Interface) represents storage that handled by an external CSI driver. // +optional CSI *CSIPersistentVolumeSource } @@ -298,7 +298,7 @@ type PersistentVolumeSpec struct { MountOptions []string // volumeMode defines if a volume is intended to be used with a formatted filesystem // or to remain in raw block state. Value of Filesystem is implied when not included in spec. - // This is an alpha feature and may change in the future. + // This is a beta feature. // +optional VolumeMode *PersistentVolumeMode // NodeAffinity defines constraints that limit what nodes this volume can be accessed from. @@ -410,7 +410,7 @@ type PersistentVolumeClaimSpec struct { StorageClassName *string // volumeMode defines what type of volume is required by the claim. // Value of Filesystem is implied when not included in claim spec. - // This is an alpha feature and may change in the future. + // This is a beta feature. // +optional VolumeMode *PersistentVolumeMode // This field requires the VolumeSnapshotDataSource alpha feature gate to be @@ -935,6 +935,30 @@ type GlusterfsVolumeSource struct { ReadOnly bool } +// Represents a Glusterfs mount that lasts the lifetime of a pod. +// Glusterfs volumes do not support ownership management or SELinux relabeling. +type GlusterfsPersistentVolumeSource struct { + // EndpointsName is the endpoint name that details Glusterfs topology. + // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod + EndpointsName string + + // Path is the Glusterfs volume path. + // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod + Path string + + // ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. + // Defaults to false. + // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod + // +optional + ReadOnly bool + + // EndpointsNamespace is the namespace that contains Glusterfs endpoint. + // If this field is empty, the EndpointNamespace defaults to the same namespace as the bound PVC. + // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod + // +optional + EndpointsNamespace *string +} + // Represents a Rados Block Device mount that lasts the lifetime of a pod. // RBD volumes support ownership management and SELinux relabeling. type RBDVolumeSource struct { @@ -1523,7 +1547,7 @@ type LocalVolumeSource struct { FSType *string } -// Represents storage that is managed by an external CSI volume driver (Beta feature) +// Represents storage that is managed by an external CSI volume driver. type CSIPersistentVolumeSource struct { // Driver is the name of the driver to use for this volume. // Required. @@ -1944,7 +1968,7 @@ type Container struct { // +optional VolumeMounts []VolumeMount // volumeDevices is the list of block devices to be used by the container. - // This is an alpha feature and may change in the future. + // This is a beta feature. // +optional VolumeDevices []VolumeDevice // +optional diff --git a/pkg/apis/core/v1/BUILD b/pkg/apis/core/v1/BUILD index 16e3e13d8f0..b7e881dd95c 100644 --- a/pkg/apis/core/v1/BUILD +++ b/pkg/apis/core/v1/BUILD @@ -36,6 +36,7 @@ go_test( srcs = [ "conversion_test.go", "defaults_test.go", + "main_test.go", ], embed = [":go_default_library"], deps = [ @@ -44,6 +45,7 @@ go_test( "//pkg/apis/apps:go_default_library", "//pkg/apis/core:go_default_library", "//pkg/apis/core/fuzzer:go_default_library", + "//pkg/features:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/apitesting/fuzzer:go_default_library", @@ -55,6 +57,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/pkg/apis/core/v1/defaults_test.go b/pkg/apis/core/v1/defaults_test.go index 9c5da00b3b8..b082014864d 100644 --- a/pkg/apis/core/v1/defaults_test.go +++ b/pkg/apis/core/v1/defaults_test.go @@ -27,8 +27,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" corev1 "k8s.io/kubernetes/pkg/apis/core/v1" + "k8s.io/kubernetes/pkg/features" utilpointer "k8s.io/utils/pointer" // enforce that all types are installed @@ -803,6 +805,7 @@ func TestSetDefaultSecret(t *testing.T) { } func TestSetDefaultPersistentVolume(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, false)() pv := &v1.PersistentVolume{} obj2 := roundTrip(t, runtime.Object(pv)) pv2 := obj2.(*v1.PersistentVolume) @@ -822,10 +825,7 @@ func TestSetDefaultPersistentVolume(t *testing.T) { } // When feature gate is enabled, field should be defaulted - err := utilfeature.DefaultFeatureGate.Set("BlockVolume=true") - if err != nil { - t.Fatalf("Failed to enable feature gate for BlockVolume: %v", err) - } + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() obj3 := roundTrip(t, runtime.Object(pv)).(*v1.PersistentVolume) outputMode3 := obj3.Spec.VolumeMode @@ -834,15 +834,10 @@ func TestSetDefaultPersistentVolume(t *testing.T) { } else if *outputMode3 != defaultMode { t.Errorf("Expected VolumeMode to be defaulted to: %+v, got: %+v", defaultMode, outputMode3) } - - err = utilfeature.DefaultFeatureGate.Set("BlockVolume=false") - if err != nil { - t.Fatalf("Failed to disable feature gate for BlockVolume: %v", err) - } - } func TestSetDefaultPersistentVolumeClaim(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, false)() pvc := &v1.PersistentVolumeClaim{} obj2 := roundTrip(t, runtime.Object(pvc)) pvc2 := obj2.(*v1.PersistentVolumeClaim) @@ -859,10 +854,7 @@ func TestSetDefaultPersistentVolumeClaim(t *testing.T) { } // When feature gate is enabled, field should be defaulted - err := utilfeature.DefaultFeatureGate.Set("BlockVolume=true") - if err != nil { - t.Fatalf("Failed to enable feature gate for BlockVolume: %v", err) - } + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() obj3 := roundTrip(t, runtime.Object(pvc)).(*v1.PersistentVolumeClaim) outputMode3 := obj3.Spec.VolumeMode @@ -871,11 +863,6 @@ func TestSetDefaultPersistentVolumeClaim(t *testing.T) { } else if *outputMode3 != defaultMode { t.Errorf("Expected VolumeMode to be defaulted to: %+v, got: %+v", defaultMode, outputMode3) } - - err = utilfeature.DefaultFeatureGate.Set("BlockVolume=false") - if err != nil { - t.Fatalf("Failed to disable feature gate for BlockVolume: %v", err) - } } func TestSetDefaulEndpointsProtocol(t *testing.T) { diff --git a/pkg/apis/core/v1/main_test.go b/pkg/apis/core/v1/main_test.go new file mode 100644 index 00000000000..e46b01929f7 --- /dev/null +++ b/pkg/apis/core/v1/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/apis/core/v1/zz_generated.conversion.go b/pkg/apis/core/v1/zz_generated.conversion.go index d56c950d734..43940142556 100644 --- a/pkg/apis/core/v1/zz_generated.conversion.go +++ b/pkg/apis/core/v1/zz_generated.conversion.go @@ -610,6 +610,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*v1.GlusterfsPersistentVolumeSource)(nil), (*core.GlusterfsPersistentVolumeSource)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_GlusterfsPersistentVolumeSource_To_core_GlusterfsPersistentVolumeSource(a.(*v1.GlusterfsPersistentVolumeSource), b.(*core.GlusterfsPersistentVolumeSource), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*core.GlusterfsPersistentVolumeSource)(nil), (*v1.GlusterfsPersistentVolumeSource)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_core_GlusterfsPersistentVolumeSource_To_v1_GlusterfsPersistentVolumeSource(a.(*core.GlusterfsPersistentVolumeSource), b.(*v1.GlusterfsPersistentVolumeSource), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*v1.GlusterfsVolumeSource)(nil), (*core.GlusterfsVolumeSource)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1_GlusterfsVolumeSource_To_core_GlusterfsVolumeSource(a.(*v1.GlusterfsVolumeSource), b.(*core.GlusterfsVolumeSource), scope) }); err != nil { @@ -3602,6 +3612,32 @@ func Convert_core_GitRepoVolumeSource_To_v1_GitRepoVolumeSource(in *core.GitRepo return autoConvert_core_GitRepoVolumeSource_To_v1_GitRepoVolumeSource(in, out, s) } +func autoConvert_v1_GlusterfsPersistentVolumeSource_To_core_GlusterfsPersistentVolumeSource(in *v1.GlusterfsPersistentVolumeSource, out *core.GlusterfsPersistentVolumeSource, s conversion.Scope) error { + out.EndpointsName = in.EndpointsName + out.Path = in.Path + out.ReadOnly = in.ReadOnly + out.EndpointsNamespace = (*string)(unsafe.Pointer(in.EndpointsNamespace)) + return nil +} + +// Convert_v1_GlusterfsPersistentVolumeSource_To_core_GlusterfsPersistentVolumeSource is an autogenerated conversion function. +func Convert_v1_GlusterfsPersistentVolumeSource_To_core_GlusterfsPersistentVolumeSource(in *v1.GlusterfsPersistentVolumeSource, out *core.GlusterfsPersistentVolumeSource, s conversion.Scope) error { + return autoConvert_v1_GlusterfsPersistentVolumeSource_To_core_GlusterfsPersistentVolumeSource(in, out, s) +} + +func autoConvert_core_GlusterfsPersistentVolumeSource_To_v1_GlusterfsPersistentVolumeSource(in *core.GlusterfsPersistentVolumeSource, out *v1.GlusterfsPersistentVolumeSource, s conversion.Scope) error { + out.EndpointsName = in.EndpointsName + out.Path = in.Path + out.ReadOnly = in.ReadOnly + out.EndpointsNamespace = (*string)(unsafe.Pointer(in.EndpointsNamespace)) + return nil +} + +// Convert_core_GlusterfsPersistentVolumeSource_To_v1_GlusterfsPersistentVolumeSource is an autogenerated conversion function. +func Convert_core_GlusterfsPersistentVolumeSource_To_v1_GlusterfsPersistentVolumeSource(in *core.GlusterfsPersistentVolumeSource, out *v1.GlusterfsPersistentVolumeSource, s conversion.Scope) error { + return autoConvert_core_GlusterfsPersistentVolumeSource_To_v1_GlusterfsPersistentVolumeSource(in, out, s) +} + func autoConvert_v1_GlusterfsVolumeSource_To_core_GlusterfsVolumeSource(in *v1.GlusterfsVolumeSource, out *core.GlusterfsVolumeSource, s conversion.Scope) error { out.EndpointsName = in.EndpointsName out.Path = in.Path @@ -4932,7 +4968,7 @@ func autoConvert_v1_PersistentVolumeSource_To_core_PersistentVolumeSource(in *v1 out.GCEPersistentDisk = (*core.GCEPersistentDiskVolumeSource)(unsafe.Pointer(in.GCEPersistentDisk)) out.AWSElasticBlockStore = (*core.AWSElasticBlockStoreVolumeSource)(unsafe.Pointer(in.AWSElasticBlockStore)) out.HostPath = (*core.HostPathVolumeSource)(unsafe.Pointer(in.HostPath)) - out.Glusterfs = (*core.GlusterfsVolumeSource)(unsafe.Pointer(in.Glusterfs)) + out.Glusterfs = (*core.GlusterfsPersistentVolumeSource)(unsafe.Pointer(in.Glusterfs)) out.NFS = (*core.NFSVolumeSource)(unsafe.Pointer(in.NFS)) out.RBD = (*core.RBDPersistentVolumeSource)(unsafe.Pointer(in.RBD)) out.ISCSI = (*core.ISCSIPersistentVolumeSource)(unsafe.Pointer(in.ISCSI)) @@ -4963,7 +4999,7 @@ func autoConvert_core_PersistentVolumeSource_To_v1_PersistentVolumeSource(in *co out.GCEPersistentDisk = (*v1.GCEPersistentDiskVolumeSource)(unsafe.Pointer(in.GCEPersistentDisk)) out.AWSElasticBlockStore = (*v1.AWSElasticBlockStoreVolumeSource)(unsafe.Pointer(in.AWSElasticBlockStore)) out.HostPath = (*v1.HostPathVolumeSource)(unsafe.Pointer(in.HostPath)) - out.Glusterfs = (*v1.GlusterfsVolumeSource)(unsafe.Pointer(in.Glusterfs)) + out.Glusterfs = (*v1.GlusterfsPersistentVolumeSource)(unsafe.Pointer(in.Glusterfs)) out.NFS = (*v1.NFSVolumeSource)(unsafe.Pointer(in.NFS)) out.RBD = (*v1.RBDPersistentVolumeSource)(unsafe.Pointer(in.RBD)) out.Quobyte = (*v1.QuobyteVolumeSource)(unsafe.Pointer(in.Quobyte)) diff --git a/pkg/apis/core/validation/BUILD b/pkg/apis/core/validation/BUILD index 9ea47dc5d6f..a815e5b0ac7 100644 --- a/pkg/apis/core/validation/BUILD +++ b/pkg/apis/core/validation/BUILD @@ -39,7 +39,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -47,6 +47,7 @@ go_test( name = "go_default_test", srcs = [ "events_test.go", + "main_test.go", "validation_test.go", ], embed = [":go_default_library"], diff --git a/pkg/apis/core/validation/main_test.go b/pkg/apis/core/validation/main_test.go new file mode 100644 index 00000000000..ad488a1caaa --- /dev/null +++ b/pkg/apis/core/validation/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validation + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go index 7af94bd42f7..6f0302c37fa 100644 --- a/pkg/apis/core/validation/validation.go +++ b/pkg/apis/core/validation/validation.go @@ -27,7 +27,7 @@ import ( "regexp" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -903,6 +903,26 @@ func validateGlusterfsVolumeSource(glusterfs *core.GlusterfsVolumeSource, fldPat } return allErrs } +func validateGlusterfsPersistentVolumeSource(glusterfs *core.GlusterfsPersistentVolumeSource, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if len(glusterfs.EndpointsName) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("endpoints"), "")) + } + if len(glusterfs.Path) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("path"), "")) + } + if glusterfs.EndpointsNamespace != nil { + endpointNs := glusterfs.EndpointsNamespace + if *endpointNs == "" { + allErrs = append(allErrs, field.Invalid(fldPath.Child("endpointsNamespace"), *endpointNs, "if the endpointnamespace is set, it must be a valid namespace name")) + } else { + for _, msg := range ValidateNamespaceName(*endpointNs, false) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("endpointsNamespace"), *endpointNs, msg)) + } + } + } + return allErrs +} func validateFlockerVolumeSource(flocker *core.FlockerVolumeSource, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} @@ -1423,24 +1443,27 @@ func validateStorageOSPersistentVolumeSource(storageos *core.StorageOSPersistent return allErrs } +func ValidateCSIDriverName(driverName string, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if len(driverName) == 0 { + allErrs = append(allErrs, field.Required(fldPath, "")) + } + + if len(driverName) > 63 { + allErrs = append(allErrs, field.TooLong(fldPath, driverName, 63)) + } + + if !csiDriverNameRexp.MatchString(driverName) { + allErrs = append(allErrs, field.Invalid(fldPath, driverName, validation.RegexError(csiDriverNameRexpErrMsg, csiDriverNameRexpFmt, "csi-hostpath"))) + } + return allErrs +} + func validateCSIPersistentVolumeSource(csi *core.CSIPersistentVolumeSource, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} - if !utilfeature.DefaultFeatureGate.Enabled(features.CSIPersistentVolume) { - allErrs = append(allErrs, field.Forbidden(fldPath, "CSIPersistentVolume disabled by feature-gate")) - } - - if len(csi.Driver) == 0 { - allErrs = append(allErrs, field.Required(fldPath.Child("driver"), "")) - } - - if len(csi.Driver) > 63 { - allErrs = append(allErrs, field.TooLong(fldPath.Child("driver"), csi.Driver, 63)) - } - - if !csiDriverNameRexp.MatchString(csi.Driver) { - allErrs = append(allErrs, field.Invalid(fldPath.Child("driver"), csi.Driver, validation.RegexError(csiDriverNameRexpErrMsg, csiDriverNameRexpFmt, "csi-hostpath"))) - } + allErrs = append(allErrs, ValidateCSIDriverName(csi.Driver, fldPath.Child("driver"))...) if len(csi.VolumeHandle) == 0 { allErrs = append(allErrs, field.Required(fldPath.Child("volumeHandle"), "")) @@ -1567,7 +1590,7 @@ func ValidatePersistentVolume(pv *core.PersistentVolume) field.ErrorList { allErrs = append(allErrs, field.Forbidden(specPath.Child("glusterfs"), "may not specify more than 1 volume type")) } else { numVolumes++ - allErrs = append(allErrs, validateGlusterfsVolumeSource(pv.Spec.Glusterfs, specPath.Child("glusterfs"))...) + allErrs = append(allErrs, validateGlusterfsPersistentVolumeSource(pv.Spec.Glusterfs, specPath.Child("glusterfs"))...) } } if pv.Spec.Flocker != nil { @@ -4281,7 +4304,7 @@ func ValidateNodeUpdate(node, oldNode *core.Node) field.ErrorList { // We made allowed changes to oldNode, and now we compare oldNode to node. Any remaining differences indicate changes to protected fields. // TODO: Add a 'real' error type for this error and provide print actual diffs. if !apiequality.Semantic.DeepEqual(oldNode, node) { - glog.V(4).Infof("Update failed validation %#v vs %#v", oldNode, node) + klog.V(4).Infof("Update failed validation %#v vs %#v", oldNode, node) allErrs = append(allErrs, field.Forbidden(field.NewPath(""), "node updates may only change labels, taints, or capacity (or configSource, if the DynamicKubeletConfig feature gate is enabled)")) } diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go index 0ec84db5862..94bff049cda 100644 --- a/pkg/apis/core/validation/validation_test.go +++ b/pkg/apis/core/validation/validation_test.go @@ -66,9 +66,11 @@ func testVolume(name string, namespace string, spec core.PersistentVolumeSpec) * func TestValidatePersistentVolumes(t *testing.T) { validMode := core.PersistentVolumeFilesystem + invalidMode := core.PersistentVolumeMode("fakeVolumeMode") scenarios := map[string]struct { isExpectedFailure bool volume *core.PersistentVolume + disableBlock bool }{ "good-volume": { isExpectedFailure: false, @@ -147,6 +149,22 @@ func TestValidatePersistentVolumes(t *testing.T) { PersistentVolumeReclaimPolicy: core.PersistentVolumeReclaimRetain, }), }, + "good-volume-with-volume-mode": { + isExpectedFailure: false, + volume: testVolume("foo", "", core.PersistentVolumeSpec{ + Capacity: core.ResourceList{ + core.ResourceName(core.ResourceStorage): resource.MustParse("10G"), + }, + AccessModes: []core.PersistentVolumeAccessMode{core.ReadWriteOnce}, + PersistentVolumeSource: core.PersistentVolumeSource{ + HostPath: &core.HostPathVolumeSource{ + Path: "/foo", + Type: newHostPathType(string(core.HostPathDirectory)), + }, + }, + VolumeMode: &validMode, + }), + }, "invalid-accessmode": { isExpectedFailure: true, volume: testVolume("foo", "", core.PersistentVolumeSpec{ @@ -178,6 +196,22 @@ func TestValidatePersistentVolumes(t *testing.T) { PersistentVolumeReclaimPolicy: "fakeReclaimPolicy", }), }, + "invalid-volume-mode": { + isExpectedFailure: true, + volume: testVolume("foo", "", core.PersistentVolumeSpec{ + Capacity: core.ResourceList{ + core.ResourceName(core.ResourceStorage): resource.MustParse("10G"), + }, + AccessModes: []core.PersistentVolumeAccessMode{core.ReadWriteOnce}, + PersistentVolumeSource: core.PersistentVolumeSource{ + HostPath: &core.HostPathVolumeSource{ + Path: "/foo", + Type: newHostPathType(string(core.HostPathDirectory)), + }, + }, + VolumeMode: &invalidMode, + }), + }, "unexpected-namespace": { isExpectedFailure: true, volume: testVolume("foo", "unexpected-namespace", core.PersistentVolumeSpec{ @@ -336,9 +370,8 @@ func TestValidatePersistentVolumes(t *testing.T) { StorageClassName: "-invalid-", }), }, - // VolumeMode alpha feature disabled - // TODO: remove when no longer alpha - "alpha disabled valid volume mode": { + "feature disabled valid volume mode": { + disableBlock: true, isExpectedFailure: true, volume: testVolume("foo", "", core.PersistentVolumeSpec{ Capacity: core.ResourceList{ @@ -400,13 +433,16 @@ func TestValidatePersistentVolumes(t *testing.T) { } for name, scenario := range scenarios { - errs := ValidatePersistentVolume(scenario.volume) - if len(errs) == 0 && scenario.isExpectedFailure { - t.Errorf("Unexpected success for scenario: %s", name) - } - if len(errs) > 0 && !scenario.isExpectedFailure { - t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) - } + t.Run(name, func(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, !scenario.disableBlock)() + errs := ValidatePersistentVolume(scenario.volume) + if len(errs) == 0 && scenario.isExpectedFailure { + t.Errorf("Unexpected success for scenario: %s", name) + } + if len(errs) > 0 && !scenario.isExpectedFailure { + t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) + } + }) } } @@ -550,29 +586,31 @@ func TestValidateLocalVolumesDisabled(t *testing.T) { }, } - utilfeature.DefaultFeatureGate.Set("PersistentLocalVolumes=false") for name, scenario := range scenarios { - errs := ValidatePersistentVolume(scenario.volume) - if len(errs) == 0 && scenario.isExpectedFailure { - t.Errorf("Unexpected success for scenario: %s", name) - } - if len(errs) > 0 && !scenario.isExpectedFailure { - t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) - } + t.Run(name+" PersistentLocalVolumes disabled", func(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PersistentLocalVolumes, false)() + errs := ValidatePersistentVolume(scenario.volume) + if len(errs) == 0 && scenario.isExpectedFailure { + t.Errorf("Unexpected success for scenario: %s", name) + } + if len(errs) > 0 && !scenario.isExpectedFailure { + t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) + } + }) } - utilfeature.DefaultFeatureGate.Set("PersistentLocalVolumes=true") - utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") for name, scenario := range scenarios { - errs := ValidatePersistentVolume(scenario.volume) - if len(errs) == 0 && scenario.isExpectedFailure { - t.Errorf("Unexpected success for scenario: %s", name) - } - if len(errs) > 0 && !scenario.isExpectedFailure { - t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) - } + t.Run(name+" VolumeScheduling disabled", func(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, false)() + errs := ValidatePersistentVolume(scenario.volume) + if len(errs) == 0 && scenario.isExpectedFailure { + t.Errorf("Unexpected success for scenario: %s", name) + } + if len(errs) > 0 && !scenario.isExpectedFailure { + t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) + } + }) } - utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") } func testVolumeWithNodeAffinity(affinity *core.VolumeNodeAffinity) *core.PersistentVolume { @@ -789,10 +827,12 @@ func testVolumeClaimStorageClassInAnnotationAndSpec(name, namespace, scNameInAnn func TestValidatePersistentVolumeClaim(t *testing.T) { invalidClassName := "-invalid-" validClassName := "valid" + invalidMode := core.PersistentVolumeMode("fakeVolumeMode") validMode := core.PersistentVolumeFilesystem scenarios := map[string]struct { isExpectedFailure bool claim *core.PersistentVolumeClaim + disableBlock bool }{ "good-claim": { isExpectedFailure: false, @@ -815,6 +855,7 @@ func TestValidatePersistentVolumeClaim(t *testing.T) { }, }, StorageClassName: &validClassName, + VolumeMode: &validMode, }), }, "invalid-claim-zero-capacity": { @@ -986,9 +1027,8 @@ func TestValidatePersistentVolumeClaim(t *testing.T) { StorageClassName: &invalidClassName, }), }, - // VolumeMode alpha feature disabled - // TODO: remove when no longer alpha - "disabled alpha valid volume mode": { + "feature disabled valid volume mode": { + disableBlock: true, isExpectedFailure: true, claim: testVolumeClaim("foo", "ns", core.PersistentVolumeClaimSpec{ Selector: &metav1.LabelSelector{ @@ -1012,16 +1052,34 @@ func TestValidatePersistentVolumeClaim(t *testing.T) { VolumeMode: &validMode, }), }, + "invalid-volume-mode": { + isExpectedFailure: true, + claim: testVolumeClaim("foo", "ns", core.PersistentVolumeClaimSpec{ + AccessModes: []core.PersistentVolumeAccessMode{ + core.ReadWriteOnce, + core.ReadOnlyMany, + }, + Resources: core.ResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceName(core.ResourceStorage): resource.MustParse("10G"), + }, + }, + VolumeMode: &invalidMode, + }), + }, } for name, scenario := range scenarios { - errs := ValidatePersistentVolumeClaim(scenario.claim) - if len(errs) == 0 && scenario.isExpectedFailure { - t.Errorf("Unexpected success for scenario: %s", name) - } - if len(errs) > 0 && !scenario.isExpectedFailure { - t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) - } + t.Run(name, func(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, !scenario.disableBlock)() + errs := ValidatePersistentVolumeClaim(scenario.claim) + if len(errs) == 0 && scenario.isExpectedFailure { + t.Errorf("Unexpected success for scenario: %s", name) + } + if len(errs) > 0 && !scenario.isExpectedFailure { + t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) + } + }) } } @@ -1104,15 +1162,17 @@ func TestAlphaPVVolumeModeUpdate(t *testing.T) { } for name, scenario := range scenarios { - // ensure we have a resource version specified for updates - toggleBlockVolumeFeature(scenario.enableBlock, t) - errs := ValidatePersistentVolumeUpdate(scenario.newPV, scenario.oldPV) - if len(errs) == 0 && scenario.isExpectedFailure { - t.Errorf("Unexpected success for scenario: %s", name) - } - if len(errs) > 0 && !scenario.isExpectedFailure { - t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) - } + t.Run(name, func(t *testing.T) { + // ensure we have a resource version specified for updates + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, scenario.enableBlock)() + errs := ValidatePersistentVolumeUpdate(scenario.newPV, scenario.oldPV) + if len(errs) == 0 && scenario.isExpectedFailure { + t.Errorf("Unexpected success for scenario: %s", name) + } + if len(errs) > 0 && !scenario.isExpectedFailure { + t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) + } + }) } } @@ -1553,52 +1613,20 @@ func TestValidatePersistentVolumeClaimUpdate(t *testing.T) { } for name, scenario := range scenarios { - // ensure we have a resource version specified for updates - togglePVExpandFeature(scenario.enableResize, t) - toggleBlockVolumeFeature(scenario.enableBlock, t) - scenario.oldClaim.ResourceVersion = "1" - scenario.newClaim.ResourceVersion = "1" - errs := ValidatePersistentVolumeClaimUpdate(scenario.newClaim, scenario.oldClaim) - if len(errs) == 0 && scenario.isExpectedFailure { - t.Errorf("Unexpected success for scenario: %s", name) - } - if len(errs) > 0 && !scenario.isExpectedFailure { - t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) - } - } -} - -func toggleBlockVolumeFeature(toggleFlag bool, t *testing.T) { - if toggleFlag { - // Enable alpha feature BlockVolume - err := utilfeature.DefaultFeatureGate.Set("BlockVolume=true") - if err != nil { - t.Errorf("Failed to enable feature gate for BlockVolume: %v", err) - return - } - } else { - err := utilfeature.DefaultFeatureGate.Set("BlockVolume=false") - if err != nil { - t.Errorf("Failed to disable feature gate for BlockVolume: %v", err) - return - } - } -} - -func togglePVExpandFeature(toggleFlag bool, t *testing.T) { - if toggleFlag { - // Enable alpha feature LocalStorageCapacityIsolation - err := utilfeature.DefaultFeatureGate.Set("ExpandPersistentVolumes=true") - if err != nil { - t.Errorf("Failed to enable feature gate for ExpandPersistentVolumes: %v", err) - return - } - } else { - err := utilfeature.DefaultFeatureGate.Set("ExpandPersistentVolumes=false") - if err != nil { - t.Errorf("Failed to disable feature gate for ExpandPersistentVolumes: %v", err) - return - } + t.Run(name, func(t *testing.T) { + // ensure we have a resource version specified for updates + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExpandPersistentVolumes, scenario.enableResize)() + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, scenario.enableBlock)() + scenario.oldClaim.ResourceVersion = "1" + scenario.newClaim.ResourceVersion = "1" + errs := ValidatePersistentVolumeClaimUpdate(scenario.newClaim, scenario.oldClaim) + if len(errs) == 0 && scenario.isExpectedFailure { + t.Errorf("Unexpected success for scenario: %s", name) + } + if len(errs) > 0 && !scenario.isExpectedFailure { + t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) + } + }) } } @@ -1774,6 +1802,60 @@ func TestValidateGlusterfs(t *testing.T) { } } +func TestValidateGlusterfsPersistentVolumeSource(t *testing.T) { + var epNs *string + namespace := "" + epNs = &namespace + + testCases := []struct { + name string + gfs *core.GlusterfsPersistentVolumeSource + errtype field.ErrorType + errfield string + }{ + { + name: "missing endpointname", + gfs: &core.GlusterfsPersistentVolumeSource{EndpointsName: "", Path: "/tmp"}, + errtype: field.ErrorTypeRequired, + errfield: "endpoints", + }, + { + name: "missing path", + gfs: &core.GlusterfsPersistentVolumeSource{EndpointsName: "my-endpoint", Path: ""}, + errtype: field.ErrorTypeRequired, + errfield: "path", + }, + { + name: "non null endpointnamespace with empty string", + gfs: &core.GlusterfsPersistentVolumeSource{EndpointsName: "my-endpoint", Path: "/tmp", EndpointsNamespace: epNs}, + errtype: field.ErrorTypeInvalid, + errfield: "endpointsNamespace", + }, + { + name: "missing endpointname and path", + gfs: &core.GlusterfsPersistentVolumeSource{EndpointsName: "", Path: ""}, + errtype: field.ErrorTypeRequired, + errfield: "endpoints", + }, + } + + for i, tc := range testCases { + errs := validateGlusterfsPersistentVolumeSource(tc.gfs, field.NewPath("field")) + + if len(errs) > 0 && tc.errtype == "" { + t.Errorf("[%d: %q] unexpected error(s): %v", i, tc.name, errs) + } else if len(errs) == 0 && tc.errtype != "" { + t.Errorf("[%d: %q] expected error type %v", i, tc.name, tc.errtype) + } else if len(errs) >= 1 { + if errs[0].Type != tc.errtype { + t.Errorf("[%d: %q] expected error type %v, got %v", i, tc.name, tc.errtype, errs[0].Type) + } else if !strings.HasSuffix(errs[0].Field, "."+tc.errfield) { + t.Errorf("[%d: %q] expected error on field %q, got %q", i, tc.name, tc.errfield, errs[0].Field) + } + } + } +} + func TestValidateCSIVolumeSource(t *testing.T) { testCases := []struct { name string @@ -1871,11 +1953,7 @@ func TestValidateCSIVolumeSource(t *testing.T) { }, } - err := utilfeature.DefaultFeatureGate.Set("CSIPersistentVolume=true") - if err != nil { - t.Errorf("Failed to enable feature gate for CSIPersistentVolumes: %v", err) - return - } + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIPersistentVolume, true)() for i, tc := range testCases { errs := validateCSIPersistentVolumeSource(tc.csi, field.NewPath("field")) @@ -1892,11 +1970,6 @@ func TestValidateCSIVolumeSource(t *testing.T) { } } } - err = utilfeature.DefaultFeatureGate.Set("CSIPersistentVolume=false") - if err != nil { - t.Errorf("Failed to disable feature gate for CSIPersistentVolumes: %v", err) - return - } } // This test is a little too top-to-bottom. Ideally we would test each volume @@ -3710,20 +3783,14 @@ func TestValidateVolumes(t *testing.T) { // Validate HugePages medium type for EmptyDir when HugePages feature is enabled/disabled hugePagesCase := core.VolumeSource{EmptyDir: &core.EmptyDirVolumeSource{Medium: core.StorageMediumHugePages}} - // Enable alpha feature HugePages - err := utilfeature.DefaultFeatureGate.Set("HugePages=true") - if err != nil { - t.Errorf("Failed to enable feature gate for HugePages: %v", err) - } + // Enable HugePages + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, true)() if errs := validateVolumeSource(&hugePagesCase, field.NewPath("field").Index(0), "working"); len(errs) != 0 { t.Errorf("Unexpected error when HugePages feature is enabled.") } - // Disable alpha feature HugePages - err = utilfeature.DefaultFeatureGate.Set("HugePages=false") - if err != nil { - t.Errorf("Failed to disable feature gate for HugePages: %v", err) - } + // Disable feature HugePages + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, false)() if errs := validateVolumeSource(&hugePagesCase, field.NewPath("field").Index(0), "failing"); len(errs) == 0 { t.Errorf("Expected error when HugePages feature is disabled got nothing.") } @@ -3828,12 +3895,8 @@ func TestAlphaHugePagesIsolation(t *testing.T) { }, }, } - // Enable alpha feature HugePages - err := utilfeature.DefaultFeatureGate.Set("HugePages=true") - if err != nil { - t.Errorf("Failed to enable feature gate for HugePages: %v", err) - return - } + // Enable feature HugePages + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, true)() for i := range successCases { pod := &successCases[i] if errs := ValidatePod(pod); len(errs) != 0 { @@ -3846,13 +3909,9 @@ func TestAlphaHugePagesIsolation(t *testing.T) { t.Errorf("Expected error for case[%d], pod: %v", i, pod.Name) } } - // Disable alpha feature HugePages - err = utilfeature.DefaultFeatureGate.Set("HugePages=false") - if err != nil { - t.Errorf("Failed to disable feature gate for HugePages: %v", err) - return - } - // Disable alpha feature HugePages and ensure all success cases fail + // Disable feature HugePages + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, false)() + // Disable feature HugePages and ensure all success cases fail for i := range successCases { pod := &successCases[i] if errs := ValidatePod(pod); len(errs) == 0 { @@ -3861,13 +3920,9 @@ func TestAlphaHugePagesIsolation(t *testing.T) { } } -func TestAlphaPVCVolumeMode(t *testing.T) { - // Enable alpha feature BlockVolume for PVC - err := utilfeature.DefaultFeatureGate.Set("BlockVolume=true") - if err != nil { - t.Errorf("Failed to enable feature gate for BlockVolume: %v", err) - return - } +func TestPVCVolumeMode(t *testing.T) { + // Enable feature BlockVolume for PVC + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() block := core.PersistentVolumeBlock file := core.PersistentVolumeFilesystem @@ -3898,13 +3953,9 @@ func TestAlphaPVCVolumeMode(t *testing.T) { } } -func TestAlphaPVVolumeMode(t *testing.T) { - // Enable alpha feature BlockVolume for PV - err := utilfeature.DefaultFeatureGate.Set("BlockVolume=true") - if err != nil { - t.Errorf("Failed to enable feature gate for BlockVolume: %v", err) - return - } +func TestPVVolumeMode(t *testing.T) { + // Enable feature BlockVolume for PVC + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() block := core.PersistentVolumeBlock file := core.PersistentVolumeFilesystem @@ -4013,23 +4064,15 @@ func TestAlphaLocalStorageCapacityIsolation(t *testing.T) { testCases := []core.VolumeSource{ {EmptyDir: &core.EmptyDirVolumeSource{SizeLimit: resource.NewQuantity(int64(5), resource.BinarySI)}}, } - // Enable alpha feature LocalStorageCapacityIsolation - err := utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=true") - if err != nil { - t.Errorf("Failed to enable feature gate for LocalStorageCapacityIsolation: %v", err) - return - } + // Enable feature LocalStorageCapacityIsolation + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, true)() for _, tc := range testCases { if errs := validateVolumeSource(&tc, field.NewPath("spec"), "tmpvol"); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } - // Disable alpha feature LocalStorageCapacityIsolation - err = utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=false") - if err != nil { - t.Errorf("Failed to disable feature gate for LocalStorageCapacityIsolation: %v", err) - return - } + // Disable feature LocalStorageCapacityIsolation + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, false)() for _, tc := range testCases { if errs := validateVolumeSource(&tc, field.NewPath("spec"), "tmpvol"); len(errs) == 0 { t.Errorf("expected failure: %v", errs) @@ -4043,21 +4086,13 @@ func TestAlphaLocalStorageCapacityIsolation(t *testing.T) { resource.BinarySI), }, } - // Enable alpha feature LocalStorageCapacityIsolation - err = utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=true") - if err != nil { - t.Errorf("Failed to enable feature gate for LocalStorageCapacityIsolation: %v", err) - return - } + // Enable feature LocalStorageCapacityIsolation + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, true)() if errs := ValidateResourceRequirements(&containerLimitCase, field.NewPath("resources")); len(errs) != 0 { t.Errorf("expected success: %v", errs) } - // Disable alpha feature LocalStorageCapacityIsolation - err = utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=false") - if err != nil { - t.Errorf("Failed to disable feature gate for LocalStorageCapacityIsolation: %v", err) - return - } + // Disable feature LocalStorageCapacityIsolation + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, false)() if errs := ValidateResourceRequirements(&containerLimitCase, field.NewPath("resources")); len(errs) == 0 { t.Errorf("expected failure: %v", errs) } @@ -4092,22 +4127,14 @@ func TestValidateResourceQuotaWithAlphaLocalStorageCapacityIsolation(t *testing. Spec: spec, } - // Enable alpha feature LocalStorageCapacityIsolation - err := utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=true") - if err != nil { - t.Errorf("Failed to enable feature gate for LocalStorageCapacityIsolation: %v", err) - return - } + // Enable feature LocalStorageCapacityIsolation + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, true)() if errs := ValidateResourceQuota(resourceQuota); len(errs) != 0 { t.Errorf("expected success: %v", errs) } - // Disable alpha feature LocalStorageCapacityIsolation - err = utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=false") - if err != nil { - t.Errorf("Failed to disable feature gate for LocalStorageCapacityIsolation: %v", err) - return - } + // Disable feature LocalStorageCapacityIsolation + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, false)() errs := ValidateResourceQuota(resourceQuota) if len(errs) == 0 { t.Errorf("expected failure for %s", resourceQuota.Name) @@ -4240,24 +4267,16 @@ func TestLocalStorageEnvWithFeatureGate(t *testing.T) { }, }, } - // Enable alpha feature LocalStorageCapacityIsolation - err := utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=true") - if err != nil { - t.Errorf("Failed to enable feature gate for LocalStorageCapacityIsolation: %v", err) - return - } + // Enable feature LocalStorageCapacityIsolation + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, true)() for _, testCase := range testCases { if errs := validateEnvVarValueFrom(testCase, field.NewPath("field")); len(errs) != 0 { t.Errorf("expected success, got: %v", errs) } } - // Disable alpha feature LocalStorageCapacityIsolation - err = utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=false") - if err != nil { - t.Errorf("Failed to disable feature gate for LocalStorageCapacityIsolation: %v", err) - return - } + // Disable feature LocalStorageCapacityIsolation + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, false)() for _, testCase := range testCases { if errs := validateEnvVarValueFrom(testCase, field.NewPath("field")); len(errs) == 0 { t.Errorf("expected failure for %v", testCase.Name) @@ -5101,12 +5120,8 @@ func TestAlphaValidateVolumeDevices(t *testing.T) { {Name: "abc-123", MountPath: "/this/path/exists"}, } - // enable Alpha BlockVolume - err1 := utilfeature.DefaultFeatureGate.Set("BlockVolume=true") - if err1 != nil { - t.Errorf("Failed to enable feature gate for BlockVolume: %v", err1) - return - } + // enable BlockVolume + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() // Success Cases: // Validate normal success cases - only PVC volumeSource if errs := ValidateVolumeDevices(successCase, GetVolumeMountMap(goodVolumeMounts), vols, field.NewPath("field")); len(errs) != 0 { @@ -5121,12 +5136,8 @@ func TestAlphaValidateVolumeDevices(t *testing.T) { } } - // disable Alpha BlockVolume - err2 := utilfeature.DefaultFeatureGate.Set("BlockVolume=false") - if err2 != nil { - t.Errorf("Failed to disable feature gate for BlockVolume: %v", err2) - return - } + // disable BlockVolume + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, false)() if errs := ValidateVolumeDevices(disabledAlphaVolDevice, GetVolumeMountMap(goodVolumeMounts), vols, field.NewPath("field")); len(errs) == 0 { t.Errorf("expected failure: %v", errs) } @@ -11147,13 +11158,8 @@ func TestValidateLimitRangeForLocalStorage(t *testing.T) { }, } - // Enable alpha feature LocalStorageCapacityIsolation - err := utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=true") - if err != nil { - t.Errorf("Failed to enable feature gate for LocalStorageCapacityIsolation: %v", err) - return - } - + // Enable feature LocalStorageCapacityIsolation + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, true)() for _, testCase := range testCases { limitRange := &core.LimitRange{ObjectMeta: metav1.ObjectMeta{Name: testCase.name, Namespace: "foo"}, Spec: testCase.spec} if errs := ValidateLimitRange(limitRange); len(errs) != 0 { @@ -11161,12 +11167,8 @@ func TestValidateLimitRangeForLocalStorage(t *testing.T) { } } - // Disable alpha feature LocalStorageCapacityIsolation - err = utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=false") - if err != nil { - t.Errorf("Failed to disable feature gate for LocalStorageCapacityIsolation: %v", err) - return - } + // Disable feature LocalStorageCapacityIsolation + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, false)() for _, testCase := range testCases { limitRange := &core.LimitRange{ObjectMeta: metav1.ObjectMeta{Name: testCase.name, Namespace: "foo"}, Spec: testCase.spec} if errs := ValidateLimitRange(limitRange); len(errs) == 0 { @@ -11523,17 +11525,20 @@ func TestValidatePersistentVolumeClaimStatusUpdate(t *testing.T) { }, } for name, scenario := range scenarios { - // ensure we have a resource version specified for updates - togglePVExpandFeature(scenario.enableResize, t) - scenario.oldClaim.ResourceVersion = "1" - scenario.newClaim.ResourceVersion = "1" - errs := ValidatePersistentVolumeClaimStatusUpdate(scenario.newClaim, scenario.oldClaim) - if len(errs) == 0 && scenario.isExpectedFailure { - t.Errorf("Unexpected success for scenario: %s", name) - } - if len(errs) > 0 && !scenario.isExpectedFailure { - t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) - } + t.Run(name, func(t *testing.T) { + // ensure we have a resource version specified for updates + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExpandPersistentVolumes, scenario.enableResize)() + + scenario.oldClaim.ResourceVersion = "1" + scenario.newClaim.ResourceVersion = "1" + errs := ValidatePersistentVolumeClaimStatusUpdate(scenario.newClaim, scenario.oldClaim) + if len(errs) == 0 && scenario.isExpectedFailure { + t.Errorf("Unexpected success for scenario: %s", name) + } + if len(errs) > 0 && !scenario.isExpectedFailure { + t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) + } + }) } } @@ -11703,13 +11708,13 @@ func TestValidateResourceQuota(t *testing.T) { Spec: nonBestEffortSpec, }, } - utilfeature.DefaultFeatureGate.Set("ResourceQuotaScopeSelectors=true") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ResourceQuotaScopeSelectors, true)() for _, successCase := range successCases { if errs := ValidateResourceQuota(&successCase); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } - utilfeature.DefaultFeatureGate.Set("ResourceQuotaScopeSelectors=false") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ResourceQuotaScopeSelectors, false)() errorCases := map[string]struct { R core.ResourceQuota diff --git a/pkg/apis/core/zz_generated.deepcopy.go b/pkg/apis/core/zz_generated.deepcopy.go index 808dad27495..a4801c2e313 100644 --- a/pkg/apis/core/zz_generated.deepcopy.go +++ b/pkg/apis/core/zz_generated.deepcopy.go @@ -1498,6 +1498,27 @@ func (in *GitRepoVolumeSource) DeepCopy() *GitRepoVolumeSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlusterfsPersistentVolumeSource) DeepCopyInto(out *GlusterfsPersistentVolumeSource) { + *out = *in + if in.EndpointsNamespace != nil { + in, out := &in.EndpointsNamespace, &out.EndpointsNamespace + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlusterfsPersistentVolumeSource. +func (in *GlusterfsPersistentVolumeSource) DeepCopy() *GlusterfsPersistentVolumeSource { + if in == nil { + return nil + } + out := new(GlusterfsPersistentVolumeSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GlusterfsVolumeSource) DeepCopyInto(out *GlusterfsVolumeSource) { *out = *in @@ -2808,8 +2829,8 @@ func (in *PersistentVolumeSource) DeepCopyInto(out *PersistentVolumeSource) { } if in.Glusterfs != nil { in, out := &in.Glusterfs, &out.Glusterfs - *out = new(GlusterfsVolumeSource) - **out = **in + *out = new(GlusterfsPersistentVolumeSource) + (*in).DeepCopyInto(*out) } if in.NFS != nil { in, out := &in.NFS, &out.NFS diff --git a/pkg/apis/imagepolicy/OWNERS b/pkg/apis/imagepolicy/OWNERS index 96ae42e027a..02b8511e465 100755 --- a/pkg/apis/imagepolicy/OWNERS +++ b/pkg/apis/imagepolicy/OWNERS @@ -1,4 +1,7 @@ +# approval on api packages bubbles to api-approvers reviewers: -- deads2k -- mbohlool -- jianhuiz +- sig-auth-policy-approvers +- sig-auth-policy-reviewers +labels: +- sig/auth + diff --git a/pkg/apis/policy/OWNERS b/pkg/apis/policy/OWNERS index 872e16329b0..5780104890d 100755 --- a/pkg/apis/policy/OWNERS +++ b/pkg/apis/policy/OWNERS @@ -1,7 +1,8 @@ -approvers: -- sig-apps-api-approvers +# approval on api packages bubbles to api-approvers reviewers: -- sig-apps-reviewers -- pweil- -- liggitt -- tallclair +- sig-apps-api-approvers +- sig-auth-policy-approvers +- sig-auth-policy-reviewers +labels: +- sig/auth + diff --git a/pkg/apis/rbac/OWNERS b/pkg/apis/rbac/OWNERS index 45b66b0a8ca..ff4a7f4bf9a 100755 --- a/pkg/apis/rbac/OWNERS +++ b/pkg/apis/rbac/OWNERS @@ -1,16 +1,7 @@ +# approval on api packages bubbles to api-approvers reviewers: -- thockin -- lavalamp -- smarterclayton -- deads2k -- sttts -- ncdc -- dims -- krousey -- mml -- mbohlool -- david-mcmahon -- lixiaobing10051267 -- jianhuiz -- liggitt -- enj +- sig-auth-authorizers-approvers +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/pkg/apis/storage/util/BUILD b/pkg/apis/storage/util/BUILD index 1b3f629507a..0cddfad64b4 100644 --- a/pkg/apis/storage/util/BUILD +++ b/pkg/apis/storage/util/BUILD @@ -36,11 +36,16 @@ filegroup( go_test( name = "go_default_test", - srcs = ["util_test.go"], + srcs = [ + "main_test.go", + "util_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", "//pkg/apis/storage:go_default_library", + "//pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/pkg/apis/storage/util/main_test.go b/pkg/apis/storage/util/main_test.go new file mode 100644 index 00000000000..6af02d0a11d --- /dev/null +++ b/pkg/apis/storage/util/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/apis/storage/util/util_test.go b/pkg/apis/storage/util/util_test.go index 8b62ddfab3b..d2cf1402105 100644 --- a/pkg/apis/storage/util/util_test.go +++ b/pkg/apis/storage/util/util_test.go @@ -21,8 +21,10 @@ import ( "testing" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/storage" + "k8s.io/kubernetes/pkg/features" ) func TestDropAlphaFields(t *testing.T) { @@ -39,9 +41,7 @@ func TestDropAlphaFields(t *testing.T) { } // Test that field gets dropped when feature gate is not set - if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false"); err != nil { - t.Fatalf("Failed to set feature gate for VolumeScheduling: %v", err) - } + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, false)() class := &storage.StorageClass{ VolumeBindingMode: &bindingMode, AllowedTopologies: allowedTopologies, @@ -59,9 +59,7 @@ func TestDropAlphaFields(t *testing.T) { VolumeBindingMode: &bindingMode, AllowedTopologies: allowedTopologies, } - if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true"); err != nil { - t.Fatalf("Failed to set feature gate for VolumeScheduling: %v", err) - } + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() DropDisabledAlphaFields(class) if class.VolumeBindingMode != &bindingMode { t.Errorf("VolumeBindingMode field got unexpectantly modified: %+v", class.VolumeBindingMode) @@ -69,8 +67,4 @@ func TestDropAlphaFields(t *testing.T) { if !reflect.DeepEqual(class.AllowedTopologies, allowedTopologies) { t.Errorf("AllowedTopologies field got unexpectantly modified: %+v", class.AllowedTopologies) } - - if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false"); err != nil { - t.Fatalf("Failed to disable feature gate for VolumeScheduling: %v", err) - } } diff --git a/pkg/apis/storage/v1/BUILD b/pkg/apis/storage/v1/BUILD index f84557ef2cd..11449ae5f12 100644 --- a/pkg/apis/storage/v1/BUILD +++ b/pkg/apis/storage/v1/BUILD @@ -47,13 +47,18 @@ filegroup( go_test( name = "go_default_test", - srcs = ["defaults_test.go"], + srcs = [ + "defaults_test.go", + "main_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/storage/install:go_default_library", + "//pkg/features:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/pkg/apis/storage/v1/defaults_test.go b/pkg/apis/storage/v1/defaults_test.go index 4143ad75e2a..82ea60f0bd1 100644 --- a/pkg/apis/storage/v1/defaults_test.go +++ b/pkg/apis/storage/v1/defaults_test.go @@ -23,8 +23,10 @@ import ( storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/runtime" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" _ "k8s.io/kubernetes/pkg/apis/storage/install" + "k8s.io/kubernetes/pkg/features" ) func roundTrip(t *testing.T, obj runtime.Object) runtime.Object { @@ -51,25 +53,9 @@ func roundTrip(t *testing.T, obj runtime.Object) runtime.Object { func TestSetDefaultVolumeBindingMode(t *testing.T) { class := &storagev1.StorageClass{} - // When feature gate is disabled, field should not be defaulted - err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - if err != nil { - t.Fatalf("Failed to enable feature gate for VolumeScheduling: %v", err) - } - output := roundTrip(t, runtime.Object(class)).(*storagev1.StorageClass) - if output.VolumeBindingMode != nil { - t.Errorf("Expected VolumeBindingMode to not be defaulted, got: %+v", output.VolumeBindingMode) - } - - class = &storagev1.StorageClass{} - // When feature gate is enabled, field should be defaulted - err = utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - if err != nil { - t.Fatalf("Failed to enable feature gate for VolumeScheduling: %v", err) - } defaultMode := storagev1.VolumeBindingImmediate - output = roundTrip(t, runtime.Object(class)).(*storagev1.StorageClass) + output := roundTrip(t, runtime.Object(class)).(*storagev1.StorageClass) outMode := output.VolumeBindingMode if outMode == nil { t.Errorf("Expected VolumeBindingMode to be defaulted to: %+v, got: nil", defaultMode) @@ -77,8 +63,12 @@ func TestSetDefaultVolumeBindingMode(t *testing.T) { t.Errorf("Expected VolumeBindingMode to be defaulted to: %+v, got: %+v", defaultMode, outMode) } - err = utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - if err != nil { - t.Fatalf("Failed to disable feature gate for VolumeScheduling: %v", err) + class = &storagev1.StorageClass{} + + // When feature gate is disabled, field should not be defaulted + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, false)() + output = roundTrip(t, runtime.Object(class)).(*storagev1.StorageClass) + if output.VolumeBindingMode != nil { + t.Errorf("Expected VolumeBindingMode to not be defaulted, got: %+v", output.VolumeBindingMode) } } diff --git a/pkg/apis/storage/v1/main_test.go b/pkg/apis/storage/v1/main_test.go new file mode 100644 index 00000000000..e46b01929f7 --- /dev/null +++ b/pkg/apis/storage/v1/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/apis/storage/v1/zz_generated.conversion.go b/pkg/apis/storage/v1/zz_generated.conversion.go index be7a27582e6..436a8626719 100644 --- a/pkg/apis/storage/v1/zz_generated.conversion.go +++ b/pkg/apis/storage/v1/zz_generated.conversion.go @@ -58,6 +58,66 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*v1.VolumeAttachment)(nil), (*storage.VolumeAttachment)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_VolumeAttachment_To_storage_VolumeAttachment(a.(*v1.VolumeAttachment), b.(*storage.VolumeAttachment), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*storage.VolumeAttachment)(nil), (*v1.VolumeAttachment)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_storage_VolumeAttachment_To_v1_VolumeAttachment(a.(*storage.VolumeAttachment), b.(*v1.VolumeAttachment), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1.VolumeAttachmentList)(nil), (*storage.VolumeAttachmentList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_VolumeAttachmentList_To_storage_VolumeAttachmentList(a.(*v1.VolumeAttachmentList), b.(*storage.VolumeAttachmentList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*storage.VolumeAttachmentList)(nil), (*v1.VolumeAttachmentList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_storage_VolumeAttachmentList_To_v1_VolumeAttachmentList(a.(*storage.VolumeAttachmentList), b.(*v1.VolumeAttachmentList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1.VolumeAttachmentSource)(nil), (*storage.VolumeAttachmentSource)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_VolumeAttachmentSource_To_storage_VolumeAttachmentSource(a.(*v1.VolumeAttachmentSource), b.(*storage.VolumeAttachmentSource), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*storage.VolumeAttachmentSource)(nil), (*v1.VolumeAttachmentSource)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_storage_VolumeAttachmentSource_To_v1_VolumeAttachmentSource(a.(*storage.VolumeAttachmentSource), b.(*v1.VolumeAttachmentSource), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1.VolumeAttachmentSpec)(nil), (*storage.VolumeAttachmentSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_VolumeAttachmentSpec_To_storage_VolumeAttachmentSpec(a.(*v1.VolumeAttachmentSpec), b.(*storage.VolumeAttachmentSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*storage.VolumeAttachmentSpec)(nil), (*v1.VolumeAttachmentSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_storage_VolumeAttachmentSpec_To_v1_VolumeAttachmentSpec(a.(*storage.VolumeAttachmentSpec), b.(*v1.VolumeAttachmentSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1.VolumeAttachmentStatus)(nil), (*storage.VolumeAttachmentStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_VolumeAttachmentStatus_To_storage_VolumeAttachmentStatus(a.(*v1.VolumeAttachmentStatus), b.(*storage.VolumeAttachmentStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*storage.VolumeAttachmentStatus)(nil), (*v1.VolumeAttachmentStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_storage_VolumeAttachmentStatus_To_v1_VolumeAttachmentStatus(a.(*storage.VolumeAttachmentStatus), b.(*v1.VolumeAttachmentStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1.VolumeError)(nil), (*storage.VolumeError)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_VolumeError_To_storage_VolumeError(a.(*v1.VolumeError), b.(*storage.VolumeError), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*storage.VolumeError)(nil), (*v1.VolumeError)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_storage_VolumeError_To_v1_VolumeError(a.(*storage.VolumeError), b.(*v1.VolumeError), scope) + }); err != nil { + return err + } return nil } @@ -116,3 +176,153 @@ func autoConvert_storage_StorageClassList_To_v1_StorageClassList(in *storage.Sto func Convert_storage_StorageClassList_To_v1_StorageClassList(in *storage.StorageClassList, out *v1.StorageClassList, s conversion.Scope) error { return autoConvert_storage_StorageClassList_To_v1_StorageClassList(in, out, s) } + +func autoConvert_v1_VolumeAttachment_To_storage_VolumeAttachment(in *v1.VolumeAttachment, out *storage.VolumeAttachment, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1_VolumeAttachmentSpec_To_storage_VolumeAttachmentSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1_VolumeAttachmentStatus_To_storage_VolumeAttachmentStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1_VolumeAttachment_To_storage_VolumeAttachment is an autogenerated conversion function. +func Convert_v1_VolumeAttachment_To_storage_VolumeAttachment(in *v1.VolumeAttachment, out *storage.VolumeAttachment, s conversion.Scope) error { + return autoConvert_v1_VolumeAttachment_To_storage_VolumeAttachment(in, out, s) +} + +func autoConvert_storage_VolumeAttachment_To_v1_VolumeAttachment(in *storage.VolumeAttachment, out *v1.VolumeAttachment, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_storage_VolumeAttachmentSpec_To_v1_VolumeAttachmentSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_storage_VolumeAttachmentStatus_To_v1_VolumeAttachmentStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_storage_VolumeAttachment_To_v1_VolumeAttachment is an autogenerated conversion function. +func Convert_storage_VolumeAttachment_To_v1_VolumeAttachment(in *storage.VolumeAttachment, out *v1.VolumeAttachment, s conversion.Scope) error { + return autoConvert_storage_VolumeAttachment_To_v1_VolumeAttachment(in, out, s) +} + +func autoConvert_v1_VolumeAttachmentList_To_storage_VolumeAttachmentList(in *v1.VolumeAttachmentList, out *storage.VolumeAttachmentList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]storage.VolumeAttachment)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1_VolumeAttachmentList_To_storage_VolumeAttachmentList is an autogenerated conversion function. +func Convert_v1_VolumeAttachmentList_To_storage_VolumeAttachmentList(in *v1.VolumeAttachmentList, out *storage.VolumeAttachmentList, s conversion.Scope) error { + return autoConvert_v1_VolumeAttachmentList_To_storage_VolumeAttachmentList(in, out, s) +} + +func autoConvert_storage_VolumeAttachmentList_To_v1_VolumeAttachmentList(in *storage.VolumeAttachmentList, out *v1.VolumeAttachmentList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]v1.VolumeAttachment)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_storage_VolumeAttachmentList_To_v1_VolumeAttachmentList is an autogenerated conversion function. +func Convert_storage_VolumeAttachmentList_To_v1_VolumeAttachmentList(in *storage.VolumeAttachmentList, out *v1.VolumeAttachmentList, s conversion.Scope) error { + return autoConvert_storage_VolumeAttachmentList_To_v1_VolumeAttachmentList(in, out, s) +} + +func autoConvert_v1_VolumeAttachmentSource_To_storage_VolumeAttachmentSource(in *v1.VolumeAttachmentSource, out *storage.VolumeAttachmentSource, s conversion.Scope) error { + out.PersistentVolumeName = (*string)(unsafe.Pointer(in.PersistentVolumeName)) + return nil +} + +// Convert_v1_VolumeAttachmentSource_To_storage_VolumeAttachmentSource is an autogenerated conversion function. +func Convert_v1_VolumeAttachmentSource_To_storage_VolumeAttachmentSource(in *v1.VolumeAttachmentSource, out *storage.VolumeAttachmentSource, s conversion.Scope) error { + return autoConvert_v1_VolumeAttachmentSource_To_storage_VolumeAttachmentSource(in, out, s) +} + +func autoConvert_storage_VolumeAttachmentSource_To_v1_VolumeAttachmentSource(in *storage.VolumeAttachmentSource, out *v1.VolumeAttachmentSource, s conversion.Scope) error { + out.PersistentVolumeName = (*string)(unsafe.Pointer(in.PersistentVolumeName)) + return nil +} + +// Convert_storage_VolumeAttachmentSource_To_v1_VolumeAttachmentSource is an autogenerated conversion function. +func Convert_storage_VolumeAttachmentSource_To_v1_VolumeAttachmentSource(in *storage.VolumeAttachmentSource, out *v1.VolumeAttachmentSource, s conversion.Scope) error { + return autoConvert_storage_VolumeAttachmentSource_To_v1_VolumeAttachmentSource(in, out, s) +} + +func autoConvert_v1_VolumeAttachmentSpec_To_storage_VolumeAttachmentSpec(in *v1.VolumeAttachmentSpec, out *storage.VolumeAttachmentSpec, s conversion.Scope) error { + out.Attacher = in.Attacher + if err := Convert_v1_VolumeAttachmentSource_To_storage_VolumeAttachmentSource(&in.Source, &out.Source, s); err != nil { + return err + } + out.NodeName = in.NodeName + return nil +} + +// Convert_v1_VolumeAttachmentSpec_To_storage_VolumeAttachmentSpec is an autogenerated conversion function. +func Convert_v1_VolumeAttachmentSpec_To_storage_VolumeAttachmentSpec(in *v1.VolumeAttachmentSpec, out *storage.VolumeAttachmentSpec, s conversion.Scope) error { + return autoConvert_v1_VolumeAttachmentSpec_To_storage_VolumeAttachmentSpec(in, out, s) +} + +func autoConvert_storage_VolumeAttachmentSpec_To_v1_VolumeAttachmentSpec(in *storage.VolumeAttachmentSpec, out *v1.VolumeAttachmentSpec, s conversion.Scope) error { + out.Attacher = in.Attacher + if err := Convert_storage_VolumeAttachmentSource_To_v1_VolumeAttachmentSource(&in.Source, &out.Source, s); err != nil { + return err + } + out.NodeName = in.NodeName + return nil +} + +// Convert_storage_VolumeAttachmentSpec_To_v1_VolumeAttachmentSpec is an autogenerated conversion function. +func Convert_storage_VolumeAttachmentSpec_To_v1_VolumeAttachmentSpec(in *storage.VolumeAttachmentSpec, out *v1.VolumeAttachmentSpec, s conversion.Scope) error { + return autoConvert_storage_VolumeAttachmentSpec_To_v1_VolumeAttachmentSpec(in, out, s) +} + +func autoConvert_v1_VolumeAttachmentStatus_To_storage_VolumeAttachmentStatus(in *v1.VolumeAttachmentStatus, out *storage.VolumeAttachmentStatus, s conversion.Scope) error { + out.Attached = in.Attached + out.AttachmentMetadata = *(*map[string]string)(unsafe.Pointer(&in.AttachmentMetadata)) + out.AttachError = (*storage.VolumeError)(unsafe.Pointer(in.AttachError)) + out.DetachError = (*storage.VolumeError)(unsafe.Pointer(in.DetachError)) + return nil +} + +// Convert_v1_VolumeAttachmentStatus_To_storage_VolumeAttachmentStatus is an autogenerated conversion function. +func Convert_v1_VolumeAttachmentStatus_To_storage_VolumeAttachmentStatus(in *v1.VolumeAttachmentStatus, out *storage.VolumeAttachmentStatus, s conversion.Scope) error { + return autoConvert_v1_VolumeAttachmentStatus_To_storage_VolumeAttachmentStatus(in, out, s) +} + +func autoConvert_storage_VolumeAttachmentStatus_To_v1_VolumeAttachmentStatus(in *storage.VolumeAttachmentStatus, out *v1.VolumeAttachmentStatus, s conversion.Scope) error { + out.Attached = in.Attached + out.AttachmentMetadata = *(*map[string]string)(unsafe.Pointer(&in.AttachmentMetadata)) + out.AttachError = (*v1.VolumeError)(unsafe.Pointer(in.AttachError)) + out.DetachError = (*v1.VolumeError)(unsafe.Pointer(in.DetachError)) + return nil +} + +// Convert_storage_VolumeAttachmentStatus_To_v1_VolumeAttachmentStatus is an autogenerated conversion function. +func Convert_storage_VolumeAttachmentStatus_To_v1_VolumeAttachmentStatus(in *storage.VolumeAttachmentStatus, out *v1.VolumeAttachmentStatus, s conversion.Scope) error { + return autoConvert_storage_VolumeAttachmentStatus_To_v1_VolumeAttachmentStatus(in, out, s) +} + +func autoConvert_v1_VolumeError_To_storage_VolumeError(in *v1.VolumeError, out *storage.VolumeError, s conversion.Scope) error { + out.Time = in.Time + out.Message = in.Message + return nil +} + +// Convert_v1_VolumeError_To_storage_VolumeError is an autogenerated conversion function. +func Convert_v1_VolumeError_To_storage_VolumeError(in *v1.VolumeError, out *storage.VolumeError, s conversion.Scope) error { + return autoConvert_v1_VolumeError_To_storage_VolumeError(in, out, s) +} + +func autoConvert_storage_VolumeError_To_v1_VolumeError(in *storage.VolumeError, out *v1.VolumeError, s conversion.Scope) error { + out.Time = in.Time + out.Message = in.Message + return nil +} + +// Convert_storage_VolumeError_To_v1_VolumeError is an autogenerated conversion function. +func Convert_storage_VolumeError_To_v1_VolumeError(in *storage.VolumeError, out *v1.VolumeError, s conversion.Scope) error { + return autoConvert_storage_VolumeError_To_v1_VolumeError(in, out, s) +} diff --git a/pkg/apis/storage/v1beta1/BUILD b/pkg/apis/storage/v1beta1/BUILD index 3006ca9307a..4695a2ddb21 100644 --- a/pkg/apis/storage/v1beta1/BUILD +++ b/pkg/apis/storage/v1beta1/BUILD @@ -47,13 +47,18 @@ filegroup( go_test( name = "go_default_test", - srcs = ["defaults_test.go"], + srcs = [ + "defaults_test.go", + "main_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/storage/install:go_default_library", + "//pkg/features:go_default_library", "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/pkg/apis/storage/v1beta1/defaults_test.go b/pkg/apis/storage/v1beta1/defaults_test.go index e846c7d5733..2e1086dad89 100644 --- a/pkg/apis/storage/v1beta1/defaults_test.go +++ b/pkg/apis/storage/v1beta1/defaults_test.go @@ -23,8 +23,10 @@ import ( storagev1beta1 "k8s.io/api/storage/v1beta1" "k8s.io/apimachinery/pkg/runtime" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" _ "k8s.io/kubernetes/pkg/apis/storage/install" + "k8s.io/kubernetes/pkg/features" ) func roundTrip(t *testing.T, obj runtime.Object) runtime.Object { @@ -51,25 +53,9 @@ func roundTrip(t *testing.T, obj runtime.Object) runtime.Object { func TestSetDefaultVolumeBindingMode(t *testing.T) { class := &storagev1beta1.StorageClass{} - // When feature gate is disabled, field should not be defaulted - err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - if err != nil { - t.Fatalf("Failed to enable feature gate for VolumeScheduling: %v", err) - } - output := roundTrip(t, runtime.Object(class)).(*storagev1beta1.StorageClass) - if output.VolumeBindingMode != nil { - t.Errorf("Expected VolumeBindingMode to not be defaulted, got: %+v", output.VolumeBindingMode) - } - - class = &storagev1beta1.StorageClass{} - // When feature gate is enabled, field should be defaulted - err = utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - if err != nil { - t.Fatalf("Failed to enable feature gate for VolumeScheduling: %v", err) - } defaultMode := storagev1beta1.VolumeBindingImmediate - output = roundTrip(t, runtime.Object(class)).(*storagev1beta1.StorageClass) + output := roundTrip(t, runtime.Object(class)).(*storagev1beta1.StorageClass) outMode := output.VolumeBindingMode if outMode == nil { t.Errorf("Expected VolumeBindingMode to be defaulted to: %+v, got: nil", defaultMode) @@ -77,8 +63,13 @@ func TestSetDefaultVolumeBindingMode(t *testing.T) { t.Errorf("Expected VolumeBindingMode to be defaulted to: %+v, got: %+v", defaultMode, outMode) } - err = utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - if err != nil { - t.Fatalf("Failed to disable feature gate for VolumeScheduling: %v", err) + class = &storagev1beta1.StorageClass{} + + // When feature gate is disabled, field should not be defaulted + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, false)() + output = roundTrip(t, runtime.Object(class)).(*storagev1beta1.StorageClass) + if output.VolumeBindingMode != nil { + t.Errorf("Expected VolumeBindingMode to not be defaulted, got: %+v", output.VolumeBindingMode) } + } diff --git a/pkg/apis/storage/v1beta1/main_test.go b/pkg/apis/storage/v1beta1/main_test.go new file mode 100644 index 00000000000..4613d1156c4 --- /dev/null +++ b/pkg/apis/storage/v1beta1/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/apis/storage/validation/BUILD b/pkg/apis/storage/validation/BUILD index 90dd3fe6824..fec90d3adcb 100644 --- a/pkg/apis/storage/validation/BUILD +++ b/pkg/apis/storage/validation/BUILD @@ -26,13 +26,18 @@ go_library( go_test( name = "go_default_test", - srcs = ["validation_test.go"], + srcs = [ + "main_test.go", + "validation_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", "//pkg/apis/storage:go_default_library", + "//pkg/features:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/pkg/apis/storage/validation/main_test.go b/pkg/apis/storage/validation/main_test.go new file mode 100644 index 00000000000..ad488a1caaa --- /dev/null +++ b/pkg/apis/storage/validation/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validation + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/apis/storage/validation/validation.go b/pkg/apis/storage/validation/validation.go index c6e50ed2f5b..48d9ecde476 100644 --- a/pkg/apis/storage/validation/validation.go +++ b/pkg/apis/storage/validation/validation.go @@ -133,7 +133,7 @@ func validateAllowVolumeExpansion(allowExpand *bool, fldPath *field.Path) field. return allErrs } -// ValidateVolumeAttachment validates a VolumeAttachment. +// ValidateVolumeAttachment validates a VolumeAttachment. This function is common for v1 and v1beta1 objects, func ValidateVolumeAttachment(volumeAttachment *storage.VolumeAttachment) field.ErrorList { allErrs := apivalidation.ValidateObjectMeta(&volumeAttachment.ObjectMeta, false, apivalidation.ValidateClassName, field.NewPath("metadata")) allErrs = append(allErrs, validateVolumeAttachmentSpec(&volumeAttachment.Spec, field.NewPath("spec"))...) @@ -141,6 +141,20 @@ func ValidateVolumeAttachment(volumeAttachment *storage.VolumeAttachment) field. return allErrs } +// ValidateVolumeAttachmentV1 validates a v1/VolumeAttachment. It contains only extra checks missing in +// ValidateVolumeAttachment. +func ValidateVolumeAttachmentV1(volumeAttachment *storage.VolumeAttachment) field.ErrorList { + allErrs := apivalidation.ValidateCSIDriverName(volumeAttachment.Spec.Attacher, field.NewPath("spec.attacher")) + + if volumeAttachment.Spec.Source.PersistentVolumeName != nil { + pvName := *volumeAttachment.Spec.Source.PersistentVolumeName + for _, msg := range apivalidation.ValidatePersistentVolumeName(pvName, false) { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec.source.persistentVolumeName"), pvName, msg)) + } + } + return allErrs +} + // ValidateVolumeAttachmentSpec tests that the specified VolumeAttachmentSpec // has valid data. func validateVolumeAttachmentSpec( @@ -218,6 +232,7 @@ func ValidateVolumeAttachmentUpdate(new, old *storage.VolumeAttachment) field.Er allErrs := ValidateVolumeAttachment(new) // Spec is read-only + // If this ever relaxes in the future, make sure to increment the Generation number in PrepareForUpdate if !apiequality.Semantic.DeepEqual(old.Spec, new.Spec) { allErrs = append(allErrs, field.Invalid(field.NewPath("spec"), new.Spec, "field is immutable")) } diff --git a/pkg/apis/storage/validation/validation_test.go b/pkg/apis/storage/validation/validation_test.go index ce3b09f1e83..d9940300c98 100644 --- a/pkg/apis/storage/validation/validation_test.go +++ b/pkg/apis/storage/validation/validation_test.go @@ -23,8 +23,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/storage" + "k8s.io/kubernetes/pkg/features" ) var ( @@ -151,21 +153,13 @@ func TestAlphaExpandPersistentVolumesFeatureValidation(t *testing.T) { VolumeBindingMode: &immediateMode1, } - // Enable alpha feature ExpandPersistentVolumes - err := utilfeature.DefaultFeatureGate.Set("ExpandPersistentVolumes=true") - if err != nil { - t.Errorf("Failed to enable feature gate for ExpandPersistentVolumes: %v", err) - return - } + // Enable feature ExpandPersistentVolumes + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExpandPersistentVolumes, true)() if errs := ValidateStorageClass(testSC); len(errs) != 0 { t.Errorf("expected success: %v", errs) } - // Disable alpha feature ExpandPersistentVolumes - err = utilfeature.DefaultFeatureGate.Set("ExpandPersistentVolumes=false") - if err != nil { - t.Errorf("Failed to disable feature gate for ExpandPersistentVolumes: %v", err) - return - } + // Disable feature ExpandPersistentVolumes + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExpandPersistentVolumes, false)() if errs := ValidateStorageClass(testSC); len(errs) == 0 { t.Errorf("expected failure, but got no error") } @@ -224,14 +218,9 @@ func TestVolumeAttachmentValidation(t *testing.T) { Spec: storage.VolumeAttachmentSpec{ Attacher: "", NodeName: "mynode", - }, - }, - { - // Invalid attacher name - ObjectMeta: metav1.ObjectMeta{Name: "foo"}, - Spec: storage.VolumeAttachmentSpec{ - Attacher: "invalid!@#$%^&*()", - NodeName: "mynode", + Source: storage.VolumeAttachmentSource{ + PersistentVolumeName: &volumeName, + }, }, }, { @@ -240,6 +229,9 @@ func TestVolumeAttachmentValidation(t *testing.T) { Spec: storage.VolumeAttachmentSpec{ Attacher: "myattacher", NodeName: "", + Source: storage.VolumeAttachmentSource{ + PersistentVolumeName: &volumeName, + }, }, }, { @@ -378,7 +370,7 @@ func TestVolumeAttachmentUpdateValidation(t *testing.T) { for _, volumeAttachment := range successCases { if errs := ValidateVolumeAttachmentUpdate(&volumeAttachment, &old); len(errs) != 0 { - t.Errorf("expected success: %v", errs) + t.Errorf("expected success: %+v", errs) } } @@ -445,7 +437,61 @@ func TestVolumeAttachmentUpdateValidation(t *testing.T) { for _, volumeAttachment := range errorCases { if errs := ValidateVolumeAttachmentUpdate(&volumeAttachment, &old); len(errs) == 0 { - t.Errorf("Expected failure for test: %v", volumeAttachment) + t.Errorf("Expected failure for test: %+v", volumeAttachment) + } + } +} + +func TestVolumeAttachmentValidationV1(t *testing.T) { + volumeName := "pv-name" + invalidVolumeName := "-invalid-@#$%^&*()-" + successCases := []storage.VolumeAttachment{ + { + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: storage.VolumeAttachmentSpec{ + Attacher: "myattacher", + Source: storage.VolumeAttachmentSource{ + PersistentVolumeName: &volumeName, + }, + NodeName: "mynode", + }, + }, + } + + for _, volumeAttachment := range successCases { + if errs := ValidateVolumeAttachmentV1(&volumeAttachment); len(errs) != 0 { + t.Errorf("expected success: %+v", errs) + } + } + + errorCases := []storage.VolumeAttachment{ + { + // Invalid attacher name + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: storage.VolumeAttachmentSpec{ + Attacher: "invalid-@#$%^&*()", + NodeName: "mynode", + Source: storage.VolumeAttachmentSource{ + PersistentVolumeName: &volumeName, + }, + }, + }, + { + // Invalid PV name + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: storage.VolumeAttachmentSpec{ + Attacher: "myattacher", + NodeName: "mynode", + Source: storage.VolumeAttachmentSource{ + PersistentVolumeName: &invalidVolumeName, + }, + }, + }, + } + + for _, volumeAttachment := range errorCases { + if errs := ValidateVolumeAttachmentV1(&volumeAttachment); len(errs) == 0 { + t.Errorf("Expected failure for test: %+v", volumeAttachment) } } } @@ -468,10 +514,7 @@ func TestValidateVolumeBindingModeAlphaDisabled(t *testing.T) { "invalid mode": makeClass(&invalidMode, nil), } - err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - if err != nil { - t.Fatalf("Failed to enable feature gate for VolumeScheduling: %v", err) - } + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, false)() for testName, storageClass := range errorCases { if errs := ValidateStorageClass(storageClass); len(errs) == 0 { t.Errorf("Expected failure for test: %v", testName) @@ -505,11 +548,7 @@ func TestValidateVolumeBindingMode(t *testing.T) { } // TODO: remove when feature gate not required - err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - if err != nil { - t.Fatalf("Failed to enable feature gate for VolumeScheduling: %v", err) - } - + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() for testName, testCase := range cases { errs := ValidateStorageClass(testCase.class) if testCase.shouldSucceed && len(errs) != 0 { @@ -519,11 +558,6 @@ func TestValidateVolumeBindingMode(t *testing.T) { t.Errorf("Expected failure for test %q, got success", testName) } } - - err = utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - if err != nil { - t.Fatalf("Failed to disable feature gate for VolumeScheduling: %v", err) - } } type updateTest struct { @@ -572,11 +606,7 @@ func TestValidateUpdateVolumeBindingMode(t *testing.T) { } // TODO: remove when feature gate not required - err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - if err != nil { - t.Fatalf("Failed to enable feature gate for VolumeScheduling: %v", err) - } - + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() for testName, testCase := range cases { errs := ValidateStorageClassUpdate(testCase.newClass, testCase.oldClass) if testCase.shouldSucceed && len(errs) != 0 { @@ -586,11 +616,6 @@ func TestValidateUpdateVolumeBindingMode(t *testing.T) { t.Errorf("Expected failure for %v, got success", testName) } } - - err = utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - if err != nil { - t.Fatalf("Failed to disable feature gate for VolumeScheduling: %v", err) - } } func TestValidateAllowedTopologies(t *testing.T) { @@ -884,12 +909,7 @@ func TestValidateAllowedTopologies(t *testing.T) { }, } - // TODO: remove when feature gate not required - err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - if err != nil { - t.Fatalf("Failed to enable feature gate for VolumeScheduling: %v", err) - } - + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() for testName, testCase := range cases { errs := ValidateStorageClass(testCase.class) if testCase.shouldSucceed && len(errs) != 0 { @@ -900,11 +920,7 @@ func TestValidateAllowedTopologies(t *testing.T) { } } - err = utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - if err != nil { - t.Fatalf("Failed to disable feature gate for VolumeScheduling: %v", err) - } - + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, false)() for testName, testCase := range cases { errs := ValidateStorageClass(testCase.class) if len(errs) == 0 && testCase.class.AllowedTopologies != nil { diff --git a/pkg/auth/authorizer/abac/BUILD b/pkg/auth/authorizer/abac/BUILD index e059fd56714..4d55745831e 100644 --- a/pkg/auth/authorizer/abac/BUILD +++ b/pkg/auth/authorizer/abac/BUILD @@ -17,7 +17,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/auth/authorizer/abac/abac.go b/pkg/auth/authorizer/abac/abac.go index 86e7f8ed3e0..8f49c98246b 100644 --- a/pkg/auth/authorizer/abac/abac.go +++ b/pkg/auth/authorizer/abac/abac.go @@ -25,7 +25,7 @@ import ( "os" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/authentication/user" @@ -105,7 +105,7 @@ func NewFromFile(path string) (policyList, error) { } if unversionedLines > 0 { - glog.Warningf("Policy file %s contained unversioned rules. See docs/admin/authorization.md#abac-mode for ABAC file format details.", path) + klog.Warningf("Policy file %s contained unversioned rules. See docs/admin/authorization.md#abac-mode for ABAC file format details.", path) } if err := scanner.Err(); err != nil { diff --git a/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/initializerconfiguration.go b/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/initializerconfiguration.go index b792b5470a3..3cae439c4a3 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/initializerconfiguration.go +++ b/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/initializerconfiguration.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -72,10 +74,15 @@ func (c *initializerConfigurations) Get(name string, options v1.GetOptions) (res // List takes label and field selectors, and returns the list of InitializerConfigurations that match those selectors. func (c *initializerConfigurations) List(opts v1.ListOptions) (result *admissionregistration.InitializerConfigurationList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &admissionregistration.InitializerConfigurationList{} err = c.client.Get(). Resource("initializerconfigurations"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *initializerConfigurations) List(opts v1.ListOptions) (result *admission // Watch returns a watch.Interface that watches the requested initializerConfigurations. func (c *initializerConfigurations) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("initializerconfigurations"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *initializerConfigurations) Delete(name string, options *v1.DeleteOption // DeleteCollection deletes a collection of objects. func (c *initializerConfigurations) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("initializerconfigurations"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/mutatingwebhookconfiguration.go b/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/mutatingwebhookconfiguration.go index c35d780fbfe..6f3c0d1aa13 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/mutatingwebhookconfiguration.go +++ b/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/mutatingwebhookconfiguration.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -72,10 +74,15 @@ func (c *mutatingWebhookConfigurations) Get(name string, options v1.GetOptions) // List takes label and field selectors, and returns the list of MutatingWebhookConfigurations that match those selectors. func (c *mutatingWebhookConfigurations) List(opts v1.ListOptions) (result *admissionregistration.MutatingWebhookConfigurationList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &admissionregistration.MutatingWebhookConfigurationList{} err = c.client.Get(). Resource("mutatingwebhookconfigurations"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *mutatingWebhookConfigurations) List(opts v1.ListOptions) (result *admis // Watch returns a watch.Interface that watches the requested mutatingWebhookConfigurations. func (c *mutatingWebhookConfigurations) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("mutatingwebhookconfigurations"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *mutatingWebhookConfigurations) Delete(name string, options *v1.DeleteOp // DeleteCollection deletes a collection of objects. func (c *mutatingWebhookConfigurations) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("mutatingwebhookconfigurations"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/validatingwebhookconfiguration.go b/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/validatingwebhookconfiguration.go index 980d9b5b8a9..183b85bdb77 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/validatingwebhookconfiguration.go +++ b/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/validatingwebhookconfiguration.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -72,10 +74,15 @@ func (c *validatingWebhookConfigurations) Get(name string, options v1.GetOptions // List takes label and field selectors, and returns the list of ValidatingWebhookConfigurations that match those selectors. func (c *validatingWebhookConfigurations) List(opts v1.ListOptions) (result *admissionregistration.ValidatingWebhookConfigurationList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &admissionregistration.ValidatingWebhookConfigurationList{} err = c.client.Get(). Resource("validatingwebhookconfigurations"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *validatingWebhookConfigurations) List(opts v1.ListOptions) (result *adm // Watch returns a watch.Interface that watches the requested validatingWebhookConfigurations. func (c *validatingWebhookConfigurations) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("validatingwebhookconfigurations"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *validatingWebhookConfigurations) Delete(name string, options *v1.Delete // DeleteCollection deletes a collection of objects. func (c *validatingWebhookConfigurations) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("validatingwebhookconfigurations"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/controllerrevision.go b/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/controllerrevision.go index 33a28c3a327..bd3b933bb19 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/controllerrevision.go +++ b/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/controllerrevision.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -75,11 +77,16 @@ func (c *controllerRevisions) Get(name string, options v1.GetOptions) (result *a // List takes label and field selectors, and returns the list of ControllerRevisions that match those selectors. func (c *controllerRevisions) List(opts v1.ListOptions) (result *apps.ControllerRevisionList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &apps.ControllerRevisionList{} err = c.client.Get(). Namespace(c.ns). Resource("controllerrevisions"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *controllerRevisions) List(opts v1.ListOptions) (result *apps.Controller // Watch returns a watch.Interface that watches the requested controllerRevisions. func (c *controllerRevisions) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("controllerrevisions"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *controllerRevisions) Delete(name string, options *v1.DeleteOptions) err // DeleteCollection deletes a collection of objects. func (c *controllerRevisions) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("controllerrevisions"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/daemonset.go b/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/daemonset.go index ac411caea45..f4b2c3ed31d 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/daemonset.go +++ b/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/daemonset.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *daemonSets) Get(name string, options v1.GetOptions) (result *apps.Daemo // List takes label and field selectors, and returns the list of DaemonSets that match those selectors. func (c *daemonSets) List(opts v1.ListOptions) (result *apps.DaemonSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &apps.DaemonSetList{} err = c.client.Get(). Namespace(c.ns). Resource("daemonsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *daemonSets) List(opts v1.ListOptions) (result *apps.DaemonSetList, err // Watch returns a watch.Interface that watches the requested daemonSets. func (c *daemonSets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("daemonsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *daemonSets) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *daemonSets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("daemonsets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/deployment.go b/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/deployment.go index 0fe3a2e2c51..1bf0bf03bc2 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/deployment.go +++ b/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/deployment.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *deployments) Get(name string, options v1.GetOptions) (result *apps.Depl // List takes label and field selectors, and returns the list of Deployments that match those selectors. func (c *deployments) List(opts v1.ListOptions) (result *apps.DeploymentList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &apps.DeploymentList{} err = c.client.Get(). Namespace(c.ns). Resource("deployments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *deployments) List(opts v1.ListOptions) (result *apps.DeploymentList, er // Watch returns a watch.Interface that watches the requested deployments. func (c *deployments) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("deployments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *deployments) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *deployments) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("deployments"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/replicaset.go b/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/replicaset.go index f213722d68b..ea7527f2b19 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/replicaset.go +++ b/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/replicaset.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *replicaSets) Get(name string, options v1.GetOptions) (result *apps.Repl // List takes label and field selectors, and returns the list of ReplicaSets that match those selectors. func (c *replicaSets) List(opts v1.ListOptions) (result *apps.ReplicaSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &apps.ReplicaSetList{} err = c.client.Get(). Namespace(c.ns). Resource("replicasets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *replicaSets) List(opts v1.ListOptions) (result *apps.ReplicaSetList, er // Watch returns a watch.Interface that watches the requested replicaSets. func (c *replicaSets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("replicasets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *replicaSets) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *replicaSets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("replicasets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/statefulset.go b/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/statefulset.go index b9eb391e9a0..d1a72132927 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/statefulset.go +++ b/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/statefulset.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *statefulSets) Get(name string, options v1.GetOptions) (result *apps.Sta // List takes label and field selectors, and returns the list of StatefulSets that match those selectors. func (c *statefulSets) List(opts v1.ListOptions) (result *apps.StatefulSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &apps.StatefulSetList{} err = c.client.Get(). Namespace(c.ns). Resource("statefulsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *statefulSets) List(opts v1.ListOptions) (result *apps.StatefulSetList, // Watch returns a watch.Interface that watches the requested statefulSets. func (c *statefulSets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("statefulsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *statefulSets) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *statefulSets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("statefulsets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/auditregistration/internalversion/auditsink.go b/pkg/client/clientset_generated/internalclientset/typed/auditregistration/internalversion/auditsink.go index 799ec09c30a..8bbb0375af2 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/auditregistration/internalversion/auditsink.go +++ b/pkg/client/clientset_generated/internalclientset/typed/auditregistration/internalversion/auditsink.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -72,10 +74,15 @@ func (c *auditSinks) Get(name string, options v1.GetOptions) (result *auditregis // List takes label and field selectors, and returns the list of AuditSinks that match those selectors. func (c *auditSinks) List(opts v1.ListOptions) (result *auditregistration.AuditSinkList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &auditregistration.AuditSinkList{} err = c.client.Get(). Resource("auditsinks"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *auditSinks) List(opts v1.ListOptions) (result *auditregistration.AuditS // Watch returns a watch.Interface that watches the requested auditSinks. func (c *auditSinks) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("auditsinks"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *auditSinks) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *auditSinks) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("auditsinks"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/autoscaling/internalversion/horizontalpodautoscaler.go b/pkg/client/clientset_generated/internalclientset/typed/autoscaling/internalversion/horizontalpodautoscaler.go index 39ec41254f8..2c5293c67ce 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/autoscaling/internalversion/horizontalpodautoscaler.go +++ b/pkg/client/clientset_generated/internalclientset/typed/autoscaling/internalversion/horizontalpodautoscaler.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *horizontalPodAutoscalers) Get(name string, options v1.GetOptions) (resu // List takes label and field selectors, and returns the list of HorizontalPodAutoscalers that match those selectors. func (c *horizontalPodAutoscalers) List(opts v1.ListOptions) (result *autoscaling.HorizontalPodAutoscalerList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &autoscaling.HorizontalPodAutoscalerList{} err = c.client.Get(). Namespace(c.ns). Resource("horizontalpodautoscalers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *horizontalPodAutoscalers) List(opts v1.ListOptions) (result *autoscalin // Watch returns a watch.Interface that watches the requested horizontalPodAutoscalers. func (c *horizontalPodAutoscalers) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("horizontalpodautoscalers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *horizontalPodAutoscalers) Delete(name string, options *v1.DeleteOptions // DeleteCollection deletes a collection of objects. func (c *horizontalPodAutoscalers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("horizontalpodautoscalers"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/batch/internalversion/cronjob.go b/pkg/client/clientset_generated/internalclientset/typed/batch/internalversion/cronjob.go index 72cb6cffa71..04f959524aa 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/batch/internalversion/cronjob.go +++ b/pkg/client/clientset_generated/internalclientset/typed/batch/internalversion/cronjob.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *cronJobs) Get(name string, options v1.GetOptions) (result *batch.CronJo // List takes label and field selectors, and returns the list of CronJobs that match those selectors. func (c *cronJobs) List(opts v1.ListOptions) (result *batch.CronJobList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &batch.CronJobList{} err = c.client.Get(). Namespace(c.ns). Resource("cronjobs"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *cronJobs) List(opts v1.ListOptions) (result *batch.CronJobList, err err // Watch returns a watch.Interface that watches the requested cronJobs. func (c *cronJobs) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("cronjobs"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *cronJobs) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *cronJobs) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("cronjobs"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/batch/internalversion/job.go b/pkg/client/clientset_generated/internalclientset/typed/batch/internalversion/job.go index e9f9dc545ca..fe5cd67e80f 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/batch/internalversion/job.go +++ b/pkg/client/clientset_generated/internalclientset/typed/batch/internalversion/job.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *jobs) Get(name string, options v1.GetOptions) (result *batch.Job, err e // List takes label and field selectors, and returns the list of Jobs that match those selectors. func (c *jobs) List(opts v1.ListOptions) (result *batch.JobList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &batch.JobList{} err = c.client.Get(). Namespace(c.ns). Resource("jobs"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *jobs) List(opts v1.ListOptions) (result *batch.JobList, err error) { // Watch returns a watch.Interface that watches the requested jobs. func (c *jobs) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("jobs"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *jobs) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *jobs) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("jobs"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/certificates/internalversion/certificatesigningrequest.go b/pkg/client/clientset_generated/internalclientset/typed/certificates/internalversion/certificatesigningrequest.go index 8a5b7b38077..b1584f20195 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/certificates/internalversion/certificatesigningrequest.go +++ b/pkg/client/clientset_generated/internalclientset/typed/certificates/internalversion/certificatesigningrequest.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -73,10 +75,15 @@ func (c *certificateSigningRequests) Get(name string, options v1.GetOptions) (re // List takes label and field selectors, and returns the list of CertificateSigningRequests that match those selectors. func (c *certificateSigningRequests) List(opts v1.ListOptions) (result *certificates.CertificateSigningRequestList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &certificates.CertificateSigningRequestList{} err = c.client.Get(). Resource("certificatesigningrequests"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *certificateSigningRequests) List(opts v1.ListOptions) (result *certific // Watch returns a watch.Interface that watches the requested certificateSigningRequests. func (c *certificateSigningRequests) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("certificatesigningrequests"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *certificateSigningRequests) Delete(name string, options *v1.DeleteOptio // DeleteCollection deletes a collection of objects. func (c *certificateSigningRequests) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("certificatesigningrequests"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/coordination/internalversion/lease.go b/pkg/client/clientset_generated/internalclientset/typed/coordination/internalversion/lease.go index a2023c38b90..fd4987e4897 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/coordination/internalversion/lease.go +++ b/pkg/client/clientset_generated/internalclientset/typed/coordination/internalversion/lease.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -75,11 +77,16 @@ func (c *leases) Get(name string, options v1.GetOptions) (result *coordination.L // List takes label and field selectors, and returns the list of Leases that match those selectors. func (c *leases) List(opts v1.ListOptions) (result *coordination.LeaseList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &coordination.LeaseList{} err = c.client.Get(). Namespace(c.ns). Resource("leases"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *leases) List(opts v1.ListOptions) (result *coordination.LeaseList, err // Watch returns a watch.Interface that watches the requested leases. func (c *leases) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("leases"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *leases) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *leases) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("leases"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/componentstatus.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/componentstatus.go index 07b75daab4c..d3d690798fb 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/componentstatus.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/componentstatus.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -72,10 +74,15 @@ func (c *componentStatuses) Get(name string, options v1.GetOptions) (result *cor // List takes label and field selectors, and returns the list of ComponentStatuses that match those selectors. func (c *componentStatuses) List(opts v1.ListOptions) (result *core.ComponentStatusList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.ComponentStatusList{} err = c.client.Get(). Resource("componentstatuses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *componentStatuses) List(opts v1.ListOptions) (result *core.ComponentSta // Watch returns a watch.Interface that watches the requested componentStatuses. func (c *componentStatuses) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("componentstatuses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *componentStatuses) Delete(name string, options *v1.DeleteOptions) error // DeleteCollection deletes a collection of objects. func (c *componentStatuses) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("componentstatuses"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/configmap.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/configmap.go index af1fc1fa17c..77eed84b525 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/configmap.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/configmap.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -75,11 +77,16 @@ func (c *configMaps) Get(name string, options v1.GetOptions) (result *core.Confi // List takes label and field selectors, and returns the list of ConfigMaps that match those selectors. func (c *configMaps) List(opts v1.ListOptions) (result *core.ConfigMapList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.ConfigMapList{} err = c.client.Get(). Namespace(c.ns). Resource("configmaps"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *configMaps) List(opts v1.ListOptions) (result *core.ConfigMapList, err // Watch returns a watch.Interface that watches the requested configMaps. func (c *configMaps) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("configmaps"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *configMaps) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *configMaps) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("configmaps"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/endpoints.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/endpoints.go index acb3fd21e57..64c1c473120 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/endpoints.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/endpoints.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -75,11 +77,16 @@ func (c *endpoints) Get(name string, options v1.GetOptions) (result *core.Endpoi // List takes label and field selectors, and returns the list of Endpoints that match those selectors. func (c *endpoints) List(opts v1.ListOptions) (result *core.EndpointsList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.EndpointsList{} err = c.client.Get(). Namespace(c.ns). Resource("endpoints"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *endpoints) List(opts v1.ListOptions) (result *core.EndpointsList, err e // Watch returns a watch.Interface that watches the requested endpoints. func (c *endpoints) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("endpoints"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *endpoints) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *endpoints) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("endpoints"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/event.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/event.go index 16dbe659642..ed123a58df5 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/event.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/event.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -75,11 +77,16 @@ func (c *events) Get(name string, options v1.GetOptions) (result *core.Event, er // List takes label and field selectors, and returns the list of Events that match those selectors. func (c *events) List(opts v1.ListOptions) (result *core.EventList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.EventList{} err = c.client.Get(). Namespace(c.ns). Resource("events"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *events) List(opts v1.ListOptions) (result *core.EventList, err error) { // Watch returns a watch.Interface that watches the requested events. func (c *events) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("events"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *events) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *events) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("events"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/limitrange.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/limitrange.go index d16b9b5ae1a..4f765a5b8b4 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/limitrange.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/limitrange.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -75,11 +77,16 @@ func (c *limitRanges) Get(name string, options v1.GetOptions) (result *core.Limi // List takes label and field selectors, and returns the list of LimitRanges that match those selectors. func (c *limitRanges) List(opts v1.ListOptions) (result *core.LimitRangeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.LimitRangeList{} err = c.client.Get(). Namespace(c.ns). Resource("limitranges"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *limitRanges) List(opts v1.ListOptions) (result *core.LimitRangeList, er // Watch returns a watch.Interface that watches the requested limitRanges. func (c *limitRanges) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("limitranges"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *limitRanges) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *limitRanges) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("limitranges"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/namespace.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/namespace.go index 5a0f413d35e..33e2c106988 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/namespace.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/namespace.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -73,10 +75,15 @@ func (c *namespaces) Get(name string, options v1.GetOptions) (result *core.Names // List takes label and field selectors, and returns the list of Namespaces that match those selectors. func (c *namespaces) List(opts v1.ListOptions) (result *core.NamespaceList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.NamespaceList{} err = c.client.Get(). Resource("namespaces"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *namespaces) List(opts v1.ListOptions) (result *core.NamespaceList, err // Watch returns a watch.Interface that watches the requested namespaces. func (c *namespaces) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("namespaces"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *namespaces) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *namespaces) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("namespaces"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/node.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/node.go index 515663c4d69..c49058ab1f8 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/node.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/node.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -73,10 +75,15 @@ func (c *nodes) Get(name string, options v1.GetOptions) (result *core.Node, err // List takes label and field selectors, and returns the list of Nodes that match those selectors. func (c *nodes) List(opts v1.ListOptions) (result *core.NodeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.NodeList{} err = c.client.Get(). Resource("nodes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *nodes) List(opts v1.ListOptions) (result *core.NodeList, err error) { // Watch returns a watch.Interface that watches the requested nodes. func (c *nodes) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("nodes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *nodes) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *nodes) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("nodes"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/persistentvolume.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/persistentvolume.go index fb717d8a3f6..017ad6eee11 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/persistentvolume.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/persistentvolume.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -73,10 +75,15 @@ func (c *persistentVolumes) Get(name string, options v1.GetOptions) (result *cor // List takes label and field selectors, and returns the list of PersistentVolumes that match those selectors. func (c *persistentVolumes) List(opts v1.ListOptions) (result *core.PersistentVolumeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.PersistentVolumeList{} err = c.client.Get(). Resource("persistentvolumes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *persistentVolumes) List(opts v1.ListOptions) (result *core.PersistentVo // Watch returns a watch.Interface that watches the requested persistentVolumes. func (c *persistentVolumes) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("persistentvolumes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *persistentVolumes) Delete(name string, options *v1.DeleteOptions) error // DeleteCollection deletes a collection of objects. func (c *persistentVolumes) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("persistentvolumes"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/persistentvolumeclaim.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/persistentvolumeclaim.go index a0111d67cdc..ed260b9f4cd 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/persistentvolumeclaim.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/persistentvolumeclaim.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *persistentVolumeClaims) Get(name string, options v1.GetOptions) (result // List takes label and field selectors, and returns the list of PersistentVolumeClaims that match those selectors. func (c *persistentVolumeClaims) List(opts v1.ListOptions) (result *core.PersistentVolumeClaimList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.PersistentVolumeClaimList{} err = c.client.Get(). Namespace(c.ns). Resource("persistentvolumeclaims"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *persistentVolumeClaims) List(opts v1.ListOptions) (result *core.Persist // Watch returns a watch.Interface that watches the requested persistentVolumeClaims. func (c *persistentVolumeClaims) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("persistentvolumeclaims"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *persistentVolumeClaims) Delete(name string, options *v1.DeleteOptions) // DeleteCollection deletes a collection of objects. func (c *persistentVolumeClaims) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("persistentvolumeclaims"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/pod.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/pod.go index b73b5b46587..faa40c200a6 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/pod.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/pod.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *pods) Get(name string, options v1.GetOptions) (result *core.Pod, err er // List takes label and field selectors, and returns the list of Pods that match those selectors. func (c *pods) List(opts v1.ListOptions) (result *core.PodList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.PodList{} err = c.client.Get(). Namespace(c.ns). Resource("pods"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *pods) List(opts v1.ListOptions) (result *core.PodList, err error) { // Watch returns a watch.Interface that watches the requested pods. func (c *pods) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("pods"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *pods) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *pods) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("pods"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/podtemplate.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/podtemplate.go index a9ef5caec88..6ef974d2b3b 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/podtemplate.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/podtemplate.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -75,11 +77,16 @@ func (c *podTemplates) Get(name string, options v1.GetOptions) (result *core.Pod // List takes label and field selectors, and returns the list of PodTemplates that match those selectors. func (c *podTemplates) List(opts v1.ListOptions) (result *core.PodTemplateList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.PodTemplateList{} err = c.client.Get(). Namespace(c.ns). Resource("podtemplates"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *podTemplates) List(opts v1.ListOptions) (result *core.PodTemplateList, // Watch returns a watch.Interface that watches the requested podTemplates. func (c *podTemplates) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("podtemplates"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *podTemplates) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *podTemplates) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("podtemplates"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/replicationcontroller.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/replicationcontroller.go index 225f020fee1..cda95f2527e 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/replicationcontroller.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/replicationcontroller.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -80,11 +82,16 @@ func (c *replicationControllers) Get(name string, options v1.GetOptions) (result // List takes label and field selectors, and returns the list of ReplicationControllers that match those selectors. func (c *replicationControllers) List(opts v1.ListOptions) (result *core.ReplicationControllerList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.ReplicationControllerList{} err = c.client.Get(). Namespace(c.ns). Resource("replicationcontrollers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -92,11 +99,16 @@ func (c *replicationControllers) List(opts v1.ListOptions) (result *core.Replica // Watch returns a watch.Interface that watches the requested replicationControllers. func (c *replicationControllers) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("replicationcontrollers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -154,10 +166,15 @@ func (c *replicationControllers) Delete(name string, options *v1.DeleteOptions) // DeleteCollection deletes a collection of objects. func (c *replicationControllers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("replicationcontrollers"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/resourcequota.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/resourcequota.go index 990451072d0..5ad2d4b8ca1 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/resourcequota.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/resourcequota.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *resourceQuotas) Get(name string, options v1.GetOptions) (result *core.R // List takes label and field selectors, and returns the list of ResourceQuotas that match those selectors. func (c *resourceQuotas) List(opts v1.ListOptions) (result *core.ResourceQuotaList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.ResourceQuotaList{} err = c.client.Get(). Namespace(c.ns). Resource("resourcequotas"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *resourceQuotas) List(opts v1.ListOptions) (result *core.ResourceQuotaLi // Watch returns a watch.Interface that watches the requested resourceQuotas. func (c *resourceQuotas) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("resourcequotas"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *resourceQuotas) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *resourceQuotas) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("resourcequotas"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/secret.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/secret.go index c3fe44d5d87..aea79ff0367 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/secret.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/secret.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -75,11 +77,16 @@ func (c *secrets) Get(name string, options v1.GetOptions) (result *core.Secret, // List takes label and field selectors, and returns the list of Secrets that match those selectors. func (c *secrets) List(opts v1.ListOptions) (result *core.SecretList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.SecretList{} err = c.client.Get(). Namespace(c.ns). Resource("secrets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *secrets) List(opts v1.ListOptions) (result *core.SecretList, err error) // Watch returns a watch.Interface that watches the requested secrets. func (c *secrets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("secrets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *secrets) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *secrets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("secrets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/service.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/service.go index e05f5337629..df8830871c7 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/service.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/service.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *services) Get(name string, options v1.GetOptions) (result *core.Service // List takes label and field selectors, and returns the list of Services that match those selectors. func (c *services) List(opts v1.ListOptions) (result *core.ServiceList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.ServiceList{} err = c.client.Get(). Namespace(c.ns). Resource("services"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *services) List(opts v1.ListOptions) (result *core.ServiceList, err erro // Watch returns a watch.Interface that watches the requested services. func (c *services) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("services"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *services) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *services) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("services"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/serviceaccount.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/serviceaccount.go index c2b6012c4d0..99578c1ee7d 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/serviceaccount.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/serviceaccount.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -75,11 +77,16 @@ func (c *serviceAccounts) Get(name string, options v1.GetOptions) (result *core. // List takes label and field selectors, and returns the list of ServiceAccounts that match those selectors. func (c *serviceAccounts) List(opts v1.ListOptions) (result *core.ServiceAccountList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &core.ServiceAccountList{} err = c.client.Get(). Namespace(c.ns). Resource("serviceaccounts"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *serviceAccounts) List(opts v1.ListOptions) (result *core.ServiceAccount // Watch returns a watch.Interface that watches the requested serviceAccounts. func (c *serviceAccounts) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("serviceaccounts"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *serviceAccounts) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *serviceAccounts) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("serviceaccounts"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion/ingress.go b/pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion/ingress.go index bbe7577ccb7..2b8f27e39aa 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion/ingress.go +++ b/pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion/ingress.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *ingresses) Get(name string, options v1.GetOptions) (result *extensions. // List takes label and field selectors, and returns the list of Ingresses that match those selectors. func (c *ingresses) List(opts v1.ListOptions) (result *extensions.IngressList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &extensions.IngressList{} err = c.client.Get(). Namespace(c.ns). Resource("ingresses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *ingresses) List(opts v1.ListOptions) (result *extensions.IngressList, e // Watch returns a watch.Interface that watches the requested ingresses. func (c *ingresses) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("ingresses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *ingresses) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *ingresses) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("ingresses"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/networking/internalversion/networkpolicy.go b/pkg/client/clientset_generated/internalclientset/typed/networking/internalversion/networkpolicy.go index 6a0cdbc3bf6..d0e8533bf77 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/networking/internalversion/networkpolicy.go +++ b/pkg/client/clientset_generated/internalclientset/typed/networking/internalversion/networkpolicy.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -75,11 +77,16 @@ func (c *networkPolicies) Get(name string, options v1.GetOptions) (result *netwo // List takes label and field selectors, and returns the list of NetworkPolicies that match those selectors. func (c *networkPolicies) List(opts v1.ListOptions) (result *networking.NetworkPolicyList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &networking.NetworkPolicyList{} err = c.client.Get(). Namespace(c.ns). Resource("networkpolicies"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *networkPolicies) List(opts v1.ListOptions) (result *networking.NetworkP // Watch returns a watch.Interface that watches the requested networkPolicies. func (c *networkPolicies) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("networkpolicies"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *networkPolicies) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *networkPolicies) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("networkpolicies"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/poddisruptionbudget.go b/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/poddisruptionbudget.go index 7e7abfcb27c..5985a5be0ab 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/poddisruptionbudget.go +++ b/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/poddisruptionbudget.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *podDisruptionBudgets) Get(name string, options v1.GetOptions) (result * // List takes label and field selectors, and returns the list of PodDisruptionBudgets that match those selectors. func (c *podDisruptionBudgets) List(opts v1.ListOptions) (result *policy.PodDisruptionBudgetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &policy.PodDisruptionBudgetList{} err = c.client.Get(). Namespace(c.ns). Resource("poddisruptionbudgets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *podDisruptionBudgets) List(opts v1.ListOptions) (result *policy.PodDisr // Watch returns a watch.Interface that watches the requested podDisruptionBudgets. func (c *podDisruptionBudgets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("poddisruptionbudgets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *podDisruptionBudgets) Delete(name string, options *v1.DeleteOptions) er // DeleteCollection deletes a collection of objects. func (c *podDisruptionBudgets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("poddisruptionbudgets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/podsecuritypolicy.go b/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/podsecuritypolicy.go index 17c162fa47a..9ee7b09530c 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/podsecuritypolicy.go +++ b/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/podsecuritypolicy.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -72,10 +74,15 @@ func (c *podSecurityPolicies) Get(name string, options v1.GetOptions) (result *p // List takes label and field selectors, and returns the list of PodSecurityPolicies that match those selectors. func (c *podSecurityPolicies) List(opts v1.ListOptions) (result *policy.PodSecurityPolicyList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &policy.PodSecurityPolicyList{} err = c.client.Get(). Resource("podsecuritypolicies"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *podSecurityPolicies) List(opts v1.ListOptions) (result *policy.PodSecur // Watch returns a watch.Interface that watches the requested podSecurityPolicies. func (c *podSecurityPolicies) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("podsecuritypolicies"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *podSecurityPolicies) Delete(name string, options *v1.DeleteOptions) err // DeleteCollection deletes a collection of objects. func (c *podSecurityPolicies) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("podsecuritypolicies"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/clusterrole.go b/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/clusterrole.go index 985e90f16aa..7e9394d4a93 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/clusterrole.go +++ b/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/clusterrole.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -72,10 +74,15 @@ func (c *clusterRoles) Get(name string, options v1.GetOptions) (result *rbac.Clu // List takes label and field selectors, and returns the list of ClusterRoles that match those selectors. func (c *clusterRoles) List(opts v1.ListOptions) (result *rbac.ClusterRoleList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &rbac.ClusterRoleList{} err = c.client.Get(). Resource("clusterroles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *clusterRoles) List(opts v1.ListOptions) (result *rbac.ClusterRoleList, // Watch returns a watch.Interface that watches the requested clusterRoles. func (c *clusterRoles) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("clusterroles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *clusterRoles) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *clusterRoles) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("clusterroles"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/clusterrolebinding.go b/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/clusterrolebinding.go index 01dcf227524..f6a06750a66 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/clusterrolebinding.go +++ b/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/clusterrolebinding.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -72,10 +74,15 @@ func (c *clusterRoleBindings) Get(name string, options v1.GetOptions) (result *r // List takes label and field selectors, and returns the list of ClusterRoleBindings that match those selectors. func (c *clusterRoleBindings) List(opts v1.ListOptions) (result *rbac.ClusterRoleBindingList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &rbac.ClusterRoleBindingList{} err = c.client.Get(). Resource("clusterrolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *clusterRoleBindings) List(opts v1.ListOptions) (result *rbac.ClusterRol // Watch returns a watch.Interface that watches the requested clusterRoleBindings. func (c *clusterRoleBindings) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("clusterrolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *clusterRoleBindings) Delete(name string, options *v1.DeleteOptions) err // DeleteCollection deletes a collection of objects. func (c *clusterRoleBindings) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("clusterrolebindings"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/role.go b/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/role.go index 7272550e8db..903388150ef 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/role.go +++ b/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/role.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -75,11 +77,16 @@ func (c *roles) Get(name string, options v1.GetOptions) (result *rbac.Role, err // List takes label and field selectors, and returns the list of Roles that match those selectors. func (c *roles) List(opts v1.ListOptions) (result *rbac.RoleList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &rbac.RoleList{} err = c.client.Get(). Namespace(c.ns). Resource("roles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *roles) List(opts v1.ListOptions) (result *rbac.RoleList, err error) { // Watch returns a watch.Interface that watches the requested roles. func (c *roles) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("roles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *roles) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *roles) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("roles"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/rolebinding.go b/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/rolebinding.go index bed5dcdf9a9..0315e69bb39 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/rolebinding.go +++ b/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/rolebinding.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -75,11 +77,16 @@ func (c *roleBindings) Get(name string, options v1.GetOptions) (result *rbac.Rol // List takes label and field selectors, and returns the list of RoleBindings that match those selectors. func (c *roleBindings) List(opts v1.ListOptions) (result *rbac.RoleBindingList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &rbac.RoleBindingList{} err = c.client.Get(). Namespace(c.ns). Resource("rolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *roleBindings) List(opts v1.ListOptions) (result *rbac.RoleBindingList, // Watch returns a watch.Interface that watches the requested roleBindings. func (c *roleBindings) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("rolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *roleBindings) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *roleBindings) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("rolebindings"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/scheduling/internalversion/priorityclass.go b/pkg/client/clientset_generated/internalclientset/typed/scheduling/internalversion/priorityclass.go index 8d5f992da64..18a4329946c 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/scheduling/internalversion/priorityclass.go +++ b/pkg/client/clientset_generated/internalclientset/typed/scheduling/internalversion/priorityclass.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -72,10 +74,15 @@ func (c *priorityClasses) Get(name string, options v1.GetOptions) (result *sched // List takes label and field selectors, and returns the list of PriorityClasses that match those selectors. func (c *priorityClasses) List(opts v1.ListOptions) (result *scheduling.PriorityClassList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &scheduling.PriorityClassList{} err = c.client.Get(). Resource("priorityclasses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *priorityClasses) List(opts v1.ListOptions) (result *scheduling.Priority // Watch returns a watch.Interface that watches the requested priorityClasses. func (c *priorityClasses) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("priorityclasses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *priorityClasses) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *priorityClasses) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("priorityclasses"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/settings/internalversion/podpreset.go b/pkg/client/clientset_generated/internalclientset/typed/settings/internalversion/podpreset.go index 334c2180c02..bf99269060d 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/settings/internalversion/podpreset.go +++ b/pkg/client/clientset_generated/internalclientset/typed/settings/internalversion/podpreset.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -75,11 +77,16 @@ func (c *podPresets) Get(name string, options v1.GetOptions) (result *settings.P // List takes label and field selectors, and returns the list of PodPresets that match those selectors. func (c *podPresets) List(opts v1.ListOptions) (result *settings.PodPresetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &settings.PodPresetList{} err = c.client.Get(). Namespace(c.ns). Resource("podpresets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *podPresets) List(opts v1.ListOptions) (result *settings.PodPresetList, // Watch returns a watch.Interface that watches the requested podPresets. func (c *podPresets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("podpresets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *podPresets) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *podPresets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("podpresets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/storage/internalversion/storageclass.go b/pkg/client/clientset_generated/internalclientset/typed/storage/internalversion/storageclass.go index 4939d7af8c3..5f40850a807 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/storage/internalversion/storageclass.go +++ b/pkg/client/clientset_generated/internalclientset/typed/storage/internalversion/storageclass.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -72,10 +74,15 @@ func (c *storageClasses) Get(name string, options v1.GetOptions) (result *storag // List takes label and field selectors, and returns the list of StorageClasses that match those selectors. func (c *storageClasses) List(opts v1.ListOptions) (result *storage.StorageClassList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &storage.StorageClassList{} err = c.client.Get(). Resource("storageclasses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *storageClasses) List(opts v1.ListOptions) (result *storage.StorageClass // Watch returns a watch.Interface that watches the requested storageClasses. func (c *storageClasses) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("storageclasses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *storageClasses) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *storageClasses) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("storageclasses"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/clientset_generated/internalclientset/typed/storage/internalversion/volumeattachment.go b/pkg/client/clientset_generated/internalclientset/typed/storage/internalversion/volumeattachment.go index 2ce2c9784cf..571cd99809a 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/storage/internalversion/volumeattachment.go +++ b/pkg/client/clientset_generated/internalclientset/typed/storage/internalversion/volumeattachment.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -73,10 +75,15 @@ func (c *volumeAttachments) Get(name string, options v1.GetOptions) (result *sto // List takes label and field selectors, and returns the list of VolumeAttachments that match those selectors. func (c *volumeAttachments) List(opts v1.ListOptions) (result *storage.VolumeAttachmentList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &storage.VolumeAttachmentList{} err = c.client.Get(). Resource("volumeattachments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *volumeAttachments) List(opts v1.ListOptions) (result *storage.VolumeAtt // Watch returns a watch.Interface that watches the requested volumeAttachments. func (c *volumeAttachments) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("volumeattachments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *volumeAttachments) Delete(name string, options *v1.DeleteOptions) error // DeleteCollection deletes a collection of objects. func (c *volumeAttachments) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("volumeattachments"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/pkg/client/tests/remotecommand_test.go b/pkg/client/tests/remotecommand_test.go index 143ee2e7d5b..010cdff92d3 100644 --- a/pkg/client/tests/remotecommand_test.go +++ b/pkg/client/tests/remotecommand_test.go @@ -108,7 +108,7 @@ func (ex *fakeExecutor) run(name string, uid types.UID, container string, cmd [] return nil } -func fakeServer(t *testing.T, testName string, exec bool, stdinData, stdoutData, stderrData, errorData string, tty bool, messageCount int, serverProtocols []string) http.HandlerFunc { +func fakeServer(t *testing.T, requestReceived chan struct{}, testName string, exec bool, stdinData, stdoutData, stderrData, errorData string, tty bool, messageCount int, serverProtocols []string) http.HandlerFunc { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { executor := &fakeExecutor{ t: t, @@ -134,6 +134,7 @@ func fakeServer(t *testing.T, testName string, exec bool, stdinData, stdoutData, if e, a := strings.Repeat(stdinData, messageCount), executor.stdinReceived.String(); e != a { t.Errorf("%s: stdin: expected %q, got %q", testName, e, a) } + close(requestReceived) }) } @@ -165,6 +166,15 @@ func TestStream(t *testing.T) { ClientProtocols: []string{remotecommandconsts.StreamProtocolV2Name}, ServerProtocols: []string{remotecommandconsts.StreamProtocolV2Name}, }, + { + TestName: "oversized stdin", + Stdin: strings.Repeat("a", 20*1024*1024), + Stdout: "b", + Stderr: "", + MessageCount: 1, + ClientProtocols: []string{remotecommandconsts.StreamProtocolV2Name}, + ServerProtocols: []string{remotecommandconsts.StreamProtocolV2Name}, + }, { TestName: "in/out/tty", Stdin: "a", @@ -218,7 +228,8 @@ func TestStream(t *testing.T) { localOut := &bytes.Buffer{} localErr := &bytes.Buffer{} - server := httptest.NewServer(fakeServer(t, name, exec, testCase.Stdin, testCase.Stdout, testCase.Stderr, testCase.Error, testCase.Tty, testCase.MessageCount, testCase.ServerProtocols)) + requestReceived := make(chan struct{}) + server := httptest.NewServer(fakeServer(t, requestReceived, name, exec, testCase.Stdin, testCase.Stdout, testCase.Stderr, testCase.Error, testCase.Tty, testCase.MessageCount, testCase.ServerProtocols)) url, _ := url.ParseRequestURI(server.URL) config := restclient.ContentConfig{ @@ -305,6 +316,12 @@ func TestStream(t *testing.T) { } } + select { + case <-requestReceived: + case <-time.After(time.Minute): + t.Errorf("%s: expected fakeServerInstance to receive request", name) + } + server.Close() } } diff --git a/pkg/cloudprovider/providers/aws/BUILD b/pkg/cloudprovider/providers/aws/BUILD index 22c40fd0ce4..ca34b569b24 100644 --- a/pkg/cloudprovider/providers/aws/BUILD +++ b/pkg/cloudprovider/providers/aws/BUILD @@ -58,9 +58,9 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/service/elbv2:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/kms:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/sts:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/gopkg.in/gcfg.v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index b80dff30e51..d4f38dc75f4 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -42,8 +42,8 @@ import ( "github.com/aws/aws-sdk-go/service/elbv2" "github.com/aws/aws-sdk-go/service/kms" "github.com/aws/aws-sdk-go/service/sts" - "github.com/golang/glog" gcfg "gopkg.in/gcfg.v1" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -967,7 +967,7 @@ func init() { Client: ec2metadata.New(sess), } } else { - glog.Infof("Using AWS assumed role %v", cfg.Global.RoleARN) + klog.Infof("Using AWS assumed role %v", cfg.Global.RoleARN) provider = &stscreds.AssumeRoleProvider{ Client: sts.New(sess), RoleARN: cfg.Global.RoleARN, @@ -1004,7 +1004,7 @@ func readAWSCloudConfig(config io.Reader) (*CloudConfig, error) { func updateConfigZone(cfg *CloudConfig, metadata EC2Metadata) error { if cfg.Global.Zone == "" { if metadata != nil { - glog.Info("Zone not specified in configuration file; querying AWS metadata service") + klog.Info("Zone not specified in configuration file; querying AWS metadata service") var err error cfg.Global.Zone, err = getAvailabilityZone(metadata) if err != nil { @@ -1038,7 +1038,7 @@ func azToRegion(az string) (string, error) { func newAWSCloud(cfg CloudConfig, awsServices Services) (*Cloud, error) { // We have some state in the Cloud object - in particular the attaching map // Log so that if we are building multiple Cloud objects, it is obvious! - glog.Infof("Building AWS cloudprovider") + klog.Infof("Building AWS cloudprovider") metadata, err := awsServices.Metadata() if err != nil { @@ -1070,7 +1070,7 @@ func newAWSCloud(cfg CloudConfig, awsServices Services) (*Cloud, error) { return nil, fmt.Errorf("not a valid AWS zone (unknown region): %s", zone) } } else { - glog.Warningf("Strict AWS zone checking is disabled. Proceeding with zone: %s", zone) + klog.Warningf("Strict AWS zone checking is disabled. Proceeding with zone: %s", zone) } ec2, err := awsServices.Compute(regionName) @@ -1117,7 +1117,7 @@ func newAWSCloud(cfg CloudConfig, awsServices Services) (*Cloud, error) { if cfg.Global.VPC != "" && (cfg.Global.SubnetID != "" || cfg.Global.RoleARN != "") && tagged { // When the master is running on a different AWS account, cloud provider or on-premise // build up a dummy instance and use the VPC from the nodes account - glog.Info("Master is configured to run on a different AWS account, different cloud provider or on-premises") + klog.Info("Master is configured to run on a different AWS account, different cloud provider or on-premises") awsCloud.selfAWSInstance = &awsInstance{ nodeName: "master-dummy", vpcID: cfg.Global.VPC, @@ -1161,7 +1161,7 @@ func (c *Cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, c.clientBuilder = clientBuilder c.kubeClient = clientBuilder.ClientOrDie("aws-cloud-provider") c.eventBroadcaster = record.NewBroadcaster() - c.eventBroadcaster.StartLogging(glog.Infof) + c.eventBroadcaster.StartLogging(klog.Infof) c.eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: c.kubeClient.CoreV1().Events("")}) c.eventRecorder = c.eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "aws-cloud-provider"}) } @@ -1232,7 +1232,7 @@ func (c *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.No if err != nil { //TODO: It would be nice to be able to determine the reason for the failure, // but the AWS client masks all failures with the same error description. - glog.V(4).Info("Could not determine public IP from AWS metadata.") + klog.V(4).Info("Could not determine public IP from AWS metadata.") } else { addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: externalIP}) } @@ -1241,7 +1241,7 @@ func (c *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.No if err != nil || len(internalDNS) == 0 { //TODO: It would be nice to be able to determine the reason for the failure, // but the AWS client masks all failures with the same error description. - glog.V(4).Info("Could not determine private DNS from AWS metadata.") + klog.V(4).Info("Could not determine private DNS from AWS metadata.") } else { addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalDNS, Address: internalDNS}) addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: internalDNS}) @@ -1251,7 +1251,7 @@ func (c *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.No if err != nil || len(externalDNS) == 0 { //TODO: It would be nice to be able to determine the reason for the failure, // but the AWS client masks all failures with the same error description. - glog.V(4).Info("Could not determine public DNS from AWS metadata.") + klog.V(4).Info("Could not determine public DNS from AWS metadata.") } else { addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalDNS, Address: externalDNS}) } @@ -1360,7 +1360,7 @@ func (c *Cloud) InstanceExistsByProviderID(ctx context.Context, providerID strin state := instances[0].State.Name if *state == ec2.InstanceStateNameTerminated { - glog.Warningf("the instance %s is terminated", instanceID) + klog.Warningf("the instance %s is terminated", instanceID) return false, nil } @@ -1383,7 +1383,7 @@ func (c *Cloud) InstanceShutdownByProviderID(ctx context.Context, providerID str return false, err } if len(instances) == 0 { - glog.Warningf("the instance %s does not exist anymore", providerID) + klog.Warningf("the instance %s does not exist anymore", providerID) // returns false, because otherwise node is not deleted from cluster // false means that it will continue to check InstanceExistsByProviderID return false, nil @@ -1485,7 +1485,7 @@ func (c *Cloud) GetCandidateZonesForDynamicVolume() (sets.String, error) { } if master { - glog.V(4).Infof("Ignoring master instance %q in zone discovery", aws.StringValue(instance.InstanceId)) + klog.V(4).Infof("Ignoring master instance %q in zone discovery", aws.StringValue(instance.InstanceId)) continue } @@ -1495,7 +1495,7 @@ func (c *Cloud) GetCandidateZonesForDynamicVolume() (sets.String, error) { } } - glog.V(2).Infof("Found instances in zones %s", zones) + klog.V(2).Infof("Found instances in zones %s", zones) return zones, nil } @@ -1614,7 +1614,7 @@ func (c *Cloud) getMountDevice( name = name[8:] } if len(name) < 1 || len(name) > 2 { - glog.Warningf("Unexpected EBS DeviceName: %q", aws.StringValue(blockDevice.DeviceName)) + klog.Warningf("Unexpected EBS DeviceName: %q", aws.StringValue(blockDevice.DeviceName)) } deviceMappings[mountDevice(name)] = EBSVolumeID(aws.StringValue(blockDevice.Ebs.VolumeId)) } @@ -1633,7 +1633,7 @@ func (c *Cloud) getMountDevice( for mountDevice, mappingVolumeID := range deviceMappings { if volumeID == mappingVolumeID { if assign { - glog.Warningf("Got assignment call for already-assigned volume: %s@%s", mountDevice, mappingVolumeID) + klog.Warningf("Got assignment call for already-assigned volume: %s@%s", mountDevice, mappingVolumeID) } return mountDevice, true, nil } @@ -1658,7 +1658,7 @@ func (c *Cloud) getMountDevice( chosen, err := deviceAllocator.GetNext(deviceMappings) if err != nil { - glog.Warningf("Could not assign a mount device. mappings=%v, error: %v", deviceMappings, err) + klog.Warningf("Could not assign a mount device. mappings=%v, error: %v", deviceMappings, err) return "", false, fmt.Errorf("too many EBS volumes attached to node %s", i.nodeName) } @@ -1668,7 +1668,7 @@ func (c *Cloud) getMountDevice( c.attaching[i.nodeName] = attaching } attaching[chosen] = volumeID - glog.V(2).Infof("Assigned mount device %s -> volume %s", chosen, volumeID) + klog.V(2).Infof("Assigned mount device %s -> volume %s", chosen, volumeID) return chosen, false, nil } @@ -1688,10 +1688,10 @@ func (c *Cloud) endAttaching(i *awsInstance, volumeID EBSVolumeID, mountDevice m // attached to the instance (as reported by the EC2 API). So if endAttaching comes after // a 10 second poll delay, we might well have had a concurrent request to allocate a mountpoint, // which because we allocate sequentially is _very_ likely to get the immediately freed volume - glog.Infof("endAttaching on device %q assigned to different volume: %q vs %q", mountDevice, volumeID, existingVolumeID) + klog.Infof("endAttaching on device %q assigned to different volume: %q vs %q", mountDevice, volumeID, existingVolumeID) return false } - glog.V(2).Infof("Releasing in-process attachment entry: %s -> volume %s", mountDevice, volumeID) + klog.V(2).Infof("Releasing in-process attachment entry: %s -> volume %s", mountDevice, volumeID) delete(c.attaching[i.nodeName], mountDevice) return true } @@ -1815,7 +1815,7 @@ func (d *awsDisk) modifyVolume(requestGiB int64) (int64, error) { func (c *Cloud) applyUnSchedulableTaint(nodeName types.NodeName, reason string) { node, fetchErr := c.kubeClient.CoreV1().Nodes().Get(string(nodeName), metav1.GetOptions{}) if fetchErr != nil { - glog.Errorf("Error fetching node %s with %v", nodeName, fetchErr) + klog.Errorf("Error fetching node %s with %v", nodeName, fetchErr) return } @@ -1826,7 +1826,7 @@ func (c *Cloud) applyUnSchedulableTaint(nodeName types.NodeName, reason string) } err := controller.AddOrUpdateTaintOnNode(c.kubeClient, string(nodeName), taint) if err != nil { - glog.Errorf("Error applying taint to node %s with error %v", nodeName, err) + klog.Errorf("Error applying taint to node %s with error %v", nodeName, err) return } c.eventRecorder.Eventf(node, v1.EventTypeWarning, volumeAttachmentStuck, reason) @@ -1854,7 +1854,7 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, if isAWSErrorVolumeNotFound(err) { if status == "detached" { // The disk doesn't exist, assume it's detached, log warning and stop waiting - glog.Warningf("Waiting for volume %q to be detached but the volume does not exist", d.awsID) + klog.Warningf("Waiting for volume %q to be detached but the volume does not exist", d.awsID) stateStr := "detached" attachment = &ec2.VolumeAttachment{ State: &stateStr, @@ -1863,7 +1863,7 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, } if status == "attached" { // The disk doesn't exist, complain, give up waiting and report error - glog.Warningf("Waiting for volume %q to be attached but the volume does not exist", d.awsID) + klog.Warningf("Waiting for volume %q to be attached but the volume does not exist", d.awsID) return false, err } } @@ -1873,7 +1873,7 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, return false, err } - glog.Warningf("Ignoring error from describe volume for volume %q; will retry: %q", d.awsID, err) + klog.Warningf("Ignoring error from describe volume for volume %q; will retry: %q", d.awsID, err) return false, nil } @@ -1881,20 +1881,20 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, if len(info.Attachments) > 1 { // Shouldn't happen; log so we know if it is - glog.Warningf("Found multiple attachments for volume %q: %v", d.awsID, info) + klog.Warningf("Found multiple attachments for volume %q: %v", d.awsID, info) } attachmentStatus := "" for _, a := range info.Attachments { if attachmentStatus != "" { // Shouldn't happen; log so we know if it is - glog.Warningf("Found multiple attachments for volume %q: %v", d.awsID, info) + klog.Warningf("Found multiple attachments for volume %q: %v", d.awsID, info) } if a.State != nil { attachment = a attachmentStatus = *a.State } else { // Shouldn't happen; log so we know if it is - glog.Warningf("Ignoring nil attachment state for volume %q: %v", d.awsID, a) + klog.Warningf("Ignoring nil attachment state for volume %q: %v", d.awsID, a) } } if attachmentStatus == "" { @@ -1905,7 +1905,7 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, return true, nil } // continue waiting - glog.V(2).Infof("Waiting for volume %q state: actual=%s, desired=%s", d.awsID, attachmentStatus, status) + klog.V(2).Infof("Waiting for volume %q state: actual=%s, desired=%s", d.awsID, attachmentStatus, status) return false, nil }) @@ -1963,11 +1963,11 @@ func wrapAttachError(err error, disk *awsDisk, instance string) error { if awsError.Code() == "VolumeInUse" { info, err := disk.describeVolume() if err != nil { - glog.Errorf("Error describing volume %q: %q", disk.awsID, err) + klog.Errorf("Error describing volume %q: %q", disk.awsID, err) } else { for _, a := range info.Attachments { if disk.awsID != EBSVolumeID(aws.StringValue(a.VolumeId)) { - glog.Warningf("Expected to get attachment info of volume %q but instead got info of %q", disk.awsID, aws.StringValue(a.VolumeId)) + klog.Warningf("Expected to get attachment info of volume %q but instead got info of %q", disk.awsID, aws.StringValue(a.VolumeId)) } else if aws.StringValue(a.State) == "attached" { return fmt.Errorf("Error attaching EBS volume %q to instance %q: %q. The volume is currently attached to instance %q", disk.awsID, instance, awsError, aws.StringValue(a.InstanceId)) } @@ -2001,7 +2001,7 @@ func (c *Cloud) AttachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) defer func() { if attachEnded { if !c.endAttaching(awsInstance, disk.awsID, mountDevice) { - glog.Errorf("endAttaching called for disk %q when attach not in progress", disk.awsID) + klog.Errorf("endAttaching called for disk %q when attach not in progress", disk.awsID) } } }() @@ -2020,7 +2020,7 @@ func (c *Cloud) AttachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) if !alreadyAttached { available, err := c.checkIfAvailable(disk, "attaching", awsInstance.awsID) if err != nil { - glog.Error(err) + klog.Error(err) } if !available { @@ -2042,7 +2042,7 @@ func (c *Cloud) AttachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) if da, ok := c.deviceAllocators[awsInstance.nodeName]; ok { da.Deprioritize(mountDevice) } - glog.V(2).Infof("AttachVolume volume=%q instance=%q request returned %v", disk.awsID, awsInstance.awsID, attachResponse) + klog.V(2).Infof("AttachVolume volume=%q instance=%q request returned %v", disk.awsID, awsInstance.awsID, attachResponse) } attachment, err := disk.waitForAttachmentStatus("attached") @@ -2080,7 +2080,7 @@ func (c *Cloud) DetachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) if err != nil { if isAWSErrorVolumeNotFound(err) { // Someone deleted the volume being detached; complain, but do nothing else and return success - glog.Warningf("DetachDisk %s called for node %s but volume does not exist; assuming the volume is detached", diskName, nodeName) + klog.Warningf("DetachDisk %s called for node %s but volume does not exist; assuming the volume is detached", diskName, nodeName) return "", nil } @@ -2088,7 +2088,7 @@ func (c *Cloud) DetachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) } if !attached && diskInfo.ec2Instance != nil { - glog.Warningf("DetachDisk %s called for node %s but volume is attached to node %s", diskName, nodeName, diskInfo.nodeName) + klog.Warningf("DetachDisk %s called for node %s but volume is attached to node %s", diskName, nodeName, diskInfo.nodeName) return "", nil } @@ -2104,7 +2104,7 @@ func (c *Cloud) DetachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) } if !alreadyAttached { - glog.Warningf("DetachDisk called on non-attached disk: %s", diskName) + klog.Warningf("DetachDisk called on non-attached disk: %s", diskName) // TODO: Continue? Tolerate non-attached error from the AWS DetachVolume call? } @@ -2131,7 +2131,7 @@ func (c *Cloud) DetachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) } if attachment != nil { // We expect it to be nil, it is (maybe) interesting if it is not - glog.V(2).Infof("waitForAttachmentStatus returned non-nil attachment with state=detached: %v", attachment) + klog.V(2).Infof("waitForAttachmentStatus returned non-nil attachment with state=detached: %v", attachment) } if mountDevice != "" { @@ -2268,10 +2268,10 @@ func (c *Cloud) DeleteDisk(volumeName KubernetesVolumeID) (bool, error) { available, err := c.checkIfAvailable(awsDisk, "deleting", "") if err != nil { if isAWSErrorVolumeNotFound(err) { - glog.V(2).Infof("Volume %s not found when deleting it, assuming it's deleted", awsDisk.awsID) + klog.V(2).Infof("Volume %s not found when deleting it, assuming it's deleted", awsDisk.awsID) return false, nil } - glog.Error(err) + klog.Error(err) } if !available { @@ -2285,7 +2285,7 @@ func (c *Cloud) checkIfAvailable(disk *awsDisk, opName string, instance string) info, err := disk.describeVolume() if err != nil { - glog.Errorf("Error describing volume %q: %q", disk.awsID, err) + klog.Errorf("Error describing volume %q: %q", disk.awsID, err) // if for some reason we can not describe volume we will return error return false, err } @@ -2305,7 +2305,7 @@ func (c *Cloud) checkIfAvailable(disk *awsDisk, opName string, instance string) attachedInstance, ierr := c.getInstanceByID(instanceID) attachErr := fmt.Sprintf("%s since volume is currently attached to %q", opError, instanceID) if ierr != nil { - glog.Error(attachErr) + klog.Error(attachErr) return false, errors.New(attachErr) } devicePath := aws.StringValue(attachment.Device) @@ -2386,7 +2386,7 @@ func (c *Cloud) DiskIsAttached(diskName KubernetesVolumeID, nodeName types.NodeN if err != nil { if isAWSErrorVolumeNotFound(err) { // The disk doesn't exist, can't be attached - glog.Warningf("DiskIsAttached called for volume %s on node %s but the volume does not exist", diskName, nodeName) + klog.Warningf("DiskIsAttached called for volume %s on node %s but the volume does not exist", diskName, nodeName) return false, nil } @@ -2423,7 +2423,7 @@ func (c *Cloud) DisksAreAttached(nodeDisks map[types.NodeName][]KubernetesVolume } if len(awsInstances) == 0 { - glog.V(2).Infof("DisksAreAttached found no instances matching node names; will assume disks not attached") + klog.V(2).Infof("DisksAreAttached found no instances matching node names; will assume disks not attached") return attached, nil } @@ -2518,7 +2518,7 @@ func (c *Cloud) describeLoadBalancer(name string) (*elb.LoadBalancerDescription, var ret *elb.LoadBalancerDescription for _, loadBalancer := range response.LoadBalancerDescriptions { if ret != nil { - glog.Errorf("Found multiple load balancers with name: %s", name) + klog.Errorf("Found multiple load balancers with name: %s", name) } ret = loadBalancer } @@ -2603,7 +2603,7 @@ func (c *Cloud) findSecurityGroup(securityGroupID string) (*ec2.SecurityGroup, e groups, err := c.ec2.DescribeSecurityGroups(describeSecurityGroupsRequest) if err != nil { - glog.Warningf("Error retrieving security group: %q", err) + klog.Warningf("Error retrieving security group: %q", err) return nil, err } @@ -2650,7 +2650,7 @@ func ipPermissionExists(newPermission, existing *ec2.IpPermission, compareGroupU } // Check only if newPermission is a subset of existing. Usually it has zero or one elements. // Not doing actual CIDR math yet; not clear it's needed, either. - glog.V(4).Infof("Comparing %v to %v", newPermission, existing) + klog.V(4).Infof("Comparing %v to %v", newPermission, existing) if len(newPermission.IpRanges) > len(existing.IpRanges) { return false } @@ -2685,7 +2685,7 @@ func ipPermissionExists(newPermission, existing *ec2.IpPermission, compareGroupU } func isEqualUserGroupPair(l, r *ec2.UserIdGroupPair, compareGroupUserIDs bool) bool { - glog.V(2).Infof("Comparing %v to %v", *l.GroupId, *r.GroupId) + klog.V(2).Infof("Comparing %v to %v", *l.GroupId, *r.GroupId) if isEqualStringPointer(l.GroupId, r.GroupId) { if compareGroupUserIDs { if isEqualStringPointer(l.UserId, r.UserId) { @@ -2710,7 +2710,7 @@ func (c *Cloud) setSecurityGroupIngress(securityGroupID string, permissions IPPe group, err := c.findSecurityGroup(securityGroupID) if err != nil { - glog.Warningf("Error retrieving security group %q", err) + klog.Warningf("Error retrieving security group %q", err) return false, err } @@ -2718,7 +2718,7 @@ func (c *Cloud) setSecurityGroupIngress(securityGroupID string, permissions IPPe return false, fmt.Errorf("security group not found: %s", securityGroupID) } - glog.V(2).Infof("Existing security group ingress: %s %v", securityGroupID, group.IpPermissions) + klog.V(2).Infof("Existing security group ingress: %s %v", securityGroupID, group.IpPermissions) actual := NewIPPermissionSet(group.IpPermissions...) @@ -2749,7 +2749,7 @@ func (c *Cloud) setSecurityGroupIngress(securityGroupID string, permissions IPPe // don't want to accidentally open more than intended while we're // applying changes. if add.Len() != 0 { - glog.V(2).Infof("Adding security group ingress: %s %v", securityGroupID, add.List()) + klog.V(2).Infof("Adding security group ingress: %s %v", securityGroupID, add.List()) request := &ec2.AuthorizeSecurityGroupIngressInput{} request.GroupId = &securityGroupID @@ -2760,7 +2760,7 @@ func (c *Cloud) setSecurityGroupIngress(securityGroupID string, permissions IPPe } } if remove.Len() != 0 { - glog.V(2).Infof("Remove security group ingress: %s %v", securityGroupID, remove.List()) + klog.V(2).Infof("Remove security group ingress: %s %v", securityGroupID, remove.List()) request := &ec2.RevokeSecurityGroupIngressInput{} request.GroupId = &securityGroupID @@ -2785,7 +2785,7 @@ func (c *Cloud) addSecurityGroupIngress(securityGroupID string, addPermissions [ group, err := c.findSecurityGroup(securityGroupID) if err != nil { - glog.Warningf("Error retrieving security group: %q", err) + klog.Warningf("Error retrieving security group: %q", err) return false, err } @@ -2793,7 +2793,7 @@ func (c *Cloud) addSecurityGroupIngress(securityGroupID string, addPermissions [ return false, fmt.Errorf("security group not found: %s", securityGroupID) } - glog.V(2).Infof("Existing security group ingress: %s %v", securityGroupID, group.IpPermissions) + klog.V(2).Infof("Existing security group ingress: %s %v", securityGroupID, group.IpPermissions) changes := []*ec2.IpPermission{} for _, addPermission := range addPermissions { @@ -2821,14 +2821,14 @@ func (c *Cloud) addSecurityGroupIngress(securityGroupID string, addPermissions [ return false, nil } - glog.V(2).Infof("Adding security group ingress: %s %v", securityGroupID, changes) + klog.V(2).Infof("Adding security group ingress: %s %v", securityGroupID, changes) request := &ec2.AuthorizeSecurityGroupIngressInput{} request.GroupId = &securityGroupID request.IpPermissions = changes _, err = c.ec2.AuthorizeSecurityGroupIngress(request) if err != nil { - glog.Warningf("Error authorizing security group ingress %q", err) + klog.Warningf("Error authorizing security group ingress %q", err) return false, fmt.Errorf("error authorizing security group ingress: %q", err) } @@ -2846,12 +2846,12 @@ func (c *Cloud) removeSecurityGroupIngress(securityGroupID string, removePermiss group, err := c.findSecurityGroup(securityGroupID) if err != nil { - glog.Warningf("Error retrieving security group: %q", err) + klog.Warningf("Error retrieving security group: %q", err) return false, err } if group == nil { - glog.Warning("Security group not found: ", securityGroupID) + klog.Warning("Security group not found: ", securityGroupID) return false, nil } @@ -2881,14 +2881,14 @@ func (c *Cloud) removeSecurityGroupIngress(securityGroupID string, removePermiss return false, nil } - glog.V(2).Infof("Removing security group ingress: %s %v", securityGroupID, changes) + klog.V(2).Infof("Removing security group ingress: %s %v", securityGroupID, changes) request := &ec2.RevokeSecurityGroupIngressInput{} request.GroupId = &securityGroupID request.IpPermissions = changes _, err = c.ec2.RevokeSecurityGroupIngress(request) if err != nil { - glog.Warningf("Error revoking security group ingress: %q", err) + klog.Warningf("Error revoking security group ingress: %q", err) return false, err } @@ -2924,7 +2924,7 @@ func (c *Cloud) ensureSecurityGroup(name string, description string, additionalT if len(securityGroups) >= 1 { if len(securityGroups) > 1 { - glog.Warningf("Found multiple security groups with name: %q", name) + klog.Warningf("Found multiple security groups with name: %q", name) } err := c.tagging.readRepairClusterTags( c.ec2, aws.StringValue(securityGroups[0].GroupId), @@ -2947,12 +2947,12 @@ func (c *Cloud) ensureSecurityGroup(name string, description string, additionalT switch err := err.(type) { case awserr.Error: if err.Code() == "InvalidGroup.Duplicate" && attempt < MaxReadThenCreateRetries { - glog.V(2).Infof("Got InvalidGroup.Duplicate while creating security group (race?); will retry") + klog.V(2).Infof("Got InvalidGroup.Duplicate while creating security group (race?); will retry") ignore = true } } if !ignore { - glog.Errorf("Error creating security group: %q", err) + klog.Errorf("Error creating security group: %q", err) return "", err } time.Sleep(1 * time.Second) @@ -3011,7 +3011,7 @@ func (c *Cloud) findSubnets() ([]*ec2.Subnet, error) { } // Fall back to the current instance subnets, if nothing is tagged - glog.Warningf("No tagged subnets found; will fall-back to the current subnet only. This is likely to be an error in a future version of k8s.") + klog.Warningf("No tagged subnets found; will fall-back to the current subnet only. This is likely to be an error in a future version of k8s.") request = &ec2.DescribeSubnetsInput{} filters = []*ec2.Filter{newEc2Filter("subnet-id", c.selfAWSInstance.subnetID)} @@ -3048,7 +3048,7 @@ func (c *Cloud) findELBSubnets(internalELB bool) ([]string, error) { az := aws.StringValue(subnet.AvailabilityZone) id := aws.StringValue(subnet.SubnetId) if az == "" || id == "" { - glog.Warningf("Ignoring subnet with empty az/id: %v", subnet) + klog.Warningf("Ignoring subnet with empty az/id: %v", subnet) continue } @@ -3057,7 +3057,7 @@ func (c *Cloud) findELBSubnets(internalELB bool) ([]string, error) { return nil, err } if !internalELB && !isPublic { - glog.V(2).Infof("Ignoring private subnet for public ELB %q", id) + klog.V(2).Infof("Ignoring private subnet for public ELB %q", id) continue } @@ -3088,12 +3088,12 @@ func (c *Cloud) findELBSubnets(internalELB bool) ([]string, error) { // If we have two subnets for the same AZ we arbitrarily choose the one that is first lexicographically. // TODO: Should this be an error. if strings.Compare(*existing.SubnetId, *subnet.SubnetId) > 0 { - glog.Warningf("Found multiple subnets in AZ %q; choosing %q between subnets %q and %q", az, *subnet.SubnetId, *existing.SubnetId, *subnet.SubnetId) + klog.Warningf("Found multiple subnets in AZ %q; choosing %q between subnets %q and %q", az, *subnet.SubnetId, *existing.SubnetId, *subnet.SubnetId) subnetsByAZ[az] = subnet continue } - glog.Warningf("Found multiple subnets in AZ %q; choosing %q between subnets %q and %q", az, *existing.SubnetId, *existing.SubnetId, *subnet.SubnetId) + klog.Warningf("Found multiple subnets in AZ %q; choosing %q between subnets %q and %q", az, *existing.SubnetId, *existing.SubnetId, *subnet.SubnetId) continue } @@ -3122,7 +3122,7 @@ func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) { for _, table := range rt { for _, assoc := range table.Associations { if aws.BoolValue(assoc.Main) == true { - glog.V(4).Infof("Assuming implicit use of main routing table %s for %s", + klog.V(4).Infof("Assuming implicit use of main routing table %s for %s", aws.StringValue(table.RouteTableId), subnetID) subnetTable = table break @@ -3195,7 +3195,7 @@ func (c *Cloud) buildELBSecurityGroupList(serviceName types.NamespacedName, load sgDescription := fmt.Sprintf("Security group for Kubernetes ELB %s (%v)", loadBalancerName, serviceName) securityGroupID, err = c.ensureSecurityGroup(sgName, sgDescription, getLoadBalancerAdditionalTags(annotations)) if err != nil { - glog.Errorf("Error creating load balancer security group: %q", err) + klog.Errorf("Error creating load balancer security group: %q", err) return nil, err } } @@ -3263,7 +3263,7 @@ func buildListener(port v1.ServicePort, annotations map[string]string, sslPorts // EnsureLoadBalancer implements LoadBalancer.EnsureLoadBalancer func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiService *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { annotations := apiService.Annotations - glog.V(2).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)", + klog.V(2).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)", clusterName, apiService.Namespace, apiService.Name, c.region, apiService.Spec.LoadBalancerIP, apiService.Spec.Ports, annotations) if apiService.Spec.SessionAffinity != v1.ServiceAffinityNone { @@ -3285,7 +3285,7 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS return nil, fmt.Errorf("Only TCP LoadBalancer is supported for AWS ELB") } if port.NodePort == 0 { - glog.Errorf("Ignoring port without NodePort defined: %v", port) + klog.Errorf("Ignoring port without NodePort defined: %v", port) continue } @@ -3342,7 +3342,7 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS // Find the subnets that the ELB will live in subnetIDs, err := c.findELBSubnets(internalELB) if err != nil { - glog.Errorf("Error listing subnets in VPC: %q", err) + klog.Errorf("Error listing subnets in VPC: %q", err) return nil, err } // Bail out early if there are no subnets @@ -3381,7 +3381,7 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS err = c.updateInstanceSecurityGroupsForNLB(v2Mappings, instances, loadBalancerName, sourceRangeCidrs) if err != nil { - glog.Warningf("Error opening ingress rules for the load balancer to the instances: %q", err) + klog.Warningf("Error opening ingress rules for the load balancer to the instances: %q", err) return nil, err } @@ -3503,7 +3503,7 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS // Find the subnets that the ELB will live in subnetIDs, err := c.findELBSubnets(internalELB) if err != nil { - glog.Errorf("Error listing subnets in VPC: %q", err) + klog.Errorf("Error listing subnets in VPC: %q", err) return nil, err } @@ -3590,13 +3590,13 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS } if path, healthCheckNodePort := service.GetServiceHealthCheckPathPort(apiService); path != "" { - glog.V(4).Infof("service %v (%v) needs health checks on :%d%s)", apiService.Name, loadBalancerName, healthCheckNodePort, path) + klog.V(4).Infof("service %v (%v) needs health checks on :%d%s)", apiService.Name, loadBalancerName, healthCheckNodePort, path) err = c.ensureLoadBalancerHealthCheck(loadBalancer, "HTTP", healthCheckNodePort, path, annotations) if err != nil { return nil, fmt.Errorf("Failed to ensure health check for localized service %v on node port %v: %q", loadBalancerName, healthCheckNodePort, err) } } else { - glog.V(4).Infof("service %v does not need custom health checks", apiService.Name) + klog.V(4).Infof("service %v does not need custom health checks", apiService.Name) // We only configure a TCP health-check on the first port var tcpHealthCheckPort int32 for _, listener := range listeners { @@ -3615,17 +3615,17 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS err = c.updateInstanceSecurityGroupsForLoadBalancer(loadBalancer, instances) if err != nil { - glog.Warningf("Error opening ingress rules for the load balancer to the instances: %q", err) + klog.Warningf("Error opening ingress rules for the load balancer to the instances: %q", err) return nil, err } err = c.ensureLoadBalancerInstances(aws.StringValue(loadBalancer.LoadBalancerName), loadBalancer.Instances, instances) if err != nil { - glog.Warningf("Error registering instances with the load balancer: %q", err) + klog.Warningf("Error registering instances with the load balancer: %q", err) return nil, err } - glog.V(1).Infof("Loadbalancer %s (%v) has DNS name %s", loadBalancerName, serviceName, aws.StringValue(loadBalancer.DNSName)) + klog.V(1).Infof("Loadbalancer %s (%v) has DNS name %s", loadBalancerName, serviceName, aws.StringValue(loadBalancer.DNSName)) // TODO: Wait for creation? @@ -3682,7 +3682,7 @@ func toStatus(lb *elb.LoadBalancerDescription) *v1.LoadBalancerStatus { func v2toStatus(lb *elbv2.LoadBalancer) *v1.LoadBalancerStatus { status := &v1.LoadBalancerStatus{} if lb == nil { - glog.Error("[BUG] v2toStatus got nil input, this is a Kubernetes bug, please report") + klog.Error("[BUG] v2toStatus got nil input, this is a Kubernetes bug, please report") return status } @@ -3709,7 +3709,7 @@ func findSecurityGroupForInstance(instance *ec2.Instance, taggedSecurityGroups m for _, group := range instance.SecurityGroups { groupID := aws.StringValue(group.GroupId) if groupID == "" { - glog.Warningf("Ignoring security group without id for instance %q: %v", instanceID, group) + klog.Warningf("Ignoring security group without id for instance %q: %v", instanceID, group) continue } _, isTagged := taggedSecurityGroups[groupID] @@ -3741,7 +3741,7 @@ func findSecurityGroupForInstance(instance *ec2.Instance, taggedSecurityGroups m return untagged[0], nil } - glog.Warningf("No security group found for instance %q", instanceID) + klog.Warningf("No security group found for instance %q", instanceID) return nil, nil } @@ -3762,7 +3762,7 @@ func (c *Cloud) getTaggedSecurityGroups() (map[string]*ec2.SecurityGroup, error) id := aws.StringValue(group.GroupId) if id == "" { - glog.Warningf("Ignoring group without id: %v", group) + klog.Warningf("Ignoring group without id: %v", group) continue } m[id] = group @@ -3785,7 +3785,7 @@ func (c *Cloud) updateInstanceSecurityGroupsForLoadBalancer(lb *elb.LoadBalancer } if loadBalancerSecurityGroupID != "" { // We create LBs with one SG - glog.Warningf("Multiple security groups for load balancer: %q", aws.StringValue(lb.LoadBalancerName)) + klog.Warningf("Multiple security groups for load balancer: %q", aws.StringValue(lb.LoadBalancerName)) } loadBalancerSecurityGroupID = *securityGroup } @@ -3834,12 +3834,12 @@ func (c *Cloud) updateInstanceSecurityGroupsForLoadBalancer(lb *elb.LoadBalancer } if securityGroup == nil { - glog.Warning("Ignoring instance without security group: ", aws.StringValue(instance.InstanceId)) + klog.Warning("Ignoring instance without security group: ", aws.StringValue(instance.InstanceId)) continue } id := aws.StringValue(securityGroup.GroupId) if id == "" { - glog.Warningf("found security group without id: %v", securityGroup) + klog.Warningf("found security group without id: %v", securityGroup) continue } @@ -3850,7 +3850,7 @@ func (c *Cloud) updateInstanceSecurityGroupsForLoadBalancer(lb *elb.LoadBalancer for _, actualGroup := range actualGroups { actualGroupID := aws.StringValue(actualGroup.GroupId) if actualGroupID == "" { - glog.Warning("Ignoring group without ID: ", actualGroup) + klog.Warning("Ignoring group without ID: ", actualGroup) continue } @@ -3866,9 +3866,9 @@ func (c *Cloud) updateInstanceSecurityGroupsForLoadBalancer(lb *elb.LoadBalancer for instanceSecurityGroupID, add := range instanceSecurityGroupIds { if add { - glog.V(2).Infof("Adding rule for traffic from the load balancer (%s) to instances (%s)", loadBalancerSecurityGroupID, instanceSecurityGroupID) + klog.V(2).Infof("Adding rule for traffic from the load balancer (%s) to instances (%s)", loadBalancerSecurityGroupID, instanceSecurityGroupID) } else { - glog.V(2).Infof("Removing rule for traffic from the load balancer (%s) to instance (%s)", loadBalancerSecurityGroupID, instanceSecurityGroupID) + klog.V(2).Infof("Removing rule for traffic from the load balancer (%s) to instance (%s)", loadBalancerSecurityGroupID, instanceSecurityGroupID) } sourceGroupID := &ec2.UserIdGroupPair{} sourceGroupID.GroupId = &loadBalancerSecurityGroupID @@ -3887,7 +3887,7 @@ func (c *Cloud) updateInstanceSecurityGroupsForLoadBalancer(lb *elb.LoadBalancer return err } if !changed { - glog.Warning("Allowing ingress was not needed; concurrent change? groupId=", instanceSecurityGroupID) + klog.Warning("Allowing ingress was not needed; concurrent change? groupId=", instanceSecurityGroupID) } } else { changed, err := c.removeSecurityGroupIngress(instanceSecurityGroupID, permissions) @@ -3895,7 +3895,7 @@ func (c *Cloud) updateInstanceSecurityGroupsForLoadBalancer(lb *elb.LoadBalancer return err } if !changed { - glog.Warning("Revoking ingress was not needed; concurrent change? groupId=", instanceSecurityGroupID) + klog.Warning("Revoking ingress was not needed; concurrent change? groupId=", instanceSecurityGroupID) } } } @@ -3913,7 +3913,7 @@ func (c *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin return err } if lb == nil { - glog.Info("Load balancer already deleted: ", loadBalancerName) + klog.Info("Load balancer already deleted: ", loadBalancerName) return nil } @@ -4037,7 +4037,7 @@ func (c *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin return err } if !changed { - glog.Warning("Revoking ingress was not needed; concurrent change? groupId=", *matchingGroups[i].GroupId) + klog.Warning("Revoking ingress was not needed; concurrent change? groupId=", *matchingGroups[i].GroupId) } } @@ -4055,7 +4055,7 @@ func (c *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin } if lb == nil { - glog.Info("Load balancer already deleted: ", loadBalancerName) + klog.Info("Load balancer already deleted: ", loadBalancerName) return nil } @@ -4063,7 +4063,7 @@ func (c *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin // De-authorize the load balancer security group from the instances security group err = c.updateInstanceSecurityGroupsForLoadBalancer(lb, nil) if err != nil { - glog.Errorf("Error deregistering load balancer from instance security groups: %q", err) + klog.Errorf("Error deregistering load balancer from instance security groups: %q", err) return err } } @@ -4076,7 +4076,7 @@ func (c *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin _, err = c.elb.DeleteLoadBalancer(request) if err != nil { // TODO: Check if error was because load balancer was concurrently deleted - glog.Errorf("Error deleting load balancer: %q", err) + klog.Errorf("Error deleting load balancer: %q", err) return err } } @@ -4094,7 +4094,7 @@ func (c *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin continue } if aws.StringValue(securityGroupID) == "" { - glog.Warning("Ignoring empty security group in ", service.Name) + klog.Warning("Ignoring empty security group in ", service.Name) continue } securityGroupIDs[*securityGroupID] = struct{}{} @@ -4113,7 +4113,7 @@ func (c *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin ignore := false if awsError, ok := err.(awserr.Error); ok { if awsError.Code() == "DependencyViolation" { - glog.V(2).Infof("Ignoring DependencyViolation while deleting load-balancer security group (%s), assuming because LB is in process of deleting", securityGroupID) + klog.V(2).Infof("Ignoring DependencyViolation while deleting load-balancer security group (%s), assuming because LB is in process of deleting", securityGroupID) ignore = true } } @@ -4124,7 +4124,7 @@ func (c *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin } if len(securityGroupIDs) == 0 { - glog.V(2).Info("Deleted all security groups for load balancer: ", service.Name) + klog.V(2).Info("Deleted all security groups for load balancer: ", service.Name) break } @@ -4137,7 +4137,7 @@ func (c *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin return fmt.Errorf("timed out deleting ELB: %s. Could not delete security groups %v", service.Name, strings.Join(ids, ",")) } - glog.V(2).Info("Waiting for load-balancer to delete so we can delete security groups: ", service.Name) + klog.V(2).Info("Waiting for load-balancer to delete so we can delete security groups: ", service.Name) time.Sleep(10 * time.Second) } @@ -4268,14 +4268,14 @@ func (c *Cloud) getInstancesByNodeNames(nodeNames []string, states ...string) ([ instances, err := c.describeInstances(filters) if err != nil { - glog.V(2).Infof("Failed to describe instances %v", nodeNames) + klog.V(2).Infof("Failed to describe instances %v", nodeNames) return nil, err } ec2Instances = append(ec2Instances, instances...) } if len(ec2Instances) == 0 { - glog.V(3).Infof("Failed to find any instances %v", nodeNames) + klog.V(3).Infof("Failed to find any instances %v", nodeNames) return nil, nil } return ec2Instances, nil diff --git a/pkg/cloudprovider/providers/aws/aws_fakes.go b/pkg/cloudprovider/providers/aws/aws_fakes.go index f398ab87cd8..28946fa35c0 100644 --- a/pkg/cloudprovider/providers/aws/aws_fakes.go +++ b/pkg/cloudprovider/providers/aws/aws_fakes.go @@ -25,7 +25,7 @@ import ( "github.com/aws/aws-sdk-go/service/elb" "github.com/aws/aws-sdk-go/service/elbv2" "github.com/aws/aws-sdk-go/service/kms" - "github.com/golang/glog" + "k8s.io/klog" ) // FakeAWSServices is an fake AWS session used for testing @@ -141,7 +141,7 @@ func (ec2i *FakeEC2Impl) DescribeInstances(request *ec2.DescribeInstancesInput) for _, instance := range ec2i.aws.instances { if request.InstanceIds != nil { if instance.InstanceId == nil { - glog.Warning("Instance with no instance id: ", instance) + klog.Warning("Instance with no instance id: ", instance) continue } diff --git a/pkg/cloudprovider/providers/aws/aws_instancegroups.go b/pkg/cloudprovider/providers/aws/aws_instancegroups.go index df6f3084ddf..6c4c59b039e 100644 --- a/pkg/cloudprovider/providers/aws/aws_instancegroups.go +++ b/pkg/cloudprovider/providers/aws/aws_instancegroups.go @@ -21,7 +21,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" - "github.com/golang/glog" + "k8s.io/klog" ) // AWSCloud implements InstanceGroups @@ -64,7 +64,7 @@ func DescribeInstanceGroup(asg ASG, instanceGroupName string) (InstanceGroupInfo return nil, nil } if len(response.AutoScalingGroups) > 1 { - glog.Warning("AWS returned multiple autoscaling groups with name ", instanceGroupName) + klog.Warning("AWS returned multiple autoscaling groups with name ", instanceGroupName) } group := response.AutoScalingGroups[0] return &awsInstanceGroup{group: group}, nil diff --git a/pkg/cloudprovider/providers/aws/aws_loadbalancer.go b/pkg/cloudprovider/providers/aws/aws_loadbalancer.go index 27d1443304c..0c2e7553067 100644 --- a/pkg/cloudprovider/providers/aws/aws_loadbalancer.go +++ b/pkg/cloudprovider/providers/aws/aws_loadbalancer.go @@ -28,7 +28,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elb" "github.com/aws/aws-sdk-go/service/elbv2" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -134,7 +134,7 @@ func (c *Cloud) ensureLoadBalancerv2(namespacedName types.NamespacedName, loadBa }) } - glog.Infof("Creating load balancer for %v with name: %s", namespacedName, loadBalancerName) + klog.Infof("Creating load balancer for %v with name: %s", namespacedName, loadBalancerName) createResponse, err := c.elbv2.CreateLoadBalancer(createRequest) if err != nil { return nil, fmt.Errorf("Error creating load balancer: %q", err) @@ -340,7 +340,7 @@ func createTargetName(namespacedName types.NamespacedName, frontendPort, nodePor func (c *Cloud) createListenerV2(loadBalancerArn *string, mapping nlbPortMapping, namespacedName types.NamespacedName, instanceIDs []string, vpcID string) (listener *elbv2.Listener, targetGroupArn *string, err error) { targetName := createTargetName(namespacedName, mapping.FrontendPort, mapping.TrafficPort) - glog.Infof("Creating load balancer target group for %v with name: %s", namespacedName, targetName) + klog.Infof("Creating load balancer target group for %v with name: %s", namespacedName, targetName) target, err := c.ensureTargetGroup( nil, mapping, @@ -361,7 +361,7 @@ func (c *Cloud) createListenerV2(loadBalancerArn *string, mapping nlbPortMapping Type: aws.String(elbv2.ActionTypeEnumForward), }}, } - glog.Infof("Creating load balancer listener for %v", namespacedName) + klog.Infof("Creating load balancer listener for %v", namespacedName) createListenerOutput, err := c.elbv2.CreateListener(createListernerInput) if err != nil { return nil, aws.String(""), fmt.Errorf("Error creating load balancer listener: %q", err) @@ -617,7 +617,7 @@ func (c *Cloud) updateInstanceSecurityGroupsForNLBTraffic(actualGroups []*ec2.Se for _, actualGroup := range actualGroups { actualGroupID := aws.StringValue(actualGroup.GroupId) if actualGroupID == "" { - glog.Warning("Ignoring group without ID: ", actualGroup) + klog.Warning("Ignoring group without ID: ", actualGroup) continue } @@ -652,17 +652,17 @@ func (c *Cloud) updateInstanceSecurityGroupsForNLBTraffic(actualGroups []*ec2.Se for port, add := range portMap { if add { if clientTraffic { - glog.V(2).Infof("Adding rule for client MTU discovery from the network load balancer (%s) to instances (%s)", clientCidrs, instanceSecurityGroupID) - glog.V(2).Infof("Adding rule for client traffic from the network load balancer (%s) to instances (%s)", clientCidrs, instanceSecurityGroupID) + klog.V(2).Infof("Adding rule for client MTU discovery from the network load balancer (%s) to instances (%s)", clientCidrs, instanceSecurityGroupID) + klog.V(2).Infof("Adding rule for client traffic from the network load balancer (%s) to instances (%s)", clientCidrs, instanceSecurityGroupID) } else { - glog.V(2).Infof("Adding rule for health check traffic from the network load balancer (%s) to instances (%s)", clientCidrs, instanceSecurityGroupID) + klog.V(2).Infof("Adding rule for health check traffic from the network load balancer (%s) to instances (%s)", clientCidrs, instanceSecurityGroupID) } } else { if clientTraffic { - glog.V(2).Infof("Removing rule for client MTU discovery from the network load balancer (%s) to instances (%s)", clientCidrs, instanceSecurityGroupID) - glog.V(2).Infof("Removing rule for client traffic from the network load balancer (%s) to instance (%s)", clientCidrs, instanceSecurityGroupID) + klog.V(2).Infof("Removing rule for client MTU discovery from the network load balancer (%s) to instances (%s)", clientCidrs, instanceSecurityGroupID) + klog.V(2).Infof("Removing rule for client traffic from the network load balancer (%s) to instance (%s)", clientCidrs, instanceSecurityGroupID) } - glog.V(2).Infof("Removing rule for health check traffic from the network load balancer (%s) to instance (%s)", clientCidrs, instanceSecurityGroupID) + klog.V(2).Infof("Removing rule for health check traffic from the network load balancer (%s) to instance (%s)", clientCidrs, instanceSecurityGroupID) } if clientTraffic { @@ -717,7 +717,7 @@ func (c *Cloud) updateInstanceSecurityGroupsForNLBTraffic(actualGroups []*ec2.Se return err } if !changed { - glog.Warning("Allowing ingress was not needed; concurrent change? groupId=", instanceSecurityGroupID) + klog.Warning("Allowing ingress was not needed; concurrent change? groupId=", instanceSecurityGroupID) } } @@ -727,7 +727,7 @@ func (c *Cloud) updateInstanceSecurityGroupsForNLBTraffic(actualGroups []*ec2.Se return err } if !changed { - glog.Warning("Revoking ingress was not needed; concurrent change? groupId=", instanceSecurityGroupID) + klog.Warning("Revoking ingress was not needed; concurrent change? groupId=", instanceSecurityGroupID) } } @@ -750,12 +750,12 @@ func (c *Cloud) updateInstanceSecurityGroupsForNLBTraffic(actualGroups []*ec2.Se group, err := c.findSecurityGroup(instanceSecurityGroupID) if err != nil { - glog.Warningf("Error retrieving security group: %q", err) + klog.Warningf("Error retrieving security group: %q", err) return err } if group == nil { - glog.Warning("Security group not found: ", instanceSecurityGroupID) + klog.Warning("Security group not found: ", instanceSecurityGroupID) return nil } @@ -776,21 +776,21 @@ func (c *Cloud) updateInstanceSecurityGroupsForNLBTraffic(actualGroups []*ec2.Se // the icmp permission is missing changed, err := c.addSecurityGroupIngress(instanceSecurityGroupID, []*ec2.IpPermission{mtuPermission}) if err != nil { - glog.Warningf("Error adding MTU permission to security group: %q", err) + klog.Warningf("Error adding MTU permission to security group: %q", err) return err } if !changed { - glog.Warning("Allowing ingress was not needed; concurrent change? groupId=", instanceSecurityGroupID) + klog.Warning("Allowing ingress was not needed; concurrent change? groupId=", instanceSecurityGroupID) } } else if icmpExists && permCount == 0 { // there is no additional permissions, remove icmp changed, err := c.removeSecurityGroupIngress(instanceSecurityGroupID, []*ec2.IpPermission{mtuPermission}) if err != nil { - glog.Warningf("Error removing MTU permission to security group: %q", err) + klog.Warningf("Error removing MTU permission to security group: %q", err) return err } if !changed { - glog.Warning("Revoking ingress was not needed; concurrent change? groupId=", instanceSecurityGroupID) + klog.Warning("Revoking ingress was not needed; concurrent change? groupId=", instanceSecurityGroupID) } } } @@ -869,13 +869,13 @@ func (c *Cloud) updateInstanceSecurityGroupsForNLB(mappings []nlbPortMapping, in } if securityGroup == nil { - glog.Warningf("Ignoring instance without security group: %s", aws.StringValue(instance.InstanceId)) + klog.Warningf("Ignoring instance without security group: %s", aws.StringValue(instance.InstanceId)) continue } id := aws.StringValue(securityGroup.GroupId) if id == "" { - glog.Warningf("found security group without id: %v", securityGroup) + klog.Warningf("found security group without id: %v", securityGroup) continue } @@ -942,7 +942,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala }) } - glog.Infof("Creating load balancer for %v with name: %s", namespacedName, loadBalancerName) + klog.Infof("Creating load balancer for %v with name: %s", namespacedName, loadBalancerName) _, err := c.elb.CreateLoadBalancer(createRequest) if err != nil { return nil, err @@ -955,7 +955,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala } for _, listener := range listeners { - glog.V(2).Infof("Adjusting AWS loadbalancer proxy protocol on node port %d. Setting to true", *listener.InstancePort) + klog.V(2).Infof("Adjusting AWS loadbalancer proxy protocol on node port %d. Setting to true", *listener.InstancePort) err := c.setBackendPolicies(loadBalancerName, *listener.InstancePort, []*string{aws.String(ProxyProtocolPolicyName)}) if err != nil { return nil, err @@ -979,7 +979,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala request := &elb.DetachLoadBalancerFromSubnetsInput{} request.LoadBalancerName = aws.String(loadBalancerName) request.Subnets = stringSetToPointers(removals) - glog.V(2).Info("Detaching load balancer from removed subnets") + klog.V(2).Info("Detaching load balancer from removed subnets") _, err := c.elb.DetachLoadBalancerFromSubnets(request) if err != nil { return nil, fmt.Errorf("error detaching AWS loadbalancer from subnets: %q", err) @@ -991,7 +991,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala request := &elb.AttachLoadBalancerToSubnetsInput{} request.LoadBalancerName = aws.String(loadBalancerName) request.Subnets = stringSetToPointers(additions) - glog.V(2).Info("Attaching load balancer to added subnets") + klog.V(2).Info("Attaching load balancer to added subnets") _, err := c.elb.AttachLoadBalancerToSubnets(request) if err != nil { return nil, fmt.Errorf("error attaching AWS loadbalancer to subnets: %q", err) @@ -1014,7 +1014,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala } else { request.SecurityGroups = aws.StringSlice(securityGroupIDs) } - glog.V(2).Info("Applying updated security groups to load balancer") + klog.V(2).Info("Applying updated security groups to load balancer") _, err := c.elb.ApplySecurityGroupsToLoadBalancer(request) if err != nil { return nil, fmt.Errorf("error applying AWS loadbalancer security groups: %q", err) @@ -1032,7 +1032,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala for _, listenerDescription := range listenerDescriptions { actual := listenerDescription.Listener if actual == nil { - glog.Warning("Ignoring empty listener in AWS loadbalancer: ", loadBalancerName) + klog.Warning("Ignoring empty listener in AWS loadbalancer: ", loadBalancerName) continue } @@ -1074,7 +1074,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala request := &elb.DeleteLoadBalancerListenersInput{} request.LoadBalancerName = aws.String(loadBalancerName) request.LoadBalancerPorts = removals - glog.V(2).Info("Deleting removed load balancer listeners") + klog.V(2).Info("Deleting removed load balancer listeners") _, err := c.elb.DeleteLoadBalancerListeners(request) if err != nil { return nil, fmt.Errorf("error deleting AWS loadbalancer listeners: %q", err) @@ -1086,7 +1086,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala request := &elb.CreateLoadBalancerListenersInput{} request.LoadBalancerName = aws.String(loadBalancerName) request.Listeners = additions - glog.V(2).Info("Creating added load balancer listeners") + klog.V(2).Info("Creating added load balancer listeners") _, err := c.elb.CreateLoadBalancerListeners(request) if err != nil { return nil, fmt.Errorf("error creating AWS loadbalancer listeners: %q", err) @@ -1138,7 +1138,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala } if setPolicy { - glog.V(2).Infof("Adjusting AWS loadbalancer proxy protocol on node port %d. Setting to %t", instancePort, proxyProtocol) + klog.V(2).Infof("Adjusting AWS loadbalancer proxy protocol on node port %d. Setting to %t", instancePort, proxyProtocol) err := c.setBackendPolicies(loadBalancerName, instancePort, proxyPolicies) if err != nil { return nil, err @@ -1152,7 +1152,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala // corresponding listener anymore for instancePort, found := range foundBackends { if !found { - glog.V(2).Infof("Adjusting AWS loadbalancer proxy protocol on node port %d. Setting to false", instancePort) + klog.V(2).Infof("Adjusting AWS loadbalancer proxy protocol on node port %d. Setting to false", instancePort) err := c.setBackendPolicies(loadBalancerName, instancePort, []*string{}) if err != nil { return nil, err @@ -1164,7 +1164,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala { // Add additional tags - glog.V(2).Infof("Creating additional load balancer tags for %s", loadBalancerName) + klog.V(2).Infof("Creating additional load balancer tags for %s", loadBalancerName) tags := getLoadBalancerAdditionalTags(annotations) if len(tags) > 0 { err := c.addLoadBalancerTags(loadBalancerName, tags) @@ -1183,7 +1183,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala describeAttributesRequest.LoadBalancerName = aws.String(loadBalancerName) describeAttributesOutput, err := c.elb.DescribeLoadBalancerAttributes(describeAttributesRequest) if err != nil { - glog.Warning("Unable to retrieve load balancer attributes during attribute sync") + klog.Warning("Unable to retrieve load balancer attributes during attribute sync") return nil, err } @@ -1191,7 +1191,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala // Update attributes if they're dirty if !reflect.DeepEqual(loadBalancerAttributes, foundAttributes) { - glog.V(2).Infof("Updating load-balancer attributes for %q", loadBalancerName) + klog.V(2).Infof("Updating load-balancer attributes for %q", loadBalancerName) modifyAttributesRequest := &elb.ModifyLoadBalancerAttributesInput{} modifyAttributesRequest.LoadBalancerName = aws.String(loadBalancerName) @@ -1207,7 +1207,7 @@ func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBala if dirty { loadBalancer, err = c.describeLoadBalancer(loadBalancerName) if err != nil { - glog.Warning("Unable to retrieve load balancer after creation/update") + klog.Warning("Unable to retrieve load balancer after creation/update") return nil, err } } @@ -1352,7 +1352,7 @@ func (c *Cloud) ensureLoadBalancerInstances(loadBalancerName string, lbInstances if err != nil { return err } - glog.V(1).Infof("Instances added to load-balancer %s", loadBalancerName) + klog.V(1).Infof("Instances added to load-balancer %s", loadBalancerName) } if len(removeInstances) > 0 { @@ -1363,7 +1363,7 @@ func (c *Cloud) ensureLoadBalancerInstances(loadBalancerName string, lbInstances if err != nil { return err } - glog.V(1).Infof("Instances removed from load-balancer %s", loadBalancerName) + klog.V(1).Infof("Instances removed from load-balancer %s", loadBalancerName) } return nil @@ -1382,7 +1382,7 @@ func (c *Cloud) getLoadBalancerTLSPorts(loadBalancer *elb.LoadBalancerDescriptio } func (c *Cloud) ensureSSLNegotiationPolicy(loadBalancer *elb.LoadBalancerDescription, policyName string) error { - glog.V(2).Info("Describing load balancer policies on load balancer") + klog.V(2).Info("Describing load balancer policies on load balancer") result, err := c.elb.DescribeLoadBalancerPolicies(&elb.DescribeLoadBalancerPoliciesInput{ LoadBalancerName: loadBalancer.LoadBalancerName, PolicyNames: []*string{ @@ -1403,7 +1403,7 @@ func (c *Cloud) ensureSSLNegotiationPolicy(loadBalancer *elb.LoadBalancerDescrip return nil } - glog.V(2).Infof("Creating SSL negotiation policy '%s' on load balancer", fmt.Sprintf(SSLNegotiationPolicyNameFormat, policyName)) + klog.V(2).Infof("Creating SSL negotiation policy '%s' on load balancer", fmt.Sprintf(SSLNegotiationPolicyNameFormat, policyName)) // there is an upper limit of 98 policies on an ELB, we're pretty safe from // running into it _, err = c.elb.CreateLoadBalancerPolicy(&elb.CreateLoadBalancerPolicyInput{ @@ -1432,7 +1432,7 @@ func (c *Cloud) setSSLNegotiationPolicy(loadBalancerName, sslPolicyName string, aws.String(policyName), }, } - glog.V(2).Infof("Setting SSL negotiation policy '%s' on load balancer", policyName) + klog.V(2).Infof("Setting SSL negotiation policy '%s' on load balancer", policyName) _, err := c.elb.SetLoadBalancerPoliciesOfListener(request) if err != nil { return fmt.Errorf("error setting SSL negotiation policy '%s' on load balancer: %q", policyName, err) @@ -1452,7 +1452,7 @@ func (c *Cloud) createProxyProtocolPolicy(loadBalancerName string) error { }, }, } - glog.V(2).Info("Creating proxy protocol policy on load balancer") + klog.V(2).Info("Creating proxy protocol policy on load balancer") _, err := c.elb.CreateLoadBalancerPolicy(request) if err != nil { return fmt.Errorf("error creating proxy protocol policy on load balancer: %q", err) @@ -1468,9 +1468,9 @@ func (c *Cloud) setBackendPolicies(loadBalancerName string, instancePort int64, PolicyNames: policies, } if len(policies) > 0 { - glog.V(2).Infof("Adding AWS loadbalancer backend policies on node port %d", instancePort) + klog.V(2).Infof("Adding AWS loadbalancer backend policies on node port %d", instancePort) } else { - glog.V(2).Infof("Removing AWS loadbalancer backend policies on node port %d", instancePort) + klog.V(2).Infof("Removing AWS loadbalancer backend policies on node port %d", instancePort) } _, err := c.elb.SetLoadBalancerPoliciesForBackendServer(request) if err != nil { diff --git a/pkg/cloudprovider/providers/aws/aws_routes.go b/pkg/cloudprovider/providers/aws/aws_routes.go index 658f6e89857..2827596dce4 100644 --- a/pkg/cloudprovider/providers/aws/aws_routes.go +++ b/pkg/cloudprovider/providers/aws/aws_routes.go @@ -22,7 +22,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/golang/glog" + "k8s.io/klog" cloudprovider "k8s.io/cloud-provider" ) @@ -117,7 +117,7 @@ func (c *Cloud) ListRoutes(ctx context.Context, clusterName string) ([]*cloudpro route.TargetNode = mapInstanceToNodeName(instance) routes = append(routes, route) } else { - glog.Warningf("unable to find instance ID %s in the list of instances being routed to", instanceID) + klog.Warningf("unable to find instance ID %s in the list of instances being routed to", instanceID) } } } @@ -172,7 +172,7 @@ func (c *Cloud) CreateRoute(ctx context.Context, clusterName string, nameHint st } if deleteRoute != nil { - glog.Infof("deleting blackholed route: %s", aws.StringValue(deleteRoute.DestinationCidrBlock)) + klog.Infof("deleting blackholed route: %s", aws.StringValue(deleteRoute.DestinationCidrBlock)) request := &ec2.DeleteRouteInput{} request.DestinationCidrBlock = deleteRoute.DestinationCidrBlock diff --git a/pkg/cloudprovider/providers/aws/instances.go b/pkg/cloudprovider/providers/aws/instances.go index c0b03a5922c..60b10abc6a8 100644 --- a/pkg/cloudprovider/providers/aws/instances.go +++ b/pkg/cloudprovider/providers/aws/instances.go @@ -26,7 +26,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" ) @@ -110,12 +110,12 @@ func mapToAWSInstanceIDsTolerant(nodes []*v1.Node) []awsInstanceID { var instanceIDs []awsInstanceID for _, node := range nodes { if node.Spec.ProviderID == "" { - glog.Warningf("node %q did not have ProviderID set", node.Name) + klog.Warningf("node %q did not have ProviderID set", node.Name) continue } instanceID, err := kubernetesInstanceID(node.Spec.ProviderID).mapToAWSInstanceID() if err != nil { - glog.Warningf("unable to parse ProviderID %q for node %q", node.Spec.ProviderID, node.Name) + klog.Warningf("unable to parse ProviderID %q for node %q", node.Spec.ProviderID, node.Name) continue } instanceIDs = append(instanceIDs, instanceID) @@ -156,7 +156,7 @@ type instanceCache struct { func (c *instanceCache) describeAllInstancesUncached() (*allInstancesSnapshot, error) { now := time.Now() - glog.V(4).Infof("EC2 DescribeInstances - fetching all instances") + klog.V(4).Infof("EC2 DescribeInstances - fetching all instances") filters := []*ec2.Filter{} instances, err := c.cloud.describeInstances(filters) @@ -177,7 +177,7 @@ func (c *instanceCache) describeAllInstancesUncached() (*allInstancesSnapshot, e if c.snapshot != nil && snapshot.olderThan(c.snapshot) { // If this happens a lot, we could run this function in a mutex and only return one result - glog.Infof("Not caching concurrent AWS DescribeInstances results") + klog.Infof("Not caching concurrent AWS DescribeInstances results") } else { c.snapshot = snapshot } @@ -210,7 +210,7 @@ func (c *instanceCache) describeAllInstancesCached(criteria cacheCriteria) (*all return nil, err } } else { - glog.V(6).Infof("EC2 DescribeInstances - using cached results") + klog.V(6).Infof("EC2 DescribeInstances - using cached results") } return snapshot, nil @@ -236,7 +236,7 @@ func (s *allInstancesSnapshot) MeetsCriteria(criteria cacheCriteria) bool { // Sub() is technically broken by time changes until we have monotonic time now := time.Now() if now.Sub(s.timestamp) > criteria.MaxAge { - glog.V(6).Infof("instanceCache snapshot cannot be used as is older than MaxAge=%s", criteria.MaxAge) + klog.V(6).Infof("instanceCache snapshot cannot be used as is older than MaxAge=%s", criteria.MaxAge) return false } } @@ -244,7 +244,7 @@ func (s *allInstancesSnapshot) MeetsCriteria(criteria cacheCriteria) bool { if len(criteria.HasInstances) != 0 { for _, id := range criteria.HasInstances { if nil == s.instances[id] { - glog.V(6).Infof("instanceCache snapshot cannot be used as does not contain instance %s", id) + klog.V(6).Infof("instanceCache snapshot cannot be used as does not contain instance %s", id) return false } } diff --git a/pkg/cloudprovider/providers/aws/log_handler.go b/pkg/cloudprovider/providers/aws/log_handler.go index 86aa30628db..9328fd284ac 100644 --- a/pkg/cloudprovider/providers/aws/log_handler.go +++ b/pkg/cloudprovider/providers/aws/log_handler.go @@ -18,23 +18,23 @@ package aws import ( "github.com/aws/aws-sdk-go/aws/request" - "github.com/golang/glog" + "k8s.io/klog" ) // Handler for aws-sdk-go that logs all requests func awsHandlerLogger(req *request.Request) { service, name := awsServiceAndName(req) - glog.V(4).Infof("AWS request: %s %s", service, name) + klog.V(4).Infof("AWS request: %s %s", service, name) } func awsSendHandlerLogger(req *request.Request) { service, name := awsServiceAndName(req) - glog.V(4).Infof("AWS API Send: %s %s %v %v", service, name, req.Operation, req.Params) + klog.V(4).Infof("AWS API Send: %s %s %v %v", service, name, req.Operation, req.Params) } func awsValidateResponseHandlerLogger(req *request.Request) { service, name := awsServiceAndName(req) - glog.V(4).Infof("AWS API ValidateResponse: %s %s %v %v %s", service, name, req.Operation, req.Params, req.HTTPResponse.Status) + klog.V(4).Infof("AWS API ValidateResponse: %s %s %v %v %s", service, name, req.Operation, req.Params, req.HTTPResponse.Status) } func awsServiceAndName(req *request.Request) (string, string) { diff --git a/pkg/cloudprovider/providers/aws/regions.go b/pkg/cloudprovider/providers/aws/regions.go index 74d64c68de0..f19bab6eb55 100644 --- a/pkg/cloudprovider/providers/aws/regions.go +++ b/pkg/cloudprovider/providers/aws/regions.go @@ -19,7 +19,7 @@ package aws import ( "sync" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/sets" awscredentialprovider "k8s.io/kubernetes/pkg/credentialprovider/aws" @@ -72,11 +72,11 @@ func recognizeRegion(region string) { } if awsRegions.Has(region) { - glog.V(6).Infof("found AWS region %q again - ignoring", region) + klog.V(6).Infof("found AWS region %q again - ignoring", region) return } - glog.V(4).Infof("found AWS region %q", region) + klog.V(4).Infof("found AWS region %q", region) awscredentialprovider.RegisterCredentialsProvider(region) diff --git a/pkg/cloudprovider/providers/aws/retry_handler.go b/pkg/cloudprovider/providers/aws/retry_handler.go index d6b382ccc35..0fe6c2a5753 100644 --- a/pkg/cloudprovider/providers/aws/retry_handler.go +++ b/pkg/cloudprovider/providers/aws/retry_handler.go @@ -24,7 +24,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/request" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -52,7 +52,7 @@ func (c *CrossRequestRetryDelay) BeforeSign(r *request.Request) { now := time.Now() delay := c.backoff.ComputeDelayForRequest(now) if delay > 0 { - glog.Warningf("Inserting delay before AWS request (%s) to avoid RequestLimitExceeded: %s", + klog.Warningf("Inserting delay before AWS request (%s) to avoid RequestLimitExceeded: %s", describeRequest(r), delay.String()) if sleepFn := r.Config.SleepDelay; sleepFn != nil { @@ -96,7 +96,7 @@ func (c *CrossRequestRetryDelay) AfterRetry(r *request.Request) { if awsError.Code() == "RequestLimitExceeded" { c.backoff.ReportError() recordAWSThrottlesMetric(operationName(r)) - glog.Warningf("Got RequestLimitExceeded error on AWS request (%s)", + klog.Warningf("Got RequestLimitExceeded error on AWS request (%s)", describeRequest(r)) } } diff --git a/pkg/cloudprovider/providers/aws/tags.go b/pkg/cloudprovider/providers/aws/tags.go index 5773983c7a3..de6cad543e3 100644 --- a/pkg/cloudprovider/providers/aws/tags.go +++ b/pkg/cloudprovider/providers/aws/tags.go @@ -22,7 +22,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/wait" ) @@ -74,7 +74,7 @@ func (t *awsTagging) init(legacyClusterID string, clusterID string) error { t.ClusterID = clusterID if clusterID != "" { - glog.Infof("AWS cloud filtering on ClusterID: %v", clusterID) + klog.Infof("AWS cloud filtering on ClusterID: %v", clusterID) } else { return fmt.Errorf("AWS cloud failed to find ClusterID") } @@ -92,7 +92,7 @@ func (t *awsTagging) initFromTags(tags []*ec2.Tag) error { } if legacyClusterID == "" && newClusterID == "" { - glog.Errorf("Tag %q nor %q not found; Kubernetes may behave unexpectedly.", TagNameKubernetesClusterLegacy, TagNameKubernetesClusterPrefix+"...") + klog.Errorf("Tag %q nor %q not found; Kubernetes may behave unexpectedly.", TagNameKubernetesClusterLegacy, TagNameKubernetesClusterPrefix+"...") } return t.init(legacyClusterID, newClusterID) @@ -168,7 +168,7 @@ func (t *awsTagging) readRepairClusterTags(client EC2, resourceID string, lifecy continue } if actual == "" { - glog.Warningf("Resource %q was missing expected cluster tag %q. Will add (with value %q)", resourceID, k, expected) + klog.Warningf("Resource %q was missing expected cluster tag %q. Will add (with value %q)", resourceID, k, expected) addTags[k] = expected } else { return fmt.Errorf("resource %q has tag belonging to another cluster: %q=%q (expected %q)", resourceID, k, actual, expected) @@ -223,7 +223,7 @@ func (t *awsTagging) createTags(client EC2, resourceID string, lifecycle Resourc // We could check that the error is retryable, but the error code changes based on what we are tagging // SecurityGroup: InvalidGroup.NotFound - glog.V(2).Infof("Failed to create tags; will retry. Error was %q", err) + klog.V(2).Infof("Failed to create tags; will retry. Error was %q", err) lastErr = err return false, nil }) diff --git a/pkg/cloudprovider/providers/aws/volumes.go b/pkg/cloudprovider/providers/aws/volumes.go index d7c046c5cd0..7031c5a52c1 100644 --- a/pkg/cloudprovider/providers/aws/volumes.go +++ b/pkg/cloudprovider/providers/aws/volumes.go @@ -24,7 +24,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/types" ) @@ -121,7 +121,7 @@ func (c *Cloud) checkIfAttachedToNode(diskName KubernetesVolumeID, nodeName type info, err := disk.describeVolume() if err != nil { - glog.Warningf("Error describing volume %s with %v", diskName, err) + klog.Warningf("Error describing volume %s with %v", diskName, err) awsDiskInfo.volumeState = "unknown" return awsDiskInfo, false, err } @@ -138,7 +138,7 @@ func (c *Cloud) checkIfAttachedToNode(diskName KubernetesVolumeID, nodeName type // has been deleted if err != nil { fetchErr := fmt.Errorf("Error fetching instance %s for volume %s", instanceID, diskName) - glog.Warning(fetchErr) + klog.Warning(fetchErr) return awsDiskInfo, false, fetchErr } diff --git a/pkg/cloudprovider/providers/azure/BUILD b/pkg/cloudprovider/providers/azure/BUILD index 1ead263abda..d4116a63893 100644 --- a/pkg/cloudprovider/providers/azure/BUILD +++ b/pkg/cloudprovider/providers/azure/BUILD @@ -66,10 +66,10 @@ go_library( "//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/github.com/rubiojr/go-vhd/vhd:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/pkg/cloudprovider/providers/azure/auth/BUILD b/pkg/cloudprovider/providers/azure/auth/BUILD index cc733d385aa..a0eca412154 100644 --- a/pkg/cloudprovider/providers/azure/auth/BUILD +++ b/pkg/cloudprovider/providers/azure/auth/BUILD @@ -8,8 +8,8 @@ go_library( deps = [ "//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/crypto/pkcs12:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/cloudprovider/providers/azure/auth/azure_auth.go b/pkg/cloudprovider/providers/azure/auth/azure_auth.go index 08f894d7216..6a651eb05c0 100644 --- a/pkg/cloudprovider/providers/azure/auth/azure_auth.go +++ b/pkg/cloudprovider/providers/azure/auth/azure_auth.go @@ -24,8 +24,8 @@ import ( "github.com/Azure/go-autorest/autorest/adal" "github.com/Azure/go-autorest/autorest/azure" - "github.com/golang/glog" "golang.org/x/crypto/pkcs12" + "k8s.io/klog" ) // AzureAuthConfig holds auth related part of cloud config @@ -55,18 +55,18 @@ type AzureAuthConfig struct { // GetServicePrincipalToken creates a new service principal token based on the configuration func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.ServicePrincipalToken, error) { if config.UseManagedIdentityExtension { - glog.V(2).Infoln("azure: using managed identity extension to retrieve access token") + klog.V(2).Infoln("azure: using managed identity extension to retrieve access token") msiEndpoint, err := adal.GetMSIVMEndpoint() if err != nil { return nil, fmt.Errorf("Getting the managed service identity endpoint: %v", err) } if len(config.UserAssignedIdentityID) > 0 { - glog.V(4).Info("azure: using User Assigned MSI ID to retrieve access token") + klog.V(4).Info("azure: using User Assigned MSI ID to retrieve access token") return adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, env.ServiceManagementEndpoint, config.UserAssignedIdentityID) } - glog.V(4).Info("azure: using System Assigned MSI to retrieve access token") + klog.V(4).Info("azure: using System Assigned MSI to retrieve access token") return adal.NewServicePrincipalTokenFromMSI( msiEndpoint, env.ServiceManagementEndpoint) @@ -78,7 +78,7 @@ func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) ( } if len(config.AADClientSecret) > 0 { - glog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token") + klog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token") return adal.NewServicePrincipalToken( *oauthConfig, config.AADClientID, @@ -87,7 +87,7 @@ func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) ( } if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 { - glog.V(2).Infoln("azure: using jwt client_assertion (client_cert+client_private_key) to retrieve access token") + klog.V(2).Infoln("azure: using jwt client_assertion (client_cert+client_private_key) to retrieve access token") certData, err := ioutil.ReadFile(config.AADClientCertPath) if err != nil { return nil, fmt.Errorf("reading the client certificate from file %s: %v", config.AADClientCertPath, err) diff --git a/pkg/cloudprovider/providers/azure/azure.go b/pkg/cloudprovider/providers/azure/azure.go index 01e07593797..e2205c37fbe 100644 --- a/pkg/cloudprovider/providers/azure/azure.go +++ b/pkg/cloudprovider/providers/azure/azure.go @@ -41,8 +41,8 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure" - "github.com/ghodss/yaml" - "github.com/golang/glog" + "k8s.io/klog" + "sigs.k8s.io/yaml" ) const ( @@ -255,11 +255,11 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) { config.CloudProviderRateLimitQPSWrite, config.CloudProviderRateLimitBucketWrite) - glog.V(2).Infof("Azure cloudprovider (read ops) using rate limit config: QPS=%g, bucket=%d", + klog.V(2).Infof("Azure cloudprovider (read ops) using rate limit config: QPS=%g, bucket=%d", config.CloudProviderRateLimitQPS, config.CloudProviderRateLimitBucket) - glog.V(2).Infof("Azure cloudprovider (write ops) using rate limit config: QPS=%g, bucket=%d", + klog.V(2).Infof("Azure cloudprovider (write ops) using rate limit config: QPS=%g, bucket=%d", config.CloudProviderRateLimitQPSWrite, config.CloudProviderRateLimitBucketWrite) } @@ -321,7 +321,7 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) { Duration: time.Duration(az.CloudProviderBackoffDuration) * time.Second, Jitter: az.CloudProviderBackoffJitter, } - glog.V(2).Infof("Azure cloudprovider using try backoff: retries=%d, exponent=%f, duration=%d, jitter=%f", + klog.V(2).Infof("Azure cloudprovider using try backoff: retries=%d, exponent=%f, duration=%d, jitter=%f", az.CloudProviderBackoffRetries, az.CloudProviderBackoffExponent, az.CloudProviderBackoffDuration, @@ -479,7 +479,7 @@ func initDiskControllers(az *Cloud) error { // SetInformers sets informers for Azure cloud provider. func (az *Cloud) SetInformers(informerFactory informers.SharedInformerFactory) { - glog.Infof("Setting up informers for Azure cloud provider") + klog.Infof("Setting up informers for Azure cloud provider") nodeInformer := informerFactory.Core().V1().Nodes().Informer() nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { @@ -502,12 +502,12 @@ func (az *Cloud) SetInformers(informerFactory informers.SharedInformerFactory) { if !isNode { deletedState, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Received unexpected object: %v", obj) + klog.Errorf("Received unexpected object: %v", obj) return } node, ok = deletedState.Obj.(*v1.Node) if !ok { - glog.Errorf("DeletedFinalStateUnknown contained non-Node object: %v", deletedState.Obj) + klog.Errorf("DeletedFinalStateUnknown contained non-Node object: %v", deletedState.Obj) return } } diff --git a/pkg/cloudprovider/providers/azure/azure_backoff.go b/pkg/cloudprovider/providers/azure/azure_backoff.go index ab268d311ea..c0a6f4a34ee 100644 --- a/pkg/cloudprovider/providers/azure/azure_backoff.go +++ b/pkg/cloudprovider/providers/azure/azure_backoff.go @@ -23,7 +23,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -64,10 +64,10 @@ func (az *Cloud) GetVirtualMachineWithRetry(name types.NodeName) (compute.Virtua return true, cloudprovider.InstanceNotFound } if retryErr != nil { - glog.Errorf("GetVirtualMachineWithRetry(%s): backoff failure, will retry, err=%v", name, retryErr) + klog.Errorf("GetVirtualMachineWithRetry(%s): backoff failure, will retry, err=%v", name, retryErr) return false, nil } - glog.V(2).Infof("GetVirtualMachineWithRetry(%s): backoff success", name) + klog.V(2).Infof("GetVirtualMachineWithRetry(%s): backoff success", name) return true, nil }) if err == wait.ErrWaitTimeout { @@ -86,12 +86,12 @@ func (az *Cloud) VirtualMachineClientListWithRetry(resourceGroup string) ([]comp defer cancel() allNodes, retryErr = az.VirtualMachinesClient.List(ctx, resourceGroup) if retryErr != nil { - glog.Errorf("VirtualMachinesClient.List(%v) - backoff: failure, will retry,err=%v", + klog.Errorf("VirtualMachinesClient.List(%v) - backoff: failure, will retry,err=%v", resourceGroup, retryErr) return false, retryErr } - glog.V(2).Infof("VirtualMachinesClient.List(%v) - backoff: success", resourceGroup) + klog.V(2).Infof("VirtualMachinesClient.List(%v) - backoff: success", resourceGroup) return true, nil }) if err != nil { @@ -108,10 +108,10 @@ func (az *Cloud) GetIPForMachineWithRetry(name types.NodeName) (string, string, var retryErr error ip, publicIP, retryErr = az.getIPForMachine(name) if retryErr != nil { - glog.Errorf("GetIPForMachineWithRetry(%s): backoff failure, will retry,err=%v", name, retryErr) + klog.Errorf("GetIPForMachineWithRetry(%s): backoff failure, will retry,err=%v", name, retryErr) return false, nil } - glog.V(2).Infof("GetIPForMachineWithRetry(%s): backoff success", name) + klog.V(2).Infof("GetIPForMachineWithRetry(%s): backoff success", name) return true, nil }) return ip, publicIP, err @@ -124,7 +124,7 @@ func (az *Cloud) CreateOrUpdateSGWithRetry(service *v1.Service, sg network.Secur defer cancel() resp, err := az.SecurityGroupsClient.CreateOrUpdate(ctx, az.ResourceGroup, *sg.Name, sg) - glog.V(10).Infof("SecurityGroupsClient.CreateOrUpdate(%s): end", *sg.Name) + klog.V(10).Infof("SecurityGroupsClient.CreateOrUpdate(%s): end", *sg.Name) done, err := az.processHTTPRetryResponse(service, "CreateOrUpdateSecurityGroup", resp, err) if done && err == nil { // Invalidate the cache right after updating @@ -141,7 +141,7 @@ func (az *Cloud) CreateOrUpdateLBWithRetry(service *v1.Service, lb network.LoadB defer cancel() resp, err := az.LoadBalancerClient.CreateOrUpdate(ctx, az.ResourceGroup, *lb.Name, lb) - glog.V(10).Infof("LoadBalancerClient.CreateOrUpdate(%s): end", *lb.Name) + klog.V(10).Infof("LoadBalancerClient.CreateOrUpdate(%s): end", *lb.Name) done, err := az.processHTTPRetryResponse(service, "CreateOrUpdateLoadBalancer", resp, err) if done && err == nil { // Invalidate the cache right after updating @@ -163,12 +163,12 @@ func (az *Cloud) ListLBWithRetry(service *v1.Service) ([]network.LoadBalancer, e allLBs, retryErr = az.LoadBalancerClient.List(ctx, az.ResourceGroup) if retryErr != nil { az.Event(service, v1.EventTypeWarning, "ListLoadBalancers", retryErr.Error()) - glog.Errorf("LoadBalancerClient.List(%v) - backoff: failure, will retry,err=%v", + klog.Errorf("LoadBalancerClient.List(%v) - backoff: failure, will retry,err=%v", az.ResourceGroup, retryErr) return false, retryErr } - glog.V(2).Infof("LoadBalancerClient.List(%v) - backoff: success", az.ResourceGroup) + klog.V(2).Infof("LoadBalancerClient.List(%v) - backoff: success", az.ResourceGroup) return true, nil }) if err != nil { @@ -190,12 +190,12 @@ func (az *Cloud) ListPIPWithRetry(service *v1.Service, pipResourceGroup string) allPIPs, retryErr = az.PublicIPAddressesClient.List(ctx, pipResourceGroup) if retryErr != nil { az.Event(service, v1.EventTypeWarning, "ListPublicIPs", retryErr.Error()) - glog.Errorf("PublicIPAddressesClient.List(%v) - backoff: failure, will retry,err=%v", + klog.Errorf("PublicIPAddressesClient.List(%v) - backoff: failure, will retry,err=%v", pipResourceGroup, retryErr) return false, retryErr } - glog.V(2).Infof("PublicIPAddressesClient.List(%v) - backoff: success", pipResourceGroup) + klog.V(2).Infof("PublicIPAddressesClient.List(%v) - backoff: success", pipResourceGroup) return true, nil }) if err != nil { @@ -212,7 +212,7 @@ func (az *Cloud) CreateOrUpdatePIPWithRetry(service *v1.Service, pipResourceGrou defer cancel() resp, err := az.PublicIPAddressesClient.CreateOrUpdate(ctx, pipResourceGroup, *pip.Name, pip) - glog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s, %s): end", pipResourceGroup, *pip.Name) + klog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s, %s): end", pipResourceGroup, *pip.Name) return az.processHTTPRetryResponse(service, "CreateOrUpdatePublicIPAddress", resp, err) }) } @@ -224,7 +224,7 @@ func (az *Cloud) CreateOrUpdateInterfaceWithRetry(service *v1.Service, nic netwo defer cancel() resp, err := az.InterfacesClient.CreateOrUpdate(ctx, az.ResourceGroup, *nic.Name, nic) - glog.V(10).Infof("InterfacesClient.CreateOrUpdate(%s): end", *nic.Name) + klog.V(10).Infof("InterfacesClient.CreateOrUpdate(%s): end", *nic.Name) return az.processHTTPRetryResponse(service, "CreateOrUpdateInterface", resp, err) }) } @@ -274,7 +274,7 @@ func (az *Cloud) CreateOrUpdateRouteWithRetry(route network.Route) error { defer cancel() resp, err := az.RoutesClient.CreateOrUpdate(ctx, az.ResourceGroup, az.RouteTableName, *route.Name, route) - glog.V(10).Infof("RoutesClient.CreateOrUpdate(%s): end", *route.Name) + klog.V(10).Infof("RoutesClient.CreateOrUpdate(%s): end", *route.Name) return az.processHTTPRetryResponse(nil, "", resp, err) }) } @@ -286,7 +286,7 @@ func (az *Cloud) DeleteRouteWithRetry(routeName string) error { defer cancel() resp, err := az.RoutesClient.Delete(ctx, az.ResourceGroup, az.RouteTableName, routeName) - glog.V(10).Infof("RoutesClient.Delete(%s): end", az.RouteTableName) + klog.V(10).Infof("RoutesClient.Delete(%s): end", az.RouteTableName) return az.processHTTPRetryResponse(nil, "", resp, err) }) } @@ -298,7 +298,7 @@ func (az *Cloud) CreateOrUpdateVMWithRetry(resourceGroup, vmName string, newVM c defer cancel() resp, err := az.VirtualMachinesClient.CreateOrUpdate(ctx, resourceGroup, vmName, newVM) - glog.V(10).Infof("VirtualMachinesClient.CreateOrUpdate(%s): end", vmName) + klog.V(10).Infof("VirtualMachinesClient.CreateOrUpdate(%s): end", vmName) return az.processHTTPRetryResponse(nil, "", resp, err) }) } @@ -307,7 +307,7 @@ func (az *Cloud) CreateOrUpdateVMWithRetry(resourceGroup, vmName string, newVM c func (az *Cloud) UpdateVmssVMWithRetry(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string, parameters compute.VirtualMachineScaleSetVM) error { return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) { resp, err := az.VirtualMachineScaleSetVMsClient.Update(ctx, resourceGroupName, VMScaleSetName, instanceID, parameters) - glog.V(10).Infof("VirtualMachinesClient.CreateOrUpdate(%s,%s): end", VMScaleSetName, instanceID) + klog.V(10).Infof("VirtualMachinesClient.CreateOrUpdate(%s,%s): end", VMScaleSetName, instanceID) return az.processHTTPRetryResponse(nil, "", resp, err) }) } @@ -345,10 +345,10 @@ func (az *Cloud) processHTTPRetryResponse(service *v1.Service, reason string, re if shouldRetryHTTPRequest(resp, err) { if err != nil { az.Event(service, v1.EventTypeWarning, reason, err.Error()) - glog.Errorf("processHTTPRetryResponse: backoff failure, will retry, err=%v", err) + klog.Errorf("processHTTPRetryResponse: backoff failure, will retry, err=%v", err) } else { az.Event(service, v1.EventTypeWarning, reason, fmt.Sprintf("Azure HTTP response %d", resp.StatusCode)) - glog.Errorf("processHTTPRetryResponse: backoff failure, will retry, HTTP response=%d", resp.StatusCode) + klog.Errorf("processHTTPRetryResponse: backoff failure, will retry, HTTP response=%d", resp.StatusCode) } // suppress the error object so that backoff process continues diff --git a/pkg/cloudprovider/providers/azure/azure_blobDiskController.go b/pkg/cloudprovider/providers/azure/azure_blobDiskController.go index 6ecc8bf4437..9d0be714894 100644 --- a/pkg/cloudprovider/providers/azure/azure_blobDiskController.go +++ b/pkg/cloudprovider/providers/azure/azure_blobDiskController.go @@ -30,8 +30,8 @@ import ( "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage" azstorage "github.com/Azure/azure-sdk-for-go/storage" "github.com/Azure/go-autorest/autorest/to" - "github.com/golang/glog" "github.com/rubiojr/go-vhd/vhd" + "k8s.io/klog" kwait "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/pkg/volume" @@ -68,7 +68,7 @@ func newBlobDiskController(common *controllerCommon) (*BlobDiskController, error // get accounts accounts, err := c.getAllStorageAccounts() if err != nil { - glog.Errorf("azureDisk - getAllStorageAccounts error: %v", err) + klog.Errorf("azureDisk - getAllStorageAccounts error: %v", err) c.accounts = make(map[string]*storageAccountState) return &c, nil } @@ -97,13 +97,13 @@ func (c *BlobDiskController) CreateVolume(blobName, accountName, accountType, lo return "", "", 0, err } - glog.V(4).Infof("azureDisk - created vhd blob uri: %s", diskURI) + klog.V(4).Infof("azureDisk - created vhd blob uri: %s", diskURI) return diskName, diskURI, requestGB, err } // DeleteVolume deletes a VHD blob func (c *BlobDiskController) DeleteVolume(diskURI string) error { - glog.V(4).Infof("azureDisk - begin to delete volume %s", diskURI) + klog.V(4).Infof("azureDisk - begin to delete volume %s", diskURI) accountName, blob, err := c.common.cloud.getBlobNameAndAccountFromURI(diskURI) if err != nil { return fmt.Errorf("failed to parse vhd URI %v", err) @@ -114,7 +114,7 @@ func (c *BlobDiskController) DeleteVolume(diskURI string) error { } err = c.common.cloud.deleteVhdBlob(accountName, key, blob) if err != nil { - glog.Warningf("azureDisk - failed to delete blob %s err: %v", diskURI, err) + klog.Warningf("azureDisk - failed to delete blob %s err: %v", diskURI, err) detail := err.Error() if strings.Contains(detail, errLeaseIDMissing) { // disk is still being used @@ -123,7 +123,7 @@ func (c *BlobDiskController) DeleteVolume(diskURI string) error { } return fmt.Errorf("failed to delete vhd %v, account %s, blob %s, err: %v", diskURI, accountName, blob, err) } - glog.V(4).Infof("azureDisk - blob %s deleted", diskURI) + klog.V(4).Infof("azureDisk - blob %s deleted", diskURI) return nil } @@ -153,7 +153,7 @@ func (c *BlobDiskController) createVHDBlobDisk(blobClient azstorage.BlobStorageC tags := make(map[string]string) tags["createdby"] = "k8sAzureDataDisk" - glog.V(4).Infof("azureDisk - creating page blob %s in container %s account %s", vhdName, containerName, accountName) + klog.V(4).Infof("azureDisk - creating page blob %s in container %s account %s", vhdName, containerName, accountName) blob := container.GetBlobReference(vhdName) blob.Properties.ContentLength = vhdSize @@ -185,7 +185,7 @@ func (c *BlobDiskController) createVHDBlobDisk(blobClient azstorage.BlobStorageC End: uint64(vhdSize - 1), } if err = blob.WriteRange(blobRange, bytes.NewBuffer(h[:vhd.VHD_HEADER_SIZE]), nil); err != nil { - glog.Infof("azureDisk - failed to put header page for data disk %s in container %s account %s, error was %s\n", + klog.Infof("azureDisk - failed to put header page for data disk %s in container %s account %s, error was %s\n", vhdName, containerName, accountName, err.Error()) return "", "", err } @@ -215,7 +215,7 @@ func (c *BlobDiskController) deleteVhdBlob(accountName, accountKey, blobName str //CreateBlobDisk : create a blob disk in a node func (c *BlobDiskController) CreateBlobDisk(dataDiskName string, storageAccountType storage.SkuName, sizeGB int) (string, error) { - glog.V(4).Infof("azureDisk - creating blob data disk named:%s on StorageAccountType:%s", dataDiskName, storageAccountType) + klog.V(4).Infof("azureDisk - creating blob data disk named:%s on StorageAccountType:%s", dataDiskName, storageAccountType) storageAccountName, err := c.findSANameForDisk(storageAccountType) if err != nil { @@ -247,7 +247,7 @@ func (c *BlobDiskController) DeleteBlobDisk(diskURI string) error { _, ok := c.accounts[storageAccountName] if !ok { // the storage account is specified by user - glog.V(4).Infof("azureDisk - deleting volume %s", diskURI) + klog.V(4).Infof("azureDisk - deleting volume %s", diskURI) return c.DeleteVolume(diskURI) } @@ -256,7 +256,7 @@ func (c *BlobDiskController) DeleteBlobDisk(diskURI string) error { return err } - glog.V(4).Infof("azureDisk - About to delete vhd file %s on storage account %s container %s", vhdName, storageAccountName, vhdContainerName) + klog.V(4).Infof("azureDisk - About to delete vhd file %s on storage account %s container %s", vhdName, storageAccountName, vhdContainerName) container := blobSvc.GetContainerReference(vhdContainerName) blob := container.GetBlobReference(vhdName) @@ -266,7 +266,7 @@ func (c *BlobDiskController) DeleteBlobDisk(diskURI string) error { if diskCount, err := c.getDiskCount(storageAccountName); err != nil { c.accounts[storageAccountName].diskCount = int32(diskCount) } else { - glog.Warningf("azureDisk - failed to get disk count for %s however the delete disk operation was ok", storageAccountName) + klog.Warningf("azureDisk - failed to get disk count for %s however the delete disk operation was ok", storageAccountName) return nil // we have failed to acquire a new count. not an error condition } } @@ -291,7 +291,7 @@ func (c *BlobDiskController) getStorageAccountKey(SAName string) (string, error) for _, v := range *listKeysResult.Keys { if v.Value != nil && *v.Value == "key1" { if _, ok := c.accounts[SAName]; !ok { - glog.Warningf("azureDisk - account %s was not cached while getting keys", SAName) + klog.Warningf("azureDisk - account %s was not cached while getting keys", SAName) return *v.Value, nil } } @@ -366,7 +366,7 @@ func (c *BlobDiskController) ensureDefaultContainer(storageAccountName string) e _, provisionState, err := c.getStorageAccountState(storageAccountName) if err != nil { - glog.V(4).Infof("azureDisk - GetStorageAccount:%s err %s", storageAccountName, err.Error()) + klog.V(4).Infof("azureDisk - GetStorageAccount:%s err %s", storageAccountName, err.Error()) return false, nil // error performing the query - retryable } @@ -374,7 +374,7 @@ func (c *BlobDiskController) ensureDefaultContainer(storageAccountName string) e return true, nil } - glog.V(4).Infof("azureDisk - GetStorageAccount:%s not ready yet (not flagged Succeeded by ARM)", storageAccountName) + klog.V(4).Infof("azureDisk - GetStorageAccount:%s not ready yet (not flagged Succeeded by ARM)", storageAccountName) return false, nil // back off and see if the account becomes ready on next retry }) // we have failed to ensure that account is ready for us to create @@ -397,7 +397,7 @@ func (c *BlobDiskController) ensureDefaultContainer(storageAccountName string) e return err } if bCreated { - glog.V(2).Infof("azureDisk - storage account:%s had no default container(%s) and it was created \n", storageAccountName, vhdContainerName) + klog.V(2).Infof("azureDisk - storage account:%s had no default container(%s) and it was created \n", storageAccountName, vhdContainerName) } // flag so we no longer have to check on ARM @@ -429,7 +429,7 @@ func (c *BlobDiskController) getDiskCount(SAName string) (int, error) { if err != nil { return 0, err } - glog.V(4).Infof("azure-Disk - refreshed data count for account %s and found %v", SAName, len(response.Blobs)) + klog.V(4).Infof("azure-Disk - refreshed data count for account %s and found %v", SAName, len(response.Blobs)) c.accounts[SAName].diskCount = int32(len(response.Blobs)) return int(c.accounts[SAName].diskCount), nil @@ -449,13 +449,13 @@ func (c *BlobDiskController) getAllStorageAccounts() (map[string]*storageAccount accounts := make(map[string]*storageAccountState) for _, v := range *accountListResult.Value { if v.Name == nil || v.Sku == nil { - glog.Info("azureDisk - accountListResult Name or Sku is nil") + klog.Info("azureDisk - accountListResult Name or Sku is nil") continue } if !strings.HasPrefix(*v.Name, sharedDiskAccountNamePrefix) { continue } - glog.Infof("azureDisk - identified account %s as part of shared PVC accounts", *v.Name) + klog.Infof("azureDisk - identified account %s as part of shared PVC accounts", *v.Name) sastate := &storageAccountState{ name: *v.Name, @@ -486,7 +486,7 @@ func (c *BlobDiskController) createStorageAccount(storageAccountName string, sto return fmt.Errorf("azureDisk - can not create new storage account, current storage accounts count:%v Max is:%v", len(c.accounts), maxStorageAccounts) } - glog.V(2).Infof("azureDisk - Creating storage account %s type %s", storageAccountName, string(storageAccountType)) + klog.V(2).Infof("azureDisk - Creating storage account %s type %s", storageAccountName, string(storageAccountType)) cp := storage.AccountCreateParameters{ Sku: &storage.Sku{Name: storageAccountType}, @@ -542,7 +542,7 @@ func (c *BlobDiskController) findSANameForDisk(storageAccountType storage.SkuNam countAccounts = countAccounts + 1 // empty account if dCount == 0 { - glog.V(2).Infof("azureDisk - account %s identified for a new disk is because it has 0 allocated disks", v.name) + klog.V(2).Infof("azureDisk - account %s identified for a new disk is because it has 0 allocated disks", v.name) return v.name, nil // short circuit, avg is good and no need to adjust } // if this account is less allocated @@ -555,7 +555,7 @@ func (c *BlobDiskController) findSANameForDisk(storageAccountType storage.SkuNam // if we failed to find storageaccount if SAName == "" { - glog.V(2).Infof("azureDisk - failed to identify a suitable account for new disk and will attempt to create new account") + klog.V(2).Infof("azureDisk - failed to identify a suitable account for new disk and will attempt to create new account") SAName = generateStorageAccountName(sharedDiskAccountNamePrefix) err := c.createStorageAccount(SAName, storageAccountType, c.common.location, true) if err != nil { @@ -571,7 +571,7 @@ func (c *BlobDiskController) findSANameForDisk(storageAccountType storage.SkuNam // avg are not create and we should create more accounts if we can if aboveAvg && countAccounts < maxStorageAccounts { - glog.V(2).Infof("azureDisk - shared storageAccounts utilization(%v) > grow-at-avg-utilization (%v). New storage account will be created", avgUtilization, storageAccountUtilizationBeforeGrowing) + klog.V(2).Infof("azureDisk - shared storageAccounts utilization(%v) > grow-at-avg-utilization (%v). New storage account will be created", avgUtilization, storageAccountUtilizationBeforeGrowing) SAName = generateStorageAccountName(sharedDiskAccountNamePrefix) err := c.createStorageAccount(SAName, storageAccountType, c.common.location, true) if err != nil { @@ -582,7 +582,7 @@ func (c *BlobDiskController) findSANameForDisk(storageAccountType storage.SkuNam // averages are not ok and we are at capacity (max storage accounts allowed) if aboveAvg && countAccounts == maxStorageAccounts { - glog.Infof("azureDisk - shared storageAccounts utilization(%v) > grow-at-avg-utilization (%v). But k8s maxed on SAs for PVC(%v). k8s will now exceed grow-at-avg-utilization without adding accounts", + klog.Infof("azureDisk - shared storageAccounts utilization(%v) > grow-at-avg-utilization (%v). But k8s maxed on SAs for PVC(%v). k8s will now exceed grow-at-avg-utilization without adding accounts", avgUtilization, storageAccountUtilizationBeforeGrowing, maxStorageAccounts) } diff --git a/pkg/cloudprovider/providers/azure/azure_client.go b/pkg/cloudprovider/providers/azure/azure_client.go index 1f1de845b5c..94a97034275 100644 --- a/pkg/cloudprovider/providers/azure/azure_client.go +++ b/pkg/cloudprovider/providers/azure/azure_client.go @@ -27,7 +27,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/adal" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/client-go/util/flowcontrol" ) @@ -179,9 +179,9 @@ func (az *azVirtualMachinesClient) CreateOrUpdate(ctx context.Context, resourceG return } - glog.V(10).Infof("azVirtualMachinesClient.CreateOrUpdate(%q, %q): start", resourceGroupName, VMName) + klog.V(10).Infof("azVirtualMachinesClient.CreateOrUpdate(%q, %q): start", resourceGroupName, VMName) defer func() { - glog.V(10).Infof("azVirtualMachinesClient.CreateOrUpdate(%q, %q): end", resourceGroupName, VMName) + klog.V(10).Infof("azVirtualMachinesClient.CreateOrUpdate(%q, %q): end", resourceGroupName, VMName) }() mc := newMetricContext("vm", "create_or_update", resourceGroupName, az.client.SubscriptionID) @@ -201,9 +201,9 @@ func (az *azVirtualMachinesClient) Get(ctx context.Context, resourceGroupName st return } - glog.V(10).Infof("azVirtualMachinesClient.Get(%q, %q): start", resourceGroupName, VMName) + klog.V(10).Infof("azVirtualMachinesClient.Get(%q, %q): start", resourceGroupName, VMName) defer func() { - glog.V(10).Infof("azVirtualMachinesClient.Get(%q, %q): end", resourceGroupName, VMName) + klog.V(10).Infof("azVirtualMachinesClient.Get(%q, %q): end", resourceGroupName, VMName) }() mc := newMetricContext("vm", "get", resourceGroupName, az.client.SubscriptionID) @@ -218,9 +218,9 @@ func (az *azVirtualMachinesClient) List(ctx context.Context, resourceGroupName s return } - glog.V(10).Infof("azVirtualMachinesClient.List(%q): start", resourceGroupName) + klog.V(10).Infof("azVirtualMachinesClient.List(%q): start", resourceGroupName) defer func() { - glog.V(10).Infof("azVirtualMachinesClient.List(%q): end", resourceGroupName) + klog.V(10).Infof("azVirtualMachinesClient.List(%q): end", resourceGroupName) }() mc := newMetricContext("vm", "list", resourceGroupName, az.client.SubscriptionID) @@ -270,9 +270,9 @@ func (az *azInterfacesClient) CreateOrUpdate(ctx context.Context, resourceGroupN return } - glog.V(10).Infof("azInterfacesClient.CreateOrUpdate(%q,%q): start", resourceGroupName, networkInterfaceName) + klog.V(10).Infof("azInterfacesClient.CreateOrUpdate(%q,%q): start", resourceGroupName, networkInterfaceName) defer func() { - glog.V(10).Infof("azInterfacesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, networkInterfaceName) + klog.V(10).Infof("azInterfacesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, networkInterfaceName) }() mc := newMetricContext("interfaces", "create_or_update", resourceGroupName, az.client.SubscriptionID) @@ -293,9 +293,9 @@ func (az *azInterfacesClient) Get(ctx context.Context, resourceGroupName string, return } - glog.V(10).Infof("azInterfacesClient.Get(%q,%q): start", resourceGroupName, networkInterfaceName) + klog.V(10).Infof("azInterfacesClient.Get(%q,%q): start", resourceGroupName, networkInterfaceName) defer func() { - glog.V(10).Infof("azInterfacesClient.Get(%q,%q): end", resourceGroupName, networkInterfaceName) + klog.V(10).Infof("azInterfacesClient.Get(%q,%q): end", resourceGroupName, networkInterfaceName) }() mc := newMetricContext("interfaces", "get", resourceGroupName, az.client.SubscriptionID) @@ -310,9 +310,9 @@ func (az *azInterfacesClient) GetVirtualMachineScaleSetNetworkInterface(ctx cont return } - glog.V(10).Infof("azInterfacesClient.GetVirtualMachineScaleSetNetworkInterface(%q,%q,%q,%q): start", resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName) + klog.V(10).Infof("azInterfacesClient.GetVirtualMachineScaleSetNetworkInterface(%q,%q,%q,%q): start", resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName) defer func() { - glog.V(10).Infof("azInterfacesClient.GetVirtualMachineScaleSetNetworkInterface(%q,%q,%q,%q): end", resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName) + klog.V(10).Infof("azInterfacesClient.GetVirtualMachineScaleSetNetworkInterface(%q,%q,%q,%q): end", resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName) }() mc := newMetricContext("interfaces", "get_vmss_ni", resourceGroupName, az.client.SubscriptionID) @@ -349,9 +349,9 @@ func (az *azLoadBalancersClient) CreateOrUpdate(ctx context.Context, resourceGro return nil, err } - glog.V(10).Infof("azLoadBalancersClient.CreateOrUpdate(%q,%q): start", resourceGroupName, loadBalancerName) + klog.V(10).Infof("azLoadBalancersClient.CreateOrUpdate(%q,%q): start", resourceGroupName, loadBalancerName) defer func() { - glog.V(10).Infof("azLoadBalancersClient.CreateOrUpdate(%q,%q): end", resourceGroupName, loadBalancerName) + klog.V(10).Infof("azLoadBalancersClient.CreateOrUpdate(%q,%q): end", resourceGroupName, loadBalancerName) }() mc := newMetricContext("load_balancers", "create_or_update", resourceGroupName, az.client.SubscriptionID) @@ -373,9 +373,9 @@ func (az *azLoadBalancersClient) Delete(ctx context.Context, resourceGroupName s return nil, err } - glog.V(10).Infof("azLoadBalancersClient.Delete(%q,%q): start", resourceGroupName, loadBalancerName) + klog.V(10).Infof("azLoadBalancersClient.Delete(%q,%q): start", resourceGroupName, loadBalancerName) defer func() { - glog.V(10).Infof("azLoadBalancersClient.Delete(%q,%q): end", resourceGroupName, loadBalancerName) + klog.V(10).Infof("azLoadBalancersClient.Delete(%q,%q): end", resourceGroupName, loadBalancerName) }() mc := newMetricContext("load_balancers", "delete", resourceGroupName, az.client.SubscriptionID) @@ -396,9 +396,9 @@ func (az *azLoadBalancersClient) Get(ctx context.Context, resourceGroupName stri return } - glog.V(10).Infof("azLoadBalancersClient.Get(%q,%q): start", resourceGroupName, loadBalancerName) + klog.V(10).Infof("azLoadBalancersClient.Get(%q,%q): start", resourceGroupName, loadBalancerName) defer func() { - glog.V(10).Infof("azLoadBalancersClient.Get(%q,%q): end", resourceGroupName, loadBalancerName) + klog.V(10).Infof("azLoadBalancersClient.Get(%q,%q): end", resourceGroupName, loadBalancerName) }() mc := newMetricContext("load_balancers", "get", resourceGroupName, az.client.SubscriptionID) @@ -413,9 +413,9 @@ func (az *azLoadBalancersClient) List(ctx context.Context, resourceGroupName str return nil, err } - glog.V(10).Infof("azLoadBalancersClient.List(%q): start", resourceGroupName) + klog.V(10).Infof("azLoadBalancersClient.List(%q): start", resourceGroupName) defer func() { - glog.V(10).Infof("azLoadBalancersClient.List(%q): end", resourceGroupName) + klog.V(10).Infof("azLoadBalancersClient.List(%q): end", resourceGroupName) }() mc := newMetricContext("load_balancers", "list", resourceGroupName, az.client.SubscriptionID) @@ -465,9 +465,9 @@ func (az *azPublicIPAddressesClient) CreateOrUpdate(ctx context.Context, resourc return nil, err } - glog.V(10).Infof("azPublicIPAddressesClient.CreateOrUpdate(%q,%q): start", resourceGroupName, publicIPAddressName) + klog.V(10).Infof("azPublicIPAddressesClient.CreateOrUpdate(%q,%q): start", resourceGroupName, publicIPAddressName) defer func() { - glog.V(10).Infof("azPublicIPAddressesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, publicIPAddressName) + klog.V(10).Infof("azPublicIPAddressesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, publicIPAddressName) }() mc := newMetricContext("public_ip_addresses", "create_or_update", resourceGroupName, az.client.SubscriptionID) @@ -489,9 +489,9 @@ func (az *azPublicIPAddressesClient) Delete(ctx context.Context, resourceGroupNa return nil, err } - glog.V(10).Infof("azPublicIPAddressesClient.Delete(%q,%q): start", resourceGroupName, publicIPAddressName) + klog.V(10).Infof("azPublicIPAddressesClient.Delete(%q,%q): start", resourceGroupName, publicIPAddressName) defer func() { - glog.V(10).Infof("azPublicIPAddressesClient.Delete(%q,%q): end", resourceGroupName, publicIPAddressName) + klog.V(10).Infof("azPublicIPAddressesClient.Delete(%q,%q): end", resourceGroupName, publicIPAddressName) }() mc := newMetricContext("public_ip_addresses", "delete", resourceGroupName, az.client.SubscriptionID) @@ -512,9 +512,9 @@ func (az *azPublicIPAddressesClient) Get(ctx context.Context, resourceGroupName return } - glog.V(10).Infof("azPublicIPAddressesClient.Get(%q,%q): start", resourceGroupName, publicIPAddressName) + klog.V(10).Infof("azPublicIPAddressesClient.Get(%q,%q): start", resourceGroupName, publicIPAddressName) defer func() { - glog.V(10).Infof("azPublicIPAddressesClient.Get(%q,%q): end", resourceGroupName, publicIPAddressName) + klog.V(10).Infof("azPublicIPAddressesClient.Get(%q,%q): end", resourceGroupName, publicIPAddressName) }() mc := newMetricContext("public_ip_addresses", "get", resourceGroupName, az.client.SubscriptionID) @@ -528,9 +528,9 @@ func (az *azPublicIPAddressesClient) List(ctx context.Context, resourceGroupName return nil, createRateLimitErr(false, "PublicIPList") } - glog.V(10).Infof("azPublicIPAddressesClient.List(%q): start", resourceGroupName) + klog.V(10).Infof("azPublicIPAddressesClient.List(%q): start", resourceGroupName) defer func() { - glog.V(10).Infof("azPublicIPAddressesClient.List(%q): end", resourceGroupName) + klog.V(10).Infof("azPublicIPAddressesClient.List(%q): end", resourceGroupName) }() mc := newMetricContext("public_ip_addresses", "list", resourceGroupName, az.client.SubscriptionID) @@ -580,9 +580,9 @@ func (az *azSubnetsClient) CreateOrUpdate(ctx context.Context, resourceGroupName return } - glog.V(10).Infof("azSubnetsClient.CreateOrUpdate(%q,%q,%q): start", resourceGroupName, virtualNetworkName, subnetName) + klog.V(10).Infof("azSubnetsClient.CreateOrUpdate(%q,%q,%q): start", resourceGroupName, virtualNetworkName, subnetName) defer func() { - glog.V(10).Infof("azSubnetsClient.CreateOrUpdate(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName) + klog.V(10).Infof("azSubnetsClient.CreateOrUpdate(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName) }() mc := newMetricContext("subnets", "create_or_update", resourceGroupName, az.client.SubscriptionID) @@ -604,9 +604,9 @@ func (az *azSubnetsClient) Delete(ctx context.Context, resourceGroupName string, return } - glog.V(10).Infof("azSubnetsClient.Delete(%q,%q,%q): start", resourceGroupName, virtualNetworkName, subnetName) + klog.V(10).Infof("azSubnetsClient.Delete(%q,%q,%q): start", resourceGroupName, virtualNetworkName, subnetName) defer func() { - glog.V(10).Infof("azSubnetsClient.Delete(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName) + klog.V(10).Infof("azSubnetsClient.Delete(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName) }() mc := newMetricContext("subnets", "delete", resourceGroupName, az.client.SubscriptionID) @@ -627,9 +627,9 @@ func (az *azSubnetsClient) Get(ctx context.Context, resourceGroupName string, vi return } - glog.V(10).Infof("azSubnetsClient.Get(%q,%q,%q): start", resourceGroupName, virtualNetworkName, subnetName) + klog.V(10).Infof("azSubnetsClient.Get(%q,%q,%q): start", resourceGroupName, virtualNetworkName, subnetName) defer func() { - glog.V(10).Infof("azSubnetsClient.Get(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName) + klog.V(10).Infof("azSubnetsClient.Get(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName) }() mc := newMetricContext("subnets", "get", resourceGroupName, az.client.SubscriptionID) @@ -643,9 +643,9 @@ func (az *azSubnetsClient) List(ctx context.Context, resourceGroupName string, v return nil, createRateLimitErr(false, "SubnetList") } - glog.V(10).Infof("azSubnetsClient.List(%q,%q): start", resourceGroupName, virtualNetworkName) + klog.V(10).Infof("azSubnetsClient.List(%q,%q): start", resourceGroupName, virtualNetworkName) defer func() { - glog.V(10).Infof("azSubnetsClient.List(%q,%q): end", resourceGroupName, virtualNetworkName) + klog.V(10).Infof("azSubnetsClient.List(%q,%q): end", resourceGroupName, virtualNetworkName) }() mc := newMetricContext("subnets", "list", resourceGroupName, az.client.SubscriptionID) @@ -695,9 +695,9 @@ func (az *azSecurityGroupsClient) CreateOrUpdate(ctx context.Context, resourceGr return } - glog.V(10).Infof("azSecurityGroupsClient.CreateOrUpdate(%q,%q): start", resourceGroupName, networkSecurityGroupName) + klog.V(10).Infof("azSecurityGroupsClient.CreateOrUpdate(%q,%q): start", resourceGroupName, networkSecurityGroupName) defer func() { - glog.V(10).Infof("azSecurityGroupsClient.CreateOrUpdate(%q,%q): end", resourceGroupName, networkSecurityGroupName) + klog.V(10).Infof("azSecurityGroupsClient.CreateOrUpdate(%q,%q): end", resourceGroupName, networkSecurityGroupName) }() mc := newMetricContext("security_groups", "create_or_update", resourceGroupName, az.client.SubscriptionID) @@ -719,9 +719,9 @@ func (az *azSecurityGroupsClient) Delete(ctx context.Context, resourceGroupName return } - glog.V(10).Infof("azSecurityGroupsClient.Delete(%q,%q): start", resourceGroupName, networkSecurityGroupName) + klog.V(10).Infof("azSecurityGroupsClient.Delete(%q,%q): start", resourceGroupName, networkSecurityGroupName) defer func() { - glog.V(10).Infof("azSecurityGroupsClient.Delete(%q,%q): end", resourceGroupName, networkSecurityGroupName) + klog.V(10).Infof("azSecurityGroupsClient.Delete(%q,%q): end", resourceGroupName, networkSecurityGroupName) }() mc := newMetricContext("security_groups", "delete", resourceGroupName, az.client.SubscriptionID) @@ -742,9 +742,9 @@ func (az *azSecurityGroupsClient) Get(ctx context.Context, resourceGroupName str return } - glog.V(10).Infof("azSecurityGroupsClient.Get(%q,%q): start", resourceGroupName, networkSecurityGroupName) + klog.V(10).Infof("azSecurityGroupsClient.Get(%q,%q): start", resourceGroupName, networkSecurityGroupName) defer func() { - glog.V(10).Infof("azSecurityGroupsClient.Get(%q,%q): end", resourceGroupName, networkSecurityGroupName) + klog.V(10).Infof("azSecurityGroupsClient.Get(%q,%q): end", resourceGroupName, networkSecurityGroupName) }() mc := newMetricContext("security_groups", "get", resourceGroupName, az.client.SubscriptionID) @@ -758,9 +758,9 @@ func (az *azSecurityGroupsClient) List(ctx context.Context, resourceGroupName st return nil, createRateLimitErr(false, "NSGList") } - glog.V(10).Infof("azSecurityGroupsClient.List(%q): start", resourceGroupName) + klog.V(10).Infof("azSecurityGroupsClient.List(%q): start", resourceGroupName) defer func() { - glog.V(10).Infof("azSecurityGroupsClient.List(%q): end", resourceGroupName) + klog.V(10).Infof("azSecurityGroupsClient.List(%q): end", resourceGroupName) }() mc := newMetricContext("security_groups", "list", resourceGroupName, az.client.SubscriptionID) @@ -810,9 +810,9 @@ func (az *azVirtualMachineScaleSetsClient) CreateOrUpdate(ctx context.Context, r return } - glog.V(10).Infof("azVirtualMachineScaleSetsClient.CreateOrUpdate(%q,%q): start", resourceGroupName, VMScaleSetName) + klog.V(10).Infof("azVirtualMachineScaleSetsClient.CreateOrUpdate(%q,%q): start", resourceGroupName, VMScaleSetName) defer func() { - glog.V(10).Infof("azVirtualMachineScaleSetsClient.CreateOrUpdate(%q,%q): end", resourceGroupName, VMScaleSetName) + klog.V(10).Infof("azVirtualMachineScaleSetsClient.CreateOrUpdate(%q,%q): end", resourceGroupName, VMScaleSetName) }() mc := newMetricContext("vmss", "create_or_update", resourceGroupName, az.client.SubscriptionID) @@ -833,9 +833,9 @@ func (az *azVirtualMachineScaleSetsClient) Get(ctx context.Context, resourceGrou return } - glog.V(10).Infof("azVirtualMachineScaleSetsClient.Get(%q,%q): start", resourceGroupName, VMScaleSetName) + klog.V(10).Infof("azVirtualMachineScaleSetsClient.Get(%q,%q): start", resourceGroupName, VMScaleSetName) defer func() { - glog.V(10).Infof("azVirtualMachineScaleSetsClient.Get(%q,%q): end", resourceGroupName, VMScaleSetName) + klog.V(10).Infof("azVirtualMachineScaleSetsClient.Get(%q,%q): end", resourceGroupName, VMScaleSetName) }() mc := newMetricContext("vmss", "get", resourceGroupName, az.client.SubscriptionID) @@ -850,9 +850,9 @@ func (az *azVirtualMachineScaleSetsClient) List(ctx context.Context, resourceGro return } - glog.V(10).Infof("azVirtualMachineScaleSetsClient.List(%q): start", resourceGroupName) + klog.V(10).Infof("azVirtualMachineScaleSetsClient.List(%q): start", resourceGroupName) defer func() { - glog.V(10).Infof("azVirtualMachineScaleSetsClient.List(%q): end", resourceGroupName) + klog.V(10).Infof("azVirtualMachineScaleSetsClient.List(%q): end", resourceGroupName) }() mc := newMetricContext("vmss", "list", resourceGroupName, az.client.SubscriptionID) @@ -881,9 +881,9 @@ func (az *azVirtualMachineScaleSetsClient) UpdateInstances(ctx context.Context, return } - glog.V(10).Infof("azVirtualMachineScaleSetsClient.UpdateInstances(%q,%q,%v): start", resourceGroupName, VMScaleSetName, VMInstanceIDs) + klog.V(10).Infof("azVirtualMachineScaleSetsClient.UpdateInstances(%q,%q,%v): start", resourceGroupName, VMScaleSetName, VMInstanceIDs) defer func() { - glog.V(10).Infof("azVirtualMachineScaleSetsClient.UpdateInstances(%q,%q,%v): end", resourceGroupName, VMScaleSetName, VMInstanceIDs) + klog.V(10).Infof("azVirtualMachineScaleSetsClient.UpdateInstances(%q,%q,%v): end", resourceGroupName, VMScaleSetName, VMInstanceIDs) }() mc := newMetricContext("vmss", "update_instances", resourceGroupName, az.client.SubscriptionID) @@ -925,9 +925,9 @@ func (az *azVirtualMachineScaleSetVMsClient) Get(ctx context.Context, resourceGr return } - glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Get(%q,%q,%q): start", resourceGroupName, VMScaleSetName, instanceID) + klog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Get(%q,%q,%q): start", resourceGroupName, VMScaleSetName, instanceID) defer func() { - glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Get(%q,%q,%q): end", resourceGroupName, VMScaleSetName, instanceID) + klog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Get(%q,%q,%q): end", resourceGroupName, VMScaleSetName, instanceID) }() mc := newMetricContext("vmssvm", "get", resourceGroupName, az.client.SubscriptionID) @@ -942,9 +942,9 @@ func (az *azVirtualMachineScaleSetVMsClient) GetInstanceView(ctx context.Context return } - glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.GetInstanceView(%q,%q,%q): start", resourceGroupName, VMScaleSetName, instanceID) + klog.V(10).Infof("azVirtualMachineScaleSetVMsClient.GetInstanceView(%q,%q,%q): start", resourceGroupName, VMScaleSetName, instanceID) defer func() { - glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.GetInstanceView(%q,%q,%q): end", resourceGroupName, VMScaleSetName, instanceID) + klog.V(10).Infof("azVirtualMachineScaleSetVMsClient.GetInstanceView(%q,%q,%q): end", resourceGroupName, VMScaleSetName, instanceID) }() mc := newMetricContext("vmssvm", "get_instance_view", resourceGroupName, az.client.SubscriptionID) @@ -959,9 +959,9 @@ func (az *azVirtualMachineScaleSetVMsClient) List(ctx context.Context, resourceG return } - glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.List(%q,%q,%q): start", resourceGroupName, virtualMachineScaleSetName, filter) + klog.V(10).Infof("azVirtualMachineScaleSetVMsClient.List(%q,%q,%q): start", resourceGroupName, virtualMachineScaleSetName, filter) defer func() { - glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.List(%q,%q,%q): end", resourceGroupName, virtualMachineScaleSetName, filter) + klog.V(10).Infof("azVirtualMachineScaleSetVMsClient.List(%q,%q,%q): end", resourceGroupName, virtualMachineScaleSetName, filter) }() mc := newMetricContext("vmssvm", "list", resourceGroupName, az.client.SubscriptionID) @@ -989,9 +989,9 @@ func (az *azVirtualMachineScaleSetVMsClient) Update(ctx context.Context, resourc return } - glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Update(%q,%q,%q): start", resourceGroupName, VMScaleSetName, instanceID) + klog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Update(%q,%q,%q): start", resourceGroupName, VMScaleSetName, instanceID) defer func() { - glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Update(%q,%q,%q): end", resourceGroupName, VMScaleSetName, instanceID) + klog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Update(%q,%q,%q): end", resourceGroupName, VMScaleSetName, instanceID) }() mc := newMetricContext("vmssvm", "update", resourceGroupName, az.client.SubscriptionID) @@ -1034,9 +1034,9 @@ func (az *azRoutesClient) CreateOrUpdate(ctx context.Context, resourceGroupName return } - glog.V(10).Infof("azRoutesClient.CreateOrUpdate(%q,%q,%q): start", resourceGroupName, routeTableName, routeName) + klog.V(10).Infof("azRoutesClient.CreateOrUpdate(%q,%q,%q): start", resourceGroupName, routeTableName, routeName) defer func() { - glog.V(10).Infof("azRoutesClient.CreateOrUpdate(%q,%q,%q): end", resourceGroupName, routeTableName, routeName) + klog.V(10).Infof("azRoutesClient.CreateOrUpdate(%q,%q,%q): end", resourceGroupName, routeTableName, routeName) }() mc := newMetricContext("routes", "create_or_update", resourceGroupName, az.client.SubscriptionID) @@ -1058,9 +1058,9 @@ func (az *azRoutesClient) Delete(ctx context.Context, resourceGroupName string, return } - glog.V(10).Infof("azRoutesClient.Delete(%q,%q,%q): start", resourceGroupName, routeTableName, routeName) + klog.V(10).Infof("azRoutesClient.Delete(%q,%q,%q): start", resourceGroupName, routeTableName, routeName) defer func() { - glog.V(10).Infof("azRoutesClient.Delete(%q,%q,%q): end", resourceGroupName, routeTableName, routeName) + klog.V(10).Infof("azRoutesClient.Delete(%q,%q,%q): end", resourceGroupName, routeTableName, routeName) }() mc := newMetricContext("routes", "delete", resourceGroupName, az.client.SubscriptionID) @@ -1103,9 +1103,9 @@ func (az *azRouteTablesClient) CreateOrUpdate(ctx context.Context, resourceGroup return } - glog.V(10).Infof("azRouteTablesClient.CreateOrUpdate(%q,%q): start", resourceGroupName, routeTableName) + klog.V(10).Infof("azRouteTablesClient.CreateOrUpdate(%q,%q): start", resourceGroupName, routeTableName) defer func() { - glog.V(10).Infof("azRouteTablesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, routeTableName) + klog.V(10).Infof("azRouteTablesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, routeTableName) }() mc := newMetricContext("route_tables", "create_or_update", resourceGroupName, az.client.SubscriptionID) @@ -1126,9 +1126,9 @@ func (az *azRouteTablesClient) Get(ctx context.Context, resourceGroupName string return } - glog.V(10).Infof("azRouteTablesClient.Get(%q,%q): start", resourceGroupName, routeTableName) + klog.V(10).Infof("azRouteTablesClient.Get(%q,%q): start", resourceGroupName, routeTableName) defer func() { - glog.V(10).Infof("azRouteTablesClient.Get(%q,%q): end", resourceGroupName, routeTableName) + klog.V(10).Infof("azRouteTablesClient.Get(%q,%q): end", resourceGroupName, routeTableName) }() mc := newMetricContext("route_tables", "get", resourceGroupName, az.client.SubscriptionID) @@ -1164,9 +1164,9 @@ func (az *azStorageAccountClient) Create(ctx context.Context, resourceGroupName return } - glog.V(10).Infof("azStorageAccountClient.Create(%q,%q): start", resourceGroupName, accountName) + klog.V(10).Infof("azStorageAccountClient.Create(%q,%q): start", resourceGroupName, accountName) defer func() { - glog.V(10).Infof("azStorageAccountClient.Create(%q,%q): end", resourceGroupName, accountName) + klog.V(10).Infof("azStorageAccountClient.Create(%q,%q): end", resourceGroupName, accountName) }() mc := newMetricContext("storage_account", "create", resourceGroupName, az.client.SubscriptionID) @@ -1186,9 +1186,9 @@ func (az *azStorageAccountClient) Delete(ctx context.Context, resourceGroupName return } - glog.V(10).Infof("azStorageAccountClient.Delete(%q,%q): start", resourceGroupName, accountName) + klog.V(10).Infof("azStorageAccountClient.Delete(%q,%q): start", resourceGroupName, accountName) defer func() { - glog.V(10).Infof("azStorageAccountClient.Delete(%q,%q): end", resourceGroupName, accountName) + klog.V(10).Infof("azStorageAccountClient.Delete(%q,%q): end", resourceGroupName, accountName) }() mc := newMetricContext("storage_account", "delete", resourceGroupName, az.client.SubscriptionID) @@ -1203,9 +1203,9 @@ func (az *azStorageAccountClient) ListKeys(ctx context.Context, resourceGroupNam return } - glog.V(10).Infof("azStorageAccountClient.ListKeys(%q,%q): start", resourceGroupName, accountName) + klog.V(10).Infof("azStorageAccountClient.ListKeys(%q,%q): start", resourceGroupName, accountName) defer func() { - glog.V(10).Infof("azStorageAccountClient.ListKeys(%q,%q): end", resourceGroupName, accountName) + klog.V(10).Infof("azStorageAccountClient.ListKeys(%q,%q): end", resourceGroupName, accountName) }() mc := newMetricContext("storage_account", "list_keys", resourceGroupName, az.client.SubscriptionID) @@ -1220,9 +1220,9 @@ func (az *azStorageAccountClient) ListByResourceGroup(ctx context.Context, resou return } - glog.V(10).Infof("azStorageAccountClient.ListByResourceGroup(%q): start", resourceGroupName) + klog.V(10).Infof("azStorageAccountClient.ListByResourceGroup(%q): start", resourceGroupName) defer func() { - glog.V(10).Infof("azStorageAccountClient.ListByResourceGroup(%q): end", resourceGroupName) + klog.V(10).Infof("azStorageAccountClient.ListByResourceGroup(%q): end", resourceGroupName) }() mc := newMetricContext("storage_account", "list_by_resource_group", resourceGroupName, az.client.SubscriptionID) @@ -1237,9 +1237,9 @@ func (az *azStorageAccountClient) GetProperties(ctx context.Context, resourceGro return } - glog.V(10).Infof("azStorageAccountClient.GetProperties(%q,%q): start", resourceGroupName, accountName) + klog.V(10).Infof("azStorageAccountClient.GetProperties(%q,%q): start", resourceGroupName, accountName) defer func() { - glog.V(10).Infof("azStorageAccountClient.GetProperties(%q,%q): end", resourceGroupName, accountName) + klog.V(10).Infof("azStorageAccountClient.GetProperties(%q,%q): end", resourceGroupName, accountName) }() mc := newMetricContext("storage_account", "get_properties", resourceGroupName, az.client.SubscriptionID) @@ -1275,9 +1275,9 @@ func (az *azDisksClient) CreateOrUpdate(ctx context.Context, resourceGroupName s return } - glog.V(10).Infof("azDisksClient.CreateOrUpdate(%q,%q): start", resourceGroupName, diskName) + klog.V(10).Infof("azDisksClient.CreateOrUpdate(%q,%q): start", resourceGroupName, diskName) defer func() { - glog.V(10).Infof("azDisksClient.CreateOrUpdate(%q,%q): end", resourceGroupName, diskName) + klog.V(10).Infof("azDisksClient.CreateOrUpdate(%q,%q): end", resourceGroupName, diskName) }() mc := newMetricContext("disks", "create_or_update", resourceGroupName, az.client.SubscriptionID) @@ -1299,9 +1299,9 @@ func (az *azDisksClient) Delete(ctx context.Context, resourceGroupName string, d return } - glog.V(10).Infof("azDisksClient.Delete(%q,%q): start", resourceGroupName, diskName) + klog.V(10).Infof("azDisksClient.Delete(%q,%q): start", resourceGroupName, diskName) defer func() { - glog.V(10).Infof("azDisksClient.Delete(%q,%q): end", resourceGroupName, diskName) + klog.V(10).Infof("azDisksClient.Delete(%q,%q): end", resourceGroupName, diskName) }() mc := newMetricContext("disks", "delete", resourceGroupName, az.client.SubscriptionID) @@ -1322,9 +1322,9 @@ func (az *azDisksClient) Get(ctx context.Context, resourceGroupName string, disk return } - glog.V(10).Infof("azDisksClient.Get(%q,%q): start", resourceGroupName, diskName) + klog.V(10).Infof("azDisksClient.Get(%q,%q): start", resourceGroupName, diskName) defer func() { - glog.V(10).Infof("azDisksClient.Get(%q,%q): end", resourceGroupName, diskName) + klog.V(10).Infof("azDisksClient.Get(%q,%q): end", resourceGroupName, diskName) }() mc := newMetricContext("disks", "get", resourceGroupName, az.client.SubscriptionID) @@ -1360,9 +1360,9 @@ func (az *azVirtualMachineSizesClient) List(ctx context.Context, location string return } - glog.V(10).Infof("azVirtualMachineSizesClient.List(%q): start", location) + klog.V(10).Infof("azVirtualMachineSizesClient.List(%q): start", location) defer func() { - glog.V(10).Infof("azVirtualMachineSizesClient.List(%q): end", location) + klog.V(10).Infof("azVirtualMachineSizesClient.List(%q): end", location) }() mc := newMetricContext("vmsizes", "list", "", az.client.SubscriptionID) diff --git a/pkg/cloudprovider/providers/azure/azure_controller_common.go b/pkg/cloudprovider/providers/azure/azure_controller_common.go index 9be915cfc37..7109ea73bd4 100644 --- a/pkg/cloudprovider/providers/azure/azure_controller_common.go +++ b/pkg/cloudprovider/providers/azure/azure_controller_common.go @@ -21,7 +21,7 @@ import ( "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/types" kwait "k8s.io/apimachinery/pkg/util/wait" @@ -119,7 +119,7 @@ func (c *controllerCommon) getNodeDataDisks(nodeName types.NodeName) ([]compute. func (c *controllerCommon) GetDiskLun(diskName, diskURI string, nodeName types.NodeName) (int32, error) { disks, err := c.getNodeDataDisks(nodeName) if err != nil { - glog.Errorf("error of getting data disks for node %q: %v", nodeName, err) + klog.Errorf("error of getting data disks for node %q: %v", nodeName, err) return -1, err } @@ -128,7 +128,7 @@ func (c *controllerCommon) GetDiskLun(diskName, diskURI string, nodeName types.N (disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && *disk.Vhd.URI == diskURI) || (disk.ManagedDisk != nil && *disk.ManagedDisk.ID == diskURI) { // found the disk - glog.V(2).Infof("azureDisk - find disk: lun %d name %q uri %q", *disk.Lun, diskName, diskURI) + klog.V(2).Infof("azureDisk - find disk: lun %d name %q uri %q", *disk.Lun, diskName, diskURI) return *disk.Lun, nil } } @@ -139,7 +139,7 @@ func (c *controllerCommon) GetDiskLun(diskName, diskURI string, nodeName types.N func (c *controllerCommon) GetNextDiskLun(nodeName types.NodeName) (int32, error) { disks, err := c.getNodeDataDisks(nodeName) if err != nil { - glog.Errorf("error of getting data disks for node %q: %v", nodeName, err) + klog.Errorf("error of getting data disks for node %q: %v", nodeName, err) return -1, err } @@ -168,7 +168,7 @@ func (c *controllerCommon) DisksAreAttached(diskNames []string, nodeName types.N if err != nil { if err == cloudprovider.InstanceNotFound { // if host doesn't exist, no need to detach - glog.Warningf("azureDisk - Cannot find node %q, DisksAreAttached will assume disks %v are not attached to it.", + klog.Warningf("azureDisk - Cannot find node %q, DisksAreAttached will assume disks %v are not attached to it.", nodeName, diskNames) return attached, nil } diff --git a/pkg/cloudprovider/providers/azure/azure_controller_standard.go b/pkg/cloudprovider/providers/azure/azure_controller_standard.go index 9c62e2877df..505b19af8ec 100644 --- a/pkg/cloudprovider/providers/azure/azure_controller_standard.go +++ b/pkg/cloudprovider/providers/azure/azure_controller_standard.go @@ -21,7 +21,7 @@ import ( "strings" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/types" ) @@ -73,19 +73,19 @@ func (as *availabilitySet) AttachDisk(isManagedDisk bool, diskName, diskURI stri }, }, } - glog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s)", nodeResourceGroup, vmName, diskName) + klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s)", nodeResourceGroup, vmName, diskName) ctx, cancel := getContextWithCancel() defer cancel() if _, err := as.VirtualMachinesClient.CreateOrUpdate(ctx, nodeResourceGroup, vmName, newVM); err != nil { - glog.Errorf("azureDisk - attach disk(%s) failed, err: %v", diskName, err) + klog.Errorf("azureDisk - attach disk(%s) failed, err: %v", diskName, err) detail := err.Error() if strings.Contains(detail, errLeaseFailed) || strings.Contains(detail, errDiskBlobNotFound) { // if lease cannot be acquired or disk not found, immediately detach the disk and return the original error - glog.V(2).Infof("azureDisk - err %v, try detach disk(%s)", err, diskName) + klog.V(2).Infof("azureDisk - err %v, try detach disk(%s)", err, diskName) as.DetachDiskByName(diskName, diskURI, nodeName) } } else { - glog.V(2).Infof("azureDisk - attach disk(%s) succeeded", diskName) + klog.V(2).Infof("azureDisk - attach disk(%s) succeeded", diskName) // Invalidate the cache right after updating as.cloud.vmCache.Delete(vmName) } @@ -98,7 +98,7 @@ func (as *availabilitySet) DetachDiskByName(diskName, diskURI string, nodeName t vm, err := as.getVirtualMachine(nodeName) if err != nil { // if host doesn't exist, no need to detach - glog.Warningf("azureDisk - cannot find node %s, skip detaching disk %s", nodeName, diskName) + klog.Warningf("azureDisk - cannot find node %s, skip detaching disk %s", nodeName, diskName) return nil } @@ -115,7 +115,7 @@ func (as *availabilitySet) DetachDiskByName(diskName, diskURI string, nodeName t (disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && *disk.Vhd.URI == diskURI) || (disk.ManagedDisk != nil && diskURI != "" && *disk.ManagedDisk.ID == diskURI) { // found the disk - glog.V(2).Infof("azureDisk - detach disk: name %q uri %q", diskName, diskURI) + klog.V(2).Infof("azureDisk - detach disk: name %q uri %q", diskName, diskURI) disks = append(disks[:i], disks[i+1:]...) bFoundDisk = true break @@ -134,13 +134,13 @@ func (as *availabilitySet) DetachDiskByName(diskName, diskURI string, nodeName t }, }, } - glog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s)", nodeResourceGroup, vmName, diskName) + klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s)", nodeResourceGroup, vmName, diskName) ctx, cancel := getContextWithCancel() defer cancel() if _, err := as.VirtualMachinesClient.CreateOrUpdate(ctx, nodeResourceGroup, vmName, newVM); err != nil { - glog.Errorf("azureDisk - detach disk(%s) failed, err: %v", diskName, err) + klog.Errorf("azureDisk - detach disk(%s) failed, err: %v", diskName, err) } else { - glog.V(2).Infof("azureDisk - detach disk(%s) succeeded", diskName) + klog.V(2).Infof("azureDisk - detach disk(%s) succeeded", diskName) // Invalidate the cache right after updating as.cloud.vmCache.Delete(vmName) } diff --git a/pkg/cloudprovider/providers/azure/azure_controller_vmss.go b/pkg/cloudprovider/providers/azure/azure_controller_vmss.go index d65b1be4729..53bbde77a99 100644 --- a/pkg/cloudprovider/providers/azure/azure_controller_vmss.go +++ b/pkg/cloudprovider/providers/azure/azure_controller_vmss.go @@ -21,7 +21,7 @@ import ( "strings" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/types" ) @@ -71,16 +71,16 @@ func (ss *scaleSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nod ctx, cancel := getContextWithCancel() defer cancel() - glog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s)", nodeResourceGroup, nodeName, diskName) + klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s)", nodeResourceGroup, nodeName, diskName) if _, err := ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, vm); err != nil { detail := err.Error() if strings.Contains(detail, errLeaseFailed) || strings.Contains(detail, errDiskBlobNotFound) { // if lease cannot be acquired or disk not found, immediately detach the disk and return the original error - glog.Infof("azureDisk - err %s, try detach disk(%s)", detail, diskName) + klog.Infof("azureDisk - err %s, try detach disk(%s)", detail, diskName) ss.DetachDiskByName(diskName, diskURI, nodeName) } } else { - glog.V(2).Infof("azureDisk - attach disk(%s) succeeded", diskName) + klog.V(2).Infof("azureDisk - attach disk(%s) succeeded", diskName) // Invalidate the cache right after updating key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID)) ss.vmssVMCache.Delete(key) @@ -112,7 +112,7 @@ func (ss *scaleSet) DetachDiskByName(diskName, diskURI string, nodeName types.No (disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && *disk.Vhd.URI == diskURI) || (disk.ManagedDisk != nil && diskURI != "" && *disk.ManagedDisk.ID == diskURI) { // found the disk - glog.V(2).Infof("azureDisk - detach disk: name %q uri %q", diskName, diskURI) + klog.V(2).Infof("azureDisk - detach disk: name %q uri %q", diskName, diskURI) disks = append(disks[:i], disks[i+1:]...) bFoundDisk = true break @@ -126,11 +126,11 @@ func (ss *scaleSet) DetachDiskByName(diskName, diskURI string, nodeName types.No vm.StorageProfile.DataDisks = &disks ctx, cancel := getContextWithCancel() defer cancel() - glog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s)", nodeResourceGroup, nodeName, diskName) + klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s)", nodeResourceGroup, nodeName, diskName) if _, err := ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, vm); err != nil { - glog.Errorf("azureDisk - detach disk(%s) from %s failed, err: %v", diskName, nodeName, err) + klog.Errorf("azureDisk - detach disk(%s) from %s failed, err: %v", diskName, nodeName, err) } else { - glog.V(2).Infof("azureDisk - detach disk(%s) succeeded", diskName) + klog.V(2).Infof("azureDisk - detach disk(%s) succeeded", diskName) // Invalidate the cache right after updating key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID)) ss.vmssVMCache.Delete(key) diff --git a/pkg/cloudprovider/providers/azure/azure_file.go b/pkg/cloudprovider/providers/azure/azure_file.go index f5bec74cb25..ab87cf3625f 100644 --- a/pkg/cloudprovider/providers/azure/azure_file.go +++ b/pkg/cloudprovider/providers/azure/azure_file.go @@ -21,7 +21,7 @@ import ( azs "github.com/Azure/azure-sdk-for-go/storage" "github.com/Azure/go-autorest/autorest/azure" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -82,7 +82,7 @@ func (f *azureFileClient) resizeFileShare(accountName, accountKey, name string, } share := fileClient.GetShareReference(name) if share.Properties.Quota >= sizeGiB { - glog.Warningf("file share size(%dGi) is already greater or equal than requested size(%dGi), accountName: %s, shareName: %s", + klog.Warningf("file share size(%dGi) is already greater or equal than requested size(%dGi), accountName: %s, shareName: %s", share.Properties.Quota, sizeGiB, accountName, name) return nil } @@ -90,7 +90,7 @@ func (f *azureFileClient) resizeFileShare(accountName, accountKey, name string, if err = share.SetProperties(nil); err != nil { return fmt.Errorf("failed to set quota on file share %s, err: %v", name, err) } - glog.V(4).Infof("resize file share completed, accountName: %s, shareName: %s, sizeGiB: %d", accountName, name, sizeGiB) + klog.V(4).Infof("resize file share completed, accountName: %s, shareName: %s, sizeGiB: %d", accountName, name, sizeGiB) return nil } diff --git a/pkg/cloudprovider/providers/azure/azure_instances.go b/pkg/cloudprovider/providers/azure/azure_instances.go index d966b21b3eb..158ffb976ca 100644 --- a/pkg/cloudprovider/providers/azure/azure_instances.go +++ b/pkg/cloudprovider/providers/azure/azure_instances.go @@ -25,8 +25,8 @@ import ( "k8s.io/api/core/v1" cloudprovider "k8s.io/cloud-provider" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" ) const ( @@ -43,14 +43,14 @@ func (az *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.N return nil, err } if unmanaged { - glog.V(4).Infof("NodeAddresses: omitting unmanaged node %q", name) + klog.V(4).Infof("NodeAddresses: omitting unmanaged node %q", name) return nil, nil } addressGetter := func(nodeName types.NodeName) ([]v1.NodeAddress, error) { ip, publicIP, err := az.GetIPForMachineWithRetry(nodeName) if err != nil { - glog.V(2).Infof("NodeAddresses(%s) abort backoff: %v", nodeName, err) + klog.V(2).Infof("NodeAddresses(%s) abort backoff: %v", nodeName, err) return nil, err } @@ -132,7 +132,7 @@ func (az *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.N func (az *Cloud) NodeAddressesByProviderID(ctx context.Context, providerID string) ([]v1.NodeAddress, error) { // Returns nil for unmanaged nodes because azure cloud provider couldn't fetch information for them. if az.IsNodeUnmanagedByProviderID(providerID) { - glog.V(4).Infof("NodeAddressesByProviderID: omitting unmanaged node %q", providerID) + klog.V(4).Infof("NodeAddressesByProviderID: omitting unmanaged node %q", providerID) return nil, nil } @@ -149,7 +149,7 @@ func (az *Cloud) NodeAddressesByProviderID(ctx context.Context, providerID strin func (az *Cloud) InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error) { // Returns true for unmanaged nodes because azure cloud provider always assumes them exists. if az.IsNodeUnmanagedByProviderID(providerID) { - glog.V(4).Infof("InstanceExistsByProviderID: assuming unmanaged node %q exists", providerID) + klog.V(4).Infof("InstanceExistsByProviderID: assuming unmanaged node %q exists", providerID) return true, nil } @@ -180,7 +180,7 @@ func (az *Cloud) InstanceShutdownByProviderID(ctx context.Context, providerID st if err != nil { return false, err } - glog.V(5).Infof("InstanceShutdownByProviderID gets power status %q for node %q", powerStatus, nodeName) + klog.V(5).Infof("InstanceShutdownByProviderID gets power status %q for node %q", powerStatus, nodeName) return strings.ToLower(powerStatus) == vmPowerStateStopped || strings.ToLower(powerStatus) == vmPowerStateDeallocated, nil } @@ -210,7 +210,7 @@ func (az *Cloud) InstanceID(ctx context.Context, name types.NodeName) (string, e } if unmanaged { // InstanceID is same with nodeName for unmanaged nodes. - glog.V(4).Infof("InstanceID: getting ID %q for unmanaged node %q", name, name) + klog.V(4).Infof("InstanceID: getting ID %q for unmanaged node %q", name, name) return nodeName, nil } @@ -264,7 +264,7 @@ func (az *Cloud) InstanceID(ctx context.Context, name types.NodeName) (string, e func (az *Cloud) InstanceTypeByProviderID(ctx context.Context, providerID string) (string, error) { // Returns "" for unmanaged nodes because azure cloud provider couldn't fetch information for them. if az.IsNodeUnmanagedByProviderID(providerID) { - glog.V(4).Infof("InstanceTypeByProviderID: omitting unmanaged node %q", providerID) + klog.V(4).Infof("InstanceTypeByProviderID: omitting unmanaged node %q", providerID) return "", nil } @@ -287,7 +287,7 @@ func (az *Cloud) InstanceType(ctx context.Context, name types.NodeName) (string, return "", err } if unmanaged { - glog.V(4).Infof("InstanceType: omitting unmanaged node %q", name) + klog.V(4).Infof("InstanceType: omitting unmanaged node %q", name) return "", nil } diff --git a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go index 506a64dd2f7..77bfdd20de2 100644 --- a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go +++ b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go @@ -31,7 +31,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network" "github.com/Azure/go-autorest/autorest/to" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -101,7 +101,7 @@ func (az *Cloud) GetLoadBalancer(ctx context.Context, clusterName string, servic } if !exists { serviceName := getServiceName(service) - glog.V(5).Infof("getloadbalancer (cluster:%s) (service:%s) - doesn't exist", clusterName, serviceName) + klog.V(5).Infof("getloadbalancer (cluster:%s) (service:%s) - doesn't exist", clusterName, serviceName) return nil, false, nil } return status, true, nil @@ -120,7 +120,7 @@ func (az *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, ser // the service may be switched from an internal LB to a public one, or vise versa. // Here we'll firstly ensure service do not lie in the opposite LB. serviceName := getServiceName(service) - glog.V(5).Infof("ensureloadbalancer(%s): START clusterName=%q", serviceName, clusterName) + klog.V(5).Infof("ensureloadbalancer(%s): START clusterName=%q", serviceName, clusterName) lb, err := az.reconcileLoadBalancer(clusterName, service, nodes, true /* wantLb */) if err != nil { @@ -136,7 +136,7 @@ func (az *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, ser if lbStatus != nil && len(lbStatus.Ingress) > 0 { serviceIP = &lbStatus.Ingress[0].IP } - glog.V(2).Infof("EnsureLoadBalancer: reconciling security group for service %q with IP %q, wantLb = true", serviceName, logSafe(serviceIP)) + klog.V(2).Infof("EnsureLoadBalancer: reconciling security group for service %q with IP %q, wantLb = true", serviceName, logSafe(serviceIP)) if _, err := az.reconcileSecurityGroup(clusterName, service, serviceIP, true /* wantLb */); err != nil { return nil, err } @@ -169,14 +169,14 @@ func (az *Cloud) UpdateLoadBalancer(ctx context.Context, clusterName string, ser func (az *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *v1.Service) error { isInternal := requiresInternalLoadBalancer(service) serviceName := getServiceName(service) - glog.V(5).Infof("delete(%s): START clusterName=%q", serviceName, clusterName) + klog.V(5).Infof("delete(%s): START clusterName=%q", serviceName, clusterName) serviceIPToCleanup, err := az.findServiceIPAddress(ctx, clusterName, service, isInternal) if err != nil { return err } - glog.V(2).Infof("EnsureLoadBalancerDeleted: reconciling security group for service %q with IP %q, wantLb = false", serviceName, serviceIPToCleanup) + klog.V(2).Infof("EnsureLoadBalancerDeleted: reconciling security group for service %q with IP %q, wantLb = false", serviceName, serviceIPToCleanup) if _, err := az.reconcileSecurityGroup(clusterName, service, &serviceIPToCleanup, false /* wantLb */); err != nil { return err } @@ -189,7 +189,7 @@ func (az *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName stri return err } - glog.V(2).Infof("delete(%s): FINISH", serviceName) + klog.V(2).Infof("delete(%s): FINISH", serviceName) return nil } @@ -279,13 +279,13 @@ func (az *Cloud) getServiceLoadBalancer(service *v1.Service, clusterName string, func (az *Cloud) selectLoadBalancer(clusterName string, service *v1.Service, existingLBs *[]network.LoadBalancer, nodes []*v1.Node) (selectedLB *network.LoadBalancer, existsLb bool, err error) { isInternal := requiresInternalLoadBalancer(service) serviceName := getServiceName(service) - glog.V(2).Infof("selectLoadBalancer for service (%s): isInternal(%v) - start", serviceName, isInternal) + klog.V(2).Infof("selectLoadBalancer for service (%s): isInternal(%v) - start", serviceName, isInternal) vmSetNames, err := az.vmSet.GetVMSetNames(service, nodes) if err != nil { - glog.Errorf("az.selectLoadBalancer: cluster(%s) service(%s) isInternal(%t) - az.GetVMSetNames failed, err=(%v)", clusterName, serviceName, isInternal, err) + klog.Errorf("az.selectLoadBalancer: cluster(%s) service(%s) isInternal(%t) - az.GetVMSetNames failed, err=(%v)", clusterName, serviceName, isInternal, err) return nil, false, err } - glog.Infof("selectLoadBalancer: cluster(%s) service(%s) isInternal(%t) - vmSetNames %v", clusterName, serviceName, isInternal, *vmSetNames) + klog.Infof("selectLoadBalancer: cluster(%s) service(%s) isInternal(%t) - vmSetNames %v", clusterName, serviceName, isInternal, *vmSetNames) mapExistingLBs := map[string]network.LoadBalancer{} for _, lb := range *existingLBs { @@ -320,13 +320,13 @@ func (az *Cloud) selectLoadBalancer(clusterName string, service *v1.Service, exi if selectedLB == nil { err = fmt.Errorf("selectLoadBalancer: cluster(%s) service(%s) isInternal(%t) - unable to find load balancer for selected VM sets %v", clusterName, serviceName, isInternal, *vmSetNames) - glog.Error(err) + klog.Error(err) return nil, false, err } // validate if the selected LB has not exceeded the MaximumLoadBalancerRuleCount if az.Config.MaximumLoadBalancerRuleCount != 0 && selectedLBRuleCount >= az.Config.MaximumLoadBalancerRuleCount { err = fmt.Errorf("selectLoadBalancer: cluster(%s) service(%s) isInternal(%t) - all available load balancers have exceeded maximum rule limit %d, vmSetNames (%v)", clusterName, serviceName, isInternal, selectedLBRuleCount, *vmSetNames) - glog.Error(err) + klog.Error(err) return selectedLB, existsLb, err } @@ -335,11 +335,11 @@ func (az *Cloud) selectLoadBalancer(clusterName string, service *v1.Service, exi func (az *Cloud) getServiceLoadBalancerStatus(service *v1.Service, lb *network.LoadBalancer) (status *v1.LoadBalancerStatus, err error) { if lb == nil { - glog.V(10).Info("getServiceLoadBalancerStatus: lb is nil") + klog.V(10).Info("getServiceLoadBalancerStatus: lb is nil") return nil, nil } if lb.FrontendIPConfigurations == nil || *lb.FrontendIPConfigurations == nil { - glog.V(10).Info("getServiceLoadBalancerStatus: lb.FrontendIPConfigurations is nil") + klog.V(10).Info("getServiceLoadBalancerStatus: lb.FrontendIPConfigurations is nil") return nil, nil } isInternal := requiresInternalLoadBalancer(service) @@ -371,7 +371,7 @@ func (az *Cloud) getServiceLoadBalancerStatus(service *v1.Service, lb *network.L } } - glog.V(2).Infof("getServiceLoadBalancerStatus gets ingress IP %q from frontendIPConfiguration %q for service %q", to.String(lbIP), lbFrontendIPConfigName, serviceName) + klog.V(2).Infof("getServiceLoadBalancerStatus gets ingress IP %q from frontendIPConfiguration %q for service %q", to.String(lbIP), lbFrontendIPConfigName, serviceName) return &v1.LoadBalancerStatus{Ingress: []v1.LoadBalancerIngress{{IP: to.String(lbIP)}}}, nil } } @@ -434,11 +434,11 @@ func (az *Cloud) findServiceIPAddress(ctx context.Context, clusterName string, s return "", err } if !existsLb { - glog.V(2).Infof("Expected to find an IP address for service %s but did not. Assuming it has been removed", service.Name) + klog.V(2).Infof("Expected to find an IP address for service %s but did not. Assuming it has been removed", service.Name) return "", nil } if len(lbStatus.Ingress) < 1 { - glog.V(2).Infof("Expected to find an IP address for service %s but it had no ingresses. Assuming it has been removed", service.Name) + klog.V(2).Infof("Expected to find an IP address for service %s but it had no ingresses. Assuming it has been removed", service.Name) return "", nil } @@ -473,14 +473,14 @@ func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domai } } - glog.V(2).Infof("ensurePublicIPExists for service(%s): pip(%s) - creating", serviceName, *pip.Name) - glog.V(10).Infof("CreateOrUpdatePIPWithRetry(%s, %q): start", pipResourceGroup, *pip.Name) + klog.V(2).Infof("ensurePublicIPExists for service(%s): pip(%s) - creating", serviceName, *pip.Name) + klog.V(10).Infof("CreateOrUpdatePIPWithRetry(%s, %q): start", pipResourceGroup, *pip.Name) err = az.CreateOrUpdatePIPWithRetry(service, pipResourceGroup, pip) if err != nil { - glog.V(2).Infof("ensure(%s) abort backoff: pip(%s) - creating", serviceName, *pip.Name) + klog.V(2).Infof("ensure(%s) abort backoff: pip(%s) - creating", serviceName, *pip.Name) return nil, err } - glog.V(10).Infof("CreateOrUpdatePIPWithRetry(%s, %q): end", pipResourceGroup, *pip.Name) + klog.V(10).Infof("CreateOrUpdatePIPWithRetry(%s, %q): end", pipResourceGroup, *pip.Name) ctx, cancel := getContextWithCancel() defer cancel() @@ -570,14 +570,14 @@ func (az *Cloud) isFrontendIPChanged(clusterName string, config network.Frontend func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node, wantLb bool) (*network.LoadBalancer, error) { isInternal := requiresInternalLoadBalancer(service) serviceName := getServiceName(service) - glog.V(2).Infof("reconcileLoadBalancer for service(%s) - wantLb(%t): started", serviceName, wantLb) + klog.V(2).Infof("reconcileLoadBalancer for service(%s) - wantLb(%t): started", serviceName, wantLb) lb, _, _, err := az.getServiceLoadBalancer(service, clusterName, nodes, wantLb) if err != nil { - glog.Errorf("reconcileLoadBalancer: failed to get load balancer for service %q, error: %v", serviceName, err) + klog.Errorf("reconcileLoadBalancer: failed to get load balancer for service %q, error: %v", serviceName, err) return nil, err } lbName := *lb.Name - glog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) wantLb(%t) resolved load balancer name", serviceName, lbName, wantLb) + klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) wantLb(%t) resolved load balancer name", serviceName, lbName, wantLb) lbFrontendIPConfigName := az.getFrontendIPConfigName(service, subnet(service)) lbFrontendIPConfigID := az.getFrontendIPConfigID(lbName, lbFrontendIPConfigName) lbBackendPoolName := getBackendPoolName(clusterName) @@ -600,18 +600,18 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, foundBackendPool := false for _, bp := range newBackendPools { if strings.EqualFold(*bp.Name, lbBackendPoolName) { - glog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found wanted backendpool. not adding anything", serviceName, wantLb) + klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found wanted backendpool. not adding anything", serviceName, wantLb) foundBackendPool = true break } else { - glog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found other backendpool %s", serviceName, wantLb, *bp.Name) + klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found other backendpool %s", serviceName, wantLb, *bp.Name) } } if !foundBackendPool { newBackendPools = append(newBackendPools, network.BackendAddressPool{ Name: to.StringPtr(lbBackendPoolName), }) - glog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - adding backendpool", serviceName, wantLb) + klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - adding backendpool", serviceName, wantLb) dirtyLb = true lb.BackendAddressPools = &newBackendPools @@ -629,7 +629,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, for i := len(newConfigs) - 1; i >= 0; i-- { config := newConfigs[i] if az.serviceOwnsFrontendIP(config, service) { - glog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb frontendconfig(%s) - dropping", serviceName, wantLb, lbFrontendIPConfigName) + klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb frontendconfig(%s) - dropping", serviceName, wantLb, lbFrontendIPConfigName) newConfigs = append(newConfigs[:i], newConfigs[i+1:]...) dirtyConfigs = true } @@ -642,7 +642,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, return nil, err } if isFipChanged { - glog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb frontendconfig(%s) - dropping", serviceName, wantLb, *config.Name) + klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb frontendconfig(%s) - dropping", serviceName, wantLb, *config.Name) newConfigs = append(newConfigs[:i], newConfigs[i+1:]...) dirtyConfigs = true } @@ -705,7 +705,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, Name: to.StringPtr(lbFrontendIPConfigName), FrontendIPConfigurationPropertiesFormat: fipConfigurationProperties, }) - glog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb frontendconfig(%s) - adding", serviceName, wantLb, lbFrontendIPConfigName) + klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb frontendconfig(%s) - adding", serviceName, wantLb, lbFrontendIPConfigName) dirtyConfigs = true } } @@ -726,15 +726,15 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, for i := len(updatedProbes) - 1; i >= 0; i-- { existingProbe := updatedProbes[i] if az.serviceOwnsRule(service, *existingProbe.Name) { - glog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb probe(%s) - considering evicting", serviceName, wantLb, *existingProbe.Name) + klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb probe(%s) - considering evicting", serviceName, wantLb, *existingProbe.Name) keepProbe := false if findProbe(expectedProbes, existingProbe) { - glog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb probe(%s) - keeping", serviceName, wantLb, *existingProbe.Name) + klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb probe(%s) - keeping", serviceName, wantLb, *existingProbe.Name) keepProbe = true } if !keepProbe { updatedProbes = append(updatedProbes[:i], updatedProbes[i+1:]...) - glog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb probe(%s) - dropping", serviceName, wantLb, *existingProbe.Name) + klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb probe(%s) - dropping", serviceName, wantLb, *existingProbe.Name) dirtyProbes = true } } @@ -743,11 +743,11 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, for _, expectedProbe := range expectedProbes { foundProbe := false if findProbe(updatedProbes, expectedProbe) { - glog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb probe(%s) - already exists", serviceName, wantLb, *expectedProbe.Name) + klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb probe(%s) - already exists", serviceName, wantLb, *expectedProbe.Name) foundProbe = true } if !foundProbe { - glog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb probe(%s) - adding", serviceName, wantLb, *expectedProbe.Name) + klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb probe(%s) - adding", serviceName, wantLb, *expectedProbe.Name) updatedProbes = append(updatedProbes, expectedProbe) dirtyProbes = true } @@ -768,13 +768,13 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, existingRule := updatedRules[i] if az.serviceOwnsRule(service, *existingRule.Name) { keepRule := false - glog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - considering evicting", serviceName, wantLb, *existingRule.Name) + klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - considering evicting", serviceName, wantLb, *existingRule.Name) if findRule(expectedRules, existingRule) { - glog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - keeping", serviceName, wantLb, *existingRule.Name) + klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - keeping", serviceName, wantLb, *existingRule.Name) keepRule = true } if !keepRule { - glog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - dropping", serviceName, wantLb, *existingRule.Name) + klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - dropping", serviceName, wantLb, *existingRule.Name) updatedRules = append(updatedRules[:i], updatedRules[i+1:]...) dirtyRules = true } @@ -784,11 +784,11 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, for _, expectedRule := range expectedRules { foundRule := false if findRule(updatedRules, expectedRule) { - glog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - already exists", serviceName, wantLb, *expectedRule.Name) + klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - already exists", serviceName, wantLb, *expectedRule.Name) foundRule = true } if !foundRule { - glog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) adding", serviceName, wantLb, *expectedRule.Name) + klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) adding", serviceName, wantLb, *expectedRule.Name) updatedRules = append(updatedRules, expectedRule) dirtyRules = true } @@ -805,31 +805,31 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, if lb.FrontendIPConfigurations == nil || len(*lb.FrontendIPConfigurations) == 0 { // When FrontendIPConfigurations is empty, we need to delete the Azure load balancer resource itself, // because an Azure load balancer cannot have an empty FrontendIPConfigurations collection - glog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName) + klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName) // Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB. vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName) - glog.V(10).Infof("EnsureBackendPoolDeleted(%s, %s): start", lbBackendPoolID, vmSetName) + klog.V(10).Infof("EnsureBackendPoolDeleted(%s, %s): start", lbBackendPoolID, vmSetName) err := az.vmSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools) if err != nil { - glog.Errorf("EnsureBackendPoolDeleted(%s, %s) failed: %v", lbBackendPoolID, vmSetName, err) + klog.Errorf("EnsureBackendPoolDeleted(%s, %s) failed: %v", lbBackendPoolID, vmSetName, err) return nil, err } - glog.V(10).Infof("EnsureBackendPoolDeleted(%s, %s): end", lbBackendPoolID, vmSetName) + klog.V(10).Infof("EnsureBackendPoolDeleted(%s, %s): end", lbBackendPoolID, vmSetName) // Remove the LB. - glog.V(10).Infof("reconcileLoadBalancer: az.DeleteLBWithRetry(%q): start", lbName) + klog.V(10).Infof("reconcileLoadBalancer: az.DeleteLBWithRetry(%q): start", lbName) err = az.DeleteLBWithRetry(service, lbName) if err != nil { - glog.V(2).Infof("reconcileLoadBalancer for service(%s) abort backoff: lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName) + klog.V(2).Infof("reconcileLoadBalancer for service(%s) abort backoff: lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName) return nil, err } - glog.V(10).Infof("az.DeleteLBWithRetry(%q): end", lbName) + klog.V(10).Infof("az.DeleteLBWithRetry(%q): end", lbName) } else { - glog.V(2).Infof("reconcileLoadBalancer: reconcileLoadBalancer for service(%s): lb(%s) - updating", serviceName, lbName) + klog.V(2).Infof("reconcileLoadBalancer: reconcileLoadBalancer for service(%s): lb(%s) - updating", serviceName, lbName) err := az.CreateOrUpdateLBWithRetry(service, *lb) if err != nil { - glog.V(2).Infof("reconcileLoadBalancer for service(%s) abort backoff: lb(%s) - updating", serviceName, lbName) + klog.V(2).Infof("reconcileLoadBalancer for service(%s) abort backoff: lb(%s) - updating", serviceName, lbName) return nil, err } @@ -837,7 +837,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, // Refresh updated lb which will be used later in other places. newLB, exist, err := az.getAzureLoadBalancer(lbName) if err != nil { - glog.V(2).Infof("reconcileLoadBalancer for service(%s): getAzureLoadBalancer(%s) failed: %v", serviceName, lbName, err) + klog.V(2).Infof("reconcileLoadBalancer for service(%s): getAzureLoadBalancer(%s) failed: %v", serviceName, lbName, err) return nil, err } if !exist { @@ -857,7 +857,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, } } - glog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) finished", serviceName, lbName) + klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) finished", serviceName, lbName) return lb, nil } @@ -881,7 +881,7 @@ func (az *Cloud) reconcileLoadBalancerRule( for _, port := range ports { lbRuleName := az.getLoadBalancerRuleName(service, port, subnet(service)) - glog.V(2).Infof("reconcileLoadBalancerRule lb name (%s) rule name (%s)", lbName, lbRuleName) + klog.V(2).Infof("reconcileLoadBalancerRule lb name (%s) rule name (%s)", lbName, lbRuleName) transportProto, _, probeProto, err := getProtocolsFromKubernetesProtocol(port.Protocol) if err != nil { @@ -956,12 +956,12 @@ func (az *Cloud) reconcileLoadBalancerRule( // This entails adding required, missing SecurityRules and removing stale rules. func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, lbIP *string, wantLb bool) (*network.SecurityGroup, error) { serviceName := getServiceName(service) - glog.V(5).Infof("reconcileSecurityGroup(%s): START clusterName=%q", serviceName, clusterName) + klog.V(5).Infof("reconcileSecurityGroup(%s): START clusterName=%q", serviceName, clusterName) ports := service.Spec.Ports if ports == nil { if useSharedSecurityRule(service) { - glog.V(2).Infof("Attempting to reconcile security group for service %s, but service uses shared rule and we don't know which port it's for", service.Name) + klog.V(2).Infof("Attempting to reconcile security group for service %s, but service uses shared rule and we don't know which port it's for", service.Name) return nil, fmt.Errorf("No port info for reconciling shared rule for service %s", service.Name) } ports = []v1.ServicePort{} @@ -1034,7 +1034,7 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, } for _, r := range expectedSecurityRules { - glog.V(10).Infof("Expecting security rule for %s: %s:%s -> %s:%s", service.Name, *r.SourceAddressPrefix, *r.SourcePortRange, *r.DestinationAddressPrefix, *r.DestinationPortRange) + klog.V(10).Infof("Expecting security rule for %s: %s:%s -> %s:%s", service.Name, *r.SourceAddressPrefix, *r.SourcePortRange, *r.DestinationAddressPrefix, *r.DestinationPortRange) } // update security rules @@ -1045,7 +1045,7 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, } for _, r := range updatedRules { - glog.V(10).Infof("Existing security rule while processing %s: %s:%s -> %s:%s", service.Name, logSafe(r.SourceAddressPrefix), logSafe(r.SourcePortRange), logSafeCollection(r.DestinationAddressPrefix, r.DestinationAddressPrefixes), logSafe(r.DestinationPortRange)) + klog.V(10).Infof("Existing security rule while processing %s: %s:%s -> %s:%s", service.Name, logSafe(r.SourceAddressPrefix), logSafe(r.SourcePortRange), logSafeCollection(r.DestinationAddressPrefix, r.DestinationAddressPrefixes), logSafe(r.DestinationPortRange)) } // update security rules: remove unwanted rules that belong privately @@ -1053,14 +1053,14 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, for i := len(updatedRules) - 1; i >= 0; i-- { existingRule := updatedRules[i] if az.serviceOwnsRule(service, *existingRule.Name) { - glog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - considering evicting", serviceName, wantLb, *existingRule.Name) + klog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - considering evicting", serviceName, wantLb, *existingRule.Name) keepRule := false if findSecurityRule(expectedSecurityRules, existingRule) { - glog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - keeping", serviceName, wantLb, *existingRule.Name) + klog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - keeping", serviceName, wantLb, *existingRule.Name) keepRule = true } if !keepRule { - glog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - dropping", serviceName, wantLb, *existingRule.Name) + klog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - dropping", serviceName, wantLb, *existingRule.Name) updatedRules = append(updatedRules[:i], updatedRules[i+1:]...) dirtySg = true } @@ -1074,17 +1074,17 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, sharedRuleName := az.getSecurityRuleName(service, port, sourceAddressPrefix) sharedIndex, sharedRule, sharedRuleFound := findSecurityRuleByName(updatedRules, sharedRuleName) if !sharedRuleFound { - glog.V(4).Infof("Expected to find shared rule %s for service %s being deleted, but did not", sharedRuleName, service.Name) + klog.V(4).Infof("Expected to find shared rule %s for service %s being deleted, but did not", sharedRuleName, service.Name) return nil, fmt.Errorf("Expected to find shared rule %s for service %s being deleted, but did not", sharedRuleName, service.Name) } if sharedRule.DestinationAddressPrefixes == nil { - glog.V(4).Infof("Expected to have array of destinations in shared rule for service %s being deleted, but did not", service.Name) + klog.V(4).Infof("Expected to have array of destinations in shared rule for service %s being deleted, but did not", service.Name) return nil, fmt.Errorf("Expected to have array of destinations in shared rule for service %s being deleted, but did not", service.Name) } existingPrefixes := *sharedRule.DestinationAddressPrefixes addressIndex, found := findIndex(existingPrefixes, destinationIPAddress) if !found { - glog.V(4).Infof("Expected to find destination address %s in shared rule %s for service %s being deleted, but did not", destinationIPAddress, sharedRuleName, service.Name) + klog.V(4).Infof("Expected to find destination address %s in shared rule %s for service %s being deleted, but did not", destinationIPAddress, sharedRuleName, service.Name) return nil, fmt.Errorf("Expected to find destination address %s in shared rule %s for service %s being deleted, but did not", destinationIPAddress, sharedRuleName, service.Name) } if len(existingPrefixes) == 1 { @@ -1114,7 +1114,7 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, for _, expectedRule := range expectedSecurityRules { foundRule := false if findSecurityRule(updatedRules, expectedRule) { - glog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - already exists", serviceName, wantLb, *expectedRule.Name) + klog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - already exists", serviceName, wantLb, *expectedRule.Name) foundRule = true } if foundRule && allowsConsolidation(expectedRule) { @@ -1123,7 +1123,7 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, dirtySg = true } if !foundRule { - glog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - adding", serviceName, wantLb, *expectedRule.Name) + klog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - adding", serviceName, wantLb, *expectedRule.Name) nextAvailablePriority, err := getNextAvailablePriority(updatedRules) if err != nil { @@ -1137,16 +1137,16 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, } for _, r := range updatedRules { - glog.V(10).Infof("Updated security rule while processing %s: %s:%s -> %s:%s", service.Name, logSafe(r.SourceAddressPrefix), logSafe(r.SourcePortRange), logSafeCollection(r.DestinationAddressPrefix, r.DestinationAddressPrefixes), logSafe(r.DestinationPortRange)) + klog.V(10).Infof("Updated security rule while processing %s: %s:%s -> %s:%s", service.Name, logSafe(r.SourceAddressPrefix), logSafe(r.SourcePortRange), logSafeCollection(r.DestinationAddressPrefix, r.DestinationAddressPrefixes), logSafe(r.DestinationPortRange)) } if dirtySg { sg.SecurityRules = &updatedRules - glog.V(2).Infof("reconcileSecurityGroup for service(%s): sg(%s) - updating", serviceName, *sg.Name) - glog.V(10).Infof("CreateOrUpdateSGWithRetry(%q): start", *sg.Name) + klog.V(2).Infof("reconcileSecurityGroup for service(%s): sg(%s) - updating", serviceName, *sg.Name) + klog.V(10).Infof("CreateOrUpdateSGWithRetry(%q): start", *sg.Name) err := az.CreateOrUpdateSGWithRetry(service, sg) if err != nil { - glog.V(2).Infof("ensure(%s) abort backoff: sg(%s) - updating", serviceName, *sg.Name) + klog.V(2).Infof("ensure(%s) abort backoff: sg(%s) - updating", serviceName, *sg.Name) // TODO (Nov 2017): remove when augmented security rules are out of preview // we could try to parse the response but it's not worth it for bridging a preview errorDescription := err.Error() @@ -1157,7 +1157,7 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, // END TODO return nil, err } - glog.V(10).Infof("CreateOrUpdateSGWithRetry(%q): end", *sg.Name) + klog.V(10).Infof("CreateOrUpdateSGWithRetry(%q): end", *sg.Name) } return &sg, nil } @@ -1331,13 +1331,13 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lb * // This is the only case we should preserve the // Public ip resource with match service tag } else { - glog.V(2).Infof("reconcilePublicIP for service(%s): pip(%s) - deleting", serviceName, pipName) + klog.V(2).Infof("reconcilePublicIP for service(%s): pip(%s) - deleting", serviceName, pipName) err := az.safeDeletePublicIP(service, pipResourceGroup, &pip, lb) if err != nil { - glog.Errorf("safeDeletePublicIP(%s) failed with error: %v", pipName, err) + klog.Errorf("safeDeletePublicIP(%s) failed with error: %v", pipName, err) return nil, err } - glog.V(2).Infof("reconcilePublicIP for service(%s): pip(%s) - finished", serviceName, pipName) + klog.V(2).Infof("reconcilePublicIP for service(%s): pip(%s) - finished", serviceName, pipName) } } @@ -1416,21 +1416,21 @@ func (az *Cloud) safeDeletePublicIP(service *v1.Service, pipResourceGroup string if frontendIPConfigUpdated || loadBalancerRuleUpdated { err := az.CreateOrUpdateLBWithRetry(service, *lb) if err != nil { - glog.Errorf("safeDeletePublicIP for service(%s) failed with error: %v", getServiceName(service), err) + klog.Errorf("safeDeletePublicIP for service(%s) failed with error: %v", getServiceName(service), err) return err } } } pipName := to.String(pip.Name) - glog.V(10).Infof("DeletePublicIPWithRetry(%s, %q): start", pipResourceGroup, pipName) + klog.V(10).Infof("DeletePublicIPWithRetry(%s, %q): start", pipResourceGroup, pipName) err := az.DeletePublicIPWithRetry(service, pipResourceGroup, pipName) if err != nil { if err = ignoreStatusNotFoundFromError(err); err != nil { return err } } - glog.V(10).Infof("DeletePublicIPWithRetry(%s, %q): end", pipResourceGroup, pipName) + klog.V(10).Infof("DeletePublicIPWithRetry(%s, %q): end", pipResourceGroup, pipName) return nil } diff --git a/pkg/cloudprovider/providers/azure/azure_managedDiskController.go b/pkg/cloudprovider/providers/azure/azure_managedDiskController.go index 1bbd65ac19a..110636531d3 100644 --- a/pkg/cloudprovider/providers/azure/azure_managedDiskController.go +++ b/pkg/cloudprovider/providers/azure/azure_managedDiskController.go @@ -25,7 +25,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute" "github.com/Azure/go-autorest/autorest/to" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -75,7 +75,7 @@ func newManagedDiskController(common *controllerCommon) (*ManagedDiskController, //CreateManagedDisk : create managed disk func (c *ManagedDiskController) CreateManagedDisk(options *ManagedDiskOptions) (string, error) { var err error - glog.V(4).Infof("azureDisk - creating new managed Name:%s StorageAccountType:%s Size:%v", options.DiskName, options.StorageAccountType, options.SizeGB) + klog.V(4).Infof("azureDisk - creating new managed Name:%s StorageAccountType:%s Size:%v", options.DiskName, options.StorageAccountType, options.SizeGB) var createZones *[]string if len(options.AvailabilityZone) > 0 { @@ -171,9 +171,9 @@ func (c *ManagedDiskController) CreateManagedDisk(options *ManagedDiskOptions) ( }) if err != nil { - glog.V(2).Infof("azureDisk - created new MD Name:%s StorageAccountType:%s Size:%v but was unable to confirm provisioningState in poll process", options.DiskName, options.StorageAccountType, options.SizeGB) + klog.V(2).Infof("azureDisk - created new MD Name:%s StorageAccountType:%s Size:%v but was unable to confirm provisioningState in poll process", options.DiskName, options.StorageAccountType, options.SizeGB) } else { - glog.V(2).Infof("azureDisk - created new MD Name:%s StorageAccountType:%s Size:%v", options.DiskName, options.StorageAccountType, options.SizeGB) + klog.V(2).Infof("azureDisk - created new MD Name:%s StorageAccountType:%s Size:%v", options.DiskName, options.StorageAccountType, options.SizeGB) } return diskID, nil @@ -197,7 +197,7 @@ func (c *ManagedDiskController) DeleteManagedDisk(diskURI string) error { // We don't need poll here, k8s will immediately stop referencing the disk // the disk will be eventually deleted - cleanly - by ARM - glog.V(2).Infof("azureDisk - deleted a managed disk: %s", diskURI) + klog.V(2).Infof("azureDisk - deleted a managed disk: %s", diskURI) return nil } @@ -244,7 +244,7 @@ func (c *ManagedDiskController) ResizeDisk(diskURI string, oldSize resource.Quan requestGiB := int32(util.RoundUpSize(requestBytes, 1024*1024*1024)) newSizeQuant := resource.MustParse(fmt.Sprintf("%dGi", requestGiB)) - glog.V(2).Infof("azureDisk - begin to resize disk(%s) with new size(%d), old size(%v)", diskName, requestGiB, oldSize) + klog.V(2).Infof("azureDisk - begin to resize disk(%s) with new size(%d), old size(%v)", diskName, requestGiB, oldSize) // If disk already of greater or equal size than requested we return if *result.DiskProperties.DiskSizeGB >= requestGiB { return newSizeQuant, nil @@ -258,7 +258,7 @@ func (c *ManagedDiskController) ResizeDisk(diskURI string, oldSize resource.Quan return oldSize, err } - glog.V(2).Infof("azureDisk - resize disk(%s) with new size(%d) completed", diskName, requestGiB) + klog.V(2).Infof("azureDisk - resize disk(%s) with new size(%d) completed", diskName, requestGiB) return newSizeQuant, nil } @@ -295,7 +295,7 @@ func (c *Cloud) GetAzureDiskLabels(diskURI string) (map[string]string, error) { diskName := path.Base(diskURI) resourceGroup, err := getResourceGroupFromDiskURI(diskURI) if err != nil { - glog.Errorf("Failed to get resource group for AzureDisk %q: %v", diskName, err) + klog.Errorf("Failed to get resource group for AzureDisk %q: %v", diskName, err) return nil, err } @@ -304,13 +304,13 @@ func (c *Cloud) GetAzureDiskLabels(diskURI string) (map[string]string, error) { defer cancel() disk, err := c.DisksClient.Get(ctx, resourceGroup, diskName) if err != nil { - glog.Errorf("Failed to get information for AzureDisk %q: %v", diskName, err) + klog.Errorf("Failed to get information for AzureDisk %q: %v", diskName, err) return nil, err } // Check whether availability zone is specified. if disk.Zones == nil || len(*disk.Zones) == 0 { - glog.V(4).Infof("Azure disk %q is not zoned", diskName) + klog.V(4).Infof("Azure disk %q is not zoned", diskName) return nil, nil } @@ -321,7 +321,7 @@ func (c *Cloud) GetAzureDiskLabels(diskURI string) (map[string]string, error) { } zone := c.makeZone(zoneID) - glog.V(4).Infof("Got zone %q for Azure disk %q", zone, diskName) + klog.V(4).Infof("Got zone %q for Azure disk %q", zone, diskName) labels := map[string]string{ kubeletapis.LabelZoneRegion: c.Location, kubeletapis.LabelZoneFailureDomain: zone, diff --git a/pkg/cloudprovider/providers/azure/azure_routes.go b/pkg/cloudprovider/providers/azure/azure_routes.go index a737de6c7f5..36219b2f917 100644 --- a/pkg/cloudprovider/providers/azure/azure_routes.go +++ b/pkg/cloudprovider/providers/azure/azure_routes.go @@ -24,13 +24,13 @@ import ( "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network" "github.com/Azure/go-autorest/autorest/to" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" ) // ListRoutes lists all managed routes that belong to the specified clusterName func (az *Cloud) ListRoutes(ctx context.Context, clusterName string) ([]*cloudprovider.Route, error) { - glog.V(10).Infof("ListRoutes: START clusterName=%q", clusterName) + klog.V(10).Infof("ListRoutes: START clusterName=%q", clusterName) routeTable, existsRouteTable, err := az.getRouteTable() routes, err := processRoutes(routeTable, existsRouteTable, err) if err != nil { @@ -72,7 +72,7 @@ func processRoutes(routeTable network.RouteTable, exists bool, err error) ([]*cl for i, route := range *routeTable.Routes { instance := mapRouteNameToNodeName(*route.Name) cidr := *route.AddressPrefix - glog.V(10).Infof("ListRoutes: * instance=%q, cidr=%q", instance, cidr) + klog.V(10).Infof("ListRoutes: * instance=%q, cidr=%q", instance, cidr) kubeRoutes[i] = &cloudprovider.Route{ Name: *route.Name, @@ -82,13 +82,13 @@ func processRoutes(routeTable network.RouteTable, exists bool, err error) ([]*cl } } - glog.V(10).Info("ListRoutes: FINISH") + klog.V(10).Info("ListRoutes: FINISH") return kubeRoutes, nil } func (az *Cloud) createRouteTableIfNotExists(clusterName string, kubeRoute *cloudprovider.Route) error { if _, existsRouteTable, err := az.getRouteTable(); err != nil { - glog.V(2).Infof("createRouteTableIfNotExists error: couldn't get routetable. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) + klog.V(2).Infof("createRouteTableIfNotExists error: couldn't get routetable. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) return err } else if existsRouteTable { return nil @@ -103,17 +103,17 @@ func (az *Cloud) createRouteTable() error { RouteTablePropertiesFormat: &network.RouteTablePropertiesFormat{}, } - glog.V(3).Infof("createRouteTableIfNotExists: creating routetable. routeTableName=%q", az.RouteTableName) + klog.V(3).Infof("createRouteTableIfNotExists: creating routetable. routeTableName=%q", az.RouteTableName) ctx, cancel := getContextWithCancel() defer cancel() resp, err := az.RouteTablesClient.CreateOrUpdate(ctx, az.ResourceGroup, az.RouteTableName, routeTable) - glog.V(10).Infof("RouteTablesClient.CreateOrUpdate(%q): end", az.RouteTableName) + klog.V(10).Infof("RouteTablesClient.CreateOrUpdate(%q): end", az.RouteTableName) if az.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { - glog.V(2).Infof("createRouteTableIfNotExists backing off: creating routetable. routeTableName=%q", az.RouteTableName) + klog.V(2).Infof("createRouteTableIfNotExists backing off: creating routetable. routeTableName=%q", az.RouteTableName) retryErr := az.CreateOrUpdateRouteTableWithRetry(routeTable) if retryErr != nil { err = retryErr - glog.V(2).Infof("createRouteTableIfNotExists abort backoff: creating routetable. routeTableName=%q", az.RouteTableName) + klog.V(2).Infof("createRouteTableIfNotExists abort backoff: creating routetable. routeTableName=%q", az.RouteTableName) } } if err != nil { @@ -136,14 +136,14 @@ func (az *Cloud) CreateRoute(ctx context.Context, clusterName string, nameHint s return err } if unmanaged { - glog.V(2).Infof("CreateRoute: omitting unmanaged node %q", kubeRoute.TargetNode) + klog.V(2).Infof("CreateRoute: omitting unmanaged node %q", kubeRoute.TargetNode) az.routeCIDRsLock.Lock() defer az.routeCIDRsLock.Unlock() az.routeCIDRs[nodeName] = kubeRoute.DestinationCIDR return nil } - glog.V(2).Infof("CreateRoute: creating route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) + klog.V(2).Infof("CreateRoute: creating route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) if err := az.createRouteTableIfNotExists(clusterName, kubeRoute); err != nil { return err } @@ -162,24 +162,24 @@ func (az *Cloud) CreateRoute(ctx context.Context, clusterName string, nameHint s }, } - glog.V(3).Infof("CreateRoute: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR) + klog.V(3).Infof("CreateRoute: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR) ctx, cancel := getContextWithCancel() defer cancel() resp, err := az.RoutesClient.CreateOrUpdate(ctx, az.ResourceGroup, az.RouteTableName, *route.Name, route) - glog.V(10).Infof("RoutesClient.CreateOrUpdate(%q): end", az.RouteTableName) + klog.V(10).Infof("RoutesClient.CreateOrUpdate(%q): end", az.RouteTableName) if az.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { - glog.V(2).Infof("CreateRoute backing off: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR) + klog.V(2).Infof("CreateRoute backing off: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR) retryErr := az.CreateOrUpdateRouteWithRetry(route) if retryErr != nil { err = retryErr - glog.V(2).Infof("CreateRoute abort backoff: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR) + klog.V(2).Infof("CreateRoute abort backoff: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR) } } if err != nil { return err } - glog.V(2).Infof("CreateRoute: route created. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) + klog.V(2).Infof("CreateRoute: route created. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) return nil } @@ -193,34 +193,34 @@ func (az *Cloud) DeleteRoute(ctx context.Context, clusterName string, kubeRoute return err } if unmanaged { - glog.V(2).Infof("DeleteRoute: omitting unmanaged node %q", kubeRoute.TargetNode) + klog.V(2).Infof("DeleteRoute: omitting unmanaged node %q", kubeRoute.TargetNode) az.routeCIDRsLock.Lock() defer az.routeCIDRsLock.Unlock() delete(az.routeCIDRs, nodeName) return nil } - glog.V(2).Infof("DeleteRoute: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) + klog.V(2).Infof("DeleteRoute: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) ctx, cancel := getContextWithCancel() defer cancel() routeName := mapNodeNameToRouteName(kubeRoute.TargetNode) resp, err := az.RoutesClient.Delete(ctx, az.ResourceGroup, az.RouteTableName, routeName) - glog.V(10).Infof("RoutesClient.Delete(%q): end", az.RouteTableName) + klog.V(10).Infof("RoutesClient.Delete(%q): end", az.RouteTableName) if az.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { - glog.V(2).Infof("DeleteRoute backing off: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) + klog.V(2).Infof("DeleteRoute backing off: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) retryErr := az.DeleteRouteWithRetry(routeName) if retryErr != nil { err = retryErr - glog.V(2).Infof("DeleteRoute abort backoff: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) + klog.V(2).Infof("DeleteRoute abort backoff: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) } } if err != nil { return err } - glog.V(2).Infof("DeleteRoute: route deleted. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) + klog.V(2).Infof("DeleteRoute: route deleted. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) return nil } diff --git a/pkg/cloudprovider/providers/azure/azure_standard.go b/pkg/cloudprovider/providers/azure/azure_standard.go index dc565276b1b..95126f58b0f 100644 --- a/pkg/cloudprovider/providers/azure/azure_standard.go +++ b/pkg/cloudprovider/providers/azure/azure_standard.go @@ -32,11 +32,11 @@ import ( "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network" "github.com/Azure/go-autorest/autorest/to" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/klog" ) const ( @@ -333,10 +333,10 @@ func (as *availabilitySet) GetInstanceIDByNodeName(name string) (string, error) } if err != nil { if as.CloudProviderBackoff { - glog.V(2).Infof("GetInstanceIDByNodeName(%s) backing off", name) + klog.V(2).Infof("GetInstanceIDByNodeName(%s) backing off", name) machine, err = as.GetVirtualMachineWithRetry(types.NodeName(name)) if err != nil { - glog.V(2).Infof("GetInstanceIDByNodeName(%s) abort backoff", name) + klog.V(2).Infof("GetInstanceIDByNodeName(%s) abort backoff", name) return "", err } } else { @@ -381,7 +381,7 @@ func (as *availabilitySet) GetNodeNameByProviderID(providerID string) (types.Nod func (as *availabilitySet) GetInstanceTypeByNodeName(name string) (string, error) { machine, err := as.getVirtualMachine(types.NodeName(name)) if err != nil { - glog.Errorf("as.GetInstanceTypeByNodeName(%s) failed: as.getVirtualMachine(%s) err=%v", name, name, err) + klog.Errorf("as.GetInstanceTypeByNodeName(%s) failed: as.getVirtualMachine(%s) err=%v", name, name, err) return "", err } @@ -433,7 +433,7 @@ func (as *availabilitySet) GetIPByNodeName(name string) (string, string, error) ipConfig, err := getPrimaryIPConfig(nic) if err != nil { - glog.Errorf("as.GetIPByNodeName(%s) failed: getPrimaryIPConfig(%v), err=%v", name, nic, err) + klog.Errorf("as.GetIPByNodeName(%s) failed: getPrimaryIPConfig(%v), err=%v", name, nic, err) return "", "", err } @@ -462,7 +462,7 @@ func (as *availabilitySet) GetIPByNodeName(name string) (string, string, error) func (as *availabilitySet) getAgentPoolAvailabiliySets(nodes []*v1.Node) (agentPoolAvailabilitySets *[]string, err error) { vms, err := as.VirtualMachineClientListWithRetry(as.ResourceGroup) if err != nil { - glog.Errorf("as.getNodeAvailabilitySet - VirtualMachineClientListWithRetry failed, err=%v", err) + klog.Errorf("as.getNodeAvailabilitySet - VirtualMachineClientListWithRetry failed, err=%v", err) return nil, err } vmNameToAvailabilitySetID := make(map[string]string, len(vms)) @@ -481,7 +481,7 @@ func (as *availabilitySet) getAgentPoolAvailabiliySets(nodes []*v1.Node) (agentP } asID, ok := vmNameToAvailabilitySetID[nodeName] if !ok { - glog.Errorf("as.getNodeAvailabilitySet - Node(%s) has no availability sets", nodeName) + klog.Errorf("as.getNodeAvailabilitySet - Node(%s) has no availability sets", nodeName) return nil, fmt.Errorf("Node (%s) - has no availability sets", nodeName) } if availabilitySetIDs.Has(asID) { @@ -490,7 +490,7 @@ func (as *availabilitySet) getAgentPoolAvailabiliySets(nodes []*v1.Node) (agentP } asName, err := getLastSegment(asID) if err != nil { - glog.Errorf("as.getNodeAvailabilitySet - Node (%s)- getLastSegment(%s), err=%v", nodeName, asID, err) + klog.Errorf("as.getNodeAvailabilitySet - Node (%s)- getLastSegment(%s), err=%v", nodeName, asID, err) return nil, err } // AvailabilitySet ID is currently upper cased in a indeterministic way @@ -516,11 +516,11 @@ func (as *availabilitySet) GetVMSetNames(service *v1.Service, nodes []*v1.Node) } availabilitySetNames, err = as.getAgentPoolAvailabiliySets(nodes) if err != nil { - glog.Errorf("as.GetVMSetNames - getAgentPoolAvailabiliySets failed err=(%v)", err) + klog.Errorf("as.GetVMSetNames - getAgentPoolAvailabiliySets failed err=(%v)", err) return nil, err } if len(*availabilitySetNames) == 0 { - glog.Errorf("as.GetVMSetNames - No availability sets found for nodes in the cluster, node count(%d)", len(nodes)) + klog.Errorf("as.GetVMSetNames - No availability sets found for nodes in the cluster, node count(%d)", len(nodes)) return nil, fmt.Errorf("No availability sets found for nodes, node count(%d)", len(nodes)) } // sort the list to have deterministic selection @@ -540,7 +540,7 @@ func (as *availabilitySet) GetVMSetNames(service *v1.Service, nodes []*v1.Node) } } if !found { - glog.Errorf("as.GetVMSetNames - Availability set (%s) in service annotation not found", serviceAvailabilitySetNames[sasx]) + klog.Errorf("as.GetVMSetNames - Availability set (%s) in service annotation not found", serviceAvailabilitySetNames[sasx]) return nil, fmt.Errorf("availability set (%s) - not found", serviceAvailabilitySetNames[sasx]) } } @@ -581,7 +581,7 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri machine, err := as.GetVirtualMachineWithRetry(types.NodeName(nodeName)) if err != nil { - glog.V(2).Infof("GetPrimaryInterface(%s, %s) abort backoff", nodeName, vmSetName) + klog.V(2).Infof("GetPrimaryInterface(%s, %s) abort backoff", nodeName, vmSetName) return network.Interface{}, err } @@ -608,7 +608,7 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri if vmSetName != "" && !as.useStandardLoadBalancer() { expectedAvailabilitySetName := as.getAvailabilitySetID(nodeResourceGroup, vmSetName) if machine.AvailabilitySet == nil || !strings.EqualFold(*machine.AvailabilitySet.ID, expectedAvailabilitySetName) { - glog.V(3).Infof( + klog.V(3).Infof( "GetPrimaryInterface: nic (%s) is not in the availabilitySet(%s)", nicName, vmSetName) return network.Interface{}, errNotInVMSet } @@ -637,16 +637,16 @@ func (as *availabilitySet) ensureHostInPool(service *v1.Service, nodeName types. nic, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName) if err != nil { if err == errNotInVMSet { - glog.V(3).Infof("ensureHostInPool skips node %s because it is not in the vmSet %s", nodeName, vmSetName) + klog.V(3).Infof("ensureHostInPool skips node %s because it is not in the vmSet %s", nodeName, vmSetName) return nil } - glog.Errorf("error: az.ensureHostInPool(%s), az.vmSet.GetPrimaryInterface.Get(%s, %s), err=%v", nodeName, vmName, vmSetName, err) + klog.Errorf("error: az.ensureHostInPool(%s), az.vmSet.GetPrimaryInterface.Get(%s, %s), err=%v", nodeName, vmName, vmSetName, err) return err } if nic.ProvisioningState != nil && *nic.ProvisioningState == nicFailedState { - glog.V(3).Infof("ensureHostInPool skips node %s because its primary nic %s is in Failed state", nodeName, *nic.Name) + klog.V(3).Infof("ensureHostInPool skips node %s because its primary nic %s is in Failed state", nodeName, *nic.Name) return nil } @@ -679,7 +679,7 @@ func (as *availabilitySet) ensureHostInPool(service *v1.Service, nodeName types. if len(matches) == 2 { lbName := matches[1] if strings.HasSuffix(lbName, InternalLoadBalancerNameSuffix) == isInternal { - glog.V(4).Infof("Node %q has already been added to LB %q, omit adding it to a new one", nodeName, lbName) + klog.V(4).Infof("Node %q has already been added to LB %q, omit adding it to a new one", nodeName, lbName) return nil } } @@ -694,17 +694,17 @@ func (as *availabilitySet) ensureHostInPool(service *v1.Service, nodeName types. primaryIPConfig.LoadBalancerBackendAddressPools = &newBackendPools nicName := *nic.Name - glog.V(3).Infof("nicupdate(%s): nic(%s) - updating", serviceName, nicName) + klog.V(3).Infof("nicupdate(%s): nic(%s) - updating", serviceName, nicName) ctx, cancel := getContextWithCancel() defer cancel() resp, err := as.InterfacesClient.CreateOrUpdate(ctx, as.ResourceGroup, *nic.Name, nic) - glog.V(10).Infof("InterfacesClient.CreateOrUpdate(%q): end", *nic.Name) + klog.V(10).Infof("InterfacesClient.CreateOrUpdate(%q): end", *nic.Name) if as.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { - glog.V(2).Infof("nicupdate(%s) backing off: nic(%s) - updating, err=%v", serviceName, nicName, err) + klog.V(2).Infof("nicupdate(%s) backing off: nic(%s) - updating, err=%v", serviceName, nicName, err) retryErr := as.CreateOrUpdateInterfaceWithRetry(service, nic) if retryErr != nil { err = retryErr - glog.V(2).Infof("nicupdate(%s) abort backoff: nic(%s) - updating", serviceName, nicName) + klog.V(2).Infof("nicupdate(%s) abort backoff: nic(%s) - updating", serviceName, nicName) } } if err != nil { @@ -721,12 +721,12 @@ func (as *availabilitySet) EnsureHostsInPool(service *v1.Service, nodes []*v1.No for _, node := range nodes { localNodeName := node.Name if as.useStandardLoadBalancer() && as.excludeMasterNodesFromStandardLB() && isMasterNode(node) { - glog.V(4).Infof("Excluding master node %q from load balancer backendpool %q", localNodeName, backendPoolID) + klog.V(4).Infof("Excluding master node %q from load balancer backendpool %q", localNodeName, backendPoolID) continue } if as.ShouldNodeExcludedFromLoadBalancer(node) { - glog.V(4).Infof("Excluding unmanaged/external-resource-group node %q", localNodeName) + klog.V(4).Infof("Excluding unmanaged/external-resource-group node %q", localNodeName) continue } diff --git a/pkg/cloudprovider/providers/azure/azure_storage.go b/pkg/cloudprovider/providers/azure/azure_storage.go index 7c95487be26..19b91fc99ba 100644 --- a/pkg/cloudprovider/providers/azure/azure_storage.go +++ b/pkg/cloudprovider/providers/azure/azure_storage.go @@ -20,7 +20,7 @@ import ( "fmt" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -46,7 +46,7 @@ func (az *Cloud) CreateFileShare(shareName, accountName, accountType, accountKin if err := az.createFileShare(account, key, shareName, requestGiB); err != nil { return "", "", fmt.Errorf("failed to create share %s in account %s: %v", shareName, account, err) } - glog.V(4).Infof("created share %s in account %s", shareName, account) + klog.V(4).Infof("created share %s in account %s", shareName, account) return account, key, nil } @@ -55,7 +55,7 @@ func (az *Cloud) DeleteFileShare(accountName, accountKey, shareName string) erro if err := az.deleteFileShare(accountName, accountKey, shareName); err != nil { return err } - glog.V(4).Infof("share %s deleted", shareName) + klog.V(4).Infof("share %s deleted", shareName) return nil } diff --git a/pkg/cloudprovider/providers/azure/azure_storageaccount.go b/pkg/cloudprovider/providers/azure/azure_storageaccount.go index f0d50091474..34871a11198 100644 --- a/pkg/cloudprovider/providers/azure/azure_storageaccount.go +++ b/pkg/cloudprovider/providers/azure/azure_storageaccount.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage" "github.com/Azure/go-autorest/autorest/to" - "github.com/golang/glog" + "k8s.io/klog" ) type accountWithLocation struct { @@ -100,7 +100,7 @@ func (az *Cloud) ensureStorageAccount(accountName, accountType, accountKind, res if len(accounts) > 0 { accountName = accounts[0].Name - glog.V(4).Infof("found a matching account %s type %s location %s", accounts[0].Name, accounts[0].StorageType, accounts[0].Location) + klog.V(4).Infof("found a matching account %s type %s location %s", accounts[0].Name, accounts[0].StorageType, accounts[0].Location) } if len(accountName) == 0 { @@ -118,7 +118,7 @@ func (az *Cloud) ensureStorageAccount(accountName, accountType, accountKind, res if accountKind != "" { kind = storage.Kind(accountKind) } - glog.V(2).Infof("azure - no matching account found, begin to create a new account %s in resource group %s, location: %s, accountType: %s, accountKind: %s", + klog.V(2).Infof("azure - no matching account found, begin to create a new account %s in resource group %s, location: %s, accountType: %s, accountKind: %s", accountName, resourceGroup, location, accountType, kind) cp := storage.AccountCreateParameters{ Sku: &storage.Sku{Name: storage.SkuName(accountType)}, diff --git a/pkg/cloudprovider/providers/azure/azure_vmss.go b/pkg/cloudprovider/providers/azure/azure_vmss.go index ca09f46e88d..4d5df404628 100644 --- a/pkg/cloudprovider/providers/azure/azure_vmss.go +++ b/pkg/cloudprovider/providers/azure/azure_vmss.go @@ -27,7 +27,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network" "github.com/Azure/go-autorest/autorest/to" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -113,7 +113,7 @@ func (ss *scaleSet) getVmssVM(nodeName string) (ssName, instanceID string, vm co return "", "", vm, err } - glog.V(4).Infof("getVmssVM gets scaleSetName (%q) and instanceID (%q) for node %q", ssName, instanceID, nodeName) + klog.V(4).Infof("getVmssVM gets scaleSetName (%q) and instanceID (%q) for node %q", ssName, instanceID, nodeName) key := buildVmssCacheKey(resourceGroup, ss.makeVmssVMName(ssName, instanceID)) cachedVM, err := ss.vmssVMCache.Get(key) if err != nil { @@ -121,7 +121,7 @@ func (ss *scaleSet) getVmssVM(nodeName string) (ssName, instanceID string, vm co } if cachedVM == nil { - glog.Errorf("Can't find node (%q) in any scale sets", nodeName) + klog.Errorf("Can't find node (%q) in any scale sets", nodeName) return ssName, instanceID, vm, cloudprovider.InstanceNotFound } @@ -159,7 +159,7 @@ func (ss *scaleSet) getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceI } if cachedVM == nil { - glog.Errorf("couldn't find vmss virtual machine by scaleSetName (%s) and instanceID (%s)", scaleSetName, instanceID) + klog.Errorf("couldn't find vmss virtual machine by scaleSetName (%s) and instanceID (%s)", scaleSetName, instanceID) return vm, cloudprovider.InstanceNotFound } @@ -172,7 +172,7 @@ func (ss *scaleSet) getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceI func (ss *scaleSet) GetInstanceIDByNodeName(name string) (string, error) { managedByAS, err := ss.isNodeManagedByAvailabilitySet(name) if err != nil { - glog.Errorf("Failed to check isNodeManagedByAvailabilitySet: %v", err) + klog.Errorf("Failed to check isNodeManagedByAvailabilitySet: %v", err) return "", err } if managedByAS { @@ -193,7 +193,7 @@ func (ss *scaleSet) GetNodeNameByProviderID(providerID string) (types.NodeName, // NodeName is not part of providerID for vmss instances. scaleSetName, err := extractScaleSetNameByProviderID(providerID) if err != nil { - glog.V(4).Infof("Can not extract scale set name from providerID (%s), assuming it is mananaged by availability set: %v", providerID, err) + klog.V(4).Infof("Can not extract scale set name from providerID (%s), assuming it is mananaged by availability set: %v", providerID, err) return ss.availabilitySet.GetNodeNameByProviderID(providerID) } @@ -204,7 +204,7 @@ func (ss *scaleSet) GetNodeNameByProviderID(providerID string) (types.NodeName, instanceID, err := getLastSegment(providerID) if err != nil { - glog.V(4).Infof("Can not extract instanceID from providerID (%s), assuming it is mananaged by availability set: %v", providerID, err) + klog.V(4).Infof("Can not extract instanceID from providerID (%s), assuming it is mananaged by availability set: %v", providerID, err) return ss.availabilitySet.GetNodeNameByProviderID(providerID) } @@ -225,7 +225,7 @@ func (ss *scaleSet) GetNodeNameByProviderID(providerID string) (types.NodeName, func (ss *scaleSet) GetInstanceTypeByNodeName(name string) (string, error) { managedByAS, err := ss.isNodeManagedByAvailabilitySet(name) if err != nil { - glog.Errorf("Failed to check isNodeManagedByAvailabilitySet: %v", err) + klog.Errorf("Failed to check isNodeManagedByAvailabilitySet: %v", err) return "", err } if managedByAS { @@ -250,7 +250,7 @@ func (ss *scaleSet) GetInstanceTypeByNodeName(name string) (string, error) { func (ss *scaleSet) GetZoneByNodeName(name string) (cloudprovider.Zone, error) { managedByAS, err := ss.isNodeManagedByAvailabilitySet(name) if err != nil { - glog.Errorf("Failed to check isNodeManagedByAvailabilitySet: %v", err) + klog.Errorf("Failed to check isNodeManagedByAvailabilitySet: %v", err) return cloudprovider.Zone{}, err } if managedByAS { @@ -294,13 +294,13 @@ func (ss *scaleSet) GetPrimaryVMSetName() string { func (ss *scaleSet) GetIPByNodeName(nodeName string) (string, string, error) { nic, err := ss.GetPrimaryInterface(nodeName) if err != nil { - glog.Errorf("error: ss.GetIPByNodeName(%s), GetPrimaryInterface(%q), err=%v", nodeName, nodeName, err) + klog.Errorf("error: ss.GetIPByNodeName(%s), GetPrimaryInterface(%q), err=%v", nodeName, nodeName, err) return "", "", err } ipConfig, err := getPrimaryIPConfig(nic) if err != nil { - glog.Errorf("error: ss.GetIPByNodeName(%s), getPrimaryIPConfig(%v), err=%v", nodeName, nic, err) + klog.Errorf("error: ss.GetIPByNodeName(%s), getPrimaryIPConfig(%v), err=%v", nodeName, nic, err) return "", "", err } @@ -390,7 +390,7 @@ func (ss *scaleSet) listScaleSets(resourceGroup string) ([]string, error) { allScaleSets, err := ss.VirtualMachineScaleSetsClient.List(ctx, resourceGroup) if err != nil { - glog.Errorf("VirtualMachineScaleSetsClient.List failed: %v", err) + klog.Errorf("VirtualMachineScaleSetsClient.List failed: %v", err) return nil, err } @@ -410,7 +410,7 @@ func (ss *scaleSet) listScaleSetVMs(scaleSetName, resourceGroup string) ([]compu allVMs, err := ss.VirtualMachineScaleSetVMsClient.List(ctx, resourceGroup, scaleSetName, "", "", string(compute.InstanceView)) if err != nil { - glog.Errorf("VirtualMachineScaleSetVMsClient.List failed: %v", err) + klog.Errorf("VirtualMachineScaleSetVMsClient.List failed: %v", err) return nil, err } @@ -437,7 +437,7 @@ func (ss *scaleSet) getAgentPoolScaleSets(nodes []*v1.Node) (*[]string, error) { } if ssName == "" { - glog.V(3).Infof("Node %q is not belonging to any known scale sets", nodeName) + klog.V(3).Infof("Node %q is not belonging to any known scale sets", nodeName) continue } @@ -461,11 +461,11 @@ func (ss *scaleSet) GetVMSetNames(service *v1.Service, nodes []*v1.Node) (vmSetN scaleSetNames, err := ss.getAgentPoolScaleSets(nodes) if err != nil { - glog.Errorf("ss.GetVMSetNames - getAgentPoolScaleSets failed err=(%v)", err) + klog.Errorf("ss.GetVMSetNames - getAgentPoolScaleSets failed err=(%v)", err) return nil, err } if len(*scaleSetNames) == 0 { - glog.Errorf("ss.GetVMSetNames - No scale sets found for nodes in the cluster, node count(%d)", len(nodes)) + klog.Errorf("ss.GetVMSetNames - No scale sets found for nodes in the cluster, node count(%d)", len(nodes)) return nil, fmt.Errorf("No scale sets found for nodes, node count(%d)", len(nodes)) } @@ -487,7 +487,7 @@ func (ss *scaleSet) GetVMSetNames(service *v1.Service, nodes []*v1.Node) (vmSetN } } if !found { - glog.Errorf("ss.GetVMSetNames - scale set (%s) in service annotation not found", serviceVMSetNames[sasx]) + klog.Errorf("ss.GetVMSetNames - scale set (%s) in service annotation not found", serviceVMSetNames[sasx]) return nil, fmt.Errorf("scale set (%s) - not found", serviceVMSetNames[sasx]) } } @@ -511,7 +511,7 @@ func extractResourceGroupByVMSSNicID(nicID string) (string, error) { func (ss *scaleSet) GetPrimaryInterface(nodeName string) (network.Interface, error) { managedByAS, err := ss.isNodeManagedByAvailabilitySet(nodeName) if err != nil { - glog.Errorf("Failed to check isNodeManagedByAvailabilitySet: %v", err) + klog.Errorf("Failed to check isNodeManagedByAvailabilitySet: %v", err) return network.Interface{}, err } if managedByAS { @@ -526,19 +526,19 @@ func (ss *scaleSet) GetPrimaryInterface(nodeName string) (network.Interface, err return ss.availabilitySet.GetPrimaryInterface(nodeName) } - glog.Errorf("error: ss.GetPrimaryInterface(%s), ss.getVmssVM(%s), err=%v", nodeName, nodeName, err) + klog.Errorf("error: ss.GetPrimaryInterface(%s), ss.getVmssVM(%s), err=%v", nodeName, nodeName, err) return network.Interface{}, err } primaryInterfaceID, err := ss.getPrimaryInterfaceID(vm) if err != nil { - glog.Errorf("error: ss.GetPrimaryInterface(%s), ss.getPrimaryInterfaceID(), err=%v", nodeName, err) + klog.Errorf("error: ss.GetPrimaryInterface(%s), ss.getPrimaryInterfaceID(), err=%v", nodeName, err) return network.Interface{}, err } nicName, err := getLastSegment(primaryInterfaceID) if err != nil { - glog.Errorf("error: ss.GetPrimaryInterface(%s), getLastSegment(%s), err=%v", nodeName, primaryInterfaceID, err) + klog.Errorf("error: ss.GetPrimaryInterface(%s), getLastSegment(%s), err=%v", nodeName, primaryInterfaceID, err) return network.Interface{}, err } resourceGroup, err := extractResourceGroupByVMSSNicID(primaryInterfaceID) @@ -550,7 +550,7 @@ func (ss *scaleSet) GetPrimaryInterface(nodeName string) (network.Interface, err defer cancel() nic, err := ss.InterfacesClient.GetVirtualMachineScaleSetNetworkInterface(ctx, resourceGroup, ssName, instanceID, nicName, "") if err != nil { - glog.Errorf("error: ss.GetPrimaryInterface(%s), ss.GetVirtualMachineScaleSetNetworkInterface.Get(%s, %s, %s), err=%v", nodeName, resourceGroup, ssName, nicName, err) + klog.Errorf("error: ss.GetPrimaryInterface(%s), ss.GetVirtualMachineScaleSetNetworkInterface.Get(%s, %s, %s), err=%v", nodeName, resourceGroup, ssName, nicName, err) return network.Interface{}, err } @@ -572,10 +572,10 @@ func (ss *scaleSet) getScaleSetWithRetry(service *v1.Service, name string) (comp cached, retryErr := ss.vmssCache.Get(name) if retryErr != nil { ss.Event(service, v1.EventTypeWarning, "GetVirtualMachineScaleSet", retryErr.Error()) - glog.Errorf("backoff: failure for scale set %q, will retry,err=%v", name, retryErr) + klog.Errorf("backoff: failure for scale set %q, will retry,err=%v", name, retryErr) return false, nil } - glog.V(4).Infof("backoff: success for scale set %q", name) + klog.V(4).Infof("backoff: success for scale set %q", name) if cached != nil { exists = true @@ -627,7 +627,7 @@ func (ss *scaleSet) createOrUpdateVMSSWithRetry(service *v1.Service, virtualMach ctx, cancel := getContextWithCancel() defer cancel() resp, err := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, ss.ResourceGroup, *virtualMachineScaleSet.Name, virtualMachineScaleSet) - glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%s): end", *virtualMachineScaleSet.Name) + klog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%s): end", *virtualMachineScaleSet.Name) return ss.processHTTPRetryResponse(service, "CreateOrUpdateVMSS", resp, err) }) } @@ -638,7 +638,7 @@ func (ss *scaleSet) updateVMSSInstancesWithRetry(service *v1.Service, scaleSetNa ctx, cancel := getContextWithCancel() defer cancel() resp, err := ss.VirtualMachineScaleSetsClient.UpdateInstances(ctx, ss.ResourceGroup, scaleSetName, vmInstanceIDs) - glog.V(10).Infof("VirtualMachineScaleSetsClient.UpdateInstances(%s): end", scaleSetName) + klog.V(10).Infof("VirtualMachineScaleSetsClient.UpdateInstances(%s): end", scaleSetName) return ss.processHTTPRetryResponse(service, "CreateOrUpdateVMSSInstance", resp, err) }) } @@ -650,18 +650,18 @@ func (ss *scaleSet) getNodesScaleSets(nodes []*v1.Node) (map[string]sets.String, for _, curNode := range nodes { if ss.useStandardLoadBalancer() && ss.excludeMasterNodesFromStandardLB() && isMasterNode(curNode) { - glog.V(4).Infof("Excluding master node %q from load balancer backendpool", curNode.Name) + klog.V(4).Infof("Excluding master node %q from load balancer backendpool", curNode.Name) continue } if ss.ShouldNodeExcludedFromLoadBalancer(curNode) { - glog.V(4).Infof("Excluding unmanaged/external-resource-group node %q", curNode.Name) + klog.V(4).Infof("Excluding unmanaged/external-resource-group node %q", curNode.Name) continue } curScaleSetName, err := extractScaleSetNameByProviderID(curNode.Spec.ProviderID) if err != nil { - glog.V(4).Infof("Node %q is not belonging to any scale sets, assuming it is belong to availability sets", curNode.Name) + klog.V(4).Infof("Node %q is not belonging to any scale sets, assuming it is belong to availability sets", curNode.Name) standardNodes = append(standardNodes, curNode) continue } @@ -672,7 +672,7 @@ func (ss *scaleSet) getNodesScaleSets(nodes []*v1.Node) (map[string]sets.String, instanceID, err := getLastSegment(curNode.Spec.ProviderID) if err != nil { - glog.Errorf("Failed to get instance ID for node %q: %v", curNode.Spec.ProviderID, err) + klog.Errorf("Failed to get instance ID for node %q: %v", curNode.Spec.ProviderID, err) return nil, nil, err } @@ -685,16 +685,16 @@ func (ss *scaleSet) getNodesScaleSets(nodes []*v1.Node) (map[string]sets.String, // ensureHostsInVMSetPool ensures the given Node's primary IP configurations are // participating in the vmSet's LoadBalancer Backend Pool. func (ss *scaleSet) ensureHostsInVMSetPool(service *v1.Service, backendPoolID string, vmSetName string, instanceIDs []string, isInternal bool) error { - glog.V(3).Infof("ensuring hosts %q of scaleset %q in LB backendpool %q", instanceIDs, vmSetName, backendPoolID) + klog.V(3).Infof("ensuring hosts %q of scaleset %q in LB backendpool %q", instanceIDs, vmSetName, backendPoolID) serviceName := getServiceName(service) virtualMachineScaleSet, exists, err := ss.getScaleSetWithRetry(service, vmSetName) if err != nil { - glog.Errorf("ss.getScaleSetWithRetry(%s) for service %q failed: %v", vmSetName, serviceName, err) + klog.Errorf("ss.getScaleSetWithRetry(%s) for service %q failed: %v", vmSetName, serviceName, err) return err } if !exists { errorMessage := fmt.Errorf("Scale set %q not found", vmSetName) - glog.Errorf("%v", errorMessage) + klog.Errorf("%v", errorMessage) return errorMessage } @@ -735,7 +735,7 @@ func (ss *scaleSet) ensureHostsInVMSetPool(service *v1.Service, backendPoolID st if len(matches) == 2 { lbName := matches[1] if strings.HasSuffix(lbName, InternalLoadBalancerNameSuffix) == isInternal { - glog.V(4).Infof("vmss %q has already been added to LB %q, omit adding it to a new one", vmSetName, lbName) + klog.V(4).Infof("vmss %q has already been added to LB %q, omit adding it to a new one", vmSetName, lbName) return nil } } @@ -750,15 +750,15 @@ func (ss *scaleSet) ensureHostsInVMSetPool(service *v1.Service, backendPoolID st ctx, cancel := getContextWithCancel() defer cancel() - glog.V(3).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate for service (%s): scale set (%s) - updating", serviceName, vmSetName) + klog.V(3).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate for service (%s): scale set (%s) - updating", serviceName, vmSetName) resp, err := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, ss.ResourceGroup, vmSetName, virtualMachineScaleSet) - glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", vmSetName) + klog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", vmSetName) if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { - glog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate for service (%s): scale set (%s) - updating, err=%v", serviceName, vmSetName, err) + klog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate for service (%s): scale set (%s) - updating, err=%v", serviceName, vmSetName, err) retryErr := ss.createOrUpdateVMSSWithRetry(service, virtualMachineScaleSet) if retryErr != nil { err = retryErr - glog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate for service (%s) abort backoff: scale set (%s) - updating", serviceName, vmSetName) + klog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate for service (%s) abort backoff: scale set (%s) - updating", serviceName, vmSetName) } } if err != nil { @@ -773,13 +773,13 @@ func (ss *scaleSet) ensureHostsInVMSetPool(service *v1.Service, backendPoolID st ctx, cancel := getContextWithCancel() defer cancel() instanceResp, err := ss.VirtualMachineScaleSetsClient.UpdateInstances(ctx, ss.ResourceGroup, vmSetName, vmInstanceIDs) - glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", vmSetName) + klog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", vmSetName) if ss.CloudProviderBackoff && shouldRetryHTTPRequest(instanceResp, err) { - glog.V(2).Infof("VirtualMachineScaleSetsClient.UpdateInstances for service (%s): scale set (%s) - updating, err=%v", serviceName, vmSetName, err) + klog.V(2).Infof("VirtualMachineScaleSetsClient.UpdateInstances for service (%s): scale set (%s) - updating, err=%v", serviceName, vmSetName, err) retryErr := ss.updateVMSSInstancesWithRetry(service, vmSetName, vmInstanceIDs) if retryErr != nil { err = retryErr - glog.V(2).Infof("VirtualMachineScaleSetsClient.UpdateInstances for service (%s) abort backoff: scale set (%s) - updating", serviceName, vmSetName) + klog.V(2).Infof("VirtualMachineScaleSetsClient.UpdateInstances for service (%s) abort backoff: scale set (%s) - updating", serviceName, vmSetName) } } if err != nil { @@ -795,7 +795,7 @@ func (ss *scaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac serviceName := getServiceName(service) scalesets, standardNodes, err := ss.getNodesScaleSets(nodes) if err != nil { - glog.Errorf("getNodesScaleSets() for service %q failed: %v", serviceName, err) + klog.Errorf("getNodesScaleSets() for service %q failed: %v", serviceName, err) return err } @@ -807,14 +807,14 @@ func (ss *scaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac if instanceIDs.Len() == 0 { // This may happen when scaling a vmss capacity to 0. - glog.V(3).Infof("scale set %q has 0 nodes, adding it to load balancer anyway", ssName) + klog.V(3).Infof("scale set %q has 0 nodes, adding it to load balancer anyway", ssName) // InstanceIDs is required to update vmss, use * instead here since there are no nodes actually. instanceIDs.Insert("*") } err := ss.ensureHostsInVMSetPool(service, backendPoolID, ssName, instanceIDs.List(), isInternal) if err != nil { - glog.Errorf("ensureHostsInVMSetPool() with scaleSet %q for service %q failed: %v", ssName, serviceName, err) + klog.Errorf("ensureHostsInVMSetPool() with scaleSet %q for service %q failed: %v", ssName, serviceName, err) return err } } @@ -822,7 +822,7 @@ func (ss *scaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac if ss.useStandardLoadBalancer() && len(standardNodes) > 0 { err := ss.availabilitySet.EnsureHostsInPool(service, standardNodes, backendPoolID, "", isInternal) if err != nil { - glog.Errorf("availabilitySet.EnsureHostsInPool() for service %q failed: %v", serviceName, err) + klog.Errorf("availabilitySet.EnsureHostsInPool() for service %q failed: %v", serviceName, err) return err } } @@ -832,14 +832,14 @@ func (ss *scaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac // ensureScaleSetBackendPoolDeleted ensures the loadBalancer backendAddressPools deleted from the specified scaleset. func (ss *scaleSet) ensureScaleSetBackendPoolDeleted(service *v1.Service, poolID, ssName string) error { - glog.V(3).Infof("ensuring backend pool %q deleted from scaleset %q", poolID, ssName) + klog.V(3).Infof("ensuring backend pool %q deleted from scaleset %q", poolID, ssName) virtualMachineScaleSet, exists, err := ss.getScaleSetWithRetry(service, ssName) if err != nil { - glog.Errorf("ss.ensureScaleSetBackendPoolDeleted(%s, %s) getScaleSetWithRetry(%s) failed: %v", poolID, ssName, ssName, err) + klog.Errorf("ss.ensureScaleSetBackendPoolDeleted(%s, %s) getScaleSetWithRetry(%s) failed: %v", poolID, ssName, ssName, err) return err } if !exists { - glog.V(2).Infof("ss.ensureScaleSetBackendPoolDeleted(%s, %s), scale set %s has already been non-exist", poolID, ssName, ssName) + klog.V(2).Infof("ss.ensureScaleSetBackendPoolDeleted(%s, %s), scale set %s has already been non-exist", poolID, ssName, ssName) return nil } @@ -866,7 +866,7 @@ func (ss *scaleSet) ensureScaleSetBackendPoolDeleted(service *v1.Service, poolID for i := len(existingBackendPools) - 1; i >= 0; i-- { curPool := existingBackendPools[i] if strings.EqualFold(poolID, *curPool.ID) { - glog.V(10).Infof("ensureScaleSetBackendPoolDeleted gets unwanted backend pool %q for scale set %q", poolID, ssName) + klog.V(10).Infof("ensureScaleSetBackendPoolDeleted gets unwanted backend pool %q for scale set %q", poolID, ssName) foundPool = true newBackendPools = append(existingBackendPools[:i], existingBackendPools[i+1:]...) } @@ -878,17 +878,17 @@ func (ss *scaleSet) ensureScaleSetBackendPoolDeleted(service *v1.Service, poolID // Update scale set with backoff. primaryIPConfiguration.LoadBalancerBackendAddressPools = &newBackendPools - glog.V(3).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating", ssName) + klog.V(3).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating", ssName) ctx, cancel := getContextWithCancel() defer cancel() resp, err := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, ss.ResourceGroup, ssName, virtualMachineScaleSet) - glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", ssName) + klog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", ssName) if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { - glog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating, err=%v", ssName, err) + klog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating, err=%v", ssName, err) retryErr := ss.createOrUpdateVMSSWithRetry(service, virtualMachineScaleSet) if retryErr != nil { err = retryErr - glog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate abort backoff: scale set (%s) - updating", ssName) + klog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate abort backoff: scale set (%s) - updating", ssName) } } if err != nil { @@ -903,13 +903,13 @@ func (ss *scaleSet) ensureScaleSetBackendPoolDeleted(service *v1.Service, poolID instanceCtx, instanceCancel := getContextWithCancel() defer instanceCancel() instanceResp, err := ss.VirtualMachineScaleSetsClient.UpdateInstances(instanceCtx, ss.ResourceGroup, ssName, vmInstanceIDs) - glog.V(10).Infof("VirtualMachineScaleSetsClient.UpdateInstances(%q): end", ssName) + klog.V(10).Infof("VirtualMachineScaleSetsClient.UpdateInstances(%q): end", ssName) if ss.CloudProviderBackoff && shouldRetryHTTPRequest(instanceResp, err) { - glog.V(2).Infof("VirtualMachineScaleSetsClient.UpdateInstances scale set (%s) - updating, err=%v", ssName, err) + klog.V(2).Infof("VirtualMachineScaleSetsClient.UpdateInstances scale set (%s) - updating, err=%v", ssName, err) retryErr := ss.updateVMSSInstancesWithRetry(service, ssName, vmInstanceIDs) if retryErr != nil { err = retryErr - glog.V(2).Infof("VirtualMachineScaleSetsClient.UpdateInstances abort backoff: scale set (%s) - updating", ssName) + klog.V(2).Infof("VirtualMachineScaleSetsClient.UpdateInstances abort backoff: scale set (%s) - updating", ssName) } } if err != nil { @@ -921,14 +921,14 @@ func (ss *scaleSet) ensureScaleSetBackendPoolDeleted(service *v1.Service, poolID if len(newBackendPools) == 0 { updateCtx, updateCancel := getContextWithCancel() defer updateCancel() - glog.V(3).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating second time", ssName) + klog.V(3).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating second time", ssName) resp, err = ss.VirtualMachineScaleSetsClient.CreateOrUpdate(updateCtx, ss.ResourceGroup, ssName, virtualMachineScaleSet) - glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", ssName) + klog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", ssName) if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { - glog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating, err=%v", ssName, err) + klog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating, err=%v", ssName, err) retryErr := ss.createOrUpdateVMSSWithRetry(service, virtualMachineScaleSet) if retryErr != nil { - glog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate abort backoff: scale set (%s) - updating", ssName) + klog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate abort backoff: scale set (%s) - updating", ssName) } } } @@ -952,7 +952,7 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(service *v1.Service, poolID, vmSetN ssName, err := extractScaleSetNameByProviderID(*ipConfigurations.ID) if err != nil { - glog.V(4).Infof("backend IP configuration %q is not belonging to any vmss, omit it", *ipConfigurations.ID) + klog.V(4).Infof("backend IP configuration %q is not belonging to any vmss, omit it", *ipConfigurations.ID) continue } @@ -970,7 +970,7 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(service *v1.Service, poolID, vmSetN err := ss.ensureScaleSetBackendPoolDeleted(service, poolID, ssName) if err != nil { - glog.Errorf("ensureScaleSetBackendPoolDeleted() with scaleSet %q failed: %v", ssName, err) + klog.Errorf("ensureScaleSetBackendPoolDeleted() with scaleSet %q failed: %v", ssName, err) return err } } diff --git a/pkg/cloudprovider/providers/azure/azure_vmss_cache.go b/pkg/cloudprovider/providers/azure/azure_vmss_cache.go index 73bc59ea4e7..a9a46ba703b 100644 --- a/pkg/cloudprovider/providers/azure/azure_vmss_cache.go +++ b/pkg/cloudprovider/providers/azure/azure_vmss_cache.go @@ -21,7 +21,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/sets" ) @@ -50,7 +50,7 @@ func (ss *scaleSet) makeVmssVMName(scaleSetName, instanceID string) string { func extractVmssVMName(name string) (string, string, error) { split := strings.SplitAfter(name, vmssNameSeparator) if len(split) < 2 { - glog.V(3).Infof("Failed to extract vmssVMName %q", name) + klog.V(3).Infof("Failed to extract vmssVMName %q", name) return "", "", ErrorNotVmssInstance } @@ -74,7 +74,7 @@ func (ss *scaleSet) newVmssCache() (*timedCache, error) { } if !exists { - glog.V(2).Infof("Virtual machine scale set %q not found with message: %q", key, message) + klog.V(2).Infof("Virtual machine scale set %q not found with message: %q", key, message) return nil, nil } @@ -107,7 +107,7 @@ func (ss *scaleSet) newNodeNameToScaleSetMappingCache() (*timedCache, error) { for _, vm := range vms { if vm.OsProfile == nil || vm.OsProfile.ComputerName == nil { - glog.Warningf("failed to get computerName for vmssVM (%q)", ssName) + klog.Warningf("failed to get computerName for vmssVM (%q)", ssName) continue } @@ -195,7 +195,7 @@ func (ss *scaleSet) newVmssVMCache() (*timedCache, error) { } if !exists { - glog.V(2).Infof("Virtual machine scale set VM %q not found with message: %q", key, message) + klog.V(2).Infof("Virtual machine scale set VM %q not found with message: %q", key, message) return nil, nil } @@ -210,7 +210,7 @@ func (ss *scaleSet) newVmssVMCache() (*timedCache, error) { return nil, realErr } if !exists { - glog.V(2).Infof("Virtual machine scale set VM %q not found with message: %q", key, message) + klog.V(2).Infof("Virtual machine scale set VM %q not found with message: %q", key, message) return nil, nil } diff --git a/pkg/cloudprovider/providers/azure/azure_wrap.go b/pkg/cloudprovider/providers/azure/azure_wrap.go index e2fa8f83d13..cf3632956ab 100644 --- a/pkg/cloudprovider/providers/azure/azure_wrap.go +++ b/pkg/cloudprovider/providers/azure/azure_wrap.go @@ -26,9 +26,9 @@ import ( "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network" "github.com/Azure/go-autorest/autorest" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" ) var ( @@ -117,7 +117,7 @@ func (az *Cloud) getPublicIPAddress(pipResourceGroup string, pipName string) (pi } if !exists { - glog.V(2).Infof("Public IP %q not found with message: %q", pipName, message) + klog.V(2).Infof("Public IP %q not found with message: %q", pipName, message) return pip, false, nil } @@ -144,7 +144,7 @@ func (az *Cloud) getSubnet(virtualNetworkName string, subnetName string) (subnet } if !exists { - glog.V(2).Infof("Subnet %q not found with message: %q", subnetName, message) + klog.V(2).Infof("Subnet %q not found with message: %q", subnetName, message) return subnet, false, nil } @@ -204,7 +204,7 @@ func (az *Cloud) newVMCache() (*timedCache, error) { } if !exists { - glog.V(2).Infof("Virtual machine %q not found with message: %q", key, message) + klog.V(2).Infof("Virtual machine %q not found with message: %q", key, message) return nil, nil } @@ -226,7 +226,7 @@ func (az *Cloud) newLBCache() (*timedCache, error) { } if !exists { - glog.V(2).Infof("Load balancer %q not found with message: %q", key, message) + klog.V(2).Infof("Load balancer %q not found with message: %q", key, message) return nil, nil } @@ -247,7 +247,7 @@ func (az *Cloud) newNSGCache() (*timedCache, error) { } if !exists { - glog.V(2).Infof("Security group %q not found with message: %q", key, message) + klog.V(2).Infof("Security group %q not found with message: %q", key, message) return nil, nil } @@ -268,7 +268,7 @@ func (az *Cloud) newRouteTableCache() (*timedCache, error) { } if !exists { - glog.V(2).Infof("Route table %q not found with message: %q", key, message) + klog.V(2).Infof("Route table %q not found with message: %q", key, message) return nil, nil } diff --git a/pkg/cloudprovider/providers/azure/azure_zones.go b/pkg/cloudprovider/providers/azure/azure_zones.go index ca8b0b51763..3c2d84571af 100644 --- a/pkg/cloudprovider/providers/azure/azure_zones.go +++ b/pkg/cloudprovider/providers/azure/azure_zones.go @@ -22,9 +22,9 @@ import ( "strconv" "strings" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" ) // makeZone returns the zone value in format of -. @@ -66,7 +66,7 @@ func (az *Cloud) GetZone(ctx context.Context) (cloudprovider.Zone, error) { } zone = az.makeZone(zoneID) } else { - glog.V(3).Infof("Availability zone is not enabled for the node, falling back to fault domain") + klog.V(3).Infof("Availability zone is not enabled for the node, falling back to fault domain") zone = metadata.Compute.FaultDomain } @@ -82,7 +82,7 @@ func (az *Cloud) GetZone(ctx context.Context) (cloudprovider.Zone, error) { func (az *Cloud) GetZoneByProviderID(ctx context.Context, providerID string) (cloudprovider.Zone, error) { // Returns nil for unmanaged nodes because azure cloud provider couldn't fetch information for them. if az.IsNodeUnmanagedByProviderID(providerID) { - glog.V(2).Infof("GetZoneByProviderID: omitting unmanaged node %q", providerID) + klog.V(2).Infof("GetZoneByProviderID: omitting unmanaged node %q", providerID) return cloudprovider.Zone{}, nil } @@ -104,7 +104,7 @@ func (az *Cloud) GetZoneByNodeName(ctx context.Context, nodeName types.NodeName) return cloudprovider.Zone{}, err } if unmanaged { - glog.V(2).Infof("GetZoneByNodeName: omitting unmanaged node %q", nodeName) + klog.V(2).Infof("GetZoneByNodeName: omitting unmanaged node %q", nodeName) return cloudprovider.Zone{}, nil } diff --git a/pkg/cloudprovider/providers/cloudstack/BUILD b/pkg/cloudprovider/providers/cloudstack/BUILD index 1e519fcde58..65cd15f2572 100644 --- a/pkg/cloudprovider/providers/cloudstack/BUILD +++ b/pkg/cloudprovider/providers/cloudstack/BUILD @@ -22,10 +22,10 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//vendor/github.com/d2g/dhcp4:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/kardianos/osext:go_default_library", "//vendor/github.com/xanzy/go-cloudstack/cloudstack:go_default_library", "//vendor/gopkg.in/gcfg.v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:android": [ "//vendor/github.com/d2g/dhcp4client:go_default_library", diff --git a/pkg/cloudprovider/providers/cloudstack/cloudstack.go b/pkg/cloudprovider/providers/cloudstack/cloudstack.go index f0a08b77bd2..4769adb647f 100644 --- a/pkg/cloudprovider/providers/cloudstack/cloudstack.go +++ b/pkg/cloudprovider/providers/cloudstack/cloudstack.go @@ -24,12 +24,12 @@ import ( "os" "path/filepath" - "github.com/golang/glog" "github.com/kardianos/osext" "github.com/xanzy/go-cloudstack/cloudstack" "gopkg.in/gcfg.v1" "k8s.io/apimachinery/pkg/types" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" ) // ProviderName is the name of this cloud provider. @@ -98,10 +98,10 @@ func newCSCloud(cfg *CSConfig) (*CSCloud, error) { // In CloudStack your metadata is always served by the DHCP server. dhcpServer, err := findDHCPServer() if err == nil { - glog.V(4).Infof("Found metadata server: %v", dhcpServer) + klog.V(4).Infof("Found metadata server: %v", dhcpServer) cs.metadata = &metadata{dhcpServer: dhcpServer, zone: cs.zone} } else { - glog.Errorf("Error searching metadata server: %v", err) + klog.Errorf("Error searching metadata server: %v", err) } } @@ -111,7 +111,7 @@ func newCSCloud(cfg *CSConfig) (*CSCloud, error) { if cs.client == nil { if cs.metadata != nil { - glog.V(2).Infof("No API URL, key and secret are provided, so only using metadata!") + klog.V(2).Infof("No API URL, key and secret are provided, so only using metadata!") } else { return nil, errors.New("no cloud provider config given") } @@ -208,7 +208,7 @@ func (cs *CSCloud) GetZone(ctx context.Context) (cloudprovider.Zone, error) { cs.zone = instance.Zonename } - glog.V(2).Infof("Current zone is %v", cs.zone) + klog.V(2).Infof("Current zone is %v", cs.zone) zone.FailureDomain = cs.zone zone.Region = cs.zone @@ -230,7 +230,7 @@ func (cs *CSCloud) GetZoneByProviderID(ctx context.Context, providerID string) ( return zone, fmt.Errorf("error retrieving zone: %v", err) } - glog.V(2).Infof("Current zone is %v", cs.zone) + klog.V(2).Infof("Current zone is %v", cs.zone) zone.FailureDomain = instance.Zonename zone.Region = instance.Zonename @@ -252,7 +252,7 @@ func (cs *CSCloud) GetZoneByNodeName(ctx context.Context, nodeName types.NodeNam return zone, fmt.Errorf("error retrieving zone: %v", err) } - glog.V(2).Infof("Current zone is %v", cs.zone) + klog.V(2).Infof("Current zone is %v", cs.zone) zone.FailureDomain = instance.Zonename zone.Region = instance.Zonename diff --git a/pkg/cloudprovider/providers/cloudstack/cloudstack_instances.go b/pkg/cloudprovider/providers/cloudstack/cloudstack_instances.go index e318b7837f4..55a81bdaa6f 100644 --- a/pkg/cloudprovider/providers/cloudstack/cloudstack_instances.go +++ b/pkg/cloudprovider/providers/cloudstack/cloudstack_instances.go @@ -21,11 +21,11 @@ import ( "errors" "fmt" - "github.com/golang/glog" "github.com/xanzy/go-cloudstack/cloudstack" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" ) // NodeAddresses returns the addresses of the specified instance. @@ -78,7 +78,7 @@ func (cs *CSCloud) nodeAddresses(instance *cloudstack.VirtualMachine) ([]v1.Node } else { // Since there is no sane way to determine the external IP if the host isn't // using static NAT, we will just fire a log message and omit the external IP. - glog.V(4).Infof("Could not determine the public IP of host %v (%v)", instance.Name, instance.Id) + klog.V(4).Infof("Could not determine the public IP of host %v (%v)", instance.Name, instance.Id) } return addresses, nil diff --git a/pkg/cloudprovider/providers/cloudstack/cloudstack_loadbalancer.go b/pkg/cloudprovider/providers/cloudstack/cloudstack_loadbalancer.go index c561b4ab333..64971d02cf2 100644 --- a/pkg/cloudprovider/providers/cloudstack/cloudstack_loadbalancer.go +++ b/pkg/cloudprovider/providers/cloudstack/cloudstack_loadbalancer.go @@ -21,8 +21,8 @@ import ( "fmt" "strconv" - "github.com/golang/glog" "github.com/xanzy/go-cloudstack/cloudstack" + "k8s.io/klog" "k8s.io/api/core/v1" cloudprovider "k8s.io/cloud-provider" @@ -43,7 +43,7 @@ type loadBalancer struct { // GetLoadBalancer returns whether the specified load balancer exists, and if so, what its status is. func (cs *CSCloud) GetLoadBalancer(ctx context.Context, clusterName string, service *v1.Service) (*v1.LoadBalancerStatus, bool, error) { - glog.V(4).Infof("GetLoadBalancer(%v, %v, %v)", clusterName, service.Namespace, service.Name) + klog.V(4).Infof("GetLoadBalancer(%v, %v, %v)", clusterName, service.Namespace, service.Name) // Get the load balancer details and existing rules. lb, err := cs.getLoadBalancer(service) @@ -56,7 +56,7 @@ func (cs *CSCloud) GetLoadBalancer(ctx context.Context, clusterName string, serv return nil, false, nil } - glog.V(4).Infof("Found a load balancer associated with IP %v", lb.ipAddr) + klog.V(4).Infof("Found a load balancer associated with IP %v", lb.ipAddr) status := &v1.LoadBalancerStatus{} status.Ingress = append(status.Ingress, v1.LoadBalancerIngress{IP: lb.ipAddr}) @@ -66,7 +66,7 @@ func (cs *CSCloud) GetLoadBalancer(ctx context.Context, clusterName string, serv // EnsureLoadBalancer creates a new load balancer, or updates the existing one. Returns the status of the balancer. func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (status *v1.LoadBalancerStatus, err error) { - glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v)", clusterName, service.Namespace, service.Name, service.Spec.LoadBalancerIP, service.Spec.Ports, nodes) + klog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v)", clusterName, service.Namespace, service.Name, service.Spec.LoadBalancerIP, service.Spec.Ports, nodes) if len(service.Spec.Ports) == 0 { return nil, fmt.Errorf("requested load balancer with no ports") @@ -104,14 +104,14 @@ func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, s defer func(lb *loadBalancer) { if err != nil { if err := lb.releaseLoadBalancerIP(); err != nil { - glog.Errorf(err.Error()) + klog.Errorf(err.Error()) } } }(lb) } } - glog.V(4).Infof("Load balancer %v is associated with IP %v", lb.name, lb.ipAddr) + klog.V(4).Infof("Load balancer %v is associated with IP %v", lb.name, lb.ipAddr) for _, port := range service.Spec.Ports { // All ports have their own load balancer rule, so add the port to lbName to keep the names unique. @@ -123,14 +123,14 @@ func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, s return nil, err } if exists && !needsUpdate { - glog.V(4).Infof("Load balancer rule %v is up-to-date", lbRuleName) + klog.V(4).Infof("Load balancer rule %v is up-to-date", lbRuleName) // Delete the rule from the map, to prevent it being deleted. delete(lb.rules, lbRuleName) continue } if needsUpdate { - glog.V(4).Infof("Updating load balancer rule: %v", lbRuleName) + klog.V(4).Infof("Updating load balancer rule: %v", lbRuleName) if err := lb.updateLoadBalancerRule(lbRuleName); err != nil { return nil, err } @@ -139,13 +139,13 @@ func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, s continue } - glog.V(4).Infof("Creating load balancer rule: %v", lbRuleName) + klog.V(4).Infof("Creating load balancer rule: %v", lbRuleName) lbRule, err := lb.createLoadBalancerRule(lbRuleName, port) if err != nil { return nil, err } - glog.V(4).Infof("Assigning hosts (%v) to load balancer rule: %v", lb.hostIDs, lbRuleName) + klog.V(4).Infof("Assigning hosts (%v) to load balancer rule: %v", lb.hostIDs, lbRuleName) if err = lb.assignHostsToRule(lbRule, lb.hostIDs); err != nil { return nil, err } @@ -154,7 +154,7 @@ func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, s // Cleanup any rules that are now still in the rules map, as they are no longer needed. for _, lbRule := range lb.rules { - glog.V(4).Infof("Deleting obsolete load balancer rule: %v", lbRule.Name) + klog.V(4).Infof("Deleting obsolete load balancer rule: %v", lbRule.Name) if err := lb.deleteLoadBalancerRule(lbRule); err != nil { return nil, err } @@ -168,7 +168,7 @@ func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, s // UpdateLoadBalancer updates hosts under the specified load balancer. func (cs *CSCloud) UpdateLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) error { - glog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v, %v)", clusterName, service.Namespace, service.Name, nodes) + klog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v, %v)", clusterName, service.Namespace, service.Name, nodes) // Get the load balancer details and existing rules. lb, err := cs.getLoadBalancer(service) @@ -194,14 +194,14 @@ func (cs *CSCloud) UpdateLoadBalancer(ctx context.Context, clusterName string, s assign, remove := symmetricDifference(lb.hostIDs, l.LoadBalancerRuleInstances) if len(assign) > 0 { - glog.V(4).Infof("Assigning new hosts (%v) to load balancer rule: %v", assign, lbRule.Name) + klog.V(4).Infof("Assigning new hosts (%v) to load balancer rule: %v", assign, lbRule.Name) if err := lb.assignHostsToRule(lbRule, assign); err != nil { return err } } if len(remove) > 0 { - glog.V(4).Infof("Removing old hosts (%v) from load balancer rule: %v", assign, lbRule.Name) + klog.V(4).Infof("Removing old hosts (%v) from load balancer rule: %v", assign, lbRule.Name) if err := lb.removeHostsFromRule(lbRule, remove); err != nil { return err } @@ -214,7 +214,7 @@ func (cs *CSCloud) UpdateLoadBalancer(ctx context.Context, clusterName string, s // EnsureLoadBalancerDeleted deletes the specified load balancer if it exists, returning // nil if the load balancer specified either didn't exist or was successfully deleted. func (cs *CSCloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *v1.Service) error { - glog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v, %v)", clusterName, service.Namespace, service.Name) + klog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v, %v)", clusterName, service.Namespace, service.Name) // Get the load balancer details and existing rules. lb, err := cs.getLoadBalancer(service) @@ -223,14 +223,14 @@ func (cs *CSCloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName st } for _, lbRule := range lb.rules { - glog.V(4).Infof("Deleting load balancer rule: %v", lbRule.Name) + klog.V(4).Infof("Deleting load balancer rule: %v", lbRule.Name) if err := lb.deleteLoadBalancerRule(lbRule); err != nil { return err } } if lb.ipAddr != "" && lb.ipAddr != service.Spec.LoadBalancerIP { - glog.V(4).Infof("Releasing load balancer IP: %v", lb.ipAddr) + klog.V(4).Infof("Releasing load balancer IP: %v", lb.ipAddr) if err := lb.releaseLoadBalancerIP(); err != nil { return err } @@ -270,14 +270,14 @@ func (cs *CSCloud) getLoadBalancer(service *v1.Service) (*loadBalancer, error) { lb.rules[lbRule.Name] = lbRule if lb.ipAddr != "" && lb.ipAddr != lbRule.Publicip { - glog.Warningf("Load balancer for service %v/%v has rules associated with different IP's: %v, %v", service.Namespace, service.Name, lb.ipAddr, lbRule.Publicip) + klog.Warningf("Load balancer for service %v/%v has rules associated with different IP's: %v, %v", service.Namespace, service.Name, lb.ipAddr, lbRule.Publicip) } lb.ipAddr = lbRule.Publicip lb.ipAddrID = lbRule.Publicipid } - glog.V(4).Infof("Load balancer %v contains %d rule(s)", lb.name, len(lb.rules)) + klog.V(4).Infof("Load balancer %v contains %d rule(s)", lb.name, len(lb.rules)) return lb, nil } @@ -335,7 +335,7 @@ func (lb *loadBalancer) getLoadBalancerIP(loadBalancerIP string) error { // getPublicIPAddressID retrieves the ID of the given IP, and sets the address and it's ID. func (lb *loadBalancer) getPublicIPAddress(loadBalancerIP string) error { - glog.V(4).Infof("Retrieve load balancer IP details: %v", loadBalancerIP) + klog.V(4).Infof("Retrieve load balancer IP details: %v", loadBalancerIP) p := lb.Address.NewListPublicIpAddressesParams() p.SetIpaddress(loadBalancerIP) @@ -362,7 +362,7 @@ func (lb *loadBalancer) getPublicIPAddress(loadBalancerIP string) error { // associatePublicIPAddress associates a new IP and sets the address and it's ID. func (lb *loadBalancer) associatePublicIPAddress() error { - glog.V(4).Infof("Allocate new IP for load balancer: %v", lb.name) + klog.V(4).Infof("Allocate new IP for load balancer: %v", lb.name) // If a network belongs to a VPC, the IP address needs to be associated with // the VPC instead of with the network. network, count, err := lb.Network.GetNetworkByID(lb.networkID, cloudstack.WithProject(lb.projectID)) diff --git a/pkg/cloudprovider/providers/cloudstack/metadata.go b/pkg/cloudprovider/providers/cloudstack/metadata.go index 1c75b107941..f2decccdf86 100644 --- a/pkg/cloudprovider/providers/cloudstack/metadata.go +++ b/pkg/cloudprovider/providers/cloudstack/metadata.go @@ -25,10 +25,10 @@ import ( "net/http" "github.com/d2g/dhcp4" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" ) type metadata struct { @@ -143,7 +143,7 @@ func (m *metadata) GetZone(ctx context.Context) (cloudprovider.Zone, error) { m.zone = zoneName } - glog.V(2).Infof("Current zone is %v", zone) + klog.V(2).Infof("Current zone is %v", zone) zone.FailureDomain = m.zone zone.Region = m.zone diff --git a/pkg/cloudprovider/providers/gce/BUILD b/pkg/cloudprovider/providers/gce/BUILD index d648f30c13c..0e590564707 100644 --- a/pkg/cloudprovider/providers/gce/BUILD +++ b/pkg/cloudprovider/providers/gce/BUILD @@ -80,7 +80,6 @@ go_library( "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//vendor/cloud.google.com/go/compute/metadata:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/golang.org/x/oauth2:go_default_library", "//vendor/golang.org/x/oauth2/google:go_default_library", @@ -91,6 +90,7 @@ go_library( "//vendor/google.golang.org/api/googleapi:go_default_library", "//vendor/google.golang.org/api/tpu/v1:go_default_library", "//vendor/gopkg.in/gcfg.v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/cloudprovider/providers/gce/cloud/BUILD b/pkg/cloudprovider/providers/gce/cloud/BUILD index de46184e547..e3c289daa6f 100644 --- a/pkg/cloudprovider/providers/gce/cloud/BUILD +++ b/pkg/cloudprovider/providers/gce/cloud/BUILD @@ -19,11 +19,11 @@ go_library( deps = [ "//pkg/cloudprovider/providers/gce/cloud/filter:go_default_library", "//pkg/cloudprovider/providers/gce/cloud/meta:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/api/compute/v0.alpha:go_default_library", "//vendor/google.golang.org/api/compute/v0.beta:go_default_library", "//vendor/google.golang.org/api/compute/v1:go_default_library", "//vendor/google.golang.org/api/googleapi:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/cloudprovider/providers/gce/cloud/filter/BUILD b/pkg/cloudprovider/providers/gce/cloud/filter/BUILD index a2e0c7941cf..0932666c50b 100644 --- a/pkg/cloudprovider/providers/gce/cloud/filter/BUILD +++ b/pkg/cloudprovider/providers/gce/cloud/filter/BUILD @@ -5,7 +5,7 @@ go_library( srcs = ["filter.go"], importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/filter", visibility = ["//visibility:public"], - deps = ["//vendor/github.com/golang/glog:go_default_library"], + deps = ["//vendor/k8s.io/klog:go_default_library"], ) go_test( diff --git a/pkg/cloudprovider/providers/gce/cloud/filter/filter.go b/pkg/cloudprovider/providers/gce/cloud/filter/filter.go index c08005726c8..b65ab6391a7 100644 --- a/pkg/cloudprovider/providers/gce/cloud/filter/filter.go +++ b/pkg/cloudprovider/providers/gce/cloud/filter/filter.go @@ -34,7 +34,7 @@ import ( "regexp" "strings" - "github.com/golang/glog" + "k8s.io/klog" ) var ( @@ -221,7 +221,7 @@ func (fp *filterPredicate) String() string { func (fp *filterPredicate) match(o interface{}) bool { v, err := extractValue(fp.fieldName, o) - glog.V(6).Infof("extractValue(%q, %#v) = %v, %v", fp.fieldName, o, v, err) + klog.V(6).Infof("extractValue(%q, %#v) = %v, %v", fp.fieldName, o, v, err) if err != nil { return false } @@ -234,7 +234,7 @@ func (fp *filterPredicate) match(o interface{}) bool { } re, err := regexp.Compile(*fp.s) if err != nil { - glog.Errorf("Match regexp %q is invalid: %v", *fp.s, err) + klog.Errorf("Match regexp %q is invalid: %v", *fp.s, err) return false } match = re.Match([]byte(x)) diff --git a/pkg/cloudprovider/providers/gce/cloud/gen.go b/pkg/cloudprovider/providers/gce/cloud/gen.go index a25cac9909c..a3e7cd2c657 100644 --- a/pkg/cloudprovider/providers/gce/cloud/gen.go +++ b/pkg/cloudprovider/providers/gce/cloud/gen.go @@ -25,8 +25,8 @@ import ( "net/http" "sync" - "github.com/golang/glog" "google.golang.org/api/googleapi" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/filter" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta" @@ -48,7 +48,7 @@ type Cloud interface { RegionBackendServices() RegionBackendServices AlphaRegionBackendServices() AlphaRegionBackendServices Disks() Disks - BetaRegionDisks() BetaRegionDisks + RegionDisks() RegionDisks Firewalls() Firewalls ForwardingRules() ForwardingRules AlphaForwardingRules() AlphaForwardingRules @@ -89,7 +89,7 @@ func NewGCE(s *Service) *GCE { gceRegionBackendServices: &GCERegionBackendServices{s}, gceAlphaRegionBackendServices: &GCEAlphaRegionBackendServices{s}, gceDisks: &GCEDisks{s}, - gceBetaRegionDisks: &GCEBetaRegionDisks{s}, + gceRegionDisks: &GCERegionDisks{s}, gceFirewalls: &GCEFirewalls{s}, gceForwardingRules: &GCEForwardingRules{s}, gceAlphaForwardingRules: &GCEAlphaForwardingRules{s}, @@ -134,7 +134,7 @@ type GCE struct { gceRegionBackendServices *GCERegionBackendServices gceAlphaRegionBackendServices *GCEAlphaRegionBackendServices gceDisks *GCEDisks - gceBetaRegionDisks *GCEBetaRegionDisks + gceRegionDisks *GCERegionDisks gceFirewalls *GCEFirewalls gceForwardingRules *GCEForwardingRules gceAlphaForwardingRules *GCEAlphaForwardingRules @@ -212,9 +212,9 @@ func (gce *GCE) Disks() Disks { return gce.gceDisks } -// BetaRegionDisks returns the interface for the beta RegionDisks. -func (gce *GCE) BetaRegionDisks() BetaRegionDisks { - return gce.gceBetaRegionDisks +// RegionDisks returns the interface for the ga RegionDisks. +func (gce *GCE) RegionDisks() RegionDisks { + return gce.gceRegionDisks } // Firewalls returns the interface for the ga Firewalls. @@ -381,7 +381,7 @@ func NewMockGCE(projectRouter ProjectRouter) *MockGCE { MockRegionBackendServices: NewMockRegionBackendServices(projectRouter, mockRegionBackendServicesObjs), MockAlphaRegionBackendServices: NewMockAlphaRegionBackendServices(projectRouter, mockRegionBackendServicesObjs), MockDisks: NewMockDisks(projectRouter, mockDisksObjs), - MockBetaRegionDisks: NewMockBetaRegionDisks(projectRouter, mockRegionDisksObjs), + MockRegionDisks: NewMockRegionDisks(projectRouter, mockRegionDisksObjs), MockFirewalls: NewMockFirewalls(projectRouter, mockFirewallsObjs), MockForwardingRules: NewMockForwardingRules(projectRouter, mockForwardingRulesObjs), MockAlphaForwardingRules: NewMockAlphaForwardingRules(projectRouter, mockForwardingRulesObjs), @@ -426,7 +426,7 @@ type MockGCE struct { MockRegionBackendServices *MockRegionBackendServices MockAlphaRegionBackendServices *MockAlphaRegionBackendServices MockDisks *MockDisks - MockBetaRegionDisks *MockBetaRegionDisks + MockRegionDisks *MockRegionDisks MockFirewalls *MockFirewalls MockForwardingRules *MockForwardingRules MockAlphaForwardingRules *MockAlphaForwardingRules @@ -504,9 +504,9 @@ func (mock *MockGCE) Disks() Disks { return mock.MockDisks } -// BetaRegionDisks returns the interface for the beta RegionDisks. -func (mock *MockGCE) BetaRegionDisks() BetaRegionDisks { - return mock.MockBetaRegionDisks +// RegionDisks returns the interface for the ga RegionDisks. +func (mock *MockGCE) RegionDisks() RegionDisks { + return mock.MockRegionDisks } // Firewalls returns the interface for the ga Firewalls. @@ -649,7 +649,7 @@ func (m *MockAddressesObj) ToAlpha() *alpha.Address { // Convert the object via JSON copying to the type that was requested. ret := &alpha.Address{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *alpha.Address via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *alpha.Address via JSON: %v", m.Obj, err) } return ret } @@ -662,7 +662,7 @@ func (m *MockAddressesObj) ToBeta() *beta.Address { // Convert the object via JSON copying to the type that was requested. ret := &beta.Address{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *beta.Address via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *beta.Address via JSON: %v", m.Obj, err) } return ret } @@ -675,7 +675,7 @@ func (m *MockAddressesObj) ToGA() *ga.Address { // Convert the object via JSON copying to the type that was requested. ret := &ga.Address{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.Address via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.Address via JSON: %v", m.Obj, err) } return ret } @@ -695,7 +695,7 @@ func (m *MockBackendServicesObj) ToAlpha() *alpha.BackendService { // Convert the object via JSON copying to the type that was requested. ret := &alpha.BackendService{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *alpha.BackendService via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *alpha.BackendService via JSON: %v", m.Obj, err) } return ret } @@ -708,7 +708,7 @@ func (m *MockBackendServicesObj) ToBeta() *beta.BackendService { // Convert the object via JSON copying to the type that was requested. ret := &beta.BackendService{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *beta.BackendService via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *beta.BackendService via JSON: %v", m.Obj, err) } return ret } @@ -721,7 +721,7 @@ func (m *MockBackendServicesObj) ToGA() *ga.BackendService { // Convert the object via JSON copying to the type that was requested. ret := &ga.BackendService{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.BackendService via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.BackendService via JSON: %v", m.Obj, err) } return ret } @@ -741,7 +741,7 @@ func (m *MockDisksObj) ToGA() *ga.Disk { // Convert the object via JSON copying to the type that was requested. ret := &ga.Disk{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.Disk via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.Disk via JSON: %v", m.Obj, err) } return ret } @@ -761,7 +761,7 @@ func (m *MockFirewallsObj) ToGA() *ga.Firewall { // Convert the object via JSON copying to the type that was requested. ret := &ga.Firewall{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.Firewall via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.Firewall via JSON: %v", m.Obj, err) } return ret } @@ -781,7 +781,7 @@ func (m *MockForwardingRulesObj) ToAlpha() *alpha.ForwardingRule { // Convert the object via JSON copying to the type that was requested. ret := &alpha.ForwardingRule{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *alpha.ForwardingRule via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *alpha.ForwardingRule via JSON: %v", m.Obj, err) } return ret } @@ -794,7 +794,7 @@ func (m *MockForwardingRulesObj) ToGA() *ga.ForwardingRule { // Convert the object via JSON copying to the type that was requested. ret := &ga.ForwardingRule{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.ForwardingRule via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.ForwardingRule via JSON: %v", m.Obj, err) } return ret } @@ -814,7 +814,7 @@ func (m *MockGlobalAddressesObj) ToGA() *ga.Address { // Convert the object via JSON copying to the type that was requested. ret := &ga.Address{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.Address via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.Address via JSON: %v", m.Obj, err) } return ret } @@ -834,7 +834,7 @@ func (m *MockGlobalForwardingRulesObj) ToGA() *ga.ForwardingRule { // Convert the object via JSON copying to the type that was requested. ret := &ga.ForwardingRule{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.ForwardingRule via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.ForwardingRule via JSON: %v", m.Obj, err) } return ret } @@ -854,7 +854,7 @@ func (m *MockHealthChecksObj) ToAlpha() *alpha.HealthCheck { // Convert the object via JSON copying to the type that was requested. ret := &alpha.HealthCheck{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *alpha.HealthCheck via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *alpha.HealthCheck via JSON: %v", m.Obj, err) } return ret } @@ -867,7 +867,7 @@ func (m *MockHealthChecksObj) ToBeta() *beta.HealthCheck { // Convert the object via JSON copying to the type that was requested. ret := &beta.HealthCheck{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *beta.HealthCheck via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *beta.HealthCheck via JSON: %v", m.Obj, err) } return ret } @@ -880,7 +880,7 @@ func (m *MockHealthChecksObj) ToGA() *ga.HealthCheck { // Convert the object via JSON copying to the type that was requested. ret := &ga.HealthCheck{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.HealthCheck via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.HealthCheck via JSON: %v", m.Obj, err) } return ret } @@ -900,7 +900,7 @@ func (m *MockHttpHealthChecksObj) ToGA() *ga.HttpHealthCheck { // Convert the object via JSON copying to the type that was requested. ret := &ga.HttpHealthCheck{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.HttpHealthCheck via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.HttpHealthCheck via JSON: %v", m.Obj, err) } return ret } @@ -920,7 +920,7 @@ func (m *MockHttpsHealthChecksObj) ToGA() *ga.HttpsHealthCheck { // Convert the object via JSON copying to the type that was requested. ret := &ga.HttpsHealthCheck{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.HttpsHealthCheck via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.HttpsHealthCheck via JSON: %v", m.Obj, err) } return ret } @@ -940,7 +940,7 @@ func (m *MockInstanceGroupsObj) ToGA() *ga.InstanceGroup { // Convert the object via JSON copying to the type that was requested. ret := &ga.InstanceGroup{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.InstanceGroup via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.InstanceGroup via JSON: %v", m.Obj, err) } return ret } @@ -960,7 +960,7 @@ func (m *MockInstancesObj) ToAlpha() *alpha.Instance { // Convert the object via JSON copying to the type that was requested. ret := &alpha.Instance{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *alpha.Instance via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *alpha.Instance via JSON: %v", m.Obj, err) } return ret } @@ -973,7 +973,7 @@ func (m *MockInstancesObj) ToBeta() *beta.Instance { // Convert the object via JSON copying to the type that was requested. ret := &beta.Instance{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *beta.Instance via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *beta.Instance via JSON: %v", m.Obj, err) } return ret } @@ -986,7 +986,7 @@ func (m *MockInstancesObj) ToGA() *ga.Instance { // Convert the object via JSON copying to the type that was requested. ret := &ga.Instance{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.Instance via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.Instance via JSON: %v", m.Obj, err) } return ret } @@ -1006,7 +1006,7 @@ func (m *MockNetworkEndpointGroupsObj) ToAlpha() *alpha.NetworkEndpointGroup { // Convert the object via JSON copying to the type that was requested. ret := &alpha.NetworkEndpointGroup{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *alpha.NetworkEndpointGroup via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *alpha.NetworkEndpointGroup via JSON: %v", m.Obj, err) } return ret } @@ -1019,7 +1019,7 @@ func (m *MockNetworkEndpointGroupsObj) ToBeta() *beta.NetworkEndpointGroup { // Convert the object via JSON copying to the type that was requested. ret := &beta.NetworkEndpointGroup{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *beta.NetworkEndpointGroup via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *beta.NetworkEndpointGroup via JSON: %v", m.Obj, err) } return ret } @@ -1039,7 +1039,7 @@ func (m *MockProjectsObj) ToGA() *ga.Project { // Convert the object via JSON copying to the type that was requested. ret := &ga.Project{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.Project via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.Project via JSON: %v", m.Obj, err) } return ret } @@ -1059,7 +1059,7 @@ func (m *MockRegionBackendServicesObj) ToAlpha() *alpha.BackendService { // Convert the object via JSON copying to the type that was requested. ret := &alpha.BackendService{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *alpha.BackendService via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *alpha.BackendService via JSON: %v", m.Obj, err) } return ret } @@ -1072,7 +1072,7 @@ func (m *MockRegionBackendServicesObj) ToGA() *ga.BackendService { // Convert the object via JSON copying to the type that was requested. ret := &ga.BackendService{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.BackendService via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.BackendService via JSON: %v", m.Obj, err) } return ret } @@ -1084,15 +1084,15 @@ type MockRegionDisksObj struct { Obj interface{} } -// ToBeta retrieves the given version of the object. -func (m *MockRegionDisksObj) ToBeta() *beta.Disk { - if ret, ok := m.Obj.(*beta.Disk); ok { +// ToGA retrieves the given version of the object. +func (m *MockRegionDisksObj) ToGA() *ga.Disk { + if ret, ok := m.Obj.(*ga.Disk); ok { return ret } // Convert the object via JSON copying to the type that was requested. - ret := &beta.Disk{} + ret := &ga.Disk{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *beta.Disk via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.Disk via JSON: %v", m.Obj, err) } return ret } @@ -1112,7 +1112,7 @@ func (m *MockRegionsObj) ToGA() *ga.Region { // Convert the object via JSON copying to the type that was requested. ret := &ga.Region{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.Region via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.Region via JSON: %v", m.Obj, err) } return ret } @@ -1132,7 +1132,7 @@ func (m *MockRoutesObj) ToGA() *ga.Route { // Convert the object via JSON copying to the type that was requested. ret := &ga.Route{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.Route via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.Route via JSON: %v", m.Obj, err) } return ret } @@ -1152,7 +1152,7 @@ func (m *MockSecurityPoliciesObj) ToBeta() *beta.SecurityPolicy { // Convert the object via JSON copying to the type that was requested. ret := &beta.SecurityPolicy{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *beta.SecurityPolicy via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *beta.SecurityPolicy via JSON: %v", m.Obj, err) } return ret } @@ -1172,7 +1172,7 @@ func (m *MockSslCertificatesObj) ToGA() *ga.SslCertificate { // Convert the object via JSON copying to the type that was requested. ret := &ga.SslCertificate{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.SslCertificate via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.SslCertificate via JSON: %v", m.Obj, err) } return ret } @@ -1192,7 +1192,7 @@ func (m *MockTargetHttpProxiesObj) ToGA() *ga.TargetHttpProxy { // Convert the object via JSON copying to the type that was requested. ret := &ga.TargetHttpProxy{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.TargetHttpProxy via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.TargetHttpProxy via JSON: %v", m.Obj, err) } return ret } @@ -1212,7 +1212,7 @@ func (m *MockTargetHttpsProxiesObj) ToGA() *ga.TargetHttpsProxy { // Convert the object via JSON copying to the type that was requested. ret := &ga.TargetHttpsProxy{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.TargetHttpsProxy via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.TargetHttpsProxy via JSON: %v", m.Obj, err) } return ret } @@ -1232,7 +1232,7 @@ func (m *MockTargetPoolsObj) ToGA() *ga.TargetPool { // Convert the object via JSON copying to the type that was requested. ret := &ga.TargetPool{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.TargetPool via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.TargetPool via JSON: %v", m.Obj, err) } return ret } @@ -1252,7 +1252,7 @@ func (m *MockUrlMapsObj) ToGA() *ga.UrlMap { // Convert the object via JSON copying to the type that was requested. ret := &ga.UrlMap{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.UrlMap via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.UrlMap via JSON: %v", m.Obj, err) } return ret } @@ -1272,7 +1272,7 @@ func (m *MockZonesObj) ToGA() *ga.Zone { // Convert the object via JSON copying to the type that was requested. ret := &ga.Zone{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *ga.Zone via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *ga.Zone via JSON: %v", m.Obj, err) } return ret } @@ -1332,7 +1332,7 @@ type MockAddresses struct { func (m *MockAddresses) Get(ctx context.Context, key *meta.Key) (*ga.Address, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAddresses.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockAddresses.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -1344,12 +1344,12 @@ func (m *MockAddresses) Get(ctx context.Context, key *meta.Key) (*ga.Address, er defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockAddresses.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAddresses.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockAddresses.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockAddresses.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -1357,7 +1357,7 @@ func (m *MockAddresses) Get(ctx context.Context, key *meta.Key) (*ga.Address, er Code: http.StatusNotFound, Message: fmt.Sprintf("MockAddresses %v not found", key), } - glog.V(5).Infof("MockAddresses.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAddresses.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -1365,7 +1365,7 @@ func (m *MockAddresses) Get(ctx context.Context, key *meta.Key) (*ga.Address, er func (m *MockAddresses) List(ctx context.Context, region string, fl *filter.F) ([]*ga.Address, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, region, fl, m); intercept { - glog.V(5).Infof("MockAddresses.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) + klog.V(5).Infof("MockAddresses.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) return objs, err } } @@ -1375,7 +1375,7 @@ func (m *MockAddresses) List(ctx context.Context, region string, fl *filter.F) ( if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockAddresses.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) + klog.V(5).Infof("MockAddresses.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) return nil, *m.ListError } @@ -1391,7 +1391,7 @@ func (m *MockAddresses) List(ctx context.Context, region string, fl *filter.F) ( objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockAddresses.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) + klog.V(5).Infof("MockAddresses.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) return objs, nil } @@ -1399,7 +1399,7 @@ func (m *MockAddresses) List(ctx context.Context, region string, fl *filter.F) ( func (m *MockAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga.Address) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -1411,7 +1411,7 @@ func (m *MockAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga.Addre defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -1419,7 +1419,7 @@ func (m *MockAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga.Addre Code: http.StatusConflict, Message: fmt.Sprintf("MockAddresses %v exists", key), } - glog.V(5).Infof("MockAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -1428,7 +1428,7 @@ func (m *MockAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga.Addre obj.SelfLink = SelfLink(meta.VersionGA, projectID, "addresses", key) m.Objects[*key] = &MockAddressesObj{obj} - glog.V(5).Infof("MockAddresses.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockAddresses.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -1436,7 +1436,7 @@ func (m *MockAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga.Addre func (m *MockAddresses) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -1448,7 +1448,7 @@ func (m *MockAddresses) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -1456,12 +1456,12 @@ func (m *MockAddresses) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockAddresses %v not found", key), } - glog.V(5).Infof("MockAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockAddresses.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockAddresses.Delete(%v, %v) = nil", ctx, key) return nil } @@ -1477,10 +1477,10 @@ type GCEAddresses struct { // Get the Address named by key. func (g *GCEAddresses) Get(ctx context.Context, key *meta.Key) (*ga.Address, error) { - glog.V(5).Infof("GCEAddresses.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAddresses.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAddresses.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAddresses.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Addresses") @@ -1490,21 +1490,21 @@ func (g *GCEAddresses) Get(ctx context.Context, key *meta.Key) (*ga.Address, err Version: meta.Version("ga"), Service: "Addresses", } - glog.V(5).Infof("GCEAddresses.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAddresses.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAddresses.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAddresses.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.Addresses.Get(projectID, key.Region, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEAddresses.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEAddresses.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all Address objects. func (g *GCEAddresses) List(ctx context.Context, region string, fl *filter.F) ([]*ga.Address, error) { - glog.V(5).Infof("GCEAddresses.List(%v, %v, %v) called", ctx, region, fl) + klog.V(5).Infof("GCEAddresses.List(%v, %v, %v) called", ctx, region, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Addresses") rk := &RateLimitKey{ ProjectID: projectID, @@ -1515,30 +1515,30 @@ func (g *GCEAddresses) List(ctx context.Context, region string, fl *filter.F) ([ if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEAddresses.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) + klog.V(5).Infof("GCEAddresses.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) call := g.s.GA.Addresses.List(projectID, region) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.Address f := func(l *ga.AddressList) error { - glog.V(5).Infof("GCEAddresses.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEAddresses.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEAddresses.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEAddresses.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -1546,9 +1546,9 @@ func (g *GCEAddresses) List(ctx context.Context, region string, fl *filter.F) ([ // Insert Address with key of value obj. func (g *GCEAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga.Address) error { - glog.V(5).Infof("GCEAddresses.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEAddresses.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEAddresses.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAddresses.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Addresses") @@ -1558,9 +1558,9 @@ func (g *GCEAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga.Addres Version: meta.Version("ga"), Service: "Addresses", } - glog.V(5).Infof("GCEAddresses.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAddresses.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAddresses.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAddresses.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -1569,20 +1569,20 @@ func (g *GCEAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga.Addres op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAddresses.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAddresses.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAddresses.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEAddresses.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the Address referenced by key. func (g *GCEAddresses) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEAddresses.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAddresses.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAddresses.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAddresses.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Addresses") @@ -1592,9 +1592,9 @@ func (g *GCEAddresses) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "Addresses", } - glog.V(5).Infof("GCEAddresses.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAddresses.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAddresses.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAddresses.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.Addresses.Delete(projectID, key.Region, key.Name) @@ -1602,12 +1602,12 @@ func (g *GCEAddresses) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } @@ -1666,7 +1666,7 @@ type MockAlphaAddresses struct { func (m *MockAlphaAddresses) Get(ctx context.Context, key *meta.Key) (*alpha.Address, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaAddresses.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaAddresses.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -1678,12 +1678,12 @@ func (m *MockAlphaAddresses) Get(ctx context.Context, key *meta.Key) (*alpha.Add defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockAlphaAddresses.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaAddresses.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToAlpha() - glog.V(5).Infof("MockAlphaAddresses.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockAlphaAddresses.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -1691,7 +1691,7 @@ func (m *MockAlphaAddresses) Get(ctx context.Context, key *meta.Key) (*alpha.Add Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaAddresses %v not found", key), } - glog.V(5).Infof("MockAlphaAddresses.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaAddresses.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -1699,7 +1699,7 @@ func (m *MockAlphaAddresses) Get(ctx context.Context, key *meta.Key) (*alpha.Add func (m *MockAlphaAddresses) List(ctx context.Context, region string, fl *filter.F) ([]*alpha.Address, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, region, fl, m); intercept { - glog.V(5).Infof("MockAlphaAddresses.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) + klog.V(5).Infof("MockAlphaAddresses.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) return objs, err } } @@ -1709,7 +1709,7 @@ func (m *MockAlphaAddresses) List(ctx context.Context, region string, fl *filter if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockAlphaAddresses.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) + klog.V(5).Infof("MockAlphaAddresses.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) return nil, *m.ListError } @@ -1725,7 +1725,7 @@ func (m *MockAlphaAddresses) List(ctx context.Context, region string, fl *filter objs = append(objs, obj.ToAlpha()) } - glog.V(5).Infof("MockAlphaAddresses.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) + klog.V(5).Infof("MockAlphaAddresses.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) return objs, nil } @@ -1733,7 +1733,7 @@ func (m *MockAlphaAddresses) List(ctx context.Context, region string, fl *filter func (m *MockAlphaAddresses) Insert(ctx context.Context, key *meta.Key, obj *alpha.Address) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockAlphaAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -1745,7 +1745,7 @@ func (m *MockAlphaAddresses) Insert(ctx context.Context, key *meta.Key, obj *alp defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockAlphaAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -1753,7 +1753,7 @@ func (m *MockAlphaAddresses) Insert(ctx context.Context, key *meta.Key, obj *alp Code: http.StatusConflict, Message: fmt.Sprintf("MockAlphaAddresses %v exists", key), } - glog.V(5).Infof("MockAlphaAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -1762,7 +1762,7 @@ func (m *MockAlphaAddresses) Insert(ctx context.Context, key *meta.Key, obj *alp obj.SelfLink = SelfLink(meta.VersionAlpha, projectID, "addresses", key) m.Objects[*key] = &MockAddressesObj{obj} - glog.V(5).Infof("MockAlphaAddresses.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockAlphaAddresses.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -1770,7 +1770,7 @@ func (m *MockAlphaAddresses) Insert(ctx context.Context, key *meta.Key, obj *alp func (m *MockAlphaAddresses) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -1782,7 +1782,7 @@ func (m *MockAlphaAddresses) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockAlphaAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -1790,12 +1790,12 @@ func (m *MockAlphaAddresses) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaAddresses %v not found", key), } - glog.V(5).Infof("MockAlphaAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockAlphaAddresses.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockAlphaAddresses.Delete(%v, %v) = nil", ctx, key) return nil } @@ -1811,10 +1811,10 @@ type GCEAlphaAddresses struct { // Get the Address named by key. func (g *GCEAlphaAddresses) Get(ctx context.Context, key *meta.Key) (*alpha.Address, error) { - glog.V(5).Infof("GCEAlphaAddresses.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaAddresses.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaAddresses.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaAddresses.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "Addresses") @@ -1824,21 +1824,21 @@ func (g *GCEAlphaAddresses) Get(ctx context.Context, key *meta.Key) (*alpha.Addr Version: meta.Version("alpha"), Service: "Addresses", } - glog.V(5).Infof("GCEAlphaAddresses.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaAddresses.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaAddresses.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaAddresses.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Alpha.Addresses.Get(projectID, key.Region, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEAlphaAddresses.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEAlphaAddresses.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all Address objects. func (g *GCEAlphaAddresses) List(ctx context.Context, region string, fl *filter.F) ([]*alpha.Address, error) { - glog.V(5).Infof("GCEAlphaAddresses.List(%v, %v, %v) called", ctx, region, fl) + klog.V(5).Infof("GCEAlphaAddresses.List(%v, %v, %v) called", ctx, region, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "Addresses") rk := &RateLimitKey{ ProjectID: projectID, @@ -1849,30 +1849,30 @@ func (g *GCEAlphaAddresses) List(ctx context.Context, region string, fl *filter. if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEAlphaAddresses.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) + klog.V(5).Infof("GCEAlphaAddresses.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) call := g.s.Alpha.Addresses.List(projectID, region) if fl != filter.None { call.Filter(fl.String()) } var all []*alpha.Address f := func(l *alpha.AddressList) error { - glog.V(5).Infof("GCEAlphaAddresses.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEAlphaAddresses.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEAlphaAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEAlphaAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEAlphaAddresses.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEAlphaAddresses.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEAlphaAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEAlphaAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -1880,9 +1880,9 @@ func (g *GCEAlphaAddresses) List(ctx context.Context, region string, fl *filter. // Insert Address with key of value obj. func (g *GCEAlphaAddresses) Insert(ctx context.Context, key *meta.Key, obj *alpha.Address) error { - glog.V(5).Infof("GCEAlphaAddresses.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEAlphaAddresses.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEAlphaAddresses.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaAddresses.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "Addresses") @@ -1892,9 +1892,9 @@ func (g *GCEAlphaAddresses) Insert(ctx context.Context, key *meta.Key, obj *alph Version: meta.Version("alpha"), Service: "Addresses", } - glog.V(5).Infof("GCEAlphaAddresses.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaAddresses.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaAddresses.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaAddresses.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -1903,20 +1903,20 @@ func (g *GCEAlphaAddresses) Insert(ctx context.Context, key *meta.Key, obj *alph op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaAddresses.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaAddresses.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaAddresses.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEAlphaAddresses.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the Address referenced by key. func (g *GCEAlphaAddresses) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEAlphaAddresses.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaAddresses.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaAddresses.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaAddresses.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "Addresses") @@ -1926,9 +1926,9 @@ func (g *GCEAlphaAddresses) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("alpha"), Service: "Addresses", } - glog.V(5).Infof("GCEAlphaAddresses.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaAddresses.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaAddresses.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaAddresses.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.Addresses.Delete(projectID, key.Region, key.Name) @@ -1936,12 +1936,12 @@ func (g *GCEAlphaAddresses) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } @@ -2000,7 +2000,7 @@ type MockBetaAddresses struct { func (m *MockBetaAddresses) Get(ctx context.Context, key *meta.Key) (*beta.Address, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaAddresses.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaAddresses.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -2012,12 +2012,12 @@ func (m *MockBetaAddresses) Get(ctx context.Context, key *meta.Key) (*beta.Addre defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockBetaAddresses.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBetaAddresses.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToBeta() - glog.V(5).Infof("MockBetaAddresses.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockBetaAddresses.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -2025,7 +2025,7 @@ func (m *MockBetaAddresses) Get(ctx context.Context, key *meta.Key) (*beta.Addre Code: http.StatusNotFound, Message: fmt.Sprintf("MockBetaAddresses %v not found", key), } - glog.V(5).Infof("MockBetaAddresses.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBetaAddresses.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -2033,7 +2033,7 @@ func (m *MockBetaAddresses) Get(ctx context.Context, key *meta.Key) (*beta.Addre func (m *MockBetaAddresses) List(ctx context.Context, region string, fl *filter.F) ([]*beta.Address, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, region, fl, m); intercept { - glog.V(5).Infof("MockBetaAddresses.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) + klog.V(5).Infof("MockBetaAddresses.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) return objs, err } } @@ -2043,7 +2043,7 @@ func (m *MockBetaAddresses) List(ctx context.Context, region string, fl *filter. if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockBetaAddresses.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) + klog.V(5).Infof("MockBetaAddresses.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) return nil, *m.ListError } @@ -2059,7 +2059,7 @@ func (m *MockBetaAddresses) List(ctx context.Context, region string, fl *filter. objs = append(objs, obj.ToBeta()) } - glog.V(5).Infof("MockBetaAddresses.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) + klog.V(5).Infof("MockBetaAddresses.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) return objs, nil } @@ -2067,7 +2067,7 @@ func (m *MockBetaAddresses) List(ctx context.Context, region string, fl *filter. func (m *MockBetaAddresses) Insert(ctx context.Context, key *meta.Key, obj *beta.Address) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockBetaAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -2079,7 +2079,7 @@ func (m *MockBetaAddresses) Insert(ctx context.Context, key *meta.Key, obj *beta defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockBetaAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -2087,7 +2087,7 @@ func (m *MockBetaAddresses) Insert(ctx context.Context, key *meta.Key, obj *beta Code: http.StatusConflict, Message: fmt.Sprintf("MockBetaAddresses %v exists", key), } - glog.V(5).Infof("MockBetaAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -2096,7 +2096,7 @@ func (m *MockBetaAddresses) Insert(ctx context.Context, key *meta.Key, obj *beta obj.SelfLink = SelfLink(meta.VersionBeta, projectID, "addresses", key) m.Objects[*key] = &MockAddressesObj{obj} - glog.V(5).Infof("MockBetaAddresses.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockBetaAddresses.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -2104,7 +2104,7 @@ func (m *MockBetaAddresses) Insert(ctx context.Context, key *meta.Key, obj *beta func (m *MockBetaAddresses) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -2116,7 +2116,7 @@ func (m *MockBetaAddresses) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockBetaAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -2124,12 +2124,12 @@ func (m *MockBetaAddresses) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockBetaAddresses %v not found", key), } - glog.V(5).Infof("MockBetaAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockBetaAddresses.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockBetaAddresses.Delete(%v, %v) = nil", ctx, key) return nil } @@ -2145,10 +2145,10 @@ type GCEBetaAddresses struct { // Get the Address named by key. func (g *GCEBetaAddresses) Get(ctx context.Context, key *meta.Key) (*beta.Address, error) { - glog.V(5).Infof("GCEBetaAddresses.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBetaAddresses.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaAddresses.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaAddresses.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "Addresses") @@ -2158,21 +2158,21 @@ func (g *GCEBetaAddresses) Get(ctx context.Context, key *meta.Key) (*beta.Addres Version: meta.Version("beta"), Service: "Addresses", } - glog.V(5).Infof("GCEBetaAddresses.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaAddresses.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaAddresses.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaAddresses.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Beta.Addresses.Get(projectID, key.Region, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEBetaAddresses.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEBetaAddresses.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all Address objects. func (g *GCEBetaAddresses) List(ctx context.Context, region string, fl *filter.F) ([]*beta.Address, error) { - glog.V(5).Infof("GCEBetaAddresses.List(%v, %v, %v) called", ctx, region, fl) + klog.V(5).Infof("GCEBetaAddresses.List(%v, %v, %v) called", ctx, region, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "Addresses") rk := &RateLimitKey{ ProjectID: projectID, @@ -2183,30 +2183,30 @@ func (g *GCEBetaAddresses) List(ctx context.Context, region string, fl *filter.F if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEBetaAddresses.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) + klog.V(5).Infof("GCEBetaAddresses.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) call := g.s.Beta.Addresses.List(projectID, region) if fl != filter.None { call.Filter(fl.String()) } var all []*beta.Address f := func(l *beta.AddressList) error { - glog.V(5).Infof("GCEBetaAddresses.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEBetaAddresses.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEBetaAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEBetaAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEBetaAddresses.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEBetaAddresses.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEBetaAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEBetaAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -2214,9 +2214,9 @@ func (g *GCEBetaAddresses) List(ctx context.Context, region string, fl *filter.F // Insert Address with key of value obj. func (g *GCEBetaAddresses) Insert(ctx context.Context, key *meta.Key, obj *beta.Address) error { - glog.V(5).Infof("GCEBetaAddresses.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEBetaAddresses.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEBetaAddresses.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaAddresses.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "Addresses") @@ -2226,9 +2226,9 @@ func (g *GCEBetaAddresses) Insert(ctx context.Context, key *meta.Key, obj *beta. Version: meta.Version("beta"), Service: "Addresses", } - glog.V(5).Infof("GCEBetaAddresses.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaAddresses.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaAddresses.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaAddresses.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -2237,20 +2237,20 @@ func (g *GCEBetaAddresses) Insert(ctx context.Context, key *meta.Key, obj *beta. op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaAddresses.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaAddresses.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaAddresses.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEBetaAddresses.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the Address referenced by key. func (g *GCEBetaAddresses) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEBetaAddresses.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBetaAddresses.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaAddresses.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaAddresses.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "Addresses") @@ -2260,9 +2260,9 @@ func (g *GCEBetaAddresses) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("beta"), Service: "Addresses", } - glog.V(5).Infof("GCEBetaAddresses.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaAddresses.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaAddresses.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaAddresses.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.Addresses.Delete(projectID, key.Region, key.Name) @@ -2270,12 +2270,12 @@ func (g *GCEBetaAddresses) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBetaAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBetaAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } @@ -2334,7 +2334,7 @@ type MockGlobalAddresses struct { func (m *MockGlobalAddresses) Get(ctx context.Context, key *meta.Key) (*ga.Address, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockGlobalAddresses.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockGlobalAddresses.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -2346,12 +2346,12 @@ func (m *MockGlobalAddresses) Get(ctx context.Context, key *meta.Key) (*ga.Addre defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockGlobalAddresses.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockGlobalAddresses.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockGlobalAddresses.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockGlobalAddresses.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -2359,7 +2359,7 @@ func (m *MockGlobalAddresses) Get(ctx context.Context, key *meta.Key) (*ga.Addre Code: http.StatusNotFound, Message: fmt.Sprintf("MockGlobalAddresses %v not found", key), } - glog.V(5).Infof("MockGlobalAddresses.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockGlobalAddresses.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -2367,7 +2367,7 @@ func (m *MockGlobalAddresses) Get(ctx context.Context, key *meta.Key) (*ga.Addre func (m *MockGlobalAddresses) List(ctx context.Context, fl *filter.F) ([]*ga.Address, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockGlobalAddresses.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockGlobalAddresses.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -2377,7 +2377,7 @@ func (m *MockGlobalAddresses) List(ctx context.Context, fl *filter.F) ([]*ga.Add if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockGlobalAddresses.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockGlobalAddresses.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -2390,7 +2390,7 @@ func (m *MockGlobalAddresses) List(ctx context.Context, fl *filter.F) ([]*ga.Add objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockGlobalAddresses.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockGlobalAddresses.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -2398,7 +2398,7 @@ func (m *MockGlobalAddresses) List(ctx context.Context, fl *filter.F) ([]*ga.Add func (m *MockGlobalAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga.Address) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockGlobalAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockGlobalAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -2410,7 +2410,7 @@ func (m *MockGlobalAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockGlobalAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockGlobalAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -2418,7 +2418,7 @@ func (m *MockGlobalAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga Code: http.StatusConflict, Message: fmt.Sprintf("MockGlobalAddresses %v exists", key), } - glog.V(5).Infof("MockGlobalAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockGlobalAddresses.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -2427,7 +2427,7 @@ func (m *MockGlobalAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga obj.SelfLink = SelfLink(meta.VersionGA, projectID, "addresses", key) m.Objects[*key] = &MockGlobalAddressesObj{obj} - glog.V(5).Infof("MockGlobalAddresses.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockGlobalAddresses.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -2435,7 +2435,7 @@ func (m *MockGlobalAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga func (m *MockGlobalAddresses) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockGlobalAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockGlobalAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -2447,7 +2447,7 @@ func (m *MockGlobalAddresses) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockGlobalAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockGlobalAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -2455,12 +2455,12 @@ func (m *MockGlobalAddresses) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockGlobalAddresses %v not found", key), } - glog.V(5).Infof("MockGlobalAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockGlobalAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockGlobalAddresses.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockGlobalAddresses.Delete(%v, %v) = nil", ctx, key) return nil } @@ -2476,10 +2476,10 @@ type GCEGlobalAddresses struct { // Get the Address named by key. func (g *GCEGlobalAddresses) Get(ctx context.Context, key *meta.Key) (*ga.Address, error) { - glog.V(5).Infof("GCEGlobalAddresses.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEGlobalAddresses.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEGlobalAddresses.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEGlobalAddresses.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "GlobalAddresses") @@ -2489,21 +2489,21 @@ func (g *GCEGlobalAddresses) Get(ctx context.Context, key *meta.Key) (*ga.Addres Version: meta.Version("ga"), Service: "GlobalAddresses", } - glog.V(5).Infof("GCEGlobalAddresses.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEGlobalAddresses.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEGlobalAddresses.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEGlobalAddresses.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.GlobalAddresses.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEGlobalAddresses.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEGlobalAddresses.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all Address objects. func (g *GCEGlobalAddresses) List(ctx context.Context, fl *filter.F) ([]*ga.Address, error) { - glog.V(5).Infof("GCEGlobalAddresses.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEGlobalAddresses.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "GlobalAddresses") rk := &RateLimitKey{ ProjectID: projectID, @@ -2514,30 +2514,30 @@ func (g *GCEGlobalAddresses) List(ctx context.Context, fl *filter.F) ([]*ga.Addr if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEGlobalAddresses.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEGlobalAddresses.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.GlobalAddresses.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.Address f := func(l *ga.AddressList) error { - glog.V(5).Infof("GCEGlobalAddresses.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEGlobalAddresses.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEGlobalAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEGlobalAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEGlobalAddresses.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEGlobalAddresses.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEGlobalAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEGlobalAddresses.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -2545,9 +2545,9 @@ func (g *GCEGlobalAddresses) List(ctx context.Context, fl *filter.F) ([]*ga.Addr // Insert Address with key of value obj. func (g *GCEGlobalAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga.Address) error { - glog.V(5).Infof("GCEGlobalAddresses.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEGlobalAddresses.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEGlobalAddresses.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEGlobalAddresses.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "GlobalAddresses") @@ -2557,9 +2557,9 @@ func (g *GCEGlobalAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga. Version: meta.Version("ga"), Service: "GlobalAddresses", } - glog.V(5).Infof("GCEGlobalAddresses.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEGlobalAddresses.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEGlobalAddresses.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEGlobalAddresses.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -2568,20 +2568,20 @@ func (g *GCEGlobalAddresses) Insert(ctx context.Context, key *meta.Key, obj *ga. op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEGlobalAddresses.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEGlobalAddresses.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEGlobalAddresses.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEGlobalAddresses.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the Address referenced by key. func (g *GCEGlobalAddresses) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEGlobalAddresses.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEGlobalAddresses.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEGlobalAddresses.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEGlobalAddresses.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "GlobalAddresses") @@ -2591,9 +2591,9 @@ func (g *GCEGlobalAddresses) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "GlobalAddresses", } - glog.V(5).Infof("GCEGlobalAddresses.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEGlobalAddresses.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEGlobalAddresses.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEGlobalAddresses.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.GlobalAddresses.Delete(projectID, key.Name) @@ -2602,12 +2602,12 @@ func (g *GCEGlobalAddresses) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEGlobalAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEGlobalAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEGlobalAddresses.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEGlobalAddresses.Delete(%v, %v) = %v", ctx, key, err) return err } @@ -2672,7 +2672,7 @@ type MockBackendServices struct { func (m *MockBackendServices) Get(ctx context.Context, key *meta.Key) (*ga.BackendService, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBackendServices.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockBackendServices.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -2684,12 +2684,12 @@ func (m *MockBackendServices) Get(ctx context.Context, key *meta.Key) (*ga.Backe defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockBackendServices.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockBackendServices.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -2697,7 +2697,7 @@ func (m *MockBackendServices) Get(ctx context.Context, key *meta.Key) (*ga.Backe Code: http.StatusNotFound, Message: fmt.Sprintf("MockBackendServices %v not found", key), } - glog.V(5).Infof("MockBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -2705,7 +2705,7 @@ func (m *MockBackendServices) Get(ctx context.Context, key *meta.Key) (*ga.Backe func (m *MockBackendServices) List(ctx context.Context, fl *filter.F) ([]*ga.BackendService, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockBackendServices.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockBackendServices.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -2715,7 +2715,7 @@ func (m *MockBackendServices) List(ctx context.Context, fl *filter.F) ([]*ga.Bac if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockBackendServices.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockBackendServices.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -2728,7 +2728,7 @@ func (m *MockBackendServices) List(ctx context.Context, fl *filter.F) ([]*ga.Bac objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockBackendServices.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockBackendServices.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -2736,7 +2736,7 @@ func (m *MockBackendServices) List(ctx context.Context, fl *filter.F) ([]*ga.Bac func (m *MockBackendServices) Insert(ctx context.Context, key *meta.Key, obj *ga.BackendService) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -2748,7 +2748,7 @@ func (m *MockBackendServices) Insert(ctx context.Context, key *meta.Key, obj *ga defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -2756,7 +2756,7 @@ func (m *MockBackendServices) Insert(ctx context.Context, key *meta.Key, obj *ga Code: http.StatusConflict, Message: fmt.Sprintf("MockBackendServices %v exists", key), } - glog.V(5).Infof("MockBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -2765,7 +2765,7 @@ func (m *MockBackendServices) Insert(ctx context.Context, key *meta.Key, obj *ga obj.SelfLink = SelfLink(meta.VersionGA, projectID, "backendServices", key) m.Objects[*key] = &MockBackendServicesObj{obj} - glog.V(5).Infof("MockBackendServices.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockBackendServices.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -2773,7 +2773,7 @@ func (m *MockBackendServices) Insert(ctx context.Context, key *meta.Key, obj *ga func (m *MockBackendServices) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -2785,7 +2785,7 @@ func (m *MockBackendServices) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -2793,12 +2793,12 @@ func (m *MockBackendServices) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockBackendServices %v not found", key), } - glog.V(5).Infof("MockBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockBackendServices.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockBackendServices.Delete(%v, %v) = nil", ctx, key) return nil } @@ -2838,10 +2838,10 @@ type GCEBackendServices struct { // Get the BackendService named by key. func (g *GCEBackendServices) Get(ctx context.Context, key *meta.Key) (*ga.BackendService, error) { - glog.V(5).Infof("GCEBackendServices.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBackendServices.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBackendServices.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBackendServices.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "BackendServices") @@ -2851,21 +2851,21 @@ func (g *GCEBackendServices) Get(ctx context.Context, key *meta.Key) (*ga.Backen Version: meta.Version("ga"), Service: "BackendServices", } - glog.V(5).Infof("GCEBackendServices.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBackendServices.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBackendServices.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBackendServices.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.BackendServices.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEBackendServices.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEBackendServices.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all BackendService objects. func (g *GCEBackendServices) List(ctx context.Context, fl *filter.F) ([]*ga.BackendService, error) { - glog.V(5).Infof("GCEBackendServices.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEBackendServices.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "BackendServices") rk := &RateLimitKey{ ProjectID: projectID, @@ -2876,30 +2876,30 @@ func (g *GCEBackendServices) List(ctx context.Context, fl *filter.F) ([]*ga.Back if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEBackendServices.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEBackendServices.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.BackendServices.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.BackendService f := func(l *ga.BackendServiceList) error { - glog.V(5).Infof("GCEBackendServices.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEBackendServices.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEBackendServices.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEBackendServices.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -2907,9 +2907,9 @@ func (g *GCEBackendServices) List(ctx context.Context, fl *filter.F) ([]*ga.Back // Insert BackendService with key of value obj. func (g *GCEBackendServices) Insert(ctx context.Context, key *meta.Key, obj *ga.BackendService) error { - glog.V(5).Infof("GCEBackendServices.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEBackendServices.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEBackendServices.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBackendServices.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "BackendServices") @@ -2919,9 +2919,9 @@ func (g *GCEBackendServices) Insert(ctx context.Context, key *meta.Key, obj *ga. Version: meta.Version("ga"), Service: "BackendServices", } - glog.V(5).Infof("GCEBackendServices.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBackendServices.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBackendServices.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBackendServices.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -2930,20 +2930,20 @@ func (g *GCEBackendServices) Insert(ctx context.Context, key *meta.Key, obj *ga. op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBackendServices.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBackendServices.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBackendServices.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEBackendServices.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the BackendService referenced by key. func (g *GCEBackendServices) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEBackendServices.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBackendServices.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBackendServices.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBackendServices.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "BackendServices") @@ -2953,9 +2953,9 @@ func (g *GCEBackendServices) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "BackendServices", } - glog.V(5).Infof("GCEBackendServices.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBackendServices.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBackendServices.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBackendServices.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.BackendServices.Delete(projectID, key.Name) @@ -2964,21 +2964,21 @@ func (g *GCEBackendServices) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } // GetHealth is a method on GCEBackendServices. func (g *GCEBackendServices) GetHealth(ctx context.Context, key *meta.Key, arg0 *ga.ResourceGroupReference) (*ga.BackendServiceGroupHealth, error) { - glog.V(5).Infof("GCEBackendServices.GetHealth(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBackendServices.GetHealth(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBackendServices.GetHealth(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBackendServices.GetHealth(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "BackendServices") @@ -2988,25 +2988,25 @@ func (g *GCEBackendServices) GetHealth(ctx context.Context, key *meta.Key, arg0 Version: meta.Version("ga"), Service: "BackendServices", } - glog.V(5).Infof("GCEBackendServices.GetHealth(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBackendServices.GetHealth(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBackendServices.GetHealth(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBackendServices.GetHealth(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.BackendServices.GetHealth(projectID, key.Name, arg0) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEBackendServices.GetHealth(%v, %v, ...) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEBackendServices.GetHealth(%v, %v, ...) = %+v, %v", ctx, key, v, err) return v, err } // Patch is a method on GCEBackendServices. func (g *GCEBackendServices) Patch(ctx context.Context, key *meta.Key, arg0 *ga.BackendService) error { - glog.V(5).Infof("GCEBackendServices.Patch(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBackendServices.Patch(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBackendServices.Patch(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBackendServices.Patch(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "BackendServices") @@ -3016,30 +3016,30 @@ func (g *GCEBackendServices) Patch(ctx context.Context, key *meta.Key, arg0 *ga. Version: meta.Version("ga"), Service: "BackendServices", } - glog.V(5).Infof("GCEBackendServices.Patch(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBackendServices.Patch(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBackendServices.Patch(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBackendServices.Patch(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.BackendServices.Patch(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBackendServices.Patch(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBackendServices.Patch(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBackendServices.Patch(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBackendServices.Patch(%v, %v, ...) = %+v", ctx, key, err) return err } // Update is a method on GCEBackendServices. func (g *GCEBackendServices) Update(ctx context.Context, key *meta.Key, arg0 *ga.BackendService) error { - glog.V(5).Infof("GCEBackendServices.Update(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBackendServices.Update(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBackendServices.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBackendServices.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "BackendServices") @@ -3049,21 +3049,21 @@ func (g *GCEBackendServices) Update(ctx context.Context, key *meta.Key, arg0 *ga Version: meta.Version("ga"), Service: "BackendServices", } - glog.V(5).Infof("GCEBackendServices.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBackendServices.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBackendServices.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBackendServices.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.BackendServices.Update(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -3126,7 +3126,7 @@ type MockBetaBackendServices struct { func (m *MockBetaBackendServices) Get(ctx context.Context, key *meta.Key) (*beta.BackendService, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaBackendServices.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaBackendServices.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -3138,12 +3138,12 @@ func (m *MockBetaBackendServices) Get(ctx context.Context, key *meta.Key) (*beta defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockBetaBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBetaBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToBeta() - glog.V(5).Infof("MockBetaBackendServices.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockBetaBackendServices.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -3151,7 +3151,7 @@ func (m *MockBetaBackendServices) Get(ctx context.Context, key *meta.Key) (*beta Code: http.StatusNotFound, Message: fmt.Sprintf("MockBetaBackendServices %v not found", key), } - glog.V(5).Infof("MockBetaBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBetaBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -3159,7 +3159,7 @@ func (m *MockBetaBackendServices) Get(ctx context.Context, key *meta.Key) (*beta func (m *MockBetaBackendServices) List(ctx context.Context, fl *filter.F) ([]*beta.BackendService, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockBetaBackendServices.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockBetaBackendServices.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -3169,7 +3169,7 @@ func (m *MockBetaBackendServices) List(ctx context.Context, fl *filter.F) ([]*be if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockBetaBackendServices.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockBetaBackendServices.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -3182,7 +3182,7 @@ func (m *MockBetaBackendServices) List(ctx context.Context, fl *filter.F) ([]*be objs = append(objs, obj.ToBeta()) } - glog.V(5).Infof("MockBetaBackendServices.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockBetaBackendServices.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -3190,7 +3190,7 @@ func (m *MockBetaBackendServices) List(ctx context.Context, fl *filter.F) ([]*be func (m *MockBetaBackendServices) Insert(ctx context.Context, key *meta.Key, obj *beta.BackendService) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockBetaBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -3202,7 +3202,7 @@ func (m *MockBetaBackendServices) Insert(ctx context.Context, key *meta.Key, obj defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockBetaBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -3210,7 +3210,7 @@ func (m *MockBetaBackendServices) Insert(ctx context.Context, key *meta.Key, obj Code: http.StatusConflict, Message: fmt.Sprintf("MockBetaBackendServices %v exists", key), } - glog.V(5).Infof("MockBetaBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -3219,7 +3219,7 @@ func (m *MockBetaBackendServices) Insert(ctx context.Context, key *meta.Key, obj obj.SelfLink = SelfLink(meta.VersionBeta, projectID, "backendServices", key) m.Objects[*key] = &MockBackendServicesObj{obj} - glog.V(5).Infof("MockBetaBackendServices.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockBetaBackendServices.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -3227,7 +3227,7 @@ func (m *MockBetaBackendServices) Insert(ctx context.Context, key *meta.Key, obj func (m *MockBetaBackendServices) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -3239,7 +3239,7 @@ func (m *MockBetaBackendServices) Delete(ctx context.Context, key *meta.Key) err defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockBetaBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -3247,12 +3247,12 @@ func (m *MockBetaBackendServices) Delete(ctx context.Context, key *meta.Key) err Code: http.StatusNotFound, Message: fmt.Sprintf("MockBetaBackendServices %v not found", key), } - glog.V(5).Infof("MockBetaBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockBetaBackendServices.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockBetaBackendServices.Delete(%v, %v) = nil", ctx, key) return nil } @@ -3284,10 +3284,10 @@ type GCEBetaBackendServices struct { // Get the BackendService named by key. func (g *GCEBetaBackendServices) Get(ctx context.Context, key *meta.Key) (*beta.BackendService, error) { - glog.V(5).Infof("GCEBetaBackendServices.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBetaBackendServices.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaBackendServices.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaBackendServices.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "BackendServices") @@ -3297,21 +3297,21 @@ func (g *GCEBetaBackendServices) Get(ctx context.Context, key *meta.Key) (*beta. Version: meta.Version("beta"), Service: "BackendServices", } - glog.V(5).Infof("GCEBetaBackendServices.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaBackendServices.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaBackendServices.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaBackendServices.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Beta.BackendServices.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEBetaBackendServices.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEBetaBackendServices.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all BackendService objects. func (g *GCEBetaBackendServices) List(ctx context.Context, fl *filter.F) ([]*beta.BackendService, error) { - glog.V(5).Infof("GCEBetaBackendServices.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEBetaBackendServices.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "BackendServices") rk := &RateLimitKey{ ProjectID: projectID, @@ -3322,30 +3322,30 @@ func (g *GCEBetaBackendServices) List(ctx context.Context, fl *filter.F) ([]*bet if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEBetaBackendServices.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEBetaBackendServices.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.Beta.BackendServices.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*beta.BackendService f := func(l *beta.BackendServiceList) error { - glog.V(5).Infof("GCEBetaBackendServices.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEBetaBackendServices.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEBetaBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEBetaBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEBetaBackendServices.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEBetaBackendServices.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEBetaBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEBetaBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -3353,9 +3353,9 @@ func (g *GCEBetaBackendServices) List(ctx context.Context, fl *filter.F) ([]*bet // Insert BackendService with key of value obj. func (g *GCEBetaBackendServices) Insert(ctx context.Context, key *meta.Key, obj *beta.BackendService) error { - glog.V(5).Infof("GCEBetaBackendServices.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEBetaBackendServices.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEBetaBackendServices.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaBackendServices.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "BackendServices") @@ -3365,9 +3365,9 @@ func (g *GCEBetaBackendServices) Insert(ctx context.Context, key *meta.Key, obj Version: meta.Version("beta"), Service: "BackendServices", } - glog.V(5).Infof("GCEBetaBackendServices.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaBackendServices.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaBackendServices.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaBackendServices.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -3376,20 +3376,20 @@ func (g *GCEBetaBackendServices) Insert(ctx context.Context, key *meta.Key, obj op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaBackendServices.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaBackendServices.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaBackendServices.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEBetaBackendServices.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the BackendService referenced by key. func (g *GCEBetaBackendServices) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEBetaBackendServices.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBetaBackendServices.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaBackendServices.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaBackendServices.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "BackendServices") @@ -3399,9 +3399,9 @@ func (g *GCEBetaBackendServices) Delete(ctx context.Context, key *meta.Key) erro Version: meta.Version("beta"), Service: "BackendServices", } - glog.V(5).Infof("GCEBetaBackendServices.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaBackendServices.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaBackendServices.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaBackendServices.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.BackendServices.Delete(projectID, key.Name) @@ -3410,21 +3410,21 @@ func (g *GCEBetaBackendServices) Delete(ctx context.Context, key *meta.Key) erro op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBetaBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBetaBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } // SetSecurityPolicy is a method on GCEBetaBackendServices. func (g *GCEBetaBackendServices) SetSecurityPolicy(ctx context.Context, key *meta.Key, arg0 *beta.SecurityPolicyReference) error { - glog.V(5).Infof("GCEBetaBackendServices.SetSecurityPolicy(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaBackendServices.SetSecurityPolicy(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaBackendServices.SetSecurityPolicy(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaBackendServices.SetSecurityPolicy(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "BackendServices") @@ -3434,30 +3434,30 @@ func (g *GCEBetaBackendServices) SetSecurityPolicy(ctx context.Context, key *met Version: meta.Version("beta"), Service: "BackendServices", } - glog.V(5).Infof("GCEBetaBackendServices.SetSecurityPolicy(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaBackendServices.SetSecurityPolicy(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaBackendServices.SetSecurityPolicy(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaBackendServices.SetSecurityPolicy(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.BackendServices.SetSecurityPolicy(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaBackendServices.SetSecurityPolicy(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaBackendServices.SetSecurityPolicy(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaBackendServices.SetSecurityPolicy(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaBackendServices.SetSecurityPolicy(%v, %v, ...) = %+v", ctx, key, err) return err } // Update is a method on GCEBetaBackendServices. func (g *GCEBetaBackendServices) Update(ctx context.Context, key *meta.Key, arg0 *beta.BackendService) error { - glog.V(5).Infof("GCEBetaBackendServices.Update(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaBackendServices.Update(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaBackendServices.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaBackendServices.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "BackendServices") @@ -3467,21 +3467,21 @@ func (g *GCEBetaBackendServices) Update(ctx context.Context, key *meta.Key, arg0 Version: meta.Version("beta"), Service: "BackendServices", } - glog.V(5).Infof("GCEBetaBackendServices.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaBackendServices.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaBackendServices.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaBackendServices.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.BackendServices.Update(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -3544,7 +3544,7 @@ type MockAlphaBackendServices struct { func (m *MockAlphaBackendServices) Get(ctx context.Context, key *meta.Key) (*alpha.BackendService, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaBackendServices.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaBackendServices.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -3556,12 +3556,12 @@ func (m *MockAlphaBackendServices) Get(ctx context.Context, key *meta.Key) (*alp defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockAlphaBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToAlpha() - glog.V(5).Infof("MockAlphaBackendServices.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockAlphaBackendServices.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -3569,7 +3569,7 @@ func (m *MockAlphaBackendServices) Get(ctx context.Context, key *meta.Key) (*alp Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaBackendServices %v not found", key), } - glog.V(5).Infof("MockAlphaBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -3577,7 +3577,7 @@ func (m *MockAlphaBackendServices) Get(ctx context.Context, key *meta.Key) (*alp func (m *MockAlphaBackendServices) List(ctx context.Context, fl *filter.F) ([]*alpha.BackendService, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockAlphaBackendServices.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockAlphaBackendServices.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -3587,7 +3587,7 @@ func (m *MockAlphaBackendServices) List(ctx context.Context, fl *filter.F) ([]*a if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockAlphaBackendServices.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockAlphaBackendServices.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -3600,7 +3600,7 @@ func (m *MockAlphaBackendServices) List(ctx context.Context, fl *filter.F) ([]*a objs = append(objs, obj.ToAlpha()) } - glog.V(5).Infof("MockAlphaBackendServices.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockAlphaBackendServices.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -3608,7 +3608,7 @@ func (m *MockAlphaBackendServices) List(ctx context.Context, fl *filter.F) ([]*a func (m *MockAlphaBackendServices) Insert(ctx context.Context, key *meta.Key, obj *alpha.BackendService) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockAlphaBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -3620,7 +3620,7 @@ func (m *MockAlphaBackendServices) Insert(ctx context.Context, key *meta.Key, ob defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockAlphaBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -3628,7 +3628,7 @@ func (m *MockAlphaBackendServices) Insert(ctx context.Context, key *meta.Key, ob Code: http.StatusConflict, Message: fmt.Sprintf("MockAlphaBackendServices %v exists", key), } - glog.V(5).Infof("MockAlphaBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -3637,7 +3637,7 @@ func (m *MockAlphaBackendServices) Insert(ctx context.Context, key *meta.Key, ob obj.SelfLink = SelfLink(meta.VersionAlpha, projectID, "backendServices", key) m.Objects[*key] = &MockBackendServicesObj{obj} - glog.V(5).Infof("MockAlphaBackendServices.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockAlphaBackendServices.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -3645,7 +3645,7 @@ func (m *MockAlphaBackendServices) Insert(ctx context.Context, key *meta.Key, ob func (m *MockAlphaBackendServices) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -3657,7 +3657,7 @@ func (m *MockAlphaBackendServices) Delete(ctx context.Context, key *meta.Key) er defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockAlphaBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -3665,12 +3665,12 @@ func (m *MockAlphaBackendServices) Delete(ctx context.Context, key *meta.Key) er Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaBackendServices %v not found", key), } - glog.V(5).Infof("MockAlphaBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockAlphaBackendServices.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockAlphaBackendServices.Delete(%v, %v) = nil", ctx, key) return nil } @@ -3702,10 +3702,10 @@ type GCEAlphaBackendServices struct { // Get the BackendService named by key. func (g *GCEAlphaBackendServices) Get(ctx context.Context, key *meta.Key) (*alpha.BackendService, error) { - glog.V(5).Infof("GCEAlphaBackendServices.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaBackendServices.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaBackendServices.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaBackendServices.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "BackendServices") @@ -3715,21 +3715,21 @@ func (g *GCEAlphaBackendServices) Get(ctx context.Context, key *meta.Key) (*alph Version: meta.Version("alpha"), Service: "BackendServices", } - glog.V(5).Infof("GCEAlphaBackendServices.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaBackendServices.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaBackendServices.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaBackendServices.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Alpha.BackendServices.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEAlphaBackendServices.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEAlphaBackendServices.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all BackendService objects. func (g *GCEAlphaBackendServices) List(ctx context.Context, fl *filter.F) ([]*alpha.BackendService, error) { - glog.V(5).Infof("GCEAlphaBackendServices.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEAlphaBackendServices.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "BackendServices") rk := &RateLimitKey{ ProjectID: projectID, @@ -3740,30 +3740,30 @@ func (g *GCEAlphaBackendServices) List(ctx context.Context, fl *filter.F) ([]*al if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEAlphaBackendServices.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEAlphaBackendServices.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.Alpha.BackendServices.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*alpha.BackendService f := func(l *alpha.BackendServiceList) error { - glog.V(5).Infof("GCEAlphaBackendServices.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEAlphaBackendServices.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEAlphaBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEAlphaBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEAlphaBackendServices.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEAlphaBackendServices.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEAlphaBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEAlphaBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -3771,9 +3771,9 @@ func (g *GCEAlphaBackendServices) List(ctx context.Context, fl *filter.F) ([]*al // Insert BackendService with key of value obj. func (g *GCEAlphaBackendServices) Insert(ctx context.Context, key *meta.Key, obj *alpha.BackendService) error { - glog.V(5).Infof("GCEAlphaBackendServices.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEAlphaBackendServices.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEAlphaBackendServices.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaBackendServices.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "BackendServices") @@ -3783,9 +3783,9 @@ func (g *GCEAlphaBackendServices) Insert(ctx context.Context, key *meta.Key, obj Version: meta.Version("alpha"), Service: "BackendServices", } - glog.V(5).Infof("GCEAlphaBackendServices.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaBackendServices.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaBackendServices.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaBackendServices.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -3794,20 +3794,20 @@ func (g *GCEAlphaBackendServices) Insert(ctx context.Context, key *meta.Key, obj op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaBackendServices.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaBackendServices.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaBackendServices.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEAlphaBackendServices.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the BackendService referenced by key. func (g *GCEAlphaBackendServices) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEAlphaBackendServices.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaBackendServices.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaBackendServices.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaBackendServices.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "BackendServices") @@ -3817,9 +3817,9 @@ func (g *GCEAlphaBackendServices) Delete(ctx context.Context, key *meta.Key) err Version: meta.Version("alpha"), Service: "BackendServices", } - glog.V(5).Infof("GCEAlphaBackendServices.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaBackendServices.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaBackendServices.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaBackendServices.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.BackendServices.Delete(projectID, key.Name) @@ -3828,21 +3828,21 @@ func (g *GCEAlphaBackendServices) Delete(ctx context.Context, key *meta.Key) err op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } // SetSecurityPolicy is a method on GCEAlphaBackendServices. func (g *GCEAlphaBackendServices) SetSecurityPolicy(ctx context.Context, key *meta.Key, arg0 *alpha.SecurityPolicyReference) error { - glog.V(5).Infof("GCEAlphaBackendServices.SetSecurityPolicy(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEAlphaBackendServices.SetSecurityPolicy(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaBackendServices.SetSecurityPolicy(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaBackendServices.SetSecurityPolicy(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "BackendServices") @@ -3852,30 +3852,30 @@ func (g *GCEAlphaBackendServices) SetSecurityPolicy(ctx context.Context, key *me Version: meta.Version("alpha"), Service: "BackendServices", } - glog.V(5).Infof("GCEAlphaBackendServices.SetSecurityPolicy(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaBackendServices.SetSecurityPolicy(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaBackendServices.SetSecurityPolicy(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaBackendServices.SetSecurityPolicy(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.BackendServices.SetSecurityPolicy(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaBackendServices.SetSecurityPolicy(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaBackendServices.SetSecurityPolicy(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaBackendServices.SetSecurityPolicy(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaBackendServices.SetSecurityPolicy(%v, %v, ...) = %+v", ctx, key, err) return err } // Update is a method on GCEAlphaBackendServices. func (g *GCEAlphaBackendServices) Update(ctx context.Context, key *meta.Key, arg0 *alpha.BackendService) error { - glog.V(5).Infof("GCEAlphaBackendServices.Update(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEAlphaBackendServices.Update(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaBackendServices.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaBackendServices.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "BackendServices") @@ -3885,21 +3885,21 @@ func (g *GCEAlphaBackendServices) Update(ctx context.Context, key *meta.Key, arg Version: meta.Version("alpha"), Service: "BackendServices", } - glog.V(5).Infof("GCEAlphaBackendServices.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaBackendServices.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaBackendServices.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaBackendServices.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.BackendServices.Update(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -3962,7 +3962,7 @@ type MockRegionBackendServices struct { func (m *MockRegionBackendServices) Get(ctx context.Context, key *meta.Key) (*ga.BackendService, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockRegionBackendServices.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockRegionBackendServices.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -3974,12 +3974,12 @@ func (m *MockRegionBackendServices) Get(ctx context.Context, key *meta.Key) (*ga defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockRegionBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockRegionBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockRegionBackendServices.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockRegionBackendServices.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -3987,7 +3987,7 @@ func (m *MockRegionBackendServices) Get(ctx context.Context, key *meta.Key) (*ga Code: http.StatusNotFound, Message: fmt.Sprintf("MockRegionBackendServices %v not found", key), } - glog.V(5).Infof("MockRegionBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockRegionBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -3995,7 +3995,7 @@ func (m *MockRegionBackendServices) Get(ctx context.Context, key *meta.Key) (*ga func (m *MockRegionBackendServices) List(ctx context.Context, region string, fl *filter.F) ([]*ga.BackendService, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, region, fl, m); intercept { - glog.V(5).Infof("MockRegionBackendServices.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) + klog.V(5).Infof("MockRegionBackendServices.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) return objs, err } } @@ -4005,7 +4005,7 @@ func (m *MockRegionBackendServices) List(ctx context.Context, region string, fl if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockRegionBackendServices.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) + klog.V(5).Infof("MockRegionBackendServices.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) return nil, *m.ListError } @@ -4021,7 +4021,7 @@ func (m *MockRegionBackendServices) List(ctx context.Context, region string, fl objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockRegionBackendServices.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) + klog.V(5).Infof("MockRegionBackendServices.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) return objs, nil } @@ -4029,7 +4029,7 @@ func (m *MockRegionBackendServices) List(ctx context.Context, region string, fl func (m *MockRegionBackendServices) Insert(ctx context.Context, key *meta.Key, obj *ga.BackendService) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockRegionBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockRegionBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -4041,7 +4041,7 @@ func (m *MockRegionBackendServices) Insert(ctx context.Context, key *meta.Key, o defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockRegionBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockRegionBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -4049,7 +4049,7 @@ func (m *MockRegionBackendServices) Insert(ctx context.Context, key *meta.Key, o Code: http.StatusConflict, Message: fmt.Sprintf("MockRegionBackendServices %v exists", key), } - glog.V(5).Infof("MockRegionBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockRegionBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -4058,7 +4058,7 @@ func (m *MockRegionBackendServices) Insert(ctx context.Context, key *meta.Key, o obj.SelfLink = SelfLink(meta.VersionGA, projectID, "backendServices", key) m.Objects[*key] = &MockRegionBackendServicesObj{obj} - glog.V(5).Infof("MockRegionBackendServices.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockRegionBackendServices.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -4066,7 +4066,7 @@ func (m *MockRegionBackendServices) Insert(ctx context.Context, key *meta.Key, o func (m *MockRegionBackendServices) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -4078,7 +4078,7 @@ func (m *MockRegionBackendServices) Delete(ctx context.Context, key *meta.Key) e defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -4086,12 +4086,12 @@ func (m *MockRegionBackendServices) Delete(ctx context.Context, key *meta.Key) e Code: http.StatusNotFound, Message: fmt.Sprintf("MockRegionBackendServices %v not found", key), } - glog.V(5).Infof("MockRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockRegionBackendServices.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockRegionBackendServices.Delete(%v, %v) = nil", ctx, key) return nil } @@ -4123,10 +4123,10 @@ type GCERegionBackendServices struct { // Get the BackendService named by key. func (g *GCERegionBackendServices) Get(ctx context.Context, key *meta.Key) (*ga.BackendService, error) { - glog.V(5).Infof("GCERegionBackendServices.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCERegionBackendServices.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCERegionBackendServices.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCERegionBackendServices.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "RegionBackendServices") @@ -4136,21 +4136,21 @@ func (g *GCERegionBackendServices) Get(ctx context.Context, key *meta.Key) (*ga. Version: meta.Version("ga"), Service: "RegionBackendServices", } - glog.V(5).Infof("GCERegionBackendServices.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCERegionBackendServices.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCERegionBackendServices.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCERegionBackendServices.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.RegionBackendServices.Get(projectID, key.Region, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCERegionBackendServices.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCERegionBackendServices.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all BackendService objects. func (g *GCERegionBackendServices) List(ctx context.Context, region string, fl *filter.F) ([]*ga.BackendService, error) { - glog.V(5).Infof("GCERegionBackendServices.List(%v, %v, %v) called", ctx, region, fl) + klog.V(5).Infof("GCERegionBackendServices.List(%v, %v, %v) called", ctx, region, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "RegionBackendServices") rk := &RateLimitKey{ ProjectID: projectID, @@ -4161,30 +4161,30 @@ func (g *GCERegionBackendServices) List(ctx context.Context, region string, fl * if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCERegionBackendServices.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) + klog.V(5).Infof("GCERegionBackendServices.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) call := g.s.GA.RegionBackendServices.List(projectID, region) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.BackendService f := func(l *ga.BackendServiceList) error { - glog.V(5).Infof("GCERegionBackendServices.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCERegionBackendServices.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCERegionBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCERegionBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCERegionBackendServices.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCERegionBackendServices.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCERegionBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCERegionBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -4192,9 +4192,9 @@ func (g *GCERegionBackendServices) List(ctx context.Context, region string, fl * // Insert BackendService with key of value obj. func (g *GCERegionBackendServices) Insert(ctx context.Context, key *meta.Key, obj *ga.BackendService) error { - glog.V(5).Infof("GCERegionBackendServices.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCERegionBackendServices.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCERegionBackendServices.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCERegionBackendServices.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "RegionBackendServices") @@ -4204,9 +4204,9 @@ func (g *GCERegionBackendServices) Insert(ctx context.Context, key *meta.Key, ob Version: meta.Version("ga"), Service: "RegionBackendServices", } - glog.V(5).Infof("GCERegionBackendServices.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCERegionBackendServices.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCERegionBackendServices.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCERegionBackendServices.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -4215,20 +4215,20 @@ func (g *GCERegionBackendServices) Insert(ctx context.Context, key *meta.Key, ob op, err := call.Do() if err != nil { - glog.V(4).Infof("GCERegionBackendServices.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCERegionBackendServices.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCERegionBackendServices.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCERegionBackendServices.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the BackendService referenced by key. func (g *GCERegionBackendServices) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCERegionBackendServices.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCERegionBackendServices.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCERegionBackendServices.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCERegionBackendServices.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "RegionBackendServices") @@ -4238,9 +4238,9 @@ func (g *GCERegionBackendServices) Delete(ctx context.Context, key *meta.Key) er Version: meta.Version("ga"), Service: "RegionBackendServices", } - glog.V(5).Infof("GCERegionBackendServices.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCERegionBackendServices.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCERegionBackendServices.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCERegionBackendServices.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.RegionBackendServices.Delete(projectID, key.Region, key.Name) @@ -4248,21 +4248,21 @@ func (g *GCERegionBackendServices) Delete(ctx context.Context, key *meta.Key) er op, err := call.Do() if err != nil { - glog.V(4).Infof("GCERegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCERegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCERegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCERegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } // GetHealth is a method on GCERegionBackendServices. func (g *GCERegionBackendServices) GetHealth(ctx context.Context, key *meta.Key, arg0 *ga.ResourceGroupReference) (*ga.BackendServiceGroupHealth, error) { - glog.V(5).Infof("GCERegionBackendServices.GetHealth(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCERegionBackendServices.GetHealth(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCERegionBackendServices.GetHealth(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCERegionBackendServices.GetHealth(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "RegionBackendServices") @@ -4272,25 +4272,25 @@ func (g *GCERegionBackendServices) GetHealth(ctx context.Context, key *meta.Key, Version: meta.Version("ga"), Service: "RegionBackendServices", } - glog.V(5).Infof("GCERegionBackendServices.GetHealth(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCERegionBackendServices.GetHealth(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCERegionBackendServices.GetHealth(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCERegionBackendServices.GetHealth(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.RegionBackendServices.GetHealth(projectID, key.Region, key.Name, arg0) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCERegionBackendServices.GetHealth(%v, %v, ...) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCERegionBackendServices.GetHealth(%v, %v, ...) = %+v, %v", ctx, key, v, err) return v, err } // Update is a method on GCERegionBackendServices. func (g *GCERegionBackendServices) Update(ctx context.Context, key *meta.Key, arg0 *ga.BackendService) error { - glog.V(5).Infof("GCERegionBackendServices.Update(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCERegionBackendServices.Update(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCERegionBackendServices.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCERegionBackendServices.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "RegionBackendServices") @@ -4300,21 +4300,21 @@ func (g *GCERegionBackendServices) Update(ctx context.Context, key *meta.Key, ar Version: meta.Version("ga"), Service: "RegionBackendServices", } - glog.V(5).Infof("GCERegionBackendServices.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCERegionBackendServices.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCERegionBackendServices.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCERegionBackendServices.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.RegionBackendServices.Update(projectID, key.Region, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCERegionBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCERegionBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCERegionBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCERegionBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -4377,7 +4377,7 @@ type MockAlphaRegionBackendServices struct { func (m *MockAlphaRegionBackendServices) Get(ctx context.Context, key *meta.Key) (*alpha.BackendService, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaRegionBackendServices.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaRegionBackendServices.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -4389,12 +4389,12 @@ func (m *MockAlphaRegionBackendServices) Get(ctx context.Context, key *meta.Key) defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockAlphaRegionBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaRegionBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToAlpha() - glog.V(5).Infof("MockAlphaRegionBackendServices.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockAlphaRegionBackendServices.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -4402,7 +4402,7 @@ func (m *MockAlphaRegionBackendServices) Get(ctx context.Context, key *meta.Key) Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaRegionBackendServices %v not found", key), } - glog.V(5).Infof("MockAlphaRegionBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaRegionBackendServices.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -4410,7 +4410,7 @@ func (m *MockAlphaRegionBackendServices) Get(ctx context.Context, key *meta.Key) func (m *MockAlphaRegionBackendServices) List(ctx context.Context, region string, fl *filter.F) ([]*alpha.BackendService, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, region, fl, m); intercept { - glog.V(5).Infof("MockAlphaRegionBackendServices.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) + klog.V(5).Infof("MockAlphaRegionBackendServices.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) return objs, err } } @@ -4420,7 +4420,7 @@ func (m *MockAlphaRegionBackendServices) List(ctx context.Context, region string if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockAlphaRegionBackendServices.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) + klog.V(5).Infof("MockAlphaRegionBackendServices.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) return nil, *m.ListError } @@ -4436,7 +4436,7 @@ func (m *MockAlphaRegionBackendServices) List(ctx context.Context, region string objs = append(objs, obj.ToAlpha()) } - glog.V(5).Infof("MockAlphaRegionBackendServices.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) + klog.V(5).Infof("MockAlphaRegionBackendServices.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) return objs, nil } @@ -4444,7 +4444,7 @@ func (m *MockAlphaRegionBackendServices) List(ctx context.Context, region string func (m *MockAlphaRegionBackendServices) Insert(ctx context.Context, key *meta.Key, obj *alpha.BackendService) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockAlphaRegionBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaRegionBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -4456,7 +4456,7 @@ func (m *MockAlphaRegionBackendServices) Insert(ctx context.Context, key *meta.K defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockAlphaRegionBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaRegionBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -4464,7 +4464,7 @@ func (m *MockAlphaRegionBackendServices) Insert(ctx context.Context, key *meta.K Code: http.StatusConflict, Message: fmt.Sprintf("MockAlphaRegionBackendServices %v exists", key), } - glog.V(5).Infof("MockAlphaRegionBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaRegionBackendServices.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -4473,7 +4473,7 @@ func (m *MockAlphaRegionBackendServices) Insert(ctx context.Context, key *meta.K obj.SelfLink = SelfLink(meta.VersionAlpha, projectID, "backendServices", key) m.Objects[*key] = &MockRegionBackendServicesObj{obj} - glog.V(5).Infof("MockAlphaRegionBackendServices.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockAlphaRegionBackendServices.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -4481,7 +4481,7 @@ func (m *MockAlphaRegionBackendServices) Insert(ctx context.Context, key *meta.K func (m *MockAlphaRegionBackendServices) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -4493,7 +4493,7 @@ func (m *MockAlphaRegionBackendServices) Delete(ctx context.Context, key *meta.K defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockAlphaRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -4501,12 +4501,12 @@ func (m *MockAlphaRegionBackendServices) Delete(ctx context.Context, key *meta.K Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaRegionBackendServices %v not found", key), } - glog.V(5).Infof("MockAlphaRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockAlphaRegionBackendServices.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockAlphaRegionBackendServices.Delete(%v, %v) = nil", ctx, key) return nil } @@ -4538,10 +4538,10 @@ type GCEAlphaRegionBackendServices struct { // Get the BackendService named by key. func (g *GCEAlphaRegionBackendServices) Get(ctx context.Context, key *meta.Key) (*alpha.BackendService, error) { - glog.V(5).Infof("GCEAlphaRegionBackendServices.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaRegionBackendServices.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaRegionBackendServices.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaRegionBackendServices.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "RegionBackendServices") @@ -4551,21 +4551,21 @@ func (g *GCEAlphaRegionBackendServices) Get(ctx context.Context, key *meta.Key) Version: meta.Version("alpha"), Service: "RegionBackendServices", } - glog.V(5).Infof("GCEAlphaRegionBackendServices.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaRegionBackendServices.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaRegionBackendServices.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Alpha.RegionBackendServices.Get(projectID, key.Region, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEAlphaRegionBackendServices.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all BackendService objects. func (g *GCEAlphaRegionBackendServices) List(ctx context.Context, region string, fl *filter.F) ([]*alpha.BackendService, error) { - glog.V(5).Infof("GCEAlphaRegionBackendServices.List(%v, %v, %v) called", ctx, region, fl) + klog.V(5).Infof("GCEAlphaRegionBackendServices.List(%v, %v, %v) called", ctx, region, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "RegionBackendServices") rk := &RateLimitKey{ ProjectID: projectID, @@ -4576,30 +4576,30 @@ func (g *GCEAlphaRegionBackendServices) List(ctx context.Context, region string, if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEAlphaRegionBackendServices.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) + klog.V(5).Infof("GCEAlphaRegionBackendServices.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) call := g.s.Alpha.RegionBackendServices.List(projectID, region) if fl != filter.None { call.Filter(fl.String()) } var all []*alpha.BackendService f := func(l *alpha.BackendServiceList) error { - glog.V(5).Infof("GCEAlphaRegionBackendServices.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEAlphaRegionBackendServices.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEAlphaRegionBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEAlphaRegionBackendServices.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEAlphaRegionBackendServices.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEAlphaRegionBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEAlphaRegionBackendServices.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -4607,9 +4607,9 @@ func (g *GCEAlphaRegionBackendServices) List(ctx context.Context, region string, // Insert BackendService with key of value obj. func (g *GCEAlphaRegionBackendServices) Insert(ctx context.Context, key *meta.Key, obj *alpha.BackendService) error { - glog.V(5).Infof("GCEAlphaRegionBackendServices.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEAlphaRegionBackendServices.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEAlphaRegionBackendServices.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaRegionBackendServices.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "RegionBackendServices") @@ -4619,9 +4619,9 @@ func (g *GCEAlphaRegionBackendServices) Insert(ctx context.Context, key *meta.Ke Version: meta.Version("alpha"), Service: "RegionBackendServices", } - glog.V(5).Infof("GCEAlphaRegionBackendServices.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaRegionBackendServices.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaRegionBackendServices.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -4630,20 +4630,20 @@ func (g *GCEAlphaRegionBackendServices) Insert(ctx context.Context, key *meta.Ke op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaRegionBackendServices.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaRegionBackendServices.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the BackendService referenced by key. func (g *GCEAlphaRegionBackendServices) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEAlphaRegionBackendServices.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaRegionBackendServices.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaRegionBackendServices.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaRegionBackendServices.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "RegionBackendServices") @@ -4653,9 +4653,9 @@ func (g *GCEAlphaRegionBackendServices) Delete(ctx context.Context, key *meta.Ke Version: meta.Version("alpha"), Service: "RegionBackendServices", } - glog.V(5).Infof("GCEAlphaRegionBackendServices.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaRegionBackendServices.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaRegionBackendServices.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.RegionBackendServices.Delete(projectID, key.Region, key.Name) @@ -4663,21 +4663,21 @@ func (g *GCEAlphaRegionBackendServices) Delete(ctx context.Context, key *meta.Ke op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.Delete(%v, %v) = %v", ctx, key, err) return err } // GetHealth is a method on GCEAlphaRegionBackendServices. func (g *GCEAlphaRegionBackendServices) GetHealth(ctx context.Context, key *meta.Key, arg0 *alpha.ResourceGroupReference) (*alpha.BackendServiceGroupHealth, error) { - glog.V(5).Infof("GCEAlphaRegionBackendServices.GetHealth(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEAlphaRegionBackendServices.GetHealth(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaRegionBackendServices.GetHealth(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaRegionBackendServices.GetHealth(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "RegionBackendServices") @@ -4687,25 +4687,25 @@ func (g *GCEAlphaRegionBackendServices) GetHealth(ctx context.Context, key *meta Version: meta.Version("alpha"), Service: "RegionBackendServices", } - glog.V(5).Infof("GCEAlphaRegionBackendServices.GetHealth(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaRegionBackendServices.GetHealth(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaRegionBackendServices.GetHealth(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.GetHealth(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Alpha.RegionBackendServices.GetHealth(projectID, key.Region, key.Name, arg0) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEAlphaRegionBackendServices.GetHealth(%v, %v, ...) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.GetHealth(%v, %v, ...) = %+v, %v", ctx, key, v, err) return v, err } // Update is a method on GCEAlphaRegionBackendServices. func (g *GCEAlphaRegionBackendServices) Update(ctx context.Context, key *meta.Key, arg0 *alpha.BackendService) error { - glog.V(5).Infof("GCEAlphaRegionBackendServices.Update(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEAlphaRegionBackendServices.Update(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaRegionBackendServices.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaRegionBackendServices.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "RegionBackendServices") @@ -4715,21 +4715,21 @@ func (g *GCEAlphaRegionBackendServices) Update(ctx context.Context, key *meta.Ke Version: meta.Version("alpha"), Service: "RegionBackendServices", } - glog.V(5).Infof("GCEAlphaRegionBackendServices.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaRegionBackendServices.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaRegionBackendServices.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.RegionBackendServices.Update(projectID, key.Region, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaRegionBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaRegionBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaRegionBackendServices.Update(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -4790,7 +4790,7 @@ type MockDisks struct { func (m *MockDisks) Get(ctx context.Context, key *meta.Key) (*ga.Disk, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockDisks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockDisks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -4802,12 +4802,12 @@ func (m *MockDisks) Get(ctx context.Context, key *meta.Key) (*ga.Disk, error) { defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockDisks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockDisks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockDisks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockDisks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -4815,7 +4815,7 @@ func (m *MockDisks) Get(ctx context.Context, key *meta.Key) (*ga.Disk, error) { Code: http.StatusNotFound, Message: fmt.Sprintf("MockDisks %v not found", key), } - glog.V(5).Infof("MockDisks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockDisks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -4823,7 +4823,7 @@ func (m *MockDisks) Get(ctx context.Context, key *meta.Key) (*ga.Disk, error) { func (m *MockDisks) List(ctx context.Context, zone string, fl *filter.F) ([]*ga.Disk, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, zone, fl, m); intercept { - glog.V(5).Infof("MockDisks.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) + klog.V(5).Infof("MockDisks.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) return objs, err } } @@ -4833,7 +4833,7 @@ func (m *MockDisks) List(ctx context.Context, zone string, fl *filter.F) ([]*ga. if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockDisks.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) + klog.V(5).Infof("MockDisks.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) return nil, *m.ListError } @@ -4849,7 +4849,7 @@ func (m *MockDisks) List(ctx context.Context, zone string, fl *filter.F) ([]*ga. objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockDisks.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) + klog.V(5).Infof("MockDisks.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) return objs, nil } @@ -4857,7 +4857,7 @@ func (m *MockDisks) List(ctx context.Context, zone string, fl *filter.F) ([]*ga. func (m *MockDisks) Insert(ctx context.Context, key *meta.Key, obj *ga.Disk) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockDisks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockDisks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -4869,7 +4869,7 @@ func (m *MockDisks) Insert(ctx context.Context, key *meta.Key, obj *ga.Disk) err defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockDisks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockDisks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -4877,7 +4877,7 @@ func (m *MockDisks) Insert(ctx context.Context, key *meta.Key, obj *ga.Disk) err Code: http.StatusConflict, Message: fmt.Sprintf("MockDisks %v exists", key), } - glog.V(5).Infof("MockDisks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockDisks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -4886,7 +4886,7 @@ func (m *MockDisks) Insert(ctx context.Context, key *meta.Key, obj *ga.Disk) err obj.SelfLink = SelfLink(meta.VersionGA, projectID, "disks", key) m.Objects[*key] = &MockDisksObj{obj} - glog.V(5).Infof("MockDisks.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockDisks.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -4894,7 +4894,7 @@ func (m *MockDisks) Insert(ctx context.Context, key *meta.Key, obj *ga.Disk) err func (m *MockDisks) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockDisks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockDisks.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -4906,7 +4906,7 @@ func (m *MockDisks) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockDisks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockDisks.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -4914,12 +4914,12 @@ func (m *MockDisks) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockDisks %v not found", key), } - glog.V(5).Infof("MockDisks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockDisks.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockDisks.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockDisks.Delete(%v, %v) = nil", ctx, key) return nil } @@ -4943,10 +4943,10 @@ type GCEDisks struct { // Get the Disk named by key. func (g *GCEDisks) Get(ctx context.Context, key *meta.Key) (*ga.Disk, error) { - glog.V(5).Infof("GCEDisks.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEDisks.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEDisks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEDisks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Disks") @@ -4956,21 +4956,21 @@ func (g *GCEDisks) Get(ctx context.Context, key *meta.Key) (*ga.Disk, error) { Version: meta.Version("ga"), Service: "Disks", } - glog.V(5).Infof("GCEDisks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEDisks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEDisks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEDisks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.Disks.Get(projectID, key.Zone, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEDisks.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEDisks.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all Disk objects. func (g *GCEDisks) List(ctx context.Context, zone string, fl *filter.F) ([]*ga.Disk, error) { - glog.V(5).Infof("GCEDisks.List(%v, %v, %v) called", ctx, zone, fl) + klog.V(5).Infof("GCEDisks.List(%v, %v, %v) called", ctx, zone, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Disks") rk := &RateLimitKey{ ProjectID: projectID, @@ -4981,30 +4981,30 @@ func (g *GCEDisks) List(ctx context.Context, zone string, fl *filter.F) ([]*ga.D if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEDisks.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) + klog.V(5).Infof("GCEDisks.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) call := g.s.GA.Disks.List(projectID, zone) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.Disk f := func(l *ga.DiskList) error { - glog.V(5).Infof("GCEDisks.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEDisks.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEDisks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEDisks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEDisks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEDisks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEDisks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEDisks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -5012,9 +5012,9 @@ func (g *GCEDisks) List(ctx context.Context, zone string, fl *filter.F) ([]*ga.D // Insert Disk with key of value obj. func (g *GCEDisks) Insert(ctx context.Context, key *meta.Key, obj *ga.Disk) error { - glog.V(5).Infof("GCEDisks.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEDisks.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEDisks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEDisks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Disks") @@ -5024,9 +5024,9 @@ func (g *GCEDisks) Insert(ctx context.Context, key *meta.Key, obj *ga.Disk) erro Version: meta.Version("ga"), Service: "Disks", } - glog.V(5).Infof("GCEDisks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEDisks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEDisks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEDisks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -5035,20 +5035,20 @@ func (g *GCEDisks) Insert(ctx context.Context, key *meta.Key, obj *ga.Disk) erro op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEDisks.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEDisks.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEDisks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEDisks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the Disk referenced by key. func (g *GCEDisks) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEDisks.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEDisks.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEDisks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEDisks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Disks") @@ -5058,9 +5058,9 @@ func (g *GCEDisks) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "Disks", } - glog.V(5).Infof("GCEDisks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEDisks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEDisks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEDisks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.Disks.Delete(projectID, key.Zone, key.Name) @@ -5068,21 +5068,21 @@ func (g *GCEDisks) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEDisks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEDisks.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEDisks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEDisks.Delete(%v, %v) = %v", ctx, key, err) return err } // Resize is a method on GCEDisks. func (g *GCEDisks) Resize(ctx context.Context, key *meta.Key, arg0 *ga.DisksResizeRequest) error { - glog.V(5).Infof("GCEDisks.Resize(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEDisks.Resize(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEDisks.Resize(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEDisks.Resize(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Disks") @@ -5092,36 +5092,36 @@ func (g *GCEDisks) Resize(ctx context.Context, key *meta.Key, arg0 *ga.DisksResi Version: meta.Version("ga"), Service: "Disks", } - glog.V(5).Infof("GCEDisks.Resize(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEDisks.Resize(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEDisks.Resize(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEDisks.Resize(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.Disks.Resize(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEDisks.Resize(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEDisks.Resize(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEDisks.Resize(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEDisks.Resize(%v, %v, ...) = %+v", ctx, key, err) return err } -// BetaRegionDisks is an interface that allows for mocking of RegionDisks. -type BetaRegionDisks interface { - Get(ctx context.Context, key *meta.Key) (*beta.Disk, error) - List(ctx context.Context, region string, fl *filter.F) ([]*beta.Disk, error) - Insert(ctx context.Context, key *meta.Key, obj *beta.Disk) error +// RegionDisks is an interface that allows for mocking of RegionDisks. +type RegionDisks interface { + Get(ctx context.Context, key *meta.Key) (*ga.Disk, error) + List(ctx context.Context, region string, fl *filter.F) ([]*ga.Disk, error) + Insert(ctx context.Context, key *meta.Key, obj *ga.Disk) error Delete(ctx context.Context, key *meta.Key) error - Resize(context.Context, *meta.Key, *beta.RegionDisksResizeRequest) error + Resize(context.Context, *meta.Key, *ga.RegionDisksResizeRequest) error } -// NewMockBetaRegionDisks returns a new mock for RegionDisks. -func NewMockBetaRegionDisks(pr ProjectRouter, objs map[meta.Key]*MockRegionDisksObj) *MockBetaRegionDisks { - mock := &MockBetaRegionDisks{ +// NewMockRegionDisks returns a new mock for RegionDisks. +func NewMockRegionDisks(pr ProjectRouter, objs map[meta.Key]*MockRegionDisksObj) *MockRegionDisks { + mock := &MockRegionDisks{ ProjectRouter: pr, Objects: objs, @@ -5132,8 +5132,8 @@ func NewMockBetaRegionDisks(pr ProjectRouter, objs map[meta.Key]*MockRegionDisks return mock } -// MockBetaRegionDisks is the mock for RegionDisks. -type MockBetaRegionDisks struct { +// MockRegionDisks is the mock for RegionDisks. +type MockRegionDisks struct { Lock sync.Mutex ProjectRouter ProjectRouter @@ -5152,11 +5152,11 @@ type MockBetaRegionDisks struct { // order to add your own logic. Return (true, _, _) to prevent the normal // execution flow of the mock. Return (false, nil, nil) to continue with // normal mock behavior/ after the hook function executes. - GetHook func(ctx context.Context, key *meta.Key, m *MockBetaRegionDisks) (bool, *beta.Disk, error) - ListHook func(ctx context.Context, region string, fl *filter.F, m *MockBetaRegionDisks) (bool, []*beta.Disk, error) - InsertHook func(ctx context.Context, key *meta.Key, obj *beta.Disk, m *MockBetaRegionDisks) (bool, error) - DeleteHook func(ctx context.Context, key *meta.Key, m *MockBetaRegionDisks) (bool, error) - ResizeHook func(context.Context, *meta.Key, *beta.RegionDisksResizeRequest, *MockBetaRegionDisks) error + GetHook func(ctx context.Context, key *meta.Key, m *MockRegionDisks) (bool, *ga.Disk, error) + ListHook func(ctx context.Context, region string, fl *filter.F, m *MockRegionDisks) (bool, []*ga.Disk, error) + InsertHook func(ctx context.Context, key *meta.Key, obj *ga.Disk, m *MockRegionDisks) (bool, error) + DeleteHook func(ctx context.Context, key *meta.Key, m *MockRegionDisks) (bool, error) + ResizeHook func(context.Context, *meta.Key, *ga.RegionDisksResizeRequest, *MockRegionDisks) error // X is extra state that can be used as part of the mock. Generated code // will not use this field. @@ -5164,10 +5164,10 @@ type MockBetaRegionDisks struct { } // Get returns the object from the mock. -func (m *MockBetaRegionDisks) Get(ctx context.Context, key *meta.Key) (*beta.Disk, error) { +func (m *MockRegionDisks) Get(ctx context.Context, key *meta.Key) (*ga.Disk, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaRegionDisks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockRegionDisks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -5179,28 +5179,28 @@ func (m *MockBetaRegionDisks) Get(ctx context.Context, key *meta.Key) (*beta.Dis defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockBetaRegionDisks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockRegionDisks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { - typedObj := obj.ToBeta() - glog.V(5).Infof("MockBetaRegionDisks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + typedObj := obj.ToGA() + klog.V(5).Infof("MockRegionDisks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } err := &googleapi.Error{ Code: http.StatusNotFound, - Message: fmt.Sprintf("MockBetaRegionDisks %v not found", key), + Message: fmt.Sprintf("MockRegionDisks %v not found", key), } - glog.V(5).Infof("MockBetaRegionDisks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockRegionDisks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } // List all of the objects in the mock in the given region. -func (m *MockBetaRegionDisks) List(ctx context.Context, region string, fl *filter.F) ([]*beta.Disk, error) { +func (m *MockRegionDisks) List(ctx context.Context, region string, fl *filter.F) ([]*ga.Disk, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, region, fl, m); intercept { - glog.V(5).Infof("MockBetaRegionDisks.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) + klog.V(5).Infof("MockRegionDisks.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) return objs, err } } @@ -5210,31 +5210,31 @@ func (m *MockBetaRegionDisks) List(ctx context.Context, region string, fl *filte if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockBetaRegionDisks.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) + klog.V(5).Infof("MockRegionDisks.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) return nil, *m.ListError } - var objs []*beta.Disk + var objs []*ga.Disk for key, obj := range m.Objects { if key.Region != region { continue } - if !fl.Match(obj.ToBeta()) { + if !fl.Match(obj.ToGA()) { continue } - objs = append(objs, obj.ToBeta()) + objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockBetaRegionDisks.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) + klog.V(5).Infof("MockRegionDisks.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) return objs, nil } // Insert is a mock for inserting/creating a new object. -func (m *MockBetaRegionDisks) Insert(ctx context.Context, key *meta.Key, obj *beta.Disk) error { +func (m *MockRegionDisks) Insert(ctx context.Context, key *meta.Key, obj *ga.Disk) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockBetaRegionDisks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockRegionDisks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -5246,32 +5246,32 @@ func (m *MockBetaRegionDisks) Insert(ctx context.Context, key *meta.Key, obj *be defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockBetaRegionDisks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockRegionDisks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { err := &googleapi.Error{ Code: http.StatusConflict, - Message: fmt.Sprintf("MockBetaRegionDisks %v exists", key), + Message: fmt.Sprintf("MockRegionDisks %v exists", key), } - glog.V(5).Infof("MockBetaRegionDisks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockRegionDisks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } obj.Name = key.Name - projectID := m.ProjectRouter.ProjectID(ctx, "beta", "disks") - obj.SelfLink = SelfLink(meta.VersionBeta, projectID, "disks", key) + projectID := m.ProjectRouter.ProjectID(ctx, "ga", "disks") + obj.SelfLink = SelfLink(meta.VersionGA, projectID, "disks", key) m.Objects[*key] = &MockRegionDisksObj{obj} - glog.V(5).Infof("MockBetaRegionDisks.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockRegionDisks.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } // Delete is a mock for deleting the object. -func (m *MockBetaRegionDisks) Delete(ctx context.Context, key *meta.Key) error { +func (m *MockRegionDisks) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaRegionDisks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockRegionDisks.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -5283,207 +5283,207 @@ func (m *MockBetaRegionDisks) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockBetaRegionDisks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockRegionDisks.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { err := &googleapi.Error{ Code: http.StatusNotFound, - Message: fmt.Sprintf("MockBetaRegionDisks %v not found", key), + Message: fmt.Sprintf("MockRegionDisks %v not found", key), } - glog.V(5).Infof("MockBetaRegionDisks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockRegionDisks.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockBetaRegionDisks.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockRegionDisks.Delete(%v, %v) = nil", ctx, key) return nil } // Obj wraps the object for use in the mock. -func (m *MockBetaRegionDisks) Obj(o *beta.Disk) *MockRegionDisksObj { +func (m *MockRegionDisks) Obj(o *ga.Disk) *MockRegionDisksObj { return &MockRegionDisksObj{o} } // Resize is a mock for the corresponding method. -func (m *MockBetaRegionDisks) Resize(ctx context.Context, key *meta.Key, arg0 *beta.RegionDisksResizeRequest) error { +func (m *MockRegionDisks) Resize(ctx context.Context, key *meta.Key, arg0 *ga.RegionDisksResizeRequest) error { if m.ResizeHook != nil { return m.ResizeHook(ctx, key, arg0, m) } return nil } -// GCEBetaRegionDisks is a simplifying adapter for the GCE RegionDisks. -type GCEBetaRegionDisks struct { +// GCERegionDisks is a simplifying adapter for the GCE RegionDisks. +type GCERegionDisks struct { s *Service } // Get the Disk named by key. -func (g *GCEBetaRegionDisks) Get(ctx context.Context, key *meta.Key) (*beta.Disk, error) { - glog.V(5).Infof("GCEBetaRegionDisks.Get(%v, %v): called", ctx, key) +func (g *GCERegionDisks) Get(ctx context.Context, key *meta.Key) (*ga.Disk, error) { + klog.V(5).Infof("GCERegionDisks.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaRegionDisks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCERegionDisks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } - projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "RegionDisks") + projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "RegionDisks") rk := &RateLimitKey{ ProjectID: projectID, Operation: "Get", - Version: meta.Version("beta"), + Version: meta.Version("ga"), Service: "RegionDisks", } - glog.V(5).Infof("GCEBetaRegionDisks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCERegionDisks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaRegionDisks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCERegionDisks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } - call := g.s.Beta.RegionDisks.Get(projectID, key.Region, key.Name) + call := g.s.GA.RegionDisks.Get(projectID, key.Region, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEBetaRegionDisks.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCERegionDisks.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all Disk objects. -func (g *GCEBetaRegionDisks) List(ctx context.Context, region string, fl *filter.F) ([]*beta.Disk, error) { - glog.V(5).Infof("GCEBetaRegionDisks.List(%v, %v, %v) called", ctx, region, fl) - projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "RegionDisks") +func (g *GCERegionDisks) List(ctx context.Context, region string, fl *filter.F) ([]*ga.Disk, error) { + klog.V(5).Infof("GCERegionDisks.List(%v, %v, %v) called", ctx, region, fl) + projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "RegionDisks") rk := &RateLimitKey{ ProjectID: projectID, Operation: "List", - Version: meta.Version("beta"), + Version: meta.Version("ga"), Service: "RegionDisks", } if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEBetaRegionDisks.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) - call := g.s.Beta.RegionDisks.List(projectID, region) + klog.V(5).Infof("GCERegionDisks.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) + call := g.s.GA.RegionDisks.List(projectID, region) if fl != filter.None { call.Filter(fl.String()) } - var all []*beta.Disk - f := func(l *beta.DiskList) error { - glog.V(5).Infof("GCEBetaRegionDisks.List(%v, ..., %v): page %+v", ctx, fl, l) + var all []*ga.Disk + f := func(l *ga.DiskList) error { + klog.V(5).Infof("GCERegionDisks.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEBetaRegionDisks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCERegionDisks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEBetaRegionDisks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCERegionDisks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEBetaRegionDisks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCERegionDisks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil } // Insert Disk with key of value obj. -func (g *GCEBetaRegionDisks) Insert(ctx context.Context, key *meta.Key, obj *beta.Disk) error { - glog.V(5).Infof("GCEBetaRegionDisks.Insert(%v, %v, %+v): called", ctx, key, obj) +func (g *GCERegionDisks) Insert(ctx context.Context, key *meta.Key, obj *ga.Disk) error { + klog.V(5).Infof("GCERegionDisks.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEBetaRegionDisks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCERegionDisks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } - projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "RegionDisks") + projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "RegionDisks") rk := &RateLimitKey{ ProjectID: projectID, Operation: "Insert", - Version: meta.Version("beta"), + Version: meta.Version("ga"), Service: "RegionDisks", } - glog.V(5).Infof("GCEBetaRegionDisks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCERegionDisks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaRegionDisks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCERegionDisks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name - call := g.s.Beta.RegionDisks.Insert(projectID, key.Region, obj) + call := g.s.GA.RegionDisks.Insert(projectID, key.Region, obj) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaRegionDisks.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCERegionDisks.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaRegionDisks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCERegionDisks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the Disk referenced by key. -func (g *GCEBetaRegionDisks) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEBetaRegionDisks.Delete(%v, %v): called", ctx, key) +func (g *GCERegionDisks) Delete(ctx context.Context, key *meta.Key) error { + klog.V(5).Infof("GCERegionDisks.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaRegionDisks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCERegionDisks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } - projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "RegionDisks") + projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "RegionDisks") rk := &RateLimitKey{ ProjectID: projectID, Operation: "Delete", - Version: meta.Version("beta"), + Version: meta.Version("ga"), Service: "RegionDisks", } - glog.V(5).Infof("GCEBetaRegionDisks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCERegionDisks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaRegionDisks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCERegionDisks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } - call := g.s.Beta.RegionDisks.Delete(projectID, key.Region, key.Name) + call := g.s.GA.RegionDisks.Delete(projectID, key.Region, key.Name) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaRegionDisks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCERegionDisks.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaRegionDisks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCERegionDisks.Delete(%v, %v) = %v", ctx, key, err) return err } -// Resize is a method on GCEBetaRegionDisks. -func (g *GCEBetaRegionDisks) Resize(ctx context.Context, key *meta.Key, arg0 *beta.RegionDisksResizeRequest) error { - glog.V(5).Infof("GCEBetaRegionDisks.Resize(%v, %v, ...): called", ctx, key) +// Resize is a method on GCERegionDisks. +func (g *GCERegionDisks) Resize(ctx context.Context, key *meta.Key, arg0 *ga.RegionDisksResizeRequest) error { + klog.V(5).Infof("GCERegionDisks.Resize(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaRegionDisks.Resize(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCERegionDisks.Resize(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } - projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "RegionDisks") + projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "RegionDisks") rk := &RateLimitKey{ ProjectID: projectID, Operation: "Resize", - Version: meta.Version("beta"), + Version: meta.Version("ga"), Service: "RegionDisks", } - glog.V(5).Infof("GCEBetaRegionDisks.Resize(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCERegionDisks.Resize(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaRegionDisks.Resize(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCERegionDisks.Resize(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } - call := g.s.Beta.RegionDisks.Resize(projectID, key.Region, key.Name, arg0) + call := g.s.GA.RegionDisks.Resize(projectID, key.Region, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaRegionDisks.Resize(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCERegionDisks.Resize(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaRegionDisks.Resize(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCERegionDisks.Resize(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -5544,7 +5544,7 @@ type MockFirewalls struct { func (m *MockFirewalls) Get(ctx context.Context, key *meta.Key) (*ga.Firewall, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockFirewalls.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockFirewalls.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -5556,12 +5556,12 @@ func (m *MockFirewalls) Get(ctx context.Context, key *meta.Key) (*ga.Firewall, e defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockFirewalls.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockFirewalls.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockFirewalls.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockFirewalls.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -5569,7 +5569,7 @@ func (m *MockFirewalls) Get(ctx context.Context, key *meta.Key) (*ga.Firewall, e Code: http.StatusNotFound, Message: fmt.Sprintf("MockFirewalls %v not found", key), } - glog.V(5).Infof("MockFirewalls.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockFirewalls.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -5577,7 +5577,7 @@ func (m *MockFirewalls) Get(ctx context.Context, key *meta.Key) (*ga.Firewall, e func (m *MockFirewalls) List(ctx context.Context, fl *filter.F) ([]*ga.Firewall, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockFirewalls.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockFirewalls.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -5587,7 +5587,7 @@ func (m *MockFirewalls) List(ctx context.Context, fl *filter.F) ([]*ga.Firewall, if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockFirewalls.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockFirewalls.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -5600,7 +5600,7 @@ func (m *MockFirewalls) List(ctx context.Context, fl *filter.F) ([]*ga.Firewall, objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockFirewalls.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockFirewalls.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -5608,7 +5608,7 @@ func (m *MockFirewalls) List(ctx context.Context, fl *filter.F) ([]*ga.Firewall, func (m *MockFirewalls) Insert(ctx context.Context, key *meta.Key, obj *ga.Firewall) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockFirewalls.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockFirewalls.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -5620,7 +5620,7 @@ func (m *MockFirewalls) Insert(ctx context.Context, key *meta.Key, obj *ga.Firew defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockFirewalls.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockFirewalls.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -5628,7 +5628,7 @@ func (m *MockFirewalls) Insert(ctx context.Context, key *meta.Key, obj *ga.Firew Code: http.StatusConflict, Message: fmt.Sprintf("MockFirewalls %v exists", key), } - glog.V(5).Infof("MockFirewalls.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockFirewalls.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -5637,7 +5637,7 @@ func (m *MockFirewalls) Insert(ctx context.Context, key *meta.Key, obj *ga.Firew obj.SelfLink = SelfLink(meta.VersionGA, projectID, "firewalls", key) m.Objects[*key] = &MockFirewallsObj{obj} - glog.V(5).Infof("MockFirewalls.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockFirewalls.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -5645,7 +5645,7 @@ func (m *MockFirewalls) Insert(ctx context.Context, key *meta.Key, obj *ga.Firew func (m *MockFirewalls) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockFirewalls.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockFirewalls.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -5657,7 +5657,7 @@ func (m *MockFirewalls) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockFirewalls.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockFirewalls.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -5665,12 +5665,12 @@ func (m *MockFirewalls) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockFirewalls %v not found", key), } - glog.V(5).Infof("MockFirewalls.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockFirewalls.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockFirewalls.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockFirewalls.Delete(%v, %v) = nil", ctx, key) return nil } @@ -5694,10 +5694,10 @@ type GCEFirewalls struct { // Get the Firewall named by key. func (g *GCEFirewalls) Get(ctx context.Context, key *meta.Key) (*ga.Firewall, error) { - glog.V(5).Infof("GCEFirewalls.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEFirewalls.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEFirewalls.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEFirewalls.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Firewalls") @@ -5707,21 +5707,21 @@ func (g *GCEFirewalls) Get(ctx context.Context, key *meta.Key) (*ga.Firewall, er Version: meta.Version("ga"), Service: "Firewalls", } - glog.V(5).Infof("GCEFirewalls.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEFirewalls.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEFirewalls.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEFirewalls.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.Firewalls.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEFirewalls.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEFirewalls.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all Firewall objects. func (g *GCEFirewalls) List(ctx context.Context, fl *filter.F) ([]*ga.Firewall, error) { - glog.V(5).Infof("GCEFirewalls.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEFirewalls.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Firewalls") rk := &RateLimitKey{ ProjectID: projectID, @@ -5732,30 +5732,30 @@ func (g *GCEFirewalls) List(ctx context.Context, fl *filter.F) ([]*ga.Firewall, if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEFirewalls.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEFirewalls.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.Firewalls.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.Firewall f := func(l *ga.FirewallList) error { - glog.V(5).Infof("GCEFirewalls.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEFirewalls.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEFirewalls.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEFirewalls.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEFirewalls.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEFirewalls.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEFirewalls.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEFirewalls.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -5763,9 +5763,9 @@ func (g *GCEFirewalls) List(ctx context.Context, fl *filter.F) ([]*ga.Firewall, // Insert Firewall with key of value obj. func (g *GCEFirewalls) Insert(ctx context.Context, key *meta.Key, obj *ga.Firewall) error { - glog.V(5).Infof("GCEFirewalls.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEFirewalls.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEFirewalls.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEFirewalls.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Firewalls") @@ -5775,9 +5775,9 @@ func (g *GCEFirewalls) Insert(ctx context.Context, key *meta.Key, obj *ga.Firewa Version: meta.Version("ga"), Service: "Firewalls", } - glog.V(5).Infof("GCEFirewalls.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEFirewalls.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEFirewalls.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEFirewalls.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -5786,20 +5786,20 @@ func (g *GCEFirewalls) Insert(ctx context.Context, key *meta.Key, obj *ga.Firewa op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEFirewalls.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEFirewalls.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEFirewalls.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEFirewalls.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the Firewall referenced by key. func (g *GCEFirewalls) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEFirewalls.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEFirewalls.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEFirewalls.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEFirewalls.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Firewalls") @@ -5809,9 +5809,9 @@ func (g *GCEFirewalls) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "Firewalls", } - glog.V(5).Infof("GCEFirewalls.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEFirewalls.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEFirewalls.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEFirewalls.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.Firewalls.Delete(projectID, key.Name) @@ -5820,21 +5820,21 @@ func (g *GCEFirewalls) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEFirewalls.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEFirewalls.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEFirewalls.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEFirewalls.Delete(%v, %v) = %v", ctx, key, err) return err } // Update is a method on GCEFirewalls. func (g *GCEFirewalls) Update(ctx context.Context, key *meta.Key, arg0 *ga.Firewall) error { - glog.V(5).Infof("GCEFirewalls.Update(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEFirewalls.Update(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEFirewalls.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEFirewalls.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Firewalls") @@ -5844,21 +5844,21 @@ func (g *GCEFirewalls) Update(ctx context.Context, key *meta.Key, arg0 *ga.Firew Version: meta.Version("ga"), Service: "Firewalls", } - glog.V(5).Infof("GCEFirewalls.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEFirewalls.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEFirewalls.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEFirewalls.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.Firewalls.Update(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEFirewalls.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEFirewalls.Update(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEFirewalls.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEFirewalls.Update(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -5917,7 +5917,7 @@ type MockForwardingRules struct { func (m *MockForwardingRules) Get(ctx context.Context, key *meta.Key) (*ga.ForwardingRule, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockForwardingRules.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockForwardingRules.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -5929,12 +5929,12 @@ func (m *MockForwardingRules) Get(ctx context.Context, key *meta.Key) (*ga.Forwa defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockForwardingRules.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockForwardingRules.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockForwardingRules.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockForwardingRules.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -5942,7 +5942,7 @@ func (m *MockForwardingRules) Get(ctx context.Context, key *meta.Key) (*ga.Forwa Code: http.StatusNotFound, Message: fmt.Sprintf("MockForwardingRules %v not found", key), } - glog.V(5).Infof("MockForwardingRules.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockForwardingRules.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -5950,7 +5950,7 @@ func (m *MockForwardingRules) Get(ctx context.Context, key *meta.Key) (*ga.Forwa func (m *MockForwardingRules) List(ctx context.Context, region string, fl *filter.F) ([]*ga.ForwardingRule, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, region, fl, m); intercept { - glog.V(5).Infof("MockForwardingRules.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) + klog.V(5).Infof("MockForwardingRules.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) return objs, err } } @@ -5960,7 +5960,7 @@ func (m *MockForwardingRules) List(ctx context.Context, region string, fl *filte if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockForwardingRules.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) + klog.V(5).Infof("MockForwardingRules.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) return nil, *m.ListError } @@ -5976,7 +5976,7 @@ func (m *MockForwardingRules) List(ctx context.Context, region string, fl *filte objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockForwardingRules.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) + klog.V(5).Infof("MockForwardingRules.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) return objs, nil } @@ -5984,7 +5984,7 @@ func (m *MockForwardingRules) List(ctx context.Context, region string, fl *filte func (m *MockForwardingRules) Insert(ctx context.Context, key *meta.Key, obj *ga.ForwardingRule) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -5996,7 +5996,7 @@ func (m *MockForwardingRules) Insert(ctx context.Context, key *meta.Key, obj *ga defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -6004,7 +6004,7 @@ func (m *MockForwardingRules) Insert(ctx context.Context, key *meta.Key, obj *ga Code: http.StatusConflict, Message: fmt.Sprintf("MockForwardingRules %v exists", key), } - glog.V(5).Infof("MockForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -6013,7 +6013,7 @@ func (m *MockForwardingRules) Insert(ctx context.Context, key *meta.Key, obj *ga obj.SelfLink = SelfLink(meta.VersionGA, projectID, "forwardingRules", key) m.Objects[*key] = &MockForwardingRulesObj{obj} - glog.V(5).Infof("MockForwardingRules.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockForwardingRules.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -6021,7 +6021,7 @@ func (m *MockForwardingRules) Insert(ctx context.Context, key *meta.Key, obj *ga func (m *MockForwardingRules) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -6033,7 +6033,7 @@ func (m *MockForwardingRules) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -6041,12 +6041,12 @@ func (m *MockForwardingRules) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockForwardingRules %v not found", key), } - glog.V(5).Infof("MockForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockForwardingRules.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockForwardingRules.Delete(%v, %v) = nil", ctx, key) return nil } @@ -6062,10 +6062,10 @@ type GCEForwardingRules struct { // Get the ForwardingRule named by key. func (g *GCEForwardingRules) Get(ctx context.Context, key *meta.Key) (*ga.ForwardingRule, error) { - glog.V(5).Infof("GCEForwardingRules.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEForwardingRules.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEForwardingRules.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEForwardingRules.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "ForwardingRules") @@ -6075,21 +6075,21 @@ func (g *GCEForwardingRules) Get(ctx context.Context, key *meta.Key) (*ga.Forwar Version: meta.Version("ga"), Service: "ForwardingRules", } - glog.V(5).Infof("GCEForwardingRules.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEForwardingRules.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEForwardingRules.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEForwardingRules.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.ForwardingRules.Get(projectID, key.Region, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEForwardingRules.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEForwardingRules.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all ForwardingRule objects. func (g *GCEForwardingRules) List(ctx context.Context, region string, fl *filter.F) ([]*ga.ForwardingRule, error) { - glog.V(5).Infof("GCEForwardingRules.List(%v, %v, %v) called", ctx, region, fl) + klog.V(5).Infof("GCEForwardingRules.List(%v, %v, %v) called", ctx, region, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "ForwardingRules") rk := &RateLimitKey{ ProjectID: projectID, @@ -6100,30 +6100,30 @@ func (g *GCEForwardingRules) List(ctx context.Context, region string, fl *filter if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEForwardingRules.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) + klog.V(5).Infof("GCEForwardingRules.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) call := g.s.GA.ForwardingRules.List(projectID, region) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.ForwardingRule f := func(l *ga.ForwardingRuleList) error { - glog.V(5).Infof("GCEForwardingRules.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEForwardingRules.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEForwardingRules.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEForwardingRules.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEForwardingRules.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEForwardingRules.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEForwardingRules.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEForwardingRules.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -6131,9 +6131,9 @@ func (g *GCEForwardingRules) List(ctx context.Context, region string, fl *filter // Insert ForwardingRule with key of value obj. func (g *GCEForwardingRules) Insert(ctx context.Context, key *meta.Key, obj *ga.ForwardingRule) error { - glog.V(5).Infof("GCEForwardingRules.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEForwardingRules.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEForwardingRules.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEForwardingRules.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "ForwardingRules") @@ -6143,9 +6143,9 @@ func (g *GCEForwardingRules) Insert(ctx context.Context, key *meta.Key, obj *ga. Version: meta.Version("ga"), Service: "ForwardingRules", } - glog.V(5).Infof("GCEForwardingRules.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEForwardingRules.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEForwardingRules.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEForwardingRules.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -6154,20 +6154,20 @@ func (g *GCEForwardingRules) Insert(ctx context.Context, key *meta.Key, obj *ga. op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEForwardingRules.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEForwardingRules.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEForwardingRules.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEForwardingRules.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the ForwardingRule referenced by key. func (g *GCEForwardingRules) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEForwardingRules.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEForwardingRules.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEForwardingRules.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEForwardingRules.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "ForwardingRules") @@ -6177,9 +6177,9 @@ func (g *GCEForwardingRules) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "ForwardingRules", } - glog.V(5).Infof("GCEForwardingRules.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEForwardingRules.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEForwardingRules.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEForwardingRules.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.ForwardingRules.Delete(projectID, key.Region, key.Name) @@ -6187,12 +6187,12 @@ func (g *GCEForwardingRules) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } @@ -6251,7 +6251,7 @@ type MockAlphaForwardingRules struct { func (m *MockAlphaForwardingRules) Get(ctx context.Context, key *meta.Key) (*alpha.ForwardingRule, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaForwardingRules.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaForwardingRules.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -6263,12 +6263,12 @@ func (m *MockAlphaForwardingRules) Get(ctx context.Context, key *meta.Key) (*alp defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockAlphaForwardingRules.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaForwardingRules.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToAlpha() - glog.V(5).Infof("MockAlphaForwardingRules.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockAlphaForwardingRules.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -6276,7 +6276,7 @@ func (m *MockAlphaForwardingRules) Get(ctx context.Context, key *meta.Key) (*alp Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaForwardingRules %v not found", key), } - glog.V(5).Infof("MockAlphaForwardingRules.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaForwardingRules.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -6284,7 +6284,7 @@ func (m *MockAlphaForwardingRules) Get(ctx context.Context, key *meta.Key) (*alp func (m *MockAlphaForwardingRules) List(ctx context.Context, region string, fl *filter.F) ([]*alpha.ForwardingRule, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, region, fl, m); intercept { - glog.V(5).Infof("MockAlphaForwardingRules.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) + klog.V(5).Infof("MockAlphaForwardingRules.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) return objs, err } } @@ -6294,7 +6294,7 @@ func (m *MockAlphaForwardingRules) List(ctx context.Context, region string, fl * if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockAlphaForwardingRules.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) + klog.V(5).Infof("MockAlphaForwardingRules.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) return nil, *m.ListError } @@ -6310,7 +6310,7 @@ func (m *MockAlphaForwardingRules) List(ctx context.Context, region string, fl * objs = append(objs, obj.ToAlpha()) } - glog.V(5).Infof("MockAlphaForwardingRules.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) + klog.V(5).Infof("MockAlphaForwardingRules.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) return objs, nil } @@ -6318,7 +6318,7 @@ func (m *MockAlphaForwardingRules) List(ctx context.Context, region string, fl * func (m *MockAlphaForwardingRules) Insert(ctx context.Context, key *meta.Key, obj *alpha.ForwardingRule) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockAlphaForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -6330,7 +6330,7 @@ func (m *MockAlphaForwardingRules) Insert(ctx context.Context, key *meta.Key, ob defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockAlphaForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -6338,7 +6338,7 @@ func (m *MockAlphaForwardingRules) Insert(ctx context.Context, key *meta.Key, ob Code: http.StatusConflict, Message: fmt.Sprintf("MockAlphaForwardingRules %v exists", key), } - glog.V(5).Infof("MockAlphaForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -6347,7 +6347,7 @@ func (m *MockAlphaForwardingRules) Insert(ctx context.Context, key *meta.Key, ob obj.SelfLink = SelfLink(meta.VersionAlpha, projectID, "forwardingRules", key) m.Objects[*key] = &MockForwardingRulesObj{obj} - glog.V(5).Infof("MockAlphaForwardingRules.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockAlphaForwardingRules.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -6355,7 +6355,7 @@ func (m *MockAlphaForwardingRules) Insert(ctx context.Context, key *meta.Key, ob func (m *MockAlphaForwardingRules) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -6367,7 +6367,7 @@ func (m *MockAlphaForwardingRules) Delete(ctx context.Context, key *meta.Key) er defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockAlphaForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -6375,12 +6375,12 @@ func (m *MockAlphaForwardingRules) Delete(ctx context.Context, key *meta.Key) er Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaForwardingRules %v not found", key), } - glog.V(5).Infof("MockAlphaForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockAlphaForwardingRules.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockAlphaForwardingRules.Delete(%v, %v) = nil", ctx, key) return nil } @@ -6396,10 +6396,10 @@ type GCEAlphaForwardingRules struct { // Get the ForwardingRule named by key. func (g *GCEAlphaForwardingRules) Get(ctx context.Context, key *meta.Key) (*alpha.ForwardingRule, error) { - glog.V(5).Infof("GCEAlphaForwardingRules.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaForwardingRules.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaForwardingRules.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaForwardingRules.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "ForwardingRules") @@ -6409,21 +6409,21 @@ func (g *GCEAlphaForwardingRules) Get(ctx context.Context, key *meta.Key) (*alph Version: meta.Version("alpha"), Service: "ForwardingRules", } - glog.V(5).Infof("GCEAlphaForwardingRules.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaForwardingRules.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaForwardingRules.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaForwardingRules.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Alpha.ForwardingRules.Get(projectID, key.Region, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEAlphaForwardingRules.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEAlphaForwardingRules.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all ForwardingRule objects. func (g *GCEAlphaForwardingRules) List(ctx context.Context, region string, fl *filter.F) ([]*alpha.ForwardingRule, error) { - glog.V(5).Infof("GCEAlphaForwardingRules.List(%v, %v, %v) called", ctx, region, fl) + klog.V(5).Infof("GCEAlphaForwardingRules.List(%v, %v, %v) called", ctx, region, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "ForwardingRules") rk := &RateLimitKey{ ProjectID: projectID, @@ -6434,30 +6434,30 @@ func (g *GCEAlphaForwardingRules) List(ctx context.Context, region string, fl *f if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEAlphaForwardingRules.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) + klog.V(5).Infof("GCEAlphaForwardingRules.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) call := g.s.Alpha.ForwardingRules.List(projectID, region) if fl != filter.None { call.Filter(fl.String()) } var all []*alpha.ForwardingRule f := func(l *alpha.ForwardingRuleList) error { - glog.V(5).Infof("GCEAlphaForwardingRules.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEAlphaForwardingRules.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEAlphaForwardingRules.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEAlphaForwardingRules.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEAlphaForwardingRules.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEAlphaForwardingRules.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEAlphaForwardingRules.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEAlphaForwardingRules.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -6465,9 +6465,9 @@ func (g *GCEAlphaForwardingRules) List(ctx context.Context, region string, fl *f // Insert ForwardingRule with key of value obj. func (g *GCEAlphaForwardingRules) Insert(ctx context.Context, key *meta.Key, obj *alpha.ForwardingRule) error { - glog.V(5).Infof("GCEAlphaForwardingRules.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEAlphaForwardingRules.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEAlphaForwardingRules.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaForwardingRules.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "ForwardingRules") @@ -6477,9 +6477,9 @@ func (g *GCEAlphaForwardingRules) Insert(ctx context.Context, key *meta.Key, obj Version: meta.Version("alpha"), Service: "ForwardingRules", } - glog.V(5).Infof("GCEAlphaForwardingRules.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaForwardingRules.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaForwardingRules.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaForwardingRules.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -6488,20 +6488,20 @@ func (g *GCEAlphaForwardingRules) Insert(ctx context.Context, key *meta.Key, obj op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaForwardingRules.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaForwardingRules.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaForwardingRules.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEAlphaForwardingRules.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the ForwardingRule referenced by key. func (g *GCEAlphaForwardingRules) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEAlphaForwardingRules.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaForwardingRules.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaForwardingRules.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaForwardingRules.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "ForwardingRules") @@ -6511,9 +6511,9 @@ func (g *GCEAlphaForwardingRules) Delete(ctx context.Context, key *meta.Key) err Version: meta.Version("alpha"), Service: "ForwardingRules", } - glog.V(5).Infof("GCEAlphaForwardingRules.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaForwardingRules.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaForwardingRules.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaForwardingRules.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.ForwardingRules.Delete(projectID, key.Region, key.Name) @@ -6521,12 +6521,12 @@ func (g *GCEAlphaForwardingRules) Delete(ctx context.Context, key *meta.Key) err op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } @@ -6587,7 +6587,7 @@ type MockGlobalForwardingRules struct { func (m *MockGlobalForwardingRules) Get(ctx context.Context, key *meta.Key) (*ga.ForwardingRule, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockGlobalForwardingRules.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockGlobalForwardingRules.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -6599,12 +6599,12 @@ func (m *MockGlobalForwardingRules) Get(ctx context.Context, key *meta.Key) (*ga defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockGlobalForwardingRules.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockGlobalForwardingRules.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockGlobalForwardingRules.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockGlobalForwardingRules.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -6612,7 +6612,7 @@ func (m *MockGlobalForwardingRules) Get(ctx context.Context, key *meta.Key) (*ga Code: http.StatusNotFound, Message: fmt.Sprintf("MockGlobalForwardingRules %v not found", key), } - glog.V(5).Infof("MockGlobalForwardingRules.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockGlobalForwardingRules.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -6620,7 +6620,7 @@ func (m *MockGlobalForwardingRules) Get(ctx context.Context, key *meta.Key) (*ga func (m *MockGlobalForwardingRules) List(ctx context.Context, fl *filter.F) ([]*ga.ForwardingRule, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockGlobalForwardingRules.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockGlobalForwardingRules.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -6630,7 +6630,7 @@ func (m *MockGlobalForwardingRules) List(ctx context.Context, fl *filter.F) ([]* if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockGlobalForwardingRules.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockGlobalForwardingRules.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -6643,7 +6643,7 @@ func (m *MockGlobalForwardingRules) List(ctx context.Context, fl *filter.F) ([]* objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockGlobalForwardingRules.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockGlobalForwardingRules.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -6651,7 +6651,7 @@ func (m *MockGlobalForwardingRules) List(ctx context.Context, fl *filter.F) ([]* func (m *MockGlobalForwardingRules) Insert(ctx context.Context, key *meta.Key, obj *ga.ForwardingRule) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockGlobalForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockGlobalForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -6663,7 +6663,7 @@ func (m *MockGlobalForwardingRules) Insert(ctx context.Context, key *meta.Key, o defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockGlobalForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockGlobalForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -6671,7 +6671,7 @@ func (m *MockGlobalForwardingRules) Insert(ctx context.Context, key *meta.Key, o Code: http.StatusConflict, Message: fmt.Sprintf("MockGlobalForwardingRules %v exists", key), } - glog.V(5).Infof("MockGlobalForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockGlobalForwardingRules.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -6680,7 +6680,7 @@ func (m *MockGlobalForwardingRules) Insert(ctx context.Context, key *meta.Key, o obj.SelfLink = SelfLink(meta.VersionGA, projectID, "forwardingRules", key) m.Objects[*key] = &MockGlobalForwardingRulesObj{obj} - glog.V(5).Infof("MockGlobalForwardingRules.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockGlobalForwardingRules.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -6688,7 +6688,7 @@ func (m *MockGlobalForwardingRules) Insert(ctx context.Context, key *meta.Key, o func (m *MockGlobalForwardingRules) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockGlobalForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockGlobalForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -6700,7 +6700,7 @@ func (m *MockGlobalForwardingRules) Delete(ctx context.Context, key *meta.Key) e defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockGlobalForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockGlobalForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -6708,12 +6708,12 @@ func (m *MockGlobalForwardingRules) Delete(ctx context.Context, key *meta.Key) e Code: http.StatusNotFound, Message: fmt.Sprintf("MockGlobalForwardingRules %v not found", key), } - glog.V(5).Infof("MockGlobalForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockGlobalForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockGlobalForwardingRules.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockGlobalForwardingRules.Delete(%v, %v) = nil", ctx, key) return nil } @@ -6737,10 +6737,10 @@ type GCEGlobalForwardingRules struct { // Get the ForwardingRule named by key. func (g *GCEGlobalForwardingRules) Get(ctx context.Context, key *meta.Key) (*ga.ForwardingRule, error) { - glog.V(5).Infof("GCEGlobalForwardingRules.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEGlobalForwardingRules.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEGlobalForwardingRules.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEGlobalForwardingRules.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "GlobalForwardingRules") @@ -6750,21 +6750,21 @@ func (g *GCEGlobalForwardingRules) Get(ctx context.Context, key *meta.Key) (*ga. Version: meta.Version("ga"), Service: "GlobalForwardingRules", } - glog.V(5).Infof("GCEGlobalForwardingRules.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEGlobalForwardingRules.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEGlobalForwardingRules.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEGlobalForwardingRules.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.GlobalForwardingRules.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEGlobalForwardingRules.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEGlobalForwardingRules.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all ForwardingRule objects. func (g *GCEGlobalForwardingRules) List(ctx context.Context, fl *filter.F) ([]*ga.ForwardingRule, error) { - glog.V(5).Infof("GCEGlobalForwardingRules.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEGlobalForwardingRules.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "GlobalForwardingRules") rk := &RateLimitKey{ ProjectID: projectID, @@ -6775,30 +6775,30 @@ func (g *GCEGlobalForwardingRules) List(ctx context.Context, fl *filter.F) ([]*g if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEGlobalForwardingRules.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEGlobalForwardingRules.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.GlobalForwardingRules.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.ForwardingRule f := func(l *ga.ForwardingRuleList) error { - glog.V(5).Infof("GCEGlobalForwardingRules.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEGlobalForwardingRules.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEGlobalForwardingRules.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEGlobalForwardingRules.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEGlobalForwardingRules.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEGlobalForwardingRules.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEGlobalForwardingRules.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEGlobalForwardingRules.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -6806,9 +6806,9 @@ func (g *GCEGlobalForwardingRules) List(ctx context.Context, fl *filter.F) ([]*g // Insert ForwardingRule with key of value obj. func (g *GCEGlobalForwardingRules) Insert(ctx context.Context, key *meta.Key, obj *ga.ForwardingRule) error { - glog.V(5).Infof("GCEGlobalForwardingRules.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEGlobalForwardingRules.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEGlobalForwardingRules.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEGlobalForwardingRules.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "GlobalForwardingRules") @@ -6818,9 +6818,9 @@ func (g *GCEGlobalForwardingRules) Insert(ctx context.Context, key *meta.Key, ob Version: meta.Version("ga"), Service: "GlobalForwardingRules", } - glog.V(5).Infof("GCEGlobalForwardingRules.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEGlobalForwardingRules.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEGlobalForwardingRules.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEGlobalForwardingRules.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -6829,20 +6829,20 @@ func (g *GCEGlobalForwardingRules) Insert(ctx context.Context, key *meta.Key, ob op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEGlobalForwardingRules.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEGlobalForwardingRules.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEGlobalForwardingRules.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEGlobalForwardingRules.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the ForwardingRule referenced by key. func (g *GCEGlobalForwardingRules) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEGlobalForwardingRules.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEGlobalForwardingRules.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEGlobalForwardingRules.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEGlobalForwardingRules.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "GlobalForwardingRules") @@ -6852,9 +6852,9 @@ func (g *GCEGlobalForwardingRules) Delete(ctx context.Context, key *meta.Key) er Version: meta.Version("ga"), Service: "GlobalForwardingRules", } - glog.V(5).Infof("GCEGlobalForwardingRules.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEGlobalForwardingRules.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEGlobalForwardingRules.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEGlobalForwardingRules.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.GlobalForwardingRules.Delete(projectID, key.Name) @@ -6863,21 +6863,21 @@ func (g *GCEGlobalForwardingRules) Delete(ctx context.Context, key *meta.Key) er op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEGlobalForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEGlobalForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEGlobalForwardingRules.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEGlobalForwardingRules.Delete(%v, %v) = %v", ctx, key, err) return err } // SetTarget is a method on GCEGlobalForwardingRules. func (g *GCEGlobalForwardingRules) SetTarget(ctx context.Context, key *meta.Key, arg0 *ga.TargetReference) error { - glog.V(5).Infof("GCEGlobalForwardingRules.SetTarget(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEGlobalForwardingRules.SetTarget(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEGlobalForwardingRules.SetTarget(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEGlobalForwardingRules.SetTarget(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "GlobalForwardingRules") @@ -6887,21 +6887,21 @@ func (g *GCEGlobalForwardingRules) SetTarget(ctx context.Context, key *meta.Key, Version: meta.Version("ga"), Service: "GlobalForwardingRules", } - glog.V(5).Infof("GCEGlobalForwardingRules.SetTarget(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEGlobalForwardingRules.SetTarget(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEGlobalForwardingRules.SetTarget(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEGlobalForwardingRules.SetTarget(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.GlobalForwardingRules.SetTarget(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEGlobalForwardingRules.SetTarget(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEGlobalForwardingRules.SetTarget(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEGlobalForwardingRules.SetTarget(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEGlobalForwardingRules.SetTarget(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -6962,7 +6962,7 @@ type MockHealthChecks struct { func (m *MockHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.HealthCheck, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockHealthChecks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockHealthChecks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -6974,12 +6974,12 @@ func (m *MockHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.HealthCh defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockHealthChecks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockHealthChecks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -6987,7 +6987,7 @@ func (m *MockHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.HealthCh Code: http.StatusNotFound, Message: fmt.Sprintf("MockHealthChecks %v not found", key), } - glog.V(5).Infof("MockHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -6995,7 +6995,7 @@ func (m *MockHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.HealthCh func (m *MockHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.HealthCheck, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockHealthChecks.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockHealthChecks.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -7005,7 +7005,7 @@ func (m *MockHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.Health if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockHealthChecks.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockHealthChecks.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -7018,7 +7018,7 @@ func (m *MockHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.Health objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockHealthChecks.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockHealthChecks.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -7026,7 +7026,7 @@ func (m *MockHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.Health func (m *MockHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga.HealthCheck) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -7038,7 +7038,7 @@ func (m *MockHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga.He defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -7046,7 +7046,7 @@ func (m *MockHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga.He Code: http.StatusConflict, Message: fmt.Sprintf("MockHealthChecks %v exists", key), } - glog.V(5).Infof("MockHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -7055,7 +7055,7 @@ func (m *MockHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga.He obj.SelfLink = SelfLink(meta.VersionGA, projectID, "healthChecks", key) m.Objects[*key] = &MockHealthChecksObj{obj} - glog.V(5).Infof("MockHealthChecks.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockHealthChecks.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -7063,7 +7063,7 @@ func (m *MockHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga.He func (m *MockHealthChecks) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -7075,7 +7075,7 @@ func (m *MockHealthChecks) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -7083,12 +7083,12 @@ func (m *MockHealthChecks) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockHealthChecks %v not found", key), } - glog.V(5).Infof("MockHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockHealthChecks.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockHealthChecks.Delete(%v, %v) = nil", ctx, key) return nil } @@ -7112,10 +7112,10 @@ type GCEHealthChecks struct { // Get the HealthCheck named by key. func (g *GCEHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.HealthCheck, error) { - glog.V(5).Infof("GCEHealthChecks.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEHealthChecks.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEHealthChecks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEHealthChecks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HealthChecks") @@ -7125,21 +7125,21 @@ func (g *GCEHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.HealthChe Version: meta.Version("ga"), Service: "HealthChecks", } - glog.V(5).Infof("GCEHealthChecks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEHealthChecks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEHealthChecks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEHealthChecks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.HealthChecks.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEHealthChecks.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEHealthChecks.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all HealthCheck objects. func (g *GCEHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.HealthCheck, error) { - glog.V(5).Infof("GCEHealthChecks.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEHealthChecks.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HealthChecks") rk := &RateLimitKey{ ProjectID: projectID, @@ -7150,30 +7150,30 @@ func (g *GCEHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.HealthC if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEHealthChecks.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEHealthChecks.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.HealthChecks.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.HealthCheck f := func(l *ga.HealthCheckList) error { - glog.V(5).Infof("GCEHealthChecks.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEHealthChecks.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEHealthChecks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEHealthChecks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -7181,9 +7181,9 @@ func (g *GCEHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.HealthC // Insert HealthCheck with key of value obj. func (g *GCEHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga.HealthCheck) error { - glog.V(5).Infof("GCEHealthChecks.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEHealthChecks.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEHealthChecks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEHealthChecks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HealthChecks") @@ -7193,9 +7193,9 @@ func (g *GCEHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga.Hea Version: meta.Version("ga"), Service: "HealthChecks", } - glog.V(5).Infof("GCEHealthChecks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEHealthChecks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEHealthChecks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEHealthChecks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -7204,20 +7204,20 @@ func (g *GCEHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga.Hea op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEHealthChecks.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEHealthChecks.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEHealthChecks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEHealthChecks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the HealthCheck referenced by key. func (g *GCEHealthChecks) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEHealthChecks.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEHealthChecks.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEHealthChecks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEHealthChecks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HealthChecks") @@ -7227,9 +7227,9 @@ func (g *GCEHealthChecks) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "HealthChecks", } - glog.V(5).Infof("GCEHealthChecks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEHealthChecks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEHealthChecks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEHealthChecks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.HealthChecks.Delete(projectID, key.Name) @@ -7238,21 +7238,21 @@ func (g *GCEHealthChecks) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } // Update is a method on GCEHealthChecks. func (g *GCEHealthChecks) Update(ctx context.Context, key *meta.Key, arg0 *ga.HealthCheck) error { - glog.V(5).Infof("GCEHealthChecks.Update(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEHealthChecks.Update(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEHealthChecks.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEHealthChecks.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HealthChecks") @@ -7262,21 +7262,21 @@ func (g *GCEHealthChecks) Update(ctx context.Context, key *meta.Key, arg0 *ga.He Version: meta.Version("ga"), Service: "HealthChecks", } - glog.V(5).Infof("GCEHealthChecks.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEHealthChecks.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEHealthChecks.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEHealthChecks.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.HealthChecks.Update(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -7337,7 +7337,7 @@ type MockBetaHealthChecks struct { func (m *MockBetaHealthChecks) Get(ctx context.Context, key *meta.Key) (*beta.HealthCheck, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaHealthChecks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaHealthChecks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -7349,12 +7349,12 @@ func (m *MockBetaHealthChecks) Get(ctx context.Context, key *meta.Key) (*beta.He defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockBetaHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBetaHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToBeta() - glog.V(5).Infof("MockBetaHealthChecks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockBetaHealthChecks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -7362,7 +7362,7 @@ func (m *MockBetaHealthChecks) Get(ctx context.Context, key *meta.Key) (*beta.He Code: http.StatusNotFound, Message: fmt.Sprintf("MockBetaHealthChecks %v not found", key), } - glog.V(5).Infof("MockBetaHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBetaHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -7370,7 +7370,7 @@ func (m *MockBetaHealthChecks) Get(ctx context.Context, key *meta.Key) (*beta.He func (m *MockBetaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*beta.HealthCheck, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockBetaHealthChecks.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockBetaHealthChecks.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -7380,7 +7380,7 @@ func (m *MockBetaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*beta. if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockBetaHealthChecks.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockBetaHealthChecks.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -7393,7 +7393,7 @@ func (m *MockBetaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*beta. objs = append(objs, obj.ToBeta()) } - glog.V(5).Infof("MockBetaHealthChecks.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockBetaHealthChecks.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -7401,7 +7401,7 @@ func (m *MockBetaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*beta. func (m *MockBetaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *beta.HealthCheck) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockBetaHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -7413,7 +7413,7 @@ func (m *MockBetaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *b defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockBetaHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -7421,7 +7421,7 @@ func (m *MockBetaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *b Code: http.StatusConflict, Message: fmt.Sprintf("MockBetaHealthChecks %v exists", key), } - glog.V(5).Infof("MockBetaHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -7430,7 +7430,7 @@ func (m *MockBetaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *b obj.SelfLink = SelfLink(meta.VersionBeta, projectID, "healthChecks", key) m.Objects[*key] = &MockHealthChecksObj{obj} - glog.V(5).Infof("MockBetaHealthChecks.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockBetaHealthChecks.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -7438,7 +7438,7 @@ func (m *MockBetaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *b func (m *MockBetaHealthChecks) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -7450,7 +7450,7 @@ func (m *MockBetaHealthChecks) Delete(ctx context.Context, key *meta.Key) error defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockBetaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -7458,12 +7458,12 @@ func (m *MockBetaHealthChecks) Delete(ctx context.Context, key *meta.Key) error Code: http.StatusNotFound, Message: fmt.Sprintf("MockBetaHealthChecks %v not found", key), } - glog.V(5).Infof("MockBetaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockBetaHealthChecks.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockBetaHealthChecks.Delete(%v, %v) = nil", ctx, key) return nil } @@ -7487,10 +7487,10 @@ type GCEBetaHealthChecks struct { // Get the HealthCheck named by key. func (g *GCEBetaHealthChecks) Get(ctx context.Context, key *meta.Key) (*beta.HealthCheck, error) { - glog.V(5).Infof("GCEBetaHealthChecks.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBetaHealthChecks.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaHealthChecks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaHealthChecks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "HealthChecks") @@ -7500,21 +7500,21 @@ func (g *GCEBetaHealthChecks) Get(ctx context.Context, key *meta.Key) (*beta.Hea Version: meta.Version("beta"), Service: "HealthChecks", } - glog.V(5).Infof("GCEBetaHealthChecks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaHealthChecks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaHealthChecks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaHealthChecks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Beta.HealthChecks.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEBetaHealthChecks.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEBetaHealthChecks.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all HealthCheck objects. func (g *GCEBetaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*beta.HealthCheck, error) { - glog.V(5).Infof("GCEBetaHealthChecks.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEBetaHealthChecks.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "HealthChecks") rk := &RateLimitKey{ ProjectID: projectID, @@ -7525,30 +7525,30 @@ func (g *GCEBetaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*beta.H if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEBetaHealthChecks.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEBetaHealthChecks.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.Beta.HealthChecks.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*beta.HealthCheck f := func(l *beta.HealthCheckList) error { - glog.V(5).Infof("GCEBetaHealthChecks.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEBetaHealthChecks.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEBetaHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEBetaHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEBetaHealthChecks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEBetaHealthChecks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEBetaHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEBetaHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -7556,9 +7556,9 @@ func (g *GCEBetaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*beta.H // Insert HealthCheck with key of value obj. func (g *GCEBetaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *beta.HealthCheck) error { - glog.V(5).Infof("GCEBetaHealthChecks.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEBetaHealthChecks.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEBetaHealthChecks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaHealthChecks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "HealthChecks") @@ -7568,9 +7568,9 @@ func (g *GCEBetaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *be Version: meta.Version("beta"), Service: "HealthChecks", } - glog.V(5).Infof("GCEBetaHealthChecks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaHealthChecks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaHealthChecks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaHealthChecks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -7579,20 +7579,20 @@ func (g *GCEBetaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *be op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaHealthChecks.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaHealthChecks.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaHealthChecks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEBetaHealthChecks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the HealthCheck referenced by key. func (g *GCEBetaHealthChecks) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEBetaHealthChecks.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBetaHealthChecks.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaHealthChecks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaHealthChecks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "HealthChecks") @@ -7602,9 +7602,9 @@ func (g *GCEBetaHealthChecks) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("beta"), Service: "HealthChecks", } - glog.V(5).Infof("GCEBetaHealthChecks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaHealthChecks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaHealthChecks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaHealthChecks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.HealthChecks.Delete(projectID, key.Name) @@ -7613,21 +7613,21 @@ func (g *GCEBetaHealthChecks) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBetaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBetaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } // Update is a method on GCEBetaHealthChecks. func (g *GCEBetaHealthChecks) Update(ctx context.Context, key *meta.Key, arg0 *beta.HealthCheck) error { - glog.V(5).Infof("GCEBetaHealthChecks.Update(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaHealthChecks.Update(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaHealthChecks.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaHealthChecks.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "HealthChecks") @@ -7637,21 +7637,21 @@ func (g *GCEBetaHealthChecks) Update(ctx context.Context, key *meta.Key, arg0 *b Version: meta.Version("beta"), Service: "HealthChecks", } - glog.V(5).Infof("GCEBetaHealthChecks.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaHealthChecks.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaHealthChecks.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaHealthChecks.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.HealthChecks.Update(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -7712,7 +7712,7 @@ type MockAlphaHealthChecks struct { func (m *MockAlphaHealthChecks) Get(ctx context.Context, key *meta.Key) (*alpha.HealthCheck, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaHealthChecks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaHealthChecks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -7724,12 +7724,12 @@ func (m *MockAlphaHealthChecks) Get(ctx context.Context, key *meta.Key) (*alpha. defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockAlphaHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToAlpha() - glog.V(5).Infof("MockAlphaHealthChecks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockAlphaHealthChecks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -7737,7 +7737,7 @@ func (m *MockAlphaHealthChecks) Get(ctx context.Context, key *meta.Key) (*alpha. Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaHealthChecks %v not found", key), } - glog.V(5).Infof("MockAlphaHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -7745,7 +7745,7 @@ func (m *MockAlphaHealthChecks) Get(ctx context.Context, key *meta.Key) (*alpha. func (m *MockAlphaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*alpha.HealthCheck, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockAlphaHealthChecks.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockAlphaHealthChecks.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -7755,7 +7755,7 @@ func (m *MockAlphaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*alph if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockAlphaHealthChecks.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockAlphaHealthChecks.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -7768,7 +7768,7 @@ func (m *MockAlphaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*alph objs = append(objs, obj.ToAlpha()) } - glog.V(5).Infof("MockAlphaHealthChecks.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockAlphaHealthChecks.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -7776,7 +7776,7 @@ func (m *MockAlphaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*alph func (m *MockAlphaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *alpha.HealthCheck) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockAlphaHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -7788,7 +7788,7 @@ func (m *MockAlphaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj * defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockAlphaHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -7796,7 +7796,7 @@ func (m *MockAlphaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj * Code: http.StatusConflict, Message: fmt.Sprintf("MockAlphaHealthChecks %v exists", key), } - glog.V(5).Infof("MockAlphaHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -7805,7 +7805,7 @@ func (m *MockAlphaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj * obj.SelfLink = SelfLink(meta.VersionAlpha, projectID, "healthChecks", key) m.Objects[*key] = &MockHealthChecksObj{obj} - glog.V(5).Infof("MockAlphaHealthChecks.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockAlphaHealthChecks.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -7813,7 +7813,7 @@ func (m *MockAlphaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj * func (m *MockAlphaHealthChecks) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -7825,7 +7825,7 @@ func (m *MockAlphaHealthChecks) Delete(ctx context.Context, key *meta.Key) error defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockAlphaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -7833,12 +7833,12 @@ func (m *MockAlphaHealthChecks) Delete(ctx context.Context, key *meta.Key) error Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaHealthChecks %v not found", key), } - glog.V(5).Infof("MockAlphaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockAlphaHealthChecks.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockAlphaHealthChecks.Delete(%v, %v) = nil", ctx, key) return nil } @@ -7862,10 +7862,10 @@ type GCEAlphaHealthChecks struct { // Get the HealthCheck named by key. func (g *GCEAlphaHealthChecks) Get(ctx context.Context, key *meta.Key) (*alpha.HealthCheck, error) { - glog.V(5).Infof("GCEAlphaHealthChecks.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaHealthChecks.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaHealthChecks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaHealthChecks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "HealthChecks") @@ -7875,21 +7875,21 @@ func (g *GCEAlphaHealthChecks) Get(ctx context.Context, key *meta.Key) (*alpha.H Version: meta.Version("alpha"), Service: "HealthChecks", } - glog.V(5).Infof("GCEAlphaHealthChecks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaHealthChecks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaHealthChecks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaHealthChecks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Alpha.HealthChecks.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEAlphaHealthChecks.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEAlphaHealthChecks.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all HealthCheck objects. func (g *GCEAlphaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*alpha.HealthCheck, error) { - glog.V(5).Infof("GCEAlphaHealthChecks.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEAlphaHealthChecks.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "HealthChecks") rk := &RateLimitKey{ ProjectID: projectID, @@ -7900,30 +7900,30 @@ func (g *GCEAlphaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*alpha if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEAlphaHealthChecks.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEAlphaHealthChecks.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.Alpha.HealthChecks.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*alpha.HealthCheck f := func(l *alpha.HealthCheckList) error { - glog.V(5).Infof("GCEAlphaHealthChecks.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEAlphaHealthChecks.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEAlphaHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEAlphaHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEAlphaHealthChecks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEAlphaHealthChecks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEAlphaHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEAlphaHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -7931,9 +7931,9 @@ func (g *GCEAlphaHealthChecks) List(ctx context.Context, fl *filter.F) ([]*alpha // Insert HealthCheck with key of value obj. func (g *GCEAlphaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *alpha.HealthCheck) error { - glog.V(5).Infof("GCEAlphaHealthChecks.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEAlphaHealthChecks.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEAlphaHealthChecks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaHealthChecks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "HealthChecks") @@ -7943,9 +7943,9 @@ func (g *GCEAlphaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *a Version: meta.Version("alpha"), Service: "HealthChecks", } - glog.V(5).Infof("GCEAlphaHealthChecks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaHealthChecks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaHealthChecks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaHealthChecks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -7954,20 +7954,20 @@ func (g *GCEAlphaHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *a op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaHealthChecks.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaHealthChecks.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaHealthChecks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEAlphaHealthChecks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the HealthCheck referenced by key. func (g *GCEAlphaHealthChecks) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEAlphaHealthChecks.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaHealthChecks.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaHealthChecks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaHealthChecks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "HealthChecks") @@ -7977,9 +7977,9 @@ func (g *GCEAlphaHealthChecks) Delete(ctx context.Context, key *meta.Key) error Version: meta.Version("alpha"), Service: "HealthChecks", } - glog.V(5).Infof("GCEAlphaHealthChecks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaHealthChecks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaHealthChecks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaHealthChecks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.HealthChecks.Delete(projectID, key.Name) @@ -7988,21 +7988,21 @@ func (g *GCEAlphaHealthChecks) Delete(ctx context.Context, key *meta.Key) error op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } // Update is a method on GCEAlphaHealthChecks. func (g *GCEAlphaHealthChecks) Update(ctx context.Context, key *meta.Key, arg0 *alpha.HealthCheck) error { - glog.V(5).Infof("GCEAlphaHealthChecks.Update(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEAlphaHealthChecks.Update(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaHealthChecks.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaHealthChecks.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "HealthChecks") @@ -8012,21 +8012,21 @@ func (g *GCEAlphaHealthChecks) Update(ctx context.Context, key *meta.Key, arg0 * Version: meta.Version("alpha"), Service: "HealthChecks", } - glog.V(5).Infof("GCEAlphaHealthChecks.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaHealthChecks.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaHealthChecks.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaHealthChecks.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.HealthChecks.Update(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -8087,7 +8087,7 @@ type MockHttpHealthChecks struct { func (m *MockHttpHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.HttpHealthCheck, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockHttpHealthChecks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockHttpHealthChecks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -8099,12 +8099,12 @@ func (m *MockHttpHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.Http defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockHttpHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockHttpHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockHttpHealthChecks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockHttpHealthChecks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -8112,7 +8112,7 @@ func (m *MockHttpHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.Http Code: http.StatusNotFound, Message: fmt.Sprintf("MockHttpHealthChecks %v not found", key), } - glog.V(5).Infof("MockHttpHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockHttpHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -8120,7 +8120,7 @@ func (m *MockHttpHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.Http func (m *MockHttpHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.HttpHealthCheck, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockHttpHealthChecks.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockHttpHealthChecks.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -8130,7 +8130,7 @@ func (m *MockHttpHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.Ht if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockHttpHealthChecks.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockHttpHealthChecks.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -8143,7 +8143,7 @@ func (m *MockHttpHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.Ht objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockHttpHealthChecks.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockHttpHealthChecks.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -8151,7 +8151,7 @@ func (m *MockHttpHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.Ht func (m *MockHttpHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga.HttpHealthCheck) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockHttpHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockHttpHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -8163,7 +8163,7 @@ func (m *MockHttpHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *g defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockHttpHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockHttpHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -8171,7 +8171,7 @@ func (m *MockHttpHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *g Code: http.StatusConflict, Message: fmt.Sprintf("MockHttpHealthChecks %v exists", key), } - glog.V(5).Infof("MockHttpHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockHttpHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -8180,7 +8180,7 @@ func (m *MockHttpHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *g obj.SelfLink = SelfLink(meta.VersionGA, projectID, "httpHealthChecks", key) m.Objects[*key] = &MockHttpHealthChecksObj{obj} - glog.V(5).Infof("MockHttpHealthChecks.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockHttpHealthChecks.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -8188,7 +8188,7 @@ func (m *MockHttpHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *g func (m *MockHttpHealthChecks) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockHttpHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockHttpHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -8200,7 +8200,7 @@ func (m *MockHttpHealthChecks) Delete(ctx context.Context, key *meta.Key) error defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockHttpHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockHttpHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -8208,12 +8208,12 @@ func (m *MockHttpHealthChecks) Delete(ctx context.Context, key *meta.Key) error Code: http.StatusNotFound, Message: fmt.Sprintf("MockHttpHealthChecks %v not found", key), } - glog.V(5).Infof("MockHttpHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockHttpHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockHttpHealthChecks.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockHttpHealthChecks.Delete(%v, %v) = nil", ctx, key) return nil } @@ -8237,10 +8237,10 @@ type GCEHttpHealthChecks struct { // Get the HttpHealthCheck named by key. func (g *GCEHttpHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.HttpHealthCheck, error) { - glog.V(5).Infof("GCEHttpHealthChecks.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEHttpHealthChecks.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEHttpHealthChecks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEHttpHealthChecks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HttpHealthChecks") @@ -8250,21 +8250,21 @@ func (g *GCEHttpHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.HttpH Version: meta.Version("ga"), Service: "HttpHealthChecks", } - glog.V(5).Infof("GCEHttpHealthChecks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEHttpHealthChecks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEHttpHealthChecks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEHttpHealthChecks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.HttpHealthChecks.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEHttpHealthChecks.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEHttpHealthChecks.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all HttpHealthCheck objects. func (g *GCEHttpHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.HttpHealthCheck, error) { - glog.V(5).Infof("GCEHttpHealthChecks.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEHttpHealthChecks.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HttpHealthChecks") rk := &RateLimitKey{ ProjectID: projectID, @@ -8275,30 +8275,30 @@ func (g *GCEHttpHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.Htt if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEHttpHealthChecks.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEHttpHealthChecks.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.HttpHealthChecks.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.HttpHealthCheck f := func(l *ga.HttpHealthCheckList) error { - glog.V(5).Infof("GCEHttpHealthChecks.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEHttpHealthChecks.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEHttpHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEHttpHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEHttpHealthChecks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEHttpHealthChecks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEHttpHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEHttpHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -8306,9 +8306,9 @@ func (g *GCEHttpHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.Htt // Insert HttpHealthCheck with key of value obj. func (g *GCEHttpHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga.HttpHealthCheck) error { - glog.V(5).Infof("GCEHttpHealthChecks.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEHttpHealthChecks.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEHttpHealthChecks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEHttpHealthChecks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HttpHealthChecks") @@ -8318,9 +8318,9 @@ func (g *GCEHttpHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga Version: meta.Version("ga"), Service: "HttpHealthChecks", } - glog.V(5).Infof("GCEHttpHealthChecks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEHttpHealthChecks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEHttpHealthChecks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEHttpHealthChecks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -8329,20 +8329,20 @@ func (g *GCEHttpHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEHttpHealthChecks.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEHttpHealthChecks.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEHttpHealthChecks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEHttpHealthChecks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the HttpHealthCheck referenced by key. func (g *GCEHttpHealthChecks) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEHttpHealthChecks.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEHttpHealthChecks.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEHttpHealthChecks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEHttpHealthChecks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HttpHealthChecks") @@ -8352,9 +8352,9 @@ func (g *GCEHttpHealthChecks) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "HttpHealthChecks", } - glog.V(5).Infof("GCEHttpHealthChecks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEHttpHealthChecks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEHttpHealthChecks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEHttpHealthChecks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.HttpHealthChecks.Delete(projectID, key.Name) @@ -8363,21 +8363,21 @@ func (g *GCEHttpHealthChecks) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEHttpHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEHttpHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEHttpHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEHttpHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } // Update is a method on GCEHttpHealthChecks. func (g *GCEHttpHealthChecks) Update(ctx context.Context, key *meta.Key, arg0 *ga.HttpHealthCheck) error { - glog.V(5).Infof("GCEHttpHealthChecks.Update(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEHttpHealthChecks.Update(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEHttpHealthChecks.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEHttpHealthChecks.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HttpHealthChecks") @@ -8387,21 +8387,21 @@ func (g *GCEHttpHealthChecks) Update(ctx context.Context, key *meta.Key, arg0 *g Version: meta.Version("ga"), Service: "HttpHealthChecks", } - glog.V(5).Infof("GCEHttpHealthChecks.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEHttpHealthChecks.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEHttpHealthChecks.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEHttpHealthChecks.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.HttpHealthChecks.Update(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEHttpHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEHttpHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEHttpHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEHttpHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -8462,7 +8462,7 @@ type MockHttpsHealthChecks struct { func (m *MockHttpsHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.HttpsHealthCheck, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockHttpsHealthChecks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockHttpsHealthChecks.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -8474,12 +8474,12 @@ func (m *MockHttpsHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.Htt defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockHttpsHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockHttpsHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockHttpsHealthChecks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockHttpsHealthChecks.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -8487,7 +8487,7 @@ func (m *MockHttpsHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.Htt Code: http.StatusNotFound, Message: fmt.Sprintf("MockHttpsHealthChecks %v not found", key), } - glog.V(5).Infof("MockHttpsHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockHttpsHealthChecks.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -8495,7 +8495,7 @@ func (m *MockHttpsHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.Htt func (m *MockHttpsHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.HttpsHealthCheck, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockHttpsHealthChecks.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockHttpsHealthChecks.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -8505,7 +8505,7 @@ func (m *MockHttpsHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.H if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockHttpsHealthChecks.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockHttpsHealthChecks.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -8518,7 +8518,7 @@ func (m *MockHttpsHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.H objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockHttpsHealthChecks.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockHttpsHealthChecks.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -8526,7 +8526,7 @@ func (m *MockHttpsHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.H func (m *MockHttpsHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga.HttpsHealthCheck) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockHttpsHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockHttpsHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -8538,7 +8538,7 @@ func (m *MockHttpsHealthChecks) Insert(ctx context.Context, key *meta.Key, obj * defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockHttpsHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockHttpsHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -8546,7 +8546,7 @@ func (m *MockHttpsHealthChecks) Insert(ctx context.Context, key *meta.Key, obj * Code: http.StatusConflict, Message: fmt.Sprintf("MockHttpsHealthChecks %v exists", key), } - glog.V(5).Infof("MockHttpsHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockHttpsHealthChecks.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -8555,7 +8555,7 @@ func (m *MockHttpsHealthChecks) Insert(ctx context.Context, key *meta.Key, obj * obj.SelfLink = SelfLink(meta.VersionGA, projectID, "httpsHealthChecks", key) m.Objects[*key] = &MockHttpsHealthChecksObj{obj} - glog.V(5).Infof("MockHttpsHealthChecks.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockHttpsHealthChecks.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -8563,7 +8563,7 @@ func (m *MockHttpsHealthChecks) Insert(ctx context.Context, key *meta.Key, obj * func (m *MockHttpsHealthChecks) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockHttpsHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockHttpsHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -8575,7 +8575,7 @@ func (m *MockHttpsHealthChecks) Delete(ctx context.Context, key *meta.Key) error defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockHttpsHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockHttpsHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -8583,12 +8583,12 @@ func (m *MockHttpsHealthChecks) Delete(ctx context.Context, key *meta.Key) error Code: http.StatusNotFound, Message: fmt.Sprintf("MockHttpsHealthChecks %v not found", key), } - glog.V(5).Infof("MockHttpsHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockHttpsHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockHttpsHealthChecks.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockHttpsHealthChecks.Delete(%v, %v) = nil", ctx, key) return nil } @@ -8612,10 +8612,10 @@ type GCEHttpsHealthChecks struct { // Get the HttpsHealthCheck named by key. func (g *GCEHttpsHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.HttpsHealthCheck, error) { - glog.V(5).Infof("GCEHttpsHealthChecks.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEHttpsHealthChecks.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEHttpsHealthChecks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEHttpsHealthChecks.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HttpsHealthChecks") @@ -8625,21 +8625,21 @@ func (g *GCEHttpsHealthChecks) Get(ctx context.Context, key *meta.Key) (*ga.Http Version: meta.Version("ga"), Service: "HttpsHealthChecks", } - glog.V(5).Infof("GCEHttpsHealthChecks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEHttpsHealthChecks.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEHttpsHealthChecks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEHttpsHealthChecks.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.HttpsHealthChecks.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEHttpsHealthChecks.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEHttpsHealthChecks.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all HttpsHealthCheck objects. func (g *GCEHttpsHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.HttpsHealthCheck, error) { - glog.V(5).Infof("GCEHttpsHealthChecks.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEHttpsHealthChecks.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HttpsHealthChecks") rk := &RateLimitKey{ ProjectID: projectID, @@ -8650,30 +8650,30 @@ func (g *GCEHttpsHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.Ht if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEHttpsHealthChecks.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEHttpsHealthChecks.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.HttpsHealthChecks.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.HttpsHealthCheck f := func(l *ga.HttpsHealthCheckList) error { - glog.V(5).Infof("GCEHttpsHealthChecks.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEHttpsHealthChecks.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEHttpsHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEHttpsHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEHttpsHealthChecks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEHttpsHealthChecks.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEHttpsHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEHttpsHealthChecks.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -8681,9 +8681,9 @@ func (g *GCEHttpsHealthChecks) List(ctx context.Context, fl *filter.F) ([]*ga.Ht // Insert HttpsHealthCheck with key of value obj. func (g *GCEHttpsHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *ga.HttpsHealthCheck) error { - glog.V(5).Infof("GCEHttpsHealthChecks.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEHttpsHealthChecks.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEHttpsHealthChecks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEHttpsHealthChecks.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HttpsHealthChecks") @@ -8693,9 +8693,9 @@ func (g *GCEHttpsHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *g Version: meta.Version("ga"), Service: "HttpsHealthChecks", } - glog.V(5).Infof("GCEHttpsHealthChecks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEHttpsHealthChecks.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEHttpsHealthChecks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEHttpsHealthChecks.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -8704,20 +8704,20 @@ func (g *GCEHttpsHealthChecks) Insert(ctx context.Context, key *meta.Key, obj *g op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEHttpsHealthChecks.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEHttpsHealthChecks.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEHttpsHealthChecks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEHttpsHealthChecks.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the HttpsHealthCheck referenced by key. func (g *GCEHttpsHealthChecks) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEHttpsHealthChecks.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEHttpsHealthChecks.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEHttpsHealthChecks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEHttpsHealthChecks.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HttpsHealthChecks") @@ -8727,9 +8727,9 @@ func (g *GCEHttpsHealthChecks) Delete(ctx context.Context, key *meta.Key) error Version: meta.Version("ga"), Service: "HttpsHealthChecks", } - glog.V(5).Infof("GCEHttpsHealthChecks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEHttpsHealthChecks.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEHttpsHealthChecks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEHttpsHealthChecks.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.HttpsHealthChecks.Delete(projectID, key.Name) @@ -8738,21 +8738,21 @@ func (g *GCEHttpsHealthChecks) Delete(ctx context.Context, key *meta.Key) error op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEHttpsHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEHttpsHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEHttpsHealthChecks.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEHttpsHealthChecks.Delete(%v, %v) = %v", ctx, key, err) return err } // Update is a method on GCEHttpsHealthChecks. func (g *GCEHttpsHealthChecks) Update(ctx context.Context, key *meta.Key, arg0 *ga.HttpsHealthCheck) error { - glog.V(5).Infof("GCEHttpsHealthChecks.Update(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEHttpsHealthChecks.Update(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEHttpsHealthChecks.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEHttpsHealthChecks.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "HttpsHealthChecks") @@ -8762,21 +8762,21 @@ func (g *GCEHttpsHealthChecks) Update(ctx context.Context, key *meta.Key, arg0 * Version: meta.Version("ga"), Service: "HttpsHealthChecks", } - glog.V(5).Infof("GCEHttpsHealthChecks.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEHttpsHealthChecks.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEHttpsHealthChecks.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEHttpsHealthChecks.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.HttpsHealthChecks.Update(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEHttpsHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEHttpsHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEHttpsHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEHttpsHealthChecks.Update(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -8843,7 +8843,7 @@ type MockInstanceGroups struct { func (m *MockInstanceGroups) Get(ctx context.Context, key *meta.Key) (*ga.InstanceGroup, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockInstanceGroups.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockInstanceGroups.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -8855,12 +8855,12 @@ func (m *MockInstanceGroups) Get(ctx context.Context, key *meta.Key) (*ga.Instan defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockInstanceGroups.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockInstanceGroups.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockInstanceGroups.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockInstanceGroups.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -8868,7 +8868,7 @@ func (m *MockInstanceGroups) Get(ctx context.Context, key *meta.Key) (*ga.Instan Code: http.StatusNotFound, Message: fmt.Sprintf("MockInstanceGroups %v not found", key), } - glog.V(5).Infof("MockInstanceGroups.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockInstanceGroups.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -8876,7 +8876,7 @@ func (m *MockInstanceGroups) Get(ctx context.Context, key *meta.Key) (*ga.Instan func (m *MockInstanceGroups) List(ctx context.Context, zone string, fl *filter.F) ([]*ga.InstanceGroup, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, zone, fl, m); intercept { - glog.V(5).Infof("MockInstanceGroups.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) + klog.V(5).Infof("MockInstanceGroups.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) return objs, err } } @@ -8886,7 +8886,7 @@ func (m *MockInstanceGroups) List(ctx context.Context, zone string, fl *filter.F if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockInstanceGroups.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) + klog.V(5).Infof("MockInstanceGroups.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) return nil, *m.ListError } @@ -8902,7 +8902,7 @@ func (m *MockInstanceGroups) List(ctx context.Context, zone string, fl *filter.F objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockInstanceGroups.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) + klog.V(5).Infof("MockInstanceGroups.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) return objs, nil } @@ -8910,7 +8910,7 @@ func (m *MockInstanceGroups) List(ctx context.Context, zone string, fl *filter.F func (m *MockInstanceGroups) Insert(ctx context.Context, key *meta.Key, obj *ga.InstanceGroup) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockInstanceGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockInstanceGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -8922,7 +8922,7 @@ func (m *MockInstanceGroups) Insert(ctx context.Context, key *meta.Key, obj *ga. defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockInstanceGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockInstanceGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -8930,7 +8930,7 @@ func (m *MockInstanceGroups) Insert(ctx context.Context, key *meta.Key, obj *ga. Code: http.StatusConflict, Message: fmt.Sprintf("MockInstanceGroups %v exists", key), } - glog.V(5).Infof("MockInstanceGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockInstanceGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -8939,7 +8939,7 @@ func (m *MockInstanceGroups) Insert(ctx context.Context, key *meta.Key, obj *ga. obj.SelfLink = SelfLink(meta.VersionGA, projectID, "instanceGroups", key) m.Objects[*key] = &MockInstanceGroupsObj{obj} - glog.V(5).Infof("MockInstanceGroups.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockInstanceGroups.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -8947,7 +8947,7 @@ func (m *MockInstanceGroups) Insert(ctx context.Context, key *meta.Key, obj *ga. func (m *MockInstanceGroups) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockInstanceGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockInstanceGroups.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -8959,7 +8959,7 @@ func (m *MockInstanceGroups) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockInstanceGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockInstanceGroups.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -8967,12 +8967,12 @@ func (m *MockInstanceGroups) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockInstanceGroups %v not found", key), } - glog.V(5).Infof("MockInstanceGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockInstanceGroups.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockInstanceGroups.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockInstanceGroups.Delete(%v, %v) = nil", ctx, key) return nil } @@ -9020,10 +9020,10 @@ type GCEInstanceGroups struct { // Get the InstanceGroup named by key. func (g *GCEInstanceGroups) Get(ctx context.Context, key *meta.Key) (*ga.InstanceGroup, error) { - glog.V(5).Infof("GCEInstanceGroups.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEInstanceGroups.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEInstanceGroups.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEInstanceGroups.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "InstanceGroups") @@ -9033,21 +9033,21 @@ func (g *GCEInstanceGroups) Get(ctx context.Context, key *meta.Key) (*ga.Instanc Version: meta.Version("ga"), Service: "InstanceGroups", } - glog.V(5).Infof("GCEInstanceGroups.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEInstanceGroups.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEInstanceGroups.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.InstanceGroups.Get(projectID, key.Zone, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEInstanceGroups.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEInstanceGroups.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all InstanceGroup objects. func (g *GCEInstanceGroups) List(ctx context.Context, zone string, fl *filter.F) ([]*ga.InstanceGroup, error) { - glog.V(5).Infof("GCEInstanceGroups.List(%v, %v, %v) called", ctx, zone, fl) + klog.V(5).Infof("GCEInstanceGroups.List(%v, %v, %v) called", ctx, zone, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "InstanceGroups") rk := &RateLimitKey{ ProjectID: projectID, @@ -9058,30 +9058,30 @@ func (g *GCEInstanceGroups) List(ctx context.Context, zone string, fl *filter.F) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEInstanceGroups.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) + klog.V(5).Infof("GCEInstanceGroups.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) call := g.s.GA.InstanceGroups.List(projectID, zone) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.InstanceGroup f := func(l *ga.InstanceGroupList) error { - glog.V(5).Infof("GCEInstanceGroups.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEInstanceGroups.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEInstanceGroups.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEInstanceGroups.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEInstanceGroups.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEInstanceGroups.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEInstanceGroups.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEInstanceGroups.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -9089,9 +9089,9 @@ func (g *GCEInstanceGroups) List(ctx context.Context, zone string, fl *filter.F) // Insert InstanceGroup with key of value obj. func (g *GCEInstanceGroups) Insert(ctx context.Context, key *meta.Key, obj *ga.InstanceGroup) error { - glog.V(5).Infof("GCEInstanceGroups.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEInstanceGroups.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEInstanceGroups.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEInstanceGroups.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "InstanceGroups") @@ -9101,9 +9101,9 @@ func (g *GCEInstanceGroups) Insert(ctx context.Context, key *meta.Key, obj *ga.I Version: meta.Version("ga"), Service: "InstanceGroups", } - glog.V(5).Infof("GCEInstanceGroups.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEInstanceGroups.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEInstanceGroups.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -9112,20 +9112,20 @@ func (g *GCEInstanceGroups) Insert(ctx context.Context, key *meta.Key, obj *ga.I op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEInstanceGroups.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEInstanceGroups.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEInstanceGroups.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the InstanceGroup referenced by key. func (g *GCEInstanceGroups) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEInstanceGroups.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEInstanceGroups.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEInstanceGroups.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEInstanceGroups.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "InstanceGroups") @@ -9135,9 +9135,9 @@ func (g *GCEInstanceGroups) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "InstanceGroups", } - glog.V(5).Infof("GCEInstanceGroups.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEInstanceGroups.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEInstanceGroups.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.InstanceGroups.Delete(projectID, key.Zone, key.Name) @@ -9145,21 +9145,21 @@ func (g *GCEInstanceGroups) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEInstanceGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEInstanceGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.Delete(%v, %v) = %v", ctx, key, err) return err } // AddInstances is a method on GCEInstanceGroups. func (g *GCEInstanceGroups) AddInstances(ctx context.Context, key *meta.Key, arg0 *ga.InstanceGroupsAddInstancesRequest) error { - glog.V(5).Infof("GCEInstanceGroups.AddInstances(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEInstanceGroups.AddInstances(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEInstanceGroups.AddInstances(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEInstanceGroups.AddInstances(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "InstanceGroups") @@ -9169,30 +9169,30 @@ func (g *GCEInstanceGroups) AddInstances(ctx context.Context, key *meta.Key, arg Version: meta.Version("ga"), Service: "InstanceGroups", } - glog.V(5).Infof("GCEInstanceGroups.AddInstances(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEInstanceGroups.AddInstances(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEInstanceGroups.AddInstances(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.AddInstances(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.InstanceGroups.AddInstances(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEInstanceGroups.AddInstances(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.AddInstances(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEInstanceGroups.AddInstances(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.AddInstances(%v, %v, ...) = %+v", ctx, key, err) return err } // ListInstances is a method on GCEInstanceGroups. func (g *GCEInstanceGroups) ListInstances(ctx context.Context, key *meta.Key, arg0 *ga.InstanceGroupsListInstancesRequest, fl *filter.F) ([]*ga.InstanceWithNamedPorts, error) { - glog.V(5).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "InstanceGroups") @@ -9202,41 +9202,41 @@ func (g *GCEInstanceGroups) ListInstances(ctx context.Context, key *meta.Key, ar Version: meta.Version("ga"), Service: "InstanceGroups", } - glog.V(5).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.InstanceGroups.ListInstances(projectID, key.Zone, key.Name, arg0) var all []*ga.InstanceWithNamedPorts f := func(l *ga.InstanceGroupsListInstances) error { - glog.V(5).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...): page %+v", ctx, key, l) + klog.V(5).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...): page %+v", ctx, key, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...) = %v, %v", ctx, key, nil, err) + klog.V(4).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...) = %v, %v", ctx, key, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...) = [%v items], %v", ctx, key, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...) = [%v items], %v", ctx, key, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...) = %v, %v", ctx, key, asStr, nil) + klog.V(5).Infof("GCEInstanceGroups.ListInstances(%v, %v, ...) = %v, %v", ctx, key, asStr, nil) } return all, nil } // RemoveInstances is a method on GCEInstanceGroups. func (g *GCEInstanceGroups) RemoveInstances(ctx context.Context, key *meta.Key, arg0 *ga.InstanceGroupsRemoveInstancesRequest) error { - glog.V(5).Infof("GCEInstanceGroups.RemoveInstances(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEInstanceGroups.RemoveInstances(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEInstanceGroups.RemoveInstances(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEInstanceGroups.RemoveInstances(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "InstanceGroups") @@ -9246,30 +9246,30 @@ func (g *GCEInstanceGroups) RemoveInstances(ctx context.Context, key *meta.Key, Version: meta.Version("ga"), Service: "InstanceGroups", } - glog.V(5).Infof("GCEInstanceGroups.RemoveInstances(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEInstanceGroups.RemoveInstances(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEInstanceGroups.RemoveInstances(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.RemoveInstances(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.InstanceGroups.RemoveInstances(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEInstanceGroups.RemoveInstances(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.RemoveInstances(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEInstanceGroups.RemoveInstances(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.RemoveInstances(%v, %v, ...) = %+v", ctx, key, err) return err } // SetNamedPorts is a method on GCEInstanceGroups. func (g *GCEInstanceGroups) SetNamedPorts(ctx context.Context, key *meta.Key, arg0 *ga.InstanceGroupsSetNamedPortsRequest) error { - glog.V(5).Infof("GCEInstanceGroups.SetNamedPorts(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEInstanceGroups.SetNamedPorts(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEInstanceGroups.SetNamedPorts(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEInstanceGroups.SetNamedPorts(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "InstanceGroups") @@ -9279,21 +9279,21 @@ func (g *GCEInstanceGroups) SetNamedPorts(ctx context.Context, key *meta.Key, ar Version: meta.Version("ga"), Service: "InstanceGroups", } - glog.V(5).Infof("GCEInstanceGroups.SetNamedPorts(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEInstanceGroups.SetNamedPorts(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEInstanceGroups.SetNamedPorts(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.SetNamedPorts(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.InstanceGroups.SetNamedPorts(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEInstanceGroups.SetNamedPorts(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.SetNamedPorts(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEInstanceGroups.SetNamedPorts(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEInstanceGroups.SetNamedPorts(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -9356,7 +9356,7 @@ type MockInstances struct { func (m *MockInstances) Get(ctx context.Context, key *meta.Key) (*ga.Instance, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockInstances.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockInstances.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -9368,12 +9368,12 @@ func (m *MockInstances) Get(ctx context.Context, key *meta.Key) (*ga.Instance, e defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockInstances.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockInstances.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockInstances.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockInstances.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -9381,7 +9381,7 @@ func (m *MockInstances) Get(ctx context.Context, key *meta.Key) (*ga.Instance, e Code: http.StatusNotFound, Message: fmt.Sprintf("MockInstances %v not found", key), } - glog.V(5).Infof("MockInstances.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockInstances.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -9389,7 +9389,7 @@ func (m *MockInstances) Get(ctx context.Context, key *meta.Key) (*ga.Instance, e func (m *MockInstances) List(ctx context.Context, zone string, fl *filter.F) ([]*ga.Instance, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, zone, fl, m); intercept { - glog.V(5).Infof("MockInstances.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) + klog.V(5).Infof("MockInstances.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) return objs, err } } @@ -9399,7 +9399,7 @@ func (m *MockInstances) List(ctx context.Context, zone string, fl *filter.F) ([] if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockInstances.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) + klog.V(5).Infof("MockInstances.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) return nil, *m.ListError } @@ -9415,7 +9415,7 @@ func (m *MockInstances) List(ctx context.Context, zone string, fl *filter.F) ([] objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockInstances.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) + klog.V(5).Infof("MockInstances.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) return objs, nil } @@ -9423,7 +9423,7 @@ func (m *MockInstances) List(ctx context.Context, zone string, fl *filter.F) ([] func (m *MockInstances) Insert(ctx context.Context, key *meta.Key, obj *ga.Instance) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -9435,7 +9435,7 @@ func (m *MockInstances) Insert(ctx context.Context, key *meta.Key, obj *ga.Insta defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -9443,7 +9443,7 @@ func (m *MockInstances) Insert(ctx context.Context, key *meta.Key, obj *ga.Insta Code: http.StatusConflict, Message: fmt.Sprintf("MockInstances %v exists", key), } - glog.V(5).Infof("MockInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -9452,7 +9452,7 @@ func (m *MockInstances) Insert(ctx context.Context, key *meta.Key, obj *ga.Insta obj.SelfLink = SelfLink(meta.VersionGA, projectID, "instances", key) m.Objects[*key] = &MockInstancesObj{obj} - glog.V(5).Infof("MockInstances.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockInstances.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -9460,7 +9460,7 @@ func (m *MockInstances) Insert(ctx context.Context, key *meta.Key, obj *ga.Insta func (m *MockInstances) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockInstances.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -9472,7 +9472,7 @@ func (m *MockInstances) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockInstances.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -9480,12 +9480,12 @@ func (m *MockInstances) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockInstances %v not found", key), } - glog.V(5).Infof("MockInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockInstances.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockInstances.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockInstances.Delete(%v, %v) = nil", ctx, key) return nil } @@ -9517,10 +9517,10 @@ type GCEInstances struct { // Get the Instance named by key. func (g *GCEInstances) Get(ctx context.Context, key *meta.Key) (*ga.Instance, error) { - glog.V(5).Infof("GCEInstances.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEInstances.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEInstances.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEInstances.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Instances") @@ -9530,21 +9530,21 @@ func (g *GCEInstances) Get(ctx context.Context, key *meta.Key) (*ga.Instance, er Version: meta.Version("ga"), Service: "Instances", } - glog.V(5).Infof("GCEInstances.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEInstances.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEInstances.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEInstances.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.Instances.Get(projectID, key.Zone, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEInstances.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEInstances.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all Instance objects. func (g *GCEInstances) List(ctx context.Context, zone string, fl *filter.F) ([]*ga.Instance, error) { - glog.V(5).Infof("GCEInstances.List(%v, %v, %v) called", ctx, zone, fl) + klog.V(5).Infof("GCEInstances.List(%v, %v, %v) called", ctx, zone, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Instances") rk := &RateLimitKey{ ProjectID: projectID, @@ -9555,30 +9555,30 @@ func (g *GCEInstances) List(ctx context.Context, zone string, fl *filter.F) ([]* if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEInstances.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) + klog.V(5).Infof("GCEInstances.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) call := g.s.GA.Instances.List(projectID, zone) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.Instance f := func(l *ga.InstanceList) error { - glog.V(5).Infof("GCEInstances.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEInstances.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEInstances.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEInstances.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEInstances.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEInstances.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEInstances.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEInstances.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -9586,9 +9586,9 @@ func (g *GCEInstances) List(ctx context.Context, zone string, fl *filter.F) ([]* // Insert Instance with key of value obj. func (g *GCEInstances) Insert(ctx context.Context, key *meta.Key, obj *ga.Instance) error { - glog.V(5).Infof("GCEInstances.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEInstances.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEInstances.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEInstances.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Instances") @@ -9598,9 +9598,9 @@ func (g *GCEInstances) Insert(ctx context.Context, key *meta.Key, obj *ga.Instan Version: meta.Version("ga"), Service: "Instances", } - glog.V(5).Infof("GCEInstances.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEInstances.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEInstances.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEInstances.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -9609,20 +9609,20 @@ func (g *GCEInstances) Insert(ctx context.Context, key *meta.Key, obj *ga.Instan op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEInstances.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEInstances.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEInstances.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEInstances.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the Instance referenced by key. func (g *GCEInstances) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEInstances.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEInstances.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEInstances.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEInstances.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Instances") @@ -9632,9 +9632,9 @@ func (g *GCEInstances) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "Instances", } - glog.V(5).Infof("GCEInstances.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEInstances.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEInstances.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEInstances.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.Instances.Delete(projectID, key.Zone, key.Name) @@ -9642,21 +9642,21 @@ func (g *GCEInstances) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEInstances.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEInstances.Delete(%v, %v) = %v", ctx, key, err) return err } // AttachDisk is a method on GCEInstances. func (g *GCEInstances) AttachDisk(ctx context.Context, key *meta.Key, arg0 *ga.AttachedDisk) error { - glog.V(5).Infof("GCEInstances.AttachDisk(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEInstances.AttachDisk(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEInstances.AttachDisk(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEInstances.AttachDisk(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Instances") @@ -9666,30 +9666,30 @@ func (g *GCEInstances) AttachDisk(ctx context.Context, key *meta.Key, arg0 *ga.A Version: meta.Version("ga"), Service: "Instances", } - glog.V(5).Infof("GCEInstances.AttachDisk(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEInstances.AttachDisk(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEInstances.AttachDisk(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEInstances.AttachDisk(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.Instances.AttachDisk(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEInstances.AttachDisk(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEInstances.AttachDisk(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEInstances.AttachDisk(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEInstances.AttachDisk(%v, %v, ...) = %+v", ctx, key, err) return err } // DetachDisk is a method on GCEInstances. func (g *GCEInstances) DetachDisk(ctx context.Context, key *meta.Key, arg0 string) error { - glog.V(5).Infof("GCEInstances.DetachDisk(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEInstances.DetachDisk(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEInstances.DetachDisk(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEInstances.DetachDisk(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Instances") @@ -9699,21 +9699,21 @@ func (g *GCEInstances) DetachDisk(ctx context.Context, key *meta.Key, arg0 strin Version: meta.Version("ga"), Service: "Instances", } - glog.V(5).Infof("GCEInstances.DetachDisk(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEInstances.DetachDisk(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEInstances.DetachDisk(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEInstances.DetachDisk(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.Instances.DetachDisk(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEInstances.DetachDisk(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEInstances.DetachDisk(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEInstances.DetachDisk(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEInstances.DetachDisk(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -9778,7 +9778,7 @@ type MockBetaInstances struct { func (m *MockBetaInstances) Get(ctx context.Context, key *meta.Key) (*beta.Instance, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaInstances.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaInstances.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -9790,12 +9790,12 @@ func (m *MockBetaInstances) Get(ctx context.Context, key *meta.Key) (*beta.Insta defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockBetaInstances.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBetaInstances.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToBeta() - glog.V(5).Infof("MockBetaInstances.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockBetaInstances.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -9803,7 +9803,7 @@ func (m *MockBetaInstances) Get(ctx context.Context, key *meta.Key) (*beta.Insta Code: http.StatusNotFound, Message: fmt.Sprintf("MockBetaInstances %v not found", key), } - glog.V(5).Infof("MockBetaInstances.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBetaInstances.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -9811,7 +9811,7 @@ func (m *MockBetaInstances) Get(ctx context.Context, key *meta.Key) (*beta.Insta func (m *MockBetaInstances) List(ctx context.Context, zone string, fl *filter.F) ([]*beta.Instance, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, zone, fl, m); intercept { - glog.V(5).Infof("MockBetaInstances.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) + klog.V(5).Infof("MockBetaInstances.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) return objs, err } } @@ -9821,7 +9821,7 @@ func (m *MockBetaInstances) List(ctx context.Context, zone string, fl *filter.F) if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockBetaInstances.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) + klog.V(5).Infof("MockBetaInstances.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) return nil, *m.ListError } @@ -9837,7 +9837,7 @@ func (m *MockBetaInstances) List(ctx context.Context, zone string, fl *filter.F) objs = append(objs, obj.ToBeta()) } - glog.V(5).Infof("MockBetaInstances.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) + klog.V(5).Infof("MockBetaInstances.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) return objs, nil } @@ -9845,7 +9845,7 @@ func (m *MockBetaInstances) List(ctx context.Context, zone string, fl *filter.F) func (m *MockBetaInstances) Insert(ctx context.Context, key *meta.Key, obj *beta.Instance) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockBetaInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -9857,7 +9857,7 @@ func (m *MockBetaInstances) Insert(ctx context.Context, key *meta.Key, obj *beta defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockBetaInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -9865,7 +9865,7 @@ func (m *MockBetaInstances) Insert(ctx context.Context, key *meta.Key, obj *beta Code: http.StatusConflict, Message: fmt.Sprintf("MockBetaInstances %v exists", key), } - glog.V(5).Infof("MockBetaInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -9874,7 +9874,7 @@ func (m *MockBetaInstances) Insert(ctx context.Context, key *meta.Key, obj *beta obj.SelfLink = SelfLink(meta.VersionBeta, projectID, "instances", key) m.Objects[*key] = &MockInstancesObj{obj} - glog.V(5).Infof("MockBetaInstances.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockBetaInstances.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -9882,7 +9882,7 @@ func (m *MockBetaInstances) Insert(ctx context.Context, key *meta.Key, obj *beta func (m *MockBetaInstances) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaInstances.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -9894,7 +9894,7 @@ func (m *MockBetaInstances) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockBetaInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaInstances.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -9902,12 +9902,12 @@ func (m *MockBetaInstances) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockBetaInstances %v not found", key), } - glog.V(5).Infof("MockBetaInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaInstances.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockBetaInstances.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockBetaInstances.Delete(%v, %v) = nil", ctx, key) return nil } @@ -9947,10 +9947,10 @@ type GCEBetaInstances struct { // Get the Instance named by key. func (g *GCEBetaInstances) Get(ctx context.Context, key *meta.Key) (*beta.Instance, error) { - glog.V(5).Infof("GCEBetaInstances.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBetaInstances.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaInstances.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaInstances.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "Instances") @@ -9960,21 +9960,21 @@ func (g *GCEBetaInstances) Get(ctx context.Context, key *meta.Key) (*beta.Instan Version: meta.Version("beta"), Service: "Instances", } - glog.V(5).Infof("GCEBetaInstances.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaInstances.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaInstances.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Beta.Instances.Get(projectID, key.Zone, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEBetaInstances.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEBetaInstances.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all Instance objects. func (g *GCEBetaInstances) List(ctx context.Context, zone string, fl *filter.F) ([]*beta.Instance, error) { - glog.V(5).Infof("GCEBetaInstances.List(%v, %v, %v) called", ctx, zone, fl) + klog.V(5).Infof("GCEBetaInstances.List(%v, %v, %v) called", ctx, zone, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "Instances") rk := &RateLimitKey{ ProjectID: projectID, @@ -9985,30 +9985,30 @@ func (g *GCEBetaInstances) List(ctx context.Context, zone string, fl *filter.F) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEBetaInstances.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) + klog.V(5).Infof("GCEBetaInstances.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) call := g.s.Beta.Instances.List(projectID, zone) if fl != filter.None { call.Filter(fl.String()) } var all []*beta.Instance f := func(l *beta.InstanceList) error { - glog.V(5).Infof("GCEBetaInstances.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEBetaInstances.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEBetaInstances.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEBetaInstances.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEBetaInstances.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEBetaInstances.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEBetaInstances.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEBetaInstances.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -10016,9 +10016,9 @@ func (g *GCEBetaInstances) List(ctx context.Context, zone string, fl *filter.F) // Insert Instance with key of value obj. func (g *GCEBetaInstances) Insert(ctx context.Context, key *meta.Key, obj *beta.Instance) error { - glog.V(5).Infof("GCEBetaInstances.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEBetaInstances.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEBetaInstances.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaInstances.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "Instances") @@ -10028,9 +10028,9 @@ func (g *GCEBetaInstances) Insert(ctx context.Context, key *meta.Key, obj *beta. Version: meta.Version("beta"), Service: "Instances", } - glog.V(5).Infof("GCEBetaInstances.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaInstances.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaInstances.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -10039,20 +10039,20 @@ func (g *GCEBetaInstances) Insert(ctx context.Context, key *meta.Key, obj *beta. op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaInstances.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaInstances.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEBetaInstances.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the Instance referenced by key. func (g *GCEBetaInstances) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEBetaInstances.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBetaInstances.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaInstances.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaInstances.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "Instances") @@ -10062,9 +10062,9 @@ func (g *GCEBetaInstances) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("beta"), Service: "Instances", } - glog.V(5).Infof("GCEBetaInstances.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaInstances.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaInstances.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.Instances.Delete(projectID, key.Zone, key.Name) @@ -10072,21 +10072,21 @@ func (g *GCEBetaInstances) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.Delete(%v, %v) = %v", ctx, key, err) return err } // AttachDisk is a method on GCEBetaInstances. func (g *GCEBetaInstances) AttachDisk(ctx context.Context, key *meta.Key, arg0 *beta.AttachedDisk) error { - glog.V(5).Infof("GCEBetaInstances.AttachDisk(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaInstances.AttachDisk(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaInstances.AttachDisk(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaInstances.AttachDisk(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "Instances") @@ -10096,30 +10096,30 @@ func (g *GCEBetaInstances) AttachDisk(ctx context.Context, key *meta.Key, arg0 * Version: meta.Version("beta"), Service: "Instances", } - glog.V(5).Infof("GCEBetaInstances.AttachDisk(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaInstances.AttachDisk(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaInstances.AttachDisk(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.AttachDisk(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.Instances.AttachDisk(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaInstances.AttachDisk(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.AttachDisk(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaInstances.AttachDisk(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.AttachDisk(%v, %v, ...) = %+v", ctx, key, err) return err } // DetachDisk is a method on GCEBetaInstances. func (g *GCEBetaInstances) DetachDisk(ctx context.Context, key *meta.Key, arg0 string) error { - glog.V(5).Infof("GCEBetaInstances.DetachDisk(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaInstances.DetachDisk(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaInstances.DetachDisk(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaInstances.DetachDisk(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "Instances") @@ -10129,30 +10129,30 @@ func (g *GCEBetaInstances) DetachDisk(ctx context.Context, key *meta.Key, arg0 s Version: meta.Version("beta"), Service: "Instances", } - glog.V(5).Infof("GCEBetaInstances.DetachDisk(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaInstances.DetachDisk(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaInstances.DetachDisk(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.DetachDisk(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.Instances.DetachDisk(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaInstances.DetachDisk(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.DetachDisk(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaInstances.DetachDisk(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.DetachDisk(%v, %v, ...) = %+v", ctx, key, err) return err } // UpdateNetworkInterface is a method on GCEBetaInstances. func (g *GCEBetaInstances) UpdateNetworkInterface(ctx context.Context, key *meta.Key, arg0 string, arg1 *beta.NetworkInterface) error { - glog.V(5).Infof("GCEBetaInstances.UpdateNetworkInterface(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaInstances.UpdateNetworkInterface(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaInstances.UpdateNetworkInterface(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaInstances.UpdateNetworkInterface(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "Instances") @@ -10162,21 +10162,21 @@ func (g *GCEBetaInstances) UpdateNetworkInterface(ctx context.Context, key *meta Version: meta.Version("beta"), Service: "Instances", } - glog.V(5).Infof("GCEBetaInstances.UpdateNetworkInterface(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaInstances.UpdateNetworkInterface(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaInstances.UpdateNetworkInterface(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.UpdateNetworkInterface(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.Instances.UpdateNetworkInterface(projectID, key.Zone, key.Name, arg0, arg1) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaInstances.UpdateNetworkInterface(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.UpdateNetworkInterface(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaInstances.UpdateNetworkInterface(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaInstances.UpdateNetworkInterface(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -10241,7 +10241,7 @@ type MockAlphaInstances struct { func (m *MockAlphaInstances) Get(ctx context.Context, key *meta.Key) (*alpha.Instance, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaInstances.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaInstances.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -10253,12 +10253,12 @@ func (m *MockAlphaInstances) Get(ctx context.Context, key *meta.Key) (*alpha.Ins defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockAlphaInstances.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaInstances.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToAlpha() - glog.V(5).Infof("MockAlphaInstances.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockAlphaInstances.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -10266,7 +10266,7 @@ func (m *MockAlphaInstances) Get(ctx context.Context, key *meta.Key) (*alpha.Ins Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaInstances %v not found", key), } - glog.V(5).Infof("MockAlphaInstances.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaInstances.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -10274,7 +10274,7 @@ func (m *MockAlphaInstances) Get(ctx context.Context, key *meta.Key) (*alpha.Ins func (m *MockAlphaInstances) List(ctx context.Context, zone string, fl *filter.F) ([]*alpha.Instance, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, zone, fl, m); intercept { - glog.V(5).Infof("MockAlphaInstances.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) + klog.V(5).Infof("MockAlphaInstances.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) return objs, err } } @@ -10284,7 +10284,7 @@ func (m *MockAlphaInstances) List(ctx context.Context, zone string, fl *filter.F if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockAlphaInstances.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) + klog.V(5).Infof("MockAlphaInstances.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) return nil, *m.ListError } @@ -10300,7 +10300,7 @@ func (m *MockAlphaInstances) List(ctx context.Context, zone string, fl *filter.F objs = append(objs, obj.ToAlpha()) } - glog.V(5).Infof("MockAlphaInstances.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) + klog.V(5).Infof("MockAlphaInstances.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) return objs, nil } @@ -10308,7 +10308,7 @@ func (m *MockAlphaInstances) List(ctx context.Context, zone string, fl *filter.F func (m *MockAlphaInstances) Insert(ctx context.Context, key *meta.Key, obj *alpha.Instance) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockAlphaInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -10320,7 +10320,7 @@ func (m *MockAlphaInstances) Insert(ctx context.Context, key *meta.Key, obj *alp defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockAlphaInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -10328,7 +10328,7 @@ func (m *MockAlphaInstances) Insert(ctx context.Context, key *meta.Key, obj *alp Code: http.StatusConflict, Message: fmt.Sprintf("MockAlphaInstances %v exists", key), } - glog.V(5).Infof("MockAlphaInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaInstances.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -10337,7 +10337,7 @@ func (m *MockAlphaInstances) Insert(ctx context.Context, key *meta.Key, obj *alp obj.SelfLink = SelfLink(meta.VersionAlpha, projectID, "instances", key) m.Objects[*key] = &MockInstancesObj{obj} - glog.V(5).Infof("MockAlphaInstances.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockAlphaInstances.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -10345,7 +10345,7 @@ func (m *MockAlphaInstances) Insert(ctx context.Context, key *meta.Key, obj *alp func (m *MockAlphaInstances) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaInstances.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -10357,7 +10357,7 @@ func (m *MockAlphaInstances) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockAlphaInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaInstances.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -10365,12 +10365,12 @@ func (m *MockAlphaInstances) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaInstances %v not found", key), } - glog.V(5).Infof("MockAlphaInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaInstances.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockAlphaInstances.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockAlphaInstances.Delete(%v, %v) = nil", ctx, key) return nil } @@ -10410,10 +10410,10 @@ type GCEAlphaInstances struct { // Get the Instance named by key. func (g *GCEAlphaInstances) Get(ctx context.Context, key *meta.Key) (*alpha.Instance, error) { - glog.V(5).Infof("GCEAlphaInstances.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaInstances.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaInstances.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaInstances.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "Instances") @@ -10423,21 +10423,21 @@ func (g *GCEAlphaInstances) Get(ctx context.Context, key *meta.Key) (*alpha.Inst Version: meta.Version("alpha"), Service: "Instances", } - glog.V(5).Infof("GCEAlphaInstances.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaInstances.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaInstances.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Alpha.Instances.Get(projectID, key.Zone, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEAlphaInstances.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEAlphaInstances.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all Instance objects. func (g *GCEAlphaInstances) List(ctx context.Context, zone string, fl *filter.F) ([]*alpha.Instance, error) { - glog.V(5).Infof("GCEAlphaInstances.List(%v, %v, %v) called", ctx, zone, fl) + klog.V(5).Infof("GCEAlphaInstances.List(%v, %v, %v) called", ctx, zone, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "Instances") rk := &RateLimitKey{ ProjectID: projectID, @@ -10448,30 +10448,30 @@ func (g *GCEAlphaInstances) List(ctx context.Context, zone string, fl *filter.F) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEAlphaInstances.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) + klog.V(5).Infof("GCEAlphaInstances.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) call := g.s.Alpha.Instances.List(projectID, zone) if fl != filter.None { call.Filter(fl.String()) } var all []*alpha.Instance f := func(l *alpha.InstanceList) error { - glog.V(5).Infof("GCEAlphaInstances.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEAlphaInstances.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEAlphaInstances.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEAlphaInstances.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEAlphaInstances.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEAlphaInstances.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEAlphaInstances.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEAlphaInstances.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -10479,9 +10479,9 @@ func (g *GCEAlphaInstances) List(ctx context.Context, zone string, fl *filter.F) // Insert Instance with key of value obj. func (g *GCEAlphaInstances) Insert(ctx context.Context, key *meta.Key, obj *alpha.Instance) error { - glog.V(5).Infof("GCEAlphaInstances.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEAlphaInstances.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEAlphaInstances.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaInstances.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "Instances") @@ -10491,9 +10491,9 @@ func (g *GCEAlphaInstances) Insert(ctx context.Context, key *meta.Key, obj *alph Version: meta.Version("alpha"), Service: "Instances", } - glog.V(5).Infof("GCEAlphaInstances.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaInstances.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaInstances.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -10502,20 +10502,20 @@ func (g *GCEAlphaInstances) Insert(ctx context.Context, key *meta.Key, obj *alph op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaInstances.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaInstances.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEAlphaInstances.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the Instance referenced by key. func (g *GCEAlphaInstances) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEAlphaInstances.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaInstances.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaInstances.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaInstances.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "Instances") @@ -10525,9 +10525,9 @@ func (g *GCEAlphaInstances) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("alpha"), Service: "Instances", } - glog.V(5).Infof("GCEAlphaInstances.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaInstances.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaInstances.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.Instances.Delete(projectID, key.Zone, key.Name) @@ -10535,21 +10535,21 @@ func (g *GCEAlphaInstances) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaInstances.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.Delete(%v, %v) = %v", ctx, key, err) return err } // AttachDisk is a method on GCEAlphaInstances. func (g *GCEAlphaInstances) AttachDisk(ctx context.Context, key *meta.Key, arg0 *alpha.AttachedDisk) error { - glog.V(5).Infof("GCEAlphaInstances.AttachDisk(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEAlphaInstances.AttachDisk(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaInstances.AttachDisk(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaInstances.AttachDisk(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "Instances") @@ -10559,30 +10559,30 @@ func (g *GCEAlphaInstances) AttachDisk(ctx context.Context, key *meta.Key, arg0 Version: meta.Version("alpha"), Service: "Instances", } - glog.V(5).Infof("GCEAlphaInstances.AttachDisk(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaInstances.AttachDisk(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaInstances.AttachDisk(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.AttachDisk(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.Instances.AttachDisk(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaInstances.AttachDisk(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.AttachDisk(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaInstances.AttachDisk(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.AttachDisk(%v, %v, ...) = %+v", ctx, key, err) return err } // DetachDisk is a method on GCEAlphaInstances. func (g *GCEAlphaInstances) DetachDisk(ctx context.Context, key *meta.Key, arg0 string) error { - glog.V(5).Infof("GCEAlphaInstances.DetachDisk(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEAlphaInstances.DetachDisk(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaInstances.DetachDisk(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaInstances.DetachDisk(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "Instances") @@ -10592,30 +10592,30 @@ func (g *GCEAlphaInstances) DetachDisk(ctx context.Context, key *meta.Key, arg0 Version: meta.Version("alpha"), Service: "Instances", } - glog.V(5).Infof("GCEAlphaInstances.DetachDisk(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaInstances.DetachDisk(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaInstances.DetachDisk(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.DetachDisk(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.Instances.DetachDisk(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaInstances.DetachDisk(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.DetachDisk(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaInstances.DetachDisk(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.DetachDisk(%v, %v, ...) = %+v", ctx, key, err) return err } // UpdateNetworkInterface is a method on GCEAlphaInstances. func (g *GCEAlphaInstances) UpdateNetworkInterface(ctx context.Context, key *meta.Key, arg0 string, arg1 *alpha.NetworkInterface) error { - glog.V(5).Infof("GCEAlphaInstances.UpdateNetworkInterface(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEAlphaInstances.UpdateNetworkInterface(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaInstances.UpdateNetworkInterface(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaInstances.UpdateNetworkInterface(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "Instances") @@ -10625,21 +10625,21 @@ func (g *GCEAlphaInstances) UpdateNetworkInterface(ctx context.Context, key *met Version: meta.Version("alpha"), Service: "Instances", } - glog.V(5).Infof("GCEAlphaInstances.UpdateNetworkInterface(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaInstances.UpdateNetworkInterface(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaInstances.UpdateNetworkInterface(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.UpdateNetworkInterface(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.Instances.UpdateNetworkInterface(projectID, key.Zone, key.Name, arg0, arg1) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaInstances.UpdateNetworkInterface(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.UpdateNetworkInterface(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaInstances.UpdateNetworkInterface(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaInstances.UpdateNetworkInterface(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -10707,7 +10707,7 @@ type MockAlphaNetworkEndpointGroups struct { func (m *MockAlphaNetworkEndpointGroups) Get(ctx context.Context, key *meta.Key) (*alpha.NetworkEndpointGroup, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -10719,12 +10719,12 @@ func (m *MockAlphaNetworkEndpointGroups) Get(ctx context.Context, key *meta.Key) defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToAlpha() - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -10732,7 +10732,7 @@ func (m *MockAlphaNetworkEndpointGroups) Get(ctx context.Context, key *meta.Key) Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaNetworkEndpointGroups %v not found", key), } - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -10740,7 +10740,7 @@ func (m *MockAlphaNetworkEndpointGroups) Get(ctx context.Context, key *meta.Key) func (m *MockAlphaNetworkEndpointGroups) List(ctx context.Context, zone string, fl *filter.F) ([]*alpha.NetworkEndpointGroup, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, zone, fl, m); intercept { - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) return objs, err } } @@ -10750,7 +10750,7 @@ func (m *MockAlphaNetworkEndpointGroups) List(ctx context.Context, zone string, if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) return nil, *m.ListError } @@ -10766,7 +10766,7 @@ func (m *MockAlphaNetworkEndpointGroups) List(ctx context.Context, zone string, objs = append(objs, obj.ToAlpha()) } - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) return objs, nil } @@ -10774,7 +10774,7 @@ func (m *MockAlphaNetworkEndpointGroups) List(ctx context.Context, zone string, func (m *MockAlphaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.Key, obj *alpha.NetworkEndpointGroup) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -10786,7 +10786,7 @@ func (m *MockAlphaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.K defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -10794,7 +10794,7 @@ func (m *MockAlphaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.K Code: http.StatusConflict, Message: fmt.Sprintf("MockAlphaNetworkEndpointGroups %v exists", key), } - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -10803,7 +10803,7 @@ func (m *MockAlphaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.K obj.SelfLink = SelfLink(meta.VersionAlpha, projectID, "networkEndpointGroups", key) m.Objects[*key] = &MockNetworkEndpointGroupsObj{obj} - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -10811,7 +10811,7 @@ func (m *MockAlphaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.K func (m *MockAlphaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -10823,7 +10823,7 @@ func (m *MockAlphaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.K defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -10831,12 +10831,12 @@ func (m *MockAlphaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.K Code: http.StatusNotFound, Message: fmt.Sprintf("MockAlphaNetworkEndpointGroups %v not found", key), } - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.Delete(%v, %v) = nil", ctx, key) return nil } @@ -10844,7 +10844,7 @@ func (m *MockAlphaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.K func (m *MockAlphaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl *filter.F) (map[string][]*alpha.NetworkEndpointGroup, error) { if m.AggregatedListHook != nil { if intercept, objs, err := m.AggregatedListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -10854,7 +10854,7 @@ func (m *MockAlphaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl if m.AggregatedListError != nil { err := *m.AggregatedListError - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = nil, %v", ctx, fl, err) return nil, err } @@ -10863,7 +10863,7 @@ func (m *MockAlphaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl res, err := ParseResourceURL(obj.ToAlpha().SelfLink) location := res.Key.Zone if err != nil { - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = nil, %v", ctx, fl, err) return nil, err } if !fl.Match(obj.ToAlpha()) { @@ -10871,7 +10871,7 @@ func (m *MockAlphaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl } objs[location] = append(objs[location], obj.ToAlpha()) } - glog.V(5).Infof("MockAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -10911,10 +10911,10 @@ type GCEAlphaNetworkEndpointGroups struct { // Get the NetworkEndpointGroup named by key. func (g *GCEAlphaNetworkEndpointGroups) Get(ctx context.Context, key *meta.Key) (*alpha.NetworkEndpointGroup, error) { - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaNetworkEndpointGroups.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaNetworkEndpointGroups.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "NetworkEndpointGroups") @@ -10924,21 +10924,21 @@ func (g *GCEAlphaNetworkEndpointGroups) Get(ctx context.Context, key *meta.Key) Version: meta.Version("alpha"), Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Alpha.NetworkEndpointGroups.Get(projectID, key.Zone, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all NetworkEndpointGroup objects. func (g *GCEAlphaNetworkEndpointGroups) List(ctx context.Context, zone string, fl *filter.F) ([]*alpha.NetworkEndpointGroup, error) { - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.List(%v, %v, %v) called", ctx, zone, fl) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.List(%v, %v, %v) called", ctx, zone, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "NetworkEndpointGroups") rk := &RateLimitKey{ ProjectID: projectID, @@ -10949,30 +10949,30 @@ func (g *GCEAlphaNetworkEndpointGroups) List(ctx context.Context, zone string, f if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) call := g.s.Alpha.NetworkEndpointGroups.List(projectID, zone) if fl != filter.None { call.Filter(fl.String()) } var all []*alpha.NetworkEndpointGroup f := func(l *alpha.NetworkEndpointGroupList) error { - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -10980,9 +10980,9 @@ func (g *GCEAlphaNetworkEndpointGroups) List(ctx context.Context, zone string, f // Insert NetworkEndpointGroup with key of value obj. func (g *GCEAlphaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.Key, obj *alpha.NetworkEndpointGroup) error { - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEAlphaNetworkEndpointGroups.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaNetworkEndpointGroups.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "NetworkEndpointGroups") @@ -10992,9 +10992,9 @@ func (g *GCEAlphaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.Ke Version: meta.Version("alpha"), Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -11003,20 +11003,20 @@ func (g *GCEAlphaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.Ke op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the NetworkEndpointGroup referenced by key. func (g *GCEAlphaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaNetworkEndpointGroups.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaNetworkEndpointGroups.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "NetworkEndpointGroups") @@ -11026,9 +11026,9 @@ func (g *GCEAlphaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.Ke Version: meta.Version("alpha"), Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.NetworkEndpointGroups.Delete(projectID, key.Zone, key.Name) @@ -11036,18 +11036,18 @@ func (g *GCEAlphaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.Ke op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) return err } // AggregatedList lists all resources of the given type across all locations. func (g *GCEAlphaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl *filter.F) (map[string][]*alpha.NetworkEndpointGroup, error) { - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "NetworkEndpointGroups") rk := &RateLimitKey{ @@ -11057,9 +11057,9 @@ func (g *GCEAlphaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl * Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v): RateLimiter error: %v", ctx, fl, err) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v): RateLimiter error: %v", ctx, fl, err) return nil, err } @@ -11072,33 +11072,33 @@ func (g *GCEAlphaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl * all := map[string][]*alpha.NetworkEndpointGroup{} f := func(l *alpha.NetworkEndpointGroupAggregatedList) error { for k, v := range l.Items { - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v): page[%v]%+v", ctx, fl, k, v) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v): page[%v]%+v", ctx, fl, k, v) all[k] = append(all[k], v.NetworkEndpointGroups...) } return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AggregatedList(%v, %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil } // AttachNetworkEndpoints is a method on GCEAlphaNetworkEndpointGroups. func (g *GCEAlphaNetworkEndpointGroups) AttachNetworkEndpoints(ctx context.Context, key *meta.Key, arg0 *alpha.NetworkEndpointGroupsAttachEndpointsRequest) error { - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "NetworkEndpointGroups") @@ -11108,30 +11108,30 @@ func (g *GCEAlphaNetworkEndpointGroups) AttachNetworkEndpoints(ctx context.Conte Version: meta.Version("alpha"), Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.NetworkEndpointGroups.AttachNetworkEndpoints(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) return err } // DetachNetworkEndpoints is a method on GCEAlphaNetworkEndpointGroups. func (g *GCEAlphaNetworkEndpointGroups) DetachNetworkEndpoints(ctx context.Context, key *meta.Key, arg0 *alpha.NetworkEndpointGroupsDetachEndpointsRequest) error { - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "NetworkEndpointGroups") @@ -11141,30 +11141,30 @@ func (g *GCEAlphaNetworkEndpointGroups) DetachNetworkEndpoints(ctx context.Conte Version: meta.Version("alpha"), Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Alpha.NetworkEndpointGroups.DetachNetworkEndpoints(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) return err } // ListNetworkEndpoints is a method on GCEAlphaNetworkEndpointGroups. func (g *GCEAlphaNetworkEndpointGroups) ListNetworkEndpoints(ctx context.Context, key *meta.Key, arg0 *alpha.NetworkEndpointGroupsListEndpointsRequest, fl *filter.F) ([]*alpha.NetworkEndpointWithHealthStatus, error) { - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "alpha", "NetworkEndpointGroups") @@ -11174,31 +11174,31 @@ func (g *GCEAlphaNetworkEndpointGroups) ListNetworkEndpoints(ctx context.Context Version: meta.Version("alpha"), Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Alpha.NetworkEndpointGroups.ListNetworkEndpoints(projectID, key.Zone, key.Name, arg0) var all []*alpha.NetworkEndpointWithHealthStatus f := func(l *alpha.NetworkEndpointGroupsListNetworkEndpoints) error { - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): page %+v", ctx, key, l) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): page %+v", ctx, key, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...) = %v, %v", ctx, key, nil, err) + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...) = %v, %v", ctx, key, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...) = [%v items], %v", ctx, key, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...) = [%v items], %v", ctx, key, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...) = %v, %v", ctx, key, asStr, nil) + klog.V(5).Infof("GCEAlphaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...) = %v, %v", ctx, key, asStr, nil) } return all, nil } @@ -11267,7 +11267,7 @@ type MockBetaNetworkEndpointGroups struct { func (m *MockBetaNetworkEndpointGroups) Get(ctx context.Context, key *meta.Key) (*beta.NetworkEndpointGroup, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaNetworkEndpointGroups.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -11279,12 +11279,12 @@ func (m *MockBetaNetworkEndpointGroups) Get(ctx context.Context, key *meta.Key) defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockBetaNetworkEndpointGroups.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToBeta() - glog.V(5).Infof("MockBetaNetworkEndpointGroups.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -11292,7 +11292,7 @@ func (m *MockBetaNetworkEndpointGroups) Get(ctx context.Context, key *meta.Key) Code: http.StatusNotFound, Message: fmt.Sprintf("MockBetaNetworkEndpointGroups %v not found", key), } - glog.V(5).Infof("MockBetaNetworkEndpointGroups.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -11300,7 +11300,7 @@ func (m *MockBetaNetworkEndpointGroups) Get(ctx context.Context, key *meta.Key) func (m *MockBetaNetworkEndpointGroups) List(ctx context.Context, zone string, fl *filter.F) ([]*beta.NetworkEndpointGroup, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, zone, fl, m); intercept { - glog.V(5).Infof("MockBetaNetworkEndpointGroups.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) return objs, err } } @@ -11310,7 +11310,7 @@ func (m *MockBetaNetworkEndpointGroups) List(ctx context.Context, zone string, f if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockBetaNetworkEndpointGroups.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) return nil, *m.ListError } @@ -11326,7 +11326,7 @@ func (m *MockBetaNetworkEndpointGroups) List(ctx context.Context, zone string, f objs = append(objs, obj.ToBeta()) } - glog.V(5).Infof("MockBetaNetworkEndpointGroups.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) return objs, nil } @@ -11334,7 +11334,7 @@ func (m *MockBetaNetworkEndpointGroups) List(ctx context.Context, zone string, f func (m *MockBetaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.Key, obj *beta.NetworkEndpointGroup) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockBetaNetworkEndpointGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -11346,7 +11346,7 @@ func (m *MockBetaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.Ke defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockBetaNetworkEndpointGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -11354,7 +11354,7 @@ func (m *MockBetaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.Ke Code: http.StatusConflict, Message: fmt.Sprintf("MockBetaNetworkEndpointGroups %v exists", key), } - glog.V(5).Infof("MockBetaNetworkEndpointGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -11363,7 +11363,7 @@ func (m *MockBetaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.Ke obj.SelfLink = SelfLink(meta.VersionBeta, projectID, "networkEndpointGroups", key) m.Objects[*key] = &MockNetworkEndpointGroupsObj{obj} - glog.V(5).Infof("MockBetaNetworkEndpointGroups.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -11371,7 +11371,7 @@ func (m *MockBetaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.Ke func (m *MockBetaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -11383,7 +11383,7 @@ func (m *MockBetaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.Ke defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockBetaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -11391,12 +11391,12 @@ func (m *MockBetaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.Ke Code: http.StatusNotFound, Message: fmt.Sprintf("MockBetaNetworkEndpointGroups %v not found", key), } - glog.V(5).Infof("MockBetaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockBetaNetworkEndpointGroups.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.Delete(%v, %v) = nil", ctx, key) return nil } @@ -11404,7 +11404,7 @@ func (m *MockBetaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.Ke func (m *MockBetaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl *filter.F) (map[string][]*beta.NetworkEndpointGroup, error) { if m.AggregatedListHook != nil { if intercept, objs, err := m.AggregatedListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockBetaNetworkEndpointGroups.AggregatedList(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.AggregatedList(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -11414,7 +11414,7 @@ func (m *MockBetaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl * if m.AggregatedListError != nil { err := *m.AggregatedListError - glog.V(5).Infof("MockBetaNetworkEndpointGroups.AggregatedList(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.AggregatedList(%v, %v) = nil, %v", ctx, fl, err) return nil, err } @@ -11423,7 +11423,7 @@ func (m *MockBetaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl * res, err := ParseResourceURL(obj.ToBeta().SelfLink) location := res.Key.Zone if err != nil { - glog.V(5).Infof("MockBetaNetworkEndpointGroups.AggregatedList(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.AggregatedList(%v, %v) = nil, %v", ctx, fl, err) return nil, err } if !fl.Match(obj.ToBeta()) { @@ -11431,7 +11431,7 @@ func (m *MockBetaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl * } objs[location] = append(objs[location], obj.ToBeta()) } - glog.V(5).Infof("MockBetaNetworkEndpointGroups.AggregatedList(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockBetaNetworkEndpointGroups.AggregatedList(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -11471,10 +11471,10 @@ type GCEBetaNetworkEndpointGroups struct { // Get the NetworkEndpointGroup named by key. func (g *GCEBetaNetworkEndpointGroups) Get(ctx context.Context, key *meta.Key) (*beta.NetworkEndpointGroup, error) { - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaNetworkEndpointGroups.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaNetworkEndpointGroups.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "NetworkEndpointGroups") @@ -11484,21 +11484,21 @@ func (g *GCEBetaNetworkEndpointGroups) Get(ctx context.Context, key *meta.Key) ( Version: meta.Version("beta"), Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Beta.NetworkEndpointGroups.Get(projectID, key.Zone, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all NetworkEndpointGroup objects. func (g *GCEBetaNetworkEndpointGroups) List(ctx context.Context, zone string, fl *filter.F) ([]*beta.NetworkEndpointGroup, error) { - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.List(%v, %v, %v) called", ctx, zone, fl) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.List(%v, %v, %v) called", ctx, zone, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "NetworkEndpointGroups") rk := &RateLimitKey{ ProjectID: projectID, @@ -11509,30 +11509,30 @@ func (g *GCEBetaNetworkEndpointGroups) List(ctx context.Context, zone string, fl if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) call := g.s.Beta.NetworkEndpointGroups.List(projectID, zone) if fl != filter.None { call.Filter(fl.String()) } var all []*beta.NetworkEndpointGroup f := func(l *beta.NetworkEndpointGroupList) error { - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -11540,9 +11540,9 @@ func (g *GCEBetaNetworkEndpointGroups) List(ctx context.Context, zone string, fl // Insert NetworkEndpointGroup with key of value obj. func (g *GCEBetaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.Key, obj *beta.NetworkEndpointGroup) error { - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEBetaNetworkEndpointGroups.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaNetworkEndpointGroups.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "NetworkEndpointGroups") @@ -11552,9 +11552,9 @@ func (g *GCEBetaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.Key Version: meta.Version("beta"), Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -11563,20 +11563,20 @@ func (g *GCEBetaNetworkEndpointGroups) Insert(ctx context.Context, key *meta.Key op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the NetworkEndpointGroup referenced by key. func (g *GCEBetaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaNetworkEndpointGroups.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaNetworkEndpointGroups.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "NetworkEndpointGroups") @@ -11586,9 +11586,9 @@ func (g *GCEBetaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.Key Version: meta.Version("beta"), Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.NetworkEndpointGroups.Delete(projectID, key.Zone, key.Name) @@ -11596,18 +11596,18 @@ func (g *GCEBetaNetworkEndpointGroups) Delete(ctx context.Context, key *meta.Key op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.Delete(%v, %v) = %v", ctx, key, err) return err } // AggregatedList lists all resources of the given type across all locations. func (g *GCEBetaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl *filter.F) (map[string][]*beta.NetworkEndpointGroup, error) { - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "NetworkEndpointGroups") rk := &RateLimitKey{ @@ -11617,9 +11617,9 @@ func (g *GCEBetaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl *f Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v): RateLimiter error: %v", ctx, fl, err) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v): RateLimiter error: %v", ctx, fl, err) return nil, err } @@ -11632,33 +11632,33 @@ func (g *GCEBetaNetworkEndpointGroups) AggregatedList(ctx context.Context, fl *f all := map[string][]*beta.NetworkEndpointGroup{} f := func(l *beta.NetworkEndpointGroupAggregatedList) error { for k, v := range l.Items { - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v): page[%v]%+v", ctx, fl, k, v) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v): page[%v]%+v", ctx, fl, k, v) all[k] = append(all[k], v.NetworkEndpointGroups...) } return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.AggregatedList(%v, %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil } // AttachNetworkEndpoints is a method on GCEBetaNetworkEndpointGroups. func (g *GCEBetaNetworkEndpointGroups) AttachNetworkEndpoints(ctx context.Context, key *meta.Key, arg0 *beta.NetworkEndpointGroupsAttachEndpointsRequest) error { - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "NetworkEndpointGroups") @@ -11668,30 +11668,30 @@ func (g *GCEBetaNetworkEndpointGroups) AttachNetworkEndpoints(ctx context.Contex Version: meta.Version("beta"), Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.NetworkEndpointGroups.AttachNetworkEndpoints(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.AttachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) return err } // DetachNetworkEndpoints is a method on GCEBetaNetworkEndpointGroups. func (g *GCEBetaNetworkEndpointGroups) DetachNetworkEndpoints(ctx context.Context, key *meta.Key, arg0 *beta.NetworkEndpointGroupsDetachEndpointsRequest) error { - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "NetworkEndpointGroups") @@ -11701,30 +11701,30 @@ func (g *GCEBetaNetworkEndpointGroups) DetachNetworkEndpoints(ctx context.Contex Version: meta.Version("beta"), Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.NetworkEndpointGroups.DetachNetworkEndpoints(projectID, key.Zone, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.DetachNetworkEndpoints(%v, %v, ...) = %+v", ctx, key, err) return err } // ListNetworkEndpoints is a method on GCEBetaNetworkEndpointGroups. func (g *GCEBetaNetworkEndpointGroups) ListNetworkEndpoints(ctx context.Context, key *meta.Key, arg0 *beta.NetworkEndpointGroupsListEndpointsRequest, fl *filter.F) ([]*beta.NetworkEndpointWithHealthStatus, error) { - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "NetworkEndpointGroups") @@ -11734,31 +11734,31 @@ func (g *GCEBetaNetworkEndpointGroups) ListNetworkEndpoints(ctx context.Context, Version: meta.Version("beta"), Service: "NetworkEndpointGroups", } - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Beta.NetworkEndpointGroups.ListNetworkEndpoints(projectID, key.Zone, key.Name, arg0) var all []*beta.NetworkEndpointWithHealthStatus f := func(l *beta.NetworkEndpointGroupsListNetworkEndpoints) error { - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): page %+v", ctx, key, l) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...): page %+v", ctx, key, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...) = %v, %v", ctx, key, nil, err) + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...) = %v, %v", ctx, key, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...) = [%v items], %v", ctx, key, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...) = [%v items], %v", ctx, key, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...) = %v, %v", ctx, key, asStr, nil) + klog.V(5).Infof("GCEBetaNetworkEndpointGroups.ListNetworkEndpoints(%v, %v, ...) = %v, %v", ctx, key, asStr, nil) } return all, nil } @@ -11859,7 +11859,7 @@ type MockRegions struct { func (m *MockRegions) Get(ctx context.Context, key *meta.Key) (*ga.Region, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockRegions.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockRegions.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -11871,12 +11871,12 @@ func (m *MockRegions) Get(ctx context.Context, key *meta.Key) (*ga.Region, error defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockRegions.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockRegions.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockRegions.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockRegions.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -11884,7 +11884,7 @@ func (m *MockRegions) Get(ctx context.Context, key *meta.Key) (*ga.Region, error Code: http.StatusNotFound, Message: fmt.Sprintf("MockRegions %v not found", key), } - glog.V(5).Infof("MockRegions.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockRegions.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -11892,7 +11892,7 @@ func (m *MockRegions) Get(ctx context.Context, key *meta.Key) (*ga.Region, error func (m *MockRegions) List(ctx context.Context, fl *filter.F) ([]*ga.Region, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockRegions.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockRegions.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -11902,7 +11902,7 @@ func (m *MockRegions) List(ctx context.Context, fl *filter.F) ([]*ga.Region, err if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockRegions.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockRegions.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -11915,7 +11915,7 @@ func (m *MockRegions) List(ctx context.Context, fl *filter.F) ([]*ga.Region, err objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockRegions.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockRegions.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -11931,10 +11931,10 @@ type GCERegions struct { // Get the Region named by key. func (g *GCERegions) Get(ctx context.Context, key *meta.Key) (*ga.Region, error) { - glog.V(5).Infof("GCERegions.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCERegions.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCERegions.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCERegions.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Regions") @@ -11944,21 +11944,21 @@ func (g *GCERegions) Get(ctx context.Context, key *meta.Key) (*ga.Region, error) Version: meta.Version("ga"), Service: "Regions", } - glog.V(5).Infof("GCERegions.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCERegions.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCERegions.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCERegions.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.Regions.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCERegions.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCERegions.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all Region objects. func (g *GCERegions) List(ctx context.Context, fl *filter.F) ([]*ga.Region, error) { - glog.V(5).Infof("GCERegions.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCERegions.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Regions") rk := &RateLimitKey{ ProjectID: projectID, @@ -11969,30 +11969,30 @@ func (g *GCERegions) List(ctx context.Context, fl *filter.F) ([]*ga.Region, erro if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCERegions.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCERegions.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.Regions.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.Region f := func(l *ga.RegionList) error { - glog.V(5).Infof("GCERegions.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCERegions.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCERegions.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCERegions.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCERegions.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCERegions.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCERegions.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCERegions.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -12053,7 +12053,7 @@ type MockRoutes struct { func (m *MockRoutes) Get(ctx context.Context, key *meta.Key) (*ga.Route, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockRoutes.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockRoutes.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -12065,12 +12065,12 @@ func (m *MockRoutes) Get(ctx context.Context, key *meta.Key) (*ga.Route, error) defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockRoutes.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockRoutes.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockRoutes.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockRoutes.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -12078,7 +12078,7 @@ func (m *MockRoutes) Get(ctx context.Context, key *meta.Key) (*ga.Route, error) Code: http.StatusNotFound, Message: fmt.Sprintf("MockRoutes %v not found", key), } - glog.V(5).Infof("MockRoutes.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockRoutes.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -12086,7 +12086,7 @@ func (m *MockRoutes) Get(ctx context.Context, key *meta.Key) (*ga.Route, error) func (m *MockRoutes) List(ctx context.Context, fl *filter.F) ([]*ga.Route, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockRoutes.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockRoutes.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -12096,7 +12096,7 @@ func (m *MockRoutes) List(ctx context.Context, fl *filter.F) ([]*ga.Route, error if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockRoutes.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockRoutes.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -12109,7 +12109,7 @@ func (m *MockRoutes) List(ctx context.Context, fl *filter.F) ([]*ga.Route, error objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockRoutes.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockRoutes.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -12117,7 +12117,7 @@ func (m *MockRoutes) List(ctx context.Context, fl *filter.F) ([]*ga.Route, error func (m *MockRoutes) Insert(ctx context.Context, key *meta.Key, obj *ga.Route) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockRoutes.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockRoutes.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -12129,7 +12129,7 @@ func (m *MockRoutes) Insert(ctx context.Context, key *meta.Key, obj *ga.Route) e defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockRoutes.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockRoutes.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -12137,7 +12137,7 @@ func (m *MockRoutes) Insert(ctx context.Context, key *meta.Key, obj *ga.Route) e Code: http.StatusConflict, Message: fmt.Sprintf("MockRoutes %v exists", key), } - glog.V(5).Infof("MockRoutes.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockRoutes.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -12146,7 +12146,7 @@ func (m *MockRoutes) Insert(ctx context.Context, key *meta.Key, obj *ga.Route) e obj.SelfLink = SelfLink(meta.VersionGA, projectID, "routes", key) m.Objects[*key] = &MockRoutesObj{obj} - glog.V(5).Infof("MockRoutes.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockRoutes.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -12154,7 +12154,7 @@ func (m *MockRoutes) Insert(ctx context.Context, key *meta.Key, obj *ga.Route) e func (m *MockRoutes) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockRoutes.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockRoutes.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -12166,7 +12166,7 @@ func (m *MockRoutes) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockRoutes.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockRoutes.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -12174,12 +12174,12 @@ func (m *MockRoutes) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockRoutes %v not found", key), } - glog.V(5).Infof("MockRoutes.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockRoutes.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockRoutes.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockRoutes.Delete(%v, %v) = nil", ctx, key) return nil } @@ -12195,10 +12195,10 @@ type GCERoutes struct { // Get the Route named by key. func (g *GCERoutes) Get(ctx context.Context, key *meta.Key) (*ga.Route, error) { - glog.V(5).Infof("GCERoutes.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCERoutes.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCERoutes.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCERoutes.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Routes") @@ -12208,21 +12208,21 @@ func (g *GCERoutes) Get(ctx context.Context, key *meta.Key) (*ga.Route, error) { Version: meta.Version("ga"), Service: "Routes", } - glog.V(5).Infof("GCERoutes.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCERoutes.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCERoutes.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCERoutes.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.Routes.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCERoutes.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCERoutes.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all Route objects. func (g *GCERoutes) List(ctx context.Context, fl *filter.F) ([]*ga.Route, error) { - glog.V(5).Infof("GCERoutes.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCERoutes.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Routes") rk := &RateLimitKey{ ProjectID: projectID, @@ -12233,30 +12233,30 @@ func (g *GCERoutes) List(ctx context.Context, fl *filter.F) ([]*ga.Route, error) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCERoutes.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCERoutes.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.Routes.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.Route f := func(l *ga.RouteList) error { - glog.V(5).Infof("GCERoutes.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCERoutes.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCERoutes.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCERoutes.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCERoutes.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCERoutes.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCERoutes.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCERoutes.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -12264,9 +12264,9 @@ func (g *GCERoutes) List(ctx context.Context, fl *filter.F) ([]*ga.Route, error) // Insert Route with key of value obj. func (g *GCERoutes) Insert(ctx context.Context, key *meta.Key, obj *ga.Route) error { - glog.V(5).Infof("GCERoutes.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCERoutes.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCERoutes.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCERoutes.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Routes") @@ -12276,9 +12276,9 @@ func (g *GCERoutes) Insert(ctx context.Context, key *meta.Key, obj *ga.Route) er Version: meta.Version("ga"), Service: "Routes", } - glog.V(5).Infof("GCERoutes.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCERoutes.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCERoutes.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCERoutes.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -12287,20 +12287,20 @@ func (g *GCERoutes) Insert(ctx context.Context, key *meta.Key, obj *ga.Route) er op, err := call.Do() if err != nil { - glog.V(4).Infof("GCERoutes.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCERoutes.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCERoutes.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCERoutes.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the Route referenced by key. func (g *GCERoutes) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCERoutes.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCERoutes.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCERoutes.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCERoutes.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Routes") @@ -12310,9 +12310,9 @@ func (g *GCERoutes) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "Routes", } - glog.V(5).Infof("GCERoutes.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCERoutes.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCERoutes.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCERoutes.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.Routes.Delete(projectID, key.Name) @@ -12321,12 +12321,12 @@ func (g *GCERoutes) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCERoutes.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCERoutes.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCERoutes.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCERoutes.Delete(%v, %v) = %v", ctx, key, err) return err } @@ -12395,7 +12395,7 @@ type MockBetaSecurityPolicies struct { func (m *MockBetaSecurityPolicies) Get(ctx context.Context, key *meta.Key) (*beta.SecurityPolicy, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaSecurityPolicies.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaSecurityPolicies.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -12407,12 +12407,12 @@ func (m *MockBetaSecurityPolicies) Get(ctx context.Context, key *meta.Key) (*bet defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockBetaSecurityPolicies.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBetaSecurityPolicies.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToBeta() - glog.V(5).Infof("MockBetaSecurityPolicies.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockBetaSecurityPolicies.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -12420,7 +12420,7 @@ func (m *MockBetaSecurityPolicies) Get(ctx context.Context, key *meta.Key) (*bet Code: http.StatusNotFound, Message: fmt.Sprintf("MockBetaSecurityPolicies %v not found", key), } - glog.V(5).Infof("MockBetaSecurityPolicies.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockBetaSecurityPolicies.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -12428,7 +12428,7 @@ func (m *MockBetaSecurityPolicies) Get(ctx context.Context, key *meta.Key) (*bet func (m *MockBetaSecurityPolicies) List(ctx context.Context, fl *filter.F) ([]*beta.SecurityPolicy, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockBetaSecurityPolicies.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockBetaSecurityPolicies.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -12438,7 +12438,7 @@ func (m *MockBetaSecurityPolicies) List(ctx context.Context, fl *filter.F) ([]*b if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockBetaSecurityPolicies.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockBetaSecurityPolicies.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -12451,7 +12451,7 @@ func (m *MockBetaSecurityPolicies) List(ctx context.Context, fl *filter.F) ([]*b objs = append(objs, obj.ToBeta()) } - glog.V(5).Infof("MockBetaSecurityPolicies.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockBetaSecurityPolicies.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -12459,7 +12459,7 @@ func (m *MockBetaSecurityPolicies) List(ctx context.Context, fl *filter.F) ([]*b func (m *MockBetaSecurityPolicies) Insert(ctx context.Context, key *meta.Key, obj *beta.SecurityPolicy) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockBetaSecurityPolicies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaSecurityPolicies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -12471,7 +12471,7 @@ func (m *MockBetaSecurityPolicies) Insert(ctx context.Context, key *meta.Key, ob defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockBetaSecurityPolicies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaSecurityPolicies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -12479,7 +12479,7 @@ func (m *MockBetaSecurityPolicies) Insert(ctx context.Context, key *meta.Key, ob Code: http.StatusConflict, Message: fmt.Sprintf("MockBetaSecurityPolicies %v exists", key), } - glog.V(5).Infof("MockBetaSecurityPolicies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockBetaSecurityPolicies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -12488,7 +12488,7 @@ func (m *MockBetaSecurityPolicies) Insert(ctx context.Context, key *meta.Key, ob obj.SelfLink = SelfLink(meta.VersionBeta, projectID, "securityPolicies", key) m.Objects[*key] = &MockSecurityPoliciesObj{obj} - glog.V(5).Infof("MockBetaSecurityPolicies.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockBetaSecurityPolicies.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -12496,7 +12496,7 @@ func (m *MockBetaSecurityPolicies) Insert(ctx context.Context, key *meta.Key, ob func (m *MockBetaSecurityPolicies) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockBetaSecurityPolicies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaSecurityPolicies.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -12508,7 +12508,7 @@ func (m *MockBetaSecurityPolicies) Delete(ctx context.Context, key *meta.Key) er defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockBetaSecurityPolicies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaSecurityPolicies.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -12516,12 +12516,12 @@ func (m *MockBetaSecurityPolicies) Delete(ctx context.Context, key *meta.Key) er Code: http.StatusNotFound, Message: fmt.Sprintf("MockBetaSecurityPolicies %v not found", key), } - glog.V(5).Infof("MockBetaSecurityPolicies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockBetaSecurityPolicies.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockBetaSecurityPolicies.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockBetaSecurityPolicies.Delete(%v, %v) = nil", ctx, key) return nil } @@ -12577,10 +12577,10 @@ type GCEBetaSecurityPolicies struct { // Get the SecurityPolicy named by key. func (g *GCEBetaSecurityPolicies) Get(ctx context.Context, key *meta.Key) (*beta.SecurityPolicy, error) { - glog.V(5).Infof("GCEBetaSecurityPolicies.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBetaSecurityPolicies.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaSecurityPolicies.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaSecurityPolicies.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "SecurityPolicies") @@ -12590,21 +12590,21 @@ func (g *GCEBetaSecurityPolicies) Get(ctx context.Context, key *meta.Key) (*beta Version: meta.Version("beta"), Service: "SecurityPolicies", } - glog.V(5).Infof("GCEBetaSecurityPolicies.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaSecurityPolicies.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Beta.SecurityPolicies.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEBetaSecurityPolicies.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all SecurityPolicy objects. func (g *GCEBetaSecurityPolicies) List(ctx context.Context, fl *filter.F) ([]*beta.SecurityPolicy, error) { - glog.V(5).Infof("GCEBetaSecurityPolicies.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEBetaSecurityPolicies.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "SecurityPolicies") rk := &RateLimitKey{ ProjectID: projectID, @@ -12615,30 +12615,30 @@ func (g *GCEBetaSecurityPolicies) List(ctx context.Context, fl *filter.F) ([]*be if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEBetaSecurityPolicies.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEBetaSecurityPolicies.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.Beta.SecurityPolicies.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*beta.SecurityPolicy f := func(l *beta.SecurityPolicyList) error { - glog.V(5).Infof("GCEBetaSecurityPolicies.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEBetaSecurityPolicies.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEBetaSecurityPolicies.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEBetaSecurityPolicies.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEBetaSecurityPolicies.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEBetaSecurityPolicies.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -12646,9 +12646,9 @@ func (g *GCEBetaSecurityPolicies) List(ctx context.Context, fl *filter.F) ([]*be // Insert SecurityPolicy with key of value obj. func (g *GCEBetaSecurityPolicies) Insert(ctx context.Context, key *meta.Key, obj *beta.SecurityPolicy) error { - glog.V(5).Infof("GCEBetaSecurityPolicies.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEBetaSecurityPolicies.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEBetaSecurityPolicies.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaSecurityPolicies.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "SecurityPolicies") @@ -12658,9 +12658,9 @@ func (g *GCEBetaSecurityPolicies) Insert(ctx context.Context, key *meta.Key, obj Version: meta.Version("beta"), Service: "SecurityPolicies", } - glog.V(5).Infof("GCEBetaSecurityPolicies.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaSecurityPolicies.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -12669,20 +12669,20 @@ func (g *GCEBetaSecurityPolicies) Insert(ctx context.Context, key *meta.Key, obj op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaSecurityPolicies.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the SecurityPolicy referenced by key. func (g *GCEBetaSecurityPolicies) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEBetaSecurityPolicies.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEBetaSecurityPolicies.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaSecurityPolicies.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaSecurityPolicies.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "SecurityPolicies") @@ -12692,9 +12692,9 @@ func (g *GCEBetaSecurityPolicies) Delete(ctx context.Context, key *meta.Key) err Version: meta.Version("beta"), Service: "SecurityPolicies", } - glog.V(5).Infof("GCEBetaSecurityPolicies.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaSecurityPolicies.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.SecurityPolicies.Delete(projectID, key.Name) @@ -12703,21 +12703,21 @@ func (g *GCEBetaSecurityPolicies) Delete(ctx context.Context, key *meta.Key) err op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaSecurityPolicies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.Delete(%v, %v) = %v", ctx, key, err) return err } // AddRule is a method on GCEBetaSecurityPolicies. func (g *GCEBetaSecurityPolicies) AddRule(ctx context.Context, key *meta.Key, arg0 *beta.SecurityPolicyRule) error { - glog.V(5).Infof("GCEBetaSecurityPolicies.AddRule(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaSecurityPolicies.AddRule(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaSecurityPolicies.AddRule(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaSecurityPolicies.AddRule(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "SecurityPolicies") @@ -12727,30 +12727,30 @@ func (g *GCEBetaSecurityPolicies) AddRule(ctx context.Context, key *meta.Key, ar Version: meta.Version("beta"), Service: "SecurityPolicies", } - glog.V(5).Infof("GCEBetaSecurityPolicies.AddRule(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaSecurityPolicies.AddRule(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.AddRule(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.AddRule(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.SecurityPolicies.AddRule(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.AddRule(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.AddRule(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaSecurityPolicies.AddRule(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.AddRule(%v, %v, ...) = %+v", ctx, key, err) return err } // GetRule is a method on GCEBetaSecurityPolicies. func (g *GCEBetaSecurityPolicies) GetRule(ctx context.Context, key *meta.Key) (*beta.SecurityPolicyRule, error) { - glog.V(5).Infof("GCEBetaSecurityPolicies.GetRule(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaSecurityPolicies.GetRule(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaSecurityPolicies.GetRule(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaSecurityPolicies.GetRule(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "SecurityPolicies") @@ -12760,25 +12760,25 @@ func (g *GCEBetaSecurityPolicies) GetRule(ctx context.Context, key *meta.Key) (* Version: meta.Version("beta"), Service: "SecurityPolicies", } - glog.V(5).Infof("GCEBetaSecurityPolicies.GetRule(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaSecurityPolicies.GetRule(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.GetRule(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.GetRule(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.Beta.SecurityPolicies.GetRule(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEBetaSecurityPolicies.GetRule(%v, %v, ...) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.GetRule(%v, %v, ...) = %+v, %v", ctx, key, v, err) return v, err } // Patch is a method on GCEBetaSecurityPolicies. func (g *GCEBetaSecurityPolicies) Patch(ctx context.Context, key *meta.Key, arg0 *beta.SecurityPolicy) error { - glog.V(5).Infof("GCEBetaSecurityPolicies.Patch(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaSecurityPolicies.Patch(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaSecurityPolicies.Patch(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaSecurityPolicies.Patch(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "SecurityPolicies") @@ -12788,30 +12788,30 @@ func (g *GCEBetaSecurityPolicies) Patch(ctx context.Context, key *meta.Key, arg0 Version: meta.Version("beta"), Service: "SecurityPolicies", } - glog.V(5).Infof("GCEBetaSecurityPolicies.Patch(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaSecurityPolicies.Patch(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.Patch(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.Patch(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.SecurityPolicies.Patch(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.Patch(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.Patch(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaSecurityPolicies.Patch(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.Patch(%v, %v, ...) = %+v", ctx, key, err) return err } // PatchRule is a method on GCEBetaSecurityPolicies. func (g *GCEBetaSecurityPolicies) PatchRule(ctx context.Context, key *meta.Key, arg0 *beta.SecurityPolicyRule) error { - glog.V(5).Infof("GCEBetaSecurityPolicies.PatchRule(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaSecurityPolicies.PatchRule(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaSecurityPolicies.PatchRule(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaSecurityPolicies.PatchRule(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "SecurityPolicies") @@ -12821,30 +12821,30 @@ func (g *GCEBetaSecurityPolicies) PatchRule(ctx context.Context, key *meta.Key, Version: meta.Version("beta"), Service: "SecurityPolicies", } - glog.V(5).Infof("GCEBetaSecurityPolicies.PatchRule(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaSecurityPolicies.PatchRule(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.PatchRule(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.PatchRule(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.SecurityPolicies.PatchRule(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.PatchRule(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.PatchRule(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaSecurityPolicies.PatchRule(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.PatchRule(%v, %v, ...) = %+v", ctx, key, err) return err } // RemoveRule is a method on GCEBetaSecurityPolicies. func (g *GCEBetaSecurityPolicies) RemoveRule(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEBetaSecurityPolicies.RemoveRule(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEBetaSecurityPolicies.RemoveRule(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEBetaSecurityPolicies.RemoveRule(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEBetaSecurityPolicies.RemoveRule(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "beta", "SecurityPolicies") @@ -12854,21 +12854,21 @@ func (g *GCEBetaSecurityPolicies) RemoveRule(ctx context.Context, key *meta.Key) Version: meta.Version("beta"), Service: "SecurityPolicies", } - glog.V(5).Infof("GCEBetaSecurityPolicies.RemoveRule(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEBetaSecurityPolicies.RemoveRule(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.RemoveRule(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.RemoveRule(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.Beta.SecurityPolicies.RemoveRule(projectID, key.Name) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEBetaSecurityPolicies.RemoveRule(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.RemoveRule(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEBetaSecurityPolicies.RemoveRule(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEBetaSecurityPolicies.RemoveRule(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -12927,7 +12927,7 @@ type MockSslCertificates struct { func (m *MockSslCertificates) Get(ctx context.Context, key *meta.Key) (*ga.SslCertificate, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockSslCertificates.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockSslCertificates.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -12939,12 +12939,12 @@ func (m *MockSslCertificates) Get(ctx context.Context, key *meta.Key) (*ga.SslCe defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockSslCertificates.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockSslCertificates.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockSslCertificates.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockSslCertificates.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -12952,7 +12952,7 @@ func (m *MockSslCertificates) Get(ctx context.Context, key *meta.Key) (*ga.SslCe Code: http.StatusNotFound, Message: fmt.Sprintf("MockSslCertificates %v not found", key), } - glog.V(5).Infof("MockSslCertificates.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockSslCertificates.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -12960,7 +12960,7 @@ func (m *MockSslCertificates) Get(ctx context.Context, key *meta.Key) (*ga.SslCe func (m *MockSslCertificates) List(ctx context.Context, fl *filter.F) ([]*ga.SslCertificate, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockSslCertificates.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockSslCertificates.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -12970,7 +12970,7 @@ func (m *MockSslCertificates) List(ctx context.Context, fl *filter.F) ([]*ga.Ssl if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockSslCertificates.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockSslCertificates.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -12983,7 +12983,7 @@ func (m *MockSslCertificates) List(ctx context.Context, fl *filter.F) ([]*ga.Ssl objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockSslCertificates.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockSslCertificates.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -12991,7 +12991,7 @@ func (m *MockSslCertificates) List(ctx context.Context, fl *filter.F) ([]*ga.Ssl func (m *MockSslCertificates) Insert(ctx context.Context, key *meta.Key, obj *ga.SslCertificate) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockSslCertificates.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockSslCertificates.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -13003,7 +13003,7 @@ func (m *MockSslCertificates) Insert(ctx context.Context, key *meta.Key, obj *ga defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockSslCertificates.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockSslCertificates.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -13011,7 +13011,7 @@ func (m *MockSslCertificates) Insert(ctx context.Context, key *meta.Key, obj *ga Code: http.StatusConflict, Message: fmt.Sprintf("MockSslCertificates %v exists", key), } - glog.V(5).Infof("MockSslCertificates.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockSslCertificates.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -13020,7 +13020,7 @@ func (m *MockSslCertificates) Insert(ctx context.Context, key *meta.Key, obj *ga obj.SelfLink = SelfLink(meta.VersionGA, projectID, "sslCertificates", key) m.Objects[*key] = &MockSslCertificatesObj{obj} - glog.V(5).Infof("MockSslCertificates.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockSslCertificates.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -13028,7 +13028,7 @@ func (m *MockSslCertificates) Insert(ctx context.Context, key *meta.Key, obj *ga func (m *MockSslCertificates) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockSslCertificates.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockSslCertificates.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -13040,7 +13040,7 @@ func (m *MockSslCertificates) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockSslCertificates.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockSslCertificates.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -13048,12 +13048,12 @@ func (m *MockSslCertificates) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockSslCertificates %v not found", key), } - glog.V(5).Infof("MockSslCertificates.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockSslCertificates.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockSslCertificates.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockSslCertificates.Delete(%v, %v) = nil", ctx, key) return nil } @@ -13069,10 +13069,10 @@ type GCESslCertificates struct { // Get the SslCertificate named by key. func (g *GCESslCertificates) Get(ctx context.Context, key *meta.Key) (*ga.SslCertificate, error) { - glog.V(5).Infof("GCESslCertificates.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCESslCertificates.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCESslCertificates.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCESslCertificates.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "SslCertificates") @@ -13082,21 +13082,21 @@ func (g *GCESslCertificates) Get(ctx context.Context, key *meta.Key) (*ga.SslCer Version: meta.Version("ga"), Service: "SslCertificates", } - glog.V(5).Infof("GCESslCertificates.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCESslCertificates.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCESslCertificates.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCESslCertificates.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.SslCertificates.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCESslCertificates.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCESslCertificates.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all SslCertificate objects. func (g *GCESslCertificates) List(ctx context.Context, fl *filter.F) ([]*ga.SslCertificate, error) { - glog.V(5).Infof("GCESslCertificates.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCESslCertificates.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "SslCertificates") rk := &RateLimitKey{ ProjectID: projectID, @@ -13107,30 +13107,30 @@ func (g *GCESslCertificates) List(ctx context.Context, fl *filter.F) ([]*ga.SslC if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCESslCertificates.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCESslCertificates.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.SslCertificates.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.SslCertificate f := func(l *ga.SslCertificateList) error { - glog.V(5).Infof("GCESslCertificates.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCESslCertificates.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCESslCertificates.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCESslCertificates.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCESslCertificates.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCESslCertificates.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCESslCertificates.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCESslCertificates.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -13138,9 +13138,9 @@ func (g *GCESslCertificates) List(ctx context.Context, fl *filter.F) ([]*ga.SslC // Insert SslCertificate with key of value obj. func (g *GCESslCertificates) Insert(ctx context.Context, key *meta.Key, obj *ga.SslCertificate) error { - glog.V(5).Infof("GCESslCertificates.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCESslCertificates.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCESslCertificates.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCESslCertificates.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "SslCertificates") @@ -13150,9 +13150,9 @@ func (g *GCESslCertificates) Insert(ctx context.Context, key *meta.Key, obj *ga. Version: meta.Version("ga"), Service: "SslCertificates", } - glog.V(5).Infof("GCESslCertificates.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCESslCertificates.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCESslCertificates.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCESslCertificates.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -13161,20 +13161,20 @@ func (g *GCESslCertificates) Insert(ctx context.Context, key *meta.Key, obj *ga. op, err := call.Do() if err != nil { - glog.V(4).Infof("GCESslCertificates.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCESslCertificates.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCESslCertificates.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCESslCertificates.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the SslCertificate referenced by key. func (g *GCESslCertificates) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCESslCertificates.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCESslCertificates.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCESslCertificates.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCESslCertificates.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "SslCertificates") @@ -13184,9 +13184,9 @@ func (g *GCESslCertificates) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "SslCertificates", } - glog.V(5).Infof("GCESslCertificates.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCESslCertificates.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCESslCertificates.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCESslCertificates.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.SslCertificates.Delete(projectID, key.Name) @@ -13195,12 +13195,12 @@ func (g *GCESslCertificates) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCESslCertificates.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCESslCertificates.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCESslCertificates.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCESslCertificates.Delete(%v, %v) = %v", ctx, key, err) return err } @@ -13261,7 +13261,7 @@ type MockTargetHttpProxies struct { func (m *MockTargetHttpProxies) Get(ctx context.Context, key *meta.Key) (*ga.TargetHttpProxy, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockTargetHttpProxies.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockTargetHttpProxies.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -13273,12 +13273,12 @@ func (m *MockTargetHttpProxies) Get(ctx context.Context, key *meta.Key) (*ga.Tar defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockTargetHttpProxies.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockTargetHttpProxies.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockTargetHttpProxies.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockTargetHttpProxies.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -13286,7 +13286,7 @@ func (m *MockTargetHttpProxies) Get(ctx context.Context, key *meta.Key) (*ga.Tar Code: http.StatusNotFound, Message: fmt.Sprintf("MockTargetHttpProxies %v not found", key), } - glog.V(5).Infof("MockTargetHttpProxies.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockTargetHttpProxies.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -13294,7 +13294,7 @@ func (m *MockTargetHttpProxies) Get(ctx context.Context, key *meta.Key) (*ga.Tar func (m *MockTargetHttpProxies) List(ctx context.Context, fl *filter.F) ([]*ga.TargetHttpProxy, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockTargetHttpProxies.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockTargetHttpProxies.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -13304,7 +13304,7 @@ func (m *MockTargetHttpProxies) List(ctx context.Context, fl *filter.F) ([]*ga.T if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockTargetHttpProxies.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockTargetHttpProxies.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -13317,7 +13317,7 @@ func (m *MockTargetHttpProxies) List(ctx context.Context, fl *filter.F) ([]*ga.T objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockTargetHttpProxies.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockTargetHttpProxies.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -13325,7 +13325,7 @@ func (m *MockTargetHttpProxies) List(ctx context.Context, fl *filter.F) ([]*ga.T func (m *MockTargetHttpProxies) Insert(ctx context.Context, key *meta.Key, obj *ga.TargetHttpProxy) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockTargetHttpProxies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockTargetHttpProxies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -13337,7 +13337,7 @@ func (m *MockTargetHttpProxies) Insert(ctx context.Context, key *meta.Key, obj * defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockTargetHttpProxies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockTargetHttpProxies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -13345,7 +13345,7 @@ func (m *MockTargetHttpProxies) Insert(ctx context.Context, key *meta.Key, obj * Code: http.StatusConflict, Message: fmt.Sprintf("MockTargetHttpProxies %v exists", key), } - glog.V(5).Infof("MockTargetHttpProxies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockTargetHttpProxies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -13354,7 +13354,7 @@ func (m *MockTargetHttpProxies) Insert(ctx context.Context, key *meta.Key, obj * obj.SelfLink = SelfLink(meta.VersionGA, projectID, "targetHttpProxies", key) m.Objects[*key] = &MockTargetHttpProxiesObj{obj} - glog.V(5).Infof("MockTargetHttpProxies.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockTargetHttpProxies.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -13362,7 +13362,7 @@ func (m *MockTargetHttpProxies) Insert(ctx context.Context, key *meta.Key, obj * func (m *MockTargetHttpProxies) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockTargetHttpProxies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockTargetHttpProxies.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -13374,7 +13374,7 @@ func (m *MockTargetHttpProxies) Delete(ctx context.Context, key *meta.Key) error defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockTargetHttpProxies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockTargetHttpProxies.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -13382,12 +13382,12 @@ func (m *MockTargetHttpProxies) Delete(ctx context.Context, key *meta.Key) error Code: http.StatusNotFound, Message: fmt.Sprintf("MockTargetHttpProxies %v not found", key), } - glog.V(5).Infof("MockTargetHttpProxies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockTargetHttpProxies.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockTargetHttpProxies.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockTargetHttpProxies.Delete(%v, %v) = nil", ctx, key) return nil } @@ -13411,10 +13411,10 @@ type GCETargetHttpProxies struct { // Get the TargetHttpProxy named by key. func (g *GCETargetHttpProxies) Get(ctx context.Context, key *meta.Key) (*ga.TargetHttpProxy, error) { - glog.V(5).Infof("GCETargetHttpProxies.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCETargetHttpProxies.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCETargetHttpProxies.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetHttpProxies.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetHttpProxies") @@ -13424,21 +13424,21 @@ func (g *GCETargetHttpProxies) Get(ctx context.Context, key *meta.Key) (*ga.Targ Version: meta.Version("ga"), Service: "TargetHttpProxies", } - glog.V(5).Infof("GCETargetHttpProxies.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetHttpProxies.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetHttpProxies.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpProxies.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.TargetHttpProxies.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCETargetHttpProxies.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCETargetHttpProxies.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all TargetHttpProxy objects. func (g *GCETargetHttpProxies) List(ctx context.Context, fl *filter.F) ([]*ga.TargetHttpProxy, error) { - glog.V(5).Infof("GCETargetHttpProxies.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCETargetHttpProxies.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetHttpProxies") rk := &RateLimitKey{ ProjectID: projectID, @@ -13449,30 +13449,30 @@ func (g *GCETargetHttpProxies) List(ctx context.Context, fl *filter.F) ([]*ga.Ta if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCETargetHttpProxies.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCETargetHttpProxies.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.TargetHttpProxies.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.TargetHttpProxy f := func(l *ga.TargetHttpProxyList) error { - glog.V(5).Infof("GCETargetHttpProxies.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCETargetHttpProxies.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCETargetHttpProxies.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCETargetHttpProxies.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCETargetHttpProxies.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCETargetHttpProxies.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCETargetHttpProxies.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCETargetHttpProxies.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -13480,9 +13480,9 @@ func (g *GCETargetHttpProxies) List(ctx context.Context, fl *filter.F) ([]*ga.Ta // Insert TargetHttpProxy with key of value obj. func (g *GCETargetHttpProxies) Insert(ctx context.Context, key *meta.Key, obj *ga.TargetHttpProxy) error { - glog.V(5).Infof("GCETargetHttpProxies.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCETargetHttpProxies.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCETargetHttpProxies.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetHttpProxies.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetHttpProxies") @@ -13492,9 +13492,9 @@ func (g *GCETargetHttpProxies) Insert(ctx context.Context, key *meta.Key, obj *g Version: meta.Version("ga"), Service: "TargetHttpProxies", } - glog.V(5).Infof("GCETargetHttpProxies.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetHttpProxies.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetHttpProxies.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpProxies.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -13503,20 +13503,20 @@ func (g *GCETargetHttpProxies) Insert(ctx context.Context, key *meta.Key, obj *g op, err := call.Do() if err != nil { - glog.V(4).Infof("GCETargetHttpProxies.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpProxies.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCETargetHttpProxies.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCETargetHttpProxies.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the TargetHttpProxy referenced by key. func (g *GCETargetHttpProxies) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCETargetHttpProxies.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCETargetHttpProxies.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCETargetHttpProxies.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetHttpProxies.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetHttpProxies") @@ -13526,9 +13526,9 @@ func (g *GCETargetHttpProxies) Delete(ctx context.Context, key *meta.Key) error Version: meta.Version("ga"), Service: "TargetHttpProxies", } - glog.V(5).Infof("GCETargetHttpProxies.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetHttpProxies.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetHttpProxies.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpProxies.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.TargetHttpProxies.Delete(projectID, key.Name) @@ -13537,21 +13537,21 @@ func (g *GCETargetHttpProxies) Delete(ctx context.Context, key *meta.Key) error op, err := call.Do() if err != nil { - glog.V(4).Infof("GCETargetHttpProxies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpProxies.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCETargetHttpProxies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpProxies.Delete(%v, %v) = %v", ctx, key, err) return err } // SetUrlMap is a method on GCETargetHttpProxies. func (g *GCETargetHttpProxies) SetUrlMap(ctx context.Context, key *meta.Key, arg0 *ga.UrlMapReference) error { - glog.V(5).Infof("GCETargetHttpProxies.SetUrlMap(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCETargetHttpProxies.SetUrlMap(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCETargetHttpProxies.SetUrlMap(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetHttpProxies.SetUrlMap(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetHttpProxies") @@ -13561,21 +13561,21 @@ func (g *GCETargetHttpProxies) SetUrlMap(ctx context.Context, key *meta.Key, arg Version: meta.Version("ga"), Service: "TargetHttpProxies", } - glog.V(5).Infof("GCETargetHttpProxies.SetUrlMap(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetHttpProxies.SetUrlMap(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetHttpProxies.SetUrlMap(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpProxies.SetUrlMap(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.TargetHttpProxies.SetUrlMap(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCETargetHttpProxies.SetUrlMap(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpProxies.SetUrlMap(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCETargetHttpProxies.SetUrlMap(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpProxies.SetUrlMap(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -13638,7 +13638,7 @@ type MockTargetHttpsProxies struct { func (m *MockTargetHttpsProxies) Get(ctx context.Context, key *meta.Key) (*ga.TargetHttpsProxy, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockTargetHttpsProxies.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockTargetHttpsProxies.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -13650,12 +13650,12 @@ func (m *MockTargetHttpsProxies) Get(ctx context.Context, key *meta.Key) (*ga.Ta defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockTargetHttpsProxies.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockTargetHttpsProxies.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockTargetHttpsProxies.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockTargetHttpsProxies.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -13663,7 +13663,7 @@ func (m *MockTargetHttpsProxies) Get(ctx context.Context, key *meta.Key) (*ga.Ta Code: http.StatusNotFound, Message: fmt.Sprintf("MockTargetHttpsProxies %v not found", key), } - glog.V(5).Infof("MockTargetHttpsProxies.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockTargetHttpsProxies.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -13671,7 +13671,7 @@ func (m *MockTargetHttpsProxies) Get(ctx context.Context, key *meta.Key) (*ga.Ta func (m *MockTargetHttpsProxies) List(ctx context.Context, fl *filter.F) ([]*ga.TargetHttpsProxy, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockTargetHttpsProxies.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockTargetHttpsProxies.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -13681,7 +13681,7 @@ func (m *MockTargetHttpsProxies) List(ctx context.Context, fl *filter.F) ([]*ga. if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockTargetHttpsProxies.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockTargetHttpsProxies.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -13694,7 +13694,7 @@ func (m *MockTargetHttpsProxies) List(ctx context.Context, fl *filter.F) ([]*ga. objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockTargetHttpsProxies.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockTargetHttpsProxies.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -13702,7 +13702,7 @@ func (m *MockTargetHttpsProxies) List(ctx context.Context, fl *filter.F) ([]*ga. func (m *MockTargetHttpsProxies) Insert(ctx context.Context, key *meta.Key, obj *ga.TargetHttpsProxy) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockTargetHttpsProxies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockTargetHttpsProxies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -13714,7 +13714,7 @@ func (m *MockTargetHttpsProxies) Insert(ctx context.Context, key *meta.Key, obj defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockTargetHttpsProxies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockTargetHttpsProxies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -13722,7 +13722,7 @@ func (m *MockTargetHttpsProxies) Insert(ctx context.Context, key *meta.Key, obj Code: http.StatusConflict, Message: fmt.Sprintf("MockTargetHttpsProxies %v exists", key), } - glog.V(5).Infof("MockTargetHttpsProxies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockTargetHttpsProxies.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -13731,7 +13731,7 @@ func (m *MockTargetHttpsProxies) Insert(ctx context.Context, key *meta.Key, obj obj.SelfLink = SelfLink(meta.VersionGA, projectID, "targetHttpsProxies", key) m.Objects[*key] = &MockTargetHttpsProxiesObj{obj} - glog.V(5).Infof("MockTargetHttpsProxies.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockTargetHttpsProxies.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -13739,7 +13739,7 @@ func (m *MockTargetHttpsProxies) Insert(ctx context.Context, key *meta.Key, obj func (m *MockTargetHttpsProxies) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockTargetHttpsProxies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockTargetHttpsProxies.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -13751,7 +13751,7 @@ func (m *MockTargetHttpsProxies) Delete(ctx context.Context, key *meta.Key) erro defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockTargetHttpsProxies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockTargetHttpsProxies.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -13759,12 +13759,12 @@ func (m *MockTargetHttpsProxies) Delete(ctx context.Context, key *meta.Key) erro Code: http.StatusNotFound, Message: fmt.Sprintf("MockTargetHttpsProxies %v not found", key), } - glog.V(5).Infof("MockTargetHttpsProxies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockTargetHttpsProxies.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockTargetHttpsProxies.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockTargetHttpsProxies.Delete(%v, %v) = nil", ctx, key) return nil } @@ -13796,10 +13796,10 @@ type GCETargetHttpsProxies struct { // Get the TargetHttpsProxy named by key. func (g *GCETargetHttpsProxies) Get(ctx context.Context, key *meta.Key) (*ga.TargetHttpsProxy, error) { - glog.V(5).Infof("GCETargetHttpsProxies.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCETargetHttpsProxies.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCETargetHttpsProxies.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetHttpsProxies.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetHttpsProxies") @@ -13809,21 +13809,21 @@ func (g *GCETargetHttpsProxies) Get(ctx context.Context, key *meta.Key) (*ga.Tar Version: meta.Version("ga"), Service: "TargetHttpsProxies", } - glog.V(5).Infof("GCETargetHttpsProxies.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetHttpsProxies.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetHttpsProxies.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpsProxies.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.TargetHttpsProxies.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCETargetHttpsProxies.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCETargetHttpsProxies.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all TargetHttpsProxy objects. func (g *GCETargetHttpsProxies) List(ctx context.Context, fl *filter.F) ([]*ga.TargetHttpsProxy, error) { - glog.V(5).Infof("GCETargetHttpsProxies.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCETargetHttpsProxies.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetHttpsProxies") rk := &RateLimitKey{ ProjectID: projectID, @@ -13834,30 +13834,30 @@ func (g *GCETargetHttpsProxies) List(ctx context.Context, fl *filter.F) ([]*ga.T if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCETargetHttpsProxies.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCETargetHttpsProxies.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.TargetHttpsProxies.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.TargetHttpsProxy f := func(l *ga.TargetHttpsProxyList) error { - glog.V(5).Infof("GCETargetHttpsProxies.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCETargetHttpsProxies.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCETargetHttpsProxies.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCETargetHttpsProxies.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCETargetHttpsProxies.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCETargetHttpsProxies.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCETargetHttpsProxies.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCETargetHttpsProxies.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -13865,9 +13865,9 @@ func (g *GCETargetHttpsProxies) List(ctx context.Context, fl *filter.F) ([]*ga.T // Insert TargetHttpsProxy with key of value obj. func (g *GCETargetHttpsProxies) Insert(ctx context.Context, key *meta.Key, obj *ga.TargetHttpsProxy) error { - glog.V(5).Infof("GCETargetHttpsProxies.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCETargetHttpsProxies.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCETargetHttpsProxies.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetHttpsProxies.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetHttpsProxies") @@ -13877,9 +13877,9 @@ func (g *GCETargetHttpsProxies) Insert(ctx context.Context, key *meta.Key, obj * Version: meta.Version("ga"), Service: "TargetHttpsProxies", } - glog.V(5).Infof("GCETargetHttpsProxies.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetHttpsProxies.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetHttpsProxies.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpsProxies.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -13888,20 +13888,20 @@ func (g *GCETargetHttpsProxies) Insert(ctx context.Context, key *meta.Key, obj * op, err := call.Do() if err != nil { - glog.V(4).Infof("GCETargetHttpsProxies.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpsProxies.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCETargetHttpsProxies.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCETargetHttpsProxies.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the TargetHttpsProxy referenced by key. func (g *GCETargetHttpsProxies) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCETargetHttpsProxies.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCETargetHttpsProxies.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCETargetHttpsProxies.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetHttpsProxies.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetHttpsProxies") @@ -13911,9 +13911,9 @@ func (g *GCETargetHttpsProxies) Delete(ctx context.Context, key *meta.Key) error Version: meta.Version("ga"), Service: "TargetHttpsProxies", } - glog.V(5).Infof("GCETargetHttpsProxies.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetHttpsProxies.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetHttpsProxies.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpsProxies.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.TargetHttpsProxies.Delete(projectID, key.Name) @@ -13922,21 +13922,21 @@ func (g *GCETargetHttpsProxies) Delete(ctx context.Context, key *meta.Key) error op, err := call.Do() if err != nil { - glog.V(4).Infof("GCETargetHttpsProxies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpsProxies.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCETargetHttpsProxies.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpsProxies.Delete(%v, %v) = %v", ctx, key, err) return err } // SetSslCertificates is a method on GCETargetHttpsProxies. func (g *GCETargetHttpsProxies) SetSslCertificates(ctx context.Context, key *meta.Key, arg0 *ga.TargetHttpsProxiesSetSslCertificatesRequest) error { - glog.V(5).Infof("GCETargetHttpsProxies.SetSslCertificates(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCETargetHttpsProxies.SetSslCertificates(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCETargetHttpsProxies.SetSslCertificates(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetHttpsProxies.SetSslCertificates(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetHttpsProxies") @@ -13946,30 +13946,30 @@ func (g *GCETargetHttpsProxies) SetSslCertificates(ctx context.Context, key *met Version: meta.Version("ga"), Service: "TargetHttpsProxies", } - glog.V(5).Infof("GCETargetHttpsProxies.SetSslCertificates(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetHttpsProxies.SetSslCertificates(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetHttpsProxies.SetSslCertificates(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpsProxies.SetSslCertificates(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.TargetHttpsProxies.SetSslCertificates(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCETargetHttpsProxies.SetSslCertificates(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpsProxies.SetSslCertificates(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCETargetHttpsProxies.SetSslCertificates(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpsProxies.SetSslCertificates(%v, %v, ...) = %+v", ctx, key, err) return err } // SetUrlMap is a method on GCETargetHttpsProxies. func (g *GCETargetHttpsProxies) SetUrlMap(ctx context.Context, key *meta.Key, arg0 *ga.UrlMapReference) error { - glog.V(5).Infof("GCETargetHttpsProxies.SetUrlMap(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCETargetHttpsProxies.SetUrlMap(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCETargetHttpsProxies.SetUrlMap(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetHttpsProxies.SetUrlMap(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetHttpsProxies") @@ -13979,21 +13979,21 @@ func (g *GCETargetHttpsProxies) SetUrlMap(ctx context.Context, key *meta.Key, ar Version: meta.Version("ga"), Service: "TargetHttpsProxies", } - glog.V(5).Infof("GCETargetHttpsProxies.SetUrlMap(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetHttpsProxies.SetUrlMap(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetHttpsProxies.SetUrlMap(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpsProxies.SetUrlMap(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.TargetHttpsProxies.SetUrlMap(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCETargetHttpsProxies.SetUrlMap(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpsProxies.SetUrlMap(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCETargetHttpsProxies.SetUrlMap(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCETargetHttpsProxies.SetUrlMap(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -14056,7 +14056,7 @@ type MockTargetPools struct { func (m *MockTargetPools) Get(ctx context.Context, key *meta.Key) (*ga.TargetPool, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockTargetPools.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockTargetPools.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -14068,12 +14068,12 @@ func (m *MockTargetPools) Get(ctx context.Context, key *meta.Key) (*ga.TargetPoo defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockTargetPools.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockTargetPools.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockTargetPools.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockTargetPools.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -14081,7 +14081,7 @@ func (m *MockTargetPools) Get(ctx context.Context, key *meta.Key) (*ga.TargetPoo Code: http.StatusNotFound, Message: fmt.Sprintf("MockTargetPools %v not found", key), } - glog.V(5).Infof("MockTargetPools.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockTargetPools.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -14089,7 +14089,7 @@ func (m *MockTargetPools) Get(ctx context.Context, key *meta.Key) (*ga.TargetPoo func (m *MockTargetPools) List(ctx context.Context, region string, fl *filter.F) ([]*ga.TargetPool, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, region, fl, m); intercept { - glog.V(5).Infof("MockTargetPools.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) + klog.V(5).Infof("MockTargetPools.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) return objs, err } } @@ -14099,7 +14099,7 @@ func (m *MockTargetPools) List(ctx context.Context, region string, fl *filter.F) if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockTargetPools.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) + klog.V(5).Infof("MockTargetPools.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) return nil, *m.ListError } @@ -14115,7 +14115,7 @@ func (m *MockTargetPools) List(ctx context.Context, region string, fl *filter.F) objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockTargetPools.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) + klog.V(5).Infof("MockTargetPools.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) return objs, nil } @@ -14123,7 +14123,7 @@ func (m *MockTargetPools) List(ctx context.Context, region string, fl *filter.F) func (m *MockTargetPools) Insert(ctx context.Context, key *meta.Key, obj *ga.TargetPool) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockTargetPools.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockTargetPools.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -14135,7 +14135,7 @@ func (m *MockTargetPools) Insert(ctx context.Context, key *meta.Key, obj *ga.Tar defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockTargetPools.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockTargetPools.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -14143,7 +14143,7 @@ func (m *MockTargetPools) Insert(ctx context.Context, key *meta.Key, obj *ga.Tar Code: http.StatusConflict, Message: fmt.Sprintf("MockTargetPools %v exists", key), } - glog.V(5).Infof("MockTargetPools.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockTargetPools.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -14152,7 +14152,7 @@ func (m *MockTargetPools) Insert(ctx context.Context, key *meta.Key, obj *ga.Tar obj.SelfLink = SelfLink(meta.VersionGA, projectID, "targetPools", key) m.Objects[*key] = &MockTargetPoolsObj{obj} - glog.V(5).Infof("MockTargetPools.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockTargetPools.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -14160,7 +14160,7 @@ func (m *MockTargetPools) Insert(ctx context.Context, key *meta.Key, obj *ga.Tar func (m *MockTargetPools) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockTargetPools.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockTargetPools.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -14172,7 +14172,7 @@ func (m *MockTargetPools) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockTargetPools.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockTargetPools.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -14180,12 +14180,12 @@ func (m *MockTargetPools) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockTargetPools %v not found", key), } - glog.V(5).Infof("MockTargetPools.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockTargetPools.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockTargetPools.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockTargetPools.Delete(%v, %v) = nil", ctx, key) return nil } @@ -14217,10 +14217,10 @@ type GCETargetPools struct { // Get the TargetPool named by key. func (g *GCETargetPools) Get(ctx context.Context, key *meta.Key) (*ga.TargetPool, error) { - glog.V(5).Infof("GCETargetPools.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCETargetPools.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCETargetPools.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetPools.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetPools") @@ -14230,21 +14230,21 @@ func (g *GCETargetPools) Get(ctx context.Context, key *meta.Key) (*ga.TargetPool Version: meta.Version("ga"), Service: "TargetPools", } - glog.V(5).Infof("GCETargetPools.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetPools.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetPools.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetPools.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.TargetPools.Get(projectID, key.Region, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCETargetPools.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCETargetPools.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all TargetPool objects. func (g *GCETargetPools) List(ctx context.Context, region string, fl *filter.F) ([]*ga.TargetPool, error) { - glog.V(5).Infof("GCETargetPools.List(%v, %v, %v) called", ctx, region, fl) + klog.V(5).Infof("GCETargetPools.List(%v, %v, %v) called", ctx, region, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetPools") rk := &RateLimitKey{ ProjectID: projectID, @@ -14255,30 +14255,30 @@ func (g *GCETargetPools) List(ctx context.Context, region string, fl *filter.F) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCETargetPools.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) + klog.V(5).Infof("GCETargetPools.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) call := g.s.GA.TargetPools.List(projectID, region) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.TargetPool f := func(l *ga.TargetPoolList) error { - glog.V(5).Infof("GCETargetPools.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCETargetPools.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCETargetPools.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCETargetPools.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCETargetPools.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCETargetPools.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCETargetPools.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCETargetPools.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -14286,9 +14286,9 @@ func (g *GCETargetPools) List(ctx context.Context, region string, fl *filter.F) // Insert TargetPool with key of value obj. func (g *GCETargetPools) Insert(ctx context.Context, key *meta.Key, obj *ga.TargetPool) error { - glog.V(5).Infof("GCETargetPools.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCETargetPools.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCETargetPools.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetPools.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetPools") @@ -14298,9 +14298,9 @@ func (g *GCETargetPools) Insert(ctx context.Context, key *meta.Key, obj *ga.Targ Version: meta.Version("ga"), Service: "TargetPools", } - glog.V(5).Infof("GCETargetPools.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetPools.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetPools.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetPools.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -14309,20 +14309,20 @@ func (g *GCETargetPools) Insert(ctx context.Context, key *meta.Key, obj *ga.Targ op, err := call.Do() if err != nil { - glog.V(4).Infof("GCETargetPools.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCETargetPools.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCETargetPools.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCETargetPools.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the TargetPool referenced by key. func (g *GCETargetPools) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCETargetPools.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCETargetPools.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCETargetPools.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetPools.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetPools") @@ -14332,9 +14332,9 @@ func (g *GCETargetPools) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "TargetPools", } - glog.V(5).Infof("GCETargetPools.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetPools.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetPools.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetPools.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.TargetPools.Delete(projectID, key.Region, key.Name) @@ -14342,21 +14342,21 @@ func (g *GCETargetPools) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCETargetPools.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCETargetPools.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCETargetPools.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCETargetPools.Delete(%v, %v) = %v", ctx, key, err) return err } // AddInstance is a method on GCETargetPools. func (g *GCETargetPools) AddInstance(ctx context.Context, key *meta.Key, arg0 *ga.TargetPoolsAddInstanceRequest) error { - glog.V(5).Infof("GCETargetPools.AddInstance(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCETargetPools.AddInstance(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCETargetPools.AddInstance(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetPools.AddInstance(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetPools") @@ -14366,30 +14366,30 @@ func (g *GCETargetPools) AddInstance(ctx context.Context, key *meta.Key, arg0 *g Version: meta.Version("ga"), Service: "TargetPools", } - glog.V(5).Infof("GCETargetPools.AddInstance(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetPools.AddInstance(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetPools.AddInstance(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetPools.AddInstance(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.TargetPools.AddInstance(projectID, key.Region, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCETargetPools.AddInstance(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCETargetPools.AddInstance(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCETargetPools.AddInstance(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCETargetPools.AddInstance(%v, %v, ...) = %+v", ctx, key, err) return err } // RemoveInstance is a method on GCETargetPools. func (g *GCETargetPools) RemoveInstance(ctx context.Context, key *meta.Key, arg0 *ga.TargetPoolsRemoveInstanceRequest) error { - glog.V(5).Infof("GCETargetPools.RemoveInstance(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCETargetPools.RemoveInstance(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCETargetPools.RemoveInstance(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCETargetPools.RemoveInstance(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "TargetPools") @@ -14399,21 +14399,21 @@ func (g *GCETargetPools) RemoveInstance(ctx context.Context, key *meta.Key, arg0 Version: meta.Version("ga"), Service: "TargetPools", } - glog.V(5).Infof("GCETargetPools.RemoveInstance(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCETargetPools.RemoveInstance(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCETargetPools.RemoveInstance(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCETargetPools.RemoveInstance(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.TargetPools.RemoveInstance(projectID, key.Region, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCETargetPools.RemoveInstance(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCETargetPools.RemoveInstance(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCETargetPools.RemoveInstance(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCETargetPools.RemoveInstance(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -14474,7 +14474,7 @@ type MockUrlMaps struct { func (m *MockUrlMaps) Get(ctx context.Context, key *meta.Key) (*ga.UrlMap, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockUrlMaps.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockUrlMaps.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -14486,12 +14486,12 @@ func (m *MockUrlMaps) Get(ctx context.Context, key *meta.Key) (*ga.UrlMap, error defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockUrlMaps.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockUrlMaps.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockUrlMaps.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockUrlMaps.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -14499,7 +14499,7 @@ func (m *MockUrlMaps) Get(ctx context.Context, key *meta.Key) (*ga.UrlMap, error Code: http.StatusNotFound, Message: fmt.Sprintf("MockUrlMaps %v not found", key), } - glog.V(5).Infof("MockUrlMaps.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockUrlMaps.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -14507,7 +14507,7 @@ func (m *MockUrlMaps) Get(ctx context.Context, key *meta.Key) (*ga.UrlMap, error func (m *MockUrlMaps) List(ctx context.Context, fl *filter.F) ([]*ga.UrlMap, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockUrlMaps.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockUrlMaps.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -14517,7 +14517,7 @@ func (m *MockUrlMaps) List(ctx context.Context, fl *filter.F) ([]*ga.UrlMap, err if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockUrlMaps.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockUrlMaps.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -14530,7 +14530,7 @@ func (m *MockUrlMaps) List(ctx context.Context, fl *filter.F) ([]*ga.UrlMap, err objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockUrlMaps.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockUrlMaps.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -14538,7 +14538,7 @@ func (m *MockUrlMaps) List(ctx context.Context, fl *filter.F) ([]*ga.UrlMap, err func (m *MockUrlMaps) Insert(ctx context.Context, key *meta.Key, obj *ga.UrlMap) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("MockUrlMaps.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockUrlMaps.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -14550,7 +14550,7 @@ func (m *MockUrlMaps) Insert(ctx context.Context, key *meta.Key, obj *ga.UrlMap) defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("MockUrlMaps.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockUrlMaps.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -14558,7 +14558,7 @@ func (m *MockUrlMaps) Insert(ctx context.Context, key *meta.Key, obj *ga.UrlMap) Code: http.StatusConflict, Message: fmt.Sprintf("MockUrlMaps %v exists", key), } - glog.V(5).Infof("MockUrlMaps.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("MockUrlMaps.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -14567,7 +14567,7 @@ func (m *MockUrlMaps) Insert(ctx context.Context, key *meta.Key, obj *ga.UrlMap) obj.SelfLink = SelfLink(meta.VersionGA, projectID, "urlMaps", key) m.Objects[*key] = &MockUrlMapsObj{obj} - glog.V(5).Infof("MockUrlMaps.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("MockUrlMaps.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } @@ -14575,7 +14575,7 @@ func (m *MockUrlMaps) Insert(ctx context.Context, key *meta.Key, obj *ga.UrlMap) func (m *MockUrlMaps) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("MockUrlMaps.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockUrlMaps.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -14587,7 +14587,7 @@ func (m *MockUrlMaps) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("MockUrlMaps.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockUrlMaps.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -14595,12 +14595,12 @@ func (m *MockUrlMaps) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("MockUrlMaps %v not found", key), } - glog.V(5).Infof("MockUrlMaps.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("MockUrlMaps.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("MockUrlMaps.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("MockUrlMaps.Delete(%v, %v) = nil", ctx, key) return nil } @@ -14624,10 +14624,10 @@ type GCEUrlMaps struct { // Get the UrlMap named by key. func (g *GCEUrlMaps) Get(ctx context.Context, key *meta.Key) (*ga.UrlMap, error) { - glog.V(5).Infof("GCEUrlMaps.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEUrlMaps.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEUrlMaps.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEUrlMaps.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "UrlMaps") @@ -14637,21 +14637,21 @@ func (g *GCEUrlMaps) Get(ctx context.Context, key *meta.Key) (*ga.UrlMap, error) Version: meta.Version("ga"), Service: "UrlMaps", } - glog.V(5).Infof("GCEUrlMaps.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEUrlMaps.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEUrlMaps.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEUrlMaps.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.UrlMaps.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEUrlMaps.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEUrlMaps.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all UrlMap objects. func (g *GCEUrlMaps) List(ctx context.Context, fl *filter.F) ([]*ga.UrlMap, error) { - glog.V(5).Infof("GCEUrlMaps.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEUrlMaps.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "UrlMaps") rk := &RateLimitKey{ ProjectID: projectID, @@ -14662,30 +14662,30 @@ func (g *GCEUrlMaps) List(ctx context.Context, fl *filter.F) ([]*ga.UrlMap, erro if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEUrlMaps.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEUrlMaps.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.UrlMaps.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.UrlMap f := func(l *ga.UrlMapList) error { - glog.V(5).Infof("GCEUrlMaps.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEUrlMaps.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEUrlMaps.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEUrlMaps.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEUrlMaps.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEUrlMaps.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEUrlMaps.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEUrlMaps.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -14693,9 +14693,9 @@ func (g *GCEUrlMaps) List(ctx context.Context, fl *filter.F) ([]*ga.UrlMap, erro // Insert UrlMap with key of value obj. func (g *GCEUrlMaps) Insert(ctx context.Context, key *meta.Key, obj *ga.UrlMap) error { - glog.V(5).Infof("GCEUrlMaps.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("GCEUrlMaps.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("GCEUrlMaps.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEUrlMaps.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "UrlMaps") @@ -14705,9 +14705,9 @@ func (g *GCEUrlMaps) Insert(ctx context.Context, key *meta.Key, obj *ga.UrlMap) Version: meta.Version("ga"), Service: "UrlMaps", } - glog.V(5).Infof("GCEUrlMaps.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEUrlMaps.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEUrlMaps.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEUrlMaps.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -14716,20 +14716,20 @@ func (g *GCEUrlMaps) Insert(ctx context.Context, key *meta.Key, obj *ga.UrlMap) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEUrlMaps.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEUrlMaps.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEUrlMaps.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("GCEUrlMaps.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } // Delete the UrlMap referenced by key. func (g *GCEUrlMaps) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("GCEUrlMaps.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEUrlMaps.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEUrlMaps.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEUrlMaps.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "UrlMaps") @@ -14739,9 +14739,9 @@ func (g *GCEUrlMaps) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("ga"), Service: "UrlMaps", } - glog.V(5).Infof("GCEUrlMaps.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEUrlMaps.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEUrlMaps.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEUrlMaps.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.UrlMaps.Delete(projectID, key.Name) @@ -14750,21 +14750,21 @@ func (g *GCEUrlMaps) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEUrlMaps.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEUrlMaps.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEUrlMaps.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("GCEUrlMaps.Delete(%v, %v) = %v", ctx, key, err) return err } // Update is a method on GCEUrlMaps. func (g *GCEUrlMaps) Update(ctx context.Context, key *meta.Key, arg0 *ga.UrlMap) error { - glog.V(5).Infof("GCEUrlMaps.Update(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("GCEUrlMaps.Update(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEUrlMaps.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEUrlMaps.Update(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "UrlMaps") @@ -14774,21 +14774,21 @@ func (g *GCEUrlMaps) Update(ctx context.Context, key *meta.Key, arg0 *ga.UrlMap) Version: meta.Version("ga"), Service: "UrlMaps", } - glog.V(5).Infof("GCEUrlMaps.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEUrlMaps.Update(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEUrlMaps.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEUrlMaps.Update(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } call := g.s.GA.UrlMaps.Update(projectID, key.Name, arg0) call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("GCEUrlMaps.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEUrlMaps.Update(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("GCEUrlMaps.Update(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("GCEUrlMaps.Update(%v, %v, ...) = %+v", ctx, key, err) return err } @@ -14839,7 +14839,7 @@ type MockZones struct { func (m *MockZones) Get(ctx context.Context, key *meta.Key) (*ga.Zone, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("MockZones.Get(%v, %s) = %+v, %v", ctx, key, obj, err) + klog.V(5).Infof("MockZones.Get(%v, %s) = %+v, %v", ctx, key, obj, err) return obj, err } } @@ -14851,12 +14851,12 @@ func (m *MockZones) Get(ctx context.Context, key *meta.Key) (*ga.Zone, error) { defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("MockZones.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockZones.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.ToGA() - glog.V(5).Infof("MockZones.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("MockZones.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -14864,7 +14864,7 @@ func (m *MockZones) Get(ctx context.Context, key *meta.Key) (*ga.Zone, error) { Code: http.StatusNotFound, Message: fmt.Sprintf("MockZones %v not found", key), } - glog.V(5).Infof("MockZones.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("MockZones.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } @@ -14872,7 +14872,7 @@ func (m *MockZones) Get(ctx context.Context, key *meta.Key) (*ga.Zone, error) { func (m *MockZones) List(ctx context.Context, fl *filter.F) ([]*ga.Zone, error) { if m.ListHook != nil { if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("MockZones.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("MockZones.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -14882,7 +14882,7 @@ func (m *MockZones) List(ctx context.Context, fl *filter.F) ([]*ga.Zone, error) if m.ListError != nil { err := *m.ListError - glog.V(5).Infof("MockZones.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("MockZones.List(%v, %v) = nil, %v", ctx, fl, err) return nil, *m.ListError } @@ -14895,7 +14895,7 @@ func (m *MockZones) List(ctx context.Context, fl *filter.F) ([]*ga.Zone, error) objs = append(objs, obj.ToGA()) } - glog.V(5).Infof("MockZones.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("MockZones.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } @@ -14911,10 +14911,10 @@ type GCEZones struct { // Get the Zone named by key. func (g *GCEZones) Get(ctx context.Context, key *meta.Key) (*ga.Zone, error) { - glog.V(5).Infof("GCEZones.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("GCEZones.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("GCEZones.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("GCEZones.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Zones") @@ -14924,21 +14924,21 @@ func (g *GCEZones) Get(ctx context.Context, key *meta.Key) (*ga.Zone, error) { Version: meta.Version("ga"), Service: "Zones", } - glog.V(5).Infof("GCEZones.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("GCEZones.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("GCEZones.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("GCEZones.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } call := g.s.GA.Zones.Get(projectID, key.Name) call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("GCEZones.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("GCEZones.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } // List all Zone objects. func (g *GCEZones) List(ctx context.Context, fl *filter.F) ([]*ga.Zone, error) { - glog.V(5).Infof("GCEZones.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("GCEZones.List(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "Zones") rk := &RateLimitKey{ ProjectID: projectID, @@ -14949,30 +14949,30 @@ func (g *GCEZones) List(ctx context.Context, fl *filter.F) ([]*ga.Zone, error) { if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { return nil, err } - glog.V(5).Infof("GCEZones.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("GCEZones.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.GA.Zones.List(projectID) if fl != filter.None { call.Filter(fl.String()) } var all []*ga.Zone f := func(l *ga.ZoneList) error { - glog.V(5).Infof("GCEZones.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("GCEZones.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("GCEZones.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("GCEZones.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("GCEZones.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("GCEZones.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("GCEZones.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("GCEZones.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil diff --git a/pkg/cloudprovider/providers/gce/cloud/gen/main.go b/pkg/cloudprovider/providers/gce/cloud/gen/main.go index 6ff2383c465..ff45ccb3e11 100644 --- a/pkg/cloudprovider/providers/gce/cloud/gen/main.go +++ b/pkg/cloudprovider/providers/gce/cloud/gen/main.go @@ -99,7 +99,7 @@ import ( "sync" "google.golang.org/api/googleapi" - "github.com/golang/glog" + "k8s.io/klog" "{{.PackageRoot}}/filter" "{{.PackageRoot}}/meta" @@ -219,7 +219,7 @@ func (m *Mock{{.Service}}Obj) ToAlpha() *{{.Alpha.FQObjectType}} { // Convert the object via JSON copying to the type that was requested. ret := &{{.Alpha.FQObjectType}}{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *{{.Alpha.FQObjectType}} via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *{{.Alpha.FQObjectType}} via JSON: %v", m.Obj, err) } return ret } @@ -233,7 +233,7 @@ func (m *Mock{{.Service}}Obj) ToBeta() *{{.Beta.FQObjectType}} { // Convert the object via JSON copying to the type that was requested. ret := &{{.Beta.FQObjectType}}{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *{{.Beta.FQObjectType}} via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *{{.Beta.FQObjectType}} via JSON: %v", m.Obj, err) } return ret } @@ -247,7 +247,7 @@ func (m *Mock{{.Service}}Obj) ToGA() *{{.GA.FQObjectType}} { // Convert the object via JSON copying to the type that was requested. ret := &{{.GA.FQObjectType}}{} if err := copyViaJSON(ret, m.Obj); err != nil { - glog.Errorf("Could not convert %T to *{{.GA.FQObjectType}} via JSON: %v", m.Obj, err) + klog.Errorf("Could not convert %T to *{{.GA.FQObjectType}} via JSON: %v", m.Obj, err) } return ret } @@ -394,7 +394,7 @@ type {{.MockWrapType}} struct { func (m *{{.MockWrapType}}) Get(ctx context.Context, key *meta.Key) (*{{.FQObjectType}}, error) { if m.GetHook != nil { if intercept, obj, err := m.GetHook(ctx, key, m); intercept { - glog.V(5).Infof("{{.MockWrapType}}.Get(%v, %s) = %+v, %v", ctx, key, obj ,err) + klog.V(5).Infof("{{.MockWrapType}}.Get(%v, %s) = %+v, %v", ctx, key, obj ,err) return obj, err } } @@ -406,12 +406,12 @@ func (m *{{.MockWrapType}}) Get(ctx context.Context, key *meta.Key) (*{{.FQObjec defer m.Lock.Unlock() if err, ok := m.GetError[*key]; ok { - glog.V(5).Infof("{{.MockWrapType}}.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("{{.MockWrapType}}.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } if obj, ok := m.Objects[*key]; ok { typedObj := obj.To{{.VersionTitle}}() - glog.V(5).Infof("{{.MockWrapType}}.Get(%v, %s) = %+v, nil", ctx, key, typedObj) + klog.V(5).Infof("{{.MockWrapType}}.Get(%v, %s) = %+v, nil", ctx, key, typedObj) return typedObj, nil } @@ -419,7 +419,7 @@ func (m *{{.MockWrapType}}) Get(ctx context.Context, key *meta.Key) (*{{.FQObjec Code: http.StatusNotFound, Message: fmt.Sprintf("{{.MockWrapType}} %v not found", key), } - glog.V(5).Infof("{{.MockWrapType}}.Get(%v, %s) = nil, %v", ctx, key, err) + klog.V(5).Infof("{{.MockWrapType}}.Get(%v, %s) = nil, %v", ctx, key, err) return nil, err } {{- end}} @@ -440,15 +440,15 @@ func (m *{{.MockWrapType}}) List(ctx context.Context, zone string, fl *filter.F) if m.ListHook != nil { {{if .KeyIsGlobal -}} if intercept, objs, err := m.ListHook(ctx, fl, m); intercept { - glog.V(5).Infof("{{.MockWrapType}}.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("{{.MockWrapType}}.List(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) {{- end -}} {{- if .KeyIsRegional -}} if intercept, objs, err := m.ListHook(ctx, region, fl, m); intercept { - glog.V(5).Infof("{{.MockWrapType}}.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) + klog.V(5).Infof("{{.MockWrapType}}.List(%v, %q, %v) = [%v items], %v", ctx, region, fl, len(objs), err) {{- end -}} {{- if .KeyIsZonal -}} if intercept, objs, err := m.ListHook(ctx, zone, fl, m); intercept { - glog.V(5).Infof("{{.MockWrapType}}.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) + klog.V(5).Infof("{{.MockWrapType}}.List(%v, %q, %v) = [%v items], %v", ctx, zone, fl, len(objs), err) {{- end}} return objs, err } @@ -460,13 +460,13 @@ func (m *{{.MockWrapType}}) List(ctx context.Context, zone string, fl *filter.F) if m.ListError != nil { err := *m.ListError {{if .KeyIsGlobal -}} - glog.V(5).Infof("{{.MockWrapType}}.List(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("{{.MockWrapType}}.List(%v, %v) = nil, %v", ctx, fl, err) {{- end -}} {{- if .KeyIsRegional -}} - glog.V(5).Infof("{{.MockWrapType}}.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) + klog.V(5).Infof("{{.MockWrapType}}.List(%v, %q, %v) = nil, %v", ctx, region, fl, err) {{- end -}} {{- if .KeyIsZonal -}} - glog.V(5).Infof("{{.MockWrapType}}.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) + klog.V(5).Infof("{{.MockWrapType}}.List(%v, %q, %v) = nil, %v", ctx, zone, fl, err) {{- end}} return nil, *m.ListError @@ -495,13 +495,13 @@ func (m *{{.MockWrapType}}) List(ctx context.Context, zone string, fl *filter.F) } {{if .KeyIsGlobal -}} - glog.V(5).Infof("{{.MockWrapType}}.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("{{.MockWrapType}}.List(%v, %v) = [%v items], nil", ctx, fl, len(objs)) {{- end -}} {{- if .KeyIsRegional -}} - glog.V(5).Infof("{{.MockWrapType}}.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) + klog.V(5).Infof("{{.MockWrapType}}.List(%v, %q, %v) = [%v items], nil", ctx, region, fl, len(objs)) {{- end -}} {{- if .KeyIsZonal -}} - glog.V(5).Infof("{{.MockWrapType}}.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) + klog.V(5).Infof("{{.MockWrapType}}.List(%v, %q, %v) = [%v items], nil", ctx, zone, fl, len(objs)) {{- end}} return objs, nil } @@ -512,7 +512,7 @@ func (m *{{.MockWrapType}}) List(ctx context.Context, zone string, fl *filter.F) func (m *{{.MockWrapType}}) Insert(ctx context.Context, key *meta.Key, obj *{{.FQObjectType}}) error { if m.InsertHook != nil { if intercept, err := m.InsertHook(ctx, key, obj, m); intercept { - glog.V(5).Infof("{{.MockWrapType}}.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("{{.MockWrapType}}.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } } @@ -524,7 +524,7 @@ func (m *{{.MockWrapType}}) Insert(ctx context.Context, key *meta.Key, obj *{{.F defer m.Lock.Unlock() if err, ok := m.InsertError[*key]; ok { - glog.V(5).Infof("{{.MockWrapType}}.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("{{.MockWrapType}}.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } if _, ok := m.Objects[*key]; ok { @@ -532,7 +532,7 @@ func (m *{{.MockWrapType}}) Insert(ctx context.Context, key *meta.Key, obj *{{.F Code: http.StatusConflict, Message: fmt.Sprintf("{{.MockWrapType}} %v exists", key), } - glog.V(5).Infof("{{.MockWrapType}}.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) + klog.V(5).Infof("{{.MockWrapType}}.Insert(%v, %v, %+v) = %v", ctx, key, obj, err) return err } @@ -541,7 +541,7 @@ func (m *{{.MockWrapType}}) Insert(ctx context.Context, key *meta.Key, obj *{{.F obj.SelfLink = SelfLink(meta.Version{{.VersionTitle}}, projectID, "{{.Resource}}", key) m.Objects[*key] = &Mock{{.Service}}Obj{obj} - glog.V(5).Infof("{{.MockWrapType}}.Insert(%v, %v, %+v) = nil", ctx, key, obj) + klog.V(5).Infof("{{.MockWrapType}}.Insert(%v, %v, %+v) = nil", ctx, key, obj) return nil } {{- end}} @@ -551,7 +551,7 @@ func (m *{{.MockWrapType}}) Insert(ctx context.Context, key *meta.Key, obj *{{.F func (m *{{.MockWrapType}}) Delete(ctx context.Context, key *meta.Key) error { if m.DeleteHook != nil { if intercept, err := m.DeleteHook(ctx, key, m); intercept { - glog.V(5).Infof("{{.MockWrapType}}.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("{{.MockWrapType}}.Delete(%v, %v) = %v", ctx, key, err) return err } } @@ -563,7 +563,7 @@ func (m *{{.MockWrapType}}) Delete(ctx context.Context, key *meta.Key) error { defer m.Lock.Unlock() if err, ok := m.DeleteError[*key]; ok { - glog.V(5).Infof("{{.MockWrapType}}.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("{{.MockWrapType}}.Delete(%v, %v) = %v", ctx, key, err) return err } if _, ok := m.Objects[*key]; !ok { @@ -571,12 +571,12 @@ func (m *{{.MockWrapType}}) Delete(ctx context.Context, key *meta.Key) error { Code: http.StatusNotFound, Message: fmt.Sprintf("{{.MockWrapType}} %v not found", key), } - glog.V(5).Infof("{{.MockWrapType}}.Delete(%v, %v) = %v", ctx, key, err) + klog.V(5).Infof("{{.MockWrapType}}.Delete(%v, %v) = %v", ctx, key, err) return err } delete(m.Objects, *key) - glog.V(5).Infof("{{.MockWrapType}}.Delete(%v, %v) = nil", ctx, key) + klog.V(5).Infof("{{.MockWrapType}}.Delete(%v, %v) = nil", ctx, key) return nil } {{- end}} @@ -586,7 +586,7 @@ func (m *{{.MockWrapType}}) Delete(ctx context.Context, key *meta.Key) error { func (m *{{.MockWrapType}}) AggregatedList(ctx context.Context, fl *filter.F) (map[string][]*{{.FQObjectType}}, error) { if m.AggregatedListHook != nil { if intercept, objs, err := m.AggregatedListHook(ctx, fl, m); intercept { - glog.V(5).Infof("{{.MockWrapType}}.AggregatedList(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) + klog.V(5).Infof("{{.MockWrapType}}.AggregatedList(%v, %v) = [%v items], %v", ctx, fl, len(objs), err) return objs, err } } @@ -596,7 +596,7 @@ func (m *{{.MockWrapType}}) AggregatedList(ctx context.Context, fl *filter.F) (m if m.AggregatedListError != nil { err := *m.AggregatedListError - glog.V(5).Infof("{{.MockWrapType}}.AggregatedList(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("{{.MockWrapType}}.AggregatedList(%v, %v) = nil, %v", ctx, fl, err) return nil, err } @@ -610,7 +610,7 @@ func (m *{{.MockWrapType}}) AggregatedList(ctx context.Context, fl *filter.F) (m location := res.Key.Zone {{- end}} if err != nil { - glog.V(5).Infof("{{.MockWrapType}}.AggregatedList(%v, %v) = nil, %v", ctx, fl, err) + klog.V(5).Infof("{{.MockWrapType}}.AggregatedList(%v, %v) = nil, %v", ctx, fl, err) return nil, err } if !fl.Match(obj.To{{.VersionTitle}}()) { @@ -618,7 +618,7 @@ func (m *{{.MockWrapType}}) AggregatedList(ctx context.Context, fl *filter.F) (m } objs[location] = append(objs[location], obj.To{{.VersionTitle}}()) } - glog.V(5).Infof("{{.MockWrapType}}.AggregatedList(%v, %v) = [%v items], nil", ctx, fl, len(objs)) + klog.V(5).Infof("{{.MockWrapType}}.AggregatedList(%v, %v) = [%v items], nil", ctx, fl, len(objs)) return objs, nil } {{- end}} @@ -659,10 +659,10 @@ type {{.GCEWrapType}} struct { {{- if .GenerateGet}} // Get the {{.Object}} named by key. func (g *{{.GCEWrapType}}) Get(ctx context.Context, key *meta.Key) (*{{.FQObjectType}}, error) { - glog.V(5).Infof("{{.GCEWrapType}}.Get(%v, %v): called", ctx, key) + klog.V(5).Infof("{{.GCEWrapType}}.Get(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("{{.GCEWrapType}}.Get(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("{{.GCEWrapType}}.Get(%v, %v): key is invalid (%#v)", ctx, key, key) return nil, fmt.Errorf("invalid GCE key (%#v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "{{.Version}}", "{{.Service}}") @@ -672,9 +672,9 @@ func (g *{{.GCEWrapType}}) Get(ctx context.Context, key *meta.Key) (*{{.FQObject Version: meta.Version("{{.Version}}"), Service: "{{.Service}}", } - glog.V(5).Infof("{{.GCEWrapType}}.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("{{.GCEWrapType}}.Get(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("{{.GCEWrapType}}.Get(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("{{.GCEWrapType}}.Get(%v, %v): RateLimiter error: %v", ctx, key, err) return nil, err } {{- if .KeyIsGlobal}} @@ -688,7 +688,7 @@ func (g *{{.GCEWrapType}}) Get(ctx context.Context, key *meta.Key) (*{{.FQObject {{- end}} call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("{{.GCEWrapType}}.Get(%v, %v) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("{{.GCEWrapType}}.Get(%v, %v) = %+v, %v", ctx, key, v, err) return v, err } {{- end}} @@ -697,15 +697,15 @@ func (g *{{.GCEWrapType}}) Get(ctx context.Context, key *meta.Key) (*{{.FQObject // List all {{.Object}} objects. {{- if .KeyIsGlobal}} func (g *{{.GCEWrapType}}) List(ctx context.Context, fl *filter.F) ([]*{{.FQObjectType}}, error) { - glog.V(5).Infof("{{.GCEWrapType}}.List(%v, %v) called", ctx, fl) + klog.V(5).Infof("{{.GCEWrapType}}.List(%v, %v) called", ctx, fl) {{- end -}} {{- if .KeyIsRegional}} func (g *{{.GCEWrapType}}) List(ctx context.Context, region string, fl *filter.F) ([]*{{.FQObjectType}}, error) { - glog.V(5).Infof("{{.GCEWrapType}}.List(%v, %v, %v) called", ctx, region, fl) + klog.V(5).Infof("{{.GCEWrapType}}.List(%v, %v, %v) called", ctx, region, fl) {{- end -}} {{- if .KeyIsZonal}} func (g *{{.GCEWrapType}}) List(ctx context.Context, zone string, fl *filter.F) ([]*{{.FQObjectType}}, error) { - glog.V(5).Infof("{{.GCEWrapType}}.List(%v, %v, %v) called", ctx, zone, fl) + klog.V(5).Infof("{{.GCEWrapType}}.List(%v, %v, %v) called", ctx, zone, fl) {{- end}} projectID := g.s.ProjectRouter.ProjectID(ctx, "{{.Version}}", "{{.Service}}") rk := &RateLimitKey{ @@ -718,15 +718,15 @@ func (g *{{.GCEWrapType}}) List(ctx context.Context, zone string, fl *filter.F) return nil, err } {{- if .KeyIsGlobal}} - glog.V(5).Infof("{{.GCEWrapType}}.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("{{.GCEWrapType}}.List(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) call := g.s.{{.VersionTitle}}.{{.Service}}.List(projectID) {{- end -}} {{- if .KeyIsRegional}} - glog.V(5).Infof("{{.GCEWrapType}}.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) + klog.V(5).Infof("{{.GCEWrapType}}.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, region, fl, projectID, rk) call := g.s.{{.VersionTitle}}.{{.Service}}.List(projectID, region) {{- end -}} {{- if .KeyIsZonal}} - glog.V(5).Infof("{{.GCEWrapType}}.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) + klog.V(5).Infof("{{.GCEWrapType}}.List(%v, %v, %v): projectID = %v, rk = %+v", ctx, zone, fl, projectID, rk) call := g.s.{{.VersionTitle}}.{{.Service}}.List(projectID, zone) {{- end}} if fl != filter.None { @@ -734,23 +734,23 @@ func (g *{{.GCEWrapType}}) List(ctx context.Context, zone string, fl *filter.F) } var all []*{{.FQObjectType}} f := func(l *{{.ObjectListType}}) error { - glog.V(5).Infof("{{.GCEWrapType}}.List(%v, ..., %v): page %+v", ctx, fl, l) + klog.V(5).Infof("{{.GCEWrapType}}.List(%v, ..., %v): page %+v", ctx, fl, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("{{.GCEWrapType}}.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("{{.GCEWrapType}}.List(%v, ..., %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("{{.GCEWrapType}}.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("{{.GCEWrapType}}.List(%v, ..., %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("{{.GCEWrapType}}.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("{{.GCEWrapType}}.List(%v, ..., %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil @@ -760,9 +760,9 @@ func (g *{{.GCEWrapType}}) List(ctx context.Context, zone string, fl *filter.F) {{- if .GenerateInsert}} // Insert {{.Object}} with key of value obj. func (g *{{.GCEWrapType}}) Insert(ctx context.Context, key *meta.Key, obj *{{.FQObjectType}}) error { - glog.V(5).Infof("{{.GCEWrapType}}.Insert(%v, %v, %+v): called", ctx, key, obj) + klog.V(5).Infof("{{.GCEWrapType}}.Insert(%v, %v, %+v): called", ctx, key, obj) if !key.Valid() { - glog.V(2).Infof("{{.GCEWrapType}}.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("{{.GCEWrapType}}.Insert(%v, %v, ...): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "{{.Version}}", "{{.Service}}") @@ -772,9 +772,9 @@ func (g *{{.GCEWrapType}}) Insert(ctx context.Context, key *meta.Key, obj *{{.FQ Version: meta.Version("{{.Version}}"), Service: "{{.Service}}", } - glog.V(5).Infof("{{.GCEWrapType}}.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("{{.GCEWrapType}}.Insert(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("{{.GCEWrapType}}.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("{{.GCEWrapType}}.Insert(%v, %v, ...): RateLimiter error: %v", ctx, key, err) return err } obj.Name = key.Name @@ -791,12 +791,12 @@ func (g *{{.GCEWrapType}}) Insert(ctx context.Context, key *meta.Key, obj *{{.FQ op, err := call.Do() if err != nil { - glog.V(4).Infof("{{.GCEWrapType}}.Insert(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("{{.GCEWrapType}}.Insert(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("{{.GCEWrapType}}.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) + klog.V(4).Infof("{{.GCEWrapType}}.Insert(%v, %v, %+v) = %+v", ctx, key, obj, err) return err } {{- end}} @@ -804,9 +804,9 @@ func (g *{{.GCEWrapType}}) Insert(ctx context.Context, key *meta.Key, obj *{{.FQ {{- if .GenerateDelete}} // Delete the {{.Object}} referenced by key. func (g *{{.GCEWrapType}}) Delete(ctx context.Context, key *meta.Key) error { - glog.V(5).Infof("{{.GCEWrapType}}.Delete(%v, %v): called", ctx, key) + klog.V(5).Infof("{{.GCEWrapType}}.Delete(%v, %v): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("{{.GCEWrapType}}.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("{{.GCEWrapType}}.Delete(%v, %v): key is invalid (%#v)", ctx, key, key) return fmt.Errorf("invalid GCE key (%+v)", key) } projectID := g.s.ProjectRouter.ProjectID(ctx, "{{.Version}}", "{{.Service}}") @@ -816,9 +816,9 @@ func (g *{{.GCEWrapType}}) Delete(ctx context.Context, key *meta.Key) error { Version: meta.Version("{{.Version}}"), Service: "{{.Service}}", } - glog.V(5).Infof("{{.GCEWrapType}}.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("{{.GCEWrapType}}.Delete(%v, %v): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("{{.GCEWrapType}}.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("{{.GCEWrapType}}.Delete(%v, %v): RateLimiter error: %v", ctx, key, err) return err } {{- if .KeyIsGlobal}} @@ -834,12 +834,12 @@ func (g *{{.GCEWrapType}}) Delete(ctx context.Context, key *meta.Key) error { op, err := call.Do() if err != nil { - glog.V(4).Infof("{{.GCEWrapType}}.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("{{.GCEWrapType}}.Delete(%v, %v) = %v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("{{.GCEWrapType}}.Delete(%v, %v) = %v", ctx, key, err) + klog.V(4).Infof("{{.GCEWrapType}}.Delete(%v, %v) = %v", ctx, key, err) return err } {{end -}} @@ -847,7 +847,7 @@ func (g *{{.GCEWrapType}}) Delete(ctx context.Context, key *meta.Key) error { {{- if .AggregatedList}} // AggregatedList lists all resources of the given type across all locations. func (g *{{.GCEWrapType}}) AggregatedList(ctx context.Context, fl *filter.F) (map[string][]*{{.FQObjectType}}, error) { - glog.V(5).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v) called", ctx, fl) + klog.V(5).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v) called", ctx, fl) projectID := g.s.ProjectRouter.ProjectID(ctx, "{{.Version}}", "{{.Service}}") rk := &RateLimitKey{ @@ -857,9 +857,9 @@ func (g *{{.GCEWrapType}}) AggregatedList(ctx context.Context, fl *filter.F) (ma Service: "{{.Service}}", } - glog.V(5).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) + klog.V(5).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v): projectID = %v, rk = %+v", ctx, fl, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(5).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v): RateLimiter error: %v", ctx, fl, err) + klog.V(5).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v): RateLimiter error: %v", ctx, fl, err) return nil, err } @@ -872,23 +872,23 @@ func (g *{{.GCEWrapType}}) AggregatedList(ctx context.Context, fl *filter.F) (ma all := map[string][]*{{.FQObjectType}}{} f := func(l *{{.ObjectAggregatedListType}}) error { for k, v := range l.Items { - glog.V(5).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v): page[%v]%+v", ctx, fl, k, v) + klog.V(5).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v): page[%v]%+v", ctx, fl, k, v) all[k] = append(all[k], v.{{.AggregatedListField}}...) } return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v) = %v, %v", ctx, fl, nil, err) + klog.V(4).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v) = %v, %v", ctx, fl, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v) = [%v items], %v", ctx, fl, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v) = [%v items], %v", ctx, fl, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v) = %v, %v", ctx, fl, asStr, nil) + klog.V(5).Infof("{{.GCEWrapType}}.AggregatedList(%v, %v) = %v, %v", ctx, fl, asStr, nil) } return all, nil } @@ -898,10 +898,10 @@ func (g *{{.GCEWrapType}}) AggregatedList(ctx context.Context, fl *filter.F) (ma {{- range .}} // {{.Name}} is a method on {{.GCEWrapType}}. func (g *{{.GCEWrapType}}) {{.FcnArgs}} { - glog.V(5).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...): called", ctx, key) + klog.V(5).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...): called", ctx, key) if !key.Valid() { - glog.V(2).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + klog.V(2).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...): key is invalid (%#v)", ctx, key, key) {{- if .IsOperation}} return fmt.Errorf("invalid GCE key (%+v)", key) {{- else if .IsGet}} @@ -917,10 +917,10 @@ func (g *{{.GCEWrapType}}) {{.FcnArgs}} { Version: meta.Version("{{.Version}}"), Service: "{{.Service}}", } - glog.V(5).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + klog.V(5).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { - glog.V(4).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + klog.V(4).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...): RateLimiter error: %v", ctx, key, err) {{- if .IsOperation}} return err {{- else}} @@ -940,36 +940,36 @@ func (g *{{.GCEWrapType}}) {{.FcnArgs}} { call.Context(ctx) op, err := call.Do() if err != nil { - glog.V(4).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...) = %+v", ctx, key, err) return err } err = g.s.WaitForCompletion(ctx, op) - glog.V(4).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...) = %+v", ctx, key, err) + klog.V(4).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...) = %+v", ctx, key, err) return err {{- else if .IsGet}} call.Context(ctx) v, err := call.Do() - glog.V(4).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...) = %+v, %v", ctx, key, v, err) + klog.V(4).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...) = %+v, %v", ctx, key, v, err) return v, err {{- else if .IsPaged}} var all []*{{.Version}}.{{.ItemType}} f := func(l *{{.Version}}.{{.ReturnType}}) error { - glog.V(5).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...): page %+v", ctx, key, l) + klog.V(5).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...): page %+v", ctx, key, l) all = append(all, l.Items...) return nil } if err := call.Pages(ctx, f); err != nil { - glog.V(4).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...) = %v, %v", ctx, key, nil, err) + klog.V(4).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...) = %v, %v", ctx, key, nil, err) return nil, err } - if glog.V(4) { - glog.V(4).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...) = [%v items], %v", ctx, key, len(all), nil) - } else if glog.V(5) { + if klog.V(4) { + klog.V(4).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...) = [%v items], %v", ctx, key, len(all), nil) + } else if klog.V(5) { var asStr []string for _, o := range all { asStr = append(asStr, fmt.Sprintf("%+v", o)) } - glog.V(5).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...) = %v, %v", ctx, key, asStr, nil) + klog.V(5).Infof("{{.GCEWrapType}}.{{.Name}}(%v, %v, ...) = %v, %v", ctx, key, asStr, nil) } return all, nil {{- end}} diff --git a/pkg/cloudprovider/providers/gce/cloud/gen_test.go b/pkg/cloudprovider/providers/gce/cloud/gen_test.go index dc1b4cc6120..902b4ee780d 100644 --- a/pkg/cloudprovider/providers/gce/cloud/gen_test.go +++ b/pkg/cloudprovider/providers/gce/cloud/gen_test.go @@ -1351,58 +1351,58 @@ func TestRegionDisksGroup(t *testing.T) { mock := NewMockGCE(pr) var key *meta.Key - keyBeta := meta.RegionalKey("key-beta", "location") - key = keyBeta + keyGA := meta.RegionalKey("key-ga", "location") + key = keyGA // Ignore unused variables. _, _, _ = ctx, mock, key // Get not found. - if _, err := mock.BetaRegionDisks().Get(ctx, key); err == nil { - t.Errorf("BetaRegionDisks().Get(%v, %v) = _, nil; want error", ctx, key) + if _, err := mock.RegionDisks().Get(ctx, key); err == nil { + t.Errorf("RegionDisks().Get(%v, %v) = _, nil; want error", ctx, key) } // Insert. { - obj := &beta.Disk{} - if err := mock.BetaRegionDisks().Insert(ctx, keyBeta, obj); err != nil { - t.Errorf("BetaRegionDisks().Insert(%v, %v, %v) = %v; want nil", ctx, keyBeta, obj, err) + obj := &ga.Disk{} + if err := mock.RegionDisks().Insert(ctx, keyGA, obj); err != nil { + t.Errorf("RegionDisks().Insert(%v, %v, %v) = %v; want nil", ctx, keyGA, obj, err) } } // Get across versions. - if obj, err := mock.BetaRegionDisks().Get(ctx, key); err != nil { - t.Errorf("BetaRegionDisks().Get(%v, %v) = %v, %v; want nil", ctx, key, obj, err) + if obj, err := mock.RegionDisks().Get(ctx, key); err != nil { + t.Errorf("RegionDisks().Get(%v, %v) = %v, %v; want nil", ctx, key, obj, err) } // List. - mock.MockBetaRegionDisks.Objects[*keyBeta] = mock.MockBetaRegionDisks.Obj(&beta.Disk{Name: keyBeta.Name}) + mock.MockRegionDisks.Objects[*keyGA] = mock.MockRegionDisks.Obj(&ga.Disk{Name: keyGA.Name}) want := map[string]bool{ - "key-beta": true, + "key-ga": true, } _ = want // ignore unused variables. { - objs, err := mock.BetaRegionDisks().List(ctx, location, filter.None) + objs, err := mock.RegionDisks().List(ctx, location, filter.None) if err != nil { - t.Errorf("BetaRegionDisks().List(%v, %v, %v) = %v, %v; want _, nil", ctx, location, filter.None, objs, err) + t.Errorf("RegionDisks().List(%v, %v, %v) = %v, %v; want _, nil", ctx, location, filter.None, objs, err) } else { got := map[string]bool{} for _, obj := range objs { got[obj.Name] = true } if !reflect.DeepEqual(got, want) { - t.Errorf("BetaRegionDisks().List(); got %+v, want %+v", got, want) + t.Errorf("RegionDisks().List(); got %+v, want %+v", got, want) } } } // Delete across versions. - if err := mock.BetaRegionDisks().Delete(ctx, keyBeta); err != nil { - t.Errorf("BetaRegionDisks().Delete(%v, %v) = %v; want nil", ctx, keyBeta, err) + if err := mock.RegionDisks().Delete(ctx, keyGA); err != nil { + t.Errorf("RegionDisks().Delete(%v, %v) = %v; want nil", ctx, keyGA, err) } // Delete not found. - if err := mock.BetaRegionDisks().Delete(ctx, keyBeta); err == nil { - t.Errorf("BetaRegionDisks().Delete(%v, %v) = nil; want error", ctx, keyBeta) + if err := mock.RegionDisks().Delete(ctx, keyGA); err == nil { + t.Errorf("RegionDisks().Delete(%v, %v) = nil; want error", ctx, keyGA) } } diff --git a/pkg/cloudprovider/providers/gce/cloud/meta/meta.go b/pkg/cloudprovider/providers/gce/cloud/meta/meta.go index 6a85207ed03..04d13e0bfd2 100644 --- a/pkg/cloudprovider/providers/gce/cloud/meta/meta.go +++ b/pkg/cloudprovider/providers/gce/cloud/meta/meta.go @@ -169,9 +169,9 @@ var AllServices = []*ServiceInfo{ Object: "Disk", Service: "RegionDisks", Resource: "disks", - version: VersionBeta, + version: VersionGA, keyType: Regional, - serviceType: reflect.TypeOf(&beta.RegionDisksService{}), + serviceType: reflect.TypeOf(&ga.RegionDisksService{}), additionalMethods: []string{ "Resize", }, diff --git a/pkg/cloudprovider/providers/gce/cloud/op.go b/pkg/cloudprovider/providers/gce/cloud/op.go index 2933fe223b7..eb45c769e46 100644 --- a/pkg/cloudprovider/providers/gce/cloud/op.go +++ b/pkg/cloudprovider/providers/gce/cloud/op.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/golang/glog" + "k8s.io/klog" alpha "google.golang.org/api/compute/v0.alpha" beta "google.golang.org/api/compute/v0.beta" @@ -67,13 +67,13 @@ func (o *gaOperation) isDone(ctx context.Context) (bool, error) { switch o.key.Type() { case meta.Regional: op, err = o.s.GA.RegionOperations.Get(o.projectID, o.key.Region, o.key.Name).Context(ctx).Do() - glog.V(5).Infof("GA.RegionOperations.Get(%v, %v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Region, o.key.Name, op, err, ctx) + klog.V(5).Infof("GA.RegionOperations.Get(%v, %v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Region, o.key.Name, op, err, ctx) case meta.Zonal: op, err = o.s.GA.ZoneOperations.Get(o.projectID, o.key.Zone, o.key.Name).Context(ctx).Do() - glog.V(5).Infof("GA.ZoneOperations.Get(%v, %v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Zone, o.key.Name, op, err, ctx) + klog.V(5).Infof("GA.ZoneOperations.Get(%v, %v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Zone, o.key.Name, op, err, ctx) case meta.Global: op, err = o.s.GA.GlobalOperations.Get(o.projectID, o.key.Name).Context(ctx).Do() - glog.V(5).Infof("GA.GlobalOperations.Get(%v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Name, op, err, ctx) + klog.V(5).Infof("GA.GlobalOperations.Get(%v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Name, op, err, ctx) default: return false, fmt.Errorf("invalid key type: %#v", o.key) } @@ -124,13 +124,13 @@ func (o *alphaOperation) isDone(ctx context.Context) (bool, error) { switch o.key.Type() { case meta.Regional: op, err = o.s.Alpha.RegionOperations.Get(o.projectID, o.key.Region, o.key.Name).Context(ctx).Do() - glog.V(5).Infof("Alpha.RegionOperations.Get(%v, %v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Region, o.key.Name, op, err, ctx) + klog.V(5).Infof("Alpha.RegionOperations.Get(%v, %v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Region, o.key.Name, op, err, ctx) case meta.Zonal: op, err = o.s.Alpha.ZoneOperations.Get(o.projectID, o.key.Zone, o.key.Name).Context(ctx).Do() - glog.V(5).Infof("Alpha.ZoneOperations.Get(%v, %v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Zone, o.key.Name, op, err, ctx) + klog.V(5).Infof("Alpha.ZoneOperations.Get(%v, %v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Zone, o.key.Name, op, err, ctx) case meta.Global: op, err = o.s.Alpha.GlobalOperations.Get(o.projectID, o.key.Name).Context(ctx).Do() - glog.V(5).Infof("Alpha.GlobalOperations.Get(%v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Name, op, err, ctx) + klog.V(5).Infof("Alpha.GlobalOperations.Get(%v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Name, op, err, ctx) default: return false, fmt.Errorf("invalid key type: %#v", o.key) } @@ -181,13 +181,13 @@ func (o *betaOperation) isDone(ctx context.Context) (bool, error) { switch o.key.Type() { case meta.Regional: op, err = o.s.Beta.RegionOperations.Get(o.projectID, o.key.Region, o.key.Name).Context(ctx).Do() - glog.V(5).Infof("Beta.RegionOperations.Get(%v, %v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Region, o.key.Name, op, err, ctx) + klog.V(5).Infof("Beta.RegionOperations.Get(%v, %v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Region, o.key.Name, op, err, ctx) case meta.Zonal: op, err = o.s.Beta.ZoneOperations.Get(o.projectID, o.key.Zone, o.key.Name).Context(ctx).Do() - glog.V(5).Infof("Beta.ZoneOperations.Get(%v, %v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Zone, o.key.Name, op, err, ctx) + klog.V(5).Infof("Beta.ZoneOperations.Get(%v, %v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Zone, o.key.Name, op, err, ctx) case meta.Global: op, err = o.s.Beta.GlobalOperations.Get(o.projectID, o.key.Name).Context(ctx).Do() - glog.V(5).Infof("Beta.GlobalOperations.Get(%v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Name, op, err, ctx) + klog.V(5).Infof("Beta.GlobalOperations.Get(%v, %v) = %+v, %v; ctx = %v", o.projectID, o.key.Name, op, err, ctx) default: return false, fmt.Errorf("invalid key type: %#v", o.key) } diff --git a/pkg/cloudprovider/providers/gce/cloud/service.go b/pkg/cloudprovider/providers/gce/cloud/service.go index 2f332dfff85..4d7b4c557f2 100644 --- a/pkg/cloudprovider/providers/gce/cloud/service.go +++ b/pkg/cloudprovider/providers/gce/cloud/service.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/golang/glog" + "k8s.io/klog" alpha "google.golang.org/api/compute/v0.alpha" beta "google.golang.org/api/compute/v0.beta" @@ -69,7 +69,7 @@ func (s *Service) wrapOperation(anyOp interface{}) (operation, error) { func (s *Service) WaitForCompletion(ctx context.Context, genericOp interface{}) error { op, err := s.wrapOperation(genericOp) if err != nil { - glog.Errorf("wrapOperation(%+v) error: %v", genericOp, err) + klog.Errorf("wrapOperation(%+v) error: %v", genericOp, err) return err } @@ -86,18 +86,18 @@ func (s *Service) pollOperation(ctx context.Context, op operation) error { // returning ctx.Err(). select { case <-ctx.Done(): - glog.V(5).Infof("op.pollOperation(%v, %v) not completed, poll count = %d, ctx.Err = %v", ctx, op, pollCount, ctx.Err()) + klog.V(5).Infof("op.pollOperation(%v, %v) not completed, poll count = %d, ctx.Err = %v", ctx, op, pollCount, ctx.Err()) return ctx.Err() default: // ctx is not canceled, continue immediately } pollCount++ - glog.V(5).Infof("op.isDone(%v) waiting; op = %v, poll count = %d", ctx, op, pollCount) + klog.V(5).Infof("op.isDone(%v) waiting; op = %v, poll count = %d", ctx, op, pollCount) s.RateLimiter.Accept(ctx, op.rateLimitKey()) done, err := op.isDone(ctx) if err != nil { - glog.V(5).Infof("op.isDone(%v) error; op = %v, poll count = %d, err = %v, retrying", ctx, op, pollCount, err) + klog.V(5).Infof("op.isDone(%v) error; op = %v, poll count = %d, err = %v, retrying", ctx, op, pollCount, err) } if done { @@ -105,6 +105,6 @@ func (s *Service) pollOperation(ctx context.Context, op operation) error { } } - glog.V(5).Infof("op.isDone(%v) complete; op = %v, poll count = %d, op.err = %v", ctx, op, pollCount, op.error()) + klog.V(5).Infof("op.isDone(%v) complete; op = %v, poll count = %d, op.err = %v", ctx, op, pollCount, op.error()) return op.error() } diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go index adba37d59b6..616fe5f5477 100644 --- a/pkg/cloudprovider/providers/gce/gce.go +++ b/pkg/cloudprovider/providers/gce/gce.go @@ -30,13 +30,13 @@ import ( gcfg "gopkg.in/gcfg.v1" "cloud.google.com/go/compute/metadata" - "github.com/golang/glog" "golang.org/x/oauth2" "golang.org/x/oauth2/google" computealpha "google.golang.org/api/compute/v0.alpha" computebeta "google.golang.org/api/compute/v0.beta" compute "google.golang.org/api/compute/v1" container "google.golang.org/api/container/v1" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -262,7 +262,7 @@ func newGCECloud(config io.Reader) (gceCloud *Cloud, err error) { if err != nil { return nil, err } - glog.Infof("Using GCE provider config %+v", configFile) + klog.Infof("Using GCE provider config %+v", configFile) } cloudConfig, err = generateCloudConfig(configFile) @@ -275,7 +275,7 @@ func newGCECloud(config io.Reader) (gceCloud *Cloud, err error) { func readConfig(reader io.Reader) (*ConfigFile, error) { cfg := &ConfigFile{} if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, reader)); err != nil { - glog.Errorf("Couldn't read config: %v", err) + klog.Errorf("Couldn't read config: %v", err) return nil, err } return cfg, nil @@ -466,7 +466,7 @@ func CreateGCECloud(config *CloudConfig) (*Cloud, error) { } else { // Other consumers may use the cloudprovider without utilizing the wrapped GCE API functions // or functions requiring network/subnetwork URLs (e.g. Kubelet). - glog.Warningf("No network name or URL specified.") + klog.Warningf("No network name or URL specified.") } if config.SubnetworkURL != "" { @@ -480,20 +480,20 @@ func CreateGCECloud(config *CloudConfig) (*Cloud, error) { if networkName := lastComponent(networkURL); networkName != "" { var n *compute.Network if n, err = getNetwork(service, netProjID, networkName); err != nil { - glog.Warningf("Could not retrieve network %q; err: %v", networkName, err) + klog.Warningf("Could not retrieve network %q; err: %v", networkName, err) } else { switch typeOfNetwork(n) { case netTypeLegacy: - glog.Infof("Network %q is type legacy - no subnetwork", networkName) + klog.Infof("Network %q is type legacy - no subnetwork", networkName) isLegacyNetwork = true case netTypeCustom: - glog.Warningf("Network %q is type custom - cannot auto select a subnetwork", networkName) + klog.Warningf("Network %q is type custom - cannot auto select a subnetwork", networkName) case netTypeAuto: subnetURL, err = determineSubnetURL(service, netProjID, networkName, config.Region) if err != nil { - glog.Warningf("Could not determine subnetwork for network %q and region %v; err: %v", networkName, config.Region, err) + klog.Warningf("Could not determine subnetwork for network %q and region %v; err: %v", networkName, config.Region, err) } else { - glog.Infof("Auto selecting subnetwork %q", subnetURL) + klog.Infof("Auto selecting subnetwork %q", subnetURL) } } } @@ -507,7 +507,7 @@ func CreateGCECloud(config *CloudConfig) (*Cloud, error) { } } if len(config.ManagedZones) > 1 { - glog.Infof("managing multiple zones: %v", config.ManagedZones) + klog.Infof("managing multiple zones: %v", config.ManagedZones) } operationPollRateLimiter := flowcontrol.NewTokenBucketRateLimiter(5, 5) // 5 qps, 5 burst. @@ -588,7 +588,7 @@ func tryConvertToProjectNames(configProject, configNetworkProject string, servic if isProjectNumber(projID) { projName, err := getProjectID(service, projID) if err != nil { - glog.Warningf("Failed to retrieve project %v while trying to retrieve its name. err %v", projID, err) + klog.Warningf("Failed to retrieve project %v while trying to retrieve its name. err %v", projID, err) } else { projID = projName } @@ -601,7 +601,7 @@ func tryConvertToProjectNames(configProject, configNetworkProject string, servic if isProjectNumber(netProjID) { netProjName, err := getProjectID(service, netProjID) if err != nil { - glog.Warningf("Failed to retrieve network project %v while trying to retrieve its name. err %v", netProjID, err) + klog.Warningf("Failed to retrieve network project %v while trying to retrieve its name. err %v", netProjID, err) } else { netProjID = netProjName } @@ -692,7 +692,7 @@ func (g *Cloud) IsLegacyNetwork() bool { // SetInformers sets up the zone handlers we need watching for node changes. func (g *Cloud) SetInformers(informerFactory informers.SharedInformerFactory) { - glog.Infof("Setting up informers for Cloud") + klog.Infof("Setting up informers for Cloud") nodeInformer := informerFactory.Core().V1().Nodes().Informer() nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { @@ -715,12 +715,12 @@ func (g *Cloud) SetInformers(informerFactory informers.SharedInformerFactory) { if !isNode { deletedState, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Received unexpected object: %v", obj) + klog.Errorf("Received unexpected object: %v", obj) return } node, ok = deletedState.Obj.(*v1.Node) if !ok { - glog.Errorf("DeletedFinalStateUnknown contained non-Node object: %v", deletedState.Obj) + klog.Errorf("DeletedFinalStateUnknown contained non-Node object: %v", deletedState.Obj) return } } @@ -871,12 +871,12 @@ func newOauthClient(tokenSource oauth2.TokenSource) (*http.Client, error) { oauth2.NoContext, compute.CloudPlatformScope, compute.ComputeScope) - glog.Infof("Using DefaultTokenSource %#v", tokenSource) + klog.Infof("Using DefaultTokenSource %#v", tokenSource) if err != nil { return nil, err } } else { - glog.Infof("Using existing Token Source %#v", tokenSource) + klog.Infof("Using existing Token Source %#v", tokenSource) } backoff := wait.Backoff{ @@ -887,7 +887,7 @@ func newOauthClient(tokenSource oauth2.TokenSource) (*http.Client, error) { } if err := wait.ExponentialBackoff(backoff, func() (bool, error) { if _, err := tokenSource.Token(); err != nil { - glog.Errorf("error fetching initial token: %v", err) + klog.Errorf("error fetching initial token: %v", err) return false, nil } return true, nil diff --git a/pkg/cloudprovider/providers/gce/gce_address_manager.go b/pkg/cloudprovider/providers/gce/gce_address_manager.go index 449b33a0d21..51b9bc5e718 100644 --- a/pkg/cloudprovider/providers/gce/gce_address_manager.go +++ b/pkg/cloudprovider/providers/gce/gce_address_manager.go @@ -22,7 +22,7 @@ import ( compute "google.golang.org/api/compute/v1" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" ) @@ -62,7 +62,7 @@ func (am *addressManager) HoldAddress() (string, error) { // could be reserving another address; therefore, it would need to be deleted. In the normal // case of using a controller address, retrieving the address by name results in the fewest API // calls since it indicates whether a Delete is necessary before Reserve. - glog.V(4).Infof("%v: attempting hold of IP %q Type %q", am.logPrefix, am.targetIP, am.addressType) + klog.V(4).Infof("%v: attempting hold of IP %q Type %q", am.logPrefix, am.targetIP, am.addressType) // Get the address in case it was orphaned earlier addr, err := am.svc.GetRegionAddress(am.name, am.region) if err != nil && !isNotFound(err) { @@ -73,20 +73,20 @@ func (am *addressManager) HoldAddress() (string, error) { // If address exists, check if the address had the expected attributes. validationError := am.validateAddress(addr) if validationError == nil { - glog.V(4).Infof("%v: address %q already reserves IP %q Type %q. No further action required.", am.logPrefix, addr.Name, addr.Address, addr.AddressType) + klog.V(4).Infof("%v: address %q already reserves IP %q Type %q. No further action required.", am.logPrefix, addr.Name, addr.Address, addr.AddressType) return addr.Address, nil } - glog.V(2).Infof("%v: deleting existing address because %v", am.logPrefix, validationError) + klog.V(2).Infof("%v: deleting existing address because %v", am.logPrefix, validationError) err := am.svc.DeleteRegionAddress(addr.Name, am.region) if err != nil { if isNotFound(err) { - glog.V(4).Infof("%v: address %q was not found. Ignoring.", am.logPrefix, addr.Name) + klog.V(4).Infof("%v: address %q was not found. Ignoring.", am.logPrefix, addr.Name) } else { return "", err } } else { - glog.V(4).Infof("%v: successfully deleted previous address %q", am.logPrefix, addr.Name) + klog.V(4).Infof("%v: successfully deleted previous address %q", am.logPrefix, addr.Name) } } @@ -96,23 +96,23 @@ func (am *addressManager) HoldAddress() (string, error) { // ReleaseAddress will release the address if it's owned by the controller. func (am *addressManager) ReleaseAddress() error { if !am.tryRelease { - glog.V(4).Infof("%v: not attempting release of address %q.", am.logPrefix, am.targetIP) + klog.V(4).Infof("%v: not attempting release of address %q.", am.logPrefix, am.targetIP) return nil } - glog.V(4).Infof("%v: releasing address %q named %q", am.logPrefix, am.targetIP, am.name) + klog.V(4).Infof("%v: releasing address %q named %q", am.logPrefix, am.targetIP, am.name) // Controller only ever tries to unreserve the address named with the load balancer's name. err := am.svc.DeleteRegionAddress(am.name, am.region) if err != nil { if isNotFound(err) { - glog.Warningf("%v: address %q was not found. Ignoring.", am.logPrefix, am.name) + klog.Warningf("%v: address %q was not found. Ignoring.", am.logPrefix, am.name) return nil } return err } - glog.V(4).Infof("%v: successfully released IP %q named %q", am.logPrefix, am.targetIP, am.name) + klog.V(4).Infof("%v: successfully released IP %q named %q", am.logPrefix, am.targetIP, am.name) return nil } @@ -130,7 +130,7 @@ func (am *addressManager) ensureAddressReservation() (string, error) { reserveErr := am.svc.ReserveRegionAddress(newAddr, am.region) if reserveErr == nil { if newAddr.Address != "" { - glog.V(4).Infof("%v: successfully reserved IP %q with name %q", am.logPrefix, newAddr.Address, newAddr.Name) + klog.V(4).Infof("%v: successfully reserved IP %q with name %q", am.logPrefix, newAddr.Address, newAddr.Name) return newAddr.Address, nil } @@ -139,7 +139,7 @@ func (am *addressManager) ensureAddressReservation() (string, error) { return "", err } - glog.V(4).Infof("%v: successfully created address %q which reserved IP %q", am.logPrefix, addr.Name, addr.Address) + klog.V(4).Infof("%v: successfully created address %q which reserved IP %q", am.logPrefix, addr.Name, addr.Address) return addr.Address, nil } else if !isHTTPErrorCode(reserveErr, http.StatusConflict) && !isHTTPErrorCode(reserveErr, http.StatusBadRequest) { // If the IP is already reserved: @@ -169,10 +169,10 @@ func (am *addressManager) ensureAddressReservation() (string, error) { if am.isManagedAddress(addr) { // The address with this name is checked at the beginning of 'HoldAddress()', but for some reason // it was re-created by this point. May be possible that two controllers are running. - glog.Warningf("%v: address %q unexpectedly existed with IP %q.", am.logPrefix, addr.Name, am.targetIP) + klog.Warningf("%v: address %q unexpectedly existed with IP %q.", am.logPrefix, addr.Name, am.targetIP) } else { // If the retrieved address is not named with the loadbalancer name, then the controller does not own it, but will allow use of it. - glog.V(4).Infof("%v: address %q was already reserved with name: %q, description: %q", am.logPrefix, am.targetIP, addr.Name, addr.Description) + klog.V(4).Infof("%v: address %q was already reserved with name: %q, description: %q", am.logPrefix, am.targetIP, addr.Name, addr.Description) am.tryRelease = false } diff --git a/pkg/cloudprovider/providers/gce/gce_addresses.go b/pkg/cloudprovider/providers/gce/gce_addresses.go index b595ae62825..044258f1b4e 100644 --- a/pkg/cloudprovider/providers/gce/gce_addresses.go +++ b/pkg/cloudprovider/providers/gce/gce_addresses.go @@ -19,7 +19,7 @@ package gce import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" computealpha "google.golang.org/api/compute/v0.alpha" computebeta "google.golang.org/api/compute/v0.beta" @@ -149,7 +149,7 @@ func (g *Cloud) GetRegionAddressByIP(region, ipAddress string) (*compute.Address } if len(addrs) > 1 { - glog.Warningf("More than one addresses matching the IP %q: %v", ipAddress, addrNames(addrs)) + klog.Warningf("More than one addresses matching the IP %q: %v", ipAddress, addrNames(addrs)) } for _, addr := range addrs { if addr.Address == ipAddress { @@ -173,7 +173,7 @@ func (g *Cloud) GetBetaRegionAddressByIP(region, ipAddress string) (*computebeta } if len(addrs) > 1 { - glog.Warningf("More than one addresses matching the IP %q: %v", ipAddress, addrNames(addrs)) + klog.Warningf("More than one addresses matching the IP %q: %v", ipAddress, addrNames(addrs)) } for _, addr := range addrs { if addr.Address == ipAddress { diff --git a/pkg/cloudprovider/providers/gce/gce_annotations.go b/pkg/cloudprovider/providers/gce/gce_annotations.go index 4f3281c3e47..39e632e0795 100644 --- a/pkg/cloudprovider/providers/gce/gce_annotations.go +++ b/pkg/cloudprovider/providers/gce/gce_annotations.go @@ -19,7 +19,7 @@ package gce import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" @@ -90,7 +90,7 @@ func GetLoadBalancerAnnotationBackendShare(service *v1.Service) bool { // Check for deprecated annotation key if l, exists := service.Annotations[deprecatedServiceAnnotationILBBackendShare]; exists && l == "true" { - glog.Warningf("Annotation %q is deprecated and replaced with an alpha-specific key: %q", deprecatedServiceAnnotationILBBackendShare, ServiceAnnotationILBBackendShare) + klog.Warningf("Annotation %q is deprecated and replaced with an alpha-specific key: %q", deprecatedServiceAnnotationILBBackendShare, ServiceAnnotationILBBackendShare) return true } diff --git a/pkg/cloudprovider/providers/gce/gce_clusterid.go b/pkg/cloudprovider/providers/gce/gce_clusterid.go index 23dd6cf5e21..2f40167788d 100644 --- a/pkg/cloudprovider/providers/gce/gce_clusterid.go +++ b/pkg/cloudprovider/providers/gce/gce_clusterid.go @@ -25,7 +25,6 @@ import ( "sync" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -33,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + "k8s.io/klog" ) const ( @@ -77,7 +77,7 @@ func (g *Cloud) watchClusterID(stop <-chan struct{}) { AddFunc: func(obj interface{}) { m, ok := obj.(*v1.ConfigMap) if !ok || m == nil { - glog.Errorf("Expected v1.ConfigMap, item=%+v, typeIsOk=%v", obj, ok) + klog.Errorf("Expected v1.ConfigMap, item=%+v, typeIsOk=%v", obj, ok) return } if m.Namespace != UIDNamespace || @@ -85,13 +85,13 @@ func (g *Cloud) watchClusterID(stop <-chan struct{}) { return } - glog.V(4).Infof("Observed new configmap for clusteriD: %v, %v; setting local values", m.Name, m.Data) + klog.V(4).Infof("Observed new configmap for clusteriD: %v, %v; setting local values", m.Name, m.Data) g.ClusterID.update(m) }, UpdateFunc: func(old, cur interface{}) { m, ok := cur.(*v1.ConfigMap) if !ok || m == nil { - glog.Errorf("Expected v1.ConfigMap, item=%+v, typeIsOk=%v", cur, ok) + klog.Errorf("Expected v1.ConfigMap, item=%+v, typeIsOk=%v", cur, ok) return } @@ -104,7 +104,7 @@ func (g *Cloud) watchClusterID(stop <-chan struct{}) { return } - glog.V(4).Infof("Observed updated configmap for clusteriD %v, %v; setting local values", m.Name, m.Data) + klog.V(4).Infof("Observed updated configmap for clusteriD %v, %v; setting local values", m.Name, m.Data) g.ClusterID.update(m) }, } @@ -185,7 +185,7 @@ func (ci *ClusterID) getOrInitialize() error { return err } - glog.V(4).Infof("Creating clusteriD: %v", newID) + klog.V(4).Infof("Creating clusteriD: %v", newID) cfg := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: UIDConfigMapName, @@ -198,11 +198,11 @@ func (ci *ClusterID) getOrInitialize() error { } if _, err := ci.client.CoreV1().ConfigMaps(UIDNamespace).Create(cfg); err != nil { - glog.Errorf("GCE cloud provider failed to create %v config map to store cluster id: %v", ci.cfgMapKey, err) + klog.Errorf("GCE cloud provider failed to create %v config map to store cluster id: %v", ci.cfgMapKey, err) return err } - glog.V(2).Infof("Created a config map containing clusteriD: %v", newID) + klog.V(2).Infof("Created a config map containing clusteriD: %v", newID) ci.update(cfg) return nil } @@ -219,7 +219,7 @@ func (ci *ClusterID) getConfigMap() (bool, error) { m, ok := item.(*v1.ConfigMap) if !ok || m == nil { err = fmt.Errorf("Expected v1.ConfigMap, item=%+v, typeIsOk=%v", item, ok) - glog.Error(err) + klog.Error(err) return false, err } ci.update(m) diff --git a/pkg/cloudprovider/providers/gce/gce_clusters.go b/pkg/cloudprovider/providers/gce/gce_clusters.go index 53295000c66..379f5396a25 100644 --- a/pkg/cloudprovider/providers/gce/gce_clusters.go +++ b/pkg/cloudprovider/providers/gce/gce_clusters.go @@ -20,8 +20,8 @@ import ( "context" "fmt" - "github.com/golang/glog" "google.golang.org/api/container/v1" + "k8s.io/klog" ) func newClustersMetricContext(request, zone string) *metricContext { @@ -97,7 +97,7 @@ func (g *Cloud) getClustersInLocation(zoneOrRegion string) ([]*container.Cluster return nil, mc.Observe(err) } if list.Header.Get("nextPageToken") != "" { - glog.Errorf("Failed to get all clusters for request, received next page token %s", list.Header.Get("nextPageToken")) + klog.Errorf("Failed to get all clusters for request, received next page token %s", list.Header.Get("nextPageToken")) } return list.Clusters, mc.Observe(nil) diff --git a/pkg/cloudprovider/providers/gce/gce_disks.go b/pkg/cloudprovider/providers/gce/gce_disks.go index 8f8c47c7434..24dd76fc96d 100644 --- a/pkg/cloudprovider/providers/gce/gce_disks.go +++ b/pkg/cloudprovider/providers/gce/gce_disks.go @@ -33,11 +33,10 @@ import ( "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" - "github.com/golang/glog" - computebeta "google.golang.org/api/compute/v0.beta" compute "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta" "k8s.io/kubernetes/pkg/features" @@ -123,7 +122,7 @@ func (manager *gceServiceManager) CreateDiskOnCloudProvider( diskType string, zone string) error { diskTypeURI, err := manager.getDiskTypeURI( - manager.gce.region /* diskRegion */, singleZone{zone}, diskType, false /* useBetaAPI */) + manager.gce.region /* diskRegion */, singleZone{zone}, diskType) if err != nil { return err } @@ -152,7 +151,7 @@ func (manager *gceServiceManager) CreateRegionalDiskOnCloudProvider( } diskTypeURI, err := manager.getDiskTypeURI( - manager.gce.region /* diskRegion */, multiZone{replicaZones}, diskType, true /* useBetaAPI */) + manager.gce.region /* diskRegion */, multiZone{replicaZones}, diskType) if err != nil { return err } @@ -162,7 +161,7 @@ func (manager *gceServiceManager) CreateRegionalDiskOnCloudProvider( fullyQualifiedReplicaZones, manager.getReplicaZoneURI(replicaZone, true)) } - diskToCreateBeta := &computebeta.Disk{ + diskToCreate := &compute.Disk{ Name: name, SizeGb: sizeGb, Description: tagsStr, @@ -172,7 +171,7 @@ func (manager *gceServiceManager) CreateRegionalDiskOnCloudProvider( ctx, cancel := cloud.ContextWithCallTimeout() defer cancel() - return manager.gce.c.BetaRegionDisks().Insert(ctx, meta.RegionalKey(name, manager.gce.region), diskToCreateBeta) + return manager.gce.c.RegionDisks().Insert(ctx, meta.RegionalKey(name, manager.gce.region), diskToCreate) } func (manager *gceServiceManager) AttachDiskOnCloudProvider( @@ -254,7 +253,7 @@ func (manager *gceServiceManager) GetRegionalDiskFromCloudProvider( ctx, cancel := cloud.ContextWithCallTimeout() defer cancel() - diskBeta, err := manager.gce.c.BetaRegionDisks().Get(ctx, meta.RegionalKey(diskName, manager.gce.region)) + diskBeta, err := manager.gce.c.RegionDisks().Get(ctx, meta.RegionalKey(diskName, manager.gce.region)) if err != nil { return nil, err } @@ -291,7 +290,7 @@ func (manager *gceServiceManager) DeleteRegionalDiskOnCloudProvider( ctx, cancel := cloud.ContextWithCallTimeout() defer cancel() - return manager.gce.c.BetaRegionDisks().Delete(ctx, meta.RegionalKey(diskName, manager.gce.region)) + return manager.gce.c.RegionDisks().Delete(ctx, meta.RegionalKey(diskName, manager.gce.region)) } func (manager *gceServiceManager) getDiskSourceURI(disk *Disk) (string, error) { @@ -329,14 +328,9 @@ func (manager *gceServiceManager) getDiskSourceURI(disk *Disk) (string, error) { } func (manager *gceServiceManager) getDiskTypeURI( - diskRegion string, diskZoneInfo zoneType, diskType string, useBetaAPI bool) (string, error) { + diskRegion string, diskZoneInfo zoneType, diskType string) (string, error) { - var getProjectsAPIEndpoint string - if useBetaAPI { - getProjectsAPIEndpoint = manager.getProjectsAPIEndpointBeta() - } else { - getProjectsAPIEndpoint = manager.getProjectsAPIEndpoint() - } + getProjectsAPIEndpoint := manager.getProjectsAPIEndpoint() switch zoneInfo := diskZoneInfo.(type) { case singleZone: @@ -406,7 +400,7 @@ func (manager *gceServiceManager) getRegionFromZone(zoneInfo zoneType) (string, region, err := GetGCERegion(zone) if err != nil { - glog.Warningf("failed to parse GCE region from zone %q: %v", zone, err) + klog.Warningf("failed to parse GCE region from zone %q: %v", zone, err) region = manager.gce.region } @@ -428,13 +422,13 @@ func (manager *gceServiceManager) RegionalResizeDiskOnCloudProvider(disk *Disk, return fmt.Errorf("the regional PD feature is only available with the %s Kubernetes feature gate enabled", features.GCERegionalPersistentDisk) } - resizeServiceRequest := &computebeta.RegionDisksResizeRequest{ + resizeServiceRequest := &compute.RegionDisksResizeRequest{ SizeGb: sizeGb, } ctx, cancel := cloud.ContextWithCallTimeout() defer cancel() - return manager.gce.c.BetaRegionDisks().Resize(ctx, meta.RegionalKey(disk.Name, disk.Region), resizeServiceRequest) + return manager.gce.c.RegionDisks().Resize(ctx, meta.RegionalKey(disk.Name, disk.Region), resizeServiceRequest) } // Disks is interface for manipulation with GCE PDs. @@ -575,7 +569,7 @@ func (g *Cloud) DetachDisk(devicePath string, nodeName types.NodeName) error { if err != nil { if err == cloudprovider.InstanceNotFound { // If instance no longer exists, safe to assume volume is not attached. - glog.Warningf( + klog.Warningf( "Instance %q does not exist. DetachDisk will assume PD %q is not attached to it.", instanceName, devicePath) @@ -596,7 +590,7 @@ func (g *Cloud) DiskIsAttached(diskName string, nodeName types.NodeName) (bool, if err != nil { if err == cloudprovider.InstanceNotFound { // If instance no longer exists, safe to assume volume is not attached. - glog.Warningf( + klog.Warningf( "Instance %q does not exist. DiskIsAttached will assume PD %q is not attached to it.", instanceName, diskName) @@ -628,7 +622,7 @@ func (g *Cloud) DisksAreAttached(diskNames []string, nodeName types.NodeName) (m if err != nil { if err == cloudprovider.InstanceNotFound { // If instance no longer exists, safe to assume volume is not attached. - glog.Warningf( + klog.Warningf( "Instance %q does not exist. DisksAreAttached will assume PD %v are not attached to it.", instanceName, diskNames) @@ -682,7 +676,7 @@ func (g *Cloud) CreateDisk( mc.Observe(err) if isGCEError(err, "alreadyExists") { - glog.Warningf("GCE PD %q already exists, reusing", name) + klog.Warningf("GCE PD %q already exists, reusing", name) return nil } return err @@ -723,7 +717,7 @@ func (g *Cloud) CreateRegionalDisk( mc.Observe(err) if isGCEError(err, "alreadyExists") { - glog.Warningf("GCE PD %q already exists, reusing", name) + klog.Warningf("GCE PD %q already exists, reusing", name) return nil } return err @@ -827,7 +821,7 @@ func (g *Cloud) GetAutoLabelsForPD(name string, zone string) (map[string]string, if utilfeature.DefaultFeatureGate.Enabled(features.GCERegionalPersistentDisk) { zoneSet, err := volumeutil.LabelZonesToSet(zone) if err != nil { - glog.Warningf("Failed to parse zone field: %q. Will use raw field.", zone) + klog.Warningf("Failed to parse zone field: %q. Will use raw field.", zone) } if len(zoneSet) > 1 { @@ -961,7 +955,7 @@ func (g *Cloud) GetDiskByNameUnknownZone(diskName string) (*Disk, error) { switch zoneInfo := disk.ZoneInfo.(type) { case multiZone: if zoneInfo.replicaZones.Has(zone) { - glog.Warningf("GCE PD name (%q) was found in multiple zones (%q), but ok because it is a RegionalDisk.", + klog.Warningf("GCE PD name (%q) was found in multiple zones (%q), but ok because it is a RegionalDisk.", diskName, zoneInfo.replicaZones) continue } @@ -975,7 +969,7 @@ func (g *Cloud) GetDiskByNameUnknownZone(diskName string) (*Disk, error) { if found != nil { return found, nil } - glog.Warningf("GCE persistent disk %q not found in managed zones (%s)", + klog.Warningf("GCE persistent disk %q not found in managed zones (%s)", diskName, strings.Join(g.managedZones, ",")) return nil, cloudprovider.DiskNotFound diff --git a/pkg/cloudprovider/providers/gce/gce_disks_test.go b/pkg/cloudprovider/providers/gce/gce_disks_test.go index 98a95b64ea2..6b45c868684 100644 --- a/pkg/cloudprovider/providers/gce/gce_disks_test.go +++ b/pkg/cloudprovider/providers/gce/gce_disks_test.go @@ -110,7 +110,7 @@ func TestCreateRegionalDisk_Basic(t *testing.T) { tags := make(map[string]string) tags["test-tag"] = "test-value" - expectedDiskTypeURI := gceComputeAPIEndpointBeta + "projects/" + fmt.Sprintf( + expectedDiskTypeURI := gceComputeAPIEndpoint + "projects/" + fmt.Sprintf( diskTypeURITemplateRegional, gceProjectID, gceRegion, diskType) expectedDescription := "{\"test-tag\":\"test-value\"}" @@ -723,9 +723,9 @@ func (manager *FakeServiceManager) CreateRegionalDiskOnCloudProvider( tagsStr string, diskType string, zones sets.String) error { - manager.createDiskCalled = true - diskTypeURI := gceComputeAPIEndpointBeta + "projects/" + fmt.Sprintf(diskTypeURITemplateRegional, manager.gceProjectID, manager.gceRegion, diskType) + manager.createDiskCalled = true + diskTypeURI := gceComputeAPIEndpoint + "projects/" + fmt.Sprintf(diskTypeURITemplateRegional, manager.gceProjectID, manager.gceRegion, diskType) switch t := manager.targetAPI; t { case targetStable: diskToCreateV1 := &compute.Disk{ @@ -737,10 +737,6 @@ func (manager *FakeServiceManager) CreateRegionalDiskOnCloudProvider( manager.diskToCreateStable = diskToCreateV1 manager.regionalDisks[diskToCreateV1.Name] = zones return nil - case targetBeta: - return fmt.Errorf("regionalDisk CreateDisk op not supported in beta") - case targetAlpha: - return fmt.Errorf("regionalDisk CreateDisk op not supported in alpha") default: return fmt.Errorf("unexpected type: %T", t) } diff --git a/pkg/cloudprovider/providers/gce/gce_healthchecks.go b/pkg/cloudprovider/providers/gce/gce_healthchecks.go index 10e1f72a359..d314376db12 100644 --- a/pkg/cloudprovider/providers/gce/gce_healthchecks.go +++ b/pkg/cloudprovider/providers/gce/gce_healthchecks.go @@ -17,7 +17,7 @@ limitations under the License. package gce import ( - "github.com/golang/glog" + "k8s.io/klog" computealpha "google.golang.org/api/compute/v0.alpha" computebeta "google.golang.org/api/compute/v0.beta" @@ -42,7 +42,7 @@ var ( func init() { if v, err := utilversion.ParseGeneric("1.7.2"); err != nil { - glog.Fatalf("Failed to parse version for minNodesHealthCheckVersion: %v", err) + klog.Fatalf("Failed to parse version for minNodesHealthCheckVersion: %v", err) } else { minNodesHealthCheckVersion = v } @@ -274,7 +274,7 @@ func GetNodesHealthCheckPath() string { func isAtLeastMinNodesHealthCheckVersion(vstring string) bool { version, err := utilversion.ParseGeneric(vstring) if err != nil { - glog.Errorf("vstring (%s) is not a valid version string: %v", vstring, err) + klog.Errorf("vstring (%s) is not a valid version string: %v", vstring, err) return false } return version.AtLeast(minNodesHealthCheckVersion) diff --git a/pkg/cloudprovider/providers/gce/gce_instances.go b/pkg/cloudprovider/providers/gce/gce_instances.go index f97d89648f8..e8345abfae5 100644 --- a/pkg/cloudprovider/providers/gce/gce_instances.go +++ b/pkg/cloudprovider/providers/gce/gce_instances.go @@ -25,9 +25,9 @@ import ( "time" "cloud.google.com/go/compute/metadata" - "github.com/golang/glog" computebeta "google.golang.org/api/compute/v0.beta" compute "google.golang.org/api/compute/v1" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -97,7 +97,7 @@ func (g *Cloud) NodeAddresses(_ context.Context, _ types.NodeName) ([]v1.NodeAdd } if internalDNSFull, err := metadata.Get("instance/hostname"); err != nil { - glog.Warningf("couldn't get full internal DNS name: %v", err) + klog.Warningf("couldn't get full internal DNS name: %v", err) } else { addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalDNS, Address: internalDNSFull}, @@ -234,7 +234,7 @@ func (g *Cloud) AddSSHKeyToAllInstances(ctx context.Context, user string, keyDat return wait.Poll(2*time.Second, 30*time.Second, func() (bool, error) { project, err := g.c.Projects().Get(ctx, g.projectID) if err != nil { - glog.Errorf("Could not get project: %v", err) + klog.Errorf("Could not get project: %v", err) return false, nil } keyString := fmt.Sprintf("%s:%s %s@%s", user, strings.TrimSpace(string(keyData)), user, user) @@ -243,7 +243,7 @@ func (g *Cloud) AddSSHKeyToAllInstances(ctx context.Context, user string, keyDat if item.Key == "sshKeys" { if strings.Contains(*item.Value, keyString) { // We've already added the key - glog.Info("SSHKey already in project metadata") + klog.Info("SSHKey already in project metadata") return true, nil } value := *item.Value + "\n" + keyString @@ -254,7 +254,7 @@ func (g *Cloud) AddSSHKeyToAllInstances(ctx context.Context, user string, keyDat } if !found { // This is super unlikely, so log. - glog.Infof("Failed to find sshKeys metadata, creating a new item") + klog.Infof("Failed to find sshKeys metadata, creating a new item") project.CommonInstanceMetadata.Items = append(project.CommonInstanceMetadata.Items, &compute.MetadataItems{ Key: "sshKeys", @@ -267,10 +267,10 @@ func (g *Cloud) AddSSHKeyToAllInstances(ctx context.Context, user string, keyDat mc.Observe(err) if err != nil { - glog.Errorf("Could not Set Metadata: %v", err) + klog.Errorf("Could not Set Metadata: %v", err) return false, nil } - glog.Infof("Successfully added sshKey to project metadata") + klog.Infof("Successfully added sshKey to project metadata") return true, nil }) } @@ -278,7 +278,7 @@ func (g *Cloud) AddSSHKeyToAllInstances(ctx context.Context, user string, keyDat // GetAllCurrentZones returns all the zones in which k8s nodes are currently running func (g *Cloud) GetAllCurrentZones() (sets.String, error) { if g.nodeInformerSynced == nil { - glog.Warningf("Cloud object does not have informers set, should only happen in E2E binary.") + klog.Warningf("Cloud object does not have informers set, should only happen in E2E binary.") return g.GetAllZonesFromCloudProvider() } g.nodeZonesLock.Lock() @@ -407,7 +407,7 @@ func (g *Cloud) AddAliasToInstance(nodeName types.NodeName, alias *net.IPNet) er return fmt.Errorf("instance %q has no network interfaces", nodeName) case 1: default: - glog.Warningf("Instance %q has more than one network interface, using only the first (%v)", + klog.Warningf("Instance %q has more than one network interface, using only the first (%v)", nodeName, instance.NetworkInterfaces) } @@ -437,7 +437,7 @@ func (g *Cloud) getInstancesByNames(names []string) ([]*gceInstance, error) { for _, name := range names { name = canonicalizeInstanceName(name) if !strings.HasPrefix(name, g.nodeInstancePrefix) { - glog.Warningf("Instance %q does not conform to prefix %q, removing filter", name, g.nodeInstancePrefix) + klog.Warningf("Instance %q does not conform to prefix %q, removing filter", name, g.nodeInstancePrefix) nodeInstancePrefix = "" } found[name] = nil @@ -459,7 +459,7 @@ func (g *Cloud) getInstancesByNames(names []string) ([]*gceInstance, error) { continue } if found[inst.Name] != nil { - glog.Errorf("Instance name %q was duplicated (in zone %q and %q)", inst.Name, zone, found[inst.Name].Zone) + klog.Errorf("Instance name %q was duplicated (in zone %q and %q)", inst.Name, zone, found[inst.Name].Zone) continue } found[inst.Name] = &gceInstance{ @@ -480,7 +480,7 @@ func (g *Cloud) getInstancesByNames(names []string) ([]*gceInstance, error) { failed = append(failed, k) } } - glog.Errorf("Failed to retrieve instances: %v", failed) + klog.Errorf("Failed to retrieve instances: %v", failed) return nil, cloudprovider.InstanceNotFound } @@ -501,7 +501,7 @@ func (g *Cloud) getInstanceByName(name string) (*gceInstance, error) { if isHTTPErrorCode(err, http.StatusNotFound) { continue } - glog.Errorf("getInstanceByName: failed to get instance %s in zone %s; err: %v", name, zone, err) + klog.Errorf("getInstanceByName: failed to get instance %s in zone %s; err: %v", name, zone, err) return nil, err } return instance, nil @@ -561,7 +561,7 @@ func (g *Cloud) isCurrentInstance(instanceID string) bool { currentInstanceID, err := getInstanceIDViaMetadata() if err != nil { // Log and swallow error - glog.Errorf("Failed to fetch instanceID via Metadata: %v", err) + klog.Errorf("Failed to fetch instanceID via Metadata: %v", err) return false } @@ -583,7 +583,7 @@ func (g *Cloud) computeHostTags(hosts []*gceInstance) ([]string, error) { nodeInstancePrefix := g.nodeInstancePrefix for _, host := range hosts { if !strings.HasPrefix(host.Name, g.nodeInstancePrefix) { - glog.Warningf("instance %v does not conform to prefix '%s', ignoring filter", host, g.nodeInstancePrefix) + klog.Warningf("instance %v does not conform to prefix '%s', ignoring filter", host, g.nodeInstancePrefix) nodeInstancePrefix = "" } diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer.go index b6e55d6e9dd..35a2c6952f4 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer.go @@ -24,7 +24,7 @@ import ( "sort" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" cloudprovider "k8s.io/cloud-provider" @@ -114,7 +114,7 @@ func (g *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, svc return nil, err } - glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v): ensure %v loadbalancer", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, desiredScheme) + klog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v): ensure %v loadbalancer", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, desiredScheme) existingFwdRule, err := g.GetRegionForwardingRule(loadBalancerName, g.region) if err != nil && !isNotFound(err) { @@ -126,14 +126,14 @@ func (g *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, svc // If the loadbalancer type changes between INTERNAL and EXTERNAL, the old load balancer should be deleted. if existingScheme != desiredScheme { - glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v): deleting existing %v loadbalancer", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, existingScheme) + klog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v): deleting existing %v loadbalancer", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, existingScheme) switch existingScheme { case cloud.SchemeInternal: err = g.ensureInternalLoadBalancerDeleted(clusterName, clusterID, svc) default: err = g.ensureExternalLoadBalancerDeleted(clusterName, clusterID, svc) } - glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v): done deleting existing %v loadbalancer. err: %v", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, existingScheme, err) + klog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v): done deleting existing %v loadbalancer. err: %v", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, existingScheme, err) if err != nil { return nil, err } @@ -150,7 +150,7 @@ func (g *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, svc default: status, err = g.ensureExternalLoadBalancer(clusterName, clusterID, svc, existingFwdRule, nodes) } - glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v): done ensuring loadbalancer. err: %v", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, err) + klog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v): done ensuring loadbalancer. err: %v", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, err) return status, err } @@ -163,7 +163,7 @@ func (g *Cloud) UpdateLoadBalancer(ctx context.Context, clusterName string, svc return err } - glog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v, %v, %v): updating with %d nodes", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, len(nodes)) + klog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v, %v, %v): updating with %d nodes", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, len(nodes)) switch scheme { case cloud.SchemeInternal: @@ -171,7 +171,7 @@ func (g *Cloud) UpdateLoadBalancer(ctx context.Context, clusterName string, svc default: err = g.updateExternalLoadBalancer(clusterName, svc, nodes) } - glog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v, %v, %v): done updating. err: %v", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, err) + klog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v, %v, %v): done updating. err: %v", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, err) return err } @@ -184,7 +184,7 @@ func (g *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin return err } - glog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v, %v, %v, %v): deleting loadbalancer", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region) + klog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v, %v, %v, %v): deleting loadbalancer", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region) switch scheme { case cloud.SchemeInternal: @@ -192,7 +192,7 @@ func (g *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin default: err = g.ensureExternalLoadBalancerDeleted(clusterName, clusterID, svc) } - glog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v, %v, %v, %v): done deleting loadbalancer. err: %v", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, err) + klog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v, %v, %v, %v): done deleting loadbalancer. err: %v", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, err) return err } diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go index 62531716662..6b92e71ef77 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go @@ -31,9 +31,9 @@ import ( "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" netsets "k8s.io/kubernetes/pkg/util/net/sets" - "github.com/golang/glog" computealpha "google.golang.org/api/compute/v0.alpha" compute "google.golang.org/api/compute/v1" + "k8s.io/klog" ) // ensureExternalLoadBalancer is the external implementation of LoadBalancer.EnsureLoadBalancer. @@ -66,16 +66,16 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, serviceName := types.NamespacedName{Namespace: apiService.Namespace, Name: apiService.Name} lbRefStr := fmt.Sprintf("%v(%v)", loadBalancerName, serviceName) - glog.V(2).Infof("ensureExternalLoadBalancer(%s, %v, %v, %v, %v, %v)", lbRefStr, g.region, requestedIP, portStr, hostNames, apiService.Annotations) + klog.V(2).Infof("ensureExternalLoadBalancer(%s, %v, %v, %v, %v, %v)", lbRefStr, g.region, requestedIP, portStr, hostNames, apiService.Annotations) // Check the current and the desired network tiers. If they do not match, // tear down the existing resources with the wrong tier. netTier, err := g.getServiceNetworkTier(apiService) if err != nil { - glog.Errorf("ensureExternalLoadBalancer(%s): Failed to get the desired network tier: %v.", lbRefStr, err) + klog.Errorf("ensureExternalLoadBalancer(%s): Failed to get the desired network tier: %v.", lbRefStr, err) return nil, err } - glog.V(4).Infof("ensureExternalLoadBalancer(%s): Desired network tier %q.", lbRefStr, netTier) + klog.V(4).Infof("ensureExternalLoadBalancer(%s): Desired network tier %q.", lbRefStr, netTier) if g.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) { g.deleteWrongNetworkTieredResources(loadBalancerName, lbRefStr, netTier) } @@ -86,7 +86,7 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, return nil, err } if !fwdRuleExists { - glog.V(2).Infof("ensureExternalLoadBalancer(%s): Forwarding rule %v doesn't exist.", lbRefStr, loadBalancerName) + klog.V(2).Infof("ensureExternalLoadBalancer(%s): Forwarding rule %v doesn't exist.", lbRefStr, loadBalancerName) } // Make sure we know which IP address will be used and have properly reserved @@ -121,14 +121,14 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, } if isSafeToReleaseIP { if err := g.DeleteRegionAddress(loadBalancerName, g.region); err != nil && !isNotFound(err) { - glog.Errorf("ensureExternalLoadBalancer(%s): Failed to release static IP %s in region %v: %v.", lbRefStr, ipAddressToUse, g.region, err) + klog.Errorf("ensureExternalLoadBalancer(%s): Failed to release static IP %s in region %v: %v.", lbRefStr, ipAddressToUse, g.region, err) } else if isNotFound(err) { - glog.V(2).Infof("ensureExternalLoadBalancer(%s): IP address %s is not reserved.", lbRefStr, ipAddressToUse) + klog.V(2).Infof("ensureExternalLoadBalancer(%s): IP address %s is not reserved.", lbRefStr, ipAddressToUse) } else { - glog.Infof("ensureExternalLoadBalancer(%s): Released static IP %s.", lbRefStr, ipAddressToUse) + klog.Infof("ensureExternalLoadBalancer(%s): Released static IP %s.", lbRefStr, ipAddressToUse) } } else { - glog.Warningf("ensureExternalLoadBalancer(%s): Orphaning static IP %s in region %v: %v.", lbRefStr, ipAddressToUse, g.region, err) + klog.Warningf("ensureExternalLoadBalancer(%s): Orphaning static IP %s in region %v: %v.", lbRefStr, ipAddressToUse, g.region, err) } }() @@ -149,7 +149,7 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, if err != nil { return nil, fmt.Errorf("failed to ensure a static IP for load balancer (%s): %v", lbRefStr, err) } - glog.Infof("ensureExternalLoadBalancer(%s): Ensured IP address %s (tier: %s).", lbRefStr, ipAddr, netTier) + klog.Infof("ensureExternalLoadBalancer(%s): Ensured IP address %s (tier: %s).", lbRefStr, ipAddr, netTier) // If the IP was not owned by the user, but it already existed, it // could indicate that the previous update cycle failed. We can use // this IP and try to run through the process again, but we should @@ -177,17 +177,17 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, // Unlike forwarding rules and target pools, firewalls can be updated // without needing to be deleted and recreated. if firewallExists { - glog.Infof("ensureExternalLoadBalancer(%s): Updating firewall.", lbRefStr) + klog.Infof("ensureExternalLoadBalancer(%s): Updating firewall.", lbRefStr) if err := g.updateFirewall(apiService, MakeFirewallName(loadBalancerName), g.region, desc, sourceRanges, ports, hosts); err != nil { return nil, err } - glog.Infof("ensureExternalLoadBalancer(%s): Updated firewall.", lbRefStr) + klog.Infof("ensureExternalLoadBalancer(%s): Updated firewall.", lbRefStr) } else { - glog.Infof("ensureExternalLoadBalancer(%s): Creating firewall.", lbRefStr) + klog.Infof("ensureExternalLoadBalancer(%s): Creating firewall.", lbRefStr) if err := g.createFirewall(apiService, MakeFirewallName(loadBalancerName), g.region, desc, sourceRanges, ports, hosts); err != nil { return nil, err } - glog.Infof("ensureExternalLoadBalancer(%s): Created firewall.", lbRefStr) + klog.Infof("ensureExternalLoadBalancer(%s): Created firewall.", lbRefStr) } } @@ -196,7 +196,7 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, return nil, err } if !tpExists { - glog.Infof("ensureExternalLoadBalancer(%s): Target pool for service doesn't exist.", lbRefStr) + klog.Infof("ensureExternalLoadBalancer(%s): Target pool for service doesn't exist.", lbRefStr) } // Check which health check needs to create and which health check needs to delete. @@ -207,12 +207,12 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, return nil, fmt.Errorf("error checking HTTP health check for load balancer (%s): %v", lbRefStr, err) } if path, healthCheckNodePort := apiservice.GetServiceHealthCheckPathPort(apiService); path != "" { - glog.V(4).Infof("ensureExternalLoadBalancer(%s): Service needs local traffic health checks on: %d%s.", lbRefStr, healthCheckNodePort, path) + klog.V(4).Infof("ensureExternalLoadBalancer(%s): Service needs local traffic health checks on: %d%s.", lbRefStr, healthCheckNodePort, path) if hcLocalTrafficExisting == nil { // This logic exists to detect a transition for non-OnlyLocal to OnlyLocal service // turn on the tpNeedsRecreation flag to delete/recreate fwdrule/tpool updating the // target pool to use local traffic health check. - glog.V(2).Infof("ensureExternalLoadBalancer(%s): Updating from nodes health checks to local traffic health checks.", lbRefStr) + klog.V(2).Infof("ensureExternalLoadBalancer(%s): Updating from nodes health checks to local traffic health checks.", lbRefStr) if supportsNodesHealthCheck { hcToDelete = makeHTTPHealthCheck(MakeNodesHealthCheckName(clusterID), GetNodesHealthCheckPath(), GetNodesHealthCheckPort()) } @@ -220,12 +220,12 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, } hcToCreate = makeHTTPHealthCheck(loadBalancerName, path, healthCheckNodePort) } else { - glog.V(4).Infof("ensureExternalLoadBalancer(%s): Service needs nodes health checks.", lbRefStr) + klog.V(4).Infof("ensureExternalLoadBalancer(%s): Service needs nodes health checks.", lbRefStr) if hcLocalTrafficExisting != nil { // This logic exists to detect a transition from OnlyLocal to non-OnlyLocal service // and turn on the tpNeedsRecreation flag to delete/recreate fwdrule/tpool updating the // target pool to use nodes health check. - glog.V(2).Infof("ensureExternalLoadBalancer(%s): Updating from local traffic health checks to nodes health checks.", lbRefStr) + klog.V(2).Infof("ensureExternalLoadBalancer(%s): Updating from local traffic health checks to nodes health checks.", lbRefStr) hcToDelete = hcLocalTrafficExisting tpNeedsRecreation = true } @@ -248,7 +248,7 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, if err := g.DeleteRegionForwardingRule(loadBalancerName, g.region); err != nil && !isNotFound(err) { return nil, fmt.Errorf("failed to delete existing forwarding rule for load balancer (%s) update: %v", lbRefStr, err) } - glog.Infof("ensureExternalLoadBalancer(%s): Deleted forwarding rule.", lbRefStr) + klog.Infof("ensureExternalLoadBalancer(%s): Deleted forwarding rule.", lbRefStr) } if err := g.ensureTargetPoolAndHealthCheck(tpExists, tpNeedsRecreation, apiService, loadBalancerName, clusterID, ipAddressToUse, hosts, hcToCreate, hcToDelete); err != nil { @@ -256,7 +256,7 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, } if tpNeedsRecreation || fwdRuleNeedsUpdate { - glog.Infof("ensureExternalLoadBalancer(%s): Creating forwarding rule, IP %s (tier: %s).", lbRefStr, ipAddressToUse, netTier) + klog.Infof("ensureExternalLoadBalancer(%s): Creating forwarding rule, IP %s (tier: %s).", lbRefStr, ipAddressToUse, netTier) if err := createForwardingRule(g, loadBalancerName, serviceName.String(), g.region, ipAddressToUse, g.targetPoolURL(loadBalancerName), ports, netTier); err != nil { return nil, fmt.Errorf("failed to create forwarding rule for load balancer (%s): %v", lbRefStr, err) } @@ -265,7 +265,7 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, // of a user-requested IP, the "is user-owned" flag will be set, // preventing it from actually being released. isSafeToReleaseIP = true - glog.Infof("ensureExternalLoadBalancer(%s): Created forwarding rule, IP %s.", lbRefStr, ipAddressToUse) + klog.Infof("ensureExternalLoadBalancer(%s): Created forwarding rule, IP %s.", lbRefStr, ipAddressToUse) } status := &v1.LoadBalancerStatus{} @@ -295,7 +295,7 @@ func (g *Cloud) ensureExternalLoadBalancerDeleted(clusterName, clusterID string, if path, _ := apiservice.GetServiceHealthCheckPathPort(service); path != "" { hcToDelete, err := g.GetHTTPHealthCheck(loadBalancerName) if err != nil && !isHTTPErrorCode(err, http.StatusNotFound) { - glog.Infof("ensureExternalLoadBalancerDeleted(%s): Failed to retrieve health check:%v.", lbRefStr, err) + klog.Infof("ensureExternalLoadBalancerDeleted(%s): Failed to retrieve health check:%v.", lbRefStr, err) return err } // If we got 'StatusNotFound' LB was already deleted and it's safe to ignore. @@ -313,11 +313,11 @@ func (g *Cloud) ensureExternalLoadBalancerDeleted(clusterName, clusterID string, errs := utilerrors.AggregateGoroutines( func() error { - glog.Infof("ensureExternalLoadBalancerDeleted(%s): Deleting firewall rule.", lbRefStr) + klog.Infof("ensureExternalLoadBalancerDeleted(%s): Deleting firewall rule.", lbRefStr) fwName := MakeFirewallName(loadBalancerName) err := ignoreNotFound(g.DeleteFirewall(fwName)) if isForbidden(err) && g.OnXPN() { - glog.V(4).Infof("ensureExternalLoadBalancerDeleted(%s): Do not have permission to delete firewall rule %v (on XPN). Raising event.", lbRefStr, fwName) + klog.V(4).Infof("ensureExternalLoadBalancerDeleted(%s): Do not have permission to delete firewall rule %v (on XPN). Raising event.", lbRefStr, fwName) g.raiseFirewallChangeNeededEvent(service, FirewallToGCloudDeleteCmd(fwName, g.NetworkProjectID())) return nil } @@ -327,17 +327,17 @@ func (g *Cloud) ensureExternalLoadBalancerDeleted(clusterName, clusterID string, // possible that EnsureLoadBalancer left one around in a failed // creation/update attempt, so make sure we clean it up here just in case. func() error { - glog.Infof("ensureExternalLoadBalancerDeleted(%s): Deleting IP address.", lbRefStr) + klog.Infof("ensureExternalLoadBalancerDeleted(%s): Deleting IP address.", lbRefStr) return ignoreNotFound(g.DeleteRegionAddress(loadBalancerName, g.region)) }, func() error { - glog.Infof("ensureExternalLoadBalancerDeleted(%s): Deleting forwarding rule.", lbRefStr) + klog.Infof("ensureExternalLoadBalancerDeleted(%s): Deleting forwarding rule.", lbRefStr) // The forwarding rule must be deleted before either the target pool can, // unfortunately, so we have to do these two serially. if err := ignoreNotFound(g.DeleteRegionForwardingRule(loadBalancerName, g.region)); err != nil { return err } - glog.Infof("ensureExternalLoadBalancerDeleted(%s): Deleting target pool.", lbRefStr) + klog.Infof("ensureExternalLoadBalancerDeleted(%s): Deleting target pool.", lbRefStr) if err := g.DeleteExternalTargetPoolAndChecks(service, loadBalancerName, g.region, clusterID, hcNames...); err != nil { return err } @@ -356,9 +356,9 @@ func (g *Cloud) DeleteExternalTargetPoolAndChecks(service *v1.Service, name, reg lbRefStr := fmt.Sprintf("%v(%v)", name, serviceName) if err := g.DeleteTargetPool(name, region); err != nil && isHTTPErrorCode(err, http.StatusNotFound) { - glog.Infof("DeleteExternalTargetPoolAndChecks(%v): Target pool already deleted. Continuing to delete other resources.", lbRefStr) + klog.Infof("DeleteExternalTargetPoolAndChecks(%v): Target pool already deleted. Continuing to delete other resources.", lbRefStr) } else if err != nil { - glog.Warningf("DeleteExternalTargetPoolAndChecks(%v): Failed to delete target pool, got error %s.", lbRefStr, err.Error()) + klog.Warningf("DeleteExternalTargetPoolAndChecks(%v): Failed to delete target pool, got error %s.", lbRefStr, err.Error()) return err } @@ -373,14 +373,14 @@ func (g *Cloud) DeleteExternalTargetPoolAndChecks(service *v1.Service, name, reg g.sharedResourceLock.Lock() defer g.sharedResourceLock.Unlock() } - glog.Infof("DeleteExternalTargetPoolAndChecks(%v): Deleting health check %v.", lbRefStr, hcName) + klog.Infof("DeleteExternalTargetPoolAndChecks(%v): Deleting health check %v.", lbRefStr, hcName) if err := g.DeleteHTTPHealthCheck(hcName); err != nil { // Delete nodes health checks will fail if any other target pool is using it. if isInUsedByError(err) { - glog.V(4).Infof("DeleteExternalTargetPoolAndChecks(%v): Health check %v is in used: %v.", lbRefStr, hcName, err) + klog.V(4).Infof("DeleteExternalTargetPoolAndChecks(%v): Health check %v is in used: %v.", lbRefStr, hcName, err) return nil } else if !isHTTPErrorCode(err, http.StatusNotFound) { - glog.Warningf("DeleteExternalTargetPoolAndChecks(%v): Failed to delete health check %v: %v.", lbRefStr, hcName, err) + klog.Warningf("DeleteExternalTargetPoolAndChecks(%v): Failed to delete health check %v: %v.", lbRefStr, hcName, err) return err } // StatusNotFound could happen when: @@ -390,15 +390,15 @@ func (g *Cloud) DeleteExternalTargetPoolAndChecks(service *v1.Service, name, reg // - This is a retry and in previous round we failed to delete the healthcheck firewall // after deleted the healthcheck. // We continue to delete the healthcheck firewall to prevent leaking. - glog.V(4).Infof("DeleteExternalTargetPoolAndChecks(%v): Health check %v is already deleted.", lbRefStr, hcName) + klog.V(4).Infof("DeleteExternalTargetPoolAndChecks(%v): Health check %v is already deleted.", lbRefStr, hcName) } // If health check is deleted without error, it means no load-balancer is using it. // So we should delete the health check firewall as well. fwName := MakeHealthCheckFirewallName(clusterID, hcName, isNodesHealthCheck) - glog.Infof("DeleteExternalTargetPoolAndChecks(%v): Deleting health check firewall %v.", lbRefStr, fwName) + klog.Infof("DeleteExternalTargetPoolAndChecks(%v): Deleting health check firewall %v.", lbRefStr, fwName) if err := ignoreNotFound(g.DeleteFirewall(fwName)); err != nil { if isForbidden(err) && g.OnXPN() { - glog.V(4).Infof("DeleteExternalTargetPoolAndChecks(%v): Do not have permission to delete firewall rule %v (on XPN). Raising event.", lbRefStr, fwName) + klog.V(4).Infof("DeleteExternalTargetPoolAndChecks(%v): Do not have permission to delete firewall rule %v (on XPN). Raising event.", lbRefStr, fwName) g.raiseFirewallChangeNeededEvent(service, FirewallToGCloudDeleteCmd(fwName, g.NetworkProjectID())) return nil } @@ -429,7 +429,7 @@ func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP // case we shouldn't delete it anyway). existingAddress, err := s.GetRegionAddressByIP(region, requestedIP) if err != nil && !isNotFound(err) { - glog.Errorf("verifyUserRequestedIP: failed to check whether the requested IP %q for LB %s exists: %v", requestedIP, lbRef, err) + klog.Errorf("verifyUserRequestedIP: failed to check whether the requested IP %q for LB %s exists: %v", requestedIP, lbRef, err) return false, err } if err == nil { @@ -443,23 +443,23 @@ func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP } netTier := cloud.NetworkTierGCEValueToType(netTierStr) if netTier != desiredNetTier { - glog.Errorf("verifyUserRequestedIP: requested static IP %q (name: %s) for LB %s has network tier %s, need %s.", requestedIP, existingAddress.Name, lbRef, netTier, desiredNetTier) + klog.Errorf("verifyUserRequestedIP: requested static IP %q (name: %s) for LB %s has network tier %s, need %s.", requestedIP, existingAddress.Name, lbRef, netTier, desiredNetTier) return false, fmt.Errorf("requrested IP %q belongs to the %s network tier; expected %s", requestedIP, netTier, desiredNetTier) } - glog.V(4).Infof("verifyUserRequestedIP: the requested static IP %q (name: %s, tier: %s) for LB %s exists.", requestedIP, existingAddress.Name, netTier, lbRef) + klog.V(4).Infof("verifyUserRequestedIP: the requested static IP %q (name: %s, tier: %s) for LB %s exists.", requestedIP, existingAddress.Name, netTier, lbRef) return true, nil } if requestedIP == fwdRuleIP { // The requested IP is not a static IP, but is currently assigned // to this forwarding rule, so we can just use it. - glog.V(4).Infof("verifyUserRequestedIP: the requested IP %q is not static, but is currently in use by for LB %s", requestedIP, lbRef) + klog.V(4).Infof("verifyUserRequestedIP: the requested IP %q is not static, but is currently in use by for LB %s", requestedIP, lbRef) return false, nil } // The requested IP is not static and it is not assigned to the // current forwarding rule. It might be attached to a different // rule or it might not be part of this project at all. Either // way, we can't use it. - glog.Errorf("verifyUserRequestedIP: requested IP %q for LB %s is neither static nor assigned to the LB", requestedIP, lbRef) + klog.Errorf("verifyUserRequestedIP: requested IP %q for LB %s is neither static nor assigned to the LB", requestedIP, lbRef) return false, fmt.Errorf("requested ip %q is neither static nor assigned to the LB", requestedIP) } @@ -476,7 +476,7 @@ func (g *Cloud) ensureTargetPoolAndHealthCheck(tpExists, tpNeedsRecreation bool, if err := g.DeleteExternalTargetPoolAndChecks(svc, loadBalancerName, g.region, clusterID, hcNames...); err != nil { return fmt.Errorf("failed to delete existing target pool for load balancer (%s) update: %v", lbRefStr, err) } - glog.Infof("ensureTargetPoolAndHealthCheck(%s): Deleted target pool.", lbRefStr) + klog.Infof("ensureTargetPoolAndHealthCheck(%s): Deleted target pool.", lbRefStr) } // Once we've deleted the resources (if necessary), build them back up (or for // the first time if they're new). @@ -489,23 +489,23 @@ func (g *Cloud) ensureTargetPoolAndHealthCheck(tpExists, tpNeedsRecreation bool, return fmt.Errorf("failed to create target pool for load balancer (%s): %v", lbRefStr, err) } if hcToCreate != nil { - glog.Infof("ensureTargetPoolAndHealthCheck(%s): Created health checks %v.", lbRefStr, hcToCreate.Name) + klog.Infof("ensureTargetPoolAndHealthCheck(%s): Created health checks %v.", lbRefStr, hcToCreate.Name) } if len(hosts) <= maxTargetPoolCreateInstances { - glog.Infof("ensureTargetPoolAndHealthCheck(%s): Created target pool.", lbRefStr) + klog.Infof("ensureTargetPoolAndHealthCheck(%s): Created target pool.", lbRefStr) } else { - glog.Infof("ensureTargetPoolAndHealthCheck(%s): Created initial target pool (now updating the remaining %d hosts).", lbRefStr, len(hosts)-maxTargetPoolCreateInstances) + klog.Infof("ensureTargetPoolAndHealthCheck(%s): Created initial target pool (now updating the remaining %d hosts).", lbRefStr, len(hosts)-maxTargetPoolCreateInstances) if err := g.updateTargetPool(loadBalancerName, hosts); err != nil { return fmt.Errorf("failed to update target pool for load balancer (%s): %v", lbRefStr, err) } - glog.Infof("ensureTargetPoolAndHealthCheck(%s): Updated target pool (with %d hosts).", lbRefStr, len(hosts)-maxTargetPoolCreateInstances) + klog.Infof("ensureTargetPoolAndHealthCheck(%s): Updated target pool (with %d hosts).", lbRefStr, len(hosts)-maxTargetPoolCreateInstances) } } else if tpExists { // Ensure hosts are updated even if there is no other changes required on target pool. if err := g.updateTargetPool(loadBalancerName, hosts); err != nil { return fmt.Errorf("failed to update target pool for load balancer (%s): %v", lbRefStr, err) } - glog.Infof("ensureTargetPoolAndHealthCheck(%s): Updated target pool (with %d hosts).", lbRefStr, len(hosts)) + klog.Infof("ensureTargetPoolAndHealthCheck(%s): Updated target pool (with %d hosts).", lbRefStr, len(hosts)) if hcToCreate != nil { if hc, err := g.ensureHTTPHealthCheck(hcToCreate.Name, hcToCreate.RequestPath, int32(hcToCreate.Port)); err != nil || hc == nil { return fmt.Errorf("Failed to ensure health check for %v port %d path %v: %v", loadBalancerName, hcToCreate.Port, hcToCreate.RequestPath, err) @@ -513,7 +513,7 @@ func (g *Cloud) ensureTargetPoolAndHealthCheck(tpExists, tpNeedsRecreation bool, } } else { // Panic worthy. - glog.Errorf("ensureTargetPoolAndHealthCheck(%s): target pool not exists and doesn't need to be created.", lbRefStr) + klog.Errorf("ensureTargetPoolAndHealthCheck(%s): target pool not exists and doesn't need to be created.", lbRefStr) } return nil } @@ -547,7 +547,7 @@ func (g *Cloud) createTargetPoolAndHealthCheck(svc *v1.Service, name, serviceNam for _, host := range hosts { instances = append(instances, host.makeComparableHostPath()) } - glog.Infof("Creating targetpool %v with %d healthchecks", name, len(hcLinks)) + klog.Infof("Creating targetpool %v with %d healthchecks", name, len(hcLinks)) pool := &compute.TargetPool{ Name: name, Description: fmt.Sprintf(`{"kubernetes.io/service-name":"%s"}`, serviceName), @@ -605,7 +605,7 @@ func (g *Cloud) updateTargetPool(loadBalancerName string, hosts []*gceInstance) return err } if len(updatedPool.Instances) != len(hosts) { - glog.Errorf("Unexpected number of instances (%d) in target pool %s after updating (expected %d). Instances in updated pool: %s", + klog.Errorf("Unexpected number of instances (%d) in target pool %s after updating (expected %d). Instances in updated pool: %s", len(updatedPool.Instances), loadBalancerName, len(hosts), strings.Join(updatedPool.Instances, ",")) return fmt.Errorf("Unexpected number of instances (%d) in target pool %s after update (expected %d)", len(updatedPool.Instances), loadBalancerName, len(hosts)) } @@ -665,28 +665,28 @@ func (g *Cloud) ensureHTTPHealthCheck(name, path string, port int32) (hc *comput newHC := makeHTTPHealthCheck(name, path, port) hc, err = g.GetHTTPHealthCheck(name) if hc == nil || err != nil && isHTTPErrorCode(err, http.StatusNotFound) { - glog.Infof("Did not find health check %v, creating port %v path %v", name, port, path) + klog.Infof("Did not find health check %v, creating port %v path %v", name, port, path) if err = g.CreateHTTPHealthCheck(newHC); err != nil { return nil, err } hc, err = g.GetHTTPHealthCheck(name) if err != nil { - glog.Errorf("Failed to get http health check %v", err) + klog.Errorf("Failed to get http health check %v", err) return nil, err } - glog.Infof("Created HTTP health check %v healthCheckNodePort: %d", name, port) + klog.Infof("Created HTTP health check %v healthCheckNodePort: %d", name, port) return hc, nil } // Validate health check fields - glog.V(4).Infof("Checking http health check params %s", name) + klog.V(4).Infof("Checking http health check params %s", name) if needToUpdateHTTPHealthChecks(hc, newHC) { - glog.Warningf("Health check %v exists but parameters have drifted - updating...", name) + klog.Warningf("Health check %v exists but parameters have drifted - updating...", name) newHC = mergeHTTPHealthChecks(hc, newHC) if err := g.UpdateHTTPHealthCheck(newHC); err != nil { - glog.Warningf("Failed to reconcile http health check %v parameters", name) + klog.Warningf("Failed to reconcile http health check %v parameters", name) return nil, err } - glog.V(4).Infof("Corrected health check %v parameters successful", name) + klog.V(4).Infof("Corrected health check %v parameters successful", name) hc, err = g.GetHTTPHealthCheck(name) if err != nil { return nil, err @@ -714,7 +714,7 @@ func (g *Cloud) forwardingRuleNeedsUpdate(name, region string, loadBalancerIP st // TODO: we report loadbalancer IP through status, so we want to verify if // that matches the forwarding rule as well. if loadBalancerIP != "" && loadBalancerIP != fwd.IPAddress { - glog.Infof("LoadBalancer ip for forwarding rule %v was expected to be %v, but was actually %v", fwd.Name, fwd.IPAddress, loadBalancerIP) + klog.Infof("LoadBalancer ip for forwarding rule %v was expected to be %v, but was actually %v", fwd.Name, fwd.IPAddress, loadBalancerIP) return true, true, fwd.IPAddress, nil } portRange, err := loadBalancerPortRange(ports) @@ -724,12 +724,12 @@ func (g *Cloud) forwardingRuleNeedsUpdate(name, region string, loadBalancerIP st return true, false, "", err } if portRange != fwd.PortRange { - glog.Infof("LoadBalancer port range for forwarding rule %v was expected to be %v, but was actually %v", fwd.Name, fwd.PortRange, portRange) + klog.Infof("LoadBalancer port range for forwarding rule %v was expected to be %v, but was actually %v", fwd.Name, fwd.PortRange, portRange) return true, true, fwd.IPAddress, nil } // The service controller verified all the protocols match on the ports, just check the first one if string(ports[0].Protocol) != fwd.IPProtocol { - glog.Infof("LoadBalancer protocol for forwarding rule %v was expected to be %v, but was actually %v", fwd.Name, fwd.IPProtocol, string(ports[0].Protocol)) + klog.Infof("LoadBalancer protocol for forwarding rule %v was expected to be %v, but was actually %v", fwd.Name, fwd.IPProtocol, string(ports[0].Protocol)) return true, true, fwd.IPAddress, nil } @@ -757,7 +757,7 @@ func (g *Cloud) targetPoolNeedsRecreation(name, region string, affinityType v1.S // target pool (which results in downtime). Fix this when we have formally // defined the defaults on either side. if tp.SessionAffinity != "" && translateAffinityType(affinityType) != tp.SessionAffinity { - glog.Infof("LoadBalancer target pool %v changed affinity from %v to %v", name, tp.SessionAffinity, affinityType) + klog.Infof("LoadBalancer target pool %v changed affinity from %v to %v", name, tp.SessionAffinity, affinityType) return true, true, nil } return true, false, nil @@ -814,7 +814,7 @@ func translateAffinityType(affinityType v1.ServiceAffinity) string { case v1.ServiceAffinityNone: return gceAffinityTypeNone default: - glog.Errorf("Unexpected affinity type: %v", affinityType) + klog.Errorf("Unexpected affinity type: %v", affinityType) return gceAffinityTypeNone } } @@ -846,7 +846,7 @@ func (g *Cloud) firewallNeedsUpdate(name, serviceName, region, ipAddress string, actualSourceRanges, err := netsets.ParseIPNets(fw.SourceRanges...) if err != nil { // This really shouldn't happen... GCE has returned something unexpected - glog.Warningf("Error parsing firewall SourceRanges: %v", fw.SourceRanges) + klog.Warningf("Error parsing firewall SourceRanges: %v", fw.SourceRanges) // We don't return the error, because we can hopefully recover from this by reconfiguring the firewall return true, true, nil } @@ -872,11 +872,11 @@ func (g *Cloud) ensureHTTPHealthCheckFirewall(svc *v1.Service, serviceName, ipAd if !isHTTPErrorCode(err, http.StatusNotFound) { return fmt.Errorf("error getting firewall for health checks: %v", err) } - glog.Infof("Creating firewall %v for health checks.", fwName) + klog.Infof("Creating firewall %v for health checks.", fwName) if err := g.createFirewall(svc, fwName, region, desc, sourceRanges, ports, hosts); err != nil { return err } - glog.Infof("Created firewall %v for health checks.", fwName) + klog.Infof("Created firewall %v for health checks.", fwName) return nil } // Validate firewall fields. @@ -885,12 +885,12 @@ func (g *Cloud) ensureHTTPHealthCheckFirewall(svc *v1.Service, serviceName, ipAd fw.Allowed[0].IPProtocol != string(ports[0].Protocol) || !equalStringSets(fw.Allowed[0].Ports, []string{strconv.Itoa(int(ports[0].Port))}) || !equalStringSets(fw.SourceRanges, sourceRanges.StringSlice()) { - glog.Warningf("Firewall %v exists but parameters have drifted - updating...", fwName) + klog.Warningf("Firewall %v exists but parameters have drifted - updating...", fwName) if err := g.updateFirewall(svc, fwName, region, desc, sourceRanges, ports, hosts); err != nil { - glog.Warningf("Failed to reconcile firewall %v parameters.", fwName) + klog.Warningf("Failed to reconcile firewall %v parameters.", fwName) return err } - glog.V(4).Infof("Corrected firewall %v parameters successful", fwName) + klog.V(4).Infof("Corrected firewall %v parameters successful", fwName) } return nil } @@ -943,7 +943,7 @@ func (g *Cloud) createFirewall(svc *v1.Service, name, region, desc string, sourc if isHTTPErrorCode(err, http.StatusConflict) { return nil } else if isForbidden(err) && g.OnXPN() { - glog.V(4).Infof("createFirewall(%v): do not have permission to create firewall rule (on XPN). Raising event.", firewall.Name) + klog.V(4).Infof("createFirewall(%v): do not have permission to create firewall rule (on XPN). Raising event.", firewall.Name) g.raiseFirewallChangeNeededEvent(svc, FirewallToGCloudCreateCmd(firewall, g.NetworkProjectID())) return nil } @@ -962,7 +962,7 @@ func (g *Cloud) updateFirewall(svc *v1.Service, name, region, desc string, sourc if isHTTPErrorCode(err, http.StatusConflict) { return nil } else if isForbidden(err) && g.OnXPN() { - glog.V(4).Infof("updateFirewall(%v): do not have permission to update firewall rule (on XPN). Raising event.", firewall.Name) + klog.V(4).Infof("updateFirewall(%v): do not have permission to update firewall rule (on XPN). Raising event.", firewall.Name) g.raiseFirewallChangeNeededEvent(svc, FirewallToGCloudUpdateCmd(firewall, g.NetworkProjectID())) return nil } @@ -1091,7 +1091,7 @@ func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logP if existingTier == desiredNetTier { return nil } - glog.V(2).Infof("%s: Network tiers do not match; existing forwarding rule: %q, desired: %q. Deleting the forwarding rule", + klog.V(2).Infof("%s: Network tiers do not match; existing forwarding rule: %q, desired: %q. Deleting the forwarding rule", logPrefix, existingTier, desiredNetTier) err = s.DeleteRegionForwardingRule(name, region) return ignoreNotFound(err) @@ -1119,7 +1119,7 @@ func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix s if existingTier == desiredNetTier { return nil } - glog.V(2).Infof("%s: Network tiers do not match; existing address: %q, desired: %q. Deleting the address", + klog.V(2).Infof("%s: Network tiers do not match; existing address: %q, desired: %q. Deleting the address", logPrefix, existingTier, desiredNetTier) err = s.DeleteRegionAddress(name, region) return ignoreNotFound(err) diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer_internal.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer_internal.go index 49a28855207..87d1be64b49 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer_internal.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer_internal.go @@ -22,11 +22,11 @@ import ( "strconv" "strings" - "github.com/golang/glog" compute "google.golang.org/api/compute/v1" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" v1_service "k8s.io/kubernetes/pkg/api/v1/service" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" ) @@ -102,7 +102,7 @@ func (g *Cloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v if err != nil { return nil, err } - glog.V(2).Infof("ensureInternalLoadBalancer(%v): reserved IP %q for the forwarding rule", loadBalancerName, ipToUse) + klog.V(2).Infof("ensureInternalLoadBalancer(%v): reserved IP %q for the forwarding rule", loadBalancerName, ipToUse) } // Ensure firewall rules if necessary @@ -130,7 +130,7 @@ func (g *Cloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v fwdRuleDeleted := false if existingFwdRule != nil && !fwdRuleEqual(existingFwdRule, expectedFwdRule) { - glog.V(2).Infof("ensureInternalLoadBalancer(%v): deleting existing forwarding rule with IP address %v", loadBalancerName, existingFwdRule.IPAddress) + klog.V(2).Infof("ensureInternalLoadBalancer(%v): deleting existing forwarding rule with IP address %v", loadBalancerName, existingFwdRule.IPAddress) if err = ignoreNotFound(g.DeleteRegionForwardingRule(loadBalancerName, g.region)); err != nil { return nil, err } @@ -145,11 +145,11 @@ func (g *Cloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v // If we previously deleted the forwarding rule or it never existed, finally create it. if fwdRuleDeleted || existingFwdRule == nil { - glog.V(2).Infof("ensureInternalLoadBalancer(%v): creating forwarding rule", loadBalancerName) + klog.V(2).Infof("ensureInternalLoadBalancer(%v): creating forwarding rule", loadBalancerName) if err = g.CreateRegionForwardingRule(expectedFwdRule, g.region); err != nil { return nil, err } - glog.V(2).Infof("ensureInternalLoadBalancer(%v): created forwarding rule", loadBalancerName) + klog.V(2).Infof("ensureInternalLoadBalancer(%v): created forwarding rule", loadBalancerName) } // Delete the previous internal load balancer resources if necessary @@ -160,7 +160,7 @@ func (g *Cloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v if addrMgr != nil { // Now that the controller knows the forwarding rule exists, we can release the address. if err := addrMgr.ReleaseAddress(); err != nil { - glog.Errorf("ensureInternalLoadBalancer: failed to release address reservation, possibly causing an orphan: %v", err) + klog.Errorf("ensureInternalLoadBalancer: failed to release address reservation, possibly causing an orphan: %v", err) } } @@ -178,9 +178,9 @@ func (g *Cloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v func (g *Cloud) clearPreviousInternalResources(svc *v1.Service, loadBalancerName string, existingBackendService *compute.BackendService, expectedBSName, expectedHCName string) { // If a new backend service was created, delete the old one. if existingBackendService.Name != expectedBSName { - glog.V(2).Infof("clearPreviousInternalResources(%v): expected backend service %q does not match previous %q - deleting backend service", loadBalancerName, expectedBSName, existingBackendService.Name) + klog.V(2).Infof("clearPreviousInternalResources(%v): expected backend service %q does not match previous %q - deleting backend service", loadBalancerName, expectedBSName, existingBackendService.Name) if err := g.teardownInternalBackendService(existingBackendService.Name); err != nil && !isNotFound(err) { - glog.Warningf("clearPreviousInternalResources: could not delete old backend service: %v, err: %v", existingBackendService.Name, err) + klog.Warningf("clearPreviousInternalResources: could not delete old backend service: %v, err: %v", existingBackendService.Name, err) } } @@ -188,13 +188,13 @@ func (g *Cloud) clearPreviousInternalResources(svc *v1.Service, loadBalancerName if len(existingBackendService.HealthChecks) == 1 { existingHCName := getNameFromLink(existingBackendService.HealthChecks[0]) if existingHCName != expectedHCName { - glog.V(2).Infof("clearPreviousInternalResources(%v): expected health check %q does not match previous %q - deleting health check", loadBalancerName, expectedHCName, existingHCName) + klog.V(2).Infof("clearPreviousInternalResources(%v): expected health check %q does not match previous %q - deleting health check", loadBalancerName, expectedHCName, existingHCName) if err := g.teardownInternalHealthCheckAndFirewall(svc, existingHCName); err != nil { - glog.Warningf("clearPreviousInternalResources: could not delete existing healthcheck: %v, err: %v", existingHCName, err) + klog.Warningf("clearPreviousInternalResources: could not delete existing healthcheck: %v, err: %v", existingHCName, err) } } } else if len(existingBackendService.HealthChecks) > 1 { - glog.Warningf("clearPreviousInternalResources(%v): more than one health check on the backend service %v, %v", loadBalancerName, existingBackendService.Name, existingBackendService.HealthChecks) + klog.Warningf("clearPreviousInternalResources(%v): more than one health check on the backend service %v, %v", loadBalancerName, existingBackendService.Name, existingBackendService.HealthChecks) } } @@ -229,24 +229,24 @@ func (g *Cloud) ensureInternalLoadBalancerDeleted(clusterName, clusterID string, g.sharedResourceLock.Lock() defer g.sharedResourceLock.Unlock() - glog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): attempting delete of region internal address", loadBalancerName) + klog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): attempting delete of region internal address", loadBalancerName) ensureAddressDeleted(g, loadBalancerName, g.region) - glog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): deleting region internal forwarding rule", loadBalancerName) + klog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): deleting region internal forwarding rule", loadBalancerName) if err := ignoreNotFound(g.DeleteRegionForwardingRule(loadBalancerName, g.region)); err != nil { return err } backendServiceName := makeBackendServiceName(loadBalancerName, clusterID, sharedBackend, scheme, protocol, svc.Spec.SessionAffinity) - glog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): deleting region backend service %v", loadBalancerName, backendServiceName) + klog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): deleting region backend service %v", loadBalancerName, backendServiceName) if err := g.teardownInternalBackendService(backendServiceName); err != nil { return err } - glog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): deleting firewall for traffic", loadBalancerName) + klog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): deleting firewall for traffic", loadBalancerName) if err := ignoreNotFound(g.DeleteFirewall(loadBalancerName)); err != nil { if isForbidden(err) && g.OnXPN() { - glog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): could not delete traffic firewall on XPN cluster. Raising event.", loadBalancerName) + klog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): could not delete traffic firewall on XPN cluster. Raising event.", loadBalancerName) g.raiseFirewallChangeNeededEvent(svc, FirewallToGCloudDeleteCmd(loadBalancerName, g.NetworkProjectID())) } else { return err @@ -254,7 +254,7 @@ func (g *Cloud) ensureInternalLoadBalancerDeleted(clusterName, clusterID string, } hcName := makeHealthCheckName(loadBalancerName, clusterID, sharedHealthCheck) - glog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): deleting health check %v and its firewall", loadBalancerName, hcName) + klog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): deleting health check %v and its firewall", loadBalancerName, hcName) if err := g.teardownInternalHealthCheckAndFirewall(svc, hcName); err != nil { return err } @@ -271,49 +271,49 @@ func (g *Cloud) ensureInternalLoadBalancerDeleted(clusterName, clusterID string, func (g *Cloud) teardownInternalBackendService(bsName string) error { if err := g.DeleteRegionBackendService(bsName, g.region); err != nil { if isNotFound(err) { - glog.V(2).Infof("teardownInternalBackendService(%v): backend service already deleted. err: %v", bsName, err) + klog.V(2).Infof("teardownInternalBackendService(%v): backend service already deleted. err: %v", bsName, err) return nil } else if isInUsedByError(err) { - glog.V(2).Infof("teardownInternalBackendService(%v): backend service in use.", bsName) + klog.V(2).Infof("teardownInternalBackendService(%v): backend service in use.", bsName) return nil } else { return fmt.Errorf("failed to delete backend service: %v, err: %v", bsName, err) } } - glog.V(2).Infof("teardownInternalBackendService(%v): backend service deleted", bsName) + klog.V(2).Infof("teardownInternalBackendService(%v): backend service deleted", bsName) return nil } func (g *Cloud) teardownInternalHealthCheckAndFirewall(svc *v1.Service, hcName string) error { if err := g.DeleteHealthCheck(hcName); err != nil { if isNotFound(err) { - glog.V(2).Infof("teardownInternalHealthCheckAndFirewall(%v): health check does not exist.", hcName) + klog.V(2).Infof("teardownInternalHealthCheckAndFirewall(%v): health check does not exist.", hcName) // Purposely do not early return - double check the firewall does not exist } else if isInUsedByError(err) { - glog.V(2).Infof("teardownInternalHealthCheckAndFirewall(%v): health check in use.", hcName) + klog.V(2).Infof("teardownInternalHealthCheckAndFirewall(%v): health check in use.", hcName) return nil } else { return fmt.Errorf("failed to delete health check: %v, err: %v", hcName, err) } } - glog.V(2).Infof("teardownInternalHealthCheckAndFirewall(%v): health check deleted", hcName) + klog.V(2).Infof("teardownInternalHealthCheckAndFirewall(%v): health check deleted", hcName) hcFirewallName := makeHealthCheckFirewallNameFromHC(hcName) if err := ignoreNotFound(g.DeleteFirewall(hcFirewallName)); err != nil { if isForbidden(err) && g.OnXPN() { - glog.V(2).Infof("teardownInternalHealthCheckAndFirewall(%v): could not delete health check traffic firewall on XPN cluster. Raising Event.", hcName) + klog.V(2).Infof("teardownInternalHealthCheckAndFirewall(%v): could not delete health check traffic firewall on XPN cluster. Raising Event.", hcName) g.raiseFirewallChangeNeededEvent(svc, FirewallToGCloudDeleteCmd(hcFirewallName, g.NetworkProjectID())) return nil } return fmt.Errorf("failed to delete health check firewall: %v, err: %v", hcFirewallName, err) } - glog.V(2).Infof("teardownInternalHealthCheckAndFirewall(%v): health check firewall deleted", hcFirewallName) + klog.V(2).Infof("teardownInternalHealthCheckAndFirewall(%v): health check firewall deleted", hcFirewallName) return nil } func (g *Cloud) ensureInternalFirewall(svc *v1.Service, fwName, fwDesc string, sourceRanges []string, ports []string, protocol v1.Protocol, nodes []*v1.Node) error { - glog.V(2).Infof("ensureInternalFirewall(%v): checking existing firewall", fwName) + klog.V(2).Infof("ensureInternalFirewall(%v): checking existing firewall", fwName) targetTags, err := g.GetNodeTags(nodeNames(nodes)) if err != nil { return err @@ -339,10 +339,10 @@ func (g *Cloud) ensureInternalFirewall(svc *v1.Service, fwName, fwDesc string, s } if existingFirewall == nil { - glog.V(2).Infof("ensureInternalFirewall(%v): creating firewall", fwName) + klog.V(2).Infof("ensureInternalFirewall(%v): creating firewall", fwName) err = g.CreateFirewall(expectedFirewall) if err != nil && isForbidden(err) && g.OnXPN() { - glog.V(2).Infof("ensureInternalFirewall(%v): do not have permission to create firewall rule (on XPN). Raising event.", fwName) + klog.V(2).Infof("ensureInternalFirewall(%v): do not have permission to create firewall rule (on XPN). Raising event.", fwName) g.raiseFirewallChangeNeededEvent(svc, FirewallToGCloudCreateCmd(expectedFirewall, g.NetworkProjectID())) return nil } @@ -353,10 +353,10 @@ func (g *Cloud) ensureInternalFirewall(svc *v1.Service, fwName, fwDesc string, s return nil } - glog.V(2).Infof("ensureInternalFirewall(%v): updating firewall", fwName) + klog.V(2).Infof("ensureInternalFirewall(%v): updating firewall", fwName) err = g.UpdateFirewall(expectedFirewall) if err != nil && isForbidden(err) && g.OnXPN() { - glog.V(2).Infof("ensureInternalFirewall(%v): do not have permission to update firewall rule (on XPN). Raising event.", fwName) + klog.V(2).Infof("ensureInternalFirewall(%v): do not have permission to update firewall rule (on XPN). Raising event.", fwName) g.raiseFirewallChangeNeededEvent(svc, FirewallToGCloudUpdateCmd(expectedFirewall, g.NetworkProjectID())) return nil } @@ -383,7 +383,7 @@ func (g *Cloud) ensureInternalFirewalls(loadBalancerName, ipAddress, clusterID s } func (g *Cloud) ensureInternalHealthCheck(name string, svcName types.NamespacedName, shared bool, path string, port int32) (*compute.HealthCheck, error) { - glog.V(2).Infof("ensureInternalHealthCheck(%v, %v, %v): checking existing health check", name, path, port) + klog.V(2).Infof("ensureInternalHealthCheck(%v, %v, %v): checking existing health check", name, path, port) expectedHC := newInternalLBHealthCheck(name, svcName, shared, path, port) hc, err := g.GetHealthCheck(name) @@ -392,27 +392,27 @@ func (g *Cloud) ensureInternalHealthCheck(name string, svcName types.NamespacedN } if hc == nil { - glog.V(2).Infof("ensureInternalHealthCheck: did not find health check %v, creating one with port %v path %v", name, port, path) + klog.V(2).Infof("ensureInternalHealthCheck: did not find health check %v, creating one with port %v path %v", name, port, path) if err = g.CreateHealthCheck(expectedHC); err != nil { return nil, err } hc, err = g.GetHealthCheck(name) if err != nil { - glog.Errorf("Failed to get http health check %v", err) + klog.Errorf("Failed to get http health check %v", err) return nil, err } - glog.V(2).Infof("ensureInternalHealthCheck: created health check %v", name) + klog.V(2).Infof("ensureInternalHealthCheck: created health check %v", name) return hc, nil } if needToUpdateHealthChecks(hc, expectedHC) { - glog.V(2).Infof("ensureInternalHealthCheck: health check %v exists but parameters have drifted - updating...", name) + klog.V(2).Infof("ensureInternalHealthCheck: health check %v exists but parameters have drifted - updating...", name) expectedHC = mergeHealthChecks(hc, expectedHC) if err := g.UpdateHealthCheck(expectedHC); err != nil { - glog.Warningf("Failed to reconcile http health check %v parameters", name) + klog.Warningf("Failed to reconcile http health check %v parameters", name) return nil, err } - glog.V(2).Infof("ensureInternalHealthCheck: corrected health check %v parameters successful", name) + klog.V(2).Infof("ensureInternalHealthCheck: corrected health check %v parameters successful", name) hc, err = g.GetHealthCheck(name) if err != nil { return nil, err @@ -422,7 +422,7 @@ func (g *Cloud) ensureInternalHealthCheck(name string, svcName types.NamespacedN } func (g *Cloud) ensureInternalInstanceGroup(name, zone string, nodes []*v1.Node) (string, error) { - glog.V(2).Infof("ensureInternalInstanceGroup(%v, %v): checking group that it contains %v nodes", name, zone, len(nodes)) + klog.V(2).Infof("ensureInternalInstanceGroup(%v, %v): checking group that it contains %v nodes", name, zone, len(nodes)) ig, err := g.GetInstanceGroup(name, zone) if err != nil && !isNotFound(err) { return "", err @@ -435,7 +435,7 @@ func (g *Cloud) ensureInternalInstanceGroup(name, zone string, nodes []*v1.Node) gceNodes := sets.NewString() if ig == nil { - glog.V(2).Infof("ensureInternalInstanceGroup(%v, %v): creating instance group", name, zone) + klog.V(2).Infof("ensureInternalInstanceGroup(%v, %v): creating instance group", name, zone) newIG := &compute.InstanceGroup{Name: name} if err = g.CreateInstanceGroup(newIG, zone); err != nil { return "", err @@ -461,7 +461,7 @@ func (g *Cloud) ensureInternalInstanceGroup(name, zone string, nodes []*v1.Node) addNodes := kubeNodes.Difference(gceNodes).List() if len(removeNodes) != 0 { - glog.V(2).Infof("ensureInternalInstanceGroup(%v, %v): removing nodes: %v", name, zone, removeNodes) + klog.V(2).Infof("ensureInternalInstanceGroup(%v, %v): removing nodes: %v", name, zone, removeNodes) instanceRefs := g.ToInstanceReferences(zone, removeNodes) // Possible we'll receive 404's here if the instance was deleted before getting to this point. if err = g.RemoveInstancesFromInstanceGroup(name, zone, instanceRefs); err != nil && !isNotFound(err) { @@ -470,7 +470,7 @@ func (g *Cloud) ensureInternalInstanceGroup(name, zone string, nodes []*v1.Node) } if len(addNodes) != 0 { - glog.V(2).Infof("ensureInternalInstanceGroup(%v, %v): adding nodes: %v", name, zone, addNodes) + klog.V(2).Infof("ensureInternalInstanceGroup(%v, %v): adding nodes: %v", name, zone, addNodes) instanceRefs := g.ToInstanceReferences(zone, addNodes) if err = g.AddInstancesToInstanceGroup(name, zone, instanceRefs); err != nil { return "", err @@ -484,7 +484,7 @@ func (g *Cloud) ensureInternalInstanceGroup(name, zone string, nodes []*v1.Node) // where a K8s node exists. It also ensures that each node belongs to an instance group func (g *Cloud) ensureInternalInstanceGroups(name string, nodes []*v1.Node) ([]string, error) { zonedNodes := splitNodesByZone(nodes) - glog.V(2).Infof("ensureInternalInstanceGroups(%v): %d nodes over %d zones in region %v", name, len(nodes), len(zonedNodes), g.region) + klog.V(2).Infof("ensureInternalInstanceGroups(%v): %d nodes over %d zones in region %v", name, len(nodes), len(zonedNodes), g.region) var igLinks []string for zone, nodes := range zonedNodes { igLink, err := g.ensureInternalInstanceGroup(name, zone, nodes) @@ -504,7 +504,7 @@ func (g *Cloud) ensureInternalInstanceGroupsDeleted(name string) error { return err } - glog.V(2).Infof("ensureInternalInstanceGroupsDeleted(%v): attempting delete instance group in all %d zones", name, len(zones)) + klog.V(2).Infof("ensureInternalInstanceGroupsDeleted(%v): attempting delete instance group in all %d zones", name, len(zones)) for _, z := range zones { if err := g.DeleteInstanceGroup(name, z.Name); err != nil && !isNotFoundOrInUse(err) { return err @@ -514,7 +514,7 @@ func (g *Cloud) ensureInternalInstanceGroupsDeleted(name string) error { } func (g *Cloud) ensureInternalBackendService(name, description string, affinityType v1.ServiceAffinity, scheme cloud.LbScheme, protocol v1.Protocol, igLinks []string, hcLink string) error { - glog.V(2).Infof("ensureInternalBackendService(%v, %v, %v): checking existing backend service with %d groups", name, scheme, protocol, len(igLinks)) + klog.V(2).Infof("ensureInternalBackendService(%v, %v, %v): checking existing backend service with %d groups", name, scheme, protocol, len(igLinks)) bs, err := g.GetRegionBackendService(name, g.region) if err != nil && !isNotFound(err) { return err @@ -533,12 +533,12 @@ func (g *Cloud) ensureInternalBackendService(name, description string, affinityT // Create backend service if none was found if bs == nil { - glog.V(2).Infof("ensureInternalBackendService: creating backend service %v", name) + klog.V(2).Infof("ensureInternalBackendService: creating backend service %v", name) err := g.CreateRegionBackendService(expectedBS, g.region) if err != nil { return err } - glog.V(2).Infof("ensureInternalBackendService: created backend service %v successfully", name) + klog.V(2).Infof("ensureInternalBackendService: created backend service %v successfully", name) return nil } @@ -546,19 +546,19 @@ func (g *Cloud) ensureInternalBackendService(name, description string, affinityT return nil } - glog.V(2).Infof("ensureInternalBackendService: updating backend service %v", name) + klog.V(2).Infof("ensureInternalBackendService: updating backend service %v", name) // Set fingerprint for optimistic locking expectedBS.Fingerprint = bs.Fingerprint if err := g.UpdateRegionBackendService(expectedBS, g.region); err != nil { return err } - glog.V(2).Infof("ensureInternalBackendService: updated backend service %v successfully", name) + klog.V(2).Infof("ensureInternalBackendService: updated backend service %v successfully", name) return nil } // ensureInternalBackendServiceGroups updates backend services if their list of backend instance groups is incorrect. func (g *Cloud) ensureInternalBackendServiceGroups(name string, igLinks []string) error { - glog.V(2).Infof("ensureInternalBackendServiceGroups(%v): checking existing backend service's groups", name) + klog.V(2).Infof("ensureInternalBackendServiceGroups(%v): checking existing backend service's groups", name) bs, err := g.GetRegionBackendService(name, g.region) if err != nil { return err @@ -572,11 +572,11 @@ func (g *Cloud) ensureInternalBackendServiceGroups(name string, igLinks []string // Set the backend service's backends to the updated list. bs.Backends = backends - glog.V(2).Infof("ensureInternalBackendServiceGroups: updating backend service %v", name) + klog.V(2).Infof("ensureInternalBackendServiceGroups: updating backend service %v", name) if err := g.UpdateRegionBackendService(bs, g.region); err != nil { return err } - glog.V(2).Infof("ensureInternalBackendServiceGroups: updated backend service %v successfully", name) + klog.V(2).Infof("ensureInternalBackendServiceGroups: updated backend service %v successfully", name) return nil } diff --git a/pkg/cloudprovider/providers/gce/gce_routes.go b/pkg/cloudprovider/providers/gce/gce_routes.go index 703e2dd3cf2..cc3cbfc53a0 100644 --- a/pkg/cloudprovider/providers/gce/gce_routes.go +++ b/pkg/cloudprovider/providers/gce/gce_routes.go @@ -22,9 +22,9 @@ import ( "net/http" "path" - "github.com/golang/glog" compute "google.golang.org/api/compute/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" cloudprovider "k8s.io/cloud-provider" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" @@ -83,7 +83,7 @@ func (g *Cloud) CreateRoute(ctx context.Context, clusterName string, nameHint st } err = g.c.Routes().Insert(ctx, meta.GlobalKey(cr.Name), cr) if isHTTPErrorCode(err, http.StatusConflict) { - glog.Infof("Route %q already exists.", cr.Name) + klog.Infof("Route %q already exists.", cr.Name) err = nil } return mc.Observe(err) diff --git a/pkg/cloudprovider/providers/gce/gce_tpu.go b/pkg/cloudprovider/providers/gce/gce_tpu.go index fda62851cfb..b9fae1da2fd 100644 --- a/pkg/cloudprovider/providers/gce/gce_tpu.go +++ b/pkg/cloudprovider/providers/gce/gce_tpu.go @@ -23,9 +23,9 @@ import ( "net/http" "time" - "github.com/golang/glog" "google.golang.org/api/googleapi" tpuapi "google.golang.org/api/tpu/v1" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/wait" ) @@ -61,7 +61,7 @@ func (g *Cloud) CreateTPU(ctx context.Context, name, zone string, node *tpuapi.N if err != nil { return nil, err } - glog.V(2).Infof("Creating Cloud TPU %q in zone %q with operation %q", name, zone, op.Name) + klog.V(2).Infof("Creating Cloud TPU %q in zone %q with operation %q", name, zone, op.Name) op, err = g.waitForTPUOp(ctx, op) if err != nil { @@ -94,7 +94,7 @@ func (g *Cloud) DeleteTPU(ctx context.Context, name, zone string) error { if err != nil { return err } - glog.V(2).Infof("Deleting Cloud TPU %q in zone %q with operation %q", name, zone, op.Name) + klog.V(2).Infof("Deleting Cloud TPU %q in zone %q with operation %q", name, zone, op.Name) op, err = g.waitForTPUOp(ctx, op) if err != nil { @@ -149,18 +149,18 @@ func (g *Cloud) waitForTPUOp(ctx context.Context, op *tpuapi.Operation) (*tpuapi // Check if context has been cancelled. select { case <-ctx.Done(): - glog.V(3).Infof("Context for operation %q has been cancelled: %s", op.Name, ctx.Err()) + klog.V(3).Infof("Context for operation %q has been cancelled: %s", op.Name, ctx.Err()) return true, ctx.Err() default: } - glog.V(3).Infof("Waiting for operation %q to complete...", op.Name) + klog.V(3).Infof("Waiting for operation %q to complete...", op.Name) start := time.Now() g.operationPollRateLimiter.Accept() duration := time.Now().Sub(start) if duration > 5*time.Second { - glog.V(2).Infof("Getting operation %q throttled for %v", op.Name, duration) + klog.V(2).Infof("Getting operation %q throttled for %v", op.Name, duration) } var err error @@ -169,7 +169,7 @@ func (g *Cloud) waitForTPUOp(ctx context.Context, op *tpuapi.Operation) (*tpuapi return true, err } if op.Done { - glog.V(3).Infof("Operation %q has completed", op.Name) + klog.V(3).Infof("Operation %q has completed", op.Name) return true, nil } return false, nil diff --git a/pkg/cloudprovider/providers/openstack/BUILD b/pkg/cloudprovider/providers/openstack/BUILD index 65997dff181..6e45765123f 100644 --- a/pkg/cloudprovider/providers/openstack/BUILD +++ b/pkg/cloudprovider/providers/openstack/BUILD @@ -34,7 +34,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/gophercloud/gophercloud:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions:go_default_library", @@ -62,6 +61,7 @@ go_library( "//vendor/github.com/mitchellh/mapstructure:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/gopkg.in/gcfg.v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/cloudprovider/providers/openstack/metadata.go b/pkg/cloudprovider/providers/openstack/metadata.go index 6ef8eb98d85..2ba32f59d59 100644 --- a/pkg/cloudprovider/providers/openstack/metadata.go +++ b/pkg/cloudprovider/providers/openstack/metadata.go @@ -27,7 +27,7 @@ import ( "path/filepath" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/utils/exec" ) @@ -121,7 +121,7 @@ func getMetadataFromConfigDrive(metadataVersion string) (*Metadata, error) { } defer os.Remove(mntdir) - glog.V(4).Infof("Attempting to mount configdrive %s on %s", dev, mntdir) + klog.V(4).Infof("Attempting to mount configdrive %s on %s", dev, mntdir) mounter := mount.New("" /* default mount path */) err = mounter.Mount(dev, mntdir, "iso9660", []string{"ro"}) @@ -133,7 +133,7 @@ func getMetadataFromConfigDrive(metadataVersion string) (*Metadata, error) { } defer mounter.Unmount(mntdir) - glog.V(4).Infof("Configdrive mounted on %s", mntdir) + klog.V(4).Infof("Configdrive mounted on %s", mntdir) configDrivePath := getConfigDrivePath(metadataVersion) f, err := os.Open( @@ -149,7 +149,7 @@ func getMetadataFromConfigDrive(metadataVersion string) (*Metadata, error) { func getMetadataFromMetadataService(metadataVersion string) (*Metadata, error) { // Try to get JSON from metadata server. metadataURL := getMetadataURL(metadataVersion) - glog.V(4).Infof("Attempting to fetch metadata from %s", metadataURL) + klog.V(4).Infof("Attempting to fetch metadata from %s", metadataURL) resp, err := http.Get(metadataURL) if err != nil { return nil, fmt.Errorf("error fetching %s: %v", metadataURL, err) diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/pkg/cloudprovider/providers/openstack/openstack.go index fbcee0f42a9..9f13db944e1 100644 --- a/pkg/cloudprovider/providers/openstack/openstack.go +++ b/pkg/cloudprovider/providers/openstack/openstack.go @@ -41,12 +41,12 @@ import ( "github.com/mitchellh/mapstructure" "gopkg.in/gcfg.v1" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" netutil "k8s.io/apimachinery/pkg/util/net" certutil "k8s.io/client-go/util/cert" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" ) @@ -277,7 +277,7 @@ func readInstanceID(searchOrder string) (string, error) { if err == nil { instanceID := string(idBytes) instanceID = strings.TrimSpace(instanceID) - glog.V(3).Infof("Got instance id from %s: %s", instanceIDFile, instanceID) + klog.V(3).Infof("Got instance id from %s: %s", instanceIDFile, instanceID) if instanceID != "" { return instanceID, nil } @@ -584,10 +584,10 @@ func (os *OpenStack) HasClusterID() bool { // LoadBalancer initializes a LbaasV2 object func (os *OpenStack) LoadBalancer() (cloudprovider.LoadBalancer, bool) { - glog.V(4).Info("openstack.LoadBalancer() called") + klog.V(4).Info("openstack.LoadBalancer() called") if reflect.DeepEqual(os.lbOpts, LoadBalancerOpts{}) { - glog.V(4).Info("LoadBalancer section is empty/not defined in cloud-config") + klog.V(4).Info("LoadBalancer section is empty/not defined in cloud-config") return nil, false } @@ -610,11 +610,11 @@ func (os *OpenStack) LoadBalancer() (cloudprovider.LoadBalancer, bool) { // Currently kubernetes OpenStack cloud provider just support LBaaS v2. lbVersion := os.lbOpts.LBVersion if lbVersion != "" && lbVersion != "v2" { - glog.Warningf("Config error: currently only support LBaaS v2, unrecognised lb-version \"%v\"", lbVersion) + klog.Warningf("Config error: currently only support LBaaS v2, unrecognised lb-version \"%v\"", lbVersion) return nil, false } - glog.V(1).Info("Claiming to support LoadBalancer") + klog.V(1).Info("Claiming to support LoadBalancer") return &LbaasV2{LoadBalancer{network, compute, lb, os.lbOpts}}, true } @@ -635,7 +635,7 @@ func isNotFound(err error) bool { // Zones indicates that we support zones func (os *OpenStack) Zones() (cloudprovider.Zones, bool) { - glog.V(1).Info("Claiming to support Zones") + klog.V(1).Info("Claiming to support Zones") return os, true } @@ -650,7 +650,7 @@ func (os *OpenStack) GetZone(ctx context.Context) (cloudprovider.Zone, error) { FailureDomain: md.AvailabilityZone, Region: os.region, } - glog.V(4).Infof("Current zone is %v", zone) + klog.V(4).Infof("Current zone is %v", zone) return zone, nil } @@ -677,7 +677,7 @@ func (os *OpenStack) GetZoneByProviderID(ctx context.Context, providerID string) FailureDomain: srv.Metadata[availabilityZone], Region: os.region, } - glog.V(4).Infof("The instance %s in zone %v", srv.Name, zone) + klog.V(4).Infof("The instance %s in zone %v", srv.Name, zone) return zone, nil } @@ -702,13 +702,13 @@ func (os *OpenStack) GetZoneByNodeName(ctx context.Context, nodeName types.NodeN FailureDomain: srv.Metadata[availabilityZone], Region: os.region, } - glog.V(4).Infof("The instance %s in zone %v", srv.Name, zone) + klog.V(4).Infof("The instance %s in zone %v", srv.Name, zone) return zone, nil } // Routes initializes routes support func (os *OpenStack) Routes() (cloudprovider.Routes, bool) { - glog.V(4).Info("openstack.Routes() called") + klog.V(4).Info("openstack.Routes() called") network, err := os.NewNetworkV2() if err != nil { @@ -717,12 +717,12 @@ func (os *OpenStack) Routes() (cloudprovider.Routes, bool) { netExts, err := networkExtensions(network) if err != nil { - glog.Warningf("Failed to list neutron extensions: %v", err) + klog.Warningf("Failed to list neutron extensions: %v", err) return nil, false } if !netExts["extraroute"] { - glog.V(3).Info("Neutron extraroute extension not found, required for Routes support") + klog.V(3).Info("Neutron extraroute extension not found, required for Routes support") return nil, false } @@ -733,11 +733,11 @@ func (os *OpenStack) Routes() (cloudprovider.Routes, bool) { r, err := NewRoutes(compute, network, os.routeOpts) if err != nil { - glog.Warningf("Error initialising Routes support: %v", err) + klog.Warningf("Error initialising Routes support: %v", err) return nil, false } - glog.V(1).Info("Claiming to support Routes") + klog.V(1).Info("Claiming to support Routes") return r, true } @@ -755,21 +755,21 @@ func (os *OpenStack) volumeService(forceVersion string) (volumeService, error) { if err != nil { return nil, err } - glog.V(3).Info("Using Blockstorage API V1") + klog.V(3).Info("Using Blockstorage API V1") return &VolumesV1{sClient, os.bsOpts}, nil case "v2": sClient, err := os.NewBlockStorageV2() if err != nil { return nil, err } - glog.V(3).Info("Using Blockstorage API V2") + klog.V(3).Info("Using Blockstorage API V2") return &VolumesV2{sClient, os.bsOpts}, nil case "v3": sClient, err := os.NewBlockStorageV3() if err != nil { return nil, err } - glog.V(3).Info("Using Blockstorage API V3") + klog.V(3).Info("Using Blockstorage API V3") return &VolumesV3{sClient, os.bsOpts}, nil case "auto": // Currently kubernetes support Cinder v1 / Cinder v2 / Cinder v3. @@ -777,17 +777,17 @@ func (os *OpenStack) volumeService(forceVersion string) (volumeService, error) { // If kubernetes can't initialize cinder v2 client, try to initialize cinder v1 client. // Return appropriate message when kubernetes can't initialize them. if sClient, err := os.NewBlockStorageV3(); err == nil { - glog.V(3).Info("Using Blockstorage API V3") + klog.V(3).Info("Using Blockstorage API V3") return &VolumesV3{sClient, os.bsOpts}, nil } if sClient, err := os.NewBlockStorageV2(); err == nil { - glog.V(3).Info("Using Blockstorage API V2") + klog.V(3).Info("Using Blockstorage API V2") return &VolumesV2{sClient, os.bsOpts}, nil } if sClient, err := os.NewBlockStorageV1(); err == nil { - glog.V(3).Info("Using Blockstorage API V1") + klog.V(3).Info("Using Blockstorage API V1") return &VolumesV1{sClient, os.bsOpts}, nil } diff --git a/pkg/cloudprovider/providers/openstack/openstack_instances.go b/pkg/cloudprovider/providers/openstack/openstack_instances.go index c85f74a9be9..c52ce21998e 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_instances.go +++ b/pkg/cloudprovider/providers/openstack/openstack_instances.go @@ -21,9 +21,9 @@ import ( "fmt" "regexp" - "github.com/golang/glog" "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -42,15 +42,15 @@ const ( // Instances returns an implementation of Instances for OpenStack. func (os *OpenStack) Instances() (cloudprovider.Instances, bool) { - glog.V(4).Info("openstack.Instances() called") + klog.V(4).Info("openstack.Instances() called") compute, err := os.NewComputeV2() if err != nil { - glog.Errorf("unable to access compute v2 API : %v", err) + klog.Errorf("unable to access compute v2 API : %v", err) return nil, false } - glog.V(4).Info("Claiming to support Instances") + klog.V(4).Info("Claiming to support Instances") return &Instances{ compute: compute, @@ -75,14 +75,14 @@ func (i *Instances) AddSSHKeyToAllInstances(ctx context.Context, user string, ke // NodeAddresses implements Instances.NodeAddresses func (i *Instances) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.NodeAddress, error) { - glog.V(4).Infof("NodeAddresses(%v) called", name) + klog.V(4).Infof("NodeAddresses(%v) called", name) addrs, err := getAddressesByName(i.compute, name) if err != nil { return nil, err } - glog.V(4).Infof("NodeAddresses(%v) => %v", name, addrs) + klog.V(4).Infof("NodeAddresses(%v) => %v", name, addrs) return addrs, nil } diff --git a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go b/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go index d5209768262..26e6b095a2c 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go +++ b/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go @@ -24,7 +24,6 @@ import ( "strings" "time" - "github.com/golang/glog" "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external" @@ -38,6 +37,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" "github.com/gophercloud/gophercloud/pagination" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -521,16 +521,16 @@ func nodeAddressForLB(node *v1.Node) (string, error) { //getStringFromServiceAnnotation searches a given v1.Service for a specific annotationKey and either returns the annotation's value or a specified defaultSetting func getStringFromServiceAnnotation(service *v1.Service, annotationKey string, defaultSetting string) string { - glog.V(4).Infof("getStringFromServiceAnnotation(%v, %v, %v)", service, annotationKey, defaultSetting) + klog.V(4).Infof("getStringFromServiceAnnotation(%v, %v, %v)", service, annotationKey, defaultSetting) if annotationValue, ok := service.Annotations[annotationKey]; ok { //if there is an annotation for this setting, set the "setting" var to it // annotationValue can be empty, it is working as designed // it makes possible for instance provisioning loadbalancer without floatingip - glog.V(4).Infof("Found a Service Annotation: %v = %v", annotationKey, annotationValue) + klog.V(4).Infof("Found a Service Annotation: %v = %v", annotationKey, annotationValue) return annotationValue } //if there is no annotation, set "settings" var to the value from cloud config - glog.V(4).Infof("Could not find a Service Annotation; falling back on cloud-config setting: %v = %v", annotationKey, defaultSetting) + klog.V(4).Infof("Could not find a Service Annotation; falling back on cloud-config setting: %v = %v", annotationKey, defaultSetting) return defaultSetting } @@ -641,7 +641,7 @@ func getFloatingNetworkIDForLB(client *gophercloud.ServiceClient) (string, error } if err == ErrMultipleResults { - glog.V(4).Infof("find multiple external networks, pick the first one when there are no explicit configuration.") + klog.V(4).Infof("find multiple external networks, pick the first one when there are no explicit configuration.") return floatingNetworkIds[0], nil } return "", err @@ -661,7 +661,7 @@ func getFloatingNetworkIDForLB(client *gophercloud.ServiceClient) (string, error // EnsureLoadBalancer creates a new load balancer 'name', or updates the existing one. func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string, apiService *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { - glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)", clusterName, apiService.Namespace, apiService.Name, apiService.Spec.LoadBalancerIP, apiService.Spec.Ports, nodes, apiService.Annotations) + klog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)", clusterName, apiService.Namespace, apiService.Name, apiService.Spec.LoadBalancerIP, apiService.Spec.Ports, nodes, apiService.Annotations) if len(nodes) == 0 { return nil, fmt.Errorf("there are no available nodes for LoadBalancer service %s/%s", apiService.Namespace, apiService.Name) @@ -673,7 +673,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string // The LB needs to be configured with instance addresses on the same subnet, so get SubnetID by one node. subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0]) if err != nil { - glog.Warningf("Failed to find subnet-id for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) + klog.Warningf("Failed to find subnet-id for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) return nil, fmt.Errorf("no subnet-id for service %s/%s : subnet-id not set in cloud provider config, "+ "and failed to find subnet-id from OpenStack: %v", apiService.Namespace, apiService.Name, err) } @@ -690,7 +690,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string var err error floatingPool, err = getFloatingNetworkIDForLB(lbaas.network) if err != nil { - glog.Warningf("Failed to find floating-network-id for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) + klog.Warningf("Failed to find floating-network-id for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) } } @@ -698,11 +698,11 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string internal := getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerInternal, "false") switch internal { case "true": - glog.V(4).Infof("Ensure an internal loadbalancer service.") + klog.V(4).Infof("Ensure an internal loadbalancer service.") internalAnnotation = true case "false": if len(floatingPool) != 0 { - glog.V(4).Infof("Ensure an external loadbalancer service, using floatingPool: %v", floatingPool) + klog.V(4).Infof("Ensure an external loadbalancer service, using floatingPool: %v", floatingPool) internalAnnotation = false } else { return nil, fmt.Errorf("floating-network-id or loadbalancer.openstack.org/floating-network-id should be specified when ensuring an external loadbalancer service") @@ -746,14 +746,14 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string if err != ErrNotFound { return nil, fmt.Errorf("error getting loadbalancer %s: %v", name, err) } - glog.V(2).Infof("Creating loadbalancer %s", name) + klog.V(2).Infof("Creating loadbalancer %s", name) loadbalancer, err = lbaas.createLoadBalancer(apiService, name, internalAnnotation) if err != nil { // Unknown error, retry later return nil, fmt.Errorf("error creating loadbalancer %s: %v", name, err) } } else { - glog.V(2).Infof("LoadBalancer %s already exists", name) + klog.V(2).Infof("LoadBalancer %s already exists", name) } provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) @@ -773,7 +773,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string for portIndex, port := range ports { listener := getListenerForPort(oldListeners, port) if listener == nil { - glog.V(4).Infof("Creating listener for port %d", int(port.Port)) + klog.V(4).Infof("Creating listener for port %d", int(port.Port)) listener, err = listeners.Create(lbaas.lb, listeners.CreateOpts{ Name: fmt.Sprintf("listener_%s_%d", name, portIndex), Protocol: listeners.Protocol(port.Protocol), @@ -790,7 +790,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string } } - glog.V(4).Infof("Listener for %s port %d: %s", string(port.Protocol), int(port.Port), listener.ID) + klog.V(4).Infof("Listener for %s port %d: %s", string(port.Protocol), int(port.Port), listener.ID) // After all ports have been processed, remaining listeners are removed as obsolete. // Pop valid listeners. @@ -801,7 +801,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string return nil, fmt.Errorf("error getting pool for listener %s: %v", listener.ID, err) } if pool == nil { - glog.V(4).Infof("Creating pool for listener %s", listener.ID) + klog.V(4).Infof("Creating pool for listener %s", listener.ID) pool, err = v2pools.Create(lbaas.lb, v2pools.CreateOpts{ Name: fmt.Sprintf("pool_%s_%d", name, portIndex), Protocol: v2pools.Protocol(port.Protocol), @@ -820,7 +820,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string } - glog.V(4).Infof("Pool for listener %s: %s", listener.ID, pool.ID) + klog.V(4).Infof("Pool for listener %s: %s", listener.ID, pool.ID) members, err := getMembersByPoolID(lbaas.lb, pool.ID) if err != nil && !isNotFound(err) { return nil, fmt.Errorf("error getting pool members %s: %v", pool.ID, err) @@ -830,7 +830,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string if err != nil { if err == ErrNotFound { // Node failure, do not create member - glog.Warningf("Failed to create LB pool member for node %s: %v", node.Name, err) + klog.Warningf("Failed to create LB pool member for node %s: %v", node.Name, err) continue } else { return nil, fmt.Errorf("error getting address for node %s: %v", node.Name, err) @@ -838,7 +838,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string } if !memberExists(members, addr, int(port.NodePort)) { - glog.V(4).Infof("Creating member for pool %s", pool.ID) + klog.V(4).Infof("Creating member for pool %s", pool.ID) _, err := v2pools.CreateMember(lbaas.lb, pool.ID, v2pools.CreateMemberOpts{ Name: fmt.Sprintf("member_%s_%d_%s", name, portIndex, node.Name), ProtocolPort: int(port.NodePort), @@ -858,12 +858,12 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string members = popMember(members, addr, int(port.NodePort)) } - glog.V(4).Infof("Ensured pool %s has member for %s at %s", pool.ID, node.Name, addr) + klog.V(4).Infof("Ensured pool %s has member for %s at %s", pool.ID, node.Name, addr) } // Delete obsolete members for this pool for _, member := range members { - glog.V(4).Infof("Deleting obsolete member %s for pool %s address %s", member.ID, pool.ID, member.Address) + klog.V(4).Infof("Deleting obsolete member %s for pool %s address %s", member.ID, pool.ID, member.Address) err := v2pools.DeleteMember(lbaas.lb, pool.ID, member.ID).ExtractErr() if err != nil && !isNotFound(err) { return nil, fmt.Errorf("error deleting obsolete member %s for pool %s address %s: %v", member.ID, pool.ID, member.Address, err) @@ -876,7 +876,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string monitorID := pool.MonitorID if monitorID == "" && lbaas.opts.CreateMonitor { - glog.V(4).Infof("Creating monitor for pool %s", pool.ID) + klog.V(4).Infof("Creating monitor for pool %s", pool.ID) monitor, err := v2monitors.Create(lbaas.lb, v2monitors.CreateOpts{ Name: fmt.Sprintf("monitor_%s_%d", name, portIndex), PoolID: pool.ID, @@ -894,17 +894,17 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string } monitorID = monitor.ID } else if lbaas.opts.CreateMonitor == false { - glog.V(4).Infof("Do not create monitor for pool %s when create-monitor is false", pool.ID) + klog.V(4).Infof("Do not create monitor for pool %s when create-monitor is false", pool.ID) } if monitorID != "" { - glog.V(4).Infof("Monitor for pool %s: %s", pool.ID, monitorID) + klog.V(4).Infof("Monitor for pool %s: %s", pool.ID, monitorID) } } // All remaining listeners are obsolete, delete for _, listener := range oldListeners { - glog.V(4).Infof("Deleting obsolete listener %s:", listener.ID) + klog.V(4).Infof("Deleting obsolete listener %s:", listener.ID) // get pool for listener pool, err := getPoolByListenerID(lbaas.lb, loadbalancer.ID, listener.ID) if err != nil && err != ErrNotFound { @@ -914,7 +914,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string // get and delete monitor monitorID := pool.MonitorID if monitorID != "" { - glog.V(4).Infof("Deleting obsolete monitor %s for pool %s", monitorID, pool.ID) + klog.V(4).Infof("Deleting obsolete monitor %s for pool %s", monitorID, pool.ID) err = v2monitors.Delete(lbaas.lb, monitorID).ExtractErr() if err != nil && !isNotFound(err) { return nil, fmt.Errorf("error deleting obsolete monitor %s for pool %s: %v", monitorID, pool.ID, err) @@ -931,7 +931,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string } if members != nil { for _, member := range members { - glog.V(4).Infof("Deleting obsolete member %s for pool %s address %s", member.ID, pool.ID, member.Address) + klog.V(4).Infof("Deleting obsolete member %s for pool %s address %s", member.ID, pool.ID, member.Address) err := v2pools.DeleteMember(lbaas.lb, pool.ID, member.ID).ExtractErr() if err != nil && !isNotFound(err) { return nil, fmt.Errorf("error deleting obsolete member %s for pool %s address %s: %v", member.ID, pool.ID, member.Address, err) @@ -942,7 +942,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string } } } - glog.V(4).Infof("Deleting obsolete pool %s for listener %s", pool.ID, listener.ID) + klog.V(4).Infof("Deleting obsolete pool %s for listener %s", pool.ID, listener.ID) // delete pool err = v2pools.Delete(lbaas.lb, pool.ID).ExtractErr() if err != nil && !isNotFound(err) { @@ -962,7 +962,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string if err != nil { return nil, fmt.Errorf("failed to loadbalance ACTIVE provisioning status %v: %v", provisioningStatus, err) } - glog.V(2).Infof("Deleted obsolete listener: %s", listener.ID) + klog.V(2).Infof("Deleted obsolete listener: %s", listener.ID) } portID := loadbalancer.VipPortID @@ -971,7 +971,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string return nil, fmt.Errorf("error getting floating ip for port %s: %v", portID, err) } if floatIP == nil && floatingPool != "" && !internalAnnotation { - glog.V(4).Infof("Creating floating ip for loadbalancer %s port %s", loadbalancer.ID, portID) + klog.V(4).Infof("Creating floating ip for loadbalancer %s port %s", loadbalancer.ID, portID) floatIPOpts := floatingips.CreateOpts{ FloatingNetworkID: floatingPool, PortID: portID, @@ -1019,7 +1019,7 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *v1.Ser return fmt.Errorf("failed to find node-security-group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) } } - glog.V(4).Infof("find node-security-group %v for loadbalancer service %s/%s", lbaas.opts.NodeSecurityGroupIDs, apiService.Namespace, apiService.Name) + klog.V(4).Infof("find node-security-group %v for loadbalancer service %s/%s", lbaas.opts.NodeSecurityGroupIDs, apiService.Namespace, apiService.Name) // get service ports ports := apiService.Spec.Ports @@ -1183,7 +1183,7 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *v1.Ser // UpdateLoadBalancer updates hosts under the specified load balancer. func (lbaas *LbaasV2) UpdateLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) error { loadBalancerName := lbaas.GetLoadBalancerName(ctx, clusterName, service) - glog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v)", clusterName, loadBalancerName, nodes) + klog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v)", clusterName, loadBalancerName, nodes) lbaas.opts.SubnetID = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerSubnetID, lbaas.opts.SubnetID) if len(lbaas.opts.SubnetID) == 0 && len(nodes) > 0 { @@ -1191,7 +1191,7 @@ func (lbaas *LbaasV2) UpdateLoadBalancer(ctx context.Context, clusterName string // The LB needs to be configured with instance addresses on the same subnet, so get SubnetID by one node. subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0]) if err != nil { - glog.Warningf("Failed to find subnet-id for loadbalancer service %s/%s: %v", service.Namespace, service.Name, err) + klog.Warningf("Failed to find subnet-id for loadbalancer service %s/%s: %v", service.Namespace, service.Name, err) return fmt.Errorf("no subnet-id for service %s/%s : subnet-id not set in cloud provider config, "+ "and failed to find subnet-id from OpenStack: %v", service.Namespace, service.Name, err) } @@ -1332,7 +1332,7 @@ func (lbaas *LbaasV2) updateSecurityGroup(clusterName string, apiService *v1.Ser if err != nil { return fmt.Errorf("failed to find node-security-group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) } - glog.V(4).Infof("find node-security-group %v for loadbalancer service %s/%s", lbaas.opts.NodeSecurityGroupIDs, apiService.Namespace, apiService.Name) + klog.V(4).Infof("find node-security-group %v for loadbalancer service %s/%s", lbaas.opts.NodeSecurityGroupIDs, apiService.Namespace, apiService.Name) original := sets.NewString(originalNodeSecurityGroupIDs...) current := sets.NewString(lbaas.opts.NodeSecurityGroupIDs...) @@ -1406,7 +1406,7 @@ func (lbaas *LbaasV2) updateSecurityGroup(clusterName string, apiService *v1.Ser // EnsureLoadBalancerDeleted deletes the specified load balancer func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *v1.Service) error { loadBalancerName := lbaas.GetLoadBalancerName(ctx, clusterName, service) - glog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v)", clusterName, loadBalancerName) + klog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v)", clusterName, loadBalancerName) loadbalancer, err := getLoadbalancerByName(lbaas.lb, loadBalancerName) if err != nil && err != ErrNotFound { @@ -1550,7 +1550,7 @@ func (lbaas *LbaasV2) EnsureSecurityGroupDeleted(clusterName string, service *v1 // Just happen when nodes have not Security Group, or should not happen // UpdateLoadBalancer and EnsureLoadBalancer can set lbaas.opts.NodeSecurityGroupIDs when it is empty // And service controller call UpdateLoadBalancer to set lbaas.opts.NodeSecurityGroupIDs when controller manager service is restarted. - glog.Warningf("Can not find node-security-group from all the nodes of this cluster when delete loadbalancer service %s/%s", + klog.Warningf("Can not find node-security-group from all the nodes of this cluster when delete loadbalancer service %s/%s", service.Namespace, service.Name) } else { // Delete the rules in the Node Security Group diff --git a/pkg/cloudprovider/providers/openstack/openstack_routes.go b/pkg/cloudprovider/providers/openstack/openstack_routes.go index f9434e3b30b..0cb31a435a7 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_routes.go +++ b/pkg/cloudprovider/providers/openstack/openstack_routes.go @@ -26,9 +26,9 @@ import ( "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers" neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" ) var errNoRouterID = errors.New("router-id not set in cloud provider config") @@ -55,7 +55,7 @@ func NewRoutes(compute *gophercloud.ServiceClient, network *gophercloud.ServiceC // ListRoutes lists all managed routes that belong to the specified clusterName func (r *Routes) ListRoutes(ctx context.Context, clusterName string) ([]*cloudprovider.Route, error) { - glog.V(4).Infof("ListRoutes(%v)", clusterName) + klog.V(4).Infof("ListRoutes(%v)", clusterName) nodeNamesByAddr := make(map[string]types.NodeName) err := foreachServer(r.compute, servers.ListOpts{}, func(srv *servers.Server) (bool, error) { @@ -109,12 +109,12 @@ func updateRoutes(network *gophercloud.ServiceClient, router *routers.Router, ne } unwinder := func() { - glog.V(4).Info("Reverting routes change to router ", router.ID) + klog.V(4).Info("Reverting routes change to router ", router.ID) _, err := routers.Update(network, router.ID, routers.UpdateOpts{ Routes: origRoutes, }).Extract() if err != nil { - glog.Warning("Unable to reset routes during error unwind: ", err) + klog.Warning("Unable to reset routes during error unwind: ", err) } } @@ -132,12 +132,12 @@ func updateAllowedAddressPairs(network *gophercloud.ServiceClient, port *neutron } unwinder := func() { - glog.V(4).Info("Reverting allowed-address-pairs change to port ", port.ID) + klog.V(4).Info("Reverting allowed-address-pairs change to port ", port.ID) _, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{ AllowedAddressPairs: &origPairs, }).Extract() if err != nil { - glog.Warning("Unable to reset allowed-address-pairs during error unwind: ", err) + klog.Warning("Unable to reset allowed-address-pairs during error unwind: ", err) } } @@ -146,7 +146,7 @@ func updateAllowedAddressPairs(network *gophercloud.ServiceClient, port *neutron // CreateRoute creates the described managed route func (r *Routes) CreateRoute(ctx context.Context, clusterName string, nameHint string, route *cloudprovider.Route) error { - glog.V(4).Infof("CreateRoute(%v, %v, %v)", clusterName, nameHint, route) + klog.V(4).Infof("CreateRoute(%v, %v, %v)", clusterName, nameHint, route) onFailure := newCaller() @@ -158,7 +158,7 @@ func (r *Routes) CreateRoute(ctx context.Context, clusterName string, nameHint s return err } - glog.V(4).Infof("Using nexthop %v for node %v", addr, route.TargetNode) + klog.V(4).Infof("Using nexthop %v for node %v", addr, route.TargetNode) router, err := routers.Get(r.network, r.opts.RouterID).Extract() if err != nil { @@ -169,7 +169,7 @@ func (r *Routes) CreateRoute(ctx context.Context, clusterName string, nameHint s for _, item := range routes { if item.DestinationCIDR == route.DestinationCIDR && item.NextHop == addr { - glog.V(4).Infof("Skipping existing route: %v", route) + klog.V(4).Infof("Skipping existing route: %v", route) return nil } } @@ -198,7 +198,7 @@ func (r *Routes) CreateRoute(ctx context.Context, clusterName string, nameHint s found := false for _, item := range port.AllowedAddressPairs { if item.IPAddress == route.DestinationCIDR { - glog.V(4).Info("Found existing allowed-address-pair: ", item) + klog.V(4).Info("Found existing allowed-address-pair: ", item) found = true break } @@ -215,14 +215,14 @@ func (r *Routes) CreateRoute(ctx context.Context, clusterName string, nameHint s defer onFailure.call(unwind) } - glog.V(4).Infof("Route created: %v", route) + klog.V(4).Infof("Route created: %v", route) onFailure.disarm() return nil } // DeleteRoute deletes the specified managed route func (r *Routes) DeleteRoute(ctx context.Context, clusterName string, route *cloudprovider.Route) error { - glog.V(4).Infof("DeleteRoute(%v, %v)", clusterName, route) + klog.V(4).Infof("DeleteRoute(%v, %v)", clusterName, route) onFailure := newCaller() @@ -255,7 +255,7 @@ func (r *Routes) DeleteRoute(ctx context.Context, clusterName string, route *clo } if index == -1 { - glog.V(4).Infof("Skipping non-existent route: %v", route) + klog.V(4).Infof("Skipping non-existent route: %v", route) return nil } @@ -301,7 +301,7 @@ func (r *Routes) DeleteRoute(ctx context.Context, clusterName string, route *clo defer onFailure.call(unwind) } - glog.V(4).Infof("Route deleted: %v", route) + klog.V(4).Infof("Route deleted: %v", route) onFailure.disarm() return nil } diff --git a/pkg/cloudprovider/providers/openstack/openstack_volumes.go b/pkg/cloudprovider/providers/openstack/openstack_volumes.go index d3e5cf1d3b6..68b8f92a812 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_volumes.go +++ b/pkg/cloudprovider/providers/openstack/openstack_volumes.go @@ -42,7 +42,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach" "github.com/prometheus/client_golang/prometheus" - "github.com/golang/glog" + "k8s.io/klog" ) type volumeService interface { @@ -334,19 +334,19 @@ func (os *OpenStack) AttachDisk(instanceID, volumeID string) (string, error) { if volume.AttachedServerID != "" { if instanceID == volume.AttachedServerID { - glog.V(4).Infof("Disk %s is already attached to instance %s", volumeID, instanceID) + klog.V(4).Infof("Disk %s is already attached to instance %s", volumeID, instanceID) return volume.ID, nil } nodeName, err := os.GetNodeNameByID(volume.AttachedServerID) attachErr := fmt.Sprintf("disk %s path %s is attached to a different instance (%s)", volumeID, volume.AttachedDevice, volume.AttachedServerID) if err != nil { - glog.Error(attachErr) + klog.Error(attachErr) return "", errors.New(attachErr) } // using volume.AttachedDevice may cause problems because cinder does not report device path correctly see issue #33128 devicePath := volume.AttachedDevice danglingErr := volumeutil.NewDanglingError(attachErr, nodeName, devicePath) - glog.V(2).Infof("Found dangling volume %s attached to node %s", volumeID, nodeName) + klog.V(2).Infof("Found dangling volume %s attached to node %s", volumeID, nodeName) return "", danglingErr } @@ -360,7 +360,7 @@ func (os *OpenStack) AttachDisk(instanceID, volumeID string) (string, error) { if err != nil { return "", fmt.Errorf("failed to attach %s volume to %s compute: %v", volumeID, instanceID, err) } - glog.V(2).Infof("Successfully attached %s volume to %s compute", volumeID, instanceID) + klog.V(2).Infof("Successfully attached %s volume to %s compute", volumeID, instanceID) return volume.ID, nil } @@ -372,7 +372,7 @@ func (os *OpenStack) DetachDisk(instanceID, volumeID string) error { } if volume.Status == volumeAvailableStatus { // "available" is fine since that means the volume is detached from instance already. - glog.V(2).Infof("volume: %s has been detached from compute: %s ", volume.ID, instanceID) + klog.V(2).Infof("volume: %s has been detached from compute: %s ", volume.ID, instanceID) return nil } @@ -396,7 +396,7 @@ func (os *OpenStack) DetachDisk(instanceID, volumeID string) error { if err != nil { return fmt.Errorf("failed to delete volume %s from compute %s attached %v", volume.ID, instanceID, err) } - glog.V(2).Infof("Successfully detached volume: %s from compute: %s", volume.ID, instanceID) + klog.V(2).Infof("Successfully detached volume: %s from compute: %s", volume.ID, instanceID) return nil } @@ -468,7 +468,7 @@ func (os *OpenStack) CreateVolume(name string, size int, vtype, availability str return "", "", "", os.bsOpts.IgnoreVolumeAZ, fmt.Errorf("failed to create a %d GB volume: %v", size, err) } - glog.Infof("Created volume %v in Availability Zone: %v Region: %v Ignore volume AZ: %v", volumeID, volumeAZ, os.region, os.bsOpts.IgnoreVolumeAZ) + klog.Infof("Created volume %v in Availability Zone: %v Region: %v Ignore volume AZ: %v", volumeID, volumeAZ, os.region, os.bsOpts.IgnoreVolumeAZ) return volumeID, volumeAZ, os.region, os.bsOpts.IgnoreVolumeAZ, nil } @@ -490,13 +490,13 @@ func (os *OpenStack) GetDevicePathBySerialID(volumeID string) string { for _, f := range files { for _, c := range candidateDeviceNodes { if c == f.Name() { - glog.V(4).Infof("Found disk attached as %q; full devicepath: %s\n", f.Name(), path.Join("/dev/disk/by-id/", f.Name())) + klog.V(4).Infof("Found disk attached as %q; full devicepath: %s\n", f.Name(), path.Join("/dev/disk/by-id/", f.Name())) return path.Join("/dev/disk/by-id/", f.Name()) } } } - glog.V(4).Infof("Failed to find device for the volumeID: %q by serial ID", volumeID) + klog.V(4).Infof("Failed to find device for the volumeID: %q by serial ID", volumeID) return "" } @@ -511,14 +511,14 @@ func (os *OpenStack) getDevicePathFromInstanceMetadata(volumeID string) string { newtonMetadataVersion) if err != nil { - glog.V(4).Infof( + klog.V(4).Infof( "Could not retrieve instance metadata. Error: %v", err) return "" } for _, device := range instanceMetadata.Devices { if device.Type == "disk" && device.Serial == volumeID { - glog.V(4).Infof( + klog.V(4).Infof( "Found disk metadata for volumeID %q. Bus: %q, Address: %q", volumeID, device.Bus, device.Address) @@ -527,7 +527,7 @@ func (os *OpenStack) getDevicePathFromInstanceMetadata(volumeID string) string { device.Bus, device.Address) diskPaths, err := filepath.Glob(diskPattern) if err != nil { - glog.Errorf( + klog.Errorf( "could not retrieve disk path for volumeID: %q. Error filepath.Glob(%q): %v", volumeID, diskPattern, err) return "" @@ -537,14 +537,14 @@ func (os *OpenStack) getDevicePathFromInstanceMetadata(volumeID string) string { return diskPaths[0] } - glog.Errorf( + klog.Errorf( "expecting to find one disk path for volumeID %q, found %d: %v", volumeID, len(diskPaths), diskPaths) return "" } } - glog.V(4).Infof( + klog.V(4).Infof( "Could not retrieve device metadata for volumeID: %q", volumeID) return "" } @@ -558,7 +558,7 @@ func (os *OpenStack) GetDevicePath(volumeID string) string { } if devicePath == "" { - glog.Warningf("Failed to find device for the volumeID: %q", volumeID) + klog.Warningf("Failed to find device for the volumeID: %q", volumeID) } return devicePath @@ -610,7 +610,7 @@ func (os *OpenStack) GetAttachmentDiskPath(instanceID, volumeID string) (string, // DiskIsAttached queries if a volume is attached to a compute instance func (os *OpenStack) DiskIsAttached(instanceID, volumeID string) (bool, error) { if instanceID == "" { - glog.Warningf("calling DiskIsAttached with empty instanceid: %s %s", instanceID, volumeID) + klog.Warningf("calling DiskIsAttached with empty instanceid: %s %s", instanceID, volumeID) } volume, err := os.getVolume(volumeID) if err != nil { @@ -717,7 +717,7 @@ func (os *OpenStack) GetLabelsForVolume(ctx context.Context, pv *v1.PersistentVo labels := make(map[string]string) labels[kubeletapis.LabelZoneFailureDomain] = volume.AvailabilityZone labels[kubeletapis.LabelZoneRegion] = os.region - glog.V(4).Infof("The Volume %s has labels %v", pv.Spec.Cinder.VolumeID, labels) + klog.V(4).Infof("The Volume %s has labels %v", pv.Spec.Cinder.VolumeID, labels) return labels, nil } diff --git a/pkg/cloudprovider/providers/photon/BUILD b/pkg/cloudprovider/providers/photon/BUILD index df4311f04ef..57a17c416f6 100644 --- a/pkg/cloudprovider/providers/photon/BUILD +++ b/pkg/cloudprovider/providers/photon/BUILD @@ -15,9 +15,9 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/vmware/photon-controller-go-sdk/photon:go_default_library", "//vendor/gopkg.in/gcfg.v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/cloudprovider/providers/photon/photon.go b/pkg/cloudprovider/providers/photon/photon.go index bcf5e313449..24f8d2b8dd8 100644 --- a/pkg/cloudprovider/providers/photon/photon.go +++ b/pkg/cloudprovider/providers/photon/photon.go @@ -34,12 +34,12 @@ import ( "os" "strings" - "github.com/golang/glog" "github.com/vmware/photon-controller-go-sdk/photon" "gopkg.in/gcfg.v1" "k8s.io/api/core/v1" k8stypes "k8s.io/apimachinery/pkg/types" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" ) @@ -135,7 +135,7 @@ func init() { cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { cfg, err := readConfig(config) if err != nil { - glog.Errorf("Photon Cloud Provider: failed to read in cloud provider config file. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: failed to read in cloud provider config file. Error[%v]", err) return nil, err } return newPCCloud(cfg) @@ -146,13 +146,13 @@ func init() { func getVMIDbyNodename(pc *PCCloud, nodeName string) (string, error) { photonClient, err := getPhotonClient(pc) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to get photon client for getVMIDbyNodename, error: [%v]", err) + klog.Errorf("Photon Cloud Provider: Failed to get photon client for getVMIDbyNodename, error: [%v]", err) return "", err } vmList, err := photonClient.Projects.GetVMs(pc.projID, nil) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to GetVMs from project %s with nodeName %s, error: [%v]", pc.projID, nodeName, err) + klog.Errorf("Photon Cloud Provider: Failed to GetVMs from project %s with nodeName %s, error: [%v]", pc.projID, nodeName, err) return "", err } @@ -169,24 +169,24 @@ func getVMIDbyNodename(pc *PCCloud, nodeName string) (string, error) { func getVMIDbyIP(pc *PCCloud, IPAddress string) (string, error) { photonClient, err := getPhotonClient(pc) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to get photon client for getVMIDbyNodename, error: [%v]", err) + klog.Errorf("Photon Cloud Provider: Failed to get photon client for getVMIDbyNodename, error: [%v]", err) return "", err } vmList, err := photonClient.Projects.GetVMs(pc.projID, nil) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to GetVMs for project %s. error: [%v]", pc.projID, err) + klog.Errorf("Photon Cloud Provider: Failed to GetVMs for project %s. error: [%v]", pc.projID, err) return "", err } for _, vm := range vmList.Items { task, err := photonClient.VMs.GetNetworks(vm.ID) if err != nil { - glog.Warningf("Photon Cloud Provider: GetNetworks failed for vm.ID %s, error [%v]", vm.ID, err) + klog.Warningf("Photon Cloud Provider: GetNetworks failed for vm.ID %s, error [%v]", vm.ID, err) } else { task, err = photonClient.Tasks.Wait(task.ID) if err != nil { - glog.Warningf("Photon Cloud Provider: Wait task for GetNetworks failed for vm.ID %s, error [%v]", vm.ID, err) + klog.Warningf("Photon Cloud Provider: Wait task for GetNetworks failed for vm.ID %s, error [%v]", vm.ID, err) } else { networkConnections := task.ResourceProperties.(map[string]interface{}) networks := networkConnections["networkConnections"].([]interface{}) @@ -221,25 +221,25 @@ func getPhotonClient(pc *PCCloud) (*photon.Client, error) { // work around before metadata is available file, err := os.Open("/etc/kubernetes/pc_login_info") if err != nil { - glog.Errorf("Photon Cloud Provider: Authentication is enabled but found no username/password at /etc/kubernetes/pc_login_info. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: Authentication is enabled but found no username/password at /etc/kubernetes/pc_login_info. Error[%v]", err) return nil, err } defer file.Close() scanner := bufio.NewScanner(file) if !scanner.Scan() { - glog.Error("Photon Cloud Provider: Empty username inside /etc/kubernetes/pc_login_info.") + klog.Error("Photon Cloud Provider: Empty username inside /etc/kubernetes/pc_login_info.") return nil, fmt.Errorf("Failed to create authentication enabled client with invalid username") } username := scanner.Text() if !scanner.Scan() { - glog.Error("Photon Cloud Provider: Empty password set inside /etc/kubernetes/pc_login_info.") + klog.Error("Photon Cloud Provider: Empty password set inside /etc/kubernetes/pc_login_info.") return nil, fmt.Errorf("Failed to create authentication enabled client with invalid password") } password := scanner.Text() token_options, err := pc.photonClient.Auth.GetTokensByPassword(username, password) if err != nil { - glog.Error("Photon Cloud Provider: failed to get tokens by password") + klog.Error("Photon Cloud Provider: failed to get tokens by password") return nil, err } @@ -254,10 +254,10 @@ func getPhotonClient(pc *PCCloud) (*photon.Client, error) { status, err := pc.photonClient.Status.Get() if err != nil { - glog.Errorf("Photon Cloud Provider: new client creation failed. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: new client creation failed. Error[%v]", err) return nil, err } - glog.V(2).Infof("Photon Cloud Provider: Status of the new photon controller client: %v", status) + klog.V(2).Infof("Photon Cloud Provider: Status of the new photon controller client: %v", status) return pc.photonClient, nil } @@ -269,7 +269,7 @@ func newPCCloud(cfg PCConfig) (*PCCloud, error) { // Get local hostname hostname, err := os.Hostname() if err != nil { - glog.Errorf("Photon Cloud Provider: get hostname failed. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: get hostname failed. Error[%v]", err) return nil, err } pc := PCCloud{ @@ -307,14 +307,14 @@ func (pc *PCCloud) NodeAddresses(ctx context.Context, nodeName k8stypes.NodeName if name == pc.localK8sHostname { ifaces, err := net.Interfaces() if err != nil { - glog.Errorf("Photon Cloud Provider: net.Interfaces() failed for NodeAddresses. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: net.Interfaces() failed for NodeAddresses. Error[%v]", err) return nodeAddrs, err } for _, i := range ifaces { addrs, err := i.Addrs() if err != nil { - glog.Warningf("Photon Cloud Provider: Failed to extract addresses for NodeAddresses. Error[%v]", err) + klog.Warningf("Photon Cloud Provider: Failed to extract addresses for NodeAddresses. Error[%v]", err) } else { for _, addr := range addrs { if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { @@ -348,20 +348,20 @@ func (pc *PCCloud) NodeAddresses(ctx context.Context, nodeName k8stypes.NodeName // This is assumed to be done by master only. vmID, err := getInstanceID(pc, name) if err != nil { - glog.Errorf("Photon Cloud Provider: getInstanceID failed for NodeAddresses. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: getInstanceID failed for NodeAddresses. Error[%v]", err) return nodeAddrs, err } photonClient, err := getPhotonClient(pc) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to get photon client for NodeAddresses, error: [%v]", err) + klog.Errorf("Photon Cloud Provider: Failed to get photon client for NodeAddresses, error: [%v]", err) return nodeAddrs, err } // Retrieve the Photon VM's IP addresses from the Photon Controller endpoint based on the VM ID vmList, err := photonClient.Projects.GetVMs(pc.projID, nil) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to GetVMs for project %s. Error[%v]", pc.projID, err) + klog.Errorf("Photon Cloud Provider: Failed to GetVMs for project %s. Error[%v]", pc.projID, err) return nodeAddrs, err } @@ -369,12 +369,12 @@ func (pc *PCCloud) NodeAddresses(ctx context.Context, nodeName k8stypes.NodeName if vm.ID == vmID { task, err := photonClient.VMs.GetNetworks(vm.ID) if err != nil { - glog.Errorf("Photon Cloud Provider: GetNetworks failed for node %s with vm.ID %s. Error[%v]", name, vm.ID, err) + klog.Errorf("Photon Cloud Provider: GetNetworks failed for node %s with vm.ID %s. Error[%v]", name, vm.ID, err) return nodeAddrs, err } else { task, err = photonClient.Tasks.Wait(task.ID) if err != nil { - glog.Errorf("Photon Cloud Provider: Wait task for GetNetworks failed for node %s with vm.ID %s. Error[%v]", name, vm.ID, err) + klog.Errorf("Photon Cloud Provider: Wait task for GetNetworks failed for node %s with vm.ID %s. Error[%v]", name, vm.ID, err) return nodeAddrs, err } else { networkConnections := task.ResourceProperties.(map[string]interface{}) @@ -414,7 +414,7 @@ func (pc *PCCloud) NodeAddresses(ctx context.Context, nodeName k8stypes.NodeName } } - glog.Errorf("Failed to find the node %s from Photon Controller endpoint", name) + klog.Errorf("Failed to find the node %s from Photon Controller endpoint", name) return nodeAddrs, fmt.Errorf("Failed to find the node %s from Photon Controller endpoint", name) } @@ -474,7 +474,7 @@ func (pc *PCCloud) InstanceID(ctx context.Context, nodeName k8stypes.NodeName) ( // We assume only master need to get InstanceID of a node other than itself ID, err := getInstanceID(pc, name) if err != nil { - glog.Errorf("Photon Cloud Provider: getInstanceID failed for InstanceID. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: getInstanceID failed for InstanceID. Error[%v]", err) return ID, err } else { return ID, nil @@ -544,7 +544,7 @@ func (pc *PCCloud) HasClusterID() bool { func (pc *PCCloud) AttachDisk(ctx context.Context, pdID string, nodeName k8stypes.NodeName) error { photonClient, err := getPhotonClient(pc) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to get photon client for AttachDisk, error: [%v]", err) + klog.Errorf("Photon Cloud Provider: Failed to get photon client for AttachDisk, error: [%v]", err) return err } @@ -554,19 +554,19 @@ func (pc *PCCloud) AttachDisk(ctx context.Context, pdID string, nodeName k8stype vmID, err := pc.InstanceID(ctx, nodeName) if err != nil { - glog.Errorf("Photon Cloud Provider: pc.InstanceID failed for AttachDisk. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: pc.InstanceID failed for AttachDisk. Error[%v]", err) return err } task, err := photonClient.VMs.AttachDisk(vmID, operation) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to attach disk with pdID %s. Error[%v]", pdID, err) + klog.Errorf("Photon Cloud Provider: Failed to attach disk with pdID %s. Error[%v]", pdID, err) return err } _, err = photonClient.Tasks.Wait(task.ID) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to wait for task to attach disk with pdID %s. Error[%v]", pdID, err) + klog.Errorf("Photon Cloud Provider: Failed to wait for task to attach disk with pdID %s. Error[%v]", pdID, err) return err } @@ -577,7 +577,7 @@ func (pc *PCCloud) AttachDisk(ctx context.Context, pdID string, nodeName k8stype func (pc *PCCloud) DetachDisk(ctx context.Context, pdID string, nodeName k8stypes.NodeName) error { photonClient, err := getPhotonClient(pc) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to get photon client for DetachDisk, error: [%v]", err) + klog.Errorf("Photon Cloud Provider: Failed to get photon client for DetachDisk, error: [%v]", err) return err } @@ -587,19 +587,19 @@ func (pc *PCCloud) DetachDisk(ctx context.Context, pdID string, nodeName k8stype vmID, err := pc.InstanceID(ctx, nodeName) if err != nil { - glog.Errorf("Photon Cloud Provider: pc.InstanceID failed for DetachDisk. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: pc.InstanceID failed for DetachDisk. Error[%v]", err) return err } task, err := photonClient.VMs.DetachDisk(vmID, operation) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to detach disk with pdID %s. Error[%v]", pdID, err) + klog.Errorf("Photon Cloud Provider: Failed to detach disk with pdID %s. Error[%v]", pdID, err) return err } _, err = photonClient.Tasks.Wait(task.ID) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to wait for task to detach disk with pdID %s. Error[%v]", pdID, err) + klog.Errorf("Photon Cloud Provider: Failed to wait for task to detach disk with pdID %s. Error[%v]", pdID, err) return err } @@ -610,23 +610,23 @@ func (pc *PCCloud) DetachDisk(ctx context.Context, pdID string, nodeName k8stype func (pc *PCCloud) DiskIsAttached(ctx context.Context, pdID string, nodeName k8stypes.NodeName) (bool, error) { photonClient, err := getPhotonClient(pc) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to get photon client for DiskIsAttached, error: [%v]", err) + klog.Errorf("Photon Cloud Provider: Failed to get photon client for DiskIsAttached, error: [%v]", err) return false, err } disk, err := photonClient.Disks.Get(pdID) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to Get disk with pdID %s. Error[%v]", pdID, err) + klog.Errorf("Photon Cloud Provider: Failed to Get disk with pdID %s. Error[%v]", pdID, err) return false, err } vmID, err := pc.InstanceID(ctx, nodeName) if err == cloudprovider.InstanceNotFound { - glog.Infof("Instance %q does not exist, disk %s will be detached automatically.", nodeName, pdID) + klog.Infof("Instance %q does not exist, disk %s will be detached automatically.", nodeName, pdID) return false, nil } if err != nil { - glog.Errorf("Photon Cloud Provider: pc.InstanceID failed for DiskIsAttached. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: pc.InstanceID failed for DiskIsAttached. Error[%v]", err) return false, err } @@ -644,7 +644,7 @@ func (pc *PCCloud) DisksAreAttached(ctx context.Context, pdIDs []string, nodeNam attached := make(map[string]bool) photonClient, err := getPhotonClient(pc) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to get photon client for DisksAreAttached, error: [%v]", err) + klog.Errorf("Photon Cloud Provider: Failed to get photon client for DisksAreAttached, error: [%v]", err) return attached, err } @@ -654,19 +654,19 @@ func (pc *PCCloud) DisksAreAttached(ctx context.Context, pdIDs []string, nodeNam vmID, err := pc.InstanceID(ctx, nodeName) if err == cloudprovider.InstanceNotFound { - glog.Infof("Instance %q does not exist, its disks will be detached automatically.", nodeName) + klog.Infof("Instance %q does not exist, its disks will be detached automatically.", nodeName) // make all the disks as detached. return attached, nil } if err != nil { - glog.Errorf("Photon Cloud Provider: pc.InstanceID failed for DiskIsAttached. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: pc.InstanceID failed for DiskIsAttached. Error[%v]", err) return attached, err } for _, pdID := range pdIDs { disk, err := photonClient.Disks.Get(pdID) if err != nil { - glog.Warningf("Photon Cloud Provider: failed to get VMs for persistent disk %s, err [%v]", pdID, err) + klog.Warningf("Photon Cloud Provider: failed to get VMs for persistent disk %s, err [%v]", pdID, err) } else { for _, vm := range disk.VMs { if vm == vmID { @@ -683,7 +683,7 @@ func (pc *PCCloud) DisksAreAttached(ctx context.Context, pdIDs []string, nodeNam func (pc *PCCloud) CreateDisk(volumeOptions *VolumeOptions) (pdID string, err error) { photonClient, err := getPhotonClient(pc) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to get photon client for CreateDisk, error: [%v]", err) + klog.Errorf("Photon Cloud Provider: Failed to get photon client for CreateDisk, error: [%v]", err) return "", err } @@ -695,13 +695,13 @@ func (pc *PCCloud) CreateDisk(volumeOptions *VolumeOptions) (pdID string, err er task, err := photonClient.Projects.CreateDisk(pc.projID, &diskSpec) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to CreateDisk. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: Failed to CreateDisk. Error[%v]", err) return "", err } waitTask, err := photonClient.Tasks.Wait(task.ID) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to wait for task to CreateDisk. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: Failed to wait for task to CreateDisk. Error[%v]", err) return "", err } @@ -712,19 +712,19 @@ func (pc *PCCloud) CreateDisk(volumeOptions *VolumeOptions) (pdID string, err er func (pc *PCCloud) DeleteDisk(pdID string) error { photonClient, err := getPhotonClient(pc) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to get photon client for DeleteDisk, error: [%v]", err) + klog.Errorf("Photon Cloud Provider: Failed to get photon client for DeleteDisk, error: [%v]", err) return err } task, err := photonClient.Disks.Delete(pdID) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to DeleteDisk. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: Failed to DeleteDisk. Error[%v]", err) return err } _, err = photonClient.Tasks.Wait(task.ID) if err != nil { - glog.Errorf("Photon Cloud Provider: Failed to wait for task to DeleteDisk. Error[%v]", err) + klog.Errorf("Photon Cloud Provider: Failed to wait for task to DeleteDisk. Error[%v]", err) return err } diff --git a/pkg/cloudprovider/providers/vsphere/BUILD b/pkg/cloudprovider/providers/vsphere/BUILD index 3eb30f03eb2..6e3034f86d0 100644 --- a/pkg/cloudprovider/providers/vsphere/BUILD +++ b/pkg/cloudprovider/providers/vsphere/BUILD @@ -27,12 +27,12 @@ go_library( "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/vmware/govmomi/vapi/rest:go_default_library", "//vendor/github.com/vmware/govmomi/vapi/tags:go_default_library", "//vendor/github.com/vmware/govmomi/vim25:go_default_library", "//vendor/github.com/vmware/govmomi/vim25/mo:go_default_library", "//vendor/gopkg.in/gcfg.v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/cloudprovider/providers/vsphere/credentialmanager.go b/pkg/cloudprovider/providers/vsphere/credentialmanager.go index 95862a8a5aa..a3b651495f6 100644 --- a/pkg/cloudprovider/providers/vsphere/credentialmanager.go +++ b/pkg/cloudprovider/providers/vsphere/credentialmanager.go @@ -19,10 +19,10 @@ package vsphere import ( "errors" "fmt" - "github.com/golang/glog" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/listers/core/v1" + "k8s.io/klog" "net/http" "strings" "sync" @@ -71,12 +71,12 @@ func (secretCredentialManager *SecretCredentialManager) GetCredential(server str return nil, err } // Handle secrets deletion by finding credentials from cache - glog.Warningf("secret %q not found in namespace %q", secretCredentialManager.SecretName, secretCredentialManager.SecretNamespace) + klog.Warningf("secret %q not found in namespace %q", secretCredentialManager.SecretName, secretCredentialManager.SecretNamespace) } credential, found := secretCredentialManager.Cache.GetCredential(server) if !found { - glog.Errorf("credentials not found for server %q", server) + klog.Errorf("credentials not found for server %q", server) return nil, ErrCredentialsNotFound } return &credential, nil @@ -88,13 +88,13 @@ func (secretCredentialManager *SecretCredentialManager) updateCredentialsMap() e } secret, err := secretCredentialManager.SecretLister.Secrets(secretCredentialManager.SecretNamespace).Get(secretCredentialManager.SecretName) if err != nil { - glog.Errorf("Cannot get secret %s in namespace %s. error: %q", secretCredentialManager.SecretName, secretCredentialManager.SecretNamespace, err) + klog.Errorf("Cannot get secret %s in namespace %s. error: %q", secretCredentialManager.SecretName, secretCredentialManager.SecretNamespace, err) return err } cacheSecret := secretCredentialManager.Cache.GetSecret() if cacheSecret != nil && cacheSecret.GetResourceVersion() == secret.GetResourceVersion() { - glog.V(4).Infof("VCP SecretCredentialManager: Secret %q will not be updated in cache. Since, secrets have same resource version %q", secretCredentialManager.SecretName, cacheSecret.GetResourceVersion()) + klog.V(4).Infof("VCP SecretCredentialManager: Secret %q will not be updated in cache. Since, secrets have same resource version %q", secretCredentialManager.SecretName, cacheSecret.GetResourceVersion()) return nil } secretCredentialManager.Cache.UpdateSecret(secret) @@ -150,13 +150,13 @@ func parseConfig(data map[string][]byte, config map[string]*Credential) error { } config[vcServer].User = string(credentialValue) } else { - glog.Errorf("Unknown secret key %s", credentialKey) + klog.Errorf("Unknown secret key %s", credentialKey) return ErrUnknownSecretKey } } for vcServer, credential := range config { if credential.User == "" || credential.Password == "" { - glog.Errorf("Username/Password is missing for server %s", vcServer) + klog.Errorf("Username/Password is missing for server %s", vcServer) return ErrCredentialMissing } } diff --git a/pkg/cloudprovider/providers/vsphere/nodemanager.go b/pkg/cloudprovider/providers/vsphere/nodemanager.go index 8d62eebb15b..92f4c55d5e0 100644 --- a/pkg/cloudprovider/providers/vsphere/nodemanager.go +++ b/pkg/cloudprovider/providers/vsphere/nodemanager.go @@ -22,9 +22,9 @@ import ( "strings" "sync" - "github.com/golang/glog" "k8s.io/api/core/v1" k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib" ) @@ -81,11 +81,11 @@ func (nm *NodeManager) DiscoverNode(node *v1.Node) error { queueChannel = make(chan *VmSearch, QUEUE_SIZE) nodeUUID, err := GetNodeUUID(node) if err != nil { - glog.Errorf("Node Discovery failed to get node uuid for node %s with error: %v", node.Name, err) + klog.Errorf("Node Discovery failed to get node uuid for node %s with error: %v", node.Name, err) return err } - glog.V(4).Infof("Discovering node %s with uuid %s", node.ObjectMeta.Name, nodeUUID) + klog.V(4).Infof("Discovering node %s with uuid %s", node.ObjectMeta.Name, nodeUUID) vmFound := false globalErr = nil @@ -124,7 +124,7 @@ func (nm *NodeManager) DiscoverNode(node *v1.Node) error { err := nm.vcConnect(ctx, vsi) if err != nil { - glog.V(4).Info("Discovering node error vc:", err) + klog.V(4).Info("Discovering node error vc:", err) setGlobalErr(err) continue } @@ -132,7 +132,7 @@ func (nm *NodeManager) DiscoverNode(node *v1.Node) error { if vsi.cfg.Datacenters == "" { datacenterObjs, err = vclib.GetAllDatacenter(ctx, vsi.conn) if err != nil { - glog.V(4).Info("Discovering node error dc:", err) + klog.V(4).Info("Discovering node error dc:", err) setGlobalErr(err) continue } @@ -145,7 +145,7 @@ func (nm *NodeManager) DiscoverNode(node *v1.Node) error { } datacenterObj, err := vclib.GetDatacenter(ctx, vsi.conn, dc) if err != nil { - glog.V(4).Info("Discovering node error dc:", err) + klog.V(4).Info("Discovering node error dc:", err) setGlobalErr(err) continue } @@ -159,7 +159,7 @@ func (nm *NodeManager) DiscoverNode(node *v1.Node) error { break } - glog.V(4).Infof("Finding node %s in vc=%s and datacenter=%s", node.Name, vc, datacenterObj.Name()) + klog.V(4).Infof("Finding node %s in vc=%s and datacenter=%s", node.Name, vc, datacenterObj.Name()) queueChannel <- &VmSearch{ vc: vc, datacenter: datacenterObj, @@ -176,18 +176,18 @@ func (nm *NodeManager) DiscoverNode(node *v1.Node) error { defer cancel() vm, err := res.datacenter.GetVMByUUID(ctx, nodeUUID) if err != nil { - glog.V(4).Infof("Error while looking for vm=%+v in vc=%s and datacenter=%s: %v", + klog.V(4).Infof("Error while looking for vm=%+v in vc=%s and datacenter=%s: %v", vm, res.vc, res.datacenter.Name(), err) if err != vclib.ErrNoVMFound { setGlobalErr(err) } else { - glog.V(4).Infof("Did not find node %s in vc=%s and datacenter=%s", + klog.V(4).Infof("Did not find node %s in vc=%s and datacenter=%s", node.Name, res.vc, res.datacenter.Name()) } continue } if vm != nil { - glog.V(4).Infof("Found node %s as vm=%+v in vc=%s and datacenter=%s", + klog.V(4).Infof("Found node %s as vm=%+v in vc=%s and datacenter=%s", node.Name, vm, res.vc, res.datacenter.Name()) nodeInfo := &NodeInfo{dataCenter: res.datacenter, vm: vm, vcServer: res.vc, vmUUID: nodeUUID} @@ -210,7 +210,7 @@ func (nm *NodeManager) DiscoverNode(node *v1.Node) error { return *globalErr } - glog.V(4).Infof("Discovery Node: %q vm not found", node.Name) + klog.V(4).Infof("Discovery Node: %q vm not found", node.Name) return vclib.ErrNoVMFound } @@ -276,19 +276,19 @@ func (nm *NodeManager) GetNodeInfo(nodeName k8stypes.NodeName) (NodeInfo, error) var err error if nodeInfo == nil { // Rediscover node if no NodeInfo found. - glog.V(4).Infof("No VM found for node %q. Initiating rediscovery.", convertToString(nodeName)) + klog.V(4).Infof("No VM found for node %q. Initiating rediscovery.", convertToString(nodeName)) err = nm.RediscoverNode(nodeName) if err != nil { - glog.Errorf("Error %q node info for node %q not found", err, convertToString(nodeName)) + klog.Errorf("Error %q node info for node %q not found", err, convertToString(nodeName)) return NodeInfo{}, err } nodeInfo = getNodeInfo(nodeName) } else { // Renew the found NodeInfo to avoid stale vSphere connection. - glog.V(4).Infof("Renewing NodeInfo %+v for node %q", nodeInfo, convertToString(nodeName)) + klog.V(4).Infof("Renewing NodeInfo %+v for node %q", nodeInfo, convertToString(nodeName)) nodeInfo, err = nm.renewNodeInfo(nodeInfo, true) if err != nil { - glog.Errorf("Error %q occurred while renewing NodeInfo for %q", err, convertToString(nodeName)) + klog.Errorf("Error %q occurred while renewing NodeInfo for %q", err, convertToString(nodeName)) return NodeInfo{}, err } nm.addNodeInfo(convertToString(nodeName), nodeInfo) @@ -309,7 +309,7 @@ func (nm *NodeManager) GetNodeDetails() ([]NodeDetails, error) { if err != nil { return nil, err } - glog.V(4).Infof("Updated NodeInfo %v for node %q.", nodeInfo, nodeName) + klog.V(4).Infof("Updated NodeInfo %v for node %q.", nodeInfo, nodeName) nodeDetails = append(nodeDetails, NodeDetails{nodeName, nodeInfo.vm, nodeInfo.vmUUID}) } return nodeDetails, nil @@ -324,7 +324,7 @@ func (nm *NodeManager) addNodeInfo(nodeName string, nodeInfo *NodeInfo) { func (nm *NodeManager) GetVSphereInstance(nodeName k8stypes.NodeName) (VSphereInstance, error) { nodeInfo, err := nm.GetNodeInfo(nodeName) if err != nil { - glog.V(4).Infof("node info for node %q not found", convertToString(nodeName)) + klog.V(4).Infof("node info for node %q not found", convertToString(nodeName)) return VSphereInstance{}, err } vsphereInstance := nm.vsphereInstanceMap[nodeInfo.vcServer] @@ -379,16 +379,16 @@ func (nm *NodeManager) vcConnect(ctx context.Context, vsphereInstance *VSphereIn credentialManager := nm.CredentialManager() if !vclib.IsInvalidCredentialsError(err) || credentialManager == nil { - glog.Errorf("Cannot connect to vCenter with err: %v", err) + klog.Errorf("Cannot connect to vCenter with err: %v", err) return err } - glog.V(4).Infof("Invalid credentials. Cannot connect to server %q. Fetching credentials from secrets.", vsphereInstance.conn.Hostname) + klog.V(4).Infof("Invalid credentials. Cannot connect to server %q. Fetching credentials from secrets.", vsphereInstance.conn.Hostname) // Get latest credentials from SecretCredentialManager credentials, err := credentialManager.GetCredential(vsphereInstance.conn.Hostname) if err != nil { - glog.Errorf("Failed to get credentials from Secret Credential Manager with err: %v", err) + klog.Errorf("Failed to get credentials from Secret Credential Manager with err: %v", err) return err } vsphereInstance.conn.UpdateCredentials(credentials.User, credentials.Password) @@ -412,19 +412,19 @@ func (nm *NodeManager) GetNodeInfoWithNodeObject(node *v1.Node) (NodeInfo, error var err error if nodeInfo == nil { // Rediscover node if no NodeInfo found. - glog.V(4).Infof("No VM found for node %q. Initiating rediscovery.", nodeName) + klog.V(4).Infof("No VM found for node %q. Initiating rediscovery.", nodeName) err = nm.DiscoverNode(node) if err != nil { - glog.Errorf("Error %q node info for node %q not found", err, nodeName) + klog.Errorf("Error %q node info for node %q not found", err, nodeName) return NodeInfo{}, err } nodeInfo = getNodeInfo(nodeName) } else { // Renew the found NodeInfo to avoid stale vSphere connection. - glog.V(4).Infof("Renewing NodeInfo %+v for node %q", nodeInfo, nodeName) + klog.V(4).Infof("Renewing NodeInfo %+v for node %q", nodeInfo, nodeName) nodeInfo, err = nm.renewNodeInfo(nodeInfo, true) if err != nil { - glog.Errorf("Error %q occurred while renewing NodeInfo for %q", err, nodeName) + klog.Errorf("Error %q occurred while renewing NodeInfo for %q", err, nodeName) return NodeInfo{}, err } nm.addNodeInfo(nodeName, nodeInfo) diff --git a/pkg/cloudprovider/providers/vsphere/vclib/BUILD b/pkg/cloudprovider/providers/vsphere/vclib/BUILD index 934054d1ec0..a81f8385a16 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/BUILD +++ b/pkg/cloudprovider/providers/vsphere/vclib/BUILD @@ -25,7 +25,6 @@ go_library( importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib", deps = [ "//pkg/version:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/github.com/vmware/govmomi/find:go_default_library", "//vendor/github.com/vmware/govmomi/object:go_default_library", @@ -38,6 +37,7 @@ go_library( "//vendor/github.com/vmware/govmomi/vim25/mo:go_default_library", "//vendor/github.com/vmware/govmomi/vim25/soap:go_default_library", "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/cloudprovider/providers/vsphere/vclib/connection.go b/pkg/cloudprovider/providers/vsphere/vclib/connection.go index be4c5e3f538..28f2228d656 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/connection.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/connection.go @@ -25,11 +25,11 @@ import ( neturl "net/url" "sync" - "github.com/golang/glog" "github.com/vmware/govmomi/session" "github.com/vmware/govmomi/sts" "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/soap" + "k8s.io/klog" "k8s.io/kubernetes/pkg/version" ) @@ -62,7 +62,7 @@ func (connection *VSphereConnection) Connect(ctx context.Context) error { if connection.Client == nil { connection.Client, err = connection.NewClient(ctx) if err != nil { - glog.Errorf("Failed to create govmomi client. err: %+v", err) + klog.Errorf("Failed to create govmomi client. err: %+v", err) return err } return nil @@ -70,17 +70,17 @@ func (connection *VSphereConnection) Connect(ctx context.Context) error { m := session.NewManager(connection.Client) userSession, err := m.UserSession(ctx) if err != nil { - glog.Errorf("Error while obtaining user session. err: %+v", err) + klog.Errorf("Error while obtaining user session. err: %+v", err) return err } if userSession != nil { return nil } - glog.Warningf("Creating new client session since the existing session is not valid or not authenticated") + klog.Warningf("Creating new client session since the existing session is not valid or not authenticated") connection.Client, err = connection.NewClient(ctx) if err != nil { - glog.Errorf("Failed to create govmomi client. err: %+v", err) + klog.Errorf("Failed to create govmomi client. err: %+v", err) return err } return nil @@ -98,21 +98,21 @@ func (connection *VSphereConnection) login(ctx context.Context, client *vim25.Cl // decide to use LoginByToken if the username value is PEM encoded. b, _ := pem.Decode([]byte(connection.Username)) if b == nil { - glog.V(3).Infof("SessionManager.Login with username '%s'", connection.Username) + klog.V(3).Infof("SessionManager.Login with username '%s'", connection.Username) return m.Login(ctx, neturl.UserPassword(connection.Username, connection.Password)) } - glog.V(3).Infof("SessionManager.LoginByToken with certificate '%s'", connection.Username) + klog.V(3).Infof("SessionManager.LoginByToken with certificate '%s'", connection.Username) cert, err := tls.X509KeyPair([]byte(connection.Username), []byte(connection.Password)) if err != nil { - glog.Errorf("Failed to load X509 key pair. err: %+v", err) + klog.Errorf("Failed to load X509 key pair. err: %+v", err) return err } tokens, err := sts.NewClient(ctx, client) if err != nil { - glog.Errorf("Failed to create STS client. err: %+v", err) + klog.Errorf("Failed to create STS client. err: %+v", err) return err } @@ -122,7 +122,7 @@ func (connection *VSphereConnection) login(ctx context.Context, client *vim25.Cl signer, err := tokens.Issue(ctx, req) if err != nil { - glog.Errorf("Failed to issue SAML token. err: %+v", err) + klog.Errorf("Failed to issue SAML token. err: %+v", err) return err } @@ -144,15 +144,15 @@ func (connection *VSphereConnection) Logout(ctx context.Context) { hasActiveSession, err := m.SessionIsActive(ctx) if err != nil { - glog.Errorf("Logout failed: %s", err) + klog.Errorf("Logout failed: %s", err) return } if !hasActiveSession { - glog.Errorf("No active session, cannot logout") + klog.Errorf("No active session, cannot logout") return } if err := m.Logout(ctx); err != nil { - glog.Errorf("Logout failed: %s", err) + klog.Errorf("Logout failed: %s", err) } } @@ -160,7 +160,7 @@ func (connection *VSphereConnection) Logout(ctx context.Context) { func (connection *VSphereConnection) NewClient(ctx context.Context) (*vim25.Client, error) { url, err := soap.ParseURL(net.JoinHostPort(connection.Hostname, connection.Port)) if err != nil { - glog.Errorf("Failed to parse URL: %s. err: %+v", url, err) + klog.Errorf("Failed to parse URL: %s. err: %+v", url, err) return nil, err } @@ -177,7 +177,7 @@ func (connection *VSphereConnection) NewClient(ctx context.Context) (*vim25.Clie client, err := vim25.NewClient(ctx, sc) if err != nil { - glog.Errorf("Failed to create new client. err: %+v", err) + klog.Errorf("Failed to create new client. err: %+v", err) return nil, err } @@ -188,10 +188,10 @@ func (connection *VSphereConnection) NewClient(ctx context.Context) (*vim25.Clie if err != nil { return nil, err } - if glog.V(3) { + if klog.V(3) { s, err := session.NewManager(client).UserSession(ctx) if err == nil { - glog.Infof("New session ID for '%s' = %s", s.UserName, s.Key) + klog.Infof("New session ID for '%s' = %s", s.UserName, s.Key) } } diff --git a/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go b/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go index 9a4eddd074d..778e0f68290 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go @@ -23,12 +23,12 @@ import ( "path/filepath" "strings" - "github.com/golang/glog" "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" + "k8s.io/klog" ) // Datacenter extends the govmomi Datacenter object @@ -42,7 +42,7 @@ func GetDatacenter(ctx context.Context, connection *VSphereConnection, datacente finder := find.NewFinder(connection.Client, false) datacenter, err := finder.Datacenter(ctx, datacenterPath) if err != nil { - glog.Errorf("Failed to find the datacenter: %s. err: %+v", datacenterPath, err) + klog.Errorf("Failed to find the datacenter: %s. err: %+v", datacenterPath, err) return nil, err } dc := Datacenter{datacenter} @@ -55,7 +55,7 @@ func GetAllDatacenter(ctx context.Context, connection *VSphereConnection) ([]*Da finder := find.NewFinder(connection.Client, false) datacenters, err := finder.DatacenterList(ctx, "*") if err != nil { - glog.Errorf("Failed to find the datacenter. err: %+v", err) + klog.Errorf("Failed to find the datacenter. err: %+v", err) return nil, err } for _, datacenter := range datacenters { @@ -71,11 +71,11 @@ func (dc *Datacenter) GetVMByUUID(ctx context.Context, vmUUID string) (*VirtualM vmUUID = strings.ToLower(strings.TrimSpace(vmUUID)) svm, err := s.FindByUuid(ctx, dc.Datacenter, vmUUID, true, nil) if err != nil { - glog.Errorf("Failed to find VM by UUID. VM UUID: %s, err: %+v", vmUUID, err) + klog.Errorf("Failed to find VM by UUID. VM UUID: %s, err: %+v", vmUUID, err) return nil, err } if svm == nil { - glog.Errorf("Unable to find VM by UUID. VM UUID: %s", vmUUID) + klog.Errorf("Unable to find VM by UUID. VM UUID: %s", vmUUID) return nil, ErrNoVMFound } virtualMachine := VirtualMachine{object.NewVirtualMachine(dc.Client(), svm.Reference()), dc} @@ -89,11 +89,11 @@ func (dc *Datacenter) GetHostByVMUUID(ctx context.Context, vmUUID string) (*type pc := property.DefaultCollector(virtualMachine.Client()) err = pc.RetrieveOne(ctx, virtualMachine.Reference(), []string{"summary.runtime.host"}, &vmMo) if err != nil { - glog.Errorf("Failed to retrive VM runtime host, err: %v", err) + klog.Errorf("Failed to retrive VM runtime host, err: %v", err) return nil, err } host := vmMo.Summary.Runtime.Host - glog.Infof("%s host is %s", virtualMachine.Reference(), host) + klog.Infof("%s host is %s", virtualMachine.Reference(), host) return host, nil } @@ -103,7 +103,7 @@ func (dc *Datacenter) GetVMByPath(ctx context.Context, vmPath string) (*VirtualM finder := getFinder(dc) vm, err := finder.VirtualMachine(ctx, vmPath) if err != nil { - glog.Errorf("Failed to find VM by Path. VM Path: %s, err: %+v", vmPath, err) + klog.Errorf("Failed to find VM by Path. VM Path: %s, err: %+v", vmPath, err) return nil, err } virtualMachine := VirtualMachine{vm, dc} @@ -116,7 +116,7 @@ func (dc *Datacenter) GetAllDatastores(ctx context.Context) (map[string]*Datasto finder := getFinder(dc) datastores, err := finder.DatastoreList(ctx, "*") if err != nil { - glog.Errorf("Failed to get all the datastores. err: %+v", err) + klog.Errorf("Failed to get all the datastores. err: %+v", err) return nil, err } var dsList []types.ManagedObjectReference @@ -129,7 +129,7 @@ func (dc *Datacenter) GetAllDatastores(ctx context.Context) (map[string]*Datasto properties := []string{DatastoreInfoProperty} err = pc.Retrieve(ctx, dsList, properties, &dsMoList) if err != nil { - glog.Errorf("Failed to get Datastore managed objects from datastore objects."+ + klog.Errorf("Failed to get Datastore managed objects from datastore objects."+ " dsObjList: %+v, properties: %+v, err: %v", dsList, properties, err) return nil, err } @@ -141,7 +141,7 @@ func (dc *Datacenter) GetAllDatastores(ctx context.Context) (map[string]*Datasto dc}, dsMo.Info.GetDatastoreInfo()} } - glog.V(9).Infof("dsURLInfoMap : %+v", dsURLInfoMap) + klog.V(9).Infof("dsURLInfoMap : %+v", dsURLInfoMap) return dsURLInfoMap, nil } @@ -150,7 +150,7 @@ func (dc *Datacenter) GetDatastoreByPath(ctx context.Context, vmDiskPath string) datastorePathObj := new(object.DatastorePath) isSuccess := datastorePathObj.FromString(vmDiskPath) if !isSuccess { - glog.Errorf("Failed to parse vmDiskPath: %s", vmDiskPath) + klog.Errorf("Failed to parse vmDiskPath: %s", vmDiskPath) return nil, errors.New("Failed to parse vmDiskPath") } @@ -162,7 +162,7 @@ func (dc *Datacenter) GetDatastoreByName(ctx context.Context, name string) (*Dat finder := getFinder(dc) ds, err := finder.Datastore(ctx, name) if err != nil { - glog.Errorf("Failed while searching for datastore: %s. err: %+v", name, err) + klog.Errorf("Failed while searching for datastore: %s. err: %+v", name, err) return nil, err } datastore := Datastore{ds, dc} @@ -176,7 +176,7 @@ func (dc *Datacenter) GetResourcePool(ctx context.Context, resourcePoolPath stri var err error resourcePool, err = finder.ResourcePoolOrDefault(ctx, resourcePoolPath) if err != nil { - glog.Errorf("Failed to get the ResourcePool for path '%s'. err: %+v", resourcePoolPath, err) + klog.Errorf("Failed to get the ResourcePool for path '%s'. err: %+v", resourcePoolPath, err) return nil, err } return resourcePool, nil @@ -188,7 +188,7 @@ func (dc *Datacenter) GetFolderByPath(ctx context.Context, folderPath string) (* finder := getFinder(dc) vmFolder, err := finder.Folder(ctx, folderPath) if err != nil { - glog.Errorf("Failed to get the folder reference for %s. err: %+v", folderPath, err) + klog.Errorf("Failed to get the folder reference for %s. err: %+v", folderPath, err) return nil, err } folder := Folder{vmFolder, dc} @@ -200,7 +200,7 @@ func (dc *Datacenter) GetVMMoList(ctx context.Context, vmObjList []*VirtualMachi var vmMoList []mo.VirtualMachine var vmRefs []types.ManagedObjectReference if len(vmObjList) < 1 { - glog.Errorf("VirtualMachine Object list is empty") + klog.Errorf("VirtualMachine Object list is empty") return nil, fmt.Errorf("VirtualMachine Object list is empty") } @@ -210,7 +210,7 @@ func (dc *Datacenter) GetVMMoList(ctx context.Context, vmObjList []*VirtualMachi pc := property.DefaultCollector(dc.Client()) err := pc.Retrieve(ctx, vmRefs, properties, &vmMoList) if err != nil { - glog.Errorf("Failed to get VM managed objects from VM objects. vmObjList: %+v, properties: %+v, err: %v", vmObjList, properties, err) + klog.Errorf("Failed to get VM managed objects from VM objects. vmObjList: %+v, properties: %+v, err: %v", vmObjList, properties, err) return nil, err } return vmMoList, nil @@ -226,7 +226,7 @@ func (dc *Datacenter) GetVirtualDiskPage83Data(ctx context.Context, diskPath str diskUUID, err := vdm.QueryVirtualDiskUuid(ctx, diskPath, dc.Datacenter) if err != nil { - glog.Warningf("QueryVirtualDiskUuid failed for diskPath: %q. err: %+v", diskPath, err) + klog.Warningf("QueryVirtualDiskUuid failed for diskPath: %q. err: %+v", diskPath, err) return "", err } diskUUID = formatVirtualDiskUUID(diskUUID) @@ -238,7 +238,7 @@ func (dc *Datacenter) GetDatastoreMoList(ctx context.Context, dsObjList []*Datas var dsMoList []mo.Datastore var dsRefs []types.ManagedObjectReference if len(dsObjList) < 1 { - glog.Errorf("Datastore Object list is empty") + klog.Errorf("Datastore Object list is empty") return nil, fmt.Errorf("Datastore Object list is empty") } @@ -248,7 +248,7 @@ func (dc *Datacenter) GetDatastoreMoList(ctx context.Context, dsObjList []*Datas pc := property.DefaultCollector(dc.Client()) err := pc.Retrieve(ctx, dsRefs, properties, &dsMoList) if err != nil { - glog.Errorf("Failed to get Datastore managed objects from datastore objects. dsObjList: %+v, properties: %+v, err: %v", dsObjList, properties, err) + klog.Errorf("Failed to get Datastore managed objects from datastore objects. dsObjList: %+v, properties: %+v, err: %v", dsObjList, properties, err) return nil, err } return dsMoList, nil @@ -266,27 +266,27 @@ func (dc *Datacenter) CheckDisksAttached(ctx context.Context, nodeVolumes map[st vm, err := dc.GetVMByPath(ctx, nodeName) if err != nil { if IsNotFound(err) { - glog.Warningf("Node %q does not exist, vSphere CP will assume disks %v are not attached to it.", nodeName, volPaths) + klog.Warningf("Node %q does not exist, vSphere CP will assume disks %v are not attached to it.", nodeName, volPaths) } continue } vmList = append(vmList, vm) } if len(vmList) == 0 { - glog.V(2).Infof("vSphere CP will assume no disks are attached to any node.") + klog.V(2).Infof("vSphere CP will assume no disks are attached to any node.") return attached, nil } vmMoList, err := dc.GetVMMoList(ctx, vmList, []string{"config.hardware.device", "name"}) if err != nil { // When there is an error fetching instance information // it is safer to return nil and let volume information not be touched. - glog.Errorf("Failed to get VM Managed object for nodes: %+v. err: +%v", vmList, err) + klog.Errorf("Failed to get VM Managed object for nodes: %+v. err: +%v", vmList, err) return nil, err } for _, vmMo := range vmMoList { if vmMo.Config == nil { - glog.Errorf("Config is not available for VM: %q", vmMo.Name) + klog.Errorf("Config is not available for VM: %q", vmMo.Name) continue } for nodeName, volPaths := range nodeVolumes { diff --git a/pkg/cloudprovider/providers/vsphere/vclib/datastore.go b/pkg/cloudprovider/providers/vsphere/vclib/datastore.go index 8d21789f9ce..a57685bc76c 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/datastore.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/datastore.go @@ -20,12 +20,12 @@ import ( "context" "fmt" - "github.com/golang/glog" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" + "k8s.io/klog" ) // Datastore extends the govmomi Datastore object @@ -59,7 +59,7 @@ func (ds *Datastore) CreateDirectory(ctx context.Context, directoryPath string, } return err } - glog.V(LogLevel).Infof("Created dir with path as %+q", directoryPath) + klog.V(LogLevel).Infof("Created dir with path as %+q", directoryPath) return nil } @@ -69,7 +69,7 @@ func (ds *Datastore) GetType(ctx context.Context) (string, error) { pc := property.DefaultCollector(ds.Client()) err := pc.RetrieveOne(ctx, ds.Datastore.Reference(), []string{"summary"}, &dsMo) if err != nil { - glog.Errorf("Failed to retrieve datastore summary property. err: %v", err) + klog.Errorf("Failed to retrieve datastore summary property. err: %v", err) return "", err } return dsMo.Summary.Type, nil @@ -80,7 +80,7 @@ func (ds *Datastore) GetType(ctx context.Context) (string, error) { func (ds *Datastore) IsCompatibleWithStoragePolicy(ctx context.Context, storagePolicyID string) (bool, string, error) { pbmClient, err := NewPbmClient(ctx, ds.Client()) if err != nil { - glog.Errorf("Failed to get new PbmClient Object. err: %v", err) + klog.Errorf("Failed to get new PbmClient Object. err: %v", err) return false, "", err } return pbmClient.IsDatastoreCompatible(ctx, storagePolicyID, ds) diff --git a/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/BUILD b/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/BUILD index 377202f4cd6..4273c972f2c 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/BUILD +++ b/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/BUILD @@ -15,9 +15,9 @@ go_library( importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers", deps = [ "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/vmware/govmomi/object:go_default_library", "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vdm.go b/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vdm.go index a643241bdb6..9a1c4715cf2 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vdm.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vdm.go @@ -20,9 +20,9 @@ import ( "context" "time" - "github.com/golang/glog" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib" ) @@ -55,13 +55,13 @@ func (diskManager virtualDiskManager) Create(ctx context.Context, datastore *vcl task, err := vdm.CreateVirtualDisk(ctx, diskManager.diskPath, datastore.Datacenter.Datacenter, vmDiskSpec) if err != nil { vclib.RecordvSphereMetric(vclib.APICreateVolume, requestTime, err) - glog.Errorf("Failed to create virtual disk: %s. err: %+v", diskManager.diskPath, err) + klog.Errorf("Failed to create virtual disk: %s. err: %+v", diskManager.diskPath, err) return "", err } taskInfo, err := task.WaitForResult(ctx, nil) vclib.RecordvSphereMetric(vclib.APICreateVolume, requestTime, err) if err != nil { - glog.Errorf("Failed to complete virtual disk creation: %s. err: %+v", diskManager.diskPath, err) + klog.Errorf("Failed to complete virtual disk creation: %s. err: %+v", diskManager.diskPath, err) return "", err } canonicalDiskPath = taskInfo.Result.(string) @@ -77,14 +77,14 @@ func (diskManager virtualDiskManager) Delete(ctx context.Context, datacenter *vc // Delete virtual disk task, err := virtualDiskManager.DeleteVirtualDisk(ctx, diskPath, datacenter.Datacenter) if err != nil { - glog.Errorf("Failed to delete virtual disk. err: %v", err) + klog.Errorf("Failed to delete virtual disk. err: %v", err) vclib.RecordvSphereMetric(vclib.APIDeleteVolume, requestTime, err) return err } err = task.Wait(ctx) vclib.RecordvSphereMetric(vclib.APIDeleteVolume, requestTime, err) if err != nil { - glog.Errorf("Failed to delete virtual disk. err: %v", err) + klog.Errorf("Failed to delete virtual disk. err: %v", err) return err } return nil diff --git a/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/virtualdisk.go b/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/virtualdisk.go index fe905cc79f7..6d7f7f5fe90 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/virtualdisk.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/virtualdisk.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib" ) @@ -65,7 +65,7 @@ func (virtualDisk *VirtualDisk) Create(ctx context.Context, datastore *vclib.Dat virtualDisk.VolumeOptions.DiskFormat = vclib.ThinDiskType } if !virtualDisk.VolumeOptions.VerifyVolumeOptions() { - glog.Error("VolumeOptions verification failed. volumeOptions: ", virtualDisk.VolumeOptions) + klog.Error("VolumeOptions verification failed. volumeOptions: ", virtualDisk.VolumeOptions) return "", vclib.ErrInvalidVolumeOptions } if virtualDisk.VolumeOptions.StoragePolicyID != "" && virtualDisk.VolumeOptions.StoragePolicyName != "" { diff --git a/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vmdm.go b/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vmdm.go index 12fba7f7024..637ac514bfd 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vmdm.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vmdm.go @@ -22,9 +22,9 @@ import ( "hash/fnv" "strings" - "github.com/golang/glog" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib" ) @@ -43,26 +43,26 @@ func (vmdisk vmDiskManager) Create(ctx context.Context, datastore *vclib.Datasto } pbmClient, err := vclib.NewPbmClient(ctx, datastore.Client()) if err != nil { - glog.Errorf("Error occurred while creating new pbmClient, err: %+v", err) + klog.Errorf("Error occurred while creating new pbmClient, err: %+v", err) return "", err } if vmdisk.volumeOptions.StoragePolicyID == "" && vmdisk.volumeOptions.StoragePolicyName != "" { vmdisk.volumeOptions.StoragePolicyID, err = pbmClient.ProfileIDByName(ctx, vmdisk.volumeOptions.StoragePolicyName) if err != nil { - glog.Errorf("Error occurred while getting Profile Id from Profile Name: %s, err: %+v", vmdisk.volumeOptions.StoragePolicyName, err) + klog.Errorf("Error occurred while getting Profile Id from Profile Name: %s, err: %+v", vmdisk.volumeOptions.StoragePolicyName, err) return "", err } } if vmdisk.volumeOptions.StoragePolicyID != "" { compatible, faultMessage, err := datastore.IsCompatibleWithStoragePolicy(ctx, vmdisk.volumeOptions.StoragePolicyID) if err != nil { - glog.Errorf("Error occurred while checking datastore compatibility with storage policy id: %s, err: %+v", vmdisk.volumeOptions.StoragePolicyID, err) + klog.Errorf("Error occurred while checking datastore compatibility with storage policy id: %s, err: %+v", vmdisk.volumeOptions.StoragePolicyID, err) return "", err } if !compatible { - glog.Errorf("Datastore: %s is not compatible with Policy: %s", datastore.Name(), vmdisk.volumeOptions.StoragePolicyName) + klog.Errorf("Datastore: %s is not compatible with Policy: %s", datastore.Name(), vmdisk.volumeOptions.StoragePolicyName) return "", fmt.Errorf("User specified datastore is not compatible with the storagePolicy: %q. Failed with faults: %+q", vmdisk.volumeOptions.StoragePolicyName, faultMessage) } } @@ -79,7 +79,7 @@ func (vmdisk vmDiskManager) Create(ctx context.Context, datastore *vclib.Datasto return "", err } if dsType != vclib.VSANDatastoreType { - glog.Errorf("The specified datastore: %q is not a VSAN datastore", datastore.Name()) + klog.Errorf("The specified datastore: %q is not a VSAN datastore", datastore.Name()) return "", fmt.Errorf("The specified datastore: %q is not a VSAN datastore."+ " The policy parameters will work only with VSAN Datastore."+ " So, please specify a valid VSAN datastore in Storage class definition.", datastore.Name()) @@ -90,7 +90,7 @@ func (vmdisk vmDiskManager) Create(ctx context.Context, datastore *vclib.Datasto ObjectData: vmdisk.volumeOptions.VSANStorageProfileData, } } else { - glog.Errorf("Both volumeOptions.StoragePolicyID and volumeOptions.VSANStorageProfileData are not set. One of them should be set") + klog.Errorf("Both volumeOptions.StoragePolicyID and volumeOptions.VSANStorageProfileData are not set. One of them should be set") return "", fmt.Errorf("Both volumeOptions.StoragePolicyID and volumeOptions.VSANStorageProfileData are not set. One of them should be set") } var dummyVM *vclib.VirtualMachine @@ -102,10 +102,10 @@ func (vmdisk vmDiskManager) Create(ctx context.Context, datastore *vclib.Datasto dummyVM, err = datastore.Datacenter.GetVMByPath(ctx, vmdisk.vmOptions.VMFolder.InventoryPath+"/"+dummyVMFullName) if err != nil { // Create a dummy VM - glog.V(1).Infof("Creating Dummy VM: %q", dummyVMFullName) + klog.V(1).Infof("Creating Dummy VM: %q", dummyVMFullName) dummyVM, err = vmdisk.createDummyVM(ctx, datastore.Datacenter, dummyVMFullName) if err != nil { - glog.Errorf("Failed to create Dummy VM. err: %v", err) + klog.Errorf("Failed to create Dummy VM. err: %v", err) return "", err } } @@ -114,7 +114,7 @@ func (vmdisk vmDiskManager) Create(ctx context.Context, datastore *vclib.Datasto virtualMachineConfigSpec := types.VirtualMachineConfigSpec{} disk, _, err := dummyVM.CreateDiskSpec(ctx, vmdisk.diskPath, datastore, vmdisk.volumeOptions) if err != nil { - glog.Errorf("Failed to create Disk Spec. err: %v", err) + klog.Errorf("Failed to create Disk Spec. err: %v", err) return "", err } deviceConfigSpec := &types.VirtualDeviceConfigSpec{ @@ -128,7 +128,7 @@ func (vmdisk vmDiskManager) Create(ctx context.Context, datastore *vclib.Datasto fileAlreadyExist := false task, err := dummyVM.Reconfigure(ctx, virtualMachineConfigSpec) if err != nil { - glog.Errorf("Failed to reconfig. err: %v", err) + klog.Errorf("Failed to reconfig. err: %v", err) return "", err } err = task.Wait(ctx) @@ -136,9 +136,9 @@ func (vmdisk vmDiskManager) Create(ctx context.Context, datastore *vclib.Datasto fileAlreadyExist = isAlreadyExists(vmdisk.diskPath, err) if fileAlreadyExist { //Skip error and continue to detach the disk as the disk was already created on the datastore. - glog.V(vclib.LogLevel).Infof("File: %v already exists", vmdisk.diskPath) + klog.V(vclib.LogLevel).Infof("File: %v already exists", vmdisk.diskPath) } else { - glog.Errorf("Failed to attach the disk to VM: %q with err: %+v", dummyVMFullName, err) + klog.Errorf("Failed to attach the disk to VM: %q with err: %+v", dummyVMFullName, err) return "", err } } @@ -147,16 +147,16 @@ func (vmdisk vmDiskManager) Create(ctx context.Context, datastore *vclib.Datasto if err != nil { if vclib.DiskNotFoundErrMsg == err.Error() && fileAlreadyExist { // Skip error if disk was already detached from the dummy VM but still present on the datastore. - glog.V(vclib.LogLevel).Infof("File: %v is already detached", vmdisk.diskPath) + klog.V(vclib.LogLevel).Infof("File: %v is already detached", vmdisk.diskPath) } else { - glog.Errorf("Failed to detach the disk: %q from VM: %q with err: %+v", vmdisk.diskPath, dummyVMFullName, err) + klog.Errorf("Failed to detach the disk: %q from VM: %q with err: %+v", vmdisk.diskPath, dummyVMFullName, err) return "", err } } // Delete the dummy VM err = dummyVM.DeleteVM(ctx) if err != nil { - glog.Errorf("Failed to destroy the vm: %q with err: %+v", dummyVMFullName, err) + klog.Errorf("Failed to destroy the vm: %q with err: %+v", dummyVMFullName, err) } return vmdisk.diskPath, nil } @@ -195,13 +195,13 @@ func (vmdisk vmDiskManager) createDummyVM(ctx context.Context, datacenter *vclib task, err := vmdisk.vmOptions.VMFolder.CreateVM(ctx, virtualMachineConfigSpec, vmdisk.vmOptions.VMResourcePool, nil) if err != nil { - glog.Errorf("Failed to create VM. err: %+v", err) + klog.Errorf("Failed to create VM. err: %+v", err) return nil, err } dummyVMTaskInfo, err := task.WaitForResult(ctx, nil) if err != nil { - glog.Errorf("Error occurred while waiting for create VM task result. err: %+v", err) + klog.Errorf("Error occurred while waiting for create VM task result. err: %+v", err) return nil, err } @@ -214,11 +214,11 @@ func (vmdisk vmDiskManager) createDummyVM(ctx context.Context, datacenter *vclib func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder, dc *vclib.Datacenter) error { vmList, err := folder.GetVirtualMachines(ctx) if err != nil { - glog.V(4).Infof("Failed to get virtual machines in the kubernetes cluster: %s, err: %+v", folder.InventoryPath, err) + klog.V(4).Infof("Failed to get virtual machines in the kubernetes cluster: %s, err: %+v", folder.InventoryPath, err) return err } if vmList == nil || len(vmList) == 0 { - glog.Errorf("No virtual machines found in the kubernetes cluster: %s", folder.InventoryPath) + klog.Errorf("No virtual machines found in the kubernetes cluster: %s", folder.InventoryPath) return fmt.Errorf("No virtual machines found in the kubernetes cluster: %s", folder.InventoryPath) } var dummyVMList []*vclib.VirtualMachine @@ -226,7 +226,7 @@ func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder, dc *vclib.Datace for _, vm := range vmList { vmName, err := vm.ObjectName(ctx) if err != nil { - glog.V(4).Infof("Unable to get name from VM with err: %+v", err) + klog.V(4).Infof("Unable to get name from VM with err: %+v", err) continue } if strings.HasPrefix(vmName, vclib.DummyVMPrefixName) { @@ -237,7 +237,7 @@ func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder, dc *vclib.Datace for _, vm := range dummyVMList { err = vm.DeleteVM(ctx) if err != nil { - glog.V(4).Infof("Unable to delete dummy VM with err: %+v", err) + klog.V(4).Infof("Unable to delete dummy VM with err: %+v", err) continue } } diff --git a/pkg/cloudprovider/providers/vsphere/vclib/folder.go b/pkg/cloudprovider/providers/vsphere/vclib/folder.go index 4e66de88dbb..1e4f8e4e888 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/folder.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/folder.go @@ -19,8 +19,8 @@ package vclib import ( "context" - "github.com/golang/glog" "github.com/vmware/govmomi/object" + "k8s.io/klog" ) // Folder extends the govmomi Folder object @@ -33,7 +33,7 @@ type Folder struct { func (folder *Folder) GetVirtualMachines(ctx context.Context) ([]*VirtualMachine, error) { vmFolders, err := folder.Children(ctx) if err != nil { - glog.Errorf("Failed to get children from Folder: %s. err: %+v", folder.InventoryPath, err) + klog.Errorf("Failed to get children from Folder: %s. err: %+v", folder.InventoryPath, err) return nil, err } var vmObjList []*VirtualMachine diff --git a/pkg/cloudprovider/providers/vsphere/vclib/pbm.go b/pkg/cloudprovider/providers/vsphere/vclib/pbm.go index 0a494f7d731..8070f200423 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/pbm.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/pbm.go @@ -20,8 +20,8 @@ import ( "context" "fmt" - "github.com/golang/glog" "github.com/vmware/govmomi/pbm" + "k8s.io/klog" pbmtypes "github.com/vmware/govmomi/pbm/types" "github.com/vmware/govmomi/vim25" @@ -36,7 +36,7 @@ type PbmClient struct { func NewPbmClient(ctx context.Context, client *vim25.Client) (*PbmClient, error) { pbmClient, err := pbm.NewClient(ctx, client) if err != nil { - glog.Errorf("Failed to create new Pbm Client. err: %+v", err) + klog.Errorf("Failed to create new Pbm Client. err: %+v", err) return nil, err } return &PbmClient{pbmClient}, nil @@ -60,7 +60,7 @@ func (pbmClient *PbmClient) IsDatastoreCompatible(ctx context.Context, storagePo } compatibilityResult, err := pbmClient.CheckRequirements(ctx, hubs, nil, req) if err != nil { - glog.Errorf("Error occurred for CheckRequirements call. err %+v", err) + klog.Errorf("Error occurred for CheckRequirements call. err %+v", err) return false, "", err } if compatibilityResult != nil && len(compatibilityResult) > 0 { @@ -70,7 +70,7 @@ func (pbmClient *PbmClient) IsDatastoreCompatible(ctx context.Context, storagePo } dsName, err := datastore.ObjectName(ctx) if err != nil { - glog.Errorf("Failed to get datastore ObjectName") + klog.Errorf("Failed to get datastore ObjectName") return false, "", err } if compatibilityResult[0].Error[0].LocalizedMessage == "" { @@ -92,7 +92,7 @@ func (pbmClient *PbmClient) GetCompatibleDatastores(ctx context.Context, dc *Dat ) compatibilityResult, err := pbmClient.GetPlacementCompatibilityResult(ctx, storagePolicyID, datastores) if err != nil { - glog.Errorf("Error occurred while retrieving placement compatibility result for datastores: %+v with storagePolicyID: %s. err: %+v", datastores, storagePolicyID, err) + klog.Errorf("Error occurred while retrieving placement compatibility result for datastores: %+v with storagePolicyID: %s. err: %+v", datastores, storagePolicyID, err) return nil, "", err } compatibleHubs := compatibilityResult.CompatibleDatastores() @@ -114,7 +114,7 @@ func (pbmClient *PbmClient) GetCompatibleDatastores(ctx context.Context, dc *Dat } // Return an error if there are no compatible datastores. if len(compatibleHubs) < 1 { - glog.Errorf("No compatible datastores found that satisfy the storage policy requirements: %s", storagePolicyID) + klog.Errorf("No compatible datastores found that satisfy the storage policy requirements: %s", storagePolicyID) return nil, localizedMessagesForNotCompatibleDatastores, fmt.Errorf("No compatible datastores found that satisfy the storage policy requirements") } return compatibleDatastoreList, localizedMessagesForNotCompatibleDatastores, nil @@ -138,7 +138,7 @@ func (pbmClient *PbmClient) GetPlacementCompatibilityResult(ctx context.Context, } res, err := pbmClient.CheckRequirements(ctx, hubs, nil, req) if err != nil { - glog.Errorf("Error occurred for CheckRequirements call. err: %+v", err) + klog.Errorf("Error occurred for CheckRequirements call. err: %+v", err) return nil, err } return res, nil @@ -162,7 +162,7 @@ func getDsMorNameMap(ctx context.Context, datastores []*DatastoreInfo) map[strin if err == nil { dsMorNameMap[ds.Reference().Value] = dsObjectName } else { - glog.Errorf("Error occurred while getting datastore object name. err: %+v", err) + klog.Errorf("Error occurred while getting datastore object name. err: %+v", err) } } return dsMorNameMap diff --git a/pkg/cloudprovider/providers/vsphere/vclib/utils.go b/pkg/cloudprovider/providers/vsphere/vclib/utils.go index 36ea8d6c6ef..161e0ebfe72 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/utils.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/utils.go @@ -22,12 +22,12 @@ import ( "regexp" "strings" - "github.com/golang/glog" "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" + "k8s.io/klog" ) // IsNotFound return true if err is NotFoundError or DefaultNotFoundError @@ -140,7 +140,7 @@ func GetPathFromVMDiskPath(vmDiskPath string) string { datastorePathObj := new(object.DatastorePath) isSuccess := datastorePathObj.FromString(vmDiskPath) if !isSuccess { - glog.Errorf("Failed to parse vmDiskPath: %s", vmDiskPath) + klog.Errorf("Failed to parse vmDiskPath: %s", vmDiskPath) return "" } return datastorePathObj.Path @@ -151,7 +151,7 @@ func GetDatastorePathObjFromVMDiskPath(vmDiskPath string) (*object.DatastorePath datastorePathObj := new(object.DatastorePath) isSuccess := datastorePathObj.FromString(vmDiskPath) if !isSuccess { - glog.Errorf("Failed to parse volPath: %s", vmDiskPath) + klog.Errorf("Failed to parse volPath: %s", vmDiskPath) return nil, fmt.Errorf("Failed to parse volPath: %s", vmDiskPath) } return datastorePathObj, nil diff --git a/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go b/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go index 01654b3d1ef..f6e28cb1036 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go @@ -22,12 +22,12 @@ import ( "strings" "time" - "github.com/golang/glog" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" + "k8s.io/klog" ) // VirtualMachine extends the govmomi VirtualMachine object @@ -52,7 +52,7 @@ func (vm *VirtualMachine) IsDiskAttached(ctx context.Context, diskPath string) ( func (vm *VirtualMachine) DeleteVM(ctx context.Context) error { destroyTask, err := vm.Destroy(ctx) if err != nil { - glog.Errorf("Failed to delete the VM: %q. err: %+v", vm.InventoryPath, err) + klog.Errorf("Failed to delete the VM: %q. err: %+v", vm.InventoryPath, err) return err } return destroyTask.Wait(ctx) @@ -69,7 +69,7 @@ func (vm *VirtualMachine) AttachDisk(ctx context.Context, vmDiskPath string, vol vmDiskPath = RemoveStorageClusterORFolderNameFromVDiskPath(vmDiskPath) attached, err := vm.IsDiskAttached(ctx, vmDiskPath) if err != nil { - glog.Errorf("Error occurred while checking if disk is attached on VM: %q. vmDiskPath: %q, err: %+v", vm.InventoryPath, vmDiskPath, err) + klog.Errorf("Error occurred while checking if disk is attached on VM: %q. vmDiskPath: %q, err: %+v", vm.InventoryPath, vmDiskPath, err) return "", err } // If disk is already attached, return the disk UUID @@ -81,31 +81,31 @@ func (vm *VirtualMachine) AttachDisk(ctx context.Context, vmDiskPath string, vol if volumeOptions.StoragePolicyName != "" { pbmClient, err := NewPbmClient(ctx, vm.Client()) if err != nil { - glog.Errorf("Error occurred while creating new pbmClient. err: %+v", err) + klog.Errorf("Error occurred while creating new pbmClient. err: %+v", err) return "", err } volumeOptions.StoragePolicyID, err = pbmClient.ProfileIDByName(ctx, volumeOptions.StoragePolicyName) if err != nil { - glog.Errorf("Failed to get Profile ID by name: %s. err: %+v", volumeOptions.StoragePolicyName, err) + klog.Errorf("Failed to get Profile ID by name: %s. err: %+v", volumeOptions.StoragePolicyName, err) return "", err } } dsObj, err := vm.Datacenter.GetDatastoreByPath(ctx, vmDiskPathCopy) if err != nil { - glog.Errorf("Failed to get datastore from vmDiskPath: %q. err: %+v", vmDiskPath, err) + klog.Errorf("Failed to get datastore from vmDiskPath: %q. err: %+v", vmDiskPath, err) return "", err } // If disk is not attached, create a disk spec for disk to be attached to the VM. disk, newSCSIController, err := vm.CreateDiskSpec(ctx, vmDiskPath, dsObj, volumeOptions) if err != nil { - glog.Errorf("Error occurred while creating disk spec. err: %+v", err) + klog.Errorf("Error occurred while creating disk spec. err: %+v", err) return "", err } vmDevices, err := vm.Device(ctx) if err != nil { - glog.Errorf("Failed to retrieve VM devices for VM: %q. err: %+v", vm.InventoryPath, err) + klog.Errorf("Failed to retrieve VM devices for VM: %q. err: %+v", vm.InventoryPath, err) return "", err } virtualMachineConfigSpec := types.VirtualMachineConfigSpec{} @@ -125,7 +125,7 @@ func (vm *VirtualMachine) AttachDisk(ctx context.Context, vmDiskPath string, vol task, err := vm.Reconfigure(ctx, virtualMachineConfigSpec) if err != nil { RecordvSphereMetric(APIAttachVolume, requestTime, err) - glog.Errorf("Failed to attach the disk with storagePolicy: %q on VM: %q. err - %+v", volumeOptions.StoragePolicyID, vm.InventoryPath, err) + klog.Errorf("Failed to attach the disk with storagePolicy: %q on VM: %q. err - %+v", volumeOptions.StoragePolicyID, vm.InventoryPath, err) if newSCSIController != nil { vm.deleteController(ctx, newSCSIController, vmDevices) } @@ -134,7 +134,7 @@ func (vm *VirtualMachine) AttachDisk(ctx context.Context, vmDiskPath string, vol err = task.Wait(ctx) RecordvSphereMetric(APIAttachVolume, requestTime, err) if err != nil { - glog.Errorf("Failed to attach the disk with storagePolicy: %+q on VM: %q. err - %+v", volumeOptions.StoragePolicyID, vm.InventoryPath, err) + klog.Errorf("Failed to attach the disk with storagePolicy: %+q on VM: %q. err - %+v", volumeOptions.StoragePolicyID, vm.InventoryPath, err) if newSCSIController != nil { vm.deleteController(ctx, newSCSIController, vmDevices) } @@ -144,7 +144,7 @@ func (vm *VirtualMachine) AttachDisk(ctx context.Context, vmDiskPath string, vol // Once disk is attached, get the disk UUID. diskUUID, err := vm.Datacenter.GetVirtualDiskPage83Data(ctx, vmDiskPath) if err != nil { - glog.Errorf("Error occurred while getting Disk Info from VM: %q. err: %v", vm.InventoryPath, err) + klog.Errorf("Error occurred while getting Disk Info from VM: %q. err: %v", vm.InventoryPath, err) vm.DetachDisk(ctx, vmDiskPath) if newSCSIController != nil { vm.deleteController(ctx, newSCSIController, vmDevices) @@ -159,11 +159,11 @@ func (vm *VirtualMachine) DetachDisk(ctx context.Context, vmDiskPath string) err vmDiskPath = RemoveStorageClusterORFolderNameFromVDiskPath(vmDiskPath) device, err := vm.getVirtualDeviceByPath(ctx, vmDiskPath) if err != nil { - glog.Errorf("Disk ID not found for VM: %q with diskPath: %q", vm.InventoryPath, vmDiskPath) + klog.Errorf("Disk ID not found for VM: %q with diskPath: %q", vm.InventoryPath, vmDiskPath) return err } if device == nil { - glog.Errorf("No virtual device found with diskPath: %q on VM: %q", vmDiskPath, vm.InventoryPath) + klog.Errorf("No virtual device found with diskPath: %q on VM: %q", vmDiskPath, vm.InventoryPath) return fmt.Errorf("No virtual device found with diskPath: %q on VM: %q", vmDiskPath, vm.InventoryPath) } // Detach disk from VM @@ -171,7 +171,7 @@ func (vm *VirtualMachine) DetachDisk(ctx context.Context, vmDiskPath string) err err = vm.RemoveDevice(ctx, true, device) RecordvSphereMetric(APIDetachVolume, requestTime, err) if err != nil { - glog.Errorf("Error occurred while removing disk device for VM: %q. err: %v", vm.InventoryPath, err) + klog.Errorf("Error occurred while removing disk device for VM: %q. err: %v", vm.InventoryPath, err) return err } return nil @@ -181,7 +181,7 @@ func (vm *VirtualMachine) DetachDisk(ctx context.Context, vmDiskPath string) err func (vm *VirtualMachine) GetResourcePool(ctx context.Context) (*object.ResourcePool, error) { vmMoList, err := vm.Datacenter.GetVMMoList(ctx, []*VirtualMachine{vm}, []string{"resourcePool"}) if err != nil { - glog.Errorf("Failed to get resource pool from VM: %q. err: %+v", vm.InventoryPath, err) + klog.Errorf("Failed to get resource pool from VM: %q. err: %+v", vm.InventoryPath, err) return nil, err } return object.NewResourcePool(vm.Client(), vmMoList[0].ResourcePool.Reference()), nil @@ -192,7 +192,7 @@ func (vm *VirtualMachine) GetResourcePool(ctx context.Context) (*object.Resource func (vm *VirtualMachine) IsActive(ctx context.Context) (bool, error) { vmMoList, err := vm.Datacenter.GetVMMoList(ctx, []*VirtualMachine{vm}, []string{"summary"}) if err != nil { - glog.Errorf("Failed to get VM Managed object with property summary. err: +%v", err) + klog.Errorf("Failed to get VM Managed object with property summary. err: +%v", err) return false, err } if vmMoList[0].Summary.Runtime.PowerState == ActivePowerState { @@ -206,14 +206,14 @@ func (vm *VirtualMachine) IsActive(ctx context.Context) (bool, error) { func (vm *VirtualMachine) GetAllAccessibleDatastores(ctx context.Context) ([]*DatastoreInfo, error) { host, err := vm.HostSystem(ctx) if err != nil { - glog.Errorf("Failed to get host system for VM: %q. err: %+v", vm.InventoryPath, err) + klog.Errorf("Failed to get host system for VM: %q. err: %+v", vm.InventoryPath, err) return nil, err } var hostSystemMo mo.HostSystem s := object.NewSearchIndex(vm.Client()) err = s.Properties(ctx, host.Reference(), []string{DatastoreProperty}, &hostSystemMo) if err != nil { - glog.Errorf("Failed to retrieve datastores for host: %+v. err: %+v", host, err) + klog.Errorf("Failed to retrieve datastores for host: %+v. err: %+v", host, err) return nil, err } var dsRefList []types.ManagedObjectReference @@ -226,11 +226,11 @@ func (vm *VirtualMachine) GetAllAccessibleDatastores(ctx context.Context) ([]*Da properties := []string{DatastoreInfoProperty} err = pc.Retrieve(ctx, dsRefList, properties, &dsMoList) if err != nil { - glog.Errorf("Failed to get Datastore managed objects from datastore objects."+ + klog.Errorf("Failed to get Datastore managed objects from datastore objects."+ " dsObjList: %+v, properties: %+v, err: %v", dsRefList, properties, err) return nil, err } - glog.V(9).Infof("Result dsMoList: %+v", dsMoList) + klog.V(9).Infof("Result dsMoList: %+v", dsMoList) var dsObjList []*DatastoreInfo for _, dsMo := range dsMoList { dsObjList = append(dsObjList, @@ -247,7 +247,7 @@ func (vm *VirtualMachine) CreateDiskSpec(ctx context.Context, diskPath string, d var newSCSIController types.BaseVirtualDevice vmDevices, err := vm.Device(ctx) if err != nil { - glog.Errorf("Failed to retrieve VM devices. err: %+v", err) + klog.Errorf("Failed to retrieve VM devices. err: %+v", err) return nil, nil, err } // find SCSI controller of particular type from VM devices @@ -256,20 +256,20 @@ func (vm *VirtualMachine) CreateDiskSpec(ctx context.Context, diskPath string, d if scsiController == nil { newSCSIController, err = vm.createAndAttachSCSIController(ctx, volumeOptions.SCSIControllerType) if err != nil { - glog.Errorf("Failed to create SCSI controller for VM :%q with err: %+v", vm.InventoryPath, err) + klog.Errorf("Failed to create SCSI controller for VM :%q with err: %+v", vm.InventoryPath, err) return nil, nil, err } // Get VM device list vmDevices, err := vm.Device(ctx) if err != nil { - glog.Errorf("Failed to retrieve VM devices. err: %v", err) + klog.Errorf("Failed to retrieve VM devices. err: %v", err) return nil, nil, err } // verify scsi controller in virtual machine scsiControllersOfRequiredType := getSCSIControllersOfType(vmDevices, volumeOptions.SCSIControllerType) scsiController = getAvailableSCSIController(scsiControllersOfRequiredType) if scsiController == nil { - glog.Errorf("Cannot find SCSI controller of type: %q in VM", volumeOptions.SCSIControllerType) + klog.Errorf("Cannot find SCSI controller of type: %q in VM", volumeOptions.SCSIControllerType) // attempt clean up of scsi controller vm.deleteController(ctx, newSCSIController, vmDevices) return nil, nil, fmt.Errorf("Cannot find SCSI controller of type: %q in VM", volumeOptions.SCSIControllerType) @@ -278,7 +278,7 @@ func (vm *VirtualMachine) CreateDiskSpec(ctx context.Context, diskPath string, d disk := vmDevices.CreateDisk(scsiController, dsObj.Reference(), diskPath) unitNumber, err := getNextUnitNumber(vmDevices, scsiController) if err != nil { - glog.Errorf("Cannot attach disk to VM, unitNumber limit reached - %+v.", err) + klog.Errorf("Cannot attach disk to VM, unitNumber limit reached - %+v.", err) return nil, nil, err } *disk.UnitNumber = unitNumber @@ -307,7 +307,7 @@ func (vm *VirtualMachine) CreateDiskSpec(ctx context.Context, diskPath string, d func (vm *VirtualMachine) GetVirtualDiskPath(ctx context.Context) (string, error) { vmDevices, err := vm.Device(ctx) if err != nil { - glog.Errorf("Failed to get the devices for VM: %q. err: %+v", vm.InventoryPath, err) + klog.Errorf("Failed to get the devices for VM: %q. err: %+v", vm.InventoryPath, err) return "", err } // filter vm devices to retrieve device for the given vmdk file identified by disk path @@ -327,18 +327,18 @@ func (vm *VirtualMachine) createAndAttachSCSIController(ctx context.Context, dis // Get VM device list vmDevices, err := vm.Device(ctx) if err != nil { - glog.Errorf("Failed to retrieve VM devices for VM: %q. err: %+v", vm.InventoryPath, err) + klog.Errorf("Failed to retrieve VM devices for VM: %q. err: %+v", vm.InventoryPath, err) return nil, err } allSCSIControllers := getSCSIControllers(vmDevices) if len(allSCSIControllers) >= SCSIControllerLimit { // we reached the maximum number of controllers we can attach - glog.Errorf("SCSI Controller Limit of %d has been reached, cannot create another SCSI controller", SCSIControllerLimit) + klog.Errorf("SCSI Controller Limit of %d has been reached, cannot create another SCSI controller", SCSIControllerLimit) return nil, fmt.Errorf("SCSI Controller Limit of %d has been reached, cannot create another SCSI controller", SCSIControllerLimit) } newSCSIController, err := vmDevices.CreateSCSIController(diskControllerType) if err != nil { - glog.Errorf("Failed to create new SCSI controller on VM: %q. err: %+v", vm.InventoryPath, err) + klog.Errorf("Failed to create new SCSI controller on VM: %q. err: %+v", vm.InventoryPath, err) return nil, err } configNewSCSIController := newSCSIController.(types.BaseVirtualSCSIController).GetVirtualSCSIController() @@ -349,7 +349,7 @@ func (vm *VirtualMachine) createAndAttachSCSIController(ctx context.Context, dis // add the scsi controller to virtual machine err = vm.AddDevice(context.TODO(), newSCSIController) if err != nil { - glog.V(LogLevel).Infof("Cannot add SCSI controller to VM: %q. err: %+v", vm.InventoryPath, err) + klog.V(LogLevel).Infof("Cannot add SCSI controller to VM: %q. err: %+v", vm.InventoryPath, err) // attempt clean up of scsi controller vm.deleteController(ctx, newSCSIController, vmDevices) return nil, err @@ -361,7 +361,7 @@ func (vm *VirtualMachine) createAndAttachSCSIController(ctx context.Context, dis func (vm *VirtualMachine) getVirtualDeviceByPath(ctx context.Context, diskPath string) (types.BaseVirtualDevice, error) { vmDevices, err := vm.Device(ctx) if err != nil { - glog.Errorf("Failed to get the devices for VM: %q. err: %+v", vm.InventoryPath, err) + klog.Errorf("Failed to get the devices for VM: %q. err: %+v", vm.InventoryPath, err) return nil, err } @@ -371,7 +371,7 @@ func (vm *VirtualMachine) getVirtualDeviceByPath(ctx context.Context, diskPath s virtualDevice := device.GetVirtualDevice() if backing, ok := virtualDevice.Backing.(*types.VirtualDiskFlatVer2BackingInfo); ok { if matchVirtualDiskAndVolPath(backing.FileName, diskPath) { - glog.V(LogLevel).Infof("Found VirtualDisk backing with filename %q for diskPath %q", backing.FileName, diskPath) + klog.V(LogLevel).Infof("Found VirtualDisk backing with filename %q for diskPath %q", backing.FileName, diskPath) return device, nil } } @@ -396,7 +396,7 @@ func (vm *VirtualMachine) deleteController(ctx context.Context, controllerDevice device := controllerDeviceList[len(controllerDeviceList)-1] err := vm.RemoveDevice(ctx, true, device) if err != nil { - glog.Errorf("Error occurred while removing device on VM: %q. err: %+v", vm.InventoryPath, err) + klog.Errorf("Error occurred while removing device on VM: %q. err: %+v", vm.InventoryPath, err) return err } return nil diff --git a/pkg/cloudprovider/providers/vsphere/vclib/volumeoptions.go b/pkg/cloudprovider/providers/vsphere/vclib/volumeoptions.go index eceba70496c..989ed446815 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/volumeoptions.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/volumeoptions.go @@ -19,7 +19,7 @@ package vclib import ( "strings" - "github.com/golang/glog" + "k8s.io/klog" ) // VolumeOptions specifies various options for a volume. @@ -59,7 +59,7 @@ func DiskformatValidOptions() string { // CheckDiskFormatSupported checks if the diskFormat is valid func CheckDiskFormatSupported(diskFormat string) bool { if DiskFormatValidType[diskFormat] == "" { - glog.Errorf("Not a valid Disk Format. Valid options are %+q", DiskformatValidOptions()) + klog.Errorf("Not a valid Disk Format. Valid options are %+q", DiskformatValidOptions()) return false } return true @@ -82,7 +82,7 @@ func CheckControllerSupported(ctrlType string) bool { return true } } - glog.Errorf("Not a valid SCSI Controller Type. Valid options are %q", SCSIControllerTypeValidOptions()) + klog.Errorf("Not a valid SCSI Controller Type. Valid options are %q", SCSIControllerTypeValidOptions()) return false } diff --git a/pkg/cloudprovider/providers/vsphere/vsphere.go b/pkg/cloudprovider/providers/vsphere/vsphere.go index 7e22407ed5b..26180e6aff9 100644 --- a/pkg/cloudprovider/providers/vsphere/vsphere.go +++ b/pkg/cloudprovider/providers/vsphere/vsphere.go @@ -33,7 +33,6 @@ import ( "gopkg.in/gcfg.v1" - "github.com/golang/glog" "github.com/vmware/govmomi/vapi/rest" "github.com/vmware/govmomi/vapi/tags" "github.com/vmware/govmomi/vim25/mo" @@ -42,6 +41,7 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/tools/cache" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers" @@ -264,13 +264,13 @@ func (vs *VSphere) SetInformers(informerFactory informers.SharedInformerFactory) // Only on controller node it is required to register listeners. // Register callbacks for node updates - glog.V(4).Infof("Setting up node informers for vSphere Cloud Provider") + klog.V(4).Infof("Setting up node informers for vSphere Cloud Provider") nodeInformer := informerFactory.Core().V1().Nodes().Informer() nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: vs.NodeAdded, DeleteFunc: vs.NodeDeleted, }) - glog.V(4).Infof("Node informers in vSphere cloud provider initialized") + klog.V(4).Infof("Node informers in vSphere cloud provider initialized") } @@ -280,12 +280,12 @@ func newWorkerNode() (*VSphere, error) { vs := VSphere{} vs.hostName, err = os.Hostname() if err != nil { - glog.Errorf("Failed to get hostname. err: %+v", err) + klog.Errorf("Failed to get hostname. err: %+v", err) return nil, err } vs.vmUUID, err = GetVMUUID() if err != nil { - glog.Errorf("Failed to get uuid. err: %+v", err) + klog.Errorf("Failed to get uuid. err: %+v", err) return nil, err } return &vs, nil @@ -296,18 +296,18 @@ func populateVsphereInstanceMap(cfg *VSphereConfig) (map[string]*VSphereInstance isSecretInfoProvided := true if cfg.Global.SecretName == "" || cfg.Global.SecretNamespace == "" { - glog.Warningf("SecretName and/or SecretNamespace is not provided. " + + klog.Warningf("SecretName and/or SecretNamespace is not provided. " + "VCP will use username and password from config file") isSecretInfoProvided = false } if isSecretInfoProvided { if cfg.Global.User != "" { - glog.Warning("Global.User and Secret info provided. VCP will use secret to get credentials") + klog.Warning("Global.User and Secret info provided. VCP will use secret to get credentials") cfg.Global.User = "" } if cfg.Global.Password != "" { - glog.Warning("Global.Password and Secret info provided. VCP will use secret to get credentials") + klog.Warning("Global.Password and Secret info provided. VCP will use secret to get credentials") cfg.Global.Password = "" } } @@ -315,28 +315,28 @@ func populateVsphereInstanceMap(cfg *VSphereConfig) (map[string]*VSphereInstance // Check if the vsphere.conf is in old format. In this // format the cfg.VirtualCenter will be nil or empty. if cfg.VirtualCenter == nil || len(cfg.VirtualCenter) == 0 { - glog.V(4).Infof("Config is not per virtual center and is in old format.") + klog.V(4).Infof("Config is not per virtual center and is in old format.") if !isSecretInfoProvided { if cfg.Global.User == "" { - glog.Error("Global.User is empty!") + klog.Error("Global.User is empty!") return nil, ErrUsernameMissing } if cfg.Global.Password == "" { - glog.Error("Global.Password is empty!") + klog.Error("Global.Password is empty!") return nil, ErrPasswordMissing } } if cfg.Global.WorkingDir == "" { - glog.Error("Global.WorkingDir is empty!") + klog.Error("Global.WorkingDir is empty!") return nil, errors.New("Global.WorkingDir is empty!") } if cfg.Global.VCenterIP == "" { - glog.Error("Global.VCenterIP is empty!") + klog.Error("Global.VCenterIP is empty!") return nil, errors.New("Global.VCenterIP is empty!") } if cfg.Global.Datacenter == "" { - glog.Error("Global.Datacenter is empty!") + klog.Error("Global.Datacenter is empty!") return nil, errors.New("Global.Datacenter is empty!") } cfg.Workspace.VCenterIP = cfg.Global.VCenterIP @@ -375,14 +375,14 @@ func populateVsphereInstanceMap(cfg *VSphereConfig) (map[string]*VSphereInstance if cfg.Workspace.VCenterIP == "" || cfg.Workspace.Folder == "" || cfg.Workspace.Datacenter == "" { msg := fmt.Sprintf("All fields in workspace are mandatory."+ " vsphere.conf does not have the workspace specified correctly. cfg.Workspace: %+v", cfg.Workspace) - glog.Error(msg) + klog.Error(msg) return nil, errors.New(msg) } for vcServer, vcConfig := range cfg.VirtualCenter { - glog.V(4).Infof("Initializing vc server %s", vcServer) + klog.V(4).Infof("Initializing vc server %s", vcServer) if vcServer == "" { - glog.Error("vsphere.conf does not have the VirtualCenter IP address specified") + klog.Error("vsphere.conf does not have the VirtualCenter IP address specified") return nil, errors.New("vsphere.conf does not have the VirtualCenter IP address specified") } @@ -390,24 +390,24 @@ func populateVsphereInstanceMap(cfg *VSphereConfig) (map[string]*VSphereInstance if vcConfig.User == "" { vcConfig.User = cfg.Global.User if vcConfig.User == "" { - glog.Errorf("vcConfig.User is empty for vc %s!", vcServer) + klog.Errorf("vcConfig.User is empty for vc %s!", vcServer) return nil, ErrUsernameMissing } } if vcConfig.Password == "" { vcConfig.Password = cfg.Global.Password if vcConfig.Password == "" { - glog.Errorf("vcConfig.Password is empty for vc %s!", vcServer) + klog.Errorf("vcConfig.Password is empty for vc %s!", vcServer) return nil, ErrPasswordMissing } } } else { if vcConfig.User != "" { - glog.Warningf("vcConfig.User for server %s and Secret info provided. VCP will use secret to get credentials", vcServer) + klog.Warningf("vcConfig.User for server %s and Secret info provided. VCP will use secret to get credentials", vcServer) vcConfig.User = "" } if vcConfig.Password != "" { - glog.Warningf("vcConfig.Password for server %s and Secret info provided. VCP will use secret to get credentials", vcServer) + klog.Warningf("vcConfig.Password for server %s and Secret info provided. VCP will use secret to get credentials", vcServer) vcConfig.Password = "" } } @@ -461,7 +461,7 @@ func newControllerNode(cfg VSphereConfig) (*VSphere, error) { } vs.hostName, err = os.Hostname() if err != nil { - glog.Errorf("Failed to get hostname. err: %+v", err) + klog.Errorf("Failed to get hostname. err: %+v", err) return nil, err } if cfg.Global.VMUUID != "" { @@ -469,7 +469,7 @@ func newControllerNode(cfg VSphereConfig) (*VSphere, error) { } else { vs.vmUUID, err = getVMUUID() if err != nil { - glog.Errorf("Failed to get uuid. err: %+v", err) + klog.Errorf("Failed to get uuid. err: %+v", err) return nil, err } } @@ -487,7 +487,7 @@ func buildVSphereFromConfig(cfg VSphereConfig) (*VSphere, error) { if cfg.Disk.SCSIControllerType == "" { cfg.Disk.SCSIControllerType = vclib.PVSCSIControllerType } else if !vclib.CheckControllerSupported(cfg.Disk.SCSIControllerType) { - glog.Errorf("%v is not a supported SCSI Controller type. Please configure 'lsilogic-sas' OR 'pvscsi'", cfg.Disk.SCSIControllerType) + klog.Errorf("%v is not a supported SCSI Controller type. Please configure 'lsilogic-sas' OR 'pvscsi'", cfg.Disk.SCSIControllerType) return nil, errors.New("Controller type not supported. Please configure 'lsilogic-sas' OR 'pvscsi'") } if cfg.Global.WorkingDir != "" { @@ -532,13 +532,13 @@ func getLocalIP() ([]v1.NodeAddress, error) { addrs := []v1.NodeAddress{} ifaces, err := net.Interfaces() if err != nil { - glog.Errorf("net.Interfaces() failed for NodeAddresses - %v", err) + klog.Errorf("net.Interfaces() failed for NodeAddresses - %v", err) return nil, err } for _, i := range ifaces { localAddrs, err := i.Addrs() if err != nil { - glog.Warningf("Failed to extract addresses for NodeAddresses - %v", err) + klog.Warningf("Failed to extract addresses for NodeAddresses - %v", err) } else { for _, addr := range localAddrs { if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { @@ -558,7 +558,7 @@ func getLocalIP() ([]v1.NodeAddress, error) { }, ) } - glog.V(4).Infof("Find local IP address %v and set type to %v", ipnet.IP.String(), addressType) + klog.V(4).Infof("Find local IP address %v and set type to %v", ipnet.IP.String(), addressType) } } } @@ -570,7 +570,7 @@ func getLocalIP() ([]v1.NodeAddress, error) { func (vs *VSphere) getVSphereInstance(nodeName k8stypes.NodeName) (*VSphereInstance, error) { vsphereIns, err := vs.nodeManager.GetVSphereInstance(nodeName) if err != nil { - glog.Errorf("Cannot find node %q in cache. Node not found!!!", nodeName) + klog.Errorf("Cannot find node %q in cache. Node not found!!!", nodeName) return nil, err } return &vsphereIns, nil @@ -579,13 +579,13 @@ func (vs *VSphere) getVSphereInstance(nodeName k8stypes.NodeName) (*VSphereInsta func (vs *VSphere) getVSphereInstanceForServer(vcServer string, ctx context.Context) (*VSphereInstance, error) { vsphereIns, ok := vs.vsphereInstanceMap[vcServer] if !ok { - glog.Errorf("cannot find vcServer %q in cache. VC not found!!!", vcServer) + klog.Errorf("cannot find vcServer %q in cache. VC not found!!!", vcServer) return nil, errors.New(fmt.Sprintf("Cannot find node %q in vsphere configuration map", vcServer)) } // Ensure client is logged in and session is valid err := vs.nodeManager.vcConnect(ctx, vsphereIns) if err != nil { - glog.Errorf("failed connecting to vcServer %q with error %+v", vcServer, err) + klog.Errorf("failed connecting to vcServer %q with error %+v", vcServer, err) return nil, err } @@ -635,12 +635,12 @@ func (vs *VSphere) NodeAddresses(ctx context.Context, nodeName k8stypes.NodeName vm, err := vs.getVMFromNodeName(ctx, nodeName) if err != nil { - glog.Errorf("Failed to get VM object for node: %q. err: +%v", convertToString(nodeName), err) + klog.Errorf("Failed to get VM object for node: %q. err: +%v", convertToString(nodeName), err) return nil, err } vmMoList, err := vm.Datacenter.GetVMMoList(ctx, []*vclib.VirtualMachine{vm}, []string{"guest.net"}) if err != nil { - glog.Errorf("Failed to get VM Managed object with property guest.net for node: %q. err: +%v", convertToString(nodeName), err) + klog.Errorf("Failed to get VM Managed object with property guest.net for node: %q. err: +%v", convertToString(nodeName), err) return nil, err } // retrieve VM's ip(s) @@ -694,7 +694,7 @@ func convertToK8sType(vmName string) k8stypes.NodeName { func (vs *VSphere) InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error) { nodeName, err := vs.GetNodeNameFromProviderID(providerID) if err != nil { - glog.Errorf("Error while getting nodename for providerID %s", providerID) + klog.Errorf("Error while getting nodename for providerID %s", providerID) return false, err } _, err = vs.InstanceID(ctx, convertToK8sType(nodeName)) @@ -709,7 +709,7 @@ func (vs *VSphere) InstanceExistsByProviderID(ctx context.Context, providerID st func (vs *VSphere) InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error) { nodeName, err := vs.GetNodeNameFromProviderID(providerID) if err != nil { - glog.Errorf("Error while getting nodename for providerID %s", providerID) + klog.Errorf("Error while getting nodename for providerID %s", providerID) return false, err } @@ -723,12 +723,12 @@ func (vs *VSphere) InstanceShutdownByProviderID(ctx context.Context, providerID } vm, err := vs.getVMFromNodeName(ctx, convertToK8sType(nodeName)) if err != nil { - glog.Errorf("Failed to get VM object for node: %q. err: +%v", nodeName, err) + klog.Errorf("Failed to get VM object for node: %q. err: +%v", nodeName, err) return false, err } isActive, err := vm.IsActive(ctx) if err != nil { - glog.Errorf("Failed to check whether node %q is active. err: %+v.", nodeName, err) + klog.Errorf("Failed to check whether node %q is active. err: %+v.", nodeName, err) return false, err } return !isActive, nil @@ -761,18 +761,18 @@ func (vs *VSphere) InstanceID(ctx context.Context, nodeName k8stypes.NodeName) ( } vm, err := vs.getVMFromNodeName(ctx, nodeName) if err != nil { - glog.Errorf("Failed to get VM object for node: %q. err: +%v", convertToString(nodeName), err) + klog.Errorf("Failed to get VM object for node: %q. err: +%v", convertToString(nodeName), err) return "", err } isActive, err := vm.IsActive(ctx) if err != nil { - glog.Errorf("Failed to check whether node %q is active. err: %+v.", convertToString(nodeName), err) + klog.Errorf("Failed to check whether node %q is active. err: %+v.", convertToString(nodeName), err) return "", err } if isActive { return vs.vmUUID, nil } - glog.Warningf("The VM: %s is not in %s state", convertToString(nodeName), vclib.ActivePowerState) + klog.Warningf("The VM: %s is not in %s state", convertToString(nodeName), vclib.ActivePowerState) return "", cloudprovider.InstanceNotFound } @@ -781,7 +781,7 @@ func (vs *VSphere) InstanceID(ctx context.Context, nodeName k8stypes.NodeName) ( if vclib.IsManagedObjectNotFoundError(err) { err = vs.nodeManager.RediscoverNode(nodeName) if err == nil { - glog.V(4).Infof("InstanceID: Found node %q", convertToString(nodeName)) + klog.V(4).Infof("InstanceID: Found node %q", convertToString(nodeName)) instanceID, err = instanceIDInternal() } else if err == vclib.ErrNoVMFound { return "", cloudprovider.InstanceNotFound @@ -820,7 +820,7 @@ func (vs *VSphere) LoadBalancer() (cloudprovider.LoadBalancer, bool) { // Zones returns an implementation of Zones for vSphere. func (vs *VSphere) Zones() (cloudprovider.Zones, bool) { if vs.cfg == nil { - glog.V(1).Info("The vSphere cloud provider does not support zones") + klog.V(1).Info("The vSphere cloud provider does not support zones") return nil, false } return vs, true @@ -852,13 +852,13 @@ func (vs *VSphere) AttachDisk(vmDiskPath string, storagePolicyName string, nodeN vm, err := vs.getVMFromNodeName(ctx, nodeName) if err != nil { - glog.Errorf("Failed to get VM object for node: %q. err: +%v", convertToString(nodeName), err) + klog.Errorf("Failed to get VM object for node: %q. err: +%v", convertToString(nodeName), err) return "", err } diskUUID, err = vm.AttachDisk(ctx, vmDiskPath, &vclib.VolumeOptions{SCSIControllerType: vclib.PVSCSIControllerType, StoragePolicyName: storagePolicyName}) if err != nil { - glog.Errorf("Failed to attach disk: %s for node: %s. err: +%v", vmDiskPath, convertToString(nodeName), err) + klog.Errorf("Failed to attach disk: %s for node: %s. err: +%v", vmDiskPath, convertToString(nodeName), err) return "", err } return diskUUID, nil @@ -869,13 +869,13 @@ func (vs *VSphere) AttachDisk(vmDiskPath string, storagePolicyName string, nodeN if vclib.IsManagedObjectNotFoundError(err) { err = vs.nodeManager.RediscoverNode(nodeName) if err == nil { - glog.V(4).Infof("AttachDisk: Found node %q", convertToString(nodeName)) + klog.V(4).Infof("AttachDisk: Found node %q", convertToString(nodeName)) diskUUID, err = attachDiskInternal(vmDiskPath, storagePolicyName, nodeName) - glog.V(4).Infof("AttachDisk: Retry: diskUUID %s, err +%v", diskUUID, err) + klog.V(4).Infof("AttachDisk: Retry: diskUUID %s, err +%v", diskUUID, err) } } } - glog.V(4).Infof("AttachDisk executed for node %s and volume %s with diskUUID %s. Err: %s", convertToString(nodeName), vmDiskPath, diskUUID, err) + klog.V(4).Infof("AttachDisk executed for node %s and volume %s with diskUUID %s. Err: %s", convertToString(nodeName), vmDiskPath, diskUUID, err) vclib.RecordvSphereMetric(vclib.OperationAttachVolume, requestTime, err) return diskUUID, err } @@ -893,7 +893,7 @@ func (vs *VSphere) DetachDisk(volPath string, nodeName k8stypes.NodeName) error if err != nil { // If node doesn't exist, disk is already detached from node. if err == vclib.ErrNoVMFound { - glog.Infof("Node %q does not exist, disk %s is already detached from node.", convertToString(nodeName), volPath) + klog.Infof("Node %q does not exist, disk %s is already detached from node.", convertToString(nodeName), volPath) return nil } return err @@ -907,16 +907,16 @@ func (vs *VSphere) DetachDisk(volPath string, nodeName k8stypes.NodeName) error if err != nil { // If node doesn't exist, disk is already detached from node. if err == vclib.ErrNoVMFound { - glog.Infof("Node %q does not exist, disk %s is already detached from node.", convertToString(nodeName), volPath) + klog.Infof("Node %q does not exist, disk %s is already detached from node.", convertToString(nodeName), volPath) return nil } - glog.Errorf("Failed to get VM object for node: %q. err: +%v", convertToString(nodeName), err) + klog.Errorf("Failed to get VM object for node: %q. err: +%v", convertToString(nodeName), err) return err } err = vm.DetachDisk(ctx, volPath) if err != nil { - glog.Errorf("Failed to detach disk: %s for node: %s. err: +%v", volPath, convertToString(nodeName), err) + klog.Errorf("Failed to detach disk: %s for node: %s. err: +%v", volPath, convertToString(nodeName), err) return err } return nil @@ -960,22 +960,22 @@ func (vs *VSphere) DiskIsAttached(volPath string, nodeName k8stypes.NodeName) (b vm, err := vs.getVMFromNodeName(ctx, nodeName) if err != nil { if err == vclib.ErrNoVMFound { - glog.Warningf("Node %q does not exist, vsphere CP will assume disk %v is not attached to it.", nodeName, volPath) + klog.Warningf("Node %q does not exist, vsphere CP will assume disk %v is not attached to it.", nodeName, volPath) // make the disk as detached and return false without error. return false, nil } - glog.Errorf("Failed to get VM object for node: %q. err: +%v", vSphereInstance, err) + klog.Errorf("Failed to get VM object for node: %q. err: +%v", vSphereInstance, err) return false, err } volPath = vclib.RemoveStorageClusterORFolderNameFromVDiskPath(volPath) attached, err := vm.IsDiskAttached(ctx, volPath) if err != nil { - glog.Errorf("DiskIsAttached failed to determine whether disk %q is still attached on node %q", + klog.Errorf("DiskIsAttached failed to determine whether disk %q is still attached on node %q", volPath, vSphereInstance) } - glog.V(4).Infof("DiskIsAttached result: %v and error: %q, for volume: %q", attached, err, volPath) + klog.V(4).Infof("DiskIsAttached result: %v and error: %q, for volume: %q", attached, err, volPath) return attached, err } requestTime := time.Now() @@ -1024,7 +1024,7 @@ func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string) for nodeName := range nodeVolumes { nodeInfo, err := vs.nodeManager.GetNodeInfo(nodeName) if err != nil { - glog.Errorf("Failed to get node info: %+v. err: %+v", nodeInfo.vm, err) + klog.Errorf("Failed to get node info: %+v. err: %+v", nodeInfo.vm, err) return nodesToRetry, err } VC_DC := nodeInfo.vcServer + nodeInfo.dataCenter.String() @@ -1042,7 +1042,7 @@ func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string) globalErrMutex.Lock() globalErr = err globalErrMutex.Unlock() - glog.Errorf("Failed to check disk attached for nodes: %+v. err: %+v", nodes, err) + klog.Errorf("Failed to check disk attached for nodes: %+v. err: %+v", nodes, err) } } nodesToRetryMutex.Lock() @@ -1065,7 +1065,7 @@ func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string) return nodesToRetry, nil } - glog.V(4).Infof("Starting DisksAreAttached API for vSphere with nodeVolumes: %+v", nodeVolumes) + klog.V(4).Infof("Starting DisksAreAttached API for vSphere with nodeVolumes: %+v", nodeVolumes) // Create context ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1078,7 +1078,7 @@ func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string) // Convert VolPaths into canonical form so that it can be compared with the VM device path. vmVolumes, err := vs.convertVolPathsToDevicePaths(ctx, nodeVolumes) if err != nil { - glog.Errorf("Failed to convert volPaths to devicePaths: %+v. err: %+v", nodeVolumes, err) + klog.Errorf("Failed to convert volPaths to devicePaths: %+v. err: %+v", nodeVolumes, err) return nil, err } attached := make(map[string]map[string]bool) @@ -1094,10 +1094,10 @@ func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string) err = vs.nodeManager.RediscoverNode(nodeName) if err != nil { if err == vclib.ErrNoVMFound { - glog.V(4).Infof("node %s not found. err: %+v", nodeName, err) + klog.V(4).Infof("node %s not found. err: %+v", nodeName, err) continue } - glog.Errorf("Failed to rediscover node %s. err: %+v", nodeName, err) + klog.Errorf("Failed to rediscover node %s. err: %+v", nodeName, err) return nil, err } remainingNodesVolumes[nodeName] = nodeVolumes[nodeName] @@ -1107,7 +1107,7 @@ func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string) if len(remainingNodesVolumes) != 0 { nodesToRetry, err = disksAreAttach(ctx, remainingNodesVolumes, attached, true) if err != nil || len(nodesToRetry) != 0 { - glog.Errorf("Failed to retry disksAreAttach for nodes %+v. err: %+v", remainingNodesVolumes, err) + klog.Errorf("Failed to retry disksAreAttach for nodes %+v. err: %+v", remainingNodesVolumes, err) return nil, err } } @@ -1116,7 +1116,7 @@ func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string) disksAttached[convertToK8sType(nodeName)] = volPaths } } - glog.V(4).Infof("DisksAreAttach successfully executed. result: %+v", attached) + klog.V(4).Infof("DisksAreAttach successfully executed. result: %+v", attached) return disksAttached, nil } requestTime := time.Now() @@ -1130,7 +1130,7 @@ func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string) // return value will be [DatastoreCluster/sharedVmfs-0] kubevols/.vmdk // else return value will be [sharedVmfs-0] kubevols/.vmdk func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVolumePath string, err error) { - glog.V(1).Infof("Starting to create a vSphere volume with volumeOptions: %+v", volumeOptions) + klog.V(1).Infof("Starting to create a vSphere volume with volumeOptions: %+v", volumeOptions) createVolumeInternal := func(volumeOptions *vclib.VolumeOptions) (canonicalVolumePath string, err error) { var datastore string // If datastore not specified, then use default datastore @@ -1160,21 +1160,21 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo // This routine will get executed for every 5 minutes and gets initiated only once in its entire lifetime. cleanUpRoutineInitLock.Lock() if !cleanUpRoutineInitialized { - glog.V(1).Infof("Starting a clean up routine to remove stale dummy VM's") + klog.V(1).Infof("Starting a clean up routine to remove stale dummy VM's") go vs.cleanUpDummyVMs(DummyVMPrefixName) cleanUpRoutineInitialized = true } cleanUpRoutineInitLock.Unlock() vmOptions, err = vs.setVMOptions(ctx, dc, vs.cfg.Workspace.ResourcePoolPath) if err != nil { - glog.Errorf("Failed to set VM options requires to create a vsphere volume. err: %+v", err) + klog.Errorf("Failed to set VM options requires to create a vsphere volume. err: %+v", err) return "", err } } if volumeOptions.StoragePolicyName != "" && volumeOptions.Datastore == "" { datastore, err = getPbmCompatibleDatastore(ctx, dc, volumeOptions.StoragePolicyName, vs.nodeManager) if err != nil { - glog.Errorf("Failed to get pbm compatible datastore with storagePolicy: %s. err: %+v", volumeOptions.StoragePolicyName, err) + klog.Errorf("Failed to get pbm compatible datastore with storagePolicy: %s. err: %+v", volumeOptions.StoragePolicyName, err) return "", err } } else { @@ -1182,7 +1182,7 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo // if the given datastore is a shared datastore across all node VMs. sharedDsList, err := getSharedDatastoresInK8SCluster(ctx, dc, vs.nodeManager) if err != nil { - glog.Errorf("Failed to get shared datastore: %+v", err) + klog.Errorf("Failed to get shared datastore: %+v", err) return "", err } found := false @@ -1205,7 +1205,7 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo kubeVolsPath := filepath.Clean(ds.Path(VolDir)) + "/" err = ds.CreateDirectory(ctx, kubeVolsPath, false) if err != nil && err != vclib.ErrFileAlreadyExist { - glog.Errorf("Cannot create dir %#v. err %s", kubeVolsPath, err) + klog.Errorf("Cannot create dir %#v. err %s", kubeVolsPath, err) return "", err } volumePath := kubeVolsPath + volumeOptions.Name + ".vmdk" @@ -1216,13 +1216,13 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo } volumePath, err = disk.Create(ctx, ds) if err != nil { - glog.Errorf("Failed to create a vsphere volume with volumeOptions: %+v on datastore: %s. err: %+v", volumeOptions, datastore, err) + klog.Errorf("Failed to create a vsphere volume with volumeOptions: %+v on datastore: %s. err: %+v", volumeOptions, datastore, err) return "", err } // Get the canonical path for the volume path. canonicalVolumePath, err = getcanonicalVolumePath(ctx, dc, volumePath) if err != nil { - glog.Errorf("Failed to get canonical vsphere volume path for volume: %s with volumeOptions: %+v on datastore: %s. err: %+v", volumePath, volumeOptions, datastore, err) + klog.Errorf("Failed to get canonical vsphere volume path for volume: %s with volumeOptions: %+v on datastore: %s. err: %+v", volumePath, volumeOptions, datastore, err) return "", err } if filepath.Base(datastore) != datastore { @@ -1234,13 +1234,13 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo requestTime := time.Now() canonicalVolumePath, err = createVolumeInternal(volumeOptions) vclib.RecordCreateVolumeMetric(volumeOptions, requestTime, err) - glog.V(4).Infof("The canonical volume path for the newly created vSphere volume is %q", canonicalVolumePath) + klog.V(4).Infof("The canonical volume path for the newly created vSphere volume is %q", canonicalVolumePath) return canonicalVolumePath, err } // DeleteVolume deletes a volume given volume name. func (vs *VSphere) DeleteVolume(vmDiskPath string) error { - glog.V(1).Infof("Starting to delete vSphere volume with vmDiskPath: %s", vmDiskPath) + klog.V(1).Infof("Starting to delete vSphere volume with vmDiskPath: %s", vmDiskPath) deleteVolumeInternal := func(vmDiskPath string) error { // Create context ctx, cancel := context.WithCancel(context.Background()) @@ -1260,7 +1260,7 @@ func (vs *VSphere) DeleteVolume(vmDiskPath string) error { } err = disk.Delete(ctx, dc) if err != nil { - glog.Errorf("Failed to delete vsphere volume with vmDiskPath: %s. err: %+v", vmDiskPath, err) + klog.Errorf("Failed to delete vsphere volume with vmDiskPath: %s. err: %+v", vmDiskPath, err) } return err } @@ -1279,11 +1279,11 @@ func (vs *VSphere) HasClusterID() bool { func (vs *VSphere) NodeAdded(obj interface{}) { node, ok := obj.(*v1.Node) if node == nil || !ok { - glog.Warningf("NodeAdded: unrecognized object %+v", obj) + klog.Warningf("NodeAdded: unrecognized object %+v", obj) return } - glog.V(4).Infof("Node added: %+v", node) + klog.V(4).Infof("Node added: %+v", node) vs.nodeManager.RegisterNode(node) } @@ -1291,11 +1291,11 @@ func (vs *VSphere) NodeAdded(obj interface{}) { func (vs *VSphere) NodeDeleted(obj interface{}) { node, ok := obj.(*v1.Node) if node == nil || !ok { - glog.Warningf("NodeDeleted: unrecognized object %+v", obj) + klog.Warningf("NodeDeleted: unrecognized object %+v", obj) return } - glog.V(4).Infof("Node deleted: %+v", node) + klog.V(4).Infof("Node deleted: %+v", node) vs.nodeManager.UnRegisterNode(node) } @@ -1320,23 +1320,23 @@ func withTagsClient(ctx context.Context, connection *vclib.VSphereConnection, f func (vs *VSphere) GetZone(ctx context.Context) (cloudprovider.Zone, error) { nodeName, err := vs.CurrentNodeName(ctx, vs.hostName) if err != nil { - glog.Errorf("Cannot get node name.") + klog.Errorf("Cannot get node name.") return cloudprovider.Zone{}, err } zone := cloudprovider.Zone{} vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx) if err != nil { - glog.Errorf("Cannot connent to vsphere. Get zone for node %s error", nodeName) + klog.Errorf("Cannot connent to vsphere. Get zone for node %s error", nodeName) return cloudprovider.Zone{}, err } dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter) if err != nil { - glog.Errorf("Cannot connent to datacenter. Get zone for node %s error", nodeName) + klog.Errorf("Cannot connent to datacenter. Get zone for node %s error", nodeName) return cloudprovider.Zone{}, err } vmHost, err := dc.GetHostByVMUUID(ctx, vs.vmUUID) if err != nil { - glog.Errorf("Cannot find VM runtime host. Get zone for node %s error", nodeName) + klog.Errorf("Cannot find VM runtime host. Get zone for node %s error", nodeName) return cloudprovider.Zone{}, err } @@ -1354,23 +1354,23 @@ func (vs *VSphere) GetZone(ctx context.Context) (cloudprovider.Zone, error) { obj := objects[len(objects)-1-i] tags, err := client.ListAttachedTags(ctx, obj) if err != nil { - glog.Errorf("Cannot list attached tags. Get zone for node %s: %s", nodeName, err) + klog.Errorf("Cannot list attached tags. Get zone for node %s: %s", nodeName, err) return err } for _, value := range tags { tag, err := client.GetTag(ctx, value) if err != nil { - glog.Errorf("Get tag %s: %s", value, err) + klog.Errorf("Get tag %s: %s", value, err) return err } category, err := client.GetCategory(ctx, tag.CategoryID) if err != nil { - glog.Errorf("Get category %s error", value) + klog.Errorf("Get category %s error", value) return err } found := func() { - glog.Errorf("Found %q tag (%s) for %s attached to %s", category.Name, tag.Name, vs.vmUUID, obj.Reference()) + klog.Errorf("Found %q tag (%s) for %s attached to %s", category.Name, tag.Name, vs.vmUUID, obj.Reference()) } switch { case category.Name == vs.cfg.Labels.Zone: @@ -1401,7 +1401,7 @@ func (vs *VSphere) GetZone(ctx context.Context) (cloudprovider.Zone, error) { return nil }) if err != nil { - glog.Errorf("Get zone for node %s: %s", nodeName, err) + klog.Errorf("Get zone for node %s: %s", nodeName, err) return cloudprovider.Zone{}, err } return zone, nil diff --git a/pkg/cloudprovider/providers/vsphere/vsphere_util.go b/pkg/cloudprovider/providers/vsphere/vsphere_util.go index 8ef73978a29..04f241a89d3 100644 --- a/pkg/cloudprovider/providers/vsphere/vsphere_util.go +++ b/pkg/cloudprovider/providers/vsphere/vsphere_util.go @@ -27,9 +27,9 @@ import ( "strings" "time" - "github.com/golang/glog" "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/mo" + "k8s.io/klog" "k8s.io/api/core/v1" k8stypes "k8s.io/apimachinery/pkg/types" @@ -87,27 +87,27 @@ func getAccessibleDatastores(ctx context.Context, nodeVmDetail *NodeDetails, nod // Check if the node VM is not found which indicates that the node info in the node manager is stale. // If so, rediscover the node and retry. if vclib.IsManagedObjectNotFoundError(err) { - glog.V(4).Infof("error %q ManagedObjectNotFound for node %q. Rediscovering...", err, nodeVmDetail.NodeName) + klog.V(4).Infof("error %q ManagedObjectNotFound for node %q. Rediscovering...", err, nodeVmDetail.NodeName) err = nodeManager.RediscoverNode(convertToK8sType(nodeVmDetail.NodeName)) if err == nil { - glog.V(4).Infof("Discovered node %s successfully", nodeVmDetail.NodeName) + klog.V(4).Infof("Discovered node %s successfully", nodeVmDetail.NodeName) nodeInfo, err := nodeManager.GetNodeInfo(convertToK8sType(nodeVmDetail.NodeName)) if err != nil { - glog.V(4).Infof("error %q getting node info for node %+v", err, nodeVmDetail) + klog.V(4).Infof("error %q getting node info for node %+v", err, nodeVmDetail) return nil, err } accessibleDatastores, err = nodeInfo.vm.GetAllAccessibleDatastores(ctx) if err != nil { - glog.V(4).Infof("error %q getting accessible datastores for node %+v", err, nodeVmDetail) + klog.V(4).Infof("error %q getting accessible datastores for node %+v", err, nodeVmDetail) return nil, err } } else { - glog.V(4).Infof("error %q rediscovering node %+v", err, nodeVmDetail) + klog.V(4).Infof("error %q rediscovering node %+v", err, nodeVmDetail) return nil, err } } else { - glog.V(4).Infof("error %q getting accessible datastores for node %+v", err, nodeVmDetail) + klog.V(4).Infof("error %q getting accessible datastores for node %+v", err, nodeVmDetail) return nil, err } } @@ -118,22 +118,22 @@ func getAccessibleDatastores(ctx context.Context, nodeVmDetail *NodeDetails, nod func getSharedDatastoresInK8SCluster(ctx context.Context, dc *vclib.Datacenter, nodeManager *NodeManager) ([]*vclib.DatastoreInfo, error) { nodeVmDetails, err := nodeManager.GetNodeDetails() if err != nil { - glog.Errorf("Error while obtaining Kubernetes node nodeVmDetail details. error : %+v", err) + klog.Errorf("Error while obtaining Kubernetes node nodeVmDetail details. error : %+v", err) return nil, err } if len(nodeVmDetails) == 0 { msg := fmt.Sprintf("Kubernetes node nodeVmDetail details is empty. nodeVmDetails : %+v", nodeVmDetails) - glog.Error(msg) + klog.Error(msg) return nil, fmt.Errorf(msg) } var sharedDatastores []*vclib.DatastoreInfo for _, nodeVmDetail := range nodeVmDetails { - glog.V(9).Infof("Getting accessible datastores for node %s", nodeVmDetail.NodeName) + klog.V(9).Infof("Getting accessible datastores for node %s", nodeVmDetail.NodeName) accessibleDatastores, err := getAccessibleDatastores(ctx, &nodeVmDetail, nodeManager) if err != nil { if err == vclib.ErrNoVMFound { - glog.V(9).Infof("Got NoVMFound error for node %s", nodeVmDetail.NodeName) + klog.V(9).Infof("Got NoVMFound error for node %s", nodeVmDetail.NodeName) continue } return nil, err @@ -148,19 +148,19 @@ func getSharedDatastoresInK8SCluster(ctx context.Context, dc *vclib.Datacenter, } } } - glog.V(9).Infof("sharedDatastores : %+v", sharedDatastores) + klog.V(9).Infof("sharedDatastores : %+v", sharedDatastores) sharedDatastores, err = getDatastoresForEndpointVC(ctx, dc, sharedDatastores) if err != nil { - glog.Errorf("Failed to get shared datastores from endpoint VC. err: %+v", err) + klog.Errorf("Failed to get shared datastores from endpoint VC. err: %+v", err) return nil, err } - glog.V(9).Infof("sharedDatastores at endpoint VC: %+v", sharedDatastores) + klog.V(9).Infof("sharedDatastores at endpoint VC: %+v", sharedDatastores) return sharedDatastores, nil } func intersect(list1 []*vclib.DatastoreInfo, list2 []*vclib.DatastoreInfo) []*vclib.DatastoreInfo { - glog.V(9).Infof("list1: %+v", list1) - glog.V(9).Infof("list2: %+v", list2) + klog.V(9).Infof("list1: %+v", list1) + klog.V(9).Infof("list2: %+v", list2) var sharedDs []*vclib.DatastoreInfo for _, val1 := range list1 { // Check if val1 is found in list2 @@ -202,10 +202,10 @@ func getDatastoresForEndpointVC(ctx context.Context, dc *vclib.Datacenter, share if ok { datastores = append(datastores, dsInfo) } else { - glog.V(4).Infof("Warning: Shared datastore with URL %s does not exist in endpoint VC", sharedDsInfo.Info.Url) + klog.V(4).Infof("Warning: Shared datastore with URL %s does not exist in endpoint VC", sharedDsInfo.Info.Url) } } - glog.V(9).Infof("Datastore from endpoint VC: %+v", datastores) + klog.V(9).Infof("Datastore from endpoint VC: %+v", datastores) return datastores, nil } @@ -216,32 +216,32 @@ func getPbmCompatibleDatastore(ctx context.Context, dc *vclib.Datacenter, storag } storagePolicyID, err := pbmClient.ProfileIDByName(ctx, storagePolicyName) if err != nil { - glog.Errorf("Failed to get Profile ID by name: %s. err: %+v", storagePolicyName, err) + klog.Errorf("Failed to get Profile ID by name: %s. err: %+v", storagePolicyName, err) return "", err } sharedDs, err := getSharedDatastoresInK8SCluster(ctx, dc, nodeManager) if err != nil { - glog.Errorf("Failed to get shared datastores. err: %+v", err) + klog.Errorf("Failed to get shared datastores. err: %+v", err) return "", err } if len(sharedDs) == 0 { msg := "No shared datastores found in the endpoint virtual center" - glog.Errorf(msg) + klog.Errorf(msg) return "", errors.New(msg) } compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, dc, storagePolicyID, sharedDs) if err != nil { - glog.Errorf("Failed to get compatible datastores from datastores : %+v with storagePolicy: %s. err: %+v", + klog.Errorf("Failed to get compatible datastores from datastores : %+v with storagePolicy: %s. err: %+v", sharedDs, storagePolicyID, err) return "", err } - glog.V(9).Infof("compatibleDatastores : %+v", compatibleDatastores) + klog.V(9).Infof("compatibleDatastores : %+v", compatibleDatastores) datastore, err := getMostFreeDatastoreName(ctx, dc.Client(), compatibleDatastores) if err != nil { - glog.Errorf("Failed to get most free datastore from compatible datastores: %+v. err: %+v", compatibleDatastores, err) + klog.Errorf("Failed to get most free datastore from compatible datastores: %+v. err: %+v", compatibleDatastores, err) return "", err } - glog.V(4).Infof("Most free datastore : %+s", datastore) + klog.V(4).Infof("Most free datastore : %+s", datastore) return datastore, err } @@ -251,7 +251,7 @@ func (vs *VSphere) setVMOptions(ctx context.Context, dc *vclib.Datacenter, resou if err != nil { return nil, err } - glog.V(9).Infof("Resource pool path %s, resourcePool %+v", resourcePoolPath, resourcePool) + klog.V(9).Infof("Resource pool path %s, resourcePool %+v", resourcePoolPath, resourcePool) folder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder) if err != nil { return nil, err @@ -270,18 +270,18 @@ func (vs *VSphere) cleanUpDummyVMs(dummyVMPrefix string) { time.Sleep(CleanUpDummyVMRoutineInterval * time.Minute) vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx) if err != nil { - glog.V(4).Infof("Failed to get VSphere instance with err: %+v. Retrying again...", err) + klog.V(4).Infof("Failed to get VSphere instance with err: %+v. Retrying again...", err) continue } dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter) if err != nil { - glog.V(4).Infof("Failed to get the datacenter: %s from VC. err: %+v", vs.cfg.Workspace.Datacenter, err) + klog.V(4).Infof("Failed to get the datacenter: %s from VC. err: %+v", vs.cfg.Workspace.Datacenter, err) continue } // Get the folder reference for global working directory where the dummy VM needs to be created. vmFolder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder) if err != nil { - glog.V(4).Infof("Unable to get the kubernetes folder: %q reference. err: %+v", vs.cfg.Workspace.Folder, err) + klog.V(4).Infof("Unable to get the kubernetes folder: %q reference. err: %+v", vs.cfg.Workspace.Folder, err) continue } // A write lock is acquired to make sure the cleanUp routine doesn't delete any VM's created by ongoing PVC requests. @@ -290,7 +290,7 @@ func (vs *VSphere) cleanUpDummyVMs(dummyVMPrefix string) { defer cleanUpDummyVMLock.Unlock() err = diskmanagers.CleanUpDummyVMs(ctx, vmFolder, dc) if err != nil { - glog.V(4).Infof("Unable to clean up dummy VM's in the kubernetes cluster: %q. err: %+v", vs.cfg.Workspace.Folder, err) + klog.V(4).Infof("Unable to clean up dummy VM's in the kubernetes cluster: %q. err: %+v", vs.cfg.Workspace.Folder, err) } } cleanUpDummyVMs() @@ -360,7 +360,7 @@ func convertVolPathToDevicePath(ctx context.Context, dc *vclib.Datacenter, volPa // Get the canonical volume path for volPath. canonicalVolumePath, err := getcanonicalVolumePath(ctx, dc, volPath) if err != nil { - glog.Errorf("Failed to get canonical vsphere volume path for volume: %s. err: %+v", volPath, err) + klog.Errorf("Failed to get canonical vsphere volume path for volume: %s. err: %+v", volPath, err) return "", err } // Check if the volume path contains .vmdk extension. If not, add the extension and update the nodeVolumes Map @@ -387,7 +387,7 @@ func (vs *VSphere) convertVolPathsToDevicePaths(ctx context.Context, nodeVolumes for i, volPath := range volPaths { deviceVolPath, err := convertVolPathToDevicePath(ctx, nodeInfo.dataCenter, volPath) if err != nil { - glog.Errorf("Failed to convert vsphere volume path %s to device path for volume %s. err: %+v", volPath, deviceVolPath, err) + klog.Errorf("Failed to convert vsphere volume path %s to device path for volume %s. err: %+v", volPath, deviceVolPath, err) return nil, err } volPaths[i] = deviceVolPath @@ -423,7 +423,7 @@ func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeN vmMoList, err := nodeInfo.dataCenter.GetVMMoList(ctx, vmList, []string{"config.hardware.device", "name", "config.uuid"}) if err != nil { if vclib.IsManagedObjectNotFoundError(err) && !retry { - glog.V(4).Infof("checkDiskAttached: ManagedObjectNotFound for property collector query for nodes: %+v vms: %+v", nodes, vmList) + klog.V(4).Infof("checkDiskAttached: ManagedObjectNotFound for property collector query for nodes: %+v vms: %+v", nodes, vmList) // Property Collector Query failed // VerifyVolumePaths per VM for _, nodeName := range nodes { @@ -434,13 +434,13 @@ func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeN devices, err := nodeInfo.vm.VirtualMachine.Device(ctx) if err != nil { if vclib.IsManagedObjectNotFoundError(err) { - glog.V(4).Infof("checkDiskAttached: ManagedObjectNotFound for Kubernetes node: %s with vSphere Virtual Machine reference: %v", nodeName, nodeInfo.vm) + klog.V(4).Infof("checkDiskAttached: ManagedObjectNotFound for Kubernetes node: %s with vSphere Virtual Machine reference: %v", nodeName, nodeInfo.vm) nodesToRetry = append(nodesToRetry, nodeName) continue } return nodesToRetry, err } - glog.V(4).Infof("Verifying Volume Paths by devices for node %s and VM %s", nodeName, nodeInfo.vm) + klog.V(4).Infof("Verifying Volume Paths by devices for node %s and VM %s", nodeName, nodeInfo.vm) vclib.VerifyVolumePathsForVMDevices(devices, nodeVolumes[nodeName], convertToString(nodeName), attached) } } @@ -450,14 +450,14 @@ func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeN vmMoMap := make(map[string]mo.VirtualMachine) for _, vmMo := range vmMoList { if vmMo.Config == nil { - glog.Errorf("Config is not available for VM: %q", vmMo.Name) + klog.Errorf("Config is not available for VM: %q", vmMo.Name) continue } - glog.V(9).Infof("vmMoMap vmname: %q vmuuid: %s", vmMo.Name, strings.ToLower(vmMo.Config.Uuid)) + klog.V(9).Infof("vmMoMap vmname: %q vmuuid: %s", vmMo.Name, strings.ToLower(vmMo.Config.Uuid)) vmMoMap[strings.ToLower(vmMo.Config.Uuid)] = vmMo } - glog.V(9).Infof("vmMoMap: +%v", vmMoMap) + klog.V(9).Infof("vmMoMap: +%v", vmMoMap) for _, nodeName := range nodes { node, err := vs.nodeManager.GetNode(nodeName) @@ -466,11 +466,11 @@ func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeN } nodeUUID, err := GetNodeUUID(&node) if err != nil { - glog.Errorf("Node Discovery failed to get node uuid for node %s with error: %v", node.Name, err) + klog.Errorf("Node Discovery failed to get node uuid for node %s with error: %v", node.Name, err) return nodesToRetry, err } nodeUUID = strings.ToLower(nodeUUID) - glog.V(9).Infof("Verifying volume for node %s with nodeuuid %q: %v", nodeName, nodeUUID, vmMoMap) + klog.V(9).Infof("Verifying volume for node %s with nodeuuid %q: %v", nodeName, nodeUUID, vmMoMap) vclib.VerifyVolumePathsForVM(vmMoMap[nodeUUID], nodeVolumes[nodeName], convertToString(nodeName), attached) } return nodesToRetry, nil @@ -517,7 +517,7 @@ func (vs *VSphere) GetNodeNameFromProviderID(providerID string) (string, error) var nodeName string nodes, err := vs.nodeManager.GetNodeDetails() if err != nil { - glog.Errorf("Error while obtaining Kubernetes node nodeVmDetail details. error : %+v", err) + klog.Errorf("Error while obtaining Kubernetes node nodeVmDetail details. error : %+v", err) return "", err } for _, node := range nodes { @@ -564,12 +564,12 @@ func GetUUIDFromProviderID(providerID string) string { func IsUUIDSupportedNode(node *v1.Node) (bool, error) { newVersion, err := version.ParseSemantic("v1.9.4") if err != nil { - glog.Errorf("Failed to determine whether node %+v is old with error %v", node, err) + klog.Errorf("Failed to determine whether node %+v is old with error %v", node, err) return false, err } nodeVersion, err := version.ParseSemantic(node.Status.NodeInfo.KubeletVersion) if err != nil { - glog.Errorf("Failed to determine whether node %+v is old with error %v", node, err) + klog.Errorf("Failed to determine whether node %+v is old with error %v", node, err) return false, err } if nodeVersion.LessThan(newVersion) { @@ -581,7 +581,7 @@ func IsUUIDSupportedNode(node *v1.Node) (bool, error) { func GetNodeUUID(node *v1.Node) (string, error) { oldNode, err := IsUUIDSupportedNode(node) if err != nil { - glog.Errorf("Failed to get node UUID for node %+v with error %v", node, err) + klog.Errorf("Failed to get node UUID for node %+v with error %v", node, err) return "", err } if oldNode { diff --git a/pkg/controller/.import-restrictions b/pkg/controller/.import-restrictions index 6dd2ce4cc4c..31aa4d0b504 100644 --- a/pkg/controller/.import-restrictions +++ b/pkg/controller/.import-restrictions @@ -104,7 +104,6 @@ "github.com/cloudflare/cfssl/signer/local", "github.com/davecgh/go-spew/spew", "github.com/evanphx/json-patch", - "github.com/golang/glog", "github.com/golang/groupcache/lru", "github.com/prometheus/client_golang/prometheus", "github.com/robfig/cron", diff --git a/pkg/controller/BUILD b/pkg/controller/BUILD index 5853bc011a2..b4915ac5365 100644 --- a/pkg/controller/BUILD +++ b/pkg/controller/BUILD @@ -88,8 +88,8 @@ go_library( "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/client-go/util/integer:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/groupcache/lru:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/bootstrap/BUILD b/pkg/controller/bootstrap/BUILD index b308efb9989..922cbde6c13 100644 --- a/pkg/controller/bootstrap/BUILD +++ b/pkg/controller/bootstrap/BUILD @@ -59,8 +59,8 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/gopkg.in/square/go-jose.v2:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/bootstrap/bootstrapsigner.go b/pkg/controller/bootstrap/bootstrapsigner.go index f816311e861..60952976eb5 100644 --- a/pkg/controller/bootstrap/bootstrapsigner.go +++ b/pkg/controller/bootstrap/bootstrapsigner.go @@ -20,7 +20,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "fmt" "k8s.io/api/core/v1" @@ -162,10 +162,10 @@ func (e *BootstrapSigner) Run(stopCh <-chan struct{}) { return } - glog.V(5).Infof("Starting workers") + klog.V(5).Infof("Starting workers") go wait.Until(e.serviceConfigMapQueue, 0, stopCh) <-stopCh - glog.V(1).Infof("Shutting down") + klog.V(1).Infof("Shutting down") } func (e *BootstrapSigner) pokeConfigMapSync() { @@ -198,7 +198,7 @@ func (e *BootstrapSigner) signConfigMap() { // First capture the config we are signing content, ok := newCM.Data[bootstrapapi.KubeConfigKey] if !ok { - glog.V(3).Infof("No %s key in %s/%s ConfigMap", bootstrapapi.KubeConfigKey, origCM.Namespace, origCM.Name) + klog.V(3).Infof("No %s key in %s/%s ConfigMap", bootstrapapi.KubeConfigKey, origCM.Namespace, origCM.Name) return } @@ -244,7 +244,7 @@ func (e *BootstrapSigner) signConfigMap() { func (e *BootstrapSigner) updateConfigMap(cm *v1.ConfigMap) { _, err := e.client.CoreV1().ConfigMaps(cm.Namespace).Update(cm) if err != nil && !apierrors.IsConflict(err) && !apierrors.IsNotFound(err) { - glog.V(3).Infof("Error updating ConfigMap: %v", err) + klog.V(3).Infof("Error updating ConfigMap: %v", err) } } @@ -295,7 +295,7 @@ func (e *BootstrapSigner) getTokens() map[string]string { if _, ok := ret[tokenID]; ok { // This should never happen as we ensure a consistent secret name. // But leave this in here just in case. - glog.V(1).Infof("Duplicate bootstrap tokens found for id %s, ignoring on in %s/%s", tokenID, secret.Namespace, secret.Name) + klog.V(1).Infof("Duplicate bootstrap tokens found for id %s, ignoring on in %s/%s", tokenID, secret.Namespace, secret.Name) continue } diff --git a/pkg/controller/bootstrap/tokencleaner.go b/pkg/controller/bootstrap/tokencleaner.go index 841a61e0d1d..adaee9b58d5 100644 --- a/pkg/controller/bootstrap/tokencleaner.go +++ b/pkg/controller/bootstrap/tokencleaner.go @@ -20,7 +20,6 @@ import ( "fmt" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,6 +31,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" bootstrapapi "k8s.io/cluster-bootstrap/token/api" + "k8s.io/klog" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/util/metrics" @@ -113,8 +113,8 @@ func (tc *TokenCleaner) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer tc.queue.ShutDown() - glog.Infof("Starting token cleaner controller") - defer glog.Infof("Shutting down token cleaner controller") + klog.Infof("Starting token cleaner controller") + defer klog.Infof("Shutting down token cleaner controller") if !controller.WaitForCacheSync("token_cleaner", stopCh, tc.secretSynced) { return @@ -161,7 +161,7 @@ func (tc *TokenCleaner) processNextWorkItem() bool { func (tc *TokenCleaner) syncFunc(key string) error { startTime := time.Now() defer func() { - glog.V(4).Infof("Finished syncing secret %q (%v)", key, time.Since(startTime)) + klog.V(4).Infof("Finished syncing secret %q (%v)", key, time.Since(startTime)) }() namespace, name, err := cache.SplitMetaNamespaceKey(key) @@ -171,7 +171,7 @@ func (tc *TokenCleaner) syncFunc(key string) error { ret, err := tc.secretLister.Secrets(namespace).Get(name) if apierrors.IsNotFound(err) { - glog.V(3).Infof("secret has been deleted: %v", key) + klog.V(3).Infof("secret has been deleted: %v", key) return nil } @@ -188,7 +188,7 @@ func (tc *TokenCleaner) syncFunc(key string) error { func (tc *TokenCleaner) evalSecret(o interface{}) { secret := o.(*v1.Secret) if isSecretExpired(secret) { - glog.V(3).Infof("Deleting expired secret %s/%s", secret.Namespace, secret.Name) + klog.V(3).Infof("Deleting expired secret %s/%s", secret.Namespace, secret.Name) var options *metav1.DeleteOptions if len(secret.UID) > 0 { options = &metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &secret.UID}} @@ -197,7 +197,7 @@ func (tc *TokenCleaner) evalSecret(o interface{}) { // NotFound isn't a real error (it's already been deleted) // Conflict isn't a real error (the UID precondition failed) if err != nil && !apierrors.IsConflict(err) && !apierrors.IsNotFound(err) { - glog.V(3).Infof("Error deleting Secret: %v", err) + klog.V(3).Infof("Error deleting Secret: %v", err) } } } diff --git a/pkg/controller/bootstrap/util.go b/pkg/controller/bootstrap/util.go index 985f005484a..44d024af41a 100644 --- a/pkg/controller/bootstrap/util.go +++ b/pkg/controller/bootstrap/util.go @@ -20,7 +20,7 @@ import ( "regexp" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" bootstrapapi "k8s.io/cluster-bootstrap/token/api" @@ -52,24 +52,24 @@ func parseSecretName(name string) (secretID string, ok bool) { func validateSecretForSigning(secret *v1.Secret) (tokenID, tokenSecret string, ok bool) { nameTokenID, ok := parseSecretName(secret.Name) if !ok { - glog.V(3).Infof("Invalid secret name: %s. Must be of form %s.", secret.Name, bootstrapapi.BootstrapTokenSecretPrefix) + klog.V(3).Infof("Invalid secret name: %s. Must be of form %s.", secret.Name, bootstrapapi.BootstrapTokenSecretPrefix) return "", "", false } tokenID = getSecretString(secret, bootstrapapi.BootstrapTokenIDKey) if len(tokenID) == 0 { - glog.V(3).Infof("No %s key in %s/%s Secret", bootstrapapi.BootstrapTokenIDKey, secret.Namespace, secret.Name) + klog.V(3).Infof("No %s key in %s/%s Secret", bootstrapapi.BootstrapTokenIDKey, secret.Namespace, secret.Name) return "", "", false } if nameTokenID != tokenID { - glog.V(3).Infof("Token ID (%s) doesn't match secret name: %s", tokenID, nameTokenID) + klog.V(3).Infof("Token ID (%s) doesn't match secret name: %s", tokenID, nameTokenID) return "", "", false } tokenSecret = getSecretString(secret, bootstrapapi.BootstrapTokenSecretKey) if len(tokenSecret) == 0 { - glog.V(3).Infof("No %s key in %s/%s Secret", bootstrapapi.BootstrapTokenSecretKey, secret.Namespace, secret.Name) + klog.V(3).Infof("No %s key in %s/%s Secret", bootstrapapi.BootstrapTokenSecretKey, secret.Namespace, secret.Name) return "", "", false } @@ -95,12 +95,12 @@ func isSecretExpired(secret *v1.Secret) bool { if len(expiration) > 0 { expTime, err2 := time.Parse(time.RFC3339, expiration) if err2 != nil { - glog.V(3).Infof("Unparseable expiration time (%s) in %s/%s Secret: %v. Treating as expired.", + klog.V(3).Infof("Unparseable expiration time (%s) in %s/%s Secret: %v. Treating as expired.", expiration, secret.Namespace, secret.Name, err2) return true } if time.Now().After(expTime) { - glog.V(3).Infof("Expired bootstrap token in %s/%s Secret: %v", + klog.V(3).Infof("Expired bootstrap token in %s/%s Secret: %v", secret.Namespace, secret.Name, expiration) return true } diff --git a/pkg/controller/certificates/BUILD b/pkg/controller/certificates/BUILD index 8057088ca18..26af7005886 100644 --- a/pkg/controller/certificates/BUILD +++ b/pkg/controller/certificates/BUILD @@ -25,8 +25,8 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/time/rate:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -43,6 +43,7 @@ filegroup( ":package-srcs", "//pkg/controller/certificates/approver:all-srcs", "//pkg/controller/certificates/cleaner:all-srcs", + "//pkg/controller/certificates/rootcacertpublisher:all-srcs", "//pkg/controller/certificates/signer:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/controller/certificates/OWNERS b/pkg/controller/certificates/OWNERS index e77a57187c9..470b7a1c92d 100755 --- a/pkg/controller/certificates/OWNERS +++ b/pkg/controller/certificates/OWNERS @@ -1,4 +1,7 @@ +approvers: +- sig-auth-certificates-approvers reviewers: -- deads2k -- mikedanese -- awly +- sig-auth-certificates-reviewers +labels: +- sig/auth + diff --git a/pkg/controller/certificates/certificate_controller.go b/pkg/controller/certificates/certificate_controller.go index 088faf50e2c..2ec79936ef2 100644 --- a/pkg/controller/certificates/certificate_controller.go +++ b/pkg/controller/certificates/certificate_controller.go @@ -22,8 +22,8 @@ import ( "fmt" "time" - "github.com/golang/glog" "golang.org/x/time/rate" + "k8s.io/klog" certificates "k8s.io/api/certificates/v1beta1" "k8s.io/apimachinery/pkg/api/errors" @@ -57,7 +57,7 @@ func NewCertificateController( ) *CertificateController { // Send events to the apiserver eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) cc := &CertificateController{ @@ -74,12 +74,12 @@ func NewCertificateController( csrInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { csr := obj.(*certificates.CertificateSigningRequest) - glog.V(4).Infof("Adding certificate request %s", csr.Name) + klog.V(4).Infof("Adding certificate request %s", csr.Name) cc.enqueueCertificateRequest(obj) }, UpdateFunc: func(old, new interface{}) { oldCSR := old.(*certificates.CertificateSigningRequest) - glog.V(4).Infof("Updating certificate request %s", oldCSR.Name) + klog.V(4).Infof("Updating certificate request %s", oldCSR.Name) cc.enqueueCertificateRequest(new) }, DeleteFunc: func(obj interface{}) { @@ -87,16 +87,16 @@ func NewCertificateController( if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.V(2).Infof("Couldn't get object from tombstone %#v", obj) + klog.V(2).Infof("Couldn't get object from tombstone %#v", obj) return } csr, ok = tombstone.Obj.(*certificates.CertificateSigningRequest) if !ok { - glog.V(2).Infof("Tombstone contained object that is not a CSR: %#v", obj) + klog.V(2).Infof("Tombstone contained object that is not a CSR: %#v", obj) return } } - glog.V(4).Infof("Deleting certificate request %s", csr.Name) + klog.V(4).Infof("Deleting certificate request %s", csr.Name) cc.enqueueCertificateRequest(obj) }, }) @@ -110,8 +110,8 @@ func (cc *CertificateController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer cc.queue.ShutDown() - glog.Infof("Starting certificate controller") - defer glog.Infof("Shutting down certificate controller") + klog.Infof("Starting certificate controller") + defer klog.Infof("Shutting down certificate controller") if !controller.WaitForCacheSync("certificate", stopCh, cc.csrsSynced) { return @@ -143,7 +143,7 @@ func (cc *CertificateController) processNextWorkItem() bool { if _, ignorable := err.(ignorableError); !ignorable { utilruntime.HandleError(fmt.Errorf("Sync %v failed with : %v", cKey, err)) } else { - glog.V(4).Infof("Sync %v failed with : %v", cKey, err) + klog.V(4).Infof("Sync %v failed with : %v", cKey, err) } return true } @@ -169,11 +169,11 @@ func (cc *CertificateController) enqueueCertificateRequest(obj interface{}) { func (cc *CertificateController) syncFunc(key string) error { startTime := time.Now() defer func() { - glog.V(4).Infof("Finished syncing certificate request %q (%v)", key, time.Since(startTime)) + klog.V(4).Infof("Finished syncing certificate request %q (%v)", key, time.Since(startTime)) }() csr, err := cc.csrLister.Get(key) if errors.IsNotFound(err) { - glog.V(3).Infof("csr has been deleted: %v", key) + klog.V(3).Infof("csr has been deleted: %v", key) return nil } if err != nil { diff --git a/pkg/controller/certificates/cleaner/BUILD b/pkg/controller/certificates/cleaner/BUILD index 64cc7730f86..6f195c6f6f5 100644 --- a/pkg/controller/certificates/cleaner/BUILD +++ b/pkg/controller/certificates/cleaner/BUILD @@ -14,7 +14,7 @@ go_library( "//staging/src/k8s.io/client-go/informers/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/listers/certificates/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/certificates/cleaner/cleaner.go b/pkg/controller/certificates/cleaner/cleaner.go index bfe43fa028b..ebdac2956d5 100644 --- a/pkg/controller/certificates/cleaner/cleaner.go +++ b/pkg/controller/certificates/cleaner/cleaner.go @@ -26,7 +26,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" capi "k8s.io/api/certificates/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -78,8 +78,8 @@ func NewCSRCleanerController( func (ccc *CSRCleanerController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() - glog.Infof("Starting CSR cleaner controller") - defer glog.Infof("Shutting down CSR cleaner controller") + klog.Infof("Starting CSR cleaner controller") + defer klog.Infof("Shutting down CSR cleaner controller") for i := 0; i < workers; i++ { go wait.Until(ccc.worker, pollingInterval, stopCh) @@ -92,12 +92,12 @@ func (ccc *CSRCleanerController) Run(workers int, stopCh <-chan struct{}) { func (ccc *CSRCleanerController) worker() { csrs, err := ccc.csrLister.List(labels.Everything()) if err != nil { - glog.Errorf("Unable to list CSRs: %v", err) + klog.Errorf("Unable to list CSRs: %v", err) return } for _, csr := range csrs { if err := ccc.handle(csr); err != nil { - glog.Errorf("Error while attempting to clean CSR %q: %v", csr.Name, err) + klog.Errorf("Error while attempting to clean CSR %q: %v", csr.Name, err) } } } @@ -124,7 +124,7 @@ func isIssuedExpired(csr *capi.CertificateSigningRequest) (bool, error) { } for _, c := range csr.Status.Conditions { if c.Type == capi.CertificateApproved && isIssued(csr) && isExpired { - glog.Infof("Cleaning CSR %q as the associated certificate is expired.", csr.Name) + klog.Infof("Cleaning CSR %q as the associated certificate is expired.", csr.Name) return true, nil } } @@ -138,7 +138,7 @@ func isPendingPastDeadline(csr *capi.CertificateSigningRequest) bool { // If there are no Conditions on the status, the CSR will appear via // `kubectl` as `Pending`. if len(csr.Status.Conditions) == 0 && isOlderThan(csr.CreationTimestamp, pendingExpiration) { - glog.Infof("Cleaning CSR %q as it is more than %v old and unhandled.", csr.Name, pendingExpiration) + klog.Infof("Cleaning CSR %q as it is more than %v old and unhandled.", csr.Name, pendingExpiration) return true } return false @@ -150,7 +150,7 @@ func isPendingPastDeadline(csr *capi.CertificateSigningRequest) bool { func isDeniedPastDeadline(csr *capi.CertificateSigningRequest) bool { for _, c := range csr.Status.Conditions { if c.Type == capi.CertificateDenied && isOlderThan(c.LastUpdateTime, deniedExpiration) { - glog.Infof("Cleaning CSR %q as it is more than %v old and denied.", csr.Name, deniedExpiration) + klog.Infof("Cleaning CSR %q as it is more than %v old and denied.", csr.Name, deniedExpiration) return true } } @@ -163,7 +163,7 @@ func isDeniedPastDeadline(csr *capi.CertificateSigningRequest) bool { func isIssuedPastDeadline(csr *capi.CertificateSigningRequest) bool { for _, c := range csr.Status.Conditions { if c.Type == capi.CertificateApproved && isIssued(csr) && isOlderThan(c.LastUpdateTime, approvedExpiration) { - glog.Infof("Cleaning CSR %q as it is more than %v old and approved.", csr.Name, approvedExpiration) + klog.Infof("Cleaning CSR %q as it is more than %v old and approved.", csr.Name, approvedExpiration) return true } } diff --git a/pkg/controller/certificates/rootcacertpublisher/BUILD b/pkg/controller/certificates/rootcacertpublisher/BUILD new file mode 100644 index 00000000000..343843a7e64 --- /dev/null +++ b/pkg/controller/certificates/rootcacertpublisher/BUILD @@ -0,0 +1,51 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["publisher.go"], + importpath = "k8s.io/kubernetes/pkg/controller/certificates/rootcacertpublisher", + visibility = ["//visibility:public"], + deps = [ + "//pkg/controller:go_default_library", + "//pkg/util/metrics:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["publisher_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/controller:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + ], +) diff --git a/pkg/controller/certificates/rootcacertpublisher/publisher.go b/pkg/controller/certificates/rootcacertpublisher/publisher.go new file mode 100644 index 00000000000..34036c14ed4 --- /dev/null +++ b/pkg/controller/certificates/rootcacertpublisher/publisher.go @@ -0,0 +1,222 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rootcacertpublisher + +import ( + "fmt" + "reflect" + "time" + + "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + coreinformers "k8s.io/client-go/informers/core/v1" + clientset "k8s.io/client-go/kubernetes" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/controller" + "k8s.io/kubernetes/pkg/util/metrics" +) + +// RootCACertConfigMapName is name of the configmap which stores certificates +// to access api-server +const RootCACertConfigMapName = "kube-root-ca.crt" + +// NewPublisher construct a new controller which would manage the configmap +// which stores certificates in each namespace. It will make sure certificate +// configmap exists in each namespace. +func NewPublisher(cmInformer coreinformers.ConfigMapInformer, nsInformer coreinformers.NamespaceInformer, cl clientset.Interface, rootCA []byte) (*Publisher, error) { + e := &Publisher{ + client: cl, + rootCA: rootCA, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "root-ca-cert-publisher"), + } + if cl.CoreV1().RESTClient().GetRateLimiter() != nil { + if err := metrics.RegisterMetricAndTrackRateLimiterUsage("root_ca_cert_publisher", cl.CoreV1().RESTClient().GetRateLimiter()); err != nil { + return nil, err + } + } + + cmInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + DeleteFunc: e.configMapDeleted, + UpdateFunc: e.configMapUpdated, + }) + e.cmLister = cmInformer.Lister() + e.cmListerSynced = cmInformer.Informer().HasSynced + + nsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: e.namespaceAdded, + UpdateFunc: e.namespaceUpdated, + }) + e.nsListerSynced = nsInformer.Informer().HasSynced + + e.syncHandler = e.syncNamespace + + return e, nil + +} + +// Publisher manages certificate ConfigMap objects inside Namespaces +type Publisher struct { + client clientset.Interface + rootCA []byte + + // To allow injection for testing. + syncHandler func(key string) error + + cmLister corelisters.ConfigMapLister + cmListerSynced cache.InformerSynced + + nsListerSynced cache.InformerSynced + + queue workqueue.RateLimitingInterface +} + +// Run starts process +func (c *Publisher) Run(workers int, stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + + klog.Infof("Starting root CA certificate configmap publisher") + defer klog.Infof("Shutting down root CA certificate configmap publisher") + + if !controller.WaitForCacheSync("crt configmap", stopCh, c.cmListerSynced) { + return + } + + for i := 0; i < workers; i++ { + go wait.Until(c.runWorker, time.Second, stopCh) + } + + <-stopCh +} + +func (c *Publisher) configMapDeleted(obj interface{}) { + cm, err := convertToCM(obj) + if err != nil { + utilruntime.HandleError(err) + return + } + if cm.Name != RootCACertConfigMapName { + return + } + c.queue.Add(cm.Namespace) +} + +func (c *Publisher) configMapUpdated(_, newObj interface{}) { + cm, err := convertToCM(newObj) + if err != nil { + utilruntime.HandleError(err) + return + } + if cm.Name != RootCACertConfigMapName { + return + } + c.queue.Add(cm.Namespace) +} + +func (c *Publisher) namespaceAdded(obj interface{}) { + namespace := obj.(*v1.Namespace) + c.queue.Add(namespace.Name) +} + +func (c *Publisher) namespaceUpdated(oldObj interface{}, newObj interface{}) { + newNamespace := newObj.(*v1.Namespace) + if newNamespace.Status.Phase != v1.NamespaceActive { + return + } + c.queue.Add(newNamespace.Name) +} + +func (c *Publisher) runWorker() { + for c.processNextWorkItem() { + } +} + +// processNextWorkItem deals with one key off the queue. It returns false when +// it's time to quit. +func (c *Publisher) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + + if err := c.syncHandler(key.(string)); err != nil { + utilruntime.HandleError(fmt.Errorf("syncing %q failed: %v", key, err)) + c.queue.AddRateLimited(key) + return true + } + + c.queue.Forget(key) + return true +} + +func (c *Publisher) syncNamespace(ns string) error { + startTime := time.Now() + defer func() { + klog.V(4).Infof("Finished syncing namespace %q (%v)", ns, time.Since(startTime)) + }() + + cm, err := c.cmLister.ConfigMaps(ns).Get(RootCACertConfigMapName) + switch { + case apierrs.IsNotFound(err): + _, err := c.client.CoreV1().ConfigMaps(ns).Create(&v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: RootCACertConfigMapName, + }, + Data: map[string]string{ + "ca.crt": string(c.rootCA), + }, + }) + return err + case err != nil: + return err + } + + data := map[string]string{ + "ca.crt": string(c.rootCA), + } + + if reflect.DeepEqual(cm.Data, data) { + return nil + } + + cm.Data = data + + _, err = c.client.CoreV1().ConfigMaps(ns).Update(cm) + return err +} + +func convertToCM(obj interface{}) (*v1.ConfigMap, error) { + cm, ok := obj.(*v1.ConfigMap) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + return nil, fmt.Errorf("Couldn't get object from tombstone %#v", obj) + } + cm, ok = tombstone.Obj.(*v1.ConfigMap) + if !ok { + return nil, fmt.Errorf("Tombstone contained object that is not a ConfigMap %#v", obj) + } + } + return cm, nil +} diff --git a/pkg/controller/certificates/rootcacertpublisher/publisher_test.go b/pkg/controller/certificates/rootcacertpublisher/publisher_test.go new file mode 100644 index 00000000000..c38a4be40de --- /dev/null +++ b/pkg/controller/certificates/rootcacertpublisher/publisher_test.go @@ -0,0 +1,177 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rootcacertpublisher + +import ( + "reflect" + "testing" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/kubernetes/pkg/controller" +) + +func TestConfigMapCreation(t *testing.T) { + ns := metav1.NamespaceDefault + fakeRootCA := []byte("fake-root-ca") + + caConfigMap := defaultCrtConfigMapPtr(fakeRootCA) + addFieldCM := defaultCrtConfigMapPtr(fakeRootCA) + addFieldCM.Data["test"] = "test" + modifyFieldCM := defaultCrtConfigMapPtr([]byte("abc")) + otherConfigMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other", + Namespace: ns, + ResourceVersion: "1", + }, + } + updateOtherConfigMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other", + Namespace: ns, + ResourceVersion: "1", + }, + Data: map[string]string{"aa": "bb"}, + } + + existNS := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + Status: v1.NamespaceStatus{ + Phase: v1.NamespaceActive, + }, + } + newNs := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "new"}, + Status: v1.NamespaceStatus{ + Phase: v1.NamespaceActive, + }, + } + terminatingNS := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + Status: v1.NamespaceStatus{ + Phase: v1.NamespaceTerminating, + }, + } + + type action struct { + verb string + name string + } + testcases := map[string]struct { + ExistingConfigMaps []*v1.ConfigMap + AddedNamespace *v1.Namespace + UpdatedNamespace *v1.Namespace + DeletedConfigMap *v1.ConfigMap + UpdatedConfigMap *v1.ConfigMap + ExpectActions []action + }{ + "create new namesapce": { + AddedNamespace: newNs, + ExpectActions: []action{{verb: "create", name: RootCACertConfigMapName}}, + }, + "delete other configmap": { + ExistingConfigMaps: []*v1.ConfigMap{otherConfigMap, caConfigMap}, + DeletedConfigMap: otherConfigMap, + }, + "delete ca configmap": { + ExistingConfigMaps: []*v1.ConfigMap{otherConfigMap, caConfigMap}, + DeletedConfigMap: caConfigMap, + ExpectActions: []action{{verb: "create", name: RootCACertConfigMapName}}, + }, + "update ca configmap with adding field": { + ExistingConfigMaps: []*v1.ConfigMap{caConfigMap}, + UpdatedConfigMap: addFieldCM, + ExpectActions: []action{{verb: "update", name: RootCACertConfigMapName}}, + }, + "update ca configmap with modifying field": { + ExistingConfigMaps: []*v1.ConfigMap{caConfigMap}, + UpdatedConfigMap: modifyFieldCM, + ExpectActions: []action{{verb: "update", name: RootCACertConfigMapName}}, + }, + "update with other configmap": { + ExistingConfigMaps: []*v1.ConfigMap{caConfigMap, otherConfigMap}, + UpdatedConfigMap: updateOtherConfigMap, + }, + "update namespace with terminating state": { + UpdatedNamespace: terminatingNS, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + client := fake.NewSimpleClientset(caConfigMap, existNS) + informers := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), controller.NoResyncPeriodFunc()) + cmInformer := informers.Core().V1().ConfigMaps() + nsInformer := informers.Core().V1().Namespaces() + controller, err := NewPublisher(cmInformer, nsInformer, client, fakeRootCA) + if err != nil { + t.Fatalf("error creating ServiceAccounts controller: %v", err) + } + + cmStore := cmInformer.Informer().GetStore() + + controller.syncHandler = controller.syncNamespace + + for _, s := range tc.ExistingConfigMaps { + cmStore.Add(s) + } + + if tc.AddedNamespace != nil { + controller.namespaceAdded(tc.AddedNamespace) + } + if tc.UpdatedNamespace != nil { + controller.namespaceUpdated(nil, tc.UpdatedNamespace) + } + + if tc.DeletedConfigMap != nil { + cmStore.Delete(tc.DeletedConfigMap) + controller.configMapDeleted(tc.DeletedConfigMap) + } + + if tc.UpdatedConfigMap != nil { + cmStore.Add(tc.UpdatedConfigMap) + controller.configMapUpdated(nil, tc.UpdatedConfigMap) + } + + for controller.queue.Len() != 0 { + controller.processNextWorkItem() + } + + actions := client.Actions() + if reflect.DeepEqual(actions, tc.ExpectActions) { + t.Errorf("Unexpected actions:\n%s", diff.ObjectGoPrintDiff(actions, tc.ExpectActions)) + } + }) + } +} + +func defaultCrtConfigMapPtr(rootCA []byte) *v1.ConfigMap { + tmp := v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: RootCACertConfigMapName, + }, + Data: map[string]string{ + "ca.crt": string(rootCA), + }, + } + tmp.Namespace = metav1.NamespaceDefault + return &tmp +} diff --git a/pkg/controller/client_builder.go b/pkg/controller/client_builder.go index e8477445922..caac5649d2f 100644 --- a/pkg/controller/client_builder.go +++ b/pkg/controller/client_builder.go @@ -38,7 +38,7 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/serviceaccount" - "github.com/golang/glog" + "k8s.io/klog" ) // ControllerClientBuilder allows you to get clients and configs for controllers @@ -65,7 +65,7 @@ func (b SimpleControllerClientBuilder) Config(name string) (*restclient.Config, func (b SimpleControllerClientBuilder) ConfigOrDie(name string) *restclient.Config { clientConfig, err := b.Config(name) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return clientConfig } @@ -81,7 +81,7 @@ func (b SimpleControllerClientBuilder) Client(name string) (clientset.Interface, func (b SimpleControllerClientBuilder) ClientOrDie(name string) clientset.Interface { client, err := b.Client(name) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return client } @@ -146,15 +146,15 @@ func (b SAControllerClientBuilder) Config(name string) (*restclient.Config, erro } validConfig, valid, err := b.getAuthenticatedConfig(sa, string(secret.Data[v1.ServiceAccountTokenKey])) if err != nil { - glog.Warningf("error validating API token for %s/%s in secret %s: %v", sa.Name, sa.Namespace, secret.Name, err) + klog.Warningf("error validating API token for %s/%s in secret %s: %v", sa.Name, sa.Namespace, secret.Name, err) // continue watching for good tokens return false, nil } if !valid { - glog.Warningf("secret %s contained an invalid API token for %s/%s", secret.Name, sa.Name, sa.Namespace) + klog.Warningf("secret %s contained an invalid API token for %s/%s", secret.Name, sa.Name, sa.Namespace) // try to delete the secret containing the invalid token if err := b.CoreClient.Secrets(secret.Namespace).Delete(secret.Name, &metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { - glog.Warningf("error deleting secret %s containing invalid API token for %s/%s: %v", secret.Name, sa.Name, sa.Namespace, err) + klog.Warningf("error deleting secret %s containing invalid API token for %s/%s: %v", secret.Name, sa.Name, sa.Namespace, err) } // continue watching for good tokens return false, nil @@ -208,14 +208,14 @@ func (b SAControllerClientBuilder) getAuthenticatedConfig(sa *v1.ServiceAccount, tokenReview := &v1authenticationapi.TokenReview{Spec: v1authenticationapi.TokenReviewSpec{Token: token}} if tokenResult, err := b.AuthenticationClient.TokenReviews().Create(tokenReview); err == nil { if !tokenResult.Status.Authenticated { - glog.Warningf("Token for %s/%s did not authenticate correctly", sa.Name, sa.Namespace) + klog.Warningf("Token for %s/%s did not authenticate correctly", sa.Name, sa.Namespace) return nil, false, nil } if tokenResult.Status.User.Username != username { - glog.Warningf("Token for %s/%s authenticated as unexpected username: %s", sa.Name, sa.Namespace, tokenResult.Status.User.Username) + klog.Warningf("Token for %s/%s authenticated as unexpected username: %s", sa.Name, sa.Namespace, tokenResult.Status.User.Username) return nil, false, nil } - glog.V(4).Infof("Verified credential for %s/%s", sa.Name, sa.Namespace) + klog.V(4).Infof("Verified credential for %s/%s", sa.Name, sa.Namespace) return clientConfig, true, nil } @@ -229,7 +229,7 @@ func (b SAControllerClientBuilder) getAuthenticatedConfig(sa *v1.ServiceAccount, } err = client.Get().AbsPath("/apis").Do().Error() if apierrors.IsUnauthorized(err) { - glog.Warningf("Token for %s/%s did not authenticate correctly: %v", sa.Name, sa.Namespace, err) + klog.Warningf("Token for %s/%s did not authenticate correctly: %v", sa.Name, sa.Namespace, err) return nil, false, nil } @@ -239,7 +239,7 @@ func (b SAControllerClientBuilder) getAuthenticatedConfig(sa *v1.ServiceAccount, func (b SAControllerClientBuilder) ConfigOrDie(name string) *restclient.Config { clientConfig, err := b.Config(name) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return clientConfig } @@ -255,7 +255,7 @@ func (b SAControllerClientBuilder) Client(name string) (clientset.Interface, err func (b SAControllerClientBuilder) ClientOrDie(name string) clientset.Interface { client, err := b.Client(name) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return client } diff --git a/pkg/controller/cloud/BUILD b/pkg/controller/cloud/BUILD index 58de6103e74..25e1c6fe6bc 100644 --- a/pkg/controller/cloud/BUILD +++ b/pkg/controller/cloud/BUILD @@ -43,13 +43,14 @@ go_library( "//staging/src/k8s.io/client-go/util/retry:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) go_test( name = "go_default_test", srcs = [ + "main_test.go", "node_controller_test.go", "pvlcontroller_test.go", ], @@ -58,6 +59,7 @@ go_test( "//pkg/cloudprovider/providers/fake:go_default_library", "//pkg/controller:go_default_library", "//pkg/controller/testutil:go_default_library", + "//pkg/features:go_default_library", "//pkg/kubelet/apis:go_default_library", "//pkg/scheduler/api:go_default_library", "//pkg/volume/util:go_default_library", @@ -68,14 +70,15 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/cloud/main_test.go b/pkg/controller/cloud/main_test.go new file mode 100644 index 00000000000..a2abc54d015 --- /dev/null +++ b/pkg/controller/cloud/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloud + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/controller/cloud/node_controller.go b/pkg/controller/cloud/node_controller.go index 0ffd3ede899..bfce4e59d4b 100644 --- a/pkg/controller/cloud/node_controller.go +++ b/pkg/controller/cloud/node_controller.go @@ -22,7 +22,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -84,12 +84,12 @@ func NewCloudNodeController( eventBroadcaster := record.NewBroadcaster() recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-controller"}) - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) if kubeClient != nil { - glog.V(0).Infof("Sending events to api server.") + klog.V(0).Infof("Sending events to api server.") eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) } else { - glog.V(0).Infof("No api server defined - no events will be sent to API server.") + klog.V(0).Infof("No api server defined - no events will be sent to API server.") } cnc := &CloudNodeController{ @@ -137,7 +137,7 @@ func (cnc *CloudNodeController) UpdateNodeStatus() { nodes, err := cnc.kubeClient.CoreV1().Nodes().List(metav1.ListOptions{ResourceVersion: "0"}) if err != nil { - glog.Errorf("Error monitoring node status: %v", err) + klog.Errorf("Error monitoring node status: %v", err) return } @@ -151,27 +151,27 @@ func (cnc *CloudNodeController) updateNodeAddress(node *v1.Node, instances cloud // Do not process nodes that are still tainted cloudTaint := getCloudTaint(node.Spec.Taints) if cloudTaint != nil { - glog.V(5).Infof("This node %s is still tainted. Will not process.", node.Name) + klog.V(5).Infof("This node %s is still tainted. Will not process.", node.Name) return } // Node that isn't present according to the cloud provider shouldn't have its address updated exists, err := ensureNodeExistsByProviderID(instances, node) if err != nil { // Continue to update node address when not sure the node is not exists - glog.Errorf("%v", err) + klog.Errorf("%v", err) } else if !exists { - glog.V(4).Infof("The node %s is no longer present according to the cloud provider, do not process.", node.Name) + klog.V(4).Infof("The node %s is no longer present according to the cloud provider, do not process.", node.Name) return } nodeAddresses, err := getNodeAddressesByProviderIDOrName(instances, node) if err != nil { - glog.Errorf("%v", err) + klog.Errorf("%v", err) return } if len(nodeAddresses) == 0 { - glog.V(5).Infof("Skipping node address update for node %q since cloud provider did not return any", node.Name) + klog.V(5).Infof("Skipping node address update for node %q since cloud provider did not return any", node.Name) return } @@ -195,7 +195,7 @@ func (cnc *CloudNodeController) updateNodeAddress(node *v1.Node, instances cloud // it can be found in the cloud as well (consistent with the behaviour in kubelet) if nodeIP, ok := ensureNodeProvidedIPExists(node, nodeAddresses); ok { if nodeIP == nil { - glog.Errorf("Specified Node IP not found in cloudprovider") + klog.Errorf("Specified Node IP not found in cloudprovider") return } } @@ -206,7 +206,7 @@ func (cnc *CloudNodeController) updateNodeAddress(node *v1.Node, instances cloud } _, _, err = nodeutil.PatchNodeStatus(cnc.kubeClient.CoreV1(), types.NodeName(node.Name), node, newNode) if err != nil { - glog.Errorf("Error patching node with cloud ip addresses = [%v]", err) + klog.Errorf("Error patching node with cloud ip addresses = [%v]", err) } } @@ -221,7 +221,7 @@ func (cnc *CloudNodeController) MonitorNode() { nodes, err := cnc.kubeClient.CoreV1().Nodes().List(metav1.ListOptions{ResourceVersion: "0"}) if err != nil { - glog.Errorf("Error monitoring node status: %v", err) + klog.Errorf("Error monitoring node status: %v", err) return } @@ -238,13 +238,13 @@ func (cnc *CloudNodeController) MonitorNode() { name := node.Name node, err = cnc.kubeClient.CoreV1().Nodes().Get(name, metav1.GetOptions{}) if err != nil { - glog.Errorf("Failed while getting a Node to retry updating NodeStatus. Probably Node %s was deleted.", name) + klog.Errorf("Failed while getting a Node to retry updating NodeStatus. Probably Node %s was deleted.", name) break } time.Sleep(retrySleepTime) } if currentReadyCondition == nil { - glog.Errorf("Update status of Node %v from CloudNodeController exceeds retry count or the Node was deleted.", node.Name) + klog.Errorf("Update status of Node %v from CloudNodeController exceeds retry count or the Node was deleted.", node.Name) continue } // If the known node status says that Node is NotReady, then check if the node has been removed @@ -256,14 +256,14 @@ func (cnc *CloudNodeController) MonitorNode() { // does not delete node from kubernetes cluster when instance it is shutdown see issue #46442 shutdown, err := nodectrlutil.ShutdownInCloudProvider(context.TODO(), cnc.cloud, node) if err != nil { - glog.Errorf("Error checking if node %s is shutdown: %v", node.Name, err) + klog.Errorf("Error checking if node %s is shutdown: %v", node.Name, err) } if shutdown && err == nil { // if node is shutdown add shutdown taint err = controller.AddOrUpdateTaintOnNode(cnc.kubeClient, node.Name, controller.ShutdownTaint) if err != nil { - glog.Errorf("Error patching node taints: %v", err) + klog.Errorf("Error patching node taints: %v", err) } // Continue checking the remaining nodes since the current one is shutdown. continue @@ -273,7 +273,7 @@ func (cnc *CloudNodeController) MonitorNode() { // doesn't, delete the node immediately. exists, err := ensureNodeExistsByProviderID(instances, node) if err != nil { - glog.Errorf("Error checking if node %s exists: %v", node.Name, err) + klog.Errorf("Error checking if node %s exists: %v", node.Name, err) continue } @@ -282,7 +282,7 @@ func (cnc *CloudNodeController) MonitorNode() { continue } - glog.V(2).Infof("Deleting node since it is no longer present in cloud provider: %s", node.Name) + klog.V(2).Infof("Deleting node since it is no longer present in cloud provider: %s", node.Name) ref := &v1.ObjectReference{ Kind: "Node", @@ -290,14 +290,14 @@ func (cnc *CloudNodeController) MonitorNode() { UID: types.UID(node.UID), Namespace: "", } - glog.V(2).Infof("Recording %s event message for node %s", "DeletingNode", node.Name) + klog.V(2).Infof("Recording %s event message for node %s", "DeletingNode", node.Name) cnc.recorder.Eventf(ref, v1.EventTypeNormal, fmt.Sprintf("Deleting Node %v because it's not present according to cloud provider", node.Name), "Node %s event: %s", node.Name, "DeletingNode") go func(nodeName string) { defer utilruntime.HandleCrash() if err := cnc.kubeClient.CoreV1().Nodes().Delete(nodeName, nil); err != nil { - glog.Errorf("unable to delete node %q: %v", nodeName, err) + klog.Errorf("unable to delete node %q: %v", nodeName, err) } }(node.Name) @@ -305,7 +305,7 @@ func (cnc *CloudNodeController) MonitorNode() { // if taint exist remove taint err = controller.RemoveTaintOffNode(cnc.kubeClient, node.Name, node, controller.ShutdownTaint) if err != nil { - glog.Errorf("Error patching node taints: %v", err) + klog.Errorf("Error patching node taints: %v", err) } } } @@ -326,7 +326,7 @@ func (cnc *CloudNodeController) AddCloudNode(obj interface{}) { cloudTaint := getCloudTaint(node.Spec.Taints) if cloudTaint == nil { - glog.V(2).Infof("This node %s is registered without the cloud taint. Will not process.", node.Name) + klog.V(2).Infof("This node %s is registered without the cloud taint. Will not process.", node.Name) return } @@ -365,7 +365,7 @@ func (cnc *CloudNodeController) AddCloudNode(obj interface{}) { // we should attempt to set providerID on curNode, but // we can continue if we fail since we will attempt to set // node addresses given the node name in getNodeAddressesByProviderIDOrName - glog.Errorf("failed to set node provider id: %v", err) + klog.Errorf("failed to set node provider id: %v", err) } } @@ -385,7 +385,7 @@ func (cnc *CloudNodeController) AddCloudNode(obj interface{}) { if instanceType, err := getInstanceTypeByProviderIDOrName(instances, curNode); err != nil { return err } else if instanceType != "" { - glog.V(2).Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelInstanceType, instanceType) + klog.V(2).Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelInstanceType, instanceType) curNode.ObjectMeta.Labels[kubeletapis.LabelInstanceType] = instanceType } @@ -395,11 +395,11 @@ func (cnc *CloudNodeController) AddCloudNode(obj interface{}) { return fmt.Errorf("failed to get zone from cloud provider: %v", err) } if zone.FailureDomain != "" { - glog.V(2).Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelZoneFailureDomain, zone.FailureDomain) + klog.V(2).Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelZoneFailureDomain, zone.FailureDomain) curNode.ObjectMeta.Labels[kubeletapis.LabelZoneFailureDomain] = zone.FailureDomain } if zone.Region != "" { - glog.V(2).Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelZoneRegion, zone.Region) + klog.V(2).Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelZoneRegion, zone.Region) curNode.ObjectMeta.Labels[kubeletapis.LabelZoneRegion] = zone.Region } } @@ -420,7 +420,7 @@ func (cnc *CloudNodeController) AddCloudNode(obj interface{}) { return } - glog.Infof("Successfully initialized node %s with cloud provider", node.Name) + klog.Infof("Successfully initialized node %s with cloud provider", node.Name) } func getCloudTaint(taints []v1.Taint) *v1.Taint { @@ -458,7 +458,7 @@ func ensureNodeExistsByProviderID(instances cloudprovider.Instances, node *v1.No } if providerID == "" { - glog.Warningf("Cannot find valid providerID for node name %q, assuming non existence", node.Name) + klog.Warningf("Cannot find valid providerID for node name %q, assuming non existence", node.Name) return false, nil } } diff --git a/pkg/controller/cloud/node_controller_test.go b/pkg/controller/cloud/node_controller_test.go index 5564b60b8f5..855594dcc8e 100644 --- a/pkg/controller/cloud/node_controller_test.go +++ b/pkg/controller/cloud/node_controller_test.go @@ -37,8 +37,8 @@ import ( kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" - "github.com/golang/glog" "github.com/stretchr/testify/assert" + "k8s.io/klog" ) func TestEnsureNodeExistsByProviderID(t *testing.T) { @@ -250,7 +250,7 @@ func TestNodeShutdown(t *testing.T) { recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-controller"}), nodeStatusUpdateFrequency: 1 * time.Second, } - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) cloudNodeController.Run(wait.NeverStop) @@ -349,7 +349,7 @@ func TestNodeDeleted(t *testing.T) { recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-controller"}), nodeStatusUpdateFrequency: 1 * time.Second, } - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) cloudNodeController.Run(wait.NeverStop) @@ -429,7 +429,7 @@ func TestNodeInitialized(t *testing.T) { recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-controller"}), nodeStatusUpdateFrequency: 1 * time.Second, } - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) cloudNodeController.AddCloudNode(fnh.Existing[0]) @@ -494,7 +494,7 @@ func TestNodeIgnored(t *testing.T) { nodeMonitorPeriod: 5 * time.Second, recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-controller"}), } - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) cloudNodeController.AddCloudNode(fnh.Existing[0]) assert.Equal(t, 0, len(fnh.UpdatedNodes), "Node was wrongly updated") @@ -568,7 +568,7 @@ func TestGCECondition(t *testing.T) { nodeMonitorPeriod: 1 * time.Second, recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-controller"}), } - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) cloudNodeController.AddCloudNode(fnh.Existing[0]) @@ -658,7 +658,7 @@ func TestZoneInitialized(t *testing.T) { nodeMonitorPeriod: 5 * time.Second, recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-controller"}), } - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) cloudNodeController.AddCloudNode(fnh.Existing[0]) @@ -749,7 +749,7 @@ func TestNodeAddresses(t *testing.T) { nodeStatusUpdateFrequency: 1 * time.Second, recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-controller"}), } - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) cloudNodeController.AddCloudNode(fnh.Existing[0]) @@ -864,7 +864,7 @@ func TestNodeProvidedIPAddresses(t *testing.T) { nodeStatusUpdateFrequency: 1 * time.Second, recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-controller"}), } - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) cloudNodeController.AddCloudNode(fnh.Existing[0]) @@ -1156,7 +1156,7 @@ func TestNodeProviderID(t *testing.T) { nodeStatusUpdateFrequency: 1 * time.Second, recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-controller"}), } - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) cloudNodeController.AddCloudNode(fnh.Existing[0]) @@ -1240,7 +1240,7 @@ func TestNodeProviderIDAlreadySet(t *testing.T) { nodeStatusUpdateFrequency: 1 * time.Second, recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-controller"}), } - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) cloudNodeController.AddCloudNode(fnh.Existing[0]) diff --git a/pkg/controller/cloud/pvlcontroller.go b/pkg/controller/cloud/pvlcontroller.go index 3459e74fcf4..5cce125a6ae 100644 --- a/pkg/controller/cloud/pvlcontroller.go +++ b/pkg/controller/cloud/pvlcontroller.go @@ -22,7 +22,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" @@ -109,8 +109,8 @@ func (pvlc *PersistentVolumeLabelController) Run(threadiness int, stopCh <-chan defer utilruntime.HandleCrash() defer pvlc.queue.ShutDown() - glog.Infof("Starting PersistentVolumeLabelController") - defer glog.Infof("Shutting down PersistentVolumeLabelController") + klog.Infof("Starting PersistentVolumeLabelController") + defer klog.Infof("Shutting down PersistentVolumeLabelController") go pvlc.pvlController.Run(stopCh) @@ -197,7 +197,7 @@ func (pvlc *PersistentVolumeLabelController) addLabelsAndAffinityToVolume(vol *v } volumeLabels = labels } else { - glog.V(4).Info("cloud provider does not support PVLabeler") + klog.V(4).Info("cloud provider does not support PVLabeler") } return pvlc.updateVolume(vol, volumeLabels) } @@ -244,7 +244,7 @@ func (pvlc *PersistentVolumeLabelController) createPatch(vol *v1.PersistentVolum } // Populate NodeAffinity with requirements if there are no conflicting keys found if v1helper.NodeSelectorRequirementKeysExistInNodeSelectorTerms(requirements, newVolume.Spec.NodeAffinity.Required.NodeSelectorTerms) { - glog.V(4).Infof("NodeSelectorRequirements for cloud labels %v conflict with existing NodeAffinity %v. Skipping addition of NodeSelectorRequirements for cloud labels.", + klog.V(4).Infof("NodeSelectorRequirements for cloud labels %v conflict with existing NodeAffinity %v. Skipping addition of NodeSelectorRequirements for cloud labels.", requirements, newVolume.Spec.NodeAffinity) } else { for _, req := range requirements { @@ -255,7 +255,7 @@ func (pvlc *PersistentVolumeLabelController) createPatch(vol *v1.PersistentVolum } } newVolume.Initializers = removeInitializer(newVolume.Initializers, initializerName) - glog.V(4).Infof("removed initializer on PersistentVolume %s", newVolume.Name) + klog.V(4).Infof("removed initializer on PersistentVolume %s", newVolume.Name) oldData, err := json.Marshal(vol) if err != nil { @@ -276,7 +276,7 @@ func (pvlc *PersistentVolumeLabelController) createPatch(vol *v1.PersistentVolum func (pvlc *PersistentVolumeLabelController) updateVolume(vol *v1.PersistentVolume, volLabels map[string]string) error { volName := vol.Name - glog.V(4).Infof("updating PersistentVolume %s", volName) + klog.V(4).Infof("updating PersistentVolume %s", volName) patchBytes, err := pvlc.createPatch(vol, volLabels) if err != nil { return err @@ -286,7 +286,7 @@ func (pvlc *PersistentVolumeLabelController) updateVolume(vol *v1.PersistentVolu if err != nil { return fmt.Errorf("failed to update PersistentVolume %s: %v", volName, err) } - glog.V(4).Infof("updated PersistentVolume %s", volName) + klog.V(4).Infof("updated PersistentVolume %s", volName) return nil } diff --git a/pkg/controller/cloud/pvlcontroller_test.go b/pkg/controller/cloud/pvlcontroller_test.go index e74af89d936..1931540d1ef 100644 --- a/pkg/controller/cloud/pvlcontroller_test.go +++ b/pkg/controller/cloud/pvlcontroller_test.go @@ -28,8 +28,10 @@ import ( sets "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" + "k8s.io/kubernetes/pkg/features" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" volumeutil "k8s.io/kubernetes/pkg/volume/util" @@ -449,8 +451,7 @@ func TestCreatePatch(t *testing.T) { }, } - utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - defer utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() for d, tc := range testCases { cloud := &fakecloud.FakeCloud{} client := fake.NewSimpleClientset() @@ -519,8 +520,7 @@ func TestAddLabelsToVolume(t *testing.T) { }, } - utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - defer utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() for d, tc := range testCases { labeledCh := make(chan bool, 1) diff --git a/pkg/controller/clusterroleaggregation/BUILD b/pkg/controller/clusterroleaggregation/BUILD index b2f74930f1e..2fef3c98f41 100644 --- a/pkg/controller/clusterroleaggregation/BUILD +++ b/pkg/controller/clusterroleaggregation/BUILD @@ -19,7 +19,7 @@ go_library( "//staging/src/k8s.io/client-go/listers/rbac/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/clusterroleaggregation/clusterroleaggregation_controller.go b/pkg/controller/clusterroleaggregation/clusterroleaggregation_controller.go index 05879e0e681..b5c9f873dbc 100644 --- a/pkg/controller/clusterroleaggregation/clusterroleaggregation_controller.go +++ b/pkg/controller/clusterroleaggregation/clusterroleaggregation_controller.go @@ -21,7 +21,7 @@ import ( "sort" "time" - "github.com/golang/glog" + "k8s.io/klog" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -145,8 +145,8 @@ func (c *ClusterRoleAggregationController) Run(workers int, stopCh <-chan struct defer utilruntime.HandleCrash() defer c.queue.ShutDown() - glog.Infof("Starting ClusterRoleAggregator") - defer glog.Infof("Shutting down ClusterRoleAggregator") + klog.Infof("Starting ClusterRoleAggregator") + defer klog.Infof("Shutting down ClusterRoleAggregator") if !controller.WaitForCacheSync("ClusterRoleAggregator", stopCh, c.clusterRolesSynced) { return diff --git a/pkg/controller/controller_ref_manager.go b/pkg/controller/controller_ref_manager.go index 6cf2ac18946..f63afaca6f3 100644 --- a/pkg/controller/controller_ref_manager.go +++ b/pkg/controller/controller_ref_manager.go @@ -20,7 +20,6 @@ import ( "fmt" "sync" - "github.com/golang/glog" apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -28,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog" ) type BaseControllerRefManager struct { @@ -223,7 +223,7 @@ func (m *PodControllerRefManager) AdoptPod(pod *v1.Pod) error { // ReleasePod sends a patch to free the pod from the control of the controller. // It returns the error if the patching fails. 404 and 422 errors are ignored. func (m *PodControllerRefManager) ReleasePod(pod *v1.Pod) error { - glog.V(2).Infof("patching pod %s_%s to remove its controllerRef to %s/%s:%s", + klog.V(2).Infof("patching pod %s_%s to remove its controllerRef to %s/%s:%s", pod.Namespace, pod.Name, m.controllerKind.GroupVersion(), m.controllerKind.Kind, m.Controller.GetName()) deleteOwnerRefPatch := fmt.Sprintf(`{"metadata":{"ownerReferences":[{"$patch":"delete","uid":"%s"}],"uid":"%s"}}`, m.Controller.GetUID(), pod.UID) err := m.podControl.PatchPod(pod.Namespace, pod.Name, []byte(deleteOwnerRefPatch)) @@ -345,7 +345,7 @@ func (m *ReplicaSetControllerRefManager) AdoptReplicaSet(rs *apps.ReplicaSet) er // ReleaseReplicaSet sends a patch to free the ReplicaSet from the control of the Deployment controller. // It returns the error if the patching fails. 404 and 422 errors are ignored. func (m *ReplicaSetControllerRefManager) ReleaseReplicaSet(replicaSet *apps.ReplicaSet) error { - glog.V(2).Infof("patching ReplicaSet %s_%s to remove its controllerRef to %s/%s:%s", + klog.V(2).Infof("patching ReplicaSet %s_%s to remove its controllerRef to %s/%s:%s", replicaSet.Namespace, replicaSet.Name, m.controllerKind.GroupVersion(), m.controllerKind.Kind, m.Controller.GetName()) deleteOwnerRefPatch := fmt.Sprintf(`{"metadata":{"ownerReferences":[{"$patch":"delete","uid":"%s"}],"uid":"%s"}}`, m.Controller.GetUID(), replicaSet.UID) err := m.rsControl.PatchReplicaSet(replicaSet.Namespace, replicaSet.Name, []byte(deleteOwnerRefPatch)) @@ -480,7 +480,7 @@ func (m *ControllerRevisionControllerRefManager) AdoptControllerRevision(history // ReleaseControllerRevision sends a patch to free the ControllerRevision from the control of its controller. // It returns the error if the patching fails. 404 and 422 errors are ignored. func (m *ControllerRevisionControllerRefManager) ReleaseControllerRevision(history *apps.ControllerRevision) error { - glog.V(2).Infof("patching ControllerRevision %s_%s to remove its controllerRef to %s/%s:%s", + klog.V(2).Infof("patching ControllerRevision %s_%s to remove its controllerRef to %s/%s:%s", history.Namespace, history.Name, m.controllerKind.GroupVersion(), m.controllerKind.Kind, m.Controller.GetName()) deleteOwnerRefPatch := fmt.Sprintf(`{"metadata":{"ownerReferences":[{"$patch":"delete","uid":"%s"}],"uid":"%s"}}`, m.Controller.GetUID(), history.UID) err := m.crControl.PatchControllerRevision(history.Namespace, history.Name, []byte(deleteOwnerRefPatch)) diff --git a/pkg/controller/controller_utils.go b/pkg/controller/controller_utils.go index 04ef26e0f4b..6ccc32aed0f 100644 --- a/pkg/controller/controller_utils.go +++ b/pkg/controller/controller_utils.go @@ -51,7 +51,7 @@ import ( hashutil "k8s.io/kubernetes/pkg/util/hash" taintutils "k8s.io/kubernetes/pkg/util/taints" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -170,7 +170,7 @@ func (r *ControllerExpectations) GetExpectations(controllerKey string) (*Control func (r *ControllerExpectations) DeleteExpectations(controllerKey string) { if exp, exists, err := r.GetByKey(controllerKey); err == nil && exists { if err := r.Delete(exp); err != nil { - glog.V(2).Infof("Error deleting expectations for controller %v: %v", controllerKey, err) + klog.V(2).Infof("Error deleting expectations for controller %v: %v", controllerKey, err) } } } @@ -181,24 +181,24 @@ func (r *ControllerExpectations) DeleteExpectations(controllerKey string) { func (r *ControllerExpectations) SatisfiedExpectations(controllerKey string) bool { if exp, exists, err := r.GetExpectations(controllerKey); exists { if exp.Fulfilled() { - glog.V(4).Infof("Controller expectations fulfilled %#v", exp) + klog.V(4).Infof("Controller expectations fulfilled %#v", exp) return true } else if exp.isExpired() { - glog.V(4).Infof("Controller expectations expired %#v", exp) + klog.V(4).Infof("Controller expectations expired %#v", exp) return true } else { - glog.V(4).Infof("Controller still waiting on expectations %#v", exp) + klog.V(4).Infof("Controller still waiting on expectations %#v", exp) return false } } else if err != nil { - glog.V(2).Infof("Error encountered while checking expectations %#v, forcing sync", err) + klog.V(2).Infof("Error encountered while checking expectations %#v, forcing sync", err) } else { // When a new controller is created, it doesn't have expectations. // When it doesn't see expected watch events for > TTL, the expectations expire. // - In this case it wakes up, creates/deletes controllees, and sets expectations again. // When it has satisfied expectations and no controllees need to be created/destroyed > TTL, the expectations expire. // - In this case it continues without setting expectations till it needs to create/delete controllees. - glog.V(4).Infof("Controller %v either never recorded expectations, or the ttl expired.", controllerKey) + klog.V(4).Infof("Controller %v either never recorded expectations, or the ttl expired.", controllerKey) } // Trigger a sync if we either encountered and error (which shouldn't happen since we're // getting from local store) or this controller hasn't established expectations. @@ -215,7 +215,7 @@ func (exp *ControlleeExpectations) isExpired() bool { // SetExpectations registers new expectations for the given controller. Forgets existing expectations. func (r *ControllerExpectations) SetExpectations(controllerKey string, add, del int) error { exp := &ControlleeExpectations{add: int64(add), del: int64(del), key: controllerKey, timestamp: clock.RealClock{}.Now()} - glog.V(4).Infof("Setting expectations %#v", exp) + klog.V(4).Infof("Setting expectations %#v", exp) return r.Add(exp) } @@ -232,7 +232,7 @@ func (r *ControllerExpectations) LowerExpectations(controllerKey string, add, de if exp, exists, err := r.GetExpectations(controllerKey); err == nil && exists { exp.Add(int64(-add), int64(-del)) // The expectations might've been modified since the update on the previous line. - glog.V(4).Infof("Lowered expectations %#v", exp) + klog.V(4).Infof("Lowered expectations %#v", exp) } } @@ -241,7 +241,7 @@ func (r *ControllerExpectations) RaiseExpectations(controllerKey string, add, de if exp, exists, err := r.GetExpectations(controllerKey); err == nil && exists { exp.Add(int64(add), int64(del)) // The expectations might've been modified since the update on the previous line. - glog.V(4).Infof("Raised expectations %#v", exp) + klog.V(4).Infof("Raised expectations %#v", exp) } } @@ -340,13 +340,13 @@ func (u *UIDTrackingControllerExpectations) ExpectDeletions(rcKey string, delete defer u.uidStoreLock.Unlock() if existing := u.GetUIDs(rcKey); existing != nil && existing.Len() != 0 { - glog.Errorf("Clobbering existing delete keys: %+v", existing) + klog.Errorf("Clobbering existing delete keys: %+v", existing) } expectedUIDs := sets.NewString() for _, k := range deletedKeys { expectedUIDs.Insert(k) } - glog.V(4).Infof("Controller %v waiting on deletions for: %+v", rcKey, deletedKeys) + klog.V(4).Infof("Controller %v waiting on deletions for: %+v", rcKey, deletedKeys) if err := u.uidStore.Add(&UIDSet{expectedUIDs, rcKey}); err != nil { return err } @@ -360,7 +360,7 @@ func (u *UIDTrackingControllerExpectations) DeletionObserved(rcKey, deleteKey st uids := u.GetUIDs(rcKey) if uids != nil && uids.Has(deleteKey) { - glog.V(4).Infof("Controller %v received delete for pod %v", rcKey, deleteKey) + klog.V(4).Infof("Controller %v received delete for pod %v", rcKey, deleteKey) u.ControllerExpectationsInterface.DeletionObserved(rcKey) uids.Delete(deleteKey) } @@ -375,7 +375,7 @@ func (u *UIDTrackingControllerExpectations) DeleteExpectations(rcKey string) { u.ControllerExpectationsInterface.DeleteExpectations(rcKey) if uidExp, exists, err := u.uidStore.GetByKey(rcKey); err == nil && exists { if err := u.uidStore.Delete(uidExp); err != nil { - glog.V(2).Infof("Error deleting uid expectations for controller %v: %v", rcKey, err) + klog.V(2).Infof("Error deleting uid expectations for controller %v: %v", rcKey, err) } } } @@ -581,10 +581,10 @@ func (r RealPodControl) createPods(nodeName, namespace string, template *v1.PodT } else { accessor, err := meta.Accessor(object) if err != nil { - glog.Errorf("parentObject does not have ObjectMeta, %v", err) + klog.Errorf("parentObject does not have ObjectMeta, %v", err) return nil } - glog.V(4).Infof("Controller %v created pod %v", accessor.GetName(), newPod.Name) + klog.V(4).Infof("Controller %v created pod %v", accessor.GetName(), newPod.Name) r.Recorder.Eventf(object, v1.EventTypeNormal, SuccessfulCreatePodReason, "Created pod: %v", newPod.Name) } return nil @@ -595,7 +595,7 @@ func (r RealPodControl) DeletePod(namespace string, podID string, object runtime if err != nil { return fmt.Errorf("object does not have ObjectMeta, %v", err) } - glog.V(2).Infof("Controller %v deleting pod %v/%v", accessor.GetName(), namespace, podID) + klog.V(2).Infof("Controller %v deleting pod %v/%v", accessor.GetName(), namespace, podID) if err := r.KubeClient.CoreV1().Pods(namespace).Delete(podID, nil); err != nil && !apierrors.IsNotFound(err) { r.Recorder.Eventf(object, v1.EventTypeWarning, FailedDeletePodReason, "Error deleting: %v", err) return fmt.Errorf("unable to delete pods: %v", err) @@ -806,7 +806,7 @@ func FilterActivePods(pods []*v1.Pod) []*v1.Pod { if IsPodActive(p) { result = append(result, p) } else { - glog.V(4).Infof("Ignoring inactive pod %v/%v in state %v, deletion time %v", + klog.V(4).Infof("Ignoring inactive pod %v/%v in state %v, deletion time %v", p.Namespace, p.Name, p.Status.Phase, p.DeletionTimestamp) } } @@ -1024,14 +1024,14 @@ func PatchNodeTaints(c clientset.Interface, nodeName string, oldNode *v1.Node, n // indicating that the controller identified by controllerName is waiting for syncs, followed by // either a successful or failed sync. func WaitForCacheSync(controllerName string, stopCh <-chan struct{}, cacheSyncs ...cache.InformerSynced) bool { - glog.Infof("Waiting for caches to sync for %s controller", controllerName) + klog.Infof("Waiting for caches to sync for %s controller", controllerName) if !cache.WaitForCacheSync(stopCh, cacheSyncs...) { utilruntime.HandleError(fmt.Errorf("Unable to sync caches for %s controller", controllerName)) return false } - glog.Infof("Caches are synced for %s controller", controllerName) + klog.Infof("Caches are synced for %s controller", controllerName) return true } diff --git a/pkg/controller/cronjob/BUILD b/pkg/controller/cronjob/BUILD index 82e659f74b4..15b4f652d45 100644 --- a/pkg/controller/cronjob/BUILD +++ b/pkg/controller/cronjob/BUILD @@ -21,12 +21,10 @@ go_library( "//staging/src/k8s.io/api/batch/v1:go_default_library", "//staging/src/k8s.io/api/batch/v1beta1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", @@ -34,8 +32,8 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/tools/reference:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/robfig/cron:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/cronjob/cronjob_controller.go b/pkg/controller/cronjob/cronjob_controller.go index 183645c9662..8e797b41714 100644 --- a/pkg/controller/cronjob/cronjob_controller.go +++ b/pkg/controller/cronjob/cronjob_controller.go @@ -33,16 +33,14 @@ import ( "sort" "time" - "github.com/golang/glog" + "k8s.io/klog" batchv1 "k8s.io/api/batch/v1" batchv1beta1 "k8s.io/api/batch/v1beta1" "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" @@ -68,7 +66,7 @@ type CronJobController struct { func NewCronJobController(kubeClient clientset.Interface) (*CronJobController, error) { eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) if kubeClient != nil && kubeClient.CoreV1().RESTClient().GetRateLimiter() != nil { @@ -91,11 +89,11 @@ func NewCronJobController(kubeClient clientset.Interface) (*CronJobController, e // Run the main goroutine responsible for watching and syncing jobs. func (jm *CronJobController) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() - glog.Infof("Starting CronJob Manager") + klog.Infof("Starting CronJob Manager") // Check things every 10 second. go wait.Until(jm.syncAll, 10*time.Second, stopCh) <-stopCh - glog.Infof("Shutting down CronJob Manager") + klog.Infof("Shutting down CronJob Manager") } // syncAll lists all the CronJobs and Jobs and reconciles them. @@ -110,7 +108,7 @@ func (jm *CronJobController) syncAll() { return } js := jl.Items - glog.V(4).Infof("Found %d jobs", len(js)) + klog.V(4).Infof("Found %d jobs", len(js)) sjl, err := jm.kubeClient.BatchV1beta1().CronJobs(metav1.NamespaceAll).List(metav1.ListOptions{}) if err != nil { @@ -118,20 +116,20 @@ func (jm *CronJobController) syncAll() { return } sjs := sjl.Items - glog.V(4).Infof("Found %d cronjobs", len(sjs)) + klog.V(4).Infof("Found %d cronjobs", len(sjs)) jobsBySj := groupJobsByParent(js) - glog.V(4).Infof("Found %d groups", len(jobsBySj)) + klog.V(4).Infof("Found %d groups", len(jobsBySj)) for _, sj := range sjs { - syncOne(&sj, jobsBySj[sj.UID], time.Now(), jm.jobControl, jm.sjControl, jm.podControl, jm.recorder) - cleanupFinishedJobs(&sj, jobsBySj[sj.UID], jm.jobControl, jm.sjControl, jm.podControl, jm.recorder) + syncOne(&sj, jobsBySj[sj.UID], time.Now(), jm.jobControl, jm.sjControl, jm.recorder) + cleanupFinishedJobs(&sj, jobsBySj[sj.UID], jm.jobControl, jm.sjControl, jm.recorder) } } // cleanupFinishedJobs cleanups finished jobs created by a CronJob func cleanupFinishedJobs(sj *batchv1beta1.CronJob, js []batchv1.Job, jc jobControlInterface, - sjc sjControlInterface, pc podControlInterface, recorder record.EventRecorder) { + sjc sjControlInterface, recorder record.EventRecorder) { // If neither limits are active, there is no need to do anything. if sj.Spec.FailedJobsHistoryLimit == nil && sj.Spec.SuccessfulJobsHistoryLimit == nil { return @@ -153,7 +151,6 @@ func cleanupFinishedJobs(sj *batchv1beta1.CronJob, js []batchv1.Job, jc jobContr removeOldestJobs(sj, succesfulJobs, jc, - pc, *sj.Spec.SuccessfulJobsHistoryLimit, recorder) } @@ -162,7 +159,6 @@ func cleanupFinishedJobs(sj *batchv1beta1.CronJob, js []batchv1.Job, jc jobContr removeOldestJobs(sj, failedJobs, jc, - pc, *sj.Spec.FailedJobsHistoryLimit, recorder) } @@ -170,25 +166,24 @@ func cleanupFinishedJobs(sj *batchv1beta1.CronJob, js []batchv1.Job, jc jobContr // Update the CronJob, in case jobs were removed from the list. if _, err := sjc.UpdateStatus(sj); err != nil { nameForLog := fmt.Sprintf("%s/%s", sj.Namespace, sj.Name) - glog.Infof("Unable to update status for %s (rv = %s): %v", nameForLog, sj.ResourceVersion, err) + klog.Infof("Unable to update status for %s (rv = %s): %v", nameForLog, sj.ResourceVersion, err) } } // removeOldestJobs removes the oldest jobs from a list of jobs -func removeOldestJobs(sj *batchv1beta1.CronJob, js []batchv1.Job, jc jobControlInterface, - pc podControlInterface, maxJobs int32, recorder record.EventRecorder) { +func removeOldestJobs(sj *batchv1beta1.CronJob, js []batchv1.Job, jc jobControlInterface, maxJobs int32, recorder record.EventRecorder) { numToDelete := len(js) - int(maxJobs) if numToDelete <= 0 { return } nameForLog := fmt.Sprintf("%s/%s", sj.Namespace, sj.Name) - glog.V(4).Infof("Cleaning up %d/%d jobs from %s", numToDelete, len(js), nameForLog) + klog.V(4).Infof("Cleaning up %d/%d jobs from %s", numToDelete, len(js), nameForLog) sort.Sort(byJobStartTime(js)) for i := 0; i < numToDelete; i++ { - glog.V(4).Infof("Removing job %s from %s", js[i].Name, nameForLog) - deleteJob(sj, &js[i], jc, pc, recorder, "history limit reached") + klog.V(4).Infof("Removing job %s from %s", js[i].Name, nameForLog) + deleteJob(sj, &js[i], jc, recorder) } } @@ -196,7 +191,7 @@ func removeOldestJobs(sj *batchv1beta1.CronJob, js []batchv1.Job, jc jobControlI // All known jobs created by "sj" should be included in "js". // The current time is passed in to facilitate testing. // It has no receiver, to facilitate testing. -func syncOne(sj *batchv1beta1.CronJob, js []batchv1.Job, now time.Time, jc jobControlInterface, sjc sjControlInterface, pc podControlInterface, recorder record.EventRecorder) { +func syncOne(sj *batchv1beta1.CronJob, js []batchv1.Job, now time.Time, jc jobControlInterface, sjc sjControlInterface, recorder record.EventRecorder) { nameForLog := fmt.Sprintf("%s/%s", sj.Namespace, sj.Name) childrenJobs := make(map[types.UID]bool) @@ -234,7 +229,7 @@ func syncOne(sj *batchv1beta1.CronJob, js []batchv1.Job, now time.Time, jc jobCo updatedSJ, err := sjc.UpdateStatus(sj) if err != nil { - glog.Errorf("Unable to update status for %s (rv = %s): %v", nameForLog, sj.ResourceVersion, err) + klog.Errorf("Unable to update status for %s (rv = %s): %v", nameForLog, sj.ResourceVersion, err) return } *sj = *updatedSJ @@ -246,23 +241,23 @@ func syncOne(sj *batchv1beta1.CronJob, js []batchv1.Job, now time.Time, jc jobCo } if sj.Spec.Suspend != nil && *sj.Spec.Suspend { - glog.V(4).Infof("Not starting job for %s because it is suspended", nameForLog) + klog.V(4).Infof("Not starting job for %s because it is suspended", nameForLog) return } times, err := getRecentUnmetScheduleTimes(*sj, now) if err != nil { recorder.Eventf(sj, v1.EventTypeWarning, "FailedNeedsStart", "Cannot determine if job needs to be started: %v", err) - glog.Errorf("Cannot determine if %s needs to be started: %v", nameForLog, err) + klog.Errorf("Cannot determine if %s needs to be started: %v", nameForLog, err) return } // TODO: handle multiple unmet start times, from oldest to newest, updating status as needed. if len(times) == 0 { - glog.V(4).Infof("No unmet start times for %s", nameForLog) + klog.V(4).Infof("No unmet start times for %s", nameForLog) return } if len(times) > 1 { - glog.V(4).Infof("Multiple unmet start times for %s so only starting last one", nameForLog) + klog.V(4).Infof("Multiple unmet start times for %s so only starting last one", nameForLog) } scheduledTime := times[len(times)-1] @@ -271,7 +266,7 @@ func syncOne(sj *batchv1beta1.CronJob, js []batchv1.Job, now time.Time, jc jobCo tooLate = scheduledTime.Add(time.Second * time.Duration(*sj.Spec.StartingDeadlineSeconds)).Before(now) } if tooLate { - glog.V(4).Infof("Missed starting window for %s", nameForLog) + klog.V(4).Infof("Missed starting window for %s", nameForLog) recorder.Eventf(sj, v1.EventTypeWarning, "MissSchedule", "Missed scheduled time to start a job: %s", scheduledTime.Format(time.RFC1123Z)) // TODO: Since we don't set LastScheduleTime when not scheduling, we are going to keep noticing // the miss every cycle. In order to avoid sending multiple events, and to avoid processing @@ -292,21 +287,19 @@ func syncOne(sj *batchv1beta1.CronJob, js []batchv1.Job, now time.Time, jc jobCo // TODO: for Forbid, we could use the same name for every execution, as a lock. // With replace, we could use a name that is deterministic per execution time. // But that would mean that you could not inspect prior successes or failures of Forbid jobs. - glog.V(4).Infof("Not starting job for %s because of prior execution still running and concurrency policy is Forbid", nameForLog) + klog.V(4).Infof("Not starting job for %s because of prior execution still running and concurrency policy is Forbid", nameForLog) return } if sj.Spec.ConcurrencyPolicy == batchv1beta1.ReplaceConcurrent { for _, j := range sj.Status.Active { - // TODO: this should be replaced with server side job deletion - // currently this mimics JobReaper from pkg/kubectl/stop.go - glog.V(4).Infof("Deleting job %s of %s that was still running at next scheduled start time", j.Name, nameForLog) + klog.V(4).Infof("Deleting job %s of %s that was still running at next scheduled start time", j.Name, nameForLog) job, err := jc.GetJob(j.Namespace, j.Name) if err != nil { recorder.Eventf(sj, v1.EventTypeWarning, "FailedGet", "Get job: %v", err) return } - if !deleteJob(sj, job, jc, pc, recorder, "") { + if !deleteJob(sj, job, jc, recorder) { return } } @@ -314,7 +307,7 @@ func syncOne(sj *batchv1beta1.CronJob, js []batchv1.Job, now time.Time, jc jobCo jobReq, err := getJobFromTemplate(sj, scheduledTime) if err != nil { - glog.Errorf("Unable to make Job from template in %s: %v", nameForLog, err) + klog.Errorf("Unable to make Job from template in %s: %v", nameForLog, err) return } jobResp, err := jc.CreateJob(sj.Namespace, jobReq) @@ -322,7 +315,7 @@ func syncOne(sj *batchv1beta1.CronJob, js []batchv1.Job, now time.Time, jc jobCo recorder.Eventf(sj, v1.EventTypeWarning, "FailedCreate", "Error creating job: %v", err) return } - glog.V(4).Infof("Created Job %s for %s", jobResp.Name, nameForLog) + klog.V(4).Infof("Created Job %s for %s", jobResp.Name, nameForLog) recorder.Eventf(sj, v1.EventTypeNormal, "SuccessfulCreate", "Created job %v", jobResp.Name) // ------------------------------------------------------------------ // @@ -338,62 +331,26 @@ func syncOne(sj *batchv1beta1.CronJob, js []batchv1.Job, now time.Time, jc jobCo // Add the just-started job to the status list. ref, err := getRef(jobResp) if err != nil { - glog.V(2).Infof("Unable to make object reference for job for %s", nameForLog) + klog.V(2).Infof("Unable to make object reference for job for %s", nameForLog) } else { sj.Status.Active = append(sj.Status.Active, *ref) } sj.Status.LastScheduleTime = &metav1.Time{Time: scheduledTime} if _, err := sjc.UpdateStatus(sj); err != nil { - glog.Infof("Unable to update status for %s (rv = %s): %v", nameForLog, sj.ResourceVersion, err) + klog.Infof("Unable to update status for %s (rv = %s): %v", nameForLog, sj.ResourceVersion, err) } return } // deleteJob reaps a job, deleting the job, the pods and the reference in the active list -func deleteJob(sj *batchv1beta1.CronJob, job *batchv1.Job, jc jobControlInterface, - pc podControlInterface, recorder record.EventRecorder, reason string) bool { - // TODO: this should be replaced with server side job deletion - // currently this mimics JobReaper from pkg/kubectl/stop.go +func deleteJob(sj *batchv1beta1.CronJob, job *batchv1.Job, jc jobControlInterface, recorder record.EventRecorder) bool { nameForLog := fmt.Sprintf("%s/%s", sj.Namespace, sj.Name) - // scale job down to 0 - if *job.Spec.Parallelism != 0 { - zero := int32(0) - var err error - job.Spec.Parallelism = &zero - job, err = jc.UpdateJob(job.Namespace, job) - if err != nil { - recorder.Eventf(sj, v1.EventTypeWarning, "FailedUpdate", "Update job: %v", err) - return false - } - } - // remove all pods... - selector, _ := metav1.LabelSelectorAsSelector(job.Spec.Selector) - options := metav1.ListOptions{LabelSelector: selector.String()} - podList, err := pc.ListPods(job.Namespace, options) - if err != nil { - recorder.Eventf(sj, v1.EventTypeWarning, "FailedList", "List job-pods: %v", err) - return false - } - errList := []error{} - for _, pod := range podList.Items { - glog.V(2).Infof("CronJob controller is deleting Pod %v/%v", pod.Namespace, pod.Name) - if err := pc.DeletePod(pod.Namespace, pod.Name); err != nil { - // ignores the error when the pod isn't found - if !errors.IsNotFound(err) { - errList = append(errList, err) - } - } - } - if len(errList) != 0 { - recorder.Eventf(sj, v1.EventTypeWarning, "FailedDelete", "Deleted job-pods: %v", utilerrors.NewAggregate(errList)) - return false - } - // ... the job itself... + // delete the job itself... if err := jc.DeleteJob(job.Namespace, job.Name); err != nil { recorder.Eventf(sj, v1.EventTypeWarning, "FailedDelete", "Deleted job: %v", err) - glog.Errorf("Error deleting job %s from %s: %v", job.Name, nameForLog, err) + klog.Errorf("Error deleting job %s from %s: %v", job.Name, nameForLog, err) return false } // ... and its reference from active list diff --git a/pkg/controller/cronjob/cronjob_controller_test.go b/pkg/controller/cronjob/cronjob_controller_test.go index 63cff8ef049..1ca289d441e 100644 --- a/pkg/controller/cronjob/cronjob_controller_test.go +++ b/pkg/controller/cronjob/cronjob_controller_test.go @@ -17,7 +17,6 @@ limitations under the License. package cronjob import ( - "errors" "strconv" "strings" "testing" @@ -286,10 +285,9 @@ func TestSyncOne_RunOrNot(t *testing.T) { jc := &fakeJobControl{Job: job} sjc := &fakeSJControl{} - pc := &fakePodControl{} recorder := record.NewFakeRecorder(10) - syncOne(&sj, js, tc.now, jc, sjc, pc, recorder) + syncOne(&sj, js, tc.now, jc, sjc, recorder) expectedCreates := 0 if tc.expectCreate { expectedCreates = 1 @@ -485,16 +483,6 @@ func TestCleanupFinishedJobs_DeleteOrNot(t *testing.T) { {"2016-05-19T08:00:00Z", F, F, F, F}, {"2016-05-19T09:00:00Z", F, F, F, F}, }, justBeforeTheHour(), &limitZero, &limitZero, 6}, - - "failed list pod err": { - []CleanupJobSpec{ - {"2016-05-19T04:00:00Z", T, F, F, F}, - {"2016-05-19T05:00:00Z", T, F, F, F}, - {"2016-05-19T06:00:00Z", T, T, F, F}, - {"2016-05-19T07:00:00Z", T, T, F, F}, - {"2016-05-19T08:00:00Z", T, F, F, F}, - {"2016-05-19T09:00:00Z", T, F, F, F}, - }, justBeforeTheHour(), &limitZero, &limitZero, 0}, } for name, tc := range testCases { @@ -563,14 +551,10 @@ func TestCleanupFinishedJobs_DeleteOrNot(t *testing.T) { } jc := &fakeJobControl{Job: job} - pc := &fakePodControl{} sjc := &fakeSJControl{} recorder := record.NewFakeRecorder(10) - if name == "failed list pod err" { - pc.Err = errors.New("fakePodControl err") - } - cleanupFinishedJobs(&sj, js, jc, sjc, pc, recorder) + cleanupFinishedJobs(&sj, js, jc, sjc, recorder) // Check we have actually deleted the correct jobs if len(jc.DeleteJobName) != len(jobsToDelete) { @@ -728,11 +712,10 @@ func TestSyncOne_Status(t *testing.T) { jc := &fakeJobControl{} sjc := &fakeSJControl{} - pc := &fakePodControl{} recorder := record.NewFakeRecorder(10) // Run the code - syncOne(&sj, jobs, tc.now, jc, sjc, pc, recorder) + syncOne(&sj, jobs, tc.now, jc, sjc, recorder) // Status update happens once when ranging through job list, and another one if create jobs. expectUpdates := 1 diff --git a/pkg/controller/cronjob/utils.go b/pkg/controller/cronjob/utils.go index 4ad7ae9994f..19fb91c4baa 100644 --- a/pkg/controller/cronjob/utils.go +++ b/pkg/controller/cronjob/utils.go @@ -20,8 +20,8 @@ import ( "fmt" "time" - "github.com/golang/glog" "github.com/robfig/cron" + "k8s.io/klog" batchv1 "k8s.io/api/batch/v1" batchv1beta1 "k8s.io/api/batch/v1beta1" @@ -64,7 +64,7 @@ func getParentUIDFromJob(j batchv1.Job) (types.UID, bool) { } if controllerRef.Kind != "CronJob" { - glog.V(4).Infof("Job with non-CronJob parent, name %s namespace %s", j.Name, j.Namespace) + klog.V(4).Infof("Job with non-CronJob parent, name %s namespace %s", j.Name, j.Namespace) return types.UID(""), false } @@ -78,7 +78,7 @@ func groupJobsByParent(js []batchv1.Job) map[types.UID][]batchv1.Job { for _, job := range js { parentUID, found := getParentUIDFromJob(job) if !found { - glog.V(4).Infof("Unable to get parent uid from job %s in namespace %s", job.Name, job.Namespace) + klog.V(4).Infof("Unable to get parent uid from job %s in namespace %s", job.Name, job.Namespace) continue } jobsBySj[parentUID] = append(jobsBySj[parentUID], job) diff --git a/pkg/controller/daemon/BUILD b/pkg/controller/daemon/BUILD index a2f2ca131bb..34eba68811e 100644 --- a/pkg/controller/daemon/BUILD +++ b/pkg/controller/daemon/BUILD @@ -53,7 +53,7 @@ go_library( "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//staging/src/k8s.io/client-go/util/integer:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -61,6 +61,7 @@ go_test( name = "go_default_test", srcs = [ "daemon_controller_test.go", + "main_test.go", "update_test.go", ], embed = [":go_default_library"], @@ -84,6 +85,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", diff --git a/pkg/controller/daemon/daemon_controller.go b/pkg/controller/daemon/daemon_controller.go index 7f5015e1053..1b896118f77 100644 --- a/pkg/controller/daemon/daemon_controller.go +++ b/pkg/controller/daemon/daemon_controller.go @@ -23,7 +23,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" @@ -149,7 +149,7 @@ func NewDaemonSetsController( failedPodsBackoff *flowcontrol.Backoff, ) (*DaemonSetsController, error) { eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) if kubeClient != nil && kubeClient.CoreV1().RESTClient().GetRateLimiter() != nil { @@ -176,13 +176,13 @@ func NewDaemonSetsController( daemonSetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { ds := obj.(*apps.DaemonSet) - glog.V(4).Infof("Adding daemon set %s", ds.Name) + klog.V(4).Infof("Adding daemon set %s", ds.Name) dsc.enqueueDaemonSet(ds) }, UpdateFunc: func(old, cur interface{}) { oldDS := old.(*apps.DaemonSet) curDS := cur.(*apps.DaemonSet) - glog.V(4).Infof("Updating daemon set %s", oldDS.Name) + klog.V(4).Infof("Updating daemon set %s", oldDS.Name) dsc.enqueueDaemonSet(curDS) }, DeleteFunc: dsc.deleteDaemonset, @@ -257,7 +257,7 @@ func (dsc *DaemonSetsController) deleteDaemonset(obj interface{}) { return } } - glog.V(4).Infof("Deleting daemon set %s", ds.Name) + klog.V(4).Infof("Deleting daemon set %s", ds.Name) dsc.enqueueDaemonSet(ds) } @@ -266,8 +266,8 @@ func (dsc *DaemonSetsController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer dsc.queue.ShutDown() - glog.Infof("Starting daemon sets controller") - defer glog.Infof("Shutting down daemon sets controller") + klog.Infof("Starting daemon sets controller") + defer klog.Infof("Shutting down daemon sets controller") if !controller.WaitForCacheSync("daemon sets", stopCh, dsc.podStoreSynced, dsc.nodeStoreSynced, dsc.historyStoreSynced, dsc.dsStoreSynced) { return @@ -363,7 +363,7 @@ func (dsc *DaemonSetsController) getDaemonSetsForHistory(history *apps.Controlle if len(daemonSets) > 1 { // ControllerRef will ensure we don't do anything crazy, but more than one // item in this list nevertheless constitutes user error. - glog.V(4).Infof("User error! more than one DaemonSets is selecting ControllerRevision %s/%s with labels: %#v", + klog.V(4).Infof("User error! more than one DaemonSets is selecting ControllerRevision %s/%s with labels: %#v", history.Namespace, history.Name, history.Labels) } return daemonSets @@ -386,7 +386,7 @@ func (dsc *DaemonSetsController) addHistory(obj interface{}) { if ds == nil { return } - glog.V(4).Infof("ControllerRevision %s added.", history.Name) + klog.V(4).Infof("ControllerRevision %s added.", history.Name) return } @@ -396,7 +396,7 @@ func (dsc *DaemonSetsController) addHistory(obj interface{}) { if len(daemonSets) == 0 { return } - glog.V(4).Infof("Orphan ControllerRevision %s added.", history.Name) + klog.V(4).Infof("Orphan ControllerRevision %s added.", history.Name) for _, ds := range daemonSets { dsc.enqueueDaemonSet(ds) } @@ -429,7 +429,7 @@ func (dsc *DaemonSetsController) updateHistory(old, cur interface{}) { if ds == nil { return } - glog.V(4).Infof("ControllerRevision %s updated.", curHistory.Name) + klog.V(4).Infof("ControllerRevision %s updated.", curHistory.Name) dsc.enqueueDaemonSet(ds) return } @@ -442,7 +442,7 @@ func (dsc *DaemonSetsController) updateHistory(old, cur interface{}) { if len(daemonSets) == 0 { return } - glog.V(4).Infof("Orphan ControllerRevision %s updated.", curHistory.Name) + klog.V(4).Infof("Orphan ControllerRevision %s updated.", curHistory.Name) for _, ds := range daemonSets { dsc.enqueueDaemonSet(ds) } @@ -481,7 +481,7 @@ func (dsc *DaemonSetsController) deleteHistory(obj interface{}) { if ds == nil { return } - glog.V(4).Infof("ControllerRevision %s deleted.", history.Name) + klog.V(4).Infof("ControllerRevision %s deleted.", history.Name) dsc.enqueueDaemonSet(ds) } @@ -505,7 +505,7 @@ func (dsc *DaemonSetsController) addPod(obj interface{}) { if err != nil { return } - glog.V(4).Infof("Pod %s added.", pod.Name) + klog.V(4).Infof("Pod %s added.", pod.Name) dsc.expectations.CreationObserved(dsKey) dsc.enqueueDaemonSet(ds) return @@ -519,7 +519,7 @@ func (dsc *DaemonSetsController) addPod(obj interface{}) { if len(dss) == 0 { return } - glog.V(4).Infof("Orphan Pod %s added.", pod.Name) + klog.V(4).Infof("Orphan Pod %s added.", pod.Name) for _, ds := range dss { dsc.enqueueDaemonSet(ds) } @@ -553,7 +553,7 @@ func (dsc *DaemonSetsController) updatePod(old, cur interface{}) { if ds == nil { return } - glog.V(4).Infof("Pod %s updated.", curPod.Name) + klog.V(4).Infof("Pod %s updated.", curPod.Name) dsc.enqueueDaemonSet(ds) changedToReady := !podutil.IsPodReady(oldPod) && podutil.IsPodReady(curPod) // See https://github.com/kubernetes/kubernetes/pull/38076 for more details @@ -571,7 +571,7 @@ func (dsc *DaemonSetsController) updatePod(old, cur interface{}) { if len(dss) == 0 { return } - glog.V(4).Infof("Orphan Pod %s updated.", curPod.Name) + klog.V(4).Infof("Orphan Pod %s updated.", curPod.Name) labelChanged := !reflect.DeepEqual(curPod.Labels, oldPod.Labels) if labelChanged || controllerRefChanged { for _, ds := range dss { @@ -602,10 +602,10 @@ func (dsc *DaemonSetsController) requeueSuspendedDaemonPods(node string) { dss := dsc.listSuspendedDaemonPods(node) for _, dsKey := range dss { if ns, name, err := cache.SplitMetaNamespaceKey(dsKey); err != nil { - glog.Errorf("Failed to get DaemonSet's namespace and name from %s: %v", dsKey, err) + klog.Errorf("Failed to get DaemonSet's namespace and name from %s: %v", dsKey, err) continue } else if ds, err := dsc.dsLister.DaemonSets(ns).Get(name); err != nil { - glog.Errorf("Failed to get DaemonSet %s/%s: %v", ns, name, err) + klog.Errorf("Failed to get DaemonSet %s/%s: %v", ns, name, err) continue } else { dsc.enqueueDaemonSetRateLimited(ds) @@ -682,7 +682,7 @@ func (dsc *DaemonSetsController) deletePod(obj interface{}) { if err != nil { return } - glog.V(4).Infof("Pod %s deleted.", pod.Name) + klog.V(4).Infof("Pod %s deleted.", pod.Name) dsc.expectations.DeletionObserved(dsKey) dsc.enqueueDaemonSet(ds) } @@ -691,7 +691,7 @@ func (dsc *DaemonSetsController) addNode(obj interface{}) { // TODO: it'd be nice to pass a hint with these enqueues, so that each ds would only examine the added node (unless it has other work to do, too). dsList, err := dsc.dsLister.List(labels.Everything()) if err != nil { - glog.V(4).Infof("Error enqueueing daemon sets: %v", err) + klog.V(4).Infof("Error enqueueing daemon sets: %v", err) return } node := obj.(*v1.Node) @@ -753,7 +753,7 @@ func (dsc *DaemonSetsController) updateNode(old, cur interface{}) { dsList, err := dsc.dsLister.List(labels.Everything()) if err != nil { - glog.V(4).Infof("Error listing daemon sets: %v", err) + klog.V(4).Infof("Error listing daemon sets: %v", err) return } // TODO: it'd be nice to pass a hint with these enqueues, so that each ds would only examine the added node (unless it has other work to do, too). @@ -820,7 +820,7 @@ func (dsc *DaemonSetsController) getNodesToDaemonPods(ds *apps.DaemonSet) (map[s for _, pod := range claimedPods { nodeName, err := util.GetTargetNodeName(pod) if err != nil { - glog.Warningf("Failed to get target node name of Pod %v/%v in DaemonSet %v/%v", + klog.Warningf("Failed to get target node name of Pod %v/%v in DaemonSet %v/%v", pod.Namespace, pod.Name, ds.Namespace, ds.Name) continue } @@ -899,7 +899,7 @@ func (dsc *DaemonSetsController) podsShouldBeOnNode( inBackoff := dsc.failedPodsBackoff.IsInBackOffSinceUpdate(backoffKey, now) if inBackoff { delay := dsc.failedPodsBackoff.Get(backoffKey) - glog.V(4).Infof("Deleting failed pod %s/%s on node %s has been limited by backoff - %v remaining", + klog.V(4).Infof("Deleting failed pod %s/%s on node %s has been limited by backoff - %v remaining", pod.Namespace, pod.Name, node.Name, delay) dsc.enqueueDaemonSetAfter(ds, delay) continue @@ -908,7 +908,7 @@ func (dsc *DaemonSetsController) podsShouldBeOnNode( dsc.failedPodsBackoff.Next(backoffKey, now) msg := fmt.Sprintf("Found failed daemon pod %s/%s on node %s, will try to kill it", pod.Namespace, pod.Name, node.Name) - glog.V(2).Infof(msg) + klog.V(2).Infof(msg) // Emit an event so that it's discoverable to users. dsc.eventRecorder.Eventf(ds, v1.EventTypeWarning, FailedDaemonPodReason, msg) podsToDelete = append(podsToDelete, pod.Name) @@ -1003,7 +1003,7 @@ func (dsc *DaemonSetsController) syncNodes(ds *apps.DaemonSet, podsToDelete, nod // error channel to communicate back failures. make the buffer big enough to avoid any blocking errCh := make(chan error, createDiff+deleteDiff) - glog.V(4).Infof("Nodes needing daemon pods for daemon set %s: %+v, creating %d", ds.Name, nodesNeedingDaemonPods, createDiff) + klog.V(4).Infof("Nodes needing daemon pods for daemon set %s: %+v, creating %d", ds.Name, nodesNeedingDaemonPods, createDiff) createWait := sync.WaitGroup{} // If the returned error is not nil we have a parse error. // The controller handles this via the hash. @@ -1057,7 +1057,7 @@ func (dsc *DaemonSetsController) syncNodes(ds *apps.DaemonSet, podsToDelete, nod return } if err != nil { - glog.V(2).Infof("Failed creation, decrementing expectations for set %q/%q", ds.Namespace, ds.Name) + klog.V(2).Infof("Failed creation, decrementing expectations for set %q/%q", ds.Namespace, ds.Name) dsc.expectations.CreationObserved(dsKey) errCh <- err utilruntime.HandleError(err) @@ -1068,7 +1068,7 @@ func (dsc *DaemonSetsController) syncNodes(ds *apps.DaemonSet, podsToDelete, nod // any skipped pods that we never attempted to start shouldn't be expected. skippedPods := createDiff - batchSize if errorCount < len(errCh) && skippedPods > 0 { - glog.V(2).Infof("Slow-start failure. Skipping creation of %d pods, decrementing expectations for set %q/%q", skippedPods, ds.Namespace, ds.Name) + klog.V(2).Infof("Slow-start failure. Skipping creation of %d pods, decrementing expectations for set %q/%q", skippedPods, ds.Namespace, ds.Name) for i := 0; i < skippedPods; i++ { dsc.expectations.CreationObserved(dsKey) } @@ -1078,14 +1078,14 @@ func (dsc *DaemonSetsController) syncNodes(ds *apps.DaemonSet, podsToDelete, nod } } - glog.V(4).Infof("Pods to delete for daemon set %s: %+v, deleting %d", ds.Name, podsToDelete, deleteDiff) + klog.V(4).Infof("Pods to delete for daemon set %s: %+v, deleting %d", ds.Name, podsToDelete, deleteDiff) deleteWait := sync.WaitGroup{} deleteWait.Add(deleteDiff) for i := 0; i < deleteDiff; i++ { go func(ix int) { defer deleteWait.Done() if err := dsc.podControl.DeletePod(ds.Namespace, podsToDelete[ix], ds); err != nil { - glog.V(2).Infof("Failed deletion, decrementing expectations for set %q/%q", ds.Namespace, ds.Name) + klog.V(2).Infof("Failed deletion, decrementing expectations for set %q/%q", ds.Namespace, ds.Name) dsc.expectations.DeletionObserved(dsKey) errCh <- err utilruntime.HandleError(err) @@ -1145,7 +1145,7 @@ func storeDaemonSetStatus(dsClient unversionedapps.DaemonSetInterface, ds *apps. } func (dsc *DaemonSetsController) updateDaemonSetStatus(ds *apps.DaemonSet, hash string, updateObservedGen bool) error { - glog.V(4).Infof("Updating daemon set status") + klog.V(4).Infof("Updating daemon set status") nodeToDaemonPods, err := dsc.getNodesToDaemonPods(ds) if err != nil { return fmt.Errorf("couldn't get node to daemon pod mapping for daemon set %q: %v", ds.Name, err) @@ -1208,7 +1208,7 @@ func (dsc *DaemonSetsController) updateDaemonSetStatus(ds *apps.DaemonSet, hash func (dsc *DaemonSetsController) syncDaemonSet(key string) error { startTime := time.Now() defer func() { - glog.V(4).Infof("Finished syncing daemon set %q (%v)", key, time.Since(startTime)) + klog.V(4).Infof("Finished syncing daemon set %q (%v)", key, time.Since(startTime)) }() namespace, name, err := cache.SplitMetaNamespaceKey(key) @@ -1217,7 +1217,7 @@ func (dsc *DaemonSetsController) syncDaemonSet(key string) error { } ds, err := dsc.dsLister.DaemonSets(namespace).Get(name) if errors.IsNotFound(err) { - glog.V(3).Infof("daemon set has been deleted %v", key) + klog.V(3).Infof("daemon set has been deleted %v", key) dsc.expectations.DeleteExpectations(key) return nil } @@ -1340,7 +1340,7 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *v1.Node, ds *apps. reasons, nodeInfo, err := dsc.simulate(newPod, node, ds) if err != nil { - glog.Warningf("DaemonSet Predicates failed on node %s for ds '%s/%s' due to unexpected error: %v", node.Name, ds.ObjectMeta.Namespace, ds.ObjectMeta.Name, err) + klog.Warningf("DaemonSet Predicates failed on node %s for ds '%s/%s' due to unexpected error: %v", node.Name, ds.ObjectMeta.Namespace, ds.ObjectMeta.Name, err) return false, false, false, err } @@ -1349,7 +1349,7 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *v1.Node, ds *apps. // into one result, e.g. selectedNode. var insufficientResourceErr error for _, r := range reasons { - glog.V(4).Infof("DaemonSet Predicates failed on node %s for ds '%s/%s' for reason: %v", node.Name, ds.ObjectMeta.Namespace, ds.ObjectMeta.Name, r.GetReason()) + klog.V(4).Infof("DaemonSet Predicates failed on node %s for ds '%s/%s' for reason: %v", node.Name, ds.ObjectMeta.Namespace, ds.ObjectMeta.Name, r.GetReason()) switch reason := r.(type) { case *predicates.InsufficientResourceError: insufficientResourceErr = reason @@ -1392,10 +1392,10 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *v1.Node, ds *apps. case predicates.ErrPodAffinityNotMatch, predicates.ErrServiceAffinityViolated: - glog.Warningf("unexpected predicate failure reason: %s", reason.GetReason()) + klog.Warningf("unexpected predicate failure reason: %s", reason.GetReason()) return false, false, false, fmt.Errorf("unexpected reason: DaemonSet Predicates should not return reason %s", reason.GetReason()) default: - glog.V(4).Infof("unknown predicate failure reason: %s", reason.GetReason()) + klog.V(4).Infof("unknown predicate failure reason: %s", reason.GetReason()) wantToRun, shouldSchedule, shouldContinueRunning = false, false, false emitEvent = true } diff --git a/pkg/controller/daemon/daemon_controller_test.go b/pkg/controller/daemon/daemon_controller_test.go index cad5e61e478..8352531fdbd 100644 --- a/pkg/controller/daemon/daemon_controller_test.go +++ b/pkg/controller/daemon/daemon_controller_test.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apiserver/pkg/storage/names" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" @@ -411,7 +412,7 @@ func clearExpectations(t *testing.T, manager *daemonSetsController, ds *apps.Dae func TestDeleteFinalStateUnknown(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { manager, _, _, err := newTestController() if err != nil { @@ -443,16 +444,10 @@ func markPodReady(pod *v1.Pod) { podutil.UpdatePodCondition(&pod.Status, &condition) } -func setFeatureGate(t *testing.T, feature utilfeature.Feature, enabled bool) { - if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=%t", feature, enabled)); err != nil { - t.Fatalf("Failed to set FeatureGate %v to %t: %v", feature, enabled, err) - } -} - // DaemonSets without node selectors should launch pods on every node. func TestSimpleDaemonSetLaunchesPods(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -470,12 +465,7 @@ func TestSimpleDaemonSetLaunchesPods(t *testing.T) { // When ScheduleDaemonSetPods is enabled, DaemonSets without node selectors should // launch pods on every node by NodeAffinity. func TestSimpleDaemonSetScheduleDaemonSetPodsLaunchesPods(t *testing.T) { - enabled := utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) - // Rollback feature gate. - defer func() { - setFeatureGate(t, features.ScheduleDaemonSetPods, enabled) - }() - setFeatureGate(t, features.ScheduleDaemonSetPods, true) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, true)() nodeNum := 5 for _, strategy := range updateStrategies() { @@ -552,7 +542,7 @@ func TestSimpleDaemonSetScheduleDaemonSetPodsLaunchesPods(t *testing.T) { // of 10 pods, and verify that the ds doesn't make 100 create calls per sync pass func TestSimpleDaemonSetPodCreateErrors(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -578,7 +568,7 @@ func TestSimpleDaemonSetPodCreateErrors(t *testing.T) { func TestSimpleDaemonSetUpdatesStatusAfterLaunchingPods(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -614,7 +604,7 @@ func TestSimpleDaemonSetUpdatesStatusAfterLaunchingPods(t *testing.T) { // DaemonSets should do nothing if there aren't any nodes func TestNoNodesDoesNothing(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { manager, podControl, _, err := newTestController() if err != nil { @@ -632,7 +622,7 @@ func TestNoNodesDoesNothing(t *testing.T) { // single node cluster. func TestOneNodeDaemonLaunchesPod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -650,7 +640,7 @@ func TestOneNodeDaemonLaunchesPod(t *testing.T) { // DaemonSets should place onto NotReady nodes func TestNotReadyNodeDaemonDoesLaunchPod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -706,21 +696,7 @@ func allocatableResources(memory, cpu string) v1.ResourceList { // When ScheduleDaemonSetPods is disabled, DaemonSets should not place onto nodes with insufficient free resource func TestInsufficientCapacityNodeDaemonDoesNotLaunchPod(t *testing.T) { - enabled := utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) - // Rollback feature gate. - defer func() { - if enabled { - err := utilfeature.DefaultFeatureGate.Set("ScheduleDaemonSetPods=true") - if err != nil { - t.Fatalf("Failed to enable feature gate for ScheduleDaemonSetPods: %v", err) - } - } - }() - - err := utilfeature.DefaultFeatureGate.Set("ScheduleDaemonSetPods=false") - if err != nil { - t.Fatalf("Failed to disable feature gate for ScheduleDaemonSetPods: %v", err) - } + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, false)() for _, strategy := range updateStrategies() { podSpec := resourcePodSpec("too-much-mem", "75M", "75m") ds := newDaemonSet("foo") @@ -751,7 +727,7 @@ func TestInsufficientCapacityNodeDaemonDoesNotLaunchPod(t *testing.T) { // DaemonSets should not unschedule a daemonset pod from a node with insufficient free resource func TestInsufficientCapacityNodeDaemonDoesNotUnscheduleRunningPod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { podSpec := resourcePodSpec("too-much-mem", "75M", "75m") podSpec.NodeName = "too-much-mem" @@ -792,7 +768,7 @@ func TestInsufficientCapacityNodeDaemonDoesNotUnscheduleRunningPod(t *testing.T) // DaemonSets should only place onto nodes with sufficient free resource and matched node selector func TestInsufficientCapacityNodeSufficientCapacityWithNodeLabelDaemonLaunchPod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() podSpec := resourcePodSpecWithoutNodeName("50M", "75m") ds := newDaemonSet("foo") ds.Spec.Template.Spec = podSpec @@ -819,15 +795,7 @@ func TestInsufficientCapacityNodeSufficientCapacityWithNodeLabelDaemonLaunchPod( // When ScheduleDaemonSetPods is disabled, DaemonSetPods should launch onto node with terminated pods if there // are sufficient resources. func TestSufficientCapacityWithTerminatedPodsDaemonLaunchesPod(t *testing.T) { - enabled := utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) - // Rollback feature gate. - defer func() { - if enabled { - setFeatureGate(t, features.ScheduleDaemonSetPods, true) - } - }() - - setFeatureGate(t, features.ScheduleDaemonSetPods, false) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, false)() for _, strategy := range updateStrategies() { podSpec := resourcePodSpec("too-much-mem", "75M", "75m") ds := newDaemonSet("foo") @@ -851,15 +819,7 @@ func TestSufficientCapacityWithTerminatedPodsDaemonLaunchesPod(t *testing.T) { // When ScheduleDaemonSetPods is disabled, DaemonSets should place onto nodes with sufficient free resources. func TestSufficientCapacityNodeDaemonLaunchesPod(t *testing.T) { - enabled := utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) - // Rollback feature gate. - defer func() { - if enabled { - setFeatureGate(t, features.ScheduleDaemonSetPods, true) - } - }() - - setFeatureGate(t, features.ScheduleDaemonSetPods, false) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, false)() for _, strategy := range updateStrategies() { podSpec := resourcePodSpec("not-too-much-mem", "75M", "75m") @@ -884,7 +844,7 @@ func TestSufficientCapacityNodeDaemonLaunchesPod(t *testing.T) { // DaemonSet should launch a pod on a node with taint NetworkUnavailable condition. func TestNetworkUnavailableNodeDaemonLaunchesPod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("simple") ds.Spec.UpdateStrategy = *strategy @@ -908,7 +868,7 @@ func TestNetworkUnavailableNodeDaemonLaunchesPod(t *testing.T) { // DaemonSets not take any actions when being deleted func TestDontDoAnythingIfBeingDeleted(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { podSpec := resourcePodSpec("not-too-much-mem", "75M", "75m") ds := newDaemonSet("foo") @@ -934,7 +894,7 @@ func TestDontDoAnythingIfBeingDeleted(t *testing.T) { func TestDontDoAnythingIfBeingDeletedRace(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { // Bare client says it IS deleted. ds := newDaemonSet("foo") @@ -963,15 +923,7 @@ func TestDontDoAnythingIfBeingDeletedRace(t *testing.T) { // When ScheduleDaemonSetPods is disabled, DaemonSets should not place onto nodes that would cause port conflicts. func TestPortConflictNodeDaemonDoesNotLaunchPod(t *testing.T) { - enabled := utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) - // Rollback feature gate. - defer func() { - if enabled { - setFeatureGate(t, features.ScheduleDaemonSetPods, true) - } - }() - - setFeatureGate(t, features.ScheduleDaemonSetPods, false) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, false)() for _, strategy := range updateStrategies() { podSpec := v1.PodSpec{ NodeName: "port-conflict", @@ -1005,7 +957,7 @@ func TestPortConflictNodeDaemonDoesNotLaunchPod(t *testing.T) { // Issue: https://github.com/kubernetes/kubernetes/issues/22309 func TestPortConflictWithSameDaemonPodDoesNotDeletePod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { podSpec := v1.PodSpec{ NodeName: "port-conflict", @@ -1035,7 +987,7 @@ func TestPortConflictWithSameDaemonPodDoesNotDeletePod(t *testing.T) { // DaemonSets should place onto nodes that would not cause port conflicts func TestNoPortConflictNodeDaemonLaunchesPod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { podSpec1 := v1.PodSpec{ NodeName: "no-port-conflict", @@ -1085,7 +1037,7 @@ func TestPodIsNotDeletedByDaemonsetWithEmptyLabelSelector(t *testing.T) { // should detect this misconfiguration and choose not to sync the DaemonSet. We should // not observe a deletion of the pod on node1. for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -1118,7 +1070,7 @@ func TestPodIsNotDeletedByDaemonsetWithEmptyLabelSelector(t *testing.T) { // Controller should not create pods on nodes which have daemon pods, and should remove excess pods from nodes that have extra pods. func TestDealsWithExistingPods(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -1140,7 +1092,7 @@ func TestDealsWithExistingPods(t *testing.T) { // Daemon with node selector should launch pods on nodes matching selector. func TestSelectorDaemonLaunchesPods(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { daemon := newDaemonSet("foo") daemon.Spec.UpdateStrategy = *strategy @@ -1160,7 +1112,7 @@ func TestSelectorDaemonLaunchesPods(t *testing.T) { // Daemon with node selector should delete pods from nodes that do not satisfy selector. func TestSelectorDaemonDeletesUnselectedPods(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -1184,7 +1136,7 @@ func TestSelectorDaemonDeletesUnselectedPods(t *testing.T) { // DaemonSet with node selector should launch pods on nodes matching selector, but also deal with existing pods on nodes. func TestSelectorDaemonDealsWithExistingPods(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -1212,7 +1164,7 @@ func TestSelectorDaemonDealsWithExistingPods(t *testing.T) { // DaemonSet with node selector which does not match any node labels should not launch pods. func TestBadSelectorDaemonDoesNothing(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { manager, podControl, _, err := newTestController() if err != nil { @@ -1232,7 +1184,7 @@ func TestBadSelectorDaemonDoesNothing(t *testing.T) { // DaemonSet with node name should launch pod on node with corresponding name. func TestNameDaemonSetLaunchesPods(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -1251,7 +1203,7 @@ func TestNameDaemonSetLaunchesPods(t *testing.T) { // DaemonSet with node name that does not exist should not launch pods. func TestBadNameDaemonSetDoesNothing(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -1270,7 +1222,7 @@ func TestBadNameDaemonSetDoesNothing(t *testing.T) { // DaemonSet with node selector, and node name, matching a node, should launch a pod on the node. func TestNameAndSelectorDaemonSetLaunchesPods(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -1291,7 +1243,7 @@ func TestNameAndSelectorDaemonSetLaunchesPods(t *testing.T) { // DaemonSet with node selector that matches some nodes, and node name that matches a different node, should do nothing. func TestInconsistentNameSelectorDaemonSetDoesNothing(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -1312,7 +1264,7 @@ func TestInconsistentNameSelectorDaemonSetDoesNothing(t *testing.T) { // DaemonSet with node selector, matching some nodes, should launch pods on all the nodes. func TestSelectorDaemonSetLaunchesPods(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() ds := newDaemonSet("foo") ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel manager, podControl, _, err := newTestController(ds) @@ -1329,7 +1281,7 @@ func TestSelectorDaemonSetLaunchesPods(t *testing.T) { // Daemon with node affinity should launch pods on nodes matching affinity. func TestNodeAffinityDaemonLaunchesPods(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { daemon := newDaemonSet("foo") daemon.Spec.UpdateStrategy = *strategy @@ -1365,7 +1317,7 @@ func TestNodeAffinityDaemonLaunchesPods(t *testing.T) { func TestNumberReadyStatus(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -1410,7 +1362,7 @@ func TestNumberReadyStatus(t *testing.T) { func TestObservedGeneration(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -1457,7 +1409,7 @@ func TestDaemonKillFailedPods(t *testing.T) { for _, test := range tests { t.Run(test.test, func(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -1479,7 +1431,7 @@ func TestDaemonKillFailedPods(t *testing.T) { // DaemonSet controller needs to backoff when killing failed pods to avoid hot looping and fighting with kubelet. func TestDaemonKillFailedPodsBackoff(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { t.Run(string(strategy.Type), func(t *testing.T) { ds := newDaemonSet("foo") @@ -1549,7 +1501,7 @@ func TestDaemonKillFailedPodsBackoff(t *testing.T) { // tolerate the nodes NoSchedule taint func TestNoScheduleTaintedDoesntEvicitRunningIntolerantPod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("intolerant") ds.Spec.UpdateStrategy = *strategy @@ -1573,7 +1525,7 @@ func TestNoScheduleTaintedDoesntEvicitRunningIntolerantPod(t *testing.T) { // tolerate the nodes NoExecute taint func TestNoExecuteTaintedDoesEvicitRunningIntolerantPod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("intolerant") ds.Spec.UpdateStrategy = *strategy @@ -1596,7 +1548,7 @@ func TestNoExecuteTaintedDoesEvicitRunningIntolerantPod(t *testing.T) { // DaemonSet should not launch a pod on a tainted node when the pod doesn't tolerate that taint. func TestTaintedNodeDaemonDoesNotLaunchIntolerantPod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("intolerant") ds.Spec.UpdateStrategy = *strategy @@ -1618,7 +1570,7 @@ func TestTaintedNodeDaemonDoesNotLaunchIntolerantPod(t *testing.T) { // DaemonSet should launch a pod on a tainted node when the pod can tolerate that taint. func TestTaintedNodeDaemonLaunchesToleratePod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("tolerate") ds.Spec.UpdateStrategy = *strategy @@ -1641,7 +1593,7 @@ func TestTaintedNodeDaemonLaunchesToleratePod(t *testing.T) { // DaemonSet should launch a pod on a not ready node with taint notReady:NoExecute. func TestNotReadyNodeDaemonLaunchesPod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("simple") ds.Spec.UpdateStrategy = *strategy @@ -1666,7 +1618,7 @@ func TestNotReadyNodeDaemonLaunchesPod(t *testing.T) { // DaemonSet should launch a pod on an unreachable node with taint unreachable:NoExecute. func TestUnreachableNodeDaemonLaunchesPod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("simple") ds.Spec.UpdateStrategy = *strategy @@ -1691,7 +1643,7 @@ func TestUnreachableNodeDaemonLaunchesPod(t *testing.T) { // DaemonSet should launch a pod on an untainted node when the pod has tolerations. func TestNodeDaemonLaunchesToleratePod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("tolerate") ds.Spec.UpdateStrategy = *strategy @@ -1711,7 +1663,7 @@ func TestNodeDaemonLaunchesToleratePod(t *testing.T) { // DaemonSet should launch a pod on a not ready node with taint notReady:NoExecute. func TestDaemonSetRespectsTermination(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -1742,12 +1694,8 @@ func setDaemonSetToleration(ds *apps.DaemonSet, tolerations []v1.Toleration) { // DaemonSet should launch a critical pod even when the node with OutOfDisk taints. // TODO(#48843) OutOfDisk taints will be removed in 1.10 func TestTaintOutOfDiskNodeDaemonLaunchesCriticalPod(t *testing.T) { - enabled := utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) - defer func() { - setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, enabled) - }() for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("critical") ds.Spec.UpdateStrategy = *strategy @@ -1765,12 +1713,12 @@ func TestTaintOutOfDiskNodeDaemonLaunchesCriticalPod(t *testing.T) { // NOTE: Whether or not TaintNodesByCondition is enabled, it'll add toleration to DaemonSet pods. // Without enabling critical pod annotation feature gate, we shouldn't create critical pod - setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, false) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, false)() manager.dsStore.Add(ds) syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) // With enabling critical pod annotation feature gate, we will create critical pod - setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, true) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() manager.dsStore.Add(ds) syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } @@ -1779,13 +1727,8 @@ func TestTaintOutOfDiskNodeDaemonLaunchesCriticalPod(t *testing.T) { // DaemonSet should launch a pod even when the node with MemoryPressure/DiskPressure taints. func TestTaintPressureNodeDaemonLaunchesPod(t *testing.T) { - enabled := utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition) - defer func() { - setFeatureGate(t, features.TaintNodesByCondition, enabled) - }() - for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("critical") ds.Spec.UpdateStrategy = *strategy @@ -1807,7 +1750,7 @@ func TestTaintPressureNodeDaemonLaunchesPod(t *testing.T) { manager.nodeStore.Add(node) // Enabling critical pod and taint nodes by condition feature gate should create critical pod - setFeatureGate(t, features.TaintNodesByCondition, true) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.TaintNodesByCondition, true)() manager.dsStore.Add(ds) syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } @@ -1816,10 +1759,7 @@ func TestTaintPressureNodeDaemonLaunchesPod(t *testing.T) { // When ScheduleDaemonSetPods is disabled, DaemonSet should launch a critical pod even when the node has insufficient free resource. func TestInsufficientCapacityNodeDaemonLaunchesCriticalPod(t *testing.T) { - enabled := utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) - defer func() { - setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, enabled) - }() + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, false)() for _, strategy := range updateStrategies() { podSpec := resourcePodSpec("too-much-mem", "75M", "75m") ds := newDaemonSet("critical") @@ -1839,7 +1779,7 @@ func TestInsufficientCapacityNodeDaemonLaunchesCriticalPod(t *testing.T) { }) // Without enabling critical pod annotation feature gate, we shouldn't create critical pod - setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, false) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, false)() manager.dsStore.Add(ds) switch strategy.Type { case apps.OnDeleteDaemonSetStrategyType: @@ -1851,7 +1791,7 @@ func TestInsufficientCapacityNodeDaemonLaunchesCriticalPod(t *testing.T) { } // Enabling critical pod annotation feature gate should create critical pod - setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, true) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() switch strategy.Type { case apps.OnDeleteDaemonSetStrategyType: syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 2) @@ -1865,11 +1805,7 @@ func TestInsufficientCapacityNodeDaemonLaunchesCriticalPod(t *testing.T) { // When ScheduleDaemonSetPods is disabled, DaemonSets should NOT launch a critical pod when there are port conflicts. func TestPortConflictNodeDaemonDoesNotLaunchCriticalPod(t *testing.T) { - enabled := utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) - defer func() { - setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, enabled) - }() - + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, false)() for _, strategy := range updateStrategies() { podSpec := v1.PodSpec{ NodeName: "port-conflict", @@ -1889,7 +1825,7 @@ func TestPortConflictNodeDaemonDoesNotLaunchCriticalPod(t *testing.T) { Spec: podSpec, }) - setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, true) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() ds := newDaemonSet("critical") ds.Spec.UpdateStrategy = *strategy ds.Spec.Template.Spec = podSpec @@ -1909,7 +1845,7 @@ func setDaemonSetCritical(ds *apps.DaemonSet) { func TestNodeShouldRunDaemonPod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() var shouldCreate, wantToRun, shouldContinueRunning bool if utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) { shouldCreate = true @@ -2236,7 +2172,7 @@ func TestNodeShouldRunDaemonPod(t *testing.T) { func TestUpdateNode(t *testing.T) { var enqueued bool for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() cases := []struct { test string newNode *v1.Node @@ -2545,7 +2481,7 @@ func TestDeleteNoDaemonPod(t *testing.T) { func TestGetNodesToDaemonPods(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -2611,7 +2547,7 @@ func TestGetNodesToDaemonPods(t *testing.T) { func TestAddNode(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() manager, _, _, err := newTestController() if err != nil { t.Fatalf("error creating DaemonSets controller: %v", err) @@ -2640,7 +2576,7 @@ func TestAddNode(t *testing.T) { func TestAddPod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { manager, _, _, err := newTestController() if err != nil { @@ -2686,7 +2622,7 @@ func TestAddPod(t *testing.T) { func TestAddPodOrphan(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { manager, _, _, err := newTestController() if err != nil { @@ -2718,7 +2654,7 @@ func TestAddPodOrphan(t *testing.T) { func TestUpdatePod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { manager, _, _, err := newTestController() if err != nil { @@ -2768,7 +2704,7 @@ func TestUpdatePod(t *testing.T) { func TestUpdatePodOrphanSameLabels(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { manager, _, _, err := newTestController() @@ -2795,7 +2731,7 @@ func TestUpdatePodOrphanSameLabels(t *testing.T) { func TestUpdatePodOrphanWithNewLabels(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { manager, _, _, err := newTestController() @@ -2826,7 +2762,7 @@ func TestUpdatePodOrphanWithNewLabels(t *testing.T) { func TestUpdatePodChangeControllerRef(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") @@ -2854,7 +2790,7 @@ func TestUpdatePodChangeControllerRef(t *testing.T) { func TestUpdatePodControllerRefRemoved(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { manager, _, _, err := newTestController() @@ -2882,7 +2818,7 @@ func TestUpdatePodControllerRefRemoved(t *testing.T) { func TestDeletePod(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { manager, _, _, err := newTestController() @@ -2929,7 +2865,7 @@ func TestDeletePod(t *testing.T) { func TestDeletePodOrphan(t *testing.T) { for _, f := range []bool{true, false} { - setFeatureGate(t, features.ScheduleDaemonSetPods, f) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, f)() for _, strategy := range updateStrategies() { manager, _, _, err := newTestController() diff --git a/pkg/controller/daemon/main_test.go b/pkg/controller/daemon/main_test.go new file mode 100644 index 00000000000..4fe1e9885c2 --- /dev/null +++ b/pkg/controller/daemon/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package daemon + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/controller/daemon/update.go b/pkg/controller/daemon/update.go index 74e42cc59f6..ac13b32fb27 100644 --- a/pkg/controller/daemon/update.go +++ b/pkg/controller/daemon/update.go @@ -22,7 +22,7 @@ import ( "reflect" "sort" - "github.com/golang/glog" + "k8s.io/klog" apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" @@ -55,23 +55,23 @@ func (dsc *DaemonSetsController) rollingUpdate(ds *apps.DaemonSet, hash string) // for oldPods delete all not running pods var oldPodsToDelete []string - glog.V(4).Infof("Marking all unavailable old pods for deletion") + klog.V(4).Infof("Marking all unavailable old pods for deletion") for _, pod := range oldUnavailablePods { // Skip terminating pods. We won't delete them again if pod.DeletionTimestamp != nil { continue } - glog.V(4).Infof("Marking pod %s/%s for deletion", ds.Name, pod.Name) + klog.V(4).Infof("Marking pod %s/%s for deletion", ds.Name, pod.Name) oldPodsToDelete = append(oldPodsToDelete, pod.Name) } - glog.V(4).Infof("Marking old pods for deletion") + klog.V(4).Infof("Marking old pods for deletion") for _, pod := range oldAvailablePods { if numUnavailable >= maxUnavailable { - glog.V(4).Infof("Number of unavailable DaemonSet pods: %d, is equal to or exceeds allowed maximum: %d", numUnavailable, maxUnavailable) + klog.V(4).Infof("Number of unavailable DaemonSet pods: %d, is equal to or exceeds allowed maximum: %d", numUnavailable, maxUnavailable) break } - glog.V(4).Infof("Marking pod %s/%s for deletion", ds.Name, pod.Name) + klog.V(4).Infof("Marking pod %s/%s for deletion", ds.Name, pod.Name) oldPodsToDelete = append(oldPodsToDelete, pod.Name) numUnavailable++ } @@ -364,7 +364,7 @@ func (dsc *DaemonSetsController) snapshot(ds *apps.DaemonSet, revision int64) (* if updateErr != nil { return nil, updateErr } - glog.V(2).Infof("Found a hash collision for DaemonSet %q - bumping collisionCount to %d to resolve it", ds.Name, *currDS.Status.CollisionCount) + klog.V(2).Infof("Found a hash collision for DaemonSet %q - bumping collisionCount to %d to resolve it", ds.Name, *currDS.Status.CollisionCount) return nil, outerErr } return history, err @@ -393,7 +393,7 @@ func (dsc *DaemonSetsController) getAllDaemonSetPods(ds *apps.DaemonSet, nodeToD } func (dsc *DaemonSetsController) getUnavailableNumbers(ds *apps.DaemonSet, nodeToDaemonPods map[string][]*v1.Pod) (int, int, error) { - glog.V(4).Infof("Getting unavailable numbers") + klog.V(4).Infof("Getting unavailable numbers") // TODO: get nodeList once in syncDaemonSet and pass it to other functions nodeList, err := dsc.nodeLister.List(labels.Everything()) if err != nil { @@ -432,7 +432,7 @@ func (dsc *DaemonSetsController) getUnavailableNumbers(ds *apps.DaemonSet, nodeT if err != nil { return -1, -1, fmt.Errorf("Invalid value for MaxUnavailable: %v", err) } - glog.V(4).Infof(" DaemonSet %s/%s, maxUnavailable: %d, numUnavailable: %d", ds.Namespace, ds.Name, maxUnavailable, numUnavailable) + klog.V(4).Infof(" DaemonSet %s/%s, maxUnavailable: %d, numUnavailable: %d", ds.Namespace, ds.Name, maxUnavailable, numUnavailable) return maxUnavailable, numUnavailable, nil } diff --git a/pkg/controller/daemon/util/BUILD b/pkg/controller/daemon/util/BUILD index feeb672c67c..0292ffce296 100644 --- a/pkg/controller/daemon/util/BUILD +++ b/pkg/controller/daemon/util/BUILD @@ -39,7 +39,10 @@ filegroup( go_test( name = "go_default_test", - srcs = ["daemonset_util_test.go"], + srcs = [ + "daemonset_util_test.go", + "main_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/api/testapi:go_default_library", @@ -50,5 +53,6 @@ go_test( "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/pkg/controller/daemon/util/main_test.go b/pkg/controller/daemon/util/main_test.go new file mode 100644 index 00000000000..6af02d0a11d --- /dev/null +++ b/pkg/controller/daemon/util/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/controller/deployment/BUILD b/pkg/controller/deployment/BUILD index 1567169a168..7f1964a4f7e 100644 --- a/pkg/controller/deployment/BUILD +++ b/pkg/controller/deployment/BUILD @@ -42,7 +42,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/integer:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/deployment/deployment_controller.go b/pkg/controller/deployment/deployment_controller.go index 003dfb1ff23..702e49d89c8 100644 --- a/pkg/controller/deployment/deployment_controller.go +++ b/pkg/controller/deployment/deployment_controller.go @@ -25,7 +25,7 @@ import ( "reflect" "time" - "github.com/golang/glog" + "k8s.io/klog" apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" @@ -99,7 +99,7 @@ type DeploymentController struct { // NewDeploymentController creates a new DeploymentController. func NewDeploymentController(dInformer appsinformers.DeploymentInformer, rsInformer appsinformers.ReplicaSetInformer, podInformer coreinformers.PodInformer, client clientset.Interface) (*DeploymentController, error) { eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")}) if client != nil && client.CoreV1().RESTClient().GetRateLimiter() != nil { @@ -149,8 +149,8 @@ func (dc *DeploymentController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer dc.queue.ShutDown() - glog.Infof("Starting deployment controller") - defer glog.Infof("Shutting down deployment controller") + klog.Infof("Starting deployment controller") + defer klog.Infof("Shutting down deployment controller") if !controller.WaitForCacheSync("deployment", stopCh, dc.dListerSynced, dc.rsListerSynced, dc.podListerSynced) { return @@ -165,14 +165,14 @@ func (dc *DeploymentController) Run(workers int, stopCh <-chan struct{}) { func (dc *DeploymentController) addDeployment(obj interface{}) { d := obj.(*apps.Deployment) - glog.V(4).Infof("Adding deployment %s", d.Name) + klog.V(4).Infof("Adding deployment %s", d.Name) dc.enqueueDeployment(d) } func (dc *DeploymentController) updateDeployment(old, cur interface{}) { oldD := old.(*apps.Deployment) curD := cur.(*apps.Deployment) - glog.V(4).Infof("Updating deployment %s", oldD.Name) + klog.V(4).Infof("Updating deployment %s", oldD.Name) dc.enqueueDeployment(curD) } @@ -190,7 +190,7 @@ func (dc *DeploymentController) deleteDeployment(obj interface{}) { return } } - glog.V(4).Infof("Deleting deployment %s", d.Name) + klog.V(4).Infof("Deleting deployment %s", d.Name) dc.enqueueDeployment(d) } @@ -211,7 +211,7 @@ func (dc *DeploymentController) addReplicaSet(obj interface{}) { if d == nil { return } - glog.V(4).Infof("ReplicaSet %s added.", rs.Name) + klog.V(4).Infof("ReplicaSet %s added.", rs.Name) dc.enqueueDeployment(d) return } @@ -222,7 +222,7 @@ func (dc *DeploymentController) addReplicaSet(obj interface{}) { if len(ds) == 0 { return } - glog.V(4).Infof("Orphan ReplicaSet %s added.", rs.Name) + klog.V(4).Infof("Orphan ReplicaSet %s added.", rs.Name) for _, d := range ds { dc.enqueueDeployment(d) } @@ -242,7 +242,7 @@ func (dc *DeploymentController) getDeploymentsForReplicaSet(rs *apps.ReplicaSet) if len(deployments) > 1 { // ControllerRef will ensure we don't do anything crazy, but more than one // item in this list nevertheless constitutes user error. - glog.V(4).Infof("user error! more than one deployment is selecting replica set %s/%s with labels: %#v, returning %s/%s", + klog.V(4).Infof("user error! more than one deployment is selecting replica set %s/%s with labels: %#v, returning %s/%s", rs.Namespace, rs.Name, rs.Labels, deployments[0].Namespace, deployments[0].Name) } return deployments @@ -277,7 +277,7 @@ func (dc *DeploymentController) updateReplicaSet(old, cur interface{}) { if d == nil { return } - glog.V(4).Infof("ReplicaSet %s updated.", curRS.Name) + klog.V(4).Infof("ReplicaSet %s updated.", curRS.Name) dc.enqueueDeployment(d) return } @@ -290,7 +290,7 @@ func (dc *DeploymentController) updateReplicaSet(old, cur interface{}) { if len(ds) == 0 { return } - glog.V(4).Infof("Orphan ReplicaSet %s updated.", curRS.Name) + klog.V(4).Infof("Orphan ReplicaSet %s updated.", curRS.Name) for _, d := range ds { dc.enqueueDeployment(d) } @@ -329,7 +329,7 @@ func (dc *DeploymentController) deleteReplicaSet(obj interface{}) { if d == nil { return } - glog.V(4).Infof("ReplicaSet %s deleted.", rs.Name) + klog.V(4).Infof("ReplicaSet %s deleted.", rs.Name) dc.enqueueDeployment(d) } @@ -353,7 +353,7 @@ func (dc *DeploymentController) deletePod(obj interface{}) { return } } - glog.V(4).Infof("Pod %s deleted.", pod.Name) + klog.V(4).Infof("Pod %s deleted.", pod.Name) if d := dc.getDeploymentForPod(pod); d != nil && d.Spec.Strategy.Type == apps.RecreateDeploymentStrategyType { // Sync if this Deployment now has no more Pods. rsList, err := util.ListReplicaSets(d, util.RsListFromClient(dc.client.AppsV1())) @@ -421,7 +421,7 @@ func (dc *DeploymentController) getDeploymentForPod(pod *v1.Pod) *apps.Deploymen } rs, err = dc.rsLister.ReplicaSets(pod.Namespace).Get(controllerRef.Name) if err != nil || rs.UID != controllerRef.UID { - glog.V(4).Infof("Cannot get replicaset %q for pod %q: %v", controllerRef.Name, pod.Name, err) + klog.V(4).Infof("Cannot get replicaset %q for pod %q: %v", controllerRef.Name, pod.Name, err) return nil } @@ -481,13 +481,13 @@ func (dc *DeploymentController) handleErr(err error, key interface{}) { } if dc.queue.NumRequeues(key) < maxRetries { - glog.V(2).Infof("Error syncing deployment %v: %v", key, err) + klog.V(2).Infof("Error syncing deployment %v: %v", key, err) dc.queue.AddRateLimited(key) return } utilruntime.HandleError(err) - glog.V(2).Infof("Dropping deployment %q out of the queue: %v", key, err) + klog.V(2).Infof("Dropping deployment %q out of the queue: %v", key, err) dc.queue.Forget(key) } @@ -559,9 +559,9 @@ func (dc *DeploymentController) getPodMapForDeployment(d *apps.Deployment, rsLis // This function is not meant to be invoked concurrently with the same key. func (dc *DeploymentController) syncDeployment(key string) error { startTime := time.Now() - glog.V(4).Infof("Started syncing deployment %q (%v)", key, startTime) + klog.V(4).Infof("Started syncing deployment %q (%v)", key, startTime) defer func() { - glog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Since(startTime)) + klog.V(4).Infof("Finished syncing deployment %q (%v)", key, time.Since(startTime)) }() namespace, name, err := cache.SplitMetaNamespaceKey(key) @@ -570,7 +570,7 @@ func (dc *DeploymentController) syncDeployment(key string) error { } deployment, err := dc.dLister.Deployments(namespace).Get(name) if errors.IsNotFound(err) { - glog.V(2).Infof("Deployment %v has been deleted", key) + klog.V(2).Infof("Deployment %v has been deleted", key) return nil } if err != nil { diff --git a/pkg/controller/deployment/progress.go b/pkg/controller/deployment/progress.go index 601359c79c4..e340a5be89d 100644 --- a/pkg/controller/deployment/progress.go +++ b/pkg/controller/deployment/progress.go @@ -21,7 +21,7 @@ import ( "reflect" "time" - "github.com/golang/glog" + "k8s.io/klog" apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" @@ -186,11 +186,11 @@ func (dc *DeploymentController) requeueStuckDeployment(d *apps.Deployment, newSt // Make it ratelimited so we stay on the safe side, eventually the Deployment should // transition either to a Complete or to a TimedOut condition. if after < time.Second { - glog.V(4).Infof("Queueing up deployment %q for a progress check now", d.Name) + klog.V(4).Infof("Queueing up deployment %q for a progress check now", d.Name) dc.enqueueRateLimited(d) return time.Duration(0) } - glog.V(4).Infof("Queueing up deployment %q for a progress check after %ds", d.Name, int(after.Seconds())) + klog.V(4).Infof("Queueing up deployment %q for a progress check after %ds", d.Name, int(after.Seconds())) // Add a second to avoid milliseconds skew in AddAfter. // See https://github.com/kubernetes/kubernetes/issues/39785#issuecomment-279959133 for more info. dc.enqueueAfter(d, after+time.Second) diff --git a/pkg/controller/deployment/rollback.go b/pkg/controller/deployment/rollback.go index 74396efdc3b..6593630af41 100644 --- a/pkg/controller/deployment/rollback.go +++ b/pkg/controller/deployment/rollback.go @@ -20,7 +20,7 @@ import ( "fmt" "strconv" - "github.com/golang/glog" + "k8s.io/klog" apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" @@ -49,11 +49,11 @@ func (dc *DeploymentController) rollback(d *apps.Deployment, rsList []*apps.Repl for _, rs := range allRSs { v, err := deploymentutil.Revision(rs) if err != nil { - glog.V(4).Infof("Unable to extract revision from deployment's replica set %q: %v", rs.Name, err) + klog.V(4).Infof("Unable to extract revision from deployment's replica set %q: %v", rs.Name, err) continue } if v == rollbackTo.Revision { - glog.V(4).Infof("Found replica set %q with desired revision %d", rs.Name, v) + klog.V(4).Infof("Found replica set %q with desired revision %d", rs.Name, v) // rollback by copying podTemplate.Spec from the replica set // revision number will be incremented during the next getAllReplicaSetsAndSyncRevision call // no-op if the spec matches current deployment's podTemplate.Spec @@ -75,7 +75,7 @@ func (dc *DeploymentController) rollback(d *apps.Deployment, rsList []*apps.Repl func (dc *DeploymentController) rollbackToTemplate(d *apps.Deployment, rs *apps.ReplicaSet) (bool, error) { performedRollback := false if !deploymentutil.EqualIgnoreHash(&d.Spec.Template, &rs.Spec.Template) { - glog.V(4).Infof("Rolling back deployment %q to template spec %+v", d.Name, rs.Spec.Template.Spec) + klog.V(4).Infof("Rolling back deployment %q to template spec %+v", d.Name, rs.Spec.Template.Spec) deploymentutil.SetFromReplicaSetTemplate(d, rs.Spec.Template) // set RS (the old RS we'll rolling back to) annotations back to the deployment; // otherwise, the deployment's current annotations (should be the same as current new RS) will be copied to the RS after the rollback. @@ -91,7 +91,7 @@ func (dc *DeploymentController) rollbackToTemplate(d *apps.Deployment, rs *apps. deploymentutil.SetDeploymentAnnotationsTo(d, rs) performedRollback = true } else { - glog.V(4).Infof("Rolling back to a revision that contains the same template as current deployment %q, skipping rollback...", d.Name) + klog.V(4).Infof("Rolling back to a revision that contains the same template as current deployment %q, skipping rollback...", d.Name) eventMsg := fmt.Sprintf("The rollback revision contains the same template as current deployment %q", d.Name) dc.emitRollbackWarningEvent(d, deploymentutil.RollbackTemplateUnchanged, eventMsg) } @@ -111,7 +111,7 @@ func (dc *DeploymentController) emitRollbackNormalEvent(d *apps.Deployment, mess // It is assumed that the caller will have updated the deployment template appropriately (in case // we want to rollback). func (dc *DeploymentController) updateDeploymentAndClearRollbackTo(d *apps.Deployment) error { - glog.V(4).Infof("Cleans up rollbackTo of deployment %q", d.Name) + klog.V(4).Infof("Cleans up rollbackTo of deployment %q", d.Name) setRollbackTo(d, nil) _, err := dc.client.AppsV1().Deployments(d.Namespace).Update(d) return err diff --git a/pkg/controller/deployment/rolling.go b/pkg/controller/deployment/rolling.go index 9dc77331b61..6950c3e6798 100644 --- a/pkg/controller/deployment/rolling.go +++ b/pkg/controller/deployment/rolling.go @@ -20,9 +20,9 @@ import ( "fmt" "sort" - "github.com/golang/glog" apps "k8s.io/api/apps/v1" "k8s.io/client-go/util/integer" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller" deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util" ) @@ -91,7 +91,7 @@ func (dc *DeploymentController) reconcileOldReplicaSets(allRSs []*apps.ReplicaSe } allPodsCount := deploymentutil.GetReplicaCountForReplicaSets(allRSs) - glog.V(4).Infof("New replica set %s/%s has %d available pods.", newRS.Namespace, newRS.Name, newRS.Status.AvailableReplicas) + klog.V(4).Infof("New replica set %s/%s has %d available pods.", newRS.Namespace, newRS.Name, newRS.Status.AvailableReplicas) maxUnavailable := deploymentutil.MaxUnavailable(*deployment) // Check if we can scale down. We can scale down in the following 2 cases: @@ -137,7 +137,7 @@ func (dc *DeploymentController) reconcileOldReplicaSets(allRSs []*apps.ReplicaSe if err != nil { return false, nil } - glog.V(4).Infof("Cleaned up unhealthy replicas from old RSes by %d", cleanupCount) + klog.V(4).Infof("Cleaned up unhealthy replicas from old RSes by %d", cleanupCount) // Scale down old replica sets, need check maxUnavailable to ensure we can scale down allRSs = append(oldRSs, newRS) @@ -145,7 +145,7 @@ func (dc *DeploymentController) reconcileOldReplicaSets(allRSs []*apps.ReplicaSe if err != nil { return false, nil } - glog.V(4).Infof("Scaled down old RSes of deployment %s by %d", deployment.Name, scaledDownCount) + klog.V(4).Infof("Scaled down old RSes of deployment %s by %d", deployment.Name, scaledDownCount) totalScaledDown := cleanupCount + scaledDownCount return totalScaledDown > 0, nil @@ -166,7 +166,7 @@ func (dc *DeploymentController) cleanupUnhealthyReplicas(oldRSs []*apps.ReplicaS // cannot scale down this replica set. continue } - glog.V(4).Infof("Found %d available pods in old RS %s/%s", targetRS.Status.AvailableReplicas, targetRS.Namespace, targetRS.Name) + klog.V(4).Infof("Found %d available pods in old RS %s/%s", targetRS.Status.AvailableReplicas, targetRS.Namespace, targetRS.Name) if *(targetRS.Spec.Replicas) == targetRS.Status.AvailableReplicas { // no unhealthy replicas found, no scaling required. continue @@ -200,7 +200,7 @@ func (dc *DeploymentController) scaleDownOldReplicaSetsForRollingUpdate(allRSs [ // Cannot scale down. return 0, nil } - glog.V(4).Infof("Found %d available pods in deployment %s, scaling down old RSes", availablePodCount, deployment.Name) + klog.V(4).Infof("Found %d available pods in deployment %s, scaling down old RSes", availablePodCount, deployment.Name) sort.Sort(controller.ReplicaSetsByCreationTimestamp(oldRSs)) diff --git a/pkg/controller/deployment/sync.go b/pkg/controller/deployment/sync.go index 7f2ed0d6016..b59aec417db 100644 --- a/pkg/controller/deployment/sync.go +++ b/pkg/controller/deployment/sync.go @@ -22,11 +22,11 @@ import ( "sort" "strconv" - "github.com/golang/glog" apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller" deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util" labelsutil "k8s.io/kubernetes/pkg/util/labels" @@ -248,7 +248,7 @@ func (dc *DeploymentController) getNewReplicaSet(d *apps.Deployment, rsList, old // error. _, dErr := dc.client.AppsV1().Deployments(d.Namespace).UpdateStatus(d) if dErr == nil { - glog.V(2).Infof("Found a hash collision for deployment %q - bumping collisionCount (%d->%d) to resolve it", d.Name, preCollisionCount, *d.Status.CollisionCount) + klog.V(2).Infof("Found a hash collision for deployment %q - bumping collisionCount (%d->%d) to resolve it", d.Name, preCollisionCount, *d.Status.CollisionCount) } return nil, err case err != nil: @@ -440,7 +440,7 @@ func (dc *DeploymentController) cleanupDeployment(oldRSs []*apps.ReplicaSet, dep } sort.Sort(controller.ReplicaSetsByCreationTimestamp(cleanableRSes)) - glog.V(4).Infof("Looking to cleanup old replica sets for deployment %q", deployment.Name) + klog.V(4).Infof("Looking to cleanup old replica sets for deployment %q", deployment.Name) for i := int32(0); i < diff; i++ { rs := cleanableRSes[i] @@ -448,7 +448,7 @@ func (dc *DeploymentController) cleanupDeployment(oldRSs []*apps.ReplicaSet, dep if rs.Status.Replicas != 0 || *(rs.Spec.Replicas) != 0 || rs.Generation > rs.Status.ObservedGeneration || rs.DeletionTimestamp != nil { continue } - glog.V(4).Infof("Trying to cleanup replica set %q for deployment %q", rs.Name, deployment.Name) + klog.V(4).Infof("Trying to cleanup replica set %q for deployment %q", rs.Name, deployment.Name) if err := dc.client.AppsV1().ReplicaSets(rs.Namespace).Delete(rs.Name, nil); err != nil && !errors.IsNotFound(err) { // Return error instead of aggregating and continuing DELETEs on the theory // that we may be overloading the api server. diff --git a/pkg/controller/deployment/util/BUILD b/pkg/controller/deployment/util/BUILD index faba8b5e1a6..2b563e00ae2 100644 --- a/pkg/controller/deployment/util/BUILD +++ b/pkg/controller/deployment/util/BUILD @@ -24,7 +24,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/apps/v1:go_default_library", "//staging/src/k8s.io/client-go/util/integer:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/deployment/util/deployment_util.go b/pkg/controller/deployment/util/deployment_util.go index acdb472746d..00e5c4c36e6 100644 --- a/pkg/controller/deployment/util/deployment_util.go +++ b/pkg/controller/deployment/util/deployment_util.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" @@ -186,7 +186,7 @@ func MaxRevision(allRSs []*apps.ReplicaSet) int64 { for _, rs := range allRSs { if v, err := Revision(rs); err != nil { // Skip the replica sets when it failed to parse their revision information - glog.V(4).Infof("Error: %v. Couldn't parse revision for replica set %#v, deployment controller will skip it when reconciling revisions.", err, rs) + klog.V(4).Infof("Error: %v. Couldn't parse revision for replica set %#v, deployment controller will skip it when reconciling revisions.", err, rs) } else if v > max { max = v } @@ -200,7 +200,7 @@ func LastRevision(allRSs []*apps.ReplicaSet) int64 { for _, rs := range allRSs { if v, err := Revision(rs); err != nil { // Skip the replica sets when it failed to parse their revision information - glog.V(4).Infof("Error: %v. Couldn't parse revision for replica set %#v, deployment controller will skip it when reconciling revisions.", err, rs) + klog.V(4).Infof("Error: %v. Couldn't parse revision for replica set %#v, deployment controller will skip it when reconciling revisions.", err, rs) } else if v >= max { secMax = max max = v @@ -241,7 +241,7 @@ func SetNewReplicaSetAnnotations(deployment *apps.Deployment, newRS *apps.Replic oldRevisionInt, err := strconv.ParseInt(oldRevision, 10, 64) if err != nil { if oldRevision != "" { - glog.Warningf("Updating replica set revision OldRevision not int %s", err) + klog.Warningf("Updating replica set revision OldRevision not int %s", err) return false } //If the RS annotation is empty then initialise it to 0 @@ -249,13 +249,13 @@ func SetNewReplicaSetAnnotations(deployment *apps.Deployment, newRS *apps.Replic } newRevisionInt, err := strconv.ParseInt(newRevision, 10, 64) if err != nil { - glog.Warningf("Updating replica set revision NewRevision not int %s", err) + klog.Warningf("Updating replica set revision NewRevision not int %s", err) return false } if oldRevisionInt < newRevisionInt { newRS.Annotations[RevisionAnnotation] = newRevision annotationChanged = true - glog.V(4).Infof("Updating replica set %q revision to %s", newRS.Name, newRevision) + klog.V(4).Infof("Updating replica set %q revision to %s", newRS.Name, newRevision) } // If a revision annotation already existed and this replica set was updated with a new revision // then that means we are rolling back to this replica set. We need to preserve the old revisions @@ -376,7 +376,7 @@ func getIntFromAnnotation(rs *apps.ReplicaSet, annotationKey string) (int32, boo } intValue, err := strconv.Atoi(annotationValue) if err != nil { - glog.V(2).Infof("Cannot convert the value %q with annotation key %q for the replica set %q", annotationValue, annotationKey, rs.Name) + klog.V(2).Infof("Cannot convert the value %q with annotation key %q for the replica set %q", annotationValue, annotationKey, rs.Name) return int32(0), false } return int32(intValue), true @@ -787,7 +787,7 @@ func DeploymentTimedOut(deployment *apps.Deployment, newStatus *apps.DeploymentS delta := time.Duration(*deployment.Spec.ProgressDeadlineSeconds) * time.Second timedOut := from.Add(delta).Before(now) - glog.V(4).Infof("Deployment %q timed out (%t) [last progress check: %v - now: %v]", deployment.Name, timedOut, from, now) + klog.V(4).Infof("Deployment %q timed out (%t) [last progress check: %v - now: %v]", deployment.Name, timedOut, from, now) return timedOut } diff --git a/pkg/controller/disruption/BUILD b/pkg/controller/disruption/BUILD index 37491c7227a..440966beca9 100644 --- a/pkg/controller/disruption/BUILD +++ b/pkg/controller/disruption/BUILD @@ -38,7 +38,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/disruption/disruption.go b/pkg/controller/disruption/disruption.go index 12592d2b627..8c45cc06011 100644 --- a/pkg/controller/disruption/disruption.go +++ b/pkg/controller/disruption/disruption.go @@ -49,7 +49,7 @@ import ( podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/controller" - "github.com/golang/glog" + "k8s.io/klog" ) const statusUpdateRetries = 2 @@ -285,18 +285,18 @@ func (dc *DisruptionController) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer dc.queue.ShutDown() - glog.Infof("Starting disruption controller") - defer glog.Infof("Shutting down disruption controller") + klog.Infof("Starting disruption controller") + defer klog.Infof("Shutting down disruption controller") if !controller.WaitForCacheSync("disruption", stopCh, dc.podListerSynced, dc.pdbListerSynced, dc.rcListerSynced, dc.rsListerSynced, dc.dListerSynced, dc.ssListerSynced) { return } if dc.kubeClient != nil { - glog.Infof("Sending events to api server.") + klog.Infof("Sending events to api server.") dc.broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: dc.kubeClient.CoreV1().Events("")}) } else { - glog.Infof("No api server defined - no events will be sent to API server.") + klog.Infof("No api server defined - no events will be sent to API server.") } go wait.Until(dc.worker, time.Second, stopCh) go wait.Until(dc.recheckWorker, time.Second, stopCh) @@ -306,44 +306,44 @@ func (dc *DisruptionController) Run(stopCh <-chan struct{}) { func (dc *DisruptionController) addDb(obj interface{}) { pdb := obj.(*policy.PodDisruptionBudget) - glog.V(4).Infof("add DB %q", pdb.Name) + klog.V(4).Infof("add DB %q", pdb.Name) dc.enqueuePdb(pdb) } func (dc *DisruptionController) updateDb(old, cur interface{}) { // TODO(mml) ignore updates where 'old' is equivalent to 'cur'. pdb := cur.(*policy.PodDisruptionBudget) - glog.V(4).Infof("update DB %q", pdb.Name) + klog.V(4).Infof("update DB %q", pdb.Name) dc.enqueuePdb(pdb) } func (dc *DisruptionController) removeDb(obj interface{}) { pdb := obj.(*policy.PodDisruptionBudget) - glog.V(4).Infof("remove DB %q", pdb.Name) + klog.V(4).Infof("remove DB %q", pdb.Name) dc.enqueuePdb(pdb) } func (dc *DisruptionController) addPod(obj interface{}) { pod := obj.(*v1.Pod) - glog.V(4).Infof("addPod called on pod %q", pod.Name) + klog.V(4).Infof("addPod called on pod %q", pod.Name) pdb := dc.getPdbForPod(pod) if pdb == nil { - glog.V(4).Infof("No matching pdb for pod %q", pod.Name) + klog.V(4).Infof("No matching pdb for pod %q", pod.Name) return } - glog.V(4).Infof("addPod %q -> PDB %q", pod.Name, pdb.Name) + klog.V(4).Infof("addPod %q -> PDB %q", pod.Name, pdb.Name) dc.enqueuePdb(pdb) } func (dc *DisruptionController) updatePod(old, cur interface{}) { pod := cur.(*v1.Pod) - glog.V(4).Infof("updatePod called on pod %q", pod.Name) + klog.V(4).Infof("updatePod called on pod %q", pod.Name) pdb := dc.getPdbForPod(pod) if pdb == nil { - glog.V(4).Infof("No matching pdb for pod %q", pod.Name) + klog.V(4).Infof("No matching pdb for pod %q", pod.Name) return } - glog.V(4).Infof("updatePod %q -> PDB %q", pod.Name, pdb.Name) + klog.V(4).Infof("updatePod %q -> PDB %q", pod.Name, pdb.Name) dc.enqueuePdb(pdb) } @@ -357,29 +357,29 @@ func (dc *DisruptionController) deletePod(obj interface{}) { if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Couldn't get object from tombstone %+v", obj) + klog.Errorf("Couldn't get object from tombstone %+v", obj) return } pod, ok = tombstone.Obj.(*v1.Pod) if !ok { - glog.Errorf("Tombstone contained object that is not a pod %+v", obj) + klog.Errorf("Tombstone contained object that is not a pod %+v", obj) return } } - glog.V(4).Infof("deletePod called on pod %q", pod.Name) + klog.V(4).Infof("deletePod called on pod %q", pod.Name) pdb := dc.getPdbForPod(pod) if pdb == nil { - glog.V(4).Infof("No matching pdb for pod %q", pod.Name) + klog.V(4).Infof("No matching pdb for pod %q", pod.Name) return } - glog.V(4).Infof("deletePod %q -> PDB %q", pod.Name, pdb.Name) + klog.V(4).Infof("deletePod %q -> PDB %q", pod.Name, pdb.Name) dc.enqueuePdb(pdb) } func (dc *DisruptionController) enqueuePdb(pdb *policy.PodDisruptionBudget) { key, err := controller.KeyFunc(pdb) if err != nil { - glog.Errorf("Couldn't get key for PodDisruptionBudget object %+v: %v", pdb, err) + klog.Errorf("Couldn't get key for PodDisruptionBudget object %+v: %v", pdb, err) return } dc.queue.Add(key) @@ -388,7 +388,7 @@ func (dc *DisruptionController) enqueuePdb(pdb *policy.PodDisruptionBudget) { func (dc *DisruptionController) enqueuePdbForRecheck(pdb *policy.PodDisruptionBudget, delay time.Duration) { key, err := controller.KeyFunc(pdb) if err != nil { - glog.Errorf("Couldn't get key for PodDisruptionBudget object %+v: %v", pdb, err) + klog.Errorf("Couldn't get key for PodDisruptionBudget object %+v: %v", pdb, err) return } dc.recheckQueue.AddAfter(key, delay) @@ -400,13 +400,13 @@ func (dc *DisruptionController) getPdbForPod(pod *v1.Pod) *policy.PodDisruptionB // caller. pdbs, err := dc.pdbLister.GetPodPodDisruptionBudgets(pod) if err != nil { - glog.V(4).Infof("No PodDisruptionBudgets found for pod %v, PodDisruptionBudget controller will avoid syncing.", pod.Name) + klog.V(4).Infof("No PodDisruptionBudgets found for pod %v, PodDisruptionBudget controller will avoid syncing.", pod.Name) return nil } if len(pdbs) > 1 { msg := fmt.Sprintf("Pod %q/%q matches multiple PodDisruptionBudgets. Chose %q arbitrarily.", pod.Namespace, pod.Name, pdbs[0].Name) - glog.Warning(msg) + klog.Warning(msg) dc.recorder.Event(pod, v1.EventTypeWarning, "MultiplePodDisruptionBudgets", msg) } return pdbs[0] @@ -471,7 +471,7 @@ func (dc *DisruptionController) processNextRecheckWorkItem() bool { func (dc *DisruptionController) sync(key string) error { startTime := time.Now() defer func() { - glog.V(4).Infof("Finished syncing PodDisruptionBudget %q (%v)", key, time.Since(startTime)) + klog.V(4).Infof("Finished syncing PodDisruptionBudget %q (%v)", key, time.Since(startTime)) }() namespace, name, err := cache.SplitMetaNamespaceKey(key) @@ -480,7 +480,7 @@ func (dc *DisruptionController) sync(key string) error { } pdb, err := dc.pdbLister.PodDisruptionBudgets(namespace).Get(name) if errors.IsNotFound(err) { - glog.V(4).Infof("PodDisruptionBudget %q has been deleted", key) + klog.V(4).Infof("PodDisruptionBudget %q has been deleted", key) return nil } if err != nil { @@ -488,7 +488,7 @@ func (dc *DisruptionController) sync(key string) error { } if err := dc.trySync(pdb); err != nil { - glog.Errorf("Failed to sync pdb %s/%s: %v", pdb.Namespace, pdb.Name, err) + klog.Errorf("Failed to sync pdb %s/%s: %v", pdb.Namespace, pdb.Name, err) return dc.failSafe(pdb) } @@ -656,7 +656,7 @@ func (dc *DisruptionController) buildDisruptedPodMap(pods []*v1.Pod, pdb *policy } expectedDeletion := disruptionTime.Time.Add(DeletionTimeout) if expectedDeletion.Before(currentTime) { - glog.V(1).Infof("Pod %s/%s was expected to be deleted at %s but it wasn't, updating pdb %s/%s", + klog.V(1).Infof("Pod %s/%s was expected to be deleted at %s but it wasn't, updating pdb %s/%s", pod.Namespace, pod.Name, disruptionTime.String(), pdb.Namespace, pdb.Name) dc.recorder.Eventf(pod, v1.EventTypeWarning, "NotDeleted", "Pod was expected by PDB %s/%s to be deleted but it wasn't", pdb.Namespace, pdb.Namespace) diff --git a/pkg/controller/endpoint/BUILD b/pkg/controller/endpoint/BUILD index 400da5a2086..0ff21c914a8 100644 --- a/pkg/controller/endpoint/BUILD +++ b/pkg/controller/endpoint/BUILD @@ -33,7 +33,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/endpoint/endpoints_controller.go b/pkg/controller/endpoint/endpoints_controller.go index 61c0f920547..403bd9c0eee 100644 --- a/pkg/controller/endpoint/endpoints_controller.go +++ b/pkg/controller/endpoint/endpoints_controller.go @@ -42,7 +42,7 @@ import ( "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/util/metrics" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -146,8 +146,8 @@ func (e *EndpointController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer e.queue.ShutDown() - glog.Infof("Starting endpoint controller") - defer glog.Infof("Shutting down endpoint controller") + klog.Infof("Starting endpoint controller") + defer klog.Infof("Shutting down endpoint controller") if !controller.WaitForCacheSync("endpoint", stopCh, e.podsSynced, e.servicesSynced, e.endpointsSynced) { return @@ -324,7 +324,7 @@ func (e *EndpointController) deletePod(obj interface{}) { utilruntime.HandleError(fmt.Errorf("Tombstone contained object that is not a Pod: %#v", obj)) return } - glog.V(4).Infof("Enqueuing services of deleted pod %s/%s having final state unrecorded", pod.Namespace, pod.Name) + klog.V(4).Infof("Enqueuing services of deleted pod %s/%s having final state unrecorded", pod.Namespace, pod.Name) e.addPod(pod) } @@ -368,12 +368,12 @@ func (e *EndpointController) handleErr(err error, key interface{}) { } if e.queue.NumRequeues(key) < maxRetries { - glog.V(2).Infof("Error syncing endpoints for service %q, retrying. Error: %v", key, err) + klog.V(2).Infof("Error syncing endpoints for service %q, retrying. Error: %v", key, err) e.queue.AddRateLimited(key) return } - glog.Warningf("Dropping service %q out of the queue: %v", key, err) + klog.Warningf("Dropping service %q out of the queue: %v", key, err) e.queue.Forget(key) utilruntime.HandleError(err) } @@ -381,7 +381,7 @@ func (e *EndpointController) handleErr(err error, key interface{}) { func (e *EndpointController) syncService(key string) error { startTime := time.Now() defer func() { - glog.V(4).Infof("Finished syncing service %q endpoints. (%v)", key, time.Since(startTime)) + klog.V(4).Infof("Finished syncing service %q endpoints. (%v)", key, time.Since(startTime)) }() namespace, name, err := cache.SplitMetaNamespaceKey(key) @@ -408,7 +408,7 @@ func (e *EndpointController) syncService(key string) error { return nil } - glog.V(5).Infof("About to update endpoints for service %q", key) + klog.V(5).Infof("About to update endpoints for service %q", key) pods, err := e.podLister.Pods(service.Namespace).List(labels.Set(service.Spec.Selector).AsSelectorPreValidated()) if err != nil { // Since we're getting stuff from a local cache, it is @@ -433,11 +433,11 @@ func (e *EndpointController) syncService(key string) error { for _, pod := range pods { if len(pod.Status.PodIP) == 0 { - glog.V(5).Infof("Failed to find an IP for pod %s/%s", pod.Namespace, pod.Name) + klog.V(5).Infof("Failed to find an IP for pod %s/%s", pod.Namespace, pod.Name) continue } if !tolerateUnreadyEndpoints && pod.DeletionTimestamp != nil { - glog.V(5).Infof("Pod is being deleted %s/%s", pod.Namespace, pod.Name) + klog.V(5).Infof("Pod is being deleted %s/%s", pod.Namespace, pod.Name) continue } @@ -462,7 +462,7 @@ func (e *EndpointController) syncService(key string) error { portProto := servicePort.Protocol portNum, err := podutil.FindPort(pod, servicePort) if err != nil { - glog.V(4).Infof("Failed to find port for service %s/%s: %v", service.Namespace, service.Name, err) + klog.V(4).Infof("Failed to find port for service %s/%s: %v", service.Namespace, service.Name, err) continue } @@ -496,7 +496,7 @@ func (e *EndpointController) syncService(key string) error { if !createEndpoints && apiequality.Semantic.DeepEqual(currentEndpoints.Subsets, subsets) && apiequality.Semantic.DeepEqual(currentEndpoints.Labels, service.Labels) { - glog.V(5).Infof("endpoints are equal for %s/%s, skipping update", service.Namespace, service.Name) + klog.V(5).Infof("endpoints are equal for %s/%s, skipping update", service.Namespace, service.Name) return nil } newEndpoints := currentEndpoints.DeepCopy() @@ -506,7 +506,7 @@ func (e *EndpointController) syncService(key string) error { newEndpoints.Annotations = make(map[string]string) } - glog.V(4).Infof("Update endpoints for %v/%v, ready: %d not ready: %d", service.Namespace, service.Name, totalReadyEps, totalNotReadyEps) + klog.V(4).Infof("Update endpoints for %v/%v, ready: %d not ready: %d", service.Namespace, service.Name, totalReadyEps, totalNotReadyEps) if createEndpoints { // No previous endpoints, create them _, err = e.client.CoreV1().Endpoints(service.Namespace).Create(newEndpoints) @@ -520,7 +520,7 @@ func (e *EndpointController) syncService(key string) error { // 1. namespace is terminating, endpoint creation is not allowed by default. // 2. policy is misconfigured, in which case no service would function anywhere. // Given the frequency of 1, we log at a lower level. - glog.V(5).Infof("Forbidden from creating endpoints: %v", err) + klog.V(5).Infof("Forbidden from creating endpoints: %v", err) } return err } @@ -572,7 +572,7 @@ func addEndpointSubset(subsets []v1.EndpointSubset, pod *v1.Pod, epa v1.Endpoint }) readyEps++ } else if shouldPodBeInEndpoints(pod) { - glog.V(5).Infof("Pod is out of service: %s/%s", pod.Namespace, pod.Name) + klog.V(5).Infof("Pod is out of service: %s/%s", pod.Namespace, pod.Name) subsets = append(subsets, v1.EndpointSubset{ NotReadyAddresses: []v1.EndpointAddress{epa}, Ports: ports, diff --git a/pkg/controller/garbagecollector/BUILD b/pkg/controller/garbagecollector/BUILD index 00fcc6433ce..c1c7f93abb0 100644 --- a/pkg/controller/garbagecollector/BUILD +++ b/pkg/controller/garbagecollector/BUILD @@ -43,12 +43,12 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/groupcache/lru:go_default_library", "//vendor/gonum.org/v1/gonum/graph:go_default_library", "//vendor/gonum.org/v1/gonum/graph/encoding:go_default_library", "//vendor/gonum.org/v1/gonum/graph/encoding/dot:go_default_library", "//vendor/gonum.org/v1/gonum/graph/simple:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/garbagecollector/garbagecollector.go b/pkg/controller/garbagecollector/garbagecollector.go index 5f1fff3effc..e090a84fb57 100644 --- a/pkg/controller/garbagecollector/garbagecollector.go +++ b/pkg/controller/garbagecollector/garbagecollector.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -130,8 +130,8 @@ func (gc *GarbageCollector) Run(workers int, stopCh <-chan struct{}) { defer gc.attemptToOrphan.ShutDown() defer gc.dependencyGraphBuilder.graphChanges.ShutDown() - glog.Infof("Starting garbage collector controller") - defer glog.Infof("Shutting down garbage collector controller") + klog.Infof("Starting garbage collector controller") + defer klog.Infof("Shutting down garbage collector controller") go gc.dependencyGraphBuilder.Run(stopCh) @@ -139,7 +139,7 @@ func (gc *GarbageCollector) Run(workers int, stopCh <-chan struct{}) { return } - glog.Infof("Garbage collector: all resource monitors have synced. Proceeding to collect garbage") + klog.Infof("Garbage collector: all resource monitors have synced. Proceeding to collect garbage") // gc workers for i := 0; i < workers; i++ { @@ -172,13 +172,13 @@ func (gc *GarbageCollector) Sync(discoveryClient discovery.ServerResourcesInterf // This can occur if there is an internal error in GetDeletableResources. if len(newResources) == 0 { - glog.V(2).Infof("no resources reported by discovery, skipping garbage collector sync") + klog.V(2).Infof("no resources reported by discovery, skipping garbage collector sync") return } // Decide whether discovery has reported a change. if reflect.DeepEqual(oldResources, newResources) { - glog.V(5).Infof("no resource updates from discovery, skipping garbage collector sync") + klog.V(5).Infof("no resource updates from discovery, skipping garbage collector sync") return } @@ -196,18 +196,18 @@ func (gc *GarbageCollector) Sync(discoveryClient discovery.ServerResourcesInterf if attempt > 1 { newResources = GetDeletableResources(discoveryClient) if len(newResources) == 0 { - glog.V(2).Infof("no resources reported by discovery (attempt %d)", attempt) + klog.V(2).Infof("no resources reported by discovery (attempt %d)", attempt) return false, nil } } - glog.V(2).Infof("syncing garbage collector with updated resources from discovery (attempt %d): %s", attempt, printDiff(oldResources, newResources)) + klog.V(2).Infof("syncing garbage collector with updated resources from discovery (attempt %d): %s", attempt, printDiff(oldResources, newResources)) // Resetting the REST mapper will also invalidate the underlying discovery // client. This is a leaky abstraction and assumes behavior about the REST // mapper, but we'll deal with it for now. gc.restMapper.Reset() - glog.V(4).Infof("reset restmapper") + klog.V(4).Infof("reset restmapper") // Perform the monitor resync and wait for controllers to report cache sync. // @@ -222,7 +222,7 @@ func (gc *GarbageCollector) Sync(discoveryClient discovery.ServerResourcesInterf utilruntime.HandleError(fmt.Errorf("failed to sync resource monitors (attempt %d): %v", attempt, err)) return false, nil } - glog.V(4).Infof("resynced monitors") + klog.V(4).Infof("resynced monitors") // wait for caches to fill for a while (our sync period) before attempting to rediscover resources and retry syncing. // this protects us from deadlocks where available resources changed and one of our informer caches will never fill. @@ -242,7 +242,7 @@ func (gc *GarbageCollector) Sync(discoveryClient discovery.ServerResourcesInterf // have succeeded to ensure we'll retry on subsequent syncs if an error // occurred. oldResources = newResources - glog.V(2).Infof("synced garbage collector") + klog.V(2).Infof("synced garbage collector") }, period, stopCh) } @@ -308,7 +308,7 @@ func (gc *GarbageCollector) attemptToDeleteWorker() bool { // have a way to distinguish this from a valid type we will recognize // after the next discovery sync. // For now, record the error and retry. - glog.V(5).Infof("error syncing item %s: %v", n, err) + klog.V(5).Infof("error syncing item %s: %v", n, err) } else { utilruntime.HandleError(fmt.Errorf("error syncing item %s: %v", n, err)) } @@ -318,7 +318,7 @@ func (gc *GarbageCollector) attemptToDeleteWorker() bool { // requeue if item hasn't been observed via an informer event yet. // otherwise a virtual node for an item added AND removed during watch reestablishment can get stuck in the graph and never removed. // see https://issue.k8s.io/56121 - glog.V(5).Infof("item %s hasn't been observed via informer yet", n.identity) + klog.V(5).Infof("item %s hasn't been observed via informer yet", n.identity) gc.attemptToDelete.AddRateLimited(item) } return true @@ -330,7 +330,7 @@ func (gc *GarbageCollector) attemptToDeleteWorker() bool { func (gc *GarbageCollector) isDangling(reference metav1.OwnerReference, item *node) ( dangling bool, owner *unstructured.Unstructured, err error) { if gc.absentOwnerCache.Has(reference.UID) { - glog.V(5).Infof("according to the absentOwnerCache, object %s's owner %s/%s, %s does not exist", item.identity.UID, reference.APIVersion, reference.Kind, reference.Name) + klog.V(5).Infof("according to the absentOwnerCache, object %s's owner %s/%s, %s does not exist", item.identity.UID, reference.APIVersion, reference.Kind, reference.Name) return true, nil, nil } // TODO: we need to verify the reference resource is supported by the @@ -351,14 +351,14 @@ func (gc *GarbageCollector) isDangling(reference metav1.OwnerReference, item *no switch { case errors.IsNotFound(err): gc.absentOwnerCache.Add(reference.UID) - glog.V(5).Infof("object %s's owner %s/%s, %s is not found", item.identity.UID, reference.APIVersion, reference.Kind, reference.Name) + klog.V(5).Infof("object %s's owner %s/%s, %s is not found", item.identity.UID, reference.APIVersion, reference.Kind, reference.Name) return true, nil, nil case err != nil: return false, nil, err } if owner.GetUID() != reference.UID { - glog.V(5).Infof("object %s's owner %s/%s, %s is not found, UID mismatch", item.identity.UID, reference.APIVersion, reference.Kind, reference.Name) + klog.V(5).Infof("object %s's owner %s/%s, %s is not found, UID mismatch", item.identity.UID, reference.APIVersion, reference.Kind, reference.Name) gc.absentOwnerCache.Add(reference.UID) return true, nil, nil } @@ -405,10 +405,10 @@ func ownerRefsToUIDs(refs []metav1.OwnerReference) []types.UID { } func (gc *GarbageCollector) attemptToDeleteItem(item *node) error { - glog.V(2).Infof("processing item %s", item.identity) + klog.V(2).Infof("processing item %s", item.identity) // "being deleted" is an one-way trip to the final deletion. We'll just wait for the final deletion, and then process the object's dependents. if item.isBeingDeleted() && !item.isDeletingDependents() { - glog.V(5).Infof("processing item %s returned at once, because its DeletionTimestamp is non-nil", item.identity) + klog.V(5).Infof("processing item %s returned at once, because its DeletionTimestamp is non-nil", item.identity) return nil } // TODO: It's only necessary to talk to the API server if this is a @@ -420,7 +420,7 @@ func (gc *GarbageCollector) attemptToDeleteItem(item *node) error { // the GraphBuilder can add "virtual" node for an owner that doesn't // exist yet, so we need to enqueue a virtual Delete event to remove // the virtual node from GraphBuilder.uidToNode. - glog.V(5).Infof("item %v not found, generating a virtual delete event", item.identity) + klog.V(5).Infof("item %v not found, generating a virtual delete event", item.identity) gc.dependencyGraphBuilder.enqueueVirtualDeleteEvent(item.identity) // since we're manually inserting a delete event to remove this node, // we don't need to keep tracking it as a virtual node and requeueing in attemptToDelete @@ -431,7 +431,7 @@ func (gc *GarbageCollector) attemptToDeleteItem(item *node) error { } if latest.GetUID() != item.identity.UID { - glog.V(5).Infof("UID doesn't match, item %v not found, generating a virtual delete event", item.identity) + klog.V(5).Infof("UID doesn't match, item %v not found, generating a virtual delete event", item.identity) gc.dependencyGraphBuilder.enqueueVirtualDeleteEvent(item.identity) // since we're manually inserting a delete event to remove this node, // we don't need to keep tracking it as a virtual node and requeueing in attemptToDelete @@ -448,7 +448,7 @@ func (gc *GarbageCollector) attemptToDeleteItem(item *node) error { // compute if we should delete the item ownerReferences := latest.GetOwnerReferences() if len(ownerReferences) == 0 { - glog.V(2).Infof("object %s's doesn't have an owner, continue on next item", item.identity) + klog.V(2).Infof("object %s's doesn't have an owner, continue on next item", item.identity) return nil } @@ -456,15 +456,15 @@ func (gc *GarbageCollector) attemptToDeleteItem(item *node) error { if err != nil { return err } - glog.V(5).Infof("classify references of %s.\nsolid: %#v\ndangling: %#v\nwaitingForDependentsDeletion: %#v\n", item.identity, solid, dangling, waitingForDependentsDeletion) + klog.V(5).Infof("classify references of %s.\nsolid: %#v\ndangling: %#v\nwaitingForDependentsDeletion: %#v\n", item.identity, solid, dangling, waitingForDependentsDeletion) switch { case len(solid) != 0: - glog.V(2).Infof("object %#v has at least one existing owner: %#v, will not garbage collect", solid, item.identity) + klog.V(2).Infof("object %#v has at least one existing owner: %#v, will not garbage collect", solid, item.identity) if len(dangling) == 0 && len(waitingForDependentsDeletion) == 0 { return nil } - glog.V(2).Infof("remove dangling references %#v and waiting references %#v for object %s", dangling, waitingForDependentsDeletion, item.identity) + klog.V(2).Infof("remove dangling references %#v and waiting references %#v for object %s", dangling, waitingForDependentsDeletion, item.identity) // waitingForDependentsDeletion needs to be deleted from the // ownerReferences, otherwise the referenced objects will be stuck with // the FinalizerDeletingDependents and never get deleted. @@ -483,7 +483,7 @@ func (gc *GarbageCollector) attemptToDeleteItem(item *node) error { // problem. // there are multiple workers run attemptToDeleteItem in // parallel, the circle detection can fail in a race condition. - glog.V(2).Infof("processing object %s, some of its owners and its dependent [%s] have FinalizerDeletingDependents, to prevent potential cycle, its ownerReferences are going to be modified to be non-blocking, then the object is going to be deleted with Foreground", item.identity, dep.identity) + klog.V(2).Infof("processing object %s, some of its owners and its dependent [%s] have FinalizerDeletingDependents, to prevent potential cycle, its ownerReferences are going to be modified to be non-blocking, then the object is going to be deleted with Foreground", item.identity, dep.identity) patch, err := item.unblockOwnerReferencesStrategicMergePatch() if err != nil { return err @@ -494,7 +494,7 @@ func (gc *GarbageCollector) attemptToDeleteItem(item *node) error { break } } - glog.V(2).Infof("at least one owner of object %s has FinalizerDeletingDependents, and the object itself has dependents, so it is going to be deleted in Foreground", item.identity) + klog.V(2).Infof("at least one owner of object %s has FinalizerDeletingDependents, and the object itself has dependents, so it is going to be deleted in Foreground", item.identity) // the deletion event will be observed by the graphBuilder, so the item // will be processed again in processDeletingDependentsItem. If it // doesn't have dependents, the function will remove the @@ -518,7 +518,7 @@ func (gc *GarbageCollector) attemptToDeleteItem(item *node) error { // otherwise, default to background. policy = metav1.DeletePropagationBackground } - glog.V(2).Infof("delete object %s with propagation policy %s", item.identity, policy) + klog.V(2).Infof("delete object %s with propagation policy %s", item.identity, policy) return gc.deleteObject(item.identity, &policy) } } @@ -527,12 +527,12 @@ func (gc *GarbageCollector) attemptToDeleteItem(item *node) error { func (gc *GarbageCollector) processDeletingDependentsItem(item *node) error { blockingDependents := item.blockingDependents() if len(blockingDependents) == 0 { - glog.V(2).Infof("remove DeleteDependents finalizer for item %s", item.identity) + klog.V(2).Infof("remove DeleteDependents finalizer for item %s", item.identity) return gc.removeFinalizer(item, metav1.FinalizerDeleteDependents) } for _, dep := range blockingDependents { if !dep.isDeletingDependents() { - glog.V(2).Infof("adding %s to attemptToDelete, because its owner %s is deletingDependents", dep.identity, item.identity) + klog.V(2).Infof("adding %s to attemptToDelete, because its owner %s is deletingDependents", dep.identity, item.identity) gc.attemptToDelete.Add(dep) } } @@ -570,7 +570,7 @@ func (gc *GarbageCollector) orphanDependents(owner objectReference, dependents [ if len(errorsSlice) != 0 { return fmt.Errorf("failed to orphan dependents of owner %s, got errors: %s", owner, utilerrors.NewAggregate(errorsSlice).Error()) } - glog.V(5).Infof("successfully updated all dependents of owner %s", owner) + klog.V(5).Infof("successfully updated all dependents of owner %s", owner) return nil } @@ -644,9 +644,9 @@ func GetDeletableResources(discoveryClient discovery.ServerResourcesInterface) m preferredResources, err := discoveryClient.ServerPreferredResources() if err != nil { if discovery.IsGroupDiscoveryFailedError(err) { - glog.Warningf("failed to discover some groups: %v", err.(*discovery.ErrGroupDiscoveryFailed).Groups) + klog.Warningf("failed to discover some groups: %v", err.(*discovery.ErrGroupDiscoveryFailed).Groups) } else { - glog.Warningf("failed to discover preferred resources: %v", err) + klog.Warningf("failed to discover preferred resources: %v", err) } } if preferredResources == nil { @@ -660,7 +660,7 @@ func GetDeletableResources(discoveryClient discovery.ServerResourcesInterface) m for _, rl := range deletableResources { gv, err := schema.ParseGroupVersion(rl.GroupVersion) if err != nil { - glog.Warningf("ignoring invalid discovered resource %q: %v", rl.GroupVersion, err) + klog.Warningf("ignoring invalid discovered resource %q: %v", rl.GroupVersion, err) continue } for i := range rl.APIResources { diff --git a/pkg/controller/garbagecollector/graph_builder.go b/pkg/controller/garbagecollector/graph_builder.go index 2c61c5769e3..c48f1dd655c 100644 --- a/pkg/controller/garbagecollector/graph_builder.go +++ b/pkg/controller/garbagecollector/graph_builder.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -176,16 +176,16 @@ func (gb *GraphBuilder) controllerFor(resource schema.GroupVersionResource, kind } shared, err := gb.sharedInformers.ForResource(resource) if err == nil { - glog.V(4).Infof("using a shared informer for resource %q, kind %q", resource.String(), kind.String()) + klog.V(4).Infof("using a shared informer for resource %q, kind %q", resource.String(), kind.String()) // need to clone because it's from a shared cache shared.Informer().AddEventHandlerWithResyncPeriod(handlers, ResourceResyncTime) return shared.Informer().GetController(), shared.Informer().GetStore(), nil } else { - glog.V(4).Infof("unable to use a shared informer for resource %q, kind %q: %v", resource.String(), kind.String(), err) + klog.V(4).Infof("unable to use a shared informer for resource %q, kind %q: %v", resource.String(), kind.String(), err) } // TODO: consider store in one storage. - glog.V(5).Infof("create storage for resource %s", resource) + klog.V(5).Infof("create storage for resource %s", resource) store, monitor := cache.NewInformer( listWatcher(gb.dynamicClient, resource), nil, @@ -245,7 +245,7 @@ func (gb *GraphBuilder) syncMonitors(resources map[schema.GroupVersionResource]s } } - glog.V(4).Infof("synced monitors; added %d, kept %d, removed %d", added, kept, len(toRemove)) + klog.V(4).Infof("synced monitors; added %d, kept %d, removed %d", added, kept, len(toRemove)) // NewAggregate returns nil if errs is 0-length return utilerrors.NewAggregate(errs) } @@ -277,7 +277,7 @@ func (gb *GraphBuilder) startMonitors() { started++ } } - glog.V(4).Infof("started %d new monitors, %d currently running", started, len(monitors)) + klog.V(4).Infof("started %d new monitors, %d currently running", started, len(monitors)) } // IsSynced returns true if any monitors exist AND all those monitors' @@ -289,13 +289,13 @@ func (gb *GraphBuilder) IsSynced() bool { defer gb.monitorLock.Unlock() if len(gb.monitors) == 0 { - glog.V(4).Info("garbage controller monitor not synced: no monitors") + klog.V(4).Info("garbage controller monitor not synced: no monitors") return false } for resource, monitor := range gb.monitors { if !monitor.controller.HasSynced() { - glog.V(4).Infof("garbage controller monitor not yet synced: %+v", resource) + klog.V(4).Infof("garbage controller monitor not yet synced: %+v", resource) return false } } @@ -305,8 +305,8 @@ func (gb *GraphBuilder) IsSynced() bool { // Run sets the stop channel and starts monitor execution until stopCh is // closed. Any running monitors will be stopped before Run returns. func (gb *GraphBuilder) Run(stopCh <-chan struct{}) { - glog.Infof("GraphBuilder running") - defer glog.Infof("GraphBuilder stopping") + klog.Infof("GraphBuilder running") + defer klog.Infof("GraphBuilder stopping") // Set up the stop channel. gb.monitorLock.Lock() @@ -333,7 +333,7 @@ func (gb *GraphBuilder) Run(stopCh <-chan struct{}) { // reset monitors so that the graph builder can be safely re-run/synced. gb.monitors = nil - glog.Infof("stopped %d of %d monitors", stopped, len(monitors)) + klog.Infof("stopped %d of %d monitors", stopped, len(monitors)) } var ignoredResources = map[schema.GroupResource]struct{}{ @@ -377,7 +377,7 @@ func (gb *GraphBuilder) addDependentToOwners(n *node, owners []metav1.OwnerRefer dependents: make(map[*node]struct{}), virtual: true, } - glog.V(5).Infof("add virtual node.identity: %s\n\n", ownerNode.identity) + klog.V(5).Infof("add virtual node.identity: %s\n\n", ownerNode.identity) gb.uidToNode.Write(ownerNode) } ownerNode.addDependent(n) @@ -515,7 +515,7 @@ func (gb *GraphBuilder) addUnblockedOwnersToDeleteQueue(removed []metav1.OwnerRe if ref.BlockOwnerDeletion != nil && *ref.BlockOwnerDeletion { node, found := gb.uidToNode.Read(ref.UID) if !found { - glog.V(5).Infof("cannot find %s in uidToNode", ref.UID) + klog.V(5).Infof("cannot find %s in uidToNode", ref.UID) continue } gb.attemptToDelete.Add(node) @@ -527,7 +527,7 @@ func (gb *GraphBuilder) addUnblockedOwnersToDeleteQueue(removed []metav1.OwnerRe if wasBlocked && isUnblocked { node, found := gb.uidToNode.Read(c.newRef.UID) if !found { - glog.V(5).Infof("cannot find %s in uidToNode", c.newRef.UID) + klog.V(5).Infof("cannot find %s in uidToNode", c.newRef.UID) continue } gb.attemptToDelete.Add(node) @@ -537,12 +537,12 @@ func (gb *GraphBuilder) addUnblockedOwnersToDeleteQueue(removed []metav1.OwnerRe func (gb *GraphBuilder) processTransitions(oldObj interface{}, newAccessor metav1.Object, n *node) { if startsWaitingForDependentsOrphaned(oldObj, newAccessor) { - glog.V(5).Infof("add %s to the attemptToOrphan", n.identity) + klog.V(5).Infof("add %s to the attemptToOrphan", n.identity) gb.attemptToOrphan.Add(n) return } if startsWaitingForDependentsDeleted(oldObj, newAccessor) { - glog.V(2).Infof("add %s to the attemptToDelete, because it's waiting for its dependents to be deleted", n.identity) + klog.V(2).Infof("add %s to the attemptToDelete, because it's waiting for its dependents to be deleted", n.identity) // if the n is added as a "virtual" node, its deletingDependents field is not properly set, so always set it here. n.markDeletingDependents() for dep := range n.dependents { @@ -575,7 +575,7 @@ func (gb *GraphBuilder) processGraphChanges() bool { utilruntime.HandleError(fmt.Errorf("cannot access obj: %v", err)) return true } - glog.V(5).Infof("GraphBuilder process object: %s/%s, namespace %s, name %s, uid %s, event type %v", event.gvk.GroupVersion().String(), event.gvk.Kind, accessor.GetNamespace(), accessor.GetName(), string(accessor.GetUID()), event.eventType) + klog.V(5).Infof("GraphBuilder process object: %s/%s, namespace %s, name %s, uid %s, event type %v", event.gvk.GroupVersion().String(), event.gvk.Kind, accessor.GetNamespace(), accessor.GetName(), string(accessor.GetUID()), event.eventType) // Check if the node already exists existingNode, found := gb.uidToNode.Read(accessor.GetUID()) if found { @@ -627,7 +627,7 @@ func (gb *GraphBuilder) processGraphChanges() bool { gb.processTransitions(event.oldObj, accessor, existingNode) case event.eventType == deleteEvent: if !found { - glog.V(5).Infof("%v doesn't exist in the graph, this shouldn't happen", accessor.GetUID()) + klog.V(5).Infof("%v doesn't exist in the graph, this shouldn't happen", accessor.GetUID()) return true } // removeNode updates the graph diff --git a/pkg/controller/garbagecollector/operations.go b/pkg/controller/garbagecollector/operations.go index ec9a096bffe..a6cf9dd5140 100644 --- a/pkg/controller/garbagecollector/operations.go +++ b/pkg/controller/garbagecollector/operations.go @@ -19,7 +19,7 @@ package garbagecollector import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -110,7 +110,7 @@ func (gc *GarbageCollector) removeFinalizer(owner *node, targetFinalizer string) newFinalizers = append(newFinalizers, f) } if !found { - glog.V(5).Infof("the %s finalizer is already removed from object %s", targetFinalizer, owner.identity) + klog.V(5).Infof("the %s finalizer is already removed from object %s", targetFinalizer, owner.identity) return nil } // remove the owner from dependent's OwnerReferences diff --git a/pkg/controller/job/BUILD b/pkg/controller/job/BUILD index 293b4b8bdd2..cb12dbe575b 100644 --- a/pkg/controller/job/BUILD +++ b/pkg/controller/job/BUILD @@ -35,7 +35,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/integer:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/job/job_controller.go b/pkg/controller/job/job_controller.go index df97da888a6..ebdcd7e2640 100644 --- a/pkg/controller/job/job_controller.go +++ b/pkg/controller/job/job_controller.go @@ -45,7 +45,7 @@ import ( "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/util/metrics" - "github.com/golang/glog" + "k8s.io/klog" ) const statusUpdateRetries = 3 @@ -91,7 +91,7 @@ type JobController struct { func NewJobController(podInformer coreinformers.PodInformer, jobInformer batchinformers.JobInformer, kubeClient clientset.Interface) *JobController { eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) if kubeClient != nil && kubeClient.CoreV1().RESTClient().GetRateLimiter() != nil { @@ -140,8 +140,8 @@ func (jm *JobController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer jm.queue.ShutDown() - glog.Infof("Starting job controller") - defer glog.Infof("Shutting down job controller") + klog.Infof("Starting job controller") + defer klog.Infof("Shutting down job controller") if !controller.WaitForCacheSync("job", stopCh, jm.podStoreSynced, jm.jobStoreSynced) { return @@ -343,7 +343,7 @@ func (jm *JobController) updateJob(old, cur interface{}) { total := time.Duration(*curADS) * time.Second // AddAfter will handle total < passed jm.queue.AddAfter(key, total-passed) - glog.V(4).Infof("job ActiveDeadlineSeconds updated, will rsync after %d seconds", total-passed) + klog.V(4).Infof("job ActiveDeadlineSeconds updated, will rsync after %d seconds", total-passed) } } } @@ -436,7 +436,7 @@ func (jm *JobController) getPodsForJob(j *batch.Job) ([]*v1.Pod, error) { func (jm *JobController) syncJob(key string) (bool, error) { startTime := time.Now() defer func() { - glog.V(4).Infof("Finished syncing job %q (%v)", key, time.Since(startTime)) + klog.V(4).Infof("Finished syncing job %q (%v)", key, time.Since(startTime)) }() ns, name, err := cache.SplitMetaNamespaceKey(key) @@ -449,7 +449,7 @@ func (jm *JobController) syncJob(key string) (bool, error) { sharedJob, err := jm.jobLister.Jobs(ns).Get(name) if err != nil { if errors.IsNotFound(err) { - glog.V(4).Infof("Job has been deleted: %v", key) + klog.V(4).Infof("Job has been deleted: %v", key) jm.expectations.DeleteExpectations(key) return true, nil } @@ -485,7 +485,7 @@ func (jm *JobController) syncJob(key string) (bool, error) { job.Status.StartTime = &now // enqueue a sync to check if job past ActiveDeadlineSeconds if job.Spec.ActiveDeadlineSeconds != nil { - glog.V(4).Infof("Job %s have ActiveDeadlineSeconds will sync after %d seconds", + klog.V(4).Infof("Job %s have ActiveDeadlineSeconds will sync after %d seconds", key, *job.Spec.ActiveDeadlineSeconds) jm.queue.AddAfter(key, time.Duration(*job.Spec.ActiveDeadlineSeconds)*time.Second) } @@ -614,7 +614,7 @@ func (jm *JobController) deleteJobPods(job *batch.Job, pods []*v1.Pod, errCh cha defer wait.Done() if err := jm.podControl.DeletePod(job.Namespace, pods[ix].Name, job); err != nil { defer utilruntime.HandleError(err) - glog.V(2).Infof("Failed to delete %v, job %q/%q deadline exceeded", pods[ix].Name, job.Namespace, job.Name) + klog.V(2).Infof("Failed to delete %v, job %q/%q deadline exceeded", pods[ix].Name, job.Namespace, job.Name) errCh <- err } }(i) @@ -697,7 +697,7 @@ func (jm *JobController) manageJob(activePods []*v1.Pod, succeeded int32, job *b diff := active - parallelism errCh = make(chan error, diff) jm.expectations.ExpectDeletions(jobKey, int(diff)) - glog.V(4).Infof("Too many pods running job %q, need %d, deleting %d", jobKey, parallelism, diff) + klog.V(4).Infof("Too many pods running job %q, need %d, deleting %d", jobKey, parallelism, diff) // Sort the pods in the order such that not-ready < ready, unscheduled // < scheduled, and pending < running. This ensures that we delete pods // in the earlier stages whenever possible. @@ -712,7 +712,7 @@ func (jm *JobController) manageJob(activePods []*v1.Pod, succeeded int32, job *b if err := jm.podControl.DeletePod(job.Namespace, activePods[ix].Name, job); err != nil { defer utilruntime.HandleError(err) // Decrement the expected number of deletes because the informer won't observe this deletion - glog.V(2).Infof("Failed to delete %v, decrementing expectations for job %q/%q", activePods[ix].Name, job.Namespace, job.Name) + klog.V(2).Infof("Failed to delete %v, decrementing expectations for job %q/%q", activePods[ix].Name, job.Namespace, job.Name) jm.expectations.DeletionObserved(jobKey) activeLock.Lock() active++ @@ -749,7 +749,7 @@ func (jm *JobController) manageJob(activePods []*v1.Pod, succeeded int32, job *b } jm.expectations.ExpectCreations(jobKey, int(diff)) errCh = make(chan error, diff) - glog.V(4).Infof("Too few pods running job %q, need %d, creating %d", jobKey, wantActive, diff) + klog.V(4).Infof("Too few pods running job %q, need %d, creating %d", jobKey, wantActive, diff) active += diff wait := sync.WaitGroup{} @@ -782,7 +782,7 @@ func (jm *JobController) manageJob(activePods []*v1.Pod, succeeded int32, job *b if err != nil { defer utilruntime.HandleError(err) // Decrement the expected number of creates because the informer won't observe this pod - glog.V(2).Infof("Failed creation, decrementing expectations for job %q/%q", job.Namespace, job.Name) + klog.V(2).Infof("Failed creation, decrementing expectations for job %q/%q", job.Namespace, job.Name) jm.expectations.CreationObserved(jobKey) activeLock.Lock() active-- @@ -795,7 +795,7 @@ func (jm *JobController) manageJob(activePods []*v1.Pod, succeeded int32, job *b // any skipped pods that we never attempted to start shouldn't be expected. skippedPods := diff - batchSize if errorCount < len(errCh) && skippedPods > 0 { - glog.V(2).Infof("Slow-start failure. Skipping creation of %d pods, decrementing expectations for job %q/%q", skippedPods, job.Namespace, job.Name) + klog.V(2).Infof("Slow-start failure. Skipping creation of %d pods, decrementing expectations for job %q/%q", skippedPods, job.Namespace, job.Name) active -= skippedPods for i := int32(0); i < skippedPods; i++ { // Decrement the expected number of creates because the informer won't observe this pod diff --git a/pkg/controller/namespace/BUILD b/pkg/controller/namespace/BUILD index 4da5a46751e..8a08de38ab3 100644 --- a/pkg/controller/namespace/BUILD +++ b/pkg/controller/namespace/BUILD @@ -27,7 +27,7 @@ go_library( "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/namespace/deletion/BUILD b/pkg/controller/namespace/deletion/BUILD index 61da1501e3b..0c058ee50df 100644 --- a/pkg/controller/namespace/deletion/BUILD +++ b/pkg/controller/namespace/deletion/BUILD @@ -22,7 +22,7 @@ go_library( "//staging/src/k8s.io/client-go/discovery:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/namespace/deletion/namespaced_resources_deleter.go b/pkg/controller/namespace/deletion/namespaced_resources_deleter.go index 62e09392f4a..7e28a4e9963 100644 --- a/pkg/controller/namespace/deletion/namespaced_resources_deleter.go +++ b/pkg/controller/namespace/deletion/namespaced_resources_deleter.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -110,7 +110,7 @@ func (d *namespacedResourcesDeleter) Delete(nsName string) error { return nil } - glog.V(5).Infof("namespace controller - syncNamespace - namespace: %s, finalizerToken: %s", namespace.Name, d.finalizerToken) + klog.V(5).Infof("namespace controller - syncNamespace - namespace: %s, finalizerToken: %s", namespace.Name, d.finalizerToken) // ensure that the status is up to date on the namespace // if we get a not found error, we assume the namespace is truly gone @@ -169,13 +169,13 @@ func (d *namespacedResourcesDeleter) initOpCache() { utilruntime.HandleError(fmt.Errorf("unable to get all supported resources from server: %v", err)) } if len(resources) == 0 { - glog.Fatalf("Unable to get any supported resources from server: %v", err) + klog.Fatalf("Unable to get any supported resources from server: %v", err) } deletableGroupVersionResources := []schema.GroupVersionResource{} for _, rl := range resources { gv, err := schema.ParseGroupVersion(rl.GroupVersion) if err != nil { - glog.Errorf("Failed to parse GroupVersion %q, skipping: %v", rl.GroupVersion, err) + klog.Errorf("Failed to parse GroupVersion %q, skipping: %v", rl.GroupVersion, err) continue } @@ -184,7 +184,7 @@ func (d *namespacedResourcesDeleter) initOpCache() { verbs := sets.NewString([]string(r.Verbs)...) if !verbs.Has("delete") { - glog.V(6).Infof("Skipping resource %v because it cannot be deleted.", gvr) + klog.V(6).Infof("Skipping resource %v because it cannot be deleted.", gvr) } for _, op := range []operation{operationList, operationDeleteCollection} { @@ -329,11 +329,11 @@ func (d *namespacedResourcesDeleter) finalizeNamespace(namespace *v1.Namespace) // it returns true if the operation was supported on the server. // it returns an error if the operation was supported on the server but was unable to complete. func (d *namespacedResourcesDeleter) deleteCollection(gvr schema.GroupVersionResource, namespace string) (bool, error) { - glog.V(5).Infof("namespace controller - deleteCollection - namespace: %s, gvr: %v", namespace, gvr) + klog.V(5).Infof("namespace controller - deleteCollection - namespace: %s, gvr: %v", namespace, gvr) key := operationKey{operation: operationDeleteCollection, gvr: gvr} if !d.opCache.isSupported(key) { - glog.V(5).Infof("namespace controller - deleteCollection ignored since not supported - namespace: %s, gvr: %v", namespace, gvr) + klog.V(5).Infof("namespace controller - deleteCollection ignored since not supported - namespace: %s, gvr: %v", namespace, gvr) return false, nil } @@ -355,12 +355,12 @@ func (d *namespacedResourcesDeleter) deleteCollection(gvr schema.GroupVersionRes // when working with this resource type, we will get a literal not found error rather than expected method not supported // remember next time that this resource does not support delete collection... if errors.IsMethodNotSupported(err) || errors.IsNotFound(err) { - glog.V(5).Infof("namespace controller - deleteCollection not supported - namespace: %s, gvr: %v", namespace, gvr) + klog.V(5).Infof("namespace controller - deleteCollection not supported - namespace: %s, gvr: %v", namespace, gvr) d.opCache.setNotSupported(key) return false, nil } - glog.V(5).Infof("namespace controller - deleteCollection unexpected error - namespace: %s, gvr: %v, error: %v", namespace, gvr, err) + klog.V(5).Infof("namespace controller - deleteCollection unexpected error - namespace: %s, gvr: %v, error: %v", namespace, gvr, err) return true, err } @@ -370,11 +370,11 @@ func (d *namespacedResourcesDeleter) deleteCollection(gvr schema.GroupVersionRes // a boolean if the operation is supported // an error if the operation is supported but could not be completed. func (d *namespacedResourcesDeleter) listCollection(gvr schema.GroupVersionResource, namespace string) (*unstructured.UnstructuredList, bool, error) { - glog.V(5).Infof("namespace controller - listCollection - namespace: %s, gvr: %v", namespace, gvr) + klog.V(5).Infof("namespace controller - listCollection - namespace: %s, gvr: %v", namespace, gvr) key := operationKey{operation: operationList, gvr: gvr} if !d.opCache.isSupported(key) { - glog.V(5).Infof("namespace controller - listCollection ignored since not supported - namespace: %s, gvr: %v", namespace, gvr) + klog.V(5).Infof("namespace controller - listCollection ignored since not supported - namespace: %s, gvr: %v", namespace, gvr) return nil, false, nil } @@ -390,7 +390,7 @@ func (d *namespacedResourcesDeleter) listCollection(gvr schema.GroupVersionResou // when working with this resource type, we will get a literal not found error rather than expected method not supported // remember next time that this resource does not support delete collection... if errors.IsMethodNotSupported(err) || errors.IsNotFound(err) { - glog.V(5).Infof("namespace controller - listCollection not supported - namespace: %s, gvr: %v", namespace, gvr) + klog.V(5).Infof("namespace controller - listCollection not supported - namespace: %s, gvr: %v", namespace, gvr) d.opCache.setNotSupported(key) return nil, false, nil } @@ -400,7 +400,7 @@ func (d *namespacedResourcesDeleter) listCollection(gvr schema.GroupVersionResou // deleteEachItem is a helper function that will list the collection of resources and delete each item 1 by 1. func (d *namespacedResourcesDeleter) deleteEachItem(gvr schema.GroupVersionResource, namespace string) error { - glog.V(5).Infof("namespace controller - deleteEachItem - namespace: %s, gvr: %v", namespace, gvr) + klog.V(5).Infof("namespace controller - deleteEachItem - namespace: %s, gvr: %v", namespace, gvr) unstructuredList, listSupported, err := d.listCollection(gvr, namespace) if err != nil { @@ -425,15 +425,15 @@ func (d *namespacedResourcesDeleter) deleteEachItem(gvr schema.GroupVersionResou func (d *namespacedResourcesDeleter) deleteAllContentForGroupVersionResource( gvr schema.GroupVersionResource, namespace string, namespaceDeletedAt metav1.Time) (int64, error) { - glog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - namespace: %s, gvr: %v", namespace, gvr) + klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - namespace: %s, gvr: %v", namespace, gvr) // estimate how long it will take for the resource to be deleted (needed for objects that support graceful delete) estimate, err := d.estimateGracefulTermination(gvr, namespace, namespaceDeletedAt) if err != nil { - glog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - unable to estimate - namespace: %s, gvr: %v, err: %v", namespace, gvr, err) + klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - unable to estimate - namespace: %s, gvr: %v, err: %v", namespace, gvr, err) return estimate, err } - glog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - estimate - namespace: %s, gvr: %v, estimate: %v", namespace, gvr, estimate) + klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - estimate - namespace: %s, gvr: %v, estimate: %v", namespace, gvr, estimate) // first try to delete the entire collection deleteCollectionSupported, err := d.deleteCollection(gvr, namespace) @@ -451,21 +451,21 @@ func (d *namespacedResourcesDeleter) deleteAllContentForGroupVersionResource( // verify there are no more remaining items // it is not an error condition for there to be remaining items if local estimate is non-zero - glog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - checking for no more items in namespace: %s, gvr: %v", namespace, gvr) + klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - checking for no more items in namespace: %s, gvr: %v", namespace, gvr) unstructuredList, listSupported, err := d.listCollection(gvr, namespace) if err != nil { - glog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - error verifying no items in namespace: %s, gvr: %v, err: %v", namespace, gvr, err) + klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - error verifying no items in namespace: %s, gvr: %v, err: %v", namespace, gvr, err) return estimate, err } if !listSupported { return estimate, nil } - glog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - items remaining - namespace: %s, gvr: %v, items: %v", namespace, gvr, len(unstructuredList.Items)) + klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - items remaining - namespace: %s, gvr: %v, items: %v", namespace, gvr, len(unstructuredList.Items)) if len(unstructuredList.Items) != 0 && estimate == int64(0) { // if any item has a finalizer, we treat that as a normal condition, and use a default estimation to allow for GC to complete. for _, item := range unstructuredList.Items { if len(item.GetFinalizers()) > 0 { - glog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - items remaining with finalizers - namespace: %s, gvr: %v, finalizers: %v", namespace, gvr, item.GetFinalizers()) + klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - items remaining with finalizers - namespace: %s, gvr: %v, finalizers: %v", namespace, gvr, item.GetFinalizers()) return finalizerEstimateSeconds, nil } } @@ -480,7 +480,7 @@ func (d *namespacedResourcesDeleter) deleteAllContentForGroupVersionResource( // If estimate > 0, not all resources are guaranteed to be gone. func (d *namespacedResourcesDeleter) deleteAllContent(namespace string, namespaceDeletedAt metav1.Time) (int64, error) { estimate := int64(0) - glog.V(4).Infof("namespace controller - deleteAllContent - namespace: %s", namespace) + klog.V(4).Infof("namespace controller - deleteAllContent - namespace: %s", namespace) resources, err := d.discoverResourcesFn() if err != nil { return estimate, err @@ -506,14 +506,14 @@ func (d *namespacedResourcesDeleter) deleteAllContent(namespace string, namespac if len(errs) > 0 { return estimate, utilerrors.NewAggregate(errs) } - glog.V(4).Infof("namespace controller - deleteAllContent - namespace: %s, estimate: %v", namespace, estimate) + klog.V(4).Infof("namespace controller - deleteAllContent - namespace: %s, estimate: %v", namespace, estimate) return estimate, nil } // estimateGrracefulTermination will estimate the graceful termination required for the specific entity in the namespace func (d *namespacedResourcesDeleter) estimateGracefulTermination(gvr schema.GroupVersionResource, ns string, namespaceDeletedAt metav1.Time) (int64, error) { groupResource := gvr.GroupResource() - glog.V(5).Infof("namespace controller - estimateGracefulTermination - group %s, resource: %s", groupResource.Group, groupResource.Resource) + klog.V(5).Infof("namespace controller - estimateGracefulTermination - group %s, resource: %s", groupResource.Group, groupResource.Resource) estimate := int64(0) var err error switch groupResource { @@ -534,7 +534,7 @@ func (d *namespacedResourcesDeleter) estimateGracefulTermination(gvr schema.Grou // estimateGracefulTerminationForPods determines the graceful termination period for pods in the namespace func (d *namespacedResourcesDeleter) estimateGracefulTerminationForPods(ns string) (int64, error) { - glog.V(5).Infof("namespace controller - estimateGracefulTerminationForPods - namespace %s", ns) + klog.V(5).Infof("namespace controller - estimateGracefulTerminationForPods - namespace %s", ns) estimate := int64(0) podsGetter := d.podsGetter if podsGetter == nil || reflect.ValueOf(podsGetter).IsNil() { diff --git a/pkg/controller/namespace/namespace_controller.go b/pkg/controller/namespace/namespace_controller.go index 396f1a3ead4..53d4225066d 100644 --- a/pkg/controller/namespace/namespace_controller.go +++ b/pkg/controller/namespace/namespace_controller.go @@ -35,7 +35,7 @@ import ( "k8s.io/kubernetes/pkg/controller/namespace/deletion" "k8s.io/kubernetes/pkg/util/metrics" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -140,7 +140,7 @@ func (nm *NamespaceController) worker() { if estimate, ok := err.(*deletion.ResourcesRemainingError); ok { t := estimate.Estimate/2 + 1 - glog.V(4).Infof("Content remaining in namespace %s, waiting %d seconds", key, t) + klog.V(4).Infof("Content remaining in namespace %s, waiting %d seconds", key, t) nm.queue.AddAfter(key, time.Duration(t)*time.Second) } else { // rather than wait for a full resync, re-add the namespace to the queue to be processed @@ -163,12 +163,12 @@ func (nm *NamespaceController) worker() { func (nm *NamespaceController) syncNamespaceFromKey(key string) (err error) { startTime := time.Now() defer func() { - glog.V(4).Infof("Finished syncing namespace %q (%v)", key, time.Since(startTime)) + klog.V(4).Infof("Finished syncing namespace %q (%v)", key, time.Since(startTime)) }() namespace, err := nm.lister.Get(key) if errors.IsNotFound(err) { - glog.Infof("Namespace has been deleted %v", key) + klog.Infof("Namespace has been deleted %v", key) return nil } if err != nil { @@ -183,14 +183,14 @@ func (nm *NamespaceController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer nm.queue.ShutDown() - glog.Infof("Starting namespace controller") - defer glog.Infof("Shutting down namespace controller") + klog.Infof("Starting namespace controller") + defer klog.Infof("Shutting down namespace controller") if !controller.WaitForCacheSync("namespace", stopCh, nm.listerSynced) { return } - glog.V(5).Info("Starting workers of namespace controller") + klog.V(5).Info("Starting workers of namespace controller") for i := 0; i < workers; i++ { go wait.Until(nm.worker, time.Second, stopCh) } diff --git a/pkg/controller/nodeipam/BUILD b/pkg/controller/nodeipam/BUILD index 7314b3960f3..e8c6a439638 100644 --- a/pkg/controller/nodeipam/BUILD +++ b/pkg/controller/nodeipam/BUILD @@ -44,7 +44,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/nodeipam/ipam/BUILD b/pkg/controller/nodeipam/ipam/BUILD index bbf02e01fe7..e95fd1f7c91 100644 --- a/pkg/controller/nodeipam/ipam/BUILD +++ b/pkg/controller/nodeipam/ipam/BUILD @@ -69,7 +69,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/scheme:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/nodeipam/ipam/adapter.go b/pkg/controller/nodeipam/ipam/adapter.go index c72e2e5a22c..96402cf7812 100644 --- a/pkg/controller/nodeipam/ipam/adapter.go +++ b/pkg/controller/nodeipam/ipam/adapter.go @@ -21,7 +21,7 @@ import ( "encoding/json" "net" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -48,9 +48,9 @@ func newAdapter(k8s clientset.Interface, cloud *gce.Cloud) *adapter { } broadcaster := record.NewBroadcaster() - broadcaster.StartLogging(glog.Infof) + broadcaster.StartLogging(klog.Infof) ret.recorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloudCIDRAllocator"}) - glog.V(0).Infof("Sending events to api server.") + klog.V(0).Infof("Sending events to api server.") broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{ Interface: k8s.CoreV1().Events(""), }) @@ -70,7 +70,7 @@ func (a *adapter) Alias(ctx context.Context, nodeName string) (*net.IPNet, error case 1: break default: - glog.Warningf("Node %q has more than one alias assigned (%v), defaulting to the first", nodeName, cidrs) + klog.Warningf("Node %q has more than one alias assigned (%v), defaulting to the first", nodeName, cidrs) } _, cidrRange, err := net.ParseCIDR(cidrs[0]) diff --git a/pkg/controller/nodeipam/ipam/cidr_allocator.go b/pkg/controller/nodeipam/ipam/cidr_allocator.go index 19f8bd0fde4..d667d11eec0 100644 --- a/pkg/controller/nodeipam/ipam/cidr_allocator.go +++ b/pkg/controller/nodeipam/ipam/cidr_allocator.go @@ -21,7 +21,7 @@ import ( "net" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -121,7 +121,7 @@ func listNodes(kubeClient clientset.Interface) (*v1.NodeList, error) { LabelSelector: labels.Everything().String(), }) if err != nil { - glog.Errorf("Failed to list all nodes: %v", err) + klog.Errorf("Failed to list all nodes: %v", err) return false, nil } return true, nil diff --git a/pkg/controller/nodeipam/ipam/cidrset/BUILD b/pkg/controller/nodeipam/ipam/cidrset/BUILD index 805174fd3c2..89cbf0c2ee7 100644 --- a/pkg/controller/nodeipam/ipam/cidrset/BUILD +++ b/pkg/controller/nodeipam/ipam/cidrset/BUILD @@ -10,7 +10,7 @@ go_test( name = "go_default_test", srcs = ["cidr_set_test.go"], embed = [":go_default_library"], - deps = ["//vendor/github.com/golang/glog:go_default_library"], + deps = ["//vendor/k8s.io/klog:go_default_library"], ) go_library( diff --git a/pkg/controller/nodeipam/ipam/cidrset/cidr_set_test.go b/pkg/controller/nodeipam/ipam/cidrset/cidr_set_test.go index 826eeba9bd8..c9e1651274a 100644 --- a/pkg/controller/nodeipam/ipam/cidrset/cidr_set_test.go +++ b/pkg/controller/nodeipam/ipam/cidrset/cidr_set_test.go @@ -22,7 +22,7 @@ import ( "reflect" "testing" - "github.com/golang/glog" + "k8s.io/klog" ) func TestCIDRSetFullyAllocated(t *testing.T) { @@ -478,17 +478,17 @@ func TestGetBitforCIDR(t *testing.T) { got, err := cs.getIndexForCIDR(subnetCIDR) if err == nil && tc.expectErr { - glog.Errorf("expected error but got null for %v", tc.description) + klog.Errorf("expected error but got null for %v", tc.description) continue } if err != nil && !tc.expectErr { - glog.Errorf("unexpected error: %v for %v", err, tc.description) + klog.Errorf("unexpected error: %v for %v", err, tc.description) continue } if got != tc.expectedBit { - glog.Errorf("expected %v, but got %v for %v", tc.expectedBit, got, tc.description) + klog.Errorf("expected %v, but got %v for %v", tc.expectedBit, got, tc.description) } } } diff --git a/pkg/controller/nodeipam/ipam/cloud_cidr_allocator.go b/pkg/controller/nodeipam/ipam/cloud_cidr_allocator.go index 12bf16f5a28..8c39a9f469e 100644 --- a/pkg/controller/nodeipam/ipam/cloud_cidr_allocator.go +++ b/pkg/controller/nodeipam/ipam/cloud_cidr_allocator.go @@ -23,7 +23,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -84,13 +84,13 @@ var _ CIDRAllocator = (*cloudCIDRAllocator)(nil) // NewCloudCIDRAllocator creates a new cloud CIDR allocator. func NewCloudCIDRAllocator(client clientset.Interface, cloud cloudprovider.Interface, nodeInformer informers.NodeInformer) (CIDRAllocator, error) { if client == nil { - glog.Fatalf("kubeClient is nil when starting NodeController") + klog.Fatalf("kubeClient is nil when starting NodeController") } eventBroadcaster := record.NewBroadcaster() recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cidrAllocator"}) - eventBroadcaster.StartLogging(glog.Infof) - glog.V(0).Infof("Sending events to api server.") + eventBroadcaster.StartLogging(klog.Infof) + klog.V(0).Infof("Sending events to api server.") eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")}) gceCloud, ok := cloud.(*gce.Cloud) @@ -127,15 +127,15 @@ func NewCloudCIDRAllocator(client clientset.Interface, cloud cloudprovider.Inter DeleteFunc: nodeutil.CreateDeleteNodeHandler(ca.ReleaseCIDR), }) - glog.V(0).Infof("Using cloud CIDR allocator (provider: %v)", cloud.ProviderName()) + klog.V(0).Infof("Using cloud CIDR allocator (provider: %v)", cloud.ProviderName()) return ca, nil } func (ca *cloudCIDRAllocator) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() - glog.Infof("Starting cloud CIDR allocator") - defer glog.Infof("Shutting down cloud CIDR allocator") + klog.Infof("Starting cloud CIDR allocator") + defer klog.Infof("Shutting down cloud CIDR allocator") if !controller.WaitForCacheSync("cidrallocator", stopCh, ca.nodesSynced) { return @@ -153,22 +153,22 @@ func (ca *cloudCIDRAllocator) worker(stopChan <-chan struct{}) { select { case workItem, ok := <-ca.nodeUpdateChannel: if !ok { - glog.Warning("Channel nodeCIDRUpdateChannel was unexpectedly closed") + klog.Warning("Channel nodeCIDRUpdateChannel was unexpectedly closed") return } if err := ca.updateCIDRAllocation(workItem); err == nil { - glog.V(3).Infof("Updated CIDR for %q", workItem) + klog.V(3).Infof("Updated CIDR for %q", workItem) } else { - glog.Errorf("Error updating CIDR for %q: %v", workItem, err) + klog.Errorf("Error updating CIDR for %q: %v", workItem, err) if canRetry, timeout := ca.retryParams(workItem); canRetry { - glog.V(2).Infof("Retrying update for %q after %v", workItem, timeout) + klog.V(2).Infof("Retrying update for %q after %v", workItem, timeout) time.AfterFunc(timeout, func() { // Requeue the failed node for update again. ca.nodeUpdateChannel <- workItem }) continue } - glog.Errorf("Exceeded retry count for %q, dropping from queue", workItem) + klog.Errorf("Exceeded retry count for %q, dropping from queue", workItem) } ca.removeNodeFromProcessing(workItem) case <-stopChan: @@ -193,7 +193,7 @@ func (ca *cloudCIDRAllocator) retryParams(nodeName string) (bool, time.Duration) entry, ok := ca.nodesInProcessing[nodeName] if !ok { - glog.Errorf("Cannot get retryParams for %q as entry does not exist", nodeName) + klog.Errorf("Cannot get retryParams for %q as entry does not exist", nodeName) return false, 0 } @@ -231,11 +231,11 @@ func (ca *cloudCIDRAllocator) AllocateOrOccupyCIDR(node *v1.Node) error { return nil } if !ca.insertNodeToProcessing(node.Name) { - glog.V(2).Infof("Node %v is already in a process of CIDR assignment.", node.Name) + klog.V(2).Infof("Node %v is already in a process of CIDR assignment.", node.Name) return nil } - glog.V(4).Infof("Putting node %s into the work queue", node.Name) + klog.V(4).Infof("Putting node %s into the work queue", node.Name) ca.nodeUpdateChannel <- node.Name return nil } @@ -247,7 +247,7 @@ func (ca *cloudCIDRAllocator) updateCIDRAllocation(nodeName string) error { if errors.IsNotFound(err) { return nil // node no longer available, skip processing } - glog.Errorf("Failed while getting node %v for updating Node.Spec.PodCIDR: %v", nodeName, err) + klog.Errorf("Failed while getting node %v for updating Node.Spec.PodCIDR: %v", nodeName, err) return err } @@ -267,11 +267,11 @@ func (ca *cloudCIDRAllocator) updateCIDRAllocation(nodeName string) error { podCIDR := cidr.String() if node.Spec.PodCIDR == podCIDR { - glog.V(4).Infof("Node %v already has allocated CIDR %v. It matches the proposed one.", node.Name, podCIDR) + klog.V(4).Infof("Node %v already has allocated CIDR %v. It matches the proposed one.", node.Name, podCIDR) // We don't return here, in order to set the NetworkUnavailable condition later below. } else { if node.Spec.PodCIDR != "" { - glog.Errorf("PodCIDR being reassigned! Node %v spec has %v, but cloud provider has assigned %v", node.Name, node.Spec.PodCIDR, podCIDR) + klog.Errorf("PodCIDR being reassigned! Node %v spec has %v, but cloud provider has assigned %v", node.Name, node.Spec.PodCIDR, podCIDR) // We fall through and set the CIDR despite this error. This // implements the same logic as implemented in the // rangeAllocator. @@ -280,14 +280,14 @@ func (ca *cloudCIDRAllocator) updateCIDRAllocation(nodeName string) error { } for i := 0; i < cidrUpdateRetries; i++ { if err = utilnode.PatchNodeCIDR(ca.client, types.NodeName(node.Name), podCIDR); err == nil { - glog.Infof("Set node %v PodCIDR to %v", node.Name, podCIDR) + klog.Infof("Set node %v PodCIDR to %v", node.Name, podCIDR) break } } } if err != nil { nodeutil.RecordNodeStatusChange(ca.recorder, node, "CIDRAssignmentFailed") - glog.Errorf("Failed to update node %v PodCIDR to %v after multiple attempts: %v", node.Name, podCIDR, err) + klog.Errorf("Failed to update node %v PodCIDR to %v after multiple attempts: %v", node.Name, podCIDR, err) return err } @@ -299,13 +299,13 @@ func (ca *cloudCIDRAllocator) updateCIDRAllocation(nodeName string) error { LastTransitionTime: metav1.Now(), }) if err != nil { - glog.Errorf("Error setting route status for node %v: %v", node.Name, err) + klog.Errorf("Error setting route status for node %v: %v", node.Name, err) } return err } func (ca *cloudCIDRAllocator) ReleaseCIDR(node *v1.Node) error { - glog.V(2).Infof("Node %v PodCIDR (%v) will be released by external cloud provider (not managed by controller)", + klog.V(2).Infof("Node %v PodCIDR (%v) will be released by external cloud provider (not managed by controller)", node.Name, node.Spec.PodCIDR) return nil } diff --git a/pkg/controller/nodeipam/ipam/controller.go b/pkg/controller/nodeipam/ipam/controller.go index 18d1c2f368f..1c8253d671f 100644 --- a/pkg/controller/nodeipam/ipam/controller.go +++ b/pkg/controller/nodeipam/ipam/controller.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" informers "k8s.io/client-go/informers/core/v1" @@ -99,7 +99,7 @@ func NewController( // registers the informers for node changes. This will start synchronization // of the node and cloud CIDR range allocations. func (c *Controller) Start(nodeInformer informers.NodeInformer) error { - glog.V(0).Infof("Starting IPAM controller (config=%+v)", c.config) + klog.V(0).Infof("Starting IPAM controller (config=%+v)", c.config) nodes, err := listNodes(c.adapter.k8s) if err != nil { @@ -110,9 +110,9 @@ func (c *Controller) Start(nodeInformer informers.NodeInformer) error { _, cidrRange, err := net.ParseCIDR(node.Spec.PodCIDR) if err == nil { c.set.Occupy(cidrRange) - glog.V(3).Infof("Occupying CIDR for node %q (%v)", node.Name, node.Spec.PodCIDR) + klog.V(3).Infof("Occupying CIDR for node %q (%v)", node.Name, node.Spec.PodCIDR) } else { - glog.Errorf("Node %q has an invalid CIDR (%q): %v", node.Name, node.Spec.PodCIDR, err) + klog.Errorf("Node %q has an invalid CIDR (%q): %v", node.Name, node.Spec.PodCIDR, err) } } @@ -180,7 +180,7 @@ func (c *Controller) onAdd(node *v1.Node) error { c.syncers[node.Name] = syncer go syncer.Loop(nil) } else { - glog.Warningf("Add for node %q that already exists", node.Name) + klog.Warningf("Add for node %q that already exists", node.Name) } syncer.Update(node) @@ -194,7 +194,7 @@ func (c *Controller) onUpdate(_, node *v1.Node) error { if sync, ok := c.syncers[node.Name]; ok { sync.Update(node) } else { - glog.Errorf("Received update for non-existent node %q", node.Name) + klog.Errorf("Received update for non-existent node %q", node.Name) return fmt.Errorf("unknown node %q", node.Name) } @@ -209,7 +209,7 @@ func (c *Controller) onDelete(node *v1.Node) error { syncer.Delete(node) delete(c.syncers, node.Name) } else { - glog.Warningf("Node %q was already deleted", node.Name) + klog.Warningf("Node %q was already deleted", node.Name) } return nil diff --git a/pkg/controller/nodeipam/ipam/range_allocator.go b/pkg/controller/nodeipam/ipam/range_allocator.go index de3e27a5537..9da52e63aa7 100644 --- a/pkg/controller/nodeipam/ipam/range_allocator.go +++ b/pkg/controller/nodeipam/ipam/range_allocator.go @@ -21,7 +21,7 @@ import ( "net" "sync" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -69,13 +69,13 @@ type rangeAllocator struct { // can initialize its CIDR map. NodeList is only nil in testing. func NewCIDRRangeAllocator(client clientset.Interface, nodeInformer informers.NodeInformer, clusterCIDR *net.IPNet, serviceCIDR *net.IPNet, subNetMaskSize int, nodeList *v1.NodeList) (CIDRAllocator, error) { if client == nil { - glog.Fatalf("kubeClient is nil when starting NodeController") + klog.Fatalf("kubeClient is nil when starting NodeController") } eventBroadcaster := record.NewBroadcaster() recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cidrAllocator"}) - eventBroadcaster.StartLogging(glog.Infof) - glog.V(0).Infof("Sending events to api server.") + eventBroadcaster.StartLogging(klog.Infof) + klog.V(0).Infof("Sending events to api server.") eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")}) set, err := cidrset.NewCIDRSet(clusterCIDR, subNetMaskSize) @@ -96,16 +96,16 @@ func NewCIDRRangeAllocator(client clientset.Interface, nodeInformer informers.No if serviceCIDR != nil { ra.filterOutServiceRange(serviceCIDR) } else { - glog.V(0).Info("No Service CIDR provided. Skipping filtering out service addresses.") + klog.V(0).Info("No Service CIDR provided. Skipping filtering out service addresses.") } if nodeList != nil { for _, node := range nodeList.Items { if node.Spec.PodCIDR == "" { - glog.Infof("Node %v has no CIDR, ignoring", node.Name) + klog.Infof("Node %v has no CIDR, ignoring", node.Name) continue } else { - glog.Infof("Node %v has CIDR %s, occupying it in CIDR map", + klog.Infof("Node %v has CIDR %s, occupying it in CIDR map", node.Name, node.Spec.PodCIDR) } if err := ra.occupyCIDR(&node); err != nil { @@ -154,8 +154,8 @@ func NewCIDRRangeAllocator(client clientset.Interface, nodeInformer informers.No func (r *rangeAllocator) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() - glog.Infof("Starting range CIDR allocator") - defer glog.Infof("Shutting down range CIDR allocator") + klog.Infof("Starting range CIDR allocator") + defer klog.Infof("Shutting down range CIDR allocator") if !controller.WaitForCacheSync("cidrallocator", stopCh, r.nodesSynced) { return @@ -173,7 +173,7 @@ func (r *rangeAllocator) worker(stopChan <-chan struct{}) { select { case workItem, ok := <-r.nodeCIDRUpdateChannel: if !ok { - glog.Warning("Channel nodeCIDRUpdateChannel was unexpectedly closed") + klog.Warning("Channel nodeCIDRUpdateChannel was unexpectedly closed") return } if err := r.updateCIDRAllocation(workItem); err != nil { @@ -225,7 +225,7 @@ func (r *rangeAllocator) AllocateOrOccupyCIDR(node *v1.Node) error { return nil } if !r.insertNodeToProcessing(node.Name) { - glog.V(2).Infof("Node %v is already in a process of CIDR assignment.", node.Name) + klog.V(2).Infof("Node %v is already in a process of CIDR assignment.", node.Name) return nil } if node.Spec.PodCIDR != "" { @@ -238,7 +238,7 @@ func (r *rangeAllocator) AllocateOrOccupyCIDR(node *v1.Node) error { return fmt.Errorf("failed to allocate cidr: %v", err) } - glog.V(4).Infof("Putting node %s with CIDR %s into the work queue", node.Name, podCIDR) + klog.V(4).Infof("Putting node %s with CIDR %s into the work queue", node.Name, podCIDR) r.nodeCIDRUpdateChannel <- nodeAndCIDR{ nodeName: node.Name, cidr: podCIDR, @@ -255,7 +255,7 @@ func (r *rangeAllocator) ReleaseCIDR(node *v1.Node) error { return fmt.Errorf("Failed to parse CIDR %s on Node %v: %v", node.Spec.PodCIDR, node.Name, err) } - glog.V(4).Infof("release CIDR %s", node.Spec.PodCIDR) + klog.V(4).Infof("release CIDR %s", node.Spec.PodCIDR) if err = r.cidrs.Release(podCIDR); err != nil { return fmt.Errorf("Error when releasing CIDR %v: %v", node.Spec.PodCIDR, err) } @@ -275,7 +275,7 @@ func (r *rangeAllocator) filterOutServiceRange(serviceCIDR *net.IPNet) { } if err := r.cidrs.Occupy(serviceCIDR); err != nil { - glog.Errorf("Error filtering out service cidr %v: %v", serviceCIDR, err) + klog.Errorf("Error filtering out service cidr %v: %v", serviceCIDR, err) } } @@ -289,37 +289,37 @@ func (r *rangeAllocator) updateCIDRAllocation(data nodeAndCIDR) error { node, err = r.nodeLister.Get(data.nodeName) if err != nil { - glog.Errorf("Failed while getting node %v for updating Node.Spec.PodCIDR: %v", data.nodeName, err) + klog.Errorf("Failed while getting node %v for updating Node.Spec.PodCIDR: %v", data.nodeName, err) return err } if node.Spec.PodCIDR == podCIDR { - glog.V(4).Infof("Node %v already has allocated CIDR %v. It matches the proposed one.", node.Name, podCIDR) + klog.V(4).Infof("Node %v already has allocated CIDR %v. It matches the proposed one.", node.Name, podCIDR) return nil } if node.Spec.PodCIDR != "" { - glog.Errorf("Node %v already has a CIDR allocated %v. Releasing the new one %v.", node.Name, node.Spec.PodCIDR, podCIDR) + klog.Errorf("Node %v already has a CIDR allocated %v. Releasing the new one %v.", node.Name, node.Spec.PodCIDR, podCIDR) if err := r.cidrs.Release(data.cidr); err != nil { - glog.Errorf("Error when releasing CIDR %v", podCIDR) + klog.Errorf("Error when releasing CIDR %v", podCIDR) } return nil } // If we reached here, it means that the node has no CIDR currently assigned. So we set it. for i := 0; i < cidrUpdateRetries; i++ { if err = utilnode.PatchNodeCIDR(r.client, types.NodeName(node.Name), podCIDR); err == nil { - glog.Infof("Set node %v PodCIDR to %v", node.Name, podCIDR) + klog.Infof("Set node %v PodCIDR to %v", node.Name, podCIDR) return nil } } - glog.Errorf("Failed to update node %v PodCIDR to %v after multiple attempts: %v", node.Name, podCIDR, err) + klog.Errorf("Failed to update node %v PodCIDR to %v after multiple attempts: %v", node.Name, podCIDR, err) nodeutil.RecordNodeStatusChange(r.recorder, node, "CIDRAssignmentFailed") // We accept the fact that we may leak CIDRs here. This is safer than releasing // them in case when we don't know if request went through. // NodeController restart will return all falsely allocated CIDRs to the pool. if !apierrors.IsServerTimeout(err) { - glog.Errorf("CIDR assignment for node %v failed: %v. Releasing allocated CIDR", node.Name, err) + klog.Errorf("CIDR assignment for node %v failed: %v. Releasing allocated CIDR", node.Name, err) if releaseErr := r.cidrs.Release(data.cidr); releaseErr != nil { - glog.Errorf("Error releasing allocated CIDR for node %v: %v", node.Name, releaseErr) + klog.Errorf("Error releasing allocated CIDR for node %v: %v", node.Name, releaseErr) } } return err diff --git a/pkg/controller/nodeipam/ipam/sync/BUILD b/pkg/controller/nodeipam/ipam/sync/BUILD index 53e7e31344c..0323850791f 100644 --- a/pkg/controller/nodeipam/ipam/sync/BUILD +++ b/pkg/controller/nodeipam/ipam/sync/BUILD @@ -8,7 +8,7 @@ go_library( deps = [ "//pkg/controller/nodeipam/ipam/cidrset:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -21,7 +21,7 @@ go_test( "//pkg/controller/nodeipam/ipam/test:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/nodeipam/ipam/sync/sync.go b/pkg/controller/nodeipam/ipam/sync/sync.go index 41806497ca8..ee95392b8ff 100644 --- a/pkg/controller/nodeipam/ipam/sync/sync.go +++ b/pkg/controller/nodeipam/ipam/sync/sync.go @@ -22,7 +22,7 @@ import ( "net" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/controller/nodeipam/ipam/cidrset" @@ -120,7 +120,7 @@ func New(c controller, cloudAlias cloudAlias, kubeAPI kubeAPI, mode NodeSyncMode // Loop runs the sync loop for a given node. done is an optional channel that // is closed when the Loop() returns. func (sync *NodeSync) Loop(done chan struct{}) { - glog.V(2).Infof("Starting sync loop for node %q", sync.nodeName) + klog.V(2).Infof("Starting sync loop for node %q", sync.nodeName) defer func() { if done != nil { @@ -130,13 +130,13 @@ func (sync *NodeSync) Loop(done chan struct{}) { timeout := sync.c.ResyncTimeout() delayTimer := time.NewTimer(timeout) - glog.V(4).Infof("Resync node %q in %v", sync.nodeName, timeout) + klog.V(4).Infof("Resync node %q in %v", sync.nodeName, timeout) for { select { case op, more := <-sync.opChan: if !more { - glog.V(2).Infof("Stopping sync loop") + klog.V(2).Infof("Stopping sync loop") return } sync.c.ReportResult(op.run(sync)) @@ -144,13 +144,13 @@ func (sync *NodeSync) Loop(done chan struct{}) { <-delayTimer.C } case <-delayTimer.C: - glog.V(4).Infof("Running resync for node %q", sync.nodeName) + klog.V(4).Infof("Running resync for node %q", sync.nodeName) sync.c.ReportResult((&updateOp{}).run(sync)) } timeout := sync.c.ResyncTimeout() delayTimer.Reset(timeout) - glog.V(4).Infof("Resync node %q in %v", sync.nodeName, timeout) + klog.V(4).Infof("Resync node %q in %v", sync.nodeName, timeout) } } @@ -190,15 +190,15 @@ func (op *updateOp) String() string { } func (op *updateOp) run(sync *NodeSync) error { - glog.V(3).Infof("Running updateOp %+v", op) + klog.V(3).Infof("Running updateOp %+v", op) ctx := context.Background() if op.node == nil { - glog.V(3).Infof("Getting node spec for %q", sync.nodeName) + klog.V(3).Infof("Getting node spec for %q", sync.nodeName) node, err := sync.kubeAPI.Node(ctx, sync.nodeName) if err != nil { - glog.Errorf("Error getting node %q spec: %v", sync.nodeName, err) + klog.Errorf("Error getting node %q spec: %v", sync.nodeName, err) return err } op.node = node @@ -206,7 +206,7 @@ func (op *updateOp) run(sync *NodeSync) error { aliasRange, err := sync.cloudAlias.Alias(ctx, sync.nodeName) if err != nil { - glog.Errorf("Error getting cloud alias for node %q: %v", sync.nodeName, err) + klog.Errorf("Error getting cloud alias for node %q: %v", sync.nodeName, err) return err } @@ -228,14 +228,14 @@ func (op *updateOp) run(sync *NodeSync) error { // match. func (op *updateOp) validateRange(ctx context.Context, sync *NodeSync, node *v1.Node, aliasRange *net.IPNet) error { if node.Spec.PodCIDR != aliasRange.String() { - glog.Errorf("Inconsistency detected between node PodCIDR and node alias (%v != %v)", + klog.Errorf("Inconsistency detected between node PodCIDR and node alias (%v != %v)", node.Spec.PodCIDR, aliasRange) sync.kubeAPI.EmitNodeWarningEvent(node.Name, MismatchEvent, "Node.Spec.PodCIDR != cloud alias (%v != %v)", node.Spec.PodCIDR, aliasRange) // User intervention is required in this case, as this is most likely due // to the user mucking around with their VM aliases on the side. } else { - glog.V(4).Infof("Node %q CIDR range %v is matches cloud assignment", node.Name, node.Spec.PodCIDR) + klog.V(4).Infof("Node %q CIDR range %v is matches cloud assignment", node.Name, node.Spec.PodCIDR) } return nil } @@ -249,26 +249,26 @@ func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, nod return fmt.Errorf("cannot sync from cloud in mode %q", sync.mode) } - glog.V(2).Infof("Updating node spec with alias range, node.PodCIDR = %v", aliasRange) + klog.V(2).Infof("Updating node spec with alias range, node.PodCIDR = %v", aliasRange) if err := sync.set.Occupy(aliasRange); err != nil { - glog.Errorf("Error occupying range %v for node %v", aliasRange, sync.nodeName) + klog.Errorf("Error occupying range %v for node %v", aliasRange, sync.nodeName) return err } if err := sync.kubeAPI.UpdateNodePodCIDR(ctx, node, aliasRange); err != nil { - glog.Errorf("Could not update node %q PodCIDR to %v: %v", node.Name, aliasRange, err) + klog.Errorf("Could not update node %q PodCIDR to %v: %v", node.Name, aliasRange, err) return err } - glog.V(2).Infof("Node %q PodCIDR set to %v", node.Name, aliasRange) + klog.V(2).Infof("Node %q PodCIDR set to %v", node.Name, aliasRange) if err := sync.kubeAPI.UpdateNodeNetworkUnavailable(node.Name, false); err != nil { - glog.Errorf("Could not update node NetworkUnavailable status to false: %v", err) + klog.Errorf("Could not update node NetworkUnavailable status to false: %v", err) return err } - glog.V(2).Infof("Updated node %q PodCIDR from cloud alias %v", node.Name, aliasRange) + klog.V(2).Infof("Updated node %q PodCIDR from cloud alias %v", node.Name, aliasRange) return nil } @@ -283,27 +283,27 @@ func (op *updateOp) updateAliasFromNode(ctx context.Context, sync *NodeSync, nod _, aliasRange, err := net.ParseCIDR(node.Spec.PodCIDR) if err != nil { - glog.Errorf("Could not parse PodCIDR (%q) for node %q: %v", + klog.Errorf("Could not parse PodCIDR (%q) for node %q: %v", node.Spec.PodCIDR, node.Name, err) return err } if err := sync.set.Occupy(aliasRange); err != nil { - glog.Errorf("Error occupying range %v for node %v", aliasRange, sync.nodeName) + klog.Errorf("Error occupying range %v for node %v", aliasRange, sync.nodeName) return err } if err := sync.cloudAlias.AddAlias(ctx, node.Name, aliasRange); err != nil { - glog.Errorf("Could not add alias %v for node %q: %v", aliasRange, node.Name, err) + klog.Errorf("Could not add alias %v for node %q: %v", aliasRange, node.Name, err) return err } if err := sync.kubeAPI.UpdateNodeNetworkUnavailable(node.Name, false); err != nil { - glog.Errorf("Could not update node NetworkUnavailable status to false: %v", err) + klog.Errorf("Could not update node NetworkUnavailable status to false: %v", err) return err } - glog.V(2).Infof("Updated node %q cloud alias with node spec, node.PodCIDR = %v", + klog.V(2).Infof("Updated node %q cloud alias with node spec, node.PodCIDR = %v", node.Name, node.Spec.PodCIDR) return nil @@ -326,21 +326,21 @@ func (op *updateOp) allocateRange(ctx context.Context, sync *NodeSync, node *v1. // is no durable record of the range. The missing space will be // recovered on the next restart of the controller. if err := sync.cloudAlias.AddAlias(ctx, node.Name, cidrRange); err != nil { - glog.Errorf("Could not add alias %v for node %q: %v", cidrRange, node.Name, err) + klog.Errorf("Could not add alias %v for node %q: %v", cidrRange, node.Name, err) return err } if err := sync.kubeAPI.UpdateNodePodCIDR(ctx, node, cidrRange); err != nil { - glog.Errorf("Could not update node %q PodCIDR to %v: %v", node.Name, cidrRange, err) + klog.Errorf("Could not update node %q PodCIDR to %v: %v", node.Name, cidrRange, err) return err } if err := sync.kubeAPI.UpdateNodeNetworkUnavailable(node.Name, false); err != nil { - glog.Errorf("Could not update node NetworkUnavailable status to false: %v", err) + klog.Errorf("Could not update node NetworkUnavailable status to false: %v", err) return err } - glog.V(2).Infof("Allocated PodCIDR %v for node %q", cidrRange, node.Name) + klog.V(2).Infof("Allocated PodCIDR %v for node %q", cidrRange, node.Name) return nil } @@ -358,15 +358,15 @@ func (op *deleteOp) String() string { } func (op *deleteOp) run(sync *NodeSync) error { - glog.V(3).Infof("Running deleteOp %+v", op) + klog.V(3).Infof("Running deleteOp %+v", op) if op.node.Spec.PodCIDR == "" { - glog.V(2).Infof("Node %q was deleted, node had no PodCIDR range assigned", op.node.Name) + klog.V(2).Infof("Node %q was deleted, node had no PodCIDR range assigned", op.node.Name) return nil } _, cidrRange, err := net.ParseCIDR(op.node.Spec.PodCIDR) if err != nil { - glog.Errorf("Deleted node %q has an invalid podCIDR %q: %v", + klog.Errorf("Deleted node %q has an invalid podCIDR %q: %v", op.node.Name, op.node.Spec.PodCIDR, err) sync.kubeAPI.EmitNodeWarningEvent(op.node.Name, InvalidPodCIDR, "Node %q has an invalid PodCIDR: %q", op.node.Name, op.node.Spec.PodCIDR) @@ -374,7 +374,7 @@ func (op *deleteOp) run(sync *NodeSync) error { } sync.set.Release(cidrRange) - glog.V(2).Infof("Node %q was deleted, releasing CIDR range %v", + klog.V(2).Infof("Node %q was deleted, releasing CIDR range %v", op.node.Name, op.node.Spec.PodCIDR) return nil diff --git a/pkg/controller/nodeipam/ipam/sync/sync_test.go b/pkg/controller/nodeipam/ipam/sync/sync_test.go index 4a47280d94b..8c80b2c6453 100644 --- a/pkg/controller/nodeipam/ipam/sync/sync_test.go +++ b/pkg/controller/nodeipam/ipam/sync/sync_test.go @@ -24,8 +24,8 @@ import ( "testing" "time" - "github.com/golang/glog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller/nodeipam/ipam/cidrset" "k8s.io/kubernetes/pkg/controller/nodeipam/ipam/test" @@ -88,7 +88,7 @@ func (f *fakeAPIs) EmitNodeWarningEvent(nodeName, reason, fmtStr string, args .. } func (f *fakeAPIs) ReportResult(err error) { - glog.V(2).Infof("ReportResult %v", err) + klog.V(2).Infof("ReportResult %v", err) f.results = append(f.results, err) if f.reportChan != nil { f.reportChan <- struct{}{} @@ -104,7 +104,7 @@ func (f *fakeAPIs) ResyncTimeout() time.Duration { func (f *fakeAPIs) dumpTrace() { for i, x := range f.calls { - glog.Infof("trace %v: %v", i, x) + klog.Infof("trace %v: %v", i, x) } } diff --git a/pkg/controller/nodeipam/node_ipam_controller.go b/pkg/controller/nodeipam/node_ipam_controller.go index 4c3f55575d4..8f87cf72b1e 100644 --- a/pkg/controller/nodeipam/node_ipam_controller.go +++ b/pkg/controller/nodeipam/node_ipam_controller.go @@ -20,7 +20,7 @@ import ( "net" "time" - "github.com/golang/glog" + "k8s.io/klog" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -90,13 +90,13 @@ func NewNodeIpamController( allocatorType ipam.CIDRAllocatorType) (*Controller, error) { if kubeClient == nil { - glog.Fatalf("kubeClient is nil when starting Controller") + klog.Fatalf("kubeClient is nil when starting Controller") } eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) - glog.Infof("Sending events to api server.") + klog.Infof("Sending events to api server.") eventBroadcaster.StartRecordingToSink( &v1core.EventSinkImpl{ Interface: kubeClient.CoreV1().Events(""), @@ -107,13 +107,13 @@ func NewNodeIpamController( } if clusterCIDR == nil { - glog.Fatal("Controller: Must specify --cluster-cidr if --allocate-node-cidrs is set") + klog.Fatal("Controller: Must specify --cluster-cidr if --allocate-node-cidrs is set") } mask := clusterCIDR.Mask if allocatorType != ipam.CloudAllocatorType { // Cloud CIDR allocator does not rely on clusterCIDR or nodeCIDRMaskSize for allocation. if maskSize, _ := mask.Size(); maskSize > nodeCIDRMaskSize { - glog.Fatal("Controller: Invalid --cluster-cidr, mask size of cluster CIDR must be less than --node-cidr-mask-size") + klog.Fatal("Controller: Invalid --cluster-cidr, mask size of cluster CIDR must be less than --node-cidr-mask-size") } } @@ -141,10 +141,10 @@ func NewNodeIpamController( } ipamc, err := ipam.NewController(cfg, kubeClient, cloud, clusterCIDR, serviceCIDR, nodeCIDRMaskSize) if err != nil { - glog.Fatalf("Error creating ipam controller: %v", err) + klog.Fatalf("Error creating ipam controller: %v", err) } if err := ipamc.Start(nodeInformer); err != nil { - glog.Fatalf("Error trying to Init(): %v", err) + klog.Fatalf("Error trying to Init(): %v", err) } } else { var err error @@ -165,8 +165,8 @@ func NewNodeIpamController( func (nc *Controller) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() - glog.Infof("Starting ipam controller") - defer glog.Infof("Shutting down ipam controller") + klog.Infof("Starting ipam controller") + defer klog.Infof("Shutting down ipam controller") if !controller.WaitForCacheSync("node", stopCh, nc.nodeInformerSynced) { return diff --git a/pkg/controller/nodelifecycle/BUILD b/pkg/controller/nodelifecycle/BUILD index 186919868a1..74eb860fa2a 100644 --- a/pkg/controller/nodelifecycle/BUILD +++ b/pkg/controller/nodelifecycle/BUILD @@ -43,8 +43,8 @@ go_library( "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/nodelifecycle/node_lifecycle_controller.go b/pkg/controller/nodelifecycle/node_lifecycle_controller.go index 431296a50d5..f474b55df9e 100644 --- a/pkg/controller/nodelifecycle/node_lifecycle_controller.go +++ b/pkg/controller/nodelifecycle/node_lifecycle_controller.go @@ -29,7 +29,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" coordv1beta1 "k8s.io/api/coordination/v1beta1" "k8s.io/api/core/v1" @@ -262,14 +262,14 @@ func NewNodeLifecycleController( taintNodeByCondition bool) (*Controller, error) { if kubeClient == nil { - glog.Fatalf("kubeClient is nil when starting Controller") + klog.Fatalf("kubeClient is nil when starting Controller") } eventBroadcaster := record.NewBroadcaster() recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "node-controller"}) - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) - glog.Infof("Sending events to api server.") + klog.Infof("Sending events to api server.") eventBroadcaster.StartRecordingToSink( &v1core.EventSinkImpl{ Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events(""), @@ -309,7 +309,7 @@ func NewNodeLifecycleController( nodeUpdateQueue: workqueue.New(), } if useTaintBasedEvictions { - glog.Infof("Controller is using taint based evictions.") + klog.Infof("Controller is using taint based evictions.") } nc.enterPartialDisruptionFunc = nc.ReducedQPSFunc @@ -336,12 +336,12 @@ func NewNodeLifecycleController( if !isPod { deletedState, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Received unexpected object: %v", obj) + klog.Errorf("Received unexpected object: %v", obj) return } pod, ok = deletedState.Obj.(*v1.Pod) if !ok { - glog.Errorf("DeletedFinalStateUnknown contained non-Pod object: %v", deletedState.Obj) + klog.Errorf("DeletedFinalStateUnknown contained non-Pod object: %v", deletedState.Obj) return } } @@ -375,7 +375,7 @@ func NewNodeLifecycleController( } if nc.taintNodeByCondition { - glog.Infof("Controller will taint node by condition.") + klog.Infof("Controller will taint node by condition.") nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: nodeutil.CreateAddNodeHandler(func(node *v1.Node) error { nc.nodeUpdateQueue.Add(node.Name) @@ -420,8 +420,8 @@ func NewNodeLifecycleController( func (nc *Controller) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() - glog.Infof("Starting node controller") - defer glog.Infof("Shutting down node controller") + klog.Infof("Starting node controller") + defer klog.Infof("Shutting down node controller") if !controller.WaitForCacheSync("taint", stopCh, nc.leaseInformerSynced, nc.nodeInformerSynced, nc.podInformerSynced, nc.daemonSetInformerSynced) { return @@ -459,7 +459,7 @@ func (nc *Controller) Run(stopCh <-chan struct{}) { // Incorporate the results of node health signal pushed from kubelet to master. go wait.Until(func() { if err := nc.monitorNodeHealth(); err != nil { - glog.Errorf("Error monitoring node health: %v", err) + klog.Errorf("Error monitoring node health: %v", err) } }, nc.nodeMonitorPeriod, stopCh) @@ -495,7 +495,7 @@ func (nc *Controller) doFixDeprecatedTaintKeyPass(node *v1.Node) error { return nil } - glog.Warningf("Detected deprecated taint keys: %v on node: %v, will substitute them with %v", + klog.Warningf("Detected deprecated taint keys: %v on node: %v, will substitute them with %v", taintsToDel, node.GetName(), taintsToAdd) if !nodeutil.SwapNodeControllerTaint(nc.kubeClient, taintsToAdd, taintsToDel, node) { @@ -516,7 +516,7 @@ func (nc *Controller) doNoScheduleTaintingPassWorker() { if err := nc.doNoScheduleTaintingPass(nodeName); err != nil { // TODO (k82cn): Add nodeName back to the queue. - glog.Errorf("Failed to taint NoSchedule on node <%s>, requeue it: %v", nodeName, err) + klog.Errorf("Failed to taint NoSchedule on node <%s>, requeue it: %v", nodeName, err) } nc.nodeUpdateQueue.Done(nodeName) } @@ -585,10 +585,10 @@ func (nc *Controller) doNoExecuteTaintingPass() { nc.zoneNoExecuteTainter[k].Try(func(value scheduler.TimedValue) (bool, time.Duration) { node, err := nc.nodeLister.Get(value.Value) if apierrors.IsNotFound(err) { - glog.Warningf("Node %v no longer present in nodeLister!", value.Value) + klog.Warningf("Node %v no longer present in nodeLister!", value.Value) return true, 0 } else if err != nil { - glog.Warningf("Failed to get Node %v from the nodeLister: %v", value.Value, err) + klog.Warningf("Failed to get Node %v from the nodeLister: %v", value.Value, err) // retry in 50 millisecond return false, 50 * time.Millisecond } else { @@ -607,7 +607,7 @@ func (nc *Controller) doNoExecuteTaintingPass() { oppositeTaint = *NotReadyTaintTemplate } else { // It seems that the Node is ready again, so there's no need to taint it. - glog.V(4).Infof("Node %v was in a taint queue, but it's ready now. Ignoring taint request.", value.Value) + klog.V(4).Infof("Node %v was in a taint queue, but it's ready now. Ignoring taint request.", value.Value) return true, 0 } @@ -624,9 +624,9 @@ func (nc *Controller) doEvictionPass() { nc.zonePodEvictor[k].Try(func(value scheduler.TimedValue) (bool, time.Duration) { node, err := nc.nodeLister.Get(value.Value) if apierrors.IsNotFound(err) { - glog.Warningf("Node %v no longer present in nodeLister!", value.Value) + klog.Warningf("Node %v no longer present in nodeLister!", value.Value) } else if err != nil { - glog.Warningf("Failed to get Node %v from the nodeLister: %v", value.Value, err) + klog.Warningf("Failed to get Node %v from the nodeLister: %v", value.Value, err) } else { zone := utilnode.GetZoneKey(node) evictionsNumber.WithLabelValues(zone).Inc() @@ -638,7 +638,7 @@ func (nc *Controller) doEvictionPass() { return false, 0 } if remaining { - glog.Infof("Pods awaiting deletion due to Controller eviction") + klog.Infof("Pods awaiting deletion due to Controller eviction") } return true, 0 }) @@ -662,7 +662,7 @@ func (nc *Controller) monitorNodeHealth() error { } for i := range added { - glog.V(1).Infof("Controller observed a new Node: %#v", added[i].Name) + klog.V(1).Infof("Controller observed a new Node: %#v", added[i].Name) nodeutil.RecordNodeEvent(nc.recorder, added[i].Name, string(added[i].UID), v1.EventTypeNormal, "RegisteredNode", fmt.Sprintf("Registered Node %v in Controller", added[i].Name)) nc.knownNodeSet[added[i].Name] = added[i] nc.addPodEvictorForNewZone(added[i]) @@ -674,7 +674,7 @@ func (nc *Controller) monitorNodeHealth() error { } for i := range deleted { - glog.V(1).Infof("Controller observed a Node deletion: %v", deleted[i].Name) + klog.V(1).Infof("Controller observed a Node deletion: %v", deleted[i].Name) nodeutil.RecordNodeEvent(nc.recorder, deleted[i].Name, string(deleted[i].UID), v1.EventTypeNormal, "RemovingNode", fmt.Sprintf("Removing Node %v from Controller", deleted[i].Name)) delete(nc.knownNodeSet, deleted[i].Name) } @@ -693,12 +693,12 @@ func (nc *Controller) monitorNodeHealth() error { name := node.Name node, err = nc.kubeClient.CoreV1().Nodes().Get(name, metav1.GetOptions{}) if err != nil { - glog.Errorf("Failed while getting a Node to retry updating node health. Probably Node %s was deleted.", name) + klog.Errorf("Failed while getting a Node to retry updating node health. Probably Node %s was deleted.", name) return false, err } return false, nil }); err != nil { - glog.Errorf("Update health of Node '%v' from Controller error: %v. "+ + klog.Errorf("Update health of Node '%v' from Controller error: %v. "+ "Skipping - no pods will be evicted.", node.Name, err) continue } @@ -717,10 +717,10 @@ func (nc *Controller) monitorNodeHealth() error { if taintutils.TaintExists(node.Spec.Taints, UnreachableTaintTemplate) { taintToAdd := *NotReadyTaintTemplate if !nodeutil.SwapNodeControllerTaint(nc.kubeClient, []*v1.Taint{&taintToAdd}, []*v1.Taint{UnreachableTaintTemplate}, node) { - glog.Errorf("Failed to instantly swap UnreachableTaint to NotReadyTaint. Will try again in the next cycle.") + klog.Errorf("Failed to instantly swap UnreachableTaint to NotReadyTaint. Will try again in the next cycle.") } } else if nc.markNodeForTainting(node) { - glog.V(2).Infof("Node %v is NotReady as of %v. Adding it to the Taint queue.", + klog.V(2).Infof("Node %v is NotReady as of %v. Adding it to the Taint queue.", node.Name, decisionTimestamp, ) @@ -728,7 +728,7 @@ func (nc *Controller) monitorNodeHealth() error { } else { if decisionTimestamp.After(nc.nodeHealthMap[node.Name].readyTransitionTimestamp.Add(nc.podEvictionTimeout)) { if nc.evictPods(node) { - glog.V(2).Infof("Node is NotReady. Adding Pods on Node %s to eviction queue: %v is later than %v + %v", + klog.V(2).Infof("Node is NotReady. Adding Pods on Node %s to eviction queue: %v is later than %v + %v", node.Name, decisionTimestamp, nc.nodeHealthMap[node.Name].readyTransitionTimestamp, @@ -744,10 +744,10 @@ func (nc *Controller) monitorNodeHealth() error { if taintutils.TaintExists(node.Spec.Taints, NotReadyTaintTemplate) { taintToAdd := *UnreachableTaintTemplate if !nodeutil.SwapNodeControllerTaint(nc.kubeClient, []*v1.Taint{&taintToAdd}, []*v1.Taint{NotReadyTaintTemplate}, node) { - glog.Errorf("Failed to instantly swap UnreachableTaint to NotReadyTaint. Will try again in the next cycle.") + klog.Errorf("Failed to instantly swap UnreachableTaint to NotReadyTaint. Will try again in the next cycle.") } } else if nc.markNodeForTainting(node) { - glog.V(2).Infof("Node %v is unresponsive as of %v. Adding it to the Taint queue.", + klog.V(2).Infof("Node %v is unresponsive as of %v. Adding it to the Taint queue.", node.Name, decisionTimestamp, ) @@ -755,7 +755,7 @@ func (nc *Controller) monitorNodeHealth() error { } else { if decisionTimestamp.After(nc.nodeHealthMap[node.Name].probeTimestamp.Add(nc.podEvictionTimeout)) { if nc.evictPods(node) { - glog.V(2).Infof("Node is unresponsive. Adding Pods on Node %s to eviction queues: %v is later than %v + %v", + klog.V(2).Infof("Node is unresponsive. Adding Pods on Node %s to eviction queues: %v is later than %v + %v", node.Name, decisionTimestamp, nc.nodeHealthMap[node.Name].readyTransitionTimestamp, @@ -769,20 +769,20 @@ func (nc *Controller) monitorNodeHealth() error { if nc.useTaintBasedEvictions { removed, err := nc.markNodeAsReachable(node) if err != nil { - glog.Errorf("Failed to remove taints from node %v. Will retry in next iteration.", node.Name) + klog.Errorf("Failed to remove taints from node %v. Will retry in next iteration.", node.Name) } if removed { - glog.V(2).Infof("Node %s is healthy again, removing all taints", node.Name) + klog.V(2).Infof("Node %s is healthy again, removing all taints", node.Name) } } else { if nc.cancelPodEviction(node) { - glog.V(2).Infof("Node %s is ready again, cancelled pod eviction", node.Name) + klog.V(2).Infof("Node %s is ready again, cancelled pod eviction", node.Name) } } // remove shutdown taint this is needed always depending do we use taintbased or not err := nc.markNodeAsNotShutdown(node) if err != nil { - glog.Errorf("Failed to remove taints from node %v. Will retry in next iteration.", node.Name) + klog.Errorf("Failed to remove taints from node %v. Will retry in next iteration.", node.Name) } } @@ -800,23 +800,23 @@ func (nc *Controller) monitorNodeHealth() error { // check is node shutdowned, if yes do not deleted it. Instead add taint shutdown, err := nc.nodeShutdownInCloudProvider(context.TODO(), node) if err != nil { - glog.Errorf("Error determining if node %v shutdown in cloud: %v", node.Name, err) + klog.Errorf("Error determining if node %v shutdown in cloud: %v", node.Name, err) } // node shutdown if shutdown && err == nil { err = controller.AddOrUpdateTaintOnNode(nc.kubeClient, node.Name, controller.ShutdownTaint) if err != nil { - glog.Errorf("Error patching node taints: %v", err) + klog.Errorf("Error patching node taints: %v", err) } continue } exists, err := nc.nodeExistsInCloudProvider(types.NodeName(node.Name)) if err != nil { - glog.Errorf("Error determining if node %v exists in cloud: %v", node.Name, err) + klog.Errorf("Error determining if node %v exists in cloud: %v", node.Name, err) continue } if !exists { - glog.V(2).Infof("Deleting node (no longer present in cloud provider): %s", node.Name) + klog.V(2).Infof("Deleting node (no longer present in cloud provider): %s", node.Name) nodeutil.RecordNodeEvent(nc.recorder, node.Name, string(node.UID), v1.EventTypeNormal, "DeletingNode", fmt.Sprintf("Deleting Node %v because it's not present according to cloud provider", node.Name)) go func(nodeName string) { defer utilruntime.HandleCrash() @@ -824,7 +824,7 @@ func (nc *Controller) monitorNodeHealth() error { // is gone. Delete it without worrying about grace // periods. if err := nodeutil.ForcefullyDeleteNode(nc.kubeClient, nodeName); err != nil { - glog.Errorf("Unable to forcefully delete node %q: %v", nodeName, err) + klog.Errorf("Unable to forcefully delete node %q: %v", nodeName, err) } }(node.Name) } @@ -892,21 +892,21 @@ func (nc *Controller) tryUpdateNodeHealth(node *v1.Node) (time.Duration, v1.Node } _, observedCondition := v1node.GetNodeCondition(&node.Status, v1.NodeReady) if !found { - glog.Warningf("Missing timestamp for Node %s. Assuming now as a timestamp.", node.Name) + klog.Warningf("Missing timestamp for Node %s. Assuming now as a timestamp.", node.Name) savedNodeHealth = &nodeHealthData{ status: &node.Status, probeTimestamp: nc.now(), readyTransitionTimestamp: nc.now(), } } else if savedCondition == nil && observedCondition != nil { - glog.V(1).Infof("Creating timestamp entry for newly observed Node %s", node.Name) + klog.V(1).Infof("Creating timestamp entry for newly observed Node %s", node.Name) savedNodeHealth = &nodeHealthData{ status: &node.Status, probeTimestamp: nc.now(), readyTransitionTimestamp: nc.now(), } } else if savedCondition != nil && observedCondition == nil { - glog.Errorf("ReadyCondition was removed from Status of Node %s", node.Name) + klog.Errorf("ReadyCondition was removed from Status of Node %s", node.Name) // TODO: figure out what to do in this case. For now we do the same thing as above. savedNodeHealth = &nodeHealthData{ status: &node.Status, @@ -918,15 +918,15 @@ func (nc *Controller) tryUpdateNodeHealth(node *v1.Node) (time.Duration, v1.Node // If ReadyCondition changed since the last time we checked, we update the transition timestamp to "now", // otherwise we leave it as it is. if savedCondition.LastTransitionTime != observedCondition.LastTransitionTime { - glog.V(3).Infof("ReadyCondition for Node %s transitioned from %v to %v", node.Name, savedCondition, observedCondition) + klog.V(3).Infof("ReadyCondition for Node %s transitioned from %v to %v", node.Name, savedCondition, observedCondition) transitionTime = nc.now() } else { transitionTime = savedNodeHealth.readyTransitionTimestamp } - if glog.V(5) { - glog.V(5).Infof("Node %s ReadyCondition updated. Updating timestamp: %+v vs %+v.", node.Name, savedNodeHealth.status, node.Status) + if klog.V(5) { + klog.V(5).Infof("Node %s ReadyCondition updated. Updating timestamp: %+v vs %+v.", node.Name, savedNodeHealth.status, node.Status) } else { - glog.V(3).Infof("Node %s ReadyCondition updated. Updating timestamp.", node.Name) + klog.V(3).Infof("Node %s ReadyCondition updated. Updating timestamp.", node.Name) } savedNodeHealth = &nodeHealthData{ status: &node.Status, @@ -952,7 +952,7 @@ func (nc *Controller) tryUpdateNodeHealth(node *v1.Node) (time.Duration, v1.Node // NodeReady condition or lease was last set longer ago than gracePeriod, so // update it to Unknown (regardless of its current value) in the master. if currentReadyCondition == nil { - glog.V(2).Infof("node %v is never updated by kubelet", node.Name) + klog.V(2).Infof("node %v is never updated by kubelet", node.Name) node.Status.Conditions = append(node.Status.Conditions, v1.NodeCondition{ Type: v1.NodeReady, Status: v1.ConditionUnknown, @@ -962,7 +962,7 @@ func (nc *Controller) tryUpdateNodeHealth(node *v1.Node) (time.Duration, v1.Node LastTransitionTime: nc.now(), }) } else { - glog.V(4).Infof("node %v hasn't been updated for %+v. Last ready condition is: %+v", + klog.V(4).Infof("node %v hasn't been updated for %+v. Last ready condition is: %+v", node.Name, nc.now().Time.Sub(savedNodeHealth.probeTimestamp.Time), observedReadyCondition) if observedReadyCondition.Status != v1.ConditionUnknown { currentReadyCondition.Status = v1.ConditionUnknown @@ -988,7 +988,7 @@ func (nc *Controller) tryUpdateNodeHealth(node *v1.Node) (time.Duration, v1.Node for _, nodeConditionType := range remainingNodeConditionTypes { _, currentCondition := v1node.GetNodeCondition(&node.Status, nodeConditionType) if currentCondition == nil { - glog.V(2).Infof("Condition %v of node %v was never updated by kubelet", nodeConditionType, node.Name) + klog.V(2).Infof("Condition %v of node %v was never updated by kubelet", nodeConditionType, node.Name) node.Status.Conditions = append(node.Status.Conditions, v1.NodeCondition{ Type: nodeConditionType, Status: v1.ConditionUnknown, @@ -998,7 +998,7 @@ func (nc *Controller) tryUpdateNodeHealth(node *v1.Node) (time.Duration, v1.Node LastTransitionTime: nowTimestamp, }) } else { - glog.V(4).Infof("node %v hasn't been updated for %+v. Last %v is: %+v", + klog.V(4).Infof("node %v hasn't been updated for %+v. Last %v is: %+v", node.Name, nc.now().Time.Sub(savedNodeHealth.probeTimestamp.Time), nodeConditionType, currentCondition) if currentCondition.Status != v1.ConditionUnknown { currentCondition.Status = v1.ConditionUnknown @@ -1012,7 +1012,7 @@ func (nc *Controller) tryUpdateNodeHealth(node *v1.Node) (time.Duration, v1.Node _, currentCondition := v1node.GetNodeCondition(&node.Status, v1.NodeReady) if !apiequality.Semantic.DeepEqual(currentCondition, &observedReadyCondition) { if _, err = nc.kubeClient.CoreV1().Nodes().UpdateStatus(node); err != nil { - glog.Errorf("Error updating node %s: %v", node.Name, err) + klog.Errorf("Error updating node %s: %v", node.Name, err) return gracePeriod, observedReadyCondition, currentReadyCondition, err } nc.nodeHealthMap[node.Name] = &nodeHealthData{ @@ -1041,7 +1041,7 @@ func (nc *Controller) handleDisruption(zoneToNodeConditions map[string][]*v1.Nod } newZoneStates[k] = newState if _, had := nc.zoneStates[k]; !had { - glog.Errorf("Setting initial state for unseen zone: %v", k) + klog.Errorf("Setting initial state for unseen zone: %v", k) nc.zoneStates[k] = stateInitial } } @@ -1069,12 +1069,12 @@ func (nc *Controller) handleDisruption(zoneToNodeConditions map[string][]*v1.Nod if !allAreFullyDisrupted || !allWasFullyDisrupted { // We're switching to full disruption mode if allAreFullyDisrupted { - glog.V(0).Info("Controller detected that all Nodes are not-Ready. Entering master disruption mode.") + klog.V(0).Info("Controller detected that all Nodes are not-Ready. Entering master disruption mode.") for i := range nodes { if nc.useTaintBasedEvictions { _, err := nc.markNodeAsReachable(nodes[i]) if err != nil { - glog.Errorf("Failed to remove taints from Node %v", nodes[i].Name) + klog.Errorf("Failed to remove taints from Node %v", nodes[i].Name) } } else { nc.cancelPodEviction(nodes[i]) @@ -1096,7 +1096,7 @@ func (nc *Controller) handleDisruption(zoneToNodeConditions map[string][]*v1.Nod } // We're exiting full disruption mode if allWasFullyDisrupted { - glog.V(0).Info("Controller detected that some Nodes are Ready. Exiting master disruption mode.") + klog.V(0).Info("Controller detected that some Nodes are Ready. Exiting master disruption mode.") // When exiting disruption mode update probe timestamps on all Nodes. now := nc.now() for i := range nodes { @@ -1119,7 +1119,7 @@ func (nc *Controller) handleDisruption(zoneToNodeConditions map[string][]*v1.Nod if v == newState { continue } - glog.V(0).Infof("Controller detected that zone %v is now in state %v.", k, newState) + klog.V(0).Infof("Controller detected that zone %v is now in state %v.", k, newState) nc.setLimiterInZone(k, len(zoneToNodeConditions[k]), newState) nc.zoneStates[k] = newState } @@ -1219,7 +1219,7 @@ func (nc *Controller) addPodEvictorForNewZone(node *v1.Node) { flowcontrol.NewTokenBucketRateLimiter(nc.evictionLimiterQPS, scheduler.EvictionRateLimiterBurst)) } // Init the metric for the new zone. - glog.Infof("Initializing eviction metric for zone: %v", zone) + klog.Infof("Initializing eviction metric for zone: %v", zone) evictionsNumber.WithLabelValues(zone).Add(0) } } @@ -1232,7 +1232,7 @@ func (nc *Controller) cancelPodEviction(node *v1.Node) bool { defer nc.evictorLock.Unlock() wasDeleting := nc.zonePodEvictor[zone].Remove(node.Name) if wasDeleting { - glog.V(2).Infof("Cancelling pod Eviction on Node: %v", node.Name) + klog.V(2).Infof("Cancelling pod Eviction on Node: %v", node.Name) return true } return false @@ -1257,12 +1257,12 @@ func (nc *Controller) markNodeAsReachable(node *v1.Node) (bool, error) { defer nc.evictorLock.Unlock() err := controller.RemoveTaintOffNode(nc.kubeClient, node.Name, node, UnreachableTaintTemplate) if err != nil { - glog.Errorf("Failed to remove taint from node %v: %v", node.Name, err) + klog.Errorf("Failed to remove taint from node %v: %v", node.Name, err) return false, err } err = controller.RemoveTaintOffNode(nc.kubeClient, node.Name, node, NotReadyTaintTemplate) if err != nil { - glog.Errorf("Failed to remove taint from node %v: %v", node.Name, err) + klog.Errorf("Failed to remove taint from node %v: %v", node.Name, err) return false, err } return nc.zoneNoExecuteTainter[utilnode.GetZoneKey(node)].Remove(node.Name), nil @@ -1273,7 +1273,7 @@ func (nc *Controller) markNodeAsNotShutdown(node *v1.Node) error { defer nc.evictorLock.Unlock() err := controller.RemoveTaintOffNode(nc.kubeClient, node.Name, node, controller.ShutdownTaint) if err != nil { - glog.Errorf("Failed to remove taint from node %v: %v", node.Name, err) + klog.Errorf("Failed to remove taint from node %v: %v", node.Name, err) return err } return nil diff --git a/pkg/controller/nodelifecycle/scheduler/BUILD b/pkg/controller/nodelifecycle/scheduler/BUILD index d8f2f98790a..ad28abb8e0a 100644 --- a/pkg/controller/nodelifecycle/scheduler/BUILD +++ b/pkg/controller/nodelifecycle/scheduler/BUILD @@ -26,7 +26,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/nodelifecycle/scheduler/rate_limited_queue.go b/pkg/controller/nodelifecycle/scheduler/rate_limited_queue.go index 5d562fb71fc..03a1fcb889f 100644 --- a/pkg/controller/nodelifecycle/scheduler/rate_limited_queue.go +++ b/pkg/controller/nodelifecycle/scheduler/rate_limited_queue.go @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/flowcontrol" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -236,7 +236,7 @@ func (q *RateLimitedTimedQueue) Try(fn ActionFunc) { for ok { // rate limit the queue checking if !q.limiter.TryAccept() { - glog.V(10).Infof("Try rate limited for value: %v", val) + klog.V(10).Infof("Try rate limited for value: %v", val) // Try again later break } diff --git a/pkg/controller/nodelifecycle/scheduler/taint_manager.go b/pkg/controller/nodelifecycle/scheduler/taint_manager.go index 90e43757c4d..fbf683077f1 100644 --- a/pkg/controller/nodelifecycle/scheduler/taint_manager.go +++ b/pkg/controller/nodelifecycle/scheduler/taint_manager.go @@ -38,7 +38,7 @@ import ( "k8s.io/kubernetes/pkg/apis/core/helper" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -99,7 +99,7 @@ func deletePodHandler(c clientset.Interface, emitEventFunc func(types.Namespaced return func(args *WorkArgs) error { ns := args.NamespacedName.Namespace name := args.NamespacedName.Name - glog.V(0).Infof("NoExecuteTaintManager is deleting Pod: %v", args.NamespacedName.String()) + klog.V(0).Infof("NoExecuteTaintManager is deleting Pod: %v", args.NamespacedName.String()) if emitEventFunc != nil { emitEventFunc(args.NamespacedName) } @@ -170,12 +170,12 @@ func getMinTolerationTime(tolerations []v1.Toleration) time.Duration { func NewNoExecuteTaintManager(c clientset.Interface, getPod GetPodFunc, getNode GetNodeFunc) *NoExecuteTaintManager { eventBroadcaster := record.NewBroadcaster() recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "taint-controller"}) - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) if c != nil { - glog.V(0).Infof("Sending events to api server.") + klog.V(0).Infof("Sending events to api server.") eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: c.CoreV1().Events("")}) } else { - glog.Fatalf("kubeClient is nil when starting NodeController") + klog.Fatalf("kubeClient is nil when starting NodeController") } tm := &NoExecuteTaintManager{ @@ -195,7 +195,7 @@ func NewNoExecuteTaintManager(c clientset.Interface, getPod GetPodFunc, getNode // Run starts NoExecuteTaintManager which will run in loop until `stopCh` is closed. func (tc *NoExecuteTaintManager) Run(stopCh <-chan struct{}) { - glog.V(0).Infof("Starting NoExecuteTaintManager") + klog.V(0).Infof("Starting NoExecuteTaintManager") for i := 0; i < UpdateWorkerSize; i++ { tc.nodeUpdateChannels = append(tc.nodeUpdateChannels, make(chan nodeUpdateItem, NodeUpdateChannelSize)) @@ -356,7 +356,7 @@ func (tc *NoExecuteTaintManager) processPodOnNode( } allTolerated, usedTolerations := v1helper.GetMatchingTolerations(taints, tolerations) if !allTolerated { - glog.V(2).Infof("Not all taints are tolerated after update for Pod %v on %v", podNamespacedName.String(), nodeName) + klog.V(2).Infof("Not all taints are tolerated after update for Pod %v on %v", podNamespacedName.String(), nodeName) // We're canceling scheduled work (if any), as we're going to delete the Pod right away. tc.cancelWorkWithEvent(podNamespacedName) tc.taintEvictionQueue.AddWork(NewWorkArgs(podNamespacedName.Name, podNamespacedName.Namespace), time.Now(), time.Now()) @@ -365,7 +365,7 @@ func (tc *NoExecuteTaintManager) processPodOnNode( minTolerationTime := getMinTolerationTime(usedTolerations) // getMinTolerationTime returns negative value to denote infinite toleration. if minTolerationTime < 0 { - glog.V(4).Infof("New tolerations for %v tolerate forever. Scheduled deletion won't be cancelled if already scheduled.", podNamespacedName.String()) + klog.V(4).Infof("New tolerations for %v tolerate forever. Scheduled deletion won't be cancelled if already scheduled.", podNamespacedName.String()) return } @@ -388,7 +388,7 @@ func (tc *NoExecuteTaintManager) handlePodUpdate(podUpdate podUpdateItem) { if apierrors.IsNotFound(err) { // Delete podNamespacedName := types.NamespacedName{Namespace: podUpdate.podNamespace, Name: podUpdate.podName} - glog.V(4).Infof("Noticed pod deletion: %#v", podNamespacedName) + klog.V(4).Infof("Noticed pod deletion: %#v", podNamespacedName) tc.cancelWorkWithEvent(podNamespacedName) return } @@ -403,7 +403,7 @@ func (tc *NoExecuteTaintManager) handlePodUpdate(podUpdate podUpdateItem) { // Create or Update podNamespacedName := types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name} - glog.V(4).Infof("Noticed pod update: %#v", podNamespacedName) + klog.V(4).Infof("Noticed pod update: %#v", podNamespacedName) nodeName := pod.Spec.NodeName if nodeName == "" { return @@ -427,7 +427,7 @@ func (tc *NoExecuteTaintManager) handleNodeUpdate(nodeUpdate nodeUpdateItem) { if err != nil { if apierrors.IsNotFound(err) { // Delete - glog.V(4).Infof("Noticed node deletion: %#v", nodeUpdate.nodeName) + klog.V(4).Infof("Noticed node deletion: %#v", nodeUpdate.nodeName) tc.taintedNodesLock.Lock() defer tc.taintedNodesLock.Unlock() delete(tc.taintedNodes, nodeUpdate.nodeName) @@ -438,12 +438,12 @@ func (tc *NoExecuteTaintManager) handleNodeUpdate(nodeUpdate nodeUpdateItem) { } // Create or Update - glog.V(4).Infof("Noticed node update: %#v", nodeUpdate) + klog.V(4).Infof("Noticed node update: %#v", nodeUpdate) taints := getNoExecuteTaints(node.Spec.Taints) func() { tc.taintedNodesLock.Lock() defer tc.taintedNodesLock.Unlock() - glog.V(4).Infof("Updating known taints on node %v: %v", node.Name, taints) + klog.V(4).Infof("Updating known taints on node %v: %v", node.Name, taints) if len(taints) == 0 { delete(tc.taintedNodes, node.Name) } else { @@ -452,7 +452,7 @@ func (tc *NoExecuteTaintManager) handleNodeUpdate(nodeUpdate nodeUpdateItem) { }() pods, err := getPodsAssignedToNode(tc.client, node.Name) if err != nil { - glog.Errorf(err.Error()) + klog.Errorf(err.Error()) return } if len(pods) == 0 { @@ -460,7 +460,7 @@ func (tc *NoExecuteTaintManager) handleNodeUpdate(nodeUpdate nodeUpdateItem) { } // Short circuit, to make this controller a bit faster. if len(taints) == 0 { - glog.V(4).Infof("All taints were removed from the Node %v. Cancelling all evictions...", node.Name) + klog.V(4).Infof("All taints were removed from the Node %v. Cancelling all evictions...", node.Name) for i := range pods { tc.cancelWorkWithEvent(types.NamespacedName{Namespace: pods[i].Namespace, Name: pods[i].Name}) } diff --git a/pkg/controller/nodelifecycle/scheduler/timed_workers.go b/pkg/controller/nodelifecycle/scheduler/timed_workers.go index 2eef59b041b..d995fb22a36 100644 --- a/pkg/controller/nodelifecycle/scheduler/timed_workers.go +++ b/pkg/controller/nodelifecycle/scheduler/timed_workers.go @@ -22,7 +22,7 @@ import ( "k8s.io/apimachinery/pkg/types" - "github.com/golang/glog" + "k8s.io/klog" ) // WorkArgs keeps arguments that will be passed to the function executed by the worker. @@ -107,12 +107,12 @@ func (q *TimedWorkerQueue) getWrappedWorkerFunc(key string) func(args *WorkArgs) // AddWork adds a work to the WorkerQueue which will be executed not earlier than `fireAt`. func (q *TimedWorkerQueue) AddWork(args *WorkArgs, createdAt time.Time, fireAt time.Time) { key := args.KeyFromWorkArgs() - glog.V(4).Infof("Adding TimedWorkerQueue item %v at %v to be fired at %v", key, createdAt, fireAt) + klog.V(4).Infof("Adding TimedWorkerQueue item %v at %v to be fired at %v", key, createdAt, fireAt) q.Lock() defer q.Unlock() if _, exists := q.workers[key]; exists { - glog.Warningf("Trying to add already existing work for %+v. Skipping.", args) + klog.Warningf("Trying to add already existing work for %+v. Skipping.", args) return } worker := CreateWorker(args, createdAt, fireAt, q.getWrappedWorkerFunc(key)) @@ -126,7 +126,7 @@ func (q *TimedWorkerQueue) CancelWork(key string) bool { worker, found := q.workers[key] result := false if found { - glog.V(4).Infof("Cancelling TimedWorkerQueue item %v at %v", key, time.Now()) + klog.V(4).Infof("Cancelling TimedWorkerQueue item %v at %v", key, time.Now()) if worker != nil { result = true worker.Cancel() diff --git a/pkg/controller/podautoscaler/BUILD b/pkg/controller/podautoscaler/BUILD index 801a5a52262..7ff398dea32 100644 --- a/pkg/controller/podautoscaler/BUILD +++ b/pkg/controller/podautoscaler/BUILD @@ -40,7 +40,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/podautoscaler/horizontal.go b/pkg/controller/podautoscaler/horizontal.go index 1ac7c1f9800..b043564abe8 100644 --- a/pkg/controller/podautoscaler/horizontal.go +++ b/pkg/controller/podautoscaler/horizontal.go @@ -21,7 +21,6 @@ import ( "math" "time" - "github.com/golang/glog" autoscalingv1 "k8s.io/api/autoscaling/v1" autoscalingv2 "k8s.io/api/autoscaling/v2beta2" "k8s.io/api/core/v1" @@ -46,6 +45,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/controller" metricsclient "k8s.io/kubernetes/pkg/controller/podautoscaler/metrics" @@ -108,7 +108,7 @@ func NewHorizontalController( ) *HorizontalController { broadcaster := record.NewBroadcaster() - broadcaster.StartLogging(glog.Infof) + broadcaster.StartLogging(klog.Infof) broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: evtNamespacer.Events("")}) recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "horizontal-pod-autoscaler"}) @@ -153,8 +153,8 @@ func (a *HorizontalController) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer a.queue.ShutDown() - glog.Infof("Starting HPA controller") - defer glog.Infof("Shutting down HPA controller") + klog.Infof("Starting HPA controller") + defer klog.Infof("Shutting down HPA controller") if !controller.WaitForCacheSync("HPA", stopCh, a.hpaListerSynced, a.podListerSynced) { return @@ -197,7 +197,7 @@ func (a *HorizontalController) deleteHPA(obj interface{}) { func (a *HorizontalController) worker() { for a.processNextWorkItem() { } - glog.Infof("horizontal pod autoscaler controller worker shutting down") + klog.Infof("horizontal pod autoscaler controller worker shutting down") } func (a *HorizontalController) processNextWorkItem() bool { @@ -306,7 +306,7 @@ func (a *HorizontalController) reconcileKey(key string) error { hpa, err := a.hpaLister.HorizontalPodAutoscalers(namespace).Get(name) if errors.IsNotFound(err) { - glog.Infof("Horizontal Pod Autoscaler %s has been deleted in %s", name, namespace) + klog.Infof("Horizontal Pod Autoscaler %s has been deleted in %s", name, namespace) delete(a.recommendations, key) return nil } @@ -553,7 +553,7 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho return fmt.Errorf("failed to compute desired number of replicas based on listed metrics for %s: %v", reference, err) } - glog.V(4).Infof("proposing %v desired replicas (based on %s from %s) for %s", metricDesiredReplicas, metricName, timestamp, reference) + klog.V(4).Infof("proposing %v desired replicas (based on %s from %s) for %s", metricDesiredReplicas, metricName, timestamp, reference) rescaleMetric := "" if metricDesiredReplicas > desiredReplicas { @@ -585,10 +585,10 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho } setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "SucceededRescale", "the HPA controller was able to update the target scale to %d", desiredReplicas) a.eventRecorder.Eventf(hpa, v1.EventTypeNormal, "SuccessfulRescale", "New size: %d; reason: %s", desiredReplicas, rescaleReason) - glog.Infof("Successful rescale of %s, old size: %d, new size: %d, reason: %s", + klog.Infof("Successful rescale of %s, old size: %d, new size: %d, reason: %s", hpa.Name, currentReplicas, desiredReplicas, rescaleReason) } else { - glog.V(4).Infof("decided not to scale %s to %v (last scale time was %s)", reference, desiredReplicas, hpa.Status.LastScaleTime) + klog.V(4).Infof("decided not to scale %s to %v (last scale time was %s)", reference, desiredReplicas, hpa.Status.LastScaleTime) desiredReplicas = currentReplicas } @@ -770,7 +770,7 @@ func (a *HorizontalController) updateStatus(hpa *autoscalingv2.HorizontalPodAuto a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedUpdateStatus", err.Error()) return fmt.Errorf("failed to update status for %s: %v", hpa.Name, err) } - glog.V(2).Infof("Successfully updated status for %s", hpa.Name) + klog.V(2).Infof("Successfully updated status for %s", hpa.Name) return nil } diff --git a/pkg/controller/podautoscaler/metrics/BUILD b/pkg/controller/podautoscaler/metrics/BUILD index f745c9ac0ef..673b2859f83 100644 --- a/pkg/controller/podautoscaler/metrics/BUILD +++ b/pkg/controller/podautoscaler/metrics/BUILD @@ -23,8 +23,8 @@ go_library( "//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/custom_metrics:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/external_metrics:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/heapster/metrics/api/v1/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/podautoscaler/metrics/legacy_metrics_client.go b/pkg/controller/podautoscaler/metrics/legacy_metrics_client.go index cd4a1a18ac1..0db40548257 100644 --- a/pkg/controller/podautoscaler/metrics/legacy_metrics_client.go +++ b/pkg/controller/podautoscaler/metrics/legacy_metrics_client.go @@ -22,8 +22,8 @@ import ( "strings" "time" - "github.com/golang/glog" heapster "k8s.io/heapster/metrics/api/v1/types" + "k8s.io/klog" metricsapi "k8s.io/metrics/pkg/apis/metrics/v1alpha1" autoscaling "k8s.io/api/autoscaling/v2beta2" @@ -73,7 +73,7 @@ func (h *HeapsterMetricsClient) GetResourceMetric(resource v1.ResourceName, name return nil, time.Time{}, fmt.Errorf("failed to get pod resource metrics: %v", err) } - glog.V(4).Infof("Heapster metrics result: %s", string(resultRaw)) + klog.V(4).Infof("Heapster metrics result: %s", string(resultRaw)) metrics := metricsapi.PodMetricsList{} err = json.Unmarshal(resultRaw, &metrics) @@ -94,7 +94,7 @@ func (h *HeapsterMetricsClient) GetResourceMetric(resource v1.ResourceName, name resValue, found := c.Usage[v1.ResourceName(resource)] if !found { missing = true - glog.V(2).Infof("missing resource metric %v for container %s in pod %s/%s", resource, c.Name, namespace, m.Name) + klog.V(2).Infof("missing resource metric %v for container %s in pod %s/%s", resource, c.Name, namespace, m.Name) continue } podSum += resValue.MilliValue() @@ -150,7 +150,7 @@ func (h *HeapsterMetricsClient) GetRawMetric(metricName string, namespace string return nil, time.Time{}, fmt.Errorf("failed to unmarshal heapster response: %v", err) } - glog.V(4).Infof("Heapster metrics result: %s", string(resultRaw)) + klog.V(4).Infof("Heapster metrics result: %s", string(resultRaw)) if len(metrics.Items) != len(podNames) { // if we get too many metrics or two few metrics, we have no way of knowing which metric goes to which pod diff --git a/pkg/controller/podautoscaler/metrics/rest_metrics_client.go b/pkg/controller/podautoscaler/metrics/rest_metrics_client.go index 138001088e4..1f84866fc68 100644 --- a/pkg/controller/podautoscaler/metrics/rest_metrics_client.go +++ b/pkg/controller/podautoscaler/metrics/rest_metrics_client.go @@ -20,7 +20,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" autoscaling "k8s.io/api/autoscaling/v2beta2" "k8s.io/api/core/v1" @@ -81,7 +81,7 @@ func (c *resourceMetricsClient) GetResourceMetric(resource v1.ResourceName, name resValue, found := c.Usage[v1.ResourceName(resource)] if !found { missing = true - glog.V(2).Infof("missing resource metric %v for container %s in pod %s/%s", resource, c.Name, namespace, m.Name) + klog.V(2).Infof("missing resource metric %v for container %s in pod %s/%s", resource, c.Name, namespace, m.Name) break // containers loop } podSum += resValue.MilliValue() diff --git a/pkg/controller/podgc/BUILD b/pkg/controller/podgc/BUILD index 16464f6bfd4..eff9284cb77 100644 --- a/pkg/controller/podgc/BUILD +++ b/pkg/controller/podgc/BUILD @@ -26,7 +26,7 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/podgc/gc_controller.go b/pkg/controller/podgc/gc_controller.go index dfaa015eba7..a288bc86b37 100644 --- a/pkg/controller/podgc/gc_controller.go +++ b/pkg/controller/podgc/gc_controller.go @@ -34,7 +34,7 @@ import ( "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/util/metrics" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -59,7 +59,7 @@ func NewPodGC(kubeClient clientset.Interface, podInformer coreinformers.PodInfor kubeClient: kubeClient, terminatedPodThreshold: terminatedPodThreshold, deletePod: func(namespace, name string) error { - glog.Infof("PodGC is force deleting Pod: %v/%v", namespace, name) + klog.Infof("PodGC is force deleting Pod: %v/%v", namespace, name) return kubeClient.CoreV1().Pods(namespace).Delete(name, metav1.NewDeleteOptions(0)) }, } @@ -73,8 +73,8 @@ func NewPodGC(kubeClient clientset.Interface, podInformer coreinformers.PodInfor func (gcc *PodGCController) Run(stop <-chan struct{}) { defer utilruntime.HandleCrash() - glog.Infof("Starting GC controller") - defer glog.Infof("Shutting down GC controller") + klog.Infof("Starting GC controller") + defer klog.Infof("Shutting down GC controller") if !controller.WaitForCacheSync("GC", stop, gcc.podListerSynced) { return @@ -88,7 +88,7 @@ func (gcc *PodGCController) Run(stop <-chan struct{}) { func (gcc *PodGCController) gc() { pods, err := gcc.podLister.List(labels.Everything()) if err != nil { - glog.Errorf("Error while listing all Pods: %v", err) + klog.Errorf("Error while listing all Pods: %v", err) return } if gcc.terminatedPodThreshold > 0 { @@ -122,7 +122,7 @@ func (gcc *PodGCController) gcTerminated(pods []*v1.Pod) { deleteCount = terminatedPodCount } if deleteCount > 0 { - glog.Infof("garbage collecting %v pods", deleteCount) + klog.Infof("garbage collecting %v pods", deleteCount) } var wait sync.WaitGroup @@ -141,7 +141,7 @@ func (gcc *PodGCController) gcTerminated(pods []*v1.Pod) { // gcOrphaned deletes pods that are bound to nodes that don't exist. func (gcc *PodGCController) gcOrphaned(pods []*v1.Pod) { - glog.V(4).Infof("GC'ing orphaned") + klog.V(4).Infof("GC'ing orphaned") // We want to get list of Nodes from the etcd, to make sure that it's as fresh as possible. nodes, err := gcc.kubeClient.CoreV1().Nodes().List(metav1.ListOptions{}) if err != nil { @@ -159,29 +159,29 @@ func (gcc *PodGCController) gcOrphaned(pods []*v1.Pod) { if nodeNames.Has(pod.Spec.NodeName) { continue } - glog.V(2).Infof("Found orphaned Pod %v/%v assigned to the Node %v. Deleting.", pod.Namespace, pod.Name, pod.Spec.NodeName) + klog.V(2).Infof("Found orphaned Pod %v/%v assigned to the Node %v. Deleting.", pod.Namespace, pod.Name, pod.Spec.NodeName) if err := gcc.deletePod(pod.Namespace, pod.Name); err != nil { utilruntime.HandleError(err) } else { - glog.V(0).Infof("Forced deletion of orphaned Pod %v/%v succeeded", pod.Namespace, pod.Name) + klog.V(0).Infof("Forced deletion of orphaned Pod %v/%v succeeded", pod.Namespace, pod.Name) } } } // gcUnscheduledTerminating deletes pods that are terminating and haven't been scheduled to a particular node. func (gcc *PodGCController) gcUnscheduledTerminating(pods []*v1.Pod) { - glog.V(4).Infof("GC'ing unscheduled pods which are terminating.") + klog.V(4).Infof("GC'ing unscheduled pods which are terminating.") for _, pod := range pods { if pod.DeletionTimestamp == nil || len(pod.Spec.NodeName) > 0 { continue } - glog.V(2).Infof("Found unscheduled terminating Pod %v/%v not assigned to any Node. Deleting.", pod.Namespace, pod.Name) + klog.V(2).Infof("Found unscheduled terminating Pod %v/%v not assigned to any Node. Deleting.", pod.Namespace, pod.Name) if err := gcc.deletePod(pod.Namespace, pod.Name); err != nil { utilruntime.HandleError(err) } else { - glog.V(0).Infof("Forced deletion of unscheduled terminating Pod %v/%v succeeded", pod.Namespace, pod.Name) + klog.V(0).Infof("Forced deletion of unscheduled terminating Pod %v/%v succeeded", pod.Namespace, pod.Name) } } } diff --git a/pkg/controller/replicaset/BUILD b/pkg/controller/replicaset/BUILD index 5d3a179e5a4..e833fdf6e37 100644 --- a/pkg/controller/replicaset/BUILD +++ b/pkg/controller/replicaset/BUILD @@ -38,7 +38,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/integer:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/replicaset/replica_set.go b/pkg/controller/replicaset/replica_set.go index f8743980636..03edbe1365c 100644 --- a/pkg/controller/replicaset/replica_set.go +++ b/pkg/controller/replicaset/replica_set.go @@ -35,7 +35,6 @@ import ( "sync" "time" - "github.com/golang/glog" apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -55,6 +54,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/integer" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/util/metrics" @@ -108,7 +108,7 @@ type ReplicaSetController struct { // NewReplicaSetController configures a replica set controller with the specified event recorder func NewReplicaSetController(rsInformer appsinformers.ReplicaSetInformer, podInformer coreinformers.PodInformer, kubeClient clientset.Interface, burstReplicas int) *ReplicaSetController { eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) return NewBaseController(rsInformer, podInformer, kubeClient, burstReplicas, apps.SchemeGroupVersion.WithKind("ReplicaSet"), @@ -179,8 +179,8 @@ func (rsc *ReplicaSetController) Run(workers int, stopCh <-chan struct{}) { defer rsc.queue.ShutDown() controllerName := strings.ToLower(rsc.Kind) - glog.Infof("Starting %v controller", controllerName) - defer glog.Infof("Shutting down %v controller", controllerName) + klog.Infof("Starting %v controller", controllerName) + defer klog.Infof("Shutting down %v controller", controllerName) if !controller.WaitForCacheSync(rsc.Kind, stopCh, rsc.podListerSynced, rsc.rsListerSynced) { return @@ -246,7 +246,7 @@ func (rsc *ReplicaSetController) updateRS(old, cur interface{}) { // that bad as ReplicaSets that haven't met expectations yet won't // sync, and all the listing is done using local stores. if *(oldRS.Spec.Replicas) != *(curRS.Spec.Replicas) { - glog.V(4).Infof("%v %v updated. Desired pod count change: %d->%d", rsc.Kind, curRS.Name, *(oldRS.Spec.Replicas), *(curRS.Spec.Replicas)) + klog.V(4).Infof("%v %v updated. Desired pod count change: %d->%d", rsc.Kind, curRS.Name, *(oldRS.Spec.Replicas), *(curRS.Spec.Replicas)) } rsc.enqueueReplicaSet(cur) } @@ -272,7 +272,7 @@ func (rsc *ReplicaSetController) addPod(obj interface{}) { if err != nil { return } - glog.V(4).Infof("Pod %s created: %#v.", pod.Name, pod) + klog.V(4).Infof("Pod %s created: %#v.", pod.Name, pod) rsc.expectations.CreationObserved(rsKey) rsc.enqueueReplicaSet(rs) return @@ -286,7 +286,7 @@ func (rsc *ReplicaSetController) addPod(obj interface{}) { if len(rss) == 0 { return } - glog.V(4).Infof("Orphan Pod %s created: %#v.", pod.Name, pod) + klog.V(4).Infof("Orphan Pod %s created: %#v.", pod.Name, pod) for _, rs := range rss { rsc.enqueueReplicaSet(rs) } @@ -335,7 +335,7 @@ func (rsc *ReplicaSetController) updatePod(old, cur interface{}) { if rs == nil { return } - glog.V(4).Infof("Pod %s updated, objectMeta %+v -> %+v.", curPod.Name, oldPod.ObjectMeta, curPod.ObjectMeta) + klog.V(4).Infof("Pod %s updated, objectMeta %+v -> %+v.", curPod.Name, oldPod.ObjectMeta, curPod.ObjectMeta) rsc.enqueueReplicaSet(rs) // TODO: MinReadySeconds in the Pod will generate an Available condition to be added in // the Pod status which in turn will trigger a requeue of the owning replica set thus @@ -345,7 +345,7 @@ func (rsc *ReplicaSetController) updatePod(old, cur interface{}) { // Note that this still suffers from #29229, we are just moving the problem one level // "closer" to kubelet (from the deployment to the replica set controller). if !podutil.IsPodReady(oldPod) && podutil.IsPodReady(curPod) && rs.Spec.MinReadySeconds > 0 { - glog.V(2).Infof("%v %q will be enqueued after %ds for availability check", rsc.Kind, rs.Name, rs.Spec.MinReadySeconds) + klog.V(2).Infof("%v %q will be enqueued after %ds for availability check", rsc.Kind, rs.Name, rs.Spec.MinReadySeconds) // Add a second to avoid milliseconds skew in AddAfter. // See https://github.com/kubernetes/kubernetes/issues/39785#issuecomment-279959133 for more info. rsc.enqueueReplicaSetAfter(rs, (time.Duration(rs.Spec.MinReadySeconds)*time.Second)+time.Second) @@ -360,7 +360,7 @@ func (rsc *ReplicaSetController) updatePod(old, cur interface{}) { if len(rss) == 0 { return } - glog.V(4).Infof("Orphan Pod %s updated, objectMeta %+v -> %+v.", curPod.Name, oldPod.ObjectMeta, curPod.ObjectMeta) + klog.V(4).Infof("Orphan Pod %s updated, objectMeta %+v -> %+v.", curPod.Name, oldPod.ObjectMeta, curPod.ObjectMeta) for _, rs := range rss { rsc.enqueueReplicaSet(rs) } @@ -402,7 +402,7 @@ func (rsc *ReplicaSetController) deletePod(obj interface{}) { if err != nil { return } - glog.V(4).Infof("Pod %s/%s deleted through %v, timestamp %+v: %#v.", pod.Namespace, pod.Name, utilruntime.GetCaller(), pod.DeletionTimestamp, pod) + klog.V(4).Infof("Pod %s/%s deleted through %v, timestamp %+v: %#v.", pod.Namespace, pod.Name, utilruntime.GetCaller(), pod.DeletionTimestamp, pod) rsc.expectations.DeletionObserved(rsKey, controller.PodKey(pod)) rsc.enqueueReplicaSet(rs) } @@ -474,7 +474,7 @@ func (rsc *ReplicaSetController) manageReplicas(filteredPods []*v1.Pod, rs *apps // into a performance bottleneck. We should generate a UID for the pod // beforehand and store it via ExpectCreations. rsc.expectations.ExpectCreations(rsKey, diff) - glog.V(2).Infof("Too few replicas for %v %s/%s, need %d, creating %d", rsc.Kind, rs.Namespace, rs.Name, *(rs.Spec.Replicas), diff) + klog.V(2).Infof("Too few replicas for %v %s/%s, need %d, creating %d", rsc.Kind, rs.Namespace, rs.Name, *(rs.Spec.Replicas), diff) // Batch the pod creates. Batch sizes start at SlowStartInitialBatchSize // and double with each successful iteration in a kind of "slow start". // This handles attempts to start large numbers of pods that would @@ -511,7 +511,7 @@ func (rsc *ReplicaSetController) manageReplicas(filteredPods []*v1.Pod, rs *apps // The skipped pods will be retried later. The next controller resync will // retry the slow start process. if skippedPods := diff - successfulCreations; skippedPods > 0 { - glog.V(2).Infof("Slow-start failure. Skipping creation of %d pods, decrementing expectations for %v %v/%v", skippedPods, rsc.Kind, rs.Namespace, rs.Name) + klog.V(2).Infof("Slow-start failure. Skipping creation of %d pods, decrementing expectations for %v %v/%v", skippedPods, rsc.Kind, rs.Namespace, rs.Name) for i := 0; i < skippedPods; i++ { // Decrement the expected number of creates because the informer won't observe this pod rsc.expectations.CreationObserved(rsKey) @@ -522,7 +522,7 @@ func (rsc *ReplicaSetController) manageReplicas(filteredPods []*v1.Pod, rs *apps if diff > rsc.burstReplicas { diff = rsc.burstReplicas } - glog.V(2).Infof("Too many replicas for %v %s/%s, need %d, deleting %d", rsc.Kind, rs.Namespace, rs.Name, *(rs.Spec.Replicas), diff) + klog.V(2).Infof("Too many replicas for %v %s/%s, need %d, deleting %d", rsc.Kind, rs.Namespace, rs.Name, *(rs.Spec.Replicas), diff) // Choose which Pods to delete, preferring those in earlier phases of startup. podsToDelete := getPodsToDelete(filteredPods, diff) @@ -544,7 +544,7 @@ func (rsc *ReplicaSetController) manageReplicas(filteredPods []*v1.Pod, rs *apps if err := rsc.podControl.DeletePod(rs.Namespace, targetPod.Name, rs); err != nil { // Decrement the expected number of deletes because the informer won't observe this deletion podKey := controller.PodKey(targetPod) - glog.V(2).Infof("Failed to delete %v, decrementing expectations for %v %s/%s", podKey, rsc.Kind, rs.Namespace, rs.Name) + klog.V(2).Infof("Failed to delete %v, decrementing expectations for %v %s/%s", podKey, rsc.Kind, rs.Namespace, rs.Name) rsc.expectations.DeletionObserved(rsKey, podKey) errCh <- err } @@ -572,7 +572,7 @@ func (rsc *ReplicaSetController) syncReplicaSet(key string) error { startTime := time.Now() defer func() { - glog.V(4).Infof("Finished syncing %v %q (%v)", rsc.Kind, key, time.Since(startTime)) + klog.V(4).Infof("Finished syncing %v %q (%v)", rsc.Kind, key, time.Since(startTime)) }() namespace, name, err := cache.SplitMetaNamespaceKey(key) @@ -581,7 +581,7 @@ func (rsc *ReplicaSetController) syncReplicaSet(key string) error { } rs, err := rsc.rsLister.ReplicaSets(namespace).Get(name) if errors.IsNotFound(err) { - glog.V(4).Infof("%v %v has been deleted", rsc.Kind, key) + klog.V(4).Infof("%v %v has been deleted", rsc.Kind, key) rsc.expectations.DeleteExpectations(key) return nil } diff --git a/pkg/controller/replicaset/replica_set_utils.go b/pkg/controller/replicaset/replica_set_utils.go index de915e522a3..a2f7795726a 100644 --- a/pkg/controller/replicaset/replica_set_utils.go +++ b/pkg/controller/replicaset/replica_set_utils.go @@ -22,7 +22,7 @@ import ( "fmt" "reflect" - "github.com/golang/glog" + "k8s.io/klog" apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" @@ -55,7 +55,7 @@ func updateReplicaSetStatus(c appsclient.ReplicaSetInterface, rs *apps.ReplicaSe var getErr, updateErr error var updatedRS *apps.ReplicaSet for i, rs := 0, rs; ; i++ { - glog.V(4).Infof(fmt.Sprintf("Updating status for %v: %s/%s, ", rs.Kind, rs.Namespace, rs.Name) + + klog.V(4).Infof(fmt.Sprintf("Updating status for %v: %s/%s, ", rs.Kind, rs.Namespace, rs.Name) + fmt.Sprintf("replicas %d->%d (need %d), ", rs.Status.Replicas, newStatus.Replicas, *(rs.Spec.Replicas)) + fmt.Sprintf("fullyLabeledReplicas %d->%d, ", rs.Status.FullyLabeledReplicas, newStatus.FullyLabeledReplicas) + fmt.Sprintf("readyReplicas %d->%d, ", rs.Status.ReadyReplicas, newStatus.ReadyReplicas) + diff --git a/pkg/controller/replication/BUILD b/pkg/controller/replication/BUILD index 635de92c6e0..344101c42b4 100644 --- a/pkg/controller/replication/BUILD +++ b/pkg/controller/replication/BUILD @@ -38,7 +38,7 @@ go_library( "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/replication/replication_controller.go b/pkg/controller/replication/replication_controller.go index 4b6ac5f4a73..d59f6208267 100644 --- a/pkg/controller/replication/replication_controller.go +++ b/pkg/controller/replication/replication_controller.go @@ -26,13 +26,13 @@ limitations under the License. package replication import ( - "github.com/golang/glog" "k8s.io/api/core/v1" coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/replicaset" ) @@ -51,7 +51,7 @@ type ReplicationManager struct { // NewReplicationManager configures a replication manager with the specified event recorder func NewReplicationManager(podInformer coreinformers.PodInformer, rcInformer coreinformers.ReplicationControllerInformer, kubeClient clientset.Interface, burstReplicas int) *ReplicationManager { eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) return &ReplicationManager{ *replicaset.NewBaseController(informerAdapter{rcInformer}, podInformer, clientsetAdapter{kubeClient}, burstReplicas, diff --git a/pkg/controller/resourcequota/BUILD b/pkg/controller/resourcequota/BUILD index d09d8f4a76b..cce526c9bc8 100644 --- a/pkg/controller/resourcequota/BUILD +++ b/pkg/controller/resourcequota/BUILD @@ -37,7 +37,7 @@ go_library( "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/resourcequota/resource_quota_controller.go b/pkg/controller/resourcequota/resource_quota_controller.go index 2212bfd4b16..948b31814f8 100644 --- a/pkg/controller/resourcequota/resource_quota_controller.go +++ b/pkg/controller/resourcequota/resource_quota_controller.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -180,7 +180,7 @@ func NewResourceQuotaController(options *ResourceQuotaControllerOptions) (*Resou // enqueueAll is called at the fullResyncPeriod interval to force a full recalculation of quota usage statistics func (rq *ResourceQuotaController) enqueueAll() { - defer glog.V(4).Infof("Resource quota controller queued all resource quota for full calculation of usage") + defer klog.V(4).Infof("Resource quota controller queued all resource quota for full calculation of usage") rqs, err := rq.rqLister.List(labels.Everything()) if err != nil { utilruntime.HandleError(fmt.Errorf("unable to enqueue all - error listing resource quotas: %v", err)) @@ -200,7 +200,7 @@ func (rq *ResourceQuotaController) enqueueAll() { func (rq *ResourceQuotaController) enqueueResourceQuota(obj interface{}) { key, err := controller.KeyFunc(obj) if err != nil { - glog.Errorf("Couldn't get key for object %+v: %v", obj, err) + klog.Errorf("Couldn't get key for object %+v: %v", obj, err) return } rq.queue.Add(key) @@ -209,7 +209,7 @@ func (rq *ResourceQuotaController) enqueueResourceQuota(obj interface{}) { func (rq *ResourceQuotaController) addQuota(obj interface{}) { key, err := controller.KeyFunc(obj) if err != nil { - glog.Errorf("Couldn't get key for object %+v: %v", obj, err) + klog.Errorf("Couldn't get key for object %+v: %v", obj, err) return } @@ -261,7 +261,7 @@ func (rq *ResourceQuotaController) worker(queue workqueue.RateLimitingInterface) return func() { for { if quit := workFunc(); quit { - glog.Infof("resource quota controller worker shutting down") + klog.Infof("resource quota controller worker shutting down") return } } @@ -273,8 +273,8 @@ func (rq *ResourceQuotaController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer rq.queue.ShutDown() - glog.Infof("Starting resource quota controller") - defer glog.Infof("Shutting down resource quota controller") + klog.Infof("Starting resource quota controller") + defer klog.Infof("Shutting down resource quota controller") if rq.quotaMonitor != nil { go rq.quotaMonitor.Run(stopCh) @@ -298,7 +298,7 @@ func (rq *ResourceQuotaController) Run(workers int, stopCh <-chan struct{}) { func (rq *ResourceQuotaController) syncResourceQuotaFromKey(key string) (err error) { startTime := time.Now() defer func() { - glog.V(4).Infof("Finished syncing resource quota %q (%v)", key, time.Since(startTime)) + klog.V(4).Infof("Finished syncing resource quota %q (%v)", key, time.Since(startTime)) }() namespace, name, err := cache.SplitMetaNamespaceKey(key) @@ -307,11 +307,11 @@ func (rq *ResourceQuotaController) syncResourceQuotaFromKey(key string) (err err } quota, err := rq.rqLister.ResourceQuotas(namespace).Get(name) if errors.IsNotFound(err) { - glog.Infof("Resource quota has been deleted %v", key) + klog.Infof("Resource quota has been deleted %v", key) return nil } if err != nil { - glog.Infof("Unable to retrieve resource quota %v from store: %v", key, err) + klog.Infof("Unable to retrieve resource quota %v from store: %v", key, err) return err } return rq.syncResourceQuota(quota) @@ -347,24 +347,17 @@ func (rq *ResourceQuotaController) syncResourceQuota(resourceQuota *v1.ResourceQ // Create a usage object that is based on the quota resource version that will handle updates // by default, we preserve the past usage observation, and set hard to the current spec - usage := v1.ResourceQuota{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceQuota.Name, - Namespace: resourceQuota.Namespace, - ResourceVersion: resourceQuota.ResourceVersion, - Labels: resourceQuota.Labels, - Annotations: resourceQuota.Annotations}, - Status: v1.ResourceQuotaStatus{ - Hard: hardLimits, - Used: used, - }, + usage := resourceQuota.DeepCopy() + usage.Status = v1.ResourceQuotaStatus{ + Hard: hardLimits, + Used: used, } dirty = dirty || !quota.Equals(usage.Status.Used, resourceQuota.Status.Used) // there was a change observed by this controller that requires we update quota if dirty { - _, err = rq.rqClient.ResourceQuotas(usage.Namespace).UpdateStatus(&usage) + _, err = rq.rqClient.ResourceQuotas(usage.Namespace).UpdateStatus(usage) return err } return nil @@ -426,12 +419,12 @@ func (rq *ResourceQuotaController) Sync(discoveryFunc NamespacedResourcesFunc, p // Decide whether discovery has reported a change. if reflect.DeepEqual(oldResources, newResources) { - glog.V(4).Infof("no resource updates from discovery, skipping resource quota sync") + klog.V(4).Infof("no resource updates from discovery, skipping resource quota sync") return } // Something has changed, so track the new state and perform a sync. - glog.V(2).Infof("syncing resource quota controller with updated resources from discovery: %v", newResources) + klog.V(2).Infof("syncing resource quota controller with updated resources from discovery: %v", newResources) oldResources = newResources // Ensure workers are paused to avoid processing events before informers diff --git a/pkg/controller/resourcequota/resource_quota_monitor.go b/pkg/controller/resourcequota/resource_quota_monitor.go index aa77fca731f..c3d9e1f3e67 100644 --- a/pkg/controller/resourcequota/resource_quota_monitor.go +++ b/pkg/controller/resourcequota/resource_quota_monitor.go @@ -21,7 +21,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -173,11 +173,11 @@ func (qm *QuotaMonitor) controllerFor(resource schema.GroupVersionResource) (cac } shared, err := qm.informerFactory.ForResource(resource) if err == nil { - glog.V(4).Infof("QuotaMonitor using a shared informer for resource %q", resource.String()) + klog.V(4).Infof("QuotaMonitor using a shared informer for resource %q", resource.String()) shared.Informer().AddEventHandlerWithResyncPeriod(handlers, qm.resyncPeriod()) return shared.Informer().GetController(), nil } - glog.V(4).Infof("QuotaMonitor unable to use a shared informer for resource %q: %v", resource.String(), err) + klog.V(4).Infof("QuotaMonitor unable to use a shared informer for resource %q: %v", resource.String(), err) // TODO: if we can share storage with garbage collector, it may make sense to support other resources // until that time, aggregated api servers will have to run their own controller to reconcile their own quota. @@ -225,7 +225,7 @@ func (qm *QuotaMonitor) SyncMonitors(resources map[schema.GroupVersionResource]s listResourceFunc := generic.ListResourceUsingListerFunc(listerFunc, resource) evaluator = generic.NewObjectCountEvaluator(resource.GroupResource(), listResourceFunc, "") qm.registry.Add(evaluator) - glog.Infof("QuotaMonitor created object count evaluator for %s", resource.GroupResource()) + klog.Infof("QuotaMonitor created object count evaluator for %s", resource.GroupResource()) } // track the monitor @@ -240,7 +240,7 @@ func (qm *QuotaMonitor) SyncMonitors(resources map[schema.GroupVersionResource]s } } - glog.V(4).Infof("quota synced monitors; added %d, kept %d, removed %d", added, kept, len(toRemove)) + klog.V(4).Infof("quota synced monitors; added %d, kept %d, removed %d", added, kept, len(toRemove)) // NewAggregate returns nil if errs is 0-length return utilerrors.NewAggregate(errs) } @@ -272,7 +272,7 @@ func (qm *QuotaMonitor) StartMonitors() { started++ } } - glog.V(4).Infof("QuotaMonitor started %d new monitors, %d currently running", started, len(monitors)) + klog.V(4).Infof("QuotaMonitor started %d new monitors, %d currently running", started, len(monitors)) } // IsSynced returns true if any monitors exist AND all those monitors' @@ -298,8 +298,8 @@ func (qm *QuotaMonitor) IsSynced() bool { // Run sets the stop channel and starts monitor execution until stopCh is // closed. Any running monitors will be stopped before Run returns. func (qm *QuotaMonitor) Run(stopCh <-chan struct{}) { - glog.Infof("QuotaMonitor running") - defer glog.Infof("QuotaMonitor stopping") + klog.Infof("QuotaMonitor running") + defer klog.Infof("QuotaMonitor stopping") // Set up the stop channel. qm.monitorLock.Lock() @@ -323,7 +323,7 @@ func (qm *QuotaMonitor) Run(stopCh <-chan struct{}) { close(monitor.stopCh) } } - glog.Infof("QuotaMonitor stopped %d of %d monitors", stopped, len(monitors)) + klog.Infof("QuotaMonitor stopped %d of %d monitors", stopped, len(monitors)) } func (qm *QuotaMonitor) runProcessResourceChanges() { @@ -349,7 +349,7 @@ func (qm *QuotaMonitor) processResourceChanges() bool { utilruntime.HandleError(fmt.Errorf("cannot access obj: %v", err)) return true } - glog.V(4).Infof("QuotaMonitor process object: %s, namespace %s, name %s, uid %s, event type %v", event.gvr.String(), accessor.GetNamespace(), accessor.GetName(), string(accessor.GetUID()), event.eventType) + klog.V(4).Infof("QuotaMonitor process object: %s, namespace %s, name %s, uid %s, event type %v", event.gvr.String(), accessor.GetNamespace(), accessor.GetName(), string(accessor.GetUID()), event.eventType) qm.replenishmentFunc(event.gvr.GroupResource(), accessor.GetNamespace()) return true } diff --git a/pkg/controller/route/BUILD b/pkg/controller/route/BUILD index eada3d37d3b..b756e1bc204 100644 --- a/pkg/controller/route/BUILD +++ b/pkg/controller/route/BUILD @@ -33,7 +33,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/route/route_controller.go b/pkg/controller/route/route_controller.go index 5ae499efc6e..a8fd29e39c8 100644 --- a/pkg/controller/route/route_controller.go +++ b/pkg/controller/route/route_controller.go @@ -23,7 +23,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -75,11 +75,11 @@ func New(routes cloudprovider.Routes, kubeClient clientset.Interface, nodeInform } if clusterCIDR == nil { - glog.Fatal("RouteController: Must specify clusterCIDR.") + klog.Fatal("RouteController: Must specify clusterCIDR.") } eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "route_controller"}) rc := &RouteController{ @@ -99,8 +99,8 @@ func New(routes cloudprovider.Routes, kubeClient clientset.Interface, nodeInform func (rc *RouteController) Run(stopCh <-chan struct{}, syncPeriod time.Duration) { defer utilruntime.HandleCrash() - glog.Info("Starting route controller") - defer glog.Info("Shutting down route controller") + klog.Info("Starting route controller") + defer klog.Info("Shutting down route controller") if !controller.WaitForCacheSync("route", stopCh, rc.nodeListerSynced) { return @@ -117,7 +117,7 @@ func (rc *RouteController) Run(stopCh <-chan struct{}, syncPeriod time.Duration) // trigger reconciliation for that node. go wait.NonSlidingUntil(func() { if err := rc.reconcileNodeRoutes(); err != nil { - glog.Errorf("Couldn't reconcile node routes: %v", err) + klog.Errorf("Couldn't reconcile node routes: %v", err) } }, syncPeriod, stopCh) @@ -173,7 +173,7 @@ func (rc *RouteController) reconcile(nodes []*v1.Node, routes []*cloudprovider.R // Ensure that we don't have more than maxConcurrentRouteCreations // CreateRoute calls in flight. rateLimiter <- struct{}{} - glog.Infof("Creating route for node %s %s with hint %s, throttled %v", nodeName, route.DestinationCIDR, nameHint, time.Since(startTime)) + klog.Infof("Creating route for node %s %s with hint %s, throttled %v", nodeName, route.DestinationCIDR, nameHint, time.Since(startTime)) err := rc.routes.CreateRoute(context.TODO(), rc.clusterName, nameHint, route) <-rateLimiter @@ -189,14 +189,14 @@ func (rc *RouteController) reconcile(nodes []*v1.Node, routes []*cloudprovider.R Namespace: "", }, v1.EventTypeWarning, "FailedToCreateRoute", msg) } - glog.V(4).Infof(msg) + klog.V(4).Infof(msg) return err } - glog.Infof("Created route for node %s %s with hint %s after %v", nodeName, route.DestinationCIDR, nameHint, time.Now().Sub(startTime)) + klog.Infof("Created route for node %s %s with hint %s after %v", nodeName, route.DestinationCIDR, nameHint, time.Now().Sub(startTime)) return nil }) if err != nil { - glog.Errorf("Could not create route %s %s for node %s: %v", nameHint, route.DestinationCIDR, nodeName, err) + klog.Errorf("Could not create route %s %s for node %s: %v", nameHint, route.DestinationCIDR, nodeName, err) } }(nodeName, nameHint, route) } else { @@ -216,11 +216,11 @@ func (rc *RouteController) reconcile(nodes []*v1.Node, routes []*cloudprovider.R // Delete the route. go func(route *cloudprovider.Route, startTime time.Time) { defer wg.Done() - glog.Infof("Deleting route %s %s", route.Name, route.DestinationCIDR) + klog.Infof("Deleting route %s %s", route.Name, route.DestinationCIDR) if err := rc.routes.DeleteRoute(context.TODO(), rc.clusterName, route); err != nil { - glog.Errorf("Could not delete route %s %s after %v: %v", route.Name, route.DestinationCIDR, time.Since(startTime), err) + klog.Errorf("Could not delete route %s %s after %v: %v", route.Name, route.DestinationCIDR, time.Since(startTime), err) } else { - glog.Infof("Deleted route %s %s after %v", route.Name, route.DestinationCIDR, time.Since(startTime)) + klog.Infof("Deleted route %s %s after %v", route.Name, route.DestinationCIDR, time.Since(startTime)) } }(route, time.Now()) } @@ -254,13 +254,13 @@ func (rc *RouteController) updateNetworkingCondition(nodeName types.NodeName, ro }) } if err != nil { - glog.V(4).Infof("Error updating node %s, retrying: %v", nodeName, err) + klog.V(4).Infof("Error updating node %s, retrying: %v", nodeName, err) } return err }) if err != nil { - glog.Errorf("Error updating node %s: %v", nodeName, err) + klog.Errorf("Error updating node %s: %v", nodeName, err) } return err @@ -269,7 +269,7 @@ func (rc *RouteController) updateNetworkingCondition(nodeName types.NodeName, ro func (rc *RouteController) isResponsibleForRoute(route *cloudprovider.Route) bool { _, cidr, err := net.ParseCIDR(route.DestinationCIDR) if err != nil { - glog.Errorf("Ignoring route %s, unparsable CIDR: %v", route.Name, err) + klog.Errorf("Ignoring route %s, unparsable CIDR: %v", route.Name, err) return false } // Not responsible if this route's CIDR is not within our clusterCIDR diff --git a/pkg/controller/service/BUILD b/pkg/controller/service/BUILD index e2f0c1f92e7..70bc4cdb49f 100644 --- a/pkg/controller/service/BUILD +++ b/pkg/controller/service/BUILD @@ -33,7 +33,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/service/service_controller.go b/pkg/controller/service/service_controller.go index 22583df344b..667611bc5b1 100644 --- a/pkg/controller/service/service_controller.go +++ b/pkg/controller/service/service_controller.go @@ -24,7 +24,6 @@ import ( "reflect" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/runtime" @@ -40,6 +39,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/controller" kubefeatures "k8s.io/kubernetes/pkg/features" @@ -110,7 +110,7 @@ func New( clusterName string, ) (*ServiceController, error) { broadcaster := record.NewBroadcaster() - broadcaster.StartLogging(glog.Infof) + broadcaster.StartLogging(klog.Infof) broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "service-controller"}) @@ -160,7 +160,7 @@ func New( func (s *ServiceController) enqueueService(obj interface{}) { key, err := controller.KeyFunc(obj) if err != nil { - glog.Errorf("Couldn't get key for object %#v: %v", obj, err) + klog.Errorf("Couldn't get key for object %#v: %v", obj, err) return } s.queue.Add(key) @@ -180,8 +180,8 @@ func (s *ServiceController) Run(stopCh <-chan struct{}, workers int) { defer runtime.HandleCrash() defer s.queue.ShutDown() - glog.Info("Starting service controller") - defer glog.Info("Shutting down service controller") + klog.Info("Starting service controller") + defer klog.Info("Shutting down service controller") if !controller.WaitForCacheSync("service", stopCh, s.serviceListerSynced, s.nodeListerSynced) { return @@ -287,7 +287,7 @@ func (s *ServiceController) createLoadBalancerIfNeeded(key string, service *v1.S return fmt.Errorf("error getting LB for service %s: %v", key, err) } if exists { - glog.Infof("Deleting existing load balancer for service %s that no longer needs a load balancer.", key) + klog.Infof("Deleting existing load balancer for service %s that no longer needs a load balancer.", key) s.eventRecorder.Event(service, v1.EventTypeNormal, "DeletingLoadBalancer", "Deleting load balancer") if err := s.balancer.EnsureLoadBalancerDeleted(context.TODO(), s.clusterName, service); err != nil { return err @@ -297,7 +297,7 @@ func (s *ServiceController) createLoadBalancerIfNeeded(key string, service *v1.S newState = &v1.LoadBalancerStatus{} } else { - glog.V(2).Infof("Ensuring LB for service %s", key) + klog.V(2).Infof("Ensuring LB for service %s", key) // TODO: We could do a dry-run here if wanted to avoid the spurious cloud-calls & events when we restart @@ -327,7 +327,7 @@ func (s *ServiceController) createLoadBalancerIfNeeded(key string, service *v1.S return nil } } else { - glog.V(2).Infof("Not persisting unchanged LoadBalancerStatus for service %s to registry.", key) + klog.V(2).Infof("Not persisting unchanged LoadBalancerStatus for service %s to registry.", key) } return nil @@ -344,7 +344,7 @@ func (s *ServiceController) persistUpdate(service *v1.Service) error { // out so that we can process the delete, which we should soon be receiving // if we haven't already. if errors.IsNotFound(err) { - glog.Infof("Not persisting update to service '%s/%s' that no longer exists: %v", + klog.Infof("Not persisting update to service '%s/%s' that no longer exists: %v", service.Namespace, service.Name, err) return nil } @@ -353,7 +353,7 @@ func (s *ServiceController) persistUpdate(service *v1.Service) error { if errors.IsConflict(err) { return err } - glog.Warningf("Failed to persist updated LoadBalancerStatus to service '%s/%s' after creating its load balancer: %v", + klog.Warningf("Failed to persist updated LoadBalancerStatus to service '%s/%s' after creating its load balancer: %v", service.Namespace, service.Name, err) time.Sleep(clientRetryInterval) } @@ -613,7 +613,7 @@ func getNodeConditionPredicate() corelisters.NodeConditionPredicate { // We consider the node for load balancing only when its NodeReady condition status // is ConditionTrue if cond.Type == v1.NodeReady && cond.Status != v1.ConditionTrue { - glog.V(4).Infof("Ignoring node %v with %v condition status %v", node.Name, cond.Type, cond.Status) + klog.V(4).Infof("Ignoring node %v with %v condition status %v", node.Name, cond.Type, cond.Status) return false } } @@ -626,7 +626,7 @@ func getNodeConditionPredicate() corelisters.NodeConditionPredicate { func (s *ServiceController) nodeSyncLoop() { newHosts, err := s.nodeLister.ListWithPredicate(getNodeConditionPredicate()) if err != nil { - glog.Errorf("Failed to retrieve current set of nodes from node lister: %v", err) + klog.Errorf("Failed to retrieve current set of nodes from node lister: %v", err) return } if nodeSlicesEqualForLB(newHosts, s.knownHosts) { @@ -636,7 +636,7 @@ func (s *ServiceController) nodeSyncLoop() { return } - glog.Infof("Detected change in list of current cluster nodes. New node set: %v", + klog.Infof("Detected change in list of current cluster nodes. New node set: %v", nodeNames(newHosts)) // Try updating all services, and save the ones that fail to try again next @@ -644,7 +644,7 @@ func (s *ServiceController) nodeSyncLoop() { s.servicesToUpdate = s.cache.allServices() numServices := len(s.servicesToUpdate) s.servicesToUpdate = s.updateLoadBalancerHosts(s.servicesToUpdate, newHosts) - glog.Infof("Successfully updated %d out of %d load balancers to direct traffic to the updated set of nodes", + klog.Infof("Successfully updated %d out of %d load balancers to direct traffic to the updated set of nodes", numServices-len(s.servicesToUpdate), numServices) s.knownHosts = newHosts @@ -660,7 +660,7 @@ func (s *ServiceController) updateLoadBalancerHosts(services []*v1.Service, host return } if err := s.lockedUpdateLoadBalancerHosts(service, hosts); err != nil { - glog.Errorf("External error while updating load balancer: %v.", err) + klog.Errorf("External error while updating load balancer: %v.", err) servicesToRetry = append(servicesToRetry, service) } }() @@ -689,7 +689,7 @@ func (s *ServiceController) lockedUpdateLoadBalancerHosts(service *v1.Service, h // It's only an actual error if the load balancer still exists. if _, exists, err := s.balancer.GetLoadBalancer(context.TODO(), s.clusterName, service); err != nil { - glog.Errorf("External error while checking if load balancer %q exists: name, %v", s.balancer.GetLoadBalancerName(context.TODO(), s.clusterName, service), err) + klog.Errorf("External error while checking if load balancer %q exists: name, %v", s.balancer.GetLoadBalancerName(context.TODO(), s.clusterName, service), err) } else if !exists { return nil } @@ -713,7 +713,7 @@ func (s *ServiceController) syncService(key string) error { startTime := time.Now() var cachedService *cachedService defer func() { - glog.V(4).Infof("Finished syncing service %q (%v)", key, time.Since(startTime)) + klog.V(4).Infof("Finished syncing service %q (%v)", key, time.Since(startTime)) }() namespace, name, err := cache.SplitMetaNamespaceKey(key) @@ -726,10 +726,10 @@ func (s *ServiceController) syncService(key string) error { switch { case errors.IsNotFound(err): // service absence in store means watcher caught the deletion, ensure LB info is cleaned - glog.Infof("Service has been deleted %v. Attempting to cleanup load balancer resources", key) + klog.Infof("Service has been deleted %v. Attempting to cleanup load balancer resources", key) err = s.processServiceDeletion(key) case err != nil: - glog.Infof("Unable to retrieve service %v from store: %v", key, err) + klog.Infof("Unable to retrieve service %v from store: %v", key, err) default: cachedService = s.cache.getOrCreate(key) err = s.processServiceUpdate(cachedService, service, key) @@ -744,7 +744,7 @@ func (s *ServiceController) syncService(key string) error { func (s *ServiceController) processServiceDeletion(key string) error { cachedService, ok := s.cache.get(key) if !ok { - glog.Errorf("service %s not in cache even though the watcher thought it was. Ignoring the deletion", key) + klog.Errorf("service %s not in cache even though the watcher thought it was. Ignoring the deletion", key) return nil } return s.processLoadBalancerDelete(cachedService, key) diff --git a/pkg/controller/serviceaccount/BUILD b/pkg/controller/serviceaccount/BUILD index bf7efc43f05..2e2fd79f003 100644 --- a/pkg/controller/serviceaccount/BUILD +++ b/pkg/controller/serviceaccount/BUILD @@ -34,7 +34,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -59,8 +59,8 @@ go_test( "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/gopkg.in/square/go-jose.v2/jwt:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/serviceaccount/OWNERS b/pkg/controller/serviceaccount/OWNERS index a678215e984..d914c0d7195 100755 --- a/pkg/controller/serviceaccount/OWNERS +++ b/pkg/controller/serviceaccount/OWNERS @@ -1,7 +1,7 @@ approvers: -- liggitt -- deads2k +- sig-auth-serviceaccounts-approvers reviewers: -- liggitt -- deads2k -- enj +- sig-auth-serviceaccounts-reviewers +labels: +- sig/auth + diff --git a/pkg/controller/serviceaccount/serviceaccounts_controller.go b/pkg/controller/serviceaccount/serviceaccounts_controller.go index 8027e79ae11..5f10312ae17 100644 --- a/pkg/controller/serviceaccount/serviceaccounts_controller.go +++ b/pkg/controller/serviceaccount/serviceaccounts_controller.go @@ -20,7 +20,6 @@ import ( "fmt" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,6 +31,7 @@ import ( corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/util/metrics" ) @@ -112,8 +112,8 @@ func (c *ServiceAccountsController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() - glog.Infof("Starting service account controller") - defer glog.Infof("Shutting down service account controller") + klog.Infof("Starting service account controller") + defer klog.Infof("Shutting down service account controller") if !controller.WaitForCacheSync("service account", stopCh, c.saListerSynced, c.nsListerSynced) { return @@ -183,7 +183,7 @@ func (c *ServiceAccountsController) processNextWorkItem() bool { func (c *ServiceAccountsController) syncNamespace(key string) error { startTime := time.Now() defer func() { - glog.V(4).Infof("Finished syncing namespace %q (%v)", key, time.Since(startTime)) + klog.V(4).Infof("Finished syncing namespace %q (%v)", key, time.Since(startTime)) }() ns, err := c.nsLister.Get(key) diff --git a/pkg/controller/serviceaccount/tokens_controller.go b/pkg/controller/serviceaccount/tokens_controller.go index 7b7f80ce43d..f93cd5822f5 100644 --- a/pkg/controller/serviceaccount/tokens_controller.go +++ b/pkg/controller/serviceaccount/tokens_controller.go @@ -21,7 +21,6 @@ import ( "fmt" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,6 +35,7 @@ import ( "k8s.io/client-go/tools/cache" clientretry "k8s.io/client-go/util/retry" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/registry/core/secret" "k8s.io/kubernetes/pkg/serviceaccount" @@ -173,13 +173,13 @@ func (e *TokensController) Run(workers int, stopCh <-chan struct{}) { return } - glog.V(5).Infof("Starting workers") + klog.V(5).Infof("Starting workers") for i := 0; i < workers; i++ { go wait.Until(e.syncServiceAccount, 0, stopCh) go wait.Until(e.syncSecret, 0, stopCh) } <-stopCh - glog.V(1).Infof("Shutting down") + klog.V(1).Infof("Shutting down") } func (e *TokensController) queueServiceAccountSync(obj interface{}) { @@ -207,7 +207,7 @@ func (e *TokensController) retryOrForget(queue workqueue.RateLimitingInterface, return } - glog.V(4).Infof("retried %d times: %#v", requeueCount, key) + klog.V(4).Infof("retried %d times: %#v", requeueCount, key) queue.Forget(key) } @@ -237,28 +237,28 @@ func (e *TokensController) syncServiceAccount() { saInfo, err := parseServiceAccountKey(key) if err != nil { - glog.Error(err) + klog.Error(err) return } sa, err := e.getServiceAccount(saInfo.namespace, saInfo.name, saInfo.uid, false) switch { case err != nil: - glog.Error(err) + klog.Error(err) retry = true case sa == nil: // service account no longer exists, so delete related tokens - glog.V(4).Infof("syncServiceAccount(%s/%s), service account deleted, removing tokens", saInfo.namespace, saInfo.name) + klog.V(4).Infof("syncServiceAccount(%s/%s), service account deleted, removing tokens", saInfo.namespace, saInfo.name) sa = &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: saInfo.namespace, Name: saInfo.name, UID: saInfo.uid}} retry, err = e.deleteTokens(sa) if err != nil { - glog.Errorf("error deleting serviceaccount tokens for %s/%s: %v", saInfo.namespace, saInfo.name, err) + klog.Errorf("error deleting serviceaccount tokens for %s/%s: %v", saInfo.namespace, saInfo.name, err) } default: // ensure a token exists and is referenced by this service account retry, err = e.ensureReferencedToken(sa) if err != nil { - glog.Errorf("error synchronizing serviceaccount %s/%s: %v", saInfo.namespace, saInfo.name, err) + klog.Errorf("error synchronizing serviceaccount %s/%s: %v", saInfo.namespace, saInfo.name, err) } } } @@ -278,14 +278,14 @@ func (e *TokensController) syncSecret() { secretInfo, err := parseSecretQueueKey(key) if err != nil { - glog.Error(err) + klog.Error(err) return } secret, err := e.getSecret(secretInfo.namespace, secretInfo.name, secretInfo.uid, false) switch { case err != nil: - glog.Error(err) + klog.Error(err) retry = true case secret == nil: // If the service account exists @@ -294,7 +294,7 @@ func (e *TokensController) syncSecret() { if err := clientretry.RetryOnConflict(RemoveTokenBackoff, func() error { return e.removeSecretReference(secretInfo.namespace, secretInfo.saName, secretInfo.saUID, secretInfo.name) }); err != nil { - glog.Error(err) + klog.Error(err) } } default: @@ -302,19 +302,19 @@ func (e *TokensController) syncSecret() { sa, saErr := e.getServiceAccount(secretInfo.namespace, secretInfo.saName, secretInfo.saUID, true) switch { case saErr != nil: - glog.Error(saErr) + klog.Error(saErr) retry = true case sa == nil: // Delete token - glog.V(4).Infof("syncSecret(%s/%s), service account does not exist, deleting token", secretInfo.namespace, secretInfo.name) + klog.V(4).Infof("syncSecret(%s/%s), service account does not exist, deleting token", secretInfo.namespace, secretInfo.name) if retriable, err := e.deleteToken(secretInfo.namespace, secretInfo.name, secretInfo.uid); err != nil { - glog.Errorf("error deleting serviceaccount token %s/%s for service account %s: %v", secretInfo.namespace, secretInfo.name, secretInfo.saName, err) + klog.Errorf("error deleting serviceaccount token %s/%s for service account %s: %v", secretInfo.namespace, secretInfo.name, secretInfo.saName, err) retry = retriable } default: // Update token if needed if retriable, err := e.generateTokenIfNeeded(sa, secret); err != nil { - glog.Errorf("error populating serviceaccount token %s/%s for service account %s: %v", secretInfo.namespace, secretInfo.name, secretInfo.saName, err) + klog.Errorf("error populating serviceaccount token %s/%s for service account %s: %v", secretInfo.namespace, secretInfo.name, secretInfo.saName, err) retry = retriable } } @@ -376,7 +376,7 @@ func (e *TokensController) ensureReferencedToken(serviceAccount *v1.ServiceAccou } if liveServiceAccount.ResourceVersion != serviceAccount.ResourceVersion { // Retry if our liveServiceAccount doesn't match our cache's resourceVersion (either the live lookup or our cache are stale) - glog.V(4).Infof("liveServiceAccount.ResourceVersion (%s) does not match cache (%s), retrying", liveServiceAccount.ResourceVersion, serviceAccount.ResourceVersion) + klog.V(4).Infof("liveServiceAccount.ResourceVersion (%s) does not match cache (%s), retrying", liveServiceAccount.ResourceVersion, serviceAccount.ResourceVersion) return true, nil } @@ -455,10 +455,10 @@ func (e *TokensController) ensureReferencedToken(serviceAccount *v1.ServiceAccou if !addedReference { // we weren't able to use the token, try to clean it up. - glog.V(2).Infof("deleting secret %s/%s because reference couldn't be added (%v)", secret.Namespace, secret.Name, err) + klog.V(2).Infof("deleting secret %s/%s because reference couldn't be added (%v)", secret.Namespace, secret.Name, err) deleteOpts := &metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &createdToken.UID}} if deleteErr := e.client.CoreV1().Secrets(createdToken.Namespace).Delete(createdToken.Name, deleteOpts); deleteErr != nil { - glog.Error(deleteErr) // if we fail, just log it + klog.Error(deleteErr) // if we fail, just log it } } @@ -524,7 +524,7 @@ func (e *TokensController) generateTokenIfNeeded(serviceAccount *v1.ServiceAccou if liveSecret.ResourceVersion != cachedSecret.ResourceVersion { // our view of the secret is not up to date // we'll get notified of an update event later and get to try again - glog.V(2).Infof("secret %s/%s is not up to date, skipping token population", liveSecret.Namespace, liveSecret.Name) + klog.V(2).Infof("secret %s/%s is not up to date, skipping token population", liveSecret.Namespace, liveSecret.Name) return false, nil } diff --git a/pkg/controller/serviceaccount/tokens_controller_test.go b/pkg/controller/serviceaccount/tokens_controller_test.go index cf0c4f854fa..e7b54cc0099 100644 --- a/pkg/controller/serviceaccount/tokens_controller_test.go +++ b/pkg/controller/serviceaccount/tokens_controller_test.go @@ -23,8 +23,8 @@ import ( "time" "github.com/davecgh/go-spew/spew" - "github.com/golang/glog" "gopkg.in/square/go-jose.v2/jwt" + "k8s.io/klog" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -568,7 +568,7 @@ func TestTokenCreation(t *testing.T) { } for k, tc := range testcases { - glog.Infof(k) + klog.Infof(k) // Re-seed to reset name generation utilrand.Seed(1) diff --git a/pkg/controller/statefulset/BUILD b/pkg/controller/statefulset/BUILD index d1c19f1aaa3..40a9cf224aa 100644 --- a/pkg/controller/statefulset/BUILD +++ b/pkg/controller/statefulset/BUILD @@ -41,7 +41,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/statefulset/stateful_set.go b/pkg/controller/statefulset/stateful_set.go index e88a6dae255..43f07bf3362 100644 --- a/pkg/controller/statefulset/stateful_set.go +++ b/pkg/controller/statefulset/stateful_set.go @@ -41,7 +41,7 @@ import ( "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/history" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -86,7 +86,7 @@ func NewStatefulSetController( kubeClient clientset.Interface, ) *StatefulSetController { eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "statefulset-controller"}) @@ -128,7 +128,7 @@ func NewStatefulSetController( oldPS := old.(*apps.StatefulSet) curPS := cur.(*apps.StatefulSet) if oldPS.Status.Replicas != curPS.Status.Replicas { - glog.V(4).Infof("Observed updated replica count for StatefulSet: %v, %d->%d", curPS.Name, oldPS.Status.Replicas, curPS.Status.Replicas) + klog.V(4).Infof("Observed updated replica count for StatefulSet: %v, %d->%d", curPS.Name, oldPS.Status.Replicas, curPS.Status.Replicas) } ssc.enqueueStatefulSet(cur) }, @@ -148,8 +148,8 @@ func (ssc *StatefulSetController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer ssc.queue.ShutDown() - glog.Infof("Starting stateful set controller") - defer glog.Infof("Shutting down statefulset controller") + klog.Infof("Starting stateful set controller") + defer klog.Infof("Shutting down statefulset controller") if !controller.WaitForCacheSync("stateful set", stopCh, ssc.podListerSynced, ssc.setListerSynced, ssc.pvcListerSynced, ssc.revListerSynced) { return @@ -179,7 +179,7 @@ func (ssc *StatefulSetController) addPod(obj interface{}) { if set == nil { return } - glog.V(4).Infof("Pod %s created, labels: %+v", pod.Name, pod.Labels) + klog.V(4).Infof("Pod %s created, labels: %+v", pod.Name, pod.Labels) ssc.enqueueStatefulSet(set) return } @@ -190,7 +190,7 @@ func (ssc *StatefulSetController) addPod(obj interface{}) { if len(sets) == 0 { return } - glog.V(4).Infof("Orphan Pod %s created, labels: %+v", pod.Name, pod.Labels) + klog.V(4).Infof("Orphan Pod %s created, labels: %+v", pod.Name, pod.Labels) for _, set := range sets { ssc.enqueueStatefulSet(set) } @@ -224,7 +224,7 @@ func (ssc *StatefulSetController) updatePod(old, cur interface{}) { if set == nil { return } - glog.V(4).Infof("Pod %s updated, objectMeta %+v -> %+v.", curPod.Name, oldPod.ObjectMeta, curPod.ObjectMeta) + klog.V(4).Infof("Pod %s updated, objectMeta %+v -> %+v.", curPod.Name, oldPod.ObjectMeta, curPod.ObjectMeta) ssc.enqueueStatefulSet(set) return } @@ -236,7 +236,7 @@ func (ssc *StatefulSetController) updatePod(old, cur interface{}) { if len(sets) == 0 { return } - glog.V(4).Infof("Orphan Pod %s updated, objectMeta %+v -> %+v.", curPod.Name, oldPod.ObjectMeta, curPod.ObjectMeta) + klog.V(4).Infof("Orphan Pod %s updated, objectMeta %+v -> %+v.", curPod.Name, oldPod.ObjectMeta, curPod.ObjectMeta) for _, set := range sets { ssc.enqueueStatefulSet(set) } @@ -273,7 +273,7 @@ func (ssc *StatefulSetController) deletePod(obj interface{}) { if set == nil { return } - glog.V(4).Infof("Pod %s/%s deleted through %v.", pod.Namespace, pod.Name, utilruntime.GetCaller()) + klog.V(4).Infof("Pod %s/%s deleted through %v.", pod.Namespace, pod.Name, utilruntime.GetCaller()) ssc.enqueueStatefulSet(set) } @@ -415,7 +415,7 @@ func (ssc *StatefulSetController) worker() { func (ssc *StatefulSetController) sync(key string) error { startTime := time.Now() defer func() { - glog.V(4).Infof("Finished syncing statefulset %q (%v)", key, time.Since(startTime)) + klog.V(4).Infof("Finished syncing statefulset %q (%v)", key, time.Since(startTime)) }() namespace, name, err := cache.SplitMetaNamespaceKey(key) @@ -424,7 +424,7 @@ func (ssc *StatefulSetController) sync(key string) error { } set, err := ssc.setLister.StatefulSets(namespace).Get(name) if errors.IsNotFound(err) { - glog.Infof("StatefulSet has been deleted %v", key) + klog.Infof("StatefulSet has been deleted %v", key) return nil } if err != nil { @@ -453,11 +453,11 @@ func (ssc *StatefulSetController) sync(key string) error { // syncStatefulSet syncs a tuple of (statefulset, []*v1.Pod). func (ssc *StatefulSetController) syncStatefulSet(set *apps.StatefulSet, pods []*v1.Pod) error { - glog.V(4).Infof("Syncing StatefulSet %v/%v with %d pods", set.Namespace, set.Name, len(pods)) + klog.V(4).Infof("Syncing StatefulSet %v/%v with %d pods", set.Namespace, set.Name, len(pods)) // TODO: investigate where we mutate the set during the update as it is not obvious. if err := ssc.control.UpdateStatefulSet(set.DeepCopy(), pods); err != nil { return err } - glog.V(4).Infof("Successfully synced StatefulSet %s/%s successful", set.Namespace, set.Name) + klog.V(4).Infof("Successfully synced StatefulSet %s/%s successful", set.Namespace, set.Name) return nil } diff --git a/pkg/controller/statefulset/stateful_set_control.go b/pkg/controller/statefulset/stateful_set_control.go index eb8664e5e11..de193a137e1 100644 --- a/pkg/controller/statefulset/stateful_set_control.go +++ b/pkg/controller/statefulset/stateful_set_control.go @@ -20,7 +20,7 @@ import ( "math" "sort" - "github.com/golang/glog" + "k8s.io/klog" apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" @@ -99,7 +99,7 @@ func (ssc *defaultStatefulSetControl) UpdateStatefulSet(set *apps.StatefulSet, p return err } - glog.V(4).Infof("StatefulSet %s/%s pod status replicas=%d ready=%d current=%d updated=%d", + klog.V(4).Infof("StatefulSet %s/%s pod status replicas=%d ready=%d current=%d updated=%d", set.Namespace, set.Name, status.Replicas, @@ -107,7 +107,7 @@ func (ssc *defaultStatefulSetControl) UpdateStatefulSet(set *apps.StatefulSet, p status.CurrentReplicas, status.UpdatedReplicas) - glog.V(4).Infof("StatefulSet %s/%s revisions current=%s update=%s", + klog.V(4).Infof("StatefulSet %s/%s revisions current=%s update=%s", set.Namespace, set.Name, status.CurrentRevision, @@ -351,7 +351,7 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( } if unhealthy > 0 { - glog.V(4).Infof("StatefulSet %s/%s has %d unhealthy Pods starting with %s", + klog.V(4).Infof("StatefulSet %s/%s has %d unhealthy Pods starting with %s", set.Namespace, set.Name, unhealthy, @@ -415,7 +415,7 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( // If we find a Pod that is currently terminating, we must wait until graceful deletion // completes before we continue to make progress. if isTerminating(replicas[i]) && monotonic { - glog.V(4).Infof( + klog.V(4).Infof( "StatefulSet %s/%s is waiting for Pod %s to Terminate", set.Namespace, set.Name, @@ -426,7 +426,7 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( // We must ensure that all for each Pod, when we create it, all of its predecessors, with respect to its // ordinal, are Running and Ready. if !isRunningAndReady(replicas[i]) && monotonic { - glog.V(4).Infof( + klog.V(4).Infof( "StatefulSet %s/%s is waiting for Pod %s to be Running and Ready", set.Namespace, set.Name, @@ -452,7 +452,7 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( for target := len(condemned) - 1; target >= 0; target-- { // wait for terminating pods to expire if isTerminating(condemned[target]) { - glog.V(4).Infof( + klog.V(4).Infof( "StatefulSet %s/%s is waiting for Pod %s to Terminate prior to scale down", set.Namespace, set.Name, @@ -465,14 +465,14 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( } // if we are in monotonic mode and the condemned target is not the first unhealthy Pod block if !isRunningAndReady(condemned[target]) && monotonic && condemned[target] != firstUnhealthyPod { - glog.V(4).Infof( + klog.V(4).Infof( "StatefulSet %s/%s is waiting for Pod %s to be Running and Ready prior to scale down", set.Namespace, set.Name, firstUnhealthyPod.Name) return &status, nil } - glog.V(2).Infof("StatefulSet %s/%s terminating Pod %s for scale down", + klog.V(2).Infof("StatefulSet %s/%s terminating Pod %s for scale down", set.Namespace, set.Name, condemned[target].Name) @@ -506,7 +506,7 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( // delete the Pod if it is not already terminating and does not match the update revision. if getPodRevision(replicas[target]) != updateRevision.Name && !isTerminating(replicas[target]) { - glog.V(2).Infof("StatefulSet %s/%s terminating Pod %s for update", + klog.V(2).Infof("StatefulSet %s/%s terminating Pod %s for update", set.Namespace, set.Name, replicas[target].Name) @@ -517,7 +517,7 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( // wait for unhealthy Pods on update if !isHealthy(replicas[target]) { - glog.V(4).Infof( + klog.V(4).Infof( "StatefulSet %s/%s is waiting for Pod %s to update", set.Namespace, set.Name, diff --git a/pkg/controller/testutil/BUILD b/pkg/controller/testutil/BUILD index ac20e1bbd71..b0b348c13a3 100644 --- a/pkg/controller/testutil/BUILD +++ b/pkg/controller/testutil/BUILD @@ -28,7 +28,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/reference:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/testutil/test_utils.go b/pkg/controller/testutil/test_utils.go index 7ee5e6be658..069c2d61a86 100644 --- a/pkg/controller/testutil/test_utils.go +++ b/pkg/controller/testutil/test_utils.go @@ -46,7 +46,7 @@ import ( utilnode "k8s.io/kubernetes/pkg/util/node" jsonpatch "github.com/evanphx/json-patch" - "github.com/golang/glog" + "k8s.io/klog" ) var ( @@ -299,12 +299,12 @@ func (m *FakeNodeHandler) Patch(name string, pt types.PatchType, data []byte, su originalObjJS, err := json.Marshal(nodeCopy) if err != nil { - glog.Errorf("Failed to marshal %v", nodeCopy) + klog.Errorf("Failed to marshal %v", nodeCopy) return nil, nil } var originalNode v1.Node if err = json.Unmarshal(originalObjJS, &originalNode); err != nil { - glog.Errorf("Failed to unmarshal original object: %v", err) + klog.Errorf("Failed to unmarshal original object: %v", err) return nil, nil } @@ -313,31 +313,31 @@ func (m *FakeNodeHandler) Patch(name string, pt types.PatchType, data []byte, su case types.JSONPatchType: patchObj, err := jsonpatch.DecodePatch(data) if err != nil { - glog.Error(err.Error()) + klog.Error(err.Error()) return nil, nil } if patchedObjJS, err = patchObj.Apply(originalObjJS); err != nil { - glog.Error(err.Error()) + klog.Error(err.Error()) return nil, nil } case types.MergePatchType: if patchedObjJS, err = jsonpatch.MergePatch(originalObjJS, data); err != nil { - glog.Error(err.Error()) + klog.Error(err.Error()) return nil, nil } case types.StrategicMergePatchType: if patchedObjJS, err = strategicpatch.StrategicMergePatch(originalObjJS, data, originalNode); err != nil { - glog.Error(err.Error()) + klog.Error(err.Error()) return nil, nil } default: - glog.Errorf("unknown Content-Type header for patch: %v", pt) + klog.Errorf("unknown Content-Type header for patch: %v", pt) return nil, nil } var updatedNode v1.Node if err = json.Unmarshal(patchedObjJS, &updatedNode); err != nil { - glog.Errorf("Failed to unmarshal patched object: %v", err) + klog.Errorf("Failed to unmarshal patched object: %v", err) return nil, nil } @@ -382,7 +382,7 @@ func (f *FakeRecorder) generateEvent(obj runtime.Object, timestamp metav1.Time, defer f.Unlock() ref, err := ref.GetReference(legacyscheme.Scheme, obj) if err != nil { - glog.Errorf("Encountered error while getting reference: %v", err) + klog.Errorf("Encountered error while getting reference: %v", err) return } event := f.makeEvent(ref, eventtype, reason, message) diff --git a/pkg/controller/ttl/BUILD b/pkg/controller/ttl/BUILD index 900302a8a07..3eff1a2ab62 100644 --- a/pkg/controller/ttl/BUILD +++ b/pkg/controller/ttl/BUILD @@ -24,7 +24,7 @@ go_library( "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/ttl/ttl_controller.go b/pkg/controller/ttl/ttl_controller.go index 2938ca8270b..e9b6728e67e 100644 --- a/pkg/controller/ttl/ttl_controller.go +++ b/pkg/controller/ttl/ttl_controller.go @@ -47,7 +47,7 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/kubernetes/pkg/controller" - "github.com/golang/glog" + "k8s.io/klog" ) type TTLController struct { @@ -113,8 +113,8 @@ func (ttlc *TTLController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer ttlc.queue.ShutDown() - glog.Infof("Starting TTL controller") - defer glog.Infof("Shutting down TTL controller") + klog.Infof("Starting TTL controller") + defer klog.Infof("Shutting down TTL controller") if !controller.WaitForCacheSync("TTL", stopCh, ttlc.hasSynced) { return @@ -190,7 +190,7 @@ func (ttlc *TTLController) deleteNode(obj interface{}) { func (ttlc *TTLController) enqueueNode(node *v1.Node) { key, err := controller.KeyFunc(node) if err != nil { - glog.Errorf("Couldn't get key for object %+v", node) + klog.Errorf("Couldn't get key for object %+v", node) return } ttlc.queue.Add(key) @@ -235,7 +235,7 @@ func getIntFromAnnotation(node *v1.Node, annotationKey string) (int, bool) { } intValue, err := strconv.Atoi(annotationValue) if err != nil { - glog.Warningf("Cannot convert the value %q with annotation key %q for the node %q", + klog.Warningf("Cannot convert the value %q with annotation key %q for the node %q", annotationValue, annotationKey, node.Name) return 0, false } @@ -265,10 +265,10 @@ func (ttlc *TTLController) patchNodeWithAnnotation(node *v1.Node, annotationKey } _, err = ttlc.kubeClient.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes) if err != nil { - glog.V(2).Infof("Failed to change ttl annotation for node %s: %v", node.Name, err) + klog.V(2).Infof("Failed to change ttl annotation for node %s: %v", node.Name, err) return err } - glog.V(2).Infof("Changed ttl annotation for node %s to %d seconds", node.Name, value) + klog.V(2).Infof("Changed ttl annotation for node %s to %d seconds", node.Name, value) return nil } diff --git a/pkg/controller/ttlafterfinished/BUILD b/pkg/controller/ttlafterfinished/BUILD index 3eda34912f6..21c5ec3294b 100644 --- a/pkg/controller/ttlafterfinished/BUILD +++ b/pkg/controller/ttlafterfinished/BUILD @@ -24,7 +24,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/ttlafterfinished/ttlafterfinished_controller.go b/pkg/controller/ttlafterfinished/ttlafterfinished_controller.go index 99c5e625fdc..930d2df7d1f 100644 --- a/pkg/controller/ttlafterfinished/ttlafterfinished_controller.go +++ b/pkg/controller/ttlafterfinished/ttlafterfinished_controller.go @@ -20,7 +20,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" batch "k8s.io/api/batch/v1" "k8s.io/api/core/v1" @@ -71,7 +71,7 @@ type Controller struct { // New creates an instance of Controller func New(jobInformer batchinformers.JobInformer, client clientset.Interface) *Controller { eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")}) if client != nil && client.CoreV1().RESTClient().GetRateLimiter() != nil { @@ -102,8 +102,8 @@ func (tc *Controller) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer tc.queue.ShutDown() - glog.Infof("Starting TTL after finished controller") - defer glog.Infof("Shutting down TTL after finished controller") + klog.Infof("Starting TTL after finished controller") + defer klog.Infof("Shutting down TTL after finished controller") if !controller.WaitForCacheSync("TTL after finished", stopCh, tc.jListerSynced) { return @@ -118,7 +118,7 @@ func (tc *Controller) Run(workers int, stopCh <-chan struct{}) { func (tc *Controller) addJob(obj interface{}) { job := obj.(*batch.Job) - glog.V(4).Infof("Adding job %s/%s", job.Namespace, job.Name) + klog.V(4).Infof("Adding job %s/%s", job.Namespace, job.Name) if job.DeletionTimestamp == nil && needsCleanup(job) { tc.enqueue(job) @@ -127,7 +127,7 @@ func (tc *Controller) addJob(obj interface{}) { func (tc *Controller) updateJob(old, cur interface{}) { job := cur.(*batch.Job) - glog.V(4).Infof("Updating job %s/%s", job.Namespace, job.Name) + klog.V(4).Infof("Updating job %s/%s", job.Namespace, job.Name) if job.DeletionTimestamp == nil && needsCleanup(job) { tc.enqueue(job) @@ -135,7 +135,7 @@ func (tc *Controller) updateJob(old, cur interface{}) { } func (tc *Controller) enqueue(job *batch.Job) { - glog.V(4).Infof("Add job %s/%s to cleanup", job.Namespace, job.Name) + klog.V(4).Infof("Add job %s/%s to cleanup", job.Namespace, job.Name) key, err := controller.KeyFunc(job) if err != nil { utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %v", job, err)) @@ -194,7 +194,7 @@ func (tc *Controller) processJob(key string) error { return err } - glog.V(4).Infof("Checking if Job %s/%s is ready for cleanup", namespace, name) + klog.V(4).Infof("Checking if Job %s/%s is ready for cleanup", namespace, name) // Ignore the Jobs that are already deleted or being deleted, or the ones that don't need clean up. job, err := tc.jLister.Jobs(namespace).Get(name) if errors.IsNotFound(err) { @@ -233,7 +233,7 @@ func (tc *Controller) processJob(key string) error { PropagationPolicy: &policy, Preconditions: &metav1.Preconditions{UID: &fresh.UID}, } - glog.V(4).Infof("Cleaning up Job %s/%s", namespace, name) + klog.V(4).Infof("Cleaning up Job %s/%s", namespace, name) return tc.client.BatchV1().Jobs(fresh.Namespace).Delete(fresh.Name, options) } @@ -284,10 +284,10 @@ func timeLeft(j *batch.Job, since *time.Time) (*time.Duration, error) { return nil, err } if finishAt.UTC().After(since.UTC()) { - glog.Warningf("Warning: Found Job %s/%s finished in the future. This is likely due to time skew in the cluster. Job cleanup will be deferred.", j.Namespace, j.Name) + klog.Warningf("Warning: Found Job %s/%s finished in the future. This is likely due to time skew in the cluster. Job cleanup will be deferred.", j.Namespace, j.Name) } remaining := expireAt.UTC().Sub(since.UTC()) - glog.V(4).Infof("Found Job %s/%s finished at %v, remaining TTL %v since %v, TTL will expire at %v", j.Namespace, j.Name, finishAt.UTC(), remaining, since.UTC(), expireAt.UTC()) + klog.V(4).Infof("Found Job %s/%s finished at %v, remaining TTL %v since %v, TTL will expire at %v", j.Namespace, j.Name, finishAt.UTC(), remaining, since.UTC(), expireAt.UTC()) return &remaining, nil } diff --git a/pkg/controller/util/node/BUILD b/pkg/controller/util/node/BUILD index fa48450d235..2dc7c49166d 100644 --- a/pkg/controller/util/node/BUILD +++ b/pkg/controller/util/node/BUILD @@ -22,7 +22,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/util/node/controller_utils.go b/pkg/controller/util/node/controller_utils.go index ff7ffe582bc..9be6845d0b8 100644 --- a/pkg/controller/util/node/controller_utils.go +++ b/pkg/controller/util/node/controller_utils.go @@ -40,7 +40,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/util/format" nodepkg "k8s.io/kubernetes/pkg/util/node" - "github.com/golang/glog" + "k8s.io/klog" ) var ( @@ -92,7 +92,7 @@ func DeletePods(kubeClient clientset.Interface, recorder record.EventRecorder, n continue } - glog.V(2).Infof("Starting deletion of pod %v/%v", pod.Namespace, pod.Name) + klog.V(2).Infof("Starting deletion of pod %v/%v", pod.Namespace, pod.Name) recorder.Eventf(&pod, v1.EventTypeNormal, "NodeControllerEviction", "Marking for deletion Pod %s from Node %s", pod.Name, nodeName) if err := kubeClient.CoreV1().Pods(pod.Namespace).Delete(pod.Name, nil); err != nil { return false, err @@ -138,7 +138,7 @@ func ForcefullyDeleteNode(kubeClient clientset.Interface, nodeName string) error // given node from master return true if success func MarkAllPodsNotReady(kubeClient clientset.Interface, node *v1.Node) error { nodeName := node.Name - glog.V(2).Infof("Update ready status of pods on node [%v]", nodeName) + klog.V(2).Infof("Update ready status of pods on node [%v]", nodeName) opts := metav1.ListOptions{FieldSelector: fields.OneTermEqualSelector(api.PodHostField, nodeName).String()} pods, err := kubeClient.CoreV1().Pods(metav1.NamespaceAll).List(opts) if err != nil { @@ -155,10 +155,10 @@ func MarkAllPodsNotReady(kubeClient clientset.Interface, node *v1.Node) error { for i, cond := range pod.Status.Conditions { if cond.Type == v1.PodReady { pod.Status.Conditions[i].Status = v1.ConditionFalse - glog.V(2).Infof("Updating ready status of pod %v to false", pod.Name) + klog.V(2).Infof("Updating ready status of pod %v to false", pod.Name) _, err := kubeClient.CoreV1().Pods(pod.Namespace).UpdateStatus(&pod) if err != nil { - glog.Warningf("Failed to update status for pod %q: %v", format.Pod(&pod), err) + klog.Warningf("Failed to update status for pod %q: %v", format.Pod(&pod), err) errMsg = append(errMsg, fmt.Sprintf("%v", err)) } break @@ -209,7 +209,7 @@ func RecordNodeEvent(recorder record.EventRecorder, nodeName, nodeUID, eventtype UID: types.UID(nodeUID), Namespace: "", } - glog.V(2).Infof("Recording %s event message for node %s", event, nodeName) + klog.V(2).Infof("Recording %s event message for node %s", event, nodeName) recorder.Eventf(ref, eventtype, reason, "Node %s event: %s", nodeName, event) } @@ -221,7 +221,7 @@ func RecordNodeStatusChange(recorder record.EventRecorder, node *v1.Node, newSta UID: node.UID, Namespace: "", } - glog.V(2).Infof("Recording status change %s event message for node %s", newStatus, node.Name) + klog.V(2).Infof("Recording status change %s event message for node %s", newStatus, node.Name) // TODO: This requires a transaction, either both node status is updated // and event is recorded or neither should happen, see issue #6055. recorder.Eventf(ref, v1.EventTypeNormal, newStatus, "Node %s status is now: %s", node.Name, newStatus) @@ -245,7 +245,7 @@ func SwapNodeControllerTaint(kubeClient clientset.Interface, taintsToAdd, taints err)) return false } - glog.V(4).Infof("Added %+v Taint to Node %v", taintsToAdd, node.Name) + klog.V(4).Infof("Added %+v Taint to Node %v", taintsToAdd, node.Name) err = controller.RemoveTaintOffNode(kubeClient, node.Name, node, taintsToRemove...) if err != nil { @@ -257,7 +257,7 @@ func SwapNodeControllerTaint(kubeClient clientset.Interface, taintsToAdd, taints err)) return false } - glog.V(4).Infof("Made sure that Node %+v has no %v Taint", node.Name, taintsToRemove) + klog.V(4).Infof("Made sure that Node %+v has no %v Taint", node.Name, taintsToRemove) return true } @@ -293,12 +293,12 @@ func CreateDeleteNodeHandler(f func(node *v1.Node) error) func(obj interface{}) if !isNode { deletedState, ok := originalObj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Received unexpected object: %v", originalObj) + klog.Errorf("Received unexpected object: %v", originalObj) return } originalNode, ok = deletedState.Obj.(*v1.Node) if !ok { - glog.Errorf("DeletedFinalStateUnknown contained non-Node object: %v", deletedState.Obj) + klog.Errorf("DeletedFinalStateUnknown contained non-Node object: %v", deletedState.Obj) return } } diff --git a/pkg/controller/volume/attachdetach/BUILD b/pkg/controller/volume/attachdetach/BUILD index ed2db539700..f68704cfff1 100644 --- a/pkg/controller/volume/attachdetach/BUILD +++ b/pkg/controller/volume/attachdetach/BUILD @@ -40,7 +40,7 @@ go_library( "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index e2b1b6632f9..f76e3224a6e 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -23,7 +23,6 @@ import ( "net" "time" - "github.com/golang/glog" authenticationv1 "k8s.io/api/authentication/v1" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -41,6 +40,7 @@ import ( "k8s.io/client-go/util/workqueue" cloudprovider "k8s.io/cloud-provider" csiclient "k8s.io/csi-api/pkg/client/clientset/versioned" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/metrics" @@ -143,7 +143,7 @@ func NewAttachDetachController( } eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "attachdetach-controller"}) blkutil := volumepathhandler.NewBlockVolumePathHandler() @@ -312,8 +312,8 @@ func (adc *attachDetachController) Run(stopCh <-chan struct{}) { defer runtime.HandleCrash() defer adc.pvcQueue.ShutDown() - glog.Infof("Starting attach detach controller") - defer glog.Infof("Shutting down attach detach controller") + klog.Infof("Starting attach detach controller") + defer klog.Infof("Shutting down attach detach controller") if !controller.WaitForCacheSync("attach detach", stopCh, adc.podsSynced, adc.nodesSynced, adc.pvcsSynced, adc.pvsSynced) { return @@ -321,11 +321,11 @@ func (adc *attachDetachController) Run(stopCh <-chan struct{}) { err := adc.populateActualStateOfWorld() if err != nil { - glog.Errorf("Error populating the actual state of world: %v", err) + klog.Errorf("Error populating the actual state of world: %v", err) } err = adc.populateDesiredStateOfWorld() if err != nil { - glog.Errorf("Error populating the desired state of world: %v", err) + klog.Errorf("Error populating the desired state of world: %v", err) } go adc.reconciler.Run(stopCh) go adc.desiredStateOfWorldPopulator.Run(stopCh) @@ -341,7 +341,7 @@ func (adc *attachDetachController) Run(stopCh <-chan struct{}) { } func (adc *attachDetachController) populateActualStateOfWorld() error { - glog.V(5).Infof("Populating ActualStateOfworld") + klog.V(5).Infof("Populating ActualStateOfworld") nodes, err := adc.nodeLister.List(labels.Everything()) if err != nil { return err @@ -358,7 +358,7 @@ func (adc *attachDetachController) populateActualStateOfWorld() error { // scans the pods and updates their volumes in the ActualStateOfWorld too. err = adc.actualStateOfWorld.MarkVolumeAsAttached(uniqueName, nil /* VolumeSpec */, nodeName, attachedVolume.DevicePath) if err != nil { - glog.Errorf("Failed to mark the volume as attached: %v", err) + klog.Errorf("Failed to mark the volume as attached: %v", err) continue } adc.processVolumesInUse(nodeName, node.Status.VolumesInUse) @@ -391,7 +391,7 @@ func (adc *attachDetachController) getNodeVolumeDevicePath( } func (adc *attachDetachController) populateDesiredStateOfWorld() error { - glog.V(5).Infof("Populating DesiredStateOfworld") + klog.V(5).Infof("Populating DesiredStateOfworld") pods, err := adc.podLister.List(labels.Everything()) if err != nil { @@ -406,7 +406,7 @@ func (adc *attachDetachController) populateDesiredStateOfWorld() error { // pod will be detached and the spec is irrelevant. volumeSpec, err := util.CreateVolumeSpec(podVolume, podToAdd.Namespace, adc.pvcLister, adc.pvLister) if err != nil { - glog.Errorf( + klog.Errorf( "Error creating spec for volume %q, pod %q/%q: %v", podVolume.Name, podToAdd.Namespace, @@ -417,7 +417,7 @@ func (adc *attachDetachController) populateDesiredStateOfWorld() error { nodeName := types.NodeName(podToAdd.Spec.NodeName) plugin, err := adc.volumePluginMgr.FindAttachablePluginBySpec(volumeSpec) if err != nil || plugin == nil { - glog.V(10).Infof( + klog.V(10).Infof( "Skipping volume %q for pod %q/%q: it does not implement attacher interface. err=%v", podVolume.Name, podToAdd.Namespace, @@ -427,7 +427,7 @@ func (adc *attachDetachController) populateDesiredStateOfWorld() error { } volumeName, err := volumeutil.GetUniqueVolumeNameFromSpec(plugin, volumeSpec) if err != nil { - glog.Errorf( + klog.Errorf( "Failed to find unique name for volume %q, pod %q/%q: %v", podVolume.Name, podToAdd.Namespace, @@ -438,12 +438,12 @@ func (adc *attachDetachController) populateDesiredStateOfWorld() error { if adc.actualStateOfWorld.VolumeNodeExists(volumeName, nodeName) { devicePath, err := adc.getNodeVolumeDevicePath(volumeName, nodeName) if err != nil { - glog.Errorf("Failed to find device path: %v", err) + klog.Errorf("Failed to find device path: %v", err) continue } err = adc.actualStateOfWorld.MarkVolumeAsAttached(volumeName, volumeSpec, nodeName, devicePath) if err != nil { - glog.Errorf("Failed to update volume spec for node %s: %v", nodeName, err) + klog.Errorf("Failed to update volume spec for node %s: %v", nodeName, err) } } } @@ -542,7 +542,7 @@ func (adc *attachDetachController) nodeDelete(obj interface{}) { nodeName := types.NodeName(node.Name) if err := adc.desiredStateOfWorld.DeleteNode(nodeName); err != nil { // This might happen during drain, but we still want it to appear in our logs - glog.Infof("error removing node %q from desired-state-of-world: %v", nodeName, err) + klog.Infof("error removing node %q from desired-state-of-world: %v", nodeName, err) } adc.processVolumesInUse(nodeName, node.Status.VolumesInUse) @@ -585,15 +585,15 @@ func (adc *attachDetachController) processNextItem() bool { } func (adc *attachDetachController) syncPVCByKey(key string) error { - glog.V(5).Infof("syncPVCByKey[%s]", key) + klog.V(5).Infof("syncPVCByKey[%s]", key) namespace, name, err := kcache.SplitMetaNamespaceKey(key) if err != nil { - glog.V(4).Infof("error getting namespace & name of pvc %q to get pvc from informer: %v", key, err) + klog.V(4).Infof("error getting namespace & name of pvc %q to get pvc from informer: %v", key, err) return nil } pvc, err := adc.pvcLister.PersistentVolumeClaims(namespace).Get(name) if apierrors.IsNotFound(err) { - glog.V(4).Infof("error getting pvc %q from informer: %v", key, err) + klog.V(4).Infof("error getting pvc %q from informer: %v", key, err) return nil } if err != nil { @@ -631,7 +631,7 @@ func (adc *attachDetachController) syncPVCByKey(key string) error { // mounted. func (adc *attachDetachController) processVolumesInUse( nodeName types.NodeName, volumesInUse []v1.UniqueVolumeName) { - glog.V(4).Infof("processVolumesInUse for node %q", nodeName) + klog.V(4).Infof("processVolumesInUse for node %q", nodeName) for _, attachedVolume := range adc.actualStateOfWorld.GetAttachedVolumesForNode(nodeName) { mounted := false for _, volumeInUse := range volumesInUse { @@ -642,7 +642,7 @@ func (adc *attachDetachController) processVolumesInUse( } err := adc.actualStateOfWorld.SetVolumeMountedByNode(attachedVolume.VolumeName, nodeName, mounted) if err != nil { - glog.Warningf( + klog.Warningf( "SetVolumeMountedByNode(%q, %q, %v) returned an error: %v", attachedVolume.VolumeName, nodeName, mounted, err) } @@ -731,7 +731,7 @@ func (adc *attachDetachController) GetServiceAccountTokenFunc() func(_, _ string func (adc *attachDetachController) DeleteServiceAccountTokenFunc() func(types.UID) { return func(types.UID) { - glog.Errorf("DeleteServiceAccountToken unsupported in attachDetachController") + klog.Errorf("DeleteServiceAccountToken unsupported in attachDetachController") } } diff --git a/pkg/controller/volume/attachdetach/cache/BUILD b/pkg/controller/volume/attachdetach/cache/BUILD index eadd50debaa..a2890223e0a 100644 --- a/pkg/controller/volume/attachdetach/cache/BUILD +++ b/pkg/controller/volume/attachdetach/cache/BUILD @@ -20,7 +20,7 @@ go_library( "//pkg/volume/util/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/attachdetach/cache/actual_state_of_world.go b/pkg/controller/volume/attachdetach/cache/actual_state_of_world.go index 7dfe7396274..f25fb008816 100644 --- a/pkg/controller/volume/attachdetach/cache/actual_state_of_world.go +++ b/pkg/controller/volume/attachdetach/cache/actual_state_of_world.go @@ -26,7 +26,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -304,7 +304,7 @@ func (asw *actualStateOfWorld) AddVolumeNode( // Update the fields for volume object except the nodes attached to the volumes. volumeObj.devicePath = devicePath volumeObj.spec = volumeSpec - glog.V(2).Infof("Volume %q is already added to attachedVolume list to node %q, update device path %q", + klog.V(2).Infof("Volume %q is already added to attachedVolume list to node %q, update device path %q", volumeName, nodeName, devicePath) @@ -321,7 +321,7 @@ func (asw *actualStateOfWorld) AddVolumeNode( detachRequestedTime: time.Time{}, } } else { - glog.V(5).Infof("Volume %q is already added to attachedVolume list to the node %q", + klog.V(5).Infof("Volume %q is already added to attachedVolume list to the node %q", volumeName, nodeName) } @@ -347,7 +347,7 @@ func (asw *actualStateOfWorld) SetVolumeMountedByNode( nodeObj.mountedByNode = mounted volumeObj.nodesAttachedTo[nodeName] = nodeObj - glog.V(4).Infof("SetVolumeMountedByNode volume %v to the node %q mounted %t", + klog.V(4).Infof("SetVolumeMountedByNode volume %v to the node %q mounted %t", volumeName, nodeName, mounted) @@ -361,7 +361,7 @@ func (asw *actualStateOfWorld) ResetDetachRequestTime( volumeObj, nodeObj, err := asw.getNodeAndVolume(volumeName, nodeName) if err != nil { - glog.Errorf("Failed to ResetDetachRequestTime with error: %v", err) + klog.Errorf("Failed to ResetDetachRequestTime with error: %v", err) return } nodeObj.detachRequestedTime = time.Time{} @@ -381,7 +381,7 @@ func (asw *actualStateOfWorld) SetDetachRequestTime( if nodeObj.detachRequestedTime.IsZero() { nodeObj.detachRequestedTime = time.Now() volumeObj.nodesAttachedTo[nodeName] = nodeObj - glog.V(4).Infof("Set detach request time to current time for volume %v on node %q", + klog.V(4).Infof("Set detach request time to current time for volume %v on node %q", volumeName, nodeName) } @@ -441,7 +441,7 @@ func (asw *actualStateOfWorld) addVolumeToReportAsAttached( volumeName v1.UniqueVolumeName, nodeName types.NodeName) { // In case the volume/node entry is no longer in attachedVolume list, skip the rest if _, _, err := asw.getNodeAndVolume(volumeName, nodeName); err != nil { - glog.V(4).Infof("Volume %q is no longer attached to node %q", volumeName, nodeName) + klog.V(4).Infof("Volume %q is no longer attached to node %q", volumeName, nodeName) return } nodeToUpdate, nodeToUpdateExists := asw.nodesToUpdateStatusFor[nodeName] @@ -453,7 +453,7 @@ func (asw *actualStateOfWorld) addVolumeToReportAsAttached( volumesToReportAsAttached: make(map[v1.UniqueVolumeName]v1.UniqueVolumeName), } asw.nodesToUpdateStatusFor[nodeName] = nodeToUpdate - glog.V(4).Infof("Add new node %q to nodesToUpdateStatusFor", nodeName) + klog.V(4).Infof("Add new node %q to nodesToUpdateStatusFor", nodeName) } _, nodeToUpdateVolumeExists := nodeToUpdate.volumesToReportAsAttached[volumeName] @@ -461,7 +461,7 @@ func (asw *actualStateOfWorld) addVolumeToReportAsAttached( nodeToUpdate.statusUpdateNeeded = true nodeToUpdate.volumesToReportAsAttached[volumeName] = volumeName asw.nodesToUpdateStatusFor[nodeName] = nodeToUpdate - glog.V(4).Infof("Report volume %q as attached to node %q", volumeName, nodeName) + klog.V(4).Infof("Report volume %q as attached to node %q", volumeName, nodeName) } } @@ -488,7 +488,7 @@ func (asw *actualStateOfWorld) SetNodeStatusUpdateNeeded(nodeName types.NodeName asw.Lock() defer asw.Unlock() if err := asw.updateNodeStatusUpdateNeeded(nodeName, true); err != nil { - glog.Warningf("Failed to update statusUpdateNeeded field in actual state of world: %v", err) + klog.Warningf("Failed to update statusUpdateNeeded field in actual state of world: %v", err) } } @@ -623,7 +623,7 @@ func (asw *actualStateOfWorld) GetVolumesToReportAttached() map[types.NodeName][ // of this node will be updated, so set the flag statusUpdateNeeded to false indicating // the current status is already updated. if err := asw.updateNodeStatusUpdateNeeded(nodeName, false); err != nil { - glog.Errorf("Failed to update statusUpdateNeeded field when getting volumes: %v", err) + klog.Errorf("Failed to update statusUpdateNeeded field when getting volumes: %v", err) } } diff --git a/pkg/controller/volume/attachdetach/metrics/BUILD b/pkg/controller/volume/attachdetach/metrics/BUILD index d07ac9b9576..c1efd8ca811 100644 --- a/pkg/controller/volume/attachdetach/metrics/BUILD +++ b/pkg/controller/volume/attachdetach/metrics/BUILD @@ -9,10 +9,11 @@ go_library( "//pkg/controller/volume/attachdetach/cache:go_default_library", "//pkg/controller/volume/attachdetach/util:go_default_library", "//pkg/volume:go_default_library", + "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/attachdetach/metrics/metrics.go b/pkg/controller/volume/attachdetach/metrics/metrics.go index 8167dba5f10..86f99c5e8b6 100644 --- a/pkg/controller/volume/attachdetach/metrics/metrics.go +++ b/pkg/controller/volume/attachdetach/metrics/metrics.go @@ -19,13 +19,14 @@ package metrics import ( "sync" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/labels" corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/util" "k8s.io/kubernetes/pkg/volume" + volumeutil "k8s.io/kubernetes/pkg/volume/util" ) const pluginNameNotAvailable = "N/A" @@ -119,7 +120,7 @@ func (collector *attachDetachStateCollector) Collect(ch chan<- prometheus.Metric string(nodeName), pluginName) if err != nil { - glog.Warningf("Failed to create metric : %v", err) + klog.Warningf("Failed to create metric : %v", err) } ch <- metric } @@ -134,7 +135,7 @@ func (collector *attachDetachStateCollector) Collect(ch chan<- prometheus.Metric pluginName, string(stateName)) if err != nil { - glog.Warningf("Failed to create metric : %v", err) + klog.Warningf("Failed to create metric : %v", err) } ch <- metric } @@ -144,7 +145,7 @@ func (collector *attachDetachStateCollector) Collect(ch chan<- prometheus.Metric func (collector *attachDetachStateCollector) getVolumeInUseCount() volumeCount { pods, err := collector.podLister.List(labels.Everything()) if err != nil { - glog.Errorf("Error getting pod list") + klog.Errorf("Error getting pod list") return nil } @@ -166,7 +167,8 @@ func (collector *attachDetachStateCollector) getVolumeInUseCount() volumeCount { if err != nil { continue } - nodeVolumeMap.add(pod.Spec.NodeName, volumePlugin.GetPluginName()) + pluginName := volumeutil.GetFullQualifiedPluginNameForVolume(volumePlugin.GetPluginName(), volumeSpec) + nodeVolumeMap.add(pod.Spec.NodeName, pluginName) } } return nodeVolumeMap @@ -178,7 +180,7 @@ func (collector *attachDetachStateCollector) getTotalVolumesCount() volumeCount if plugin, err := collector.volumePluginMgr.FindPluginBySpec(v.VolumeSpec); err == nil { pluginName := pluginNameNotAvailable if plugin != nil { - pluginName = plugin.GetPluginName() + pluginName = volumeutil.GetFullQualifiedPluginNameForVolume(plugin.GetPluginName(), v.VolumeSpec) } stateVolumeMap.add("desired_state_of_world", pluginName) } @@ -187,7 +189,7 @@ func (collector *attachDetachStateCollector) getTotalVolumesCount() volumeCount if plugin, err := collector.volumePluginMgr.FindPluginBySpec(v.VolumeSpec); err == nil { pluginName := pluginNameNotAvailable if plugin != nil { - pluginName = plugin.GetPluginName() + pluginName = volumeutil.GetFullQualifiedPluginNameForVolume(plugin.GetPluginName(), v.VolumeSpec) } stateVolumeMap.add("actual_state_of_world", pluginName) } diff --git a/pkg/controller/volume/attachdetach/populator/BUILD b/pkg/controller/volume/attachdetach/populator/BUILD index dffe2bbe487..b216acf90f7 100644 --- a/pkg/controller/volume/attachdetach/populator/BUILD +++ b/pkg/controller/volume/attachdetach/populator/BUILD @@ -21,7 +21,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/attachdetach/populator/desired_state_of_world_populator.go b/pkg/controller/volume/attachdetach/populator/desired_state_of_world_populator.go index 4065e25a804..2d9005b97a1 100644 --- a/pkg/controller/volume/attachdetach/populator/desired_state_of_world_populator.go +++ b/pkg/controller/volume/attachdetach/populator/desired_state_of_world_populator.go @@ -22,7 +22,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" @@ -92,7 +92,7 @@ func (dswp *desiredStateOfWorldPopulator) populatorLoopFunc() func() { // findAndAddActivePods is called periodically, independently of the main // populator loop. if time.Since(dswp.timeOfLastListPods) < dswp.listPodsRetryDuration { - glog.V(5).Infof( + klog.V(5).Infof( "Skipping findAndAddActivePods(). Not permitted until %v (listPodsRetryDuration %v).", dswp.timeOfLastListPods.Add(dswp.listPodsRetryDuration), dswp.listPodsRetryDuration) @@ -109,7 +109,7 @@ func (dswp *desiredStateOfWorldPopulator) findAndRemoveDeletedPods() { for dswPodUID, dswPodToAdd := range dswp.desiredStateOfWorld.GetPodToAdd() { dswPodKey, err := kcache.MetaNamespaceKeyFunc(dswPodToAdd.Pod) if err != nil { - glog.Errorf("MetaNamespaceKeyFunc failed for pod %q (UID %q) with: %v", dswPodKey, dswPodUID, err) + klog.Errorf("MetaNamespaceKeyFunc failed for pod %q (UID %q) with: %v", dswPodKey, dswPodUID, err) continue } @@ -124,7 +124,7 @@ func (dswp *desiredStateOfWorldPopulator) findAndRemoveDeletedPods() { case errors.IsNotFound(err): // if we can't find the pod, we need to delete it below case err != nil: - glog.Errorf("podLister Get failed for pod %q (UID %q) with %v", dswPodKey, dswPodUID, err) + klog.Errorf("podLister Get failed for pod %q (UID %q) with %v", dswPodKey, dswPodUID, err) continue default: volumeActionFlag := util.DetermineVolumeAction( @@ -136,7 +136,7 @@ func (dswp *desiredStateOfWorldPopulator) findAndRemoveDeletedPods() { informerPodUID := volutil.GetUniquePodName(informerPod) // Check whether the unique identifier of the pod from dsw matches the one retrieved from pod informer if informerPodUID == dswPodUID { - glog.V(10).Infof("Verified pod %q (UID %q) from dsw exists in pod informer.", dswPodKey, dswPodUID) + klog.V(10).Infof("Verified pod %q (UID %q) from dsw exists in pod informer.", dswPodKey, dswPodUID) continue } } @@ -144,7 +144,7 @@ func (dswp *desiredStateOfWorldPopulator) findAndRemoveDeletedPods() { // the pod from dsw does not exist in pod informer, or it does not match the unique identifier retrieved // from the informer, delete it from dsw - glog.V(1).Infof("Removing pod %q (UID %q) from dsw because it does not exist in pod informer.", dswPodKey, dswPodUID) + klog.V(1).Infof("Removing pod %q (UID %q) from dsw because it does not exist in pod informer.", dswPodKey, dswPodUID) dswp.desiredStateOfWorld.DeletePod(dswPodUID, dswPodToAdd.VolumeName, dswPodToAdd.NodeName) } } @@ -152,7 +152,7 @@ func (dswp *desiredStateOfWorldPopulator) findAndRemoveDeletedPods() { func (dswp *desiredStateOfWorldPopulator) findAndAddActivePods() { pods, err := dswp.podLister.List(labels.Everything()) if err != nil { - glog.Errorf("podLister List failed: %v", err) + klog.Errorf("podLister List failed: %v", err) return } dswp.timeOfLastListPods = time.Now() diff --git a/pkg/controller/volume/attachdetach/reconciler/BUILD b/pkg/controller/volume/attachdetach/reconciler/BUILD index dcdbf66d312..b09344c04fd 100644 --- a/pkg/controller/volume/attachdetach/reconciler/BUILD +++ b/pkg/controller/volume/attachdetach/reconciler/BUILD @@ -22,7 +22,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/attachdetach/reconciler/reconciler.go b/pkg/controller/volume/attachdetach/reconciler/reconciler.go index c52c608c2a4..deb257e00f4 100644 --- a/pkg/controller/volume/attachdetach/reconciler/reconciler.go +++ b/pkg/controller/volume/attachdetach/reconciler/reconciler.go @@ -24,11 +24,11 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/record" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/metrics" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/statusupdater" @@ -110,11 +110,11 @@ func (rc *reconciler) reconciliationLoopFunc() func() { rc.reconcile() if rc.disableReconciliationSync { - glog.V(5).Info("Skipping reconciling attached volumes still attached since it is disabled via the command line.") + klog.V(5).Info("Skipping reconciling attached volumes still attached since it is disabled via the command line.") } else if rc.syncDuration < time.Second { - glog.V(5).Info("Skipping reconciling attached volumes still attached since it is set to less than one second via the command line.") + klog.V(5).Info("Skipping reconciling attached volumes still attached since it is set to less than one second via the command line.") } else if time.Since(rc.timeOfLastSync) > rc.syncDuration { - glog.V(5).Info("Starting reconciling attached volumes still attached") + klog.V(5).Info("Starting reconciling attached volumes still attached") rc.sync() } } @@ -184,21 +184,21 @@ func (rc *reconciler) reconcile() { // may pass while at the same time the volume leaves the pending state, resulting in // double detach attempts if rc.attacherDetacher.IsOperationPending(attachedVolume.VolumeName, "") { - glog.V(10).Infof("Operation for volume %q is already running. Can't start detach for %q", attachedVolume.VolumeName, attachedVolume.NodeName) + klog.V(10).Infof("Operation for volume %q is already running. Can't start detach for %q", attachedVolume.VolumeName, attachedVolume.NodeName) continue } // Set the detach request time elapsedTime, err := rc.actualStateOfWorld.SetDetachRequestTime(attachedVolume.VolumeName, attachedVolume.NodeName) if err != nil { - glog.Errorf("Cannot trigger detach because it fails to set detach request time with error %v", err) + klog.Errorf("Cannot trigger detach because it fails to set detach request time with error %v", err) continue } // Check whether timeout has reached the maximum waiting time timeout := elapsedTime > rc.maxWaitForUnmountDuration // Check whether volume is still mounted. Skip detach if it is still mounted unless timeout if attachedVolume.MountedByNode && !timeout { - glog.V(12).Infof(attachedVolume.GenerateMsgDetailed("Cannot detach volume because it is still mounted", "")) + klog.V(12).Infof(attachedVolume.GenerateMsgDetailed("Cannot detach volume because it is still mounted", "")) continue } @@ -206,7 +206,7 @@ func (rc *reconciler) reconcile() { // If it fails to update node status, skip detach volume err = rc.actualStateOfWorld.RemoveVolumeFromReportAsAttached(attachedVolume.VolumeName, attachedVolume.NodeName) if err != nil { - glog.V(5).Infof("RemoveVolumeFromReportAsAttached failed while removing volume %q from node %q with: %v", + klog.V(5).Infof("RemoveVolumeFromReportAsAttached failed while removing volume %q from node %q with: %v", attachedVolume.VolumeName, attachedVolume.NodeName, err) @@ -216,27 +216,27 @@ func (rc *reconciler) reconcile() { err = rc.nodeStatusUpdater.UpdateNodeStatuses() if err != nil { // Skip detaching this volume if unable to update node status - glog.Errorf(attachedVolume.GenerateErrorDetailed("UpdateNodeStatuses failed while attempting to report volume as attached", err).Error()) + klog.Errorf(attachedVolume.GenerateErrorDetailed("UpdateNodeStatuses failed while attempting to report volume as attached", err).Error()) continue } // Trigger detach volume which requires verifing safe to detach step // If timeout is true, skip verifySafeToDetach check - glog.V(5).Infof(attachedVolume.GenerateMsgDetailed("Starting attacherDetacher.DetachVolume", "")) + klog.V(5).Infof(attachedVolume.GenerateMsgDetailed("Starting attacherDetacher.DetachVolume", "")) verifySafeToDetach := !timeout err = rc.attacherDetacher.DetachVolume(attachedVolume.AttachedVolume, verifySafeToDetach, rc.actualStateOfWorld) if err == nil { if !timeout { - glog.Infof(attachedVolume.GenerateMsgDetailed("attacherDetacher.DetachVolume started", "")) + klog.Infof(attachedVolume.GenerateMsgDetailed("attacherDetacher.DetachVolume started", "")) } else { metrics.RecordForcedDetachMetric() - glog.Warningf(attachedVolume.GenerateMsgDetailed("attacherDetacher.DetachVolume started", fmt.Sprintf("This volume is not safe to detach, but maxWaitForUnmountDuration %v expired, force detaching", rc.maxWaitForUnmountDuration))) + klog.Warningf(attachedVolume.GenerateMsgDetailed("attacherDetacher.DetachVolume started", fmt.Sprintf("This volume is not safe to detach, but maxWaitForUnmountDuration %v expired, force detaching", rc.maxWaitForUnmountDuration))) } } if err != nil && !exponentialbackoff.IsExponentialBackoff(err) { // Ignore exponentialbackoff.IsExponentialBackoff errors, they are expected. // Log all other errors. - glog.Errorf(attachedVolume.GenerateErrorDetailed("attacherDetacher.DetachVolume failed to start", err).Error()) + klog.Errorf(attachedVolume.GenerateErrorDetailed("attacherDetacher.DetachVolume failed to start", err).Error()) } } } @@ -246,7 +246,7 @@ func (rc *reconciler) reconcile() { // Update Node Status err := rc.nodeStatusUpdater.UpdateNodeStatuses() if err != nil { - glog.Warningf("UpdateNodeStatuses failed with: %v", err) + klog.Warningf("UpdateNodeStatuses failed with: %v", err) } } @@ -255,16 +255,16 @@ func (rc *reconciler) attachDesiredVolumes() { for _, volumeToAttach := range rc.desiredStateOfWorld.GetVolumesToAttach() { if rc.actualStateOfWorld.VolumeNodeExists(volumeToAttach.VolumeName, volumeToAttach.NodeName) { // Volume/Node exists, touch it to reset detachRequestedTime - if glog.V(5) { - glog.Infof(volumeToAttach.GenerateMsgDetailed("Volume attached--touching", "")) + if klog.V(5) { + klog.Infof(volumeToAttach.GenerateMsgDetailed("Volume attached--touching", "")) } rc.actualStateOfWorld.ResetDetachRequestTime(volumeToAttach.VolumeName, volumeToAttach.NodeName) continue } // Don't even try to start an operation if there is already one running if rc.attacherDetacher.IsOperationPending(volumeToAttach.VolumeName, "") { - if glog.V(10) { - glog.Infof("Operation for volume %q is already running. Can't start attach for %q", volumeToAttach.VolumeName, volumeToAttach.NodeName) + if klog.V(10) { + klog.Infof("Operation for volume %q is already running. Can't start attach for %q", volumeToAttach.VolumeName, volumeToAttach.NodeName) } continue } @@ -281,17 +281,17 @@ func (rc *reconciler) attachDesiredVolumes() { } // Volume/Node doesn't exist, spawn a goroutine to attach it - if glog.V(5) { - glog.Infof(volumeToAttach.GenerateMsgDetailed("Starting attacherDetacher.AttachVolume", "")) + if klog.V(5) { + klog.Infof(volumeToAttach.GenerateMsgDetailed("Starting attacherDetacher.AttachVolume", "")) } err := rc.attacherDetacher.AttachVolume(volumeToAttach.VolumeToAttach, rc.actualStateOfWorld) if err == nil { - glog.Infof(volumeToAttach.GenerateMsgDetailed("attacherDetacher.AttachVolume started", "")) + klog.Infof(volumeToAttach.GenerateMsgDetailed("attacherDetacher.AttachVolume started", "")) } if err != nil && !exponentialbackoff.IsExponentialBackoff(err) { // Ignore exponentialbackoff.IsExponentialBackoff errors, they are expected. // Log all other errors. - glog.Errorf(volumeToAttach.GenerateErrorDetailed("attacherDetacher.AttachVolume failed to start", err).Error()) + klog.Errorf(volumeToAttach.GenerateErrorDetailed("attacherDetacher.AttachVolume failed to start", err).Error()) } } } @@ -326,7 +326,7 @@ func (rc *reconciler) reportMultiAttachError(volumeToAttach cache.VolumeToAttach // Log detailed message to system admin nodeList := strings.Join(otherNodesStr, ", ") detailedMsg := volumeToAttach.GenerateMsgDetailed("Multi-Attach error", fmt.Sprintf("Volume is already exclusively attached to node %s and can't be attached to another", nodeList)) - glog.Warningf(detailedMsg) + klog.Warningf(detailedMsg) return } @@ -367,5 +367,5 @@ func (rc *reconciler) reportMultiAttachError(volumeToAttach cache.VolumeToAttach podNames = append(podNames, pod.Namespace+"/"+pod.Name) } detailedMsg := volumeToAttach.GenerateMsgDetailed("Multi-Attach error", fmt.Sprintf("Volume is already used by pods %s on node %s", strings.Join(podNames, ", "), strings.Join(otherNodesStr, ", "))) - glog.Warningf(detailedMsg) + klog.Warningf(detailedMsg) } diff --git a/pkg/controller/volume/attachdetach/statusupdater/BUILD b/pkg/controller/volume/attachdetach/statusupdater/BUILD index 97de6707a16..b72929ecdaa 100644 --- a/pkg/controller/volume/attachdetach/statusupdater/BUILD +++ b/pkg/controller/volume/attachdetach/statusupdater/BUILD @@ -20,7 +20,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/attachdetach/statusupdater/node_status_updater.go b/pkg/controller/volume/attachdetach/statusupdater/node_status_updater.go index b5cbb224464..615444d76e0 100644 --- a/pkg/controller/volume/attachdetach/statusupdater/node_status_updater.go +++ b/pkg/controller/volume/attachdetach/statusupdater/node_status_updater.go @@ -19,7 +19,7 @@ limitations under the License. package statusupdater import ( - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -65,7 +65,7 @@ func (nsu *nodeStatusUpdater) UpdateNodeStatuses() error { if errors.IsNotFound(err) { // If node does not exist, its status cannot be updated. // Do nothing so that there is no retry until node is created. - glog.V(2).Infof( + klog.V(2).Infof( "Could not update node status. Failed to find node %q in NodeInformer cache. Error: '%v'", nodeName, err) @@ -73,7 +73,7 @@ func (nsu *nodeStatusUpdater) UpdateNodeStatuses() error { } else if err != nil { // For all other errors, log error and reset flag statusUpdateNeeded // back to true to indicate this node status needs to be updated again. - glog.V(2).Infof("Error retrieving nodes from node lister. Error: %v", err) + klog.V(2).Infof("Error retrieving nodes from node lister. Error: %v", err) nsu.actualStateOfWorld.SetNodeStatusUpdateNeeded(nodeName) continue } @@ -83,7 +83,7 @@ func (nsu *nodeStatusUpdater) UpdateNodeStatuses() error { // to indicate this node status needs to be updated again nsu.actualStateOfWorld.SetNodeStatusUpdateNeeded(nodeName) - glog.V(2).Infof( + klog.V(2).Infof( "Could not update node status for %q; re-marking for update. %v", nodeName, err) @@ -103,6 +103,6 @@ func (nsu *nodeStatusUpdater) updateNodeStatus(nodeName types.NodeName, nodeObj return err } - glog.V(4).Infof("Updating status %q for node %q succeeded. VolumesAttached: %v", patchBytes, nodeName, attachedVolumes) + klog.V(4).Infof("Updating status %q for node %q succeeded. VolumesAttached: %v", patchBytes, nodeName, attachedVolumes) return nil } diff --git a/pkg/controller/volume/attachdetach/testing/BUILD b/pkg/controller/volume/attachdetach/testing/BUILD index bc06368aebc..1d52155d522 100644 --- a/pkg/controller/volume/attachdetach/testing/BUILD +++ b/pkg/controller/volume/attachdetach/testing/BUILD @@ -19,7 +19,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/attachdetach/testing/testvolumespec.go b/pkg/controller/volume/attachdetach/testing/testvolumespec.go index 7dceaba2732..a243e724b25 100644 --- a/pkg/controller/volume/attachdetach/testing/testvolumespec.go +++ b/pkg/controller/volume/attachdetach/testing/testvolumespec.go @@ -21,7 +21,6 @@ import ( "sync" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -29,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" + "k8s.io/klog" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" ) @@ -237,7 +237,7 @@ func (plugin *TestPlugin) GetVolumeName(spec *volume.Spec) (string, error) { plugin.pluginLock.Lock() defer plugin.pluginLock.Unlock() if spec == nil { - glog.Errorf("GetVolumeName called with nil volume spec") + klog.Errorf("GetVolumeName called with nil volume spec") plugin.ErrorEncountered = true } return spec.Name(), nil @@ -247,7 +247,7 @@ func (plugin *TestPlugin) CanSupport(spec *volume.Spec) bool { plugin.pluginLock.Lock() defer plugin.pluginLock.Unlock() if spec == nil { - glog.Errorf("CanSupport called with nil volume spec") + klog.Errorf("CanSupport called with nil volume spec") plugin.ErrorEncountered = true } return true @@ -261,7 +261,7 @@ func (plugin *TestPlugin) NewMounter(spec *volume.Spec, podRef *v1.Pod, opts vol plugin.pluginLock.Lock() defer plugin.pluginLock.Unlock() if spec == nil { - glog.Errorf("NewMounter called with nil volume spec") + klog.Errorf("NewMounter called with nil volume spec") plugin.ErrorEncountered = true } return nil, nil @@ -373,7 +373,7 @@ func (attacher *testPluginAttacher) Attach(spec *volume.Spec, nodeName types.Nod defer attacher.pluginLock.Unlock() if spec == nil { *attacher.ErrorEncountered = true - glog.Errorf("Attach called with nil volume spec") + klog.Errorf("Attach called with nil volume spec") return "", fmt.Errorf("Attach called with nil volume spec") } attacher.attachedVolumeMap[string(nodeName)] = append(attacher.attachedVolumeMap[string(nodeName)], spec.Name()) @@ -389,7 +389,7 @@ func (attacher *testPluginAttacher) WaitForAttach(spec *volume.Spec, devicePath defer attacher.pluginLock.Unlock() if spec == nil { *attacher.ErrorEncountered = true - glog.Errorf("WaitForAttach called with nil volume spec") + klog.Errorf("WaitForAttach called with nil volume spec") return "", fmt.Errorf("WaitForAttach called with nil volume spec") } fakePath := fmt.Sprintf("%s/%s", devicePath, spec.Name()) @@ -401,7 +401,7 @@ func (attacher *testPluginAttacher) GetDeviceMountPath(spec *volume.Spec) (strin defer attacher.pluginLock.Unlock() if spec == nil { *attacher.ErrorEncountered = true - glog.Errorf("GetDeviceMountPath called with nil volume spec") + klog.Errorf("GetDeviceMountPath called with nil volume spec") return "", fmt.Errorf("GetDeviceMountPath called with nil volume spec") } return "", nil @@ -412,7 +412,7 @@ func (attacher *testPluginAttacher) MountDevice(spec *volume.Spec, devicePath st defer attacher.pluginLock.Unlock() if spec == nil { *attacher.ErrorEncountered = true - glog.Errorf("MountDevice called with nil volume spec") + klog.Errorf("MountDevice called with nil volume spec") return fmt.Errorf("MountDevice called with nil volume spec") } return nil diff --git a/pkg/controller/volume/attachdetach/util/BUILD b/pkg/controller/volume/attachdetach/util/BUILD index 93889f81930..59c1db5809f 100644 --- a/pkg/controller/volume/attachdetach/util/BUILD +++ b/pkg/controller/volume/attachdetach/util/BUILD @@ -16,7 +16,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/attachdetach/util/util.go b/pkg/controller/volume/attachdetach/util/util.go index c17ab729906..8affeb46f08 100644 --- a/pkg/controller/volume/attachdetach/util/util.go +++ b/pkg/controller/volume/attachdetach/util/util.go @@ -19,10 +19,10 @@ package util import ( "fmt" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" @@ -32,7 +32,7 @@ import ( // specified volume. It dereference any PVC to get PV objects, if needed. func CreateVolumeSpec(podVolume v1.Volume, podNamespace string, pvcLister corelisters.PersistentVolumeClaimLister, pvLister corelisters.PersistentVolumeLister) (*volume.Spec, error) { if pvcSource := podVolume.VolumeSource.PersistentVolumeClaim; pvcSource != nil { - glog.V(10).Infof( + klog.V(10).Infof( "Found PVC, ClaimName: %q/%q", podNamespace, pvcSource.ClaimName) @@ -48,7 +48,7 @@ func CreateVolumeSpec(podVolume v1.Volume, podNamespace string, pvcLister coreli err) } - glog.V(10).Infof( + klog.V(10).Infof( "Found bound PV for PVC (ClaimName %q/%q pvcUID %v): pvName=%q", podNamespace, pvcSource.ClaimName, @@ -66,7 +66,7 @@ func CreateVolumeSpec(podVolume v1.Volume, podNamespace string, pvcLister coreli err) } - glog.V(10).Infof( + klog.V(10).Infof( "Extracted volumeSpec (%v) from bound PV (pvName %q) and PVC (ClaimName %q/%q pvcUID %v)", volumeSpec.Name(), pvName, @@ -166,7 +166,7 @@ func ProcessPodVolumes(pod *v1.Pod, addVolumes bool, desiredStateOfWorld cache.D } if len(pod.Spec.Volumes) <= 0 { - glog.V(10).Infof("Skipping processing of pod %q/%q: it has no volumes.", + klog.V(10).Infof("Skipping processing of pod %q/%q: it has no volumes.", pod.Namespace, pod.Name) return @@ -174,7 +174,7 @@ func ProcessPodVolumes(pod *v1.Pod, addVolumes bool, desiredStateOfWorld cache.D nodeName := types.NodeName(pod.Spec.NodeName) if nodeName == "" { - glog.V(10).Infof( + klog.V(10).Infof( "Skipping processing of pod %q/%q: it is not scheduled to a node.", pod.Namespace, pod.Name) @@ -183,7 +183,7 @@ func ProcessPodVolumes(pod *v1.Pod, addVolumes bool, desiredStateOfWorld cache.D // If the node the pod is scheduled to does not exist in the desired // state of the world data structure, that indicates the node is not // yet managed by the controller. Therefore, ignore the pod. - glog.V(4).Infof( + klog.V(4).Infof( "Skipping processing of pod %q/%q: it is scheduled to node %q which is not managed by the controller.", pod.Namespace, pod.Name, @@ -195,7 +195,7 @@ func ProcessPodVolumes(pod *v1.Pod, addVolumes bool, desiredStateOfWorld cache.D for _, podVolume := range pod.Spec.Volumes { volumeSpec, err := CreateVolumeSpec(podVolume, pod.Namespace, pvcLister, pvLister) if err != nil { - glog.V(10).Infof( + klog.V(10).Infof( "Error processing volume %q for pod %q/%q: %v", podVolume.Name, pod.Namespace, @@ -207,7 +207,7 @@ func ProcessPodVolumes(pod *v1.Pod, addVolumes bool, desiredStateOfWorld cache.D attachableVolumePlugin, err := volumePluginMgr.FindAttachablePluginBySpec(volumeSpec) if err != nil || attachableVolumePlugin == nil { - glog.V(10).Infof( + klog.V(10).Infof( "Skipping volume %q for pod %q/%q: it does not implement attacher interface. err=%v", podVolume.Name, pod.Namespace, @@ -222,7 +222,7 @@ func ProcessPodVolumes(pod *v1.Pod, addVolumes bool, desiredStateOfWorld cache.D _, err := desiredStateOfWorld.AddPod( uniquePodName, pod, volumeSpec, nodeName) if err != nil { - glog.V(10).Infof( + klog.V(10).Infof( "Failed to add volume %q for pod %q/%q to desiredStateOfWorld. %v", podVolume.Name, pod.Namespace, @@ -235,7 +235,7 @@ func ProcessPodVolumes(pod *v1.Pod, addVolumes bool, desiredStateOfWorld cache.D uniqueVolumeName, err := util.GetUniqueVolumeNameFromSpec( attachableVolumePlugin, volumeSpec) if err != nil { - glog.V(10).Infof( + klog.V(10).Infof( "Failed to delete volume %q for pod %q/%q from desiredStateOfWorld. GetUniqueVolumeNameFromSpec failed with %v", podVolume.Name, pod.Namespace, diff --git a/pkg/controller/volume/expand/BUILD b/pkg/controller/volume/expand/BUILD index 424d4a48371..c633196b325 100644 --- a/pkg/controller/volume/expand/BUILD +++ b/pkg/controller/volume/expand/BUILD @@ -1,9 +1,6 @@ package(default_visibility = ["//visibility:public"]) -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", @@ -39,7 +36,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/expand/cache/BUILD b/pkg/controller/volume/expand/cache/BUILD index e39803a409c..a416cc859c5 100644 --- a/pkg/controller/volume/expand/cache/BUILD +++ b/pkg/controller/volume/expand/cache/BUILD @@ -19,7 +19,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/expand/cache/volume_resize_map.go b/pkg/controller/volume/expand/cache/volume_resize_map.go index d5392c1b65a..76d85ec4d92 100644 --- a/pkg/controller/volume/expand/cache/volume_resize_map.go +++ b/pkg/controller/volume/expand/cache/volume_resize_map.go @@ -21,13 +21,13 @@ import ( "fmt" "sync" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" commontypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/strategicpatch" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/types" ) @@ -97,7 +97,7 @@ func NewVolumeResizeMap(kubeClient clientset.Interface) VolumeResizeMap { // the PVC and hopefully after a no-op resize in volume plugin, PVC will be updated with right values as well. func (resizeMap *volumeResizeMap) AddPVCUpdate(pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) { if pv.Spec.ClaimRef == nil || pvc.Namespace != pv.Spec.ClaimRef.Namespace || pvc.Name != pv.Spec.ClaimRef.Name { - glog.V(4).Infof("Persistent Volume is not bound to PVC being updated : %s", util.ClaimToClaimKey(pvc)) + klog.V(4).Infof("Persistent Volume is not bound to PVC being updated : %s", util.ClaimToClaimKey(pvc)) return } @@ -112,7 +112,7 @@ func (resizeMap *volumeResizeMap) AddPVCUpdate(pvc *v1.PersistentVolumeClaim, pv return } - glog.V(4).Infof("Adding pvc %s with Size %s/%s for resizing", util.ClaimToClaimKey(pvc), pvcSize.String(), pvcStatusSize.String()) + klog.V(4).Infof("Adding pvc %s with Size %s/%s for resizing", util.ClaimToClaimKey(pvc), pvcSize.String(), pvcStatusSize.String()) pvcRequest := &PVCWithResizeRequest{ PVC: pvc, @@ -144,7 +144,7 @@ func (resizeMap *volumeResizeMap) GetPVCsWithResizeRequest() []*PVCWithResizeReq // deleting a pvc in this map doesn't affect operations that are already inflight. func (resizeMap *volumeResizeMap) DeletePVC(pvc *v1.PersistentVolumeClaim) { pvcUniqueName := types.UniquePVCName(pvc.UID) - glog.V(5).Infof("Removing PVC %v from resize map", pvcUniqueName) + klog.V(5).Infof("Removing PVC %v from resize map", pvcUniqueName) resizeMap.Lock() defer resizeMap.Unlock() delete(resizeMap.pvcrs, pvcUniqueName) @@ -156,7 +156,7 @@ func (resizeMap *volumeResizeMap) MarkAsResized(pvcr *PVCWithResizeRequest, newS err := resizeMap.updatePVCCapacityAndConditions(pvcr, newSize, emptyCondition) if err != nil { - glog.V(4).Infof("Error updating PV spec capacity for volume %q with : %v", pvcr.QualifiedName(), err) + klog.V(4).Infof("Error updating PV spec capacity for volume %q with : %v", pvcr.QualifiedName(), err) return err } return nil @@ -205,7 +205,7 @@ func (resizeMap *volumeResizeMap) UpdatePVSize(pvcr *PVCWithResizeRequest, newSi _, updateErr := resizeMap.kubeClient.CoreV1().PersistentVolumes().Patch(pvClone.Name, commontypes.StrategicMergePatchType, patchBytes) if updateErr != nil { - glog.V(4).Infof("Error updating pv %q with error : %v", pvClone.Name, updateErr) + klog.V(4).Infof("Error updating pv %q with error : %v", pvClone.Name, updateErr) return updateErr } return nil diff --git a/pkg/controller/volume/expand/expand_controller.go b/pkg/controller/volume/expand/expand_controller.go index f40bf080a87..d53b953524d 100644 --- a/pkg/controller/volume/expand/expand_controller.go +++ b/pkg/controller/volume/expand/expand_controller.go @@ -24,7 +24,7 @@ import ( "net" "time" - "github.com/golang/glog" + "k8s.io/klog" authenticationv1 "k8s.io/api/authentication/v1" "k8s.io/api/core/v1" @@ -118,7 +118,7 @@ func NewExpandController( } eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) expc.recorder = eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "volume_expand"}) blkutil := volumepathhandler.NewBlockVolumePathHandler() @@ -150,8 +150,8 @@ func NewExpandController( func (expc *expandController) Run(stopCh <-chan struct{}) { defer runtime.HandleCrash() - glog.Infof("Starting expand controller") - defer glog.Infof("Shutting down expand controller") + klog.Infof("Starting expand controller") + defer klog.Infof("Shutting down expand controller") if !controller.WaitForCacheSync("expand", stopCh, expc.pvcsSynced, expc.pvSynced) { return @@ -204,7 +204,7 @@ func (expc *expandController) pvcUpdate(oldObj, newObj interface{}) { if newSize.Cmp(oldSize) > 0 { pv, err := getPersistentVolume(newPVC, expc.pvLister) if err != nil { - glog.V(5).Infof("Error getting Persistent Volume for PVC %q : %v", newPVC.UID, err) + klog.V(5).Infof("Error getting Persistent Volume for PVC %q : %v", newPVC.UID, err) return } @@ -219,7 +219,7 @@ func (expc *expandController) pvcUpdate(oldObj, newObj interface{}) { } expc.recorder.Event(newPVC, eventType, events.ExternalExpanding, fmt.Sprintf("Ignoring the PVC: %v.", err)) - glog.V(3).Infof("Ignoring the PVC %q (uid: %q) : %v.", + klog.V(3).Infof("Ignoring the PVC %q (uid: %q) : %v.", util.GetPersistentVolumeClaimQualifiedName(newPVC), newPVC.UID, err) return } @@ -319,7 +319,7 @@ func (expc *expandController) GetServiceAccountTokenFunc() func(_, _ string, _ * func (expc *expandController) DeleteServiceAccountTokenFunc() func(types.UID) { return func(types.UID) { - glog.Errorf("DeleteServiceAccountToken unsupported in expandController") + klog.Errorf("DeleteServiceAccountToken unsupported in expandController") } } diff --git a/pkg/controller/volume/expand/pvc_populator.go b/pkg/controller/volume/expand/pvc_populator.go index 4f29e2292e2..20ccd6350b0 100644 --- a/pkg/controller/volume/expand/pvc_populator.go +++ b/pkg/controller/volume/expand/pvc_populator.go @@ -24,7 +24,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" @@ -84,7 +84,7 @@ func (populator *pvcPopulator) Run(stopCh <-chan struct{}) { func (populator *pvcPopulator) Sync() { pvcs, err := populator.pvcLister.List(labels.Everything()) if err != nil { - glog.Errorf("Listing PVCs failed in populator : %v", err) + klog.Errorf("Listing PVCs failed in populator : %v", err) return } @@ -92,7 +92,7 @@ func (populator *pvcPopulator) Sync() { pv, err := getPersistentVolume(pvc, populator.pvLister) if err != nil { - glog.V(5).Infof("Error getting persistent volume for PVC %q : %v", pvc.UID, err) + klog.V(5).Infof("Error getting persistent volume for PVC %q : %v", pvc.UID, err) continue } @@ -110,7 +110,7 @@ func (populator *pvcPopulator) Sync() { } populator.recorder.Event(pvc, eventType, events.ExternalExpanding, fmt.Sprintf("Ignoring the PVC: %v.", err)) - glog.V(3).Infof("Ignoring the PVC %q (uid: %q) : %v.", + klog.V(3).Infof("Ignoring the PVC %q (uid: %q) : %v.", util.GetPersistentVolumeClaimQualifiedName(pvc), pvc.UID, err) continue } diff --git a/pkg/controller/volume/expand/sync_volume_resize.go b/pkg/controller/volume/expand/sync_volume_resize.go index 9ad10eb2771..c355a967472 100644 --- a/pkg/controller/volume/expand/sync_volume_resize.go +++ b/pkg/controller/volume/expand/sync_volume_resize.go @@ -19,11 +19,11 @@ package expand import ( "time" - "github.com/golang/glog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller/volume/expand/cache" "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff" "k8s.io/kubernetes/pkg/volume/util" @@ -66,7 +66,7 @@ func (rc *syncResize) Sync() { uniqueVolumeKey := v1.UniqueVolumeName(pvcWithResizeRequest.UniquePVCKey()) updatedClaim, err := markPVCResizeInProgress(pvcWithResizeRequest, rc.kubeClient) if err != nil { - glog.V(5).Infof("Error setting PVC %s in progress with error : %v", pvcWithResizeRequest.QualifiedName(), err) + klog.V(5).Infof("Error setting PVC %s in progress with error : %v", pvcWithResizeRequest.QualifiedName(), err) continue } if updatedClaim != nil { @@ -74,15 +74,15 @@ func (rc *syncResize) Sync() { } if rc.opsExecutor.IsOperationPending(uniqueVolumeKey, "") { - glog.V(10).Infof("Operation for PVC %v is already pending", pvcWithResizeRequest.QualifiedName()) + klog.V(10).Infof("Operation for PVC %v is already pending", pvcWithResizeRequest.QualifiedName()) continue } growFuncError := rc.opsExecutor.ExpandVolume(pvcWithResizeRequest, rc.resizeMap) if growFuncError != nil && !exponentialbackoff.IsExponentialBackoff(growFuncError) { - glog.Errorf("Error growing pvc %s with %v", pvcWithResizeRequest.QualifiedName(), growFuncError) + klog.Errorf("Error growing pvc %s with %v", pvcWithResizeRequest.QualifiedName(), growFuncError) } if growFuncError == nil { - glog.V(5).Infof("Started opsExecutor.ExpandVolume for volume %s", pvcWithResizeRequest.QualifiedName()) + klog.V(5).Infof("Started opsExecutor.ExpandVolume for volume %s", pvcWithResizeRequest.QualifiedName()) } } } diff --git a/pkg/controller/volume/persistentvolume/BUILD b/pkg/controller/volume/persistentvolume/BUILD index 7ac8a718840..ae16c23f113 100644 --- a/pkg/controller/volume/persistentvolume/BUILD +++ b/pkg/controller/volume/persistentvolume/BUILD @@ -58,8 +58,8 @@ go_library( "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -70,6 +70,7 @@ go_test( "delete_test.go", "framework_test.go", "index_test.go", + "main_test.go", "provision_test.go", "pv_controller_test.go", "recycle_test.go", @@ -82,6 +83,7 @@ go_test( "//pkg/api/testapi:go_default_library", "//pkg/apis/core:go_default_library", "//pkg/controller:go_default_library", + "//pkg/features:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/recyclerclient:go_default_library", @@ -96,6 +98,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", @@ -106,7 +109,7 @@ go_test( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/tools/reference:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/persistentvolume/binder_test.go b/pkg/controller/volume/persistentvolume/binder_test.go index c278c8c01d1..90d9c2ea038 100644 --- a/pkg/controller/volume/persistentvolume/binder_test.go +++ b/pkg/controller/volume/persistentvolume/binder_test.go @@ -23,6 +23,8 @@ import ( storage "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + "k8s.io/kubernetes/pkg/features" ) // Test single call to syncClaim and syncVolume methods. @@ -35,8 +37,6 @@ func TestSync(t *testing.T) { "foo": "true", "bar": "false", } - modeBlock := v1.PersistentVolumeBlock - modeFile := v1.PersistentVolumeFilesystem tests := []controllerTest{ // [Unit test set 1] User did not care which PV they get. @@ -589,9 +589,22 @@ func TestSync(t *testing.T) { newClaimArray("claim13-5", "uid13-5", "1Gi", "volume13-5", v1.ClaimBound, nil, annBoundByController, annBindCompleted), noevents, noerrors, testSyncClaim, }, + } - // All of these should bind as feature set is not enabled for BlockVolume - // meaning volumeMode will be ignored and dropped + runSyncTests(t, tests, []*storage.StorageClass{ + { + ObjectMeta: metav1.ObjectMeta{Name: classWait}, + VolumeBindingMode: &modeWait, + }, + }, []*v1.Pod{}) +} + +func TestSyncBlockVolumeDisabled(t *testing.T) { + modeBlock := v1.PersistentVolumeBlock + modeFile := v1.PersistentVolumeFilesystem + // All of these should bind as feature set is not enabled for BlockVolume + // meaning volumeMode will be ignored and dropped + tests := []controllerTest{ { // syncVolume binds a requested block claim to a block volume "14-1 - binding to volumeMode block", @@ -639,9 +652,7 @@ func TestSync(t *testing.T) { }, } - utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - defer utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, false)() runSyncTests(t, tests, []*storage.StorageClass{ { ObjectMeta: metav1.ObjectMeta{Name: classWait}, @@ -650,7 +661,7 @@ func TestSync(t *testing.T) { }, []*v1.Pod{}) } -func TestSyncAlphaBlockVolume(t *testing.T) { +func TestSyncBlockVolume(t *testing.T) { modeBlock := v1.PersistentVolumeBlock modeFile := v1.PersistentVolumeFilesystem @@ -830,12 +841,7 @@ func TestSyncAlphaBlockVolume(t *testing.T) { }, } - err := utilfeature.DefaultFeatureGate.Set("BlockVolume=true") - if err != nil { - t.Errorf("Failed to enable feature gate for BlockVolume: %v", err) - return - } - defer utilfeature.DefaultFeatureGate.Set("BlockVolume=false") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() runSyncTests(t, tests, []*storage.StorageClass{}, []*v1.Pod{}) } diff --git a/pkg/controller/volume/persistentvolume/framework_test.go b/pkg/controller/volume/persistentvolume/framework_test.go index 4626e661e2a..ef632cd5285 100644 --- a/pkg/controller/volume/persistentvolume/framework_test.go +++ b/pkg/controller/volume/persistentvolume/framework_test.go @@ -27,7 +27,7 @@ import ( "testing" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" @@ -38,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" @@ -48,6 +49,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/api/testapi" "k8s.io/kubernetes/pkg/controller" + "k8s.io/kubernetes/pkg/features" vol "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util/recyclerclient" ) @@ -159,7 +161,7 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj r.lock.Lock() defer r.lock.Unlock() - glog.V(4).Infof("reactor got operation %q on %q", action.GetVerb(), action.GetResource()) + klog.V(4).Infof("reactor got operation %q on %q", action.GetVerb(), action.GetResource()) // Inject error when requested err = r.injectReactError(action) @@ -179,11 +181,17 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj return true, nil, fmt.Errorf("Cannot create volume %s: volume already exists", volume.Name) } + // mimic apiserver defaulting + if volume.Spec.VolumeMode == nil && utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { + volume.Spec.VolumeMode = new(v1.PersistentVolumeMode) + *volume.Spec.VolumeMode = v1.PersistentVolumeFilesystem + } + // Store the updated object to appropriate places. r.volumes[volume.Name] = volume r.changedObjects = append(r.changedObjects, volume) r.changedSinceLastSync++ - glog.V(4).Infof("created volume %s", volume.Name) + klog.V(4).Infof("created volume %s", volume.Name) return true, volume, nil case action.Matches("update", "persistentvolumes"): @@ -209,7 +217,7 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj r.volumes[volume.Name] = volume r.changedObjects = append(r.changedObjects, volume) r.changedSinceLastSync++ - glog.V(4).Infof("saved updated volume %s", volume.Name) + klog.V(4).Infof("saved updated volume %s", volume.Name) return true, volume, nil case action.Matches("update", "persistentvolumeclaims"): @@ -235,23 +243,23 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj r.claims[claim.Name] = claim r.changedObjects = append(r.changedObjects, claim) r.changedSinceLastSync++ - glog.V(4).Infof("saved updated claim %s", claim.Name) + klog.V(4).Infof("saved updated claim %s", claim.Name) return true, claim, nil case action.Matches("get", "persistentvolumes"): name := action.(core.GetAction).GetName() volume, found := r.volumes[name] if found { - glog.V(4).Infof("GetVolume: found %s", volume.Name) + klog.V(4).Infof("GetVolume: found %s", volume.Name) return true, volume, nil } else { - glog.V(4).Infof("GetVolume: volume %s not found", name) + klog.V(4).Infof("GetVolume: volume %s not found", name) return true, nil, fmt.Errorf("Cannot find volume %s", name) } case action.Matches("delete", "persistentvolumes"): name := action.(core.DeleteAction).GetName() - glog.V(4).Infof("deleted volume %s", name) + klog.V(4).Infof("deleted volume %s", name) _, found := r.volumes[name] if found { delete(r.volumes, name) @@ -263,7 +271,7 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj case action.Matches("delete", "persistentvolumeclaims"): name := action.(core.DeleteAction).GetName() - glog.V(4).Infof("deleted claim %s", name) + klog.V(4).Infof("deleted claim %s", name) _, found := r.volumes[name] if found { delete(r.claims, name) @@ -286,11 +294,11 @@ func (r *volumeReactor) injectReactError(action core.Action) error { } for i, expected := range r.errors { - glog.V(4).Infof("trying to match %q %q with %q %q", expected.verb, expected.resource, action.GetVerb(), action.GetResource()) + klog.V(4).Infof("trying to match %q %q with %q %q", expected.verb, expected.resource, action.GetVerb(), action.GetResource()) if action.Matches(expected.verb, expected.resource) { // That's the action we're waiting for, remove it from injectedErrors r.errors = append(r.errors[:i], r.errors[i+1:]...) - glog.V(4).Infof("reactor found matching error at index %d: %q %q, returning %v", i, expected.verb, expected.resource, expected.error) + klog.V(4).Infof("reactor found matching error at index %d: %q %q, returning %v", i, expected.verb, expected.resource, expected.error) return expected.error } } @@ -379,14 +387,14 @@ func checkEvents(t *testing.T, expectedEvents []string, ctrl *PersistentVolumeCo select { case event, ok := <-fakeRecorder.Events: if ok { - glog.V(5).Infof("event recorder got event %s", event) + klog.V(5).Infof("event recorder got event %s", event) gotEvents = append(gotEvents, event) } else { - glog.V(5).Infof("event recorder finished") + klog.V(5).Infof("event recorder finished") finished = true } case _, _ = <-timer.C: - glog.V(5).Infof("event recorder timeout") + klog.V(5).Infof("event recorder timeout") finished = true } } @@ -426,10 +434,10 @@ func (r *volumeReactor) popChange() interface{} { switch obj.(type) { case *v1.PersistentVolume: vol, _ := obj.(*v1.PersistentVolume) - glog.V(4).Infof("reactor queue: %s", vol.Name) + klog.V(4).Infof("reactor queue: %s", vol.Name) case *v1.PersistentVolumeClaim: claim, _ := obj.(*v1.PersistentVolumeClaim) - glog.V(4).Infof("reactor queue: %s", claim.Name) + klog.V(4).Infof("reactor queue: %s", claim.Name) } } @@ -630,6 +638,7 @@ func newTestController(kubeClient clientset.Interface, informerFactory informers // newVolume returns a new volume with given attributes func newVolume(name, capacity, boundToClaimUID, boundToClaimName string, phase v1.PersistentVolumePhase, reclaimPolicy v1.PersistentVolumeReclaimPolicy, class string, annotations ...string) *v1.PersistentVolume { + fs := v1.PersistentVolumeFilesystem volume := v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -645,6 +654,7 @@ func newVolume(name, capacity, boundToClaimUID, boundToClaimName string, phase v AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce, v1.ReadOnlyMany}, PersistentVolumeReclaimPolicy: reclaimPolicy, StorageClassName: class, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: phase, @@ -740,6 +750,7 @@ func newVolumeArray(name, capacity, boundToClaimUID, boundToClaimName string, ph // newClaim returns a new claim with given attributes func newClaim(name, claimUID, capacity, boundToVolume string, phase v1.PersistentVolumeClaimPhase, class *string, annotations ...string) *v1.PersistentVolumeClaim { + fs := v1.PersistentVolumeFilesystem claim := v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -756,6 +767,7 @@ func newClaim(name, claimUID, capacity, boundToVolume string, phase v1.Persisten }, VolumeName: boundToVolume, StorageClassName: class, + VolumeMode: &fs, }, Status: v1.PersistentVolumeClaimStatus{ Phase: phase, @@ -898,7 +910,7 @@ func wrapTestWithInjectedOperation(toWrap testCall, injectBeforeOperation func(c // Inject a hook before async operation starts ctrl.preOperationHook = func(operationName string) { // Inside the hook, run the function to inject - glog.V(4).Infof("reactor: scheduleOperation reached, injecting call") + klog.V(4).Infof("reactor: scheduleOperation reached, injecting call") injectBeforeOperation(ctrl, reactor) } @@ -945,7 +957,7 @@ func evaluateTestResults(ctrl *PersistentVolumeController, reactor *volumeReacto // 3. Compare resulting volumes and claims with expected volumes and claims. func runSyncTests(t *testing.T, tests []controllerTest, storageClasses []*storage.StorageClass, pods []*v1.Pod) { for _, test := range tests { - glog.V(4).Infof("starting test %q", test.name) + klog.V(4).Infof("starting test %q", test.name) // Initialize the controller client := &fake.Clientset{} @@ -1008,7 +1020,7 @@ func runSyncTests(t *testing.T, tests []controllerTest, storageClasses []*storag // Some limit of calls in enforced to prevent endless loops. func runMultisyncTests(t *testing.T, tests []controllerTest, storageClasses []*storage.StorageClass, defaultStorageClass string) { for _, test := range tests { - glog.V(4).Infof("starting multisync test %q", test.name) + klog.V(4).Infof("starting multisync test %q", test.name) // Initialize the controller client := &fake.Clientset{} @@ -1046,7 +1058,7 @@ func runMultisyncTests(t *testing.T, tests []controllerTest, storageClasses []*s counter := 0 for { counter++ - glog.V(4).Infof("test %q: iteration %d", test.name, counter) + klog.V(4).Infof("test %q: iteration %d", test.name, counter) if counter > 100 { t.Errorf("Test %q failed: too many iterations", test.name) @@ -1064,7 +1076,7 @@ func runMultisyncTests(t *testing.T, tests []controllerTest, storageClasses []*s // Simulate "periodic sync" of everything (until it produces // no changes). firstSync = false - glog.V(4).Infof("test %q: simulating periodical sync of all claims and volumes", test.name) + klog.V(4).Infof("test %q: simulating periodical sync of all claims and volumes", test.name) reactor.syncAll() } else { // Last sync did not produce any updates, the test reached @@ -1085,7 +1097,7 @@ func runMultisyncTests(t *testing.T, tests []controllerTest, storageClasses []*s if err != nil { if err == versionConflictError { // Ignore version errors - glog.V(4).Infof("test intentionaly ignores version error.") + klog.V(4).Infof("test intentionaly ignores version error.") } else { t.Errorf("Error calling syncClaim: %v", err) // Finish the loop on the first error @@ -1102,7 +1114,7 @@ func runMultisyncTests(t *testing.T, tests []controllerTest, storageClasses []*s if err != nil { if err == versionConflictError { // Ignore version errors - glog.V(4).Infof("test intentionaly ignores version error.") + klog.V(4).Infof("test intentionaly ignores version error.") } else { t.Errorf("Error calling syncVolume: %v", err) // Finish the loop on the first error @@ -1114,7 +1126,7 @@ func runMultisyncTests(t *testing.T, tests []controllerTest, storageClasses []*s } } evaluateTestResults(ctrl, reactor, test, t) - glog.V(4).Infof("test %q finished after %d iterations", test.name, counter) + klog.V(4).Infof("test %q finished after %d iterations", test.name, counter) } } @@ -1185,7 +1197,7 @@ func (plugin *mockVolumePlugin) NewUnmounter(name string, podUID types.UID) (vol func (plugin *mockVolumePlugin) NewProvisioner(options vol.VolumeOptions) (vol.Provisioner, error) { if len(plugin.provisionCalls) > 0 { // mockVolumePlugin directly implements Provisioner interface - glog.V(4).Infof("mock plugin NewProvisioner called, returning mock provisioner") + klog.V(4).Infof("mock plugin NewProvisioner called, returning mock provisioner") plugin.provisionOptions = options return plugin, nil } else { @@ -1201,7 +1213,7 @@ func (plugin *mockVolumePlugin) Provision(selectedNode *v1.Node, allowedTopologi var pv *v1.PersistentVolume call := plugin.provisionCalls[plugin.provisionCallCounter] if !reflect.DeepEqual(call.expectedParameters, plugin.provisionOptions.Parameters) { - glog.Errorf("invalid provisioner call, expected options: %+v, got: %+v", call.expectedParameters, plugin.provisionOptions.Parameters) + klog.Errorf("invalid provisioner call, expected options: %+v, got: %+v", call.expectedParameters, plugin.provisionOptions.Parameters) return nil, fmt.Errorf("Mock plugin error: invalid provisioner call") } if call.ret == nil { @@ -1226,10 +1238,11 @@ func (plugin *mockVolumePlugin) Provision(selectedNode *v1.Node, allowedTopologi Phase: v1.VolumeAvailable, }, } + pv.Spec.VolumeMode = plugin.provisionOptions.PVC.Spec.VolumeMode } plugin.provisionCallCounter++ - glog.V(4).Infof("mock plugin Provision call nr. %d, returning %v: %v", plugin.provisionCallCounter, pv, call.ret) + klog.V(4).Infof("mock plugin Provision call nr. %d, returning %v: %v", plugin.provisionCallCounter, pv, call.ret) return pv, call.ret } @@ -1238,7 +1251,7 @@ func (plugin *mockVolumePlugin) Provision(selectedNode *v1.Node, allowedTopologi func (plugin *mockVolumePlugin) NewDeleter(spec *vol.Spec) (vol.Deleter, error) { if len(plugin.deleteCalls) > 0 { // mockVolumePlugin directly implements Deleter interface - glog.V(4).Infof("mock plugin NewDeleter called, returning mock deleter") + klog.V(4).Infof("mock plugin NewDeleter called, returning mock deleter") return plugin, nil } else { return nil, fmt.Errorf("Mock plugin error: no deleteCalls configured") @@ -1251,7 +1264,7 @@ func (plugin *mockVolumePlugin) Delete() error { } ret := plugin.deleteCalls[plugin.deleteCallCounter] plugin.deleteCallCounter++ - glog.V(4).Infof("mock plugin Delete call nr. %d, returning %v", plugin.deleteCallCounter, ret) + klog.V(4).Infof("mock plugin Delete call nr. %d, returning %v", plugin.deleteCallCounter, ret) return ret } @@ -1277,6 +1290,6 @@ func (plugin *mockVolumePlugin) Recycle(pvName string, spec *vol.Spec, eventReco } ret := plugin.recycleCalls[plugin.recycleCallCounter] plugin.recycleCallCounter++ - glog.V(4).Infof("mock plugin Recycle call nr. %d, returning %v", plugin.recycleCallCounter, ret) + klog.V(4).Infof("mock plugin Recycle call nr. %d, returning %v", plugin.recycleCallCounter, ret) return ret } diff --git a/pkg/controller/volume/persistentvolume/index.go b/pkg/controller/volume/persistentvolume/index.go index 4c84abe82e0..19195e81523 100644 --- a/pkg/controller/volume/persistentvolume/index.go +++ b/pkg/controller/volume/persistentvolume/index.go @@ -158,13 +158,13 @@ func findMatchingVolume( volumeQty := volume.Spec.Capacity[v1.ResourceStorage] - // check if volumeModes do not match (Alpha and feature gate protected) - isMisMatch, err := checkVolumeModeMisMatches(&claim.Spec, &volume.Spec) + // check if volumeModes do not match (feature gate protected) + isMismatch, err := checkVolumeModeMismatches(&claim.Spec, &volume.Spec) if err != nil { return nil, fmt.Errorf("error checking if volumeMode was a mismatch: %v", err) } // filter out mismatching volumeModes - if isMisMatch { + if isMismatch { continue } @@ -258,25 +258,24 @@ func findMatchingVolume( return nil, nil } -// checkVolumeModeMatches is a convenience method that checks volumeMode for PersistentVolume -// and PersistentVolumeClaims along with making sure that the Alpha feature gate BlockVolume is -// enabled. -// This is Alpha and could change in the future. -func checkVolumeModeMisMatches(pvcSpec *v1.PersistentVolumeClaimSpec, pvSpec *v1.PersistentVolumeSpec) (bool, error) { - if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { - if pvSpec.VolumeMode != nil && pvcSpec.VolumeMode != nil { - requestedVolumeMode := *pvcSpec.VolumeMode - pvVolumeMode := *pvSpec.VolumeMode - return requestedVolumeMode != pvVolumeMode, nil - } else { - // This also should retrun an error, this means that - // the defaulting has failed. - return true, fmt.Errorf("api defaulting for volumeMode failed") - } - } else { - // feature gate is disabled +// checkVolumeModeMismatches is a convenience method that checks volumeMode for PersistentVolume +// and PersistentVolumeClaims +func checkVolumeModeMismatches(pvcSpec *v1.PersistentVolumeClaimSpec, pvSpec *v1.PersistentVolumeSpec) (bool, error) { + if !utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { return false, nil } + + // In HA upgrades, we cannot guarantee that the apiserver is on a version >= controller-manager. + // So we default a nil volumeMode to filesystem + requestedVolumeMode := v1.PersistentVolumeFilesystem + if pvcSpec.VolumeMode != nil { + requestedVolumeMode = *pvcSpec.VolumeMode + } + pvVolumeMode := v1.PersistentVolumeFilesystem + if pvSpec.VolumeMode != nil { + pvVolumeMode = *pvSpec.VolumeMode + } + return requestedVolumeMode != pvVolumeMode, nil } // findBestMatchForClaim is a convenience method that finds a volume by the claim's AccessModes and requests for Storage diff --git a/pkg/controller/volume/persistentvolume/index_test.go b/pkg/controller/volume/persistentvolume/index_test.go index 8ad390daf89..8729891c2c7 100644 --- a/pkg/controller/volume/persistentvolume/index_test.go +++ b/pkg/controller/volume/persistentvolume/index_test.go @@ -24,13 +24,16 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/kubernetes/scheme" ref "k8s.io/client-go/tools/reference" "k8s.io/kubernetes/pkg/api/testapi" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume/util" ) func makePVC(size string, modfn func(*v1.PersistentVolumeClaim)) *v1.PersistentVolumeClaim { + fs := v1.PersistentVolumeFilesystem pvc := v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "claim01", @@ -43,6 +46,7 @@ func makePVC(size string, modfn func(*v1.PersistentVolumeClaim)) *v1.PersistentV v1.ResourceName(v1.ResourceStorage): resource.MustParse(size), }, }, + VolumeMode: &fs, }, } if modfn != nil { @@ -195,6 +199,7 @@ func TestMatchVolume(t *testing.T) { } func TestMatchingWithBoundVolumes(t *testing.T) { + fs := v1.PersistentVolumeFilesystem volumeIndex := newPersistentVolumeOrderedIndex() // two similar volumes, one is bound pv1 := &v1.PersistentVolume{ @@ -211,7 +216,8 @@ func TestMatchingWithBoundVolumes(t *testing.T) { }, AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce, v1.ReadOnlyMany}, // this one we're pretending is already bound - ClaimRef: &v1.ObjectReference{UID: "abc123"}, + ClaimRef: &v1.ObjectReference{UID: "abc123"}, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeBound, @@ -231,6 +237,7 @@ func TestMatchingWithBoundVolumes(t *testing.T) { GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}, }, AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce, v1.ReadOnlyMany}, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -252,6 +259,7 @@ func TestMatchingWithBoundVolumes(t *testing.T) { v1.ResourceName(v1.ResourceStorage): resource.MustParse("1G"), }, }, + VolumeMode: &fs, }, } @@ -326,6 +334,7 @@ func TestAllPossibleAccessModes(t *testing.T) { } func TestFindingVolumeWithDifferentAccessModes(t *testing.T) { + fs := v1.PersistentVolumeFilesystem gce := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{UID: "001", Name: "gce"}, Spec: v1.PersistentVolumeSpec{ @@ -335,6 +344,7 @@ func TestFindingVolumeWithDifferentAccessModes(t *testing.T) { v1.ReadWriteOnce, v1.ReadOnlyMany, }, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -349,6 +359,7 @@ func TestFindingVolumeWithDifferentAccessModes(t *testing.T) { AccessModes: []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, }, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -365,6 +376,7 @@ func TestFindingVolumeWithDifferentAccessModes(t *testing.T) { v1.ReadOnlyMany, v1.ReadWriteMany, }, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -379,6 +391,7 @@ func TestFindingVolumeWithDifferentAccessModes(t *testing.T) { Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse("1G")}}, + VolumeMode: &fs, }, } @@ -438,6 +451,7 @@ func TestFindingVolumeWithDifferentAccessModes(t *testing.T) { } func createTestVolumes() []*v1.PersistentVolume { + fs := v1.PersistentVolumeFilesystem // these volumes are deliberately out-of-order to test indexing and sorting return []*v1.PersistentVolume{ { @@ -456,6 +470,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadWriteOnce, v1.ReadOnlyMany, }, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -478,7 +493,8 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, // this one we're pretending is already bound - ClaimRef: &v1.ObjectReference{UID: "def456"}, + ClaimRef: &v1.ObjectReference{UID: "def456"}, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeBound, @@ -494,13 +510,14 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ResourceName(v1.ResourceStorage): resource.MustParse("5G"), }, PersistentVolumeSource: v1.PersistentVolumeSource{ - Glusterfs: &v1.GlusterfsVolumeSource{}, + Glusterfs: &v1.GlusterfsPersistentVolumeSource{}, }, AccessModes: []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, v1.ReadOnlyMany, v1.ReadWriteMany, }, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -523,7 +540,8 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadOnlyMany, }, // this one we're pretending is already bound - ClaimRef: &v1.ObjectReference{UID: "abc123"}, + ClaimRef: &v1.ObjectReference{UID: "abc123"}, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeBound, @@ -539,13 +557,14 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ResourceName(v1.ResourceStorage): resource.MustParse("10G"), }, PersistentVolumeSource: v1.PersistentVolumeSource{ - Glusterfs: &v1.GlusterfsVolumeSource{}, + Glusterfs: &v1.GlusterfsPersistentVolumeSource{}, }, AccessModes: []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, v1.ReadOnlyMany, v1.ReadWriteMany, }, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -567,6 +586,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadWriteOnce, v1.ReadOnlyMany, }, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -582,13 +602,14 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ResourceName(v1.ResourceStorage): resource.MustParse("1G"), }, PersistentVolumeSource: v1.PersistentVolumeSource{ - Glusterfs: &v1.GlusterfsVolumeSource{}, + Glusterfs: &v1.GlusterfsPersistentVolumeSource{}, }, AccessModes: []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, v1.ReadOnlyMany, v1.ReadWriteMany, }, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -612,6 +633,7 @@ func createTestVolumes() []*v1.PersistentVolume { AccessModes: []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, }, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -636,6 +658,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadWriteOnce, }, StorageClassName: classSilver, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -657,6 +680,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadWriteOnce, }, StorageClassName: classSilver, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -678,6 +702,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadWriteOnce, }, StorageClassName: classGold, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -701,6 +726,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadWriteMany, }, StorageClassName: classLarge, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -724,6 +750,7 @@ func createTestVolumes() []*v1.PersistentVolume { v1.ReadWriteMany, }, StorageClassName: classLarge, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -747,6 +774,7 @@ func createTestVolumes() []*v1.PersistentVolume { }, StorageClassName: classWait, NodeAffinity: getVolumeNodeAffinity("key1", "value1"), + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -770,6 +798,7 @@ func createTestVolumes() []*v1.PersistentVolume { }, StorageClassName: classWait, NodeAffinity: getVolumeNodeAffinity("key1", "value1"), + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -794,6 +823,7 @@ func createTestVolumes() []*v1.PersistentVolume { StorageClassName: classWait, ClaimRef: &v1.ObjectReference{Name: "claim02", Namespace: "myns"}, NodeAffinity: getVolumeNodeAffinity("key1", "value1"), + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -817,6 +847,7 @@ func createTestVolumes() []*v1.PersistentVolume { }, StorageClassName: classWait, NodeAffinity: getVolumeNodeAffinity("key1", "value3"), + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -840,6 +871,7 @@ func createTestVolumes() []*v1.PersistentVolume { }, StorageClassName: classWait, NodeAffinity: getVolumeNodeAffinity("key1", "value4"), + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumePending, @@ -863,6 +895,7 @@ func createTestVolumes() []*v1.PersistentVolume { }, StorageClassName: classWait, NodeAffinity: getVolumeNodeAffinity("key1", "value4"), + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeFailed, @@ -886,6 +919,7 @@ func createTestVolumes() []*v1.PersistentVolume { }, StorageClassName: classWait, NodeAffinity: getVolumeNodeAffinity("key1", "value4"), + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeReleased, @@ -909,12 +943,14 @@ func createTestVolumes() []*v1.PersistentVolume { }, StorageClassName: classWait, NodeAffinity: getVolumeNodeAffinity("key1", "value4"), + VolumeMode: &fs, }, }, } } func testVolume(name, size string) *v1.PersistentVolume { + fs := v1.PersistentVolumeFilesystem return &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -924,6 +960,7 @@ func testVolume(name, size string) *v1.PersistentVolume { Capacity: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse(size)}, PersistentVolumeSource: v1.PersistentVolumeSource{HostPath: &v1.HostPathVolumeSource{}}, AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -1001,34 +1038,36 @@ func createVolumeModeFilesystemTestVolume() *v1.PersistentVolume { } } +func createVolumeModeNilTestVolume() *v1.PersistentVolume { + return &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + UID: "local-1", + Name: "nil-mode", + }, + Spec: v1.PersistentVolumeSpec{ + Capacity: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): resource.MustParse("10G"), + }, + PersistentVolumeSource: v1.PersistentVolumeSource{ + Local: &v1.LocalVolumeSource{}, + }, + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + } +} + func createTestVolOrderedIndex(pv *v1.PersistentVolume) persistentVolumeOrderedIndex { volFile := newPersistentVolumeOrderedIndex() volFile.store.Add(pv) return volFile } -func toggleFeature(toggleFlag bool, featureName string, t *testing.T) { - var valueStr string - if toggleFlag { - // Enable feature - valueStr = featureName + "=true" - err := utilfeature.DefaultFeatureGate.Set(valueStr) - if err != nil { - t.Errorf("Failed to enable feature gate for %s: %v", featureName, err) - return - } - } else { - // Disable feature - valueStr = featureName + "=false" - err := utilfeature.DefaultFeatureGate.Set(valueStr) - if err != nil { - t.Errorf("Failed to disable feature gate for %s: %v", featureName, err) - return - } - } -} - -func TestAlphaVolumeModeCheck(t *testing.T) { +func TestVolumeModeCheck(t *testing.T) { blockMode := v1.PersistentVolumeBlock filesystemMode := v1.PersistentVolumeFilesystem @@ -1036,55 +1075,85 @@ func TestAlphaVolumeModeCheck(t *testing.T) { // If feature gate is enabled, VolumeMode will always be defaulted // If feature gate is disabled, VolumeMode is dropped by API and ignored scenarios := map[string]struct { - isExpectedMisMatch bool + isExpectedMismatch bool vol *v1.PersistentVolume pvc *v1.PersistentVolumeClaim enableBlock bool }{ "feature enabled - pvc block and pv filesystem": { - isExpectedMisMatch: true, + isExpectedMismatch: true, vol: createVolumeModeFilesystemTestVolume(), pvc: makeVolumeModePVC("8G", &blockMode, nil), enableBlock: true, }, "feature enabled - pvc filesystem and pv block": { - isExpectedMisMatch: true, + isExpectedMismatch: true, vol: createVolumeModeBlockTestVolume(), pvc: makeVolumeModePVC("8G", &filesystemMode, nil), enableBlock: true, }, "feature enabled - pvc block and pv block": { - isExpectedMisMatch: false, + isExpectedMismatch: false, vol: createVolumeModeBlockTestVolume(), pvc: makeVolumeModePVC("8G", &blockMode, nil), enableBlock: true, }, "feature enabled - pvc filesystem and pv filesystem": { - isExpectedMisMatch: false, + isExpectedMismatch: false, vol: createVolumeModeFilesystemTestVolume(), pvc: makeVolumeModePVC("8G", &filesystemMode, nil), enableBlock: true, }, + "feature enabled - pvc filesystem and pv nil": { + isExpectedMismatch: false, + vol: createVolumeModeNilTestVolume(), + pvc: makeVolumeModePVC("8G", &filesystemMode, nil), + enableBlock: true, + }, + "feature enabled - pvc nil and pv filesytem": { + isExpectedMismatch: false, + vol: createVolumeModeFilesystemTestVolume(), + pvc: makeVolumeModePVC("8G", nil, nil), + enableBlock: true, + }, + "feature enabled - pvc nil and pv nil": { + isExpectedMismatch: false, + vol: createVolumeModeNilTestVolume(), + pvc: makeVolumeModePVC("8G", nil, nil), + enableBlock: true, + }, + "feature enabled - pvc nil and pv block": { + isExpectedMismatch: true, + vol: createVolumeModeBlockTestVolume(), + pvc: makeVolumeModePVC("8G", nil, nil), + enableBlock: true, + }, + "feature enabled - pvc block and pv nil": { + isExpectedMismatch: true, + vol: createVolumeModeNilTestVolume(), + pvc: makeVolumeModePVC("8G", &blockMode, nil), + enableBlock: true, + }, "feature disabled - pvc block and pv filesystem": { - isExpectedMisMatch: false, + isExpectedMismatch: false, vol: createVolumeModeFilesystemTestVolume(), pvc: makeVolumeModePVC("8G", &blockMode, nil), enableBlock: false, }, "feature disabled - pvc filesystem and pv block": { - isExpectedMisMatch: false, + isExpectedMismatch: false, vol: createVolumeModeBlockTestVolume(), pvc: makeVolumeModePVC("8G", &filesystemMode, nil), enableBlock: false, }, "feature disabled - pvc block and pv block": { - isExpectedMisMatch: false, + isExpectedMismatch: false, vol: createVolumeModeBlockTestVolume(), pvc: makeVolumeModePVC("8G", &blockMode, nil), enableBlock: false, }, "feature disabled - pvc filesystem and pv filesystem": { - isExpectedMisMatch: false, + isExpectedMismatch: false, vol: createVolumeModeFilesystemTestVolume(), pvc: makeVolumeModePVC("8G", &filesystemMode, nil), enableBlock: false, @@ -1092,25 +1161,23 @@ func TestAlphaVolumeModeCheck(t *testing.T) { } for name, scenario := range scenarios { - toggleFeature(scenario.enableBlock, "BlockVolume", t) - expectedMisMatch, err := checkVolumeModeMisMatches(&scenario.pvc.Spec, &scenario.vol.Spec) + recover := utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, scenario.enableBlock) + expectedMismatch, err := checkVolumeModeMismatches(&scenario.pvc.Spec, &scenario.vol.Spec) if err != nil { - t.Errorf("Unexpected failure for checkVolumeModeMisMatches: %v", err) + t.Errorf("Unexpected failure for checkVolumeModeMismatches: %v", err) } // expected to match but either got an error or no returned pvmatch - if expectedMisMatch && !scenario.isExpectedMisMatch { + if expectedMismatch && !scenario.isExpectedMismatch { t.Errorf("Unexpected failure for scenario, expected not to mismatch on modes but did: %s", name) } - if !expectedMisMatch && scenario.isExpectedMisMatch { + if !expectedMismatch && scenario.isExpectedMismatch { t.Errorf("Unexpected failure for scenario, did not mismatch on mode when expected to mismatch: %s", name) } + recover() } - - // make sure feature gate is turned off - toggleFeature(false, "BlockVolume", t) } -func TestAlphaFilteringVolumeModes(t *testing.T) { +func TestFilteringVolumeModes(t *testing.T) { blockMode := v1.PersistentVolumeBlock filesystemMode := v1.PersistentVolumeFilesystem @@ -1185,7 +1252,7 @@ func TestAlphaFilteringVolumeModes(t *testing.T) { } for name, scenario := range scenarios { - toggleFeature(scenario.enableBlock, "BlockVolume", t) + recover := utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, scenario.enableBlock) pvmatch, err := scenario.vol.findBestMatchForClaim(scenario.pvc, false) // expected to match but either got an error or no returned pvmatch if pvmatch == nil && scenario.isExpectedMatch { @@ -1201,13 +1268,12 @@ func TestAlphaFilteringVolumeModes(t *testing.T) { if err != nil && !scenario.isExpectedMatch { t.Errorf("Unexpected failure for scenario: %s - %+v", name, err) } + recover() } - - // make sure feature gate is turned off - toggleFeature(false, "BlockVolume", t) } -func TestAlphaStorageObjectInUseProtectionFiltering(t *testing.T) { +func TestStorageObjectInUseProtectionFiltering(t *testing.T) { + fs := v1.PersistentVolumeFilesystem pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pv1", @@ -1217,6 +1283,7 @@ func TestAlphaStorageObjectInUseProtectionFiltering(t *testing.T) { Capacity: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse("1G")}, PersistentVolumeSource: v1.PersistentVolumeSource{HostPath: &v1.HostPathVolumeSource{}}, AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -1235,6 +1302,7 @@ func TestAlphaStorageObjectInUseProtectionFiltering(t *testing.T) { Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse("1G")}}, + VolumeMode: &fs, }, } @@ -1271,17 +1339,19 @@ func TestAlphaStorageObjectInUseProtectionFiltering(t *testing.T) { } for name, testCase := range satisfyingTestCases { - toggleFeature(testCase.enableStorageObjectInUseProtection, "StorageObjectInUseProtection", t) - err := checkVolumeSatisfyClaim(testCase.vol, testCase.pvc) - // expected to match but got an error - if err != nil && testCase.isExpectedMatch { - t.Errorf("%s: expected to match but got an error: %v", name, err) - } - // not expected to match but did - if err == nil && !testCase.isExpectedMatch { - t.Errorf("%s: not expected to match but did", name) - } + t.Run(name, func(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageObjectInUseProtection, testCase.enableStorageObjectInUseProtection)() + err := checkVolumeSatisfyClaim(testCase.vol, testCase.pvc) + // expected to match but got an error + if err != nil && testCase.isExpectedMatch { + t.Errorf("%s: expected to match but got an error: %v", name, err) + } + // not expected to match but did + if err == nil && !testCase.isExpectedMatch { + t.Errorf("%s: not expected to match but did", name) + } + }) } filteringTestCases := map[string]struct { @@ -1316,29 +1386,30 @@ func TestAlphaStorageObjectInUseProtectionFiltering(t *testing.T) { }, } for name, testCase := range filteringTestCases { - toggleFeature(testCase.enableStorageObjectInUseProtection, "StorageObjectInUseProtection", t) - pvmatch, err := testCase.vol.findBestMatchForClaim(testCase.pvc, false) - // expected to match but either got an error or no returned pvmatch - if pvmatch == nil && testCase.isExpectedMatch { - t.Errorf("Unexpected failure for testcase, no matching volume: %s", name) - } - if err != nil && testCase.isExpectedMatch { - t.Errorf("Unexpected failure for testcase: %s - %+v", name, err) - } - // expected to not match but either got an error or a returned pvmatch - if pvmatch != nil && !testCase.isExpectedMatch { - t.Errorf("Unexpected failure for testcase, expected no matching volume: %s", name) - } - if err != nil && !testCase.isExpectedMatch { - t.Errorf("Unexpected failure for testcase: %s - %+v", name, err) - } - } + t.Run(name, func(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageObjectInUseProtection, testCase.enableStorageObjectInUseProtection)() - // make sure feature gate is turned off - toggleFeature(false, "StorageObjectInUseProtection", t) + pvmatch, err := testCase.vol.findBestMatchForClaim(testCase.pvc, false) + // expected to match but either got an error or no returned pvmatch + if pvmatch == nil && testCase.isExpectedMatch { + t.Errorf("Unexpected failure for testcase, no matching volume: %s", name) + } + if err != nil && testCase.isExpectedMatch { + t.Errorf("Unexpected failure for testcase: %s - %+v", name, err) + } + // expected to not match but either got an error or a returned pvmatch + if pvmatch != nil && !testCase.isExpectedMatch { + t.Errorf("Unexpected failure for testcase, expected no matching volume: %s", name) + } + if err != nil && !testCase.isExpectedMatch { + t.Errorf("Unexpected failure for testcase: %s - %+v", name, err) + } + }) + } } func TestFindingPreboundVolumes(t *testing.T) { + fs := v1.PersistentVolumeFilesystem claim := &v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "claim01", @@ -1348,6 +1419,7 @@ func TestFindingPreboundVolumes(t *testing.T) { Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi")}}, + VolumeMode: &fs, }, } claimRef, err := ref.GetReference(scheme.Scheme, claim) diff --git a/pkg/controller/volume/persistentvolume/main_test.go b/pkg/controller/volume/persistentvolume/main_test.go new file mode 100644 index 00000000000..490540aa60b --- /dev/null +++ b/pkg/controller/volume/persistentvolume/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package persistentvolume + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/controller/volume/persistentvolume/metrics/BUILD b/pkg/controller/volume/persistentvolume/metrics/BUILD index bcf22a490ac..d686e4df42f 100644 --- a/pkg/controller/volume/persistentvolume/metrics/BUILD +++ b/pkg/controller/volume/persistentvolume/metrics/BUILD @@ -7,8 +7,8 @@ go_library( visibility = ["//visibility:public"], deps = [ "//staging/src/k8s.io/api/core/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/persistentvolume/metrics/metrics.go b/pkg/controller/volume/persistentvolume/metrics/metrics.go index 2c26935b3a4..1184378777c 100644 --- a/pkg/controller/volume/persistentvolume/metrics/metrics.go +++ b/pkg/controller/volume/persistentvolume/metrics/metrics.go @@ -21,8 +21,8 @@ import ( "k8s.io/api/core/v1" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" + "k8s.io/klog" ) const ( @@ -139,7 +139,7 @@ func (collector *pvAndPVCCountCollector) pvCollect(ch chan<- prometheus.Metric) float64(number), storageClassName) if err != nil { - glog.Warningf("Create bound pv number metric failed: %v", err) + klog.Warningf("Create bound pv number metric failed: %v", err) continue } ch <- metric @@ -151,7 +151,7 @@ func (collector *pvAndPVCCountCollector) pvCollect(ch chan<- prometheus.Metric) float64(number), storageClassName) if err != nil { - glog.Warningf("Create unbound pv number metric failed: %v", err) + klog.Warningf("Create unbound pv number metric failed: %v", err) continue } ch <- metric @@ -179,7 +179,7 @@ func (collector *pvAndPVCCountCollector) pvcCollect(ch chan<- prometheus.Metric) float64(number), namespace) if err != nil { - glog.Warningf("Create bound pvc number metric failed: %v", err) + klog.Warningf("Create bound pvc number metric failed: %v", err) continue } ch <- metric @@ -191,7 +191,7 @@ func (collector *pvAndPVCCountCollector) pvcCollect(ch chan<- prometheus.Metric) float64(number), namespace) if err != nil { - glog.Warningf("Create unbound pvc number metric failed: %v", err) + klog.Warningf("Create unbound pvc number metric failed: %v", err) continue } ch <- metric diff --git a/pkg/controller/volume/persistentvolume/provision_test.go b/pkg/controller/volume/persistentvolume/provision_test.go index 0337996e464..879e72dd500 100644 --- a/pkg/controller/volume/persistentvolume/provision_test.go +++ b/pkg/controller/volume/persistentvolume/provision_test.go @@ -34,6 +34,7 @@ var class2Parameters = map[string]string{ "param2": "value2", } var deleteReclaimPolicy = v1.PersistentVolumeReclaimDelete +var modeImmediate = storage.VolumeBindingImmediate var storageClasses = []*storage.StorageClass{ { TypeMeta: metav1.TypeMeta{ @@ -44,9 +45,10 @@ var storageClasses = []*storage.StorageClass{ Name: "gold", }, - Provisioner: mockPluginName, - Parameters: class1Parameters, - ReclaimPolicy: &deleteReclaimPolicy, + Provisioner: mockPluginName, + Parameters: class1Parameters, + ReclaimPolicy: &deleteReclaimPolicy, + VolumeBindingMode: &modeImmediate, }, { TypeMeta: metav1.TypeMeta{ @@ -55,9 +57,10 @@ var storageClasses = []*storage.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "silver", }, - Provisioner: mockPluginName, - Parameters: class2Parameters, - ReclaimPolicy: &deleteReclaimPolicy, + Provisioner: mockPluginName, + Parameters: class2Parameters, + ReclaimPolicy: &deleteReclaimPolicy, + VolumeBindingMode: &modeImmediate, }, { TypeMeta: metav1.TypeMeta{ @@ -66,9 +69,10 @@ var storageClasses = []*storage.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "external", }, - Provisioner: "vendor.com/my-volume", - Parameters: class1Parameters, - ReclaimPolicy: &deleteReclaimPolicy, + Provisioner: "vendor.com/my-volume", + Parameters: class1Parameters, + ReclaimPolicy: &deleteReclaimPolicy, + VolumeBindingMode: &modeImmediate, }, { TypeMeta: metav1.TypeMeta{ @@ -77,9 +81,10 @@ var storageClasses = []*storage.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "unknown-internal", }, - Provisioner: "kubernetes.io/unknown", - Parameters: class1Parameters, - ReclaimPolicy: &deleteReclaimPolicy, + Provisioner: "kubernetes.io/unknown", + Parameters: class1Parameters, + ReclaimPolicy: &deleteReclaimPolicy, + VolumeBindingMode: &modeImmediate, }, { TypeMeta: metav1.TypeMeta{ @@ -88,10 +93,11 @@ var storageClasses = []*storage.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "unsupported-mountoptions", }, - Provisioner: mockPluginName, - Parameters: class1Parameters, - ReclaimPolicy: &deleteReclaimPolicy, - MountOptions: []string{"foo"}, + Provisioner: mockPluginName, + Parameters: class1Parameters, + ReclaimPolicy: &deleteReclaimPolicy, + MountOptions: []string{"foo"}, + VolumeBindingMode: &modeImmediate, }, } diff --git a/pkg/controller/volume/persistentvolume/pv_controller.go b/pkg/controller/volume/persistentvolume/pv_controller.go index e759c457e25..e50da0ad4fa 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller.go +++ b/pkg/controller/volume/persistentvolume/pv_controller.go @@ -48,7 +48,7 @@ import ( "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/recyclerclient" - "github.com/golang/glog" + "k8s.io/klog" ) // ================================================================== @@ -238,7 +238,7 @@ type PersistentVolumeController struct { // For easier readability, it was split into syncUnboundClaim and syncBoundClaim // methods. func (ctrl *PersistentVolumeController) syncClaim(claim *v1.PersistentVolumeClaim) error { - glog.V(4).Infof("synchronizing PersistentVolumeClaim[%s]: %s", claimToClaimKey(claim), getClaimStatusForLogging(claim)) + klog.V(4).Infof("synchronizing PersistentVolumeClaim[%s]: %s", claimToClaimKey(claim), getClaimStatusForLogging(claim)) if !metav1.HasAnnotation(claim.ObjectMeta, annBindCompleted) { return ctrl.syncUnboundClaim(claim) @@ -270,11 +270,11 @@ func checkVolumeSatisfyClaim(volume *v1.PersistentVolume, claim *v1.PersistentVo return fmt.Errorf("storageClassName does not match") } - isMisMatch, err := checkVolumeModeMisMatches(&claim.Spec, &volume.Spec) + isMismatch, err := checkVolumeModeMismatches(&claim.Spec, &volume.Spec) if err != nil { return fmt.Errorf("error checking volumeMode: %v", err) } - if isMisMatch { + if isMismatch { return fmt.Errorf("incompatible volumeMode") } @@ -330,11 +330,11 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *v1.PersistentVol // [Unit test set 1] volume, err := ctrl.volumes.findBestMatchForClaim(claim, delayBinding) if err != nil { - glog.V(2).Infof("synchronizing unbound PersistentVolumeClaim[%s]: Error finding PV for claim: %v", claimToClaimKey(claim), err) + klog.V(2).Infof("synchronizing unbound PersistentVolumeClaim[%s]: Error finding PV for claim: %v", claimToClaimKey(claim), err) return fmt.Errorf("Error finding PV for claim %q: %v", claimToClaimKey(claim), err) } if volume == nil { - glog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: no volume found", claimToClaimKey(claim)) + klog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: no volume found", claimToClaimKey(claim)) // No PV could be found // OBSERVATION: pvc is "Pending", will retry switch { @@ -358,7 +358,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *v1.PersistentVol } else /* pv != nil */ { // Found a PV for this claim // OBSERVATION: pvc is "Pending", pv is "Available" - glog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume %q found: %s", claimToClaimKey(claim), volume.Name, getVolumeStatusForLogging(volume)) + klog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume %q found: %s", claimToClaimKey(claim), volume.Name, getVolumeStatusForLogging(volume)) if err = ctrl.bind(volume, claim); err != nil { // On any error saving the volume or the claim, subsequent // syncClaim will finish the binding. @@ -370,7 +370,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *v1.PersistentVol } else /* pvc.Spec.VolumeName != nil */ { // [Unit test set 2] // User asked for a specific PV. - glog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume %q requested", claimToClaimKey(claim), claim.Spec.VolumeName) + klog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume %q requested", claimToClaimKey(claim), claim.Spec.VolumeName) obj, found, err := ctrl.volumes.store.GetByKey(claim.Spec.VolumeName) if err != nil { return err @@ -379,7 +379,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *v1.PersistentVol // User asked for a PV that does not exist. // OBSERVATION: pvc is "Pending" // Retry later. - glog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume %q requested and not found, will try again next time", claimToClaimKey(claim), claim.Spec.VolumeName) + klog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume %q requested and not found, will try again next time", claimToClaimKey(claim), claim.Spec.VolumeName) if _, err = ctrl.updateClaimStatus(claim, v1.ClaimPending, nil); err != nil { return err } @@ -389,13 +389,13 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *v1.PersistentVol if !ok { return fmt.Errorf("Cannot convert object from volume cache to volume %q!?: %+v", claim.Spec.VolumeName, obj) } - glog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume %q requested and found: %s", claimToClaimKey(claim), claim.Spec.VolumeName, getVolumeStatusForLogging(volume)) + klog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume %q requested and found: %s", claimToClaimKey(claim), claim.Spec.VolumeName, getVolumeStatusForLogging(volume)) if volume.Spec.ClaimRef == nil { // User asked for a PV that is not claimed // OBSERVATION: pvc is "Pending", pv is "Available" - glog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume is unbound, binding", claimToClaimKey(claim)) + klog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume is unbound, binding", claimToClaimKey(claim)) if err = checkVolumeSatisfyClaim(volume, claim); err != nil { - glog.V(4).Infof("Can't bind the claim to volume %q: %v", volume.Name, err) + klog.V(4).Infof("Can't bind the claim to volume %q: %v", volume.Name, err) //send an event msg := fmt.Sprintf("Cannot bind to requested volume %q: %s", volume.Name, err) ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.VolumeMismatch, msg) @@ -413,7 +413,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *v1.PersistentVol } else if isVolumeBoundToClaim(volume, claim) { // User asked for a PV that is claimed by this PVC // OBSERVATION: pvc is "Pending", pv is "Bound" - glog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume already bound, finishing the binding", claimToClaimKey(claim)) + klog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume already bound, finishing the binding", claimToClaimKey(claim)) // Finish the volume binding by adding claim UID. if err = ctrl.bind(volume, claim); err != nil { @@ -425,7 +425,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *v1.PersistentVol // User asked for a PV that is claimed by someone else // OBSERVATION: pvc is "Pending", pv is "Bound" if !metav1.HasAnnotation(claim.ObjectMeta, annBoundByController) { - glog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume already bound to different claim by user, will retry later", claimToClaimKey(claim)) + klog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume already bound to different claim by user, will retry later", claimToClaimKey(claim)) // User asked for a specific PV, retry later if _, err = ctrl.updateClaimStatus(claim, v1.ClaimPending, nil); err != nil { return err @@ -434,7 +434,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *v1.PersistentVol } else { // This should never happen because someone had to remove // annBindCompleted annotation on the claim. - glog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume already bound to different claim %q by controller, THIS SHOULD NEVER HAPPEN", claimToClaimKey(claim), claimrefToClaimKey(volume.Spec.ClaimRef)) + klog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume already bound to different claim %q by controller, THIS SHOULD NEVER HAPPEN", claimToClaimKey(claim), claimrefToClaimKey(volume.Spec.ClaimRef)) return fmt.Errorf("Invalid binding of claim %q to volume %q: volume already claimed by %q", claimToClaimKey(claim), claim.Spec.VolumeName, claimrefToClaimKey(volume.Spec.ClaimRef)) } } @@ -472,13 +472,13 @@ func (ctrl *PersistentVolumeController) syncBoundClaim(claim *v1.PersistentVolum return fmt.Errorf("Cannot convert object from volume cache to volume %q!?: %#v", claim.Spec.VolumeName, obj) } - glog.V(4).Infof("synchronizing bound PersistentVolumeClaim[%s]: volume %q found: %s", claimToClaimKey(claim), claim.Spec.VolumeName, getVolumeStatusForLogging(volume)) + klog.V(4).Infof("synchronizing bound PersistentVolumeClaim[%s]: volume %q found: %s", claimToClaimKey(claim), claim.Spec.VolumeName, getVolumeStatusForLogging(volume)) if volume.Spec.ClaimRef == nil { // Claim is bound but volume has come unbound. // Or, a claim was bound and the controller has not received updated // volume yet. We can't distinguish these cases. // Bind the volume again and set all states to Bound. - glog.V(4).Infof("synchronizing bound PersistentVolumeClaim[%s]: volume is unbound, fixing", claimToClaimKey(claim)) + klog.V(4).Infof("synchronizing bound PersistentVolumeClaim[%s]: volume is unbound, fixing", claimToClaimKey(claim)) if err = ctrl.bind(volume, claim); err != nil { // Objects not saved, next syncPV or syncClaim will try again return err @@ -489,7 +489,7 @@ func (ctrl *PersistentVolumeController) syncBoundClaim(claim *v1.PersistentVolum // NOTE: syncPV can handle this so it can be left out. // NOTE: bind() call here will do nothing in most cases as // everything should be already set. - glog.V(4).Infof("synchronizing bound PersistentVolumeClaim[%s]: claim is already correctly bound", claimToClaimKey(claim)) + klog.V(4).Infof("synchronizing bound PersistentVolumeClaim[%s]: claim is already correctly bound", claimToClaimKey(claim)) if err = ctrl.bind(volume, claim); err != nil { // Objects not saved, next syncPV or syncClaim will try again return err @@ -512,12 +512,12 @@ func (ctrl *PersistentVolumeController) syncBoundClaim(claim *v1.PersistentVolum // created, updated or periodically synced. We do not differentiate between // these events. func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume) error { - glog.V(4).Infof("synchronizing PersistentVolume[%s]: %s", volume.Name, getVolumeStatusForLogging(volume)) + klog.V(4).Infof("synchronizing PersistentVolume[%s]: %s", volume.Name, getVolumeStatusForLogging(volume)) // [Unit test set 4] if volume.Spec.ClaimRef == nil { // Volume is unused - glog.V(4).Infof("synchronizing PersistentVolume[%s]: volume is unused", volume.Name) + klog.V(4).Infof("synchronizing PersistentVolume[%s]: volume is unused", volume.Name) if _, err := ctrl.updateVolumePhase(volume, v1.VolumeAvailable, ""); err != nil { // Nothing was saved; we will fall back into the same // condition in the next call to this method @@ -529,7 +529,7 @@ func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume) if volume.Spec.ClaimRef.UID == "" { // The PV is reserved for a PVC; that PVC has not yet been // bound to this PV; the PVC sync will handle it. - glog.V(4).Infof("synchronizing PersistentVolume[%s]: volume is pre-bound to claim %s", volume.Name, claimrefToClaimKey(volume.Spec.ClaimRef)) + klog.V(4).Infof("synchronizing PersistentVolume[%s]: volume is pre-bound to claim %s", volume.Name, claimrefToClaimKey(volume.Spec.ClaimRef)) if _, err := ctrl.updateVolumePhase(volume, v1.VolumeAvailable, ""); err != nil { // Nothing was saved; we will fall back into the same // condition in the next call to this method @@ -537,7 +537,7 @@ func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume) } return nil } - glog.V(4).Infof("synchronizing PersistentVolume[%s]: volume is bound to claim %s", volume.Name, claimrefToClaimKey(volume.Spec.ClaimRef)) + klog.V(4).Infof("synchronizing PersistentVolume[%s]: volume is bound to claim %s", volume.Name, claimrefToClaimKey(volume.Spec.ClaimRef)) // Get the PVC by _name_ var claim *v1.PersistentVolumeClaim claimName := claimrefToClaimKey(volume.Spec.ClaimRef) @@ -570,7 +570,7 @@ func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume) } } if !found { - glog.V(4).Infof("synchronizing PersistentVolume[%s]: claim %s not found", volume.Name, claimrefToClaimKey(volume.Spec.ClaimRef)) + klog.V(4).Infof("synchronizing PersistentVolume[%s]: claim %s not found", volume.Name, claimrefToClaimKey(volume.Spec.ClaimRef)) // Fall through with claim = nil } else { var ok bool @@ -578,12 +578,12 @@ func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume) if !ok { return fmt.Errorf("Cannot convert object from volume cache to volume %q!?: %#v", claim.Spec.VolumeName, obj) } - glog.V(4).Infof("synchronizing PersistentVolume[%s]: claim %s found: %s", volume.Name, claimrefToClaimKey(volume.Spec.ClaimRef), getClaimStatusForLogging(claim)) + klog.V(4).Infof("synchronizing PersistentVolume[%s]: claim %s found: %s", volume.Name, claimrefToClaimKey(volume.Spec.ClaimRef), getClaimStatusForLogging(claim)) } if claim != nil && claim.UID != volume.Spec.ClaimRef.UID { // The claim that the PV was pointing to was deleted, and another // with the same name created. - glog.V(4).Infof("synchronizing PersistentVolume[%s]: claim %s has different UID, the old one must have been deleted", volume.Name, claimrefToClaimKey(volume.Spec.ClaimRef)) + klog.V(4).Infof("synchronizing PersistentVolume[%s]: claim %s has different UID, the old one must have been deleted", volume.Name, claimrefToClaimKey(volume.Spec.ClaimRef)) // Treat the volume as bound to a missing claim. claim = nil } @@ -598,7 +598,7 @@ func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume) // volume. if volume.Status.Phase != v1.VolumeReleased && volume.Status.Phase != v1.VolumeFailed { // Also, log this only once: - glog.V(2).Infof("volume %q is released and reclaim policy %q will be executed", volume.Name, volume.Spec.PersistentVolumeReclaimPolicy) + klog.V(2).Infof("volume %q is released and reclaim policy %q will be executed", volume.Name, volume.Spec.PersistentVolumeReclaimPolicy) if volume, err = ctrl.updateVolumePhase(volume, v1.VolumeReleased, ""); err != nil { // Nothing was saved; we will fall back into the same condition // in the next call to this method @@ -613,7 +613,7 @@ func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume) } return nil } else if claim.Spec.VolumeName == "" { - if isMisMatch, err := checkVolumeModeMisMatches(&claim.Spec, &volume.Spec); err != nil || isMisMatch { + if isMismatch, err := checkVolumeModeMismatches(&claim.Spec, &volume.Spec); err != nil || isMismatch { // Binding for the volume won't be called in syncUnboundClaim, // because findBestMatchForClaim won't return the volume due to volumeMode mismatch. volumeMsg := fmt.Sprintf("Cannot bind PersistentVolume to requested PersistentVolumeClaim %q due to incompatible volumeMode.", claim.Name) @@ -626,10 +626,10 @@ func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume) if metav1.HasAnnotation(volume.ObjectMeta, annBoundByController) { // The binding is not completed; let PVC sync handle it - glog.V(4).Infof("synchronizing PersistentVolume[%s]: volume not bound yet, waiting for syncClaim to fix it", volume.Name) + klog.V(4).Infof("synchronizing PersistentVolume[%s]: volume not bound yet, waiting for syncClaim to fix it", volume.Name) } else { // Dangling PV; try to re-establish the link in the PVC sync - glog.V(4).Infof("synchronizing PersistentVolume[%s]: volume was bound and got unbound (by user?), waiting for syncClaim to fix it", volume.Name) + klog.V(4).Infof("synchronizing PersistentVolume[%s]: volume was bound and got unbound (by user?), waiting for syncClaim to fix it", volume.Name) } // In both cases, the volume is Bound and the claim is Pending. // Next syncClaim will fix it. To speed it up, we enqueue the claim @@ -642,7 +642,7 @@ func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume) return nil } else if claim.Spec.VolumeName == volume.Name { // Volume is bound to a claim properly, update status if necessary - glog.V(4).Infof("synchronizing PersistentVolume[%s]: all is bound", volume.Name) + klog.V(4).Infof("synchronizing PersistentVolume[%s]: all is bound", volume.Name) if _, err = ctrl.updateVolumePhase(volume, v1.VolumeBound, ""); err != nil { // Nothing was saved; we will fall back into the same // condition in the next call to this method @@ -659,7 +659,7 @@ func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume) // the user know. Don't overwrite existing Failed status! if volume.Status.Phase != v1.VolumeReleased && volume.Status.Phase != v1.VolumeFailed { // Also, log this only once: - glog.V(2).Infof("dynamically volume %q is released and it will be deleted", volume.Name) + klog.V(2).Infof("dynamically volume %q is released and it will be deleted", volume.Name) if volume, err = ctrl.updateVolumePhase(volume, v1.VolumeReleased, ""); err != nil { // Nothing was saved; we will fall back into the same condition // in the next call to this method @@ -679,14 +679,14 @@ func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume) // This is part of the normal operation of the controller; the // controller tried to use this volume for a claim but the claim // was fulfilled by another volume. We did this; fix it. - glog.V(4).Infof("synchronizing PersistentVolume[%s]: volume is bound by controller to a claim that is bound to another volume, unbinding", volume.Name) + klog.V(4).Infof("synchronizing PersistentVolume[%s]: volume is bound by controller to a claim that is bound to another volume, unbinding", volume.Name) if err = ctrl.unbindVolume(volume); err != nil { return err } return nil } else { // The PV must have been created with this ptr; leave it alone. - glog.V(4).Infof("synchronizing PersistentVolume[%s]: volume is bound by user to a claim that is bound to another volume, waiting for the claim to get unbound", volume.Name) + klog.V(4).Infof("synchronizing PersistentVolume[%s]: volume is bound by user to a claim that is bound to another volume, waiting for the claim to get unbound", volume.Name) // This just updates the volume phase and clears // volume.Spec.ClaimRef.UID. It leaves the volume pre-bound // to the claim. @@ -706,7 +706,7 @@ func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume) // phase - phase to set // volume - volume which Capacity is set into claim.Status.Capacity func (ctrl *PersistentVolumeController) updateClaimStatus(claim *v1.PersistentVolumeClaim, phase v1.PersistentVolumeClaimPhase, volume *v1.PersistentVolume) (*v1.PersistentVolumeClaim, error) { - glog.V(4).Infof("updating PersistentVolumeClaim[%s] status: set phase %s", claimToClaimKey(claim), phase) + klog.V(4).Infof("updating PersistentVolumeClaim[%s] status: set phase %s", claimToClaimKey(claim), phase) dirty := false @@ -751,21 +751,21 @@ func (ctrl *PersistentVolumeController) updateClaimStatus(claim *v1.PersistentVo if !dirty { // Nothing to do. - glog.V(4).Infof("updating PersistentVolumeClaim[%s] status: phase %s already set", claimToClaimKey(claim), phase) + klog.V(4).Infof("updating PersistentVolumeClaim[%s] status: phase %s already set", claimToClaimKey(claim), phase) return claim, nil } newClaim, err := ctrl.kubeClient.CoreV1().PersistentVolumeClaims(claimClone.Namespace).UpdateStatus(claimClone) if err != nil { - glog.V(4).Infof("updating PersistentVolumeClaim[%s] status: set phase %s failed: %v", claimToClaimKey(claim), phase, err) + klog.V(4).Infof("updating PersistentVolumeClaim[%s] status: set phase %s failed: %v", claimToClaimKey(claim), phase, err) return newClaim, err } _, err = ctrl.storeClaimUpdate(newClaim) if err != nil { - glog.V(4).Infof("updating PersistentVolumeClaim[%s] status: cannot update internal cache: %v", claimToClaimKey(claim), err) + klog.V(4).Infof("updating PersistentVolumeClaim[%s] status: cannot update internal cache: %v", claimToClaimKey(claim), err) return newClaim, err } - glog.V(2).Infof("claim %q entered phase %q", claimToClaimKey(claim), phase) + klog.V(2).Infof("claim %q entered phase %q", claimToClaimKey(claim), phase) return newClaim, nil } @@ -778,10 +778,10 @@ func (ctrl *PersistentVolumeController) updateClaimStatus(claim *v1.PersistentVo // volume - volume which Capacity is set into claim.Status.Capacity // eventtype, reason, message - event to send, see EventRecorder.Event() func (ctrl *PersistentVolumeController) updateClaimStatusWithEvent(claim *v1.PersistentVolumeClaim, phase v1.PersistentVolumeClaimPhase, volume *v1.PersistentVolume, eventtype, reason, message string) (*v1.PersistentVolumeClaim, error) { - glog.V(4).Infof("updating updateClaimStatusWithEvent[%s]: set phase %s", claimToClaimKey(claim), phase) + klog.V(4).Infof("updating updateClaimStatusWithEvent[%s]: set phase %s", claimToClaimKey(claim), phase) if claim.Status.Phase == phase { // Nothing to do. - glog.V(4).Infof("updating updateClaimStatusWithEvent[%s]: phase %s already set", claimToClaimKey(claim), phase) + klog.V(4).Infof("updating updateClaimStatusWithEvent[%s]: phase %s already set", claimToClaimKey(claim), phase) return claim, nil } @@ -792,7 +792,7 @@ func (ctrl *PersistentVolumeController) updateClaimStatusWithEvent(claim *v1.Per // Emit the event only when the status change happens, not every time // syncClaim is called. - glog.V(3).Infof("claim %q changed status to %q: %s", claimToClaimKey(claim), phase, message) + klog.V(3).Infof("claim %q changed status to %q: %s", claimToClaimKey(claim), phase, message) ctrl.eventRecorder.Event(newClaim, eventtype, reason, message) return newClaim, nil @@ -800,10 +800,10 @@ func (ctrl *PersistentVolumeController) updateClaimStatusWithEvent(claim *v1.Per // updateVolumePhase saves new volume phase to API server. func (ctrl *PersistentVolumeController) updateVolumePhase(volume *v1.PersistentVolume, phase v1.PersistentVolumePhase, message string) (*v1.PersistentVolume, error) { - glog.V(4).Infof("updating PersistentVolume[%s]: set phase %s", volume.Name, phase) + klog.V(4).Infof("updating PersistentVolume[%s]: set phase %s", volume.Name, phase) if volume.Status.Phase == phase { // Nothing to do. - glog.V(4).Infof("updating PersistentVolume[%s]: phase %s already set", volume.Name, phase) + klog.V(4).Infof("updating PersistentVolume[%s]: phase %s already set", volume.Name, phase) return volume, nil } @@ -813,15 +813,15 @@ func (ctrl *PersistentVolumeController) updateVolumePhase(volume *v1.PersistentV newVol, err := ctrl.kubeClient.CoreV1().PersistentVolumes().UpdateStatus(volumeClone) if err != nil { - glog.V(4).Infof("updating PersistentVolume[%s]: set phase %s failed: %v", volume.Name, phase, err) + klog.V(4).Infof("updating PersistentVolume[%s]: set phase %s failed: %v", volume.Name, phase, err) return newVol, err } _, err = ctrl.storeVolumeUpdate(newVol) if err != nil { - glog.V(4).Infof("updating PersistentVolume[%s]: cannot update internal cache: %v", volume.Name, err) + klog.V(4).Infof("updating PersistentVolume[%s]: cannot update internal cache: %v", volume.Name, err) return newVol, err } - glog.V(2).Infof("volume %q entered phase %q", volume.Name, phase) + klog.V(2).Infof("volume %q entered phase %q", volume.Name, phase) return newVol, err } @@ -829,10 +829,10 @@ func (ctrl *PersistentVolumeController) updateVolumePhase(volume *v1.PersistentV // given event on the volume. It saves the phase and emits the event only when // the phase has actually changed from the version saved in API server. func (ctrl *PersistentVolumeController) updateVolumePhaseWithEvent(volume *v1.PersistentVolume, phase v1.PersistentVolumePhase, eventtype, reason, message string) (*v1.PersistentVolume, error) { - glog.V(4).Infof("updating updateVolumePhaseWithEvent[%s]: set phase %s", volume.Name, phase) + klog.V(4).Infof("updating updateVolumePhaseWithEvent[%s]: set phase %s", volume.Name, phase) if volume.Status.Phase == phase { // Nothing to do. - glog.V(4).Infof("updating updateVolumePhaseWithEvent[%s]: phase %s already set", volume.Name, phase) + klog.V(4).Infof("updating updateVolumePhaseWithEvent[%s]: phase %s already set", volume.Name, phase) return volume, nil } @@ -843,7 +843,7 @@ func (ctrl *PersistentVolumeController) updateVolumePhaseWithEvent(volume *v1.Pe // Emit the event only when the status change happens, not every time // syncClaim is called. - glog.V(3).Infof("volume %q changed status to %q: %s", volume.Name, phase, message) + klog.V(3).Infof("volume %q changed status to %q: %s", volume.Name, phase, message) ctrl.eventRecorder.Event(newVol, eventtype, reason, message) return newVol, nil @@ -852,7 +852,7 @@ func (ctrl *PersistentVolumeController) updateVolumePhaseWithEvent(volume *v1.Pe // bindVolumeToClaim modifies given volume to be bound to a claim and saves it to // API server. The claim is not modified in this method! func (ctrl *PersistentVolumeController) bindVolumeToClaim(volume *v1.PersistentVolume, claim *v1.PersistentVolumeClaim) (*v1.PersistentVolume, error) { - glog.V(4).Infof("updating PersistentVolume[%s]: binding to %q", volume.Name, claimToClaimKey(claim)) + klog.V(4).Infof("updating PersistentVolume[%s]: binding to %q", volume.Name, claimToClaimKey(claim)) volumeClone, dirty, err := ctrl.getBindVolumeToClaim(volume, claim) if err != nil { @@ -864,27 +864,27 @@ func (ctrl *PersistentVolumeController) bindVolumeToClaim(volume *v1.PersistentV return ctrl.updateBindVolumeToClaim(volumeClone, claim, true) } - glog.V(4).Infof("updating PersistentVolume[%s]: already bound to %q", volume.Name, claimToClaimKey(claim)) + klog.V(4).Infof("updating PersistentVolume[%s]: already bound to %q", volume.Name, claimToClaimKey(claim)) return volume, nil } // bindVolumeToClaim modifies given volume to be bound to a claim and saves it to // API server. The claim is not modified in this method! func (ctrl *PersistentVolumeController) updateBindVolumeToClaim(volumeClone *v1.PersistentVolume, claim *v1.PersistentVolumeClaim, updateCache bool) (*v1.PersistentVolume, error) { - glog.V(2).Infof("claim %q bound to volume %q", claimToClaimKey(claim), volumeClone.Name) + klog.V(2).Infof("claim %q bound to volume %q", claimToClaimKey(claim), volumeClone.Name) newVol, err := ctrl.kubeClient.CoreV1().PersistentVolumes().Update(volumeClone) if err != nil { - glog.V(4).Infof("updating PersistentVolume[%s]: binding to %q failed: %v", volumeClone.Name, claimToClaimKey(claim), err) + klog.V(4).Infof("updating PersistentVolume[%s]: binding to %q failed: %v", volumeClone.Name, claimToClaimKey(claim), err) return newVol, err } if updateCache { _, err = ctrl.storeVolumeUpdate(newVol) if err != nil { - glog.V(4).Infof("updating PersistentVolume[%s]: cannot update internal cache: %v", volumeClone.Name, err) + klog.V(4).Infof("updating PersistentVolume[%s]: cannot update internal cache: %v", volumeClone.Name, err) return newVol, err } } - glog.V(4).Infof("updating PersistentVolume[%s]: bound to %q", newVol.Name, claimToClaimKey(claim)) + klog.V(4).Infof("updating PersistentVolume[%s]: bound to %q", newVol.Name, claimToClaimKey(claim)) return newVol, nil } @@ -928,7 +928,7 @@ func (ctrl *PersistentVolumeController) getBindVolumeToClaim(volume *v1.Persiste // bindClaimToVolume modifies the given claim to be bound to a volume and // saves it to API server. The volume is not modified in this method! func (ctrl *PersistentVolumeController) bindClaimToVolume(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) (*v1.PersistentVolumeClaim, error) { - glog.V(4).Infof("updating PersistentVolumeClaim[%s]: binding to %q", claimToClaimKey(claim), volume.Name) + klog.V(4).Infof("updating PersistentVolumeClaim[%s]: binding to %q", claimToClaimKey(claim), volume.Name) dirty := false @@ -960,22 +960,22 @@ func (ctrl *PersistentVolumeController) bindClaimToVolume(claim *v1.PersistentVo } if dirty { - glog.V(2).Infof("volume %q bound to claim %q", volume.Name, claimToClaimKey(claim)) + klog.V(2).Infof("volume %q bound to claim %q", volume.Name, claimToClaimKey(claim)) newClaim, err := ctrl.kubeClient.CoreV1().PersistentVolumeClaims(claim.Namespace).Update(claimClone) if err != nil { - glog.V(4).Infof("updating PersistentVolumeClaim[%s]: binding to %q failed: %v", claimToClaimKey(claim), volume.Name, err) + klog.V(4).Infof("updating PersistentVolumeClaim[%s]: binding to %q failed: %v", claimToClaimKey(claim), volume.Name, err) return newClaim, err } _, err = ctrl.storeClaimUpdate(newClaim) if err != nil { - glog.V(4).Infof("updating PersistentVolumeClaim[%s]: cannot update internal cache: %v", claimToClaimKey(claim), err) + klog.V(4).Infof("updating PersistentVolumeClaim[%s]: cannot update internal cache: %v", claimToClaimKey(claim), err) return newClaim, err } - glog.V(4).Infof("updating PersistentVolumeClaim[%s]: bound to %q", claimToClaimKey(claim), volume.Name) + klog.V(4).Infof("updating PersistentVolumeClaim[%s]: bound to %q", claimToClaimKey(claim), volume.Name) return newClaim, nil } - glog.V(4).Infof("updating PersistentVolumeClaim[%s]: already bound to %q", claimToClaimKey(claim), volume.Name) + klog.V(4).Infof("updating PersistentVolumeClaim[%s]: already bound to %q", claimToClaimKey(claim), volume.Name) return claim, nil } @@ -990,35 +990,35 @@ func (ctrl *PersistentVolumeController) bind(volume *v1.PersistentVolume, claim var updatedClaim *v1.PersistentVolumeClaim var updatedVolume *v1.PersistentVolume - glog.V(4).Infof("binding volume %q to claim %q", volume.Name, claimToClaimKey(claim)) + klog.V(4).Infof("binding volume %q to claim %q", volume.Name, claimToClaimKey(claim)) if updatedVolume, err = ctrl.bindVolumeToClaim(volume, claim); err != nil { - glog.V(3).Infof("error binding volume %q to claim %q: failed saving the volume: %v", volume.Name, claimToClaimKey(claim), err) + klog.V(3).Infof("error binding volume %q to claim %q: failed saving the volume: %v", volume.Name, claimToClaimKey(claim), err) return err } volume = updatedVolume if updatedVolume, err = ctrl.updateVolumePhase(volume, v1.VolumeBound, ""); err != nil { - glog.V(3).Infof("error binding volume %q to claim %q: failed saving the volume status: %v", volume.Name, claimToClaimKey(claim), err) + klog.V(3).Infof("error binding volume %q to claim %q: failed saving the volume status: %v", volume.Name, claimToClaimKey(claim), err) return err } volume = updatedVolume if updatedClaim, err = ctrl.bindClaimToVolume(claim, volume); err != nil { - glog.V(3).Infof("error binding volume %q to claim %q: failed saving the claim: %v", volume.Name, claimToClaimKey(claim), err) + klog.V(3).Infof("error binding volume %q to claim %q: failed saving the claim: %v", volume.Name, claimToClaimKey(claim), err) return err } claim = updatedClaim if updatedClaim, err = ctrl.updateClaimStatus(claim, v1.ClaimBound, volume); err != nil { - glog.V(3).Infof("error binding volume %q to claim %q: failed saving the claim status: %v", volume.Name, claimToClaimKey(claim), err) + klog.V(3).Infof("error binding volume %q to claim %q: failed saving the claim status: %v", volume.Name, claimToClaimKey(claim), err) return err } claim = updatedClaim - glog.V(4).Infof("volume %q bound to claim %q", volume.Name, claimToClaimKey(claim)) - glog.V(4).Infof("volume %q status after binding: %s", volume.Name, getVolumeStatusForLogging(volume)) - glog.V(4).Infof("claim %q status after binding: %s", claimToClaimKey(claim), getClaimStatusForLogging(claim)) + klog.V(4).Infof("volume %q bound to claim %q", volume.Name, claimToClaimKey(claim)) + klog.V(4).Infof("volume %q status after binding: %s", volume.Name, getVolumeStatusForLogging(volume)) + klog.V(4).Infof("claim %q status after binding: %s", claimToClaimKey(claim), getClaimStatusForLogging(claim)) return nil } @@ -1029,7 +1029,7 @@ func (ctrl *PersistentVolumeController) bind(volume *v1.PersistentVolume, claim // It returns on first error, it's up to the caller to implement some retry // mechanism. func (ctrl *PersistentVolumeController) unbindVolume(volume *v1.PersistentVolume) error { - glog.V(4).Infof("updating PersistentVolume[%s]: rolling back binding from %q", volume.Name, claimrefToClaimKey(volume.Spec.ClaimRef)) + klog.V(4).Infof("updating PersistentVolume[%s]: rolling back binding from %q", volume.Name, claimrefToClaimKey(volume.Spec.ClaimRef)) // Save the PV only when any modification is necessary. volumeClone := volume.DeepCopy() @@ -1050,15 +1050,15 @@ func (ctrl *PersistentVolumeController) unbindVolume(volume *v1.PersistentVolume newVol, err := ctrl.kubeClient.CoreV1().PersistentVolumes().Update(volumeClone) if err != nil { - glog.V(4).Infof("updating PersistentVolume[%s]: rollback failed: %v", volume.Name, err) + klog.V(4).Infof("updating PersistentVolume[%s]: rollback failed: %v", volume.Name, err) return err } _, err = ctrl.storeVolumeUpdate(newVol) if err != nil { - glog.V(4).Infof("updating PersistentVolume[%s]: cannot update internal cache: %v", volume.Name, err) + klog.V(4).Infof("updating PersistentVolume[%s]: cannot update internal cache: %v", volume.Name, err) return err } - glog.V(4).Infof("updating PersistentVolume[%s]: rolled back", newVol.Name) + klog.V(4).Infof("updating PersistentVolume[%s]: rolled back", newVol.Name) // Update the status _, err = ctrl.updateVolumePhase(newVol, v1.VolumeAvailable, "") @@ -1070,10 +1070,10 @@ func (ctrl *PersistentVolumeController) unbindVolume(volume *v1.PersistentVolume func (ctrl *PersistentVolumeController) reclaimVolume(volume *v1.PersistentVolume) error { switch volume.Spec.PersistentVolumeReclaimPolicy { case v1.PersistentVolumeReclaimRetain: - glog.V(4).Infof("reclaimVolume[%s]: policy is Retain, nothing to do", volume.Name) + klog.V(4).Infof("reclaimVolume[%s]: policy is Retain, nothing to do", volume.Name) case v1.PersistentVolumeReclaimRecycle: - glog.V(4).Infof("reclaimVolume[%s]: policy is Recycle", volume.Name) + klog.V(4).Infof("reclaimVolume[%s]: policy is Recycle", volume.Name) opName := fmt.Sprintf("recycle-%s[%s]", volume.Name, string(volume.UID)) ctrl.scheduleOperation(opName, func() error { ctrl.recycleVolumeOperation(volume) @@ -1081,7 +1081,7 @@ func (ctrl *PersistentVolumeController) reclaimVolume(volume *v1.PersistentVolum }) case v1.PersistentVolumeReclaimDelete: - glog.V(4).Infof("reclaimVolume[%s]: policy is Delete", volume.Name) + klog.V(4).Infof("reclaimVolume[%s]: policy is Delete", volume.Name) opName := fmt.Sprintf("delete-%s[%s]", volume.Name, string(volume.UID)) startTime := time.Now() ctrl.scheduleOperation(opName, func() error { @@ -1103,33 +1103,45 @@ func (ctrl *PersistentVolumeController) reclaimVolume(volume *v1.PersistentVolum // doRerecycleVolumeOperationcycleVolume recycles a volume. This method is // running in standalone goroutine and already has all necessary locks. func (ctrl *PersistentVolumeController) recycleVolumeOperation(volume *v1.PersistentVolume) { - glog.V(4).Infof("recycleVolumeOperation [%s] started", volume.Name) + klog.V(4).Infof("recycleVolumeOperation [%s] started", volume.Name) // This method may have been waiting for a volume lock for some time. // Previous recycleVolumeOperation might just have saved an updated version, // so read current volume state now. newVolume, err := ctrl.kubeClient.CoreV1().PersistentVolumes().Get(volume.Name, metav1.GetOptions{}) if err != nil { - glog.V(3).Infof("error reading persistent volume %q: %v", volume.Name, err) + klog.V(3).Infof("error reading persistent volume %q: %v", volume.Name, err) return } needsReclaim, err := ctrl.isVolumeReleased(newVolume) if err != nil { - glog.V(3).Infof("error reading claim for volume %q: %v", volume.Name, err) + klog.V(3).Infof("error reading claim for volume %q: %v", volume.Name, err) return } if !needsReclaim { - glog.V(3).Infof("volume %q no longer needs recycling, skipping", volume.Name) + klog.V(3).Infof("volume %q no longer needs recycling, skipping", volume.Name) return } pods, used, err := ctrl.isVolumeUsed(newVolume) if err != nil { - glog.V(3).Infof("can't recycle volume %q: %v", volume.Name, err) + klog.V(3).Infof("can't recycle volume %q: %v", volume.Name, err) return } - if used { + + // Verify the claim is in cache: if so, then it is a different PVC with the same name + // since the volume is known to be released at this moment. Ths new (cached) PVC must use + // a different PV -- we checked that the PV is unused in isVolumeReleased. + // So the old PV is safe to be recycled. + claimName := claimrefToClaimKey(volume.Spec.ClaimRef) + _, claimCached, err := ctrl.claims.GetByKey(claimName) + if err != nil { + klog.V(3).Infof("error getting the claim %s from cache", claimName) + return + } + + if used && !claimCached { msg := fmt.Sprintf("Volume is used by pods: %s", strings.Join(pods, ",")) - glog.V(3).Infof("can't recycle volume %q: %s", volume.Name, msg) + klog.V(3).Infof("can't recycle volume %q: %s", volume.Name, msg) ctrl.eventRecorder.Event(volume, v1.EventTypeNormal, events.VolumeFailedRecycle, msg) return } @@ -1144,7 +1156,7 @@ func (ctrl *PersistentVolumeController) recycleVolumeOperation(volume *v1.Persis if err != nil { // No recycler found. Emit an event and mark the volume Failed. if _, err = ctrl.updateVolumePhaseWithEvent(volume, v1.VolumeFailed, v1.EventTypeWarning, events.VolumeFailedRecycle, "No recycler plugin found for the volume!"); err != nil { - glog.V(4).Infof("recycleVolumeOperation [%s]: failed to mark volume as failed: %v", volume.Name, err) + klog.V(4).Infof("recycleVolumeOperation [%s]: failed to mark volume as failed: %v", volume.Name, err) // Save failed, retry on the next deletion attempt return } @@ -1160,7 +1172,7 @@ func (ctrl *PersistentVolumeController) recycleVolumeOperation(volume *v1.Persis // Recycler failed strerr := fmt.Sprintf("Recycle failed: %s", err) if _, err = ctrl.updateVolumePhaseWithEvent(volume, v1.VolumeFailed, v1.EventTypeWarning, events.VolumeFailedRecycle, strerr); err != nil { - glog.V(4).Infof("recycleVolumeOperation [%s]: failed to mark volume as failed: %v", volume.Name, err) + klog.V(4).Infof("recycleVolumeOperation [%s]: failed to mark volume as failed: %v", volume.Name, err) // Save failed, retry on the next deletion attempt return } @@ -1169,7 +1181,7 @@ func (ctrl *PersistentVolumeController) recycleVolumeOperation(volume *v1.Persis return } - glog.V(2).Infof("volume %q recycled", volume.Name) + klog.V(2).Infof("volume %q recycled", volume.Name) // Send an event ctrl.eventRecorder.Event(volume, v1.EventTypeNormal, events.VolumeRecycled, "Volume recycled") // Make the volume available again @@ -1178,7 +1190,7 @@ func (ctrl *PersistentVolumeController) recycleVolumeOperation(volume *v1.Persis // recycle the volume again on next update. We _could_ maintain a cache // of "recently recycled volumes" and avoid unnecessary recycling, this // is left out as future optimization. - glog.V(3).Infof("recycleVolumeOperation [%s]: failed to make recycled volume 'Available' (%v), we will recycle the volume again", volume.Name, err) + klog.V(3).Infof("recycleVolumeOperation [%s]: failed to make recycled volume 'Available' (%v), we will recycle the volume again", volume.Name, err) return } return @@ -1187,30 +1199,30 @@ func (ctrl *PersistentVolumeController) recycleVolumeOperation(volume *v1.Persis // deleteVolumeOperation deletes a volume. This method is running in standalone // goroutine and already has all necessary locks. func (ctrl *PersistentVolumeController) deleteVolumeOperation(volume *v1.PersistentVolume) (string, error) { - glog.V(4).Infof("deleteVolumeOperation [%s] started", volume.Name) + klog.V(4).Infof("deleteVolumeOperation [%s] started", volume.Name) // This method may have been waiting for a volume lock for some time. // Previous deleteVolumeOperation might just have saved an updated version, so // read current volume state now. newVolume, err := ctrl.kubeClient.CoreV1().PersistentVolumes().Get(volume.Name, metav1.GetOptions{}) if err != nil { - glog.V(3).Infof("error reading persistent volume %q: %v", volume.Name, err) + klog.V(3).Infof("error reading persistent volume %q: %v", volume.Name, err) return "", nil } needsReclaim, err := ctrl.isVolumeReleased(newVolume) if err != nil { - glog.V(3).Infof("error reading claim for volume %q: %v", volume.Name, err) + klog.V(3).Infof("error reading claim for volume %q: %v", volume.Name, err) return "", nil } if !needsReclaim { - glog.V(3).Infof("volume %q no longer needs deletion, skipping", volume.Name) + klog.V(3).Infof("volume %q no longer needs deletion, skipping", volume.Name) return "", nil } pluginName, deleted, err := ctrl.doDeleteVolume(volume) if err != nil { // Delete failed, update the volume and emit an event. - glog.V(3).Infof("deletion of volume %q failed: %v", volume.Name, err) + klog.V(3).Infof("deletion of volume %q failed: %v", volume.Name, err) if vol.IsDeletedVolumeInUse(err) { // The plugin needs more time, don't mark the volume as Failed // and send Normal event only @@ -1219,7 +1231,7 @@ func (ctrl *PersistentVolumeController) deleteVolumeOperation(volume *v1.Persist // The plugin failed, mark the volume as Failed and send Warning // event if _, err := ctrl.updateVolumePhaseWithEvent(volume, v1.VolumeFailed, v1.EventTypeWarning, events.VolumeFailedDelete, err.Error()); err != nil { - glog.V(4).Infof("deleteVolumeOperation [%s]: failed to mark volume as failed: %v", volume.Name, err) + klog.V(4).Infof("deleteVolumeOperation [%s]: failed to mark volume as failed: %v", volume.Name, err) // Save failed, retry on the next deletion attempt return pluginName, err } @@ -1234,14 +1246,14 @@ func (ctrl *PersistentVolumeController) deleteVolumeOperation(volume *v1.Persist return pluginName, nil } - glog.V(4).Infof("deleteVolumeOperation [%s]: success", volume.Name) + klog.V(4).Infof("deleteVolumeOperation [%s]: success", volume.Name) // Delete the volume if err = ctrl.kubeClient.CoreV1().PersistentVolumes().Delete(volume.Name, nil); err != nil { // Oops, could not delete the volume and therefore the controller will // try to delete the volume again on next update. We _could_ maintain a // cache of "recently deleted volumes" and avoid unnecessary deletion, // this is left out as future optimization. - glog.V(3).Infof("failed to delete volume %q from database: %v", volume.Name, err) + klog.V(3).Infof("failed to delete volume %q from database: %v", volume.Name, err) return pluginName, nil } return pluginName, nil @@ -1254,13 +1266,13 @@ func (ctrl *PersistentVolumeController) isVolumeReleased(volume *v1.PersistentVo // A volume needs reclaim if it has ClaimRef and appropriate claim does not // exist. if volume.Spec.ClaimRef == nil { - glog.V(4).Infof("isVolumeReleased[%s]: ClaimRef is nil", volume.Name) + klog.V(4).Infof("isVolumeReleased[%s]: ClaimRef is nil", volume.Name) return false, nil } if volume.Spec.ClaimRef.UID == "" { // This is a volume bound by user and the controller has not finished // binding to the real claim yet. - glog.V(4).Infof("isVolumeReleased[%s]: ClaimRef is not bound", volume.Name) + klog.V(4).Infof("isVolumeReleased[%s]: ClaimRef is not bound", volume.Name) return false, nil } @@ -1287,11 +1299,11 @@ func (ctrl *PersistentVolumeController) isVolumeReleased(volume *v1.PersistentVo return true, nil } - glog.V(4).Infof("isVolumeReleased[%s]: ClaimRef is still valid, volume is not released", volume.Name) + klog.V(4).Infof("isVolumeReleased[%s]: ClaimRef is still valid, volume is not released", volume.Name) return false, nil } - glog.V(2).Infof("isVolumeReleased[%s]: volume is released", volume.Name) + klog.V(2).Infof("isVolumeReleased[%s]: volume is released", volume.Name) return true, nil } @@ -1326,7 +1338,7 @@ func (ctrl *PersistentVolumeController) isVolumeUsed(pv *v1.PersistentVolume) ([ // 'false' when the volume cannot be deleted because of the deleter is external. No // error should be reported in this case. func (ctrl *PersistentVolumeController) doDeleteVolume(volume *v1.PersistentVolume) (string, bool, error) { - glog.V(4).Infof("doDeleteVolume [%s]", volume.Name) + klog.V(4).Infof("doDeleteVolume [%s]", volume.Name) var err error plugin, err := ctrl.findDeletablePlugin(volume) @@ -1335,13 +1347,13 @@ func (ctrl *PersistentVolumeController) doDeleteVolume(volume *v1.PersistentVolu } if plugin == nil { // External deleter is requested, do nothing - glog.V(3).Infof("external deleter for volume %q requested, ignoring", volume.Name) + klog.V(3).Infof("external deleter for volume %q requested, ignoring", volume.Name) return "", false, nil } // Plugin found pluginName := plugin.GetPluginName() - glog.V(5).Infof("found a deleter plugin %q for volume %q", pluginName, volume.Name) + klog.V(5).Infof("found a deleter plugin %q for volume %q", pluginName, volume.Name) spec := vol.NewSpecFromPersistentVolume(volume, false) deleter, err := plugin.NewDeleter(spec) if err != nil { @@ -1357,7 +1369,7 @@ func (ctrl *PersistentVolumeController) doDeleteVolume(volume *v1.PersistentVolu return pluginName, false, err } - glog.V(2).Infof("volume %q deleted", volume.Name) + klog.V(2).Infof("volume %q deleted", volume.Name) return pluginName, true, nil } @@ -1367,7 +1379,7 @@ func (ctrl *PersistentVolumeController) provisionClaim(claim *v1.PersistentVolum if !ctrl.enableDynamicProvisioning { return nil } - glog.V(4).Infof("provisionClaim[%s]: started", claimToClaimKey(claim)) + klog.V(4).Infof("provisionClaim[%s]: started", claimToClaimKey(claim)) opName := fmt.Sprintf("provision-%s[%s]", claimToClaimKey(claim), string(claim.UID)) startTime := time.Now() ctrl.scheduleOperation(opName, func() error { @@ -1383,12 +1395,12 @@ func (ctrl *PersistentVolumeController) provisionClaim(claim *v1.PersistentVolum // standalone goroutine and already has all necessary locks. func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.PersistentVolumeClaim) (string, error) { claimClass := v1helper.GetPersistentVolumeClaimClass(claim) - glog.V(4).Infof("provisionClaimOperation [%s] started, class: %q", claimToClaimKey(claim), claimClass) + klog.V(4).Infof("provisionClaimOperation [%s] started, class: %q", claimToClaimKey(claim), claimClass) plugin, storageClass, err := ctrl.findProvisionablePlugin(claim) if err != nil { ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.ProvisioningFailed, err.Error()) - glog.V(2).Infof("error finding provisioning plugin for claim %s: %v", claimToClaimKey(claim), err) + klog.V(2).Infof("error finding provisioning plugin for claim %s: %v", claimToClaimKey(claim), err) // The controller will retry provisioning the volume in every // syncVolume() call. return "", err @@ -1403,7 +1415,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis newClaim, err := ctrl.setClaimProvisioner(claim, storageClass) if err != nil { // Save failed, the controller will retry in the next sync - glog.V(2).Infof("error saving claim %s: %v", claimToClaimKey(claim), err) + klog.V(2).Infof("error saving claim %s: %v", claimToClaimKey(claim), err) return pluginName, err } claim = newClaim @@ -1414,7 +1426,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis // and wait for the external provisioner msg := fmt.Sprintf("waiting for a volume to be created, either by external provisioner %q or manually created by system administrator", storageClass.Provisioner) ctrl.eventRecorder.Event(claim, v1.EventTypeNormal, events.ExternalProvisioning, msg) - glog.V(3).Infof("provisioning claim %q: %s", claimToClaimKey(claim), msg) + klog.V(3).Infof("provisioning claim %q: %s", claimToClaimKey(claim), msg) return pluginName, nil } @@ -1428,7 +1440,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis volume, err := ctrl.kubeClient.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{}) if err == nil && volume != nil { // Volume has been already provisioned, nothing to do. - glog.V(4).Infof("provisionClaimOperation [%s]: volume already exists, skipping", claimToClaimKey(claim)) + klog.V(4).Infof("provisionClaimOperation [%s]: volume already exists, skipping", claimToClaimKey(claim)) return pluginName, err } @@ -1436,7 +1448,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis // provisioned) claimRef, err := ref.GetReference(scheme.Scheme, claim) if err != nil { - glog.V(3).Infof("unexpected error getting claim reference: %v", err) + klog.V(3).Infof("unexpected error getting claim reference: %v", err) return pluginName, err } @@ -1460,7 +1472,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis // of PV would be rejected by validation anyway if !plugin.SupportsMountOption() && len(options.MountOptions) > 0 { strerr := fmt.Sprintf("Mount options are not supported by the provisioner but StorageClass %q has mount options %v", storageClass.Name, options.MountOptions) - glog.V(2).Infof("Mount options are not supported by the provisioner but claim %q's StorageClass %q has mount options %v", claimToClaimKey(claim), storageClass.Name, options.MountOptions) + klog.V(2).Infof("Mount options are not supported by the provisioner but claim %q's StorageClass %q has mount options %v", claimToClaimKey(claim), storageClass.Name, options.MountOptions) ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.ProvisioningFailed, strerr) return pluginName, fmt.Errorf("provisioner %q doesn't support mount options", plugin.GetPluginName()) } @@ -1469,7 +1481,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis provisioner, err := plugin.NewProvisioner(options) if err != nil { strerr := fmt.Sprintf("Failed to create provisioner: %v", err) - glog.V(2).Infof("failed to create provisioner for claim %q with StorageClass %q: %v", claimToClaimKey(claim), storageClass.Name, err) + klog.V(2).Infof("failed to create provisioner for claim %q with StorageClass %q: %v", claimToClaimKey(claim), storageClass.Name, err) ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.ProvisioningFailed, strerr) return pluginName, err } @@ -1479,7 +1491,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis selectedNode, err = ctrl.NodeLister.Get(nodeName) if err != nil { strerr := fmt.Sprintf("Failed to get target node: %v", err) - glog.V(3).Infof("unexpected error getting target node %q for claim %q: %v", nodeName, claimToClaimKey(claim), err) + klog.V(3).Infof("unexpected error getting target node %q for claim %q: %v", nodeName, claimToClaimKey(claim), err) ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.ProvisioningFailed, strerr) return pluginName, err } @@ -1496,12 +1508,12 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis ctrl.rescheduleProvisioning(claim) strerr := fmt.Sprintf("Failed to provision volume with StorageClass %q: %v", storageClass.Name, err) - glog.V(2).Infof("failed to provision volume for claim %q with StorageClass %q: %v", claimToClaimKey(claim), storageClass.Name, err) + klog.V(2).Infof("failed to provision volume for claim %q with StorageClass %q: %v", claimToClaimKey(claim), storageClass.Name, err) ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.ProvisioningFailed, strerr) return pluginName, err } - glog.V(3).Infof("volume %q for claim %q created", volume.Name, claimToClaimKey(claim)) + klog.V(3).Infof("volume %q for claim %q created", volume.Name, claimToClaimKey(claim)) // Create Kubernetes PV object for the volume. if volume.Name == "" { @@ -1518,26 +1530,26 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis // Try to create the PV object several times for i := 0; i < ctrl.createProvisionedPVRetryCount; i++ { - glog.V(4).Infof("provisionClaimOperation [%s]: trying to save volume %s", claimToClaimKey(claim), volume.Name) + klog.V(4).Infof("provisionClaimOperation [%s]: trying to save volume %s", claimToClaimKey(claim), volume.Name) var newVol *v1.PersistentVolume if newVol, err = ctrl.kubeClient.CoreV1().PersistentVolumes().Create(volume); err == nil || apierrs.IsAlreadyExists(err) { // Save succeeded. if err != nil { - glog.V(3).Infof("volume %q for claim %q already exists, reusing", volume.Name, claimToClaimKey(claim)) + klog.V(3).Infof("volume %q for claim %q already exists, reusing", volume.Name, claimToClaimKey(claim)) err = nil } else { - glog.V(3).Infof("volume %q for claim %q saved", volume.Name, claimToClaimKey(claim)) + klog.V(3).Infof("volume %q for claim %q saved", volume.Name, claimToClaimKey(claim)) _, updateErr := ctrl.storeVolumeUpdate(newVol) if updateErr != nil { // We will get an "volume added" event soon, this is not a big error - glog.V(4).Infof("provisionClaimOperation [%s]: cannot update internal cache: %v", volume.Name, updateErr) + klog.V(4).Infof("provisionClaimOperation [%s]: cannot update internal cache: %v", volume.Name, updateErr) } } break } // Save failed, try again after a while. - glog.V(3).Infof("failed to save volume %q for claim %q: %v", volume.Name, claimToClaimKey(claim), err) + klog.V(3).Infof("failed to save volume %q for claim %q: %v", volume.Name, claimToClaimKey(claim), err) time.Sleep(ctrl.createProvisionedPVInterval) } @@ -1547,7 +1559,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis // Emit some event here and try to delete the storage asset several // times. strerr := fmt.Sprintf("Error creating provisioned PV object for claim %s: %v. Deleting the volume.", claimToClaimKey(claim), err) - glog.V(3).Info(strerr) + klog.V(3).Info(strerr) ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.ProvisioningFailed, strerr) var deleteErr error @@ -1556,18 +1568,18 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis _, deleted, deleteErr = ctrl.doDeleteVolume(volume) if deleteErr == nil && deleted { // Delete succeeded - glog.V(4).Infof("provisionClaimOperation [%s]: cleaning volume %s succeeded", claimToClaimKey(claim), volume.Name) + klog.V(4).Infof("provisionClaimOperation [%s]: cleaning volume %s succeeded", claimToClaimKey(claim), volume.Name) break } if !deleted { // This is unreachable code, the volume was provisioned by an // internal plugin and therefore there MUST be an internal // plugin that deletes it. - glog.Errorf("Error finding internal deleter for volume plugin %q", plugin.GetPluginName()) + klog.Errorf("Error finding internal deleter for volume plugin %q", plugin.GetPluginName()) break } // Delete failed, try again after a while. - glog.V(3).Infof("failed to delete volume %q: %v", volume.Name, deleteErr) + klog.V(3).Infof("failed to delete volume %q: %v", volume.Name, deleteErr) time.Sleep(ctrl.createProvisionedPVInterval) } @@ -1575,11 +1587,11 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis // Delete failed several times. There is an orphaned volume and there // is nothing we can do about it. strerr := fmt.Sprintf("Error cleaning provisioned volume for claim %s: %v. Please delete manually.", claimToClaimKey(claim), deleteErr) - glog.V(2).Info(strerr) + klog.V(2).Info(strerr) ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.ProvisioningCleanupFailed, strerr) } } else { - glog.V(2).Infof("volume %q provisioned for claim %q", volume.Name, claimToClaimKey(claim)) + klog.V(2).Infof("volume %q provisioned for claim %q", volume.Name, claimToClaimKey(claim)) msg := fmt.Sprintf("Successfully provisioned volume %s using %s", volume.Name, plugin.GetPluginName()) ctrl.eventRecorder.Event(claim, v1.EventTypeNormal, events.ProvisioningSucceeded, msg) } @@ -1600,12 +1612,12 @@ func (ctrl *PersistentVolumeController) rescheduleProvisioning(claim *v1.Persist delete(newClaim.Annotations, annSelectedNode) // Try to update the PVC object if _, err := ctrl.kubeClient.CoreV1().PersistentVolumeClaims(newClaim.Namespace).Update(newClaim); err != nil { - glog.V(4).Infof("Failed to delete annotation 'annSelectedNode' for PersistentVolumeClaim %q: %v", claimToClaimKey(newClaim), err) + klog.V(4).Infof("Failed to delete annotation 'annSelectedNode' for PersistentVolumeClaim %q: %v", claimToClaimKey(newClaim), err) return } if _, err := ctrl.storeClaimUpdate(newClaim); err != nil { // We will get an "claim updated" event soon, this is not a big error - glog.V(4).Infof("Updating PersistentVolumeClaim %q: cannot update internal cache: %v", claimToClaimKey(newClaim), err) + klog.V(4).Infof("Updating PersistentVolumeClaim %q: cannot update internal cache: %v", claimToClaimKey(newClaim), err) } } @@ -1618,7 +1630,7 @@ func (ctrl *PersistentVolumeController) getProvisionedVolumeNameForClaim(claim * // scheduleOperation starts given asynchronous operation on given volume. It // makes sure the operation is already not running. func (ctrl *PersistentVolumeController) scheduleOperation(operationName string, operation func() error) { - glog.V(4).Infof("scheduleOperation[%s]", operationName) + klog.V(4).Infof("scheduleOperation[%s]", operationName) // Poke test code that an operation is just about to get started. if ctrl.preOperationHook != nil { @@ -1629,11 +1641,11 @@ func (ctrl *PersistentVolumeController) scheduleOperation(operationName string, if err != nil { switch { case goroutinemap.IsAlreadyExists(err): - glog.V(4).Infof("operation %q is already running, skipping", operationName) + klog.V(4).Infof("operation %q is already running, skipping", operationName) case exponentialbackoff.IsExponentialBackoff(err): - glog.V(4).Infof("operation %q postponed due to exponential backoff", operationName) + klog.V(4).Infof("operation %q postponed due to exponential backoff", operationName) default: - glog.Errorf("error scheduling operation %q: %v", operationName, err) + klog.Errorf("error scheduling operation %q: %v", operationName, err) } } } diff --git a/pkg/controller/volume/persistentvolume/pv_controller_base.go b/pkg/controller/volume/persistentvolume/pv_controller_base.go index aaf97de0cb5..e83f975da34 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller_base.go +++ b/pkg/controller/volume/persistentvolume/pv_controller_base.go @@ -44,7 +44,7 @@ import ( "k8s.io/kubernetes/pkg/util/goroutinemap" vol "k8s.io/kubernetes/pkg/volume" - "github.com/golang/glog" + "k8s.io/klog" ) // This file contains the controller base functionality, i.e. framework to @@ -73,7 +73,7 @@ func NewController(p ControllerParameters) (*PersistentVolumeController, error) eventRecorder := p.EventRecorder if eventRecorder == nil { broadcaster := record.NewBroadcaster() - broadcaster.StartLogging(glog.Infof) + broadcaster.StartLogging(klog.Infof) broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: p.KubeClient.CoreV1().Events("")}) eventRecorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "persistentvolume-controller"}) } @@ -134,27 +134,27 @@ func NewController(p ControllerParameters) (*PersistentVolumeController, error) func (ctrl *PersistentVolumeController) initializeCaches(volumeLister corelisters.PersistentVolumeLister, claimLister corelisters.PersistentVolumeClaimLister) { volumeList, err := volumeLister.List(labels.Everything()) if err != nil { - glog.Errorf("PersistentVolumeController can't initialize caches: %v", err) + klog.Errorf("PersistentVolumeController can't initialize caches: %v", err) return } for _, volume := range volumeList { volumeClone := volume.DeepCopy() if _, err = ctrl.storeVolumeUpdate(volumeClone); err != nil { - glog.Errorf("error updating volume cache: %v", err) + klog.Errorf("error updating volume cache: %v", err) } } claimList, err := claimLister.List(labels.Everything()) if err != nil { - glog.Errorf("PersistentVolumeController can't initialize caches: %v", err) + klog.Errorf("PersistentVolumeController can't initialize caches: %v", err) return } for _, claim := range claimList { if _, err = ctrl.storeClaimUpdate(claim.DeepCopy()); err != nil { - glog.Errorf("error updating claim cache: %v", err) + klog.Errorf("error updating claim cache: %v", err) } } - glog.V(4).Infof("controller initialized") + klog.V(4).Infof("controller initialized") } // enqueueWork adds volume or claim to given work queue. @@ -165,10 +165,10 @@ func (ctrl *PersistentVolumeController) enqueueWork(queue workqueue.Interface, o } objName, err := controller.KeyFunc(obj) if err != nil { - glog.Errorf("failed to get key from object: %v", err) + klog.Errorf("failed to get key from object: %v", err) return } - glog.V(5).Infof("enqueued %q for sync", objName) + klog.V(5).Infof("enqueued %q for sync", objName) queue.Add(objName) } @@ -187,7 +187,7 @@ func (ctrl *PersistentVolumeController) updateVolume(volume *v1.PersistentVolume // is an old version. new, err := ctrl.storeVolumeUpdate(volume) if err != nil { - glog.Errorf("%v", err) + klog.Errorf("%v", err) } if !new { return @@ -198,9 +198,9 @@ func (ctrl *PersistentVolumeController) updateVolume(volume *v1.PersistentVolume if errors.IsConflict(err) { // Version conflict error happens quite often and the controller // recovers from it easily. - glog.V(3).Infof("could not sync volume %q: %+v", volume.Name, err) + klog.V(3).Infof("could not sync volume %q: %+v", volume.Name, err) } else { - glog.Errorf("could not sync volume %q: %+v", volume.Name, err) + klog.Errorf("could not sync volume %q: %+v", volume.Name, err) } } } @@ -208,7 +208,7 @@ func (ctrl *PersistentVolumeController) updateVolume(volume *v1.PersistentVolume // deleteVolume runs in worker thread and handles "volume deleted" event. func (ctrl *PersistentVolumeController) deleteVolume(volume *v1.PersistentVolume) { _ = ctrl.volumes.store.Delete(volume) - glog.V(4).Infof("volume %q deleted", volume.Name) + klog.V(4).Infof("volume %q deleted", volume.Name) if volume.Spec.ClaimRef == nil { return @@ -217,7 +217,7 @@ func (ctrl *PersistentVolumeController) deleteVolume(volume *v1.PersistentVolume // claim here in response to volume deletion prevents the claim from // waiting until the next sync period for its Lost status. claimKey := claimrefToClaimKey(volume.Spec.ClaimRef) - glog.V(5).Infof("deleteVolume[%s]: scheduling sync of claim %q", volume.Name, claimKey) + klog.V(5).Infof("deleteVolume[%s]: scheduling sync of claim %q", volume.Name, claimKey) ctrl.claimQueue.Add(claimKey) } @@ -228,7 +228,7 @@ func (ctrl *PersistentVolumeController) updateClaim(claim *v1.PersistentVolumeCl // an old version. new, err := ctrl.storeClaimUpdate(claim) if err != nil { - glog.Errorf("%v", err) + klog.Errorf("%v", err) } if !new { return @@ -238,9 +238,9 @@ func (ctrl *PersistentVolumeController) updateClaim(claim *v1.PersistentVolumeCl if errors.IsConflict(err) { // Version conflict error happens quite often and the controller // recovers from it easily. - glog.V(3).Infof("could not sync claim %q: %+v", claimToClaimKey(claim), err) + klog.V(3).Infof("could not sync claim %q: %+v", claimToClaimKey(claim), err) } else { - glog.Errorf("could not sync volume %q: %+v", claimToClaimKey(claim), err) + klog.Errorf("could not sync volume %q: %+v", claimToClaimKey(claim), err) } } } @@ -248,17 +248,17 @@ func (ctrl *PersistentVolumeController) updateClaim(claim *v1.PersistentVolumeCl // deleteClaim runs in worker thread and handles "claim deleted" event. func (ctrl *PersistentVolumeController) deleteClaim(claim *v1.PersistentVolumeClaim) { _ = ctrl.claims.Delete(claim) - glog.V(4).Infof("claim %q deleted", claimToClaimKey(claim)) + klog.V(4).Infof("claim %q deleted", claimToClaimKey(claim)) volumeName := claim.Spec.VolumeName if volumeName == "" { - glog.V(5).Infof("deleteClaim[%q]: volume not bound", claimToClaimKey(claim)) + klog.V(5).Infof("deleteClaim[%q]: volume not bound", claimToClaimKey(claim)) return } // sync the volume when its claim is deleted. Explicitly sync'ing the // volume here in response to claim deletion prevents the volume from // waiting until the next sync period for its Release. - glog.V(5).Infof("deleteClaim[%q]: scheduling sync of volume %s", claimToClaimKey(claim), volumeName) + klog.V(5).Infof("deleteClaim[%q]: scheduling sync of volume %s", claimToClaimKey(claim), volumeName) ctrl.volumeQueue.Add(volumeName) } @@ -268,8 +268,8 @@ func (ctrl *PersistentVolumeController) Run(stopCh <-chan struct{}) { defer ctrl.claimQueue.ShutDown() defer ctrl.volumeQueue.ShutDown() - glog.Infof("Starting persistent volume controller") - defer glog.Infof("Shutting down persistent volume controller") + klog.Infof("Starting persistent volume controller") + defer klog.Infof("Shutting down persistent volume controller") if !controller.WaitForCacheSync("persistent volume", stopCh, ctrl.volumeListerSynced, ctrl.claimListerSynced, ctrl.classListerSynced, ctrl.podListerSynced, ctrl.NodeListerSynced) { return @@ -296,11 +296,11 @@ func (ctrl *PersistentVolumeController) volumeWorker() { } defer ctrl.volumeQueue.Done(keyObj) key := keyObj.(string) - glog.V(5).Infof("volumeWorker[%s]", key) + klog.V(5).Infof("volumeWorker[%s]", key) _, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { - glog.V(4).Infof("error getting name of volume %q to get volume from informer: %v", key, err) + klog.V(4).Infof("error getting name of volume %q to get volume from informer: %v", key, err) return false } volume, err := ctrl.volumeLister.Get(name) @@ -311,7 +311,7 @@ func (ctrl *PersistentVolumeController) volumeWorker() { return false } if !errors.IsNotFound(err) { - glog.V(2).Infof("error getting volume %q from informer: %v", key, err) + klog.V(2).Infof("error getting volume %q from informer: %v", key, err) return false } @@ -319,18 +319,18 @@ func (ctrl *PersistentVolumeController) volumeWorker() { // "delete" volumeObj, found, err := ctrl.volumes.store.GetByKey(key) if err != nil { - glog.V(2).Infof("error getting volume %q from cache: %v", key, err) + klog.V(2).Infof("error getting volume %q from cache: %v", key, err) return false } if !found { // The controller has already processed the delete event and // deleted the volume from its cache - glog.V(2).Infof("deletion of volume %q was already processed", key) + klog.V(2).Infof("deletion of volume %q was already processed", key) return false } volume, ok := volumeObj.(*v1.PersistentVolume) if !ok { - glog.Errorf("expected volume, got %+v", volumeObj) + klog.Errorf("expected volume, got %+v", volumeObj) return false } ctrl.deleteVolume(volume) @@ -338,7 +338,7 @@ func (ctrl *PersistentVolumeController) volumeWorker() { } for { if quit := workFunc(); quit { - glog.Infof("volume worker queue shutting down") + klog.Infof("volume worker queue shutting down") return } } @@ -354,11 +354,11 @@ func (ctrl *PersistentVolumeController) claimWorker() { } defer ctrl.claimQueue.Done(keyObj) key := keyObj.(string) - glog.V(5).Infof("claimWorker[%s]", key) + klog.V(5).Infof("claimWorker[%s]", key) namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { - glog.V(4).Infof("error getting namespace & name of claim %q to get claim from informer: %v", key, err) + klog.V(4).Infof("error getting namespace & name of claim %q to get claim from informer: %v", key, err) return false } claim, err := ctrl.claimLister.PersistentVolumeClaims(namespace).Get(name) @@ -369,25 +369,25 @@ func (ctrl *PersistentVolumeController) claimWorker() { return false } if !errors.IsNotFound(err) { - glog.V(2).Infof("error getting claim %q from informer: %v", key, err) + klog.V(2).Infof("error getting claim %q from informer: %v", key, err) return false } // The claim is not in informer cache, the event must have been "delete" claimObj, found, err := ctrl.claims.GetByKey(key) if err != nil { - glog.V(2).Infof("error getting claim %q from cache: %v", key, err) + klog.V(2).Infof("error getting claim %q from cache: %v", key, err) return false } if !found { // The controller has already processed the delete event and // deleted the claim from its cache - glog.V(2).Infof("deletion of claim %q was already processed", key) + klog.V(2).Infof("deletion of claim %q was already processed", key) return false } claim, ok := claimObj.(*v1.PersistentVolumeClaim) if !ok { - glog.Errorf("expected claim, got %+v", claimObj) + klog.Errorf("expected claim, got %+v", claimObj) return false } ctrl.deleteClaim(claim) @@ -395,7 +395,7 @@ func (ctrl *PersistentVolumeController) claimWorker() { } for { if quit := workFunc(); quit { - glog.Infof("claim worker queue shutting down") + klog.Infof("claim worker queue shutting down") return } } @@ -405,11 +405,11 @@ func (ctrl *PersistentVolumeController) claimWorker() { // all consumers of PV/PVC shared informer to have a short resync period, // therefore we do our own. func (ctrl *PersistentVolumeController) resync() { - glog.V(4).Infof("resyncing PV controller") + klog.V(4).Infof("resyncing PV controller") pvcs, err := ctrl.claimLister.List(labels.NewSelector()) if err != nil { - glog.Warningf("cannot list claims: %s", err) + klog.Warningf("cannot list claims: %s", err) return } for _, pvc := range pvcs { @@ -418,7 +418,7 @@ func (ctrl *PersistentVolumeController) resync() { pvs, err := ctrl.volumeLister.List(labels.NewSelector()) if err != nil { - glog.Warningf("cannot list persistent volumes: %s", err) + klog.Warningf("cannot list persistent volumes: %s", err) return } for _, pv := range pvs { @@ -504,7 +504,7 @@ func storeObjectUpdate(store cache.Store, obj interface{}, className string) (bo if !found { // This is a new object - glog.V(4).Infof("storeObjectUpdate: adding %s %q, version %s", className, objName, objAccessor.GetResourceVersion()) + klog.V(4).Infof("storeObjectUpdate: adding %s %q, version %s", className, objName, objAccessor.GetResourceVersion()) if err = store.Add(obj); err != nil { return false, fmt.Errorf("Error adding %s %q to controller cache: %v", className, objName, err) } @@ -528,11 +528,11 @@ func storeObjectUpdate(store cache.Store, obj interface{}, className string) (bo // Throw away only older version, let the same version pass - we do want to // get periodic sync events. if oldObjResourceVersion > objResourceVersion { - glog.V(4).Infof("storeObjectUpdate: ignoring %s %q version %s", className, objName, objAccessor.GetResourceVersion()) + klog.V(4).Infof("storeObjectUpdate: ignoring %s %q version %s", className, objName, objAccessor.GetResourceVersion()) return false, nil } - glog.V(4).Infof("storeObjectUpdate updating %s %q with version %s", className, objName, objAccessor.GetResourceVersion()) + klog.V(4).Infof("storeObjectUpdate updating %s %q with version %s", className, objName, objAccessor.GetResourceVersion()) if err = store.Update(obj); err != nil { return false, fmt.Errorf("Error updating %s %q in controller cache: %v", className, objName, err) } diff --git a/pkg/controller/volume/persistentvolume/pv_controller_test.go b/pkg/controller/volume/persistentvolume/pv_controller_test.go index cc372879674..514617e5f1d 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller_test.go +++ b/pkg/controller/volume/persistentvolume/pv_controller_test.go @@ -20,18 +20,26 @@ import ( "testing" "time" - "github.com/golang/glog" - "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller" + "k8s.io/kubernetes/pkg/features" +) + +var ( + classNotHere = "not-here" + classNoMode = "no-mode" + classImmediateMode = "immediate-mode" + classWaitMode = "wait-mode" ) // Test the real controller methods (add/update/delete claim/volume) with @@ -96,7 +104,7 @@ func TestControllerSync(t *testing.T) { } for _, test := range tests { - glog.V(4).Infof("starting test %q", test.name) + klog.V(4).Infof("starting test %q", test.name) // Initialize the controller client := &fake.Clientset{} @@ -140,7 +148,7 @@ func TestControllerSync(t *testing.T) { time.Sleep(10 * time.Millisecond) } - glog.V(4).Infof("controller synced, starting test") + klog.V(4).Infof("controller synced, starting test") // Call the tested function err = test.test(ctrl, reactor, test) @@ -264,16 +272,6 @@ func makeStorageClass(scName string, mode *storagev1.VolumeBindingMode) *storage } func TestDelayBinding(t *testing.T) { - var ( - classNotHere = "not-here" - classNoMode = "no-mode" - classImmediateMode = "immediate-mode" - classWaitMode = "wait-mode" - - modeImmediate = storagev1.VolumeBindingImmediate - modeWait = storagev1.VolumeBindingWaitForFirstConsumer - ) - tests := map[string]struct { pvc *v1.PersistentVolumeClaim shouldDelay bool @@ -325,22 +323,8 @@ func TestDelayBinding(t *testing.T) { } } - // When volumeScheduling feature gate is disabled, should always be delayed - name := "volumeScheduling-feature-disabled" - shouldDelay, err := ctrl.shouldDelayBinding(makePVCClass(&classWaitMode, false)) - if err != nil { - t.Errorf("Test %q returned error: %v", name, err) - } - if shouldDelay { - t.Errorf("Test %q returned true, expected false", name) - } - - // Enable volumeScheduling feature gate - utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - defer utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - for name, test := range tests { - shouldDelay, err = ctrl.shouldDelayBinding(test.pvc) + shouldDelay, err := ctrl.shouldDelayBinding(test.pvc) if err != nil && !test.shouldFail { t.Errorf("Test %q returned error: %v", name, err) } @@ -352,3 +336,24 @@ func TestDelayBinding(t *testing.T) { } } } + +func TestDelayBindingDisabled(t *testing.T) { + // When volumeScheduling feature gate is disabled, should always be immediate + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, false)() + + client := &fake.Clientset{} + informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) + classInformer := informerFactory.Storage().V1().StorageClasses() + ctrl := &PersistentVolumeController{ + classLister: classInformer.Lister(), + } + + name := "volumeScheduling-feature-disabled" + shouldDelay, err := ctrl.shouldDelayBinding(makePVCClass(&classWaitMode, false)) + if err != nil { + t.Errorf("Test %q returned error: %v", name, err) + } + if shouldDelay { + t.Errorf("Test %q returned true, expected false", name) + } +} diff --git a/pkg/controller/volume/persistentvolume/recycle_test.go b/pkg/controller/volume/persistentvolume/recycle_test.go index 73426c44046..f0c0f7dcba9 100644 --- a/pkg/controller/volume/persistentvolume/recycle_test.go +++ b/pkg/controller/volume/persistentvolume/recycle_test.go @@ -229,6 +229,16 @@ func TestRecycleSync(t *testing.T) { // recycler simulates one recycle() call that succeeds. wrapTestWithReclaimCalls(operationRecycle, []error{nil}, testSyncVolume), }, + { + // volume is used by a completed pod, pod using claim with the same name bound to different pv is running, should recycle + "6-14 - seemingly used by running pod", + newVolumeArray("volume6-14", "1Gi", "uid6-14", "completedClaim", v1.VolumeBound, v1.PersistentVolumeReclaimRecycle, classEmpty, annBoundByController), + newVolumeArray("volume6-14", "1Gi", "", "", v1.VolumeAvailable, v1.PersistentVolumeReclaimRecycle, classEmpty), + newClaimArray("completedClaim", "uid6-14-x", "10Gi", "", v1.ClaimBound, nil), + newClaimArray("completedClaim", "uid6-14-x", "10Gi", "", v1.ClaimBound, nil), + noevents, noerrors, + wrapTestWithReclaimCalls(operationRecycle, []error{nil}, testSyncVolume), + }, } runSyncTests(t, tests, []*storage.StorageClass{}, pods) } diff --git a/pkg/controller/volume/persistentvolume/scheduler_assume_cache.go b/pkg/controller/volume/persistentvolume/scheduler_assume_cache.go index 0e4ade0faf1..292ee8f6ff2 100644 --- a/pkg/controller/volume/persistentvolume/scheduler_assume_cache.go +++ b/pkg/controller/volume/persistentvolume/scheduler_assume_cache.go @@ -21,7 +21,7 @@ import ( "strconv" "sync" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -152,7 +152,7 @@ func (c *assumeCache) add(obj interface{}) { name, err := cache.MetaNamespaceKeyFunc(obj) if err != nil { - glog.Errorf("add failed: %v", &errObjectName{err}) + klog.Errorf("add failed: %v", &errObjectName{err}) return } @@ -162,27 +162,27 @@ func (c *assumeCache) add(obj interface{}) { if objInfo, _ := c.getObjInfo(name); objInfo != nil { newVersion, err := c.getObjVersion(name, obj) if err != nil { - glog.Errorf("add: couldn't get object version: %v", err) + klog.Errorf("add: couldn't get object version: %v", err) return } storedVersion, err := c.getObjVersion(name, objInfo.latestObj) if err != nil { - glog.Errorf("add: couldn't get stored object version: %v", err) + klog.Errorf("add: couldn't get stored object version: %v", err) return } // Only update object if version is newer. // This is so we don't override assumed objects due to informer resync. if newVersion <= storedVersion { - glog.V(10).Infof("Skip adding %v %v to assume cache because version %v is not newer than %v", c.description, name, newVersion, storedVersion) + klog.V(10).Infof("Skip adding %v %v to assume cache because version %v is not newer than %v", c.description, name, newVersion, storedVersion) return } } objInfo := &objInfo{name: name, latestObj: obj, apiObj: obj} c.store.Update(objInfo) - glog.V(10).Infof("Adding %v %v to assume cache: %+v ", c.description, name, obj) + klog.V(10).Infof("Adding %v %v to assume cache: %+v ", c.description, name, obj) } func (c *assumeCache) update(oldObj interface{}, newObj interface{}) { @@ -196,7 +196,7 @@ func (c *assumeCache) delete(obj interface{}) { name, err := cache.MetaNamespaceKeyFunc(obj) if err != nil { - glog.Errorf("delete failed: %v", &errObjectName{err}) + klog.Errorf("delete failed: %v", &errObjectName{err}) return } @@ -206,7 +206,7 @@ func (c *assumeCache) delete(obj interface{}) { objInfo := &objInfo{name: name} err = c.store.Delete(objInfo) if err != nil { - glog.Errorf("delete: failed to delete %v %v: %v", c.description, name, err) + klog.Errorf("delete: failed to delete %v %v: %v", c.description, name, err) } } @@ -257,14 +257,14 @@ func (c *assumeCache) List(indexObj interface{}) []interface{} { allObjs := []interface{}{} objs, err := c.store.Index(c.indexName, &objInfo{latestObj: indexObj}) if err != nil { - glog.Errorf("list index error: %v", err) + klog.Errorf("list index error: %v", err) return nil } for _, obj := range objs { objInfo, ok := obj.(*objInfo) if !ok { - glog.Errorf("list error: %v", &errWrongType{"objInfo", obj}) + klog.Errorf("list error: %v", &errWrongType{"objInfo", obj}) continue } allObjs = append(allObjs, objInfo.latestObj) @@ -302,7 +302,7 @@ func (c *assumeCache) Assume(obj interface{}) error { // Only update the cached object objInfo.latestObj = obj - glog.V(4).Infof("Assumed %v %q, version %v", c.description, name, newVersion) + klog.V(4).Infof("Assumed %v %q, version %v", c.description, name, newVersion) return nil } @@ -313,10 +313,10 @@ func (c *assumeCache) Restore(objName string) { objInfo, err := c.getObjInfo(objName) if err != nil { // This could be expected if object got deleted - glog.V(5).Infof("Restore %v %q warning: %v", c.description, objName, err) + klog.V(5).Infof("Restore %v %q warning: %v", c.description, objName, err) } else { objInfo.latestObj = objInfo.apiObj - glog.V(4).Infof("Restored %v %q", c.description, objName) + klog.V(4).Infof("Restored %v %q", c.description, objName) } } @@ -366,7 +366,7 @@ func (c *pvAssumeCache) ListPVs(storageClassName string) []*v1.PersistentVolume for _, obj := range objs { pv, ok := obj.(*v1.PersistentVolume) if !ok { - glog.Errorf("ListPVs: %v", &errWrongType{"v1.PersistentVolume", obj}) + klog.Errorf("ListPVs: %v", &errWrongType{"v1.PersistentVolume", obj}) } pvs = append(pvs, pv) } diff --git a/pkg/controller/volume/persistentvolume/scheduler_binder.go b/pkg/controller/volume/persistentvolume/scheduler_binder.go index 8ef8b2adc1a..85fd71cf4eb 100644 --- a/pkg/controller/volume/persistentvolume/scheduler_binder.go +++ b/pkg/controller/volume/persistentvolume/scheduler_binder.go @@ -21,7 +21,7 @@ import ( "sort" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -144,7 +144,7 @@ func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, node *v1.Node) (unboundVolume podName := getPodName(pod) // Warning: Below log needs high verbosity as it can be printed several times (#60933). - glog.V(5).Infof("FindPodVolumes for pod %q, node %q", podName, node.Name) + klog.V(5).Infof("FindPodVolumes for pod %q, node %q", podName, node.Name) // Initialize to true for pods that don't have volumes unboundVolumesSatisfied = true @@ -204,7 +204,7 @@ func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, node *v1.Node) (unboundVolume func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string) (allFullyBound bool, err error) { podName := getPodName(assumedPod) - glog.V(4).Infof("AssumePodVolumes for pod %q, node %q", podName, nodeName) + klog.V(4).Infof("AssumePodVolumes for pod %q, node %q", podName, nodeName) start := time.Now() defer func() { VolumeSchedulingStageLatency.WithLabelValues("assume").Observe(time.Since(start).Seconds()) @@ -214,7 +214,7 @@ func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string) (al }() if allBound := b.arePodVolumesBound(assumedPod); allBound { - glog.V(4).Infof("AssumePodVolumes for pod %q, node %q: all PVCs bound and nothing to do", podName, nodeName) + klog.V(4).Infof("AssumePodVolumes for pod %q, node %q: all PVCs bound and nothing to do", podName, nodeName) return true, nil } @@ -227,7 +227,7 @@ func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string) (al newBindings := []*bindingInfo{} for _, binding := range claimsToBind { newPV, dirty, err := b.ctrl.getBindVolumeToClaim(binding.pv, binding.pvc) - glog.V(5).Infof("AssumePodVolumes: getBindVolumeToClaim for pod %q, PV %q, PVC %q. newPV %p, dirty %v, err: %v", + klog.V(5).Infof("AssumePodVolumes: getBindVolumeToClaim for pod %q, PV %q, PVC %q. newPV %p, dirty %v, err: %v", podName, binding.pv.Name, binding.pvc.Name, @@ -280,7 +280,7 @@ func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string) (al // by the PV controller. func (b *volumeBinder) BindPodVolumes(assumedPod *v1.Pod) (err error) { podName := getPodName(assumedPod) - glog.V(4).Infof("BindPodVolumes for pod %q, node %q", podName, assumedPod.Spec.NodeName) + klog.V(4).Infof("BindPodVolumes for pod %q, node %q", podName, assumedPod.Spec.NodeName) start := time.Now() defer func() { @@ -346,7 +346,7 @@ func (b *volumeBinder) bindAPIUpdate(podName string, bindings []*bindingInfo, cl // Do the actual prebinding. Let the PV controller take care of the rest // There is no API rollback if the actual binding fails for _, binding = range bindings { - glog.V(5).Infof("bindAPIUpdate: Pod %q, binding PV %q to PVC %q", podName, binding.pv.Name, binding.pvc.Name) + klog.V(5).Infof("bindAPIUpdate: Pod %q, binding PV %q to PVC %q", podName, binding.pv.Name, binding.pvc.Name) // TODO: does it hurt if we make an api call and nothing needs to be updated? if _, err := b.ctrl.updateBindVolumeToClaim(binding.pv, binding.pvc, false); err != nil { return err @@ -357,7 +357,7 @@ func (b *volumeBinder) bindAPIUpdate(podName string, bindings []*bindingInfo, cl // Update claims objects to trigger volume provisioning. Let the PV controller take care of the rest // PV controller is expect to signal back by removing related annotations if actual provisioning fails for _, claim = range claimsToProvision { - glog.V(5).Infof("bindAPIUpdate: Pod %q, PVC %q", podName, getPVCName(claim)) + klog.V(5).Infof("bindAPIUpdate: Pod %q, PVC %q", podName, getPVCName(claim)) if _, err := b.ctrl.kubeClient.CoreV1().PersistentVolumeClaims(claim.Namespace).Update(claim); err != nil { return err } @@ -426,7 +426,7 @@ func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*bindingInfo, claim } // All pvs and pvcs that we operated on are bound - glog.V(4).Infof("All PVCs for pod %q are bound", podName) + klog.V(4).Infof("All PVCs for pod %q are bound", podName) return true, nil } @@ -455,15 +455,15 @@ func (b *volumeBinder) isPVCBound(namespace, pvcName string) (bool, *v1.Persiste pvName := pvc.Spec.VolumeName if pvName != "" { if metav1.HasAnnotation(pvc.ObjectMeta, annBindCompleted) { - glog.V(5).Infof("PVC %q is fully bound to PV %q", pvcKey, pvName) + klog.V(5).Infof("PVC %q is fully bound to PV %q", pvcKey, pvName) return true, pvc, nil } else { - glog.V(5).Infof("PVC %q is not fully bound to PV %q", pvcKey, pvName) + klog.V(5).Infof("PVC %q is not fully bound to PV %q", pvcKey, pvName) return false, pvc, nil } } - glog.V(5).Infof("PVC %q is not bound", pvcKey) + klog.V(5).Infof("PVC %q is not bound", pvcKey) return false, pvc, nil } @@ -523,13 +523,13 @@ func (b *volumeBinder) checkBoundClaims(claims []*v1.PersistentVolumeClaim, node err = volumeutil.CheckNodeAffinity(pv, node.Labels) if err != nil { - glog.V(4).Infof("PersistentVolume %q, Node %q mismatch for Pod %q: %v", pvName, node.Name, podName, err) + klog.V(4).Infof("PersistentVolume %q, Node %q mismatch for Pod %q: %v", pvName, node.Name, podName, err) return false, nil } - glog.V(5).Infof("PersistentVolume %q, Node %q matches for Pod %q", pvName, node.Name, podName) + klog.V(5).Infof("PersistentVolume %q, Node %q matches for Pod %q", pvName, node.Name, podName) } - glog.V(4).Infof("All bound volumes for Pod %q match with Node %q", podName, node.Name) + klog.V(4).Infof("All bound volumes for Pod %q match with Node %q", podName, node.Name) return true, nil } @@ -561,7 +561,7 @@ func (b *volumeBinder) findMatchingVolumes(pod *v1.Pod, claimsToBind []*bindingI return false, nil, err } if bindingInfo.pv == nil { - glog.V(4).Infof("No matching volumes for Pod %q, PVC %q on node %q", podName, pvcName, node.Name) + klog.V(4).Infof("No matching volumes for Pod %q, PVC %q on node %q", podName, pvcName, node.Name) unboundClaims = append(unboundClaims, bindingInfo.pvc) foundMatches = false continue @@ -570,7 +570,7 @@ func (b *volumeBinder) findMatchingVolumes(pod *v1.Pod, claimsToBind []*bindingI // matching PV needs to be excluded so we don't select it again chosenPVs[bindingInfo.pv.Name] = bindingInfo.pv matchedClaims = append(matchedClaims, bindingInfo) - glog.V(5).Infof("Found matching PV %q for PVC %q on node %q for pod %q", bindingInfo.pv.Name, pvcName, node.Name, podName) + klog.V(5).Infof("Found matching PV %q for PVC %q on node %q for pod %q", bindingInfo.pv.Name, pvcName, node.Name, podName) } // Mark cache with all the matches for each PVC for this node @@ -579,7 +579,7 @@ func (b *volumeBinder) findMatchingVolumes(pod *v1.Pod, claimsToBind []*bindingI } if foundMatches { - glog.V(4).Infof("Found matching volumes for pod %q on node %q", podName, node.Name) + klog.V(4).Infof("Found matching volumes for pod %q on node %q", podName, node.Name) } return @@ -605,13 +605,13 @@ func (b *volumeBinder) checkVolumeProvisions(pod *v1.Pod, claimsToProvision []*v } provisioner := class.Provisioner if provisioner == "" || provisioner == notSupportedProvisioner { - glog.V(4).Infof("storage class %q of claim %q does not support dynamic provisioning", className, pvcName) + klog.V(4).Infof("storage class %q of claim %q does not support dynamic provisioning", className, pvcName) return false, nil } // Check if the node can satisfy the topology requirement in the class if !v1helper.MatchTopologySelectorTerms(class.AllowedTopologies, labels.Set(node.Labels)) { - glog.V(4).Infof("Node %q cannot satisfy provisioning topology requirements of claim %q", node.Name, pvcName) + klog.V(4).Infof("Node %q cannot satisfy provisioning topology requirements of claim %q", node.Name, pvcName) return false, nil } @@ -621,7 +621,7 @@ func (b *volumeBinder) checkVolumeProvisions(pod *v1.Pod, claimsToProvision []*v provisionedClaims = append(provisionedClaims, claim) } - glog.V(4).Infof("Provisioning for claims of pod %q that has no matching volumes on node %q ...", podName, node.Name) + klog.V(4).Infof("Provisioning for claims of pod %q that has no matching volumes on node %q ...", podName, node.Name) // Mark cache with all the PVCs that need provisioning for this node b.podBindingCache.UpdateProvisionedPVCs(pod, node.Name, provisionedClaims) diff --git a/pkg/controller/volume/persistentvolume/scheduler_binder_test.go b/pkg/controller/volume/persistentvolume/scheduler_binder_test.go index 8b669caa5b1..1e7af815154 100644 --- a/pkg/controller/volume/persistentvolume/scheduler_binder_test.go +++ b/pkg/controller/volume/persistentvolume/scheduler_binder_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -30,7 +30,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/diff" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" @@ -427,6 +426,7 @@ const ( ) func makeTestPVC(name, size, node string, pvcBoundState int, pvName, resourceVersion string, className *string) *v1.PersistentVolumeClaim { + fs := v1.PersistentVolumeFilesystem pvc := &v1.PersistentVolumeClaim{ TypeMeta: metav1.TypeMeta{ Kind: "PersistentVolumeClaim", @@ -446,6 +446,7 @@ func makeTestPVC(name, size, node string, pvcBoundState int, pvName, resourceVer }, }, StorageClassName: className, + VolumeMode: &fs, }, } @@ -463,6 +464,7 @@ func makeTestPVC(name, size, node string, pvcBoundState int, pvName, resourceVer } func makeBadPVC() *v1.PersistentVolumeClaim { + fs := v1.PersistentVolumeFilesystem return &v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "bad-pvc", @@ -478,11 +480,13 @@ func makeBadPVC() *v1.PersistentVolumeClaim { }, }, StorageClassName: &waitClass, + VolumeMode: &fs, }, } } func makeTestPV(name, node, capacity, version string, boundToPVC *v1.PersistentVolumeClaim, className string) *v1.PersistentVolume { + fs := v1.PersistentVolumeFilesystem pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -493,6 +497,7 @@ func makeTestPV(name, node, capacity, version string, boundToPVC *v1.PersistentV v1.ResourceName(v1.ResourceStorage): resource.MustParse(capacity), }, StorageClassName: className, + VolumeMode: &fs, }, Status: v1.PersistentVolumeStatus{ Phase: v1.VolumeAvailable, @@ -728,10 +733,6 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) { }, } - // Set feature gate - utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - defer utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - testNode := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "node1", @@ -742,7 +743,7 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) { } for name, scenario := range scenarios { - glog.V(5).Infof("Running test case %q", name) + klog.V(5).Infof("Running test case %q", name) // Setup testEnv := newTestBinder(t) @@ -844,10 +845,6 @@ func TestFindPodVolumesWithProvisioning(t *testing.T) { }, } - // Set VolumeScheduling feature gate - utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - defer utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - testNode := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "node1", @@ -964,7 +961,7 @@ func TestAssumePodVolumes(t *testing.T) { } for name, scenario := range scenarios { - glog.V(5).Infof("Running test case %q", name) + klog.V(5).Infof("Running test case %q", name) // Setup testEnv := newTestBinder(t) @@ -1094,7 +1091,7 @@ func TestBindAPIUpdate(t *testing.T) { }, } for name, scenario := range scenarios { - glog.V(4).Infof("Running test case %q", name) + klog.V(4).Infof("Running test case %q", name) // Setup testEnv := newTestBinder(t) @@ -1253,7 +1250,7 @@ func TestCheckBindings(t *testing.T) { } for name, scenario := range scenarios { - glog.V(4).Infof("Running test case %q", name) + klog.V(4).Infof("Running test case %q", name) // Setup pod := makePod(nil) @@ -1386,7 +1383,7 @@ func TestBindPodVolumes(t *testing.T) { } for name, scenario := range scenarios { - glog.V(4).Infof("Running test case %q", name) + klog.V(4).Infof("Running test case %q", name) // Setup pod := makePod(nil) @@ -1407,7 +1404,7 @@ func TestBindPodVolumes(t *testing.T) { if scenario.delayFunc != nil { go func() { time.Sleep(5 * time.Second) - glog.V(5).Infof("Running delay function") + klog.V(5).Infof("Running delay function") scenario.delayFunc(t, testEnv, pod, scenario.binding.pv, scenario.binding.pvc) }() } @@ -1426,10 +1423,6 @@ func TestBindPodVolumes(t *testing.T) { } func TestFindAssumeVolumes(t *testing.T) { - // Set feature gate - utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - defer utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - // Test case podPVCs := []*v1.PersistentVolumeClaim{unboundPVC} pvs := []*v1.PersistentVolume{pvNode2, pvNode1a, pvNode1c} diff --git a/pkg/controller/volume/persistentvolume/volume_host.go b/pkg/controller/volume/persistentvolume/volume_host.go index 298d5737a62..6e4fc3fdd38 100644 --- a/pkg/controller/volume/persistentvolume/volume_host.go +++ b/pkg/controller/volume/persistentvolume/volume_host.go @@ -20,7 +20,6 @@ import ( "fmt" "net" - "github.com/golang/glog" authenticationv1 "k8s.io/api/authentication/v1" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -28,6 +27,7 @@ import ( "k8s.io/client-go/tools/record" cloudprovider "k8s.io/cloud-provider" csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" vol "k8s.io/kubernetes/pkg/volume" ) @@ -112,7 +112,7 @@ func (ctrl *PersistentVolumeController) GetServiceAccountTokenFunc() func(_, _ s func (ctrl *PersistentVolumeController) DeleteServiceAccountTokenFunc() func(types.UID) { return func(types.UID) { - glog.Errorf("DeleteServiceAccountToken unsupported in PersistentVolumeController") + klog.Errorf("DeleteServiceAccountToken unsupported in PersistentVolumeController") } } diff --git a/pkg/controller/volume/pvcprotection/BUILD b/pkg/controller/volume/pvcprotection/BUILD index 363aac9dbcf..041d1fb4b95 100644 --- a/pkg/controller/volume/pvcprotection/BUILD +++ b/pkg/controller/volume/pvcprotection/BUILD @@ -20,7 +20,7 @@ go_library( "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -41,7 +41,7 @@ go_test( "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/pvcprotection/pvc_protection_controller.go b/pkg/controller/volume/pvcprotection/pvc_protection_controller.go index 8de612fffdf..a832476c552 100644 --- a/pkg/controller/volume/pvcprotection/pvc_protection_controller.go +++ b/pkg/controller/volume/pvcprotection/pvc_protection_controller.go @@ -20,7 +20,6 @@ import ( "fmt" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" @@ -31,6 +30,7 @@ import ( corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/util/metrics" "k8s.io/kubernetes/pkg/util/slice" @@ -96,8 +96,8 @@ func (c *Controller) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() - glog.Infof("Starting PVC protection controller") - defer glog.Infof("Shutting down PVC protection controller") + klog.Infof("Starting PVC protection controller") + defer klog.Infof("Shutting down PVC protection controller") if !controller.WaitForCacheSync("PVC protection", stopCh, c.pvcListerSynced, c.podListerSynced) { return @@ -142,15 +142,15 @@ func (c *Controller) processNextWorkItem() bool { } func (c *Controller) processPVC(pvcNamespace, pvcName string) error { - glog.V(4).Infof("Processing PVC %s/%s", pvcNamespace, pvcName) + klog.V(4).Infof("Processing PVC %s/%s", pvcNamespace, pvcName) startTime := time.Now() defer func() { - glog.V(4).Infof("Finished processing PVC %s/%s (%v)", pvcNamespace, pvcName, time.Since(startTime)) + klog.V(4).Infof("Finished processing PVC %s/%s (%v)", pvcNamespace, pvcName, time.Since(startTime)) }() pvc, err := c.pvcLister.PersistentVolumeClaims(pvcNamespace).Get(pvcName) if apierrs.IsNotFound(err) { - glog.V(4).Infof("PVC %s/%s not found, ignoring", pvcNamespace, pvcName) + klog.V(4).Infof("PVC %s/%s not found, ignoring", pvcNamespace, pvcName) return nil } if err != nil { @@ -188,10 +188,10 @@ func (c *Controller) addFinalizer(pvc *v1.PersistentVolumeClaim) error { claimClone.ObjectMeta.Finalizers = append(claimClone.ObjectMeta.Finalizers, volumeutil.PVCProtectionFinalizer) _, err := c.client.CoreV1().PersistentVolumeClaims(claimClone.Namespace).Update(claimClone) if err != nil { - glog.V(3).Infof("Error adding protection finalizer to PVC %s/%s: %v", pvc.Namespace, pvc.Name, err) + klog.V(3).Infof("Error adding protection finalizer to PVC %s/%s: %v", pvc.Namespace, pvc.Name, err) return err } - glog.V(3).Infof("Added protection finalizer to PVC %s/%s", pvc.Namespace, pvc.Name) + klog.V(3).Infof("Added protection finalizer to PVC %s/%s", pvc.Namespace, pvc.Name) return nil } @@ -200,10 +200,10 @@ func (c *Controller) removeFinalizer(pvc *v1.PersistentVolumeClaim) error { claimClone.ObjectMeta.Finalizers = slice.RemoveString(claimClone.ObjectMeta.Finalizers, volumeutil.PVCProtectionFinalizer, nil) _, err := c.client.CoreV1().PersistentVolumeClaims(claimClone.Namespace).Update(claimClone) if err != nil { - glog.V(3).Infof("Error removing protection finalizer from PVC %s/%s: %v", pvc.Namespace, pvc.Name, err) + klog.V(3).Infof("Error removing protection finalizer from PVC %s/%s: %v", pvc.Namespace, pvc.Name, err) return err } - glog.V(3).Infof("Removed protection finalizer from PVC %s/%s", pvc.Namespace, pvc.Name) + klog.V(3).Infof("Removed protection finalizer from PVC %s/%s", pvc.Namespace, pvc.Name) return nil } @@ -218,7 +218,7 @@ func (c *Controller) isBeingUsed(pvc *v1.PersistentVolumeClaim) (bool, error) { // prevents scheduling pods with deletion timestamp, so we can be // pretty sure it won't be scheduled in parallel to this check. // Therefore this pod does not block the PVC from deletion. - glog.V(4).Infof("Skipping unscheduled pod %s when checking PVC %s/%s", pod.Name, pvc.Namespace, pvc.Name) + klog.V(4).Infof("Skipping unscheduled pod %s when checking PVC %s/%s", pod.Name, pvc.Namespace, pvc.Name) continue } for _, volume := range pod.Spec.Volumes { @@ -226,13 +226,13 @@ func (c *Controller) isBeingUsed(pvc *v1.PersistentVolumeClaim) (bool, error) { continue } if volume.PersistentVolumeClaim.ClaimName == pvc.Name { - glog.V(2).Infof("Keeping PVC %s/%s, it is used by pod %s/%s", pvc.Namespace, pvc.Name, pod.Namespace, pod.Name) + klog.V(2).Infof("Keeping PVC %s/%s, it is used by pod %s/%s", pvc.Namespace, pvc.Name, pod.Namespace, pod.Name) return true, nil } } } - glog.V(3).Infof("PVC %s/%s is unused", pvc.Namespace, pvc.Name) + klog.V(3).Infof("PVC %s/%s is unused", pvc.Namespace, pvc.Name) return false, nil } @@ -248,7 +248,7 @@ func (c *Controller) pvcAddedUpdated(obj interface{}) { utilruntime.HandleError(fmt.Errorf("Couldn't get key for Persistent Volume Claim %#v: %v", pvc, err)) return } - glog.V(4).Infof("Got event on PVC %s", key) + klog.V(4).Infof("Got event on PVC %s", key) if needToAddFinalizer(pvc) || isDeletionCandidate(pvc) { c.queue.Add(key) @@ -276,7 +276,7 @@ func (c *Controller) podAddedDeletedUpdated(obj interface{}, deleted bool) { return } - glog.V(4).Infof("Got event on pod %s/%s", pod.Namespace, pod.Name) + klog.V(4).Infof("Got event on pod %s/%s", pod.Namespace, pod.Name) // Enqueue all PVCs that the pod uses for _, volume := range pod.Spec.Volumes { diff --git a/pkg/controller/volume/pvcprotection/pvc_protection_controller_test.go b/pkg/controller/volume/pvcprotection/pvc_protection_controller_test.go index 288be8f20bf..c8eb87bf652 100644 --- a/pkg/controller/volume/pvcprotection/pvc_protection_controller_test.go +++ b/pkg/controller/volume/pvcprotection/pvc_protection_controller_test.go @@ -23,7 +23,7 @@ import ( "time" "github.com/davecgh/go-spew/spew" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -374,7 +374,7 @@ func TestPVCProtectionController(t *testing.T) { break } if ctrl.queue.Len() > 0 { - glog.V(5).Infof("Test %q: %d events queue, processing one", test.name, ctrl.queue.Len()) + klog.V(5).Infof("Test %q: %d events queue, processing one", test.name, ctrl.queue.Len()) ctrl.processNextWorkItem() } if ctrl.queue.Len() > 0 { @@ -385,7 +385,7 @@ func TestPVCProtectionController(t *testing.T) { if currentActionCount < len(test.expectedActions) { // Do not log evey wait, only when the action count changes. if lastReportedActionCount < currentActionCount { - glog.V(5).Infof("Test %q: got %d actions out of %d, waiting for the rest", test.name, currentActionCount, len(test.expectedActions)) + klog.V(5).Infof("Test %q: got %d actions out of %d, waiting for the rest", test.name, currentActionCount, len(test.expectedActions)) lastReportedActionCount = currentActionCount } // The test expected more to happen, wait for the actions. diff --git a/pkg/controller/volume/pvprotection/BUILD b/pkg/controller/volume/pvprotection/BUILD index a19318bd0a3..cdea8771770 100644 --- a/pkg/controller/volume/pvprotection/BUILD +++ b/pkg/controller/volume/pvprotection/BUILD @@ -19,7 +19,7 @@ go_library( "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -54,6 +54,6 @@ go_test( "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/controller/volume/pvprotection/pv_protection_controller.go b/pkg/controller/volume/pvprotection/pv_protection_controller.go index bc19eb5e527..d14a7a06d1a 100644 --- a/pkg/controller/volume/pvprotection/pv_protection_controller.go +++ b/pkg/controller/volume/pvprotection/pv_protection_controller.go @@ -20,7 +20,6 @@ import ( "fmt" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -30,6 +29,7 @@ import ( corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/util/metrics" "k8s.io/kubernetes/pkg/util/slice" @@ -78,8 +78,8 @@ func (c *Controller) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() - glog.Infof("Starting PV protection controller") - defer glog.Infof("Shutting down PV protection controller") + klog.Infof("Starting PV protection controller") + defer klog.Infof("Shutting down PV protection controller") if !controller.WaitForCacheSync("PV protection", stopCh, c.pvListerSynced) { return @@ -120,15 +120,15 @@ func (c *Controller) processNextWorkItem() bool { } func (c *Controller) processPV(pvName string) error { - glog.V(4).Infof("Processing PV %s", pvName) + klog.V(4).Infof("Processing PV %s", pvName) startTime := time.Now() defer func() { - glog.V(4).Infof("Finished processing PV %s (%v)", pvName, time.Since(startTime)) + klog.V(4).Infof("Finished processing PV %s (%v)", pvName, time.Since(startTime)) }() pv, err := c.pvLister.Get(pvName) if apierrs.IsNotFound(err) { - glog.V(4).Infof("PV %s not found, ignoring", pvName) + klog.V(4).Infof("PV %s not found, ignoring", pvName) return nil } if err != nil { @@ -163,10 +163,10 @@ func (c *Controller) addFinalizer(pv *v1.PersistentVolume) error { pvClone.ObjectMeta.Finalizers = append(pvClone.ObjectMeta.Finalizers, volumeutil.PVProtectionFinalizer) _, err := c.client.CoreV1().PersistentVolumes().Update(pvClone) if err != nil { - glog.V(3).Infof("Error adding protection finalizer to PV %s: %v", pv.Name, err) + klog.V(3).Infof("Error adding protection finalizer to PV %s: %v", pv.Name, err) return err } - glog.V(3).Infof("Added protection finalizer to PV %s", pv.Name) + klog.V(3).Infof("Added protection finalizer to PV %s", pv.Name) return nil } @@ -175,10 +175,10 @@ func (c *Controller) removeFinalizer(pv *v1.PersistentVolume) error { pvClone.ObjectMeta.Finalizers = slice.RemoveString(pvClone.ObjectMeta.Finalizers, volumeutil.PVProtectionFinalizer, nil) _, err := c.client.CoreV1().PersistentVolumes().Update(pvClone) if err != nil { - glog.V(3).Infof("Error removing protection finalizer from PV %s: %v", pv.Name, err) + klog.V(3).Infof("Error removing protection finalizer from PV %s: %v", pv.Name, err) return err } - glog.V(3).Infof("Removed protection finalizer from PV %s", pv.Name) + klog.V(3).Infof("Removed protection finalizer from PV %s", pv.Name) return nil } @@ -200,7 +200,7 @@ func (c *Controller) pvAddedUpdated(obj interface{}) { utilruntime.HandleError(fmt.Errorf("PV informer returned non-PV object: %#v", obj)) return } - glog.V(4).Infof("Got event on PV %s", pv.Name) + klog.V(4).Infof("Got event on PV %s", pv.Name) if needToAddFinalizer(pv) || isDeletionCandidate(pv) { c.queue.Add(pv.Name) diff --git a/pkg/controller/volume/pvprotection/pv_protection_controller_test.go b/pkg/controller/volume/pvprotection/pv_protection_controller_test.go index ba7b342a3d8..75caba4dab1 100644 --- a/pkg/controller/volume/pvprotection/pv_protection_controller_test.go +++ b/pkg/controller/volume/pvprotection/pv_protection_controller_test.go @@ -23,7 +23,7 @@ import ( "time" "github.com/davecgh/go-spew/spew" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -246,7 +246,7 @@ func TestPVProtectionController(t *testing.T) { break } if ctrl.queue.Len() > 0 { - glog.V(5).Infof("Test %q: %d events queue, processing one", test.name, ctrl.queue.Len()) + klog.V(5).Infof("Test %q: %d events queue, processing one", test.name, ctrl.queue.Len()) ctrl.processNextWorkItem() } if ctrl.queue.Len() > 0 { @@ -257,7 +257,7 @@ func TestPVProtectionController(t *testing.T) { if currentActionCount < len(test.expectedActions) { // Do not log evey wait, only when the action count changes. if lastReportedActionCount < currentActionCount { - glog.V(5).Infof("Test %q: got %d actions out of %d, waiting for the rest", test.name, currentActionCount, len(test.expectedActions)) + klog.V(5).Infof("Test %q: got %d actions out of %d, waiting for the rest", test.name, currentActionCount, len(test.expectedActions)) lastReportedActionCount = currentActionCount } // The test expected more to happen, wait for the actions. diff --git a/pkg/credentialprovider/BUILD b/pkg/credentialprovider/BUILD index f83f75be26b..0dfa2dfdc59 100644 --- a/pkg/credentialprovider/BUILD +++ b/pkg/credentialprovider/BUILD @@ -18,7 +18,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/credentialprovider", deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/credentialprovider/aws/BUILD b/pkg/credentialprovider/aws/BUILD index 19857243558..5161307a712 100644 --- a/pkg/credentialprovider/aws/BUILD +++ b/pkg/credentialprovider/aws/BUILD @@ -16,7 +16,7 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/ecr:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/credentialprovider/aws/aws_credentials.go b/pkg/credentialprovider/aws/aws_credentials.go index 862c2c62c6f..89869e76eaa 100644 --- a/pkg/credentialprovider/aws/aws_credentials.go +++ b/pkg/credentialprovider/aws/aws_credentials.go @@ -26,7 +26,7 @@ import ( "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ecr" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/credentialprovider" ) @@ -47,7 +47,7 @@ func awsHandlerLogger(req *request.Request) { name = req.Operation.Name } - glog.V(3).Infof("AWS request: %s:%s in %s", service, name, *region) + klog.V(3).Infof("AWS request: %s:%s in %s", service, name, *region) } // An interface for testing purposes. @@ -101,7 +101,7 @@ func registryURL(region string) string { // This should be called only if using the AWS cloud provider. // This way, we avoid timeouts waiting for a non-existent provider. func RegisterCredentialsProvider(region string) { - glog.V(4).Infof("registering credentials provider for AWS region %q", region) + klog.V(4).Infof("registering credentials provider for AWS region %q", region) credentialprovider.RegisterCredentialProvider("aws-ecr-"+region, &lazyEcrProvider{ @@ -122,7 +122,7 @@ func (p *lazyEcrProvider) Enabled() bool { // provider only when we actually need it the first time. func (p *lazyEcrProvider) LazyProvide() *credentialprovider.DockerConfigEntry { if p.actualProvider == nil { - glog.V(2).Infof("Creating ecrProvider for %s", p.region) + klog.V(2).Infof("Creating ecrProvider for %s", p.region) p.actualProvider = &credentialprovider.CachingDockerConfigProvider{ Provider: newEcrProvider(p.region, nil), // Refresh credentials a little earlier than expiration time @@ -161,7 +161,7 @@ func newEcrProvider(region string, getter tokenGetter) *ecrProvider { // use ECR somehow? func (p *ecrProvider) Enabled() bool { if p.region == "" { - glog.Errorf("Called ecrProvider.Enabled() with no region set") + klog.Errorf("Called ecrProvider.Enabled() with no region set") return false } @@ -191,11 +191,11 @@ func (p *ecrProvider) Provide() credentialprovider.DockerConfig { params := &ecr.GetAuthorizationTokenInput{} output, err := p.getter.GetAuthorizationToken(params) if err != nil { - glog.Errorf("while requesting ECR authorization token %v", err) + klog.Errorf("while requesting ECR authorization token %v", err) return cfg } if output == nil { - glog.Errorf("Got back no ECR token") + klog.Errorf("Got back no ECR token") return cfg } @@ -204,7 +204,7 @@ func (p *ecrProvider) Provide() credentialprovider.DockerConfig { data.AuthorizationToken != nil { decodedToken, err := base64.StdEncoding.DecodeString(aws.StringValue(data.AuthorizationToken)) if err != nil { - glog.Errorf("while decoding token for endpoint %v %v", data.ProxyEndpoint, err) + klog.Errorf("while decoding token for endpoint %v %v", data.ProxyEndpoint, err) return cfg } parts := strings.SplitN(string(decodedToken), ":", 2) @@ -217,7 +217,7 @@ func (p *ecrProvider) Provide() credentialprovider.DockerConfig { Email: "not@val.id", } - glog.V(3).Infof("Adding credentials for user %s in %s", user, p.region) + klog.V(3).Infof("Adding credentials for user %s in %s", user, p.region) // Add our config entry for this region's registry URLs cfg[p.regionURL] = entry diff --git a/pkg/credentialprovider/azure/BUILD b/pkg/credentialprovider/azure/BUILD index 5d1c3800d7f..aa1b29b2c78 100644 --- a/pkg/credentialprovider/azure/BUILD +++ b/pkg/credentialprovider/azure/BUILD @@ -21,9 +21,9 @@ go_library( "//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library", "//vendor/github.com/dgrijalva/jwt-go:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/pkg/credentialprovider/azure/azure_credentials.go b/pkg/credentialprovider/azure/azure_credentials.go index ffeab5ea5ab..28e43048d22 100644 --- a/pkg/credentialprovider/azure/azure_credentials.go +++ b/pkg/credentialprovider/azure/azure_credentials.go @@ -27,9 +27,9 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/adal" "github.com/Azure/go-autorest/autorest/azure" - "github.com/ghodss/yaml" - "github.com/golang/glog" "github.com/spf13/pflag" + "k8s.io/klog" + "sigs.k8s.io/yaml" "k8s.io/kubernetes/pkg/cloudprovider/providers/azure/auth" "k8s.io/kubernetes/pkg/credentialprovider" @@ -133,7 +133,7 @@ func (a *acrProvider) loadConfig(rdr io.Reader) error { var err error a.config, err = parseConfig(rdr) if err != nil { - glog.Errorf("Failed to load azure credential file: %v", err) + klog.Errorf("Failed to load azure credential file: %v", err) } a.environment, err = auth.ParseAzureEnvironment(a.config.Cloud) @@ -146,26 +146,26 @@ func (a *acrProvider) loadConfig(rdr io.Reader) error { func (a *acrProvider) Enabled() bool { if a.file == nil || len(*a.file) == 0 { - glog.V(5).Infof("Azure config unspecified, disabling") + klog.V(5).Infof("Azure config unspecified, disabling") return false } f, err := os.Open(*a.file) if err != nil { - glog.Errorf("Failed to load config from file: %s", *a.file) + klog.Errorf("Failed to load config from file: %s", *a.file) return false } defer f.Close() err = a.loadConfig(f) if err != nil { - glog.Errorf("Failed to load config from file: %s", *a.file) + klog.Errorf("Failed to load config from file: %s", *a.file) return false } a.servicePrincipalToken, err = auth.GetServicePrincipalToken(a.config, a.environment) if err != nil { - glog.Errorf("Failed to create service principal token: %v", err) + klog.Errorf("Failed to create service principal token: %v", err) return false } @@ -179,16 +179,16 @@ func (a *acrProvider) Provide() credentialprovider.DockerConfig { defer cancel() if a.config.UseManagedIdentityExtension { - glog.V(4).Infof("listing registries") + klog.V(4).Infof("listing registries") result, err := a.registryClient.List(ctx) if err != nil { - glog.Errorf("Failed to list registries: %v", err) + klog.Errorf("Failed to list registries: %v", err) return cfg } for ix := range result { loginServer := getLoginServer(result[ix]) - glog.V(2).Infof("loginServer: %s", loginServer) + klog.V(2).Infof("loginServer: %s", loginServer) cred, err := getACRDockerEntryFromARMToken(a, loginServer) if err != nil { continue @@ -216,22 +216,22 @@ func getLoginServer(registry containerregistry.Registry) string { func getACRDockerEntryFromARMToken(a *acrProvider, loginServer string) (*credentialprovider.DockerConfigEntry, error) { armAccessToken := a.servicePrincipalToken.OAuthToken() - glog.V(4).Infof("discovering auth redirects for: %s", loginServer) + klog.V(4).Infof("discovering auth redirects for: %s", loginServer) directive, err := receiveChallengeFromLoginServer(loginServer) if err != nil { - glog.Errorf("failed to receive challenge: %s", err) + klog.Errorf("failed to receive challenge: %s", err) return nil, err } - glog.V(4).Infof("exchanging an acr refresh_token") + klog.V(4).Infof("exchanging an acr refresh_token") registryRefreshToken, err := performTokenExchange( loginServer, directive, a.config.TenantID, armAccessToken) if err != nil { - glog.Errorf("failed to perform token exchange: %s", err) + klog.Errorf("failed to perform token exchange: %s", err) return nil, err } - glog.V(4).Infof("adding ACR docker config entry for: %s", loginServer) + klog.V(4).Infof("adding ACR docker config entry for: %s", loginServer) return &credentialprovider.DockerConfigEntry{ Username: dockerTokenLoginUsernameGUID, Password: registryRefreshToken, diff --git a/pkg/credentialprovider/config.go b/pkg/credentialprovider/config.go index 433f28b1544..a43e8c2b15d 100644 --- a/pkg/credentialprovider/config.go +++ b/pkg/credentialprovider/config.go @@ -27,7 +27,7 @@ import ( "strings" "sync" - "github.com/golang/glog" + "k8s.io/klog" ) // DockerConfigJson represents ~/.docker/config.json file info @@ -95,21 +95,21 @@ func ReadDockercfgFile(searchPaths []string) (cfg DockerConfig, err error) { for _, configPath := range searchPaths { absDockerConfigFileLocation, err := filepath.Abs(filepath.Join(configPath, configFileName)) if err != nil { - glog.Errorf("while trying to canonicalize %s: %v", configPath, err) + klog.Errorf("while trying to canonicalize %s: %v", configPath, err) continue } - glog.V(4).Infof("looking for .dockercfg at %s", absDockerConfigFileLocation) + klog.V(4).Infof("looking for .dockercfg at %s", absDockerConfigFileLocation) contents, err := ioutil.ReadFile(absDockerConfigFileLocation) if os.IsNotExist(err) { continue } if err != nil { - glog.V(4).Infof("while trying to read %s: %v", absDockerConfigFileLocation, err) + klog.V(4).Infof("while trying to read %s: %v", absDockerConfigFileLocation, err) continue } cfg, err := readDockerConfigFileFromBytes(contents) if err == nil { - glog.V(4).Infof("found .dockercfg at %s", absDockerConfigFileLocation) + klog.V(4).Infof("found .dockercfg at %s", absDockerConfigFileLocation) return cfg, nil } } @@ -125,18 +125,18 @@ func ReadDockerConfigJSONFile(searchPaths []string) (cfg DockerConfig, err error for _, configPath := range searchPaths { absDockerConfigFileLocation, err := filepath.Abs(filepath.Join(configPath, configJsonFileName)) if err != nil { - glog.Errorf("while trying to canonicalize %s: %v", configPath, err) + klog.Errorf("while trying to canonicalize %s: %v", configPath, err) continue } - glog.V(4).Infof("looking for %s at %s", configJsonFileName, absDockerConfigFileLocation) + klog.V(4).Infof("looking for %s at %s", configJsonFileName, absDockerConfigFileLocation) cfg, err = ReadSpecificDockerConfigJsonFile(absDockerConfigFileLocation) if err != nil { if !os.IsNotExist(err) { - glog.V(4).Infof("while trying to read %s: %v", absDockerConfigFileLocation, err) + klog.V(4).Infof("while trying to read %s: %v", absDockerConfigFileLocation, err) } continue } - glog.V(4).Infof("found valid %s at %s", configJsonFileName, absDockerConfigFileLocation) + klog.V(4).Infof("found valid %s at %s", configJsonFileName, absDockerConfigFileLocation) return cfg, nil } return nil, fmt.Errorf("couldn't find valid %s after checking in %v", configJsonFileName, searchPaths) @@ -188,7 +188,7 @@ func ReadUrl(url string, client *http.Client, header *http.Header) (body []byte, defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - glog.V(2).Infof("body of failing http response: %v", resp.Body) + klog.V(2).Infof("body of failing http response: %v", resp.Body) return nil, &HttpError{ StatusCode: resp.StatusCode, Url: url, @@ -213,7 +213,7 @@ func ReadDockerConfigFileFromUrl(url string, client *http.Client, header *http.H func readDockerConfigFileFromBytes(contents []byte) (cfg DockerConfig, err error) { if err = json.Unmarshal(contents, &cfg); err != nil { - glog.Errorf("while trying to parse blob %q: %v", contents, err) + klog.Errorf("while trying to parse blob %q: %v", contents, err) return nil, err } return @@ -222,7 +222,7 @@ func readDockerConfigFileFromBytes(contents []byte) (cfg DockerConfig, err error func readDockerConfigJsonFileFromBytes(contents []byte) (cfg DockerConfig, err error) { var cfgJson DockerConfigJson if err = json.Unmarshal(contents, &cfgJson); err != nil { - glog.Errorf("while trying to parse blob %q: %v", contents, err) + klog.Errorf("while trying to parse blob %q: %v", contents, err) return nil, err } cfg = cfgJson.Auths diff --git a/pkg/credentialprovider/gcp/BUILD b/pkg/credentialprovider/gcp/BUILD index 6b34743bca7..1d61eaaef33 100644 --- a/pkg/credentialprovider/gcp/BUILD +++ b/pkg/credentialprovider/gcp/BUILD @@ -16,7 +16,7 @@ go_library( deps = [ "//pkg/credentialprovider:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/credentialprovider/gcp/metadata.go b/pkg/credentialprovider/gcp/metadata.go index 4c668d0b384..7620f0c040f 100644 --- a/pkg/credentialprovider/gcp/metadata.go +++ b/pkg/credentialprovider/gcp/metadata.go @@ -23,8 +23,8 @@ import ( "strings" "time" - "github.com/golang/glog" utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/klog" "k8s.io/kubernetes/pkg/credentialprovider" ) @@ -117,7 +117,7 @@ func init() { func onGCEVM() bool { data, err := ioutil.ReadFile(gceProductNameFile) if err != nil { - glog.V(2).Infof("Error while reading product_name: %v", err) + klog.V(2).Infof("Error while reading product_name: %v", err) return false } name := strings.TrimSpace(string(data)) @@ -139,7 +139,7 @@ func (g *dockerConfigKeyProvider) Provide() credentialprovider.DockerConfig { // Read the contents of the google-dockercfg metadata key and // parse them as an alternate .dockercfg if cfg, err := credentialprovider.ReadDockerConfigFileFromUrl(dockerConfigKey, g.Client, metadataHeader); err != nil { - glog.Errorf("while reading 'google-dockercfg' metadata: %v", err) + klog.Errorf("while reading 'google-dockercfg' metadata: %v", err) } else { return cfg } @@ -156,17 +156,17 @@ func (g *dockerConfigUrlKeyProvider) LazyProvide() *credentialprovider.DockerCon func (g *dockerConfigUrlKeyProvider) Provide() credentialprovider.DockerConfig { // Read the contents of the google-dockercfg-url key and load a .dockercfg from there if url, err := credentialprovider.ReadUrl(dockerConfigUrlKey, g.Client, metadataHeader); err != nil { - glog.Errorf("while reading 'google-dockercfg-url' metadata: %v", err) + klog.Errorf("while reading 'google-dockercfg-url' metadata: %v", err) } else { if strings.HasPrefix(string(url), "http") { if cfg, err := credentialprovider.ReadDockerConfigFileFromUrl(string(url), g.Client, nil); err != nil { - glog.Errorf("while reading 'google-dockercfg-url'-specified url: %s, %v", string(url), err) + klog.Errorf("while reading 'google-dockercfg-url'-specified url: %s, %v", string(url), err) } else { return cfg } } else { // TODO(mattmoor): support reading alternate scheme URLs (e.g. gs:// or s3://) - glog.Errorf("Unsupported URL scheme: %s", string(url)) + klog.Errorf("Unsupported URL scheme: %s", string(url)) } } @@ -209,7 +209,7 @@ func (g *containerRegistryProvider) Enabled() bool { value := runWithBackoff(func() ([]byte, error) { value, err := credentialprovider.ReadUrl(serviceAccounts, g.Client, metadataHeader) if err != nil { - glog.V(2).Infof("Failed to Get service accounts from gce metadata server: %v", err) + klog.V(2).Infof("Failed to Get service accounts from gce metadata server: %v", err) } return value, err }) @@ -225,20 +225,20 @@ func (g *containerRegistryProvider) Enabled() bool { } } if !defaultServiceAccountExists { - glog.V(2).Infof("'default' service account does not exist. Found following service accounts: %q", string(value)) + klog.V(2).Infof("'default' service account does not exist. Found following service accounts: %q", string(value)) return false } url := metadataScopes + "?alt=json" value = runWithBackoff(func() ([]byte, error) { value, err := credentialprovider.ReadUrl(url, g.Client, metadataHeader) if err != nil { - glog.V(2).Infof("Failed to Get scopes in default service account from gce metadata server: %v", err) + klog.V(2).Infof("Failed to Get scopes in default service account from gce metadata server: %v", err) } return value, err }) var scopes []string if err := json.Unmarshal(value, &scopes); err != nil { - glog.Errorf("Failed to unmarshal scopes: %v", err) + klog.Errorf("Failed to unmarshal scopes: %v", err) return false } for _, v := range scopes { @@ -247,7 +247,7 @@ func (g *containerRegistryProvider) Enabled() bool { return true } } - glog.Warningf("Google container registry is disabled, no storage scope is available: %s", value) + klog.Warningf("Google container registry is disabled, no storage scope is available: %s", value) return false } @@ -268,19 +268,19 @@ func (g *containerRegistryProvider) Provide() credentialprovider.DockerConfig { tokenJsonBlob, err := credentialprovider.ReadUrl(metadataToken, g.Client, metadataHeader) if err != nil { - glog.Errorf("while reading access token endpoint: %v", err) + klog.Errorf("while reading access token endpoint: %v", err) return cfg } email, err := credentialprovider.ReadUrl(metadataEmail, g.Client, metadataHeader) if err != nil { - glog.Errorf("while reading email endpoint: %v", err) + klog.Errorf("while reading email endpoint: %v", err) return cfg } var parsedBlob tokenBlob if err := json.Unmarshal([]byte(tokenJsonBlob), &parsedBlob); err != nil { - glog.Errorf("while parsing json blob %s: %v", tokenJsonBlob, err) + klog.Errorf("while parsing json blob %s: %v", tokenJsonBlob, err) return cfg } diff --git a/pkg/credentialprovider/keyring.go b/pkg/credentialprovider/keyring.go index 8a6f563d085..6f5fad5fc47 100644 --- a/pkg/credentialprovider/keyring.go +++ b/pkg/credentialprovider/keyring.go @@ -23,7 +23,7 @@ import ( "sort" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/sets" ) @@ -113,7 +113,7 @@ func (dk *BasicDockerKeyring) Add(cfg DockerConfig) { } parsed, err := url.Parse(value) if err != nil { - glog.Errorf("Entry %q in dockercfg invalid (%v), ignoring", loc, err) + klog.Errorf("Entry %q in dockercfg invalid (%v), ignoring", loc, err) continue } diff --git a/pkg/credentialprovider/plugins.go b/pkg/credentialprovider/plugins.go index c817fefa2b7..5ea3a000e8b 100644 --- a/pkg/credentialprovider/plugins.go +++ b/pkg/credentialprovider/plugins.go @@ -21,7 +21,7 @@ import ( "sort" "sync" - "github.com/golang/glog" + "k8s.io/klog" ) // All registered credential providers. @@ -38,9 +38,9 @@ func RegisterCredentialProvider(name string, provider DockerConfigProvider) { defer providersMutex.Unlock() _, found := providers[name] if found { - glog.Fatalf("Credential provider %q was registered twice", name) + klog.Fatalf("Credential provider %q was registered twice", name) } - glog.V(4).Infof("Registered credential provider %q", name) + klog.V(4).Infof("Registered credential provider %q", name) providers[name] = provider } @@ -61,7 +61,7 @@ func NewDockerKeyring() DockerKeyring { for _, key := range stringKeys { provider := providers[key] if provider.Enabled() { - glog.V(4).Infof("Registering credential provider: %v", key) + klog.V(4).Infof("Registering credential provider: %v", key) keyring.Providers = append(keyring.Providers, provider) } } diff --git a/pkg/credentialprovider/provider.go b/pkg/credentialprovider/provider.go index 422696e9b0b..16b4e601a10 100644 --- a/pkg/credentialprovider/provider.go +++ b/pkg/credentialprovider/provider.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" ) // DockerConfigProvider is the interface that registered extensions implement @@ -83,7 +83,7 @@ func (d *defaultDockerConfigProvider) Provide() DockerConfig { if cfg, err := ReadDockerConfigFile(); err == nil { return cfg } else if !os.IsNotExist(err) { - glog.V(4).Infof("Unable to parse Docker config file: %v", err) + klog.V(4).Infof("Unable to parse Docker config file: %v", err) } return DockerConfig{} } @@ -113,7 +113,7 @@ func (d *CachingDockerConfigProvider) Provide() DockerConfig { return d.cacheDockerConfig } - glog.V(2).Infof("Refreshing cache for provider: %v", reflect.TypeOf(d.Provider).String()) + klog.V(2).Infof("Refreshing cache for provider: %v", reflect.TypeOf(d.Provider).String()) d.cacheDockerConfig = d.Provider.Provide() d.expiration = time.Now().Add(d.Lifetime) return d.cacheDockerConfig diff --git a/pkg/credentialprovider/rancher/BUILD b/pkg/credentialprovider/rancher/BUILD index af1f490b103..02ab95c7fce 100644 --- a/pkg/credentialprovider/rancher/BUILD +++ b/pkg/credentialprovider/rancher/BUILD @@ -25,8 +25,8 @@ go_library( importpath = "k8s.io/kubernetes/pkg/credentialprovider/rancher", deps = [ "//pkg/credentialprovider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/rancher/go-rancher/client:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/credentialprovider/rancher/rancher_registry_credentials.go b/pkg/credentialprovider/rancher/rancher_registry_credentials.go index 3143560dfe9..d122308ed82 100644 --- a/pkg/credentialprovider/rancher/rancher_registry_credentials.go +++ b/pkg/credentialprovider/rancher/rancher_registry_credentials.go @@ -20,8 +20,8 @@ import ( "os" "time" - "github.com/golang/glog" "github.com/rancher/go-rancher/client" + "k8s.io/klog" "k8s.io/kubernetes/pkg/credentialprovider" ) @@ -102,13 +102,13 @@ func (g *rancherCredentialsGetter) getCredentials() []registryCredential { var registryCreds []registryCredential credColl, err := g.client.RegistryCredential.List(client.NewListOpts()) if err != nil { - glog.Errorf("Failed to pull registry credentials from rancher %v", err) + klog.Errorf("Failed to pull registry credentials from rancher %v", err) return registryCreds } for _, cred := range credColl.Data { registry := &client.Registry{} if err = g.client.GetLink(cred.Resource, "registry", registry); err != nil { - glog.Errorf("Failed to pull registry from rancher %v", err) + klog.Errorf("Failed to pull registry from rancher %v", err) return registryCreds } registryCred := registryCredential{ diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 3960bf081d8..cd35ecb7044 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -188,14 +188,13 @@ const ( MountContainers utilfeature.Feature = "MountContainers" // owner: @msau42 - // alpha: v1.9 + // GA: v1.13 // // Extend the default scheduler to be aware of PV topology and handle PV binding - // Before moving to beta, resolve Kubernetes issue #56180 VolumeScheduling utilfeature.Feature = "VolumeScheduling" // owner: @vladimirvivien - // beta: v1.10 + // GA: v1.13 // // Enable mount/attachment of Container Storage Interface (CSI) backed PVs CSIPersistentVolume utilfeature.Feature = "CSIPersistentVolume" @@ -218,6 +217,7 @@ const ( // owner: @screeley44 // alpha: v1.9 + // beta: v1.13 // // Enable Block volume support in containers. BlockVolume utilfeature.Feature = "BlockVolume" @@ -270,6 +270,14 @@ const ( // Enable ServiceAccountTokenVolumeProjection support in ProjectedVolumes. TokenRequestProjection utilfeature.Feature = "TokenRequestProjection" + // owner: @mikedanese + // alpha: v1.13 + // + // Migrate ServiceAccount volumes to use a projected volume consisting of a + // ServiceAccountTokenVolumeProjection. This feature adds new required flags + // to the API server. + BoundServiceAccountTokenVolume utilfeature.Feature = "BoundServiceAccountTokenVolume" + // owner: @Random-Liu // beta: v1.11 // @@ -277,7 +285,7 @@ const ( CRIContainerLogRotation utilfeature.Feature = "CRIContainerLogRotation" // owner: @verult - // beta: v1.10 + // GA: v1.13 // // Enables the regional PD feature on GCE. GCERegionalPersistentDisk utilfeature.Feature = "GCERegionalPersistentDisk" @@ -324,7 +332,7 @@ const ( VolumeSubpathEnvExpansion utilfeature.Feature = "VolumeSubpathEnvExpansion" // owner: @vikaschoudhary16 - // alpha: v1.11 + // GA: v1.13 // // // Enable probe based plugin watcher utility for discovering Kubelet plugins @@ -379,6 +387,12 @@ const ( // // Allow TTL controller to clean up Pods and Jobs after they finish. TTLAfterFinished utilfeature.Feature = "TTLAfterFinished" + + // owner: @dashpole + // alpha: v1.13 + // + // Enables the kubelet's pod resources grpc endpoint + KubeletPodResources utilfeature.Feature = "KubeletPodResources" ) func init() { @@ -415,28 +429,29 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS CPUCFSQuotaPeriod: {Default: false, PreRelease: utilfeature.Alpha}, ServiceNodeExclusion: {Default: false, PreRelease: utilfeature.Alpha}, MountContainers: {Default: false, PreRelease: utilfeature.Alpha}, - VolumeScheduling: {Default: true, PreRelease: utilfeature.Beta}, - CSIPersistentVolume: {Default: true, PreRelease: utilfeature.Beta}, + VolumeScheduling: {Default: true, PreRelease: utilfeature.GA}, + CSIPersistentVolume: {Default: true, PreRelease: utilfeature.GA}, CSIDriverRegistry: {Default: false, PreRelease: utilfeature.Alpha}, CSINodeInfo: {Default: false, PreRelease: utilfeature.Alpha}, CustomPodDNS: {Default: true, PreRelease: utilfeature.Beta}, - BlockVolume: {Default: false, PreRelease: utilfeature.Alpha}, + BlockVolume: {Default: true, PreRelease: utilfeature.Beta}, StorageObjectInUseProtection: {Default: true, PreRelease: utilfeature.GA}, - ResourceLimitsPriorityFunction: {Default: true, PreRelease: utilfeature.Beta}, + ResourceLimitsPriorityFunction: {Default: false, PreRelease: utilfeature.Alpha}, SupportIPVSProxyMode: {Default: true, PreRelease: utilfeature.GA}, SupportPodPidsLimit: {Default: false, PreRelease: utilfeature.Alpha}, HyperVContainer: {Default: false, PreRelease: utilfeature.Alpha}, ScheduleDaemonSetPods: {Default: true, PreRelease: utilfeature.Beta}, TokenRequest: {Default: true, PreRelease: utilfeature.Beta}, TokenRequestProjection: {Default: true, PreRelease: utilfeature.Beta}, + BoundServiceAccountTokenVolume: {Default: false, PreRelease: utilfeature.Alpha}, CRIContainerLogRotation: {Default: true, PreRelease: utilfeature.Beta}, - GCERegionalPersistentDisk: {Default: true, PreRelease: utilfeature.Beta}, + GCERegionalPersistentDisk: {Default: true, PreRelease: utilfeature.GA}, RunAsGroup: {Default: false, PreRelease: utilfeature.Alpha}, VolumeSubpath: {Default: true, PreRelease: utilfeature.GA}, BalanceAttachedNodeVolumes: {Default: false, PreRelease: utilfeature.Alpha}, PodReadinessGates: {Default: true, PreRelease: utilfeature.Beta}, VolumeSubpathEnvExpansion: {Default: false, PreRelease: utilfeature.Alpha}, - KubeletPluginsWatcher: {Default: true, PreRelease: utilfeature.Beta}, + KubeletPluginsWatcher: {Default: true, PreRelease: utilfeature.GA}, ResourceQuotaScopeSelectors: {Default: true, PreRelease: utilfeature.Beta}, CSIBlockVolume: {Default: false, PreRelease: utilfeature.Alpha}, RuntimeClass: {Default: false, PreRelease: utilfeature.Alpha}, @@ -445,11 +460,13 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS VolumeSnapshotDataSource: {Default: false, PreRelease: utilfeature.Alpha}, ProcMountType: {Default: false, PreRelease: utilfeature.Alpha}, TTLAfterFinished: {Default: false, PreRelease: utilfeature.Alpha}, + KubeletPodResources: {Default: false, PreRelease: utilfeature.Alpha}, // inherited features from generic apiserver, relisted here to get a conflict if it is changed // unintentionally on either side: genericfeatures.StreamingProxyRedirects: {Default: true, PreRelease: utilfeature.Beta}, genericfeatures.AdvancedAuditing: {Default: true, PreRelease: utilfeature.GA}, + genericfeatures.DynamicAuditing: {Default: false, PreRelease: utilfeature.Alpha}, genericfeatures.APIResponseCompression: {Default: false, PreRelease: utilfeature.Alpha}, genericfeatures.Initializers: {Default: false, PreRelease: utilfeature.Alpha}, genericfeatures.APIListChunking: {Default: true, PreRelease: utilfeature.Beta}, diff --git a/pkg/kubeapiserver/BUILD b/pkg/kubeapiserver/BUILD index 2e2b390e1f6..ef4c4b1e22d 100644 --- a/pkg/kubeapiserver/BUILD +++ b/pkg/kubeapiserver/BUILD @@ -50,7 +50,6 @@ filegroup( "//pkg/kubeapiserver/authorizer:all-srcs", "//pkg/kubeapiserver/options:all-srcs", "//pkg/kubeapiserver/server:all-srcs", - "//pkg/kubeapiserver/util:all-srcs", ], tags = ["automanaged"], ) diff --git a/pkg/kubeapiserver/admission/BUILD b/pkg/kubeapiserver/admission/BUILD index fb9a80a8e92..abe660e7177 100644 --- a/pkg/kubeapiserver/admission/BUILD +++ b/pkg/kubeapiserver/admission/BUILD @@ -23,7 +23,7 @@ go_library( "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/restmapper:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubeapiserver/admission/config.go b/pkg/kubeapiserver/admission/config.go index bf88b932dba..a47e4559377 100644 --- a/pkg/kubeapiserver/admission/config.go +++ b/pkg/kubeapiserver/admission/config.go @@ -21,7 +21,7 @@ import ( "net/http" "time" - "github.com/golang/glog" + "k8s.io/klog" utilwait "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/admission" @@ -37,14 +37,15 @@ import ( quotainstall "k8s.io/kubernetes/pkg/quota/v1/install" ) -// AdmissionConfig holds the configuration for initializing the admission plugins -type AdmissionConfig struct { +// Config holds the configuration needed to for initialize the admission plugins +type Config struct { CloudConfigFile string LoopbackClientConfig *rest.Config ExternalInformers externalinformers.SharedInformerFactory } -func (c *AdmissionConfig) New(proxyTransport *http.Transport, serviceResolver webhook.ServiceResolver) ([]admission.PluginInitializer, server.PostStartHookFunc, error) { +// New sets up the plugins and admission start hooks needed for admission +func (c *Config) New(proxyTransport *http.Transport, serviceResolver webhook.ServiceResolver) ([]admission.PluginInitializer, server.PostStartHookFunc, error) { webhookAuthResolverWrapper := webhook.NewDefaultAuthenticationInfoResolverWrapper(proxyTransport, c.LoopbackClientConfig) webhookPluginInitializer := webhookinit.NewPluginInitializer(webhookAuthResolverWrapper, serviceResolver) @@ -53,7 +54,7 @@ func (c *AdmissionConfig) New(proxyTransport *http.Transport, serviceResolver we var err error cloudConfig, err = ioutil.ReadFile(c.CloudConfigFile) if err != nil { - glog.Fatalf("Error reading from cloud configuration file %s: %#v", c.CloudConfigFile, err) + klog.Fatalf("Error reading from cloud configuration file %s: %#v", c.CloudConfigFile, err) } } internalClient, err := internalclientset.NewForConfig(c.LoopbackClientConfig) diff --git a/pkg/kubeapiserver/admission/initializer_test.go b/pkg/kubeapiserver/admission/initializer_test.go index 053623fb377..f5a6b144c6a 100644 --- a/pkg/kubeapiserver/admission/initializer_test.go +++ b/pkg/kubeapiserver/admission/initializer_test.go @@ -33,8 +33,8 @@ type WantsCloudConfigAdmissionPlugin struct { cloudConfig []byte } -func (self *WantsCloudConfigAdmissionPlugin) SetCloudConfig(cloudConfig []byte) { - self.cloudConfig = cloudConfig +func (p *WantsCloudConfigAdmissionPlugin) SetCloudConfig(cloudConfig []byte) { + p.cloudConfig = cloudConfig } func TestCloudConfigAdmissionPlugin(t *testing.T) { diff --git a/pkg/kubeapiserver/authenticator/OWNERS b/pkg/kubeapiserver/authenticator/OWNERS new file mode 100644 index 00000000000..c607d2aa8c5 --- /dev/null +++ b/pkg/kubeapiserver/authenticator/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authenticators-approvers +reviewers: +- sig-auth-authenticators-reviewers +labels: +- sig/auth + diff --git a/pkg/kubeapiserver/authenticator/config.go b/pkg/kubeapiserver/authenticator/config.go index 976666f5e63..b34e6b8958a 100644 --- a/pkg/kubeapiserver/authenticator/config.go +++ b/pkg/kubeapiserver/authenticator/config.go @@ -38,13 +38,15 @@ import ( "k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth" "k8s.io/apiserver/plugin/pkg/authenticator/token/oidc" "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook" + // Initialize all known client auth plugins. _ "k8s.io/client-go/plugin/pkg/client/auth" certutil "k8s.io/client-go/util/cert" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/serviceaccount" ) -type AuthenticatorConfig struct { +// Config contains the data on how to authenticate a request to the Kube API Server +type Config struct { Anonymous bool BasicAuthFile string BootstrapToken bool @@ -62,7 +64,7 @@ type AuthenticatorConfig struct { ServiceAccountKeyFiles []string ServiceAccountLookup bool ServiceAccountIssuer string - APIAudiences []string + APIAudiences authenticator.Audiences WebhookTokenAuthnConfigFile string WebhookTokenAuthnCacheTTL time.Duration @@ -78,7 +80,7 @@ type AuthenticatorConfig struct { // New returns an authenticator.Request or an error that supports the standard // Kubernetes authentication mechanisms. -func (config AuthenticatorConfig) New() (authenticator.Request, *spec.SecurityDefinitions, error) { +func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, error) { var authenticators []authenticator.Request var tokenAuthenticators []authenticator.Token securityDefinitions := spec.SecurityDefinitions{} @@ -133,14 +135,14 @@ func (config AuthenticatorConfig) New() (authenticator.Request, *spec.SecurityDe tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, tokenAuth)) } if len(config.ServiceAccountKeyFiles) > 0 { - serviceAccountAuth, err := newLegacyServiceAccountAuthenticator(config.ServiceAccountKeyFiles, config.ServiceAccountLookup, config.ServiceAccountTokenGetter) + serviceAccountAuth, err := newLegacyServiceAccountAuthenticator(config.ServiceAccountKeyFiles, config.ServiceAccountLookup, config.APIAudiences, config.ServiceAccountTokenGetter) if err != nil { return nil, nil, err } - tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, serviceAccountAuth)) + tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth) } if utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) && config.ServiceAccountIssuer != "" { - serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountIssuer, config.APIAudiences, config.ServiceAccountKeyFiles, config.ServiceAccountTokenGetter) + serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountIssuer, config.ServiceAccountKeyFiles, config.APIAudiences, config.ServiceAccountTokenGetter) if err != nil { return nil, nil, err } @@ -159,14 +161,25 @@ func (config AuthenticatorConfig) New() (authenticator.Request, *spec.SecurityDe // simply returns an error, the OpenID Connect plugin may query the provider to // update the keys, causing performance hits. if len(config.OIDCIssuerURL) > 0 && len(config.OIDCClientID) > 0 { - oidcAuth, err := newAuthenticatorFromOIDCIssuerURL(config.OIDCIssuerURL, config.OIDCClientID, config.OIDCCAFile, config.OIDCUsernameClaim, config.OIDCUsernamePrefix, config.OIDCGroupsClaim, config.OIDCGroupsPrefix, config.OIDCSigningAlgs, config.OIDCRequiredClaims) + oidcAuth, err := newAuthenticatorFromOIDCIssuerURL(oidc.Options{ + IssuerURL: config.OIDCIssuerURL, + ClientID: config.OIDCClientID, + APIAudiences: config.APIAudiences, + CAFile: config.OIDCCAFile, + UsernameClaim: config.OIDCUsernameClaim, + UsernamePrefix: config.OIDCUsernamePrefix, + GroupsClaim: config.OIDCGroupsClaim, + GroupsPrefix: config.OIDCGroupsPrefix, + SupportedSigningAlgs: config.OIDCSigningAlgs, + RequiredClaims: config.OIDCRequiredClaims, + }) if err != nil { return nil, nil, err } tokenAuthenticators = append(tokenAuthenticators, oidcAuth) } if len(config.WebhookTokenAuthnConfigFile) > 0 { - webhookTokenAuth, err := newWebhookTokenAuthenticator(config.WebhookTokenAuthnConfigFile, config.WebhookTokenAuthnCacheTTL) + webhookTokenAuth, err := newWebhookTokenAuthenticator(config.WebhookTokenAuthnConfigFile, config.WebhookTokenAuthnCacheTTL, config.APIAudiences) if err != nil { return nil, nil, err } @@ -178,7 +191,7 @@ func (config AuthenticatorConfig) New() (authenticator.Request, *spec.SecurityDe tokenAuth := tokenunion.New(tokenAuthenticators...) // Optionally cache authentication results if config.TokenSuccessCacheTTL > 0 || config.TokenFailureCacheTTL > 0 { - tokenAuth = tokencache.New(tokenAuth, config.TokenSuccessCacheTTL, config.TokenFailureCacheTTL) + tokenAuth = tokencache.New(tokenAuth, true, config.TokenSuccessCacheTTL, config.TokenFailureCacheTTL) } authenticators = append(authenticators, bearertoken.New(tokenAuth), websocket.NewProtocolAuthenticator(tokenAuth)) securityDefinitions["BearerToken"] = &spec.SecurityScheme{ @@ -238,33 +251,23 @@ func newAuthenticatorFromTokenFile(tokenAuthFile string) (authenticator.Token, e } // newAuthenticatorFromOIDCIssuerURL returns an authenticator.Token or an error. -func newAuthenticatorFromOIDCIssuerURL(issuerURL, clientID, caFile, usernameClaim, usernamePrefix, groupsClaim, groupsPrefix string, signingAlgs []string, requiredClaims map[string]string) (authenticator.Token, error) { +func newAuthenticatorFromOIDCIssuerURL(opts oidc.Options) (authenticator.Token, error) { const noUsernamePrefix = "-" - if usernamePrefix == "" && usernameClaim != "email" { + if opts.UsernamePrefix == "" && opts.UsernameClaim != "email" { // Old behavior. If a usernamePrefix isn't provided, prefix all claims other than "email" // with the issuerURL. // // See https://github.com/kubernetes/kubernetes/issues/31380 - usernamePrefix = issuerURL + "#" + opts.UsernamePrefix = opts.IssuerURL + "#" } - if usernamePrefix == noUsernamePrefix { + if opts.UsernamePrefix == noUsernamePrefix { // Special value indicating usernames shouldn't be prefixed. - usernamePrefix = "" + opts.UsernamePrefix = "" } - tokenAuthenticator, err := oidc.New(oidc.Options{ - IssuerURL: issuerURL, - ClientID: clientID, - CAFile: caFile, - UsernameClaim: usernameClaim, - UsernamePrefix: usernamePrefix, - GroupsClaim: groupsClaim, - GroupsPrefix: groupsPrefix, - SupportedSigningAlgs: signingAlgs, - RequiredClaims: requiredClaims, - }) + tokenAuthenticator, err := oidc.New(opts) if err != nil { return nil, err } @@ -273,7 +276,7 @@ func newAuthenticatorFromOIDCIssuerURL(issuerURL, clientID, caFile, usernameClai } // newLegacyServiceAccountAuthenticator returns an authenticator.Token or an error -func newLegacyServiceAccountAuthenticator(keyfiles []string, lookup bool, serviceAccountGetter serviceaccount.ServiceAccountTokenGetter) (authenticator.Token, error) { +func newLegacyServiceAccountAuthenticator(keyfiles []string, lookup bool, apiAudiences authenticator.Audiences, serviceAccountGetter serviceaccount.ServiceAccountTokenGetter) (authenticator.Token, error) { allPublicKeys := []interface{}{} for _, keyfile := range keyfiles { publicKeys, err := certutil.PublicKeysFromFile(keyfile) @@ -283,12 +286,12 @@ func newLegacyServiceAccountAuthenticator(keyfiles []string, lookup bool, servic allPublicKeys = append(allPublicKeys, publicKeys...) } - tokenAuthenticator := serviceaccount.JWTTokenAuthenticator(serviceaccount.LegacyIssuer, allPublicKeys, serviceaccount.NewLegacyValidator(lookup, serviceAccountGetter)) + tokenAuthenticator := serviceaccount.JWTTokenAuthenticator(serviceaccount.LegacyIssuer, allPublicKeys, apiAudiences, serviceaccount.NewLegacyValidator(lookup, serviceAccountGetter)) return tokenAuthenticator, nil } // newServiceAccountAuthenticator returns an authenticator.Token or an error -func newServiceAccountAuthenticator(iss string, audiences []string, keyfiles []string, serviceAccountGetter serviceaccount.ServiceAccountTokenGetter) (authenticator.Token, error) { +func newServiceAccountAuthenticator(iss string, keyfiles []string, apiAudiences authenticator.Audiences, serviceAccountGetter serviceaccount.ServiceAccountTokenGetter) (authenticator.Token, error) { allPublicKeys := []interface{}{} for _, keyfile := range keyfiles { publicKeys, err := certutil.PublicKeysFromFile(keyfile) @@ -298,7 +301,7 @@ func newServiceAccountAuthenticator(iss string, audiences []string, keyfiles []s allPublicKeys = append(allPublicKeys, publicKeys...) } - tokenAuthenticator := serviceaccount.JWTTokenAuthenticator(iss, allPublicKeys, serviceaccount.NewValidator(audiences, serviceAccountGetter)) + tokenAuthenticator := serviceaccount.JWTTokenAuthenticator(iss, allPublicKeys, apiAudiences, serviceaccount.NewValidator(serviceAccountGetter)) return tokenAuthenticator, nil } @@ -315,11 +318,11 @@ func newAuthenticatorFromClientCAFile(clientCAFile string) (authenticator.Reques return x509.New(opts, x509.CommonNameUserConversion), nil } -func newWebhookTokenAuthenticator(webhookConfigFile string, ttl time.Duration) (authenticator.Token, error) { - webhookTokenAuthenticator, err := webhook.New(webhookConfigFile, ttl) +func newWebhookTokenAuthenticator(webhookConfigFile string, ttl time.Duration, implicitAuds authenticator.Audiences) (authenticator.Token, error) { + webhookTokenAuthenticator, err := webhook.New(webhookConfigFile, implicitAuds) if err != nil { return nil, err } - return webhookTokenAuthenticator, nil + return tokencache.New(webhookTokenAuthenticator, false, ttl, ttl), nil } diff --git a/pkg/kubeapiserver/authorizer/OWNERS b/pkg/kubeapiserver/authorizer/OWNERS new file mode 100644 index 00000000000..cd0d70a0f8f --- /dev/null +++ b/pkg/kubeapiserver/authorizer/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authorizers-approvers +reviewers: +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/pkg/kubeapiserver/authorizer/config.go b/pkg/kubeapiserver/authorizer/config.go index 00224fd4825..5a3d88aabbc 100644 --- a/pkg/kubeapiserver/authorizer/config.go +++ b/pkg/kubeapiserver/authorizer/config.go @@ -33,7 +33,8 @@ import ( "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy" ) -type AuthorizationConfig struct { +// Config contains the data on how to authorize a request to the Kube API Server +type Config struct { AuthorizationModes []string // Options for ModeABAC @@ -55,7 +56,7 @@ type AuthorizationConfig struct { // New returns the right sort of union of multiple authorizer.Authorizer objects // based on the authorizationMode or an error. -func (config AuthorizationConfig) New() (authorizer.Authorizer, authorizer.RuleResolver, error) { +func (config Config) New() (authorizer.Authorizer, authorizer.RuleResolver, error) { if len(config.AuthorizationModes) == 0 { return nil, nil, fmt.Errorf("at least one authorization mode must be passed") } diff --git a/pkg/kubeapiserver/authorizer/modes/modes.go b/pkg/kubeapiserver/authorizer/modes/modes.go index 54d0a627701..501b98a95c2 100644 --- a/pkg/kubeapiserver/authorizer/modes/modes.go +++ b/pkg/kubeapiserver/authorizer/modes/modes.go @@ -19,14 +19,21 @@ package modes import "k8s.io/apimachinery/pkg/util/sets" const ( + // ModeAlwaysAllow is the mode to set all requests as authorized ModeAlwaysAllow string = "AlwaysAllow" - ModeAlwaysDeny string = "AlwaysDeny" - ModeABAC string = "ABAC" - ModeWebhook string = "Webhook" - ModeRBAC string = "RBAC" - ModeNode string = "Node" + // ModeAlwaysDeny is the mode to set no requests as authorized + ModeAlwaysDeny string = "AlwaysDeny" + // ModeABAC is the mode to use Attribute Based Access Control to authorize + ModeABAC string = "ABAC" + // ModeWebhook is the mode to make an external webhook call to authorize + ModeWebhook string = "Webhook" + // ModeRBAC is the mode to use Role Based Access Control to authorize + ModeRBAC string = "RBAC" + // ModeNode is an authorization mode that authorizes API requests made by kubelets. + ModeNode string = "Node" ) +// AuthorizationModeChoices is the list of supported authorization modes var AuthorizationModeChoices = []string{ModeAlwaysAllow, ModeAlwaysDeny, ModeABAC, ModeWebhook, ModeRBAC, ModeNode} // IsValidAuthorizationMode returns true if the given authorization mode is a valid one for the apiserver diff --git a/pkg/kubeapiserver/options/BUILD b/pkg/kubeapiserver/options/BUILD index 0a9373a0ea0..9ee6eb989f7 100644 --- a/pkg/kubeapiserver/options/BUILD +++ b/pkg/kubeapiserver/options/BUILD @@ -68,8 +68,8 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubeapiserver/options/authentication.go b/pkg/kubeapiserver/options/authentication.go index fad78831437..9230b07215f 100644 --- a/pkg/kubeapiserver/options/authentication.go +++ b/pkg/kubeapiserver/options/authentication.go @@ -17,19 +17,22 @@ limitations under the License. package options import ( + "errors" "fmt" "net/url" "strings" "time" - "github.com/golang/glog" "github.com/spf13/pflag" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/authenticator" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/flag" + "k8s.io/kubernetes/pkg/features" kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" ) @@ -170,6 +173,18 @@ func (s *BuiltInAuthenticationOptions) Validate() []error { allErrors = append(allErrors, fmt.Errorf("service-account-issuer contained a ':' but was not a valid URL: %v", err)) } } + if s.ServiceAccounts != nil && utilfeature.DefaultFeatureGate.Enabled(features.BoundServiceAccountTokenVolume) { + if !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) || !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequestProjection) { + allErrors = append(allErrors, errors.New("If the BoundServiceAccountTokenVolume feature is enabled,"+ + " the TokenRequest and TokenRequestProjection features must also be enabled")) + } + if len(s.ServiceAccounts.Issuer) == 0 { + allErrors = append(allErrors, errors.New("service-account-issuer is a required flag when BoundServiceAccountTokenVolume is enabled")) + } + if len(s.ServiceAccounts.KeyFiles) == 0 { + allErrors = append(allErrors, errors.New("service-account-key-file is a required flag when BoundServiceAccountTokenVolume is enabled")) + } + } return allErrors } @@ -292,8 +307,8 @@ func (s *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) { } } -func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() kubeauthenticator.AuthenticatorConfig { - ret := kubeauthenticator.AuthenticatorConfig{ +func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() kubeauthenticator.Config { + ret := kubeauthenticator.Config{ TokenSuccessCacheTTL: s.TokenSuccessCacheTTL, TokenFailureCacheTTL: s.TokenFailureCacheTTL, } @@ -350,10 +365,10 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() kubeauthenticato if len(s.WebHook.ConfigFile) > 0 && s.WebHook.CacheTTL > 0 { if s.TokenSuccessCacheTTL > 0 && s.WebHook.CacheTTL < s.TokenSuccessCacheTTL { - glog.Warningf("the webhook cache ttl of %s is shorter than the overall cache ttl of %s for successful token authentication attempts.", s.WebHook.CacheTTL, s.TokenSuccessCacheTTL) + klog.Warningf("the webhook cache ttl of %s is shorter than the overall cache ttl of %s for successful token authentication attempts.", s.WebHook.CacheTTL, s.TokenSuccessCacheTTL) } if s.TokenFailureCacheTTL > 0 && s.WebHook.CacheTTL < s.TokenFailureCacheTTL { - glog.Warningf("the webhook cache ttl of %s is shorter than the overall cache ttl of %s for failed token authentication attempts.", s.WebHook.CacheTTL, s.TokenFailureCacheTTL) + klog.Warningf("the webhook cache ttl of %s is shorter than the overall cache ttl of %s for failed token authentication attempts.", s.WebHook.CacheTTL, s.TokenFailureCacheTTL) } } } @@ -397,7 +412,7 @@ func (o *BuiltInAuthenticationOptions) ApplyAuthorization(authorization *BuiltIn // authorization ModeAlwaysAllow cannot be combined with AnonymousAuth. // in such a case the AnonymousAuth is stomped to false and you get a message if o.Anonymous.Allow && sets.NewString(authorization.Modes...).Has(authzmodes.ModeAlwaysAllow) { - glog.Warningf("AnonymousAuth is not allowed with the AlwaysAllow authorizer. Resetting AnonymousAuth to false. You should use a different authorizer") + klog.Warningf("AnonymousAuth is not allowed with the AlwaysAllow authorizer. Resetting AnonymousAuth to false. You should use a different authorizer") o.Anonymous.Allow = false } } diff --git a/pkg/kubeapiserver/options/authentication_test.go b/pkg/kubeapiserver/options/authentication_test.go index be0d57f29b2..ac45c457de1 100644 --- a/pkg/kubeapiserver/options/authentication_test.go +++ b/pkg/kubeapiserver/options/authentication_test.go @@ -138,7 +138,7 @@ func TestToAuthenticationConfig(t *testing.T) { TokenFailureCacheTTL: 0, } - expectConfig := kubeauthenticator.AuthenticatorConfig{ + expectConfig := kubeauthenticator.Config{ APIAudiences: authenticator.Audiences{"http://foo.bar.com"}, Anonymous: false, BasicAuthFile: "/testBasicAuthFile", diff --git a/pkg/kubeapiserver/options/authorization.go b/pkg/kubeapiserver/options/authorization.go index 585e3ecfb8f..017aa4bfea3 100644 --- a/pkg/kubeapiserver/options/authorization.go +++ b/pkg/kubeapiserver/options/authorization.go @@ -109,8 +109,8 @@ func (s *BuiltInAuthorizationOptions) AddFlags(fs *pflag.FlagSet) { "The duration to cache 'unauthorized' responses from the webhook authorizer.") } -func (s *BuiltInAuthorizationOptions) ToAuthorizationConfig(versionedInformerFactory versionedinformers.SharedInformerFactory) authorizer.AuthorizationConfig { - return authorizer.AuthorizationConfig{ +func (s *BuiltInAuthorizationOptions) ToAuthorizationConfig(versionedInformerFactory versionedinformers.SharedInformerFactory) authorizer.Config { + return authorizer.Config{ AuthorizationModes: s.Modes, PolicyFile: s.PolicyFile, WebhookConfigFile: s.WebhookConfigFile, diff --git a/pkg/kubeapiserver/util/BUILD b/pkg/kubeapiserver/util/BUILD deleted file mode 100644 index 6df04e38cd7..00000000000 --- a/pkg/kubeapiserver/util/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/pkg/kubectl/BUILD b/pkg/kubectl/BUILD index fbd9e945497..cfb06a76450 100644 --- a/pkg/kubectl/BUILD +++ b/pkg/kubectl/BUILD @@ -14,7 +14,6 @@ go_test( "rolling_updater_test.go", "rollout_status_test.go", "scale_test.go", - "sorter_test.go", ], embed = [":go_default_library"], deps = [ @@ -25,7 +24,6 @@ go_test( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/testapigroup/v1:go_default_library", @@ -57,20 +55,16 @@ go_library( "rolling_updater.go", "rollout_status.go", "scale.go", - "sorter.go", ], importpath = "k8s.io/kubernetes/pkg/kubectl", deps = [ - "//pkg/apis/core:go_default_library", - "//pkg/apis/core/v1:go_default_library", "//pkg/kubectl/apps:go_default_library", + "//pkg/kubectl/describe/versioned:go_default_library", "//pkg/kubectl/scheme:go_default_library", "//pkg/kubectl/util:go_default_library", "//pkg/kubectl/util/deployment:go_default_library", "//pkg/kubectl/util/podutils:go_default_library", "//pkg/kubectl/util/slice:go_default_library", - "//pkg/printers:go_default_library", - "//pkg/printers/internalversion:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/autoscaling/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", @@ -79,8 +73,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", @@ -96,10 +88,7 @@ go_library( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/scale:go_default_library", "//staging/src/k8s.io/client-go/util/integer:go_default_library", - "//staging/src/k8s.io/client-go/util/jsonpath:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", - "//vendor/vbom.ml/util/sortorder:go_default_library", ], ) diff --git a/pkg/kubectl/apply.go b/pkg/kubectl/apply.go index 3c82aee808e..78447d498cf 100644 --- a/pkg/kubectl/apply.go +++ b/pkg/kubectl/apply.go @@ -135,7 +135,8 @@ func CreateApplyAnnotation(obj runtime.Object, codec runtime.Encoder) error { return setOriginalConfiguration(obj, modified) } -// Create the annotation used by kubectl apply only when createAnnotation is true +// CreateOrUpdateAnnotation creates the annotation used by +// kubectl apply only when createAnnotation is true // Otherwise, only update the annotation when it already exists func CreateOrUpdateAnnotation(createAnnotation bool, obj runtime.Object, codec runtime.Encoder) error { if createAnnotation { diff --git a/pkg/kubectl/apply/strategy/BUILD b/pkg/kubectl/apply/strategy/BUILD index 34bb23eccbf..6243b3a20d0 100644 --- a/pkg/kubectl/apply/strategy/BUILD +++ b/pkg/kubectl/apply/strategy/BUILD @@ -41,11 +41,11 @@ go_test( "//pkg/kubectl/cmd/util/openapi:go_default_library", "//pkg/kubectl/cmd/util/openapi/testing:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/ginkgo/config:go_default_library", "//vendor/github.com/onsi/ginkgo/types:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/pkg/kubectl/apply/strategy/utils_test.go b/pkg/kubectl/apply/strategy/utils_test.go index 9a958b78fd6..437da605bda 100644 --- a/pkg/kubectl/apply/strategy/utils_test.go +++ b/pkg/kubectl/apply/strategy/utils_test.go @@ -23,7 +23,7 @@ import ( "path/filepath" "strings" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" "k8s.io/apimachinery/pkg/util/diff" "k8s.io/kubernetes/pkg/kubectl/apply" diff --git a/pkg/kubectl/apps/kind_visitor_test.go b/pkg/kubectl/apps/kind_visitor_test.go index 10704b65e9e..3d96920c6d9 100644 --- a/pkg/kubectl/apps/kind_visitor_test.go +++ b/pkg/kubectl/apps/kind_visitor_test.go @@ -173,7 +173,7 @@ type TestKindVisitor struct { var _ apps.KindVisitor = &TestKindVisitor{} -func (t *TestKindVisitor) Visit(kind apps.GroupKindElement) { t.visits[kind.Kind] += 1 } +func (t *TestKindVisitor) Visit(kind apps.GroupKindElement) { t.visits[kind.Kind]++ } func (t *TestKindVisitor) VisitDaemonSet(kind apps.GroupKindElement) { t.Visit(kind) } func (t *TestKindVisitor) VisitDeployment(kind apps.GroupKindElement) { t.Visit(kind) } diff --git a/pkg/kubectl/cmd/BUILD b/pkg/kubectl/cmd/BUILD index b3c2b79a4af..3672fe88d02 100644 --- a/pkg/kubectl/cmd/BUILD +++ b/pkg/kubectl/cmd/BUILD @@ -68,10 +68,7 @@ go_library( go_test( name = "go_default_test", - srcs = [ - "cmd_printing_test.go", - "cmd_test.go", - ], + srcs = ["cmd_test.go"], data = [ "//api/openapi-spec:swagger-spec", "//pkg/kubectl/cmd/plugin:testdata", @@ -80,15 +77,8 @@ go_test( ], embed = [":go_default_library"], deps = [ - "//pkg/api/legacyscheme:go_default_library", - "//pkg/apis/core:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", - "//pkg/printers:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", - "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/annotate/BUILD b/pkg/kubectl/cmd/annotate/BUILD index 97cbb1f00eb..728e940743f 100644 --- a/pkg/kubectl/cmd/annotate/BUILD +++ b/pkg/kubectl/cmd/annotate/BUILD @@ -20,8 +20,8 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/annotate/annotate.go b/pkg/kubectl/cmd/annotate/annotate.go index 00f4551a493..56fe5603fb4 100644 --- a/pkg/kubectl/cmd/annotate/annotate.go +++ b/pkg/kubectl/cmd/annotate/annotate.go @@ -22,8 +22,8 @@ import ( "io" jsonpatch "github.com/evanphx/json-patch" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -271,7 +271,7 @@ func (o AnnotateOptions) RunAnnotate() error { return err } if err := o.Recorder.Record(info.Object); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } if err := o.updateAnnotations(obj); err != nil { return err @@ -283,7 +283,7 @@ func (o AnnotateOptions) RunAnnotate() error { patchBytes, err := jsonpatch.CreateMergePatch(oldData, newData) createdPatch := err == nil if err != nil { - glog.V(2).Infof("couldn't compute patch: %v", err) + klog.V(2).Infof("couldn't compute patch: %v", err) } mapping := info.ResourceMapping() diff --git a/pkg/kubectl/cmd/apiresources/apiresources.go b/pkg/kubectl/cmd/apiresources/apiresources.go index b3f84bae973..2e3d90bf239 100644 --- a/pkg/kubectl/cmd/apiresources/apiresources.go +++ b/pkg/kubectl/cmd/apiresources/apiresources.go @@ -77,7 +77,8 @@ func NewAPIResourceOptions(ioStreams genericclioptions.IOStreams) *ApiResourcesO } } -func NewCmdApiResources(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command { +// NewCmdAPIResources creates the `api-resources` command +func NewCmdAPIResources(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command { o := NewAPIResourceOptions(ioStreams) cmd := &cobra.Command{ @@ -112,7 +113,7 @@ func (o *ApiResourcesOptions) Validate() error { func (o *ApiResourcesOptions) Complete(cmd *cobra.Command, args []string) error { if len(args) != 0 { - return cmdutil.UsageErrorf(cmd, "unexpected args: %v", args) + return cmdutil.UsageErrorf(cmd, "unexpected arguments: %v", args) } return nil } diff --git a/pkg/kubectl/cmd/apiresources/apiversions.go b/pkg/kubectl/cmd/apiresources/apiversions.go index 57760860757..738aa51b6b4 100644 --- a/pkg/kubectl/cmd/apiresources/apiversions.go +++ b/pkg/kubectl/cmd/apiresources/apiversions.go @@ -48,7 +48,8 @@ func NewApiVersionsOptions(ioStreams genericclioptions.IOStreams) *ApiVersionsOp } } -func NewCmdApiVersions(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command { +// NewCmdAPIVersions creates the `api-versions` command +func NewCmdAPIVersions(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command { o := NewApiVersionsOptions(ioStreams) cmd := &cobra.Command{ Use: "api-versions", @@ -65,7 +66,7 @@ func NewCmdApiVersions(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) func (o *ApiVersionsOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { if len(args) != 0 { - return cmdutil.UsageErrorf(cmd, "unexpected args: %v", args) + return cmdutil.UsageErrorf(cmd, "unexpected arguments: %v", args) } var err error o.discoveryClient, err = f.ToDiscoveryClient() diff --git a/pkg/kubectl/cmd/apply/BUILD b/pkg/kubectl/cmd/apply/BUILD index 4a6ca665a8b..8f954982581 100644 --- a/pkg/kubectl/cmd/apply/BUILD +++ b/pkg/kubectl/cmd/apply/BUILD @@ -38,11 +38,11 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", "//staging/src/k8s.io/client-go/discovery:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/jonboulle/clockwork:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/util/proto:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/apply/apply.go b/pkg/kubectl/cmd/apply/apply.go index de8274169ea..08919f23b2b 100644 --- a/pkg/kubectl/cmd/apply/apply.go +++ b/pkg/kubectl/cmd/apply/apply.go @@ -22,7 +22,6 @@ import ( "strings" "time" - "github.com/golang/glog" "github.com/jonboulle/clockwork" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" @@ -43,6 +42,7 @@ import ( "k8s.io/cli-runtime/pkg/genericclioptions/resource" "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" + "k8s.io/klog" oapi "k8s.io/kube-openapi/pkg/util/proto" "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/delete" @@ -347,7 +347,7 @@ func (o *ApplyOptions) Run() error { } if err := o.Recorder.Record(info.Object); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } // Get the modified configuration of the object. Embed the result @@ -389,13 +389,14 @@ func (o *ApplyOptions) Run() error { return cmdutil.AddSourceToErr("creating", info.Source, err) } info.Refresh(obj, true) - metadata, err := meta.Accessor(info.Object) - if err != nil { - return err - } - visitedUids.Insert(string(metadata.GetUID())) } + metadata, err := meta.Accessor(info.Object) + if err != nil { + return err + } + visitedUids.Insert(string(metadata.GetUID())) + count++ if printObject { @@ -410,12 +411,13 @@ func (o *ApplyOptions) Run() error { return printer.PrintObj(info.Object, o.Out) } - if !o.DryRun { - metadata, err := meta.Accessor(info.Object) - if err != nil { - return err - } + metadata, err := meta.Accessor(info.Object) + if err != nil { + return err + } + visitedUids.Insert(string(metadata.GetUID())) + if !o.DryRun { annotationMap := metadata.GetAnnotations() if _, ok := annotationMap[corev1.LastAppliedConfigAnnotation]; !ok { fmt.Fprintf(o.ErrOut, warningNoLastAppliedConfigAnnotation, o.cmdBaseName) @@ -443,8 +445,6 @@ func (o *ApplyOptions) Run() error { info.Refresh(patchedObject, true) - visitedUids.Insert(string(metadata.GetUID())) - if string(patchBytes) == "{}" && !printObject { count++ diff --git a/pkg/kubectl/cmd/apply/apply_test.go b/pkg/kubectl/cmd/apply/apply_test.go index 8c1631f684e..3bd088fae96 100644 --- a/pkg/kubectl/cmd/apply/apply_test.go +++ b/pkg/kubectl/cmd/apply/apply_test.go @@ -103,19 +103,24 @@ const ( filenameWidgetServerside = "../../../../test/fixtures/pkg/kubectl/cmd/apply/widget-serverside.yaml" ) -func readConfigMapList(t *testing.T, filename string) []byte { +func readConfigMapList(t *testing.T, filename string) [][]byte { data := readBytesFromFile(t, filename) cmList := corev1.ConfigMapList{} if err := runtime.DecodeInto(codec, data, &cmList); err != nil { t.Fatal(err) } - cmListBytes, err := runtime.Encode(codec, &cmList) - if err != nil { - t.Fatal(err) + var listCmBytes [][]byte + + for _, cm := range cmList.Items { + cmBytes, err := runtime.Encode(codec, &cm) + if err != nil { + t.Fatal(err) + } + listCmBytes = append(listCmBytes, cmBytes) } - return cmListBytes + return listCmBytes } func readBytesFromFile(t *testing.T, filename string) []byte { @@ -270,7 +275,7 @@ func walkMapPath(t *testing.T, start map[string]interface{}, path []string) map[ func TestRunApplyPrintsValidObjectList(t *testing.T) { cmdtesting.InitTestErrorHandler(t) - cmBytes := readConfigMapList(t, filenameCM) + configMapList := readConfigMapList(t, filenameCM) pathCM := "/namespaces/test/configmaps" tf := cmdtesting.NewTestFactory().WithNamespace("test") @@ -280,12 +285,21 @@ func TestRunApplyPrintsValidObjectList(t *testing.T) { NegotiatedSerializer: resource.UnstructuredPlusDefaultContentConfig().NegotiatedSerializer, Client: fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { switch p, m := req.URL.Path, req.Method; { - case strings.HasPrefix(p, pathCM) && m != "GET": - pod := ioutil.NopCloser(bytes.NewReader(cmBytes)) - return &http.Response{StatusCode: 200, Header: cmdtesting.DefaultHeader(), Body: pod}, nil - case strings.HasPrefix(p, pathCM) && m != "PATCH": - pod := ioutil.NopCloser(bytes.NewReader(cmBytes)) - return &http.Response{StatusCode: 200, Header: cmdtesting.DefaultHeader(), Body: pod}, nil + case strings.HasPrefix(p, pathCM) && m == "GET": + fallthrough + case strings.HasPrefix(p, pathCM) && m == "PATCH": + var body io.ReadCloser + + switch p { + case pathCM + "/test0": + body = ioutil.NopCloser(bytes.NewReader(configMapList[0])) + case pathCM + "/test1": + body = ioutil.NopCloser(bytes.NewReader(configMapList[1])) + default: + t.Errorf("unexpected request to %s", p) + } + + return &http.Response{StatusCode: 200, Header: cmdtesting.DefaultHeader(), Body: body}, nil default: t.Fatalf("unexpected request: %#v\n%#v", req.URL, req) return nil, nil @@ -306,6 +320,16 @@ func TestRunApplyPrintsValidObjectList(t *testing.T) { if err := runtime.DecodeInto(codec, buf.Bytes(), &cmList); err != nil { t.Fatal(err) } + + if len(cmList.Items) != 2 { + t.Fatalf("Expected 2 items in the result; got %d", len(cmList.Items)) + } + if !strings.Contains(string(cmList.Items[0].Raw), "key1") { + t.Fatalf("Did not get first ConfigMap at the first position") + } + if !strings.Contains(string(cmList.Items[1].Raw), "key2") { + t.Fatalf("Did not get second ConfigMap at the second position") + } } func TestRunApplyViewLastApplied(t *testing.T) { diff --git a/pkg/kubectl/cmd/apply/apply_view_last_applied.go b/pkg/kubectl/cmd/apply/apply_view_last_applied.go index ce4a7ff5f20..e35b0c434cb 100644 --- a/pkg/kubectl/cmd/apply/apply_view_last_applied.go +++ b/pkg/kubectl/cmd/apply/apply_view_last_applied.go @@ -21,7 +21,6 @@ import ( "encoding/json" "fmt" - "github.com/ghodss/yaml" "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/cli-runtime/pkg/genericclioptions/resource" @@ -29,6 +28,7 @@ import ( cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/kubectl/util/templates" + "sigs.k8s.io/yaml" ) type ViewLastAppliedOptions struct { diff --git a/pkg/kubectl/cmd/attach/BUILD b/pkg/kubectl/cmd/attach/BUILD index dde0b4da104..ae86f8d5bfd 100644 --- a/pkg/kubectl/cmd/attach/BUILD +++ b/pkg/kubectl/cmd/attach/BUILD @@ -18,8 +18,8 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/attach/attach.go b/pkg/kubectl/cmd/attach/attach.go index df5f24790d0..c0bfd783163 100644 --- a/pkg/kubectl/cmd/attach/attach.go +++ b/pkg/kubectl/cmd/attach/attach.go @@ -22,8 +22,8 @@ import ( "net/url" "time" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -326,7 +326,7 @@ func (o *AttachOptions) containerToAttachTo(pod *corev1.Pod) (*corev1.Container, fmt.Fprintf(o.ErrOut, "%s\n", o.SuggestedCmdUsage) } - glog.V(4).Infof("defaulting container name to %s", pod.Spec.Containers[0].Name) + klog.V(4).Infof("defaulting container name to %s", pod.Spec.Containers[0].Name) return &pod.Spec.Containers[0], nil } diff --git a/pkg/kubectl/cmd/auth/BUILD b/pkg/kubectl/cmd/auth/BUILD index 756d2692067..9be5ff830ac 100644 --- a/pkg/kubectl/cmd/auth/BUILD +++ b/pkg/kubectl/cmd/auth/BUILD @@ -32,8 +32,8 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes/typed/authorization/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/auth/OWNERS b/pkg/kubectl/cmd/auth/OWNERS new file mode 100644 index 00000000000..cd0d70a0f8f --- /dev/null +++ b/pkg/kubectl/cmd/auth/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authorizers-approvers +reviewers: +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/pkg/kubectl/cmd/auth/reconcile.go b/pkg/kubectl/cmd/auth/reconcile.go index 089639705bb..e2dafca743d 100644 --- a/pkg/kubectl/cmd/auth/reconcile.go +++ b/pkg/kubectl/cmd/auth/reconcile.go @@ -20,8 +20,8 @@ import ( "errors" "fmt" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" rbacv1 "k8s.io/api/rbac/v1" rbacv1alpha1 "k8s.io/api/rbac/v1alpha1" @@ -251,7 +251,7 @@ func (o *ReconcileOptions) RunReconcile() error { return fmt.Errorf("only rbac.authorization.k8s.io/v1 is supported: not %T", t) default: - glog.V(1).Infof("skipping %#v", info.Object.GetObjectKind()) + klog.V(1).Infof("skipping %#v", info.Object.GetObjectKind()) // skip ignored resources } diff --git a/pkg/kubectl/cmd/autoscale/BUILD b/pkg/kubectl/cmd/autoscale/BUILD index 1c9aea0718e..fdb1dce6c24 100644 --- a/pkg/kubectl/cmd/autoscale/BUILD +++ b/pkg/kubectl/cmd/autoscale/BUILD @@ -20,8 +20,8 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/autoscaling/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/autoscale/autoscale.go b/pkg/kubectl/cmd/autoscale/autoscale.go index f51c93be5f4..9f59e7bbd82 100644 --- a/pkg/kubectl/cmd/autoscale/autoscale.go +++ b/pkg/kubectl/cmd/autoscale/autoscale.go @@ -19,8 +19,8 @@ package autoscale import ( "fmt" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" autoscalingv1 "k8s.io/api/autoscaling/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -42,7 +42,7 @@ var ( autoscaleLong = templates.LongDesc(i18n.T(` Creates an autoscaler that automatically chooses and sets the number of pods that run in a kubernetes cluster. - Looks up a Deployment, ReplicaSet, or ReplicationController by name and creates an autoscaler that uses the given resource as a reference. + Looks up a Deployment, ReplicaSet, StatefulSet, or ReplicationController by name and creates an autoscaler that uses the given resource as a reference. An autoscaler can automatically increase or decrease number of pods deployed within the system as needed.`)) autoscaleExample = templates.Examples(i18n.T(` @@ -159,7 +159,7 @@ func (o *AutoscaleOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args CPUPercent: o.CpuPercent, ScaleRefName: name, ScaleRefKind: mapping.GroupVersionKind.Kind, - ScaleRefApiVersion: mapping.GroupVersionKind.GroupVersion().String(), + ScaleRefAPIVersion: mapping.GroupVersionKind.GroupVersion().String(), }, nil default: return nil, cmdutil.UsageErrorf(cmd, "Generator %s not supported. ", o.Generator) @@ -234,7 +234,7 @@ func (o *AutoscaleOptions) Run() error { } if err := o.Recorder.Record(hpa); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } if o.dryRun { diff --git a/pkg/kubectl/cmd/cmd.go b/pkg/kubectl/cmd/cmd.go index 43a541ecc9f..2cfe569e93c 100644 --- a/pkg/kubectl/cmd/cmd.go +++ b/pkg/kubectl/cmd/cmd.go @@ -284,7 +284,7 @@ __custom_func() { ) var ( - bash_completion_flags = map[string]string{ + bashCompletionFlags = map[string]string{ "namespace": "__kubectl_get_resource_namespace", "context": "__kubectl_config_get_contexts", "cluster": "__kubectl_config_get_clusters", @@ -292,10 +292,12 @@ var ( } ) +// NewDefaultKubectlCommand creates the `kubectl` command with default arguments func NewDefaultKubectlCommand() *cobra.Command { return NewDefaultKubectlCommandWithArgs(&defaultPluginHandler{}, os.Args, os.Stdin, os.Stdout, os.Stderr) } +// NewDefaultKubectlCommandWithArgs creates the `kubectl` command with arguments func NewDefaultKubectlCommandWithArgs(pluginHandler PluginHandler, args []string, in io.Reader, out, errout io.Writer) *cobra.Command { cmd := NewKubectlCommand(in, out, errout) @@ -450,7 +452,6 @@ func NewKubectlCommand(in io.Reader, out, err io.Writer) *cobra.Command { expose.NewCmdExposeService(f, ioStreams), run.NewCmdRun(f, ioStreams), set.NewCmdSet(f, ioStreams), - deprecatedAlias("run-container", run.NewCmdRun(f, ioStreams)), }, }, { @@ -528,7 +529,7 @@ func NewKubectlCommand(in io.Reader, out, err io.Writer) *cobra.Command { templates.ActsAsRootCommand(cmds, filters, groups...) - for name, completion := range bash_completion_flags { + for name, completion := range bashCompletionFlags { if cmds.Flag(name) != nil { if cmds.Flag(name).Annotations == nil { cmds.Flag(name).Annotations = map[string][]string{} @@ -544,8 +545,8 @@ func NewKubectlCommand(in io.Reader, out, err io.Writer) *cobra.Command { cmds.AddCommand(cmdconfig.NewCmdConfig(f, clientcmd.NewDefaultPathOptions(), ioStreams)) cmds.AddCommand(plugin.NewCmdPlugin(f, ioStreams)) cmds.AddCommand(version.NewCmdVersion(f, ioStreams)) - cmds.AddCommand(apiresources.NewCmdApiVersions(f, ioStreams)) - cmds.AddCommand(apiresources.NewCmdApiResources(f, ioStreams)) + cmds.AddCommand(apiresources.NewCmdAPIVersions(f, ioStreams)) + cmds.AddCommand(apiresources.NewCmdAPIResources(f, ioStreams)) cmds.AddCommand(options.NewCmdOptions(ioStreams.Out)) return cmds diff --git a/pkg/kubectl/cmd/cmd_printing_test.go b/pkg/kubectl/cmd/cmd_printing_test.go deleted file mode 100644 index 79ccde79251..00000000000 --- a/pkg/kubectl/cmd/cmd_printing_test.go +++ /dev/null @@ -1,238 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "bytes" - "testing" - - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/cli-runtime/pkg/genericclioptions" - genericprinters "k8s.io/cli-runtime/pkg/genericclioptions/printers" - "k8s.io/kubernetes/pkg/api/legacyscheme" - api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/printers" -) - -func TestIllegalPackageSourceCheckerThroughPrintFlags(t *testing.T) { - testCases := []struct { - name string - expectInternalObjErr bool - output string - obj runtime.Object - expectedOutput string - }{ - { - name: "success printer: object containing package path beginning with forbidden prefix is rejected", - expectInternalObjErr: true, - obj: internalPod(), - }, - { - name: "success printer: object containing package path with no forbidden prefix returns no error", - expectInternalObjErr: false, - obj: externalPod(), - output: "", - expectedOutput: "pod/foo succeeded\n", - }, - { - name: "name printer: object containing package path beginning with forbidden prefix is rejected", - expectInternalObjErr: true, - output: "name", - obj: internalPod(), - }, - { - name: "json printer: object containing package path beginning with forbidden prefix is rejected", - expectInternalObjErr: true, - output: "json", - obj: internalPod(), - }, - { - name: "json printer: object containing package path with no forbidden prefix returns no error", - expectInternalObjErr: false, - obj: externalPod(), - output: "json", - }, - { - name: "yaml printer: object containing package path beginning with forbidden prefix is rejected", - expectInternalObjErr: true, - output: "yaml", - obj: internalPod(), - }, - { - name: "yaml printer: object containing package path with no forbidden prefix returns no error", - expectInternalObjErr: false, - obj: externalPod(), - output: "yaml", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - printFlags := genericclioptions.NewPrintFlags("succeeded").WithTypeSetter(legacyscheme.Scheme) - printFlags.OutputFormat = &tc.output - - printer, err := printFlags.ToPrinter() - if err != nil { - t.Fatalf("unexpected error %v", err) - } - - output := bytes.NewBuffer([]byte{}) - - err = printer.PrintObj(tc.obj, output) - if err != nil { - if !tc.expectInternalObjErr { - t.Fatalf("unexpected error %v", err) - } - - if !genericprinters.IsInternalObjectError(err) { - t.Fatalf("unexpected error - expecting internal object printer error, got %q", err) - } - return - } - - if tc.expectInternalObjErr { - t.Fatalf("expected internal object printer error, but got no error") - } - - if len(tc.expectedOutput) == 0 { - return - } - - if tc.expectedOutput != output.String() { - t.Fatalf("unexpected output: expecting %q, got %q", tc.expectedOutput, output.String()) - } - }) - } -} - -func TestIllegalPackageSourceCheckerDirectlyThroughPrinters(t *testing.T) { - jsonPathPrinter, err := genericprinters.NewJSONPathPrinter("{ .metadata.name }") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - goTemplatePrinter, err := genericprinters.NewGoTemplatePrinter([]byte("{{ .metadata.name }}")) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - customColumns, err := printers.NewCustomColumnsPrinterFromSpec("NAME:.metadata.name", legacyscheme.Codecs.UniversalDecoder(), true) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - testCases := []struct { - name string - expectInternalObjErr bool - printer genericprinters.ResourcePrinter - obj runtime.Object - expectedOutput string - }{ - { - name: "json printer: object containing package path beginning with forbidden prefix is rejected", - expectInternalObjErr: true, - printer: &genericprinters.JSONPrinter{}, - obj: internalPod(), - }, - { - name: "yaml printer: object containing package path beginning with forbidden prefix is rejected", - expectInternalObjErr: true, - printer: &genericprinters.YAMLPrinter{}, - obj: internalPod(), - }, - { - name: "jsonpath printer: object containing package path beginning with forbidden prefix is rejected", - expectInternalObjErr: true, - printer: jsonPathPrinter, - obj: internalPod(), - }, - { - name: "go-template printer: object containing package path beginning with forbidden prefix is rejected", - expectInternalObjErr: true, - printer: goTemplatePrinter, - obj: internalPod(), - }, - { - name: "go-template printer: object containing package path beginning with forbidden prefix is rejected", - expectInternalObjErr: true, - printer: goTemplatePrinter, - obj: internalPod(), - }, - { - name: "custom-columns printer: object containing package path beginning with forbidden prefix is rejected", - expectInternalObjErr: true, - printer: customColumns, - obj: internalPod(), - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - output := bytes.NewBuffer([]byte{}) - - err := tc.printer.PrintObj(tc.obj, output) - if err != nil { - if !tc.expectInternalObjErr { - t.Fatalf("unexpected error %v", err) - } - - if !genericprinters.IsInternalObjectError(err) { - t.Fatalf("unexpected error - expecting internal object printer error, got %q", err) - } - return - } - - if tc.expectInternalObjErr { - t.Fatalf("expected internal object printer error, but got no error") - } - - if len(tc.expectedOutput) == 0 { - return - } - - if tc.expectedOutput != output.String() { - t.Fatalf("unexpected output: expecting %q, got %q", tc.expectedOutput, output.String()) - } - }) - } -} - -func internalPod() *api.Pod { - return &api.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "test", ResourceVersion: "10"}, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Name: "bar", - }, - }, - }, - Status: api.PodStatus{ - Phase: api.PodRunning, - }, - } -} - -func externalPod() *v1.Pod { - return &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - } -} diff --git a/pkg/kubectl/cmd/config/view.go b/pkg/kubectl/cmd/config/view.go index 83b3a847ce3..7017cb6a9cf 100644 --- a/pkg/kubectl/cmd/config/view.go +++ b/pkg/kubectl/cmd/config/view.go @@ -82,7 +82,7 @@ func NewCmdConfigView(f cmdutil.Factory, streams genericclioptions.IOStreams, Co Long: view_long, Example: view_example, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(o.Complete(cmd)) + cmdutil.CheckErr(o.Complete(cmd, args)) cmdutil.CheckErr(o.Validate()) cmdutil.CheckErr(o.Run()) }, @@ -99,7 +99,10 @@ func NewCmdConfigView(f cmdutil.Factory, streams genericclioptions.IOStreams, Co return cmd } -func (o *ViewOptions) Complete(cmd *cobra.Command) error { +func (o *ViewOptions) Complete(cmd *cobra.Command, args []string) error { + if len(args) != 0 { + return cmdutil.UsageErrorf(cmd, "unexpected arguments: %v", args) + } if o.ConfigAccess.IsExplicitFile() { if !o.Merge.Provided() { o.Merge.Set("false") diff --git a/pkg/kubectl/cmd/convert/BUILD b/pkg/kubectl/cmd/convert/BUILD index 493542234fd..930c177c9ce 100644 --- a/pkg/kubectl/cmd/convert/BUILD +++ b/pkg/kubectl/cmd/convert/BUILD @@ -2,12 +2,30 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["convert.go"], + srcs = [ + "convert.go", + "import_known_versions.go", + ], importpath = "k8s.io/kubernetes/pkg/kubectl/cmd/convert", visibility = ["//visibility:public"], deps = [ "//pkg/api/legacyscheme:go_default_library", + "//pkg/apis/apps/install:go_default_library", + "//pkg/apis/authentication/install:go_default_library", + "//pkg/apis/authorization/install:go_default_library", + "//pkg/apis/autoscaling/install:go_default_library", + "//pkg/apis/batch/install:go_default_library", + "//pkg/apis/certificates/install:go_default_library", + "//pkg/apis/coordination/install:go_default_library", "//pkg/apis/core:go_default_library", + "//pkg/apis/core/install:go_default_library", + "//pkg/apis/events/install:go_default_library", + "//pkg/apis/extensions/install:go_default_library", + "//pkg/apis/policy/install:go_default_library", + "//pkg/apis/rbac/install:go_default_library", + "//pkg/apis/scheduling/install:go_default_library", + "//pkg/apis/settings/install:go_default_library", + "//pkg/apis/storage/install:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", "//pkg/kubectl/util/i18n:go_default_library", "//pkg/kubectl/util/templates:go_default_library", @@ -18,8 +36,8 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/convert/convert.go b/pkg/kubectl/cmd/convert/convert.go index 251b15540ed..a233b8cd1cb 100644 --- a/pkg/kubectl/cmd/convert/convert.go +++ b/pkg/kubectl/cmd/convert/convert.go @@ -19,8 +19,8 @@ package convert import ( "fmt" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -134,6 +134,14 @@ func (o *ConvertOptions) Complete(f cmdutil.Factory, cmd *cobra.Command) (err er // RunConvert implements the generic Convert command func (o *ConvertOptions) RunConvert() error { + + // Convert must be removed from kubectl, since kubectl can not depend on + // Kubernetes "internal" dependencies. These "internal" dependencies can + // not be removed from convert. Another way to convert a resource is to + // "kubectl apply" it to the cluster, then "kubectl get" at the desired version. + // Another possible solution is to make convert a plugin. + fmt.Fprintf(o.ErrOut, "kubectl convert is DEPRECATED and will be removed in a future version.\nIn order to convert, kubectl apply the object to the cluster, then kubectl get at the desired version.\n") + b := o.builder(). WithScheme(scheme.Scheme). LocalParam(o.local) @@ -218,7 +226,7 @@ func asVersionedObject(infos []*resource.Info, forceList bool, specifiedOutputVe if len(actualVersion.Version) > 0 { defaultVersionInfo = fmt.Sprintf("Defaulting to %q", actualVersion.Version) } - glog.V(1).Infof("info: the output version specified is invalid. %s\n", defaultVersionInfo) + klog.V(1).Infof("info: the output version specified is invalid. %s\n", defaultVersionInfo) } return object, nil } diff --git a/pkg/kubectl/cmd/convert/import_known_versions.go b/pkg/kubectl/cmd/convert/import_known_versions.go new file mode 100644 index 00000000000..2da5f1aadd1 --- /dev/null +++ b/pkg/kubectl/cmd/convert/import_known_versions.go @@ -0,0 +1,37 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package convert + +// These imports are the API groups the client will support. +// TODO: Remove these manual install once we don't need legacy scheme in convert +import ( + _ "k8s.io/kubernetes/pkg/apis/apps/install" + _ "k8s.io/kubernetes/pkg/apis/authentication/install" + _ "k8s.io/kubernetes/pkg/apis/authorization/install" + _ "k8s.io/kubernetes/pkg/apis/autoscaling/install" + _ "k8s.io/kubernetes/pkg/apis/batch/install" + _ "k8s.io/kubernetes/pkg/apis/certificates/install" + _ "k8s.io/kubernetes/pkg/apis/coordination/install" + _ "k8s.io/kubernetes/pkg/apis/core/install" + _ "k8s.io/kubernetes/pkg/apis/events/install" + _ "k8s.io/kubernetes/pkg/apis/extensions/install" + _ "k8s.io/kubernetes/pkg/apis/policy/install" + _ "k8s.io/kubernetes/pkg/apis/rbac/install" + _ "k8s.io/kubernetes/pkg/apis/scheduling/install" + _ "k8s.io/kubernetes/pkg/apis/settings/install" + _ "k8s.io/kubernetes/pkg/apis/storage/install" +) diff --git a/pkg/kubectl/cmd/create/BUILD b/pkg/kubectl/cmd/create/BUILD index 3f043b50b5e..49ce594021e 100644 --- a/pkg/kubectl/cmd/create/BUILD +++ b/pkg/kubectl/cmd/create/BUILD @@ -47,8 +47,8 @@ go_library( "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/batch/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/create/create.go b/pkg/kubectl/cmd/create/create.go index b96b62eca83..1a426bae616 100644 --- a/pkg/kubectl/cmd/create/create.go +++ b/pkg/kubectl/cmd/create/create.go @@ -24,8 +24,8 @@ import ( "runtime" "strings" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -247,7 +247,7 @@ func (o *CreateOptions) RunCreate(f cmdutil.Factory, cmd *cobra.Command) error { } if err := o.Recorder.Record(info.Object); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } if !o.DryRun { diff --git a/pkg/kubectl/cmd/delete/BUILD b/pkg/kubectl/cmd/delete/BUILD index 1768dad67a0..f1b922a72d6 100644 --- a/pkg/kubectl/cmd/delete/BUILD +++ b/pkg/kubectl/cmd/delete/BUILD @@ -21,8 +21,8 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/delete/delete.go b/pkg/kubectl/cmd/delete/delete.go index b34bc9fa751..84b6db50bfe 100644 --- a/pkg/kubectl/cmd/delete/delete.go +++ b/pkg/kubectl/cmd/delete/delete.go @@ -21,8 +21,8 @@ import ( "strings" "time" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -259,7 +259,7 @@ func (o *DeleteOptions) DeleteResult(r *resource.Result) error { responseMetadata, err := meta.Accessor(response) if err != nil { // we don't have UID, but we didn't fail the delete, next best thing is just skipping the UID - glog.V(1).Info(err) + klog.V(1).Info(err) return nil } uidMap[resourceLocation] = responseMetadata.GetUID() @@ -301,7 +301,7 @@ func (o *DeleteOptions) DeleteResult(r *resource.Result) error { if errors.IsForbidden(err) || errors.IsMethodNotSupported(err) { // if we're forbidden from waiting, we shouldn't fail. // if the resource doesn't support a verb we need, we shouldn't fail. - glog.V(1).Info(err) + klog.V(1).Info(err) return nil } return err diff --git a/pkg/kubectl/cmd/describe/BUILD b/pkg/kubectl/cmd/describe/BUILD index 4b1aaea02b4..036e46df9c3 100644 --- a/pkg/kubectl/cmd/describe/BUILD +++ b/pkg/kubectl/cmd/describe/BUILD @@ -7,10 +7,10 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/kubectl/cmd/util:go_default_library", + "//pkg/kubectl/describe:go_default_library", "//pkg/kubectl/describe/versioned:go_default_library", "//pkg/kubectl/util/i18n:go_default_library", "//pkg/kubectl/util/templates:go_default_library", - "//pkg/printers:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", @@ -30,9 +30,9 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/kubectl/cmd/testing:go_default_library", + "//pkg/kubectl/describe:go_default_library", "//pkg/kubectl/describe/versioned:go_default_library", "//pkg/kubectl/scheme:go_default_library", - "//pkg/printers:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", diff --git a/pkg/kubectl/cmd/describe/describe.go b/pkg/kubectl/cmd/describe/describe.go index 472f1ffcff3..c1c84cd1717 100644 --- a/pkg/kubectl/cmd/describe/describe.go +++ b/pkg/kubectl/cmd/describe/describe.go @@ -29,10 +29,10 @@ import ( "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/cli-runtime/pkg/genericclioptions/resource" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/describe" describeversioned "k8s.io/kubernetes/pkg/kubectl/describe/versioned" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/kubectl/util/templates" - "k8s.io/kubernetes/pkg/printers" ) var ( @@ -74,7 +74,7 @@ type DescribeOptions struct { Selector string Namespace string - Describer func(*meta.RESTMapping) (printers.Describer, error) + Describer func(*meta.RESTMapping) (describe.Describer, error) NewBuilder func() *resource.Builder BuilderArgs []string @@ -83,7 +83,7 @@ type DescribeOptions struct { AllNamespaces bool IncludeUninitialized bool - DescriberSettings *printers.DescriberSettings + DescriberSettings *describe.DescriberSettings FilenameOptions *resource.FilenameOptions genericclioptions.IOStreams @@ -92,7 +92,7 @@ type DescribeOptions struct { func NewCmdDescribe(parent string, f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := &DescribeOptions{ FilenameOptions: &resource.FilenameOptions{}, - DescriberSettings: &printers.DescriberSettings{ + DescriberSettings: &describe.DescriberSettings{ ShowEvents: true, }, @@ -138,7 +138,7 @@ func (o *DescribeOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args [ o.BuilderArgs = args - o.Describer = func(mapping *meta.RESTMapping) (printers.Describer, error) { + o.Describer = func(mapping *meta.RESTMapping) (describe.Describer, error) { return describeversioned.DescriberFn(f, mapping) } diff --git a/pkg/kubectl/cmd/describe/describe_test.go b/pkg/kubectl/cmd/describe/describe_test.go index 7ed9fd93ae7..4cd0f16622c 100644 --- a/pkg/kubectl/cmd/describe/describe_test.go +++ b/pkg/kubectl/cmd/describe/describe_test.go @@ -27,19 +27,19 @@ import ( "k8s.io/cli-runtime/pkg/genericclioptions/resource" "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" - describe "k8s.io/kubernetes/pkg/kubectl/describe/versioned" + "k8s.io/kubernetes/pkg/kubectl/describe" + versioneddescribe "k8s.io/kubernetes/pkg/kubectl/describe/versioned" "k8s.io/kubernetes/pkg/kubectl/scheme" - "k8s.io/kubernetes/pkg/printers" ) // Verifies that schemas that are not in the master tree of Kubernetes can be retrieved via Get. func TestDescribeUnknownSchemaObject(t *testing.T) { d := &testDescriber{Output: "test output"} - oldFn := describe.DescriberFn + oldFn := versioneddescribe.DescriberFn defer func() { - describe.DescriberFn = oldFn + versioneddescribe.DescriberFn = oldFn }() - describe.DescriberFn = d.describerFor + versioneddescribe.DescriberFn = d.describerFor tf := cmdtesting.NewTestFactory().WithNamespace("non-default") defer tf.Cleanup() @@ -67,11 +67,11 @@ func TestDescribeUnknownSchemaObject(t *testing.T) { // Verifies that schemas that are not in the master tree of Kubernetes can be retrieved via Get. func TestDescribeUnknownNamespacedSchemaObject(t *testing.T) { d := &testDescriber{Output: "test output"} - oldFn := describe.DescriberFn + oldFn := versioneddescribe.DescriberFn defer func() { - describe.DescriberFn = oldFn + versioneddescribe.DescriberFn = oldFn }() - describe.DescriberFn = d.describerFor + versioneddescribe.DescriberFn = d.describerFor tf := cmdtesting.NewTestFactory() defer tf.Cleanup() @@ -99,11 +99,11 @@ func TestDescribeUnknownNamespacedSchemaObject(t *testing.T) { func TestDescribeObject(t *testing.T) { d := &testDescriber{Output: "test output"} - oldFn := describe.DescriberFn + oldFn := versioneddescribe.DescriberFn defer func() { - describe.DescriberFn = oldFn + versioneddescribe.DescriberFn = oldFn }() - describe.DescriberFn = d.describerFor + versioneddescribe.DescriberFn = d.describerFor _, _, rc := cmdtesting.TestData() tf := cmdtesting.NewTestFactory().WithNamespace("test") @@ -140,11 +140,11 @@ func TestDescribeObject(t *testing.T) { func TestDescribeListObjects(t *testing.T) { d := &testDescriber{Output: "test output"} - oldFn := describe.DescriberFn + oldFn := versioneddescribe.DescriberFn defer func() { - describe.DescriberFn = oldFn + versioneddescribe.DescriberFn = oldFn }() - describe.DescriberFn = d.describerFor + versioneddescribe.DescriberFn = d.describerFor pods, _, _ := cmdtesting.TestData() tf := cmdtesting.NewTestFactory().WithNamespace("test") @@ -167,11 +167,11 @@ func TestDescribeListObjects(t *testing.T) { func TestDescribeObjectShowEvents(t *testing.T) { d := &testDescriber{Output: "test output"} - oldFn := describe.DescriberFn + oldFn := versioneddescribe.DescriberFn defer func() { - describe.DescriberFn = oldFn + versioneddescribe.DescriberFn = oldFn }() - describe.DescriberFn = d.describerFor + versioneddescribe.DescriberFn = d.describerFor pods, _, _ := cmdtesting.TestData() tf := cmdtesting.NewTestFactory().WithNamespace("test") @@ -193,11 +193,11 @@ func TestDescribeObjectShowEvents(t *testing.T) { func TestDescribeObjectSkipEvents(t *testing.T) { d := &testDescriber{Output: "test output"} - oldFn := describe.DescriberFn + oldFn := versioneddescribe.DescriberFn defer func() { - describe.DescriberFn = oldFn + versioneddescribe.DescriberFn = oldFn }() - describe.DescriberFn = d.describerFor + versioneddescribe.DescriberFn = d.describerFor pods, _, _ := cmdtesting.TestData() tf := cmdtesting.NewTestFactory().WithNamespace("test") @@ -247,16 +247,16 @@ func TestDescribeHelpMessage(t *testing.T) { type testDescriber struct { Name, Namespace string - Settings printers.DescriberSettings + Settings describe.DescriberSettings Output string Err error } -func (t *testDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (output string, err error) { +func (t *testDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (output string, err error) { t.Namespace, t.Name = namespace, name t.Settings = describerSettings return t.Output, t.Err } -func (t *testDescriber) describerFor(restClientGetter genericclioptions.RESTClientGetter, mapping *meta.RESTMapping) (printers.Describer, error) { +func (t *testDescriber) describerFor(restClientGetter genericclioptions.RESTClientGetter, mapping *meta.RESTMapping) (describe.Describer, error) { return t, nil } diff --git a/pkg/kubectl/cmd/diff/BUILD b/pkg/kubectl/cmd/diff/BUILD index a1eaca378cc..4a99f161251 100644 --- a/pkg/kubectl/cmd/diff/BUILD +++ b/pkg/kubectl/cmd/diff/BUILD @@ -19,10 +19,10 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/jonboulle/clockwork:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/diff/diff.go b/pkg/kubectl/cmd/diff/diff.go index 076439d2647..9c52dbfb6b4 100644 --- a/pkg/kubectl/cmd/diff/diff.go +++ b/pkg/kubectl/cmd/diff/diff.go @@ -23,7 +23,6 @@ import ( "os" "path/filepath" - "github.com/ghodss/yaml" "github.com/jonboulle/clockwork" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/api/errors" @@ -40,6 +39,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/kubectl/util/templates" "k8s.io/utils/exec" + "sigs.k8s.io/yaml" ) var ( diff --git a/pkg/kubectl/cmd/drain/drain.go b/pkg/kubectl/cmd/drain/drain.go index 536b3f2a076..d24ab8ac6bb 100644 --- a/pkg/kubectl/cmd/drain/drain.go +++ b/pkg/kubectl/cmd/drain/drain.go @@ -235,9 +235,6 @@ func (o *DrainOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []st if len(args) > 0 && len(o.Selector) > 0 { return cmdutil.UsageErrorf(cmd, "error: cannot specify both a node name and a --selector option") } - if len(args) > 0 && len(args) != 1 { - return cmdutil.UsageErrorf(cmd, fmt.Sprintf("USAGE: %s [flags]", cmd.Use)) - } o.DryRun = cmdutil.GetDryRunFlag(cmd) diff --git a/pkg/kubectl/cmd/drain/drain_test.go b/pkg/kubectl/cmd/drain/drain_test.go index bd2eebb48b3..0d4d2bbf1a5 100644 --- a/pkg/kubectl/cmd/drain/drain_test.go +++ b/pkg/kubectl/cmd/drain/drain_test.go @@ -142,6 +142,22 @@ func TestCordon(t *testing.T) { arg: "bar", expectFatal: true, }, + { + description: "cordon for multiple nodes", + node: node, + expected: cordoned_node, + cmd: NewCmdCordon, + arg: "node node1 node2", + expectFatal: false, + }, + { + description: "uncordon for multiple nodes", + node: cordoned_node, + expected: node, + cmd: NewCmdUncordon, + arg: "node node1 node2", + expectFatal: false, + }, } for _, test := range tests { @@ -160,10 +176,18 @@ func TestCordon(t *testing.T) { Client: fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { m := &MyReq{req} switch { + case m.isFor("GET", "/nodes/node1"): + fallthrough + case m.isFor("GET", "/nodes/node2"): + fallthrough case m.isFor("GET", "/nodes/node"): return &http.Response{StatusCode: 200, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, test.node)}, nil case m.isFor("GET", "/nodes/bar"): return &http.Response{StatusCode: 404, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.StringBody("nope")}, nil + case m.isFor("PATCH", "/nodes/node1"): + fallthrough + case m.isFor("PATCH", "/nodes/node2"): + fallthrough case m.isFor("PATCH", "/nodes/node"): data, err := ioutil.ReadAll(req.Body) if err != nil { @@ -209,7 +233,7 @@ func TestCordon(t *testing.T) { saw_fatal = true panic(e) }) - cmd.SetArgs([]string{test.arg}) + cmd.SetArgs(strings.Split(test.arg, " ")) cmd.Execute() }() diff --git a/pkg/kubectl/cmd/expose/BUILD b/pkg/kubectl/cmd/expose/BUILD index 71c17b46e65..bd71a3589e9 100644 --- a/pkg/kubectl/cmd/expose/BUILD +++ b/pkg/kubectl/cmd/expose/BUILD @@ -24,8 +24,8 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/expose/expose.go b/pkg/kubectl/cmd/expose/expose.go index c826e2a7b74..2727e61c1bf 100644 --- a/pkg/kubectl/cmd/expose/expose.go +++ b/pkg/kubectl/cmd/expose/expose.go @@ -20,8 +20,8 @@ import ( "regexp" "strings" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -321,7 +321,7 @@ func (o *ExposeServiceOptions) RunExpose(cmd *cobra.Command, args []string) erro } if err := o.Recorder.Record(object); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } if o.DryRun { diff --git a/pkg/kubectl/cmd/get/BUILD b/pkg/kubectl/cmd/get/BUILD index a14665b829c..c63ccad3fd1 100644 --- a/pkg/kubectl/cmd/get/BUILD +++ b/pkg/kubectl/cmd/get/BUILD @@ -17,24 +17,27 @@ filegroup( go_library( name = "go_default_library", srcs = [ + "customcolumn.go", + "customcolumn_flags.go", "get.go", "get_flags.go", "humanreadable_flags.go", + "sorter.go", ], importpath = "k8s.io/kubernetes/pkg/kubectl/cmd/get", visibility = ["//visibility:public"], deps = [ "//pkg/api/legacyscheme:go_default_library", - "//pkg/apis/core:go_default_library", - "//pkg/kubectl:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", "//pkg/kubectl/cmd/util/openapi:go_default_library", "//pkg/kubectl/scheme:go_default_library", "//pkg/kubectl/util/i18n:go_default_library", + "//pkg/kubectl/util/printers:go_default_library", "//pkg/kubectl/util/templates:go_default_library", "//pkg/printers:go_default_library", "//pkg/printers/internalversion:go_default_library", "//pkg/util/interrupt:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -46,19 +49,26 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", + "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//staging/src/k8s.io/client-go/util/integer:go_default_library", + "//staging/src/k8s.io/client-go/util/jsonpath:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/vbom.ml/util/sortorder:go_default_library", ], ) go_test( name = "go_default_test", srcs = [ + "customcolumn_flags_test.go", + "customcolumn_test.go", "get_test.go", "humanreadable_flags_test.go", + "sorter_test.go", ], data = [ "//api/openapi-spec:swagger-spec", @@ -72,13 +82,16 @@ go_test( "//pkg/kubectl/cmd/util/openapi:go_default_library", "//pkg/kubectl/cmd/util/openapi/testing:go_default_library", "//pkg/kubectl/scheme:go_default_library", + "//pkg/kubectl/util/printers:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/autoscaling/v1:go_default_library", "//staging/src/k8s.io/api/batch/v1:go_default_library", "//staging/src/k8s.io/api/batch/v1beta1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/pkg/printers/customcolumn.go b/pkg/kubectl/cmd/get/customcolumn.go similarity index 98% rename from pkg/printers/customcolumn.go rename to pkg/kubectl/cmd/get/customcolumn.go index b0ab6bc239c..576d4995af4 100644 --- a/pkg/printers/customcolumn.go +++ b/pkg/kubectl/cmd/get/customcolumn.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package printers +package get import ( "bufio" @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/cli-runtime/pkg/genericclioptions/printers" "k8s.io/client-go/util/jsonpath" + utilprinters "k8s.io/kubernetes/pkg/kubectl/util/printers" ) var jsonRegexp = regexp.MustCompile("^\\{\\.?([^{}]+)\\}$|^\\.?([^{}]+)$") @@ -160,7 +161,7 @@ func (s *CustomColumnsPrinter) PrintObj(obj runtime.Object, out io.Writer) error } if w, found := out.(*tabwriter.Writer); !found { - w = GetNewTabWriter(out) + w = utilprinters.GetNewTabWriter(out) out = w defer w.Flush() } diff --git a/pkg/printers/customcolumn_flags.go b/pkg/kubectl/cmd/get/customcolumn_flags.go similarity index 91% rename from pkg/printers/customcolumn_flags.go rename to pkg/kubectl/cmd/get/customcolumn_flags.go index 599d91c2728..b8f2f4002b2 100644 --- a/pkg/printers/customcolumn_flags.go +++ b/pkg/kubectl/cmd/get/customcolumn_flags.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package printers +package get import ( "fmt" @@ -24,6 +24,7 @@ import ( "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericclioptions/printers" "k8s.io/kubernetes/pkg/kubectl/scheme" ) @@ -51,7 +52,7 @@ func (f *CustomColumnsPrintFlags) AllowedFormats() []string { // handling custom-column printing. // Returns false if the specified templateFormat does not match a supported format. // Supported format types can be found in pkg/printers/printers.go -func (f *CustomColumnsPrintFlags) ToPrinter(templateFormat string) (ResourcePrinter, error) { +func (f *CustomColumnsPrintFlags) ToPrinter(templateFormat string) (printers.ResourcePrinter, error) { if len(templateFormat) == 0 { return nil, genericclioptions.NoCompatiblePrinterError{} } @@ -79,7 +80,8 @@ func (f *CustomColumnsPrintFlags) ToPrinter(templateFormat string) (ResourcePrin return nil, fmt.Errorf("custom-columns format specified but no custom columns given") } - decoder := scheme.Codecs.UniversalDecoder() + // UniversalDecoder call must specify parameter versions; otherwise it will decode to internal versions. + decoder := scheme.Codecs.UniversalDecoder(scheme.Scheme.PrioritizedVersionsAllGroups()...) if templateFormat == "custom-columns-file" { file, err := os.Open(templateValue) diff --git a/pkg/printers/customcolumn_flags_test.go b/pkg/kubectl/cmd/get/customcolumn_flags_test.go similarity index 97% rename from pkg/printers/customcolumn_flags_test.go rename to pkg/kubectl/cmd/get/customcolumn_flags_test.go index 2b96f8f48ac..6704bdf2383 100644 --- a/pkg/printers/customcolumn_flags_test.go +++ b/pkg/kubectl/cmd/get/customcolumn_flags_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package printers +package get import ( "bytes" @@ -24,13 +24,13 @@ import ( "strings" "testing" - "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" ) func TestPrinterSupportsExpectedCustomColumnFormats(t *testing.T) { - testObject := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + testObject := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} customColumnsFile, err := ioutil.TempFile("", "printers_jsonpath_flags") if err != nil { diff --git a/pkg/printers/customcolumn_test.go b/pkg/kubectl/cmd/get/customcolumn_test.go similarity index 88% rename from pkg/printers/customcolumn_test.go rename to pkg/kubectl/cmd/get/customcolumn_test.go index f59cfdf5191..263ecdde92b 100644 --- a/pkg/printers/customcolumn_test.go +++ b/pkg/kubectl/cmd/get/customcolumn_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package printers +package get import ( "bytes" @@ -22,13 +22,16 @@ import ( "strings" "testing" - "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/kubernetes/pkg/api/legacyscheme" - api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/kubectl/scheme" + "k8s.io/kubernetes/pkg/kubectl/util/printers" ) +// UniversalDecoder call must specify parameter versions; otherwise it will decode to internal versions. +var decoder = scheme.Codecs.UniversalDecoder(scheme.Scheme.PrioritizedVersionsAllGroups()...) + func TestMassageJSONPath(t *testing.T) { tests := []struct { input string @@ -115,7 +118,7 @@ func TestNewColumnPrinterFromSpec(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - printer, err := NewCustomColumnsPrinterFromSpec(test.spec, legacyscheme.Codecs.UniversalDecoder(), test.noHeaders) + printer, err := NewCustomColumnsPrinterFromSpec(test.spec, decoder, test.noHeaders) if test.expectErr { if err == nil { t.Errorf("[%s] unexpected non-error", test.name) @@ -129,7 +132,7 @@ func TestNewColumnPrinterFromSpec(t *testing.T) { if test.noHeaders { buffer := &bytes.Buffer{} - printer.PrintObj(&api.Pod{}, buffer) + printer.PrintObj(&corev1.Pod{}, buffer) if err != nil { t.Fatalf("An error occurred printing Pod: %#v", err) } @@ -219,7 +222,7 @@ func TestNewColumnPrinterFromTemplate(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { reader := bytes.NewBufferString(test.spec) - printer, err := NewCustomColumnsPrinterFromTemplate(reader, legacyscheme.Codecs.UniversalDecoder()) + printer, err := NewCustomColumnsPrinterFromTemplate(reader, decoder) if test.expectErr { if err == nil { t.Errorf("[%s] unexpected non-error", test.name) @@ -251,7 +254,7 @@ func TestColumnPrint(t *testing.T) { FieldSpec: "{.metadata.name}", }, }, - obj: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, + obj: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, expectedOutput: `NAME foo `, @@ -263,8 +266,8 @@ foo FieldSpec: "{.metadata.name}", }, }, - obj: &v1.PodList{ - Items: []v1.Pod{ + obj: &corev1.PodList{ + Items: []corev1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, {ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, }, @@ -285,7 +288,7 @@ bar FieldSpec: "{.apiVersion}", }, }, - obj: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}, TypeMeta: metav1.TypeMeta{APIVersion: "baz"}}, + obj: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}, TypeMeta: metav1.TypeMeta{APIVersion: "baz"}}, expectedOutput: `NAME API_VERSION foo baz `, @@ -305,7 +308,7 @@ foo baz FieldSpec: "{.notFound}", }, }, - obj: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}, TypeMeta: metav1.TypeMeta{APIVersion: "baz"}}, + obj: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}, TypeMeta: metav1.TypeMeta{APIVersion: "baz"}}, expectedOutput: `NAME API_VERSION NOT_FOUND foo baz `, @@ -316,7 +319,7 @@ foo baz t.Run(test.expectedOutput, func(t *testing.T) { printer := &CustomColumnsPrinter{ Columns: test.columns, - Decoder: legacyscheme.Codecs.UniversalDecoder(), + Decoder: decoder, } buffer := &bytes.Buffer{} if err := printer.PrintObj(test.obj, buffer); err != nil { @@ -345,7 +348,7 @@ func TestIndividualPrintObjOnExistingTabWriter(t *testing.T) { FieldSpec: "{.metadata.labels.label2}", }, } - objects := []*v1.Pod{ + objects := []*corev1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "foo", Labels: map[string]string{"label1": "foo", "label2": "foo"}}}, {ObjectMeta: metav1.ObjectMeta{Name: "bar", Labels: map[string]string{"label1": "bar", "label2": "bar"}}}, } @@ -355,10 +358,10 @@ bar bar bar ` buffer := &bytes.Buffer{} - tabWriter := GetNewTabWriter(buffer) + tabWriter := printers.GetNewTabWriter(buffer) printer := &CustomColumnsPrinter{ Columns: columns, - Decoder: legacyscheme.Codecs.UniversalDecoder(), + Decoder: decoder, } for _, obj := range objects { if err := printer.PrintObj(obj, tabWriter); err != nil { diff --git a/pkg/kubectl/cmd/get/get.go b/pkg/kubectl/cmd/get/get.go index 2739b3bc8c4..8011485e412 100644 --- a/pkg/kubectl/cmd/get/get.go +++ b/pkg/kubectl/cmd/get/get.go @@ -23,9 +23,10 @@ import ( "io" "net/url" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" + corev1 "k8s.io/api/core/v1" kapierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,16 +38,15 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericclioptions/printers" "k8s.io/cli-runtime/pkg/genericclioptions/resource" "k8s.io/client-go/rest" watchtools "k8s.io/client-go/tools/watch" "k8s.io/kubernetes/pkg/api/legacyscheme" - api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/kubectl" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/util/i18n" + utilprinters "k8s.io/kubernetes/pkg/kubectl/util/printers" "k8s.io/kubernetes/pkg/kubectl/util/templates" - "k8s.io/kubernetes/pkg/printers" "k8s.io/kubernetes/pkg/util/interrupt" ) @@ -331,7 +331,7 @@ func (r *RuntimeSorter) Sort() error { case *metav1beta1.Table: includesTable = true - if err := kubectl.NewTableSorter(t, r.field).Sort(); err != nil { + if err := NewTableSorter(t, r.field).Sort(); err != nil { continue } default: @@ -354,7 +354,7 @@ func (r *RuntimeSorter) Sort() error { // if not dealing with a Table response from the server, assume // all objects are runtime.Object as usual, and sort using old method. var err error - if r.positioner, err = kubectl.SortObjects(r.decoder, r.objects, r.field); err != nil { + if r.positioner, err = SortObjects(r.decoder, r.objects, r.field); err != nil { return err } return nil @@ -374,7 +374,7 @@ func (r *RuntimeSorter) WithDecoder(decoder runtime.Decoder) *RuntimeSorter { } func NewRuntimeSorter(objects []runtime.Object, sortBy string) *RuntimeSorter { - parsedField, err := printers.RelaxedJSONPathExpression(sortBy) + parsedField, err := RelaxedJSONPathExpression(sortBy) if err != nil { parsedField = sortBy } @@ -471,7 +471,7 @@ func (o *GetOptions) Run(f cmdutil.Factory, cmd *cobra.Command, args []string) e } else { // if we are unable to decode server response into a v1beta1.Table, // fallback to client-side printing with whatever info the server returned. - glog.V(2).Infof("Unable to decode server response into a Table. Falling back to hardcoded types: %v", err) + klog.V(2).Infof("Unable to decode server response into a Table. Falling back to hardcoded types: %v", err) } } @@ -495,7 +495,7 @@ func (o *GetOptions) Run(f cmdutil.Factory, cmd *cobra.Command, args []string) e var printer printers.ResourcePrinter var lastMapping *meta.RESTMapping nonEmptyObjCount := 0 - w := printers.GetNewTabWriter(o.Out) + w := utilprinters.GetNewTabWriter(o.Out) for ix := range objs { var mapping *meta.RESTMapping var info *resource.Info @@ -555,7 +555,7 @@ func (o *GetOptions) Run(f cmdutil.Factory, cmd *cobra.Command, args []string) e internalObj, err := legacyscheme.Scheme.ConvertToVersion(info.Object, info.Mapping.GroupVersionKind.GroupKind().WithVersion(runtime.APIVersionInternal).GroupVersion()) if err != nil { // if there's an error, try to print what you have (mirrors old behavior). - glog.V(1).Info(err) + klog.V(1).Info(err) printer.PrintObj(info.Object, w) } else { printer.PrintObj(internalObj, w) @@ -645,7 +645,7 @@ func (o *GetOptions) watch(f cmdutil.Factory, cmd *cobra.Command, args []string) // print the current object if !o.WatchOnly { var objsToPrint []runtime.Object - writer := printers.GetNewTabWriter(o.Out) + writer := utilprinters.GetNewTabWriter(o.Out) if isList { objsToPrint, _ = meta.ExtractList(obj) @@ -704,7 +704,7 @@ func (o *GetOptions) watch(f cmdutil.Factory, cmd *cobra.Command, args []string) func attemptToConvertToInternal(obj runtime.Object, converter runtime.ObjectConvertor, targetVersion schema.GroupVersion) runtime.Object { internalObject, err := converter.ConvertToVersion(obj, targetVersion) if err != nil { - glog.V(1).Infof("Unable to convert %T to %v: %v", obj, targetVersion, err) + klog.V(1).Infof("Unable to convert %T to %v: %v", obj, targetVersion, err) return obj } return internalObject @@ -768,8 +768,8 @@ func (o *GetOptions) printGeneric(r *resource.Result) error { // we have more than one item, so coerce all items into a list. // we don't want an *unstructured.Unstructured list yet, as we // may be dealing with non-unstructured objects. Compose all items - // into an api.List, and then decode using an unstructured scheme. - list := api.List{ + // into an corev1.List, and then decode using an unstructured scheme. + list := corev1.List{ TypeMeta: metav1.TypeMeta{ Kind: "List", APIVersion: "v1", @@ -777,7 +777,7 @@ func (o *GetOptions) printGeneric(r *resource.Result) error { ListMeta: metav1.ListMeta{}, } for _, info := range infos { - list.Items = append(list.Items, info.Object) + list.Items = append(list.Items, runtime.RawExtension{Object: info.Object}) } listData, err := json.Marshal(list) @@ -852,7 +852,7 @@ func cmdSpecifiesOutputFmt(cmd *cobra.Command) bool { func maybeWrapSortingPrinter(printer printers.ResourcePrinter, sortBy string) printers.ResourcePrinter { if len(sortBy) != 0 { - return &kubectl.SortingPrinter{ + return &SortingPrinter{ Delegate: printer, SortField: fmt.Sprintf("%s", sortBy), } diff --git a/pkg/kubectl/cmd/get/get_flags.go b/pkg/kubectl/cmd/get/get_flags.go index 4bfeafd5b2a..93700a85524 100644 --- a/pkg/kubectl/cmd/get/get_flags.go +++ b/pkg/kubectl/cmd/get/get_flags.go @@ -24,8 +24,8 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericclioptions/printers" "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi" - "k8s.io/kubernetes/pkg/printers" ) // PrintFlags composes common printer flag structs @@ -33,7 +33,7 @@ import ( type PrintFlags struct { JSONYamlPrintFlags *genericclioptions.JSONYamlPrintFlags NamePrintFlags *genericclioptions.NamePrintFlags - CustomColumnsFlags *printers.CustomColumnsPrintFlags + CustomColumnsFlags *CustomColumnsPrintFlags HumanReadableFlags *HumanPrintFlags TemplateFlags *genericclioptions.KubeTemplatePrintFlags @@ -185,6 +185,6 @@ func NewGetPrintFlags() *PrintFlags { TemplateFlags: genericclioptions.NewKubeTemplatePrintFlags(), HumanReadableFlags: NewHumanPrintFlags(), - CustomColumnsFlags: printers.NewCustomColumnsPrintFlags(), + CustomColumnsFlags: NewCustomColumnsPrintFlags(), } } diff --git a/pkg/kubectl/cmd/get/humanreadable_flags_test.go b/pkg/kubectl/cmd/get/humanreadable_flags_test.go index 6b31c48c9dc..cc547df42ae 100644 --- a/pkg/kubectl/cmd/get/humanreadable_flags_test.go +++ b/pkg/kubectl/cmd/get/humanreadable_flags_test.go @@ -61,7 +61,7 @@ func TestHumanReadablePrinterSupportsExpectedOptions(t *testing.T) { { name: "\"wide\" output format prints", outputFormat: "wide", - expectedOutput: "NAME\\ +READY\\ +STATUS\\ +RESTARTS\\ +AGE\\ +IP\\ +NODE\\ +NOMINATED NODE\nfoo\\ +0/0\\ +0\\ +\\ +\\ +\\ +\n", + expectedOutput: "NAME\\ +READY\\ +STATUS\\ +RESTARTS\\ +AGE\\ +IP\\ +NODE\\ +NOMINATED NODE\\ +READINESS GATES\nfoo\\ +0/0\\ +0\\ +\\ +\\ +\\ +\\ +\n", }, { name: "no-headers prints output with no headers", @@ -72,7 +72,7 @@ func TestHumanReadablePrinterSupportsExpectedOptions(t *testing.T) { name: "no-headers and a \"wide\" output format prints output with no headers and additional columns", outputFormat: "wide", noHeaders: true, - expectedOutput: "foo\\ +0/0\\ +0\\ +\\ +\\ +\\ +\n", + expectedOutput: "foo\\ +0/0\\ +0\\ +\\ +\\ +\\ +\\ +\n", }, { name: "show-kind displays the resource's kind, even when printing a single type of resource", diff --git a/pkg/kubectl/sorter.go b/pkg/kubectl/cmd/get/sorter.go similarity index 85% rename from pkg/kubectl/sorter.go rename to pkg/kubectl/cmd/get/sorter.go index b7c339dfb2b..cd8bb28b8d8 100644 --- a/pkg/kubectl/sorter.go +++ b/pkg/kubectl/cmd/get/sorter.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubectl +package get import ( "fmt" @@ -22,22 +22,22 @@ import ( "reflect" "sort" - "github.com/golang/glog" + "k8s.io/klog" - "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" metav1beta1 "k8s.io/apimachinery/pkg/apis/meta/v1beta1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions/printers" "k8s.io/client-go/util/integer" "k8s.io/client-go/util/jsonpath" - "k8s.io/kubernetes/pkg/printers" "vbom.ml/util/sortorder" ) -// Sorting printer sorts list types before delegating to another printer. +// SortingPrinter sorts list types before delegating to another printer. // Non-list types are simply passed through type SortingPrinter struct { SortField string @@ -71,7 +71,7 @@ func (s *SortingPrinter) sortObj(obj runtime.Object) error { } switch list := obj.(type) { - case *v1.List: + case *corev1.List: outputList := make([]runtime.RawExtension, len(objs)) for ix := range objs { outputList[ix] = list.Items[sorter.OriginalPosition(ix)] @@ -96,7 +96,7 @@ func SortObjects(decoder runtime.Decoder, objs []runtime.Object, fieldInput stri } } - field, err := printers.RelaxedJSONPathExpression(fieldInput) + field, err := RelaxedJSONPathExpression(fieldInput) if err != nil { return nil, err } @@ -272,12 +272,12 @@ func (r *RuntimeSort) Less(i, j int) bool { iValues, err = findJSONPathResults(parser, iObj) if err != nil { - glog.Fatalf("Failed to get i values for %#v using %s (%#v)", iObj, r.field, err) + klog.Fatalf("Failed to get i values for %#v using %s (%#v)", iObj, r.field, err) } jValues, err = findJSONPathResults(parser, jObj) if err != nil { - glog.Fatalf("Failed to get j values for %#v using %s (%v)", jObj, r.field, err) + klog.Fatalf("Failed to get j values for %#v using %s (%v)", jObj, r.field, err) } if len(iValues) == 0 || len(iValues[0]) == 0 { @@ -291,12 +291,13 @@ func (r *RuntimeSort) Less(i, j int) bool { less, err := isLess(iField, jField) if err != nil { - glog.Fatalf("Field %s in %T is an unsortable type: %s, err: %v", r.field, iObj, iField.Kind().String(), err) + klog.Fatalf("Field %s in %T is an unsortable type: %s, err: %v", r.field, iObj, iField.Kind().String(), err) } return less } -// Returns the starting (original) position of a particular index. e.g. If OriginalPosition(0) returns 5 than the +// OriginalPosition returns the starting (original) position of a particular index. +// e.g. If OriginalPosition(0) returns 5 than the // the item currently at position 0 was at position 5 in the original unsorted array. func (r *RuntimeSort) OriginalPosition(ix int) int { if ix < 0 || ix > len(r.origPosition) { @@ -306,8 +307,9 @@ func (r *RuntimeSort) OriginalPosition(ix int) int { } type TableSorter struct { - field string - obj *metav1beta1.Table + field string + obj *metav1beta1.Table + parsedRows [][][]reflect.Value } func (t *TableSorter) Len() int { @@ -319,34 +321,10 @@ func (t *TableSorter) Swap(i, j int) { } func (t *TableSorter) Less(i, j int) bool { - iObj := t.obj.Rows[i].Object.Object - jObj := t.obj.Rows[j].Object.Object - - var iValues [][]reflect.Value - var jValues [][]reflect.Value - var err error - - parser := jsonpath.New("sorting").AllowMissingKeys(true) - err = parser.Parse(t.field) - if err != nil { - glog.Fatalf("sorting error: %v\n", err) - } - - // TODO(juanvallejo): this is expensive for very large sets. - // To improve runtime complexity, build an array which contains all - // resolved fields, and sort that instead. - iValues, err = findJSONPathResults(parser, iObj) - if err != nil { - glog.Fatalf("Failed to get i values for %#v using %s (%#v)", iObj, t.field, err) - } - - jValues, err = findJSONPathResults(parser, jObj) - if err != nil { - glog.Fatalf("Failed to get j values for %#v using %s (%v)", jObj, t.field, err) - } - + iValues := t.parsedRows[i] + jValues := t.parsedRows[j] if len(iValues) == 0 || len(iValues[0]) == 0 || len(jValues) == 0 || len(jValues[0]) == 0 { - glog.Fatalf("couldn't find any field with path %q in the list of objects", t.field) + klog.Fatalf("couldn't find any field with path %q in the list of objects", t.field) } iField := iValues[0][0] @@ -354,7 +332,7 @@ func (t *TableSorter) Less(i, j int) bool { less, err := isLess(iField, jField) if err != nil { - glog.Fatalf("Field %s in %T is an unsortable type: %s, err: %v", t.field, iObj, iField.Kind().String(), err) + klog.Fatalf("Field %s in %T is an unsortable type: %s, err: %v", t.field, t.parsedRows, iField.Kind().String(), err) } return less } @@ -365,16 +343,31 @@ func (t *TableSorter) Sort() error { } func NewTableSorter(table *metav1beta1.Table, field string) *TableSorter { + var parsedRows [][][]reflect.Value + + parser := jsonpath.New("sorting").AllowMissingKeys(true) + err := parser.Parse(field) + if err != nil { + klog.Fatalf("sorting error: %v\n", err) + } + + for i := range table.Rows { + parsedRow, err := findJSONPathResults(parser, table.Rows[i].Object.Object) + if err != nil { + klog.Fatalf("Failed to get values for %#v using %s (%#v)", parsedRow, field, err) + } + parsedRows = append(parsedRows, parsedRow) + } + return &TableSorter{ - obj: table, - field: field, + obj: table, + field: field, + parsedRows: parsedRows, } } - func findJSONPathResults(parser *jsonpath.JSONPath, from runtime.Object) ([][]reflect.Value, error) { if unstructuredObj, ok := from.(*unstructured.Unstructured); ok { return parser.FindResults(unstructuredObj.Object) } - return parser.FindResults(reflect.ValueOf(from).Elem().Interface()) } diff --git a/pkg/kubectl/sorter_test.go b/pkg/kubectl/cmd/get/sorter_test.go similarity index 99% rename from pkg/kubectl/sorter_test.go rename to pkg/kubectl/cmd/get/sorter_test.go index 1fcd35d14b6..c85d5d1bf5a 100644 --- a/pkg/kubectl/sorter_test.go +++ b/pkg/kubectl/cmd/get/sorter_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubectl +package get import ( "reflect" diff --git a/pkg/kubectl/cmd/label/BUILD b/pkg/kubectl/cmd/label/BUILD index 7e9abb28d4d..549899a7661 100644 --- a/pkg/kubectl/cmd/label/BUILD +++ b/pkg/kubectl/cmd/label/BUILD @@ -22,8 +22,8 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/label/label.go b/pkg/kubectl/cmd/label/label.go index ae512805ae2..11251a278d2 100644 --- a/pkg/kubectl/cmd/label/label.go +++ b/pkg/kubectl/cmd/label/label.go @@ -22,8 +22,8 @@ import ( "strings" jsonpatch "github.com/evanphx/json-patch" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -288,7 +288,7 @@ func (o *LabelOptions) RunLabel() error { return err } if err := o.Recorder.Record(obj); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } newObj, err := json.Marshal(obj) if err != nil { @@ -298,7 +298,7 @@ func (o *LabelOptions) RunLabel() error { patchBytes, err := jsonpatch.CreateMergePatch(oldData, newObj) createdPatch := err == nil if err != nil { - glog.V(2).Infof("couldn't compute patch: %v", err) + klog.V(2).Infof("couldn't compute patch: %v", err) } mapping := info.ResourceMapping() diff --git a/pkg/kubectl/cmd/logs/logs.go b/pkg/kubectl/cmd/logs/logs.go index 8ba240a950d..5f8f3162b19 100644 --- a/pkg/kubectl/cmd/logs/logs.go +++ b/pkg/kubectl/cmd/logs/logs.go @@ -181,7 +181,7 @@ func (o *LogsOptions) ToLogOptions() (*corev1.PodLogOptions, error) { logOptions.SinceSeconds = &sec } - if len(o.Selector) > 0 && o.Tail != -1 { + if len(o.Selector) > 0 && o.Tail == -1 { logOptions.TailLines = &selectorTail } else if o.Tail != -1 { logOptions.TailLines = &o.Tail diff --git a/pkg/kubectl/cmd/patch/BUILD b/pkg/kubectl/cmd/patch/BUILD index ef65d083b9e..683b0abef23 100644 --- a/pkg/kubectl/cmd/patch/BUILD +++ b/pkg/kubectl/cmd/patch/BUILD @@ -22,8 +22,8 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/patch/patch.go b/pkg/kubectl/cmd/patch/patch.go index 37eb4f51c63..cb8eb0c8c0b 100644 --- a/pkg/kubectl/cmd/patch/patch.go +++ b/pkg/kubectl/cmd/patch/patch.go @@ -22,8 +22,8 @@ import ( "strings" jsonpatch "github.com/evanphx/json-patch" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -229,10 +229,10 @@ func (o *PatchOptions) RunPatch() error { // if the recorder makes a change, compute and create another patch if mergePatch, err := o.Recorder.MakeRecordMergePatch(patchedObj); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } else if len(mergePatch) > 0 { if recordedObj, err := helper.Patch(info.Namespace, info.Name, types.MergePatchType, mergePatch, nil); err != nil { - glog.V(4).Infof("error recording reason: %v", err) + klog.V(4).Infof("error recording reason: %v", err) } else { patchedObj = recordedObj } diff --git a/pkg/kubectl/cmd/proxy/BUILD b/pkg/kubectl/cmd/proxy/BUILD index 3edb46af216..b5c08df6736 100644 --- a/pkg/kubectl/cmd/proxy/BUILD +++ b/pkg/kubectl/cmd/proxy/BUILD @@ -11,8 +11,8 @@ go_library( "//pkg/kubectl/util/i18n:go_default_library", "//pkg/kubectl/util/templates:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/proxy/proxy.go b/pkg/kubectl/cmd/proxy/proxy.go index 67019606757..bf54bb21fb8 100644 --- a/pkg/kubectl/cmd/proxy/proxy.go +++ b/pkg/kubectl/cmd/proxy/proxy.go @@ -24,9 +24,9 @@ import ( "os" "strings" - "github.com/golang/glog" "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/klog" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/proxy" "k8s.io/kubernetes/pkg/kubectl/util/i18n" @@ -119,9 +119,9 @@ func RunProxy(f cmdutil.Factory, out io.Writer, cmd *cobra.Command) error { if staticDir != "" { fileInfo, err := os.Stat(staticDir) if err != nil { - glog.Warning("Failed to stat static file directory "+staticDir+": ", err) + klog.Warning("Failed to stat static file directory "+staticDir+": ", err) } else if !fileInfo.IsDir() { - glog.Warning("Static file directory " + staticDir + " is not a directory") + klog.Warning("Static file directory " + staticDir + " is not a directory") } } @@ -137,7 +137,7 @@ func RunProxy(f cmdutil.Factory, out io.Writer, cmd *cobra.Command) error { } if cmdutil.GetFlagBool(cmd, "disable-filter") { if path == "" { - glog.Warning("Request filter disabled, your proxy is vulnerable to XSRF attacks, please be cautious") + klog.Warning("Request filter disabled, your proxy is vulnerable to XSRF attacks, please be cautious") } filter = nil } @@ -155,9 +155,9 @@ func RunProxy(f cmdutil.Factory, out io.Writer, cmd *cobra.Command) error { l, err = server.ListenUnix(path) } if err != nil { - glog.Fatal(err) + klog.Fatal(err) } fmt.Fprintf(out, "Starting to serve on %s\n", l.Addr().String()) - glog.Fatal(server.ServeOnListener(l)) + klog.Fatal(server.ServeOnListener(l)) return nil } diff --git a/pkg/kubectl/cmd/replace/BUILD b/pkg/kubectl/cmd/replace/BUILD index 6e58729442f..864f5a34747 100644 --- a/pkg/kubectl/cmd/replace/BUILD +++ b/pkg/kubectl/cmd/replace/BUILD @@ -18,8 +18,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/replace/replace.go b/pkg/kubectl/cmd/replace/replace.go index 1a72fd6a088..dcc8ddcb878 100644 --- a/pkg/kubectl/cmd/replace/replace.go +++ b/pkg/kubectl/cmd/replace/replace.go @@ -25,7 +25,7 @@ import ( "github.com/spf13/cobra" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -225,7 +225,7 @@ func (o *ReplaceOptions) Run() error { } if err := o.Recorder.Record(info.Object); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } // Serialize the object with the annotation applied. @@ -316,7 +316,7 @@ func (o *ReplaceOptions) forceReplace() error { } if err := o.Recorder.Record(info.Object); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } obj, err := resource.NewHelper(info.Client, info.Mapping).Create(info.Namespace, true, info.Object, nil) diff --git a/pkg/kubectl/cmd/rollingupdate/BUILD b/pkg/kubectl/cmd/rollingupdate/BUILD index ea784dc334b..cd16b37319b 100644 --- a/pkg/kubectl/cmd/rollingupdate/BUILD +++ b/pkg/kubectl/cmd/rollingupdate/BUILD @@ -23,8 +23,8 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/scale:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/rollingupdate/rollingupdate.go b/pkg/kubectl/cmd/rollingupdate/rollingupdate.go index 4947e720308..b39ad2049f6 100644 --- a/pkg/kubectl/cmd/rollingupdate/rollingupdate.go +++ b/pkg/kubectl/cmd/rollingupdate/rollingupdate.go @@ -21,8 +21,8 @@ import ( "fmt" "time" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" @@ -292,7 +292,7 @@ func (o *RollingUpdateOptions) Run() error { uncastVersionedObj, err := scheme.Scheme.ConvertToVersion(infos[0].Object, corev1.SchemeGroupVersion) if err != nil { - glog.V(4).Infof("Object %T is not a ReplicationController", infos[0].Object) + klog.V(4).Infof("Object %T is not a ReplicationController", infos[0].Object) return fmt.Errorf("%s contains a %v not a ReplicationController", filename, infos[0].Object.GetObjectKind().GroupVersionKind()) } switch t := uncastVersionedObj.(type) { @@ -301,7 +301,7 @@ func (o *RollingUpdateOptions) Run() error { newRc = t } if newRc == nil { - glog.V(4).Infof("Object %T is not a ReplicationController", infos[0].Object) + klog.V(4).Infof("Object %T is not a ReplicationController", infos[0].Object) return fmt.Errorf("%s contains a %v not a ReplicationController", filename, infos[0].Object.GetObjectKind().GroupVersionKind()) } } diff --git a/pkg/kubectl/cmd/run/BUILD b/pkg/kubectl/cmd/run/BUILD index bffca3654e9..180ba1b6e2c 100644 --- a/pkg/kubectl/cmd/run/BUILD +++ b/pkg/kubectl/cmd/run/BUILD @@ -33,8 +33,8 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//vendor/github.com/docker/distribution/reference:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/run/run.go b/pkg/kubectl/cmd/run/run.go index d8da7818361..fdd47b287a8 100644 --- a/pkg/kubectl/cmd/run/run.go +++ b/pkg/kubectl/cmd/run/run.go @@ -22,8 +22,8 @@ import ( "time" "github.com/docker/distribution/reference" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -316,7 +316,7 @@ func (o *RunOptions) Run(f cmdutil.Factory, cmd *cobra.Command, args []string) e if len(generatorName) == 0 { switch restartPolicy { case corev1.RestartPolicyAlways: - generatorName = generateversioned.DeploymentAppsV1Beta1GeneratorName + generatorName = generateversioned.DeploymentAppsV1GeneratorName case corev1.RestartPolicyOnFailure: generatorName = generateversioned.JobV1GeneratorName case corev1.RestartPolicyNever: @@ -674,7 +674,7 @@ func (o *RunOptions) createGeneratedObject(f cmdutil.Factory, cmd *cobra.Command } if err := o.Recorder.Record(obj); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } actualObj := obj diff --git a/pkg/kubectl/cmd/scale/BUILD b/pkg/kubectl/cmd/scale/BUILD index 1c30a322743..5c76a02f140 100644 --- a/pkg/kubectl/cmd/scale/BUILD +++ b/pkg/kubectl/cmd/scale/BUILD @@ -25,8 +25,8 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/batch/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/scale/scale.go b/pkg/kubectl/cmd/scale/scale.go index 1144e307565..997d37cefbb 100644 --- a/pkg/kubectl/cmd/scale/scale.go +++ b/pkg/kubectl/cmd/scale/scale.go @@ -20,8 +20,8 @@ import ( "fmt" "time" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" @@ -240,7 +240,7 @@ func (o *ScaleOptions) RunScale() error { // if the recorder makes a change, compute and create another patch if mergePatch, err := o.Recorder.MakeRecordMergePatch(info.Object); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } else if len(mergePatch) > 0 { client, err := o.unstructuredClientForMapping(mapping) if err != nil { @@ -248,7 +248,7 @@ func (o *ScaleOptions) RunScale() error { } helper := resource.NewHelper(client, mapping) if _, err := helper.Patch(info.Namespace, info.Name, types.MergePatchType, mergePatch, nil); err != nil { - glog.V(4).Infof("error recording reason: %v", err) + klog.V(4).Infof("error recording reason: %v", err) } } diff --git a/pkg/kubectl/cmd/set/BUILD b/pkg/kubectl/cmd/set/BUILD index cf756dbef41..aa428c673c2 100644 --- a/pkg/kubectl/cmd/set/BUILD +++ b/pkg/kubectl/cmd/set/BUILD @@ -41,8 +41,8 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/set/set_image.go b/pkg/kubectl/cmd/set/set_image.go index 3a18f36bec5..38677a803f0 100644 --- a/pkg/kubectl/cmd/set/set_image.go +++ b/pkg/kubectl/cmd/set/set_image.go @@ -19,8 +19,8 @@ package set import ( "fmt" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -256,7 +256,7 @@ func (o *SetImageOptions) Run() error { } // record this change (for rollout history) if err := o.Recorder.Record(obj); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } return runtime.Encode(scheme.DefaultJSONEncoder(), obj) diff --git a/pkg/kubectl/cmd/set/set_resources.go b/pkg/kubectl/cmd/set/set_resources.go index 6b94b6a4feb..b7328a0cb45 100644 --- a/pkg/kubectl/cmd/set/set_resources.go +++ b/pkg/kubectl/cmd/set/set_resources.go @@ -19,8 +19,8 @@ package set import ( "fmt" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -250,7 +250,7 @@ func (o *SetResourcesOptions) Run() error { } // record this change (for rollout history) if err := o.Recorder.Record(obj); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } return runtime.Encode(scheme.DefaultJSONEncoder(), obj) diff --git a/pkg/kubectl/cmd/set/set_selector.go b/pkg/kubectl/cmd/set/set_selector.go index 9babe08a2ad..167ae747170 100644 --- a/pkg/kubectl/cmd/set/set_selector.go +++ b/pkg/kubectl/cmd/set/set_selector.go @@ -19,8 +19,8 @@ package set import ( "fmt" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -171,7 +171,7 @@ func (o *SetSelectorOptions) RunSelector() error { // record this change (for rollout history) if err := o.Recorder.Record(patch.Info.Object); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } return runtime.Encode(scheme.DefaultJSONEncoder(), info.Object) diff --git a/pkg/kubectl/cmd/set/set_serviceaccount.go b/pkg/kubectl/cmd/set/set_serviceaccount.go index 45b0ae3057d..eced76f7e9c 100644 --- a/pkg/kubectl/cmd/set/set_serviceaccount.go +++ b/pkg/kubectl/cmd/set/set_serviceaccount.go @@ -20,8 +20,8 @@ import ( "errors" "fmt" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -183,7 +183,7 @@ func (o *SetServiceAccountOptions) Run() error { } // record this change (for rollout history) if err := o.Recorder.Record(obj); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } return runtime.Encode(scheme.DefaultJSONEncoder(), obj) diff --git a/pkg/kubectl/cmd/taint/BUILD b/pkg/kubectl/cmd/taint/BUILD index 19767bf0ed7..e5e42effbb9 100644 --- a/pkg/kubectl/cmd/taint/BUILD +++ b/pkg/kubectl/cmd/taint/BUILD @@ -24,8 +24,8 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/taint/taint.go b/pkg/kubectl/cmd/taint/taint.go index 13dbcfeed4e..02f56513079 100644 --- a/pkg/kubectl/cmd/taint/taint.go +++ b/pkg/kubectl/cmd/taint/taint.go @@ -21,8 +21,8 @@ import ( "fmt" "strings" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -98,15 +98,9 @@ func NewCmdTaint(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra. Long: fmt.Sprintf(taintLong, validation.DNS1123SubdomainMaxLength, validation.LabelValueMaxLength), Example: taintExample, Run: func(cmd *cobra.Command, args []string) { - if err := options.Complete(f, cmd, args); err != nil { - cmdutil.CheckErr(err) - } - if err := options.Validate(); err != nil { - cmdutil.CheckErr(cmdutil.UsageErrorf(cmd, err.Error())) - } - if err := options.RunTaint(); err != nil { - cmdutil.CheckErr(err) - } + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Validate()) + cmdutil.CheckErr(options.RunTaint()) }, ValidArgs: validArgs, } @@ -261,7 +255,7 @@ func (o TaintOptions) RunTaint() error { patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, obj) createdPatch := err == nil if err != nil { - glog.V(2).Infof("couldn't compute patch: %v", err) + klog.V(2).Infof("couldn't compute patch: %v", err) } mapping := info.ResourceMapping() diff --git a/pkg/kubectl/cmd/top/BUILD b/pkg/kubectl/cmd/top/BUILD index 2a879a973b1..0cec30b8f7b 100644 --- a/pkg/kubectl/cmd/top/BUILD +++ b/pkg/kubectl/cmd/top/BUILD @@ -23,9 +23,9 @@ go_library( "//staging/src/k8s.io/metrics/pkg/apis/metrics:go_default_library", "//staging/src/k8s.io/metrics/pkg/apis/metrics/v1beta1:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/clientset/versioned:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/top/top_pod.go b/pkg/kubectl/cmd/top/top_pod.go index c8f87d558ef..5c9a3712761 100644 --- a/pkg/kubectl/cmd/top/top_pod.go +++ b/pkg/kubectl/cmd/top/top_pod.go @@ -34,9 +34,9 @@ import ( metricsv1beta1api "k8s.io/metrics/pkg/apis/metrics/v1beta1" metricsclientset "k8s.io/metrics/pkg/client/clientset/versioned" - "github.com/golang/glog" "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/klog" ) type TopPodOptions struct { @@ -256,10 +256,10 @@ func checkPodAge(pod *v1.Pod) error { age := time.Since(pod.CreationTimestamp.Time) if age > metricsCreationDelay { message := fmt.Sprintf("Metrics not available for pod %s/%s, age: %s", pod.Namespace, pod.Name, age.String()) - glog.Warningf(message) + klog.Warningf(message) return errors.New(message) } else { - glog.V(2).Infof("Metrics not yet available for pod %s/%s, age: %s", pod.Namespace, pod.Name, age.String()) + klog.V(2).Infof("Metrics not yet available for pod %s/%s, age: %s", pod.Namespace, pod.Name, age.String()) return nil } } diff --git a/pkg/kubectl/cmd/util/BUILD b/pkg/kubectl/cmd/util/BUILD index bad60cd16c8..91f282405b5 100644 --- a/pkg/kubectl/cmd/util/BUILD +++ b/pkg/kubectl/cmd/util/BUILD @@ -39,9 +39,9 @@ go_library( "//staging/src/k8s.io/client-go/scale:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/util/crdfinder.go b/pkg/kubectl/cmd/util/crdfinder.go index d3674ae1601..aaf309dca44 100644 --- a/pkg/kubectl/cmd/util/crdfinder.go +++ b/pkg/kubectl/cmd/util/crdfinder.go @@ -17,6 +17,7 @@ limitations under the License. package util import ( + "fmt" "reflect" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,10 +35,10 @@ func CRDFromDynamic(client dynamic.Interface) CRDGetter { list, err := client.Resource(schema.GroupVersionResource{ Group: "apiextensions.k8s.io", Version: "v1beta1", - Resource: "curstomresourcedefinitions", + Resource: "customresourcedefinitions", }).List(metav1.ListOptions{}) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to list CRDs: %v", err) } if list == nil { return nil, nil diff --git a/pkg/kubectl/cmd/util/editor/BUILD b/pkg/kubectl/cmd/util/editor/BUILD index 464d84b1ff4..ac104a73023 100644 --- a/pkg/kubectl/cmd/util/editor/BUILD +++ b/pkg/kubectl/cmd/util/editor/BUILD @@ -35,8 +35,8 @@ go_library( "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/util/editor/editoptions.go b/pkg/kubectl/cmd/util/editor/editoptions.go index ab43141cf7c..2b2add45859 100644 --- a/pkg/kubectl/cmd/util/editor/editoptions.go +++ b/pkg/kubectl/cmd/util/editor/editoptions.go @@ -29,8 +29,8 @@ import ( "strings" "github.com/evanphx/json-patch" - "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/klog" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -287,7 +287,7 @@ func (o *EditOptions) Run() error { if len(results.file) > 0 { os.Remove(results.file) } - glog.V(4).Infof("User edited:\n%s", string(edited)) + klog.V(4).Infof("User edited:\n%s", string(edited)) // Apply validation schema, err := o.f.Validator(o.EnableValidation) @@ -606,12 +606,12 @@ func (o *EditOptions) visitToPatch(originalInfos []*resource.Info, patchVisitor patchType = types.MergePatchType patch, err = jsonpatch.CreateMergePatch(originalJS, editedJS) if err != nil { - glog.V(4).Infof("Unable to calculate diff, no merge is possible: %v", err) + klog.V(4).Infof("Unable to calculate diff, no merge is possible: %v", err) return err } for _, precondition := range preconditions { if !precondition(patch) { - glog.V(4).Infof("Unable to calculate diff, no merge is possible: %v", err) + klog.V(4).Infof("Unable to calculate diff, no merge is possible: %v", err) return fmt.Errorf("%s", "At least one of apiVersion, kind and name was changed") } } @@ -621,7 +621,7 @@ func (o *EditOptions) visitToPatch(originalInfos []*resource.Info, patchVisitor patchType = types.StrategicMergePatchType patch, err = strategicpatch.CreateTwoWayMergePatch(originalJS, editedJS, versionedObject, preconditions...) if err != nil { - glog.V(4).Infof("Unable to calculate diff, no merge is possible: %v", err) + klog.V(4).Infof("Unable to calculate diff, no merge is possible: %v", err) if mergepatch.IsPreconditionFailed(err) { return fmt.Errorf("%s", "At least one of apiVersion, kind and name was changed") } @@ -674,7 +674,7 @@ func (o *EditOptions) visitAnnotation(annotationVisitor resource.Visitor) error } } if err := o.Recorder.Record(info.Object); err != nil { - glog.V(4).Infof("error recording current command: %v", err) + klog.V(4).Infof("error recording current command: %v", err) } return nil diff --git a/pkg/kubectl/cmd/util/editor/editor.go b/pkg/kubectl/cmd/util/editor/editor.go index a53e8c0970a..e7229870ec7 100644 --- a/pkg/kubectl/cmd/util/editor/editor.go +++ b/pkg/kubectl/cmd/util/editor/editor.go @@ -27,7 +27,7 @@ import ( "runtime" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubectl/util/term" ) @@ -124,7 +124,7 @@ func (e Editor) Launch(path string) error { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdin - glog.V(5).Infof("Opening file with editor %v", args) + klog.V(5).Infof("Opening file with editor %v", args) if err := (term.TTY{In: os.Stdin, TryDev: true}).Safe(cmd.Run); err != nil { if err, ok := err.(*exec.Error); ok { if err.Err == exec.ErrNotFound { diff --git a/pkg/kubectl/cmd/util/helpers.go b/pkg/kubectl/cmd/util/helpers.go index da9e25d04de..9a392cde59b 100644 --- a/pkg/kubectl/cmd/util/helpers.go +++ b/pkg/kubectl/cmd/util/helpers.go @@ -27,9 +27,9 @@ import ( "time" "github.com/evanphx/json-patch" - "github.com/golang/glog" "github.com/spf13/cobra" "github.com/spf13/pflag" + "k8s.io/klog" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -88,10 +88,10 @@ func DefaultBehaviorOnFatal() { } // fatal prints the message (if provided) and then exits. If V(2) or greater, -// glog.Fatal is invoked for extended information. +// klog.Fatal is invoked for extended information. func fatal(msg string, code int) { - if glog.V(2) { - glog.FatalDepth(2, msg) + if klog.V(2) { + klog.FatalDepth(2, msg) } if len(msg) > 0 { // add newline if needed @@ -189,13 +189,13 @@ func statusCausesToAggrError(scs []metav1.StatusCause) utilerrors.Aggregate { // StandardErrorMessage translates common errors into a human readable message, or returns // false if the error is not one of the recognized types. It may also log extended -// information to glog. +// information to klog. // // This method is generic to the command in use and may be used by non-Kubectl // commands. func StandardErrorMessage(err error) (string, bool) { if debugErr, ok := err.(debugError); ok { - glog.V(4).Infof(debugErr.DebugError()) + klog.V(4).Infof(debugErr.DebugError()) } status, isStatus := err.(kerrors.APIStatus) switch { @@ -213,7 +213,7 @@ func StandardErrorMessage(err error) (string, bool) { } switch t := err.(type) { case *url.Error: - glog.V(4).Infof("Connection error: %s %s: %v", t.Op, t.URL, t.Err) + klog.V(4).Infof("Connection error: %s %s: %v", t.Op, t.URL, t.Err) switch { case strings.Contains(t.Err.Error(), "connection refused"): host := t.URL @@ -300,7 +300,7 @@ func IsFilenameSliceEmpty(filenames []string) bool { func GetFlagString(cmd *cobra.Command, flag string) string { s, err := cmd.Flags().GetString(flag) if err != nil { - glog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) + klog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) } return s } @@ -309,7 +309,7 @@ func GetFlagString(cmd *cobra.Command, flag string) string { func GetFlagStringSlice(cmd *cobra.Command, flag string) []string { s, err := cmd.Flags().GetStringSlice(flag) if err != nil { - glog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) + klog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) } return s } @@ -318,7 +318,7 @@ func GetFlagStringSlice(cmd *cobra.Command, flag string) []string { func GetFlagStringArray(cmd *cobra.Command, flag string) []string { s, err := cmd.Flags().GetStringArray(flag) if err != nil { - glog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) + klog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) } return s } @@ -326,7 +326,7 @@ func GetFlagStringArray(cmd *cobra.Command, flag string) []string { func GetFlagBool(cmd *cobra.Command, flag string) bool { b, err := cmd.Flags().GetBool(flag) if err != nil { - glog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) + klog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) } return b } @@ -335,7 +335,7 @@ func GetFlagBool(cmd *cobra.Command, flag string) bool { func GetFlagInt(cmd *cobra.Command, flag string) int { i, err := cmd.Flags().GetInt(flag) if err != nil { - glog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) + klog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) } return i } @@ -344,7 +344,7 @@ func GetFlagInt(cmd *cobra.Command, flag string) int { func GetFlagInt32(cmd *cobra.Command, flag string) int32 { i, err := cmd.Flags().GetInt32(flag) if err != nil { - glog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) + klog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) } return i } @@ -353,7 +353,7 @@ func GetFlagInt32(cmd *cobra.Command, flag string) int32 { func GetFlagInt64(cmd *cobra.Command, flag string) int64 { i, err := cmd.Flags().GetInt64(flag) if err != nil { - glog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) + klog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) } return i } @@ -361,7 +361,7 @@ func GetFlagInt64(cmd *cobra.Command, flag string) int64 { func GetFlagDuration(cmd *cobra.Command, flag string) time.Duration { d, err := cmd.Flags().GetDuration(flag) if err != nil { - glog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) + klog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) } return d } diff --git a/pkg/kubectl/cmd/version/BUILD b/pkg/kubectl/cmd/version/BUILD index 64c3e569525..514bc705144 100644 --- a/pkg/kubectl/cmd/version/BUILD +++ b/pkg/kubectl/cmd/version/BUILD @@ -13,8 +13,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", "//staging/src/k8s.io/client-go/discovery:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", + "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/pkg/kubectl/cmd/version/version.go b/pkg/kubectl/cmd/version/version.go index 8682063ead1..1d249e213d2 100644 --- a/pkg/kubectl/cmd/version/version.go +++ b/pkg/kubectl/cmd/version/version.go @@ -21,12 +21,13 @@ import ( "errors" "fmt" - "github.com/ghodss/yaml" "github.com/spf13/cobra" + "sigs.k8s.io/yaml" apimachineryversion "k8s.io/apimachinery/pkg/version" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/discovery" + "k8s.io/client-go/tools/clientcmd" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/kubectl/util/templates" @@ -83,7 +84,9 @@ func NewCmdVersion(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *co func (o *VersionOptions) Complete(f cmdutil.Factory, cmd *cobra.Command) error { var err error o.discoveryClient, err = f.ToDiscoveryClient() - if err != nil { + // if we had an empty rest.Config, continue and just print out client information. + // if we had an error other than being unable to build a rest.Config, fail. + if err != nil && !clientcmd.IsEmptyConfig(err) { return err } return nil @@ -107,7 +110,7 @@ func (o *VersionOptions) Run() error { clientVersion := version.Get() versionInfo.ClientVersion = &clientVersion - if !o.ClientOnly { + if !o.ClientOnly && o.discoveryClient != nil { // Always request fresh data from the server o.discoveryClient.Invalidate() serverVersion, serverErr = o.discoveryClient.ServerVersion() diff --git a/pkg/kubectl/cmd/wait/wait.go b/pkg/kubectl/cmd/wait/wait.go index c338fdec00d..6a1e075e816 100644 --- a/pkg/kubectl/cmd/wait/wait.go +++ b/pkg/kubectl/cmd/wait/wait.go @@ -249,6 +249,9 @@ func (o *WaitOptions) RunWait() error { func IsDeleted(info *resource.Info, o *WaitOptions) (runtime.Object, bool, error) { endTime := time.Now().Add(o.Timeout) for { + if len(info.Name) == 0 { + return info.Object, false, fmt.Errorf("resource name must be provided") + } gottenObj, err := o.DynamicClient.Resource(info.Mapping.Resource).Namespace(info.Namespace).Get(info.Name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { return info.Object, true, nil @@ -334,6 +337,9 @@ type ConditionalWait struct { func (w ConditionalWait) IsConditionMet(info *resource.Info, o *WaitOptions) (runtime.Object, bool, error) { endTime := time.Now().Add(o.Timeout) for { + if len(info.Name) == 0 { + return info.Object, false, fmt.Errorf("resource name must be provided") + } resourceVersion := "" gottenObj, err := o.DynamicClient.Resource(info.Mapping.Resource).Namespace(info.Namespace).Get(info.Name, metav1.GetOptions{}) switch { diff --git a/pkg/kubectl/cmd/wait/wait_test.go b/pkg/kubectl/cmd/wait/wait_test.go index ef34dec72ac..cea5e64f321 100644 --- a/pkg/kubectl/cmd/wait/wait_test.go +++ b/pkg/kubectl/cmd/wait/wait_test.go @@ -445,6 +445,28 @@ func TestWaitForCondition(t *testing.T) { } }, }, + { + name: "handles empty object name", + infos: []*resource.Info{ + { + Mapping: &meta.RESTMapping{ + Resource: schema.GroupVersionResource{Group: "group", Version: "version", Resource: "theresource"}, + }, + Namespace: "ns-foo", + }, + }, + fakeClient: func() *dynamicfakeclient.FakeDynamicClient { + return dynamicfakeclient.NewSimpleDynamicClient(scheme) + }, + timeout: 10 * time.Second, + expectedErr: "resource name must be provided", + + validateActions: func(t *testing.T, actions []clienttesting.Action) { + if len(actions) != 0 { + t.Fatal(spew.Sdump(actions)) + } + }, + }, { name: "times out", infos: []*resource.Info{ diff --git a/pkg/kubectl/describe/BUILD b/pkg/kubectl/describe/BUILD index 533c5dfb2c4..c120954a8d2 100644 --- a/pkg/kubectl/describe/BUILD +++ b/pkg/kubectl/describe/BUILD @@ -2,11 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["describe.go"], + srcs = ["interface.go"], importpath = "k8s.io/kubernetes/pkg/kubectl/describe", visibility = ["//visibility:public"], deps = [ - "//pkg/printers:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", ], diff --git a/pkg/kubectl/describe/interface.go b/pkg/kubectl/describe/interface.go new file mode 100644 index 00000000000..58429918787 --- /dev/null +++ b/pkg/kubectl/describe/interface.go @@ -0,0 +1,71 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package describe + +import ( + "fmt" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +const ( + // LoadBalancerWidth is the width how we describe load balancer + LoadBalancerWidth = 16 + + // LabelNodeRolePrefix is a label prefix for node roles + // It's copied over to here until it's merged in core: https://github.com/kubernetes/kubernetes/pull/39112 + LabelNodeRolePrefix = "node-role.kubernetes.io/" + + // NodeLabelRole specifies the role of a node + NodeLabelRole = "kubernetes.io/role" +) + +// DescriberFunc gives a way to display the specified RESTMapping type +type DescriberFunc func(restClientGetter genericclioptions.RESTClientGetter, mapping *meta.RESTMapping) (Describer, error) + +// Describer generates output for the named resource or an error +// if the output could not be generated. Implementers typically +// abstract the retrieval of the named object from a remote server. +type Describer interface { + Describe(namespace, name string, describerSettings DescriberSettings) (output string, err error) +} + +// DescriberSettings holds display configuration for each object +// describer to control what is printed. +type DescriberSettings struct { + ShowEvents bool +} + +// ObjectDescriber is an interface for displaying arbitrary objects with extra +// information. Use when an object is in hand (on disk, or already retrieved). +// Implementers may ignore the additional information passed on extra, or use it +// by default. ObjectDescribers may return ErrNoDescriber if no suitable describer +// is found. +type ObjectDescriber interface { + DescribeObject(object interface{}, extra ...interface{}) (output string, err error) +} + +// ErrNoDescriber is a structured error indicating the provided object or objects +// cannot be described. +type ErrNoDescriber struct { + Types []string +} + +// Error implements the error interface. +func (e ErrNoDescriber) Error() string { + return fmt.Sprintf("no describer has been defined for %v", e.Types) +} diff --git a/pkg/kubectl/describe/versioned/BUILD b/pkg/kubectl/describe/versioned/BUILD index b0c4d44ed46..21c1a0b78c7 100644 --- a/pkg/kubectl/describe/versioned/BUILD +++ b/pkg/kubectl/describe/versioned/BUILD @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -7,10 +7,47 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/kubectl/describe:go_default_library", - "//pkg/printers:go_default_library", - "//pkg/printers/internalversion:go_default_library", + "//pkg/kubectl/scheme:go_default_library", + "//pkg/kubectl/util/certificate:go_default_library", + "//pkg/kubectl/util/deployment:go_default_library", + "//pkg/kubectl/util/event:go_default_library", + "//pkg/kubectl/util/fieldpath:go_default_library", + "//pkg/kubectl/util/qos:go_default_library", + "//pkg/kubectl/util/rbac:go_default_library", + "//pkg/kubectl/util/resource:go_default_library", + "//pkg/kubectl/util/slice:go_default_library", + "//pkg/kubectl/util/storage:go_default_library", + "//staging/src/k8s.io/api/apps/v1:go_default_library", + "//staging/src/k8s.io/api/autoscaling/v2beta2:go_default_library", + "//staging/src/k8s.io/api/batch/v1:go_default_library", + "//staging/src/k8s.io/api/batch/v1beta1:go_default_library", + "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library", + "//staging/src/k8s.io/api/networking/v1:go_default_library", + "//staging/src/k8s.io/api/policy/v1beta1:go_default_library", + "//staging/src/k8s.io/api/rbac/v1:go_default_library", + "//staging/src/k8s.io/api/scheduling/v1beta1:go_default_library", + "//staging/src/k8s.io/api/storage/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/duration:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", + "//staging/src/k8s.io/client-go/dynamic:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/client-go/tools/reference:go_default_library", + "//vendor/github.com/fatih/camelcase:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -27,3 +64,26 @@ filegroup( tags = ["automanaged"], visibility = ["//visibility:public"], ) + +go_test( + name = "go_default_test", + srcs = ["describe_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/kubectl/describe:go_default_library", + "//staging/src/k8s.io/api/apps/v1:go_default_library", + "//staging/src/k8s.io/api/autoscaling/v2beta2:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/api/networking/v1:go_default_library", + "//staging/src/k8s.io/api/policy/v1beta1:go_default_library", + "//staging/src/k8s.io/api/storage/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", + ], +) diff --git a/pkg/kubectl/describe/versioned/describe.go b/pkg/kubectl/describe/versioned/describe.go index b79b9ca20e0..6f6dba6f0bc 100644 --- a/pkg/kubectl/describe/versioned/describe.go +++ b/pkg/kubectl/describe/versioned/describe.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2014 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,32 +17,4422 @@ limitations under the License. package versioned import ( + "bytes" + "crypto/x509" "fmt" + "io" + "net" + "net/url" + "reflect" + "sort" + "strconv" + "strings" + "text/tabwriter" + "time" + "github.com/fatih/camelcase" + + appsv1 "k8s.io/api/apps/v1" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" + batchv1 "k8s.io/api/batch/v1" + batchv1beta1 "k8s.io/api/batch/v1beta1" + certificatesv1beta1 "k8s.io/api/certificates/v1beta1" + corev1 "k8s.io/api/core/v1" + extensionsv1beta1 "k8s.io/api/extensions/v1beta1" + networkingv1 "k8s.io/api/networking/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" + rbacv1 "k8s.io/api/rbac/v1" + schedulingv1beta1 "k8s.io/api/scheduling/v1beta1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/dynamic" + clientset "k8s.io/client-go/kubernetes" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/reference" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubectl/describe" - "k8s.io/kubernetes/pkg/printers" - printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" + "k8s.io/kubernetes/pkg/kubectl/scheme" + "k8s.io/kubernetes/pkg/kubectl/util/certificate" + deploymentutil "k8s.io/kubernetes/pkg/kubectl/util/deployment" + "k8s.io/kubernetes/pkg/kubectl/util/event" + "k8s.io/kubernetes/pkg/kubectl/util/fieldpath" + "k8s.io/kubernetes/pkg/kubectl/util/qos" + "k8s.io/kubernetes/pkg/kubectl/util/rbac" + resourcehelper "k8s.io/kubernetes/pkg/kubectl/util/resource" + "k8s.io/kubernetes/pkg/kubectl/util/slice" + storageutil "k8s.io/kubernetes/pkg/kubectl/util/storage" +) + +// Each level has 2 spaces for PrefixWriter +const ( + LEVEL_0 = iota + LEVEL_1 + LEVEL_2 + LEVEL_3 ) // DescriberFn gives a way to easily override the function for unit testing if needed var DescriberFn describe.DescriberFunc = Describer // Describer returns a Describer for displaying the specified RESTMapping type or an error. -func Describer(restClientGetter genericclioptions.RESTClientGetter, mapping *meta.RESTMapping) (printers.Describer, error) { +func Describer(restClientGetter genericclioptions.RESTClientGetter, mapping *meta.RESTMapping) (describe.Describer, error) { clientConfig, err := restClientGetter.ToRESTConfig() if err != nil { return nil, err } // try to get a describer - if describer, ok := printersinternal.DescriberFor(mapping.GroupVersionKind.GroupKind(), clientConfig); ok { + if describer, ok := DescriberFor(mapping.GroupVersionKind.GroupKind(), clientConfig); ok { return describer, nil } // if this is a kind we don't have a describer for yet, go generic if possible - if genericDescriber, ok := printersinternal.GenericDescriberFor(mapping, clientConfig); ok { + if genericDescriber, ok := GenericDescriberFor(mapping, clientConfig); ok { return genericDescriber, nil } // otherwise return an unregistered error return nil, fmt.Errorf("no description has been implemented for %s", mapping.GroupVersionKind.String()) } + +// PrefixWriter can write text at various indentation levels. +type PrefixWriter interface { + // Write writes text with the specified indentation level. + Write(level int, format string, a ...interface{}) + // WriteLine writes an entire line with no indentation level. + WriteLine(a ...interface{}) + // Flush forces indentation to be reset. + Flush() +} + +// prefixWriter implements PrefixWriter +type prefixWriter struct { + out io.Writer +} + +var _ PrefixWriter = &prefixWriter{} + +// NewPrefixWriter creates a new PrefixWriter. +func NewPrefixWriter(out io.Writer) PrefixWriter { + return &prefixWriter{out: out} +} + +func (pw *prefixWriter) Write(level int, format string, a ...interface{}) { + levelSpace := " " + prefix := "" + for i := 0; i < level; i++ { + prefix += levelSpace + } + fmt.Fprintf(pw.out, prefix+format, a...) +} + +func (pw *prefixWriter) WriteLine(a ...interface{}) { + fmt.Fprintln(pw.out, a...) +} + +func (pw *prefixWriter) Flush() { + if f, ok := pw.out.(flusher); ok { + f.Flush() + } +} + +func describerMap(clientConfig *rest.Config) (map[schema.GroupKind]describe.Describer, error) { + c, err := clientset.NewForConfig(clientConfig) + if err != nil { + return nil, err + } + + m := map[schema.GroupKind]describe.Describer{ + {Group: corev1.GroupName, Kind: "Pod"}: &PodDescriber{c}, + {Group: corev1.GroupName, Kind: "ReplicationController"}: &ReplicationControllerDescriber{c}, + {Group: corev1.GroupName, Kind: "Secret"}: &SecretDescriber{c}, + {Group: corev1.GroupName, Kind: "Service"}: &ServiceDescriber{c}, + {Group: corev1.GroupName, Kind: "ServiceAccount"}: &ServiceAccountDescriber{c}, + {Group: corev1.GroupName, Kind: "Node"}: &NodeDescriber{c}, + {Group: corev1.GroupName, Kind: "LimitRange"}: &LimitRangeDescriber{c}, + {Group: corev1.GroupName, Kind: "ResourceQuota"}: &ResourceQuotaDescriber{c}, + {Group: corev1.GroupName, Kind: "PersistentVolume"}: &PersistentVolumeDescriber{c}, + {Group: corev1.GroupName, Kind: "PersistentVolumeClaim"}: &PersistentVolumeClaimDescriber{c}, + {Group: corev1.GroupName, Kind: "Namespace"}: &NamespaceDescriber{c}, + {Group: corev1.GroupName, Kind: "Endpoints"}: &EndpointsDescriber{c}, + {Group: corev1.GroupName, Kind: "ConfigMap"}: &ConfigMapDescriber{c}, + {Group: corev1.GroupName, Kind: "PriorityClass"}: &PriorityClassDescriber{c}, + {Group: extensionsv1beta1.GroupName, Kind: "ReplicaSet"}: &ReplicaSetDescriber{c}, + {Group: extensionsv1beta1.GroupName, Kind: "NetworkPolicy"}: &NetworkPolicyDescriber{c}, + {Group: extensionsv1beta1.GroupName, Kind: "PodSecurityPolicy"}: &PodSecurityPolicyDescriber{c}, + {Group: autoscalingv2beta2.GroupName, Kind: "HorizontalPodAutoscaler"}: &HorizontalPodAutoscalerDescriber{c}, + {Group: extensionsv1beta1.GroupName, Kind: "DaemonSet"}: &DaemonSetDescriber{c}, + {Group: extensionsv1beta1.GroupName, Kind: "Deployment"}: &DeploymentDescriber{c}, + {Group: extensionsv1beta1.GroupName, Kind: "Ingress"}: &IngressDescriber{c}, + {Group: batchv1.GroupName, Kind: "Job"}: &JobDescriber{c}, + {Group: batchv1.GroupName, Kind: "CronJob"}: &CronJobDescriber{c}, + {Group: appsv1.GroupName, Kind: "StatefulSet"}: &StatefulSetDescriber{c}, + {Group: appsv1.GroupName, Kind: "Deployment"}: &DeploymentDescriber{c}, + {Group: appsv1.GroupName, Kind: "DaemonSet"}: &DaemonSetDescriber{c}, + {Group: appsv1.GroupName, Kind: "ReplicaSet"}: &ReplicaSetDescriber{c}, + {Group: certificatesv1beta1.GroupName, Kind: "CertificateSigningRequest"}: &CertificateSigningRequestDescriber{c}, + {Group: storagev1.GroupName, Kind: "StorageClass"}: &StorageClassDescriber{c}, + {Group: policyv1beta1.GroupName, Kind: "PodDisruptionBudget"}: &PodDisruptionBudgetDescriber{c}, + {Group: rbacv1.GroupName, Kind: "Role"}: &RoleDescriber{c}, + {Group: rbacv1.GroupName, Kind: "ClusterRole"}: &ClusterRoleDescriber{c}, + {Group: rbacv1.GroupName, Kind: "RoleBinding"}: &RoleBindingDescriber{c}, + {Group: rbacv1.GroupName, Kind: "ClusterRoleBinding"}: &ClusterRoleBindingDescriber{c}, + {Group: networkingv1.GroupName, Kind: "NetworkPolicy"}: &NetworkPolicyDescriber{c}, + {Group: schedulingv1beta1.GroupName, Kind: "PriorityClass"}: &PriorityClassDescriber{c}, + } + + return m, nil +} + +// DescriberFor returns the default describe functions for each of the standard +// Kubernetes types. +func DescriberFor(kind schema.GroupKind, clientConfig *rest.Config) (describe.Describer, bool) { + describers, err := describerMap(clientConfig) + if err != nil { + klog.V(1).Info(err) + return nil, false + } + + f, ok := describers[kind] + return f, ok +} + +// GenericDescriberFor returns a generic describer for the specified mapping +// that uses only information available from runtime.Unstructured +func GenericDescriberFor(mapping *meta.RESTMapping, clientConfig *rest.Config) (describe.Describer, bool) { + // used to fetch the resource + dynamicClient, err := dynamic.NewForConfig(clientConfig) + if err != nil { + return nil, false + } + + // used to get events for the resource + clientSet, err := clientset.NewForConfig(clientConfig) + if err != nil { + return nil, false + } + eventsClient := clientSet.Core() + + return &genericDescriber{mapping, dynamicClient, eventsClient}, true +} + +type genericDescriber struct { + mapping *meta.RESTMapping + dynamic dynamic.Interface + events corev1client.EventsGetter +} + +func (g *genericDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (output string, err error) { + obj, err := g.dynamic.Resource(g.mapping.Resource).Namespace(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = g.events.Events(namespace).Search(scheme.Scheme, obj) + } + + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", obj.GetName()) + w.Write(LEVEL_0, "Namespace:\t%s\n", obj.GetNamespace()) + printLabelsMultiline(w, "Labels", obj.GetLabels()) + printAnnotationsMultiline(w, "Annotations", obj.GetAnnotations()) + printUnstructuredContent(w, LEVEL_0, obj.UnstructuredContent(), "", ".metadata.name", ".metadata.namespace", ".metadata.labels", ".metadata.annotations") + if events != nil { + DescribeEvents(events, w) + } + return nil + }) +} + +func printUnstructuredContent(w PrefixWriter, level int, content map[string]interface{}, skipPrefix string, skip ...string) { + fields := []string{} + for field := range content { + fields = append(fields, field) + } + sort.Strings(fields) + + for _, field := range fields { + value := content[field] + switch typedValue := value.(type) { + case map[string]interface{}: + skipExpr := fmt.Sprintf("%s.%s", skipPrefix, field) + if slice.ContainsString(skip, skipExpr, nil) { + continue + } + w.Write(level, "%s:\n", smartLabelFor(field)) + printUnstructuredContent(w, level+1, typedValue, skipExpr, skip...) + + case []interface{}: + skipExpr := fmt.Sprintf("%s.%s", skipPrefix, field) + if slice.ContainsString(skip, skipExpr, nil) { + continue + } + w.Write(level, "%s:\n", smartLabelFor(field)) + for _, child := range typedValue { + switch typedChild := child.(type) { + case map[string]interface{}: + printUnstructuredContent(w, level+1, typedChild, skipExpr, skip...) + default: + w.Write(level+1, "%v\n", typedChild) + } + } + + default: + skipExpr := fmt.Sprintf("%s.%s", skipPrefix, field) + if slice.ContainsString(skip, skipExpr, nil) { + continue + } + w.Write(level, "%s:\t%v\n", smartLabelFor(field), typedValue) + } + } +} + +func smartLabelFor(field string) string { + commonAcronyms := []string{"API", "URL", "UID", "OSB", "GUID"} + + parts := camelcase.Split(field) + result := make([]string, 0, len(parts)) + for _, part := range parts { + if part == "_" { + continue + } + + if slice.ContainsString(commonAcronyms, strings.ToUpper(part), nil) { + part = strings.ToUpper(part) + } else { + part = strings.Title(part) + } + result = append(result, part) + } + + return strings.Join(result, " ") +} + +// DefaultObjectDescriber can describe the default Kubernetes objects. +var DefaultObjectDescriber describe.ObjectDescriber + +func init() { + d := &Describers{} + err := d.Add( + describeLimitRange, + describeQuota, + describePod, + describeService, + describeReplicationController, + describeDaemonSet, + describeNode, + describeNamespace, + ) + if err != nil { + klog.Fatalf("Cannot register describers: %v", err) + } + DefaultObjectDescriber = d +} + +// NamespaceDescriber generates information about a namespace +type NamespaceDescriber struct { + clientset.Interface +} + +func (d *NamespaceDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + ns, err := d.Core().Namespaces().Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + resourceQuotaList, err := d.Core().ResourceQuotas(name).List(metav1.ListOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // Server does not support resource quotas. + // Not an error, will not show resource quotas information. + resourceQuotaList = nil + } else { + return "", err + } + } + limitRangeList, err := d.Core().LimitRanges(name).List(metav1.ListOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // Server does not support limit ranges. + // Not an error, will not show limit ranges information. + limitRangeList = nil + } else { + return "", err + } + } + return describeNamespace(ns, resourceQuotaList, limitRangeList) +} + +func describeNamespace(namespace *corev1.Namespace, resourceQuotaList *corev1.ResourceQuotaList, limitRangeList *corev1.LimitRangeList) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", namespace.Name) + printLabelsMultiline(w, "Labels", namespace.Labels) + printAnnotationsMultiline(w, "Annotations", namespace.Annotations) + w.Write(LEVEL_0, "Status:\t%s\n", string(namespace.Status.Phase)) + if resourceQuotaList != nil { + w.Write(LEVEL_0, "\n") + DescribeResourceQuotas(resourceQuotaList, w) + } + if limitRangeList != nil { + w.Write(LEVEL_0, "\n") + DescribeLimitRanges(limitRangeList, w) + } + return nil + }) +} + +func describeLimitRangeSpec(spec corev1.LimitRangeSpec, prefix string, w PrefixWriter) { + for i := range spec.Limits { + item := spec.Limits[i] + maxResources := item.Max + minResources := item.Min + defaultLimitResources := item.Default + defaultRequestResources := item.DefaultRequest + ratio := item.MaxLimitRequestRatio + + set := map[corev1.ResourceName]bool{} + for k := range maxResources { + set[k] = true + } + for k := range minResources { + set[k] = true + } + for k := range defaultLimitResources { + set[k] = true + } + for k := range defaultRequestResources { + set[k] = true + } + for k := range ratio { + set[k] = true + } + + for k := range set { + // if no value is set, we output - + maxValue := "-" + minValue := "-" + defaultLimitValue := "-" + defaultRequestValue := "-" + ratioValue := "-" + + maxQuantity, maxQuantityFound := maxResources[k] + if maxQuantityFound { + maxValue = maxQuantity.String() + } + + minQuantity, minQuantityFound := minResources[k] + if minQuantityFound { + minValue = minQuantity.String() + } + + defaultLimitQuantity, defaultLimitQuantityFound := defaultLimitResources[k] + if defaultLimitQuantityFound { + defaultLimitValue = defaultLimitQuantity.String() + } + + defaultRequestQuantity, defaultRequestQuantityFound := defaultRequestResources[k] + if defaultRequestQuantityFound { + defaultRequestValue = defaultRequestQuantity.String() + } + + ratioQuantity, ratioQuantityFound := ratio[k] + if ratioQuantityFound { + ratioValue = ratioQuantity.String() + } + + msg := "%s%s\t%v\t%v\t%v\t%v\t%v\t%v\n" + w.Write(LEVEL_0, msg, prefix, item.Type, k, minValue, maxValue, defaultRequestValue, defaultLimitValue, ratioValue) + } + } +} + +// DescribeLimitRanges merges a set of limit range items into a single tabular description +func DescribeLimitRanges(limitRanges *corev1.LimitRangeList, w PrefixWriter) { + if len(limitRanges.Items) == 0 { + w.Write(LEVEL_0, "No resource limits.\n") + return + } + w.Write(LEVEL_0, "Resource Limits\n Type\tResource\tMin\tMax\tDefault Request\tDefault Limit\tMax Limit/Request Ratio\n") + w.Write(LEVEL_0, " ----\t--------\t---\t---\t---------------\t-------------\t-----------------------\n") + for _, limitRange := range limitRanges.Items { + describeLimitRangeSpec(limitRange.Spec, " ", w) + } +} + +// DescribeResourceQuotas merges a set of quota items into a single tabular description of all quotas +func DescribeResourceQuotas(quotas *corev1.ResourceQuotaList, w PrefixWriter) { + if len(quotas.Items) == 0 { + w.Write(LEVEL_0, "No resource quota.\n") + return + } + sort.Sort(SortableResourceQuotas(quotas.Items)) + + w.Write(LEVEL_0, "Resource Quotas") + for _, q := range quotas.Items { + w.Write(LEVEL_0, "\n Name:\t%s\n", q.Name) + if len(q.Spec.Scopes) > 0 { + scopes := make([]string, 0, len(q.Spec.Scopes)) + for _, scope := range q.Spec.Scopes { + scopes = append(scopes, string(scope)) + } + sort.Strings(scopes) + w.Write(LEVEL_0, " Scopes:\t%s\n", strings.Join(scopes, ", ")) + for _, scope := range scopes { + helpText := helpTextForResourceQuotaScope(corev1.ResourceQuotaScope(scope)) + if len(helpText) > 0 { + w.Write(LEVEL_0, " * %s\n", helpText) + } + } + } + + w.Write(LEVEL_0, " Resource\tUsed\tHard\n") + w.Write(LEVEL_0, " --------\t---\t---\n") + + resources := make([]corev1.ResourceName, 0, len(q.Status.Hard)) + for resource := range q.Status.Hard { + resources = append(resources, resource) + } + sort.Sort(SortableResourceNames(resources)) + + for _, resource := range resources { + hardQuantity := q.Status.Hard[resource] + usedQuantity := q.Status.Used[resource] + w.Write(LEVEL_0, " %s\t%s\t%s\n", string(resource), usedQuantity.String(), hardQuantity.String()) + } + } +} + +// LimitRangeDescriber generates information about a limit range +type LimitRangeDescriber struct { + clientset.Interface +} + +func (d *LimitRangeDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + lr := d.Core().LimitRanges(namespace) + + limitRange, err := lr.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + return describeLimitRange(limitRange) +} + +func describeLimitRange(limitRange *corev1.LimitRange) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", limitRange.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", limitRange.Namespace) + w.Write(LEVEL_0, "Type\tResource\tMin\tMax\tDefault Request\tDefault Limit\tMax Limit/Request Ratio\n") + w.Write(LEVEL_0, "----\t--------\t---\t---\t---------------\t-------------\t-----------------------\n") + describeLimitRangeSpec(limitRange.Spec, "", w) + return nil + }) +} + +// ResourceQuotaDescriber generates information about a resource quota +type ResourceQuotaDescriber struct { + clientset.Interface +} + +func (d *ResourceQuotaDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + rq := d.Core().ResourceQuotas(namespace) + + resourceQuota, err := rq.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + return describeQuota(resourceQuota) +} + +func helpTextForResourceQuotaScope(scope corev1.ResourceQuotaScope) string { + switch scope { + case corev1.ResourceQuotaScopeTerminating: + return "Matches all pods that have an active deadline. These pods have a limited lifespan on a node before being actively terminated by the system." + case corev1.ResourceQuotaScopeNotTerminating: + return "Matches all pods that do not have an active deadline. These pods usually include long running pods whose container command is not expected to terminate." + case corev1.ResourceQuotaScopeBestEffort: + return "Matches all pods that do not have resource requirements set. These pods have a best effort quality of service." + case corev1.ResourceQuotaScopeNotBestEffort: + return "Matches all pods that have at least one resource requirement set. These pods have a burstable or guaranteed quality of service." + default: + return "" + } +} +func describeQuota(resourceQuota *corev1.ResourceQuota) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", resourceQuota.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", resourceQuota.Namespace) + if len(resourceQuota.Spec.Scopes) > 0 { + scopes := make([]string, 0, len(resourceQuota.Spec.Scopes)) + for _, scope := range resourceQuota.Spec.Scopes { + scopes = append(scopes, string(scope)) + } + sort.Strings(scopes) + w.Write(LEVEL_0, "Scopes:\t%s\n", strings.Join(scopes, ", ")) + for _, scope := range scopes { + helpText := helpTextForResourceQuotaScope(corev1.ResourceQuotaScope(scope)) + if len(helpText) > 0 { + w.Write(LEVEL_0, " * %s\n", helpText) + } + } + } + w.Write(LEVEL_0, "Resource\tUsed\tHard\n") + w.Write(LEVEL_0, "--------\t----\t----\n") + + resources := make([]corev1.ResourceName, 0, len(resourceQuota.Status.Hard)) + for resource := range resourceQuota.Status.Hard { + resources = append(resources, resource) + } + sort.Sort(SortableResourceNames(resources)) + + msg := "%v\t%v\t%v\n" + for i := range resources { + resource := resources[i] + hardQuantity := resourceQuota.Status.Hard[resource] + usedQuantity := resourceQuota.Status.Used[resource] + w.Write(LEVEL_0, msg, resource, usedQuantity.String(), hardQuantity.String()) + } + return nil + }) +} + +// PodDescriber generates information about a pod and the replication controllers that +// create it. +type PodDescriber struct { + clientset.Interface +} + +func (d *PodDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + pod, err := d.Core().Pods(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + if describerSettings.ShowEvents { + eventsInterface := d.Core().Events(namespace) + selector := eventsInterface.GetFieldSelector(&name, &namespace, nil, nil) + options := metav1.ListOptions{FieldSelector: selector.String()} + events, err2 := eventsInterface.List(options) + if describerSettings.ShowEvents && err2 == nil && len(events.Items) > 0 { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Pod '%v': error '%v', but found events.\n", name, err) + DescribeEvents(events, w) + return nil + }) + } + } + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + if ref, err := reference.GetReference(scheme.Scheme, pod); err != nil { + klog.Errorf("Unable to construct reference to '%#v': %v", pod, err) + } else { + ref.Kind = "" + events, _ = d.Core().Events(namespace).Search(scheme.Scheme, ref) + } + } + + return describePod(pod, events) +} + +func describePod(pod *corev1.Pod, events *corev1.EventList) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", pod.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", pod.Namespace) + if pod.Spec.Priority != nil { + w.Write(LEVEL_0, "Priority:\t%d\n", *pod.Spec.Priority) + w.Write(LEVEL_0, "PriorityClassName:\t%s\n", stringOrNone(pod.Spec.PriorityClassName)) + } + if pod.Spec.NodeName == "" { + w.Write(LEVEL_0, "Node:\t\n") + } else { + w.Write(LEVEL_0, "Node:\t%s\n", pod.Spec.NodeName+"/"+pod.Status.HostIP) + } + if pod.Status.StartTime != nil { + w.Write(LEVEL_0, "Start Time:\t%s\n", pod.Status.StartTime.Time.Format(time.RFC1123Z)) + } + printLabelsMultiline(w, "Labels", pod.Labels) + printAnnotationsMultiline(w, "Annotations", pod.Annotations) + if pod.DeletionTimestamp != nil { + w.Write(LEVEL_0, "Status:\tTerminating (lasts %s)\n", translateTimestampUntil(*pod.DeletionTimestamp)) + w.Write(LEVEL_0, "Termination Grace Period:\t%ds\n", *pod.DeletionGracePeriodSeconds) + } else { + w.Write(LEVEL_0, "Status:\t%s\n", string(pod.Status.Phase)) + } + if len(pod.Status.Reason) > 0 { + w.Write(LEVEL_0, "Reason:\t%s\n", pod.Status.Reason) + } + if len(pod.Status.Message) > 0 { + w.Write(LEVEL_0, "Message:\t%s\n", pod.Status.Message) + } + w.Write(LEVEL_0, "IP:\t%s\n", pod.Status.PodIP) + if controlledBy := printController(pod); len(controlledBy) > 0 { + w.Write(LEVEL_0, "Controlled By:\t%s\n", controlledBy) + } + if len(pod.Status.NominatedNodeName) > 0 { + w.Write(LEVEL_0, "NominatedNodeName:\t%s\n", pod.Status.NominatedNodeName) + } + + if len(pod.Spec.InitContainers) > 0 { + describeContainers("Init Containers", pod.Spec.InitContainers, pod.Status.InitContainerStatuses, EnvValueRetriever(pod), w, "") + } + describeContainers("Containers", pod.Spec.Containers, pod.Status.ContainerStatuses, EnvValueRetriever(pod), w, "") + if len(pod.Spec.ReadinessGates) > 0 { + w.Write(LEVEL_0, "Readiness Gates:\n Type\tStatus\n") + for _, g := range pod.Spec.ReadinessGates { + status := "" + for _, c := range pod.Status.Conditions { + if c.Type == g.ConditionType { + status = fmt.Sprintf("%v", c.Status) + break + } + } + w.Write(LEVEL_1, "%v \t%v \n", + g.ConditionType, + status) + } + } + if len(pod.Status.Conditions) > 0 { + w.Write(LEVEL_0, "Conditions:\n Type\tStatus\n") + for _, c := range pod.Status.Conditions { + w.Write(LEVEL_1, "%v \t%v \n", + c.Type, + c.Status) + } + } + describeVolumes(pod.Spec.Volumes, w, "") + if pod.Status.QOSClass != "" { + w.Write(LEVEL_0, "QoS Class:\t%s\n", pod.Status.QOSClass) + } else { + w.Write(LEVEL_0, "QoS Class:\t%s\n", qos.GetPodQOS(pod)) + } + printLabelsMultiline(w, "Node-Selectors", pod.Spec.NodeSelector) + printPodTolerationsMultiline(w, "Tolerations", pod.Spec.Tolerations) + if events != nil { + DescribeEvents(events, w) + } + return nil + }) +} + +func printController(controllee metav1.Object) string { + if controllerRef := metav1.GetControllerOf(controllee); controllerRef != nil { + return fmt.Sprintf("%s/%s", controllerRef.Kind, controllerRef.Name) + } + return "" +} + +func describeVolumes(volumes []corev1.Volume, w PrefixWriter, space string) { + if volumes == nil || len(volumes) == 0 { + w.Write(LEVEL_0, "%sVolumes:\t\n", space) + return + } + w.Write(LEVEL_0, "%sVolumes:\n", space) + for _, volume := range volumes { + nameIndent := "" + if len(space) > 0 { + nameIndent = " " + } + w.Write(LEVEL_1, "%s%v:\n", nameIndent, volume.Name) + switch { + case volume.VolumeSource.HostPath != nil: + printHostPathVolumeSource(volume.VolumeSource.HostPath, w) + case volume.VolumeSource.EmptyDir != nil: + printEmptyDirVolumeSource(volume.VolumeSource.EmptyDir, w) + case volume.VolumeSource.GCEPersistentDisk != nil: + printGCEPersistentDiskVolumeSource(volume.VolumeSource.GCEPersistentDisk, w) + case volume.VolumeSource.AWSElasticBlockStore != nil: + printAWSElasticBlockStoreVolumeSource(volume.VolumeSource.AWSElasticBlockStore, w) + case volume.VolumeSource.GitRepo != nil: + printGitRepoVolumeSource(volume.VolumeSource.GitRepo, w) + case volume.VolumeSource.Secret != nil: + printSecretVolumeSource(volume.VolumeSource.Secret, w) + case volume.VolumeSource.ConfigMap != nil: + printConfigMapVolumeSource(volume.VolumeSource.ConfigMap, w) + case volume.VolumeSource.NFS != nil: + printNFSVolumeSource(volume.VolumeSource.NFS, w) + case volume.VolumeSource.ISCSI != nil: + printISCSIVolumeSource(volume.VolumeSource.ISCSI, w) + case volume.VolumeSource.Glusterfs != nil: + printGlusterfsVolumeSource(volume.VolumeSource.Glusterfs, w) + case volume.VolumeSource.PersistentVolumeClaim != nil: + printPersistentVolumeClaimVolumeSource(volume.VolumeSource.PersistentVolumeClaim, w) + case volume.VolumeSource.RBD != nil: + printRBDVolumeSource(volume.VolumeSource.RBD, w) + case volume.VolumeSource.Quobyte != nil: + printQuobyteVolumeSource(volume.VolumeSource.Quobyte, w) + case volume.VolumeSource.DownwardAPI != nil: + printDownwardAPIVolumeSource(volume.VolumeSource.DownwardAPI, w) + case volume.VolumeSource.AzureDisk != nil: + printAzureDiskVolumeSource(volume.VolumeSource.AzureDisk, w) + case volume.VolumeSource.VsphereVolume != nil: + printVsphereVolumeSource(volume.VolumeSource.VsphereVolume, w) + case volume.VolumeSource.Cinder != nil: + printCinderVolumeSource(volume.VolumeSource.Cinder, w) + case volume.VolumeSource.PhotonPersistentDisk != nil: + printPhotonPersistentDiskVolumeSource(volume.VolumeSource.PhotonPersistentDisk, w) + case volume.VolumeSource.PortworxVolume != nil: + printPortworxVolumeSource(volume.VolumeSource.PortworxVolume, w) + case volume.VolumeSource.ScaleIO != nil: + printScaleIOVolumeSource(volume.VolumeSource.ScaleIO, w) + case volume.VolumeSource.CephFS != nil: + printCephFSVolumeSource(volume.VolumeSource.CephFS, w) + case volume.VolumeSource.StorageOS != nil: + printStorageOSVolumeSource(volume.VolumeSource.StorageOS, w) + case volume.VolumeSource.FC != nil: + printFCVolumeSource(volume.VolumeSource.FC, w) + case volume.VolumeSource.AzureFile != nil: + printAzureFileVolumeSource(volume.VolumeSource.AzureFile, w) + case volume.VolumeSource.FlexVolume != nil: + printFlexVolumeSource(volume.VolumeSource.FlexVolume, w) + case volume.VolumeSource.Flocker != nil: + printFlockerVolumeSource(volume.VolumeSource.Flocker, w) + case volume.VolumeSource.Projected != nil: + printProjectedVolumeSource(volume.VolumeSource.Projected, w) + default: + w.Write(LEVEL_1, "\n") + } + } +} + +func printHostPathVolumeSource(hostPath *corev1.HostPathVolumeSource, w PrefixWriter) { + hostPathType := "" + if hostPath.Type != nil { + hostPathType = string(*hostPath.Type) + } + w.Write(LEVEL_2, "Type:\tHostPath (bare host directory volume)\n"+ + " Path:\t%v\n"+ + " HostPathType:\t%v\n", + hostPath.Path, hostPathType) +} + +func printEmptyDirVolumeSource(emptyDir *corev1.EmptyDirVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tEmptyDir (a temporary directory that shares a pod's lifetime)\n"+ + " Medium:\t%v\n", emptyDir.Medium) +} + +func printGCEPersistentDiskVolumeSource(gce *corev1.GCEPersistentDiskVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tGCEPersistentDisk (a Persistent Disk resource in Google Compute Engine)\n"+ + " PDName:\t%v\n"+ + " FSType:\t%v\n"+ + " Partition:\t%v\n"+ + " ReadOnly:\t%v\n", + gce.PDName, gce.FSType, gce.Partition, gce.ReadOnly) +} + +func printAWSElasticBlockStoreVolumeSource(aws *corev1.AWSElasticBlockStoreVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tAWSElasticBlockStore (a Persistent Disk resource in AWS)\n"+ + " VolumeID:\t%v\n"+ + " FSType:\t%v\n"+ + " Partition:\t%v\n"+ + " ReadOnly:\t%v\n", + aws.VolumeID, aws.FSType, aws.Partition, aws.ReadOnly) +} + +func printGitRepoVolumeSource(git *corev1.GitRepoVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tGitRepo (a volume that is pulled from git when the pod is created)\n"+ + " Repository:\t%v\n"+ + " Revision:\t%v\n", + git.Repository, git.Revision) +} + +func printSecretVolumeSource(secret *corev1.SecretVolumeSource, w PrefixWriter) { + optional := secret.Optional != nil && *secret.Optional + w.Write(LEVEL_2, "Type:\tSecret (a volume populated by a Secret)\n"+ + " SecretName:\t%v\n"+ + " Optional:\t%v\n", + secret.SecretName, optional) +} + +func printConfigMapVolumeSource(configMap *corev1.ConfigMapVolumeSource, w PrefixWriter) { + optional := configMap.Optional != nil && *configMap.Optional + w.Write(LEVEL_2, "Type:\tConfigMap (a volume populated by a ConfigMap)\n"+ + " Name:\t%v\n"+ + " Optional:\t%v\n", + configMap.Name, optional) +} + +func printProjectedVolumeSource(projected *corev1.ProjectedVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tProjected (a volume that contains injected data from multiple sources)\n") + for _, source := range projected.Sources { + if source.Secret != nil { + w.Write(LEVEL_2, "SecretName:\t%v\n"+ + " SecretOptionalName:\t%v\n", + source.Secret.Name, source.Secret.Optional) + } else if source.DownwardAPI != nil { + w.Write(LEVEL_2, "DownwardAPI:\ttrue\n") + } else if source.ConfigMap != nil { + w.Write(LEVEL_2, "ConfigMapName:\t%v\n"+ + " ConfigMapOptional:\t%v\n", + source.ConfigMap.Name, source.ConfigMap.Optional) + } else if source.ServiceAccountToken != nil { + w.Write(LEVEL_2, "TokenExpirationSeconds:\t%v\n", + source.ServiceAccountToken.ExpirationSeconds) + } + } +} + +func printNFSVolumeSource(nfs *corev1.NFSVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tNFS (an NFS mount that lasts the lifetime of a pod)\n"+ + " Server:\t%v\n"+ + " Path:\t%v\n"+ + " ReadOnly:\t%v\n", + nfs.Server, nfs.Path, nfs.ReadOnly) +} + +func printQuobyteVolumeSource(quobyte *corev1.QuobyteVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tQuobyte (a Quobyte mount on the host that shares a pod's lifetime)\n"+ + " Registry:\t%v\n"+ + " Volume:\t%v\n"+ + " ReadOnly:\t%v\n", + quobyte.Registry, quobyte.Volume, quobyte.ReadOnly) +} + +func printPortworxVolumeSource(pwxVolume *corev1.PortworxVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tPortworxVolume (a Portworx Volume resource)\n"+ + " VolumeID:\t%v\n", + pwxVolume.VolumeID) +} + +func printISCSIVolumeSource(iscsi *corev1.ISCSIVolumeSource, w PrefixWriter) { + initiator := "" + if iscsi.InitiatorName != nil { + initiator = *iscsi.InitiatorName + } + w.Write(LEVEL_2, "Type:\tISCSI (an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod)\n"+ + " TargetPortal:\t%v\n"+ + " IQN:\t%v\n"+ + " Lun:\t%v\n"+ + " ISCSIInterface\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n"+ + " Portals:\t%v\n"+ + " DiscoveryCHAPAuth:\t%v\n"+ + " SessionCHAPAuth:\t%v\n"+ + " SecretRef:\t%v\n"+ + " InitiatorName:\t%v\n", + iscsi.TargetPortal, iscsi.IQN, iscsi.Lun, iscsi.ISCSIInterface, iscsi.FSType, iscsi.ReadOnly, iscsi.Portals, iscsi.DiscoveryCHAPAuth, iscsi.SessionCHAPAuth, iscsi.SecretRef, initiator) +} + +func printISCSIPersistentVolumeSource(iscsi *corev1.ISCSIPersistentVolumeSource, w PrefixWriter) { + initiatorName := "" + if iscsi.InitiatorName != nil { + initiatorName = *iscsi.InitiatorName + } + w.Write(LEVEL_2, "Type:\tISCSI (an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod)\n"+ + " TargetPortal:\t%v\n"+ + " IQN:\t%v\n"+ + " Lun:\t%v\n"+ + " ISCSIInterface\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n"+ + " Portals:\t%v\n"+ + " DiscoveryCHAPAuth:\t%v\n"+ + " SessionCHAPAuth:\t%v\n"+ + " SecretRef:\t%v\n"+ + " InitiatorName:\t%v\n", + iscsi.TargetPortal, iscsi.IQN, iscsi.Lun, iscsi.ISCSIInterface, iscsi.FSType, iscsi.ReadOnly, iscsi.Portals, iscsi.DiscoveryCHAPAuth, iscsi.SessionCHAPAuth, iscsi.SecretRef, initiatorName) +} + +func printGlusterfsVolumeSource(glusterfs *corev1.GlusterfsVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tGlusterfs (a Glusterfs mount on the host that shares a pod's lifetime)\n"+ + " EndpointsName:\t%v\n"+ + " Path:\t%v\n"+ + " ReadOnly:\t%v\n", + glusterfs.EndpointsName, glusterfs.Path, glusterfs.ReadOnly) +} + +func printGlusterfsPersistentVolumeSource(glusterfs *corev1.GlusterfsPersistentVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tGlusterfs (a Glusterfs mount on the host that shares a pod's lifetime)\n"+ + " EndpointsName:\t%v\n"+ + " EndpointsNamespace:\t%v\n"+ + " Path:\t%v\n"+ + " ReadOnly:\t%v\n", + glusterfs.EndpointsName, glusterfs.EndpointsNamespace, glusterfs.Path, glusterfs.ReadOnly) +} + +func printPersistentVolumeClaimVolumeSource(claim *corev1.PersistentVolumeClaimVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tPersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)\n"+ + " ClaimName:\t%v\n"+ + " ReadOnly:\t%v\n", + claim.ClaimName, claim.ReadOnly) +} + +func printRBDVolumeSource(rbd *corev1.RBDVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tRBD (a Rados Block Device mount on the host that shares a pod's lifetime)\n"+ + " CephMonitors:\t%v\n"+ + " RBDImage:\t%v\n"+ + " FSType:\t%v\n"+ + " RBDPool:\t%v\n"+ + " RadosUser:\t%v\n"+ + " Keyring:\t%v\n"+ + " SecretRef:\t%v\n"+ + " ReadOnly:\t%v\n", + rbd.CephMonitors, rbd.RBDImage, rbd.FSType, rbd.RBDPool, rbd.RadosUser, rbd.Keyring, rbd.SecretRef, rbd.ReadOnly) +} + +func printRBDPersistentVolumeSource(rbd *corev1.RBDPersistentVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tRBD (a Rados Block Device mount on the host that shares a pod's lifetime)\n"+ + " CephMonitors:\t%v\n"+ + " RBDImage:\t%v\n"+ + " FSType:\t%v\n"+ + " RBDPool:\t%v\n"+ + " RadosUser:\t%v\n"+ + " Keyring:\t%v\n"+ + " SecretRef:\t%v\n"+ + " ReadOnly:\t%v\n", + rbd.CephMonitors, rbd.RBDImage, rbd.FSType, rbd.RBDPool, rbd.RadosUser, rbd.Keyring, rbd.SecretRef, rbd.ReadOnly) +} + +func printDownwardAPIVolumeSource(d *corev1.DownwardAPIVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tDownwardAPI (a volume populated by information about the pod)\n Items:\n") + for _, mapping := range d.Items { + if mapping.FieldRef != nil { + w.Write(LEVEL_3, "%v -> %v\n", mapping.FieldRef.FieldPath, mapping.Path) + } + if mapping.ResourceFieldRef != nil { + w.Write(LEVEL_3, "%v -> %v\n", mapping.ResourceFieldRef.Resource, mapping.Path) + } + } +} + +func printAzureDiskVolumeSource(d *corev1.AzureDiskVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tAzureDisk (an Azure Data Disk mount on the host and bind mount to the pod)\n"+ + " DiskName:\t%v\n"+ + " DiskURI:\t%v\n"+ + " Kind: \t%v\n"+ + " FSType:\t%v\n"+ + " CachingMode:\t%v\n"+ + " ReadOnly:\t%v\n", + d.DiskName, d.DataDiskURI, *d.Kind, *d.FSType, *d.CachingMode, *d.ReadOnly) +} + +func printVsphereVolumeSource(vsphere *corev1.VsphereVirtualDiskVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tvSphereVolume (a Persistent Disk resource in vSphere)\n"+ + " VolumePath:\t%v\n"+ + " FSType:\t%v\n"+ + " StoragePolicyName:\t%v\n", + vsphere.VolumePath, vsphere.FSType, vsphere.StoragePolicyName) +} + +func printPhotonPersistentDiskVolumeSource(photon *corev1.PhotonPersistentDiskVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tPhotonPersistentDisk (a Persistent Disk resource in photon platform)\n"+ + " PdID:\t%v\n"+ + " FSType:\t%v\n", + photon.PdID, photon.FSType) +} + +func printCinderVolumeSource(cinder *corev1.CinderVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tCinder (a Persistent Disk resource in OpenStack)\n"+ + " VolumeID:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + " SecretRef:\t%v\n"+ + cinder.VolumeID, cinder.FSType, cinder.ReadOnly, cinder.SecretRef) +} + +func printCinderPersistentVolumeSource(cinder *corev1.CinderPersistentVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tCinder (a Persistent Disk resource in OpenStack)\n"+ + " VolumeID:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + " SecretRef:\t%v\n"+ + cinder.VolumeID, cinder.SecretRef, cinder.FSType, cinder.ReadOnly, cinder.SecretRef) +} + +func printScaleIOVolumeSource(sio *corev1.ScaleIOVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tScaleIO (a persistent volume backed by a block device in ScaleIO)\n"+ + " Gateway:\t%v\n"+ + " System:\t%v\n"+ + " Protection Domain:\t%v\n"+ + " Storage Pool:\t%v\n"+ + " Storage Mode:\t%v\n"+ + " VolumeName:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + sio.Gateway, sio.System, sio.ProtectionDomain, sio.StoragePool, sio.StorageMode, sio.VolumeName, sio.FSType, sio.ReadOnly) +} + +func printScaleIOPersistentVolumeSource(sio *corev1.ScaleIOPersistentVolumeSource, w PrefixWriter) { + var secretNS, secretName string + if sio.SecretRef != nil { + secretName = sio.SecretRef.Name + secretNS = sio.SecretRef.Namespace + } + w.Write(LEVEL_2, "Type:\tScaleIO (a persistent volume backed by a block device in ScaleIO)\n"+ + " Gateway:\t%v\n"+ + " System:\t%v\n"+ + " Protection Domain:\t%v\n"+ + " Storage Pool:\t%v\n"+ + " Storage Mode:\t%v\n"+ + " VolumeName:\t%v\n"+ + " SecretName:\t%v\n"+ + " SecretNamespace:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + sio.Gateway, sio.System, sio.ProtectionDomain, sio.StoragePool, sio.StorageMode, sio.VolumeName, secretName, secretNS, sio.FSType, sio.ReadOnly) +} + +func printLocalVolumeSource(ls *corev1.LocalVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tLocalVolume (a persistent volume backed by local storage on a node)\n"+ + " Path:\t%v\n", + ls.Path) +} + +func printCephFSVolumeSource(cephfs *corev1.CephFSVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tCephFS (a CephFS mount on the host that shares a pod's lifetime)\n"+ + " Monitors:\t%v\n"+ + " Path:\t%v\n"+ + " User:\t%v\n"+ + " SecretFile:\t%v\n"+ + " SecretRef:\t%v\n"+ + " ReadOnly:\t%v\n", + cephfs.Monitors, cephfs.Path, cephfs.User, cephfs.SecretFile, cephfs.SecretRef, cephfs.ReadOnly) +} + +func printCephFSPersistentVolumeSource(cephfs *corev1.CephFSPersistentVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tCephFS (a CephFS mount on the host that shares a pod's lifetime)\n"+ + " Monitors:\t%v\n"+ + " Path:\t%v\n"+ + " User:\t%v\n"+ + " SecretFile:\t%v\n"+ + " SecretRef:\t%v\n"+ + " ReadOnly:\t%v\n", + cephfs.Monitors, cephfs.Path, cephfs.User, cephfs.SecretFile, cephfs.SecretRef, cephfs.ReadOnly) +} + +func printStorageOSVolumeSource(storageos *corev1.StorageOSVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tStorageOS (a StorageOS Persistent Disk resource)\n"+ + " VolumeName:\t%v\n"+ + " VolumeNamespace:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + storageos.VolumeName, storageos.VolumeNamespace, storageos.FSType, storageos.ReadOnly) +} + +func printStorageOSPersistentVolumeSource(storageos *corev1.StorageOSPersistentVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tStorageOS (a StorageOS Persistent Disk resource)\n"+ + " VolumeName:\t%v\n"+ + " VolumeNamespace:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + storageos.VolumeName, storageos.VolumeNamespace, storageos.FSType, storageos.ReadOnly) +} + +func printFCVolumeSource(fc *corev1.FCVolumeSource, w PrefixWriter) { + lun := "" + if fc.Lun != nil { + lun = strconv.Itoa(int(*fc.Lun)) + } + w.Write(LEVEL_2, "Type:\tFC (a Fibre Channel disk)\n"+ + " TargetWWNs:\t%v\n"+ + " LUN:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + strings.Join(fc.TargetWWNs, ", "), lun, fc.FSType, fc.ReadOnly) +} + +func printAzureFileVolumeSource(azureFile *corev1.AzureFileVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tAzureFile (an Azure File Service mount on the host and bind mount to the pod)\n"+ + " SecretName:\t%v\n"+ + " ShareName:\t%v\n"+ + " ReadOnly:\t%v\n", + azureFile.SecretName, azureFile.ShareName, azureFile.ReadOnly) +} + +func printAzureFilePersistentVolumeSource(azureFile *corev1.AzureFilePersistentVolumeSource, w PrefixWriter) { + ns := "" + if azureFile.SecretNamespace != nil { + ns = *azureFile.SecretNamespace + } + w.Write(LEVEL_2, "Type:\tAzureFile (an Azure File Service mount on the host and bind mount to the pod)\n"+ + " SecretName:\t%v\n"+ + " SecretNamespace:\t%v\n"+ + " ShareName:\t%v\n"+ + " ReadOnly:\t%v\n", + azureFile.SecretName, ns, azureFile.ShareName, azureFile.ReadOnly) +} + +func printFlexPersistentVolumeSource(flex *corev1.FlexPersistentVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tFlexVolume (a generic volume resource that is provisioned/attached using an exec based plugin)\n"+ + " Driver:\t%v\n"+ + " FSType:\t%v\n"+ + " SecretRef:\t%v\n"+ + " ReadOnly:\t%v\n"+ + " Options:\t%v\n", + flex.Driver, flex.FSType, flex.SecretRef, flex.ReadOnly, flex.Options) +} + +func printFlexVolumeSource(flex *corev1.FlexVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tFlexVolume (a generic volume resource that is provisioned/attached using an exec based plugin)\n"+ + " Driver:\t%v\n"+ + " FSType:\t%v\n"+ + " SecretRef:\t%v\n"+ + " ReadOnly:\t%v\n"+ + " Options:\t%v\n", + flex.Driver, flex.FSType, flex.SecretRef, flex.ReadOnly, flex.Options) +} + +func printFlockerVolumeSource(flocker *corev1.FlockerVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tFlocker (a Flocker volume mounted by the Flocker agent)\n"+ + " DatasetName:\t%v\n"+ + " DatasetUUID:\t%v\n", + flocker.DatasetName, flocker.DatasetUUID) +} + +func printCSIPersistentVolumeSource(csi *corev1.CSIPersistentVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tCSI (a Container Storage Interface (CSI) volume source)\n"+ + " Driver:\t%v\n"+ + " VolumeHandle:\t%v\n"+ + " ReadOnly:\t%v\n", + csi.Driver, csi.VolumeHandle, csi.ReadOnly) + printCSIPersistentVolumeAttributesMultiline(w, "VolumeAttributes", csi.VolumeAttributes) +} + +func printCSIPersistentVolumeAttributesMultiline(w PrefixWriter, title string, annotations map[string]string) { + printCSIPersistentVolumeAttributesMultilineIndent(w, "", title, "\t", annotations, sets.NewString()) +} + +func printCSIPersistentVolumeAttributesMultilineIndent(w PrefixWriter, initialIndent, title, innerIndent string, attributes map[string]string, skip sets.String) { + w.Write(LEVEL_2, "%s%s:%s", initialIndent, title, innerIndent) + + if len(attributes) == 0 { + w.WriteLine("") + return + } + + // to print labels in the sorted order + keys := make([]string, 0, len(attributes)) + for key := range attributes { + if skip.Has(key) { + continue + } + keys = append(keys, key) + } + if len(attributes) == 0 { + w.WriteLine("") + return + } + sort.Strings(keys) + + for i, key := range keys { + if i != 0 { + w.Write(LEVEL_2, initialIndent) + w.Write(LEVEL_2, innerIndent) + } + line := fmt.Sprintf("%s=%s", key, attributes[key]) + if len(line) > maxAnnotationLen { + w.Write(LEVEL_2, "%s...\n", line[:maxAnnotationLen]) + } else { + w.Write(LEVEL_2, "%s\n", line) + } + i++ + } +} + +type PersistentVolumeDescriber struct { + clientset.Interface +} + +func (d *PersistentVolumeDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + c := d.Core().PersistentVolumes() + + pv, err := c.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = d.Core().Events(namespace).Search(scheme.Scheme, pv) + } + + return describePersistentVolume(pv, events) +} + +func printVolumeNodeAffinity(w PrefixWriter, affinity *corev1.VolumeNodeAffinity) { + w.Write(LEVEL_0, "Node Affinity:\t") + if affinity == nil || affinity.Required == nil { + w.WriteLine("") + return + } + w.WriteLine("") + + if affinity.Required != nil { + w.Write(LEVEL_1, "Required Terms:\t") + if len(affinity.Required.NodeSelectorTerms) == 0 { + w.WriteLine("") + } else { + w.WriteLine("") + for i, term := range affinity.Required.NodeSelectorTerms { + printNodeSelectorTermsMultilineWithIndent(w, LEVEL_2, fmt.Sprintf("Term %v", i), "\t", term.MatchExpressions) + } + } + } +} + +// printLabelsMultiline prints multiple labels with a user-defined alignment. +func printNodeSelectorTermsMultilineWithIndent(w PrefixWriter, indentLevel int, title, innerIndent string, reqs []corev1.NodeSelectorRequirement) { + w.Write(indentLevel, "%s:%s", title, innerIndent) + + if len(reqs) == 0 { + w.WriteLine("") + return + } + + for i, req := range reqs { + if i != 0 { + w.Write(indentLevel, "%s", innerIndent) + } + exprStr := fmt.Sprintf("%s %s", req.Key, strings.ToLower(string(req.Operator))) + if len(req.Values) > 0 { + exprStr = fmt.Sprintf("%s [%s]", exprStr, strings.Join(req.Values, ", ")) + } + w.Write(LEVEL_0, "%s\n", exprStr) + } +} + +func describePersistentVolume(pv *corev1.PersistentVolume, events *corev1.EventList) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", pv.Name) + printLabelsMultiline(w, "Labels", pv.ObjectMeta.Labels) + printAnnotationsMultiline(w, "Annotations", pv.ObjectMeta.Annotations) + w.Write(LEVEL_0, "Finalizers:\t%v\n", pv.ObjectMeta.Finalizers) + w.Write(LEVEL_0, "StorageClass:\t%s\n", storageutil.GetPersistentVolumeClass(pv)) + if pv.ObjectMeta.DeletionTimestamp != nil { + w.Write(LEVEL_0, "Status:\tTerminating (lasts %s)\n", translateTimestampUntil(*pv.ObjectMeta.DeletionTimestamp)) + } else { + w.Write(LEVEL_0, "Status:\t%v\n", pv.Status.Phase) + } + if pv.Spec.ClaimRef != nil { + w.Write(LEVEL_0, "Claim:\t%s\n", pv.Spec.ClaimRef.Namespace+"/"+pv.Spec.ClaimRef.Name) + } else { + w.Write(LEVEL_0, "Claim:\t%s\n", "") + } + w.Write(LEVEL_0, "Reclaim Policy:\t%v\n", pv.Spec.PersistentVolumeReclaimPolicy) + w.Write(LEVEL_0, "Access Modes:\t%s\n", storageutil.GetAccessModesAsString(pv.Spec.AccessModes)) + if pv.Spec.VolumeMode != nil { + w.Write(LEVEL_0, "VolumeMode:\t%v\n", *pv.Spec.VolumeMode) + } + storage := pv.Spec.Capacity[corev1.ResourceStorage] + w.Write(LEVEL_0, "Capacity:\t%s\n", storage.String()) + printVolumeNodeAffinity(w, pv.Spec.NodeAffinity) + w.Write(LEVEL_0, "Message:\t%s\n", pv.Status.Message) + w.Write(LEVEL_0, "Source:\n") + + switch { + case pv.Spec.HostPath != nil: + printHostPathVolumeSource(pv.Spec.HostPath, w) + case pv.Spec.GCEPersistentDisk != nil: + printGCEPersistentDiskVolumeSource(pv.Spec.GCEPersistentDisk, w) + case pv.Spec.AWSElasticBlockStore != nil: + printAWSElasticBlockStoreVolumeSource(pv.Spec.AWSElasticBlockStore, w) + case pv.Spec.NFS != nil: + printNFSVolumeSource(pv.Spec.NFS, w) + case pv.Spec.ISCSI != nil: + printISCSIPersistentVolumeSource(pv.Spec.ISCSI, w) + case pv.Spec.Glusterfs != nil: + printGlusterfsPersistentVolumeSource(pv.Spec.Glusterfs, w) + case pv.Spec.RBD != nil: + printRBDPersistentVolumeSource(pv.Spec.RBD, w) + case pv.Spec.Quobyte != nil: + printQuobyteVolumeSource(pv.Spec.Quobyte, w) + case pv.Spec.VsphereVolume != nil: + printVsphereVolumeSource(pv.Spec.VsphereVolume, w) + case pv.Spec.Cinder != nil: + printCinderPersistentVolumeSource(pv.Spec.Cinder, w) + case pv.Spec.AzureDisk != nil: + printAzureDiskVolumeSource(pv.Spec.AzureDisk, w) + case pv.Spec.PhotonPersistentDisk != nil: + printPhotonPersistentDiskVolumeSource(pv.Spec.PhotonPersistentDisk, w) + case pv.Spec.PortworxVolume != nil: + printPortworxVolumeSource(pv.Spec.PortworxVolume, w) + case pv.Spec.ScaleIO != nil: + printScaleIOPersistentVolumeSource(pv.Spec.ScaleIO, w) + case pv.Spec.Local != nil: + printLocalVolumeSource(pv.Spec.Local, w) + case pv.Spec.CephFS != nil: + printCephFSPersistentVolumeSource(pv.Spec.CephFS, w) + case pv.Spec.StorageOS != nil: + printStorageOSPersistentVolumeSource(pv.Spec.StorageOS, w) + case pv.Spec.FC != nil: + printFCVolumeSource(pv.Spec.FC, w) + case pv.Spec.AzureFile != nil: + printAzureFilePersistentVolumeSource(pv.Spec.AzureFile, w) + case pv.Spec.FlexVolume != nil: + printFlexPersistentVolumeSource(pv.Spec.FlexVolume, w) + case pv.Spec.Flocker != nil: + printFlockerVolumeSource(pv.Spec.Flocker, w) + case pv.Spec.CSI != nil: + printCSIPersistentVolumeSource(pv.Spec.CSI, w) + default: + w.Write(LEVEL_1, "\n") + } + + if events != nil { + DescribeEvents(events, w) + } + + return nil + }) +} + +type PersistentVolumeClaimDescriber struct { + clientset.Interface +} + +func (d *PersistentVolumeClaimDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + c := d.Core().PersistentVolumeClaims(namespace) + + pvc, err := c.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + pc := d.Core().Pods(namespace) + + mountPods, err := getMountPods(pc, pvc.Name) + if err != nil { + return "", err + } + + events, _ := d.Core().Events(namespace).Search(scheme.Scheme, pvc) + + return describePersistentVolumeClaim(pvc, events, mountPods) +} + +func getMountPods(c corev1client.PodInterface, pvcName string) ([]corev1.Pod, error) { + nsPods, err := c.List(metav1.ListOptions{}) + if err != nil { + return []corev1.Pod{}, err + } + + var pods []corev1.Pod + + for _, pod := range nsPods.Items { + pvcs := getPvcs(pod.Spec.Volumes) + + for _, pvc := range pvcs { + if pvc.PersistentVolumeClaim.ClaimName == pvcName { + pods = append(pods, pod) + } + } + } + + return pods, nil +} + +func getPvcs(volumes []corev1.Volume) []corev1.Volume { + var pvcs []corev1.Volume + + for _, volume := range volumes { + if volume.VolumeSource.PersistentVolumeClaim != nil { + pvcs = append(pvcs, volume) + } + } + + return pvcs +} + +func describePersistentVolumeClaim(pvc *corev1.PersistentVolumeClaim, events *corev1.EventList, mountPods []corev1.Pod) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", pvc.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", pvc.Namespace) + w.Write(LEVEL_0, "StorageClass:\t%s\n", storageutil.GetPersistentVolumeClaimClass(pvc)) + if pvc.ObjectMeta.DeletionTimestamp != nil { + w.Write(LEVEL_0, "Status:\tTerminating (lasts %s)\n", translateTimestampUntil(*pvc.ObjectMeta.DeletionTimestamp)) + } else { + w.Write(LEVEL_0, "Status:\t%v\n", pvc.Status.Phase) + } + w.Write(LEVEL_0, "Volume:\t%s\n", pvc.Spec.VolumeName) + printLabelsMultiline(w, "Labels", pvc.Labels) + printAnnotationsMultiline(w, "Annotations", pvc.Annotations) + w.Write(LEVEL_0, "Finalizers:\t%v\n", pvc.ObjectMeta.Finalizers) + storage := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + capacity := "" + accessModes := "" + if pvc.Spec.VolumeName != "" { + accessModes = storageutil.GetAccessModesAsString(pvc.Status.AccessModes) + storage = pvc.Status.Capacity[corev1.ResourceStorage] + capacity = storage.String() + } + w.Write(LEVEL_0, "Capacity:\t%s\n", capacity) + w.Write(LEVEL_0, "Access Modes:\t%s\n", accessModes) + if pvc.Spec.VolumeMode != nil { + w.Write(LEVEL_0, "VolumeMode:\t%v\n", *pvc.Spec.VolumeMode) + } + if len(pvc.Status.Conditions) > 0 { + w.Write(LEVEL_0, "Conditions:\n") + w.Write(LEVEL_1, "Type\tStatus\tLastProbeTime\tLastTransitionTime\tReason\tMessage\n") + w.Write(LEVEL_1, "----\t------\t-----------------\t------------------\t------\t-------\n") + for _, c := range pvc.Status.Conditions { + w.Write(LEVEL_1, "%v \t%v \t%s \t%s \t%v \t%v\n", + c.Type, + c.Status, + c.LastProbeTime.Time.Format(time.RFC1123Z), + c.LastTransitionTime.Time.Format(time.RFC1123Z), + c.Reason, + c.Message) + } + } + if events != nil { + DescribeEvents(events, w) + } + + printPodsMultiline(w, "Mounted By", mountPods) + + return nil + }) +} + +func describeContainers(label string, containers []corev1.Container, containerStatuses []corev1.ContainerStatus, + resolverFn EnvVarResolverFunc, w PrefixWriter, space string) { + statuses := map[string]corev1.ContainerStatus{} + for _, status := range containerStatuses { + statuses[status.Name] = status + } + + describeContainersLabel(containers, label, space, w) + + for _, container := range containers { + status, ok := statuses[container.Name] + describeContainerBasicInfo(container, status, ok, space, w) + describeContainerCommand(container, w) + if ok { + describeContainerState(status, w) + } + describeContainerResource(container, w) + describeContainerProbe(container, w) + if len(container.EnvFrom) > 0 { + describeContainerEnvFrom(container, resolverFn, w) + } + describeContainerEnvVars(container, resolverFn, w) + describeContainerVolumes(container, w) + } +} + +func describeContainersLabel(containers []corev1.Container, label, space string, w PrefixWriter) { + none := "" + if len(containers) == 0 { + none = " " + } + w.Write(LEVEL_0, "%s%s:%s\n", space, label, none) +} + +func describeContainerBasicInfo(container corev1.Container, status corev1.ContainerStatus, ok bool, space string, w PrefixWriter) { + nameIndent := "" + if len(space) > 0 { + nameIndent = " " + } + w.Write(LEVEL_1, "%s%v:\n", nameIndent, container.Name) + if ok { + w.Write(LEVEL_2, "Container ID:\t%s\n", status.ContainerID) + } + w.Write(LEVEL_2, "Image:\t%s\n", container.Image) + if ok { + w.Write(LEVEL_2, "Image ID:\t%s\n", status.ImageID) + } + portString := describeContainerPorts(container.Ports) + if strings.Contains(portString, ",") { + w.Write(LEVEL_2, "Ports:\t%s\n", portString) + } else { + w.Write(LEVEL_2, "Port:\t%s\n", stringOrNone(portString)) + } + hostPortString := describeContainerHostPorts(container.Ports) + if strings.Contains(hostPortString, ",") { + w.Write(LEVEL_2, "Host Ports:\t%s\n", hostPortString) + } else { + w.Write(LEVEL_2, "Host Port:\t%s\n", stringOrNone(hostPortString)) + } +} + +func describeContainerPorts(cPorts []corev1.ContainerPort) string { + ports := make([]string, 0, len(cPorts)) + for _, cPort := range cPorts { + ports = append(ports, fmt.Sprintf("%d/%s", cPort.ContainerPort, cPort.Protocol)) + } + return strings.Join(ports, ", ") +} + +func describeContainerHostPorts(cPorts []corev1.ContainerPort) string { + ports := make([]string, 0, len(cPorts)) + for _, cPort := range cPorts { + ports = append(ports, fmt.Sprintf("%d/%s", cPort.HostPort, cPort.Protocol)) + } + return strings.Join(ports, ", ") +} + +func describeContainerCommand(container corev1.Container, w PrefixWriter) { + if len(container.Command) > 0 { + w.Write(LEVEL_2, "Command:\n") + for _, c := range container.Command { + for _, s := range strings.Split(c, "\n") { + w.Write(LEVEL_3, "%s\n", s) + } + } + } + if len(container.Args) > 0 { + w.Write(LEVEL_2, "Args:\n") + for _, arg := range container.Args { + for _, s := range strings.Split(arg, "\n") { + w.Write(LEVEL_3, "%s\n", s) + } + } + } +} + +func describeContainerResource(container corev1.Container, w PrefixWriter) { + resources := container.Resources + if len(resources.Limits) > 0 { + w.Write(LEVEL_2, "Limits:\n") + } + for _, name := range SortedResourceNames(resources.Limits) { + quantity := resources.Limits[name] + w.Write(LEVEL_3, "%s:\t%s\n", name, quantity.String()) + } + + if len(resources.Requests) > 0 { + w.Write(LEVEL_2, "Requests:\n") + } + for _, name := range SortedResourceNames(resources.Requests) { + quantity := resources.Requests[name] + w.Write(LEVEL_3, "%s:\t%s\n", name, quantity.String()) + } +} + +func describeContainerState(status corev1.ContainerStatus, w PrefixWriter) { + describeStatus("State", status.State, w) + if status.LastTerminationState.Terminated != nil { + describeStatus("Last State", status.LastTerminationState, w) + } + w.Write(LEVEL_2, "Ready:\t%v\n", printBool(status.Ready)) + w.Write(LEVEL_2, "Restart Count:\t%d\n", status.RestartCount) +} + +func describeContainerProbe(container corev1.Container, w PrefixWriter) { + if container.LivenessProbe != nil { + probe := DescribeProbe(container.LivenessProbe) + w.Write(LEVEL_2, "Liveness:\t%s\n", probe) + } + if container.ReadinessProbe != nil { + probe := DescribeProbe(container.ReadinessProbe) + w.Write(LEVEL_2, "Readiness:\t%s\n", probe) + } +} + +func describeContainerVolumes(container corev1.Container, w PrefixWriter) { + // Show volumeMounts + none := "" + if len(container.VolumeMounts) == 0 { + none = "\t" + } + w.Write(LEVEL_2, "Mounts:%s\n", none) + sort.Sort(SortableVolumeMounts(container.VolumeMounts)) + for _, mount := range container.VolumeMounts { + flags := []string{} + switch { + case mount.ReadOnly: + flags = append(flags, "ro") + case !mount.ReadOnly: + flags = append(flags, "rw") + case len(mount.SubPath) > 0: + flags = append(flags, fmt.Sprintf("path=%q", mount.SubPath)) + } + w.Write(LEVEL_3, "%s from %s (%s)\n", mount.MountPath, mount.Name, strings.Join(flags, ",")) + } + // Show volumeDevices if exists + if len(container.VolumeDevices) > 0 { + w.Write(LEVEL_2, "Devices:%s\n", none) + sort.Sort(SortableVolumeDevices(container.VolumeDevices)) + for _, device := range container.VolumeDevices { + w.Write(LEVEL_3, "%s from %s\n", device.DevicePath, device.Name) + } + } +} + +func describeContainerEnvVars(container corev1.Container, resolverFn EnvVarResolverFunc, w PrefixWriter) { + none := "" + if len(container.Env) == 0 { + none = "\t" + } + w.Write(LEVEL_2, "Environment:%s\n", none) + + for _, e := range container.Env { + if e.ValueFrom == nil { + for i, s := range strings.Split(e.Value, "\n") { + if i == 0 { + w.Write(LEVEL_3, "%s:\t%s\n", e.Name, s) + } else { + w.Write(LEVEL_3, "\t%s\n", s) + } + } + continue + } + + switch { + case e.ValueFrom.FieldRef != nil: + var valueFrom string + if resolverFn != nil { + valueFrom = resolverFn(e) + } + w.Write(LEVEL_3, "%s:\t%s (%s:%s)\n", e.Name, valueFrom, e.ValueFrom.FieldRef.APIVersion, e.ValueFrom.FieldRef.FieldPath) + case e.ValueFrom.ResourceFieldRef != nil: + valueFrom, err := resourcehelper.ExtractContainerResourceValue(e.ValueFrom.ResourceFieldRef, &container) + if err != nil { + valueFrom = "" + } + resource := e.ValueFrom.ResourceFieldRef.Resource + if valueFrom == "0" && (resource == "limits.cpu" || resource == "limits.memory") { + valueFrom = "node allocatable" + } + w.Write(LEVEL_3, "%s:\t%s (%s)\n", e.Name, valueFrom, resource) + case e.ValueFrom.SecretKeyRef != nil: + optional := e.ValueFrom.SecretKeyRef.Optional != nil && *e.ValueFrom.SecretKeyRef.Optional + w.Write(LEVEL_3, "%s:\t\tOptional: %t\n", e.Name, e.ValueFrom.SecretKeyRef.Key, e.ValueFrom.SecretKeyRef.Name, optional) + case e.ValueFrom.ConfigMapKeyRef != nil: + optional := e.ValueFrom.ConfigMapKeyRef.Optional != nil && *e.ValueFrom.ConfigMapKeyRef.Optional + w.Write(LEVEL_3, "%s:\t\tOptional: %t\n", e.Name, e.ValueFrom.ConfigMapKeyRef.Key, e.ValueFrom.ConfigMapKeyRef.Name, optional) + } + } +} + +func describeContainerEnvFrom(container corev1.Container, resolverFn EnvVarResolverFunc, w PrefixWriter) { + none := "" + if len(container.EnvFrom) == 0 { + none = "\t" + } + w.Write(LEVEL_2, "Environment Variables from:%s\n", none) + + for _, e := range container.EnvFrom { + from := "" + name := "" + optional := false + if e.ConfigMapRef != nil { + from = "ConfigMap" + name = e.ConfigMapRef.Name + optional = e.ConfigMapRef.Optional != nil && *e.ConfigMapRef.Optional + } else if e.SecretRef != nil { + from = "Secret" + name = e.SecretRef.Name + optional = e.SecretRef.Optional != nil && *e.SecretRef.Optional + } + if len(e.Prefix) == 0 { + w.Write(LEVEL_3, "%s\t%s\tOptional: %t\n", name, from, optional) + } else { + w.Write(LEVEL_3, "%s\t%s with prefix '%s'\tOptional: %t\n", name, from, e.Prefix, optional) + } + } +} + +// DescribeProbe is exported for consumers in other API groups that have probes +func DescribeProbe(probe *corev1.Probe) string { + attrs := fmt.Sprintf("delay=%ds timeout=%ds period=%ds #success=%d #failure=%d", probe.InitialDelaySeconds, probe.TimeoutSeconds, probe.PeriodSeconds, probe.SuccessThreshold, probe.FailureThreshold) + switch { + case probe.Exec != nil: + return fmt.Sprintf("exec %v %s", probe.Exec.Command, attrs) + case probe.HTTPGet != nil: + url := &url.URL{} + url.Scheme = strings.ToLower(string(probe.HTTPGet.Scheme)) + if len(probe.HTTPGet.Port.String()) > 0 { + url.Host = net.JoinHostPort(probe.HTTPGet.Host, probe.HTTPGet.Port.String()) + } else { + url.Host = probe.HTTPGet.Host + } + url.Path = probe.HTTPGet.Path + return fmt.Sprintf("http-get %s %s", url.String(), attrs) + case probe.TCPSocket != nil: + return fmt.Sprintf("tcp-socket %s:%s %s", probe.TCPSocket.Host, probe.TCPSocket.Port.String(), attrs) + } + return fmt.Sprintf("unknown %s", attrs) +} + +type EnvVarResolverFunc func(e corev1.EnvVar) string + +// EnvValueFrom is exported for use by describers in other packages +func EnvValueRetriever(pod *corev1.Pod) EnvVarResolverFunc { + return func(e corev1.EnvVar) string { + gv, err := schema.ParseGroupVersion(e.ValueFrom.FieldRef.APIVersion) + if err != nil { + return "" + } + gvk := gv.WithKind("Pod") + internalFieldPath, _, err := scheme.Scheme.ConvertFieldLabel(gvk, e.ValueFrom.FieldRef.FieldPath, "") + if err != nil { + return "" // pod validation should catch this on create + } + + valueFrom, err := fieldpath.ExtractFieldPathAsString(pod, internalFieldPath) + if err != nil { + return "" // pod validation should catch this on create + } + + return valueFrom + } +} + +func describeStatus(stateName string, state corev1.ContainerState, w PrefixWriter) { + switch { + case state.Running != nil: + w.Write(LEVEL_2, "%s:\tRunning\n", stateName) + w.Write(LEVEL_3, "Started:\t%v\n", state.Running.StartedAt.Time.Format(time.RFC1123Z)) + case state.Waiting != nil: + w.Write(LEVEL_2, "%s:\tWaiting\n", stateName) + if state.Waiting.Reason != "" { + w.Write(LEVEL_3, "Reason:\t%s\n", state.Waiting.Reason) + } + case state.Terminated != nil: + w.Write(LEVEL_2, "%s:\tTerminated\n", stateName) + if state.Terminated.Reason != "" { + w.Write(LEVEL_3, "Reason:\t%s\n", state.Terminated.Reason) + } + if state.Terminated.Message != "" { + w.Write(LEVEL_3, "Message:\t%s\n", state.Terminated.Message) + } + w.Write(LEVEL_3, "Exit Code:\t%d\n", state.Terminated.ExitCode) + if state.Terminated.Signal > 0 { + w.Write(LEVEL_3, "Signal:\t%d\n", state.Terminated.Signal) + } + w.Write(LEVEL_3, "Started:\t%s\n", state.Terminated.StartedAt.Time.Format(time.RFC1123Z)) + w.Write(LEVEL_3, "Finished:\t%s\n", state.Terminated.FinishedAt.Time.Format(time.RFC1123Z)) + default: + w.Write(LEVEL_2, "%s:\tWaiting\n", stateName) + } +} + +func describeVolumeClaimTemplates(templates []corev1.PersistentVolumeClaim, w PrefixWriter) { + if len(templates) == 0 { + w.Write(LEVEL_0, "Volume Claims:\t\n") + return + } + w.Write(LEVEL_0, "Volume Claims:\n") + for _, pvc := range templates { + w.Write(LEVEL_1, "Name:\t%s\n", pvc.Name) + w.Write(LEVEL_1, "StorageClass:\t%s\n", storageutil.GetPersistentVolumeClaimClass(&pvc)) + printLabelsMultilineWithIndent(w, " ", "Labels", "\t", pvc.Labels, sets.NewString()) + printLabelsMultilineWithIndent(w, " ", "Annotations", "\t", pvc.Annotations, sets.NewString()) + if capacity, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage]; ok { + w.Write(LEVEL_1, "Capacity:\t%s\n", capacity.String()) + } else { + w.Write(LEVEL_1, "Capacity:\t%s\n", "") + } + w.Write(LEVEL_1, "Access Modes:\t%s\n", pvc.Spec.AccessModes) + } +} + +func printBoolPtr(value *bool) string { + if value != nil { + return printBool(*value) + } + + return "" +} + +func printBool(value bool) string { + if value { + return "True" + } + + return "False" +} + +// ReplicationControllerDescriber generates information about a replication controller +// and the pods it has created. +type ReplicationControllerDescriber struct { + clientset.Interface +} + +func (d *ReplicationControllerDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + rc := d.Core().ReplicationControllers(namespace) + pc := d.Core().Pods(namespace) + + controller, err := rc.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + running, waiting, succeeded, failed, err := getPodStatusForController(pc, labels.SelectorFromSet(controller.Spec.Selector), controller.UID) + if err != nil { + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = d.Core().Events(namespace).Search(scheme.Scheme, controller) + } + + return describeReplicationController(controller, events, running, waiting, succeeded, failed) +} + +func describeReplicationController(controller *corev1.ReplicationController, events *corev1.EventList, running, waiting, succeeded, failed int) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", controller.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", controller.Namespace) + w.Write(LEVEL_0, "Selector:\t%s\n", labels.FormatLabels(controller.Spec.Selector)) + printLabelsMultiline(w, "Labels", controller.Labels) + printAnnotationsMultiline(w, "Annotations", controller.Annotations) + w.Write(LEVEL_0, "Replicas:\t%d current / %d desired\n", controller.Status.Replicas, *controller.Spec.Replicas) + w.Write(LEVEL_0, "Pods Status:\t%d Running / %d Waiting / %d Succeeded / %d Failed\n", running, waiting, succeeded, failed) + DescribePodTemplate(controller.Spec.Template, w) + if len(controller.Status.Conditions) > 0 { + w.Write(LEVEL_0, "Conditions:\n Type\tStatus\tReason\n") + w.Write(LEVEL_1, "----\t------\t------\n") + for _, c := range controller.Status.Conditions { + w.Write(LEVEL_1, "%v \t%v\t%v\n", c.Type, c.Status, c.Reason) + } + } + if events != nil { + DescribeEvents(events, w) + } + return nil + }) +} + +func DescribePodTemplate(template *corev1.PodTemplateSpec, w PrefixWriter) { + w.Write(LEVEL_0, "Pod Template:\n") + if template == nil { + w.Write(LEVEL_1, "") + return + } + printLabelsMultiline(w, " Labels", template.Labels) + if len(template.Annotations) > 0 { + printAnnotationsMultiline(w, " Annotations", template.Annotations) + } + if len(template.Spec.ServiceAccountName) > 0 { + w.Write(LEVEL_1, "Service Account:\t%s\n", template.Spec.ServiceAccountName) + } + if len(template.Spec.InitContainers) > 0 { + describeContainers("Init Containers", template.Spec.InitContainers, nil, nil, w, " ") + } + describeContainers("Containers", template.Spec.Containers, nil, nil, w, " ") + describeVolumes(template.Spec.Volumes, w, " ") +} + +// ReplicaSetDescriber generates information about a ReplicaSet and the pods it has created. +type ReplicaSetDescriber struct { + clientset.Interface +} + +func (d *ReplicaSetDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + rsc := d.Apps().ReplicaSets(namespace) + pc := d.Core().Pods(namespace) + + rs, err := rsc.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector) + if err != nil { + return "", err + } + + running, waiting, succeeded, failed, getPodErr := getPodStatusForController(pc, selector, rs.UID) + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = d.Core().Events(namespace).Search(scheme.Scheme, rs) + } + + return describeReplicaSet(rs, events, running, waiting, succeeded, failed, getPodErr) +} + +func describeReplicaSet(rs *appsv1.ReplicaSet, events *corev1.EventList, running, waiting, succeeded, failed int, getPodErr error) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", rs.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", rs.Namespace) + w.Write(LEVEL_0, "Selector:\t%s\n", metav1.FormatLabelSelector(rs.Spec.Selector)) + printLabelsMultiline(w, "Labels", rs.Labels) + printAnnotationsMultiline(w, "Annotations", rs.Annotations) + if controlledBy := printController(rs); len(controlledBy) > 0 { + w.Write(LEVEL_0, "Controlled By:\t%s\n", controlledBy) + } + w.Write(LEVEL_0, "Replicas:\t%d current / %d desired\n", rs.Status.Replicas, *rs.Spec.Replicas) + w.Write(LEVEL_0, "Pods Status:\t") + if getPodErr != nil { + w.Write(LEVEL_0, "error in fetching pods: %s\n", getPodErr) + } else { + w.Write(LEVEL_0, "%d Running / %d Waiting / %d Succeeded / %d Failed\n", running, waiting, succeeded, failed) + } + DescribePodTemplate(&rs.Spec.Template, w) + if len(rs.Status.Conditions) > 0 { + w.Write(LEVEL_0, "Conditions:\n Type\tStatus\tReason\n") + w.Write(LEVEL_1, "----\t------\t------\n") + for _, c := range rs.Status.Conditions { + w.Write(LEVEL_1, "%v \t%v\t%v\n", c.Type, c.Status, c.Reason) + } + } + if events != nil { + DescribeEvents(events, w) + } + return nil + }) +} + +// JobDescriber generates information about a job and the pods it has created. +type JobDescriber struct { + clientset.Interface +} + +func (d *JobDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + job, err := d.Batch().Jobs(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = d.Core().Events(namespace).Search(scheme.Scheme, job) + } + + return describeJob(job, events) +} + +func describeJob(job *batchv1.Job, events *corev1.EventList) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", job.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", job.Namespace) + selector, _ := metav1.LabelSelectorAsSelector(job.Spec.Selector) + w.Write(LEVEL_0, "Selector:\t%s\n", selector) + printLabelsMultiline(w, "Labels", job.Labels) + printAnnotationsMultiline(w, "Annotations", job.Annotations) + if controlledBy := printController(job); len(controlledBy) > 0 { + w.Write(LEVEL_0, "Controlled By:\t%s\n", controlledBy) + } + w.Write(LEVEL_0, "Parallelism:\t%d\n", *job.Spec.Parallelism) + if job.Spec.Completions != nil { + w.Write(LEVEL_0, "Completions:\t%d\n", *job.Spec.Completions) + } else { + w.Write(LEVEL_0, "Completions:\t\n") + } + if job.Status.StartTime != nil { + w.Write(LEVEL_0, "Start Time:\t%s\n", job.Status.StartTime.Time.Format(time.RFC1123Z)) + } + if job.Status.CompletionTime != nil { + w.Write(LEVEL_0, "Completed At:\t%s\n", job.Status.CompletionTime.Time.Format(time.RFC1123Z)) + } + if job.Status.StartTime != nil && job.Status.CompletionTime != nil { + w.Write(LEVEL_0, "Duration:\t%s\n", duration.HumanDuration(job.Status.CompletionTime.Sub(job.Status.StartTime.Time))) + } + if job.Spec.ActiveDeadlineSeconds != nil { + w.Write(LEVEL_0, "Active Deadline Seconds:\t%ds\n", *job.Spec.ActiveDeadlineSeconds) + } + w.Write(LEVEL_0, "Pods Statuses:\t%d Running / %d Succeeded / %d Failed\n", job.Status.Active, job.Status.Succeeded, job.Status.Failed) + DescribePodTemplate(&job.Spec.Template, w) + if events != nil { + DescribeEvents(events, w) + } + return nil + }) +} + +// CronJobDescriber generates information about a cron job and the jobs it has created. +type CronJobDescriber struct { + client clientset.Interface +} + +func (d *CronJobDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + cronJob, err := d.client.BatchV1beta1().CronJobs(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = d.client.CoreV1().Events(namespace).Search(scheme.Scheme, cronJob) + } + return describeCronJob(cronJob, events) +} + +func describeCronJob(cronJob *batchv1beta1.CronJob, events *corev1.EventList) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", cronJob.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", cronJob.Namespace) + printLabelsMultiline(w, "Labels", cronJob.Labels) + printAnnotationsMultiline(w, "Annotations", cronJob.Annotations) + w.Write(LEVEL_0, "Schedule:\t%s\n", cronJob.Spec.Schedule) + w.Write(LEVEL_0, "Concurrency Policy:\t%s\n", cronJob.Spec.ConcurrencyPolicy) + w.Write(LEVEL_0, "Suspend:\t%s\n", printBoolPtr(cronJob.Spec.Suspend)) + if cronJob.Spec.StartingDeadlineSeconds != nil { + w.Write(LEVEL_0, "Starting Deadline Seconds:\t%ds\n", *cronJob.Spec.StartingDeadlineSeconds) + } else { + w.Write(LEVEL_0, "Starting Deadline Seconds:\t\n") + } + describeJobTemplate(cronJob.Spec.JobTemplate, w) + if cronJob.Status.LastScheduleTime != nil { + w.Write(LEVEL_0, "Last Schedule Time:\t%s\n", cronJob.Status.LastScheduleTime.Time.Format(time.RFC1123Z)) + } else { + w.Write(LEVEL_0, "Last Schedule Time:\t\n") + } + printActiveJobs(w, "Active Jobs", cronJob.Status.Active) + if events != nil { + DescribeEvents(events, w) + } + return nil + }) +} + +func describeJobTemplate(jobTemplate batchv1beta1.JobTemplateSpec, w PrefixWriter) { + if jobTemplate.Spec.Selector != nil { + selector, _ := metav1.LabelSelectorAsSelector(jobTemplate.Spec.Selector) + w.Write(LEVEL_0, "Selector:\t%s\n", selector) + } else { + w.Write(LEVEL_0, "Selector:\t\n") + } + if jobTemplate.Spec.Parallelism != nil { + w.Write(LEVEL_0, "Parallelism:\t%d\n", *jobTemplate.Spec.Parallelism) + } else { + w.Write(LEVEL_0, "Parallelism:\t\n") + } + if jobTemplate.Spec.Completions != nil { + w.Write(LEVEL_0, "Completions:\t%d\n", *jobTemplate.Spec.Completions) + } else { + w.Write(LEVEL_0, "Completions:\t\n") + } + if jobTemplate.Spec.ActiveDeadlineSeconds != nil { + w.Write(LEVEL_0, "Active Deadline Seconds:\t%ds\n", *jobTemplate.Spec.ActiveDeadlineSeconds) + } + DescribePodTemplate(&jobTemplate.Spec.Template, w) +} + +func printActiveJobs(w PrefixWriter, title string, jobs []corev1.ObjectReference) { + w.Write(LEVEL_0, "%s:\t", title) + if len(jobs) == 0 { + w.WriteLine("") + return + } + + for i, job := range jobs { + if i != 0 { + w.Write(LEVEL_0, ", ") + } + w.Write(LEVEL_0, "%s", job.Name) + } + w.WriteLine("") +} + +// DaemonSetDescriber generates information about a daemon set and the pods it has created. +type DaemonSetDescriber struct { + clientset.Interface +} + +func (d *DaemonSetDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + dc := d.Apps().DaemonSets(namespace) + pc := d.Core().Pods(namespace) + + daemon, err := dc.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + selector, err := metav1.LabelSelectorAsSelector(daemon.Spec.Selector) + if err != nil { + return "", err + } + running, waiting, succeeded, failed, err := getPodStatusForController(pc, selector, daemon.UID) + if err != nil { + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = d.Core().Events(namespace).Search(scheme.Scheme, daemon) + } + + return describeDaemonSet(daemon, events, running, waiting, succeeded, failed) +} + +func describeDaemonSet(daemon *appsv1.DaemonSet, events *corev1.EventList, running, waiting, succeeded, failed int) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", daemon.Name) + selector, err := metav1.LabelSelectorAsSelector(daemon.Spec.Selector) + if err != nil { + // this shouldn't happen if LabelSelector passed validation + return err + } + w.Write(LEVEL_0, "Selector:\t%s\n", selector) + w.Write(LEVEL_0, "Node-Selector:\t%s\n", labels.FormatLabels(daemon.Spec.Template.Spec.NodeSelector)) + printLabelsMultiline(w, "Labels", daemon.Labels) + printAnnotationsMultiline(w, "Annotations", daemon.Annotations) + w.Write(LEVEL_0, "Desired Number of Nodes Scheduled: %d\n", daemon.Status.DesiredNumberScheduled) + w.Write(LEVEL_0, "Current Number of Nodes Scheduled: %d\n", daemon.Status.CurrentNumberScheduled) + w.Write(LEVEL_0, "Number of Nodes Scheduled with Up-to-date Pods: %d\n", daemon.Status.UpdatedNumberScheduled) + w.Write(LEVEL_0, "Number of Nodes Scheduled with Available Pods: %d\n", daemon.Status.NumberAvailable) + w.Write(LEVEL_0, "Number of Nodes Misscheduled: %d\n", daemon.Status.NumberMisscheduled) + w.Write(LEVEL_0, "Pods Status:\t%d Running / %d Waiting / %d Succeeded / %d Failed\n", running, waiting, succeeded, failed) + DescribePodTemplate(&daemon.Spec.Template, w) + if events != nil { + DescribeEvents(events, w) + } + return nil + }) +} + +// SecretDescriber generates information about a secret +type SecretDescriber struct { + clientset.Interface +} + +func (d *SecretDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + c := d.Core().Secrets(namespace) + + secret, err := c.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + return describeSecret(secret) +} + +func describeSecret(secret *corev1.Secret) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", secret.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", secret.Namespace) + printLabelsMultiline(w, "Labels", secret.Labels) + skipAnnotations := sets.NewString(corev1.LastAppliedConfigAnnotation) + printAnnotationsMultilineWithFilter(w, "Annotations", secret.Annotations, skipAnnotations) + + w.Write(LEVEL_0, "\nType:\t%s\n", secret.Type) + + w.Write(LEVEL_0, "\nData\n====\n") + for k, v := range secret.Data { + switch { + case k == corev1.ServiceAccountTokenKey && secret.Type == corev1.SecretTypeServiceAccountToken: + w.Write(LEVEL_0, "%s:\t%s\n", k, string(v)) + default: + w.Write(LEVEL_0, "%s:\t%d bytes\n", k, len(v)) + } + } + + return nil + }) +} + +type IngressDescriber struct { + clientset.Interface +} + +func (i *IngressDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + c := i.Extensions().Ingresses(namespace) + ing, err := c.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + return i.describeIngress(ing, describerSettings) +} + +func (i *IngressDescriber) describeBackend(ns string, backend *extensionsv1beta1.IngressBackend) string { + endpoints, _ := i.Core().Endpoints(ns).Get(backend.ServiceName, metav1.GetOptions{}) + service, _ := i.Core().Services(ns).Get(backend.ServiceName, metav1.GetOptions{}) + spName := "" + for i := range service.Spec.Ports { + sp := &service.Spec.Ports[i] + switch backend.ServicePort.Type { + case intstr.String: + if backend.ServicePort.StrVal == sp.Name { + spName = sp.Name + } + case intstr.Int: + if int32(backend.ServicePort.IntVal) == sp.Port { + spName = sp.Name + } + } + } + return formatEndpoints(endpoints, sets.NewString(spName)) +} + +func (i *IngressDescriber) describeIngress(ing *extensionsv1beta1.Ingress, describerSettings describe.DescriberSettings) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%v\n", ing.Name) + w.Write(LEVEL_0, "Namespace:\t%v\n", ing.Namespace) + w.Write(LEVEL_0, "Address:\t%v\n", loadBalancerStatusStringer(ing.Status.LoadBalancer, true)) + def := ing.Spec.Backend + ns := ing.Namespace + if def == nil { + // Ingresses that don't specify a default backend inherit the + // default backend in the kube-system namespace. + def = &extensionsv1beta1.IngressBackend{ + ServiceName: "default-http-backend", + ServicePort: intstr.IntOrString{Type: intstr.Int, IntVal: 80}, + } + ns = metav1.NamespaceSystem + } + w.Write(LEVEL_0, "Default backend:\t%s (%s)\n", backendStringer(def), i.describeBackend(ns, def)) + if len(ing.Spec.TLS) != 0 { + describeIngressTLS(w, ing.Spec.TLS) + } + w.Write(LEVEL_0, "Rules:\n Host\tPath\tBackends\n") + w.Write(LEVEL_1, "----\t----\t--------\n") + count := 0 + for _, rules := range ing.Spec.Rules { + if rules.HTTP == nil { + continue + } + count++ + host := rules.Host + if len(host) == 0 { + host = "*" + } + w.Write(LEVEL_1, "%s\t\n", host) + for _, path := range rules.HTTP.Paths { + w.Write(LEVEL_2, "\t%s \t%s (%s)\n", path.Path, backendStringer(&path.Backend), i.describeBackend(ns, &path.Backend)) + } + } + if count == 0 { + w.Write(LEVEL_1, "%s\t%s \t%s (%s)\n", "*", "*", backendStringer(def), i.describeBackend(ns, def)) + } + describeIngressAnnotations(w, ing.Annotations) + + if describerSettings.ShowEvents { + events, _ := i.Core().Events(ing.Namespace).Search(scheme.Scheme, ing) + if events != nil { + DescribeEvents(events, w) + } + } + return nil + }) +} + +func describeIngressTLS(w PrefixWriter, ingTLS []extensionsv1beta1.IngressTLS) { + w.Write(LEVEL_0, "TLS:\n") + for _, t := range ingTLS { + if t.SecretName == "" { + w.Write(LEVEL_1, "SNI routes %v\n", strings.Join(t.Hosts, ",")) + } else { + w.Write(LEVEL_1, "%v terminates %v\n", t.SecretName, strings.Join(t.Hosts, ",")) + } + } + return +} + +// TODO: Move from annotations into Ingress status. +func describeIngressAnnotations(w PrefixWriter, annotations map[string]string) { + w.Write(LEVEL_0, "Annotations:\n") + for k, v := range annotations { + w.Write(LEVEL_1, "%v:\t%s\n", k, v) + } + return +} + +// ServiceDescriber generates information about a service. +type ServiceDescriber struct { + clientset.Interface +} + +func (d *ServiceDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + c := d.Core().Services(namespace) + + service, err := c.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + endpoints, _ := d.Core().Endpoints(namespace).Get(name, metav1.GetOptions{}) + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = d.Core().Events(namespace).Search(scheme.Scheme, service) + } + return describeService(service, endpoints, events) +} + +func buildIngressString(ingress []corev1.LoadBalancerIngress) string { + var buffer bytes.Buffer + + for i := range ingress { + if i != 0 { + buffer.WriteString(", ") + } + if ingress[i].IP != "" { + buffer.WriteString(ingress[i].IP) + } else { + buffer.WriteString(ingress[i].Hostname) + } + } + return buffer.String() +} + +func describeService(service *corev1.Service, endpoints *corev1.Endpoints, events *corev1.EventList) (string, error) { + if endpoints == nil { + endpoints = &corev1.Endpoints{} + } + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", service.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", service.Namespace) + printLabelsMultiline(w, "Labels", service.Labels) + printAnnotationsMultiline(w, "Annotations", service.Annotations) + w.Write(LEVEL_0, "Selector:\t%s\n", labels.FormatLabels(service.Spec.Selector)) + w.Write(LEVEL_0, "Type:\t%s\n", service.Spec.Type) + w.Write(LEVEL_0, "IP:\t%s\n", service.Spec.ClusterIP) + if len(service.Spec.ExternalIPs) > 0 { + w.Write(LEVEL_0, "External IPs:\t%v\n", strings.Join(service.Spec.ExternalIPs, ",")) + } + if service.Spec.LoadBalancerIP != "" { + w.Write(LEVEL_0, "IP:\t%s\n", service.Spec.LoadBalancerIP) + } + if service.Spec.ExternalName != "" { + w.Write(LEVEL_0, "External Name:\t%s\n", service.Spec.ExternalName) + } + if len(service.Status.LoadBalancer.Ingress) > 0 { + list := buildIngressString(service.Status.LoadBalancer.Ingress) + w.Write(LEVEL_0, "LoadBalancer Ingress:\t%s\n", list) + } + for i := range service.Spec.Ports { + sp := &service.Spec.Ports[i] + + name := sp.Name + if name == "" { + name = "" + } + w.Write(LEVEL_0, "Port:\t%s\t%d/%s\n", name, sp.Port, sp.Protocol) + if sp.TargetPort.Type == intstr.Type(intstr.Int) { + w.Write(LEVEL_0, "TargetPort:\t%d/%s\n", sp.TargetPort.IntVal, sp.Protocol) + } else { + w.Write(LEVEL_0, "TargetPort:\t%s/%s\n", sp.TargetPort.StrVal, sp.Protocol) + } + if sp.NodePort != 0 { + w.Write(LEVEL_0, "NodePort:\t%s\t%d/%s\n", name, sp.NodePort, sp.Protocol) + } + w.Write(LEVEL_0, "Endpoints:\t%s\n", formatEndpoints(endpoints, sets.NewString(sp.Name))) + } + w.Write(LEVEL_0, "Session Affinity:\t%s\n", service.Spec.SessionAffinity) + if service.Spec.ExternalTrafficPolicy != "" { + w.Write(LEVEL_0, "External Traffic Policy:\t%s\n", service.Spec.ExternalTrafficPolicy) + } + if service.Spec.HealthCheckNodePort != 0 { + w.Write(LEVEL_0, "HealthCheck NodePort:\t%d\n", service.Spec.HealthCheckNodePort) + } + if len(service.Spec.LoadBalancerSourceRanges) > 0 { + w.Write(LEVEL_0, "LoadBalancer Source Ranges:\t%v\n", strings.Join(service.Spec.LoadBalancerSourceRanges, ",")) + } + if events != nil { + DescribeEvents(events, w) + } + return nil + }) +} + +// EndpointsDescriber generates information about an Endpoint. +type EndpointsDescriber struct { + clientset.Interface +} + +func (d *EndpointsDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + c := d.Core().Endpoints(namespace) + + ep, err := c.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = d.Core().Events(namespace).Search(scheme.Scheme, ep) + } + + return describeEndpoints(ep, events) +} + +func describeEndpoints(ep *corev1.Endpoints, events *corev1.EventList) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", ep.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", ep.Namespace) + printLabelsMultiline(w, "Labels", ep.Labels) + printAnnotationsMultiline(w, "Annotations", ep.Annotations) + + w.Write(LEVEL_0, "Subsets:\n") + for i := range ep.Subsets { + subset := &ep.Subsets[i] + + addresses := make([]string, 0, len(subset.Addresses)) + for _, addr := range subset.Addresses { + addresses = append(addresses, addr.IP) + } + addressesString := strings.Join(addresses, ",") + if len(addressesString) == 0 { + addressesString = "" + } + w.Write(LEVEL_1, "Addresses:\t%s\n", addressesString) + + notReadyAddresses := make([]string, 0, len(subset.NotReadyAddresses)) + for _, addr := range subset.NotReadyAddresses { + notReadyAddresses = append(notReadyAddresses, addr.IP) + } + notReadyAddressesString := strings.Join(notReadyAddresses, ",") + if len(notReadyAddressesString) == 0 { + notReadyAddressesString = "" + } + w.Write(LEVEL_1, "NotReadyAddresses:\t%s\n", notReadyAddressesString) + + if len(subset.Ports) > 0 { + w.Write(LEVEL_1, "Ports:\n") + w.Write(LEVEL_2, "Name\tPort\tProtocol\n") + w.Write(LEVEL_2, "----\t----\t--------\n") + for _, port := range subset.Ports { + name := port.Name + if len(name) == 0 { + name = "" + } + w.Write(LEVEL_2, "%s\t%d\t%s\n", name, port.Port, port.Protocol) + } + } + w.Write(LEVEL_0, "\n") + } + + if events != nil { + DescribeEvents(events, w) + } + return nil + }) +} + +// ServiceAccountDescriber generates information about a service. +type ServiceAccountDescriber struct { + clientset.Interface +} + +func (d *ServiceAccountDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + c := d.Core().ServiceAccounts(namespace) + + serviceAccount, err := c.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + tokens := []corev1.Secret{} + + // missingSecrets is the set of all secrets present in the + // serviceAccount but not present in the set of existing secrets. + missingSecrets := sets.NewString() + secrets, err := d.Core().Secrets(namespace).List(metav1.ListOptions{}) + + // errors are tolerated here in order to describe the serviceAccount with all + // of the secrets that it references, even if those secrets cannot be fetched. + if err == nil { + // existingSecrets is the set of all secrets remaining on a + // service account that are not present in the "tokens" slice. + existingSecrets := sets.NewString() + + for _, s := range secrets.Items { + if s.Type == corev1.SecretTypeServiceAccountToken { + name, _ := s.Annotations[corev1.ServiceAccountNameKey] + uid, _ := s.Annotations[corev1.ServiceAccountUIDKey] + if name == serviceAccount.Name && uid == string(serviceAccount.UID) { + tokens = append(tokens, s) + } + } + existingSecrets.Insert(s.Name) + } + + for _, s := range serviceAccount.Secrets { + if !existingSecrets.Has(s.Name) { + missingSecrets.Insert(s.Name) + } + } + for _, s := range serviceAccount.ImagePullSecrets { + if !existingSecrets.Has(s.Name) { + missingSecrets.Insert(s.Name) + } + } + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = d.Core().Events(namespace).Search(scheme.Scheme, serviceAccount) + } + + return describeServiceAccount(serviceAccount, tokens, missingSecrets, events) +} + +func describeServiceAccount(serviceAccount *corev1.ServiceAccount, tokens []corev1.Secret, missingSecrets sets.String, events *corev1.EventList) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", serviceAccount.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", serviceAccount.Namespace) + printLabelsMultiline(w, "Labels", serviceAccount.Labels) + printAnnotationsMultiline(w, "Annotations", serviceAccount.Annotations) + + var ( + emptyHeader = " " + pullHeader = "Image pull secrets:" + mountHeader = "Mountable secrets: " + tokenHeader = "Tokens: " + + pullSecretNames = []string{} + mountSecretNames = []string{} + tokenSecretNames = []string{} + ) + + for _, s := range serviceAccount.ImagePullSecrets { + pullSecretNames = append(pullSecretNames, s.Name) + } + for _, s := range serviceAccount.Secrets { + mountSecretNames = append(mountSecretNames, s.Name) + } + for _, s := range tokens { + tokenSecretNames = append(tokenSecretNames, s.Name) + } + + types := map[string][]string{ + pullHeader: pullSecretNames, + mountHeader: mountSecretNames, + tokenHeader: tokenSecretNames, + } + for _, header := range sets.StringKeySet(types).List() { + names := types[header] + if len(names) == 0 { + w.Write(LEVEL_0, "%s\t\n", header) + } else { + prefix := header + for _, name := range names { + if missingSecrets.Has(name) { + w.Write(LEVEL_0, "%s\t%s (not found)\n", prefix, name) + } else { + w.Write(LEVEL_0, "%s\t%s\n", prefix, name) + } + prefix = emptyHeader + } + } + } + + if events != nil { + DescribeEvents(events, w) + } + + return nil + }) +} + +// RoleDescriber generates information about a node. +type RoleDescriber struct { + clientset.Interface +} + +func (d *RoleDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + role, err := d.Rbac().Roles(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + breakdownRules := []rbacv1.PolicyRule{} + for _, rule := range role.Rules { + breakdownRules = append(breakdownRules, rbac.BreakdownRule(rule)...) + } + + compactRules, err := rbac.CompactRules(breakdownRules) + if err != nil { + return "", err + } + sort.Stable(rbac.SortableRuleSlice(compactRules)) + + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", role.Name) + printLabelsMultiline(w, "Labels", role.Labels) + printAnnotationsMultiline(w, "Annotations", role.Annotations) + + w.Write(LEVEL_0, "PolicyRule:\n") + w.Write(LEVEL_1, "Resources\tNon-Resource URLs\tResource Names\tVerbs\n") + w.Write(LEVEL_1, "---------\t-----------------\t--------------\t-----\n") + for _, r := range compactRules { + w.Write(LEVEL_1, "%s\t%v\t%v\t%v\n", combineResourceGroup(r.Resources, r.APIGroups), r.NonResourceURLs, r.ResourceNames, r.Verbs) + } + + return nil + }) +} + +// ClusterRoleDescriber generates information about a node. +type ClusterRoleDescriber struct { + clientset.Interface +} + +func (d *ClusterRoleDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + role, err := d.Rbac().ClusterRoles().Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + breakdownRules := []rbacv1.PolicyRule{} + for _, rule := range role.Rules { + breakdownRules = append(breakdownRules, rbac.BreakdownRule(rule)...) + } + + compactRules, err := rbac.CompactRules(breakdownRules) + if err != nil { + return "", err + } + sort.Stable(rbac.SortableRuleSlice(compactRules)) + + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", role.Name) + printLabelsMultiline(w, "Labels", role.Labels) + printAnnotationsMultiline(w, "Annotations", role.Annotations) + + w.Write(LEVEL_0, "PolicyRule:\n") + w.Write(LEVEL_1, "Resources\tNon-Resource URLs\tResource Names\tVerbs\n") + w.Write(LEVEL_1, "---------\t-----------------\t--------------\t-----\n") + for _, r := range compactRules { + w.Write(LEVEL_1, "%s\t%v\t%v\t%v\n", combineResourceGroup(r.Resources, r.APIGroups), r.NonResourceURLs, r.ResourceNames, r.Verbs) + } + + return nil + }) +} + +func combineResourceGroup(resource, group []string) string { + if len(resource) == 0 { + return "" + } + parts := strings.SplitN(resource[0], "/", 2) + combine := parts[0] + + if len(group) > 0 && group[0] != "" { + combine = combine + "." + group[0] + } + + if len(parts) == 2 { + combine = combine + "/" + parts[1] + } + return combine +} + +// RoleBindingDescriber generates information about a node. +type RoleBindingDescriber struct { + clientset.Interface +} + +func (d *RoleBindingDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + binding, err := d.Rbac().RoleBindings(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", binding.Name) + printLabelsMultiline(w, "Labels", binding.Labels) + printAnnotationsMultiline(w, "Annotations", binding.Annotations) + + w.Write(LEVEL_0, "Role:\n") + w.Write(LEVEL_1, "Kind:\t%s\n", binding.RoleRef.Kind) + w.Write(LEVEL_1, "Name:\t%s\n", binding.RoleRef.Name) + + w.Write(LEVEL_0, "Subjects:\n") + w.Write(LEVEL_1, "Kind\tName\tNamespace\n") + w.Write(LEVEL_1, "----\t----\t---------\n") + for _, s := range binding.Subjects { + w.Write(LEVEL_1, "%s\t%s\t%s\n", s.Kind, s.Name, s.Namespace) + } + + return nil + }) +} + +// ClusterRoleBindingDescriber generates information about a node. +type ClusterRoleBindingDescriber struct { + clientset.Interface +} + +func (d *ClusterRoleBindingDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + binding, err := d.Rbac().ClusterRoleBindings().Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", binding.Name) + printLabelsMultiline(w, "Labels", binding.Labels) + printAnnotationsMultiline(w, "Annotations", binding.Annotations) + + w.Write(LEVEL_0, "Role:\n") + w.Write(LEVEL_1, "Kind:\t%s\n", binding.RoleRef.Kind) + w.Write(LEVEL_1, "Name:\t%s\n", binding.RoleRef.Name) + + w.Write(LEVEL_0, "Subjects:\n") + w.Write(LEVEL_1, "Kind\tName\tNamespace\n") + w.Write(LEVEL_1, "----\t----\t---------\n") + for _, s := range binding.Subjects { + w.Write(LEVEL_1, "%s\t%s\t%s\n", s.Kind, s.Name, s.Namespace) + } + + return nil + }) +} + +// NodeDescriber generates information about a node. +type NodeDescriber struct { + clientset.Interface +} + +func (d *NodeDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + mc := d.Core().Nodes() + node, err := mc.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + fieldSelector, err := fields.ParseSelector("spec.nodeName=" + name + ",status.phase!=" + string(corev1.PodSucceeded) + ",status.phase!=" + string(corev1.PodFailed)) + if err != nil { + return "", err + } + // in a policy aware setting, users may have access to a node, but not all pods + // in that case, we note that the user does not have access to the pods + canViewPods := true + nodeNonTerminatedPodsList, err := d.Core().Pods(namespace).List(metav1.ListOptions{FieldSelector: fieldSelector.String()}) + if err != nil { + if !errors.IsForbidden(err) { + return "", err + } + canViewPods = false + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + if ref, err := reference.GetReference(scheme.Scheme, node); err != nil { + klog.Errorf("Unable to construct reference to '%#v': %v", node, err) + } else { + // TODO: We haven't decided the namespace for Node object yet. + ref.UID = types.UID(ref.Name) + events, _ = d.Core().Events("").Search(scheme.Scheme, ref) + } + } + + return describeNode(node, nodeNonTerminatedPodsList, events, canViewPods) +} + +func describeNode(node *corev1.Node, nodeNonTerminatedPodsList *corev1.PodList, events *corev1.EventList, canViewPods bool) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", node.Name) + if roles := findNodeRoles(node); len(roles) > 0 { + w.Write(LEVEL_0, "Roles:\t%s\n", strings.Join(roles, ",")) + } else { + w.Write(LEVEL_0, "Roles:\t%s\n", "") + } + printLabelsMultiline(w, "Labels", node.Labels) + printAnnotationsMultiline(w, "Annotations", node.Annotations) + w.Write(LEVEL_0, "CreationTimestamp:\t%s\n", node.CreationTimestamp.Time.Format(time.RFC1123Z)) + printNodeTaintsMultiline(w, "Taints", node.Spec.Taints) + w.Write(LEVEL_0, "Unschedulable:\t%v\n", node.Spec.Unschedulable) + if len(node.Status.Conditions) > 0 { + w.Write(LEVEL_0, "Conditions:\n Type\tStatus\tLastHeartbeatTime\tLastTransitionTime\tReason\tMessage\n") + w.Write(LEVEL_1, "----\t------\t-----------------\t------------------\t------\t-------\n") + for _, c := range node.Status.Conditions { + w.Write(LEVEL_1, "%v \t%v \t%s \t%s \t%v \t%v\n", + c.Type, + c.Status, + c.LastHeartbeatTime.Time.Format(time.RFC1123Z), + c.LastTransitionTime.Time.Format(time.RFC1123Z), + c.Reason, + c.Message) + } + } + + w.Write(LEVEL_0, "Addresses:\n") + for _, address := range node.Status.Addresses { + w.Write(LEVEL_1, "%s:\t%s\n", address.Type, address.Address) + } + + printResourceList := func(resourceList corev1.ResourceList) { + resources := make([]corev1.ResourceName, 0, len(resourceList)) + for resource := range resourceList { + resources = append(resources, resource) + } + sort.Sort(SortableResourceNames(resources)) + for _, resource := range resources { + value := resourceList[resource] + w.Write(LEVEL_0, " %s:\t%s\n", resource, value.String()) + } + } + + if len(node.Status.Capacity) > 0 { + w.Write(LEVEL_0, "Capacity:\n") + printResourceList(node.Status.Capacity) + } + if len(node.Status.Allocatable) > 0 { + w.Write(LEVEL_0, "Allocatable:\n") + printResourceList(node.Status.Allocatable) + } + + w.Write(LEVEL_0, "System Info:\n") + w.Write(LEVEL_0, " Machine ID:\t%s\n", node.Status.NodeInfo.MachineID) + w.Write(LEVEL_0, " System UUID:\t%s\n", node.Status.NodeInfo.SystemUUID) + w.Write(LEVEL_0, " Boot ID:\t%s\n", node.Status.NodeInfo.BootID) + w.Write(LEVEL_0, " Kernel Version:\t%s\n", node.Status.NodeInfo.KernelVersion) + w.Write(LEVEL_0, " OS Image:\t%s\n", node.Status.NodeInfo.OSImage) + w.Write(LEVEL_0, " Operating System:\t%s\n", node.Status.NodeInfo.OperatingSystem) + w.Write(LEVEL_0, " Architecture:\t%s\n", node.Status.NodeInfo.Architecture) + w.Write(LEVEL_0, " Container Runtime Version:\t%s\n", node.Status.NodeInfo.ContainerRuntimeVersion) + w.Write(LEVEL_0, " Kubelet Version:\t%s\n", node.Status.NodeInfo.KubeletVersion) + w.Write(LEVEL_0, " Kube-Proxy Version:\t%s\n", node.Status.NodeInfo.KubeProxyVersion) + + if len(node.Spec.PodCIDR) > 0 { + w.Write(LEVEL_0, "PodCIDR:\t%s\n", node.Spec.PodCIDR) + } + if len(node.Spec.ProviderID) > 0 { + w.Write(LEVEL_0, "ProviderID:\t%s\n", node.Spec.ProviderID) + } + if canViewPods && nodeNonTerminatedPodsList != nil { + describeNodeResource(nodeNonTerminatedPodsList, node, w) + } else { + w.Write(LEVEL_0, "Pods:\tnot authorized\n") + } + if events != nil { + DescribeEvents(events, w) + } + return nil + }) +} + +type StatefulSetDescriber struct { + client clientset.Interface +} + +func (p *StatefulSetDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + ps, err := p.client.Apps().StatefulSets(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + pc := p.client.Core().Pods(namespace) + + selector, err := metav1.LabelSelectorAsSelector(ps.Spec.Selector) + if err != nil { + return "", err + } + + running, waiting, succeeded, failed, err := getPodStatusForController(pc, selector, ps.UID) + if err != nil { + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = p.client.Core().Events(namespace).Search(scheme.Scheme, ps) + } + + return describeStatefulSet(ps, selector, events, running, waiting, succeeded, failed) +} + +func describeStatefulSet(ps *appsv1.StatefulSet, selector labels.Selector, events *corev1.EventList, running, waiting, succeeded, failed int) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", ps.ObjectMeta.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", ps.ObjectMeta.Namespace) + w.Write(LEVEL_0, "CreationTimestamp:\t%s\n", ps.CreationTimestamp.Time.Format(time.RFC1123Z)) + w.Write(LEVEL_0, "Selector:\t%s\n", selector) + printLabelsMultiline(w, "Labels", ps.Labels) + printAnnotationsMultiline(w, "Annotations", ps.Annotations) + w.Write(LEVEL_0, "Replicas:\t%d desired | %d total\n", ps.Spec.Replicas, ps.Status.Replicas) + w.Write(LEVEL_0, "Update Strategy:\t%s\n", ps.Spec.UpdateStrategy.Type) + if ps.Spec.UpdateStrategy.RollingUpdate != nil { + ru := ps.Spec.UpdateStrategy.RollingUpdate + if ru.Partition != nil { + w.Write(LEVEL_1, "Partition:\t%d\n", ru.Partition) + } + } + + w.Write(LEVEL_0, "Pods Status:\t%d Running / %d Waiting / %d Succeeded / %d Failed\n", running, waiting, succeeded, failed) + DescribePodTemplate(&ps.Spec.Template, w) + describeVolumeClaimTemplates(ps.Spec.VolumeClaimTemplates, w) + if events != nil { + DescribeEvents(events, w) + } + + return nil + }) +} + +type CertificateSigningRequestDescriber struct { + client clientset.Interface +} + +func (p *CertificateSigningRequestDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + csr, err := p.client.Certificates().CertificateSigningRequests().Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + cr, err := certificate.ParseCSR(csr) + if err != nil { + return "", fmt.Errorf("Error parsing CSR: %v", err) + } + status, err := extractCSRStatus(csr) + if err != nil { + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = p.client.Core().Events(namespace).Search(scheme.Scheme, csr) + } + + return describeCertificateSigningRequest(csr, cr, status, events) +} + +func describeCertificateSigningRequest(csr *certificatesv1beta1.CertificateSigningRequest, cr *x509.CertificateRequest, status string, events *corev1.EventList) (string, error) { + printListHelper := func(w PrefixWriter, prefix, name string, values []string) { + if len(values) == 0 { + return + } + w.Write(LEVEL_0, prefix+name+":\t") + w.Write(LEVEL_0, strings.Join(values, "\n"+prefix+"\t")) + w.Write(LEVEL_0, "\n") + } + + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", csr.Name) + w.Write(LEVEL_0, "Labels:\t%s\n", labels.FormatLabels(csr.Labels)) + w.Write(LEVEL_0, "Annotations:\t%s\n", labels.FormatLabels(csr.Annotations)) + w.Write(LEVEL_0, "CreationTimestamp:\t%s\n", csr.CreationTimestamp.Time.Format(time.RFC1123Z)) + w.Write(LEVEL_0, "Requesting User:\t%s\n", csr.Spec.Username) + w.Write(LEVEL_0, "Status:\t%s\n", status) + + w.Write(LEVEL_0, "Subject:\n") + w.Write(LEVEL_0, "\tCommon Name:\t%s\n", cr.Subject.CommonName) + w.Write(LEVEL_0, "\tSerial Number:\t%s\n", cr.Subject.SerialNumber) + printListHelper(w, "\t", "Organization", cr.Subject.Organization) + printListHelper(w, "\t", "Organizational Unit", cr.Subject.OrganizationalUnit) + printListHelper(w, "\t", "Country", cr.Subject.Country) + printListHelper(w, "\t", "Locality", cr.Subject.Locality) + printListHelper(w, "\t", "Province", cr.Subject.Province) + printListHelper(w, "\t", "StreetAddress", cr.Subject.StreetAddress) + printListHelper(w, "\t", "PostalCode", cr.Subject.PostalCode) + + if len(cr.DNSNames)+len(cr.EmailAddresses)+len(cr.IPAddresses) > 0 { + w.Write(LEVEL_0, "Subject Alternative Names:\n") + printListHelper(w, "\t", "DNS Names", cr.DNSNames) + printListHelper(w, "\t", "Email Addresses", cr.EmailAddresses) + var ipaddrs []string + for _, ipaddr := range cr.IPAddresses { + ipaddrs = append(ipaddrs, ipaddr.String()) + } + printListHelper(w, "\t", "IP Addresses", ipaddrs) + } + + if events != nil { + DescribeEvents(events, w) + } + + return nil + }) +} + +// HorizontalPodAutoscalerDescriber generates information about a horizontal pod autoscaler. +type HorizontalPodAutoscalerDescriber struct { + client clientset.Interface +} + +func (d *HorizontalPodAutoscalerDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + hpa, err := d.client.AutoscalingV2beta2().HorizontalPodAutoscalers(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = d.client.Core().Events(namespace).Search(scheme.Scheme, hpa) + } + + return describeHorizontalPodAutoscaler(hpa, events, d) +} + +func describeHorizontalPodAutoscaler(hpa *autoscalingv2beta2.HorizontalPodAutoscaler, events *corev1.EventList, d *HorizontalPodAutoscalerDescriber) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", hpa.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", hpa.Namespace) + printLabelsMultiline(w, "Labels", hpa.Labels) + printAnnotationsMultiline(w, "Annotations", hpa.Annotations) + w.Write(LEVEL_0, "CreationTimestamp:\t%s\n", hpa.CreationTimestamp.Time.Format(time.RFC1123Z)) + w.Write(LEVEL_0, "Reference:\t%s/%s\n", + hpa.Spec.ScaleTargetRef.Kind, + hpa.Spec.ScaleTargetRef.Name) + w.Write(LEVEL_0, "Metrics:\t( current / target )\n") + for i, metric := range hpa.Spec.Metrics { + switch metric.Type { + case autoscalingv2beta2.ExternalMetricSourceType: + if metric.External.Target.AverageValue != nil { + current := "" + if len(hpa.Status.CurrentMetrics) > i && hpa.Status.CurrentMetrics[i].External != nil && + &hpa.Status.CurrentMetrics[i].External.Current.AverageValue != nil { + current = hpa.Status.CurrentMetrics[i].External.Current.AverageValue.String() + } + w.Write(LEVEL_1, "%q (target average value):\t%s / %s\n", metric.External.Metric.Name, current, metric.External.Target.AverageValue.String()) + } else { + current := "" + if len(hpa.Status.CurrentMetrics) > i && hpa.Status.CurrentMetrics[i].External != nil { + current = hpa.Status.CurrentMetrics[i].External.Current.Value.String() + } + w.Write(LEVEL_1, "%q (target value):\t%s / %s\n", metric.External.Metric.Name, current, metric.External.Target.Value.String()) + + } + case autoscalingv2beta2.PodsMetricSourceType: + current := "" + if len(hpa.Status.CurrentMetrics) > i && hpa.Status.CurrentMetrics[i].Pods != nil { + current = hpa.Status.CurrentMetrics[i].Pods.Current.AverageValue.String() + } + w.Write(LEVEL_1, "%q on pods:\t%s / %s\n", metric.Pods.Metric.Name, current, metric.Pods.Target.AverageValue.String()) + case autoscalingv2beta2.ObjectMetricSourceType: + current := "" + if len(hpa.Status.CurrentMetrics) > i && hpa.Status.CurrentMetrics[i].Object != nil { + current = hpa.Status.CurrentMetrics[i].Object.Current.Value.String() + } + w.Write(LEVEL_1, "%q on %s/%s:\t%s / %s\n", metric.Object.Metric.Name, metric.Object.DescribedObject.Kind, metric.Object.DescribedObject.Name, current, metric.Object.Target.Value.String()) + case autoscalingv2beta2.ResourceMetricSourceType: + w.Write(LEVEL_1, "resource %s on pods", string(metric.Resource.Name)) + if metric.Resource.Target.AverageValue != nil { + current := "" + if len(hpa.Status.CurrentMetrics) > i && hpa.Status.CurrentMetrics[i].Resource != nil { + current = hpa.Status.CurrentMetrics[i].Resource.Current.AverageValue.String() + } + w.Write(LEVEL_0, ":\t%s / %s\n", current, metric.Resource.Target.AverageValue.String()) + } else { + current := "" + if len(hpa.Status.CurrentMetrics) > i && hpa.Status.CurrentMetrics[i].Resource != nil && hpa.Status.CurrentMetrics[i].Resource.Current.AverageUtilization != nil { + current = fmt.Sprintf("%d%% (%s)", *hpa.Status.CurrentMetrics[i].Resource.Current.AverageUtilization, hpa.Status.CurrentMetrics[i].Resource.Current.AverageValue.String()) + } + + target := "" + if metric.Resource.Target.AverageUtilization != nil { + target = fmt.Sprintf("%d%%", *metric.Resource.Target.AverageUtilization) + } + w.Write(LEVEL_1, "(as a percentage of request):\t%s / %s\n", current, target) + } + default: + w.Write(LEVEL_1, "", string(metric.Type)) + } + } + minReplicas := "" + if hpa.Spec.MinReplicas != nil { + minReplicas = fmt.Sprintf("%d", *hpa.Spec.MinReplicas) + } + w.Write(LEVEL_0, "Min replicas:\t%s\n", minReplicas) + w.Write(LEVEL_0, "Max replicas:\t%d\n", hpa.Spec.MaxReplicas) + w.Write(LEVEL_0, "%s pods:\t", hpa.Spec.ScaleTargetRef.Kind) + w.Write(LEVEL_0, "%d current / %d desired\n", hpa.Status.CurrentReplicas, hpa.Status.DesiredReplicas) + + if len(hpa.Status.Conditions) > 0 { + w.Write(LEVEL_0, "Conditions:\n") + w.Write(LEVEL_1, "Type\tStatus\tReason\tMessage\n") + w.Write(LEVEL_1, "----\t------\t------\t-------\n") + for _, c := range hpa.Status.Conditions { + w.Write(LEVEL_1, "%v\t%v\t%v\t%v\n", c.Type, c.Status, c.Reason, c.Message) + } + } + + if events != nil { + DescribeEvents(events, w) + } + + return nil + }) +} + +func describeNodeResource(nodeNonTerminatedPodsList *corev1.PodList, node *corev1.Node, w PrefixWriter) { + w.Write(LEVEL_0, "Non-terminated Pods:\t(%d in total)\n", len(nodeNonTerminatedPodsList.Items)) + w.Write(LEVEL_1, "Namespace\tName\t\tCPU Requests\tCPU Limits\tMemory Requests\tMemory Limits\tAGE\n") + w.Write(LEVEL_1, "---------\t----\t\t------------\t----------\t---------------\t-------------\t---\n") + allocatable := node.Status.Capacity + if len(node.Status.Allocatable) > 0 { + allocatable = node.Status.Allocatable + } + + for _, pod := range nodeNonTerminatedPodsList.Items { + req, limit := resourcehelper.PodRequestsAndLimits(&pod) + cpuReq, cpuLimit, memoryReq, memoryLimit := req[corev1.ResourceCPU], limit[corev1.ResourceCPU], req[corev1.ResourceMemory], limit[corev1.ResourceMemory] + fractionCpuReq := float64(cpuReq.MilliValue()) / float64(allocatable.Cpu().MilliValue()) * 100 + fractionCpuLimit := float64(cpuLimit.MilliValue()) / float64(allocatable.Cpu().MilliValue()) * 100 + fractionMemoryReq := float64(memoryReq.Value()) / float64(allocatable.Memory().Value()) * 100 + fractionMemoryLimit := float64(memoryLimit.Value()) / float64(allocatable.Memory().Value()) * 100 + w.Write(LEVEL_1, "%s\t%s\t\t%s (%d%%)\t%s (%d%%)\t%s (%d%%)\t%s (%d%%)\t%s\n", pod.Namespace, pod.Name, + cpuReq.String(), int64(fractionCpuReq), cpuLimit.String(), int64(fractionCpuLimit), + memoryReq.String(), int64(fractionMemoryReq), memoryLimit.String(), int64(fractionMemoryLimit), translateTimestampSince(pod.CreationTimestamp)) + } + + w.Write(LEVEL_0, "Allocated resources:\n (Total limits may be over 100 percent, i.e., overcommitted.)\n") + w.Write(LEVEL_1, "Resource\tRequests\tLimits\n") + w.Write(LEVEL_1, "--------\t--------\t------\n") + reqs, limits := getPodsTotalRequestsAndLimits(nodeNonTerminatedPodsList) + cpuReqs, cpuLimits, memoryReqs, memoryLimits, ephemeralstorageReqs, ephemeralstorageLimits := + reqs[corev1.ResourceCPU], limits[corev1.ResourceCPU], reqs[corev1.ResourceMemory], limits[corev1.ResourceMemory], reqs[corev1.ResourceEphemeralStorage], limits[corev1.ResourceEphemeralStorage] + fractionCpuReqs := float64(0) + fractionCpuLimits := float64(0) + if allocatable.Cpu().MilliValue() != 0 { + fractionCpuReqs = float64(cpuReqs.MilliValue()) / float64(allocatable.Cpu().MilliValue()) * 100 + fractionCpuLimits = float64(cpuLimits.MilliValue()) / float64(allocatable.Cpu().MilliValue()) * 100 + } + fractionMemoryReqs := float64(0) + fractionMemoryLimits := float64(0) + if allocatable.Memory().Value() != 0 { + fractionMemoryReqs = float64(memoryReqs.Value()) / float64(allocatable.Memory().Value()) * 100 + fractionMemoryLimits = float64(memoryLimits.Value()) / float64(allocatable.Memory().Value()) * 100 + } + fractionEphemeralStorageReqs := float64(0) + fractionEphemeralStorageLimits := float64(0) + if allocatable.StorageEphemeral().Value() != 0 { + fractionEphemeralStorageReqs = float64(ephemeralstorageReqs.Value()) / float64(allocatable.StorageEphemeral().Value()) * 100 + fractionEphemeralStorageLimits = float64(ephemeralstorageLimits.Value()) / float64(allocatable.StorageEphemeral().Value()) * 100 + } + w.Write(LEVEL_1, "%s\t%s (%d%%)\t%s (%d%%)\n", + corev1.ResourceCPU, cpuReqs.String(), int64(fractionCpuReqs), cpuLimits.String(), int64(fractionCpuLimits)) + w.Write(LEVEL_1, "%s\t%s (%d%%)\t%s (%d%%)\n", + corev1.ResourceMemory, memoryReqs.String(), int64(fractionMemoryReqs), memoryLimits.String(), int64(fractionMemoryLimits)) + w.Write(LEVEL_1, "%s\t%s (%d%%)\t%s (%d%%)\n", + corev1.ResourceEphemeralStorage, ephemeralstorageReqs.String(), int64(fractionEphemeralStorageReqs), ephemeralstorageLimits.String(), int64(fractionEphemeralStorageLimits)) + extResources := make([]string, 0, len(allocatable)) + for resource := range allocatable { + if !resourcehelper.IsStandardContainerResourceName(string(resource)) && resource != corev1.ResourcePods { + extResources = append(extResources, string(resource)) + } + } + sort.Strings(extResources) + for _, ext := range extResources { + extRequests, extLimits := reqs[corev1.ResourceName(ext)], limits[corev1.ResourceName(ext)] + w.Write(LEVEL_1, "%s\t%s\t%s\n", ext, extRequests.String(), extLimits.String()) + } +} + +func getPodsTotalRequestsAndLimits(podList *corev1.PodList) (reqs map[corev1.ResourceName]resource.Quantity, limits map[corev1.ResourceName]resource.Quantity) { + reqs, limits = map[corev1.ResourceName]resource.Quantity{}, map[corev1.ResourceName]resource.Quantity{} + for _, pod := range podList.Items { + podReqs, podLimits := resourcehelper.PodRequestsAndLimits(&pod) + for podReqName, podReqValue := range podReqs { + if value, ok := reqs[podReqName]; !ok { + reqs[podReqName] = *podReqValue.Copy() + } else { + value.Add(podReqValue) + reqs[podReqName] = value + } + } + for podLimitName, podLimitValue := range podLimits { + if value, ok := limits[podLimitName]; !ok { + limits[podLimitName] = *podLimitValue.Copy() + } else { + value.Add(podLimitValue) + limits[podLimitName] = value + } + } + } + return +} + +func DescribeEvents(el *corev1.EventList, w PrefixWriter) { + if len(el.Items) == 0 { + w.Write(LEVEL_0, "Events:\t\n") + return + } + w.Flush() + sort.Sort(event.SortableEvents(el.Items)) + w.Write(LEVEL_0, "Events:\n Type\tReason\tAge\tFrom\tMessage\n") + w.Write(LEVEL_1, "----\t------\t----\t----\t-------\n") + for _, e := range el.Items { + var interval string + if e.Count > 1 { + interval = fmt.Sprintf("%s (x%d over %s)", translateTimestampSince(e.LastTimestamp), e.Count, translateTimestampSince(e.FirstTimestamp)) + } else { + interval = translateTimestampSince(e.FirstTimestamp) + } + w.Write(LEVEL_1, "%v\t%v\t%s\t%v\t%v\n", + e.Type, + e.Reason, + interval, + formatEventSource(e.Source), + strings.TrimSpace(e.Message), + ) + } +} + +// DeploymentDescriber generates information about a deployment. +type DeploymentDescriber struct { + client clientset.Interface +} + +func (dd *DeploymentDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + d, err := dd.client.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + selector, err := metav1.LabelSelectorAsSelector(d.Spec.Selector) + if err != nil { + return "", err + } + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = dd.client.CoreV1().Events(namespace).Search(scheme.Scheme, d) + } + + return describeDeployment(d, selector, d, events, dd) +} + +func describeDeployment(d *appsv1.Deployment, selector labels.Selector, internalDeployment *appsv1.Deployment, events *corev1.EventList, dd *DeploymentDescriber) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", d.ObjectMeta.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", d.ObjectMeta.Namespace) + w.Write(LEVEL_0, "CreationTimestamp:\t%s\n", d.CreationTimestamp.Time.Format(time.RFC1123Z)) + printLabelsMultiline(w, "Labels", d.Labels) + printAnnotationsMultiline(w, "Annotations", d.Annotations) + w.Write(LEVEL_0, "Selector:\t%s\n", selector) + w.Write(LEVEL_0, "Replicas:\t%d desired | %d updated | %d total | %d available | %d unavailable\n", *(d.Spec.Replicas), d.Status.UpdatedReplicas, d.Status.Replicas, d.Status.AvailableReplicas, d.Status.UnavailableReplicas) + w.Write(LEVEL_0, "StrategyType:\t%s\n", d.Spec.Strategy.Type) + w.Write(LEVEL_0, "MinReadySeconds:\t%d\n", d.Spec.MinReadySeconds) + if d.Spec.Strategy.RollingUpdate != nil { + ru := d.Spec.Strategy.RollingUpdate + w.Write(LEVEL_0, "RollingUpdateStrategy:\t%s max unavailable, %s max surge\n", ru.MaxUnavailable.String(), ru.MaxSurge.String()) + } + DescribePodTemplate(&internalDeployment.Spec.Template, w) + if len(d.Status.Conditions) > 0 { + w.Write(LEVEL_0, "Conditions:\n Type\tStatus\tReason\n") + w.Write(LEVEL_1, "----\t------\t------\n") + for _, c := range d.Status.Conditions { + w.Write(LEVEL_1, "%v \t%v\t%v\n", c.Type, c.Status, c.Reason) + } + } + oldRSs, _, newRS, err := deploymentutil.GetAllReplicaSets(d, dd.client.AppsV1()) + if err == nil { + w.Write(LEVEL_0, "OldReplicaSets:\t%s\n", printReplicaSetsByLabels(oldRSs)) + var newRSs []*appsv1.ReplicaSet + if newRS != nil { + newRSs = append(newRSs, newRS) + } + w.Write(LEVEL_0, "NewReplicaSet:\t%s\n", printReplicaSetsByLabels(newRSs)) + } + if events != nil { + DescribeEvents(events, w) + } + + return nil + }) +} + +func printReplicaSetsByLabels(matchingRSs []*appsv1.ReplicaSet) string { + // Format the matching ReplicaSets into strings. + rsStrings := make([]string, 0, len(matchingRSs)) + for _, rs := range matchingRSs { + rsStrings = append(rsStrings, fmt.Sprintf("%s (%d/%d replicas created)", rs.Name, rs.Status.Replicas, *rs.Spec.Replicas)) + } + + list := strings.Join(rsStrings, ", ") + if list == "" { + return "" + } + return list +} + +func getPodStatusForController(c corev1client.PodInterface, selector labels.Selector, uid types.UID) (running, waiting, succeeded, failed int, err error) { + options := metav1.ListOptions{LabelSelector: selector.String()} + rcPods, err := c.List(options) + if err != nil { + return + } + for _, pod := range rcPods.Items { + controllerRef := metav1.GetControllerOf(&pod) + // Skip pods that are orphans or owned by other controllers. + if controllerRef == nil || controllerRef.UID != uid { + continue + } + switch pod.Status.Phase { + case corev1.PodRunning: + running++ + case corev1.PodPending: + waiting++ + case corev1.PodSucceeded: + succeeded++ + case corev1.PodFailed: + failed++ + } + } + return +} + +// ConfigMapDescriber generates information about a ConfigMap +type ConfigMapDescriber struct { + clientset.Interface +} + +func (d *ConfigMapDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + c := d.Core().ConfigMaps(namespace) + + configMap, err := c.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", configMap.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", configMap.Namespace) + printLabelsMultiline(w, "Labels", configMap.Labels) + printAnnotationsMultiline(w, "Annotations", configMap.Annotations) + + w.Write(LEVEL_0, "\nData\n====\n") + for k, v := range configMap.Data { + w.Write(LEVEL_0, "%s:\n----\n", k) + w.Write(LEVEL_0, "%s\n", string(v)) + } + if describerSettings.ShowEvents { + events, err := d.Core().Events(namespace).Search(scheme.Scheme, configMap) + if err != nil { + return err + } + if events != nil { + DescribeEvents(events, w) + } + } + return nil + }) +} + +// NetworkPolicyDescriber generates information about a networkingv1.NetworkPolicy +type NetworkPolicyDescriber struct { + clientset.Interface +} + +func (d *NetworkPolicyDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + c := d.Networking().NetworkPolicies(namespace) + + networkPolicy, err := c.Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + return describeNetworkPolicy(networkPolicy) +} + +func describeNetworkPolicy(networkPolicy *networkingv1.NetworkPolicy) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", networkPolicy.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", networkPolicy.Namespace) + w.Write(LEVEL_0, "Created on:\t%s\n", networkPolicy.CreationTimestamp) + printLabelsMultiline(w, "Labels", networkPolicy.Labels) + printAnnotationsMultiline(w, "Annotations", networkPolicy.Annotations) + describeNetworkPolicySpec(networkPolicy.Spec, w) + return nil + }) +} + +func describeNetworkPolicySpec(nps networkingv1.NetworkPolicySpec, w PrefixWriter) { + w.Write(LEVEL_0, "Spec:\n") + w.Write(LEVEL_1, "PodSelector: ") + if len(nps.PodSelector.MatchLabels) == 0 && len(nps.PodSelector.MatchExpressions) == 0 { + w.Write(LEVEL_2, " (Allowing the specific traffic to all pods in this namespace)\n") + } else { + w.Write(LEVEL_2, "%s\n", metav1.FormatLabelSelector(&nps.PodSelector)) + } + w.Write(LEVEL_1, "Allowing ingress traffic:\n") + printNetworkPolicySpecIngressFrom(nps.Ingress, " ", w) + w.Write(LEVEL_1, "Allowing egress traffic:\n") + printNetworkPolicySpecEgressTo(nps.Egress, " ", w) + w.Write(LEVEL_1, "Policy Types: %v\n", policyTypesToString(nps.PolicyTypes)) +} + +func printNetworkPolicySpecIngressFrom(npirs []networkingv1.NetworkPolicyIngressRule, initialIndent string, w PrefixWriter) { + if len(npirs) == 0 { + w.Write(LEVEL_0, "%s%s\n", initialIndent, " (Selected pods are isolated for ingress connectivity)") + return + } + for i, npir := range npirs { + if len(npir.Ports) == 0 { + w.Write(LEVEL_0, "%s%s\n", initialIndent, "To Port: (traffic allowed to all ports)") + } else { + for _, port := range npir.Ports { + var proto corev1.Protocol + if port.Protocol != nil { + proto = *port.Protocol + } else { + proto = corev1.ProtocolTCP + } + w.Write(LEVEL_0, "%s%s: %s/%s\n", initialIndent, "To Port", port.Port, proto) + } + } + if len(npir.From) == 0 { + w.Write(LEVEL_0, "%s%s\n", initialIndent, "From: (traffic not restricted by source)") + } else { + for _, from := range npir.From { + w.Write(LEVEL_0, "%s%s\n", initialIndent, "From:") + if from.PodSelector != nil && from.NamespaceSelector != nil { + w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "NamespaceSelector", metav1.FormatLabelSelector(from.NamespaceSelector)) + w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "PodSelector", metav1.FormatLabelSelector(from.PodSelector)) + } else if from.PodSelector != nil { + w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "PodSelector", metav1.FormatLabelSelector(from.PodSelector)) + } else if from.NamespaceSelector != nil { + w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "NamespaceSelector", metav1.FormatLabelSelector(from.NamespaceSelector)) + } else if from.IPBlock != nil { + w.Write(LEVEL_1, "%sIPBlock:\n", initialIndent) + w.Write(LEVEL_2, "%sCIDR: %s\n", initialIndent, from.IPBlock.CIDR) + w.Write(LEVEL_2, "%sExcept: %v\n", initialIndent, strings.Join(from.IPBlock.Except, ", ")) + } + } + } + if i != len(npirs)-1 { + w.Write(LEVEL_0, "%s%s\n", initialIndent, "----------") + } + } +} + +func printNetworkPolicySpecEgressTo(npers []networkingv1.NetworkPolicyEgressRule, initialIndent string, w PrefixWriter) { + if len(npers) == 0 { + w.Write(LEVEL_0, "%s%s\n", initialIndent, " (Selected pods are isolated for egress connectivity)") + return + } + for i, nper := range npers { + if len(nper.Ports) == 0 { + w.Write(LEVEL_0, "%s%s\n", initialIndent, "To Port: (traffic allowed to all ports)") + } else { + for _, port := range nper.Ports { + var proto corev1.Protocol + if port.Protocol != nil { + proto = *port.Protocol + } else { + proto = corev1.ProtocolTCP + } + w.Write(LEVEL_0, "%s%s: %s/%s\n", initialIndent, "To Port", port.Port, proto) + } + } + if len(nper.To) == 0 { + w.Write(LEVEL_0, "%s%s\n", initialIndent, "To: (traffic not restricted by source)") + } else { + for _, to := range nper.To { + w.Write(LEVEL_0, "%s%s\n", initialIndent, "To:") + if to.PodSelector != nil && to.NamespaceSelector != nil { + w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "NamespaceSelector", metav1.FormatLabelSelector(to.NamespaceSelector)) + w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "PodSelector", metav1.FormatLabelSelector(to.PodSelector)) + } else if to.PodSelector != nil { + w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "PodSelector", metav1.FormatLabelSelector(to.PodSelector)) + } else if to.NamespaceSelector != nil { + w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "NamespaceSelector", metav1.FormatLabelSelector(to.NamespaceSelector)) + } else if to.IPBlock != nil { + w.Write(LEVEL_1, "%sIPBlock:\n", initialIndent) + w.Write(LEVEL_2, "%sCIDR: %s\n", initialIndent, to.IPBlock.CIDR) + w.Write(LEVEL_2, "%sExcept: %v\n", initialIndent, strings.Join(to.IPBlock.Except, ", ")) + } + } + } + if i != len(npers)-1 { + w.Write(LEVEL_0, "%s%s\n", initialIndent, "----------") + } + } +} + +type StorageClassDescriber struct { + clientset.Interface +} + +func (s *StorageClassDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + sc, err := s.Storage().StorageClasses().Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = s.Core().Events(namespace).Search(scheme.Scheme, sc) + } + + return describeStorageClass(sc, events) +} + +func describeStorageClass(sc *storagev1.StorageClass, events *corev1.EventList) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", sc.Name) + w.Write(LEVEL_0, "IsDefaultClass:\t%s\n", storageutil.IsDefaultAnnotationText(sc.ObjectMeta)) + w.Write(LEVEL_0, "Annotations:\t%s\n", labels.FormatLabels(sc.Annotations)) + w.Write(LEVEL_0, "Provisioner:\t%s\n", sc.Provisioner) + w.Write(LEVEL_0, "Parameters:\t%s\n", labels.FormatLabels(sc.Parameters)) + w.Write(LEVEL_0, "AllowVolumeExpansion:\t%s\n", printBoolPtr(sc.AllowVolumeExpansion)) + if len(sc.MountOptions) == 0 { + w.Write(LEVEL_0, "MountOptions:\t\n") + } else { + w.Write(LEVEL_0, "MountOptions:\n") + for _, option := range sc.MountOptions { + w.Write(LEVEL_1, "%s\n", option) + } + } + if sc.ReclaimPolicy != nil { + w.Write(LEVEL_0, "ReclaimPolicy:\t%s\n", *sc.ReclaimPolicy) + } + if sc.VolumeBindingMode != nil { + w.Write(LEVEL_0, "VolumeBindingMode:\t%s\n", *sc.VolumeBindingMode) + } + if sc.AllowedTopologies != nil { + printAllowedTopologies(w, sc.AllowedTopologies) + } + if events != nil { + DescribeEvents(events, w) + } + + return nil + }) +} + +func printAllowedTopologies(w PrefixWriter, topologies []corev1.TopologySelectorTerm) { + w.Write(LEVEL_0, "AllowedTopologies:\t") + if len(topologies) == 0 { + w.WriteLine("") + return + } + w.WriteLine("") + for i, term := range topologies { + printTopologySelectorTermsMultilineWithIndent(w, LEVEL_1, fmt.Sprintf("Term %d", i), "\t", term.MatchLabelExpressions) + } +} + +func printTopologySelectorTermsMultilineWithIndent(w PrefixWriter, indentLevel int, title, innerIndent string, reqs []corev1.TopologySelectorLabelRequirement) { + w.Write(indentLevel, "%s:%s", title, innerIndent) + + if len(reqs) == 0 { + w.WriteLine("") + return + } + + for i, req := range reqs { + if i != 0 { + w.Write(indentLevel, "%s", innerIndent) + } + exprStr := fmt.Sprintf("%s %s", req.Key, "in") + if len(req.Values) > 0 { + exprStr = fmt.Sprintf("%s [%s]", exprStr, strings.Join(req.Values, ", ")) + } + w.Write(LEVEL_0, "%s\n", exprStr) + } +} + +type PodDisruptionBudgetDescriber struct { + clientset.Interface +} + +func (p *PodDisruptionBudgetDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + pdb, err := p.Policy().PodDisruptionBudgets(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = p.Core().Events(namespace).Search(scheme.Scheme, pdb) + } + + return describePodDisruptionBudget(pdb, events) +} + +func describePodDisruptionBudget(pdb *policyv1beta1.PodDisruptionBudget, events *corev1.EventList) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", pdb.Name) + w.Write(LEVEL_0, "Namespace:\t%s\n", pdb.Namespace) + + if pdb.Spec.MinAvailable != nil { + w.Write(LEVEL_0, "Min available:\t%s\n", pdb.Spec.MinAvailable.String()) + } else if pdb.Spec.MaxUnavailable != nil { + w.Write(LEVEL_0, "Max unavailable:\t%s\n", pdb.Spec.MaxUnavailable.String()) + } + + if pdb.Spec.Selector != nil { + w.Write(LEVEL_0, "Selector:\t%s\n", metav1.FormatLabelSelector(pdb.Spec.Selector)) + } else { + w.Write(LEVEL_0, "Selector:\t\n") + } + w.Write(LEVEL_0, "Status:\n") + w.Write(LEVEL_2, "Allowed disruptions:\t%d\n", pdb.Status.PodDisruptionsAllowed) + w.Write(LEVEL_2, "Current:\t%d\n", pdb.Status.CurrentHealthy) + w.Write(LEVEL_2, "Desired:\t%d\n", pdb.Status.DesiredHealthy) + w.Write(LEVEL_2, "Total:\t%d\n", pdb.Status.ExpectedPods) + if events != nil { + DescribeEvents(events, w) + } + + return nil + }) +} + +// PriorityClassDescriber generates information about a PriorityClass. +type PriorityClassDescriber struct { + clientset.Interface +} + +func (s *PriorityClassDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + pc, err := s.Scheduling().PriorityClasses().Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + var events *corev1.EventList + if describerSettings.ShowEvents { + events, _ = s.Core().Events(namespace).Search(scheme.Scheme, pc) + } + + return describePriorityClass(pc, events) +} + +func describePriorityClass(pc *schedulingv1beta1.PriorityClass, events *corev1.EventList) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", pc.Name) + w.Write(LEVEL_0, "Value:\t%v\n", pc.Value) + w.Write(LEVEL_0, "GlobalDefault:\t%v\n", pc.GlobalDefault) + w.Write(LEVEL_0, "Description:\t%s\n", pc.Description) + + w.Write(LEVEL_0, "Annotations:\t%s\n", labels.FormatLabels(pc.Annotations)) + if events != nil { + DescribeEvents(events, w) + } + + return nil + }) +} + +// PodSecurityPolicyDescriber generates information about a PodSecuritypolicyv1beta1. +type PodSecurityPolicyDescriber struct { + clientset.Interface +} + +func (d *PodSecurityPolicyDescriber) Describe(namespace, name string, describerSettings describe.DescriberSettings) (string, error) { + psp, err := d.Policy().PodSecurityPolicies().Get(name, metav1.GetOptions{}) + if err != nil { + return "", err + } + + return describePodSecurityPolicy(psp) +} + +func describePodSecurityPolicy(psp *policyv1beta1.PodSecurityPolicy) (string, error) { + return tabbedString(func(out io.Writer) error { + w := NewPrefixWriter(out) + w.Write(LEVEL_0, "Name:\t%s\n", psp.Name) + + w.Write(LEVEL_0, "\nSettings:\n") + + w.Write(LEVEL_1, "Allow Privileged:\t%t\n", psp.Spec.Privileged) + w.Write(LEVEL_1, "Allow Privilege Escalation:\t%v\n", psp.Spec.AllowPrivilegeEscalation) + w.Write(LEVEL_1, "Default Add Capabilities:\t%v\n", capsToString(psp.Spec.DefaultAddCapabilities)) + w.Write(LEVEL_1, "Required Drop Capabilities:\t%s\n", capsToString(psp.Spec.RequiredDropCapabilities)) + w.Write(LEVEL_1, "Allowed Capabilities:\t%s\n", capsToString(psp.Spec.AllowedCapabilities)) + w.Write(LEVEL_1, "Allowed Volume Types:\t%s\n", fsTypeToString(psp.Spec.Volumes)) + + if len(psp.Spec.AllowedFlexVolumes) > 0 { + w.Write(LEVEL_1, "Allowed FlexVolume Types:\t%s\n", flexVolumesToString(psp.Spec.AllowedFlexVolumes)) + } + if len(psp.Spec.AllowedUnsafeSysctls) > 0 { + w.Write(LEVEL_1, "Allowed Unsafe Sysctls:\t%s\n", sysctlsToString(psp.Spec.AllowedUnsafeSysctls)) + } + if len(psp.Spec.ForbiddenSysctls) > 0 { + w.Write(LEVEL_1, "Forbidden Sysctls:\t%s\n", sysctlsToString(psp.Spec.ForbiddenSysctls)) + } + w.Write(LEVEL_1, "Allow Host Network:\t%t\n", psp.Spec.HostNetwork) + w.Write(LEVEL_1, "Allow Host Ports:\t%s\n", hostPortRangeToString(psp.Spec.HostPorts)) + w.Write(LEVEL_1, "Allow Host PID:\t%t\n", psp.Spec.HostPID) + w.Write(LEVEL_1, "Allow Host IPC:\t%t\n", psp.Spec.HostIPC) + w.Write(LEVEL_1, "Read Only Root Filesystem:\t%v\n", psp.Spec.ReadOnlyRootFilesystem) + + w.Write(LEVEL_1, "SELinux Context Strategy: %s\t\n", string(psp.Spec.SELinux.Rule)) + var user, role, seLinuxType, level string + if psp.Spec.SELinux.SELinuxOptions != nil { + user = psp.Spec.SELinux.SELinuxOptions.User + role = psp.Spec.SELinux.SELinuxOptions.Role + seLinuxType = psp.Spec.SELinux.SELinuxOptions.Type + level = psp.Spec.SELinux.SELinuxOptions.Level + } + w.Write(LEVEL_2, "User:\t%s\n", stringOrNone(user)) + w.Write(LEVEL_2, "Role:\t%s\n", stringOrNone(role)) + w.Write(LEVEL_2, "Type:\t%s\n", stringOrNone(seLinuxType)) + w.Write(LEVEL_2, "Level:\t%s\n", stringOrNone(level)) + + w.Write(LEVEL_1, "Run As User Strategy: %s\t\n", string(psp.Spec.RunAsUser.Rule)) + w.Write(LEVEL_2, "Ranges:\t%s\n", idRangeToString(psp.Spec.RunAsUser.Ranges)) + + w.Write(LEVEL_1, "FSGroup Strategy: %s\t\n", string(psp.Spec.FSGroup.Rule)) + w.Write(LEVEL_2, "Ranges:\t%s\n", idRangeToString(psp.Spec.FSGroup.Ranges)) + + w.Write(LEVEL_1, "Supplemental Groups Strategy: %s\t\n", string(psp.Spec.SupplementalGroups.Rule)) + w.Write(LEVEL_2, "Ranges:\t%s\n", idRangeToString(psp.Spec.SupplementalGroups.Ranges)) + + return nil + }) +} + +func stringOrNone(s string) string { + return stringOrDefaultValue(s, "") +} + +func stringOrDefaultValue(s, defaultValue string) string { + if len(s) > 0 { + return s + } + return defaultValue +} + +func fsTypeToString(volumes []policyv1beta1.FSType) string { + strVolumes := []string{} + for _, v := range volumes { + strVolumes = append(strVolumes, string(v)) + } + return stringOrNone(strings.Join(strVolumes, ",")) +} + +func flexVolumesToString(flexVolumes []policyv1beta1.AllowedFlexVolume) string { + volumes := []string{} + for _, flexVolume := range flexVolumes { + volumes = append(volumes, "driver="+flexVolume.Driver) + } + return stringOrDefaultValue(strings.Join(volumes, ","), "") +} + +func sysctlsToString(sysctls []string) string { + return stringOrNone(strings.Join(sysctls, ",")) +} + +func hostPortRangeToString(ranges []policyv1beta1.HostPortRange) string { + formattedString := "" + if ranges != nil { + strRanges := []string{} + for _, r := range ranges { + strRanges = append(strRanges, fmt.Sprintf("%d-%d", r.Min, r.Max)) + } + formattedString = strings.Join(strRanges, ",") + } + return stringOrNone(formattedString) +} + +func idRangeToString(ranges []policyv1beta1.IDRange) string { + formattedString := "" + if ranges != nil { + strRanges := []string{} + for _, r := range ranges { + strRanges = append(strRanges, fmt.Sprintf("%d-%d", r.Min, r.Max)) + } + formattedString = strings.Join(strRanges, ",") + } + return stringOrNone(formattedString) +} + +func capsToString(caps []corev1.Capability) string { + formattedString := "" + if caps != nil { + strCaps := []string{} + for _, c := range caps { + strCaps = append(strCaps, string(c)) + } + formattedString = strings.Join(strCaps, ",") + } + return stringOrNone(formattedString) +} + +func policyTypesToString(pts []networkingv1.PolicyType) string { + formattedString := "" + if pts != nil { + strPts := []string{} + for _, p := range pts { + strPts = append(strPts, string(p)) + } + formattedString = strings.Join(strPts, ", ") + } + return stringOrNone(formattedString) +} + +// newErrNoDescriber creates a new ErrNoDescriber with the names of the provided types. +func newErrNoDescriber(types ...reflect.Type) error { + names := make([]string, 0, len(types)) + for _, t := range types { + names = append(names, t.String()) + } + return describe.ErrNoDescriber{Types: names} +} + +// Describers implements ObjectDescriber against functions registered via Add. Those functions can +// be strongly typed. Types are exactly matched (no conversion or assignable checks). +type Describers struct { + searchFns map[reflect.Type][]typeFunc +} + +// DescribeObject implements ObjectDescriber and will attempt to print the provided object to a string, +// if at least one describer function has been registered with the exact types passed, or if any +// describer can print the exact object in its first argument (the remainder will be provided empty +// values). If no function registered with Add can satisfy the passed objects, an ErrNoDescriber will +// be returned +// TODO: reorder and partial match extra. +func (d *Describers) DescribeObject(exact interface{}, extra ...interface{}) (string, error) { + exactType := reflect.TypeOf(exact) + fns, ok := d.searchFns[exactType] + if !ok { + return "", newErrNoDescriber(exactType) + } + if len(extra) == 0 { + for _, typeFn := range fns { + if len(typeFn.Extra) == 0 { + return typeFn.Describe(exact, extra...) + } + } + typeFn := fns[0] + for _, t := range typeFn.Extra { + v := reflect.New(t).Elem() + extra = append(extra, v.Interface()) + } + return fns[0].Describe(exact, extra...) + } + + types := make([]reflect.Type, 0, len(extra)) + for _, obj := range extra { + types = append(types, reflect.TypeOf(obj)) + } + for _, typeFn := range fns { + if typeFn.Matches(types) { + return typeFn.Describe(exact, extra...) + } + } + return "", newErrNoDescriber(append([]reflect.Type{exactType}, types...)...) +} + +// Add adds one or more describer functions to the describe.Describer. The passed function must +// match the signature: +// +// func(...) (string, error) +// +// Any number of arguments may be provided. +func (d *Describers) Add(fns ...interface{}) error { + for _, fn := range fns { + fv := reflect.ValueOf(fn) + ft := fv.Type() + if ft.Kind() != reflect.Func { + return fmt.Errorf("expected func, got: %v", ft) + } + numIn := ft.NumIn() + if numIn == 0 { + return fmt.Errorf("expected at least one 'in' params, got: %v", ft) + } + if ft.NumOut() != 2 { + return fmt.Errorf("expected two 'out' params - (string, error), got: %v", ft) + } + types := make([]reflect.Type, 0, numIn) + for i := 0; i < numIn; i++ { + types = append(types, ft.In(i)) + } + if ft.Out(0) != reflect.TypeOf(string("")) { + return fmt.Errorf("expected string return, got: %v", ft) + } + var forErrorType error + // This convolution is necessary, otherwise TypeOf picks up on the fact + // that forErrorType is nil. + errorType := reflect.TypeOf(&forErrorType).Elem() + if ft.Out(1) != errorType { + return fmt.Errorf("expected error return, got: %v", ft) + } + + exact := types[0] + extra := types[1:] + if d.searchFns == nil { + d.searchFns = make(map[reflect.Type][]typeFunc) + } + fns := d.searchFns[exact] + fn := typeFunc{Extra: extra, Fn: fv} + fns = append(fns, fn) + d.searchFns[exact] = fns + } + return nil +} + +// typeFunc holds information about a describer function and the types it accepts +type typeFunc struct { + Extra []reflect.Type + Fn reflect.Value +} + +// Matches returns true when the passed types exactly match the Extra list. +func (fn typeFunc) Matches(types []reflect.Type) bool { + if len(fn.Extra) != len(types) { + return false + } + // reorder the items in array types and fn.Extra + // convert the type into string and sort them, check if they are matched + varMap := make(map[reflect.Type]bool) + for i := range fn.Extra { + varMap[fn.Extra[i]] = true + } + for i := range types { + if _, found := varMap[types[i]]; !found { + return false + } + } + return true +} + +// Describe invokes the nested function with the exact number of arguments. +func (fn typeFunc) Describe(exact interface{}, extra ...interface{}) (string, error) { + values := []reflect.Value{reflect.ValueOf(exact)} + for _, obj := range extra { + values = append(values, reflect.ValueOf(obj)) + } + out := fn.Fn.Call(values) + s := out[0].Interface().(string) + var err error + if !out[1].IsNil() { + err = out[1].Interface().(error) + } + return s, err +} + +// printLabelsMultiline prints multiple labels with a proper alignment. +func printLabelsMultiline(w PrefixWriter, title string, labels map[string]string) { + printLabelsMultilineWithIndent(w, "", title, "\t", labels, sets.NewString()) +} + +// printLabelsMultiline prints multiple labels with a user-defined alignment. +func printLabelsMultilineWithIndent(w PrefixWriter, initialIndent, title, innerIndent string, labels map[string]string, skip sets.String) { + w.Write(LEVEL_0, "%s%s:%s", initialIndent, title, innerIndent) + + if labels == nil || len(labels) == 0 { + w.WriteLine("") + return + } + + // to print labels in the sorted order + keys := make([]string, 0, len(labels)) + for key := range labels { + if skip.Has(key) { + continue + } + keys = append(keys, key) + } + if len(keys) == 0 { + w.WriteLine("") + return + } + sort.Strings(keys) + + for i, key := range keys { + if i != 0 { + w.Write(LEVEL_0, "%s", initialIndent) + w.Write(LEVEL_0, "%s", innerIndent) + } + w.Write(LEVEL_0, "%s=%s\n", key, labels[key]) + i++ + } +} + +// printTaintsMultiline prints multiple taints with a proper alignment. +func printNodeTaintsMultiline(w PrefixWriter, title string, taints []corev1.Taint) { + printTaintsMultilineWithIndent(w, "", title, "\t", taints) +} + +// printTaintsMultilineWithIndent prints multiple taints with a user-defined alignment. +func printTaintsMultilineWithIndent(w PrefixWriter, initialIndent, title, innerIndent string, taints []corev1.Taint) { + w.Write(LEVEL_0, "%s%s:%s", initialIndent, title, innerIndent) + + if taints == nil || len(taints) == 0 { + w.WriteLine("") + return + } + + // to print taints in the sorted order + sort.Slice(taints, func(i, j int) bool { + cmpKey := func(taint corev1.Taint) string { + return string(taint.Effect) + "," + taint.Key + } + return cmpKey(taints[i]) < cmpKey(taints[j]) + }) + + for i, taint := range taints { + if i != 0 { + w.Write(LEVEL_0, "%s", initialIndent) + w.Write(LEVEL_0, "%s", innerIndent) + } + w.Write(LEVEL_0, "%s\n", taint.ToString()) + } +} + +// printPodsMultiline prints multiple pods with a proper alignment. +func printPodsMultiline(w PrefixWriter, title string, pods []corev1.Pod) { + printPodsMultilineWithIndent(w, "", title, "\t", pods) +} + +// printPodsMultilineWithIndent prints multiple pods with a user-defined alignment. +func printPodsMultilineWithIndent(w PrefixWriter, initialIndent, title, innerIndent string, pods []corev1.Pod) { + w.Write(LEVEL_0, "%s%s:%s", initialIndent, title, innerIndent) + + if pods == nil || len(pods) == 0 { + w.WriteLine("") + return + } + + // to print pods in the sorted order + sort.Slice(pods, func(i, j int) bool { + cmpKey := func(pod corev1.Pod) string { + return pod.Name + } + return cmpKey(pods[i]) < cmpKey(pods[j]) + }) + + for i, pod := range pods { + if i != 0 { + w.Write(LEVEL_0, "%s", initialIndent) + w.Write(LEVEL_0, "%s", innerIndent) + } + w.Write(LEVEL_0, "%s\n", pod.Name) + } +} + +// printPodTolerationsMultiline prints multiple tolerations with a proper alignment. +func printPodTolerationsMultiline(w PrefixWriter, title string, tolerations []corev1.Toleration) { + printTolerationsMultilineWithIndent(w, "", title, "\t", tolerations) +} + +// printTolerationsMultilineWithIndent prints multiple tolerations with a user-defined alignment. +func printTolerationsMultilineWithIndent(w PrefixWriter, initialIndent, title, innerIndent string, tolerations []corev1.Toleration) { + w.Write(LEVEL_0, "%s%s:%s", initialIndent, title, innerIndent) + + if tolerations == nil || len(tolerations) == 0 { + w.WriteLine("") + return + } + + // to print tolerations in the sorted order + sort.Slice(tolerations, func(i, j int) bool { + return tolerations[i].Key < tolerations[j].Key + }) + + for i, toleration := range tolerations { + if i != 0 { + w.Write(LEVEL_0, "%s", initialIndent) + w.Write(LEVEL_0, "%s", innerIndent) + } + w.Write(LEVEL_0, "%s", toleration.Key) + if len(toleration.Value) != 0 { + w.Write(LEVEL_0, "=%s", toleration.Value) + } + if len(toleration.Effect) != 0 { + w.Write(LEVEL_0, ":%s", toleration.Effect) + } + if toleration.TolerationSeconds != nil { + w.Write(LEVEL_0, " for %ds", *toleration.TolerationSeconds) + } + w.Write(LEVEL_0, "\n") + } +} + +type flusher interface { + Flush() +} + +func tabbedString(f func(io.Writer) error) (string, error) { + out := new(tabwriter.Writer) + buf := &bytes.Buffer{} + out.Init(buf, 0, 8, 2, ' ', 0) + + err := f(out) + if err != nil { + return "", err + } + + out.Flush() + str := string(buf.String()) + return str, nil +} + +type SortableResourceNames []corev1.ResourceName + +func (list SortableResourceNames) Len() int { + return len(list) +} + +func (list SortableResourceNames) Swap(i, j int) { + list[i], list[j] = list[j], list[i] +} + +func (list SortableResourceNames) Less(i, j int) bool { + return list[i] < list[j] +} + +// SortedResourceNames returns the sorted resource names of a resource list. +func SortedResourceNames(list corev1.ResourceList) []corev1.ResourceName { + resources := make([]corev1.ResourceName, 0, len(list)) + for res := range list { + resources = append(resources, res) + } + sort.Sort(SortableResourceNames(resources)) + return resources +} + +type SortableResourceQuotas []corev1.ResourceQuota + +func (list SortableResourceQuotas) Len() int { + return len(list) +} + +func (list SortableResourceQuotas) Swap(i, j int) { + list[i], list[j] = list[j], list[i] +} + +func (list SortableResourceQuotas) Less(i, j int) bool { + return list[i].Name < list[j].Name +} + +type SortableVolumeMounts []corev1.VolumeMount + +func (list SortableVolumeMounts) Len() int { + return len(list) +} + +func (list SortableVolumeMounts) Swap(i, j int) { + list[i], list[j] = list[j], list[i] +} + +func (list SortableVolumeMounts) Less(i, j int) bool { + return list[i].MountPath < list[j].MountPath +} + +type SortableVolumeDevices []corev1.VolumeDevice + +func (list SortableVolumeDevices) Len() int { + return len(list) +} + +func (list SortableVolumeDevices) Swap(i, j int) { + list[i], list[j] = list[j], list[i] +} + +func (list SortableVolumeDevices) Less(i, j int) bool { + return list[i].DevicePath < list[j].DevicePath +} + +var maxAnnotationLen = 140 + +// printAnnotationsMultilineWithFilter prints filtered multiple annotations with a proper alignment. +func printAnnotationsMultilineWithFilter(w PrefixWriter, title string, annotations map[string]string, skip sets.String) { + printAnnotationsMultilineWithIndent(w, "", title, "\t", annotations, skip) +} + +// printAnnotationsMultiline prints multiple annotations with a proper alignment. +func printAnnotationsMultiline(w PrefixWriter, title string, annotations map[string]string) { + printAnnotationsMultilineWithIndent(w, "", title, "\t", annotations, sets.NewString()) +} + +// printAnnotationsMultilineWithIndent prints multiple annotations with a user-defined alignment. +// If annotation string is too long, we omit chars more than 200 length. +func printAnnotationsMultilineWithIndent(w PrefixWriter, initialIndent, title, innerIndent string, annotations map[string]string, skip sets.String) { + + w.Write(LEVEL_0, "%s%s:%s", initialIndent, title, innerIndent) + + if len(annotations) == 0 { + w.WriteLine("") + return + } + + // to print labels in the sorted order + keys := make([]string, 0, len(annotations)) + for key := range annotations { + if skip.Has(key) { + continue + } + keys = append(keys, key) + } + if len(annotations) == 0 { + w.WriteLine("") + return + } + sort.Strings(keys) + indent := initialIndent + innerIndent + for i, key := range keys { + if i != 0 { + w.Write(LEVEL_0, indent) + } + value := strings.TrimSuffix(annotations[key], "\n") + if (len(value)+len(key)+2) > maxAnnotationLen || strings.Contains(value, "\n") { + w.Write(LEVEL_0, "%s:\n", key) + for _, s := range strings.Split(value, "\n") { + w.Write(LEVEL_0, "%s %s\n", indent, shorten(s, maxAnnotationLen-2)) + } + } else { + w.Write(LEVEL_0, "%s: %s\n", key, value) + } + i++ + } +} + +func shorten(s string, maxLength int) string { + if len(s) > maxLength { + return s[:maxLength] + "..." + } + return s +} + +// translateTimestampUntil returns the elapsed time until timestamp in +// human-readable approximation. +func translateTimestampUntil(timestamp metav1.Time) string { + if timestamp.IsZero() { + return "" + } + + return duration.HumanDuration(time.Until(timestamp.Time)) +} + +// translateTimestampSince returns the elapsed time since timestamp in +// human-readable approximation. +func translateTimestampSince(timestamp metav1.Time) string { + if timestamp.IsZero() { + return "" + } + + return duration.HumanDuration(time.Since(timestamp.Time)) +} + +// formatEventSource formats EventSource as a comma separated string excluding Host when empty +func formatEventSource(es corev1.EventSource) string { + EventSourceString := []string{es.Component} + if len(es.Host) > 0 { + EventSourceString = append(EventSourceString, es.Host) + } + return strings.Join(EventSourceString, ", ") +} + +// Pass ports=nil for all ports. +func formatEndpoints(endpoints *corev1.Endpoints, ports sets.String) string { + if len(endpoints.Subsets) == 0 { + return "" + } + list := []string{} + max := 3 + more := false + count := 0 + for i := range endpoints.Subsets { + ss := &endpoints.Subsets[i] + if len(ss.Ports) == 0 { + // It's possible to have headless services with no ports. + for i := range ss.Addresses { + if len(list) == max { + more = true + } + if !more { + list = append(list, ss.Addresses[i].IP) + } + count++ + } + } else { + // "Normal" services with ports defined. + for i := range ss.Ports { + port := &ss.Ports[i] + if ports == nil || ports.Has(port.Name) { + for i := range ss.Addresses { + if len(list) == max { + more = true + } + addr := &ss.Addresses[i] + if !more { + hostPort := net.JoinHostPort(addr.IP, strconv.Itoa(int(port.Port))) + list = append(list, hostPort) + } + count++ + } + } + } + } + } + ret := strings.Join(list, ",") + if more { + return fmt.Sprintf("%s + %d more...", ret, count-max) + } + return ret +} + +func extractCSRStatus(csr *certificatesv1beta1.CertificateSigningRequest) (string, error) { + var approved, denied bool + for _, c := range csr.Status.Conditions { + switch c.Type { + case certificatesv1beta1.CertificateApproved: + approved = true + case certificatesv1beta1.CertificateDenied: + denied = true + default: + return "", fmt.Errorf("unknown csr condition %q", c) + } + } + var status string + // must be in order of presidence + if denied { + status += "Denied" + } else if approved { + status += "Approved" + } else { + status += "Pending" + } + if len(csr.Status.Certificate) > 0 { + status += ",Issued" + } + return status, nil +} + +// backendStringer behaves just like a string interface and converts the given backend to a string. +func backendStringer(backend *extensionsv1beta1.IngressBackend) string { + if backend == nil { + return "" + } + return fmt.Sprintf("%v:%v", backend.ServiceName, backend.ServicePort.String()) +} + +// findNodeRoles returns the roles of a given node. +// The roles are determined by looking for: +// * a node-role.kubernetes.io/="" label +// * a kubernetes.io/role="" label +func findNodeRoles(node *corev1.Node) []string { + roles := sets.NewString() + for k, v := range node.Labels { + switch { + case strings.HasPrefix(k, describe.LabelNodeRolePrefix): + if role := strings.TrimPrefix(k, describe.LabelNodeRolePrefix); len(role) > 0 { + roles.Insert(role) + } + + case k == describe.NodeLabelRole && v != "": + roles.Insert(v) + } + } + return roles.List() +} + +// loadBalancerStatusStringer behaves mostly like a string interface and converts the given status to a string. +// `wide` indicates whether the returned value is meant for --o=wide output. If not, it's clipped to 16 bytes. +func loadBalancerStatusStringer(s corev1.LoadBalancerStatus, wide bool) string { + ingress := s.Ingress + result := sets.NewString() + for i := range ingress { + if ingress[i].IP != "" { + result.Insert(ingress[i].IP) + } else if ingress[i].Hostname != "" { + result.Insert(ingress[i].Hostname) + } + } + + r := strings.Join(result.List(), ",") + if !wide && len(r) > describe.LoadBalancerWidth { + r = r[0:(describe.LoadBalancerWidth-3)] + "..." + } + return r +} diff --git a/pkg/printers/internalversion/describe_test.go b/pkg/kubectl/describe/versioned/describe_test.go similarity index 63% rename from pkg/printers/internalversion/describe_test.go rename to pkg/kubectl/describe/versioned/describe_test.go index ceded030e41..fff37f189eb 100644 --- a/pkg/printers/internalversion/describe_test.go +++ b/pkg/kubectl/describe/versioned/describe_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package internalversion +package versioned import ( "bytes" @@ -26,22 +26,20 @@ import ( "time" appsv1 "k8s.io/api/apps/v1" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" + storagev1 "k8s.io/api/storage/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/intstr" - versionedfake "k8s.io/client-go/kubernetes/fake" - "k8s.io/kubernetes/pkg/apis/apps" - "k8s.io/kubernetes/pkg/apis/autoscaling" - api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/apis/networking" - "k8s.io/kubernetes/pkg/apis/policy" - "k8s.io/kubernetes/pkg/apis/storage" - "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake" - "k8s.io/kubernetes/pkg/printers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/kubernetes/pkg/kubectl/describe" utilpointer "k8s.io/utils/pointer" ) @@ -49,23 +47,43 @@ type describeClient struct { T *testing.T Namespace string Err error - internalclientset.Interface + kubernetes.Interface } func TestDescribePod(t *testing.T) { deletionTimestamp := metav1.Time{Time: time.Now().UTC().AddDate(10, 0, 0)} gracePeriod := int64(1234) - fake := fake.NewSimpleClientset(&api.Pod{ + condition1 := corev1.PodConditionType("condition1") + condition2 := corev1.PodConditionType("condition2") + fake := fake.NewSimpleClientset(&corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", DeletionTimestamp: &deletionTimestamp, DeletionGracePeriodSeconds: &gracePeriod, }, + Spec: corev1.PodSpec{ + ReadinessGates: []corev1.PodReadinessGate{ + { + ConditionType: condition1, + }, + { + ConditionType: condition2, + }, + }, + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: condition1, + Status: corev1.ConditionTrue, + }, + }, + }, }) c := &describeClient{T: t, Namespace: "foo", Interface: fake} d := PodDescriber{c} - out, err := d.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: true}) + out, err := d.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -78,22 +96,22 @@ func TestDescribePod(t *testing.T) { } func TestDescribePodNode(t *testing.T) { - fake := fake.NewSimpleClientset(&api.Pod{ + fake := fake.NewSimpleClientset(&corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", }, - Spec: api.PodSpec{ + Spec: corev1.PodSpec{ NodeName: "all-in-one", }, - Status: api.PodStatus{ + Status: corev1.PodStatus{ HostIP: "127.0.0.1", NominatedNodeName: "nodeA", }, }) c := &describeClient{T: t, Namespace: "foo", Interface: fake} d := PodDescriber{c} - out, err := d.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: true}) + out, err := d.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -106,24 +124,24 @@ func TestDescribePodNode(t *testing.T) { } func TestDescribePodTolerations(t *testing.T) { - fake := fake.NewSimpleClientset(&api.Pod{ + fake := fake.NewSimpleClientset(&corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", }, - Spec: api.PodSpec{ - Tolerations: []api.Toleration{ - {Key: "key0", Operator: api.TolerationOpExists}, + Spec: corev1.PodSpec{ + Tolerations: []corev1.Toleration{ + {Key: "key0", Operator: corev1.TolerationOpExists}, {Key: "key1", Value: "value1"}, - {Key: "key2", Operator: api.TolerationOpEqual, Value: "value2", Effect: api.TaintEffectNoSchedule}, - {Key: "key3", Value: "value3", Effect: api.TaintEffectNoExecute, TolerationSeconds: &[]int64{300}[0]}, - {Key: "key4", Effect: api.TaintEffectNoExecute, TolerationSeconds: &[]int64{60}[0]}, + {Key: "key2", Operator: corev1.TolerationOpEqual, Value: "value2", Effect: corev1.TaintEffectNoSchedule}, + {Key: "key3", Value: "value3", Effect: corev1.TaintEffectNoExecute, TolerationSeconds: &[]int64{300}[0]}, + {Key: "key4", Effect: corev1.TaintEffectNoExecute, TolerationSeconds: &[]int64{60}[0]}, }, }, }) c := &describeClient{T: t, Namespace: "foo", Interface: fake} d := PodDescriber{c} - out, err := d.Describe("foo", "bar", printers.DescriberSettings{}) + out, err := d.Describe("foo", "bar", describe.DescriberSettings{}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -138,7 +156,7 @@ func TestDescribePodTolerations(t *testing.T) { } func TestDescribeSecret(t *testing.T) { - fake := fake.NewSimpleClientset(&api.Secret{ + fake := fake.NewSimpleClientset(&corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", @@ -150,7 +168,7 @@ func TestDescribeSecret(t *testing.T) { }) c := &describeClient{T: t, Namespace: "foo", Interface: fake} d := SecretDescriber{c} - out, err := d.Describe("foo", "bar", printers.DescriberSettings{}) + out, err := d.Describe("foo", "bar", describe.DescriberSettings{}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -163,14 +181,14 @@ func TestDescribeSecret(t *testing.T) { } func TestDescribeNamespace(t *testing.T) { - fake := fake.NewSimpleClientset(&api.Namespace{ + fake := fake.NewSimpleClientset(&corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "myns", }, }) c := &describeClient{T: t, Namespace: "", Interface: fake} d := NamespaceDescriber{c} - out, err := d.Describe("", "myns", printers.DescriberSettings{ShowEvents: true}) + out, err := d.Describe("", "myns", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -181,18 +199,18 @@ func TestDescribeNamespace(t *testing.T) { func TestDescribePodPriority(t *testing.T) { priority := int32(1000) - fake := fake.NewSimpleClientset(&api.Pod{ + fake := fake.NewSimpleClientset(&corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", }, - Spec: api.PodSpec{ + Spec: corev1.PodSpec{ PriorityClassName: "high-priority", Priority: &priority, }, }) c := &describeClient{T: t, Namespace: "", Interface: fake} d := PodDescriber{c} - out, err := d.Describe("", "bar", printers.DescriberSettings{ShowEvents: true}) + out, err := d.Describe("", "bar", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -202,7 +220,7 @@ func TestDescribePodPriority(t *testing.T) { } func TestDescribeConfigMap(t *testing.T) { - fake := fake.NewSimpleClientset(&api.ConfigMap{ + fake := fake.NewSimpleClientset(&corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "mycm", Namespace: "foo", @@ -214,7 +232,7 @@ func TestDescribeConfigMap(t *testing.T) { }) c := &describeClient{T: t, Namespace: "foo", Interface: fake} d := ConfigMapDescriber{c} - out, err := d.Describe("foo", "mycm", printers.DescriberSettings{ShowEvents: true}) + out, err := d.Describe("foo", "mycm", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -224,21 +242,21 @@ func TestDescribeConfigMap(t *testing.T) { } func TestDescribeLimitRange(t *testing.T) { - fake := fake.NewSimpleClientset(&api.LimitRange{ + fake := fake.NewSimpleClientset(&corev1.LimitRange{ ObjectMeta: metav1.ObjectMeta{ Name: "mylr", Namespace: "foo", }, - Spec: api.LimitRangeSpec{ - Limits: []api.LimitRangeItem{ + Spec: corev1.LimitRangeSpec{ + Limits: []corev1.LimitRangeItem{ { - Type: api.LimitTypePod, + Type: corev1.LimitTypePod, Max: getResourceList("100m", "10000Mi"), Min: getResourceList("5m", "100Mi"), MaxLimitRequestRatio: getResourceList("10", ""), }, { - Type: api.LimitTypeContainer, + Type: corev1.LimitTypeContainer, Max: getResourceList("100m", "10000Mi"), Min: getResourceList("5m", "100Mi"), Default: getResourceList("50m", "500Mi"), @@ -246,7 +264,7 @@ func TestDescribeLimitRange(t *testing.T) { MaxLimitRequestRatio: getResourceList("10", ""), }, { - Type: api.LimitTypePersistentVolumeClaim, + Type: corev1.LimitTypePersistentVolumeClaim, Max: getStorageResourceList("10Gi"), Min: getStorageResourceList("5Gi"), }, @@ -255,7 +273,7 @@ func TestDescribeLimitRange(t *testing.T) { }) c := &describeClient{T: t, Namespace: "foo", Interface: fake} d := LimitRangeDescriber{c} - out, err := d.Describe("foo", "mylr", printers.DescriberSettings{ShowEvents: true}) + out, err := d.Describe("foo", "mylr", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -268,21 +286,21 @@ func TestDescribeLimitRange(t *testing.T) { } } -func getStorageResourceList(storage string) api.ResourceList { - res := api.ResourceList{} +func getStorageResourceList(storage string) corev1.ResourceList { + res := corev1.ResourceList{} if storage != "" { - res[api.ResourceStorage] = resource.MustParse(storage) + res[corev1.ResourceStorage] = resource.MustParse(storage) } return res } -func getResourceList(cpu, memory string) api.ResourceList { - res := api.ResourceList{} +func getResourceList(cpu, memory string) corev1.ResourceList { + res := corev1.ResourceList{} if cpu != "" { - res[api.ResourceCPU] = resource.MustParse(cpu) + res[corev1.ResourceCPU] = resource.MustParse(cpu) } if memory != "" { - res[api.ResourceMemory] = resource.MustParse(memory) + res[corev1.ResourceMemory] = resource.MustParse(memory) } return res } @@ -290,22 +308,22 @@ func getResourceList(cpu, memory string) api.ResourceList { func TestDescribeService(t *testing.T) { testCases := []struct { name string - service *api.Service + service *corev1.Service expect []string }{ { name: "test1", - service: &api.Service{ + service: &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", }, - Spec: api.ServiceSpec{ - Type: api.ServiceTypeLoadBalancer, - Ports: []api.ServicePort{{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Ports: []corev1.ServicePort{{ Name: "port-tcp", Port: 8080, - Protocol: api.ProtocolTCP, + Protocol: corev1.ProtocolTCP, TargetPort: intstr.FromInt(9527), NodePort: 31111, }}, @@ -333,17 +351,17 @@ func TestDescribeService(t *testing.T) { }, { name: "test2", - service: &api.Service{ + service: &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", }, - Spec: api.ServiceSpec{ - Type: api.ServiceTypeLoadBalancer, - Ports: []api.ServicePort{{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Ports: []corev1.ServicePort{{ Name: "port-tcp", Port: 8080, - Protocol: api.ProtocolTCP, + Protocol: corev1.ProtocolTCP, TargetPort: intstr.FromString("targetPort"), NodePort: 31111, }}, @@ -375,7 +393,7 @@ func TestDescribeService(t *testing.T) { fake := fake.NewSimpleClientset(testCase.service) c := &describeClient{T: t, Namespace: "foo", Interface: fake} d := ServiceDescriber{c} - out, err := d.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: true}) + out, err := d.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -391,44 +409,44 @@ func TestDescribeService(t *testing.T) { func TestPodDescribeResultsSorted(t *testing.T) { // Arrange fake := fake.NewSimpleClientset( - &api.EventList{ - Items: []api.Event{ + &corev1.EventList{ + Items: []corev1.Event{ { ObjectMeta: metav1.ObjectMeta{Name: "one"}, - Source: api.EventSource{Component: "kubelet"}, + Source: corev1.EventSource{Component: "kubelet"}, Message: "Item 1", FirstTimestamp: metav1.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)), LastTimestamp: metav1.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)), Count: 1, - Type: api.EventTypeNormal, + Type: corev1.EventTypeNormal, }, { ObjectMeta: metav1.ObjectMeta{Name: "two"}, - Source: api.EventSource{Component: "scheduler"}, + Source: corev1.EventSource{Component: "scheduler"}, Message: "Item 2", FirstTimestamp: metav1.NewTime(time.Date(1987, time.June, 17, 0, 0, 0, 0, time.UTC)), LastTimestamp: metav1.NewTime(time.Date(1987, time.June, 17, 0, 0, 0, 0, time.UTC)), Count: 1, - Type: api.EventTypeNormal, + Type: corev1.EventTypeNormal, }, { ObjectMeta: metav1.ObjectMeta{Name: "three"}, - Source: api.EventSource{Component: "kubelet"}, + Source: corev1.EventSource{Component: "kubelet"}, Message: "Item 3", FirstTimestamp: metav1.NewTime(time.Date(2002, time.December, 25, 0, 0, 0, 0, time.UTC)), LastTimestamp: metav1.NewTime(time.Date(2002, time.December, 25, 0, 0, 0, 0, time.UTC)), Count: 1, - Type: api.EventTypeNormal, + Type: corev1.EventTypeNormal, }, }, }, - &api.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}}, + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}}, ) c := &describeClient{T: t, Namespace: "foo", Interface: fake} d := PodDescriber{c} // Act - out, err := d.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: true}) + out, err := d.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: true}) // Assert if err != nil { @@ -464,17 +482,17 @@ func VerifyDatesInOrder( func TestDescribeContainers(t *testing.T) { trueVal := true testCases := []struct { - container api.Container - status api.ContainerStatus + container corev1.Container + status corev1.ContainerStatus expectedElements []string }{ // Running state. { - container: api.Container{Name: "test", Image: "image"}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image"}, + status: corev1.ContainerStatus{ Name: "test", - State: api.ContainerState{ - Running: &api.ContainerStateRunning{ + State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, @@ -485,11 +503,11 @@ func TestDescribeContainers(t *testing.T) { }, // Waiting state. { - container: api.Container{Name: "test", Image: "image"}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image"}, + status: corev1.ContainerStatus{ Name: "test", - State: api.ContainerState{ - Waiting: &api.ContainerStateWaiting{ + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ Reason: "potato", }, }, @@ -500,11 +518,11 @@ func TestDescribeContainers(t *testing.T) { }, // Terminated state. { - container: api.Container{Name: "test", Image: "image"}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image"}, + status: corev1.ContainerStatus{ Name: "test", - State: api.ContainerState{ - Terminated: &api.ContainerStateTerminated{ + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ StartedAt: metav1.NewTime(time.Now()), FinishedAt: metav1.NewTime(time.Now()), Reason: "potato", @@ -518,16 +536,16 @@ func TestDescribeContainers(t *testing.T) { }, // Last Terminated { - container: api.Container{Name: "test", Image: "image"}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image"}, + status: corev1.ContainerStatus{ Name: "test", - State: api.ContainerState{ - Running: &api.ContainerStateRunning{ + State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{ StartedAt: metav1.NewTime(time.Now()), }, }, - LastTerminationState: api.ContainerState{ - Terminated: &api.ContainerStateTerminated{ + LastTerminationState: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ StartedAt: metav1.NewTime(time.Now().Add(time.Second * 3)), FinishedAt: metav1.NewTime(time.Now()), Reason: "crashing", @@ -541,8 +559,8 @@ func TestDescribeContainers(t *testing.T) { }, // No state defaults to waiting. { - container: api.Container{Name: "test", Image: "image"}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image"}, + status: corev1.ContainerStatus{ Name: "test", Ready: true, RestartCount: 7, @@ -551,8 +569,8 @@ func TestDescribeContainers(t *testing.T) { }, // Env { - container: api.Container{Name: "test", Image: "image", Env: []api.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []api.EnvFromSource{{ConfigMapRef: &api.ConfigMapEnvSource{LocalObjectReference: api.LocalObjectReference{Name: "a123"}}}}}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image", Env: []corev1.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "a123"}}}}}, + status: corev1.ContainerStatus{ Name: "test", Ready: true, RestartCount: 7, @@ -560,8 +578,8 @@ func TestDescribeContainers(t *testing.T) { expectedElements: []string{"test", "State", "Waiting", "Ready", "True", "Restart Count", "7", "Image", "image", "envname", "xyz", "a123\tConfigMap\tOptional: false"}, }, { - container: api.Container{Name: "test", Image: "image", Env: []api.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []api.EnvFromSource{{Prefix: "p_", ConfigMapRef: &api.ConfigMapEnvSource{LocalObjectReference: api.LocalObjectReference{Name: "a123"}}}}}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image", Env: []corev1.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []corev1.EnvFromSource{{Prefix: "p_", ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "a123"}}}}}, + status: corev1.ContainerStatus{ Name: "test", Ready: true, RestartCount: 7, @@ -569,8 +587,8 @@ func TestDescribeContainers(t *testing.T) { expectedElements: []string{"test", "State", "Waiting", "Ready", "True", "Restart Count", "7", "Image", "image", "envname", "xyz", "a123\tConfigMap with prefix 'p_'\tOptional: false"}, }, { - container: api.Container{Name: "test", Image: "image", Env: []api.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []api.EnvFromSource{{ConfigMapRef: &api.ConfigMapEnvSource{Optional: &trueVal, LocalObjectReference: api.LocalObjectReference{Name: "a123"}}}}}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image", Env: []corev1.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{Optional: &trueVal, LocalObjectReference: corev1.LocalObjectReference{Name: "a123"}}}}}, + status: corev1.ContainerStatus{ Name: "test", Ready: true, RestartCount: 7, @@ -578,8 +596,8 @@ func TestDescribeContainers(t *testing.T) { expectedElements: []string{"test", "State", "Waiting", "Ready", "True", "Restart Count", "7", "Image", "image", "envname", "xyz", "a123\tConfigMap\tOptional: true"}, }, { - container: api.Container{Name: "test", Image: "image", Env: []api.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []api.EnvFromSource{{SecretRef: &api.SecretEnvSource{LocalObjectReference: api.LocalObjectReference{Name: "a123"}, Optional: &trueVal}}}}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image", Env: []corev1.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []corev1.EnvFromSource{{SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "a123"}, Optional: &trueVal}}}}, + status: corev1.ContainerStatus{ Name: "test", Ready: true, RestartCount: 7, @@ -587,8 +605,8 @@ func TestDescribeContainers(t *testing.T) { expectedElements: []string{"test", "State", "Waiting", "Ready", "True", "Restart Count", "7", "Image", "image", "envname", "xyz", "a123\tSecret\tOptional: true"}, }, { - container: api.Container{Name: "test", Image: "image", Env: []api.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []api.EnvFromSource{{Prefix: "p_", SecretRef: &api.SecretEnvSource{LocalObjectReference: api.LocalObjectReference{Name: "a123"}}}}}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image", Env: []corev1.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []corev1.EnvFromSource{{Prefix: "p_", SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "a123"}}}}}, + status: corev1.ContainerStatus{ Name: "test", Ready: true, RestartCount: 7, @@ -597,8 +615,8 @@ func TestDescribeContainers(t *testing.T) { }, // Command { - container: api.Container{Name: "test", Image: "image", Command: []string{"sleep", "1000"}}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image", Command: []string{"sleep", "1000"}}, + status: corev1.ContainerStatus{ Name: "test", Ready: true, RestartCount: 7, @@ -607,8 +625,8 @@ func TestDescribeContainers(t *testing.T) { }, // Command with newline { - container: api.Container{Name: "test", Image: "image", Command: []string{"sleep", "1000\n2000"}}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image", Command: []string{"sleep", "1000\n2000"}}, + status: corev1.ContainerStatus{ Name: "test", Ready: true, RestartCount: 7, @@ -617,8 +635,8 @@ func TestDescribeContainers(t *testing.T) { }, // Args { - container: api.Container{Name: "test", Image: "image", Args: []string{"time", "1000"}}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image", Args: []string{"time", "1000"}}, + status: corev1.ContainerStatus{ Name: "test", Ready: true, RestartCount: 7, @@ -627,8 +645,8 @@ func TestDescribeContainers(t *testing.T) { }, // Args with newline { - container: api.Container{Name: "test", Image: "image", Args: []string{"time", "1000\n2000"}}, - status: api.ContainerStatus{ + container: corev1.Container{Name: "test", Image: "image", Args: []string{"time", "1000\n2000"}}, + status: corev1.ContainerStatus{ Name: "test", Ready: true, RestartCount: 7, @@ -637,18 +655,18 @@ func TestDescribeContainers(t *testing.T) { }, // Using limits. { - container: api.Container{ + container: corev1.Container{ Name: "test", Image: "image", - Resources: api.ResourceRequirements{ - Limits: api.ResourceList{ - api.ResourceName(api.ResourceCPU): resource.MustParse("1000"), - api.ResourceName(api.ResourceMemory): resource.MustParse("4G"), - api.ResourceName(api.ResourceStorage): resource.MustParse("20G"), + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("1000"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("4G"), + corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("20G"), }, }, }, - status: api.ContainerStatus{ + status: corev1.ContainerStatus{ Name: "test", Ready: true, RestartCount: 7, @@ -657,14 +675,14 @@ func TestDescribeContainers(t *testing.T) { }, // Using requests. { - container: api.Container{ + container: corev1.Container{ Name: "test", Image: "image", - Resources: api.ResourceRequirements{ - Requests: api.ResourceList{ - api.ResourceName(api.ResourceCPU): resource.MustParse("1000"), - api.ResourceName(api.ResourceMemory): resource.MustParse("4G"), - api.ResourceName(api.ResourceStorage): resource.MustParse("20G"), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("1000"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("4G"), + corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("20G"), }, }, }, @@ -672,10 +690,10 @@ func TestDescribeContainers(t *testing.T) { }, // volumeMounts read/write { - container: api.Container{ + container: corev1.Container{ Name: "test", Image: "image", - VolumeMounts: []api.VolumeMount{ + VolumeMounts: []corev1.VolumeMount{ { Name: "mounted-volume", MountPath: "/opt/", @@ -686,10 +704,10 @@ func TestDescribeContainers(t *testing.T) { }, // volumeMounts readonly { - container: api.Container{ + container: corev1.Container{ Name: "test", Image: "image", - VolumeMounts: []api.VolumeMount{ + VolumeMounts: []corev1.VolumeMount{ { Name: "mounted-volume", MountPath: "/opt/", @@ -702,10 +720,10 @@ func TestDescribeContainers(t *testing.T) { // volumeDevices { - container: api.Container{ + container: corev1.Container{ Name: "test", Image: "image", - VolumeDevices: []api.VolumeDevice{ + VolumeDevices: []corev1.VolumeDevice{ { Name: "volume-device", DevicePath: "/dev/xvda", @@ -719,12 +737,12 @@ func TestDescribeContainers(t *testing.T) { for i, testCase := range testCases { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { out := new(bytes.Buffer) - pod := api.Pod{ - Spec: api.PodSpec{ - Containers: []api.Container{testCase.container}, + pod := corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{testCase.container}, }, - Status: api.PodStatus{ - ContainerStatuses: []api.ContainerStatus{testCase.status}, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{testCase.status}, }, } writer := NewPrefixWriter(out) @@ -740,13 +758,13 @@ func TestDescribeContainers(t *testing.T) { } func TestDescribers(t *testing.T) { - first := &api.Event{} - second := &api.Pod{} - var third *api.Pod + first := &corev1.Event{} + second := &corev1.Pod{} + var third *corev1.Pod testErr := fmt.Errorf("test") d := Describers{} d.Add( - func(e *api.Event, p *api.Pod) (string, error) { + func(e *corev1.Event, p *corev1.Pod) (string, error) { if e != first { t.Errorf("first argument not equal: %#v", e) } @@ -763,8 +781,8 @@ func TestDescribers(t *testing.T) { if out, err := d.DescribeObject(first, second, third); out != "" || err == nil { t.Errorf("unexpected result: %s %v", out, err) } else { - if noDescriber, ok := err.(printers.ErrNoDescriber); ok { - if !reflect.DeepEqual(noDescriber.Types, []string{"*core.Event", "*core.Pod", "*core.Pod"}) { + if noDescriber, ok := err.(describe.ErrNoDescriber); ok { + if !reflect.DeepEqual(noDescriber.Types, []string{"*v1.Event", "*v1.Pod", "*v1.Pod"}) { t.Errorf("unexpected describer: %v", err) } } else { @@ -773,7 +791,7 @@ func TestDescribers(t *testing.T) { } d.Add( - func(e *api.Event) (string, error) { + func(e *corev1.Event) (string, error) { if e != first { t.Errorf("first argument not equal: %#v", e) } @@ -786,7 +804,7 @@ func TestDescribers(t *testing.T) { } func TestDefaultDescribers(t *testing.T) { - out, err := DefaultObjectDescriber.DescribeObject(&api.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) + out, err := DefaultObjectDescriber.DescribeObject(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -794,7 +812,7 @@ func TestDefaultDescribers(t *testing.T) { t.Errorf("unexpected output: %s", out) } - out, err = DefaultObjectDescriber.DescribeObject(&api.Service{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) + out, err = DefaultObjectDescriber.DescribeObject(&corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -802,7 +820,10 @@ func TestDefaultDescribers(t *testing.T) { t.Errorf("unexpected output: %s", out) } - out, err = DefaultObjectDescriber.DescribeObject(&api.ReplicationController{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) + out, err = DefaultObjectDescriber.DescribeObject(&corev1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: corev1.ReplicationControllerSpec{Replicas: utilpointer.Int32Ptr(1)}, + }) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -810,7 +831,7 @@ func TestDefaultDescribers(t *testing.T) { t.Errorf("unexpected output: %s", out) } - out, err = DefaultObjectDescriber.DescribeObject(&api.Node{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) + out, err = DefaultObjectDescriber.DescribeObject(&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -822,31 +843,31 @@ func TestDefaultDescribers(t *testing.T) { func TestGetPodsTotalRequests(t *testing.T) { testCases := []struct { name string - pods *api.PodList - expectedReqs map[api.ResourceName]resource.Quantity + pods *corev1.PodList + expectedReqs map[corev1.ResourceName]resource.Quantity }{ { name: "test1", - pods: &api.PodList{ - Items: []api.Pod{ + pods: &corev1.PodList{ + Items: []corev1.Pod{ { - Spec: api.PodSpec{ - Containers: []api.Container{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ { - Resources: api.ResourceRequirements{ - Requests: api.ResourceList{ - api.ResourceName(api.ResourceCPU): resource.MustParse("1"), - api.ResourceName(api.ResourceMemory): resource.MustParse("300Mi"), - api.ResourceName(api.ResourceStorage): resource.MustParse("1G"), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("1"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("300Mi"), + corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("1G"), }, }, }, { - Resources: api.ResourceRequirements{ - Requests: api.ResourceList{ - api.ResourceName(api.ResourceCPU): resource.MustParse("90m"), - api.ResourceName(api.ResourceMemory): resource.MustParse("120Mi"), - api.ResourceName(api.ResourceStorage): resource.MustParse("200M"), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("90m"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("120Mi"), + corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("200M"), }, }, }, @@ -854,23 +875,23 @@ func TestGetPodsTotalRequests(t *testing.T) { }, }, { - Spec: api.PodSpec{ - Containers: []api.Container{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ { - Resources: api.ResourceRequirements{ - Requests: api.ResourceList{ - api.ResourceName(api.ResourceCPU): resource.MustParse("60m"), - api.ResourceName(api.ResourceMemory): resource.MustParse("43Mi"), - api.ResourceName(api.ResourceStorage): resource.MustParse("500M"), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("60m"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("43Mi"), + corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("500M"), }, }, }, { - Resources: api.ResourceRequirements{ - Requests: api.ResourceList{ - api.ResourceName(api.ResourceCPU): resource.MustParse("34m"), - api.ResourceName(api.ResourceMemory): resource.MustParse("83Mi"), - api.ResourceName(api.ResourceStorage): resource.MustParse("700M"), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("34m"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("83Mi"), + corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("700M"), }, }, }, @@ -879,10 +900,10 @@ func TestGetPodsTotalRequests(t *testing.T) { }, }, }, - expectedReqs: map[api.ResourceName]resource.Quantity{ - api.ResourceName(api.ResourceCPU): resource.MustParse("1.184"), - api.ResourceName(api.ResourceMemory): resource.MustParse("546Mi"), - api.ResourceName(api.ResourceStorage): resource.MustParse("2.4G"), + expectedReqs: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("1.184"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("546Mi"), + corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("2.4G"), }, }, } @@ -898,24 +919,24 @@ func TestGetPodsTotalRequests(t *testing.T) { } func TestPersistentVolumeDescriber(t *testing.T) { - block := api.PersistentVolumeBlock - file := api.PersistentVolumeFilesystem + block := corev1.PersistentVolumeBlock + file := corev1.PersistentVolumeFilesystem deletionTimestamp := metav1.Time{Time: time.Now().UTC().AddDate(10, 0, 0)} testCases := []struct { name string plugin string - pv *api.PersistentVolume + pv *corev1.PersistentVolume expectedElements []string unexpectedElements []string }{ { name: "test0", plugin: "hostpath", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Type: new(api.HostPathType)}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + HostPath: &corev1.HostPathVolumeSource{Type: new(corev1.HostPathType)}, }, }, }, @@ -924,11 +945,11 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test1", plugin: "gce", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - GCEPersistentDisk: &api.GCEPersistentDiskVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + GCEPersistentDisk: &corev1.GCEPersistentDiskVolumeSource{}, }, VolumeMode: &file, }, @@ -938,11 +959,11 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test2", plugin: "ebs", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - AWSElasticBlockStore: &api.AWSElasticBlockStoreVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + AWSElasticBlockStore: &corev1.AWSElasticBlockStoreVolumeSource{}, }, }, }, @@ -951,11 +972,11 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test3", plugin: "nfs", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - NFS: &api.NFSVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + NFS: &corev1.NFSVolumeSource{}, }, }, }, @@ -964,11 +985,11 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test4", plugin: "iscsi", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - ISCSI: &api.ISCSIPersistentVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + ISCSI: &corev1.ISCSIPersistentVolumeSource{}, }, VolumeMode: &block, }, @@ -978,11 +999,11 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test5", plugin: "gluster", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - Glusterfs: &api.GlusterfsVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + Glusterfs: &corev1.GlusterfsPersistentVolumeSource{}, }, }, }, @@ -991,11 +1012,11 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test6", plugin: "rbd", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - RBD: &api.RBDPersistentVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + RBD: &corev1.RBDPersistentVolumeSource{}, }, }, }, @@ -1004,11 +1025,11 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test7", plugin: "quobyte", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - Quobyte: &api.QuobyteVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + Quobyte: &corev1.QuobyteVolumeSource{}, }, }, }, @@ -1017,11 +1038,11 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test8", plugin: "cinder", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - Cinder: &api.CinderPersistentVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + Cinder: &corev1.CinderPersistentVolumeSource{}, }, }, }, @@ -1030,11 +1051,11 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test9", plugin: "fc", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - FC: &api.FCVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + FC: &corev1.FCVolumeSource{}, }, VolumeMode: &block, }, @@ -1044,11 +1065,11 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test10", plugin: "local", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - Local: &api.LocalVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + Local: &corev1.LocalVolumeSource{}, }, }, }, @@ -1058,13 +1079,13 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test11", plugin: "local", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - Local: &api.LocalVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + Local: &corev1.LocalVolumeSource{}, }, - NodeAffinity: &api.VolumeNodeAffinity{}, + NodeAffinity: &corev1.VolumeNodeAffinity{}, }, }, expectedElements: []string{"Node Affinity: "}, @@ -1073,14 +1094,14 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test12", plugin: "local", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - Local: &api.LocalVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + Local: &corev1.LocalVolumeSource{}, }, - NodeAffinity: &api.VolumeNodeAffinity{ - Required: &api.NodeSelector{}, + NodeAffinity: &corev1.VolumeNodeAffinity{ + Required: &corev1.NodeSelector{}, }, }, }, @@ -1090,20 +1111,20 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test13", plugin: "local", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - Local: &api.LocalVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + Local: &corev1.LocalVolumeSource{}, }, - NodeAffinity: &api.VolumeNodeAffinity{ - Required: &api.NodeSelector{ - NodeSelectorTerms: []api.NodeSelectorTerm{ + NodeAffinity: &corev1.VolumeNodeAffinity{ + Required: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ { - MatchExpressions: []api.NodeSelectorRequirement{}, + MatchExpressions: []corev1.NodeSelectorRequirement{}, }, { - MatchExpressions: []api.NodeSelectorRequirement{}, + MatchExpressions: []corev1.NodeSelectorRequirement{}, }, }, }, @@ -1115,17 +1136,17 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test14", plugin: "local", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "bar"}, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - Local: &api.LocalVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + Local: &corev1.LocalVolumeSource{}, }, - NodeAffinity: &api.VolumeNodeAffinity{ - Required: &api.NodeSelector{ - NodeSelectorTerms: []api.NodeSelectorTerm{ + NodeAffinity: &corev1.VolumeNodeAffinity{ + Required: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ { - MatchExpressions: []api.NodeSelectorRequirement{ + MatchExpressions: []corev1.NodeSelectorRequirement{ { Key: "foo", Operator: "In", @@ -1149,14 +1170,14 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test15", plugin: "local", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", DeletionTimestamp: &deletionTimestamp, }, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - Local: &api.LocalVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + Local: &corev1.LocalVolumeSource{}, }, }, }, @@ -1165,7 +1186,7 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test16", plugin: "local", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", GenerateName: "test-GenerateName", @@ -1176,15 +1197,15 @@ func TestPersistentVolumeDescriber(t *testing.T) { Labels: map[string]string{"label1": "label1", "label2": "label2", "label3": "label3"}, Annotations: map[string]string{"annotation1": "annotation1", "annotation2": "annotation2", "annotation3": "annotation3"}, }, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - Local: &api.LocalVolumeSource{}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + Local: &corev1.LocalVolumeSource{}, }, - NodeAffinity: &api.VolumeNodeAffinity{ - Required: &api.NodeSelector{ - NodeSelectorTerms: []api.NodeSelectorTerm{ + NodeAffinity: &corev1.VolumeNodeAffinity{ + Required: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ { - MatchExpressions: []api.NodeSelectorRequirement{ + MatchExpressions: []corev1.NodeSelectorRequirement{ { Key: "foo", Operator: "In", @@ -1208,7 +1229,7 @@ func TestPersistentVolumeDescriber(t *testing.T) { { name: "test17", plugin: "local", - pv: &api.PersistentVolume{ + pv: &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", GenerateName: "test-GenerateName", @@ -1219,9 +1240,9 @@ func TestPersistentVolumeDescriber(t *testing.T) { Labels: map[string]string{"label1": "label1", "label2": "label2", "label3": "label3"}, Annotations: map[string]string{"annotation1": "annotation1", "annotation2": "annotation2", "annotation3": "annotation3"}, }, - Spec: api.PersistentVolumeSpec{ - PersistentVolumeSource: api.PersistentVolumeSource{ - CSI: &api.CSIPersistentVolumeSource{ + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ Driver: "drive", VolumeHandle: "handler", ReadOnly: true, @@ -1242,7 +1263,7 @@ func TestPersistentVolumeDescriber(t *testing.T) { t.Run(test.name, func(t *testing.T) { fake := fake.NewSimpleClientset(test.pv) c := PersistentVolumeDescriber{fake} - str, err := c.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: true}) + str, err := c.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("Unexpected error for test %s: %v", test.plugin, err) } @@ -1264,57 +1285,57 @@ func TestPersistentVolumeDescriber(t *testing.T) { } func TestPersistentVolumeClaimDescriber(t *testing.T) { - block := api.PersistentVolumeBlock - file := api.PersistentVolumeFilesystem + block := corev1.PersistentVolumeBlock + file := corev1.PersistentVolumeFilesystem goldClassName := "gold" now := time.Now() deletionTimestamp := metav1.Time{Time: time.Now().UTC().AddDate(10, 0, 0)} testCases := []struct { name string - pvc *api.PersistentVolumeClaim + pvc *corev1.PersistentVolumeClaim expectedElements []string unexpectedElements []string }{ { name: "default", - pvc: &api.PersistentVolumeClaim{ + pvc: &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, - Spec: api.PersistentVolumeClaimSpec{ + Spec: corev1.PersistentVolumeClaimSpec{ VolumeName: "volume1", StorageClassName: &goldClassName, }, - Status: api.PersistentVolumeClaimStatus{ - Phase: api.ClaimBound, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, }, }, unexpectedElements: []string{"VolumeMode", "Filesystem"}, }, { name: "filesystem", - pvc: &api.PersistentVolumeClaim{ + pvc: &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, - Spec: api.PersistentVolumeClaimSpec{ + Spec: corev1.PersistentVolumeClaimSpec{ VolumeName: "volume2", StorageClassName: &goldClassName, VolumeMode: &file, }, - Status: api.PersistentVolumeClaimStatus{ - Phase: api.ClaimBound, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, }, }, expectedElements: []string{"VolumeMode", "Filesystem"}, }, { name: "block", - pvc: &api.PersistentVolumeClaim{ + pvc: &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, - Spec: api.PersistentVolumeClaimSpec{ + Spec: corev1.PersistentVolumeClaimSpec{ VolumeName: "volume3", StorageClassName: &goldClassName, VolumeMode: &block, }, - Status: api.PersistentVolumeClaimStatus{ - Phase: api.ClaimBound, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, }, }, expectedElements: []string{"VolumeMode", "Block"}, @@ -1322,15 +1343,15 @@ func TestPersistentVolumeClaimDescriber(t *testing.T) { // Tests for Status.Condition. { name: "condition-type", - pvc: &api.PersistentVolumeClaim{ + pvc: &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, - Spec: api.PersistentVolumeClaimSpec{ + Spec: corev1.PersistentVolumeClaimSpec{ VolumeName: "volume4", StorageClassName: &goldClassName, }, - Status: api.PersistentVolumeClaimStatus{ - Conditions: []api.PersistentVolumeClaimCondition{ - {Type: api.PersistentVolumeClaimResizing}, + Status: corev1.PersistentVolumeClaimStatus{ + Conditions: []corev1.PersistentVolumeClaimCondition{ + {Type: corev1.PersistentVolumeClaimResizing}, }, }, }, @@ -1338,15 +1359,15 @@ func TestPersistentVolumeClaimDescriber(t *testing.T) { }, { name: "condition-status", - pvc: &api.PersistentVolumeClaim{ + pvc: &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, - Spec: api.PersistentVolumeClaimSpec{ + Spec: corev1.PersistentVolumeClaimSpec{ VolumeName: "volume5", StorageClassName: &goldClassName, }, - Status: api.PersistentVolumeClaimStatus{ - Conditions: []api.PersistentVolumeClaimCondition{ - {Status: api.ConditionTrue}, + Status: corev1.PersistentVolumeClaimStatus{ + Conditions: []corev1.PersistentVolumeClaimCondition{ + {Status: corev1.ConditionTrue}, }, }, }, @@ -1354,14 +1375,14 @@ func TestPersistentVolumeClaimDescriber(t *testing.T) { }, { name: "condition-last-probe-time", - pvc: &api.PersistentVolumeClaim{ + pvc: &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, - Spec: api.PersistentVolumeClaimSpec{ + Spec: corev1.PersistentVolumeClaimSpec{ VolumeName: "volume6", StorageClassName: &goldClassName, }, - Status: api.PersistentVolumeClaimStatus{ - Conditions: []api.PersistentVolumeClaimCondition{ + Status: corev1.PersistentVolumeClaimStatus{ + Conditions: []corev1.PersistentVolumeClaimCondition{ {LastProbeTime: metav1.Time{Time: now}}, }, }, @@ -1370,14 +1391,14 @@ func TestPersistentVolumeClaimDescriber(t *testing.T) { }, { name: "condition-last-transition-time", - pvc: &api.PersistentVolumeClaim{ + pvc: &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, - Spec: api.PersistentVolumeClaimSpec{ + Spec: corev1.PersistentVolumeClaimSpec{ VolumeName: "volume7", StorageClassName: &goldClassName, }, - Status: api.PersistentVolumeClaimStatus{ - Conditions: []api.PersistentVolumeClaimCondition{ + Status: corev1.PersistentVolumeClaimStatus{ + Conditions: []corev1.PersistentVolumeClaimCondition{ {LastTransitionTime: metav1.Time{Time: now}}, }, }, @@ -1386,14 +1407,14 @@ func TestPersistentVolumeClaimDescriber(t *testing.T) { }, { name: "condition-reason", - pvc: &api.PersistentVolumeClaim{ + pvc: &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, - Spec: api.PersistentVolumeClaimSpec{ + Spec: corev1.PersistentVolumeClaimSpec{ VolumeName: "volume8", StorageClassName: &goldClassName, }, - Status: api.PersistentVolumeClaimStatus{ - Conditions: []api.PersistentVolumeClaimCondition{ + Status: corev1.PersistentVolumeClaimStatus{ + Conditions: []corev1.PersistentVolumeClaimCondition{ {Reason: "OfflineResize"}, }, }, @@ -1402,14 +1423,14 @@ func TestPersistentVolumeClaimDescriber(t *testing.T) { }, { name: "condition-message", - pvc: &api.PersistentVolumeClaim{ + pvc: &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, - Spec: api.PersistentVolumeClaimSpec{ + Spec: corev1.PersistentVolumeClaimSpec{ VolumeName: "volume9", StorageClassName: &goldClassName, }, - Status: api.PersistentVolumeClaimStatus{ - Conditions: []api.PersistentVolumeClaimCondition{ + Status: corev1.PersistentVolumeClaimStatus{ + Conditions: []corev1.PersistentVolumeClaimCondition{ {Message: "User request resize"}, }, }, @@ -1418,17 +1439,17 @@ func TestPersistentVolumeClaimDescriber(t *testing.T) { }, { name: "deletion-timestamp", - pvc: &api.PersistentVolumeClaim{ + pvc: &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "foo", Name: "bar", DeletionTimestamp: &deletionTimestamp, }, - Spec: api.PersistentVolumeClaimSpec{ + Spec: corev1.PersistentVolumeClaimSpec{ VolumeName: "volume10", StorageClassName: &goldClassName, }, - Status: api.PersistentVolumeClaimStatus{}, + Status: corev1.PersistentVolumeClaimStatus{}, }, expectedElements: []string{"Terminating (lasts 10y)"}, }, @@ -1438,7 +1459,7 @@ func TestPersistentVolumeClaimDescriber(t *testing.T) { t.Run(test.name, func(t *testing.T) { fake := fake.NewSimpleClientset(test.pvc) c := PersistentVolumeClaimDescriber{fake} - str, err := c.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: true}) + str, err := c.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("Unexpected error for test %s: %v", test.name, err) } @@ -1460,8 +1481,7 @@ func TestPersistentVolumeClaimDescriber(t *testing.T) { } func TestDescribeDeployment(t *testing.T) { - fake := fake.NewSimpleClientset() - versionedFake := versionedfake.NewSimpleClientset(&appsv1.Deployment{ + fakeClient := fake.NewSimpleClientset(&appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", @@ -1478,8 +1498,8 @@ func TestDescribeDeployment(t *testing.T) { }, }, }) - d := DeploymentDescriber{fake, versionedFake} - out, err := d.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: true}) + d := DeploymentDescriber{fakeClient} + out, err := d.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -1489,9 +1509,9 @@ func TestDescribeDeployment(t *testing.T) { } func TestDescribeStorageClass(t *testing.T) { - reclaimPolicy := api.PersistentVolumeReclaimRetain - bindingMode := storage.VolumeBindingMode("bindingmode") - f := fake.NewSimpleClientset(&storage.StorageClass{ + reclaimPolicy := corev1.PersistentVolumeReclaimRetain + bindingMode := storagev1.VolumeBindingMode("bindingmode") + f := fake.NewSimpleClientset(&storagev1.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", ResourceVersion: "4", @@ -1506,9 +1526,9 @@ func TestDescribeStorageClass(t *testing.T) { }, ReclaimPolicy: &reclaimPolicy, VolumeBindingMode: &bindingMode, - AllowedTopologies: []api.TopologySelectorTerm{ + AllowedTopologies: []corev1.TopologySelectorTerm{ { - MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ + MatchLabelExpressions: []corev1.TopologySelectorLabelRequirement{ { Key: "failure-domain.beta.kubernetes.io/zone", Values: []string{"zone1"}, @@ -1520,7 +1540,7 @@ func TestDescribeStorageClass(t *testing.T) { }, }, { - MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ + MatchLabelExpressions: []corev1.TopologySelectorLabelRequirement{ { Key: "failure-domain.beta.kubernetes.io/zone", Values: []string{"zone2"}, @@ -1534,7 +1554,7 @@ func TestDescribeStorageClass(t *testing.T) { }, }) s := StorageClassDescriber{f} - out, err := s.Describe("", "foo", printers.DescriberSettings{ShowEvents: true}) + out, err := s.Describe("", "foo", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -1558,21 +1578,21 @@ func TestDescribeStorageClass(t *testing.T) { func TestDescribePodDisruptionBudget(t *testing.T) { minAvailable := intstr.FromInt(22) - f := fake.NewSimpleClientset(&policy.PodDisruptionBudget{ + f := fake.NewSimpleClientset(&policyv1beta1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns1", Name: "pdb1", CreationTimestamp: metav1.Time{Time: time.Now().Add(1.9e9)}, }, - Spec: policy.PodDisruptionBudgetSpec{ + Spec: policyv1beta1.PodDisruptionBudgetSpec{ MinAvailable: &minAvailable, }, - Status: policy.PodDisruptionBudgetStatus{ + Status: policyv1beta1.PodDisruptionBudgetStatus{ PodDisruptionsAllowed: 5, }, }) s := PodDisruptionBudgetDescriber{f} - out, err := s.Describe("ns1", "pdb1", printers.DescriberSettings{ShowEvents: true}) + out, err := s.Describe("ns1", "pdb1", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -1594,19 +1614,19 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { } tests := []struct { name string - hpa autoscaling.HorizontalPodAutoscaler + hpa autoscalingv2beta2.HorizontalPodAutoscaler }{ { "minReplicas unset", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MaxReplicas: 10, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, }, @@ -1614,31 +1634,31 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { }, { "external source type, target average value (no current)", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MinReplicas: &minReplicasVal, MaxReplicas: 10, - Metrics: []autoscaling.MetricSpec{ + Metrics: []autoscalingv2beta2.MetricSpec{ { - Type: autoscaling.ExternalMetricSourceType, - External: &autoscaling.ExternalMetricSource{ - Metric: autoscaling.MetricIdentifier{ + Type: autoscalingv2beta2.ExternalMetricSourceType, + External: &autoscalingv2beta2.ExternalMetricSource{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-external-metric", Selector: metricLabelSelector, }, - Target: autoscaling.MetricTarget{ - Type: autoscaling.AverageValueMetricType, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.AverageValueMetricType, AverageValue: resource.NewMilliQuantity(100, resource.DecimalSI), }, }, }, }, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, }, @@ -1646,42 +1666,42 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { }, { "external source type, target average value (with current)", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MinReplicas: &minReplicasVal, MaxReplicas: 10, - Metrics: []autoscaling.MetricSpec{ + Metrics: []autoscalingv2beta2.MetricSpec{ { - Type: autoscaling.ExternalMetricSourceType, - External: &autoscaling.ExternalMetricSource{ - Metric: autoscaling.MetricIdentifier{ + Type: autoscalingv2beta2.ExternalMetricSourceType, + External: &autoscalingv2beta2.ExternalMetricSource{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-external-metric", Selector: metricLabelSelector, }, - Target: autoscaling.MetricTarget{ - Type: autoscaling.AverageValueMetricType, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.AverageValueMetricType, AverageValue: resource.NewMilliQuantity(100, resource.DecimalSI), }, }, }, }, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, - CurrentMetrics: []autoscaling.MetricStatus{ + CurrentMetrics: []autoscalingv2beta2.MetricStatus{ { - Type: autoscaling.ExternalMetricSourceType, - External: &autoscaling.ExternalMetricStatus{ - Metric: autoscaling.MetricIdentifier{ + Type: autoscalingv2beta2.ExternalMetricSourceType, + External: &autoscalingv2beta2.ExternalMetricStatus{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-external-metric", Selector: metricLabelSelector, }, - Current: autoscaling.MetricValueStatus{ + Current: autoscalingv2beta2.MetricValueStatus{ AverageValue: resource.NewMilliQuantity(50, resource.DecimalSI), }, }, @@ -1692,31 +1712,31 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { }, { "external source type, target value (no current)", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MinReplicas: &minReplicasVal, MaxReplicas: 10, - Metrics: []autoscaling.MetricSpec{ + Metrics: []autoscalingv2beta2.MetricSpec{ { - Type: autoscaling.ExternalMetricSourceType, - External: &autoscaling.ExternalMetricSource{ - Metric: autoscaling.MetricIdentifier{ + Type: autoscalingv2beta2.ExternalMetricSourceType, + External: &autoscalingv2beta2.ExternalMetricSource{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-external-metric", Selector: metricLabelSelector, }, - Target: autoscaling.MetricTarget{ - Type: autoscaling.ValueMetricType, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.ValueMetricType, Value: resource.NewMilliQuantity(100, resource.DecimalSI), }, }, }, }, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, }, @@ -1724,42 +1744,42 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { }, { "external source type, target value (with current)", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MinReplicas: &minReplicasVal, MaxReplicas: 10, - Metrics: []autoscaling.MetricSpec{ + Metrics: []autoscalingv2beta2.MetricSpec{ { - Type: autoscaling.ExternalMetricSourceType, - External: &autoscaling.ExternalMetricSource{ - Metric: autoscaling.MetricIdentifier{ + Type: autoscalingv2beta2.ExternalMetricSourceType, + External: &autoscalingv2beta2.ExternalMetricSource{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-external-metric", Selector: metricLabelSelector, }, - Target: autoscaling.MetricTarget{ - Type: autoscaling.ValueMetricType, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.ValueMetricType, Value: resource.NewMilliQuantity(100, resource.DecimalSI), }, }, }, }, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, - CurrentMetrics: []autoscaling.MetricStatus{ + CurrentMetrics: []autoscalingv2beta2.MetricStatus{ { - Type: autoscaling.ExternalMetricSourceType, - External: &autoscaling.ExternalMetricStatus{ - Metric: autoscaling.MetricIdentifier{ + Type: autoscalingv2beta2.ExternalMetricSourceType, + External: &autoscalingv2beta2.ExternalMetricStatus{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-external-metric", Selector: metricLabelSelector, }, - Current: autoscaling.MetricValueStatus{ + Current: autoscalingv2beta2.MetricValueStatus{ Value: resource.NewMilliQuantity(50, resource.DecimalSI), }, }, @@ -1770,30 +1790,30 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { }, { "pods source type (no current)", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MinReplicas: &minReplicasVal, MaxReplicas: 10, - Metrics: []autoscaling.MetricSpec{ + Metrics: []autoscalingv2beta2.MetricSpec{ { - Type: autoscaling.PodsMetricSourceType, - Pods: &autoscaling.PodsMetricSource{ - Metric: autoscaling.MetricIdentifier{ + Type: autoscalingv2beta2.PodsMetricSourceType, + Pods: &autoscalingv2beta2.PodsMetricSource{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-pods-metric", }, - Target: autoscaling.MetricTarget{ - Type: autoscaling.AverageValueMetricType, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.AverageValueMetricType, AverageValue: resource.NewMilliQuantity(100, resource.DecimalSI), }, }, }, }, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, }, @@ -1801,40 +1821,40 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { }, { "pods source type (with current)", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MinReplicas: &minReplicasVal, MaxReplicas: 10, - Metrics: []autoscaling.MetricSpec{ + Metrics: []autoscalingv2beta2.MetricSpec{ { - Type: autoscaling.PodsMetricSourceType, - Pods: &autoscaling.PodsMetricSource{ - Metric: autoscaling.MetricIdentifier{ + Type: autoscalingv2beta2.PodsMetricSourceType, + Pods: &autoscalingv2beta2.PodsMetricSource{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-pods-metric", }, - Target: autoscaling.MetricTarget{ - Type: autoscaling.AverageValueMetricType, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.AverageValueMetricType, AverageValue: resource.NewMilliQuantity(100, resource.DecimalSI), }, }, }, }, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, - CurrentMetrics: []autoscaling.MetricStatus{ + CurrentMetrics: []autoscalingv2beta2.MetricStatus{ { - Type: autoscaling.PodsMetricSourceType, - Pods: &autoscaling.PodsMetricStatus{ - Metric: autoscaling.MetricIdentifier{ + Type: autoscalingv2beta2.PodsMetricSourceType, + Pods: &autoscalingv2beta2.PodsMetricStatus{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-pods-metric", }, - Current: autoscaling.MetricValueStatus{ + Current: autoscalingv2beta2.MetricValueStatus{ AverageValue: resource.NewMilliQuantity(50, resource.DecimalSI), }, }, @@ -1845,34 +1865,34 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { }, { "object source type (no current)", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MinReplicas: &minReplicasVal, MaxReplicas: 10, - Metrics: []autoscaling.MetricSpec{ + Metrics: []autoscalingv2beta2.MetricSpec{ { - Type: autoscaling.ObjectMetricSourceType, - Object: &autoscaling.ObjectMetricSource{ - DescribedObject: autoscaling.CrossVersionObjectReference{ + Type: autoscalingv2beta2.ObjectMetricSourceType, + Object: &autoscalingv2beta2.ObjectMetricSource{ + DescribedObject: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-service", Kind: "Service", }, - Metric: autoscaling.MetricIdentifier{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-service-metric", }, - Target: autoscaling.MetricTarget{ - Type: autoscaling.ValueMetricType, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.ValueMetricType, Value: resource.NewMilliQuantity(100, resource.DecimalSI), }, }, }, }, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, }, @@ -1880,48 +1900,48 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { }, { "object source type (with current)", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MinReplicas: &minReplicasVal, MaxReplicas: 10, - Metrics: []autoscaling.MetricSpec{ + Metrics: []autoscalingv2beta2.MetricSpec{ { - Type: autoscaling.ObjectMetricSourceType, - Object: &autoscaling.ObjectMetricSource{ - DescribedObject: autoscaling.CrossVersionObjectReference{ + Type: autoscalingv2beta2.ObjectMetricSourceType, + Object: &autoscalingv2beta2.ObjectMetricSource{ + DescribedObject: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-service", Kind: "Service", }, - Metric: autoscaling.MetricIdentifier{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-service-metric", }, - Target: autoscaling.MetricTarget{ - Type: autoscaling.ValueMetricType, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.ValueMetricType, Value: resource.NewMilliQuantity(100, resource.DecimalSI), }, }, }, }, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, - CurrentMetrics: []autoscaling.MetricStatus{ + CurrentMetrics: []autoscalingv2beta2.MetricStatus{ { - Type: autoscaling.ObjectMetricSourceType, - Object: &autoscaling.ObjectMetricStatus{ - DescribedObject: autoscaling.CrossVersionObjectReference{ + Type: autoscalingv2beta2.ObjectMetricSourceType, + Object: &autoscalingv2beta2.ObjectMetricStatus{ + DescribedObject: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-service", Kind: "Service", }, - Metric: autoscaling.MetricIdentifier{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-service-metric", }, - Current: autoscaling.MetricValueStatus{ + Current: autoscalingv2beta2.MetricValueStatus{ Value: resource.NewMilliQuantity(50, resource.DecimalSI), }, }, @@ -1932,28 +1952,28 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { }, { "resource source type, target average value (no current)", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MinReplicas: &minReplicasVal, MaxReplicas: 10, - Metrics: []autoscaling.MetricSpec{ + Metrics: []autoscalingv2beta2.MetricSpec{ { - Type: autoscaling.ResourceMetricSourceType, - Resource: &autoscaling.ResourceMetricSource{ - Name: api.ResourceCPU, - Target: autoscaling.MetricTarget{ - Type: autoscaling.AverageValueMetricType, + Type: autoscalingv2beta2.ResourceMetricSourceType, + Resource: &autoscalingv2beta2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.AverageValueMetricType, AverageValue: resource.NewMilliQuantity(100, resource.DecimalSI), }, }, }, }, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, }, @@ -1961,36 +1981,36 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { }, { "resource source type, target average value (with current)", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MinReplicas: &minReplicasVal, MaxReplicas: 10, - Metrics: []autoscaling.MetricSpec{ + Metrics: []autoscalingv2beta2.MetricSpec{ { - Type: autoscaling.ResourceMetricSourceType, - Resource: &autoscaling.ResourceMetricSource{ - Name: api.ResourceCPU, - Target: autoscaling.MetricTarget{ - Type: autoscaling.AverageValueMetricType, + Type: autoscalingv2beta2.ResourceMetricSourceType, + Resource: &autoscalingv2beta2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.AverageValueMetricType, AverageValue: resource.NewMilliQuantity(100, resource.DecimalSI), }, }, }, }, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, - CurrentMetrics: []autoscaling.MetricStatus{ + CurrentMetrics: []autoscalingv2beta2.MetricStatus{ { - Type: autoscaling.ResourceMetricSourceType, - Resource: &autoscaling.ResourceMetricStatus{ - Name: api.ResourceCPU, - Current: autoscaling.MetricValueStatus{ + Type: autoscalingv2beta2.ResourceMetricSourceType, + Resource: &autoscalingv2beta2.ResourceMetricStatus{ + Name: corev1.ResourceCPU, + Current: autoscalingv2beta2.MetricValueStatus{ AverageValue: resource.NewMilliQuantity(50, resource.DecimalSI), }, }, @@ -2001,28 +2021,28 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { }, { "resource source type, target utilization (no current)", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MinReplicas: &minReplicasVal, MaxReplicas: 10, - Metrics: []autoscaling.MetricSpec{ + Metrics: []autoscalingv2beta2.MetricSpec{ { - Type: autoscaling.ResourceMetricSourceType, - Resource: &autoscaling.ResourceMetricSource{ - Name: api.ResourceCPU, - Target: autoscaling.MetricTarget{ - Type: autoscaling.UtilizationMetricType, + Type: autoscalingv2beta2.ResourceMetricSourceType, + Resource: &autoscalingv2beta2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.UtilizationMetricType, AverageUtilization: &targetUtilizationVal, }, }, }, }, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, }, @@ -2030,36 +2050,36 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { }, { "resource source type, target utilization (with current)", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MinReplicas: &minReplicasVal, MaxReplicas: 10, - Metrics: []autoscaling.MetricSpec{ + Metrics: []autoscalingv2beta2.MetricSpec{ { - Type: autoscaling.ResourceMetricSourceType, - Resource: &autoscaling.ResourceMetricSource{ - Name: api.ResourceCPU, - Target: autoscaling.MetricTarget{ - Type: autoscaling.UtilizationMetricType, + Type: autoscalingv2beta2.ResourceMetricSourceType, + Resource: &autoscalingv2beta2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.UtilizationMetricType, AverageUtilization: &targetUtilizationVal, }, }, }, }, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, - CurrentMetrics: []autoscaling.MetricStatus{ + CurrentMetrics: []autoscalingv2beta2.MetricStatus{ { - Type: autoscaling.ResourceMetricSourceType, - Resource: &autoscaling.ResourceMetricStatus{ - Name: api.ResourceCPU, - Current: autoscaling.MetricValueStatus{ + Type: autoscalingv2beta2.ResourceMetricSourceType, + Resource: &autoscalingv2beta2.ResourceMetricStatus{ + Name: corev1.ResourceCPU, + Current: autoscalingv2beta2.MetricValueStatus{ AverageUtilization: ¤tUtilizationVal, AverageValue: resource.NewMilliQuantity(40, resource.DecimalSI), }, @@ -2071,71 +2091,71 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { }, { "multiple metrics", - autoscaling.HorizontalPodAutoscaler{ - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + autoscalingv2beta2.HorizontalPodAutoscaler{ + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ Name: "some-rc", Kind: "ReplicationController", }, MinReplicas: &minReplicasVal, MaxReplicas: 10, - Metrics: []autoscaling.MetricSpec{ + Metrics: []autoscalingv2beta2.MetricSpec{ { - Type: autoscaling.PodsMetricSourceType, - Pods: &autoscaling.PodsMetricSource{ - Metric: autoscaling.MetricIdentifier{ + Type: autoscalingv2beta2.PodsMetricSourceType, + Pods: &autoscalingv2beta2.PodsMetricSource{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-pods-metric", }, - Target: autoscaling.MetricTarget{ - Type: autoscaling.AverageValueMetricType, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.AverageValueMetricType, AverageValue: resource.NewMilliQuantity(100, resource.DecimalSI), }, }, }, { - Type: autoscaling.ResourceMetricSourceType, - Resource: &autoscaling.ResourceMetricSource{ - Name: api.ResourceCPU, - Target: autoscaling.MetricTarget{ - Type: autoscaling.UtilizationMetricType, + Type: autoscalingv2beta2.ResourceMetricSourceType, + Resource: &autoscalingv2beta2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.UtilizationMetricType, AverageUtilization: &targetUtilizationVal, }, }, }, { - Type: autoscaling.PodsMetricSourceType, - Pods: &autoscaling.PodsMetricSource{ - Metric: autoscaling.MetricIdentifier{ + Type: autoscalingv2beta2.PodsMetricSourceType, + Pods: &autoscalingv2beta2.PodsMetricSource{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "other-pods-metric", }, - Target: autoscaling.MetricTarget{ - Type: autoscaling.AverageValueMetricType, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.AverageValueMetricType, AverageValue: resource.NewMilliQuantity(400, resource.DecimalSI), }, }, }, }, }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ + Status: autoscalingv2beta2.HorizontalPodAutoscalerStatus{ CurrentReplicas: 4, DesiredReplicas: 5, - CurrentMetrics: []autoscaling.MetricStatus{ + CurrentMetrics: []autoscalingv2beta2.MetricStatus{ { - Type: autoscaling.PodsMetricSourceType, - Pods: &autoscaling.PodsMetricStatus{ - Metric: autoscaling.MetricIdentifier{ + Type: autoscalingv2beta2.PodsMetricSourceType, + Pods: &autoscalingv2beta2.PodsMetricStatus{ + Metric: autoscalingv2beta2.MetricIdentifier{ Name: "some-pods-metric", }, - Current: autoscaling.MetricValueStatus{ + Current: autoscalingv2beta2.MetricValueStatus{ AverageValue: resource.NewMilliQuantity(50, resource.DecimalSI), }, }, }, { - Type: autoscaling.ResourceMetricSourceType, - Resource: &autoscaling.ResourceMetricStatus{ - Name: api.ResourceCPU, - Current: autoscaling.MetricValueStatus{ + Type: autoscalingv2beta2.ResourceMetricSourceType, + Resource: &autoscalingv2beta2.ResourceMetricStatus{ + Name: corev1.ResourceCPU, + Current: autoscalingv2beta2.MetricValueStatus{ AverageUtilization: ¤tUtilizationVal, AverageValue: resource.NewMilliQuantity(40, resource.DecimalSI), }, @@ -2155,7 +2175,7 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { } fake := fake.NewSimpleClientset(&test.hpa) desc := HorizontalPodAutoscalerDescriber{fake} - str, err := desc.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: true}) + str, err := desc.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("Unexpected error for test %s: %v", test.name, err) } @@ -2169,25 +2189,25 @@ func TestDescribeHorizontalPodAutoscaler(t *testing.T) { func TestDescribeEvents(t *testing.T) { - events := &api.EventList{ - Items: []api.Event{ + events := &corev1.EventList{ + Items: []corev1.Event{ { ObjectMeta: metav1.ObjectMeta{ Namespace: "foo", }, - Source: api.EventSource{Component: "kubelet"}, + Source: corev1.EventSource{Component: "kubelet"}, Message: "Item 1", FirstTimestamp: metav1.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)), LastTimestamp: metav1.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)), Count: 1, - Type: api.EventTypeNormal, + Type: corev1.EventTypeNormal, }, }, } - m := map[string]printers.Describer{ + m := map[string]describe.Describer{ "DaemonSetDescriber": &DaemonSetDescriber{ - fake.NewSimpleClientset(&apps.DaemonSet{ + fake.NewSimpleClientset(&appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", @@ -2195,8 +2215,7 @@ func TestDescribeEvents(t *testing.T) { }, events), }, "DeploymentDescriber": &DeploymentDescriber{ - fake.NewSimpleClientset(events), - versionedfake.NewSimpleClientset(&appsv1.Deployment{ + fake.NewSimpleClientset(&appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", @@ -2205,10 +2224,10 @@ func TestDescribeEvents(t *testing.T) { Replicas: utilpointer.Int32Ptr(1), Selector: &metav1.LabelSelector{}, }, - }), + }, events), }, "EndpointsDescriber": &EndpointsDescriber{ - fake.NewSimpleClientset(&api.Endpoints{ + fake.NewSimpleClientset(&corev1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", @@ -2219,48 +2238,54 @@ func TestDescribeEvents(t *testing.T) { // - IngressDescriber // - JobDescriber "NodeDescriber": &NodeDescriber{ - fake.NewSimpleClientset(&api.Node{ + fake.NewSimpleClientset(&corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", - SelfLink: "url/url/url", + SelfLink: "url/url/url/url", }, }, events), }, "PersistentVolumeDescriber": &PersistentVolumeDescriber{ - fake.NewSimpleClientset(&api.PersistentVolume{ + fake.NewSimpleClientset(&corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", - SelfLink: "url/url/url", + SelfLink: "url/url/url/url", }, }, events), }, "PodDescriber": &PodDescriber{ - fake.NewSimpleClientset(&api.Pod{ + fake.NewSimpleClientset(&corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", - SelfLink: "url/url/url", + SelfLink: "url/url/url/url", }, }, events), }, "ReplicaSetDescriber": &ReplicaSetDescriber{ - fake.NewSimpleClientset(&apps.ReplicaSet{ + fake.NewSimpleClientset(&appsv1.ReplicaSet{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: utilpointer.Int32Ptr(1), + }, }, events), }, "ReplicationControllerDescriber": &ReplicationControllerDescriber{ - fake.NewSimpleClientset(&api.ReplicationController{ + fake.NewSimpleClientset(&corev1.ReplicationController{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", }, + Spec: corev1.ReplicationControllerSpec{ + Replicas: utilpointer.Int32Ptr(1), + }, }, events), }, "Service": &ServiceDescriber{ - fake.NewSimpleClientset(&api.Service{ + fake.NewSimpleClientset(&corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", @@ -2268,14 +2293,14 @@ func TestDescribeEvents(t *testing.T) { }, events), }, "StorageClass": &StorageClassDescriber{ - fake.NewSimpleClientset(&storage.StorageClass{ + fake.NewSimpleClientset(&storagev1.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", }, }, events), }, "HorizontalPodAutoscaler": &HorizontalPodAutoscalerDescriber{ - fake.NewSimpleClientset(&autoscaling.HorizontalPodAutoscaler{ + fake.NewSimpleClientset(&autoscalingv2beta2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", @@ -2283,7 +2308,7 @@ func TestDescribeEvents(t *testing.T) { }, events), }, "ConfigMap": &ConfigMapDescriber{ - fake.NewSimpleClientset(&api.ConfigMap{ + fake.NewSimpleClientset(&corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", @@ -2294,7 +2319,7 @@ func TestDescribeEvents(t *testing.T) { for name, d := range m { t.Run(name, func(t *testing.T) { - out, err := d.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: true}) + out, err := d.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error for %q: %v", name, err) } @@ -2305,7 +2330,7 @@ func TestDescribeEvents(t *testing.T) { t.Errorf("events not found for %q when ShowEvents=true: %s", name, out) } - out, err = d.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: false}) + out, err = d.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: false}) if err != nil { t.Errorf("unexpected error for %q: %s", name, err) } @@ -2473,31 +2498,31 @@ func TestDescribePodSecurityPolicy(t *testing.T) { "Supplemental Groups Strategy: RunAsAny", } - fake := fake.NewSimpleClientset(&policy.PodSecurityPolicy{ + fake := fake.NewSimpleClientset(&policyv1beta1.PodSecurityPolicy{ ObjectMeta: metav1.ObjectMeta{ Name: "mypsp", }, - Spec: policy.PodSecurityPolicySpec{ + Spec: policyv1beta1.PodSecurityPolicySpec{ AllowedUnsafeSysctls: []string{"kernel.*", "net.ipv4.ip_local_port_range"}, ForbiddenSysctls: []string{"net.ipv4.ip_default_ttl"}, - SELinux: policy.SELinuxStrategyOptions{ - Rule: policy.SELinuxStrategyRunAsAny, + SELinux: policyv1beta1.SELinuxStrategyOptions{ + Rule: policyv1beta1.SELinuxStrategyRunAsAny, }, - RunAsUser: policy.RunAsUserStrategyOptions{ - Rule: policy.RunAsUserStrategyRunAsAny, + RunAsUser: policyv1beta1.RunAsUserStrategyOptions{ + Rule: policyv1beta1.RunAsUserStrategyRunAsAny, }, - FSGroup: policy.FSGroupStrategyOptions{ - Rule: policy.FSGroupStrategyRunAsAny, + FSGroup: policyv1beta1.FSGroupStrategyOptions{ + Rule: policyv1beta1.FSGroupStrategyRunAsAny, }, - SupplementalGroups: policy.SupplementalGroupsStrategyOptions{ - Rule: policy.SupplementalGroupsStrategyRunAsAny, + SupplementalGroups: policyv1beta1.SupplementalGroupsStrategyOptions{ + Rule: policyv1beta1.SupplementalGroupsStrategyRunAsAny, }, }, }) c := &describeClient{T: t, Namespace: "", Interface: fake} d := PodSecurityPolicyDescriber{c} - out, err := d.Describe("", "mypsp", printers.DescriberSettings{}) + out, err := d.Describe("", "mypsp", describe.DescriberSettings{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -2510,33 +2535,33 @@ func TestDescribePodSecurityPolicy(t *testing.T) { } func TestDescribeResourceQuota(t *testing.T) { - fake := fake.NewSimpleClientset(&api.ResourceQuota{ + fake := fake.NewSimpleClientset(&corev1.ResourceQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", }, - Status: api.ResourceQuotaStatus{ - Hard: api.ResourceList{ - api.ResourceName(api.ResourceCPU): resource.MustParse("1"), - api.ResourceName(api.ResourceLimitsCPU): resource.MustParse("2"), - api.ResourceName(api.ResourceLimitsMemory): resource.MustParse("2G"), - api.ResourceName(api.ResourceMemory): resource.MustParse("1G"), - api.ResourceName(api.ResourceRequestsCPU): resource.MustParse("1"), - api.ResourceName(api.ResourceRequestsMemory): resource.MustParse("1G"), + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("1"), + corev1.ResourceName(corev1.ResourceLimitsCPU): resource.MustParse("2"), + corev1.ResourceName(corev1.ResourceLimitsMemory): resource.MustParse("2G"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1G"), + corev1.ResourceName(corev1.ResourceRequestsCPU): resource.MustParse("1"), + corev1.ResourceName(corev1.ResourceRequestsMemory): resource.MustParse("1G"), }, - Used: api.ResourceList{ - api.ResourceName(api.ResourceCPU): resource.MustParse("0"), - api.ResourceName(api.ResourceLimitsCPU): resource.MustParse("0"), - api.ResourceName(api.ResourceLimitsMemory): resource.MustParse("0G"), - api.ResourceName(api.ResourceMemory): resource.MustParse("0G"), - api.ResourceName(api.ResourceRequestsCPU): resource.MustParse("0"), - api.ResourceName(api.ResourceRequestsMemory): resource.MustParse("0G"), + Used: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("0"), + corev1.ResourceName(corev1.ResourceLimitsCPU): resource.MustParse("0"), + corev1.ResourceName(corev1.ResourceLimitsMemory): resource.MustParse("0G"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("0G"), + corev1.ResourceName(corev1.ResourceRequestsCPU): resource.MustParse("0"), + corev1.ResourceName(corev1.ResourceRequestsMemory): resource.MustParse("0G"), }, }, }) c := &describeClient{T: t, Namespace: "foo", Interface: fake} d := ResourceQuotaDescriber{c} - out, err := d.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: true}) + out, err := d.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -2603,15 +2628,15 @@ Spec: port80 := intstr.FromInt(80) port82 := intstr.FromInt(82) - protoTCP := api.ProtocolTCP + protoTCP := corev1.ProtocolTCP - versionedFake := fake.NewSimpleClientset(&networking.NetworkPolicy{ + versionedFake := fake.NewSimpleClientset(&networkingv1.NetworkPolicy{ ObjectMeta: metav1.ObjectMeta{ Name: "network-policy-1", Namespace: "default", CreationTimestamp: metav1.NewTime(expectedTime), }, - Spec: networking.NetworkPolicySpec{ + Spec: networkingv1.NetworkPolicySpec{ PodSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ "id1": "app1", @@ -2622,13 +2647,13 @@ Spec: {Key: "foo2", Operator: "NotIn", Values: []string{"bar1", "bar2"}}, }, }, - Ingress: []networking.NetworkPolicyIngressRule{ + Ingress: []networkingv1.NetworkPolicyIngressRule{ { - Ports: []networking.NetworkPolicyPort{ + Ports: []networkingv1.NetworkPolicyPort{ {Port: &port80}, {Port: &port82, Protocol: &protoTCP}, }, - From: []networking.NetworkPolicyPeer{ + From: []networkingv1.NetworkPolicyPeer{ { PodSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -2671,7 +2696,7 @@ Spec: }, }, { - IPBlock: &networking.IPBlock{ + IPBlock: &networkingv1.IPBlock{ CIDR: "192.168.0.0/16", Except: []string{"192.168.3.0/24", "192.168.4.0/24"}, }, @@ -2680,13 +2705,13 @@ Spec: }, {}, }, - Egress: []networking.NetworkPolicyEgressRule{ + Egress: []networkingv1.NetworkPolicyEgressRule{ { - Ports: []networking.NetworkPolicyPort{ + Ports: []networkingv1.NetworkPolicyPort{ {Port: &port80}, {Port: &port82, Protocol: &protoTCP}, }, - To: []networking.NetworkPolicyPeer{ + To: []networkingv1.NetworkPolicyPeer{ { PodSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -2729,7 +2754,7 @@ Spec: }, }, { - IPBlock: &networking.IPBlock{ + IPBlock: &networkingv1.IPBlock{ CIDR: "192.168.0.0/16", Except: []string{"192.168.3.0/24", "192.168.4.0/24"}, }, @@ -2738,11 +2763,11 @@ Spec: }, {}, }, - PolicyTypes: []networking.PolicyType{networking.PolicyTypeIngress, networking.PolicyTypeEgress}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress}, }, }) d := NetworkPolicyDescriber{versionedFake} - out, err := d.Describe("", "network-policy-1", printers.DescriberSettings{}) + out, err := d.Describe("", "network-policy-1", describe.DescriberSettings{}) if err != nil { t.Errorf("unexpected error: %s", err) } @@ -2752,17 +2777,17 @@ Spec: } func TestDescribeServiceAccount(t *testing.T) { - fake := fake.NewSimpleClientset(&api.ServiceAccount{ + fake := fake.NewSimpleClientset(&corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", }, - Secrets: []api.ObjectReference{ + Secrets: []corev1.ObjectReference{ { Name: "test-objectref", }, }, - ImagePullSecrets: []api.LocalObjectReference{ + ImagePullSecrets: []corev1.LocalObjectReference{ { Name: "test-local-ref", }, @@ -2770,7 +2795,7 @@ func TestDescribeServiceAccount(t *testing.T) { }) c := &describeClient{T: t, Namespace: "foo", Interface: fake} d := ServiceAccountDescriber{c} - out, err := d.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: true}) + out, err := d.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -2789,18 +2814,18 @@ Events: ` + "\n" } func TestDescribeNode(t *testing.T) { - fake := fake.NewSimpleClientset(&api.Node{ + fake := fake.NewSimpleClientset(&corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", }, - Spec: api.NodeSpec{ + Spec: corev1.NodeSpec{ Unschedulable: true, }, }) c := &describeClient{T: t, Namespace: "foo", Interface: fake} d := NodeDescriber{c} - out, err := d.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: true}) + out, err := d.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -2815,31 +2840,33 @@ func TestDescribeNode(t *testing.T) { } func TestDescribeStatefulSet(t *testing.T) { - fake := fake.NewSimpleClientset(&apps.StatefulSet{ + var partition int32 = 2 + var replicas int32 = 1 + fake := fake.NewSimpleClientset(&appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", }, - Spec: apps.StatefulSetSpec{ - Replicas: 1, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, Selector: &metav1.LabelSelector{}, - Template: api.PodTemplateSpec{ - Spec: api.PodSpec{ - Containers: []api.Container{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ {Image: "mytest-image:latest"}, }, }, }, - UpdateStrategy: apps.StatefulSetUpdateStrategy{ - Type: apps.RollingUpdateStatefulSetStrategyType, - RollingUpdate: &apps.RollingUpdateStatefulSetStrategy{ - Partition: 2, + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{ + Partition: &partition, }, }, }, }) d := StatefulSetDescriber{fake} - out, err := d.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: true}) + out, err := d.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: true}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -2861,8 +2888,9 @@ func boolPtr(b bool) *bool { } func TestControllerRef(t *testing.T) { + var replicas int32 = 1 f := fake.NewSimpleClientset( - &api.ReplicationController{ + &corev1.ReplicationController{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", Namespace: "foo", @@ -2871,19 +2899,19 @@ func TestControllerRef(t *testing.T) { TypeMeta: metav1.TypeMeta{ Kind: "ReplicationController", }, - Spec: api.ReplicationControllerSpec{ - Replicas: 1, + Spec: corev1.ReplicationControllerSpec{ + Replicas: &replicas, Selector: map[string]string{"abc": "xyz"}, - Template: &api.PodTemplateSpec{ - Spec: api.PodSpec{ - Containers: []api.Container{ + Template: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ {Image: "mytest-image:latest"}, }, }, }, }, }, - &api.Pod{ + &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "barpod", Namespace: "foo", @@ -2893,16 +2921,16 @@ func TestControllerRef(t *testing.T) { TypeMeta: metav1.TypeMeta{ Kind: "Pod", }, - Spec: api.PodSpec{ - Containers: []api.Container{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ {Image: "mytest-image:latest"}, }, }, - Status: api.PodStatus{ - Phase: api.PodRunning, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, }, }, - &api.Pod{ + &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "orphan", Namespace: "foo", @@ -2911,16 +2939,16 @@ func TestControllerRef(t *testing.T) { TypeMeta: metav1.TypeMeta{ Kind: "Pod", }, - Spec: api.PodSpec{ - Containers: []api.Container{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ {Image: "mytest-image:latest"}, }, }, - Status: api.PodStatus{ - Phase: api.PodRunning, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, }, }, - &api.Pod{ + &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "buzpod", Namespace: "foo", @@ -2930,17 +2958,17 @@ func TestControllerRef(t *testing.T) { TypeMeta: metav1.TypeMeta{ Kind: "Pod", }, - Spec: api.PodSpec{ - Containers: []api.Container{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ {Image: "mytest-image:latest"}, }, }, - Status: api.PodStatus{ - Phase: api.PodRunning, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, }, }) d := ReplicationControllerDescriber{f} - out, err := d.Describe("foo", "bar", printers.DescriberSettings{ShowEvents: false}) + out, err := d.Describe("foo", "bar", describe.DescriberSettings{ShowEvents: false}) if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/pkg/kubectl/generate/generate.go b/pkg/kubectl/generate/generate.go index 65abaafd827..1915be8cb96 100644 --- a/pkg/kubectl/generate/generate.go +++ b/pkg/kubectl/generate/generate.go @@ -102,7 +102,7 @@ func AnnotateFlags(cmd *cobra.Command, generators map[string]Generator) { } } -// EnsureFlagsValid ensures that no invalid flags are being used against a +// EnsureFlagsValid ensures that no invalid flags are being used against a func EnsureFlagsValid(cmd *cobra.Command, generators map[string]Generator, generatorInUse string) error { AnnotateFlags(cmd, generators) diff --git a/pkg/kubectl/generate/versioned/autoscale.go b/pkg/kubectl/generate/versioned/autoscale.go index 790b07e5819..7cb8dbbc50d 100644 --- a/pkg/kubectl/generate/versioned/autoscale.go +++ b/pkg/kubectl/generate/versioned/autoscale.go @@ -25,12 +25,12 @@ import ( "k8s.io/kubernetes/pkg/kubectl/generate" ) -// HorizontalPodAutoscalerV1Generator supports stable generation of a horizontal pod autoscaler. +// HorizontalPodAutoscalerGeneratorV1 supports stable generation of a horizontal pod autoscaler. type HorizontalPodAutoscalerGeneratorV1 struct { Name string ScaleRefKind string ScaleRefName string - ScaleRefApiVersion string + ScaleRefAPIVersion string MinReplicas int32 MaxReplicas int32 CPUPercent int32 @@ -53,7 +53,7 @@ func (s *HorizontalPodAutoscalerGeneratorV1) StructuredGenerate() (runtime.Objec ScaleTargetRef: autoscalingv1.CrossVersionObjectReference{ Kind: s.ScaleRefKind, Name: s.ScaleRefName, - APIVersion: s.ScaleRefApiVersion, + APIVersion: s.ScaleRefAPIVersion, }, MaxReplicas: s.MaxReplicas, }, diff --git a/pkg/kubectl/generate/versioned/autoscale_test.go b/pkg/kubectl/generate/versioned/autoscale_test.go index 49b76b5b364..c5f8ae74d45 100644 --- a/pkg/kubectl/generate/versioned/autoscale_test.go +++ b/pkg/kubectl/generate/versioned/autoscale_test.go @@ -31,7 +31,7 @@ func TestHPAGenerate(t *testing.T) { HPAName string scaleRefKind string scaleRefName string - scaleRefApiVersion string + scaleRefAPIVersion string minReplicas int32 maxReplicas int32 CPUPercent int32 @@ -46,7 +46,7 @@ func TestHPAGenerate(t *testing.T) { CPUPercent: 80, scaleRefKind: "kind", scaleRefName: "name", - scaleRefApiVersion: "apiVersion", + scaleRefAPIVersion: "apiVersion", expected: &autoscalingv1.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", @@ -68,7 +68,7 @@ func TestHPAGenerate(t *testing.T) { name: "'name' is a required parameter", scaleRefKind: "kind", scaleRefName: "name", - scaleRefApiVersion: "apiVersion", + scaleRefAPIVersion: "apiVersion", expectErr: true, }, { @@ -76,7 +76,7 @@ func TestHPAGenerate(t *testing.T) { HPAName: "foo", scaleRefKind: "kind", scaleRefName: "name", - scaleRefApiVersion: "apiVersion", + scaleRefAPIVersion: "apiVersion", expectErr: true, }, { @@ -86,7 +86,7 @@ func TestHPAGenerate(t *testing.T) { maxReplicas: 1, scaleRefKind: "kind", scaleRefName: "name", - scaleRefApiVersion: "apiVersion", + scaleRefAPIVersion: "apiVersion", expectErr: true, }, { @@ -96,7 +96,7 @@ func TestHPAGenerate(t *testing.T) { maxReplicas: -10, scaleRefKind: "kind", scaleRefName: "name", - scaleRefApiVersion: "apiVersion", + scaleRefAPIVersion: "apiVersion", expectErr: true, }, } @@ -107,7 +107,7 @@ func TestHPAGenerate(t *testing.T) { Name: tt.HPAName, ScaleRefKind: tt.scaleRefKind, ScaleRefName: tt.scaleRefName, - ScaleRefApiVersion: tt.scaleRefApiVersion, + ScaleRefAPIVersion: tt.scaleRefAPIVersion, MinReplicas: tt.minReplicas, MaxReplicas: tt.maxReplicas, CPUPercent: tt.CPUPercent, diff --git a/pkg/kubectl/generate/versioned/deployment.go b/pkg/kubectl/generate/versioned/deployment.go index d5b23eba8d3..4b2b24d575d 100644 --- a/pkg/kubectl/generate/versioned/deployment.go +++ b/pkg/kubectl/generate/versioned/deployment.go @@ -29,7 +29,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/generate" ) -// BaseDeploymentGenerator: implement the common functionality of +// BaseDeploymentGenerator implements the common functionality of // DeploymentBasicGeneratorV1, DeploymentBasicAppsGeneratorV1Beta1 and DeploymentBasicAppsGeneratorV1. To reduce // confusion, it's best to keep this struct in the same file as those // generators. diff --git a/pkg/kubectl/generate/versioned/generator.go b/pkg/kubectl/generate/versioned/generator.go index 64539a50a64..01d0e21d5a7 100644 --- a/pkg/kubectl/generate/versioned/generator.go +++ b/pkg/kubectl/generate/versioned/generator.go @@ -51,6 +51,7 @@ const ( HorizontalPodAutoscalerV1GeneratorName = "horizontalpodautoscaler/v1" DeploymentV1Beta1GeneratorName = "deployment/v1beta1" DeploymentAppsV1Beta1GeneratorName = "deployment/apps.v1beta1" + DeploymentAppsV1GeneratorName = "deployment/apps.v1" DeploymentBasicV1Beta1GeneratorName = "deployment-basic/v1beta1" DeploymentBasicAppsV1Beta1GeneratorName = "deployment-basic/apps.v1beta1" DeploymentBasicAppsV1GeneratorName = "deployment-basic/apps.v1" @@ -105,6 +106,7 @@ func DefaultGenerators(cmdName string) map[string]generate.Generator { RunPodV1GeneratorName: BasicPod{}, DeploymentV1Beta1GeneratorName: DeploymentV1Beta1{}, DeploymentAppsV1Beta1GeneratorName: DeploymentAppsV1Beta1{}, + DeploymentAppsV1GeneratorName: DeploymentAppsV1{}, JobV1GeneratorName: JobV1{}, CronJobV2Alpha1GeneratorName: CronJobV2Alpha1{}, CronJobV1Beta1GeneratorName: CronJobV1Beta1{}, @@ -146,6 +148,14 @@ func FallbackGeneratorNameIfNecessary( cmdErr io.Writer, ) (string, error) { switch generatorName { + case DeploymentAppsV1GeneratorName: + hasResource, err := HasResource(discoveryClient, appsv1.SchemeGroupVersion.WithResource("deployments")) + if err != nil { + return "", err + } + if !hasResource { + return FallbackGeneratorNameIfNecessary(DeploymentAppsV1Beta1GeneratorName, discoveryClient, cmdErr) + } case DeploymentAppsV1Beta1GeneratorName: hasResource, err := HasResource(discoveryClient, appsv1beta1.SchemeGroupVersion.WithResource("deployments")) if err != nil { diff --git a/pkg/kubectl/generate/versioned/run.go b/pkg/kubectl/generate/versioned/run.go index 4a3c00258a5..4557aef7779 100644 --- a/pkg/kubectl/generate/versioned/run.go +++ b/pkg/kubectl/generate/versioned/run.go @@ -21,6 +21,7 @@ import ( "strconv" "strings" + appsv1 "k8s.io/api/apps/v1" appsv1beta1 "k8s.io/api/apps/v1beta1" batchv1 "k8s.io/api/batch/v1" batchv1beta1 "k8s.io/api/batch/v1beta1" @@ -210,6 +211,94 @@ func (DeploymentAppsV1Beta1) Generate(genericParams map[string]interface{}) (run return &deployment, nil } +type DeploymentAppsV1 struct{} + +func (DeploymentAppsV1) ParamNames() []generate.GeneratorParam { + return []generate.GeneratorParam{ + {Name: "labels", Required: false}, + {Name: "default-name", Required: false}, + {Name: "name", Required: true}, + {Name: "replicas", Required: true}, + {Name: "image", Required: true}, + {Name: "image-pull-policy", Required: false}, + {Name: "port", Required: false}, + {Name: "hostport", Required: false}, + {Name: "stdin", Required: false}, + {Name: "tty", Required: false}, + {Name: "command", Required: false}, + {Name: "args", Required: false}, + {Name: "env", Required: false}, + {Name: "requests", Required: false}, + {Name: "limits", Required: false}, + {Name: "serviceaccount", Required: false}, + } +} + +func (DeploymentAppsV1) Generate(genericParams map[string]interface{}) (runtime.Object, error) { + args, err := getArgs(genericParams) + if err != nil { + return nil, err + } + + envs, err := getEnvs(genericParams) + if err != nil { + return nil, err + } + + params, err := getParams(genericParams) + if err != nil { + return nil, err + } + + name, err := getName(params) + if err != nil { + return nil, err + } + + labels, err := getLabels(params, name) + if err != nil { + return nil, err + } + + count, err := strconv.Atoi(params["replicas"]) + if err != nil { + return nil, err + } + + podSpec, err := makePodSpec(params, name) + if err != nil { + return nil, err + } + + imagePullPolicy := v1.PullPolicy(params["image-pull-policy"]) + if err = updatePodContainers(params, args, envs, imagePullPolicy, podSpec); err != nil { + return nil, err + } + + if err := updatePodPorts(params, podSpec); err != nil { + return nil, err + } + + count32 := int32(count) + deployment := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &count32, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: *podSpec, + }, + }, + } + return &deployment, nil +} + // getLabels returns map of labels. func getLabels(params map[string]string, name string) (map[string]string, error) { labelString, found := params["labels"] diff --git a/pkg/kubectl/generate/versioned/secret.go b/pkg/kubectl/generate/versioned/secret.go index bca194139fc..02b27519f45 100644 --- a/pkg/kubectl/generate/versioned/secret.go +++ b/pkg/kubectl/generate/versioned/secret.go @@ -205,7 +205,7 @@ func handleFromFileSources(secret *v1.Secret, fileSources []string) error { } if info.IsDir() { if strings.Contains(fileSource, "=") { - return fmt.Errorf("cannot give a key name for a directory path.") + return fmt.Errorf("cannot give a key name for a directory path") } fileList, err := ioutil.ReadDir(filePath) if err != nil { @@ -265,7 +265,7 @@ func addKeyFromLiteralToSecret(secret *v1.Secret, keyName string, data []byte) e } if _, entryExists := secret.Data[keyName]; entryExists { - return fmt.Errorf("cannot add key %s, another key by that name already exists: %v.", keyName, secret.Data) + return fmt.Errorf("cannot add key %s, another key by that name already exists: %v", keyName, secret.Data) } secret.Data[keyName] = data return nil diff --git a/pkg/kubectl/generate/versioned/secret_for_docker_registry.go b/pkg/kubectl/generate/versioned/secret_for_docker_registry.go index e8996b8f2c6..db9d38448c2 100644 --- a/pkg/kubectl/generate/versioned/secret_for_docker_registry.go +++ b/pkg/kubectl/generate/versioned/secret_for_docker_registry.go @@ -97,11 +97,11 @@ func (s SecretForDockerRegistryGeneratorV1) StructuredGenerate() (runtime.Object } } if len(s.FileSources) == 0 { - dockercfgJsonContent, err := handleDockerCfgJsonContent(s.Username, s.Password, s.Email, s.Server) + dockercfgJSONContent, err := handleDockerCfgJSONContent(s.Username, s.Password, s.Email, s.Server) if err != nil { return nil, err } - secret.Data[v1.DockerConfigJsonKey] = dockercfgJsonContent + secret.Data[v1.DockerConfigJsonKey] = dockercfgJSONContent } if s.AppendHash { h, err := hash.SecretHash(secret) @@ -146,24 +146,24 @@ func (s SecretForDockerRegistryGeneratorV1) validate() error { return nil } -// handleDockerCfgJsonContent serializes a ~/.docker/config.json file -func handleDockerCfgJsonContent(username, password, email, server string) ([]byte, error) { +// handleDockerCfgJSONContent serializes a ~/.docker/config.json file +func handleDockerCfgJSONContent(username, password, email, server string) ([]byte, error) { dockercfgAuth := DockerConfigEntry{ Username: username, Password: password, Email: email, } - dockerCfgJson := DockerConfigJson{ + dockerCfgJSON := DockerConfigJSON{ Auths: map[string]DockerConfigEntry{server: dockercfgAuth}, } - return json.Marshal(dockerCfgJson) + return json.Marshal(dockerCfgJSON) } -// DockerConfigJson represents a local docker auth config file +// DockerConfigJSON represents a local docker auth config file // for pulling images. -type DockerConfigJson struct { +type DockerConfigJSON struct { Auths DockerConfig `json:"auths"` // +optional HttpHeaders map[string]string `json:"HttpHeaders,omitempty"` diff --git a/pkg/kubectl/generate/versioned/secret_for_docker_registry_test.go b/pkg/kubectl/generate/versioned/secret_for_docker_registry_test.go index 3cb73f68f49..2b90e186ef9 100644 --- a/pkg/kubectl/generate/versioned/secret_for_docker_registry_test.go +++ b/pkg/kubectl/generate/versioned/secret_for_docker_registry_test.go @@ -26,11 +26,11 @@ import ( func TestSecretForDockerRegistryGenerate(t *testing.T) { username, password, email, server := "test-user", "test-password", "test-user@example.org", "https://index.docker.io/v1/" - secretData, err := handleDockerCfgJsonContent(username, password, email, server) + secretData, err := handleDockerCfgJSONContent(username, password, email, server) if err != nil { t.Errorf("unexpected error: %v", err) } - secretDataNoEmail, err := handleDockerCfgJsonContent(username, password, "", server) + secretDataNoEmail, err := handleDockerCfgJSONContent(username, password, "", server) if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/pkg/kubectl/generate/versioned/service.go b/pkg/kubectl/generate/versioned/service.go index bea484098e6..39d084bf9ff 100644 --- a/pkg/kubectl/generate/versioned/service.go +++ b/pkg/kubectl/generate/versioned/service.go @@ -88,7 +88,7 @@ func generateService(genericParams map[string]interface{}) (runtime.Object, erro } selectorString, found := params["selector"] if !found || len(selectorString) == 0 { - return nil, fmt.Errorf("'selector' is a required parameter.") + return nil, fmt.Errorf("'selector' is a required parameter") } selector, err := generate.ParseLabels(selectorString) if err != nil { @@ -108,7 +108,7 @@ func generateService(genericParams map[string]interface{}) (runtime.Object, erro if !found || len(name) == 0 { name, found = params["default-name"] if !found || len(name) == 0 { - return nil, fmt.Errorf("'name' is a required parameter.") + return nil, fmt.Errorf("'name' is a required parameter") } } @@ -136,7 +136,7 @@ func generateService(genericParams map[string]interface{}) (runtime.Object, erro if portString, found = params["ports"]; !found { portString, found = params["port"] if !found && !isHeadlessService { - return nil, fmt.Errorf("'ports' or 'port' is a required parameter.") + return nil, fmt.Errorf("'ports' or 'port' is a required parameter") } } diff --git a/pkg/kubectl/history.go b/pkg/kubectl/history.go index be49c2cf94a..a1a4829aeb6 100644 --- a/pkg/kubectl/history.go +++ b/pkg/kubectl/history.go @@ -34,12 +34,10 @@ import ( "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/kubernetes" clientappsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" - api "k8s.io/kubernetes/pkg/apis/core" - apiv1 "k8s.io/kubernetes/pkg/apis/core/v1" kapps "k8s.io/kubernetes/pkg/kubectl/apps" + describe "k8s.io/kubernetes/pkg/kubectl/describe/versioned" deploymentutil "k8s.io/kubernetes/pkg/kubectl/util/deployment" sliceutil "k8s.io/kubernetes/pkg/kubectl/util/slice" - printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" ) const ( @@ -168,12 +166,8 @@ func (h *DeploymentHistoryViewer) ViewHistory(namespace, name string, revision i func printTemplate(template *corev1.PodTemplateSpec) (string, error) { buf := bytes.NewBuffer([]byte{}) - internalTemplate := &api.PodTemplateSpec{} - if err := apiv1.Convert_v1_PodTemplateSpec_To_core_PodTemplateSpec(template, internalTemplate, nil); err != nil { - return "", fmt.Errorf("failed to convert podtemplate, %v", err) - } - w := printersinternal.NewPrefixWriter(buf) - printersinternal.DescribePodTemplate(internalTemplate, w) + w := describe.NewPrefixWriter(buf) + describe.DescribePodTemplate(template, w) return buf.String(), nil } diff --git a/pkg/kubectl/metricsutil/metrics_client.go b/pkg/kubectl/metricsutil/metrics_client.go index 7b7c13d0b88..73399b233b0 100644 --- a/pkg/kubectl/metricsutil/metrics_client.go +++ b/pkg/kubectl/metricsutil/metrics_client.go @@ -64,7 +64,7 @@ func NewHeapsterMetricsClient(svcClient corev1client.ServicesGetter, namespace, } } -func podMetricsUrl(namespace string, name string) (string, error) { +func podMetricsURL(namespace string, name string) (string, error) { if namespace == metav1.NamespaceAll { return fmt.Sprintf("%s/pods", metricsRoot), nil } @@ -83,7 +83,7 @@ func podMetricsUrl(namespace string, name string) (string, error) { return fmt.Sprintf("%s/namespaces/%s/pods/%s", metricsRoot, namespace, name), nil } -func nodeMetricsUrl(name string) (string, error) { +func nodeMetricsURL(name string) (string, error) { if len(name) > 0 { errs := validation.NameIsDNSSubdomain(name, false) if len(errs) > 0 { @@ -96,7 +96,7 @@ func nodeMetricsUrl(name string) (string, error) { func (cli *HeapsterMetricsClient) GetNodeMetrics(nodeName string, selector string) (*metricsapi.NodeMetricsList, error) { params := map[string]string{"labelSelector": selector} - path, err := nodeMetricsUrl(nodeName) + path, err := nodeMetricsURL(nodeName) if err != nil { return nil, err } @@ -130,7 +130,7 @@ func (cli *HeapsterMetricsClient) GetPodMetrics(namespace string, podName string if allNamespaces { namespace = metav1.NamespaceAll } - path, err := podMetricsUrl(namespace, podName) + path, err := podMetricsURL(namespace, podName) if err != nil { return nil, err } diff --git a/pkg/kubectl/polymorphichelpers/canbeautoscaled.go b/pkg/kubectl/polymorphichelpers/canbeautoscaled.go index 696e69a0f46..c91e816f7d2 100644 --- a/pkg/kubectl/polymorphichelpers/canbeautoscaled.go +++ b/pkg/kubectl/polymorphichelpers/canbeautoscaled.go @@ -31,6 +31,7 @@ func canBeAutoscaled(kind schema.GroupKind) error { corev1.SchemeGroupVersion.WithKind("ReplicationController").GroupKind(), appsv1.SchemeGroupVersion.WithKind("Deployment").GroupKind(), appsv1.SchemeGroupVersion.WithKind("ReplicaSet").GroupKind(), + appsv1.SchemeGroupVersion.WithKind("StatefulSet").GroupKind(), extensionsv1beta1.SchemeGroupVersion.WithKind("Deployment").GroupKind(), extensionsv1beta1.SchemeGroupVersion.WithKind("ReplicaSet").GroupKind(): // nothing to do here diff --git a/pkg/kubectl/polymorphichelpers/canbeautoscaled_test.go b/pkg/kubectl/polymorphichelpers/canbeautoscaled_test.go index a4cbe03f9e0..b1e1fc92f0e 100644 --- a/pkg/kubectl/polymorphichelpers/canbeautoscaled_test.go +++ b/pkg/kubectl/polymorphichelpers/canbeautoscaled_test.go @@ -38,6 +38,10 @@ func TestCanBeAutoscaled(t *testing.T) { kind: appsv1.SchemeGroupVersion.WithKind("Deployment").GroupKind(), expectErr: false, }, + { + kind: appsv1.SchemeGroupVersion.WithKind("StatefulSet").GroupKind(), + expectErr: false, + }, { kind: extensionsv1beta1.SchemeGroupVersion.WithKind("ReplicaSet").GroupKind(), expectErr: false, diff --git a/pkg/kubectl/proxy/BUILD b/pkg/kubectl/proxy/BUILD index 7d836c711bf..a272308651a 100644 --- a/pkg/kubectl/proxy/BUILD +++ b/pkg/kubectl/proxy/BUILD @@ -26,7 +26,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/proxy:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/transport:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/proxy/proxy_server.go b/pkg/kubectl/proxy/proxy_server.go index 7a8fee0e59c..85fe8fc8fd8 100644 --- a/pkg/kubectl/proxy/proxy_server.go +++ b/pkg/kubectl/proxy/proxy_server.go @@ -26,11 +26,11 @@ import ( "strings" "time" - "github.com/golang/glog" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/proxy" "k8s.io/client-go/rest" "k8s.io/client-go/transport" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubectl/util" ) @@ -87,7 +87,7 @@ func MakeRegexpArray(str string) ([]*regexp.Regexp, error) { func MakeRegexpArrayOrDie(str string) []*regexp.Regexp { result, err := MakeRegexpArray(str) if err != nil { - glog.Fatalf("Error compiling re: %v", err) + klog.Fatalf("Error compiling re: %v", err) } return result } @@ -95,7 +95,7 @@ func MakeRegexpArrayOrDie(str string) []*regexp.Regexp { func matchesRegexp(str string, regexps []*regexp.Regexp) bool { for _, re := range regexps { if re.MatchString(str) { - glog.V(6).Infof("%v matched %s", str, re) + klog.V(6).Infof("%v matched %s", str, re) return true } } @@ -135,11 +135,11 @@ func extractHost(header string) (host string) { func (f *FilterServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { host := extractHost(req.Host) if f.accept(req.Method, req.URL.Path, host) { - glog.V(3).Infof("Filter accepting %v %v %v", req.Method, req.URL.Path, host) + klog.V(3).Infof("Filter accepting %v %v %v", req.Method, req.URL.Path, host) f.delegate.ServeHTTP(rw, req) return } - glog.V(3).Infof("Filter rejecting %v %v %v", req.Method, req.URL.Path, host) + klog.V(3).Infof("Filter rejecting %v %v %v", req.Method, req.URL.Path, host) http.Error(rw, http.StatusText(http.StatusForbidden), http.StatusForbidden) } @@ -151,7 +151,7 @@ type Server struct { type responder struct{} func (r *responder) Error(w http.ResponseWriter, req *http.Request, err error) { - glog.Errorf("Error while proxying request: %v", err) + klog.Errorf("Error while proxying request: %v", err) http.Error(w, err.Error(), http.StatusInternalServerError) } diff --git a/pkg/kubectl/rolling_updater.go b/pkg/kubectl/rolling_updater.go index 58741483805..b6df08c4120 100644 --- a/pkg/kubectl/rolling_updater.go +++ b/pkg/kubectl/rolling_updater.go @@ -54,7 +54,7 @@ func valOrZero(val *int32) int32 { const ( kubectlAnnotationPrefix = "kubectl.kubernetes.io/" - sourceIdAnnotation = kubectlAnnotationPrefix + "update-source-id" + sourceIDAnnotation = kubectlAnnotationPrefix + "update-source-id" desiredReplicasAnnotation = kubectlAnnotationPrefix + "desired-replicas" originalReplicasAnnotation = kubectlAnnotationPrefix + "original-replicas" nextControllerAnnotation = kubectlAnnotationPrefix + "next-controller-id" @@ -135,7 +135,7 @@ type RollingUpdater struct { scaleAndWait func(rc *corev1.ReplicationController, retry *RetryParams, wait *RetryParams) (*corev1.ReplicationController, error) //getOrCreateTargetController gets and validates an existing controller or //makes a new one. - getOrCreateTargetController func(controller *corev1.ReplicationController, sourceId string) (*corev1.ReplicationController, bool, error) + getOrCreateTargetController func(controller *corev1.ReplicationController, sourceID string) (*corev1.ReplicationController, bool, error) // cleanup performs post deployment cleanup tasks for newRc and oldRc. cleanup func(oldRc, newRc *corev1.ReplicationController, config *RollingUpdaterConfig) error // getReadyPods returns the amount of old and new ready pods. @@ -188,8 +188,8 @@ func (r *RollingUpdater) Update(config *RollingUpdaterConfig) error { // Find an existing controller (for continuing an interrupted update) or // create a new one if necessary. - sourceId := fmt.Sprintf("%s:%s", oldRc.Name, oldRc.UID) - newRc, existed, err := r.getOrCreateTargetController(config.NewRc, sourceId) + sourceID := fmt.Sprintf("%s:%s", oldRc.Name, oldRc.UID) + newRc, existed, err := r.getOrCreateTargetController(config.NewRc, sourceID) if err != nil { return err } @@ -458,14 +458,14 @@ func (r *RollingUpdater) readyPods(oldRc, newRc *corev1.ReplicationController, m } // getOrCreateTargetControllerWithClient looks for an existing controller with -// sourceId. If found, the existing controller is returned with true +// sourceID. If found, the existing controller is returned with true // indicating that the controller already exists. If the controller isn't // found, a new one is created and returned along with false indicating the // controller was created. // -// Existing controllers are validated to ensure their sourceIdAnnotation -// matches sourceId; if there's a mismatch, an error is returned. -func (r *RollingUpdater) getOrCreateTargetControllerWithClient(controller *corev1.ReplicationController, sourceId string) (*corev1.ReplicationController, bool, error) { +// Existing controllers are validated to ensure their sourceIDAnnotation +// matches sourceID; if there's a mismatch, an error is returned. +func (r *RollingUpdater) getOrCreateTargetControllerWithClient(controller *corev1.ReplicationController, sourceID string) (*corev1.ReplicationController, bool, error) { existingRc, err := r.existingController(controller) if err != nil { if !errors.IsNotFound(err) { @@ -474,24 +474,24 @@ func (r *RollingUpdater) getOrCreateTargetControllerWithClient(controller *corev return nil, false, err } if valOrZero(controller.Spec.Replicas) <= 0 { - return nil, false, fmt.Errorf("Invalid controller spec for %s; required: > 0 replicas, actual: %d\n", controller.Name, valOrZero(controller.Spec.Replicas)) + return nil, false, fmt.Errorf("Invalid controller spec for %s; required: > 0 replicas, actual: %d", controller.Name, valOrZero(controller.Spec.Replicas)) } // The controller wasn't found, so create it. if controller.Annotations == nil { controller.Annotations = map[string]string{} } controller.Annotations[desiredReplicasAnnotation] = fmt.Sprintf("%d", valOrZero(controller.Spec.Replicas)) - controller.Annotations[sourceIdAnnotation] = sourceId + controller.Annotations[sourceIDAnnotation] = sourceID controller.Spec.Replicas = newInt32Ptr(0) newRc, err := r.rcClient.ReplicationControllers(r.ns).Create(controller) return newRc, false, err } // Validate and use the existing controller. annotations := existingRc.Annotations - source := annotations[sourceIdAnnotation] + source := annotations[sourceIDAnnotation] _, ok := annotations[desiredReplicasAnnotation] - if source != sourceId || !ok { - return nil, false, fmt.Errorf("Missing/unexpected annotations for controller %s, expected %s : %s", controller.Name, sourceId, annotations) + if source != sourceID || !ok { + return nil, false, fmt.Errorf("Missing/unexpected annotations for controller %s, expected %s : %s", controller.Name, sourceID, annotations) } return existingRc, true, nil } @@ -517,7 +517,7 @@ func (r *RollingUpdater) cleanupWithClients(oldRc, newRc *corev1.ReplicationCont return err } applyUpdate := func(rc *corev1.ReplicationController) { - delete(rc.Annotations, sourceIdAnnotation) + delete(rc.Annotations, sourceIDAnnotation) delete(rc.Annotations, desiredReplicasAnnotation) } if newRc, err = updateRcWithRetries(r.rcClient, r.ns, newRc, applyUpdate); err != nil { @@ -662,7 +662,7 @@ func AbortRollingUpdate(c *RollingUpdaterConfig) error { if c.NewRc.Annotations == nil { c.NewRc.Annotations = map[string]string{} } - c.NewRc.Annotations[sourceIdAnnotation] = fmt.Sprintf("%s:%s", c.OldRc.Name, c.OldRc.UID) + c.NewRc.Annotations[sourceIDAnnotation] = fmt.Sprintf("%s:%s", c.OldRc.Name, c.OldRc.UID) // Use the original value since the replica count change from old to new // could be asymmetric. If we don't know the original count, we can't safely @@ -841,7 +841,7 @@ func FindSourceController(r corev1client.ReplicationControllersGetter, namespace } for ix := range list.Items { rc := &list.Items[ix] - if rc.Annotations != nil && strings.HasPrefix(rc.Annotations[sourceIdAnnotation], name) { + if rc.Annotations != nil && strings.HasPrefix(rc.Annotations[sourceIDAnnotation], name) { return rc, nil } } diff --git a/pkg/kubectl/rolling_updater_test.go b/pkg/kubectl/rolling_updater_test.go index be61b20bd04..824f2ac82c1 100644 --- a/pkg/kubectl/rolling_updater_test.go +++ b/pkg/kubectl/rolling_updater_test.go @@ -86,7 +86,7 @@ func newRc(replicas int, desired int) *corev1.ReplicationController { Name: "foo-v2", Annotations: map[string]string{ desiredReplicasAnnotation: fmt.Sprintf("%d", desired), - sourceIdAnnotation: "foo-v1:7764ae47-9092-11e4-8393-42010af018ff", + sourceIDAnnotation: "foo-v1:7764ae47-9092-11e4-8393-42010af018ff", }, } return rc @@ -812,7 +812,7 @@ Scaling foo-v2 up to 2 rc.Status.Replicas = *rc.Spec.Replicas return rc, nil }, - getOrCreateTargetController: func(controller *corev1.ReplicationController, sourceId string) (*corev1.ReplicationController, bool, error) { + getOrCreateTargetController: func(controller *corev1.ReplicationController, sourceID string) (*corev1.ReplicationController, bool, error) { // Simulate a create vs. update of an existing controller. return tt.newRc, tt.newRcExists, nil }, @@ -865,7 +865,7 @@ func TestUpdate_progressTimeout(t *testing.T) { // Do nothing. return rc, nil }, - getOrCreateTargetController: func(controller *corev1.ReplicationController, sourceId string) (*corev1.ReplicationController, bool, error) { + getOrCreateTargetController: func(controller *corev1.ReplicationController, sourceID string) (*corev1.ReplicationController, bool, error) { return newRc, false, nil }, cleanup: func(oldRc, newRc *corev1.ReplicationController, config *RollingUpdaterConfig) error { @@ -909,7 +909,7 @@ func TestUpdate_assignOriginalAnnotation(t *testing.T) { scaleAndWait: func(rc *corev1.ReplicationController, retry *RetryParams, wait *RetryParams) (*corev1.ReplicationController, error) { return rc, nil }, - getOrCreateTargetController: func(controller *corev1.ReplicationController, sourceId string) (*corev1.ReplicationController, bool, error) { + getOrCreateTargetController: func(controller *corev1.ReplicationController, sourceID string) (*corev1.ReplicationController, bool, error) { return newRc, false, nil }, cleanup: func(oldRc, newRc *corev1.ReplicationController, config *RollingUpdaterConfig) error { @@ -1244,7 +1244,7 @@ func TestFindSourceController(t *testing.T) { Namespace: metav1.NamespaceDefault, Name: "foo", Annotations: map[string]string{ - sourceIdAnnotation: "bar:1234", + sourceIDAnnotation: "bar:1234", }, }, } @@ -1253,7 +1253,7 @@ func TestFindSourceController(t *testing.T) { Namespace: metav1.NamespaceDefault, Name: "bar", Annotations: map[string]string{ - sourceIdAnnotation: "foo:12345", + sourceIDAnnotation: "foo:12345", }, }, } @@ -1262,7 +1262,7 @@ func TestFindSourceController(t *testing.T) { Namespace: metav1.NamespaceDefault, Name: "baz", Annotations: map[string]string{ - sourceIdAnnotation: "baz:45667", + sourceIDAnnotation: "baz:45667", }, }, } diff --git a/pkg/kubectl/scale.go b/pkg/kubectl/scale.go index f1d46b7a110..54f96fd2c34 100644 --- a/pkg/kubectl/scale.go +++ b/pkg/kubectl/scale.go @@ -175,7 +175,7 @@ func scaleHasDesiredReplicas(sClient scaleclient.ScalesGetter, gr schema.GroupRe // or returns error when timeout happens func WaitForScaleHasDesiredReplicas(sClient scaleclient.ScalesGetter, gr schema.GroupResource, resourceName string, namespace string, newSize uint, waitForReplicas *RetryParams) error { if waitForReplicas == nil { - return fmt.Errorf("waitForReplicas parameter cannot be nil!") + return fmt.Errorf("waitForReplicas parameter cannot be nil") } err := wait.PollImmediate( waitForReplicas.Interval, diff --git a/pkg/kubectl/util/BUILD b/pkg/kubectl/util/BUILD index f1d00a04a8c..fc010ff5997 100644 --- a/pkg/kubectl/util/BUILD +++ b/pkg/kubectl/util/BUILD @@ -66,13 +66,20 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//pkg/kubectl/util/certificate:all-srcs", "//pkg/kubectl/util/deployment:all-srcs", + "//pkg/kubectl/util/event:all-srcs", + "//pkg/kubectl/util/fieldpath:all-srcs", "//pkg/kubectl/util/hash:all-srcs", "//pkg/kubectl/util/i18n:all-srcs", "//pkg/kubectl/util/logs:all-srcs", "//pkg/kubectl/util/podutils:all-srcs", "//pkg/kubectl/util/printers:all-srcs", + "//pkg/kubectl/util/qos:all-srcs", + "//pkg/kubectl/util/rbac:all-srcs", + "//pkg/kubectl/util/resource:all-srcs", "//pkg/kubectl/util/slice:all-srcs", + "//pkg/kubectl/util/storage:all-srcs", "//pkg/kubectl/util/templates:all-srcs", "//pkg/kubectl/util/term:all-srcs", ], diff --git a/pkg/kubectl/util/certificate/BUILD b/pkg/kubectl/util/certificate/BUILD new file mode 100644 index 00000000000..cac97d41705 --- /dev/null +++ b/pkg/kubectl/util/certificate/BUILD @@ -0,0 +1,23 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["certificate.go"], + importpath = "k8s.io/kubernetes/pkg/kubectl/util/certificate", + visibility = ["//visibility:public"], + deps = ["//staging/src/k8s.io/api/certificates/v1beta1:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/kubectl/util/certificate/certificate.go b/pkg/kubectl/util/certificate/certificate.go new file mode 100644 index 00000000000..201958c6fca --- /dev/null +++ b/pkg/kubectl/util/certificate/certificate.go @@ -0,0 +1,42 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "crypto/x509" + "encoding/pem" + "errors" + + certificatesv1beta1 "k8s.io/api/certificates/v1beta1" +) + +// TODO(yue9944882): Remove this helper package once it's copied to k/api + +// ParseCSR extracts the CSR from the API object and decodes it. +func ParseCSR(obj *certificatesv1beta1.CertificateSigningRequest) (*x509.CertificateRequest, error) { + // extract PEM from request object + pemBytes := obj.Spec.Request + block, _ := pem.Decode(pemBytes) + if block == nil || block.Type != "CERTIFICATE REQUEST" { + return nil, errors.New("PEM block type must be CERTIFICATE REQUEST") + } + csr, err := x509.ParseCertificateRequest(block.Bytes) + if err != nil { + return nil, err + } + return csr, nil +} diff --git a/pkg/api/events/BUILD b/pkg/kubectl/util/event/BUILD similarity index 77% rename from pkg/api/events/BUILD rename to pkg/kubectl/util/event/BUILD index 7bda51ac2fb..1c0fe6300d3 100644 --- a/pkg/api/events/BUILD +++ b/pkg/kubectl/util/event/BUILD @@ -9,8 +9,8 @@ load( go_library( name = "go_default_library", srcs = ["sorted_event_list.go"], - importpath = "k8s.io/kubernetes/pkg/api/events", - deps = ["//pkg/apis/core:go_default_library"], + importpath = "k8s.io/kubernetes/pkg/kubectl/util/event", + deps = ["//staging/src/k8s.io/api/core/v1:go_default_library"], ) go_test( @@ -18,7 +18,7 @@ go_test( srcs = ["sorted_event_list_test.go"], embed = [":go_default_library"], deps = [ - "//pkg/apis/core:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", ], ) diff --git a/pkg/api/events/sorted_event_list.go b/pkg/kubectl/util/event/sorted_event_list.go similarity index 91% rename from pkg/api/events/sorted_event_list.go rename to pkg/kubectl/util/event/sorted_event_list.go index 9976c10ce73..9967f953e68 100644 --- a/pkg/api/events/sorted_event_list.go +++ b/pkg/kubectl/util/event/sorted_event_list.go @@ -14,14 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package events +package event import ( - api "k8s.io/kubernetes/pkg/apis/core" + corev1 "k8s.io/api/core/v1" ) // SortableEvents implements sort.Interface for []api.Event based on the Timestamp field -type SortableEvents []api.Event +type SortableEvents []corev1.Event func (list SortableEvents) Len() int { return len(list) diff --git a/pkg/api/events/sorted_event_list_test.go b/pkg/kubectl/util/event/sorted_event_list_test.go similarity index 80% rename from pkg/api/events/sorted_event_list_test.go rename to pkg/kubectl/util/event/sorted_event_list_test.go index 687f6fa19f8..5c1d79257d7 100644 --- a/pkg/api/events/sorted_event_list_test.go +++ b/pkg/kubectl/util/event/sorted_event_list_test.go @@ -14,43 +14,43 @@ See the License for the specific language governing permissions and limitations under the License. */ -package events +package event import ( "sort" "testing" "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "k8s.io/kubernetes/pkg/apis/core" ) func TestSortableEvents(t *testing.T) { // Arrange - list := SortableEvents([]api.Event{ + list := SortableEvents([]corev1.Event{ { - Source: api.EventSource{Component: "kubelet"}, + Source: corev1.EventSource{Component: "kubelet"}, Message: "Item 1", FirstTimestamp: metav1.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)), LastTimestamp: metav1.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)), Count: 1, - Type: api.EventTypeNormal, + Type: corev1.EventTypeNormal, }, { - Source: api.EventSource{Component: "scheduler"}, + Source: corev1.EventSource{Component: "scheduler"}, Message: "Item 2", FirstTimestamp: metav1.NewTime(time.Date(1987, time.June, 17, 0, 0, 0, 0, time.UTC)), LastTimestamp: metav1.NewTime(time.Date(1987, time.June, 17, 0, 0, 0, 0, time.UTC)), Count: 1, - Type: api.EventTypeNormal, + Type: corev1.EventTypeNormal, }, { - Source: api.EventSource{Component: "kubelet"}, + Source: corev1.EventSource{Component: "kubelet"}, Message: "Item 3", FirstTimestamp: metav1.NewTime(time.Date(2002, time.December, 25, 0, 0, 0, 0, time.UTC)), LastTimestamp: metav1.NewTime(time.Date(2002, time.December, 25, 0, 0, 0, 0, time.UTC)), Count: 1, - Type: api.EventTypeNormal, + Type: corev1.EventTypeNormal, }, }) diff --git a/pkg/kubectl/util/fieldpath/BUILD b/pkg/kubectl/util/fieldpath/BUILD new file mode 100644 index 00000000000..188a07d00d4 --- /dev/null +++ b/pkg/kubectl/util/fieldpath/BUILD @@ -0,0 +1,27 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["fieldpath.go"], + importpath = "k8s.io/kubernetes/pkg/kubectl/util/fieldpath", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/kubectl/util/fieldpath/fieldpath.go b/pkg/kubectl/util/fieldpath/fieldpath.go new file mode 100644 index 00000000000..efd08cde0f5 --- /dev/null +++ b/pkg/kubectl/util/fieldpath/fieldpath.go @@ -0,0 +1,111 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fieldpath + +import ( + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation" +) + +// TODO(yue9944882): Remove this helper package once it's copied to k/apimachinery + +// FormatMap formats map[string]string to a string. +func FormatMap(m map[string]string) (fmtStr string) { + // output with keys in sorted order to provide stable output + keys := sets.NewString() + for key := range m { + keys.Insert(key) + } + for _, key := range keys.List() { + fmtStr += fmt.Sprintf("%v=%q\n", key, m[key]) + } + fmtStr = strings.TrimSuffix(fmtStr, "\n") + + return +} + +// ExtractFieldPathAsString extracts the field from the given object +// and returns it as a string. The object must be a pointer to an +// API type. +func ExtractFieldPathAsString(obj interface{}, fieldPath string) (string, error) { + accessor, err := meta.Accessor(obj) + if err != nil { + return "", nil + } + + if path, subscript, ok := SplitMaybeSubscriptedPath(fieldPath); ok { + switch path { + case "metadata.annotations": + if errs := validation.IsQualifiedName(strings.ToLower(subscript)); len(errs) != 0 { + return "", fmt.Errorf("invalid key subscript in %s: %s", fieldPath, strings.Join(errs, ";")) + } + return accessor.GetAnnotations()[subscript], nil + case "metadata.labels": + if errs := validation.IsQualifiedName(subscript); len(errs) != 0 { + return "", fmt.Errorf("invalid key subscript in %s: %s", fieldPath, strings.Join(errs, ";")) + } + return accessor.GetLabels()[subscript], nil + default: + return "", fmt.Errorf("fieldPath %q does not support subscript", fieldPath) + } + } + + switch fieldPath { + case "metadata.annotations": + return FormatMap(accessor.GetAnnotations()), nil + case "metadata.labels": + return FormatMap(accessor.GetLabels()), nil + case "metadata.name": + return accessor.GetName(), nil + case "metadata.namespace": + return accessor.GetNamespace(), nil + case "metadata.uid": + return string(accessor.GetUID()), nil + } + + return "", fmt.Errorf("unsupported fieldPath: %v", fieldPath) +} + +// SplitMaybeSubscriptedPath checks whether the specified fieldPath is +// subscripted, and +// - if yes, this function splits the fieldPath into path and subscript, and +// returns (path, subscript, true). +// - if no, this function returns (fieldPath, "", false). +// +// Example inputs and outputs: +// - "metadata.annotations['myKey']" --> ("metadata.annotations", "myKey", true) +// - "metadata.annotations['a[b]c']" --> ("metadata.annotations", "a[b]c", true) +// - "metadata.labels['']" --> ("metadata.labels", "", true) +// - "metadata.labels" --> ("metadata.labels", "", false) +func SplitMaybeSubscriptedPath(fieldPath string) (string, string, bool) { + if !strings.HasSuffix(fieldPath, "']") { + return fieldPath, "", false + } + s := strings.TrimSuffix(fieldPath, "']") + parts := strings.SplitN(s, "['", 2) + if len(parts) < 2 { + return fieldPath, "", false + } + if len(parts[0]) == 0 { + return fieldPath, "", false + } + return parts[0], parts[1], true +} diff --git a/pkg/kubectl/util/i18n/BUILD b/pkg/kubectl/util/i18n/BUILD index fd48ae70cb9..7cfcf07abed 100644 --- a/pkg/kubectl/util/i18n/BUILD +++ b/pkg/kubectl/util/i18n/BUILD @@ -13,7 +13,7 @@ go_library( deps = [ "//pkg/kubectl/generated:go_default_library", "//vendor/github.com/chai2010/gettext-go/gettext:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/util/i18n/i18n.go b/pkg/kubectl/util/i18n/i18n.go index 69d742ca243..a4ff9ac036d 100644 --- a/pkg/kubectl/util/i18n/i18n.go +++ b/pkg/kubectl/util/i18n/i18n.go @@ -27,7 +27,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/generated" "github.com/chai2010/gettext-go/gettext" - "github.com/golang/glog" + "k8s.io/klog" ) var knownTranslations = map[string][]string{ @@ -61,12 +61,12 @@ func loadSystemLanguage() string { } if langStr == "" { - glog.V(3).Infof("Couldn't find the LC_ALL, LC_MESSAGES or LANG environment variables, defaulting to en_US") + klog.V(3).Infof("Couldn't find the LC_ALL, LC_MESSAGES or LANG environment variables, defaulting to en_US") return "default" } pieces := strings.Split(langStr, ".") if len(pieces) != 2 { - glog.V(3).Infof("Unexpected system language (%s), defaulting to en_US", langStr) + klog.V(3).Infof("Unexpected system language (%s), defaulting to en_US", langStr) return "default" } return pieces[0] @@ -83,7 +83,7 @@ func findLanguage(root string, getLanguageFn func() string) string { } } } - glog.V(3).Infof("Couldn't find translations for %s, using default", langStr) + klog.V(3).Infof("Couldn't find translations for %s, using default", langStr) return "default" } @@ -101,7 +101,7 @@ func LoadTranslations(root string, getLanguageFn func() string) error { fmt.Sprintf("%s/%s/LC_MESSAGES/k8s.mo", root, langStr), } - glog.V(3).Infof("Setting language to %s", langStr) + klog.V(3).Infof("Setting language to %s", langStr) // TODO: list the directory and load all files. buf := new(bytes.Buffer) w := zip.NewWriter(buf) diff --git a/pkg/kubectl/util/logs/BUILD b/pkg/kubectl/util/logs/BUILD index 4a896ce16b6..9813e0015db 100644 --- a/pkg/kubectl/util/logs/BUILD +++ b/pkg/kubectl/util/logs/BUILD @@ -11,8 +11,8 @@ go_library( importpath = "k8s.io/kubernetes/pkg/kubectl/util/logs", deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubectl/util/logs/logs.go b/pkg/kubectl/util/logs/logs.go index 2b33372b5ab..200416453e1 100644 --- a/pkg/kubectl/util/logs/logs.go +++ b/pkg/kubectl/util/logs/logs.go @@ -21,41 +21,42 @@ import ( "log" "time" - "github.com/golang/glog" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" ) var logFlushFreq = pflag.Duration("log-flush-frequency", 5*time.Second, "Maximum number of seconds between log flushes") // TODO(thockin): This is temporary until we agree on log dirs and put those into each cmd. func init() { + klog.InitFlags(flag.CommandLine) flag.Set("logtostderr", "true") } -// GlogWriter serves as a bridge between the standard log package and the glog package. -type GlogWriter struct{} +// KlogWriter serves as a bridge between the standard log package and the glog package. +type KlogWriter struct{} // Write implements the io.Writer interface. -func (writer GlogWriter) Write(data []byte) (n int, err error) { - glog.InfoDepth(1, string(data)) +func (writer KlogWriter) Write(data []byte) (n int, err error) { + klog.InfoDepth(1, string(data)) return len(data), nil } // InitLogs initializes logs the way we want for kubernetes. func InitLogs() { - log.SetOutput(GlogWriter{}) + log.SetOutput(KlogWriter{}) log.SetFlags(0) // The default glog flush interval is 5 seconds. - go wait.Until(glog.Flush, *logFlushFreq, wait.NeverStop) + go wait.Until(klog.Flush, *logFlushFreq, wait.NeverStop) } // FlushLogs flushes logs immediately. func FlushLogs() { - glog.Flush() + klog.Flush() } -// NewLogger creates a new log.Logger which sends logs to glog.Info. +// NewLogger creates a new log.Logger which sends logs to klog.Info. func NewLogger(prefix string) *log.Logger { - return log.New(GlogWriter{}, prefix, 0) + return log.New(KlogWriter{}, prefix, 0) } diff --git a/pkg/kubectl/util/qos/BUILD b/pkg/kubectl/util/qos/BUILD new file mode 100644 index 00000000000..bed95210fc2 --- /dev/null +++ b/pkg/kubectl/util/qos/BUILD @@ -0,0 +1,27 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["qos.go"], + importpath = "k8s.io/kubernetes/pkg/kubectl/util/qos", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/kubectl/util/qos/qos.go b/pkg/kubectl/util/qos/qos.go new file mode 100644 index 00000000000..73b25acac2d --- /dev/null +++ b/pkg/kubectl/util/qos/qos.go @@ -0,0 +1,95 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qos + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/sets" +) + +var supportedQoSComputeResources = sets.NewString(string(corev1.ResourceCPU), string(corev1.ResourceMemory)) + +func isSupportedQoSComputeResource(name corev1.ResourceName) bool { + return supportedQoSComputeResources.Has(string(name)) +} + +// GetPodQOS returns the QoS class of a pod. +// A pod is besteffort if none of its containers have specified any requests or limits. +// A pod is guaranteed only when requests and limits are specified for all the containers and they are equal. +// A pod is burstable if limits and requests do not match across all containers. +func GetPodQOS(pod *corev1.Pod) corev1.PodQOSClass { + requests := corev1.ResourceList{} + limits := corev1.ResourceList{} + zeroQuantity := resource.MustParse("0") + isGuaranteed := true + for _, container := range pod.Spec.Containers { + // process requests + for name, quantity := range container.Resources.Requests { + if !isSupportedQoSComputeResource(name) { + continue + } + if quantity.Cmp(zeroQuantity) == 1 { + delta := quantity.Copy() + if _, exists := requests[name]; !exists { + requests[name] = *delta + } else { + delta.Add(requests[name]) + requests[name] = *delta + } + } + } + // process limits + qosLimitsFound := sets.NewString() + for name, quantity := range container.Resources.Limits { + if !isSupportedQoSComputeResource(name) { + continue + } + if quantity.Cmp(zeroQuantity) == 1 { + qosLimitsFound.Insert(string(name)) + delta := quantity.Copy() + if _, exists := limits[name]; !exists { + limits[name] = *delta + } else { + delta.Add(limits[name]) + limits[name] = *delta + } + } + } + + if !qosLimitsFound.HasAll(string(corev1.ResourceMemory), string(corev1.ResourceCPU)) { + isGuaranteed = false + } + } + if len(requests) == 0 && len(limits) == 0 { + return corev1.PodQOSBestEffort + } + // Check is requests match limits for all resources. + if isGuaranteed { + for name, req := range requests { + if lim, exists := limits[name]; !exists || lim.Cmp(req) != 0 { + isGuaranteed = false + break + } + } + } + if isGuaranteed && + len(requests) == len(limits) { + return corev1.PodQOSGuaranteed + } + return corev1.PodQOSBurstable +} diff --git a/vendor/github.com/lpabon/godbc/BUILD b/pkg/kubectl/util/rbac/BUILD similarity index 73% rename from vendor/github.com/lpabon/godbc/BUILD rename to pkg/kubectl/util/rbac/BUILD index 31c0af8f19f..6a398bc854c 100644 --- a/vendor/github.com/lpabon/godbc/BUILD +++ b/pkg/kubectl/util/rbac/BUILD @@ -2,10 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["godbc.go"], - importmap = "k8s.io/kubernetes/vendor/github.com/lpabon/godbc", - importpath = "github.com/lpabon/godbc", + srcs = ["rbac.go"], + importpath = "k8s.io/kubernetes/pkg/kubectl/util/rbac", visibility = ["//visibility:public"], + deps = ["//staging/src/k8s.io/api/rbac/v1:go_default_library"], ) filegroup( diff --git a/pkg/kubectl/util/rbac/rbac.go b/pkg/kubectl/util/rbac/rbac.go new file mode 100644 index 00000000000..a149a51afb0 --- /dev/null +++ b/pkg/kubectl/util/rbac/rbac.go @@ -0,0 +1,128 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbac + +import ( + rbacv1 "k8s.io/api/rbac/v1" + "reflect" + "strings" +) + +type simpleResource struct { + Group string + Resource string + ResourceNameExist bool + ResourceName string +} + +// CompactRules combines rules that contain a single APIGroup/Resource, differ only by verb, and contain no other attributes. +// this is a fast check, and works well with the decomposed "missing rules" list from a Covers check. +func CompactRules(rules []rbacv1.PolicyRule) ([]rbacv1.PolicyRule, error) { + compacted := make([]rbacv1.PolicyRule, 0, len(rules)) + + simpleRules := map[simpleResource]*rbacv1.PolicyRule{} + for _, rule := range rules { + if resource, isSimple := isSimpleResourceRule(&rule); isSimple { + if existingRule, ok := simpleRules[resource]; ok { + // Add the new verbs to the existing simple resource rule + if existingRule.Verbs == nil { + existingRule.Verbs = []string{} + } + existingRule.Verbs = append(existingRule.Verbs, rule.Verbs...) + } else { + // Copy the rule to accumulate matching simple resource rules into + simpleRules[resource] = rule.DeepCopy() + } + } else { + compacted = append(compacted, rule) + } + } + + // Once we've consolidated the simple resource rules, add them to the compacted list + for _, simpleRule := range simpleRules { + compacted = append(compacted, *simpleRule) + } + + return compacted, nil +} + +// isSimpleResourceRule returns true if the given rule contains verbs, a single resource, a single API group, at most one Resource Name, and no other values +func isSimpleResourceRule(rule *rbacv1.PolicyRule) (simpleResource, bool) { + resource := simpleResource{} + + // If we have "complex" rule attributes, return early without allocations or expensive comparisons + if len(rule.ResourceNames) > 1 || len(rule.NonResourceURLs) > 0 { + return resource, false + } + // If we have multiple api groups or resources, return early + if len(rule.APIGroups) != 1 || len(rule.Resources) != 1 { + return resource, false + } + + // Test if this rule only contains APIGroups/Resources/Verbs/ResourceNames + simpleRule := &rbacv1.PolicyRule{APIGroups: rule.APIGroups, Resources: rule.Resources, Verbs: rule.Verbs, ResourceNames: rule.ResourceNames} + if !reflect.DeepEqual(simpleRule, rule) { + return resource, false + } + + if len(rule.ResourceNames) == 0 { + resource = simpleResource{Group: rule.APIGroups[0], Resource: rule.Resources[0], ResourceNameExist: false} + } else { + resource = simpleResource{Group: rule.APIGroups[0], Resource: rule.Resources[0], ResourceNameExist: true, ResourceName: rule.ResourceNames[0]} + } + + return resource, true +} + +// BreakdownRule takes a rule and builds an equivalent list of rules that each have at most one verb, one +// resource, and one resource name +func BreakdownRule(rule rbacv1.PolicyRule) []rbacv1.PolicyRule { + subrules := []rbacv1.PolicyRule{} + for _, group := range rule.APIGroups { + for _, resource := range rule.Resources { + for _, verb := range rule.Verbs { + if len(rule.ResourceNames) > 0 { + for _, resourceName := range rule.ResourceNames { + subrules = append(subrules, rbacv1.PolicyRule{APIGroups: []string{group}, Resources: []string{resource}, Verbs: []string{verb}, ResourceNames: []string{resourceName}}) + } + + } else { + subrules = append(subrules, rbacv1.PolicyRule{APIGroups: []string{group}, Resources: []string{resource}, Verbs: []string{verb}}) + } + + } + } + } + + // Non-resource URLs are unique because they only combine with verbs. + for _, nonResourceURL := range rule.NonResourceURLs { + for _, verb := range rule.Verbs { + subrules = append(subrules, rbacv1.PolicyRule{NonResourceURLs: []string{nonResourceURL}, Verbs: []string{verb}}) + } + } + + return subrules +} + +// SortableRuleSlice is used to sort rule slice +type SortableRuleSlice []rbacv1.PolicyRule + +func (s SortableRuleSlice) Len() int { return len(s) } +func (s SortableRuleSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s SortableRuleSlice) Less(i, j int) bool { + return strings.Compare(s[i].String(), s[j].String()) < 0 +} diff --git a/pkg/kubectl/util/resource/BUILD b/pkg/kubectl/util/resource/BUILD new file mode 100644 index 00000000000..bbbe532b710 --- /dev/null +++ b/pkg/kubectl/util/resource/BUILD @@ -0,0 +1,27 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["resource.go"], + importpath = "k8s.io/kubernetes/pkg/kubectl/util/resource", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/kubectl/util/resource/resource.go b/pkg/kubectl/util/resource/resource.go new file mode 100644 index 00000000000..0a6f177a647 --- /dev/null +++ b/pkg/kubectl/util/resource/resource.go @@ -0,0 +1,138 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resource + +import ( + "fmt" + "math" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/sets" +) + +// PodRequestsAndLimits returns a dictionary of all defined resources summed up for all +// containers of the pod. +func PodRequestsAndLimits(pod *corev1.Pod) (reqs, limits corev1.ResourceList) { + reqs, limits = corev1.ResourceList{}, corev1.ResourceList{} + for _, container := range pod.Spec.Containers { + addResourceList(reqs, container.Resources.Requests) + addResourceList(limits, container.Resources.Limits) + } + // init containers define the minimum of any resource + for _, container := range pod.Spec.InitContainers { + maxResourceList(reqs, container.Resources.Requests) + maxResourceList(limits, container.Resources.Limits) + } + return +} + +// addResourceList adds the resources in newList to list +func addResourceList(list, new corev1.ResourceList) { + for name, quantity := range new { + if value, ok := list[name]; !ok { + list[name] = *quantity.Copy() + } else { + value.Add(quantity) + list[name] = value + } + } +} + +// maxResourceList sets list to the greater of list/newList for every resource +// either list +func maxResourceList(list, new corev1.ResourceList) { + for name, quantity := range new { + if value, ok := list[name]; !ok { + list[name] = *quantity.Copy() + continue + } else { + if quantity.Cmp(value) > 0 { + list[name] = *quantity.Copy() + } + } + } +} + +// ExtractContainerResourceValue extracts the value of a resource +// in an already known container +func ExtractContainerResourceValue(fs *corev1.ResourceFieldSelector, container *corev1.Container) (string, error) { + divisor := resource.Quantity{} + if divisor.Cmp(fs.Divisor) == 0 { + divisor = resource.MustParse("1") + } else { + divisor = fs.Divisor + } + + switch fs.Resource { + case "limits.cpu": + return convertResourceCPUToString(container.Resources.Limits.Cpu(), divisor) + case "limits.memory": + return convertResourceMemoryToString(container.Resources.Limits.Memory(), divisor) + case "limits.ephemeral-storage": + return convertResourceEphemeralStorageToString(container.Resources.Limits.StorageEphemeral(), divisor) + case "requests.cpu": + return convertResourceCPUToString(container.Resources.Requests.Cpu(), divisor) + case "requests.memory": + return convertResourceMemoryToString(container.Resources.Requests.Memory(), divisor) + case "requests.ephemeral-storage": + return convertResourceEphemeralStorageToString(container.Resources.Requests.StorageEphemeral(), divisor) + } + + return "", fmt.Errorf("Unsupported container resource : %v", fs.Resource) +} + +// convertResourceCPUToString converts cpu value to the format of divisor and returns +// ceiling of the value. +func convertResourceCPUToString(cpu *resource.Quantity, divisor resource.Quantity) (string, error) { + c := int64(math.Ceil(float64(cpu.MilliValue()) / float64(divisor.MilliValue()))) + return strconv.FormatInt(c, 10), nil +} + +// convertResourceMemoryToString converts memory value to the format of divisor and returns +// ceiling of the value. +func convertResourceMemoryToString(memory *resource.Quantity, divisor resource.Quantity) (string, error) { + m := int64(math.Ceil(float64(memory.Value()) / float64(divisor.Value()))) + return strconv.FormatInt(m, 10), nil +} + +// convertResourceEphemeralStorageToString converts ephemeral storage value to the format of divisor and returns +// ceiling of the value. +func convertResourceEphemeralStorageToString(ephemeralStorage *resource.Quantity, divisor resource.Quantity) (string, error) { + m := int64(math.Ceil(float64(ephemeralStorage.Value()) / float64(divisor.Value()))) + return strconv.FormatInt(m, 10), nil +} + +var standardContainerResources = sets.NewString( + string(corev1.ResourceCPU), + string(corev1.ResourceMemory), + string(corev1.ResourceEphemeralStorage), +) + +// IsStandardContainerResourceName returns true if the container can make a resource request +// for the specified resource +func IsStandardContainerResourceName(str string) bool { + return standardContainerResources.Has(str) || IsHugePageResourceName(corev1.ResourceName(str)) +} + +// IsHugePageResourceName returns true if the resource name has the huge page +// resource prefix. +func IsHugePageResourceName(name corev1.ResourceName) bool { + return strings.HasPrefix(string(name), corev1.ResourceHugePagesPrefix) +} diff --git a/pkg/kubectl/util/service_port.go b/pkg/kubectl/util/service_port.go index 18960b14864..bc56ab7d6a9 100644 --- a/pkg/kubectl/util/service_port.go +++ b/pkg/kubectl/util/service_port.go @@ -23,8 +23,8 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -// Lookup containerPort number from Service port number -// It implements the handling of resolving container named port, as well as ignoring targetPort when clusterIP=None +// LookupContainerPortNumberByServicePort implements +// the handling of resolving container named port, as well as ignoring targetPort when clusterIP=None // It returns an error when a named port can't find a match (with -1 returned), or when the service does not // declare such port (with the input port number returned). func LookupContainerPortNumberByServicePort(svc v1.Service, pod v1.Pod, port int32) (int32, error) { @@ -39,12 +39,10 @@ func LookupContainerPortNumberByServicePort(svc v1.Service, pod v1.Pod, port int if svcportspec.TargetPort.IntValue() == 0 { // targetPort is omitted, and the IntValue() would be zero return svcportspec.Port, nil - } else { - return int32(svcportspec.TargetPort.IntValue()), nil } - } else { - return LookupContainerPortNumberByName(pod, svcportspec.TargetPort.String()) + return int32(svcportspec.TargetPort.IntValue()), nil } + return LookupContainerPortNumberByName(pod, svcportspec.TargetPort.String()) } return port, fmt.Errorf("Service %s does not have a service port %d", svc.Name, port) } diff --git a/pkg/kubectl/util/slice/slice.go b/pkg/kubectl/util/slice/slice.go index 8130753c300..f997d5cb405 100644 --- a/pkg/kubectl/util/slice/slice.go +++ b/pkg/kubectl/util/slice/slice.go @@ -20,5 +20,19 @@ import ( "sort" ) -// Sorts []int64 in increasing order +// SortInts64 sorts []int64 in increasing order func SortInts64(a []int64) { sort.Slice(a, func(i, j int) bool { return a[i] < a[j] }) } + +// ContainsString checks if a given slice of strings contains the provided string. +// If a modifier func is provided, it is called with the slice item before the comparation. +func ContainsString(slice []string, s string, modifier func(s string) string) bool { + for _, item := range slice { + if item == s { + return true + } + if modifier != nil && modifier(item) == s { + return true + } + } + return false +} diff --git a/staging/src/k8s.io/code-generator/cmd/openapi-gen/args/BUILD b/pkg/kubectl/util/storage/BUILD similarity index 59% rename from staging/src/k8s.io/code-generator/cmd/openapi-gen/args/BUILD rename to pkg/kubectl/util/storage/BUILD index 9ba1ad9d34a..cd1ed24f700 100644 --- a/staging/src/k8s.io/code-generator/cmd/openapi-gen/args/BUILD +++ b/pkg/kubectl/util/storage/BUILD @@ -2,13 +2,12 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["args.go"], - importmap = "k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/openapi-gen/args", - importpath = "k8s.io/code-generator/cmd/openapi-gen/args", + srcs = ["storage.go"], + importpath = "k8s.io/kubernetes/pkg/kubectl/util/storage", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/gengo/args:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", ], ) diff --git a/pkg/kubectl/util/storage/storage.go b/pkg/kubectl/util/storage/storage.go new file mode 100644 index 00000000000..c62cc4ea127 --- /dev/null +++ b/pkg/kubectl/util/storage/storage.go @@ -0,0 +1,107 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strings" +) + +// TODO(yue9944882): Remove this helper package once it's copied to k/api + +// IsDefaultStorageClassAnnotation represents a StorageClass annotation that +// marks a class as the default StorageClass +const IsDefaultStorageClassAnnotation = "storageclass.kubernetes.io/is-default-class" + +// BetaIsDefaultStorageClassAnnotation is the beta version of BetaIsDefaultStorageClassAnnotation. +const BetaIsDefaultStorageClassAnnotation = "storageclass.beta.kubernetes.io/is-default-class" + +// IsDefaultAnnotationText returns a pretty Yes/No String if +// the annotation is set +func IsDefaultAnnotationText(obj metav1.ObjectMeta) string { + if obj.Annotations[IsDefaultStorageClassAnnotation] == "true" { + return "Yes" + } + if obj.Annotations[BetaIsDefaultStorageClassAnnotation] == "true" { + return "Yes" + } + + return "No" +} + +// GetAccessModesAsString returns a string representation of an array of access modes. +// modes, when present, are always in the same order: RWO,ROX,RWX. +func GetAccessModesAsString(modes []v1.PersistentVolumeAccessMode) string { + modes = removeDuplicateAccessModes(modes) + modesStr := []string{} + if containsAccessMode(modes, v1.ReadWriteOnce) { + modesStr = append(modesStr, "RWO") + } + if containsAccessMode(modes, v1.ReadOnlyMany) { + modesStr = append(modesStr, "ROX") + } + if containsAccessMode(modes, v1.ReadWriteMany) { + modesStr = append(modesStr, "RWX") + } + return strings.Join(modesStr, ",") +} + +// removeDuplicateAccessModes returns an array of access modes without any duplicates +func removeDuplicateAccessModes(modes []v1.PersistentVolumeAccessMode) []v1.PersistentVolumeAccessMode { + accessModes := []v1.PersistentVolumeAccessMode{} + for _, m := range modes { + if !containsAccessMode(accessModes, m) { + accessModes = append(accessModes, m) + } + } + return accessModes +} + +func containsAccessMode(modes []v1.PersistentVolumeAccessMode, mode v1.PersistentVolumeAccessMode) bool { + for _, m := range modes { + if m == mode { + return true + } + } + return false +} + +// GetPersistentVolumeClass returns StorageClassName. +func GetPersistentVolumeClass(volume *v1.PersistentVolume) string { + // Use beta annotation first + if class, found := volume.Annotations[v1.BetaStorageClassAnnotation]; found { + return class + } + + return volume.Spec.StorageClassName +} + +// GetPersistentVolumeClaimClass returns StorageClassName. If no storage class was +// requested, it returns "". +func GetPersistentVolumeClaimClass(claim *v1.PersistentVolumeClaim) string { + // Use beta annotation first + if class, found := claim.Annotations[v1.BetaStorageClassAnnotation]; found { + return class + } + + if claim.Spec.StorageClassName != nil { + return *claim.Spec.StorageClassName + } + + return "" +} diff --git a/pkg/kubectl/util/umask.go b/pkg/kubectl/util/umask.go index 93e14473c22..67add4e9140 100644 --- a/pkg/kubectl/util/umask.go +++ b/pkg/kubectl/util/umask.go @@ -22,6 +22,7 @@ import ( "golang.org/x/sys/unix" ) +// Umask is a wrapper for `unix.Umask()` on non-Windows platforms func Umask(mask int) (old int, err error) { return unix.Umask(mask), nil } diff --git a/pkg/kubectl/util/umask_windows.go b/pkg/kubectl/util/umask_windows.go index 7a1ba15386f..5b4f54bb795 100644 --- a/pkg/kubectl/util/umask_windows.go +++ b/pkg/kubectl/util/umask_windows.go @@ -22,6 +22,7 @@ import ( "errors" ) +// Umask returns an error on Windows func Umask(mask int) (int, error) { return 0, errors.New("platform and architecture is not supported") } diff --git a/pkg/kubectl/util/util.go b/pkg/kubectl/util/util.go index 41427780c71..0c1973dfb8e 100644 --- a/pkg/kubectl/util/util.go +++ b/pkg/kubectl/util/util.go @@ -41,6 +41,7 @@ func ParseRFC3339(s string, nowFn func() metav1.Time) (metav1.Time, error) { return metav1.Time{Time: t}, nil } +// HashObject returns the hash of a Object hash by a Codec func HashObject(obj runtime.Object, codec runtime.Codec) (string, error) { data, err := runtime.Encode(codec, obj) if err != nil { @@ -63,11 +64,11 @@ func ParseFileSource(source string) (keyName, filePath string, err error) { case numSeparators == 0: return path.Base(filepath.ToSlash(source)), source, nil case numSeparators == 1 && strings.HasPrefix(source, "="): - return "", "", fmt.Errorf("key name for file path %v missing.", strings.TrimPrefix(source, "=")) + return "", "", fmt.Errorf("key name for file path %v missing", strings.TrimPrefix(source, "=")) case numSeparators == 1 && strings.HasSuffix(source, "="): - return "", "", fmt.Errorf("file path for key name %v missing.", strings.TrimSuffix(source, "=")) + return "", "", fmt.Errorf("file path for key name %v missing", strings.TrimSuffix(source, "=")) case numSeparators > 1: - return "", "", errors.New("Key names or file paths cannot contain '='.") + return "", "", errors.New("Key names or file paths cannot contain '='") default: components := strings.Split(source, "=") return components[0], components[1], nil diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index f75821e08e7..fd5e6391c51 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -46,7 +46,8 @@ go_library( "//pkg/kubelet/apis/config:go_default_library", "//pkg/kubelet/apis/cri:go_default_library", "//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library", - "//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library", + "//pkg/kubelet/apis/pluginregistration/v1:go_default_library", + "//pkg/kubelet/apis/podresources:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", "//pkg/kubelet/certificate:go_default_library", "//pkg/kubelet/checkpointmanager:go_default_library", @@ -117,6 +118,7 @@ go_library( "//pkg/volume/validation:go_default_library", "//staging/src/k8s.io/api/authentication/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -142,11 +144,11 @@ go_library( "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", "//third_party/forked/golang/expansion:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/groupcache/lru:go_default_library", "//vendor/github.com/google/cadvisor/events:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) @@ -164,6 +166,7 @@ go_test( "kubelet_resources_test.go", "kubelet_test.go", "kubelet_volumes_test.go", + "main_test.go", "oom_watcher_test.go", "pod_container_deletor_test.go", "pod_workers_test.go", @@ -234,6 +237,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", diff --git a/pkg/kubelet/apis/BUILD b/pkg/kubelet/apis/BUILD index c4a2fb50872..926c8add6ca 100644 --- a/pkg/kubelet/apis/BUILD +++ b/pkg/kubelet/apis/BUILD @@ -13,7 +13,9 @@ go_library( "well_known_labels.go", ], importpath = "k8s.io/kubernetes/pkg/kubelet/apis", - deps = select({ + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + ] + select({ "@io_bazel_rules_go//go/platform:windows": [ "//pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", @@ -37,8 +39,10 @@ filegroup( "//pkg/kubelet/apis/cri:all-srcs", "//pkg/kubelet/apis/deviceplugin/v1alpha:all-srcs", "//pkg/kubelet/apis/deviceplugin/v1beta1:all-srcs", + "//pkg/kubelet/apis/pluginregistration/v1:all-srcs", "//pkg/kubelet/apis/pluginregistration/v1alpha1:all-srcs", "//pkg/kubelet/apis/pluginregistration/v1beta1:all-srcs", + "//pkg/kubelet/apis/podresources:all-srcs", "//pkg/kubelet/apis/stats/v1alpha1:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/kubelet/apis/config/fuzzer/fuzzer.go b/pkg/kubelet/apis/config/fuzzer/fuzzer.go index 1154789c420..4efd0f45afd 100644 --- a/pkg/kubelet/apis/config/fuzzer/fuzzer.go +++ b/pkg/kubelet/apis/config/fuzzer/fuzzer.go @@ -62,6 +62,7 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { obj.MaxPods = 110 obj.PodPidsLimit = -1 obj.NodeStatusUpdateFrequency = metav1.Duration{Duration: 10 * time.Second} + obj.NodeStatusReportFrequency = metav1.Duration{Duration: time.Minute} obj.NodeLeaseDurationSeconds = 40 obj.CPUManagerPolicy = "none" obj.CPUManagerReconcilePeriod = obj.NodeStatusUpdateFrequency diff --git a/pkg/kubelet/apis/config/helpers_test.go b/pkg/kubelet/apis/config/helpers_test.go index 412975b98bf..0ba6dad1a5b 100644 --- a/pkg/kubelet/apis/config/helpers_test.go +++ b/pkg/kubelet/apis/config/helpers_test.go @@ -197,6 +197,7 @@ var ( "MaxOpenFiles", "MaxPods", "NodeStatusUpdateFrequency.Duration", + "NodeStatusReportFrequency.Duration", "NodeLeaseDurationSeconds", "OOMScoreAdj", "PodCIDR", diff --git a/pkg/kubelet/apis/config/types.go b/pkg/kubelet/apis/config/types.go index ab232157493..941d8374866 100644 --- a/pkg/kubelet/apis/config/types.go +++ b/pkg/kubelet/apis/config/types.go @@ -151,10 +151,16 @@ type KubeletConfiguration struct { // streamingConnectionIdleTimeout is the maximum time a streaming connection // can be idle before the connection is automatically closed. StreamingConnectionIdleTimeout metav1.Duration - // nodeStatusUpdateFrequency is the frequency that kubelet posts node - // status to master. Note: be cautious when changing the constant, it - // must work with nodeMonitorGracePeriod in nodecontroller. + // nodeStatusUpdateFrequency is the frequency that kubelet computes node + // status. If node lease feature is not enabled, it is also the frequency that + // kubelet posts node status to master. In that case, be cautious when + // changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller. NodeStatusUpdateFrequency metav1.Duration + // nodeStatusReportFrequency is the frequency that kubelet posts node + // status to master if node status does not change. Kubelet will ignore this + // frequency and post node status immediately if any change is detected. It is + // only used when node lease feature is enabled. + NodeStatusReportFrequency metav1.Duration // nodeLeaseDurationSeconds is the duration the Kubelet will set on its corresponding Lease. NodeLeaseDurationSeconds int32 // imageMinimumGCAge is the minimum age for an unused image before it is diff --git a/pkg/kubelet/apis/config/v1beta1/defaults.go b/pkg/kubelet/apis/config/v1beta1/defaults.go index d5a12f338b0..0aebb8447b8 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults.go @@ -107,6 +107,16 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura if obj.StreamingConnectionIdleTimeout == zeroDuration { obj.StreamingConnectionIdleTimeout = metav1.Duration{Duration: 4 * time.Hour} } + if obj.NodeStatusReportFrequency == zeroDuration { + // For backward compatibility, NodeStatusReportFrequency's default value is + // set to NodeStatusUpdateFrequency if NodeStatusUpdateFrequency is set + // explicitly. + if obj.NodeStatusUpdateFrequency == zeroDuration { + obj.NodeStatusReportFrequency = metav1.Duration{Duration: time.Minute} + } else { + obj.NodeStatusReportFrequency = obj.NodeStatusUpdateFrequency + } + } if obj.NodeStatusUpdateFrequency == zeroDuration { obj.NodeStatusUpdateFrequency = metav1.Duration{Duration: 10 * time.Second} } diff --git a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go index 9e67bf9e9a9..b264f2423b2 100644 --- a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go @@ -251,6 +251,7 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in out.ClusterDNS = *(*[]string)(unsafe.Pointer(&in.ClusterDNS)) out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + out.NodeStatusReportFrequency = in.NodeStatusReportFrequency out.NodeLeaseDurationSeconds = in.NodeLeaseDurationSeconds out.ImageMinimumGCAge = in.ImageMinimumGCAge if err := v1.Convert_Pointer_int32_To_int32(&in.ImageGCHighThresholdPercent, &out.ImageGCHighThresholdPercent, s); err != nil { @@ -380,6 +381,7 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in out.ClusterDNS = *(*[]string)(unsafe.Pointer(&in.ClusterDNS)) out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + out.NodeStatusReportFrequency = in.NodeStatusReportFrequency out.NodeLeaseDurationSeconds = in.NodeLeaseDurationSeconds out.ImageMinimumGCAge = in.ImageMinimumGCAge if err := v1.Convert_int32_To_Pointer_int32(&in.ImageGCHighThresholdPercent, &out.ImageGCHighThresholdPercent, s); err != nil { diff --git a/pkg/kubelet/apis/config/zz_generated.deepcopy.go b/pkg/kubelet/apis/config/zz_generated.deepcopy.go index 266a862943f..1644f968c56 100644 --- a/pkg/kubelet/apis/config/zz_generated.deepcopy.go +++ b/pkg/kubelet/apis/config/zz_generated.deepcopy.go @@ -112,6 +112,7 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { } out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + out.NodeStatusReportFrequency = in.NodeStatusReportFrequency out.ImageMinimumGCAge = in.ImageMinimumGCAge out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod out.CPUManagerReconcilePeriod = in.CPUManagerReconcilePeriod diff --git a/pkg/kubelet/apis/pluginregistration/v1/BUILD b/pkg/kubelet/apis/pluginregistration/v1/BUILD new file mode 100644 index 00000000000..644c05e1be2 --- /dev/null +++ b/pkg/kubelet/apis/pluginregistration/v1/BUILD @@ -0,0 +1,34 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = [ + "api.pb.go", + "constants.go", + ], + importpath = "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1", + deps = [ + "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library", + "//vendor/github.com/gogo/protobuf/proto:go_default_library", + "//vendor/golang.org/x/net/context:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/pkg/kubelet/apis/pluginregistration/v1/api.pb.go b/pkg/kubelet/apis/pluginregistration/v1/api.pb.go new file mode 100644 index 00000000000..d478726ab25 --- /dev/null +++ b/pkg/kubelet/apis/pluginregistration/v1/api.pb.go @@ -0,0 +1,1008 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: api.proto + +/* + Package pluginregistration is a generated protocol buffer package. + + It is generated from these files: + api.proto + + It has these top-level messages: + PluginInfo + RegistrationStatus + RegistrationStatusResponse + InfoRequest +*/ +package pluginregistration + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "github.com/gogo/protobuf/gogoproto" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +import strings "strings" +import reflect "reflect" + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +// PluginInfo is the message sent from a plugin to the Kubelet pluginwatcher for plugin registration +type PluginInfo struct { + // Type of the Plugin. CSIPlugin or DevicePlugin + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + // Plugin name that uniquely identifies the plugin for the given plugin type. + // For DevicePlugin, this is the resource name that the plugin manages and + // should follow the extended resource name convention. + // For CSI, this is the CSI driver registrar name. + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + // Optional endpoint location. If found set by Kubelet component, + // Kubelet component will use this endpoint for specific requests. + // This allows the plugin to register using one endpoint and possibly use + // a different socket for control operations. CSI uses this model to delegate + // its registration external from the plugin. + Endpoint string `protobuf:"bytes,3,opt,name=endpoint,proto3" json:"endpoint,omitempty"` + // Plugin service API versions the plugin supports. + // For DevicePlugin, this maps to the deviceplugin API versions the + // plugin supports at the given socket. + // The Kubelet component communicating with the plugin should be able + // to choose any preferred version from this list, or returns an error + // if none of the listed versions is supported. + SupportedVersions []string `protobuf:"bytes,4,rep,name=supported_versions,json=supportedVersions" json:"supported_versions,omitempty"` +} + +func (m *PluginInfo) Reset() { *m = PluginInfo{} } +func (*PluginInfo) ProtoMessage() {} +func (*PluginInfo) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{0} } + +func (m *PluginInfo) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *PluginInfo) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *PluginInfo) GetEndpoint() string { + if m != nil { + return m.Endpoint + } + return "" +} + +func (m *PluginInfo) GetSupportedVersions() []string { + if m != nil { + return m.SupportedVersions + } + return nil +} + +// RegistrationStatus is the message sent from Kubelet pluginwatcher to the plugin for notification on registration status +type RegistrationStatus struct { + // True if plugin gets registered successfully at Kubelet + PluginRegistered bool `protobuf:"varint,1,opt,name=plugin_registered,json=pluginRegistered,proto3" json:"plugin_registered,omitempty"` + // Error message in case plugin fails to register, empty string otherwise + Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` +} + +func (m *RegistrationStatus) Reset() { *m = RegistrationStatus{} } +func (*RegistrationStatus) ProtoMessage() {} +func (*RegistrationStatus) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{1} } + +func (m *RegistrationStatus) GetPluginRegistered() bool { + if m != nil { + return m.PluginRegistered + } + return false +} + +func (m *RegistrationStatus) GetError() string { + if m != nil { + return m.Error + } + return "" +} + +// RegistrationStatusResponse is sent by plugin to kubelet in response to RegistrationStatus RPC +type RegistrationStatusResponse struct { +} + +func (m *RegistrationStatusResponse) Reset() { *m = RegistrationStatusResponse{} } +func (*RegistrationStatusResponse) ProtoMessage() {} +func (*RegistrationStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{2} } + +// InfoRequest is the empty request message from Kubelet +type InfoRequest struct { +} + +func (m *InfoRequest) Reset() { *m = InfoRequest{} } +func (*InfoRequest) ProtoMessage() {} +func (*InfoRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{3} } + +func init() { + proto.RegisterType((*PluginInfo)(nil), "pluginregistration.PluginInfo") + proto.RegisterType((*RegistrationStatus)(nil), "pluginregistration.RegistrationStatus") + proto.RegisterType((*RegistrationStatusResponse)(nil), "pluginregistration.RegistrationStatusResponse") + proto.RegisterType((*InfoRequest)(nil), "pluginregistration.InfoRequest") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for Registration service + +type RegistrationClient interface { + GetInfo(ctx context.Context, in *InfoRequest, opts ...grpc.CallOption) (*PluginInfo, error) + NotifyRegistrationStatus(ctx context.Context, in *RegistrationStatus, opts ...grpc.CallOption) (*RegistrationStatusResponse, error) +} + +type registrationClient struct { + cc *grpc.ClientConn +} + +func NewRegistrationClient(cc *grpc.ClientConn) RegistrationClient { + return ®istrationClient{cc} +} + +func (c *registrationClient) GetInfo(ctx context.Context, in *InfoRequest, opts ...grpc.CallOption) (*PluginInfo, error) { + out := new(PluginInfo) + err := grpc.Invoke(ctx, "/pluginregistration.Registration/GetInfo", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *registrationClient) NotifyRegistrationStatus(ctx context.Context, in *RegistrationStatus, opts ...grpc.CallOption) (*RegistrationStatusResponse, error) { + out := new(RegistrationStatusResponse) + err := grpc.Invoke(ctx, "/pluginregistration.Registration/NotifyRegistrationStatus", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for Registration service + +type RegistrationServer interface { + GetInfo(context.Context, *InfoRequest) (*PluginInfo, error) + NotifyRegistrationStatus(context.Context, *RegistrationStatus) (*RegistrationStatusResponse, error) +} + +func RegisterRegistrationServer(s *grpc.Server, srv RegistrationServer) { + s.RegisterService(&_Registration_serviceDesc, srv) +} + +func _Registration_GetInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(InfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(RegistrationServer).GetInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/pluginregistration.Registration/GetInfo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(RegistrationServer).GetInfo(ctx, req.(*InfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Registration_NotifyRegistrationStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RegistrationStatus) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(RegistrationServer).NotifyRegistrationStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/pluginregistration.Registration/NotifyRegistrationStatus", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(RegistrationServer).NotifyRegistrationStatus(ctx, req.(*RegistrationStatus)) + } + return interceptor(ctx, in, info, handler) +} + +var _Registration_serviceDesc = grpc.ServiceDesc{ + ServiceName: "pluginregistration.Registration", + HandlerType: (*RegistrationServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetInfo", + Handler: _Registration_GetInfo_Handler, + }, + { + MethodName: "NotifyRegistrationStatus", + Handler: _Registration_NotifyRegistrationStatus_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "api.proto", +} + +func (m *PluginInfo) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PluginInfo) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Type) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintApi(dAtA, i, uint64(len(m.Type))) + i += copy(dAtA[i:], m.Type) + } + if len(m.Name) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintApi(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Endpoint) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintApi(dAtA, i, uint64(len(m.Endpoint))) + i += copy(dAtA[i:], m.Endpoint) + } + if len(m.SupportedVersions) > 0 { + for _, s := range m.SupportedVersions { + dAtA[i] = 0x22 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + return i, nil +} + +func (m *RegistrationStatus) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RegistrationStatus) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.PluginRegistered { + dAtA[i] = 0x8 + i++ + if m.PluginRegistered { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if len(m.Error) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintApi(dAtA, i, uint64(len(m.Error))) + i += copy(dAtA[i:], m.Error) + } + return i, nil +} + +func (m *RegistrationStatusResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RegistrationStatusResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + +func (m *InfoRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *InfoRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + +func encodeVarintApi(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *PluginInfo) Size() (n int) { + var l int + _ = l + l = len(m.Type) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + l = len(m.Name) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + l = len(m.Endpoint) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + if len(m.SupportedVersions) > 0 { + for _, s := range m.SupportedVersions { + l = len(s) + n += 1 + l + sovApi(uint64(l)) + } + } + return n +} + +func (m *RegistrationStatus) Size() (n int) { + var l int + _ = l + if m.PluginRegistered { + n += 2 + } + l = len(m.Error) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + return n +} + +func (m *RegistrationStatusResponse) Size() (n int) { + var l int + _ = l + return n +} + +func (m *InfoRequest) Size() (n int) { + var l int + _ = l + return n +} + +func sovApi(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozApi(x uint64) (n int) { + return sovApi(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *PluginInfo) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&PluginInfo{`, + `Type:` + fmt.Sprintf("%v", this.Type) + `,`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `Endpoint:` + fmt.Sprintf("%v", this.Endpoint) + `,`, + `SupportedVersions:` + fmt.Sprintf("%v", this.SupportedVersions) + `,`, + `}`, + }, "") + return s +} +func (this *RegistrationStatus) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&RegistrationStatus{`, + `PluginRegistered:` + fmt.Sprintf("%v", this.PluginRegistered) + `,`, + `Error:` + fmt.Sprintf("%v", this.Error) + `,`, + `}`, + }, "") + return s +} +func (this *RegistrationStatusResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&RegistrationStatusResponse{`, + `}`, + }, "") + return s +} +func (this *InfoRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&InfoRequest{`, + `}`, + }, "") + return s +} +func valueToStringApi(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *PluginInfo) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PluginInfo: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PluginInfo: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Type = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Endpoint", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Endpoint = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SupportedVersions", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SupportedVersions = append(m.SupportedVersions, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RegistrationStatus) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RegistrationStatus: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RegistrationStatus: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PluginRegistered", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.PluginRegistered = bool(v != 0) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Error = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RegistrationStatusResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RegistrationStatusResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RegistrationStatusResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *InfoRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: InfoRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: InfoRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipApi(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowApi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowApi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowApi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthApi + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowApi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipApi(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthApi = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowApi = fmt.Errorf("proto: integer overflow") +) + +func init() { proto.RegisterFile("api.proto", fileDescriptorApi) } + +var fileDescriptorApi = []byte{ + // 337 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x52, 0x41, 0x4b, 0x33, 0x31, + 0x14, 0xdc, 0x7c, 0xed, 0xa7, 0xed, 0x53, 0xc1, 0x06, 0x0f, 0xcb, 0x52, 0x62, 0xd9, 0x83, 0x14, + 0xa4, 0x5b, 0xd0, 0x7f, 0xe0, 0x45, 0x04, 0x11, 0x89, 0xa0, 0xc7, 0xb2, 0xb5, 0xaf, 0x6b, 0xc0, + 0x26, 0x31, 0xc9, 0x0a, 0x3d, 0xe9, 0x4f, 0xf0, 0x67, 0xf5, 0x28, 0x9e, 0x3c, 0xda, 0xf5, 0x8f, + 0x48, 0xb3, 0x65, 0x2d, 0xb4, 0x07, 0x6f, 0x6f, 0xe6, 0x4d, 0x1e, 0x33, 0x43, 0xa0, 0x99, 0x6a, + 0x91, 0x68, 0xa3, 0x9c, 0xa2, 0x54, 0x3f, 0xe6, 0x99, 0x90, 0x06, 0x33, 0x61, 0x9d, 0x49, 0x9d, + 0x50, 0x32, 0xea, 0x65, 0xc2, 0x3d, 0xe4, 0xc3, 0xe4, 0x5e, 0x4d, 0xfa, 0x99, 0xca, 0x54, 0xdf, + 0x4b, 0x87, 0xf9, 0xd8, 0x23, 0x0f, 0xfc, 0x54, 0x9e, 0x88, 0x5f, 0x00, 0xae, 0xfd, 0x91, 0x0b, + 0x39, 0x56, 0x94, 0x42, 0xdd, 0x4d, 0x35, 0x86, 0xa4, 0x43, 0xba, 0x4d, 0xee, 0xe7, 0x05, 0x27, + 0xd3, 0x09, 0x86, 0xff, 0x4a, 0x6e, 0x31, 0xd3, 0x08, 0x1a, 0x28, 0x47, 0x5a, 0x09, 0xe9, 0xc2, + 0x9a, 0xe7, 0x2b, 0x4c, 0x7b, 0x40, 0x6d, 0xae, 0xb5, 0x32, 0x0e, 0x47, 0x83, 0x67, 0x34, 0x56, + 0x28, 0x69, 0xc3, 0x7a, 0xa7, 0xd6, 0x6d, 0xf2, 0x56, 0xb5, 0xb9, 0x5d, 0x2e, 0xe2, 0x3b, 0xa0, + 0x7c, 0xc5, 0xff, 0x8d, 0x4b, 0x5d, 0x6e, 0xe9, 0x31, 0xb4, 0xca, 0x6c, 0x83, 0x32, 0x1c, 0x1a, + 0x1c, 0x79, 0x57, 0x0d, 0xbe, 0x5f, 0x2e, 0x78, 0xc5, 0xd3, 0x03, 0xf8, 0x8f, 0xc6, 0x28, 0xb3, + 0xb4, 0x58, 0x82, 0xb8, 0x0d, 0xd1, 0xfa, 0x61, 0x8e, 0x56, 0x2b, 0x69, 0x31, 0xde, 0x83, 0x9d, + 0x45, 0x62, 0x8e, 0x4f, 0x39, 0x5a, 0x77, 0xf2, 0x41, 0x60, 0x77, 0x55, 0x4d, 0x2f, 0x61, 0xfb, + 0x1c, 0x9d, 0x2f, 0xe5, 0x30, 0x59, 0xaf, 0x39, 0x59, 0x79, 0x1c, 0xb1, 0x4d, 0x82, 0xdf, 0x56, + 0xe3, 0x80, 0x3a, 0x08, 0xaf, 0x94, 0x13, 0xe3, 0xe9, 0x86, 0xa8, 0x47, 0x9b, 0x5e, 0xaf, 0xeb, + 0xa2, 0xe4, 0x6f, 0xba, 0x2a, 0x61, 0x70, 0xd6, 0x9e, 0xcd, 0x19, 0xf9, 0x9c, 0xb3, 0xe0, 0xb5, + 0x60, 0x64, 0x56, 0x30, 0xf2, 0x5e, 0x30, 0xf2, 0x55, 0x30, 0xf2, 0xf6, 0xcd, 0x82, 0xe1, 0x96, + 0xff, 0x00, 0xa7, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe4, 0xc0, 0xe3, 0x42, 0x50, 0x02, 0x00, + 0x00, +} diff --git a/pkg/kubelet/apis/pluginregistration/v1/api.proto b/pkg/kubelet/apis/pluginregistration/v1/api.proto new file mode 100644 index 00000000000..c6bd26209d6 --- /dev/null +++ b/pkg/kubelet/apis/pluginregistration/v1/api.proto @@ -0,0 +1,60 @@ +// To regenerate api.pb.go run hack/update-generated-kubelet-plugin-registration.sh +syntax = 'proto3'; + +package pluginregistration; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; + +option (gogoproto.goproto_stringer_all) = false; +option (gogoproto.stringer_all) = true; +option (gogoproto.goproto_getters_all) = true; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (gogoproto.goproto_unrecognized_all) = false; + +// PluginInfo is the message sent from a plugin to the Kubelet pluginwatcher for plugin registration +message PluginInfo { + // Type of the Plugin. CSIPlugin or DevicePlugin + string type = 1; + // Plugin name that uniquely identifies the plugin for the given plugin type. + // For DevicePlugin, this is the resource name that the plugin manages and + // should follow the extended resource name convention. + // For CSI, this is the CSI driver registrar name. + string name = 2; + // Optional endpoint location. If found set by Kubelet component, + // Kubelet component will use this endpoint for specific requests. + // This allows the plugin to register using one endpoint and possibly use + // a different socket for control operations. CSI uses this model to delegate + // its registration external from the plugin. + string endpoint = 3; + // Plugin service API versions the plugin supports. + // For DevicePlugin, this maps to the deviceplugin API versions the + // plugin supports at the given socket. + // The Kubelet component communicating with the plugin should be able + // to choose any preferred version from this list, or returns an error + // if none of the listed versions is supported. + repeated string supported_versions = 4; +} + +// RegistrationStatus is the message sent from Kubelet pluginwatcher to the plugin for notification on registration status +message RegistrationStatus { + // True if plugin gets registered successfully at Kubelet + bool plugin_registered = 1; + // Error message in case plugin fails to register, empty string otherwise + string error = 2; +} + +// RegistrationStatusResponse is sent by plugin to kubelet in response to RegistrationStatus RPC +message RegistrationStatusResponse { +} + +// InfoRequest is the empty request message from Kubelet +message InfoRequest { +} + +// Registration is the service advertised by the Plugins. +service Registration { + rpc GetInfo(InfoRequest) returns (PluginInfo) {} + rpc NotifyRegistrationStatus(RegistrationStatus) returns (RegistrationStatusResponse) {} +} diff --git a/pkg/kubelet/apis/pluginregistration/v1/constants.go b/pkg/kubelet/apis/pluginregistration/v1/constants.go new file mode 100644 index 00000000000..7708f758fa2 --- /dev/null +++ b/pkg/kubelet/apis/pluginregistration/v1/constants.go @@ -0,0 +1,24 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pluginregistration + +const ( + // CSIPlugin identifier for registered CSI plugins + CSIPlugin = "CSIPlugin" + // DevicePlugin identifier for registered device plugins + DevicePlugin = "DevicePlugin" +) diff --git a/pkg/kubelet/apis/podresources/BUILD b/pkg/kubelet/apis/podresources/BUILD new file mode 100644 index 00000000000..8bca4b89702 --- /dev/null +++ b/pkg/kubelet/apis/podresources/BUILD @@ -0,0 +1,48 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "client.go", + "constants.go", + "server.go", + ], + importpath = "k8s.io/kubernetes/pkg/kubelet/apis/podresources", + visibility = ["//visibility:public"], + deps = [ + "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", + "//pkg/kubelet/util:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["server_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/github.com/stretchr/testify/mock:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//pkg/kubelet/apis/podresources/v1alpha1:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/kubelet/apis/podresources/client.go b/pkg/kubelet/apis/podresources/client.go new file mode 100644 index 00000000000..cb3ca222ed3 --- /dev/null +++ b/pkg/kubelet/apis/podresources/client.go @@ -0,0 +1,44 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podresources + +import ( + "context" + "fmt" + "time" + + "google.golang.org/grpc" + + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" + "k8s.io/kubernetes/pkg/kubelet/util" +) + +// GetClient returns a client for the PodResourcesLister grpc service +func GetClient(socket string, connectionTimeout time.Duration, maxMsgSize int) (podresourcesapi.PodResourcesListerClient, *grpc.ClientConn, error) { + addr, dialer, err := util.GetAddressAndDialer(socket) + if err != nil { + return nil, nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), connectionTimeout) + defer cancel() + + conn, err := grpc.DialContext(ctx, addr, grpc.WithInsecure(), grpc.WithDialer(dialer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize))) + if err != nil { + return nil, nil, fmt.Errorf("Error dialing socket %s: %v", socket, err) + } + return podresourcesapi.NewPodResourcesListerClient(conn), conn, nil +} diff --git a/pkg/kubelet/apis/podresources/constants.go b/pkg/kubelet/apis/podresources/constants.go new file mode 100644 index 00000000000..6cc4c6a261a --- /dev/null +++ b/pkg/kubelet/apis/podresources/constants.go @@ -0,0 +1,22 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podresources + +const ( + // Socket is the name of the podresources server socket + Socket = "kubelet" +) diff --git a/pkg/kubelet/apis/podresources/server.go b/pkg/kubelet/apis/podresources/server.go new file mode 100644 index 00000000000..f39e2b26ce0 --- /dev/null +++ b/pkg/kubelet/apis/podresources/server.go @@ -0,0 +1,75 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podresources + +import ( + "context" + + "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" +) + +// DevicesProvider knows how to provide the devices used by the given container +type DevicesProvider interface { + GetDevices(podUID, containerName string) []*v1alpha1.ContainerDevices +} + +// PodsProvider knows how to provide the pods admitted by the node +type PodsProvider interface { + GetPods() []*v1.Pod +} + +// podResourcesServer implements PodResourcesListerServer +type podResourcesServer struct { + podsProvider PodsProvider + devicesProvider DevicesProvider +} + +// NewPodResourcesServer returns a PodResourcesListerServer which lists pods provided by the PodsProvider +// with device information provided by the DevicesProvider +func NewPodResourcesServer(podsProvider PodsProvider, devicesProvider DevicesProvider) v1alpha1.PodResourcesListerServer { + return &podResourcesServer{ + podsProvider: podsProvider, + devicesProvider: devicesProvider, + } +} + +// List returns information about the resources assigned to pods on the node +func (p *podResourcesServer) List(ctx context.Context, req *v1alpha1.ListPodResourcesRequest) (*v1alpha1.ListPodResourcesResponse, error) { + pods := p.podsProvider.GetPods() + podResources := make([]*v1alpha1.PodResources, len(pods)) + + for i, pod := range pods { + pRes := v1alpha1.PodResources{ + Name: pod.Name, + Namespace: pod.Namespace, + Containers: make([]*v1alpha1.ContainerResources, len(pod.Spec.Containers)), + } + + for j, container := range pod.Spec.Containers { + pRes.Containers[j] = &v1alpha1.ContainerResources{ + Name: container.Name, + Devices: p.devicesProvider.GetDevices(string(pod.UID), container.Name), + } + } + podResources[i] = &pRes + } + + return &v1alpha1.ListPodResourcesResponse{ + PodResources: podResources, + }, nil +} diff --git a/pkg/kubelet/apis/podresources/server_test.go b/pkg/kubelet/apis/podresources/server_test.go new file mode 100644 index 00000000000..60c14d4c954 --- /dev/null +++ b/pkg/kubelet/apis/podresources/server_test.go @@ -0,0 +1,153 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podresources + +import ( + "context" + "testing" + + "github.com/stretchr/testify/mock" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" +) + +type mockProvider struct { + mock.Mock +} + +func (m *mockProvider) GetPods() []*v1.Pod { + args := m.Called() + return args.Get(0).([]*v1.Pod) +} + +func (m *mockProvider) GetDevices(podUID, containerName string) []*v1alpha1.ContainerDevices { + args := m.Called(podUID, containerName) + return args.Get(0).([]*v1alpha1.ContainerDevices) +} + +func TestListPodResources(t *testing.T) { + podName := "pod-name" + podNamespace := "pod-namespace" + podUID := types.UID("pod-uid") + containerName := "container-name" + + devs := []*v1alpha1.ContainerDevices{ + { + ResourceName: "resource", + DeviceIds: []string{"dev0", "dev1"}, + }, + } + + for _, tc := range []struct { + desc string + pods []*v1.Pod + devices []*v1alpha1.ContainerDevices + expectedResponse *v1alpha1.ListPodResourcesResponse + }{ + { + desc: "no pods", + pods: []*v1.Pod{}, + devices: []*v1alpha1.ContainerDevices{}, + expectedResponse: &v1alpha1.ListPodResourcesResponse{}, + }, + { + desc: "pod without devices", + pods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + UID: podUID, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: containerName, + }, + }, + }, + }, + }, + devices: []*v1alpha1.ContainerDevices{}, + expectedResponse: &v1alpha1.ListPodResourcesResponse{ + PodResources: []*v1alpha1.PodResources{ + { + Name: podName, + Namespace: podNamespace, + Containers: []*v1alpha1.ContainerResources{ + { + Name: containerName, + Devices: []*v1alpha1.ContainerDevices{}, + }, + }, + }, + }, + }, + }, + { + desc: "pod with devices", + pods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + UID: podUID, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: containerName, + }, + }, + }, + }, + }, + devices: devs, + expectedResponse: &v1alpha1.ListPodResourcesResponse{ + PodResources: []*v1alpha1.PodResources{ + { + Name: podName, + Namespace: podNamespace, + Containers: []*v1alpha1.ContainerResources{ + { + Name: containerName, + Devices: devs, + }, + }, + }, + }, + }, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + m := new(mockProvider) + m.On("GetPods").Return(tc.pods) + m.On("GetDevices", string(podUID), containerName).Return(tc.devices) + server := NewPodResourcesServer(m, m) + resp, err := server.List(context.TODO(), &v1alpha1.ListPodResourcesRequest{}) + if err != nil { + t.Errorf("want err = %v, got %q", nil, err) + } + if tc.expectedResponse.String() != resp.String() { + t.Errorf("want resp = %s, got %s", tc.expectedResponse.String(), resp.String()) + } + }) + } +} diff --git a/pkg/kubelet/apis/podresources/v1alpha1/BUILD b/pkg/kubelet/apis/podresources/v1alpha1/BUILD new file mode 100644 index 00000000000..a56deb71467 --- /dev/null +++ b/pkg/kubelet/apis/podresources/v1alpha1/BUILD @@ -0,0 +1,28 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["api.pb.go"], + importpath = "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1", + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library", + "//vendor/github.com/gogo/protobuf/proto:go_default_library", + "//vendor/golang.org/x/net/context:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/kubelet/apis/podresources/v1alpha1/api.pb.go b/pkg/kubelet/apis/podresources/v1alpha1/api.pb.go new file mode 100644 index 00000000000..df4d4300f21 --- /dev/null +++ b/pkg/kubelet/apis/podresources/v1alpha1/api.pb.go @@ -0,0 +1,1182 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: api.proto + +/* + Package v1alpha1 is a generated protocol buffer package. + + It is generated from these files: + api.proto + + It has these top-level messages: + ListPodResourcesRequest + ListPodResourcesResponse + PodResources + ContainerResources + ContainerDevices +*/ +package v1alpha1 + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "github.com/gogo/protobuf/gogoproto" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +import strings "strings" +import reflect "reflect" + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +// ListPodResourcesRequest is the request made to the PodResourcesLister service +type ListPodResourcesRequest struct { +} + +func (m *ListPodResourcesRequest) Reset() { *m = ListPodResourcesRequest{} } +func (*ListPodResourcesRequest) ProtoMessage() {} +func (*ListPodResourcesRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{0} } + +// ListPodResourcesResponse is the response returned by List function +type ListPodResourcesResponse struct { + PodResources []*PodResources `protobuf:"bytes,1,rep,name=pod_resources,json=podResources" json:"pod_resources,omitempty"` +} + +func (m *ListPodResourcesResponse) Reset() { *m = ListPodResourcesResponse{} } +func (*ListPodResourcesResponse) ProtoMessage() {} +func (*ListPodResourcesResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{1} } + +func (m *ListPodResourcesResponse) GetPodResources() []*PodResources { + if m != nil { + return m.PodResources + } + return nil +} + +// PodResources contains information about the node resources assigned to a pod +type PodResources struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Namespace string `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` + Containers []*ContainerResources `protobuf:"bytes,3,rep,name=containers" json:"containers,omitempty"` +} + +func (m *PodResources) Reset() { *m = PodResources{} } +func (*PodResources) ProtoMessage() {} +func (*PodResources) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{2} } + +func (m *PodResources) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *PodResources) GetNamespace() string { + if m != nil { + return m.Namespace + } + return "" +} + +func (m *PodResources) GetContainers() []*ContainerResources { + if m != nil { + return m.Containers + } + return nil +} + +// ContainerResources contains information about the resources assigned to a container +type ContainerResources struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Devices []*ContainerDevices `protobuf:"bytes,2,rep,name=devices" json:"devices,omitempty"` +} + +func (m *ContainerResources) Reset() { *m = ContainerResources{} } +func (*ContainerResources) ProtoMessage() {} +func (*ContainerResources) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{3} } + +func (m *ContainerResources) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *ContainerResources) GetDevices() []*ContainerDevices { + if m != nil { + return m.Devices + } + return nil +} + +// ContainerDevices contains information about the devices assigned to a container +type ContainerDevices struct { + ResourceName string `protobuf:"bytes,1,opt,name=resource_name,json=resourceName,proto3" json:"resource_name,omitempty"` + DeviceIds []string `protobuf:"bytes,2,rep,name=device_ids,json=deviceIds" json:"device_ids,omitempty"` +} + +func (m *ContainerDevices) Reset() { *m = ContainerDevices{} } +func (*ContainerDevices) ProtoMessage() {} +func (*ContainerDevices) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{4} } + +func (m *ContainerDevices) GetResourceName() string { + if m != nil { + return m.ResourceName + } + return "" +} + +func (m *ContainerDevices) GetDeviceIds() []string { + if m != nil { + return m.DeviceIds + } + return nil +} + +func init() { + proto.RegisterType((*ListPodResourcesRequest)(nil), "v1alpha1.ListPodResourcesRequest") + proto.RegisterType((*ListPodResourcesResponse)(nil), "v1alpha1.ListPodResourcesResponse") + proto.RegisterType((*PodResources)(nil), "v1alpha1.PodResources") + proto.RegisterType((*ContainerResources)(nil), "v1alpha1.ContainerResources") + proto.RegisterType((*ContainerDevices)(nil), "v1alpha1.ContainerDevices") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for PodResourcesLister service + +type PodResourcesListerClient interface { + List(ctx context.Context, in *ListPodResourcesRequest, opts ...grpc.CallOption) (*ListPodResourcesResponse, error) +} + +type podResourcesListerClient struct { + cc *grpc.ClientConn +} + +func NewPodResourcesListerClient(cc *grpc.ClientConn) PodResourcesListerClient { + return &podResourcesListerClient{cc} +} + +func (c *podResourcesListerClient) List(ctx context.Context, in *ListPodResourcesRequest, opts ...grpc.CallOption) (*ListPodResourcesResponse, error) { + out := new(ListPodResourcesResponse) + err := grpc.Invoke(ctx, "/v1alpha1.PodResourcesLister/List", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for PodResourcesLister service + +type PodResourcesListerServer interface { + List(context.Context, *ListPodResourcesRequest) (*ListPodResourcesResponse, error) +} + +func RegisterPodResourcesListerServer(s *grpc.Server, srv PodResourcesListerServer) { + s.RegisterService(&_PodResourcesLister_serviceDesc, srv) +} + +func _PodResourcesLister_List_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListPodResourcesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PodResourcesListerServer).List(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/v1alpha1.PodResourcesLister/List", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PodResourcesListerServer).List(ctx, req.(*ListPodResourcesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _PodResourcesLister_serviceDesc = grpc.ServiceDesc{ + ServiceName: "v1alpha1.PodResourcesLister", + HandlerType: (*PodResourcesListerServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "List", + Handler: _PodResourcesLister_List_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "api.proto", +} + +func (m *ListPodResourcesRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ListPodResourcesRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + +func (m *ListPodResourcesResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ListPodResourcesResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.PodResources) > 0 { + for _, msg := range m.PodResources { + dAtA[i] = 0xa + i++ + i = encodeVarintApi(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *PodResources) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PodResources) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintApi(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Namespace) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintApi(dAtA, i, uint64(len(m.Namespace))) + i += copy(dAtA[i:], m.Namespace) + } + if len(m.Containers) > 0 { + for _, msg := range m.Containers { + dAtA[i] = 0x1a + i++ + i = encodeVarintApi(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *ContainerResources) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ContainerResources) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintApi(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Devices) > 0 { + for _, msg := range m.Devices { + dAtA[i] = 0x12 + i++ + i = encodeVarintApi(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *ContainerDevices) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ContainerDevices) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.ResourceName) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintApi(dAtA, i, uint64(len(m.ResourceName))) + i += copy(dAtA[i:], m.ResourceName) + } + if len(m.DeviceIds) > 0 { + for _, s := range m.DeviceIds { + dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + return i, nil +} + +func encodeVarintApi(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *ListPodResourcesRequest) Size() (n int) { + var l int + _ = l + return n +} + +func (m *ListPodResourcesResponse) Size() (n int) { + var l int + _ = l + if len(m.PodResources) > 0 { + for _, e := range m.PodResources { + l = e.Size() + n += 1 + l + sovApi(uint64(l)) + } + } + return n +} + +func (m *PodResources) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + l = len(m.Namespace) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + if len(m.Containers) > 0 { + for _, e := range m.Containers { + l = e.Size() + n += 1 + l + sovApi(uint64(l)) + } + } + return n +} + +func (m *ContainerResources) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + if len(m.Devices) > 0 { + for _, e := range m.Devices { + l = e.Size() + n += 1 + l + sovApi(uint64(l)) + } + } + return n +} + +func (m *ContainerDevices) Size() (n int) { + var l int + _ = l + l = len(m.ResourceName) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + if len(m.DeviceIds) > 0 { + for _, s := range m.DeviceIds { + l = len(s) + n += 1 + l + sovApi(uint64(l)) + } + } + return n +} + +func sovApi(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozApi(x uint64) (n int) { + return sovApi(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *ListPodResourcesRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ListPodResourcesRequest{`, + `}`, + }, "") + return s +} +func (this *ListPodResourcesResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ListPodResourcesResponse{`, + `PodResources:` + strings.Replace(fmt.Sprintf("%v", this.PodResources), "PodResources", "PodResources", 1) + `,`, + `}`, + }, "") + return s +} +func (this *PodResources) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&PodResources{`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, + `Containers:` + strings.Replace(fmt.Sprintf("%v", this.Containers), "ContainerResources", "ContainerResources", 1) + `,`, + `}`, + }, "") + return s +} +func (this *ContainerResources) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ContainerResources{`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `Devices:` + strings.Replace(fmt.Sprintf("%v", this.Devices), "ContainerDevices", "ContainerDevices", 1) + `,`, + `}`, + }, "") + return s +} +func (this *ContainerDevices) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ContainerDevices{`, + `ResourceName:` + fmt.Sprintf("%v", this.ResourceName) + `,`, + `DeviceIds:` + fmt.Sprintf("%v", this.DeviceIds) + `,`, + `}`, + }, "") + return s +} +func valueToStringApi(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *ListPodResourcesRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ListPodResourcesRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ListPodResourcesRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ListPodResourcesResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ListPodResourcesResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ListPodResourcesResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PodResources", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PodResources = append(m.PodResources, &PodResources{}) + if err := m.PodResources[len(m.PodResources)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PodResources) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PodResources: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PodResources: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Namespace = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Containers", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Containers = append(m.Containers, &ContainerResources{}) + if err := m.Containers[len(m.Containers)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ContainerResources) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ContainerResources: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ContainerResources: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Devices", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Devices = append(m.Devices, &ContainerDevices{}) + if err := m.Devices[len(m.Devices)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ContainerDevices) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ContainerDevices: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ContainerDevices: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ResourceName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ResourceName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DeviceIds", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DeviceIds = append(m.DeviceIds, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipApi(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowApi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowApi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowApi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthApi + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowApi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipApi(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthApi = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowApi = fmt.Errorf("proto: integer overflow") +) + +func init() { proto.RegisterFile("api.proto", fileDescriptorApi) } + +var fileDescriptorApi = []byte{ + // 343 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x52, 0xb1, 0x4e, 0xc3, 0x30, + 0x10, 0xad, 0xdb, 0x0a, 0xc8, 0xd1, 0x4a, 0xc8, 0x03, 0x84, 0xaa, 0x58, 0xc5, 0x2c, 0x5d, 0x48, + 0xd5, 0xc2, 0x06, 0x13, 0xb0, 0x20, 0x21, 0x40, 0x19, 0x60, 0xa3, 0x4a, 0x13, 0xd3, 0x46, 0xa2, + 0xb1, 0x89, 0x93, 0x8e, 0x88, 0x4f, 0xe0, 0xb3, 0x3a, 0x32, 0x32, 0xd2, 0xf0, 0x23, 0x28, 0xb6, + 0xac, 0x04, 0x5a, 0x98, 0x7c, 0x77, 0xef, 0x9d, 0xdf, 0xf3, 0x9d, 0xc1, 0xf2, 0x44, 0xe8, 0x88, + 0x98, 0x27, 0x1c, 0x6f, 0xcc, 0xfa, 0xde, 0x93, 0x98, 0x78, 0xfd, 0xd6, 0xe1, 0x38, 0x4c, 0x26, + 0xe9, 0xc8, 0xf1, 0xf9, 0xb4, 0x37, 0xe6, 0x63, 0xde, 0x53, 0x84, 0x51, 0xfa, 0xa8, 0x32, 0x95, + 0xa8, 0x48, 0x37, 0xd2, 0x5d, 0xd8, 0xb9, 0x0a, 0x65, 0x72, 0xcb, 0x03, 0x97, 0x49, 0x9e, 0xc6, + 0x3e, 0x93, 0x2e, 0x7b, 0x4e, 0x99, 0x4c, 0xe8, 0x3d, 0xd8, 0xcb, 0x90, 0x14, 0x3c, 0x92, 0x0c, + 0x9f, 0x40, 0x53, 0xf0, 0x60, 0x18, 0x1b, 0xc0, 0x46, 0x9d, 0x5a, 0x77, 0x73, 0xb0, 0xed, 0x18, + 0x1f, 0xce, 0x8f, 0xb6, 0x86, 0x28, 0x65, 0xf4, 0x05, 0x1a, 0x65, 0x14, 0x63, 0xa8, 0x47, 0xde, + 0x94, 0xd9, 0xa8, 0x83, 0xba, 0x96, 0xab, 0x62, 0xdc, 0x06, 0x2b, 0x3f, 0xa5, 0xf0, 0x7c, 0x66, + 0x57, 0x15, 0x50, 0x14, 0xf0, 0x29, 0x80, 0xcf, 0xa3, 0xc4, 0x0b, 0x23, 0x16, 0x4b, 0xbb, 0xa6, + 0xb4, 0xdb, 0x85, 0xf6, 0xb9, 0xc1, 0x0a, 0x07, 0x25, 0x3e, 0x7d, 0x00, 0xbc, 0xcc, 0x58, 0xe9, + 0xe2, 0x18, 0xd6, 0x03, 0x36, 0x0b, 0xf3, 0x07, 0x56, 0x95, 0x48, 0x6b, 0x85, 0xc8, 0x85, 0x66, + 0xb8, 0x86, 0x4a, 0xef, 0x60, 0xeb, 0x37, 0x88, 0x0f, 0xa0, 0x69, 0x86, 0x35, 0x2c, 0xc9, 0x34, + 0x4c, 0xf1, 0x3a, 0x97, 0xdb, 0x03, 0xd0, 0x77, 0x0c, 0xc3, 0x40, 0x2b, 0x5a, 0xae, 0xa5, 0x2b, + 0x97, 0x81, 0x1c, 0x30, 0xc0, 0xe5, 0xb9, 0xe5, 0xcb, 0x61, 0x31, 0xbe, 0x81, 0x7a, 0x1e, 0xe1, + 0xfd, 0xc2, 0xda, 0x1f, 0x1b, 0x6d, 0xd1, 0xff, 0x28, 0x7a, 0xb3, 0xb4, 0x72, 0xd6, 0x9e, 0x2f, + 0x08, 0xfa, 0x58, 0x90, 0xca, 0x6b, 0x46, 0xd0, 0x3c, 0x23, 0xe8, 0x3d, 0x23, 0xe8, 0x33, 0x23, + 0xe8, 0xed, 0x8b, 0x54, 0x46, 0x6b, 0xea, 0xdf, 0x1c, 0x7d, 0x07, 0x00, 0x00, 0xff, 0xff, 0xc0, + 0xce, 0xf2, 0x80, 0x7d, 0x02, 0x00, 0x00, +} diff --git a/pkg/kubelet/apis/podresources/v1alpha1/api.proto b/pkg/kubelet/apis/podresources/v1alpha1/api.proto new file mode 100644 index 00000000000..eedcd59d737 --- /dev/null +++ b/pkg/kubelet/apis/podresources/v1alpha1/api.proto @@ -0,0 +1,48 @@ +// To regenerate api.pb.go run hack/update-generated-pod-resources.sh +syntax = 'proto3'; + +package v1alpha1; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; + +option (gogoproto.goproto_stringer_all) = false; +option (gogoproto.stringer_all) = true; +option (gogoproto.goproto_getters_all) = true; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (gogoproto.goproto_unrecognized_all) = false; + + +// PodResourcesLister is a service provided by the kubelet that provides information about the +// node resources consumed by pods and containers on the node +service PodResourcesLister { + rpc List(ListPodResourcesRequest) returns (ListPodResourcesResponse) {} +} + +// ListPodResourcesRequest is the request made to the PodResourcesLister service +message ListPodResourcesRequest {} + +// ListPodResourcesResponse is the response returned by List function +message ListPodResourcesResponse { + repeated PodResources pod_resources = 1; +} + +// PodResources contains information about the node resources assigned to a pod +message PodResources { + string name = 1; + string namespace = 2; + repeated ContainerResources containers = 3; +} + +// ContainerResources contains information about the resources assigned to a container +message ContainerResources { + string name = 1; + repeated ContainerDevices devices = 2; +} + +// ContainerDevices contains information about the devices assigned to a container +message ContainerDevices { + string resource_name = 1; + repeated string device_ids = 2; +} \ No newline at end of file diff --git a/pkg/kubelet/apis/well_known_labels.go b/pkg/kubelet/apis/well_known_labels.go index 5a0db552c82..869952556dd 100644 --- a/pkg/kubelet/apis/well_known_labels.go +++ b/pkg/kubelet/apis/well_known_labels.go @@ -16,6 +16,12 @@ limitations under the License. package apis +import ( + "strings" + + "k8s.io/apimachinery/pkg/util/sets" +) + const ( LabelHostname = "kubernetes.io/hostname" LabelZoneFailureDomain = "failure-domain.beta.kubernetes.io/zone" @@ -26,8 +32,78 @@ const ( LabelOS = "beta.kubernetes.io/os" LabelArch = "beta.kubernetes.io/arch" + + // GA versions of the legacy beta labels. + // TODO: update kubelet and controllers to set both beta and GA labels, then export these constants + labelZoneFailureDomainGA = "failure-domain.kubernetes.io/zone" + labelZoneRegionGA = "failure-domain.kubernetes.io/region" + labelInstanceTypeGA = "kubernetes.io/instance-type" + labelOSGA = "kubernetes.io/os" + labelArchGA = "kubernetes.io/arch" + + // LabelNamespaceSuffixKubelet is an allowed label namespace suffix kubelets can self-set ([*.]kubelet.kubernetes.io/*) + LabelNamespaceSuffixKubelet = "kubelet.kubernetes.io" + // LabelNamespaceSuffixNode is an allowed label namespace suffix kubelets can self-set ([*.]node.kubernetes.io/*) + LabelNamespaceSuffixNode = "node.kubernetes.io" + + // LabelNamespaceNodeRestriction is a forbidden label namespace that kubelets may not self-set when the NodeRestriction admission plugin is enabled + LabelNamespaceNodeRestriction = "node-restriction.kubernetes.io" ) // When the --failure-domains scheduler flag is not specified, // DefaultFailureDomains defines the set of label keys used when TopologyKey is empty in PreferredDuringScheduling anti-affinity. var DefaultFailureDomains string = LabelHostname + "," + LabelZoneFailureDomain + "," + LabelZoneRegion + +var kubeletLabels = sets.NewString( + LabelHostname, + LabelZoneFailureDomain, + LabelZoneRegion, + LabelInstanceType, + LabelOS, + LabelArch, + + labelZoneFailureDomainGA, + labelZoneRegionGA, + labelInstanceTypeGA, + labelOSGA, + labelArchGA, +) + +var kubeletLabelNamespaces = sets.NewString( + LabelNamespaceSuffixKubelet, + LabelNamespaceSuffixNode, +) + +// KubeletLabels returns the list of label keys kubelets are allowed to set on their own Node objects +func KubeletLabels() []string { + return kubeletLabels.List() +} + +// KubeletLabelNamespaces returns the list of label key namespaces kubelets are allowed to set on their own Node objects +func KubeletLabelNamespaces() []string { + return kubeletLabelNamespaces.List() +} + +// IsKubeletLabel returns true if the label key is one that kubelets are allowed to set on their own Node object. +// This checks if the key is in the KubeletLabels() list, or has a namespace in the KubeletLabelNamespaces() list. +func IsKubeletLabel(key string) bool { + if kubeletLabels.Has(key) { + return true + } + + namespace := getLabelNamespace(key) + for allowedNamespace := range kubeletLabelNamespaces { + if namespace == allowedNamespace || strings.HasSuffix(namespace, "."+allowedNamespace) { + return true + } + } + + return false +} + +func getLabelNamespace(key string) string { + if parts := strings.SplitN(key, "/", 2); len(parts) == 2 { + return parts[0] + } + return "" +} diff --git a/pkg/kubelet/cadvisor/BUILD b/pkg/kubelet/cadvisor/BUILD index 7199ed3dfd7..d3370f13cc8 100644 --- a/pkg/kubelet/cadvisor/BUILD +++ b/pkg/kubelet/cadvisor/BUILD @@ -31,13 +31,13 @@ go_library( "//vendor/github.com/google/cadvisor/info/v2:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/cache/memory:go_default_library", "//vendor/github.com/google/cadvisor/container:go_default_library", "//vendor/github.com/google/cadvisor/fs:go_default_library", "//vendor/github.com/google/cadvisor/manager:go_default_library", "//vendor/github.com/google/cadvisor/metrics:go_default_library", "//vendor/github.com/google/cadvisor/utils/sysfs:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:windows": [ "//pkg/kubelet/winstats:go_default_library", @@ -50,16 +50,19 @@ go_test( name = "go_default_test", srcs = [ "cadvisor_linux_test.go", + "main_test.go", "util_test.go", ], embed = [":go_default_library"], - deps = select({ + deps = [ + "//pkg/features:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", + ] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//pkg/apis/core/v1/helper:go_default_library", - "//pkg/features:go_default_library", "//pkg/kubelet/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/metrics:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", diff --git a/pkg/kubelet/cadvisor/cadvisor_linux.go b/pkg/kubelet/cadvisor/cadvisor_linux.go index f4a3e8864df..d15907fe869 100644 --- a/pkg/kubelet/cadvisor/cadvisor_linux.go +++ b/pkg/kubelet/cadvisor/cadvisor_linux.go @@ -26,7 +26,6 @@ import ( "path" "time" - "github.com/golang/glog" "github.com/google/cadvisor/cache/memory" cadvisormetrics "github.com/google/cadvisor/container" "github.com/google/cadvisor/events" @@ -35,6 +34,7 @@ import ( "github.com/google/cadvisor/manager" "github.com/google/cadvisor/metrics" "github.com/google/cadvisor/utils/sysfs" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/types" ) @@ -67,7 +67,7 @@ func init() { f.DefValue = defaultValue f.Value.Set(defaultValue) } else { - glog.Errorf("Expected cAdvisor flag %q not found", name) + klog.Errorf("Expected cAdvisor flag %q not found", name) } } } @@ -199,7 +199,7 @@ func (cc *cadvisorClient) getFsInfo(label string) (cadvisorapiv2.FsInfo, error) } // TODO(vmarmol): Handle this better when a label has more than one image filesystem. if len(res) > 1 { - glog.Warningf("More than one filesystem labeled %q: %#v. Only using the first one", label, res) + klog.Warningf("More than one filesystem labeled %q: %#v. Only using the first one", label, res) } return res[0], nil diff --git a/pkg/kubelet/cadvisor/main_test.go b/pkg/kubelet/cadvisor/main_test.go new file mode 100644 index 00000000000..f13d029c393 --- /dev/null +++ b/pkg/kubelet/cadvisor/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cadvisor + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/kubelet/cadvisor/util_test.go b/pkg/kubelet/cadvisor/util_test.go index dd8657f9341..5d83e861cd2 100644 --- a/pkg/kubelet/cadvisor/util_test.go +++ b/pkg/kubelet/cadvisor/util_test.go @@ -19,14 +19,15 @@ limitations under the License. package cadvisor import ( - "fmt" + "testing" + info "github.com/google/cadvisor/info/v1" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/resource" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/features" - "testing" ) func TestCapacityFromMachineInfo(t *testing.T) { @@ -42,7 +43,7 @@ func TestCapacityFromMachineInfo(t *testing.T) { } // enable the features.HugePages - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.HugePages)) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, true)() resourceList := CapacityFromMachineInfo(machineInfo) diff --git a/pkg/kubelet/certificate/BUILD b/pkg/kubelet/certificate/BUILD index fe659984028..5c85202a4b7 100644 --- a/pkg/kubelet/certificate/BUILD +++ b/pkg/kubelet/certificate/BUILD @@ -26,8 +26,8 @@ go_library( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/util/certificate:go_default_library", "//staging/src/k8s.io/client-go/util/connrotation:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/certificate/OWNERS b/pkg/kubelet/certificate/OWNERS index dcb32d2b8c8..470b7a1c92d 100644 --- a/pkg/kubelet/certificate/OWNERS +++ b/pkg/kubelet/certificate/OWNERS @@ -1,7 +1,7 @@ -reviewers: -- mikedanese -- liggitt -- awly approvers: -- mikedanese -- liggitt +- sig-auth-certificates-approvers +reviewers: +- sig-auth-certificates-reviewers +labels: +- sig/auth + diff --git a/pkg/kubelet/certificate/bootstrap/BUILD b/pkg/kubelet/certificate/bootstrap/BUILD index b0f21d50301..257ec672e18 100644 --- a/pkg/kubelet/certificate/bootstrap/BUILD +++ b/pkg/kubelet/certificate/bootstrap/BUILD @@ -11,8 +11,13 @@ go_test( srcs = ["bootstrap_test.go"], embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/client-go/util/cert:go_default_library", ], ) @@ -21,6 +26,7 @@ go_library( srcs = ["bootstrap.go"], importpath = "k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap", deps = [ + "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", @@ -34,7 +40,7 @@ go_library( "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/client-go/util/certificate:go_default_library", "//staging/src/k8s.io/client-go/util/certificate/csr:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/certificate/bootstrap/bootstrap.go b/pkg/kubelet/certificate/bootstrap/bootstrap.go index e8e87dcdc0f..efe34ef1510 100644 --- a/pkg/kubelet/certificate/bootstrap/bootstrap.go +++ b/pkg/kubelet/certificate/bootstrap/bootstrap.go @@ -18,20 +18,24 @@ package bootstrap import ( "context" + "crypto/sha512" + "crypto/x509/pkix" + "encoding/base64" "errors" "fmt" "os" "path/filepath" "time" - "github.com/golang/glog" + "k8s.io/klog" + certificates "k8s.io/api/certificates/v1beta1" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" - certificates "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + certificatesv1beta1 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -54,18 +58,18 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string, return err } if ok { - glog.V(2).Infof("Kubeconfig %s exists and is valid, skipping bootstrap", kubeconfigPath) + klog.V(2).Infof("Kubeconfig %s exists and is valid, skipping bootstrap", kubeconfigPath) return nil } - glog.V(2).Info("Using bootstrap kubeconfig to generate TLS client cert, key and kubeconfig file") + klog.V(2).Info("Using bootstrap kubeconfig to generate TLS client cert, key and kubeconfig file") bootstrapClientConfig, err := loadRESTClientConfig(bootstrapPath) if err != nil { return fmt.Errorf("unable to load bootstrap kubeconfig: %v", err) } - bootstrapClient, err := certificates.NewForConfig(bootstrapClientConfig) + bootstrapClient, err := certificatesv1beta1.NewForConfig(bootstrapClientConfig) if err != nil { return fmt.Errorf("unable to create certificates signing request client: %v", err) } @@ -89,7 +93,7 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string, // managed by the store. privKeyPath := filepath.Join(certDir, tmpPrivateKeyFile) if !verifyKeyData(keyData) { - glog.V(2).Infof("No valid private key and/or certificate found, reusing existing private key or creating a new one") + klog.V(2).Infof("No valid private key and/or certificate found, reusing existing private key or creating a new one") // Note: always call LoadOrGenerateKeyFile so that private key is // reused on next startup if CSR request fails. keyData, _, err = certutil.LoadOrGenerateKeyFile(privKeyPath) @@ -99,10 +103,10 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string, } if err := waitForServer(*bootstrapClientConfig, 1*time.Minute); err != nil { - glog.Warningf("Error waiting for apiserver to come up: %v", err) + klog.Warningf("Error waiting for apiserver to come up: %v", err) } - certData, err := csr.RequestNodeCertificate(bootstrapClient.CertificateSigningRequests(), keyData, nodeName) + certData, err := requestNodeCertificate(bootstrapClient.CertificateSigningRequests(), keyData, nodeName) if err != nil { return err } @@ -110,7 +114,7 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string, return err } if err := os.Remove(privKeyPath); err != nil && !os.IsNotExist(err) { - glog.V(2).Infof("failed cleaning up private key file %q: %v", privKeyPath, err) + klog.V(2).Infof("failed cleaning up private key file %q: %v", privKeyPath, err) } pemPath := store.CurrentPath() @@ -232,7 +236,7 @@ func waitForServer(cfg restclient.Config, deadline time.Duration) error { var connected bool wait.JitterUntil(func() { if _, err := cli.Get().AbsPath("/healthz").Do().Raw(); err != nil { - glog.Infof("Failed to connect to apiserver: %v", err) + klog.Infof("Failed to connect to apiserver: %v", err) return } cancel() @@ -244,3 +248,70 @@ func waitForServer(cfg restclient.Config, deadline time.Duration) error { } return nil } + +// requestNodeCertificate will create a certificate signing request for a node +// (Organization and CommonName for the CSR will be set as expected for node +// certificates) and send it to API server, then it will watch the object's +// status, once approved by API server, it will return the API server's issued +// certificate (pem-encoded). If there is any errors, or the watch timeouts, it +// will return an error. This is intended for use on nodes (kubelet and +// kubeadm). +func requestNodeCertificate(client certificatesv1beta1.CertificateSigningRequestInterface, privateKeyData []byte, nodeName types.NodeName) (certData []byte, err error) { + subject := &pkix.Name{ + Organization: []string{"system:nodes"}, + CommonName: "system:node:" + string(nodeName), + } + + privateKey, err := certutil.ParsePrivateKeyPEM(privateKeyData) + if err != nil { + return nil, fmt.Errorf("invalid private key for certificate request: %v", err) + } + csrData, err := certutil.MakeCSR(privateKey, subject, nil, nil) + if err != nil { + return nil, fmt.Errorf("unable to generate certificate request: %v", err) + } + + usages := []certificates.KeyUsage{ + certificates.UsageDigitalSignature, + certificates.UsageKeyEncipherment, + certificates.UsageClientAuth, + } + name := digestedName(privateKeyData, subject, usages) + req, err := csr.RequestCertificate(client, csrData, name, usages, privateKey) + if err != nil { + return nil, err + } + return csr.WaitForCertificate(client, req, 3600*time.Second) +} + +// This digest should include all the relevant pieces of the CSR we care about. +// We can't direcly hash the serialized CSR because of random padding that we +// regenerate every loop and we include usages which are not contained in the +// CSR. This needs to be kept up to date as we add new fields to the node +// certificates and with ensureCompatible. +func digestedName(privateKeyData []byte, subject *pkix.Name, usages []certificates.KeyUsage) string { + hash := sha512.New512_256() + + // Here we make sure two different inputs can't write the same stream + // to the hash. This delimiter is not in the base64.URLEncoding + // alphabet so there is no way to have spill over collisions. Without + // it 'CN:foo,ORG:bar' hashes to the same value as 'CN:foob,ORG:ar' + const delimiter = '|' + encode := base64.RawURLEncoding.EncodeToString + + write := func(data []byte) { + hash.Write([]byte(encode(data))) + hash.Write([]byte{delimiter}) + } + + write(privateKeyData) + write([]byte(subject.CommonName)) + for _, v := range subject.Organization { + write([]byte(v)) + } + for _, v := range usages { + write([]byte(v)) + } + + return "node-csr-" + encode(hash.Sum(nil)) +} diff --git a/pkg/kubelet/certificate/bootstrap/bootstrap_test.go b/pkg/kubelet/certificate/bootstrap/bootstrap_test.go index 2885719b6f1..54c1646fc5b 100644 --- a/pkg/kubelet/certificate/bootstrap/bootstrap_test.go +++ b/pkg/kubelet/certificate/bootstrap/bootstrap_test.go @@ -17,13 +17,19 @@ limitations under the License. package bootstrap import ( + "fmt" "io/ioutil" "os" "reflect" "testing" + certificates "k8s.io/api/certificates/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/watch" + certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" restclient "k8s.io/client-go/rest" + certutil "k8s.io/client-go/util/cert" ) func TestLoadRESTClientConfig(t *testing.T) { @@ -36,7 +42,7 @@ clusters: server: https://cluster-a.com name: cluster-a - cluster: - certificate-authority-data: VGVzdA== + certificate-authority-data: VGVzdA== server: https://cluster-b.com name: cluster-b contexts: @@ -83,3 +89,110 @@ users: t.Errorf("Unexpected config: %s", diff.ObjectDiff(config, expectedConfig)) } } + +func TestRequestNodeCertificateNoKeyData(t *testing.T) { + certData, err := requestNodeCertificate(&fakeClient{}, []byte{}, "fake-node-name") + if err == nil { + t.Errorf("Got no error, wanted error an error because there was an empty private key passed in.") + } + if certData != nil { + t.Errorf("Got cert data, wanted nothing as there should have been an error.") + } +} + +func TestRequestNodeCertificateErrorCreatingCSR(t *testing.T) { + client := &fakeClient{ + failureType: createError, + } + privateKeyData, err := certutil.MakeEllipticPrivateKeyPEM() + if err != nil { + t.Fatalf("Unable to generate a new private key: %v", err) + } + + certData, err := requestNodeCertificate(client, privateKeyData, "fake-node-name") + if err == nil { + t.Errorf("Got no error, wanted error an error because client.Create failed.") + } + if certData != nil { + t.Errorf("Got cert data, wanted nothing as there should have been an error.") + } +} + +func TestRequestNodeCertificate(t *testing.T) { + privateKeyData, err := certutil.MakeEllipticPrivateKeyPEM() + if err != nil { + t.Fatalf("Unable to generate a new private key: %v", err) + } + + certData, err := requestNodeCertificate(&fakeClient{}, privateKeyData, "fake-node-name") + if err != nil { + t.Errorf("Got %v, wanted no error.", err) + } + if certData == nil { + t.Errorf("Got nothing, expected a CSR.") + } +} + +type failureType int + +const ( + noError failureType = iota + createError + certificateSigningRequestDenied +) + +type fakeClient struct { + certificatesclient.CertificateSigningRequestInterface + watch *watch.FakeWatcher + failureType failureType +} + +func (c *fakeClient) Create(*certificates.CertificateSigningRequest) (*certificates.CertificateSigningRequest, error) { + if c.failureType == createError { + return nil, fmt.Errorf("fakeClient failed creating request") + } + csr := certificates.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + UID: "fake-uid", + Name: "fake-certificate-signing-request-name", + }, + } + return &csr, nil +} + +func (c *fakeClient) List(opts metav1.ListOptions) (*certificates.CertificateSigningRequestList, error) { + return &certificates.CertificateSigningRequestList{}, nil +} + +func (c *fakeClient) Watch(opts metav1.ListOptions) (watch.Interface, error) { + c.watch = watch.NewFakeWithChanSize(1, false) + c.watch.Add(c.generateCSR()) + c.watch.Stop() + return c.watch, nil +} + +func (c *fakeClient) generateCSR() *certificates.CertificateSigningRequest { + var condition certificates.CertificateSigningRequestCondition + if c.failureType == certificateSigningRequestDenied { + condition = certificates.CertificateSigningRequestCondition{ + Type: certificates.CertificateDenied, + } + } else { + condition = certificates.CertificateSigningRequestCondition{ + Type: certificates.CertificateApproved, + } + } + + csr := certificates.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + UID: "fake-uid", + }, + Status: certificates.CertificateSigningRequestStatus{ + Conditions: []certificates.CertificateSigningRequestCondition{ + condition, + }, + Certificate: []byte{}, + }, + } + return &csr +} diff --git a/pkg/kubelet/certificate/transport.go b/pkg/kubelet/certificate/transport.go index 436bb8b4c96..442f4a8a379 100644 --- a/pkg/kubelet/certificate/transport.go +++ b/pkg/kubelet/certificate/transport.go @@ -23,7 +23,7 @@ import ( "net/http" "time" - "github.com/golang/glog" + "k8s.io/klog" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/wait" @@ -105,18 +105,18 @@ func addCertRotation(stopCh <-chan struct{}, period time.Duration, clientConfig // the certificate has been deleted from disk or is otherwise corrupt if now.After(lastCertAvailable.Add(exitAfter)) { if clientCertificateManager.ServerHealthy() { - glog.Fatalf("It has been %s since a valid client cert was found and the server is responsive, exiting.", exitAfter) + klog.Fatalf("It has been %s since a valid client cert was found and the server is responsive, exiting.", exitAfter) } else { - glog.Errorf("It has been %s since a valid client cert was found, but the server is not responsive. A restart may be necessary to retrieve new initial credentials.", exitAfter) + klog.Errorf("It has been %s since a valid client cert was found, but the server is not responsive. A restart may be necessary to retrieve new initial credentials.", exitAfter) } } } else { // the certificate is expired if now.After(curr.Leaf.NotAfter) { if clientCertificateManager.ServerHealthy() { - glog.Fatalf("The currently active client certificate has expired and the server is responsive, exiting.") + klog.Fatalf("The currently active client certificate has expired and the server is responsive, exiting.") } else { - glog.Errorf("The currently active client certificate has expired, but the server is not responsive. A restart may be necessary to retrieve new initial credentials.") + klog.Errorf("The currently active client certificate has expired, but the server is not responsive. A restart may be necessary to retrieve new initial credentials.") } } lastCertAvailable = now @@ -129,7 +129,7 @@ func addCertRotation(stopCh <-chan struct{}, period time.Duration, clientConfig } lastCert = curr - glog.Infof("certificate rotation detected, shutting down client connections to start using new credentials") + klog.Infof("certificate rotation detected, shutting down client connections to start using new credentials") // The cert has been rotated. Close all existing connections to force the client // to reperform its TLS handshake with new cert. // diff --git a/pkg/kubelet/checkpoint/BUILD b/pkg/kubelet/checkpoint/BUILD index a57fff4cdbd..d8d1fe7fd30 100644 --- a/pkg/kubelet/checkpoint/BUILD +++ b/pkg/kubelet/checkpoint/BUILD @@ -10,7 +10,7 @@ go_library( "//pkg/kubelet/checkpointmanager:go_default_library", "//pkg/kubelet/checkpointmanager/checksum:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/checkpoint/checkpoint.go b/pkg/kubelet/checkpoint/checkpoint.go index bf84178dce8..f1fa9bdb7ae 100644 --- a/pkg/kubelet/checkpoint/checkpoint.go +++ b/pkg/kubelet/checkpoint/checkpoint.go @@ -20,7 +20,7 @@ import ( "encoding/json" "fmt" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/apis/core" @@ -93,14 +93,14 @@ func LoadPods(cpm checkpointmanager.CheckpointManager) ([]*v1.Pod, error) { checkpointKeys := []string{} checkpointKeys, err = cpm.ListCheckpoints() if err != nil { - glog.Errorf("Failed to list checkpoints: %v", err) + klog.Errorf("Failed to list checkpoints: %v", err) } for _, key := range checkpointKeys { checkpoint := NewPodCheckpoint(nil) err := cpm.GetCheckpoint(key, checkpoint) if err != nil { - glog.Errorf("Failed to retrieve checkpoint for pod %q: %v", key, err) + klog.Errorf("Failed to retrieve checkpoint for pod %q: %v", key, err) continue } pods = append(pods, checkpoint.GetPod()) diff --git a/pkg/kubelet/cloudresource/BUILD b/pkg/kubelet/cloudresource/BUILD index 2390abeddfc..1ab5c46dec2 100644 --- a/pkg/kubelet/cloudresource/BUILD +++ b/pkg/kubelet/cloudresource/BUILD @@ -10,7 +10,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/cloudresource/cloud_request_manager.go b/pkg/kubelet/cloudresource/cloud_request_manager.go index 8435c00a61e..dbc05f094df 100644 --- a/pkg/kubelet/cloudresource/cloud_request_manager.go +++ b/pkg/kubelet/cloudresource/cloud_request_manager.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" cloudprovider "k8s.io/cloud-provider" - "github.com/golang/glog" + "k8s.io/klog" ) var nodeAddressesRetryPeriod = 5 * time.Second @@ -85,7 +85,7 @@ func (manager *cloudResourceSyncManager) NodeAddresses() ([]v1.NodeAddress, erro for { nodeAddresses, err := manager.getNodeAddressSafe() if len(nodeAddresses) == 0 && err == nil { - glog.V(5).Infof("Waiting for %v for cloud provider to provide node addresses", nodeAddressesRetryPeriod) + klog.V(5).Infof("Waiting for %v for cloud provider to provide node addresses", nodeAddressesRetryPeriod) time.Sleep(nodeAddressesRetryPeriod) continue } @@ -94,7 +94,7 @@ func (manager *cloudResourceSyncManager) NodeAddresses() ([]v1.NodeAddress, erro } func (manager *cloudResourceSyncManager) collectNodeAddresses(ctx context.Context, nodeName types.NodeName) { - glog.V(5).Infof("Requesting node addresses from cloud provider for node %q", nodeName) + klog.V(5).Infof("Requesting node addresses from cloud provider for node %q", nodeName) instances, ok := manager.cloud.Instances() if !ok { @@ -110,10 +110,10 @@ func (manager *cloudResourceSyncManager) collectNodeAddresses(ctx context.Contex nodeAddresses, err := instances.NodeAddresses(ctx, nodeName) if err != nil { manager.setNodeAddressSafe(nil, fmt.Errorf("failed to get node address from cloud provider: %v", err)) - glog.V(2).Infof("Node addresses from cloud provider for node %q not collected", nodeName) + klog.V(2).Infof("Node addresses from cloud provider for node %q not collected", nodeName) } else { manager.setNodeAddressSafe(nodeAddresses, nil) - glog.V(5).Infof("Node addresses from cloud provider for node %q collected", nodeName) + klog.V(5).Infof("Node addresses from cloud provider for node %q collected", nodeName) } } diff --git a/pkg/kubelet/cm/BUILD b/pkg/kubelet/cm/BUILD index dca92c084e7..6ead9ea8c3d 100644 --- a/pkg/kubelet/cm/BUILD +++ b/pkg/kubelet/cm/BUILD @@ -27,6 +27,7 @@ go_library( deps = [ "//pkg/features:go_default_library", "//pkg/kubelet/apis/cri:go_default_library", + "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", "//pkg/kubelet/cm/cpumanager:go_default_library", "//pkg/kubelet/config:go_default_library", "//pkg/kubelet/container:go_default_library", @@ -40,7 +41,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:android": [ "//pkg/kubelet/cadvisor:go_default_library", diff --git a/pkg/kubelet/cm/cgroup_manager_linux.go b/pkg/kubelet/cm/cgroup_manager_linux.go index d0eb6b6d27b..f1c81369367 100644 --- a/pkg/kubelet/cm/cgroup_manager_linux.go +++ b/pkg/kubelet/cm/cgroup_manager_linux.go @@ -25,11 +25,11 @@ import ( "time" units "github.com/docker/go-units" - "github.com/golang/glog" libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups" cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs" cgroupsystemd "github.com/opencontainers/runc/libcontainer/cgroups/systemd" libcontainerconfigs "github.com/opencontainers/runc/libcontainer/configs" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -67,7 +67,10 @@ func NewCgroupName(base CgroupName, components ...string) CgroupName { panic(fmt.Errorf("invalid character in component [%q] of CgroupName", component)) } } - return CgroupName(append(base, components...)) + // copy data from the base cgroup to eliminate cases where CgroupNames share underlying slices. See #68416 + baseCopy := make([]string, len(base)) + copy(baseCopy, base) + return CgroupName(append(baseCopy, components...)) } func escapeSystemdCgroupName(part string) string { @@ -268,7 +271,7 @@ func (m *cgroupManagerImpl) Exists(name CgroupName) bool { } if len(missingPaths) > 0 { - glog.V(4).Infof("The Cgroup %v has some missing paths: %v", name, missingPaths) + klog.V(4).Infof("The Cgroup %v has some missing paths: %v", name, missingPaths) return false } @@ -347,7 +350,7 @@ func setSupportedSubsystems(cgroupConfig *libcontainerconfigs.Cgroup) error { return fmt.Errorf("Failed to find subsystem mount for required subsystem: %v", sys.Name()) } // the cgroup is not mounted, but its not required so continue... - glog.V(6).Infof("Unable to find subsystem mount for optional subsystem: %v", sys.Name()) + klog.V(6).Infof("Unable to find subsystem mount for optional subsystem: %v", sys.Name()) continue } if err := sys.Set(cgroupConfig.Paths[sys.Name()], cgroupConfig); err != nil { @@ -509,7 +512,7 @@ func (m *cgroupManagerImpl) Pids(name CgroupName) []int { // WalkFunc which is called for each file and directory in the pod cgroup dir visitor := func(path string, info os.FileInfo, err error) error { if err != nil { - glog.V(4).Infof("cgroup manager encountered error scanning cgroup path %q: %v", path, err) + klog.V(4).Infof("cgroup manager encountered error scanning cgroup path %q: %v", path, err) return filepath.SkipDir } if !info.IsDir() { @@ -517,7 +520,7 @@ func (m *cgroupManagerImpl) Pids(name CgroupName) []int { } pids, err = getCgroupProcs(path) if err != nil { - glog.V(4).Infof("cgroup manager encountered error getting procs for cgroup path %q: %v", path, err) + klog.V(4).Infof("cgroup manager encountered error getting procs for cgroup path %q: %v", path, err) return filepath.SkipDir } pidsToKill.Insert(pids...) @@ -527,7 +530,7 @@ func (m *cgroupManagerImpl) Pids(name CgroupName) []int { // container cgroups haven't been GCed yet. Get attached processes to // all such unwanted containers under the pod cgroup if err = filepath.Walk(dir, visitor); err != nil { - glog.V(4).Infof("cgroup manager encountered error scanning pids for directory: %q: %v", dir, err) + klog.V(4).Infof("cgroup manager encountered error scanning pids for directory: %q: %v", dir, err) } } return pidsToKill.List() @@ -555,7 +558,7 @@ func getStatsSupportedSubsystems(cgroupPaths map[string]string) (*libcontainercg return nil, fmt.Errorf("Failed to find subsystem mount for required subsystem: %v", sys.Name()) } // the cgroup is not mounted, but its not required so continue... - glog.V(6).Infof("Unable to find subsystem mount for optional subsystem: %v", sys.Name()) + klog.V(6).Infof("Unable to find subsystem mount for optional subsystem: %v", sys.Name()) continue } if err := sys.GetStats(cgroupPaths[sys.Name()], stats); err != nil { diff --git a/pkg/kubelet/cm/cgroup_manager_linux_test.go b/pkg/kubelet/cm/cgroup_manager_linux_test.go index eb71014c5a3..dcadae26770 100644 --- a/pkg/kubelet/cm/cgroup_manager_linux_test.go +++ b/pkg/kubelet/cm/cgroup_manager_linux_test.go @@ -20,9 +20,34 @@ package cm import ( "path" + "reflect" "testing" ) +// TestNewCgroupName tests confirms that #68416 is fixed +func TestNewCgroupName(t *testing.T) { + a := ParseCgroupfsToCgroupName("/a/") + ab := NewCgroupName(a, "b") + + expectedAB := CgroupName([]string{"a", "", "b"}) + if !reflect.DeepEqual(ab, expectedAB) { + t.Errorf("Expected %d%+v; got %d%+v", len(expectedAB), expectedAB, len(ab), ab) + } + + abc := NewCgroupName(ab, "c") + + expectedABC := CgroupName([]string{"a", "", "b", "c"}) + if !reflect.DeepEqual(abc, expectedABC) { + t.Errorf("Expected %d%+v; got %d%+v", len(expectedABC), expectedABC, len(abc), abc) + } + + _ = NewCgroupName(ab, "d") + + if !reflect.DeepEqual(abc, expectedABC) { + t.Errorf("Expected %d%+v; got %d%+v", len(expectedABC), expectedABC, len(abc), abc) + } +} + func TestCgroupNameToSystemdBasename(t *testing.T) { testCases := []struct { input CgroupName diff --git a/pkg/kubelet/cm/container_manager.go b/pkg/kubelet/cm/container_manager.go index 64728a7d6b1..a9fe926ee04 100644 --- a/pkg/kubelet/cm/container_manager.go +++ b/pkg/kubelet/cm/container_manager.go @@ -23,6 +23,7 @@ import ( // TODO: Migrate kubelet to either use its own internal objects or client library. "k8s.io/api/core/v1" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" @@ -100,6 +101,9 @@ type ContainerManager interface { // The pluginwatcher's Handlers allow to have a single module for handling // registration. GetPluginRegistrationHandler() pluginwatcher.PluginHandler + + // GetDevices returns information about the devices assigned to pods and containers + GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices } type NodeConfig struct { diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 96506163e4d..9432fc71adc 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -29,10 +29,10 @@ import ( "sync" "time" - "github.com/golang/glog" "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/cgroups/fs" "github.com/opencontainers/runc/libcontainer/configs" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -44,6 +44,7 @@ import ( "k8s.io/client-go/tools/record" kubefeatures "k8s.io/kubernetes/pkg/features" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager" @@ -180,11 +181,11 @@ func validateSystemRequirements(mountUtil mount.Interface) (features, error) { // CPU cgroup is required and so it expected to be mounted at this point. periodExists, err := utilfile.FileExists(path.Join(cpuMountPoint, "cpu.cfs_period_us")) if err != nil { - glog.Errorf("failed to detect if CPU cgroup cpu.cfs_period_us is available - %v", err) + klog.Errorf("failed to detect if CPU cgroup cpu.cfs_period_us is available - %v", err) } quotaExists, err := utilfile.FileExists(path.Join(cpuMountPoint, "cpu.cfs_quota_us")) if err != nil { - glog.Errorf("failed to detect if CPU cgroup cpu.cfs_quota_us is available - %v", err) + klog.Errorf("failed to detect if CPU cgroup cpu.cfs_quota_us is available - %v", err) } if quotaExists && periodExists { f.cpuHardcapping = true @@ -244,12 +245,12 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I if !cgroupManager.Exists(cgroupRoot) { return nil, fmt.Errorf("invalid configuration: cgroup-root %q doesn't exist: %v", cgroupRoot, err) } - glog.Infof("container manager verified user specified cgroup-root exists: %v", cgroupRoot) + klog.Infof("container manager verified user specified cgroup-root exists: %v", cgroupRoot) // Include the top level cgroup for enforcing node allocatable into cgroup-root. // This way, all sub modules can avoid having to understand the concept of node allocatable. cgroupRoot = NewCgroupName(cgroupRoot, defaultNodeAllocatableCgroupName) } - glog.Infof("Creating Container Manager object based on Node Config: %+v", nodeConfig) + klog.Infof("Creating Container Manager object based on Node Config: %+v", nodeConfig) qosContainerManager, err := NewQOSContainerManager(subsystems, cgroupRoot, nodeConfig, cgroupManager) if err != nil { @@ -268,7 +269,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I qosContainerManager: qosContainerManager, } - glog.Infof("Creating device plugin manager: %t", devicePluginEnabled) + klog.Infof("Creating device plugin manager: %t", devicePluginEnabled) if devicePluginEnabled { cm.deviceManager, err = devicemanager.NewManagerImpl() } else { @@ -288,7 +289,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I nodeConfig.KubeletRootDir, ) if err != nil { - glog.Errorf("failed to initialize cpu manager: %v", err) + klog.Errorf("failed to initialize cpu manager: %v", err) return nil, err } } @@ -370,9 +371,9 @@ func setupKernelTunables(option KernelTunableBehavior) error { case KernelTunableError: errList = append(errList, fmt.Errorf("Invalid kernel flag: %v, expected value: %v, actual value: %v", flag, expectedValue, val)) case KernelTunableWarn: - glog.V(2).Infof("Invalid kernel flag: %v, expected value: %v, actual value: %v", flag, expectedValue, val) + klog.V(2).Infof("Invalid kernel flag: %v, expected value: %v, actual value: %v", flag, expectedValue, val) case KernelTunableModify: - glog.V(2).Infof("Updating kernel flag: %v, expected value: %v, actual value: %v", flag, expectedValue, val) + klog.V(2).Infof("Updating kernel flag: %v, expected value: %v, actual value: %v", flag, expectedValue, val) err = sysctl.SetSysctl(flag, expectedValue) if err != nil { errList = append(errList, err) @@ -424,13 +425,13 @@ func (cm *containerManagerImpl) setupNode(activePods ActivePodsFunc) error { // the cgroup for docker and serve stats for the runtime. // TODO(#27097): Fix this after NodeSpec is clearly defined. cm.periodicTasks = append(cm.periodicTasks, func() { - glog.V(4).Infof("[ContainerManager]: Adding periodic tasks for docker CRI integration") + klog.V(4).Infof("[ContainerManager]: Adding periodic tasks for docker CRI integration") cont, err := getContainerNameForProcess(dockerProcessName, dockerPidFile) if err != nil { - glog.Error(err) + klog.Error(err) return } - glog.V(2).Infof("[ContainerManager]: Discovered runtime cgroups name: %s", cont) + klog.V(2).Infof("[ContainerManager]: Discovered runtime cgroups name: %s", cont) cm.Lock() defer cm.Unlock() cm.RuntimeCgroupsName = cont @@ -467,12 +468,12 @@ func (cm *containerManagerImpl) setupNode(activePods ActivePodsFunc) error { } else { cm.periodicTasks = append(cm.periodicTasks, func() { if err := ensureProcessInContainerWithOOMScore(os.Getpid(), qos.KubeletOOMScoreAdj, nil); err != nil { - glog.Error(err) + klog.Error(err) return } cont, err := getContainer(os.Getpid()) if err != nil { - glog.Errorf("failed to find cgroups of kubelet - %v", err) + klog.Errorf("failed to find cgroups of kubelet - %v", err) return } cm.Lock() @@ -579,7 +580,7 @@ func (cm *containerManagerImpl) Start(node *v1.Node, for _, cont := range cm.systemContainers { if cont.ensureStateFunc != nil { if err := cont.ensureStateFunc(cont.manager); err != nil { - glog.Warningf("[ContainerManager] Failed to ensure state of %q: %v", cont.name, err) + klog.Warningf("[ContainerManager] Failed to ensure state of %q: %v", cont.name, err) } } } @@ -652,12 +653,12 @@ func isProcessRunningInHost(pid int) (bool, error) { if err != nil { return false, fmt.Errorf("failed to find pid namespace of init process") } - glog.V(10).Infof("init pid ns is %q", initPidNs) + klog.V(10).Infof("init pid ns is %q", initPidNs) processPidNs, err := os.Readlink(fmt.Sprintf("/proc/%d/ns/pid", pid)) if err != nil { return false, fmt.Errorf("failed to find pid namespace of process %q", pid) } - glog.V(10).Infof("Pid %d pid ns is %q", pid, processPidNs) + klog.V(10).Infof("Pid %d pid ns is %q", pid, processPidNs) return initPidNs == processPidNs, nil } @@ -699,7 +700,7 @@ func getPidsForProcess(name, pidFile string) ([]int, error) { // Return error from getPidFromPidFile since that should have worked // and is the real source of the problem. - glog.V(4).Infof("unable to get pid from %s: %v", pidFile, err) + klog.V(4).Infof("unable to get pid from %s: %v", pidFile, err) return []int{}, err } @@ -737,7 +738,7 @@ func ensureProcessInContainerWithOOMScore(pid int, oomScoreAdj int, manager *fs. return err } else if !runningInHost { // Process is running inside a container. Don't touch that. - glog.V(2).Infof("pid %d is not running in the host namespaces", pid) + klog.V(2).Infof("pid %d is not running in the host namespaces", pid) return nil } @@ -758,9 +759,9 @@ func ensureProcessInContainerWithOOMScore(pid int, oomScoreAdj int, manager *fs. // Also apply oom-score-adj to processes oomAdjuster := oom.NewOOMAdjuster() - glog.V(5).Infof("attempting to apply oom_score_adj of %d to pid %d", oomScoreAdj, pid) + klog.V(5).Infof("attempting to apply oom_score_adj of %d to pid %d", oomScoreAdj, pid) if err := oomAdjuster.ApplyOOMScoreAdj(pid, oomScoreAdj); err != nil { - glog.V(3).Infof("Failed to apply oom_score_adj %d for pid %d: %v", oomScoreAdj, pid, err) + klog.V(3).Infof("Failed to apply oom_score_adj %d for pid %d: %v", oomScoreAdj, pid, err) errs = append(errs, fmt.Errorf("failed to apply oom score %d to PID %d: %v", oomScoreAdj, pid, err)) } return utilerrors.NewAggregate(errs) @@ -800,10 +801,10 @@ func getContainer(pid int) (string, error) { // in addition, you would not get memory or cpu accounting for the runtime unless accounting was enabled on its unit (or globally). if systemd, found := cgs["name=systemd"]; found { if systemd != cpu { - glog.Warningf("CPUAccounting not enabled for pid: %d", pid) + klog.Warningf("CPUAccounting not enabled for pid: %d", pid) } if systemd != memory { - glog.Warningf("MemoryAccounting not enabled for pid: %d", pid) + klog.Warningf("MemoryAccounting not enabled for pid: %d", pid) } return systemd, nil } @@ -841,14 +842,14 @@ func ensureSystemCgroups(rootCgroupPath string, manager *fs.Manager) error { pids = append(pids, pid) } - glog.Infof("Found %d PIDs in root, %d of them are not to be moved", len(allPids), len(allPids)-len(pids)) + klog.Infof("Found %d PIDs in root, %d of them are not to be moved", len(allPids), len(allPids)-len(pids)) // Check if we have moved all the non-kernel PIDs. if len(pids) == 0 { break } - glog.Infof("Moving non-kernel processes: %v", pids) + klog.Infof("Moving non-kernel processes: %v", pids) for _, pid := range pids { err := manager.Apply(pid) if err != nil { @@ -878,3 +879,7 @@ func (cm *containerManagerImpl) GetCapacity() v1.ResourceList { func (cm *containerManagerImpl) GetDevicePluginResourceCapacity() (v1.ResourceList, v1.ResourceList, []string) { return cm.deviceManager.GetCapacity() } + +func (cm *containerManagerImpl) GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices { + return cm.deviceManager.GetDevices(podUID, containerName) +} diff --git a/pkg/kubelet/cm/container_manager_stub.go b/pkg/kubelet/cm/container_manager_stub.go index 8f948c64d2a..4563dc53048 100644 --- a/pkg/kubelet/cm/container_manager_stub.go +++ b/pkg/kubelet/cm/container_manager_stub.go @@ -17,11 +17,12 @@ limitations under the License. package cm import ( - "github.com/golang/glog" "k8s.io/api/core/v1" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/resource" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -36,7 +37,7 @@ type containerManagerStub struct{} var _ ContainerManager = &containerManagerStub{} func (cm *containerManagerStub) Start(_ *v1.Node, _ ActivePodsFunc, _ config.SourcesReady, _ status.PodStatusProvider, _ internalapi.RuntimeService) error { - glog.V(2).Infof("Starting stub container manager") + klog.V(2).Infof("Starting stub container manager") return nil } @@ -105,6 +106,10 @@ func (cm *containerManagerStub) GetPodCgroupRoot() string { return "" } +func (cm *containerManagerStub) GetDevices(_, _ string) []*podresourcesapi.ContainerDevices { + return nil +} + func NewStubContainerManager() ContainerManager { return &containerManagerStub{} } diff --git a/pkg/kubelet/cm/container_manager_windows.go b/pkg/kubelet/cm/container_manager_windows.go index aadc51d0f46..a8a84d5f22d 100644 --- a/pkg/kubelet/cm/container_manager_windows.go +++ b/pkg/kubelet/cm/container_manager_windows.go @@ -24,13 +24,14 @@ package cm import ( "fmt" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/record" + "k8s.io/klog" kubefeatures "k8s.io/kubernetes/pkg/features" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" "k8s.io/kubernetes/pkg/kubelet/config" @@ -56,7 +57,7 @@ func (cm *containerManagerImpl) Start(node *v1.Node, sourcesReady config.SourcesReady, podStatusProvider status.PodStatusProvider, runtimeService internalapi.RuntimeService) error { - glog.V(2).Infof("Starting Windows container manager") + klog.V(2).Infof("Starting Windows container manager") if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LocalStorageCapacityIsolation) { rootfs, err := cm.cadvisorInterface.RootFsInfo() @@ -166,3 +167,7 @@ func (cm *containerManagerImpl) InternalContainerLifecycle() InternalContainerLi func (cm *containerManagerImpl) GetPodCgroupRoot() string { return "" } + +func (cm *containerManagerImpl) GetDevices(_, _ string) []*podresourcesapi.ContainerDevices { + return nil +} diff --git a/pkg/kubelet/cm/cpumanager/BUILD b/pkg/kubelet/cm/cpumanager/BUILD index 4b3baa3184e..4e597c9b388 100644 --- a/pkg/kubelet/cm/cpumanager/BUILD +++ b/pkg/kubelet/cm/cpumanager/BUILD @@ -22,8 +22,8 @@ go_library( "//pkg/kubelet/status:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment.go b/pkg/kubelet/cm/cpumanager/cpu_assignment.go index 3f5f33bafed..be6babab14c 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment.go @@ -20,7 +20,7 @@ import ( "fmt" "sort" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -160,7 +160,7 @@ func takeByTopology(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, num // least a socket's-worth of CPUs. for _, s := range acc.freeSockets() { if acc.needs(acc.topo.CPUsPerSocket()) { - glog.V(4).Infof("[cpumanager] takeByTopology: claiming socket [%d]", s) + klog.V(4).Infof("[cpumanager] takeByTopology: claiming socket [%d]", s) acc.take(acc.details.CPUsInSocket(s)) if acc.isSatisfied() { return acc.result, nil @@ -172,7 +172,7 @@ func takeByTopology(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, num // a core's-worth of CPUs. for _, c := range acc.freeCores() { if acc.needs(acc.topo.CPUsPerCore()) { - glog.V(4).Infof("[cpumanager] takeByTopology: claiming core [%d]", c) + klog.V(4).Infof("[cpumanager] takeByTopology: claiming core [%d]", c) acc.take(acc.details.CPUsInCore(c)) if acc.isSatisfied() { return acc.result, nil @@ -184,7 +184,7 @@ func takeByTopology(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, num // on the same sockets as the whole cores we have already taken in this // allocation. for _, c := range acc.freeCPUs() { - glog.V(4).Infof("[cpumanager] takeByTopology: claiming CPU [%d]", c) + klog.V(4).Infof("[cpumanager] takeByTopology: claiming CPU [%d]", c) if acc.needs(1) { acc.take(cpuset.NewCPUSet(c)) } diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go index 0754a4da092..4ccddd554da 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go @@ -22,10 +22,10 @@ import ( "sync" "time" - "github.com/golang/glog" cadvisorapi "github.com/google/cadvisor/info/v1" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" @@ -110,7 +110,7 @@ func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo if err != nil { return nil, err } - glog.Infof("[cpumanager] detected CPU topology: %v", topo) + klog.Infof("[cpumanager] detected CPU topology: %v", topo) reservedCPUs, ok := nodeAllocatableReservation[v1.ResourceCPU] if !ok { // The static policy cannot initialize without this information. @@ -132,7 +132,7 @@ func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo policy = NewStaticPolicy(topo, numReservedCPUs) default: - glog.Errorf("[cpumanager] Unknown policy \"%s\", falling back to default policy \"%s\"", cpuPolicyName, PolicyNone) + klog.Errorf("[cpumanager] Unknown policy \"%s\", falling back to default policy \"%s\"", cpuPolicyName, PolicyNone) policy = NewNonePolicy() } @@ -152,8 +152,8 @@ func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo } func (m *manager) Start(activePods ActivePodsFunc, podStatusProvider status.PodStatusProvider, containerRuntime runtimeService) { - glog.Infof("[cpumanager] starting with %s policy", m.policy.Name()) - glog.Infof("[cpumanager] reconciling every %v", m.reconcilePeriod) + klog.Infof("[cpumanager] starting with %s policy", m.policy.Name()) + klog.Infof("[cpumanager] reconciling every %v", m.reconcilePeriod) m.activePods = activePods m.podStatusProvider = podStatusProvider @@ -170,7 +170,7 @@ func (m *manager) AddContainer(p *v1.Pod, c *v1.Container, containerID string) e m.Lock() err := m.policy.AddContainer(m.state, p, c, containerID) if err != nil { - glog.Errorf("[cpumanager] AddContainer error: %v", err) + klog.Errorf("[cpumanager] AddContainer error: %v", err) m.Unlock() return err } @@ -180,17 +180,17 @@ func (m *manager) AddContainer(p *v1.Pod, c *v1.Container, containerID string) e if !cpus.IsEmpty() { err = m.updateContainerCPUSet(containerID, cpus) if err != nil { - glog.Errorf("[cpumanager] AddContainer error: %v", err) + klog.Errorf("[cpumanager] AddContainer error: %v", err) m.Lock() err := m.policy.RemoveContainer(m.state, containerID) if err != nil { - glog.Errorf("[cpumanager] AddContainer rollback state error: %v", err) + klog.Errorf("[cpumanager] AddContainer rollback state error: %v", err) } m.Unlock() } return err } - glog.V(5).Infof("[cpumanager] update container resources is skipped due to cpu set is empty") + klog.V(5).Infof("[cpumanager] update container resources is skipped due to cpu set is empty") return nil } @@ -200,7 +200,7 @@ func (m *manager) RemoveContainer(containerID string) error { err := m.policy.RemoveContainer(m.state, containerID) if err != nil { - glog.Errorf("[cpumanager] RemoveContainer error: %v", err) + klog.Errorf("[cpumanager] RemoveContainer error: %v", err) return err } return nil @@ -226,14 +226,14 @@ func (m *manager) reconcileState() (success []reconciledContainer, failure []rec for _, container := range allContainers { status, ok := m.podStatusProvider.GetPodStatus(pod.UID) if !ok { - glog.Warningf("[cpumanager] reconcileState: skipping pod; status not found (pod: %s, container: %s)", pod.Name, container.Name) + klog.Warningf("[cpumanager] reconcileState: skipping pod; status not found (pod: %s, container: %s)", pod.Name, container.Name) failure = append(failure, reconciledContainer{pod.Name, container.Name, ""}) break } containerID, err := findContainerIDByName(&status, container.Name) if err != nil { - glog.Warningf("[cpumanager] reconcileState: skipping container; ID not found in status (pod: %s, container: %s, error: %v)", pod.Name, container.Name, err) + klog.Warningf("[cpumanager] reconcileState: skipping container; ID not found in status (pod: %s, container: %s, error: %v)", pod.Name, container.Name, err) failure = append(failure, reconciledContainer{pod.Name, container.Name, ""}) continue } @@ -244,10 +244,10 @@ func (m *manager) reconcileState() (success []reconciledContainer, failure []rec // - container has been removed from state by RemoveContainer call (DeletionTimestamp is set) if _, ok := m.state.GetCPUSet(containerID); !ok { if status.Phase == v1.PodRunning && pod.DeletionTimestamp == nil { - glog.V(4).Infof("[cpumanager] reconcileState: container is not present in state - trying to add (pod: %s, container: %s, container id: %s)", pod.Name, container.Name, containerID) + klog.V(4).Infof("[cpumanager] reconcileState: container is not present in state - trying to add (pod: %s, container: %s, container id: %s)", pod.Name, container.Name, containerID) err := m.AddContainer(pod, &container, containerID) if err != nil { - glog.Errorf("[cpumanager] reconcileState: failed to add container (pod: %s, container: %s, container id: %s, error: %v)", pod.Name, container.Name, containerID, err) + klog.Errorf("[cpumanager] reconcileState: failed to add container (pod: %s, container: %s, container id: %s, error: %v)", pod.Name, container.Name, containerID, err) failure = append(failure, reconciledContainer{pod.Name, container.Name, containerID}) continue } @@ -261,15 +261,15 @@ func (m *manager) reconcileState() (success []reconciledContainer, failure []rec cset := m.state.GetCPUSetOrDefault(containerID) if cset.IsEmpty() { // NOTE: This should not happen outside of tests. - glog.Infof("[cpumanager] reconcileState: skipping container; assigned cpuset is empty (pod: %s, container: %s)", pod.Name, container.Name) + klog.Infof("[cpumanager] reconcileState: skipping container; assigned cpuset is empty (pod: %s, container: %s)", pod.Name, container.Name) failure = append(failure, reconciledContainer{pod.Name, container.Name, containerID}) continue } - glog.V(4).Infof("[cpumanager] reconcileState: updating container (pod: %s, container: %s, container id: %s, cpuset: \"%v\")", pod.Name, container.Name, containerID, cset) + klog.V(4).Infof("[cpumanager] reconcileState: updating container (pod: %s, container: %s, container id: %s, cpuset: \"%v\")", pod.Name, container.Name, containerID, cset) err = m.updateContainerCPUSet(containerID, cset) if err != nil { - glog.Errorf("[cpumanager] reconcileState: failed to update container (pod: %s, container: %s, container id: %s, cpuset: \"%v\", error: %v)", pod.Name, container.Name, containerID, cset, err) + klog.Errorf("[cpumanager] reconcileState: failed to update container (pod: %s, container: %s, container id: %s, cpuset: \"%v\", error: %v)", pod.Name, container.Name, containerID, cset, err) failure = append(failure, reconciledContainer{pod.Name, container.Name, containerID}) continue } diff --git a/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go b/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go index f4af5667ea6..8240bbd497d 100644 --- a/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go +++ b/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go @@ -17,8 +17,8 @@ limitations under the License. package cpumanager import ( - "github.com/golang/glog" "k8s.io/api/core/v1" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/status" ) @@ -28,21 +28,21 @@ type fakeManager struct { } func (m *fakeManager) Start(activePods ActivePodsFunc, podStatusProvider status.PodStatusProvider, containerRuntime runtimeService) { - glog.Info("[fake cpumanager] Start()") + klog.Info("[fake cpumanager] Start()") } func (m *fakeManager) Policy() Policy { - glog.Info("[fake cpumanager] Policy()") + klog.Info("[fake cpumanager] Policy()") return NewNonePolicy() } func (m *fakeManager) AddContainer(pod *v1.Pod, container *v1.Container, containerID string) error { - glog.Infof("[fake cpumanager] AddContainer (pod: %s, container: %s, container id: %s)", pod.Name, container.Name, containerID) + klog.Infof("[fake cpumanager] AddContainer (pod: %s, container: %s, container id: %s)", pod.Name, container.Name, containerID) return nil } func (m *fakeManager) RemoveContainer(containerID string) error { - glog.Infof("[fake cpumanager] RemoveContainer (container id: %s)", containerID) + klog.Infof("[fake cpumanager] RemoveContainer (container id: %s)", containerID) return nil } diff --git a/pkg/kubelet/cm/cpumanager/policy_none.go b/pkg/kubelet/cm/cpumanager/policy_none.go index 26e3335f73a..294edc6bf31 100644 --- a/pkg/kubelet/cm/cpumanager/policy_none.go +++ b/pkg/kubelet/cm/cpumanager/policy_none.go @@ -17,8 +17,8 @@ limitations under the License. package cpumanager import ( - "github.com/golang/glog" "k8s.io/api/core/v1" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" ) @@ -39,7 +39,7 @@ func (p *nonePolicy) Name() string { } func (p *nonePolicy) Start(s state.State) { - glog.Info("[cpumanager] none policy: Start") + klog.Info("[cpumanager] none policy: Start") } func (p *nonePolicy) AddContainer(s state.State, pod *v1.Pod, container *v1.Container, containerID string) error { diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go index 651e5fe79c9..d72a44ad0e9 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static.go +++ b/pkg/kubelet/cm/cpumanager/policy_static.go @@ -19,8 +19,8 @@ package cpumanager import ( "fmt" - "github.com/golang/glog" "k8s.io/api/core/v1" + "k8s.io/klog" v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" @@ -94,7 +94,7 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int) Policy panic(fmt.Sprintf("[cpumanager] unable to reserve the required amount of CPUs (size of %s did not equal %d)", reserved, numReservedCPUs)) } - glog.Infof("[cpumanager] reserved %d CPUs (\"%s\") not available for exclusive assignment", reserved.Size(), reserved) + klog.Infof("[cpumanager] reserved %d CPUs (\"%s\") not available for exclusive assignment", reserved.Size(), reserved) return &staticPolicy{ topology: topology, @@ -108,7 +108,7 @@ func (p *staticPolicy) Name() string { func (p *staticPolicy) Start(s state.State) { if err := p.validateState(s); err != nil { - glog.Errorf("[cpumanager] static policy invalid state: %s\n", err.Error()) + klog.Errorf("[cpumanager] static policy invalid state: %s\n", err.Error()) panic("[cpumanager] - please drain node and remove policy state file") } } @@ -172,17 +172,17 @@ func (p *staticPolicy) assignableCPUs(s state.State) cpuset.CPUSet { func (p *staticPolicy) AddContainer(s state.State, pod *v1.Pod, container *v1.Container, containerID string) error { if numCPUs := guaranteedCPUs(pod, container); numCPUs != 0 { - glog.Infof("[cpumanager] static policy: AddContainer (pod: %s, container: %s, container id: %s)", pod.Name, container.Name, containerID) + klog.Infof("[cpumanager] static policy: AddContainer (pod: %s, container: %s, container id: %s)", pod.Name, container.Name, containerID) // container belongs in an exclusively allocated pool if _, ok := s.GetCPUSet(containerID); ok { - glog.Infof("[cpumanager] static policy: container already present in state, skipping (container: %s, container id: %s)", container.Name, containerID) + klog.Infof("[cpumanager] static policy: container already present in state, skipping (container: %s, container id: %s)", container.Name, containerID) return nil } cpuset, err := p.allocateCPUs(s, numCPUs) if err != nil { - glog.Errorf("[cpumanager] unable to allocate %d CPUs (container id: %s, error: %v)", numCPUs, containerID, err) + klog.Errorf("[cpumanager] unable to allocate %d CPUs (container id: %s, error: %v)", numCPUs, containerID, err) return err } s.SetCPUSet(containerID, cpuset) @@ -192,7 +192,7 @@ func (p *staticPolicy) AddContainer(s state.State, pod *v1.Pod, container *v1.Co } func (p *staticPolicy) RemoveContainer(s state.State, containerID string) error { - glog.Infof("[cpumanager] static policy: RemoveContainer (container id: %s)", containerID) + klog.Infof("[cpumanager] static policy: RemoveContainer (container id: %s)", containerID) if toRelease, ok := s.GetCPUSet(containerID); ok { s.Delete(containerID) // Mutate the shared pool, adding released cpus. @@ -202,7 +202,7 @@ func (p *staticPolicy) RemoveContainer(s state.State, containerID string) error } func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int) (cpuset.CPUSet, error) { - glog.Infof("[cpumanager] allocateCpus: (numCPUs: %d)", numCPUs) + klog.Infof("[cpumanager] allocateCpus: (numCPUs: %d)", numCPUs) result, err := takeByTopology(p.topology, p.assignableCPUs(s), numCPUs) if err != nil { return cpuset.NewCPUSet(), err @@ -210,7 +210,7 @@ func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int) (cpuset.CPUSet, // Remove allocated CPUs from the shared CPUSet. s.SetDefaultCPUSet(s.GetDefaultCPUSet().Difference(result)) - glog.Infof("[cpumanager] allocateCPUs: returning \"%v\"", result) + klog.Infof("[cpumanager] allocateCPUs: returning \"%v\"", result) return result, nil } diff --git a/pkg/kubelet/cm/cpumanager/state/BUILD b/pkg/kubelet/cm/cpumanager/state/BUILD index d39211962e8..6a95d1af163 100644 --- a/pkg/kubelet/cm/cpumanager/state/BUILD +++ b/pkg/kubelet/cm/cpumanager/state/BUILD @@ -16,7 +16,7 @@ go_library( "//pkg/kubelet/checkpointmanager/checksum:go_default_library", "//pkg/kubelet/checkpointmanager/errors:go_default_library", "//pkg/kubelet/cm/cpuset:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/cm/cpumanager/state/state_checkpoint.go b/pkg/kubelet/cm/cpumanager/state/state_checkpoint.go index 6d92573b866..badafab43d8 100644 --- a/pkg/kubelet/cm/cpumanager/state/state_checkpoint.go +++ b/pkg/kubelet/cm/cpumanager/state/state_checkpoint.go @@ -21,7 +21,7 @@ import ( "path" "sync" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -97,8 +97,8 @@ func (sc *stateCheckpoint) restoreState() error { sc.cache.SetDefaultCPUSet(tmpDefaultCPUSet) sc.cache.SetCPUAssignments(tmpAssignments) - glog.V(2).Info("[cpumanager] state checkpoint: restored state from checkpoint") - glog.V(2).Infof("[cpumanager] state checkpoint: defaultCPUSet: %s", tmpDefaultCPUSet.String()) + klog.V(2).Info("[cpumanager] state checkpoint: restored state from checkpoint") + klog.V(2).Infof("[cpumanager] state checkpoint: defaultCPUSet: %s", tmpDefaultCPUSet.String()) return nil } diff --git a/pkg/kubelet/cm/cpumanager/state/state_file.go b/pkg/kubelet/cm/cpumanager/state/state_file.go index 6c2353cf10f..90d16693dc3 100644 --- a/pkg/kubelet/cm/cpumanager/state/state_file.go +++ b/pkg/kubelet/cm/cpumanager/state/state_file.go @@ -19,8 +19,8 @@ package state import ( "encoding/json" "fmt" - "github.com/golang/glog" "io/ioutil" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" "os" "sync" @@ -79,7 +79,7 @@ func (sf *stateFile) tryRestoreState() error { // If the state file does not exist or has zero length, write a new file. if os.IsNotExist(err) || len(content) == 0 { sf.storeState() - glog.Infof("[cpumanager] state file: created new state file \"%s\"", sf.stateFilePath) + klog.Infof("[cpumanager] state file: created new state file \"%s\"", sf.stateFilePath) return nil } @@ -92,7 +92,7 @@ func (sf *stateFile) tryRestoreState() error { var readState stateFileData if err = json.Unmarshal(content, &readState); err != nil { - glog.Errorf("[cpumanager] state file: could not unmarshal, corrupted state file - \"%s\"", sf.stateFilePath) + klog.Errorf("[cpumanager] state file: could not unmarshal, corrupted state file - \"%s\"", sf.stateFilePath) return err } @@ -101,13 +101,13 @@ func (sf *stateFile) tryRestoreState() error { } if tmpDefaultCPUSet, err = cpuset.Parse(readState.DefaultCPUSet); err != nil { - glog.Errorf("[cpumanager] state file: could not parse state file - [defaultCpuSet:\"%s\"]", readState.DefaultCPUSet) + klog.Errorf("[cpumanager] state file: could not parse state file - [defaultCpuSet:\"%s\"]", readState.DefaultCPUSet) return err } for containerID, cpuString := range readState.Entries { if tmpContainerCPUSet, err = cpuset.Parse(cpuString); err != nil { - glog.Errorf("[cpumanager] state file: could not parse state file - container id: %s, cpuset: \"%s\"", containerID, cpuString) + klog.Errorf("[cpumanager] state file: could not parse state file - container id: %s, cpuset: \"%s\"", containerID, cpuString) return err } tmpAssignments[containerID] = tmpContainerCPUSet @@ -116,8 +116,8 @@ func (sf *stateFile) tryRestoreState() error { sf.cache.SetDefaultCPUSet(tmpDefaultCPUSet) sf.cache.SetCPUAssignments(tmpAssignments) - glog.V(2).Infof("[cpumanager] state file: restored state from state file \"%s\"", sf.stateFilePath) - glog.V(2).Infof("[cpumanager] state file: defaultCPUSet: %s", tmpDefaultCPUSet.String()) + klog.V(2).Infof("[cpumanager] state file: restored state from state file \"%s\"", sf.stateFilePath) + klog.V(2).Infof("[cpumanager] state file: defaultCPUSet: %s", tmpDefaultCPUSet.String()) return nil } diff --git a/pkg/kubelet/cm/cpumanager/state/state_mem.go b/pkg/kubelet/cm/cpumanager/state/state_mem.go index 816c89868e5..77c5f4a525c 100644 --- a/pkg/kubelet/cm/cpumanager/state/state_mem.go +++ b/pkg/kubelet/cm/cpumanager/state/state_mem.go @@ -19,7 +19,7 @@ package state import ( "sync" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) @@ -33,7 +33,7 @@ var _ State = &stateMemory{} // NewMemoryState creates new State for keeping track of cpu/pod assignment func NewMemoryState() State { - glog.Infof("[cpumanager] initializing new in-memory state store") + klog.Infof("[cpumanager] initializing new in-memory state store") return &stateMemory{ assignments: ContainerCPUAssignments{}, defaultCPUSet: cpuset.NewCPUSet(), @@ -73,7 +73,7 @@ func (s *stateMemory) SetCPUSet(containerID string, cset cpuset.CPUSet) { defer s.Unlock() s.assignments[containerID] = cset - glog.Infof("[cpumanager] updated desired cpuset (container id: %s, cpuset: \"%s\")", containerID, cset) + klog.Infof("[cpumanager] updated desired cpuset (container id: %s, cpuset: \"%s\")", containerID, cset) } func (s *stateMemory) SetDefaultCPUSet(cset cpuset.CPUSet) { @@ -81,7 +81,7 @@ func (s *stateMemory) SetDefaultCPUSet(cset cpuset.CPUSet) { defer s.Unlock() s.defaultCPUSet = cset - glog.Infof("[cpumanager] updated default cpuset: \"%s\"", cset) + klog.Infof("[cpumanager] updated default cpuset: \"%s\"", cset) } func (s *stateMemory) SetCPUAssignments(a ContainerCPUAssignments) { @@ -89,7 +89,7 @@ func (s *stateMemory) SetCPUAssignments(a ContainerCPUAssignments) { defer s.Unlock() s.assignments = a.Clone() - glog.Infof("[cpumanager] updated cpuset assignments: \"%v\"", a) + klog.Infof("[cpumanager] updated cpuset assignments: \"%v\"", a) } func (s *stateMemory) Delete(containerID string) { @@ -97,7 +97,7 @@ func (s *stateMemory) Delete(containerID string) { defer s.Unlock() delete(s.assignments, containerID) - glog.V(2).Infof("[cpumanager] deleted cpuset assignment (container id: %s)", containerID) + klog.V(2).Infof("[cpumanager] deleted cpuset assignment (container id: %s)", containerID) } func (s *stateMemory) ClearState() { @@ -106,5 +106,5 @@ func (s *stateMemory) ClearState() { s.defaultCPUSet = cpuset.CPUSet{} s.assignments = make(ContainerCPUAssignments) - glog.V(2).Infof("[cpumanager] cleared state") + klog.V(2).Infof("[cpumanager] cleared state") } diff --git a/pkg/kubelet/cm/cpumanager/topology/BUILD b/pkg/kubelet/cm/cpumanager/topology/BUILD index e9f64078554..18ab41a33f4 100644 --- a/pkg/kubelet/cm/cpumanager/topology/BUILD +++ b/pkg/kubelet/cm/cpumanager/topology/BUILD @@ -10,8 +10,8 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/kubelet/cm/cpuset:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/cm/cpumanager/topology/topology.go b/pkg/kubelet/cm/cpumanager/topology/topology.go index 2491abe6c06..adb0aa7fc70 100644 --- a/pkg/kubelet/cm/cpumanager/topology/topology.go +++ b/pkg/kubelet/cm/cpumanager/topology/topology.go @@ -20,8 +20,8 @@ import ( "fmt" "sort" - "github.com/golang/glog" cadvisorapi "github.com/google/cadvisor/info/v1" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) @@ -156,7 +156,7 @@ func Discover(machineInfo *cadvisorapi.MachineInfo) (*CPUTopology, error) { numPhysicalCores += len(socket.Cores) for _, core := range socket.Cores { if coreID, err = getUniqueCoreID(core.Threads); err != nil { - glog.Errorf("could not get unique coreID for socket: %d core %d threads: %v", + klog.Errorf("could not get unique coreID for socket: %d core %d threads: %v", socket.Id, core.Id, core.Threads) return nil, err } diff --git a/pkg/kubelet/cm/cpuset/BUILD b/pkg/kubelet/cm/cpuset/BUILD index 89126d680e3..9b81edc1961 100644 --- a/pkg/kubelet/cm/cpuset/BUILD +++ b/pkg/kubelet/cm/cpuset/BUILD @@ -5,7 +5,7 @@ go_library( srcs = ["cpuset.go"], importpath = "k8s.io/kubernetes/pkg/kubelet/cm/cpuset", visibility = ["//visibility:public"], - deps = ["//vendor/github.com/golang/glog:go_default_library"], + deps = ["//vendor/k8s.io/klog:go_default_library"], ) go_test( diff --git a/pkg/kubelet/cm/cpuset/cpuset.go b/pkg/kubelet/cm/cpuset/cpuset.go index ccf577d819c..d87efc78593 100644 --- a/pkg/kubelet/cm/cpuset/cpuset.go +++ b/pkg/kubelet/cm/cpuset/cpuset.go @@ -19,7 +19,7 @@ package cpuset import ( "bytes" "fmt" - "github.com/golang/glog" + "k8s.io/klog" "reflect" "sort" "strconv" @@ -221,7 +221,7 @@ func (s CPUSet) String() string { func MustParse(s string) CPUSet { res, err := Parse(s) if err != nil { - glog.Fatalf("unable to parse [%s] as CPUSet: %v", s, err) + klog.Fatalf("unable to parse [%s] as CPUSet: %v", s, err) } return res } diff --git a/pkg/kubelet/cm/devicemanager/BUILD b/pkg/kubelet/cm/devicemanager/BUILD index 719dbbd65e1..885f783db79 100644 --- a/pkg/kubelet/cm/devicemanager/BUILD +++ b/pkg/kubelet/cm/devicemanager/BUILD @@ -15,7 +15,8 @@ go_library( deps = [ "//pkg/apis/core/v1/helper:go_default_library", "//pkg/kubelet/apis/deviceplugin/v1beta1:go_default_library", - "//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library", + "//pkg/kubelet/apis/pluginregistration/v1:go_default_library", + "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", "//pkg/kubelet/checkpointmanager:go_default_library", "//pkg/kubelet/checkpointmanager/errors:go_default_library", "//pkg/kubelet/cm/devicemanager/checkpoint:go_default_library", @@ -28,8 +29,8 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -42,7 +43,7 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/kubelet/apis/deviceplugin/v1beta1:go_default_library", - "//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library", + "//pkg/kubelet/apis/pluginregistration/v1:go_default_library", "//pkg/kubelet/checkpointmanager:go_default_library", "//pkg/kubelet/lifecycle:go_default_library", "//pkg/kubelet/util/pluginwatcher:go_default_library", diff --git a/pkg/kubelet/cm/devicemanager/device_plugin_stub.go b/pkg/kubelet/cm/devicemanager/device_plugin_stub.go index d4040fd1ec9..f0c7bc2641c 100644 --- a/pkg/kubelet/cm/devicemanager/device_plugin_stub.go +++ b/pkg/kubelet/cm/devicemanager/device_plugin_stub.go @@ -28,7 +28,7 @@ import ( "google.golang.org/grpc" pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" - watcherapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1alpha1" + watcherapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1" ) // Stub implementation for DevicePlugin. diff --git a/pkg/kubelet/cm/devicemanager/endpoint.go b/pkg/kubelet/cm/devicemanager/endpoint.go index eed50de6561..624643e5efb 100644 --- a/pkg/kubelet/cm/devicemanager/endpoint.go +++ b/pkg/kubelet/cm/devicemanager/endpoint.go @@ -23,8 +23,8 @@ import ( "sync" "time" - "github.com/golang/glog" "google.golang.org/grpc" + "k8s.io/klog" pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" ) @@ -59,7 +59,7 @@ type endpointImpl struct { func newEndpointImpl(socketPath, resourceName string, callback monitorCallback) (*endpointImpl, error) { client, c, err := dial(socketPath) if err != nil { - glog.Errorf("Can't create new endpoint with path %s err %v", socketPath, err) + klog.Errorf("Can't create new endpoint with path %s err %v", socketPath, err) return nil, err } @@ -95,7 +95,7 @@ func (e *endpointImpl) callback(resourceName string, devices []pluginapi.Device) func (e *endpointImpl) run() { stream, err := e.client.ListAndWatch(context.Background(), &pluginapi.Empty{}) if err != nil { - glog.Errorf(errListAndWatch, e.resourceName, err) + klog.Errorf(errListAndWatch, e.resourceName, err) return } @@ -103,12 +103,12 @@ func (e *endpointImpl) run() { for { response, err := stream.Recv() if err != nil { - glog.Errorf(errListAndWatch, e.resourceName, err) + klog.Errorf(errListAndWatch, e.resourceName, err) return } devs := response.Devices - glog.V(2).Infof("State pushed for device plugin %s", e.resourceName) + klog.V(2).Infof("State pushed for device plugin %s", e.resourceName) var newDevs []pluginapi.Device for _, d := range devs { diff --git a/pkg/kubelet/cm/devicemanager/manager.go b/pkg/kubelet/cm/devicemanager/manager.go index 8064b572b39..5434aa29f8e 100644 --- a/pkg/kubelet/cm/devicemanager/manager.go +++ b/pkg/kubelet/cm/devicemanager/manager.go @@ -25,14 +25,15 @@ import ( "sync" "time" - "github.com/golang/glog" "google.golang.org/grpc" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint" @@ -105,7 +106,7 @@ func NewManagerImpl() (*ManagerImpl, error) { } func newManagerImpl(socketPath string) (*ManagerImpl, error) { - glog.V(2).Infof("Creating Device Plugin manager at %s", socketPath) + klog.V(2).Infof("Creating Device Plugin manager at %s", socketPath) if socketPath == "" || !filepath.IsAbs(socketPath) { return nil, fmt.Errorf(errBadSocket+" %s", socketPath) @@ -169,7 +170,7 @@ func (m *ManagerImpl) removeContents(dir string) error { } stat, err := os.Stat(filePath) if err != nil { - glog.Errorf("Failed to stat file %s: %v", filePath, err) + klog.Errorf("Failed to stat file %s: %v", filePath, err) continue } if stat.IsDir() { @@ -192,7 +193,7 @@ func (m *ManagerImpl) checkpointFile() string { // podDevices and allocatedDevices information from checkpoint-ed state and // starts device plugin registration service. func (m *ManagerImpl) Start(activePods ActivePodsFunc, sourcesReady config.SourcesReady) error { - glog.V(2).Infof("Starting Device Plugin manager") + klog.V(2).Infof("Starting Device Plugin manager") m.activePods = activePods m.sourcesReady = sourcesReady @@ -200,7 +201,7 @@ func (m *ManagerImpl) Start(activePods ActivePodsFunc, sourcesReady config.Sourc // Loads in allocatedDevices information from disk. err := m.readCheckpoint() if err != nil { - glog.Warningf("Continue after failing to read checkpoint file. Device allocation info may NOT be up-to-date. Err: %v", err) + klog.Warningf("Continue after failing to read checkpoint file. Device allocation info may NOT be up-to-date. Err: %v", err) } socketPath := filepath.Join(m.socketdir, m.socketname) @@ -209,12 +210,12 @@ func (m *ManagerImpl) Start(activePods ActivePodsFunc, sourcesReady config.Sourc // Removes all stale sockets in m.socketdir. Device plugins can monitor // this and use it as a signal to re-register with the new Kubelet. if err := m.removeContents(m.socketdir); err != nil { - glog.Errorf("Fail to clean up stale contents under %s: %v", m.socketdir, err) + klog.Errorf("Fail to clean up stale contents under %s: %v", m.socketdir, err) } s, err := net.Listen("unix", socketPath) if err != nil { - glog.Errorf(errListenSocket+" %v", err) + klog.Errorf(errListenSocket+" %v", err) return err } @@ -227,7 +228,7 @@ func (m *ManagerImpl) Start(activePods ActivePodsFunc, sourcesReady config.Sourc m.server.Serve(s) }() - glog.V(2).Infof("Serving device plugin registration server on %q", socketPath) + klog.V(2).Infof("Serving device plugin registration server on %q", socketPath) return nil } @@ -235,10 +236,10 @@ func (m *ManagerImpl) Start(activePods ActivePodsFunc, sourcesReady config.Sourc // GetWatcherHandler returns the plugin handler func (m *ManagerImpl) GetWatcherHandler() watcher.PluginHandler { if f, err := os.Create(m.socketdir + "DEPRECATION"); err != nil { - glog.Errorf("Failed to create deprecation file at %s", m.socketdir) + klog.Errorf("Failed to create deprecation file at %s", m.socketdir) } else { f.Close() - glog.V(4).Infof("created deprecation file %s", f.Name()) + klog.V(4).Infof("created deprecation file %s", f.Name()) } return watcher.PluginHandler(m) @@ -246,7 +247,7 @@ func (m *ManagerImpl) GetWatcherHandler() watcher.PluginHandler { // ValidatePlugin validates a plugin if the version is correct and the name has the format of an extended resource func (m *ManagerImpl) ValidatePlugin(pluginName string, endpoint string, versions []string) error { - glog.V(2).Infof("Got Plugin %s at endpoint %s with versions %v", pluginName, endpoint, versions) + klog.V(2).Infof("Got Plugin %s at endpoint %s with versions %v", pluginName, endpoint, versions) if !m.isVersionCompatibleWithPlugin(versions) { return fmt.Errorf("manager version, %s, is not among plugin supported versions %v", pluginapi.Version, versions) @@ -263,7 +264,7 @@ func (m *ManagerImpl) ValidatePlugin(pluginName string, endpoint string, version // TODO: Start the endpoint and wait for the First ListAndWatch call // before registering the plugin func (m *ManagerImpl) RegisterPlugin(pluginName string, endpoint string) error { - glog.V(2).Infof("Registering Plugin %s at endpoint %s", pluginName, endpoint) + klog.V(2).Infof("Registering Plugin %s at endpoint %s", pluginName, endpoint) e, err := newEndpointImpl(endpoint, pluginName, m.callback) if err != nil { @@ -342,7 +343,7 @@ func (m *ManagerImpl) Allocate(node *schedulercache.NodeInfo, attrs *lifecycle.P // Register registers a device plugin. func (m *ManagerImpl) Register(ctx context.Context, r *pluginapi.RegisterRequest) (*pluginapi.Empty, error) { - glog.Infof("Got registration request from device plugin with resource name %q", r.ResourceName) + klog.Infof("Got registration request from device plugin with resource name %q", r.ResourceName) metrics.DevicePluginRegistrationCount.WithLabelValues(r.ResourceName).Inc() var versionCompatible bool for _, v := range pluginapi.SupportedVersions { @@ -353,13 +354,13 @@ func (m *ManagerImpl) Register(ctx context.Context, r *pluginapi.RegisterRequest } if !versionCompatible { errorString := fmt.Sprintf(errUnsupportedVersion, r.Version, pluginapi.SupportedVersions) - glog.Infof("Bad registration request from device plugin with resource name %q: %s", r.ResourceName, errorString) + klog.Infof("Bad registration request from device plugin with resource name %q: %s", r.ResourceName, errorString) return &pluginapi.Empty{}, fmt.Errorf(errorString) } if !v1helper.IsExtendedResourceName(v1.ResourceName(r.ResourceName)) { errorString := fmt.Sprintf(errInvalidResourceName, r.ResourceName) - glog.Infof("Bad registration request from device plugin: %s", errorString) + klog.Infof("Bad registration request from device plugin: %s", errorString) return &pluginapi.Empty{}, fmt.Errorf(errorString) } @@ -396,7 +397,7 @@ func (m *ManagerImpl) registerEndpoint(resourceName string, options *pluginapi.D defer m.mutex.Unlock() m.endpoints[resourceName] = endpointInfo{e: e, opts: options} - glog.V(2).Infof("Registered endpoint %v", e) + klog.V(2).Infof("Registered endpoint %v", e) } func (m *ManagerImpl) runEndpoint(resourceName string, e endpoint) { @@ -410,13 +411,13 @@ func (m *ManagerImpl) runEndpoint(resourceName string, e endpoint) { m.markResourceUnhealthy(resourceName) } - glog.V(2).Infof("Endpoint (%s, %v) became unhealthy", resourceName, e) + klog.V(2).Infof("Endpoint (%s, %v) became unhealthy", resourceName, e) } func (m *ManagerImpl) addEndpoint(r *pluginapi.RegisterRequest) { new, err := newEndpointImpl(filepath.Join(m.socketdir, r.Endpoint), r.ResourceName, m.callback) if err != nil { - glog.Errorf("Failed to dial device plugin with request %v: %v", r, err) + klog.Errorf("Failed to dial device plugin with request %v: %v", r, err) return } m.registerEndpoint(r.ResourceName, r.Options, new) @@ -426,7 +427,7 @@ func (m *ManagerImpl) addEndpoint(r *pluginapi.RegisterRequest) { } func (m *ManagerImpl) markResourceUnhealthy(resourceName string) { - glog.V(2).Infof("Mark all resources Unhealthy for resource %s", resourceName) + klog.V(2).Infof("Mark all resources Unhealthy for resource %s", resourceName) healthyDevices := sets.NewString() if _, ok := m.healthyDevices[resourceName]; ok { healthyDevices = m.healthyDevices[resourceName] @@ -463,7 +464,7 @@ func (m *ManagerImpl) GetCapacity() (v1.ResourceList, v1.ResourceList, []string) // should always be consistent. Otherwise, we run with the risk // of failing to garbage collect non-existing resources or devices. if !ok { - glog.Errorf("unexpected: healthyDevices and endpoints are out of sync") + klog.Errorf("unexpected: healthyDevices and endpoints are out of sync") } delete(m.endpoints, resourceName) delete(m.healthyDevices, resourceName) @@ -478,7 +479,7 @@ func (m *ManagerImpl) GetCapacity() (v1.ResourceList, v1.ResourceList, []string) eI, ok := m.endpoints[resourceName] if (ok && eI.e.stopGracePeriodExpired()) || !ok { if !ok { - glog.Errorf("unexpected: unhealthyDevices and endpoints are out of sync") + klog.Errorf("unexpected: unhealthyDevices and endpoints are out of sync") } delete(m.endpoints, resourceName) delete(m.unhealthyDevices, resourceName) @@ -524,7 +525,7 @@ func (m *ManagerImpl) readCheckpoint() error { err := m.checkpointManager.GetCheckpoint(kubeletDeviceManagerCheckpoint, cp) if err != nil { if err == errors.ErrCheckpointNotFound { - glog.Warningf("Failed to retrieve checkpoint for %q: %v", kubeletDeviceManagerCheckpoint, err) + klog.Warningf("Failed to retrieve checkpoint for %q: %v", kubeletDeviceManagerCheckpoint, err) return nil } return err @@ -561,7 +562,7 @@ func (m *ManagerImpl) updateAllocatedDevices(activePods []*v1.Pod) { if len(podsToBeRemoved) <= 0 { return } - glog.V(3).Infof("pods to be removed: %v", podsToBeRemoved.List()) + klog.V(3).Infof("pods to be removed: %v", podsToBeRemoved.List()) m.podDevices.delete(podsToBeRemoved.List()) // Regenerated allocatedDevices after we update pod allocation information. m.allocatedDevices = m.podDevices.devices() @@ -577,7 +578,7 @@ func (m *ManagerImpl) devicesToAllocate(podUID, contName, resource string, requi // This can happen if a container restarts for example. devices := m.podDevices.containerDevices(podUID, contName, resource) if devices != nil { - glog.V(3).Infof("Found pre-allocated devices for resource %s container %q in Pod %q: %v", resource, contName, podUID, devices.List()) + klog.V(3).Infof("Found pre-allocated devices for resource %s container %q in Pod %q: %v", resource, contName, podUID, devices.List()) needed = needed - devices.Len() // A pod's resource is not expected to change once admitted by the API server, // so just fail loudly here. We can revisit this part if this no longer holds. @@ -589,7 +590,7 @@ func (m *ManagerImpl) devicesToAllocate(podUID, contName, resource string, requi // No change, no work. return nil, nil } - glog.V(3).Infof("Needs to allocate %d %q for pod %q container %q", needed, resource, podUID, contName) + klog.V(3).Infof("Needs to allocate %d %q for pod %q container %q", needed, resource, podUID, contName) // Needs to allocate additional devices. if _, ok := m.healthyDevices[resource]; !ok { return nil, fmt.Errorf("can't allocate unregistered device %s", resource) @@ -640,7 +641,7 @@ func (m *ManagerImpl) allocateContainerResources(pod *v1.Pod, container *v1.Cont for k, v := range container.Resources.Limits { resource := string(k) needed := int(v.Value()) - glog.V(3).Infof("needs %d %s", needed, resource) + klog.V(3).Infof("needs %d %s", needed, resource) if !m.isDevicePluginResource(resource) { continue } @@ -684,7 +685,7 @@ func (m *ManagerImpl) allocateContainerResources(pod *v1.Pod, container *v1.Cont devs := allocDevices.UnsortedList() // TODO: refactor this part of code to just append a ContainerAllocationRequest // in a passed in AllocateRequest pointer, and issues a single Allocate call per pod. - glog.V(3).Infof("Making allocation request for devices %v for device plugin %s", devs, resource) + klog.V(3).Infof("Making allocation request for devices %v for device plugin %s", devs, resource) resp, err := eI.e.allocate(devs) metrics.DevicePluginAllocationLatency.WithLabelValues(resource).Observe(metrics.SinceInMicroseconds(startRPCTime)) if err != nil { @@ -743,7 +744,7 @@ func (m *ManagerImpl) callPreStartContainerIfNeeded(podUID, contName, resource s if eI.opts == nil || !eI.opts.PreStartRequired { m.mutex.Unlock() - glog.V(4).Infof("Plugin options indicate to skip PreStartContainer for resource: %s", resource) + klog.V(4).Infof("Plugin options indicate to skip PreStartContainer for resource: %s", resource) return nil } @@ -755,7 +756,7 @@ func (m *ManagerImpl) callPreStartContainerIfNeeded(podUID, contName, resource s m.mutex.Unlock() devs := devices.UnsortedList() - glog.V(4).Infof("Issuing an PreStartContainer call for container, %s, of pod %s", contName, podUID) + klog.V(4).Infof("Issuing an PreStartContainer call for container, %s, of pod %s", contName, podUID) _, err := eI.e.preStartContainer(devs) if err != nil { return fmt.Errorf("device plugin PreStartContainer rpc failed with err: %v", err) @@ -802,3 +803,10 @@ func (m *ManagerImpl) isDevicePluginResource(resource string) bool { } return false } + +// GetDevices returns the devices used by the specified container +func (m *ManagerImpl) GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices { + m.mutex.Lock() + defer m.mutex.Unlock() + return m.podDevices.getContainerDevices(podUID, containerName) +} diff --git a/pkg/kubelet/cm/devicemanager/manager_stub.go b/pkg/kubelet/cm/devicemanager/manager_stub.go index 1008daca3b7..e32b671ffb2 100644 --- a/pkg/kubelet/cm/devicemanager/manager_stub.go +++ b/pkg/kubelet/cm/devicemanager/manager_stub.go @@ -18,6 +18,7 @@ package devicemanager import ( "k8s.io/api/core/v1" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/config" "k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher" @@ -61,3 +62,8 @@ func (h *ManagerStub) GetCapacity() (v1.ResourceList, v1.ResourceList, []string) func (h *ManagerStub) GetWatcherHandler() pluginwatcher.PluginHandler { return nil } + +// GetDevices returns nil +func (h *ManagerStub) GetDevices(_, _ string) []*podresourcesapi.ContainerDevices { + return nil +} diff --git a/pkg/kubelet/cm/devicemanager/manager_test.go b/pkg/kubelet/cm/devicemanager/manager_test.go index 22faea96689..637779cfdbf 100644 --- a/pkg/kubelet/cm/devicemanager/manager_test.go +++ b/pkg/kubelet/cm/devicemanager/manager_test.go @@ -33,7 +33,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" - watcherapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1alpha1" + watcherapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" "k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher" diff --git a/pkg/kubelet/cm/devicemanager/pod_devices.go b/pkg/kubelet/cm/devicemanager/pod_devices.go index 7df0e201c71..4fd4b196c4e 100644 --- a/pkg/kubelet/cm/devicemanager/pod_devices.go +++ b/pkg/kubelet/cm/devicemanager/pod_devices.go @@ -17,10 +17,11 @@ limitations under the License. package devicemanager import ( - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/sets" pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -135,13 +136,13 @@ func (pdev podDevices) toCheckpointData() []checkpoint.PodDevicesEntry { for resource, devices := range resources { devIds := devices.deviceIds.UnsortedList() if devices.allocResp == nil { - glog.Errorf("Can't marshal allocResp for %v %v %v: allocation response is missing", podUID, conName, resource) + klog.Errorf("Can't marshal allocResp for %v %v %v: allocation response is missing", podUID, conName, resource) continue } allocResp, err := devices.allocResp.Marshal() if err != nil { - glog.Errorf("Can't marshal allocResp for %v %v %v: %v", podUID, conName, resource, err) + klog.Errorf("Can't marshal allocResp for %v %v %v: %v", podUID, conName, resource, err) continue } data = append(data, checkpoint.PodDevicesEntry{ @@ -159,7 +160,7 @@ func (pdev podDevices) toCheckpointData() []checkpoint.PodDevicesEntry { // Populates podDevices from the passed in checkpointData. func (pdev podDevices) fromCheckpointData(data []checkpoint.PodDevicesEntry) { for _, entry := range data { - glog.V(2).Infof("Get checkpoint entry: %v %v %v %v %v\n", + klog.V(2).Infof("Get checkpoint entry: %v %v %v %v %v\n", entry.PodUID, entry.ContainerName, entry.ResourceName, entry.DeviceIDs, entry.AllocResp) devIDs := sets.NewString() for _, devID := range entry.DeviceIDs { @@ -168,7 +169,7 @@ func (pdev podDevices) fromCheckpointData(data []checkpoint.PodDevicesEntry) { allocResp := &pluginapi.ContainerAllocateResponse{} err := allocResp.Unmarshal(entry.AllocResp) if err != nil { - glog.Errorf("Can't unmarshal allocResp for %v %v %v: %v", entry.PodUID, entry.ContainerName, entry.ResourceName, err) + klog.Errorf("Can't unmarshal allocResp for %v %v %v: %v", entry.PodUID, entry.ContainerName, entry.ResourceName, err) continue } pdev.insert(entry.PodUID, entry.ContainerName, entry.ResourceName, devIDs, allocResp) @@ -203,13 +204,13 @@ func (pdev podDevices) deviceRunContainerOptions(podUID, contName string) *Devic // Updates RunContainerOptions.Envs. for k, v := range resp.Envs { if e, ok := envsMap[k]; ok { - glog.V(4).Infof("Skip existing env %s %s", k, v) + klog.V(4).Infof("Skip existing env %s %s", k, v) if e != v { - glog.Errorf("Environment variable %s has conflicting setting: %s and %s", k, e, v) + klog.Errorf("Environment variable %s has conflicting setting: %s and %s", k, e, v) } continue } - glog.V(4).Infof("Add env %s %s", k, v) + klog.V(4).Infof("Add env %s %s", k, v) envsMap[k] = v opts.Envs = append(opts.Envs, kubecontainer.EnvVar{Name: k, Value: v}) } @@ -217,14 +218,14 @@ func (pdev podDevices) deviceRunContainerOptions(podUID, contName string) *Devic // Updates RunContainerOptions.Devices. for _, dev := range resp.Devices { if d, ok := devsMap[dev.ContainerPath]; ok { - glog.V(4).Infof("Skip existing device %s %s", dev.ContainerPath, dev.HostPath) + klog.V(4).Infof("Skip existing device %s %s", dev.ContainerPath, dev.HostPath) if d != dev.HostPath { - glog.Errorf("Container device %s has conflicting mapping host devices: %s and %s", + klog.Errorf("Container device %s has conflicting mapping host devices: %s and %s", dev.ContainerPath, d, dev.HostPath) } continue } - glog.V(4).Infof("Add device %s %s", dev.ContainerPath, dev.HostPath) + klog.V(4).Infof("Add device %s %s", dev.ContainerPath, dev.HostPath) devsMap[dev.ContainerPath] = dev.HostPath opts.Devices = append(opts.Devices, kubecontainer.DeviceInfo{ PathOnHost: dev.HostPath, @@ -236,14 +237,14 @@ func (pdev podDevices) deviceRunContainerOptions(podUID, contName string) *Devic // Updates RunContainerOptions.Mounts. for _, mount := range resp.Mounts { if m, ok := mountsMap[mount.ContainerPath]; ok { - glog.V(4).Infof("Skip existing mount %s %s", mount.ContainerPath, mount.HostPath) + klog.V(4).Infof("Skip existing mount %s %s", mount.ContainerPath, mount.HostPath) if m != mount.HostPath { - glog.Errorf("Container mount %s has conflicting mapping host mounts: %s and %s", + klog.Errorf("Container mount %s has conflicting mapping host mounts: %s and %s", mount.ContainerPath, m, mount.HostPath) } continue } - glog.V(4).Infof("Add mount %s %s", mount.ContainerPath, mount.HostPath) + klog.V(4).Infof("Add mount %s %s", mount.ContainerPath, mount.HostPath) mountsMap[mount.ContainerPath] = mount.HostPath opts.Mounts = append(opts.Mounts, kubecontainer.Mount{ Name: mount.ContainerPath, @@ -258,16 +259,34 @@ func (pdev podDevices) deviceRunContainerOptions(podUID, contName string) *Devic // Updates for Annotations for k, v := range resp.Annotations { if e, ok := annotationsMap[k]; ok { - glog.V(4).Infof("Skip existing annotation %s %s", k, v) + klog.V(4).Infof("Skip existing annotation %s %s", k, v) if e != v { - glog.Errorf("Annotation %s has conflicting setting: %s and %s", k, e, v) + klog.Errorf("Annotation %s has conflicting setting: %s and %s", k, e, v) } continue } - glog.V(4).Infof("Add annotation %s %s", k, v) + klog.V(4).Infof("Add annotation %s %s", k, v) annotationsMap[k] = v opts.Annotations = append(opts.Annotations, kubecontainer.Annotation{Name: k, Value: v}) } } return opts } + +// getContainerDevices returns the devices assigned to the provided container for all ResourceNames +func (pdev podDevices) getContainerDevices(podUID, contName string) []*podresourcesapi.ContainerDevices { + if _, podExists := pdev[podUID]; !podExists { + return nil + } + if _, contExists := pdev[podUID][contName]; !contExists { + return nil + } + cDev := []*podresourcesapi.ContainerDevices{} + for resource, allocateInfo := range pdev[podUID][contName] { + cDev = append(cDev, &podresourcesapi.ContainerDevices{ + ResourceName: resource, + DeviceIds: allocateInfo.deviceIds.UnsortedList(), + }) + } + return cDev +} diff --git a/pkg/kubelet/cm/devicemanager/types.go b/pkg/kubelet/cm/devicemanager/types.go index 35923b00d12..8396378b407 100644 --- a/pkg/kubelet/cm/devicemanager/types.go +++ b/pkg/kubelet/cm/devicemanager/types.go @@ -20,6 +20,7 @@ import ( "time" "k8s.io/api/core/v1" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/lifecycle" @@ -54,6 +55,9 @@ type Manager interface { // and inactive device plugin resources previously registered on the node. GetCapacity() (v1.ResourceList, v1.ResourceList, []string) GetWatcherHandler() watcher.PluginHandler + + // GetDevices returns information about the devices assigned to pods and containers + GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices } // DeviceRunContainerOptions contains the combined container runtime settings to consume its allocated devices. diff --git a/pkg/kubelet/cm/node_container_manager_linux.go b/pkg/kubelet/cm/node_container_manager_linux.go index 52fb1998339..8d7f16f1d19 100644 --- a/pkg/kubelet/cm/node_container_manager_linux.go +++ b/pkg/kubelet/cm/node_container_manager_linux.go @@ -23,11 +23,11 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" kubefeatures "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/events" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" @@ -48,7 +48,7 @@ func (cm *containerManagerImpl) createNodeAllocatableCgroups() error { return nil } if err := cm.cgroupManager.Create(cgroupConfig); err != nil { - glog.Errorf("Failed to create %q cgroup", cm.cgroupRoot) + klog.Errorf("Failed to create %q cgroup", cm.cgroupRoot) return err } return nil @@ -66,7 +66,7 @@ func (cm *containerManagerImpl) enforceNodeAllocatableCgroups() error { nodeAllocatable = cm.getNodeAllocatableAbsolute() } - glog.V(4).Infof("Attempting to enforce Node Allocatable with config: %+v", nc) + klog.V(4).Infof("Attempting to enforce Node Allocatable with config: %+v", nc) cgroupConfig := &CgroupConfig{ Name: cm.cgroupRoot, @@ -103,7 +103,7 @@ func (cm *containerManagerImpl) enforceNodeAllocatableCgroups() error { } // Now apply kube reserved and system reserved limits if required. if nc.EnforceNodeAllocatable.Has(kubetypes.SystemReservedEnforcementKey) { - glog.V(2).Infof("Enforcing System reserved on cgroup %q with limits: %+v", nc.SystemReservedCgroupName, nc.SystemReserved) + klog.V(2).Infof("Enforcing System reserved on cgroup %q with limits: %+v", nc.SystemReservedCgroupName, nc.SystemReserved) if err := enforceExistingCgroup(cm.cgroupManager, ParseCgroupfsToCgroupName(nc.SystemReservedCgroupName), nc.SystemReserved); err != nil { message := fmt.Sprintf("Failed to enforce System Reserved Cgroup Limits on %q: %v", nc.SystemReservedCgroupName, err) cm.recorder.Event(nodeRef, v1.EventTypeWarning, events.FailedNodeAllocatableEnforcement, message) @@ -112,7 +112,7 @@ func (cm *containerManagerImpl) enforceNodeAllocatableCgroups() error { cm.recorder.Eventf(nodeRef, v1.EventTypeNormal, events.SuccessfulNodeAllocatableEnforcement, "Updated limits on system reserved cgroup %v", nc.SystemReservedCgroupName) } if nc.EnforceNodeAllocatable.Has(kubetypes.KubeReservedEnforcementKey) { - glog.V(2).Infof("Enforcing kube reserved on cgroup %q with limits: %+v", nc.KubeReservedCgroupName, nc.KubeReserved) + klog.V(2).Infof("Enforcing kube reserved on cgroup %q with limits: %+v", nc.KubeReservedCgroupName, nc.KubeReserved) if err := enforceExistingCgroup(cm.cgroupManager, ParseCgroupfsToCgroupName(nc.KubeReservedCgroupName), nc.KubeReserved); err != nil { message := fmt.Sprintf("Failed to enforce Kube Reserved Cgroup Limits on %q: %v", nc.KubeReservedCgroupName, err) cm.recorder.Event(nodeRef, v1.EventTypeWarning, events.FailedNodeAllocatableEnforcement, message) @@ -132,7 +132,7 @@ func enforceExistingCgroup(cgroupManager CgroupManager, cName CgroupName, rl v1. if cgroupConfig.ResourceParameters == nil { return fmt.Errorf("%q cgroup is not config properly", cgroupConfig.Name) } - glog.V(4).Infof("Enforcing limits on cgroup %q with %d cpu shares and %d bytes of memory", cName, cgroupConfig.ResourceParameters.CpuShares, cgroupConfig.ResourceParameters.Memory) + klog.V(4).Infof("Enforcing limits on cgroup %q with %d cpu shares and %d bytes of memory", cName, cgroupConfig.ResourceParameters.CpuShares, cgroupConfig.ResourceParameters.Memory) if !cgroupManager.Exists(cgroupConfig.Name) { return fmt.Errorf("%q cgroup does not exist", cgroupConfig.Name) } diff --git a/pkg/kubelet/cm/pod_container_manager_linux.go b/pkg/kubelet/cm/pod_container_manager_linux.go index 844f63986de..b91c5babe92 100644 --- a/pkg/kubelet/cm/pod_container_manager_linux.go +++ b/pkg/kubelet/cm/pod_container_manager_linux.go @@ -23,11 +23,11 @@ import ( "path" "strings" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" kubefeatures "k8s.io/kubernetes/pkg/features" ) @@ -137,7 +137,7 @@ func (m *podContainerManagerImpl) killOnePid(pid int) error { // Hate parsing strings, but // vendor/github.com/opencontainers/runc/libcontainer/ // also does this. - glog.V(3).Infof("process with pid %v no longer exists", pid) + klog.V(3).Infof("process with pid %v no longer exists", pid) return nil } return err @@ -159,18 +159,18 @@ func (m *podContainerManagerImpl) tryKillingCgroupProcesses(podCgroup CgroupName // We try killing all the pids multiple times for i := 0; i < 5; i++ { if i != 0 { - glog.V(3).Infof("Attempt %v failed to kill all unwanted process. Retyring", i) + klog.V(3).Infof("Attempt %v failed to kill all unwanted process. Retyring", i) } errlist = []error{} for _, pid := range pidsToKill { - glog.V(3).Infof("Attempt to kill process with pid: %v", pid) + klog.V(3).Infof("Attempt to kill process with pid: %v", pid) if err := m.killOnePid(pid); err != nil { - glog.V(3).Infof("failed to kill process with pid: %v", pid) + klog.V(3).Infof("failed to kill process with pid: %v", pid) errlist = append(errlist, err) } } if len(errlist) == 0 { - glog.V(3).Infof("successfully killed all unwanted processes.") + klog.V(3).Infof("successfully killed all unwanted processes.") return nil } } @@ -181,7 +181,7 @@ func (m *podContainerManagerImpl) tryKillingCgroupProcesses(podCgroup CgroupName func (m *podContainerManagerImpl) Destroy(podCgroup CgroupName) error { // Try killing all the processes attached to the pod cgroup if err := m.tryKillingCgroupProcesses(podCgroup); err != nil { - glog.V(3).Infof("failed to kill all the processes attached to the %v cgroups", podCgroup) + klog.V(3).Infof("failed to kill all the processes attached to the %v cgroups", podCgroup) return fmt.Errorf("failed to kill all the processes attached to the %v cgroups : %v", podCgroup, err) } @@ -269,7 +269,7 @@ func (m *podContainerManagerImpl) GetAllPodsFromCgroups() (map[types.UID]CgroupN parts := strings.Split(basePath, podCgroupNamePrefix) // the uid is missing, so we log the unexpected cgroup not of form pod if len(parts) != 2 { - glog.Errorf("pod cgroup manager ignoring unexpected cgroup %v because it is not a pod", cgroupfsPath) + klog.Errorf("pod cgroup manager ignoring unexpected cgroup %v because it is not a pod", cgroupfsPath) continue } podUID := parts[1] diff --git a/pkg/kubelet/cm/qos_container_manager_linux.go b/pkg/kubelet/cm/qos_container_manager_linux.go index 2cfc198c3a5..cebc9756e43 100644 --- a/pkg/kubelet/cm/qos_container_manager_linux.go +++ b/pkg/kubelet/cm/qos_container_manager_linux.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/wait" @@ -138,7 +138,7 @@ func (m *qosContainerManagerImpl) Start(getNodeAllocatable func() v1.ResourceLis go wait.Until(func() { err := m.UpdateCgroups() if err != nil { - glog.Warningf("[ContainerManager] Failed to reserve QoS requests: %v", err) + klog.Warningf("[ContainerManager] Failed to reserve QoS requests: %v", err) } }, periodicQOSCgroupUpdateInterval, wait.NeverStop) @@ -222,17 +222,17 @@ func (m *qosContainerManagerImpl) setMemoryReserve(configs map[v1.PodQOSClass]*C resources := m.getNodeAllocatable() allocatableResource, ok := resources[v1.ResourceMemory] if !ok { - glog.V(2).Infof("[Container Manager] Allocatable memory value could not be determined. Not setting QOS memory limts.") + klog.V(2).Infof("[Container Manager] Allocatable memory value could not be determined. Not setting QOS memory limts.") return } allocatable := allocatableResource.Value() if allocatable == 0 { - glog.V(2).Infof("[Container Manager] Memory allocatable reported as 0, might be in standalone mode. Not setting QOS memory limts.") + klog.V(2).Infof("[Container Manager] Memory allocatable reported as 0, might be in standalone mode. Not setting QOS memory limts.") return } for qos, limits := range qosMemoryRequests { - glog.V(2).Infof("[Container Manager] %s pod requests total %d bytes (reserve %d%%)", qos, limits, percentReserve) + klog.V(2).Infof("[Container Manager] %s pod requests total %d bytes (reserve %d%%)", qos, limits, percentReserve) } // Calculate QOS memory limits @@ -252,7 +252,7 @@ func (m *qosContainerManagerImpl) retrySetMemoryReserve(configs map[v1.PodQOSCla for qos, config := range configs { stats, err := m.cgroupManager.GetResourceStats(config.Name) if err != nil { - glog.V(2).Infof("[Container Manager] %v", err) + klog.V(2).Infof("[Container Manager] %v", err) return } usage := stats.MemoryStats.Usage @@ -312,7 +312,7 @@ func (m *qosContainerManagerImpl) UpdateCgroups() error { } } if updateSuccess { - glog.V(4).Infof("[ContainerManager]: Updated QoS cgroup configuration") + klog.V(4).Infof("[ContainerManager]: Updated QoS cgroup configuration") return nil } @@ -330,12 +330,12 @@ func (m *qosContainerManagerImpl) UpdateCgroups() error { for _, config := range qosConfigs { err := m.cgroupManager.Update(config) if err != nil { - glog.Errorf("[ContainerManager]: Failed to update QoS cgroup configuration") + klog.Errorf("[ContainerManager]: Failed to update QoS cgroup configuration") return err } } - glog.V(4).Infof("[ContainerManager]: Updated QoS cgroup configuration") + klog.V(4).Infof("[ContainerManager]: Updated QoS cgroup configuration") return nil } diff --git a/pkg/kubelet/config/BUILD b/pkg/kubelet/config/BUILD index 82cb3a86ecd..8b408d66a2e 100644 --- a/pkg/kubelet/config/BUILD +++ b/pkg/kubelet/config/BUILD @@ -43,8 +43,8 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", diff --git a/pkg/kubelet/config/common.go b/pkg/kubelet/config/common.go index 8fb4199f831..933ee29e786 100644 --- a/pkg/kubelet/config/common.go +++ b/pkg/kubelet/config/common.go @@ -40,7 +40,7 @@ import ( kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/util/hash" - "github.com/golang/glog" + "k8s.io/klog" ) // Generate a pod name that is unique among nodes by appending the nodeName. @@ -59,16 +59,16 @@ func applyDefaults(pod *api.Pod, source string, isFile bool, nodeName types.Node } hash.DeepHashObject(hasher, pod) pod.UID = types.UID(hex.EncodeToString(hasher.Sum(nil)[0:])) - glog.V(5).Infof("Generated UID %q pod %q from %s", pod.UID, pod.Name, source) + klog.V(5).Infof("Generated UID %q pod %q from %s", pod.UID, pod.Name, source) } pod.Name = generatePodName(pod.Name, nodeName) - glog.V(5).Infof("Generated Name %q for UID %q from URL %s", pod.Name, pod.UID, source) + klog.V(5).Infof("Generated Name %q for UID %q from URL %s", pod.Name, pod.UID, source) if pod.Namespace == "" { pod.Namespace = metav1.NamespaceDefault } - glog.V(5).Infof("Using namespace %q for pod %q from %s", pod.Namespace, pod.Name, source) + klog.V(5).Infof("Using namespace %q for pod %q from %s", pod.Namespace, pod.Name, source) // Set the Host field to indicate this pod is scheduled on the current node. pod.Spec.NodeName = string(nodeName) @@ -132,7 +132,7 @@ func tryDecodeSinglePod(data []byte, defaultFn defaultFunc) (parsed bool, pod *v } v1Pod := &v1.Pod{} if err := k8s_api_v1.Convert_core_Pod_To_v1_Pod(newPod, v1Pod, nil); err != nil { - glog.Errorf("Pod %q failed to convert to v1", newPod.Name) + klog.Errorf("Pod %q failed to convert to v1", newPod.Name) return true, nil, err } return true, v1Pod, nil diff --git a/pkg/kubelet/config/config.go b/pkg/kubelet/config/config.go index 51c2e29c546..75d42e0e731 100644 --- a/pkg/kubelet/config/config.go +++ b/pkg/kubelet/config/config.go @@ -21,11 +21,11 @@ import ( "reflect" "sync" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/checkpoint" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -97,7 +97,7 @@ func (c *PodConfig) SeenAllSources(seenSources sets.String) bool { if c.pods == nil { return false } - glog.V(5).Infof("Looking for %v, have seen %v", c.sources.List(), seenSources) + klog.V(5).Infof("Looking for %v, have seen %v", c.sources.List(), seenSources) return seenSources.HasAll(c.sources.List()...) && c.pods.seenSources(c.sources.List()...) } @@ -279,16 +279,16 @@ func (s *podStorage) merge(source string, change interface{}) (adds, updates, de switch update.Op { case kubetypes.ADD, kubetypes.UPDATE, kubetypes.DELETE: if update.Op == kubetypes.ADD { - glog.V(4).Infof("Adding new pods from source %s : %v", source, update.Pods) + klog.V(4).Infof("Adding new pods from source %s : %v", source, update.Pods) } else if update.Op == kubetypes.DELETE { - glog.V(4).Infof("Graceful deleting pods from source %s : %v", source, update.Pods) + klog.V(4).Infof("Graceful deleting pods from source %s : %v", source, update.Pods) } else { - glog.V(4).Infof("Updating pods from source %s : %v", source, update.Pods) + klog.V(4).Infof("Updating pods from source %s : %v", source, update.Pods) } updatePodsFunc(update.Pods, pods, pods) case kubetypes.REMOVE: - glog.V(4).Infof("Removing pods from source %s : %v", source, update.Pods) + klog.V(4).Infof("Removing pods from source %s : %v", source, update.Pods) for _, value := range update.Pods { if existing, found := pods[value.UID]; found { // this is a delete @@ -300,7 +300,7 @@ func (s *podStorage) merge(source string, change interface{}) (adds, updates, de } case kubetypes.SET: - glog.V(4).Infof("Setting pods for source %s", source) + klog.V(4).Infof("Setting pods for source %s", source) s.markSourceSet(source) // Clear the old map entries by just creating a new map oldPods := pods @@ -313,13 +313,13 @@ func (s *podStorage) merge(source string, change interface{}) (adds, updates, de } } case kubetypes.RESTORE: - glog.V(4).Infof("Restoring pods for source %s", source) + klog.V(4).Infof("Restoring pods for source %s", source) for _, value := range update.Pods { restorePods = append(restorePods, value) } default: - glog.Warningf("Received invalid update type: %v", update) + klog.Warningf("Received invalid update type: %v", update) } @@ -354,7 +354,7 @@ func filterInvalidPods(pods []*v1.Pod, source string, recorder record.EventRecor // This function only checks if there is any naming conflict. name := kubecontainer.GetPodFullName(pod) if names.Has(name) { - glog.Warningf("Pod[%d] (%s) from %s failed validation due to duplicate pod name %q, ignoring", i+1, format.Pod(pod), source, pod.Name) + klog.Warningf("Pod[%d] (%s) from %s failed validation due to duplicate pod name %q, ignoring", i+1, format.Pod(pod), source, pod.Name) recorder.Eventf(pod, v1.EventTypeWarning, events.FailedValidation, "Error validating pod %s from %s due to duplicate pod name %q, ignoring", format.Pod(pod), source, pod.Name) continue } else { @@ -411,7 +411,7 @@ func isAnnotationMapEqual(existingMap, candidateMap map[string]string) bool { // recordFirstSeenTime records the first seen time of this pod. func recordFirstSeenTime(pod *v1.Pod) { - glog.V(4).Infof("Receiving a new pod %q", format.Pod(pod)) + klog.V(4).Infof("Receiving a new pod %q", format.Pod(pod)) pod.Annotations[kubetypes.ConfigFirstSeenAnnotationKey] = kubetypes.NewTimestamp().GetString() } diff --git a/pkg/kubelet/config/defaults.go b/pkg/kubelet/config/defaults.go index b90306968f1..16bd6cb5649 100644 --- a/pkg/kubelet/config/defaults.go +++ b/pkg/kubelet/config/defaults.go @@ -23,4 +23,5 @@ const ( DefaultKubeletPluginsDirName = "plugins" DefaultKubeletContainersDirName = "containers" DefaultKubeletPluginContainersDirName = "plugin-containers" + DefaultKubeletPodResourcesDirName = "pod-resources" ) diff --git a/pkg/kubelet/config/file.go b/pkg/kubelet/config/file.go index 683dc9a3c94..5eee61d3cf4 100644 --- a/pkg/kubelet/config/file.go +++ b/pkg/kubelet/config/file.go @@ -26,7 +26,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -65,7 +65,7 @@ func NewSourceFile(path string, nodeName types.NodeName, period time.Duration, u path = strings.TrimRight(path, string(os.PathSeparator)) config := newSourceFile(path, nodeName, period, updates) - glog.V(1).Infof("Watching path %q", path) + klog.V(1).Infof("Watching path %q", path) config.run() } @@ -95,17 +95,17 @@ func (s *sourceFile) run() { go func() { // Read path immediately to speed up startup. if err := s.listConfig(); err != nil { - glog.Errorf("Unable to read config path %q: %v", s.path, err) + klog.Errorf("Unable to read config path %q: %v", s.path, err) } for { select { case <-listTicker.C: if err := s.listConfig(); err != nil { - glog.Errorf("Unable to read config path %q: %v", s.path, err) + klog.Errorf("Unable to read config path %q: %v", s.path, err) } case e := <-s.watchEvents: if err := s.consumeWatchEvent(e); err != nil { - glog.Errorf("Unable to process watch event: %v", err) + klog.Errorf("Unable to process watch event: %v", err) } } } @@ -173,31 +173,31 @@ func (s *sourceFile) extractFromDir(name string) ([]*v1.Pod, error) { for _, path := range dirents { statInfo, err := os.Stat(path) if err != nil { - glog.Errorf("Can't get metadata for %q: %v", path, err) + klog.Errorf("Can't get metadata for %q: %v", path, err) continue } switch { case statInfo.Mode().IsDir(): - glog.Errorf("Not recursing into manifest path %q", path) + klog.Errorf("Not recursing into manifest path %q", path) case statInfo.Mode().IsRegular(): pod, err := s.extractFromFile(path) if err != nil { if !os.IsNotExist(err) { - glog.Errorf("Can't process manifest file %q: %v", path, err) + klog.Errorf("Can't process manifest file %q: %v", path, err) } } else { pods = append(pods, pod) } default: - glog.Errorf("Manifest path %q is not a directory or file: %v", path, statInfo.Mode()) + klog.Errorf("Manifest path %q is not a directory or file: %v", path, statInfo.Mode()) } } return pods, nil } func (s *sourceFile) extractFromFile(filename string) (pod *v1.Pod, err error) { - glog.V(3).Infof("Reading config file %q", filename) + klog.V(3).Infof("Reading config file %q", filename) defer func() { if err == nil && pod != nil { objKey, keyErr := cache.MetaNamespaceKeyFunc(pod) diff --git a/pkg/kubelet/config/file_linux.go b/pkg/kubelet/config/file_linux.go index 4e6b1a5a237..98803ec4492 100644 --- a/pkg/kubelet/config/file_linux.go +++ b/pkg/kubelet/config/file_linux.go @@ -26,8 +26,8 @@ import ( "strings" "time" - "github.com/golang/glog" "golang.org/x/exp/inotify" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -58,7 +58,7 @@ func (s *sourceFile) startWatch() { } if err := s.doWatch(); err != nil { - glog.Errorf("Unable to read config path %q: %v", s.path, err) + klog.Errorf("Unable to read config path %q: %v", s.path, err) if _, retryable := err.(*retryableError); !retryable { backOff.Next(backOffId, time.Now()) } @@ -103,13 +103,13 @@ func (s *sourceFile) doWatch() error { func (s *sourceFile) produceWatchEvent(e *inotify.Event) error { // Ignore file start with dots if strings.HasPrefix(filepath.Base(e.Name), ".") { - glog.V(4).Infof("Ignored pod manifest: %s, because it starts with dots", e.Name) + klog.V(4).Infof("Ignored pod manifest: %s, because it starts with dots", e.Name) return nil } var eventType podEventType switch { case (e.Mask & inotify.IN_ISDIR) > 0: - glog.Errorf("Not recursing into manifest path %q", s.path) + klog.Errorf("Not recursing into manifest path %q", s.path) return nil case (e.Mask & inotify.IN_CREATE) > 0: eventType = podAdd diff --git a/pkg/kubelet/config/file_unsupported.go b/pkg/kubelet/config/file_unsupported.go index 4bee74f544d..d46b5f361de 100644 --- a/pkg/kubelet/config/file_unsupported.go +++ b/pkg/kubelet/config/file_unsupported.go @@ -22,11 +22,11 @@ package config import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" ) func (s *sourceFile) startWatch() { - glog.Errorf("Watching source file is unsupported in this build") + klog.Errorf("Watching source file is unsupported in this build") } func (s *sourceFile) consumeWatchEvent(e *watchEvent) error { diff --git a/pkg/kubelet/config/http.go b/pkg/kubelet/config/http.go index 0ef43e062a5..1eb8ad042a9 100644 --- a/pkg/kubelet/config/http.go +++ b/pkg/kubelet/config/http.go @@ -29,8 +29,8 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" ) type sourceURL struct { @@ -54,7 +54,7 @@ func NewSourceURL(url string, header http.Header, nodeName types.NodeName, perio // read the manifest URL passed to kubelet. client: &http.Client{Timeout: 10 * time.Second}, } - glog.V(1).Infof("Watching URL %s", url) + klog.V(1).Infof("Watching URL %s", url) go wait.Until(config.run, period, wait.NeverStop) } @@ -63,16 +63,16 @@ func (s *sourceURL) run() { // Don't log this multiple times per minute. The first few entries should be // enough to get the point across. if s.failureLogs < 3 { - glog.Warningf("Failed to read pods from URL: %v", err) + klog.Warningf("Failed to read pods from URL: %v", err) } else if s.failureLogs == 3 { - glog.Warningf("Failed to read pods from URL. Dropping verbosity of this message to V(4): %v", err) + klog.Warningf("Failed to read pods from URL. Dropping verbosity of this message to V(4): %v", err) } else { - glog.V(4).Infof("Failed to read pods from URL: %v", err) + klog.V(4).Infof("Failed to read pods from URL: %v", err) } s.failureLogs++ } else { if s.failureLogs > 0 { - glog.Info("Successfully read pods from URL.") + klog.Info("Successfully read pods from URL.") s.failureLogs = 0 } } diff --git a/pkg/kubelet/container/BUILD b/pkg/kubelet/container/BUILD index d7b26708333..3995b79805a 100644 --- a/pkg/kubelet/container/BUILD +++ b/pkg/kubelet/container/BUILD @@ -34,7 +34,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library", "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//third_party/forked/golang/expansion:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/container/container_gc.go b/pkg/kubelet/container/container_gc.go index 72fa4bd722e..790596b2289 100644 --- a/pkg/kubelet/container/container_gc.go +++ b/pkg/kubelet/container/container_gc.go @@ -20,7 +20,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" ) // Specified a policy for garbage collecting containers. @@ -82,6 +82,6 @@ func (cgc *realContainerGC) GarbageCollect() error { } func (cgc *realContainerGC) DeleteAllUnusedContainers() error { - glog.Infof("attempting to delete unused containers") + klog.Infof("attempting to delete unused containers") return cgc.runtime.GarbageCollect(cgc.policy, cgc.sourcesReadyProvider.AllReady(), true) } diff --git a/pkg/kubelet/container/helpers.go b/pkg/kubelet/container/helpers.go index 01fe7129d38..10d848a2d87 100644 --- a/pkg/kubelet/container/helpers.go +++ b/pkg/kubelet/container/helpers.go @@ -21,7 +21,7 @@ import ( "hash/fnv" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -75,13 +75,13 @@ func ShouldContainerBeRestarted(container *v1.Container, pod *v1.Pod, podStatus } // Check RestartPolicy for dead container if pod.Spec.RestartPolicy == v1.RestartPolicyNever { - glog.V(4).Infof("Already ran container %q of pod %q, do nothing", container.Name, format.Pod(pod)) + klog.V(4).Infof("Already ran container %q of pod %q, do nothing", container.Name, format.Pod(pod)) return false } if pod.Spec.RestartPolicy == v1.RestartPolicyOnFailure { // Check the exit code. if status.ExitCode == 0 { - glog.V(4).Infof("Already successfully ran container %q of pod %q, do nothing", container.Name, format.Pod(pod)) + klog.V(4).Infof("Already successfully ran container %q of pod %q, do nothing", container.Name, format.Pod(pod)) return false } } @@ -311,7 +311,7 @@ func MakePortMappings(container *v1.Container) (ports []PortMapping) { // Protect against exposing the same protocol-port more than once in a container. if _, ok := names[pm.Name]; ok { - glog.Warningf("Port name conflicted, %q is defined more than once", pm.Name) + klog.Warningf("Port name conflicted, %q is defined more than once", pm.Name) continue } ports = append(ports, pm) diff --git a/pkg/kubelet/container/runtime.go b/pkg/kubelet/container/runtime.go index 4859342abcb..2d9fcd33fd9 100644 --- a/pkg/kubelet/container/runtime.go +++ b/pkg/kubelet/container/runtime.go @@ -25,11 +25,11 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/remotecommand" "k8s.io/client-go/util/flowcontrol" + "k8s.io/klog" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/volume" ) @@ -203,7 +203,7 @@ func BuildContainerID(typ, ID string) ContainerID { func ParseContainerID(containerID string) ContainerID { var id ContainerID if err := id.ParseString(containerID); err != nil { - glog.Error(err) + klog.Error(err) } return id } diff --git a/pkg/kubelet/dockershim/BUILD b/pkg/kubelet/dockershim/BUILD index d835f257060..5915ed02124 100644 --- a/pkg/kubelet/dockershim/BUILD +++ b/pkg/kubelet/dockershim/BUILD @@ -67,7 +67,7 @@ go_library( "//vendor/github.com/docker/docker/api/types/strslice:go_default_library", "//vendor/github.com/docker/docker/pkg/jsonmessage:go_default_library", "//vendor/github.com/docker/go-connections/nat:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:windows": [ diff --git a/pkg/kubelet/dockershim/cm/BUILD b/pkg/kubelet/dockershim/cm/BUILD index 754952527a9..80736ce2410 100644 --- a/pkg/kubelet/dockershim/cm/BUILD +++ b/pkg/kubelet/dockershim/cm/BUILD @@ -33,9 +33,9 @@ go_library( "//pkg/kubelet/qos:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/configs:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ "//pkg/kubelet/dockershim/libdocker:go_default_library", diff --git a/pkg/kubelet/dockershim/cm/container_manager_linux.go b/pkg/kubelet/dockershim/cm/container_manager_linux.go index 8484fd2d873..b59c6a08364 100644 --- a/pkg/kubelet/dockershim/cm/container_manager_linux.go +++ b/pkg/kubelet/dockershim/cm/container_manager_linux.go @@ -25,11 +25,11 @@ import ( "strconv" "time" - "github.com/golang/glog" "github.com/opencontainers/runc/libcontainer/cgroups/fs" "github.com/opencontainers/runc/libcontainer/configs" utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" kubecm "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/qos" @@ -83,19 +83,19 @@ func (m *containerManager) Start() error { func (m *containerManager) doWork() { v, err := m.client.Version() if err != nil { - glog.Errorf("Unable to get docker version: %v", err) + klog.Errorf("Unable to get docker version: %v", err) return } version, err := utilversion.ParseGeneric(v.APIVersion) if err != nil { - glog.Errorf("Unable to parse docker version %q: %v", v.APIVersion, err) + klog.Errorf("Unable to parse docker version %q: %v", v.APIVersion, err) return } // EnsureDockerInContainer does two things. // 1. Ensure processes run in the cgroups if m.cgroupsManager is not nil. // 2. Ensure processes have the OOM score applied. if err := kubecm.EnsureDockerInContainer(version, dockerOOMScoreAdj, m.cgroupsManager); err != nil { - glog.Errorf("Unable to ensure the docker processes run in the desired containers: %v", err) + klog.Errorf("Unable to ensure the docker processes run in the desired containers: %v", err) } } @@ -104,7 +104,7 @@ func createCgroupManager(name string) (*fs.Manager, error) { memoryCapacity, err := getMemoryCapacity() if err != nil { - glog.Errorf("Failed to get the memory capacity on machine: %v", err) + klog.Errorf("Failed to get the memory capacity on machine: %v", err) } else { memoryLimit = memoryCapacity * dockerMemoryLimitThresholdPercent / 100 } @@ -112,7 +112,7 @@ func createCgroupManager(name string) (*fs.Manager, error) { if err != nil || memoryLimit < minDockerMemoryLimit { memoryLimit = minDockerMemoryLimit } - glog.V(2).Infof("Configure resource-only container %q with memory limit: %d", name, memoryLimit) + klog.V(2).Infof("Configure resource-only container %q with memory limit: %d", name, memoryLimit) allowAllDevices := true cm := &fs.Manager{ diff --git a/pkg/kubelet/dockershim/docker_container.go b/pkg/kubelet/dockershim/docker_container.go index 6343e9be197..3c6b9b48497 100644 --- a/pkg/kubelet/dockershim/docker_container.go +++ b/pkg/kubelet/dockershim/docker_container.go @@ -27,7 +27,7 @@ import ( dockercontainer "github.com/docker/docker/api/types/container" dockerfilters "github.com/docker/docker/api/types/filters" dockerstrslice "github.com/docker/docker/api/types/strslice" - "github.com/golang/glog" + "k8s.io/klog" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" @@ -71,7 +71,7 @@ func (ds *dockerService) ListContainers(_ context.Context, r *runtimeapi.ListCon converted, err := toRuntimeAPIContainer(&c) if err != nil { - glog.V(4).Infof("Unable to convert docker to runtime API container: %v", err) + klog.V(4).Infof("Unable to convert docker to runtime API container: %v", err) continue } @@ -191,7 +191,7 @@ func (ds *dockerService) createContainerLogSymlink(containerID string) error { } if path == "" { - glog.V(5).Infof("Container %s log path isn't specified, will not create the symlink", containerID) + klog.V(5).Infof("Container %s log path isn't specified, will not create the symlink", containerID) return nil } @@ -199,7 +199,7 @@ func (ds *dockerService) createContainerLogSymlink(containerID string) error { // Only create the symlink when container log path is specified and log file exists. // Delete possibly existing file first if err = ds.os.Remove(path); err == nil { - glog.Warningf("Deleted previously existing symlink file: %q", path) + klog.Warningf("Deleted previously existing symlink file: %q", path) } if err = ds.os.Symlink(realPath, path); err != nil { return fmt.Errorf("failed to create symbolic link %q to the container log file %q for container %q: %v", @@ -208,14 +208,14 @@ func (ds *dockerService) createContainerLogSymlink(containerID string) error { } else { supported, err := ds.IsCRISupportedLogDriver() if err != nil { - glog.Warningf("Failed to check supported logging driver by CRI: %v", err) + klog.Warningf("Failed to check supported logging driver by CRI: %v", err) return nil } if supported { - glog.Warningf("Cannot create symbolic link because container log file doesn't exist!") + klog.Warningf("Cannot create symbolic link because container log file doesn't exist!") } else { - glog.V(5).Infof("Unsupported logging driver by CRI") + klog.V(5).Infof("Unsupported logging driver by CRI") } } diff --git a/pkg/kubelet/dockershim/docker_image.go b/pkg/kubelet/dockershim/docker_image.go index e4c450bc8b0..c1089a037a3 100644 --- a/pkg/kubelet/dockershim/docker_image.go +++ b/pkg/kubelet/dockershim/docker_image.go @@ -25,7 +25,7 @@ import ( dockerfilters "github.com/docker/docker/api/types/filters" "github.com/docker/docker/pkg/jsonmessage" - "github.com/golang/glog" + "k8s.io/klog" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" ) @@ -52,7 +52,7 @@ func (ds *dockerService) ListImages(_ context.Context, r *runtimeapi.ListImagesR for _, i := range images { apiImage, err := imageToRuntimeAPIImage(&i) if err != nil { - glog.V(5).Infof("Failed to convert docker API image %+v to runtime API image: %v", i, err) + klog.V(5).Infof("Failed to convert docker API image %+v to runtime API image: %v", i, err) continue } result = append(result, apiImage) diff --git a/pkg/kubelet/dockershim/docker_image_windows.go b/pkg/kubelet/dockershim/docker_image_windows.go index 8fd6d2c869e..e9df2f663fa 100644 --- a/pkg/kubelet/dockershim/docker_image_windows.go +++ b/pkg/kubelet/dockershim/docker_image_windows.go @@ -22,7 +22,7 @@ import ( "context" "time" - "github.com/golang/glog" + "k8s.io/klog" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/winstats" @@ -32,14 +32,14 @@ import ( func (ds *dockerService) ImageFsInfo(_ context.Context, _ *runtimeapi.ImageFsInfoRequest) (*runtimeapi.ImageFsInfoResponse, error) { info, err := ds.client.Info() if err != nil { - glog.Errorf("Failed to get docker info: %v", err) + klog.Errorf("Failed to get docker info: %v", err) return nil, err } statsClient := &winstats.StatsClient{} fsinfo, err := statsClient.GetDirFsInfo(info.DockerRootDir) if err != nil { - glog.Errorf("Failed to get dir fsInfo for %q: %v", info.DockerRootDir, err) + klog.Errorf("Failed to get dir fsInfo for %q: %v", info.DockerRootDir, err) return nil, err } diff --git a/pkg/kubelet/dockershim/docker_sandbox.go b/pkg/kubelet/dockershim/docker_sandbox.go index b114bf052b4..0443793d8db 100644 --- a/pkg/kubelet/dockershim/docker_sandbox.go +++ b/pkg/kubelet/dockershim/docker_sandbox.go @@ -18,6 +18,7 @@ package dockershim import ( "context" + "encoding/json" "fmt" "os" "strings" @@ -26,9 +27,8 @@ import ( dockertypes "github.com/docker/docker/api/types" dockercontainer "github.com/docker/docker/api/types/container" dockerfilters "github.com/docker/docker/api/types/filters" - "github.com/golang/glog" - utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors" @@ -165,7 +165,16 @@ func (ds *dockerService) RunPodSandbox(ctx context.Context, r *runtimeapi.RunPod // on the host as well, to satisfy parts of the pod spec that aren't // recognized by the CNI standard yet. cID := kubecontainer.BuildContainerID(runtimeName, createResp.ID) - err = ds.network.SetUpPod(config.GetMetadata().Namespace, config.GetMetadata().Name, cID, config.Annotations) + networkOptions := make(map[string]string) + if dnsConfig := config.GetDnsConfig(); dnsConfig != nil { + // Build DNS options. + dnsOption, err := json.Marshal(dnsConfig) + if err != nil { + return nil, fmt.Errorf("failed to marshal dns config for pod %q: %v", config.Metadata.Name, err) + } + networkOptions["dns"] = string(dnsOption) + } + err = ds.network.SetUpPod(config.GetMetadata().Namespace, config.GetMetadata().Name, cID, config.Annotations, networkOptions) if err != nil { errList := []error{fmt.Errorf("failed to set up sandbox container %q network for pod %q: %v", createResp.ID, config.Metadata.Name, err)} @@ -216,11 +225,11 @@ func (ds *dockerService) StopPodSandbox(ctx context.Context, r *runtimeapi.StopP if checkpointErr != errors.ErrCheckpointNotFound { err := ds.checkpointManager.RemoveCheckpoint(podSandboxID) if err != nil { - glog.Errorf("Failed to delete corrupt checkpoint for sandbox %q: %v", podSandboxID, err) + klog.Errorf("Failed to delete corrupt checkpoint for sandbox %q: %v", podSandboxID, err) } } if libdocker.IsContainerNotFoundError(statusErr) { - glog.Warningf("Both sandbox container and checkpoint for id %q could not be found. "+ + klog.Warningf("Both sandbox container and checkpoint for id %q could not be found. "+ "Proceed without further sandbox information.", podSandboxID) } else { return nil, utilerrors.NewAggregate([]error{ @@ -255,7 +264,7 @@ func (ds *dockerService) StopPodSandbox(ctx context.Context, r *runtimeapi.StopP if err := ds.client.StopContainer(podSandboxID, defaultSandboxGracePeriod); err != nil { // Do not return error if the container does not exist if !libdocker.IsContainerNotFoundError(err) { - glog.Errorf("Failed to stop sandbox %q: %v", podSandboxID, err) + klog.Errorf("Failed to stop sandbox %q: %v", podSandboxID, err) errList = append(errList, err) } else { // remove the checkpoint for any sandbox that is not found in the runtime @@ -372,7 +381,7 @@ func (ds *dockerService) getIP(podSandboxID string, sandbox *dockertypes.Contain // If all else fails, warn but don't return an error, as pod status // should generally not return anything except fatal errors // FIXME: handle network errors by restarting the pod somehow? - glog.Warningf("failed to read pod IP from plugin/docker: %v", err) + klog.Warningf("failed to read pod IP from plugin/docker: %v", err) return "" } @@ -489,7 +498,7 @@ func (ds *dockerService) ListPodSandbox(_ context.Context, r *runtimeapi.ListPod if filter == nil { checkpoints, err = ds.checkpointManager.ListCheckpoints() if err != nil { - glog.Errorf("Failed to list checkpoints: %v", err) + klog.Errorf("Failed to list checkpoints: %v", err) } } @@ -506,7 +515,7 @@ func (ds *dockerService) ListPodSandbox(_ context.Context, r *runtimeapi.ListPod c := containers[i] converted, err := containerToRuntimeAPISandbox(&c) if err != nil { - glog.V(4).Infof("Unable to convert docker to runtime API sandbox %+v: %v", c, err) + klog.V(4).Infof("Unable to convert docker to runtime API sandbox %+v: %v", c, err) continue } if filterOutReadySandboxes && converted.State == runtimeapi.PodSandboxState_SANDBOX_READY { @@ -526,11 +535,11 @@ func (ds *dockerService) ListPodSandbox(_ context.Context, r *runtimeapi.ListPod checkpoint := NewPodSandboxCheckpoint("", "", &CheckpointData{}) err := ds.checkpointManager.GetCheckpoint(id, checkpoint) if err != nil { - glog.Errorf("Failed to retrieve checkpoint for sandbox %q: %v", id, err) + klog.Errorf("Failed to retrieve checkpoint for sandbox %q: %v", id, err) if err == errors.ErrCorruptCheckpoint { err = ds.checkpointManager.RemoveCheckpoint(id) if err != nil { - glog.Errorf("Failed to delete corrupt checkpoint for sandbox %q: %v", id, err) + klog.Errorf("Failed to delete corrupt checkpoint for sandbox %q: %v", id, err) } } continue @@ -678,14 +687,14 @@ func toCheckpointProtocol(protocol runtimeapi.Protocol) Protocol { case runtimeapi.Protocol_SCTP: return protocolSCTP } - glog.Warningf("Unknown protocol %q: defaulting to TCP", protocol) + klog.Warningf("Unknown protocol %q: defaulting to TCP", protocol) return protocolTCP } // rewriteResolvFile rewrites resolv.conf file generated by docker. func rewriteResolvFile(resolvFilePath string, dns []string, dnsSearch []string, dnsOptions []string) error { if len(resolvFilePath) == 0 { - glog.Errorf("ResolvConfPath is empty.") + klog.Errorf("ResolvConfPath is empty.") return nil } @@ -710,9 +719,9 @@ func rewriteResolvFile(resolvFilePath string, dns []string, dnsSearch []string, resolvFileContentStr := strings.Join(resolvFileContent, "\n") resolvFileContentStr += "\n" - glog.V(4).Infof("Will attempt to re-write config file %s with: \n%s", resolvFilePath, resolvFileContent) + klog.V(4).Infof("Will attempt to re-write config file %s with: \n%s", resolvFilePath, resolvFileContent) if err := rewriteFile(resolvFilePath, resolvFileContentStr); err != nil { - glog.Errorf("resolv.conf could not be updated: %v", err) + klog.Errorf("resolv.conf could not be updated: %v", err) return err } } diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index ae1f70f5218..97f6543c4bb 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -27,7 +27,7 @@ import ( "github.com/blang/semver" dockertypes "github.com/docker/docker/api/types" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" @@ -233,7 +233,7 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon // lead to retries of the same failure, so just fail hard. return nil, err } - glog.Infof("Hairpin mode set to %q", pluginSettings.HairpinMode) + klog.Infof("Hairpin mode set to %q", pluginSettings.HairpinMode) // dockershim currently only supports CNI plugins. pluginSettings.PluginBinDirs = cni.SplitDirs(pluginSettings.PluginBinDirString) @@ -248,25 +248,25 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon return nil, fmt.Errorf("didn't find compatible CNI plugin with given settings %+v: %v", pluginSettings, err) } ds.network = network.NewPluginManager(plug) - glog.Infof("Docker cri networking managed by %v", plug.Name()) + klog.Infof("Docker cri networking managed by %v", plug.Name()) // NOTE: cgroup driver is only detectable in docker 1.11+ cgroupDriver := defaultCgroupDriver dockerInfo, err := ds.client.Info() - glog.Infof("Docker Info: %+v", dockerInfo) + klog.Infof("Docker Info: %+v", dockerInfo) if err != nil { - glog.Errorf("Failed to execute Info() call to the Docker client: %v", err) - glog.Warningf("Falling back to use the default driver: %q", cgroupDriver) + klog.Errorf("Failed to execute Info() call to the Docker client: %v", err) + klog.Warningf("Falling back to use the default driver: %q", cgroupDriver) } else if len(dockerInfo.CgroupDriver) == 0 { - glog.Warningf("No cgroup driver is set in Docker") - glog.Warningf("Falling back to use the default driver: %q", cgroupDriver) + klog.Warningf("No cgroup driver is set in Docker") + klog.Warningf("Falling back to use the default driver: %q", cgroupDriver) } else { cgroupDriver = dockerInfo.CgroupDriver } if len(kubeCgroupDriver) != 0 && kubeCgroupDriver != cgroupDriver { return nil, fmt.Errorf("misconfiguration: kubelet cgroup driver: %q is different from docker cgroup driver: %q", kubeCgroupDriver, cgroupDriver) } - glog.Infof("Setting cgroupDriver to %s", cgroupDriver) + klog.Infof("Setting cgroupDriver to %s", cgroupDriver) ds.cgroupDriver = cgroupDriver ds.versionCache = cache.NewObjectCache( func() (interface{}, error) { @@ -342,7 +342,7 @@ func (ds *dockerService) UpdateRuntimeConfig(_ context.Context, r *runtimeapi.Up return &runtimeapi.UpdateRuntimeConfigResponse{}, nil } - glog.Infof("docker cri received runtime config %+v", runtimeConfig) + klog.Infof("docker cri received runtime config %+v", runtimeConfig) if ds.network != nil && runtimeConfig.NetworkConfig.PodCidr != "" { event := make(map[string]interface{}) event[network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR] = runtimeConfig.NetworkConfig.PodCidr @@ -375,7 +375,7 @@ func (ds *dockerService) GetPodPortMappings(podSandboxID string) ([]*hostport.Po } errRem := ds.checkpointManager.RemoveCheckpoint(podSandboxID) if errRem != nil { - glog.Errorf("Failed to delete corrupt checkpoint for sandbox %q: %v", podSandboxID, errRem) + klog.Errorf("Failed to delete corrupt checkpoint for sandbox %q: %v", podSandboxID, errRem) } return nil, err } @@ -398,7 +398,7 @@ func (ds *dockerService) Start() error { if ds.startLocalStreamingServer { go func() { if err := ds.streamingServer.Start(true); err != nil { - glog.Fatalf("Streaming server stopped unexpectedly: %v", err) + klog.Fatalf("Streaming server stopped unexpectedly: %v", err) } }() } @@ -450,7 +450,7 @@ func (ds *dockerService) GenerateExpectedCgroupParent(cgroupParent string) (stri cgroupParent = path.Base(cgroupParent) } } - glog.V(3).Infof("Setting cgroup parent to: %q", cgroupParent) + klog.V(3).Infof("Setting cgroup parent to: %q", cgroupParent) return cgroupParent, nil } @@ -518,7 +518,7 @@ func toAPIProtocol(protocol Protocol) v1.Protocol { case protocolSCTP: return v1.ProtocolSCTP } - glog.Warningf("Unknown protocol %q: defaulting to TCP", protocol) + klog.Warningf("Unknown protocol %q: defaulting to TCP", protocol) return v1.ProtocolTCP } @@ -537,7 +537,7 @@ func effectiveHairpinMode(s *NetworkPluginSettings) error { // This is not a valid combination, since promiscuous-bridge only works on kubenet. Users might be using the // default values (from before the hairpin-mode flag existed) and we // should keep the old behavior. - glog.Warningf("Hairpin mode set to %q but kubenet is not enabled, falling back to %q", s.HairpinMode, kubeletconfig.HairpinVeth) + klog.Warningf("Hairpin mode set to %q but kubenet is not enabled, falling back to %q", s.HairpinMode, kubeletconfig.HairpinVeth) s.HairpinMode = kubeletconfig.HairpinVeth return nil } diff --git a/pkg/kubelet/dockershim/docker_service_test.go b/pkg/kubelet/dockershim/docker_service_test.go index cc42ef5cabb..a1a0ee3e479 100644 --- a/pkg/kubelet/dockershim/docker_service_test.go +++ b/pkg/kubelet/dockershim/docker_service_test.go @@ -27,7 +27,6 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/util/clock" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" diff --git a/pkg/kubelet/dockershim/docker_streaming.go b/pkg/kubelet/dockershim/docker_streaming.go index d65b970041a..1c4dc813b08 100644 --- a/pkg/kubelet/dockershim/docker_streaming.go +++ b/pkg/kubelet/dockershim/docker_streaming.go @@ -27,7 +27,7 @@ import ( "time" dockertypes "github.com/docker/docker/api/types" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/client-go/tools/remotecommand" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" @@ -199,7 +199,7 @@ func portForward(client libdocker.Interface, podSandboxID string, port int32, st } commandString := fmt.Sprintf("%s %s", nsenterPath, strings.Join(args, " ")) - glog.V(4).Infof("executing port forwarding command: %s", commandString) + klog.V(4).Infof("executing port forwarding command: %s", commandString) command := exec.Command(nsenterPath, args...) command.Stdout = stream diff --git a/pkg/kubelet/dockershim/exec.go b/pkg/kubelet/dockershim/exec.go index aaaff8487d3..4b0d085b5a1 100644 --- a/pkg/kubelet/dockershim/exec.go +++ b/pkg/kubelet/dockershim/exec.go @@ -22,7 +22,7 @@ import ( "time" dockertypes "github.com/docker/docker/api/types" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/client-go/tools/remotecommand" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -124,7 +124,7 @@ func (*NativeExecHandler) ExecInContainer(client libdocker.Interface, container count++ if count == 5 { - glog.Errorf("Exec session %s in container %s terminated but process still running!", execObj.ID, container.ID) + klog.Errorf("Exec session %s in container %s terminated but process still running!", execObj.ID, container.ID) break } diff --git a/pkg/kubelet/dockershim/helpers.go b/pkg/kubelet/dockershim/helpers.go index 6719422d0d9..21166b82d1f 100644 --- a/pkg/kubelet/dockershim/helpers.go +++ b/pkg/kubelet/dockershim/helpers.go @@ -26,7 +26,7 @@ import ( dockercontainer "github.com/docker/docker/api/types/container" dockerfilters "github.com/docker/docker/api/types/filters" dockernat "github.com/docker/go-connections/nat" - "github.com/golang/glog" + "k8s.io/klog" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/kubernetes/pkg/credentialprovider" @@ -142,7 +142,7 @@ func generateMountBindings(mounts []*runtimeapi.Mount) []string { case runtimeapi.MountPropagation_PROPAGATION_HOST_TO_CONTAINER: attrs = append(attrs, "rslave") default: - glog.Warningf("unknown propagation mode for hostPath %q", m.HostPath) + klog.Warningf("unknown propagation mode for hostPath %q", m.HostPath) // Falls back to "private" } @@ -175,7 +175,7 @@ func makePortsAndBindings(pm []*runtimeapi.PortMapping) (dockernat.PortSet, map[ case runtimeapi.Protocol_SCTP: protocol = "/sctp" default: - glog.Warningf("Unknown protocol %q: defaulting to TCP", port.Protocol) + klog.Warningf("Unknown protocol %q: defaulting to TCP", port.Protocol) protocol = "/tcp" } @@ -283,12 +283,12 @@ func recoverFromCreationConflictIfNeeded(client libdocker.Interface, createConfi } id := matches[1] - glog.Warningf("Unable to create pod sandbox due to conflict. Attempting to remove sandbox %q", id) + klog.Warningf("Unable to create pod sandbox due to conflict. Attempting to remove sandbox %q", id) if rmErr := client.RemoveContainer(id, dockertypes.ContainerRemoveOptions{RemoveVolumes: true}); rmErr == nil { - glog.V(2).Infof("Successfully removed conflicting container %q", id) + klog.V(2).Infof("Successfully removed conflicting container %q", id) return nil, err } else { - glog.Errorf("Failed to remove the conflicting container %q: %v", id, rmErr) + klog.Errorf("Failed to remove the conflicting container %q: %v", id, rmErr) // Return if the error is not container not found error. if !libdocker.IsContainerNotFoundError(rmErr) { return nil, err @@ -297,7 +297,7 @@ func recoverFromCreationConflictIfNeeded(client libdocker.Interface, createConfi // randomize the name to avoid conflict. createConfig.Name = randomizeName(createConfig.Name) - glog.V(2).Infof("Create the container with randomized name %s", createConfig.Name) + klog.V(2).Infof("Create the container with randomized name %s", createConfig.Name) return client.CreateContainer(createConfig) } @@ -332,7 +332,7 @@ func ensureSandboxImageExists(client libdocker.Interface, image string) error { keyring := credentialprovider.NewDockerKeyring() creds, withCredentials := keyring.Lookup(repoToPull) if !withCredentials { - glog.V(3).Infof("Pulling image %q without credentials", image) + klog.V(3).Infof("Pulling image %q without credentials", image) err := client.PullImage(image, dockertypes.AuthConfig{}, dockertypes.ImagePullOptions{}) if err != nil { diff --git a/pkg/kubelet/dockershim/helpers_unsupported.go b/pkg/kubelet/dockershim/helpers_unsupported.go index 2867898f301..d78a7eb7b53 100644 --- a/pkg/kubelet/dockershim/helpers_unsupported.go +++ b/pkg/kubelet/dockershim/helpers_unsupported.go @@ -23,7 +23,7 @@ import ( "github.com/blang/semver" dockertypes "github.com/docker/docker/api/types" - "github.com/golang/glog" + "k8s.io/klog" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" ) @@ -32,7 +32,7 @@ func DefaultMemorySwap() int64 { } func (ds *dockerService) getSecurityOpts(seccompProfile string, separator rune) ([]string, error) { - glog.Warningf("getSecurityOpts is unsupported in this build") + klog.Warningf("getSecurityOpts is unsupported in this build") return nil, nil } @@ -41,12 +41,12 @@ func (ds *dockerService) updateCreateConfig( config *runtimeapi.ContainerConfig, sandboxConfig *runtimeapi.PodSandboxConfig, podSandboxID string, securityOptSep rune, apiVersion *semver.Version) error { - glog.Warningf("updateCreateConfig is unsupported in this build") + klog.Warningf("updateCreateConfig is unsupported in this build") return nil } func (ds *dockerService) determinePodIPBySandboxID(uid string) string { - glog.Warningf("determinePodIPBySandboxID is unsupported in this build") + klog.Warningf("determinePodIPBySandboxID is unsupported in this build") return "" } diff --git a/pkg/kubelet/dockershim/helpers_windows.go b/pkg/kubelet/dockershim/helpers_windows.go index 436701546c5..d6c8ebabb96 100644 --- a/pkg/kubelet/dockershim/helpers_windows.go +++ b/pkg/kubelet/dockershim/helpers_windows.go @@ -25,7 +25,7 @@ import ( dockertypes "github.com/docker/docker/api/types" dockercontainer "github.com/docker/docker/api/types/container" dockerfilters "github.com/docker/docker/api/types/filters" - "github.com/golang/glog" + "k8s.io/klog" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" @@ -37,7 +37,7 @@ func DefaultMemorySwap() int64 { func (ds *dockerService) getSecurityOpts(seccompProfile string, separator rune) ([]string, error) { if seccompProfile != "" { - glog.Warningf("seccomp annotations are not supported on windows") + klog.Warningf("seccomp annotations are not supported on windows") } return nil, nil } diff --git a/pkg/kubelet/dockershim/libdocker/BUILD b/pkg/kubelet/dockershim/libdocker/BUILD index 25c1a0e2d38..f706d66e3e8 100644 --- a/pkg/kubelet/dockershim/libdocker/BUILD +++ b/pkg/kubelet/dockershim/libdocker/BUILD @@ -40,8 +40,8 @@ go_library( "//vendor/github.com/docker/docker/client:go_default_library", "//vendor/github.com/docker/docker/pkg/jsonmessage:go_default_library", "//vendor/github.com/docker/docker/pkg/stdcopy:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/opencontainers/go-digest:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/dockershim/libdocker/client.go b/pkg/kubelet/dockershim/libdocker/client.go index 73f0a4dc7d7..d43da1bd22e 100644 --- a/pkg/kubelet/dockershim/libdocker/client.go +++ b/pkg/kubelet/dockershim/libdocker/client.go @@ -23,7 +23,7 @@ import ( dockercontainer "github.com/docker/docker/api/types/container" dockerimagetypes "github.com/docker/docker/api/types/image" dockerapi "github.com/docker/docker/client" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -72,7 +72,7 @@ type Interface interface { // DOCKER_HOST, DOCKER_TLS_VERIFY, and DOCKER_CERT path per their spec func getDockerClient(dockerEndpoint string) (*dockerapi.Client, error) { if len(dockerEndpoint) > 0 { - glog.Infof("Connecting to docker on %s", dockerEndpoint) + klog.Infof("Connecting to docker on %s", dockerEndpoint) return dockerapi.NewClient(dockerEndpoint, "", nil, nil) } return dockerapi.NewEnvClient() @@ -99,8 +99,8 @@ func ConnectToDockerOrDie(dockerEndpoint string, requestTimeout, imagePullProgre } client, err := getDockerClient(dockerEndpoint) if err != nil { - glog.Fatalf("Couldn't connect to docker: %v", err) + klog.Fatalf("Couldn't connect to docker: %v", err) } - glog.Infof("Start docker client with request timeout=%v", requestTimeout) + klog.Infof("Start docker client with request timeout=%v", requestTimeout) return newKubeDockerClient(client, requestTimeout, imagePullProgressDeadline) } diff --git a/pkg/kubelet/dockershim/libdocker/helpers.go b/pkg/kubelet/dockershim/libdocker/helpers.go index f8a785cbd0a..a26ca48a36e 100644 --- a/pkg/kubelet/dockershim/libdocker/helpers.go +++ b/pkg/kubelet/dockershim/libdocker/helpers.go @@ -22,8 +22,8 @@ import ( dockerref "github.com/docker/distribution/reference" dockertypes "github.com/docker/docker/api/types" - "github.com/golang/glog" godigest "github.com/opencontainers/go-digest" + "k8s.io/klog" ) // ParseDockerTimestamp parses the timestamp returned by Interface from string to time.Time @@ -42,7 +42,7 @@ func matchImageTagOrSHA(inspected dockertypes.ImageInspect, image string) bool { // https://github.com/docker/distribution/blob/master/reference/reference.go#L4 named, err := dockerref.ParseNormalizedNamed(image) if err != nil { - glog.V(4).Infof("couldn't parse image reference %q: %v", image, err) + klog.V(4).Infof("couldn't parse image reference %q: %v", image, err) return false } _, isTagged := named.(dockerref.Tagged) @@ -100,7 +100,7 @@ func matchImageTagOrSHA(inspected dockertypes.ImageInspect, image string) bool { for _, repoDigest := range inspected.RepoDigests { named, err := dockerref.ParseNormalizedNamed(repoDigest) if err != nil { - glog.V(4).Infof("couldn't parse image RepoDigest reference %q: %v", repoDigest, err) + klog.V(4).Infof("couldn't parse image RepoDigest reference %q: %v", repoDigest, err) continue } if d, isDigested := named.(dockerref.Digested); isDigested { @@ -114,14 +114,14 @@ func matchImageTagOrSHA(inspected dockertypes.ImageInspect, image string) bool { // process the ID as a digest id, err := godigest.Parse(inspected.ID) if err != nil { - glog.V(4).Infof("couldn't parse image ID reference %q: %v", id, err) + klog.V(4).Infof("couldn't parse image ID reference %q: %v", id, err) return false } if digest.Digest().Algorithm().String() == id.Algorithm().String() && digest.Digest().Hex() == id.Hex() { return true } } - glog.V(4).Infof("Inspected image (%q) does not match %s", inspected.ID, image) + klog.V(4).Infof("Inspected image (%q) does not match %s", inspected.ID, image) return false } @@ -138,19 +138,19 @@ func matchImageIDOnly(inspected dockertypes.ImageInspect, image string) bool { // Otherwise, we should try actual parsing to be more correct ref, err := dockerref.Parse(image) if err != nil { - glog.V(4).Infof("couldn't parse image reference %q: %v", image, err) + klog.V(4).Infof("couldn't parse image reference %q: %v", image, err) return false } digest, isDigested := ref.(dockerref.Digested) if !isDigested { - glog.V(4).Infof("the image reference %q was not a digest reference", image) + klog.V(4).Infof("the image reference %q was not a digest reference", image) return false } id, err := godigest.Parse(inspected.ID) if err != nil { - glog.V(4).Infof("couldn't parse image ID reference %q: %v", id, err) + klog.V(4).Infof("couldn't parse image ID reference %q: %v", id, err) return false } @@ -158,7 +158,7 @@ func matchImageIDOnly(inspected dockertypes.ImageInspect, image string) bool { return true } - glog.V(4).Infof("The reference %s does not directly refer to the given image's ID (%q)", image, inspected.ID) + klog.V(4).Infof("The reference %s does not directly refer to the given image's ID (%q)", image, inspected.ID) return false } diff --git a/pkg/kubelet/dockershim/libdocker/kube_docker_client.go b/pkg/kubelet/dockershim/libdocker/kube_docker_client.go index 78671375410..0cdb9955e3d 100644 --- a/pkg/kubelet/dockershim/libdocker/kube_docker_client.go +++ b/pkg/kubelet/dockershim/libdocker/kube_docker_client.go @@ -28,7 +28,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" dockertypes "github.com/docker/docker/api/types" dockercontainer "github.com/docker/docker/api/types/container" @@ -88,8 +88,8 @@ func newKubeDockerClient(dockerClient *dockerapi.Client, requestTimeout, imagePu // Notice that this assumes that docker is running before kubelet is started. v, err := k.Version() if err != nil { - glog.Errorf("failed to retrieve docker version: %v", err) - glog.Warningf("Using empty version for docker client, this may sometimes cause compatibility issue.") + klog.Errorf("failed to retrieve docker version: %v", err) + klog.Warningf("Using empty version for docker client, this may sometimes cause compatibility issue.") } else { // Update client version with real api version. dockerClient.NegotiateAPIVersionPing(dockertypes.Ping{APIVersion: v.APIVersion}) @@ -338,14 +338,14 @@ func (p *progressReporter) start() { progress, timestamp := p.progress.get() // If there is no progress for p.imagePullProgressDeadline, cancel the operation. if time.Since(timestamp) > p.imagePullProgressDeadline { - glog.Errorf("Cancel pulling image %q because of no progress for %v, latest progress: %q", p.image, p.imagePullProgressDeadline, progress) + klog.Errorf("Cancel pulling image %q because of no progress for %v, latest progress: %q", p.image, p.imagePullProgressDeadline, progress) p.cancel() return } - glog.V(2).Infof("Pulling image %q: %q", p.image, progress) + klog.V(2).Infof("Pulling image %q: %q", p.image, progress) case <-p.stopCh: progress, _ := p.progress.get() - glog.V(2).Infof("Stop pulling image %q: %q", p.image, progress) + klog.V(2).Infof("Stop pulling image %q: %q", p.image, progress) return } } diff --git a/pkg/kubelet/dockershim/network/BUILD b/pkg/kubelet/dockershim/network/BUILD index 13ab4c3d853..330764a3c08 100644 --- a/pkg/kubelet/dockershim/network/BUILD +++ b/pkg/kubelet/dockershim/network/BUILD @@ -18,7 +18,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/kubelet/dockershim/network/cni/BUILD b/pkg/kubelet/dockershim/network/cni/BUILD index 9396ba84e03..8691c7344f2 100644 --- a/pkg/kubelet/dockershim/network/cni/BUILD +++ b/pkg/kubelet/dockershim/network/cni/BUILD @@ -16,12 +16,13 @@ go_library( importpath = "k8s.io/kubernetes/pkg/kubelet/dockershim/network/cni", deps = [ "//pkg/kubelet/apis/config:go_default_library", + "//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/dockershim/network:go_default_library", "//pkg/util/bandwidth:go_default_library", "//vendor/github.com/containernetworking/cni/libcni:go_default_library", "//vendor/github.com/containernetworking/cni/pkg/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:windows": [ diff --git a/pkg/kubelet/dockershim/network/cni/cni.go b/pkg/kubelet/dockershim/network/cni/cni.go index f177a3ed80d..77546a0974c 100644 --- a/pkg/kubelet/dockershim/network/cni/cni.go +++ b/pkg/kubelet/dockershim/network/cni/cni.go @@ -17,6 +17,7 @@ limitations under the License. package cni import ( + "encoding/json" "errors" "fmt" "math" @@ -26,8 +27,9 @@ import ( "github.com/containernetworking/cni/libcni" cnitypes "github.com/containernetworking/cni/pkg/types" - "github.com/golang/glog" + "k8s.io/klog" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/network" "k8s.io/kubernetes/pkg/util/bandwidth" @@ -90,6 +92,18 @@ type cniIpRange struct { Subnet string `json:"subnet"` } +// cniDNSConfig maps to the windows CNI dns Capability. +// see: https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md +// Note that dns capability is only used for Windows containers. +type cniDNSConfig struct { + // List of DNS servers of the cluster. + Servers []string `json:"servers,omitempty"` + // List of DNS search domains of the cluster. + Searches []string `json:"searches,omitempty"` + // List of DNS options. + Options []string `json:"options,omitempty"` +} + func SplitDirs(dirs string) []string { // Use comma rather than colon to work better with Windows too return strings.Split(dirs, ",") @@ -132,34 +146,34 @@ func getDefaultCNINetwork(confDir string, binDirs []string) (*cniNetwork, error) if strings.HasSuffix(confFile, ".conflist") { confList, err = libcni.ConfListFromFile(confFile) if err != nil { - glog.Warningf("Error loading CNI config list file %s: %v", confFile, err) + klog.Warningf("Error loading CNI config list file %s: %v", confFile, err) continue } } else { conf, err := libcni.ConfFromFile(confFile) if err != nil { - glog.Warningf("Error loading CNI config file %s: %v", confFile, err) + klog.Warningf("Error loading CNI config file %s: %v", confFile, err) continue } // Ensure the config has a "type" so we know what plugin to run. // Also catches the case where somebody put a conflist into a conf file. if conf.Network.Type == "" { - glog.Warningf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", confFile) + klog.Warningf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", confFile) continue } confList, err = libcni.ConfListFromConf(conf) if err != nil { - glog.Warningf("Error converting CNI config file %s to list: %v", confFile, err) + klog.Warningf("Error converting CNI config file %s to list: %v", confFile, err) continue } } if len(confList.Plugins) == 0 { - glog.Warningf("CNI config list %s has no networks, skipping", confFile) + klog.Warningf("CNI config list %s has no networks, skipping", confFile) continue } - glog.V(4).Infof("Using CNI configuration file %s", confFile) + klog.V(4).Infof("Using CNI configuration file %s", confFile) network := &cniNetwork{ name: confList.Name, @@ -186,7 +200,7 @@ func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfi func (plugin *cniNetworkPlugin) syncNetworkConfig() { network, err := getDefaultCNINetwork(plugin.confDir, plugin.binDirs) if err != nil { - glog.Warningf("Unable to update cni config: %s", err) + klog.Warningf("Unable to update cni config: %s", err) return } plugin.setDefaultNetwork(network) @@ -233,12 +247,12 @@ func (plugin *cniNetworkPlugin) Event(name string, details map[string]interface{ podCIDR, ok := details[network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR].(string) if !ok { - glog.Warningf("%s event didn't contain pod CIDR", network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE) + klog.Warningf("%s event didn't contain pod CIDR", network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE) return } if plugin.podCidr != "" { - glog.Warningf("Ignoring subsequent pod CIDR update to %s", podCIDR) + klog.Warningf("Ignoring subsequent pod CIDR update to %s", podCIDR) return } @@ -257,7 +271,7 @@ func (plugin *cniNetworkPlugin) Status() error { return plugin.checkInitialized() } -func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { +func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error { if err := plugin.checkInitialized(); err != nil { return err } @@ -268,12 +282,12 @@ func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubec // Windows doesn't have loNetwork. It comes only with Linux if plugin.loNetwork != nil { - if _, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath, annotations); err != nil { + if _, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath, annotations, options); err != nil { return err } } - _, err = plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, annotations) + _, err = plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, annotations, options) return err } @@ -285,7 +299,7 @@ func (plugin *cniNetworkPlugin) TearDownPod(namespace string, name string, id ku // Lack of namespace should not be fatal on teardown netnsPath, err := plugin.host.GetNetNS(id.ID) if err != nil { - glog.Warningf("CNI failed to retrieve network namespace path: %v", err) + klog.Warningf("CNI failed to retrieve network namespace path: %v", err) } return plugin.deleteFromNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, nil) @@ -295,47 +309,47 @@ func podDesc(namespace, name string, id kubecontainer.ContainerID) string { return fmt.Sprintf("%s_%s/%s", namespace, name, id.ID) } -func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations map[string]string) (cnitypes.Result, error) { - rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations) +func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations, options map[string]string) (cnitypes.Result, error) { + rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations, options) if err != nil { - glog.Errorf("Error adding network when building cni runtime conf: %v", err) + klog.Errorf("Error adding network when building cni runtime conf: %v", err) return nil, err } pdesc := podDesc(podNamespace, podName, podSandboxID) netConf, cniNet := network.NetworkConfig, network.CNIConfig - glog.V(4).Infof("Adding %s to network %s/%s netns %q", pdesc, netConf.Plugins[0].Network.Type, netConf.Name, podNetnsPath) + klog.V(4).Infof("Adding %s to network %s/%s netns %q", pdesc, netConf.Plugins[0].Network.Type, netConf.Name, podNetnsPath) res, err := cniNet.AddNetworkList(netConf, rt) if err != nil { - glog.Errorf("Error adding %s to network %s/%s: %v", pdesc, netConf.Plugins[0].Network.Type, netConf.Name, err) + klog.Errorf("Error adding %s to network %s/%s: %v", pdesc, netConf.Plugins[0].Network.Type, netConf.Name, err) return nil, err } - glog.V(4).Infof("Added %s to network %s: %v", pdesc, netConf.Name, res) + klog.V(4).Infof("Added %s to network %s: %v", pdesc, netConf.Name, res) return res, nil } func (plugin *cniNetworkPlugin) deleteFromNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations map[string]string) error { - rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations) + rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations, nil) if err != nil { - glog.Errorf("Error deleting network when building cni runtime conf: %v", err) + klog.Errorf("Error deleting network when building cni runtime conf: %v", err) return err } pdesc := podDesc(podNamespace, podName, podSandboxID) netConf, cniNet := network.NetworkConfig, network.CNIConfig - glog.V(4).Infof("Deleting %s from network %s/%s netns %q", pdesc, netConf.Plugins[0].Network.Type, netConf.Name, podNetnsPath) + klog.V(4).Infof("Deleting %s from network %s/%s netns %q", pdesc, netConf.Plugins[0].Network.Type, netConf.Name, podNetnsPath) err = cniNet.DelNetworkList(netConf, rt) // The pod may not get deleted successfully at the first time. // Ignore "no such file or directory" error in case the network has already been deleted in previous attempts. if err != nil && !strings.Contains(err.Error(), "no such file or directory") { - glog.Errorf("Error deleting %s from network %s/%s: %v", pdesc, netConf.Plugins[0].Network.Type, netConf.Name, err) + klog.Errorf("Error deleting %s from network %s/%s: %v", pdesc, netConf.Plugins[0].Network.Type, netConf.Name, err) return err } - glog.V(4).Infof("Deleted %s from network %s/%s", pdesc, netConf.Plugins[0].Network.Type, netConf.Name) + klog.V(4).Infof("Deleted %s from network %s/%s", pdesc, netConf.Plugins[0].Network.Type, netConf.Name) return nil } -func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations map[string]string) (*libcni.RuntimeConf, error) { +func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations, options map[string]string) (*libcni.RuntimeConf, error) { rt := &libcni.RuntimeConf{ ContainerID: podSandboxID.ID, NetNS: podNetnsPath, @@ -390,5 +404,17 @@ func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string // Set the PodCIDR rt.CapabilityArgs["ipRanges"] = [][]cniIpRange{{{Subnet: plugin.podCidr}}} + // Set dns capability args. + if dnsOptions, ok := options["dns"]; ok { + dnsConfig := runtimeapi.DNSConfig{} + err := json.Unmarshal([]byte(dnsOptions), &dnsConfig) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal dns config %q: %v", dnsOptions, err) + } + if dnsParam := buildDNSCapabilities(&dnsConfig); dnsParam != nil { + rt.CapabilityArgs["dns"] = *dnsParam + } + } + return rt, nil } diff --git a/pkg/kubelet/dockershim/network/cni/cni_others.go b/pkg/kubelet/dockershim/network/cni/cni_others.go index 56f75ca3a96..1712f8abe3c 100644 --- a/pkg/kubelet/dockershim/network/cni/cni_others.go +++ b/pkg/kubelet/dockershim/network/cni/cni_others.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/containernetworking/cni/libcni" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/network" ) @@ -75,3 +76,8 @@ func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name strin return &network.PodNetworkStatus{IP: ip}, nil } + +// buildDNSCapabilities builds cniDNSConfig from runtimeapi.DNSConfig. +func buildDNSCapabilities(dnsConfig *runtimeapi.DNSConfig) *cniDNSConfig { + return nil +} diff --git a/pkg/kubelet/dockershim/network/cni/cni_test.go b/pkg/kubelet/dockershim/network/cni/cni_test.go index 152fa0b161f..0fbe58ad715 100644 --- a/pkg/kubelet/dockershim/network/cni/cni_test.go +++ b/pkg/kubelet/dockershim/network/cni/cni_test.go @@ -254,7 +254,7 @@ func TestCNIPlugin(t *testing.T) { bandwidthAnnotation["kubernetes.io/egress-bandwidth"] = "1M" // Set up the pod - err = plug.SetUpPod("podNamespace", "podName", containerID, bandwidthAnnotation) + err = plug.SetUpPod("podNamespace", "podName", containerID, bandwidthAnnotation, nil) if err != nil { t.Errorf("Expected nil: %v", err) } diff --git a/pkg/kubelet/dockershim/network/cni/cni_windows.go b/pkg/kubelet/dockershim/network/cni/cni_windows.go index 0c904445501..76b78e143cb 100644 --- a/pkg/kubelet/dockershim/network/cni/cni_windows.go +++ b/pkg/kubelet/dockershim/network/cni/cni_windows.go @@ -22,7 +22,8 @@ import ( "fmt" cniTypes020 "github.com/containernetworking/cni/pkg/types/020" - "github.com/golang/glog" + "k8s.io/klog" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/network" ) @@ -42,11 +43,11 @@ func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name strin return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err) } - result, err := plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, nil) + result, err := plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, nil, nil) - glog.V(5).Infof("GetPodNetworkStatus result %+v", result) + klog.V(5).Infof("GetPodNetworkStatus result %+v", result) if err != nil { - glog.Errorf("error while adding to cni network: %s", err) + klog.Errorf("error while adding to cni network: %s", err) return nil, err } @@ -54,8 +55,21 @@ func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name strin var result020 *cniTypes020.Result result020, err = cniTypes020.GetResult(result) if err != nil { - glog.Errorf("error while cni parsing result: %s", err) + klog.Errorf("error while cni parsing result: %s", err) return nil, err } return &network.PodNetworkStatus{IP: result020.IP4.IP.IP}, nil } + +// buildDNSCapabilities builds cniDNSConfig from runtimeapi.DNSConfig. +func buildDNSCapabilities(dnsConfig *runtimeapi.DNSConfig) *cniDNSConfig { + if dnsConfig != nil { + return &cniDNSConfig{ + Servers: dnsConfig.Servers, + Searches: dnsConfig.Searches, + Options: dnsConfig.Options, + } + } + + return nil +} diff --git a/pkg/kubelet/dockershim/network/hairpin/BUILD b/pkg/kubelet/dockershim/network/hairpin/BUILD index 94401b46922..570c027ee74 100644 --- a/pkg/kubelet/dockershim/network/hairpin/BUILD +++ b/pkg/kubelet/dockershim/network/hairpin/BUILD @@ -11,7 +11,7 @@ go_library( srcs = ["hairpin.go"], importpath = "k8s.io/kubernetes/pkg/kubelet/dockershim/network/hairpin", deps = [ - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/kubelet/dockershim/network/hairpin/hairpin.go b/pkg/kubelet/dockershim/network/hairpin/hairpin.go index d933131ed31..262e78460df 100644 --- a/pkg/kubelet/dockershim/network/hairpin/hairpin.go +++ b/pkg/kubelet/dockershim/network/hairpin/hairpin.go @@ -25,7 +25,7 @@ import ( "regexp" "strconv" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/utils/exec" ) @@ -72,7 +72,7 @@ func findPairInterfaceOfContainerInterface(e exec.Interface, containerInterfaceN } func setUpInterface(ifName string) error { - glog.V(3).Infof("Enabling hairpin on interface %s", ifName) + klog.V(3).Infof("Enabling hairpin on interface %s", ifName) ifPath := path.Join(sysfsNetPath, ifName) if _, err := os.Stat(ifPath); err != nil { return err diff --git a/pkg/kubelet/dockershim/network/hostport/BUILD b/pkg/kubelet/dockershim/network/hostport/BUILD index d1f438e0ff8..cceadb121ae 100644 --- a/pkg/kubelet/dockershim/network/hostport/BUILD +++ b/pkg/kubelet/dockershim/network/hostport/BUILD @@ -23,7 +23,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/kubelet/dockershim/network/hostport/hostport.go b/pkg/kubelet/dockershim/network/hostport/hostport.go index e04107fa389..4f9f7751b3f 100644 --- a/pkg/kubelet/dockershim/network/hostport/hostport.go +++ b/pkg/kubelet/dockershim/network/hostport/hostport.go @@ -21,7 +21,7 @@ import ( "net" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" utiliptables "k8s.io/kubernetes/pkg/util/iptables" @@ -97,7 +97,7 @@ func openLocalPort(hp *hostport) (closeable, error) { default: return nil, fmt.Errorf("unknown protocol %q", hp.protocol) } - glog.V(3).Infof("Opened local port %s", hp.String()) + klog.V(3).Infof("Opened local port %s", hp.String()) return socket, nil } @@ -111,7 +111,7 @@ func portMappingToHostport(portMapping *PortMapping) hostport { // ensureKubeHostportChains ensures the KUBE-HOSTPORTS chain is setup correctly func ensureKubeHostportChains(iptables utiliptables.Interface, natInterfaceName string) error { - glog.V(4).Info("Ensuring kubelet hostport chains") + klog.V(4).Info("Ensuring kubelet hostport chains") // Ensure kubeHostportChain if _, err := iptables.EnsureChain(utiliptables.TableNAT, kubeHostportsChain); err != nil { return fmt.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, kubeHostportsChain, err) diff --git a/pkg/kubelet/dockershim/network/hostport/hostport_manager.go b/pkg/kubelet/dockershim/network/hostport/hostport_manager.go index 70bfd16dab7..ce5108fd318 100644 --- a/pkg/kubelet/dockershim/network/hostport/hostport_manager.go +++ b/pkg/kubelet/dockershim/network/hostport/hostport_manager.go @@ -25,9 +25,9 @@ import ( "strings" "sync" - "github.com/golang/glog" "k8s.io/api/core/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog" iptablesproxy "k8s.io/kubernetes/pkg/proxy/iptables" "k8s.io/kubernetes/pkg/util/conntrack" utiliptables "k8s.io/kubernetes/pkg/util/iptables" @@ -65,7 +65,7 @@ func NewHostportManager(iptables utiliptables.Interface) HostPortManager { } h.conntrackFound = conntrack.Exists(h.execer) if !h.conntrackFound { - glog.Warningf("The binary conntrack is not installed, this can cause failures in network connection cleanup.") + klog.Warningf("The binary conntrack is not installed, this can cause failures in network connection cleanup.") } return h } @@ -173,11 +173,11 @@ func (hm *hostportManager) Add(id string, podPortMapping *PodPortMapping, natInt // create a new conntrack entry without any DNAT. That will result in blackhole of the traffic even after correct // iptables rules have been added back. if hm.execer != nil && hm.conntrackFound { - glog.Infof("Starting to delete udp conntrack entries: %v, isIPv6 - %v", conntrackPortsToRemove, isIpv6) + klog.Infof("Starting to delete udp conntrack entries: %v, isIPv6 - %v", conntrackPortsToRemove, isIpv6) for _, port := range conntrackPortsToRemove { err = conntrack.ClearEntriesForPort(hm.execer, port, isIpv6, v1.ProtocolUDP) if err != nil { - glog.Errorf("Failed to clear udp conntrack for port %d, error: %v", port, err) + klog.Errorf("Failed to clear udp conntrack for port %d, error: %v", port, err) } } } @@ -246,7 +246,7 @@ func (hm *hostportManager) Remove(id string, podPortMapping *PodPortMapping) (er // syncIPTables executes iptables-restore with given lines func (hm *hostportManager) syncIPTables(lines []byte) error { - glog.V(3).Infof("Restoring iptables rules: %s", lines) + klog.V(3).Infof("Restoring iptables rules: %s", lines) err := hm.iptables.RestoreAll(lines, utiliptables.NoFlushTables, utiliptables.RestoreCounters) if err != nil { return fmt.Errorf("Failed to execute iptables-restore: %v", err) @@ -283,7 +283,7 @@ func (hm *hostportManager) openHostports(podPortMapping *PodPortMapping) (map[ho if retErr != nil { for hp, socket := range ports { if err := socket.Close(); err != nil { - glog.Errorf("Cannot clean up hostport %d for pod %s: %v", hp.port, getPodFullName(podPortMapping), err) + klog.Errorf("Cannot clean up hostport %d for pod %s: %v", hp.port, getPodFullName(podPortMapping), err) } } return nil, retErr @@ -297,7 +297,7 @@ func (hm *hostportManager) closeHostports(hostportMappings []*PortMapping) error for _, pm := range hostportMappings { hp := portMappingToHostport(pm) if socket, ok := hm.hostPortMap[hp]; ok { - glog.V(2).Infof("Closing host port %s", hp.String()) + klog.V(2).Infof("Closing host port %s", hp.String()) if err := socket.Close(); err != nil { errList = append(errList, fmt.Errorf("failed to close host port %s: %v", hp.String(), err)) continue diff --git a/pkg/kubelet/dockershim/network/hostport/hostport_syncer.go b/pkg/kubelet/dockershim/network/hostport/hostport_syncer.go index 1f9df7e9b98..5274f890dff 100644 --- a/pkg/kubelet/dockershim/network/hostport/hostport_syncer.go +++ b/pkg/kubelet/dockershim/network/hostport/hostport_syncer.go @@ -25,7 +25,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" iptablesproxy "k8s.io/kubernetes/pkg/proxy/iptables" @@ -97,7 +97,7 @@ func (h *hostportSyncer) openHostports(podHostportMapping *PodPortMapping) error if retErr != nil { for hp, socket := range ports { if err := socket.Close(); err != nil { - glog.Errorf("Cannot clean up hostport %d for pod %s: %v", hp.port, getPodFullName(podHostportMapping), err) + klog.Errorf("Cannot clean up hostport %d for pod %s: %v", hp.port, getPodFullName(podHostportMapping), err) } } return retErr @@ -188,7 +188,7 @@ func (h *hostportSyncer) OpenPodHostportsAndSync(newPortMapping *PodPortMapping, func (h *hostportSyncer) SyncHostports(natInterfaceName string, activePodPortMappings []*PodPortMapping) error { start := time.Now() defer func() { - glog.V(4).Infof("syncHostportsRules took %v", time.Since(start)) + klog.V(4).Infof("syncHostportsRules took %v", time.Since(start)) }() hostportPodMap, err := gatherAllHostports(activePodPortMappings) @@ -205,7 +205,7 @@ func (h *hostportSyncer) SyncHostports(natInterfaceName string, activePodPortMap iptablesData := bytes.NewBuffer(nil) err = h.iptables.SaveInto(utiliptables.TableNAT, iptablesData) if err != nil { // if we failed to get any rules - glog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err) + klog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err) } else { // otherwise parse the output existingNATChains = utiliptables.GetChainLines(utiliptables.TableNAT, iptablesData.Bytes()) } @@ -283,7 +283,7 @@ func (h *hostportSyncer) SyncHostports(natInterfaceName string, activePodPortMap writeLine(natRules, "COMMIT") natLines := append(natChains.Bytes(), natRules.Bytes()...) - glog.V(3).Infof("Restoring iptables rules: %s", natLines) + klog.V(3).Infof("Restoring iptables rules: %s", natLines) err = h.iptables.RestoreAll(natLines, utiliptables.NoFlushTables, utiliptables.RestoreCounters) if err != nil { return fmt.Errorf("Failed to execute iptables-restore: %v", err) @@ -309,7 +309,7 @@ func (h *hostportSyncer) cleanupHostportMap(containerPortMap map[*PortMapping]ta for hp, socket := range h.hostPortMap { if _, ok := currentHostports[hp]; !ok { socket.Close() - glog.V(3).Infof("Closed local port %s", hp.String()) + klog.V(3).Infof("Closed local port %s", hp.String()) delete(h.hostPortMap, hp) } } diff --git a/pkg/kubelet/dockershim/network/kubenet/BUILD b/pkg/kubelet/dockershim/network/kubenet/BUILD index 92613205c5b..0b4c606aebe 100644 --- a/pkg/kubelet/dockershim/network/kubenet/BUILD +++ b/pkg/kubelet/dockershim/network/kubenet/BUILD @@ -51,9 +51,9 @@ go_library( "//vendor/github.com/containernetworking/cni/libcni:go_default_library", "//vendor/github.com/containernetworking/cni/pkg/types:go_default_library", "//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/vishvananda/netlink:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go index 0aec8c38cf7..c3909b1c7f4 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go @@ -29,12 +29,12 @@ import ( "github.com/containernetworking/cni/libcni" cnitypes "github.com/containernetworking/cni/pkg/types" cnitypes020 "github.com/containernetworking/cni/pkg/types/020" - "github.com/golang/glog" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilnet "k8s.io/apimachinery/pkg/util/net" utilsets "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/network" @@ -124,10 +124,10 @@ func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode kubeletc if mtu == network.UseDefaultMTU { if link, err := findMinMTU(); err == nil { plugin.mtu = link.MTU - glog.V(5).Infof("Using interface %s MTU %d as bridge MTU", link.Name, link.MTU) + klog.V(5).Infof("Using interface %s MTU %d as bridge MTU", link.Name, link.MTU) } else { plugin.mtu = fallbackMTU - glog.Warningf("Failed to find default bridge MTU, using %d: %v", fallbackMTU, err) + klog.Warningf("Failed to find default bridge MTU, using %d: %v", fallbackMTU, err) } } else { plugin.mtu = mtu @@ -142,7 +142,7 @@ func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode kubeletc plugin.execer.Command("modprobe", "br-netfilter").CombinedOutput() err := plugin.sysctl.SetSysctl(sysctlBridgeCallIPTables, 1) if err != nil { - glog.Warningf("can't set sysctl %s: %v", sysctlBridgeCallIPTables, err) + klog.Warningf("can't set sysctl %s: %v", sysctlBridgeCallIPTables, err) } plugin.loConfig, err = libcni.ConfFromBytes([]byte(`{ @@ -234,16 +234,16 @@ func (plugin *kubenetNetworkPlugin) Event(name string, details map[string]interf podCIDR, ok := details[network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR].(string) if !ok { - glog.Warningf("%s event didn't contain pod CIDR", network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE) + klog.Warningf("%s event didn't contain pod CIDR", network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE) return } if plugin.netConfig != nil { - glog.Warningf("Ignoring subsequent pod CIDR update to %s", podCIDR) + klog.Warningf("Ignoring subsequent pod CIDR update to %s", podCIDR) return } - glog.V(5).Infof("PodCIDR is set to %q", podCIDR) + klog.V(5).Infof("PodCIDR is set to %q", podCIDR) _, cidr, err := net.ParseCIDR(podCIDR) if err == nil { setHairpin := plugin.hairpinMode == kubeletconfig.HairpinVeth @@ -251,10 +251,10 @@ func (plugin *kubenetNetworkPlugin) Event(name string, details map[string]interf cidr.IP[len(cidr.IP)-1] += 1 json := fmt.Sprintf(NET_CONFIG_TEMPLATE, BridgeName, plugin.mtu, network.DefaultInterfaceName, setHairpin, podCIDR, cidr.IP.String()) - glog.V(2).Infof("CNI network config set to %v", json) + klog.V(2).Infof("CNI network config set to %v", json) plugin.netConfig, err = libcni.ConfFromBytes([]byte(json)) if err == nil { - glog.V(5).Infof("CNI network config:\n%s", json) + klog.V(5).Infof("CNI network config:\n%s", json) // Ensure cbr0 has no conflicting addresses; CNI's 'bridge' // plugin will bail out if the bridge has an unexpected one @@ -265,7 +265,7 @@ func (plugin *kubenetNetworkPlugin) Event(name string, details map[string]interf } if err != nil { - glog.Warningf("Failed to generate CNI network config: %v", err) + klog.Warningf("Failed to generate CNI network config: %v", err) } } @@ -282,7 +282,7 @@ func (plugin *kubenetNetworkPlugin) clearBridgeAddressesExcept(keep *net.IPNet) for _, addr := range addrs { if !utilnet.IPNetEqual(addr.IPNet, keep) { - glog.V(2).Infof("Removing old address %s from %s", addr.IPNet.String(), BridgeName) + klog.V(2).Infof("Removing old address %s from %s", addr.IPNet.String(), BridgeName) netlink.AddrDel(bridge, &addr) } } @@ -300,7 +300,7 @@ func (plugin *kubenetNetworkPlugin) Capabilities() utilsets.Int { func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { // Disable DAD so we skip the kernel delay on bringing up new interfaces. if err := plugin.disableContainerDAD(id); err != nil { - glog.V(3).Infof("Failed to disable DAD in container: %v", err) + klog.V(3).Infof("Failed to disable DAD in container: %v", err) } // Bring up container loopback interface @@ -379,13 +379,13 @@ func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kube return nil } -func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { +func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error { plugin.mu.Lock() defer plugin.mu.Unlock() start := time.Now() defer func() { - glog.V(4).Infof("SetUpPod took %v for %s/%s", time.Since(start), namespace, name) + klog.V(4).Infof("SetUpPod took %v for %s/%s", time.Since(start), namespace, name) }() if err := plugin.Status(); err != nil { @@ -397,14 +397,14 @@ func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id k podIP, _ := plugin.podIPs[id] if err := plugin.teardown(namespace, name, id, podIP); err != nil { // Not a hard error or warning - glog.V(4).Infof("Failed to clean up %s/%s after SetUpPod failure: %v", namespace, name, err) + klog.V(4).Infof("Failed to clean up %s/%s after SetUpPod failure: %v", namespace, name, err) } return err } // Need to SNAT outbound traffic from cluster if err := plugin.ensureMasqRule(); err != nil { - glog.Errorf("Failed to ensure MASQ rule: %v", err) + klog.Errorf("Failed to ensure MASQ rule: %v", err) } return nil @@ -416,11 +416,11 @@ func (plugin *kubenetNetworkPlugin) teardown(namespace string, name string, id k errList := []error{} if podIP != "" { - glog.V(5).Infof("Removing pod IP %s from shaper", podIP) + klog.V(5).Infof("Removing pod IP %s from shaper", podIP) // shaper wants /32 if err := plugin.shaper().Reset(fmt.Sprintf("%s/32", podIP)); err != nil { // Possible bandwidth shaping wasn't enabled for this pod anyways - glog.V(4).Infof("Failed to remove pod IP %s from shaper: %v", podIP, err) + klog.V(4).Infof("Failed to remove pod IP %s from shaper: %v", podIP, err) } delete(plugin.podIPs, id) @@ -429,7 +429,7 @@ func (plugin *kubenetNetworkPlugin) teardown(namespace string, name string, id k if err := plugin.delContainerFromNetwork(plugin.netConfig, network.DefaultInterfaceName, namespace, name, id); err != nil { // This is to prevent returning error when TearDownPod is called twice on the same pod. This helps to reduce event pollution. if podIP != "" { - glog.Warningf("Failed to delete container from kubenet: %v", err) + klog.Warningf("Failed to delete container from kubenet: %v", err) } else { errList = append(errList, err) } @@ -457,7 +457,7 @@ func (plugin *kubenetNetworkPlugin) TearDownPod(namespace string, name string, i start := time.Now() defer func() { - glog.V(4).Infof("TearDownPod took %v for %s/%s", time.Since(start), namespace, name) + klog.V(4).Infof("TearDownPod took %v for %s/%s", time.Since(start), namespace, name) }() if plugin.netConfig == nil { @@ -472,7 +472,7 @@ func (plugin *kubenetNetworkPlugin) TearDownPod(namespace string, name string, i // Need to SNAT outbound traffic from cluster if err := plugin.ensureMasqRule(); err != nil { - glog.Errorf("Failed to ensure MASQ rule: %v", err) + klog.Errorf("Failed to ensure MASQ rule: %v", err) } return nil @@ -550,7 +550,7 @@ func (plugin *kubenetNetworkPlugin) checkRequiredCNIPluginsInOneDir(dir string) func (plugin *kubenetNetworkPlugin) buildCNIRuntimeConf(ifName string, id kubecontainer.ContainerID, needNetNs bool) (*libcni.RuntimeConf, error) { netnsPath, err := plugin.host.GetNetNS(id.ID) if needNetNs && err != nil { - glog.Errorf("Kubenet failed to retrieve network namespace path: %v", err) + klog.Errorf("Kubenet failed to retrieve network namespace path: %v", err) } return &libcni.RuntimeConf{ @@ -566,7 +566,7 @@ func (plugin *kubenetNetworkPlugin) addContainerToNetwork(config *libcni.Network return nil, fmt.Errorf("Error building CNI config: %v", err) } - glog.V(3).Infof("Adding %s/%s to '%s' with CNI '%s' plugin and runtime: %+v", namespace, name, config.Network.Name, config.Network.Type, rt) + klog.V(3).Infof("Adding %s/%s to '%s' with CNI '%s' plugin and runtime: %+v", namespace, name, config.Network.Name, config.Network.Type, rt) // The network plugin can take up to 3 seconds to execute, // so yield the lock while it runs. plugin.mu.Unlock() @@ -584,7 +584,7 @@ func (plugin *kubenetNetworkPlugin) delContainerFromNetwork(config *libcni.Netwo return fmt.Errorf("Error building CNI config: %v", err) } - glog.V(3).Infof("Removing %s/%s from '%s' with CNI '%s' plugin and runtime: %+v", namespace, name, config.Network.Name, config.Network.Type, rt) + klog.V(3).Infof("Removing %s/%s from '%s' with CNI '%s' plugin and runtime: %+v", namespace, name, config.Network.Name, config.Network.Type, rt) err = plugin.cniConfig.DelNetwork(config, rt) // The pod may not get deleted successfully at the first time. // Ignore "no such file or directory" error in case the network has already been deleted in previous attempts. @@ -609,40 +609,40 @@ func (plugin *kubenetNetworkPlugin) shaper() bandwidth.BandwidthShaper { func (plugin *kubenetNetworkPlugin) syncEbtablesDedupRules(macAddr net.HardwareAddr) { if plugin.ebtables == nil { plugin.ebtables = utilebtables.New(plugin.execer) - glog.V(3).Infof("Flushing dedup chain") + klog.V(3).Infof("Flushing dedup chain") if err := plugin.ebtables.FlushChain(utilebtables.TableFilter, dedupChain); err != nil { - glog.Errorf("Failed to flush dedup chain: %v", err) + klog.Errorf("Failed to flush dedup chain: %v", err) } } _, err := plugin.ebtables.GetVersion() if err != nil { - glog.Warningf("Failed to get ebtables version. Skip syncing ebtables dedup rules: %v", err) + klog.Warningf("Failed to get ebtables version. Skip syncing ebtables dedup rules: %v", err) return } - glog.V(3).Infof("Filtering packets with ebtables on mac address: %v, gateway: %v, pod CIDR: %v", macAddr.String(), plugin.gateway.String(), plugin.podCidr) + klog.V(3).Infof("Filtering packets with ebtables on mac address: %v, gateway: %v, pod CIDR: %v", macAddr.String(), plugin.gateway.String(), plugin.podCidr) _, err = plugin.ebtables.EnsureChain(utilebtables.TableFilter, dedupChain) if err != nil { - glog.Errorf("Failed to ensure %v chain %v", utilebtables.TableFilter, dedupChain) + klog.Errorf("Failed to ensure %v chain %v", utilebtables.TableFilter, dedupChain) return } _, err = plugin.ebtables.EnsureRule(utilebtables.Append, utilebtables.TableFilter, utilebtables.ChainOutput, "-j", string(dedupChain)) if err != nil { - glog.Errorf("Failed to ensure %v chain %v jump to %v chain: %v", utilebtables.TableFilter, utilebtables.ChainOutput, dedupChain, err) + klog.Errorf("Failed to ensure %v chain %v jump to %v chain: %v", utilebtables.TableFilter, utilebtables.ChainOutput, dedupChain, err) return } commonArgs := []string{"-p", "IPv4", "-s", macAddr.String(), "-o", "veth+"} _, err = plugin.ebtables.EnsureRule(utilebtables.Prepend, utilebtables.TableFilter, dedupChain, append(commonArgs, "--ip-src", plugin.gateway.String(), "-j", "ACCEPT")...) if err != nil { - glog.Errorf("Failed to ensure packets from cbr0 gateway to be accepted") + klog.Errorf("Failed to ensure packets from cbr0 gateway to be accepted") return } _, err = plugin.ebtables.EnsureRule(utilebtables.Append, utilebtables.TableFilter, dedupChain, append(commonArgs, "--ip-src", plugin.podCidr, "-j", "DROP")...) if err != nil { - glog.Errorf("Failed to ensure packets from podCidr but has mac address of cbr0 to get dropped.") + klog.Errorf("Failed to ensure packets from podCidr but has mac address of cbr0 to get dropped.") return } } diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go index ebb56e40ec3..d2a59126b3e 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go @@ -42,7 +42,7 @@ func (plugin *kubenetNetworkPlugin) Name() string { return "kubenet" } -func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { +func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error { return fmt.Errorf("Kubenet is not supported in this build") } diff --git a/pkg/kubelet/dockershim/network/plugins.go b/pkg/kubelet/dockershim/network/plugins.go index bcef49f77c5..c67c1a355b6 100644 --- a/pkg/kubelet/dockershim/network/plugins.go +++ b/pkg/kubelet/dockershim/network/plugins.go @@ -23,11 +23,11 @@ import ( "sync" "time" - "github.com/golang/glog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilsets "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/klog" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/network/hostport" @@ -63,7 +63,7 @@ type NetworkPlugin interface { // SetUpPod is the method called after the infra container of // the pod has been created but before the other containers of the // pod are launched. - SetUpPod(namespace string, name string, podSandboxID kubecontainer.ContainerID, annotations map[string]string) error + SetUpPod(namespace string, name string, podSandboxID kubecontainer.ContainerID, annotations, options map[string]string) error // TearDownPod is the method called before a pod's infra container will be deleted TearDownPod(namespace string, name string, podSandboxID kubecontainer.ContainerID) error @@ -156,7 +156,7 @@ func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host H if err != nil { allErrs = append(allErrs, fmt.Errorf("Network plugin %q failed init: %v", networkPluginName, err)) } else { - glog.V(1).Infof("Loaded network plugin %q", networkPluginName) + klog.V(1).Infof("Loaded network plugin %q", networkPluginName) } } else { allErrs = append(allErrs, fmt.Errorf("Network plugin %q not found.", networkPluginName)) @@ -183,12 +183,12 @@ func (plugin *NoopNetworkPlugin) Init(host Host, hairpinMode kubeletconfig.Hairp // it was built-in. utilexec.New().Command("modprobe", "br-netfilter").CombinedOutput() if err := plugin.Sysctl.SetSysctl(sysctlBridgeCallIPTables, 1); err != nil { - glog.Warningf("can't set sysctl %s: %v", sysctlBridgeCallIPTables, err) + klog.Warningf("can't set sysctl %s: %v", sysctlBridgeCallIPTables, err) } if val, err := plugin.Sysctl.GetSysctl(sysctlBridgeCallIP6Tables); err == nil { if val != 1 { if err = plugin.Sysctl.SetSysctl(sysctlBridgeCallIP6Tables, 1); err != nil { - glog.Warningf("can't set sysctl %s: %v", sysctlBridgeCallIP6Tables, err) + klog.Warningf("can't set sysctl %s: %v", sysctlBridgeCallIP6Tables, err) } } } @@ -207,7 +207,7 @@ func (plugin *NoopNetworkPlugin) Capabilities() utilsets.Int { return utilsets.NewInt() } -func (plugin *NoopNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { +func (plugin *NoopNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error { return nil } @@ -334,12 +334,12 @@ func (pm *PluginManager) podUnlock(fullPodName string) { lock, ok := pm.pods[fullPodName] if !ok { - glog.Warningf("Unbalanced pod lock unref for %s", fullPodName) + klog.Warningf("Unbalanced pod lock unref for %s", fullPodName) return } else if lock.refcount == 0 { // This should never ever happen, but handle it anyway delete(pm.pods, fullPodName) - glog.Warningf("Pod lock for %s still in map with zero refcount", fullPodName) + klog.Warningf("Pod lock for %s still in map with zero refcount", fullPodName) return } lock.refcount-- @@ -368,14 +368,14 @@ func (pm *PluginManager) GetPodNetworkStatus(podNamespace, podName string, id ku return netStatus, nil } -func (pm *PluginManager) SetUpPod(podNamespace, podName string, id kubecontainer.ContainerID, annotations map[string]string) error { +func (pm *PluginManager) SetUpPod(podNamespace, podName string, id kubecontainer.ContainerID, annotations, options map[string]string) error { defer recordOperation("set_up_pod", time.Now()) fullPodName := kubecontainer.BuildPodFullName(podName, podNamespace) pm.podLock(fullPodName).Lock() defer pm.podUnlock(fullPodName) - glog.V(3).Infof("Calling network plugin %s to set up pod %q", pm.plugin.Name(), fullPodName) - if err := pm.plugin.SetUpPod(podNamespace, podName, id, annotations); err != nil { + klog.V(3).Infof("Calling network plugin %s to set up pod %q", pm.plugin.Name(), fullPodName) + if err := pm.plugin.SetUpPod(podNamespace, podName, id, annotations, options); err != nil { return fmt.Errorf("NetworkPlugin %s failed to set up pod %q network: %v", pm.plugin.Name(), fullPodName, err) } @@ -388,7 +388,7 @@ func (pm *PluginManager) TearDownPod(podNamespace, podName string, id kubecontai pm.podLock(fullPodName).Lock() defer pm.podUnlock(fullPodName) - glog.V(3).Infof("Calling network plugin %s to tear down pod %q", pm.plugin.Name(), fullPodName) + klog.V(3).Infof("Calling network plugin %s to tear down pod %q", pm.plugin.Name(), fullPodName) if err := pm.plugin.TearDownPod(podNamespace, podName, id); err != nil { return fmt.Errorf("NetworkPlugin %s failed to teardown pod %q network: %v", pm.plugin.Name(), fullPodName, err) } diff --git a/pkg/kubelet/dockershim/network/testing/mock_network_plugin.go b/pkg/kubelet/dockershim/network/testing/mock_network_plugin.go index 43e6d744903..afb0752e1df 100644 --- a/pkg/kubelet/dockershim/network/testing/mock_network_plugin.go +++ b/pkg/kubelet/dockershim/network/testing/mock_network_plugin.go @@ -102,7 +102,7 @@ func (_mr *_MockNetworkPluginRecorder) Name() *gomock.Call { return _mr.mock.ctrl.RecordCall(_mr.mock, "Name") } -func (_m *MockNetworkPlugin) SetUpPod(_param0 string, _param1 string, _param2 container.ContainerID, annotations map[string]string) error { +func (_m *MockNetworkPlugin) SetUpPod(_param0 string, _param1 string, _param2 container.ContainerID, annotations, options map[string]string) error { ret := _m.ctrl.Call(_m, "SetUpPod", _param0, _param1, _param2) ret0, _ := ret[0].(error) return ret0 diff --git a/pkg/kubelet/dockershim/network/testing/plugins_test.go b/pkg/kubelet/dockershim/network/testing/plugins_test.go index 29ae5b148f5..42c6ba9e8e1 100644 --- a/pkg/kubelet/dockershim/network/testing/plugins_test.go +++ b/pkg/kubelet/dockershim/network/testing/plugins_test.go @@ -108,7 +108,7 @@ func TestPluginManager(t *testing.T) { // concurrently. allCreatedWg.Wait() - if err := pm.SetUpPod("", name, id, nil); err != nil { + if err := pm.SetUpPod("", name, id, nil, nil); err != nil { t.Errorf("Failed to set up pod %q: %v", name, err) return } @@ -159,7 +159,7 @@ func (p *hookableFakeNetworkPlugin) Capabilities() utilsets.Int { return utilsets.NewInt() } -func (p *hookableFakeNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { +func (p *hookableFakeNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error { if p.setupHook != nil { p.setupHook(namespace, name, id) } @@ -210,7 +210,7 @@ func TestMultiPodParallelNetworkOps(t *testing.T) { // Setup will block on the runner pod completing. If network // operations locking isn't correct (eg pod network operations // block other pods) setUpPod() will never return. - if err := pm.SetUpPod("", podName, containerID, nil); err != nil { + if err := pm.SetUpPod("", podName, containerID, nil, nil); err != nil { t.Errorf("Failed to set up waiter pod: %v", err) return } @@ -230,7 +230,7 @@ func TestMultiPodParallelNetworkOps(t *testing.T) { podName := "runner" containerID := kubecontainer.ContainerID{ID: podName} - if err := pm.SetUpPod("", podName, containerID, nil); err != nil { + if err := pm.SetUpPod("", podName, containerID, nil, nil); err != nil { t.Errorf("Failed to set up runner pod: %v", err) return } diff --git a/pkg/kubelet/dockershim/remote/BUILD b/pkg/kubelet/dockershim/remote/BUILD index a40ce09eaae..06740b1f160 100644 --- a/pkg/kubelet/dockershim/remote/BUILD +++ b/pkg/kubelet/dockershim/remote/BUILD @@ -13,8 +13,8 @@ go_library( "//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library", "//pkg/kubelet/dockershim:go_default_library", "//pkg/kubelet/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/dockershim/remote/docker_server.go b/pkg/kubelet/dockershim/remote/docker_server.go index 546c3b6c4ab..734f61cca04 100644 --- a/pkg/kubelet/dockershim/remote/docker_server.go +++ b/pkg/kubelet/dockershim/remote/docker_server.go @@ -19,8 +19,8 @@ package remote import ( "fmt" - "github.com/golang/glog" "google.golang.org/grpc" + "k8s.io/klog" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/dockershim" "k8s.io/kubernetes/pkg/kubelet/util" @@ -52,11 +52,11 @@ func NewDockerServer(endpoint string, s dockershim.CRIService) *DockerServer { func (s *DockerServer) Start() error { // Start the internal service. if err := s.service.Start(); err != nil { - glog.Errorf("Unable to start docker service") + klog.Errorf("Unable to start docker service") return err } - glog.V(2).Infof("Start dockershim grpc server") + klog.V(2).Infof("Start dockershim grpc server") l, err := util.CreateListener(s.endpoint) if err != nil { return fmt.Errorf("failed to listen on %q: %v", s.endpoint, err) @@ -70,7 +70,7 @@ func (s *DockerServer) Start() error { runtimeapi.RegisterImageServiceServer(s.server, s.service) go func() { if err := s.server.Serve(l); err != nil { - glog.Fatalf("Failed to serve connections: %v", err) + klog.Fatalf("Failed to serve connections: %v", err) } }() return nil diff --git a/pkg/kubelet/eviction/BUILD b/pkg/kubelet/eviction/BUILD index a3c139bcd60..65968751a8d 100644 --- a/pkg/kubelet/eviction/BUILD +++ b/pkg/kubelet/eviction/BUILD @@ -11,6 +11,7 @@ go_test( srcs = [ "eviction_manager_test.go", "helpers_test.go", + "main_test.go", "memory_threshold_notifier_test.go", "mock_threshold_notifier_test.go", ], @@ -28,6 +29,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//vendor/github.com/stretchr/testify/mock:go_default_library", ], @@ -67,7 +69,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//vendor/golang.org/x/sys/unix:go_default_library", diff --git a/pkg/kubelet/eviction/eviction_manager.go b/pkg/kubelet/eviction/eviction_manager.go index c1bf8837c78..7e1ede9eeb7 100644 --- a/pkg/kubelet/eviction/eviction_manager.go +++ b/pkg/kubelet/eviction/eviction_manager.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -153,7 +153,7 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd } // reject pods when under memory pressure (if pod is best effort), or if under disk pressure. - glog.Warningf("Failed to admit pod %s - node has conditions: %v", format.Pod(attrs.Pod), m.nodeConditions) + klog.Warningf("Failed to admit pod %s - node has conditions: %v", format.Pod(attrs.Pod), m.nodeConditions) return lifecycle.PodAdmitResult{ Admit: false, Reason: Reason, @@ -164,7 +164,7 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd // Start starts the control loop to observe and response to low compute resources. func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, podCleanedUpFunc PodCleanedUpFunc, monitoringInterval time.Duration) { thresholdHandler := func(message string) { - glog.Infof(message) + klog.Infof(message) m.synchronize(diskInfoProvider, podFunc) } if m.config.KernelMemcgNotification { @@ -172,7 +172,7 @@ func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePod if threshold.Signal == evictionapi.SignalMemoryAvailable || threshold.Signal == evictionapi.SignalAllocatableMemoryAvailable { notifier, err := NewMemoryThresholdNotifier(threshold, m.config.PodCgroupRoot, &CgroupNotifierFactory{}, thresholdHandler) if err != nil { - glog.Warningf("eviction manager: failed to create memory threshold notifier: %v", err) + klog.Warningf("eviction manager: failed to create memory threshold notifier: %v", err) } else { go notifier.Start() m.thresholdNotifiers = append(m.thresholdNotifiers, notifier) @@ -184,7 +184,7 @@ func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePod go func() { for { if evictedPods := m.synchronize(diskInfoProvider, podFunc); evictedPods != nil { - glog.Infof("eviction manager: pods %s evicted, waiting for pod to be cleaned up", format.Pods(evictedPods)) + klog.Infof("eviction manager: pods %s evicted, waiting for pod to be cleaned up", format.Pods(evictedPods)) m.waitForPodsCleanup(podCleanedUpFunc, evictedPods) } else { time.Sleep(monitoringInterval) @@ -223,7 +223,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act return nil } - glog.V(3).Infof("eviction manager: synchronize housekeeping") + klog.V(3).Infof("eviction manager: synchronize housekeeping") // build the ranking functions (if not yet known) // TODO: have a function in cadvisor that lets us know if global housekeeping has completed if m.dedicatedImageFs == nil { @@ -240,7 +240,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act updateStats := true summary, err := m.summaryProvider.Get(updateStats) if err != nil { - glog.Errorf("eviction manager: failed to get summary stats: %v", err) + klog.Errorf("eviction manager: failed to get summary stats: %v", err) return nil } @@ -248,7 +248,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act m.thresholdsLastUpdated = m.clock.Now() for _, notifier := range m.thresholdNotifiers { if err := notifier.UpdateThreshold(summary); err != nil { - glog.Warningf("eviction manager: failed to update %s: %v", notifier.Description(), err) + klog.Warningf("eviction manager: failed to update %s: %v", notifier.Description(), err) } } } @@ -275,7 +275,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act // the set of node conditions that are triggered by currently observed thresholds nodeConditions := nodeConditions(thresholds) if len(nodeConditions) > 0 { - glog.V(3).Infof("eviction manager: node conditions - observed: %v", nodeConditions) + klog.V(3).Infof("eviction manager: node conditions - observed: %v", nodeConditions) } // track when a node condition was last observed @@ -284,7 +284,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act // node conditions report true if it has been observed within the transition period window nodeConditions = nodeConditionsObservedSince(nodeConditionsLastObservedAt, m.config.PressureTransitionPeriod, now) if len(nodeConditions) > 0 { - glog.V(3).Infof("eviction manager: node conditions - transition period not met: %v", nodeConditions) + klog.V(3).Infof("eviction manager: node conditions - transition period not met: %v", nodeConditions) } // determine the set of thresholds we need to drive eviction behavior (i.e. all grace periods are met) @@ -314,7 +314,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act } if len(thresholds) == 0 { - glog.V(3).Infof("eviction manager: no resources are starved") + klog.V(3).Infof("eviction manager: no resources are starved") return nil } @@ -323,39 +323,39 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act thresholdToReclaim := thresholds[0] resourceToReclaim, found := signalToResource[thresholdToReclaim.Signal] if !found { - glog.V(3).Infof("eviction manager: threshold %s was crossed, but reclaim is not implemented for this threshold.", thresholdToReclaim.Signal) + klog.V(3).Infof("eviction manager: threshold %s was crossed, but reclaim is not implemented for this threshold.", thresholdToReclaim.Signal) return nil } - glog.Warningf("eviction manager: attempting to reclaim %v", resourceToReclaim) + klog.Warningf("eviction manager: attempting to reclaim %v", resourceToReclaim) // record an event about the resources we are now attempting to reclaim via eviction m.recorder.Eventf(m.nodeRef, v1.EventTypeWarning, "EvictionThresholdMet", "Attempting to reclaim %s", resourceToReclaim) // check if there are node-level resources we can reclaim to reduce pressure before evicting end-user pods. if m.reclaimNodeLevelResources(thresholdToReclaim.Signal, resourceToReclaim) { - glog.Infof("eviction manager: able to reduce %v pressure without evicting pods.", resourceToReclaim) + klog.Infof("eviction manager: able to reduce %v pressure without evicting pods.", resourceToReclaim) return nil } - glog.Infof("eviction manager: must evict pod(s) to reclaim %v", resourceToReclaim) + klog.Infof("eviction manager: must evict pod(s) to reclaim %v", resourceToReclaim) // rank the pods for eviction rank, ok := m.signalToRankFunc[thresholdToReclaim.Signal] if !ok { - glog.Errorf("eviction manager: no ranking function for signal %s", thresholdToReclaim.Signal) + klog.Errorf("eviction manager: no ranking function for signal %s", thresholdToReclaim.Signal) return nil } // the only candidates viable for eviction are those pods that had anything running. if len(activePods) == 0 { - glog.Errorf("eviction manager: eviction thresholds have been met, but no pods are active to evict") + klog.Errorf("eviction manager: eviction thresholds have been met, but no pods are active to evict") return nil } // rank the running pods for eviction for the specified resource rank(activePods, statsFunc) - glog.Infof("eviction manager: pods ranked for eviction: %s", format.Pods(activePods)) + klog.Infof("eviction manager: pods ranked for eviction: %s", format.Pods(activePods)) //record age of metrics for met thresholds that we are using for evictions. for _, t := range thresholds { @@ -377,7 +377,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act return []*v1.Pod{pod} } } - glog.Infof("eviction manager: unable to evict any pods from the node") + klog.Infof("eviction manager: unable to evict any pods from the node") return nil } @@ -389,7 +389,7 @@ func (m *managerImpl) waitForPodsCleanup(podCleanedUpFunc PodCleanedUpFunc, pods for { select { case <-timeout.C(): - glog.Warningf("eviction manager: timed out waiting for pods %s to be cleaned up", format.Pods(pods)) + klog.Warningf("eviction manager: timed out waiting for pods %s to be cleaned up", format.Pods(pods)) return case <-ticker.C(): for i, pod := range pods { @@ -397,7 +397,7 @@ func (m *managerImpl) waitForPodsCleanup(podCleanedUpFunc PodCleanedUpFunc, pods break } if i == len(pods)-1 { - glog.Infof("eviction manager: pods %s successfully cleaned up", format.Pods(pods)) + klog.Infof("eviction manager: pods %s successfully cleaned up", format.Pods(pods)) return } } @@ -411,14 +411,14 @@ func (m *managerImpl) reclaimNodeLevelResources(signalToReclaim evictionapi.Sign for _, nodeReclaimFunc := range nodeReclaimFuncs { // attempt to reclaim the pressured resource. if err := nodeReclaimFunc(); err != nil { - glog.Warningf("eviction manager: unexpected error when attempting to reduce %v pressure: %v", resourceToReclaim, err) + klog.Warningf("eviction manager: unexpected error when attempting to reduce %v pressure: %v", resourceToReclaim, err) } } if len(nodeReclaimFuncs) > 0 { summary, err := m.summaryProvider.Get(true) if err != nil { - glog.Errorf("eviction manager: failed to get summary stats after resource reclaim: %v", err) + klog.Errorf("eviction manager: failed to get summary stats after resource reclaim: %v", err) return false } @@ -502,7 +502,7 @@ func (m *managerImpl) podEphemeralStorageLimitEviction(podStats statsapi.PodStat } podEphemeralUsage, err := podLocalEphemeralStorageUsage(podStats, pod, fsStatsSet) if err != nil { - glog.Errorf("eviction manager: error getting pod disk usage %v", err) + klog.Errorf("eviction manager: error getting pod disk usage %v", err) return false } @@ -545,7 +545,7 @@ func (m *managerImpl) evictPod(pod *v1.Pod, gracePeriodOverride int64, evictMsg // do not evict such pods. Static pods are not re-admitted after evictions. // https://github.com/kubernetes/kubernetes/issues/40573 has more details. if kubelettypes.IsCriticalPod(pod) && kubepod.IsStaticPod(pod) { - glog.Errorf("eviction manager: cannot evict a critical static pod %s", format.Pod(pod)) + klog.Errorf("eviction manager: cannot evict a critical static pod %s", format.Pod(pod)) return false } status := v1.PodStatus{ @@ -558,9 +558,9 @@ func (m *managerImpl) evictPod(pod *v1.Pod, gracePeriodOverride int64, evictMsg // this is a blocking call and should only return when the pod and its containers are killed. err := m.killPodFunc(pod, status, &gracePeriodOverride) if err != nil { - glog.Errorf("eviction manager: pod %s failed to evict %v", format.Pod(pod), err) + klog.Errorf("eviction manager: pod %s failed to evict %v", format.Pod(pod), err) } else { - glog.Infof("eviction manager: pod %s is evicted successfully", format.Pod(pod)) + klog.Infof("eviction manager: pod %s is evicted successfully", format.Pod(pod)) } return true } diff --git a/pkg/kubelet/eviction/helpers.go b/pkg/kubelet/eviction/helpers.go index c078ce93d00..ef34b97a0ba 100644 --- a/pkg/kubelet/eviction/helpers.go +++ b/pkg/kubelet/eviction/helpers.go @@ -23,10 +23,10 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" @@ -732,7 +732,7 @@ func makeSignalObservations(summary *statsapi.Summary) (signalObservations, stat } } if allocatableContainer, err := getSysContainer(summary.Node.SystemContainers, statsapi.SystemContainerPods); err != nil { - glog.Errorf("eviction manager: failed to construct signal: %q error: %v", evictionapi.SignalAllocatableMemoryAvailable, err) + klog.Errorf("eviction manager: failed to construct signal: %q error: %v", evictionapi.SignalAllocatableMemoryAvailable, err) } else { if memory := allocatableContainer.Memory; memory != nil && memory.AvailableBytes != nil && memory.WorkingSetBytes != nil { result[evictionapi.SignalAllocatableMemoryAvailable] = signalObservation{ @@ -805,7 +805,7 @@ func thresholdsMet(thresholds []evictionapi.Threshold, observations signalObserv threshold := thresholds[i] observed, found := observations[threshold.Signal] if !found { - glog.Warningf("eviction manager: no observation found for eviction signal %v", threshold.Signal) + klog.Warningf("eviction manager: no observation found for eviction signal %v", threshold.Signal) continue } // determine if we have met the specified threshold @@ -828,20 +828,20 @@ func thresholdsMet(thresholds []evictionapi.Threshold, observations signalObserv } func debugLogObservations(logPrefix string, observations signalObservations) { - if !glog.V(3) { + if !klog.V(3) { return } for k, v := range observations { if !v.time.IsZero() { - glog.Infof("eviction manager: %v: signal=%v, available: %v, capacity: %v, time: %v", logPrefix, k, v.available, v.capacity, v.time) + klog.Infof("eviction manager: %v: signal=%v, available: %v, capacity: %v, time: %v", logPrefix, k, v.available, v.capacity, v.time) } else { - glog.Infof("eviction manager: %v: signal=%v, available: %v, capacity: %v", logPrefix, k, v.available, v.capacity) + klog.Infof("eviction manager: %v: signal=%v, available: %v, capacity: %v", logPrefix, k, v.available, v.capacity) } } } func debugLogThresholdsWithObservation(logPrefix string, thresholds []evictionapi.Threshold, observations signalObservations) { - if !glog.V(3) { + if !klog.V(3) { return } for i := range thresholds { @@ -849,9 +849,9 @@ func debugLogThresholdsWithObservation(logPrefix string, thresholds []evictionap observed, found := observations[threshold.Signal] if found { quantity := evictionapi.GetThresholdQuantity(threshold.Value, observed.capacity) - glog.Infof("eviction manager: %v: threshold [signal=%v, quantity=%v] observed %v", logPrefix, threshold.Signal, quantity, observed.available) + klog.Infof("eviction manager: %v: threshold [signal=%v, quantity=%v] observed %v", logPrefix, threshold.Signal, quantity, observed.available) } else { - glog.Infof("eviction manager: %v: threshold [signal=%v] had no observation", logPrefix, threshold.Signal) + klog.Infof("eviction manager: %v: threshold [signal=%v] had no observation", logPrefix, threshold.Signal) } } } @@ -862,7 +862,7 @@ func thresholdsUpdatedStats(thresholds []evictionapi.Threshold, observations, la threshold := thresholds[i] observed, found := observations[threshold.Signal] if !found { - glog.Warningf("eviction manager: no observation found for eviction signal %v", threshold.Signal) + klog.Warningf("eviction manager: no observation found for eviction signal %v", threshold.Signal) continue } last, found := lastObservations[threshold.Signal] @@ -892,7 +892,7 @@ func thresholdsMetGracePeriod(observedAt thresholdsObservedAt, now time.Time) [] for threshold, at := range observedAt { duration := now.Sub(at) if duration < threshold.GracePeriod { - glog.V(2).Infof("eviction manager: eviction criteria not yet met for %v, duration: %v", formatThreshold(threshold), duration) + klog.V(2).Infof("eviction manager: eviction criteria not yet met for %v, duration: %v", formatThreshold(threshold), duration) continue } results = append(results, threshold) diff --git a/pkg/kubelet/eviction/main_test.go b/pkg/kubelet/eviction/main_test.go new file mode 100644 index 00000000000..a5c470b3ed5 --- /dev/null +++ b/pkg/kubelet/eviction/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eviction + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/kubelet/eviction/memory_threshold_notifier.go b/pkg/kubelet/eviction/memory_threshold_notifier.go index 8d86944f39d..b60393e8c64 100644 --- a/pkg/kubelet/eviction/memory_threshold_notifier.go +++ b/pkg/kubelet/eviction/memory_threshold_notifier.go @@ -20,7 +20,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/resource" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" @@ -71,7 +71,7 @@ func NewMemoryThresholdNotifier(threshold evictionapi.Threshold, cgroupRoot stri } func (m *memoryThresholdNotifier) Start() { - glog.Infof("eviction manager: created %s", m.Description()) + klog.Infof("eviction manager: created %s", m.Description()) for range m.events { m.handler(fmt.Sprintf("eviction manager: %s crossed", m.Description())) } @@ -98,7 +98,7 @@ func (m *memoryThresholdNotifier) UpdateThreshold(summary *statsapi.Summary) err memcgThreshold.Sub(*evictionThresholdQuantity) memcgThreshold.Add(*inactiveFile) - glog.V(3).Infof("eviction manager: setting %s to %s\n", m.Description(), memcgThreshold.String()) + klog.V(3).Infof("eviction manager: setting %s to %s\n", m.Description(), memcgThreshold.String()) if m.notifier != nil { m.notifier.Stop() } diff --git a/pkg/kubelet/eviction/threshold_notifier_linux.go b/pkg/kubelet/eviction/threshold_notifier_linux.go index 4fabd7345a5..1d097fd293f 100644 --- a/pkg/kubelet/eviction/threshold_notifier_linux.go +++ b/pkg/kubelet/eviction/threshold_notifier_linux.go @@ -21,8 +21,8 @@ import ( "sync" "time" - "github.com/golang/glog" "golang.org/x/sys/unix" + "k8s.io/klog" ) const ( @@ -104,7 +104,7 @@ func (n *linuxCgroupNotifier) Start(eventCh chan<- struct{}) { Events: unix.EPOLLIN, }) if err != nil { - glog.Warningf("eviction manager: error adding epoll eventfd: %v", err) + klog.Warningf("eviction manager: error adding epoll eventfd: %v", err) return } for { @@ -115,7 +115,7 @@ func (n *linuxCgroupNotifier) Start(eventCh chan<- struct{}) { } event, err := wait(n.epfd, n.eventfd, notifierRefreshInterval) if err != nil { - glog.Warningf("eviction manager: error while waiting for memcg events: %v", err) + klog.Warningf("eviction manager: error while waiting for memcg events: %v", err) return } else if !event { // Timeout on wait. This is expected if the threshold was not crossed @@ -125,7 +125,7 @@ func (n *linuxCgroupNotifier) Start(eventCh chan<- struct{}) { buf := make([]byte, eventSize) _, err = unix.Read(n.eventfd, buf) if err != nil { - glog.Warningf("eviction manager: error reading memcg events: %v", err) + klog.Warningf("eviction manager: error reading memcg events: %v", err) return } eventCh <- struct{}{} diff --git a/pkg/kubelet/eviction/threshold_notifier_unsupported.go b/pkg/kubelet/eviction/threshold_notifier_unsupported.go index 7078c7865a9..afa92fe5fce 100644 --- a/pkg/kubelet/eviction/threshold_notifier_unsupported.go +++ b/pkg/kubelet/eviction/threshold_notifier_unsupported.go @@ -18,11 +18,11 @@ limitations under the License. package eviction -import "github.com/golang/glog" +import "k8s.io/klog" // NewCgroupNotifier creates a cgroup notifier that does nothing because cgroups do not exist on non-linux systems. func NewCgroupNotifier(path, attribute string, threshold int64) (CgroupNotifier, error) { - glog.V(5).Infof("cgroup notifications not supported") + klog.V(5).Infof("cgroup notifications not supported") return &unsupportedThresholdNotifier{}, nil } diff --git a/pkg/kubelet/images/BUILD b/pkg/kubelet/images/BUILD index 5f5594b3f3c..cbb23f945e1 100644 --- a/pkg/kubelet/images/BUILD +++ b/pkg/kubelet/images/BUILD @@ -30,7 +30,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//vendor/github.com/docker/distribution/reference:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/images/image_gc_manager.go b/pkg/kubelet/images/image_gc_manager.go index a25769a90e3..4c6feabb17b 100644 --- a/pkg/kubelet/images/image_gc_manager.go +++ b/pkg/kubelet/images/image_gc_manager.go @@ -24,7 +24,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/errors" @@ -178,7 +178,7 @@ func (im *realImageGCManager) Start() { } _, err := im.detectImages(ts) if err != nil { - glog.Warningf("[imageGCManager] Failed to monitor images: %v", err) + klog.Warningf("[imageGCManager] Failed to monitor images: %v", err) } else { im.initialized = true } @@ -189,7 +189,7 @@ func (im *realImageGCManager) Start() { go wait.Until(func() { images, err := im.runtime.ListImages() if err != nil { - glog.Warningf("[imageGCManager] Failed to update image list: %v", err) + klog.Warningf("[imageGCManager] Failed to update image list: %v", err) } else { im.imageCache.set(images) } @@ -223,7 +223,7 @@ func (im *realImageGCManager) detectImages(detectTime time.Time) (sets.String, e // Make a set of images in use by containers. for _, pod := range pods { for _, container := range pod.Containers { - glog.V(5).Infof("Pod %s/%s, container %s uses image %s(%s)", pod.Namespace, pod.Name, container.Name, container.Image, container.ImageID) + klog.V(5).Infof("Pod %s/%s, container %s uses image %s(%s)", pod.Namespace, pod.Name, container.Name, container.Image, container.ImageID) imagesInUse.Insert(container.ImageID) } } @@ -234,12 +234,12 @@ func (im *realImageGCManager) detectImages(detectTime time.Time) (sets.String, e im.imageRecordsLock.Lock() defer im.imageRecordsLock.Unlock() for _, image := range images { - glog.V(5).Infof("Adding image ID %s to currentImages", image.ID) + klog.V(5).Infof("Adding image ID %s to currentImages", image.ID) currentImages.Insert(image.ID) // New image, set it as detected now. if _, ok := im.imageRecords[image.ID]; !ok { - glog.V(5).Infof("Image ID %s is new", image.ID) + klog.V(5).Infof("Image ID %s is new", image.ID) im.imageRecords[image.ID] = &imageRecord{ firstDetected: detectTime, } @@ -247,18 +247,18 @@ func (im *realImageGCManager) detectImages(detectTime time.Time) (sets.String, e // Set last used time to now if the image is being used. if isImageUsed(image.ID, imagesInUse) { - glog.V(5).Infof("Setting Image ID %s lastUsed to %v", image.ID, now) + klog.V(5).Infof("Setting Image ID %s lastUsed to %v", image.ID, now) im.imageRecords[image.ID].lastUsed = now } - glog.V(5).Infof("Image ID %s has size %d", image.ID, image.Size) + klog.V(5).Infof("Image ID %s has size %d", image.ID, image.Size) im.imageRecords[image.ID].size = image.Size } // Remove old images from our records. for image := range im.imageRecords { if !currentImages.Has(image) { - glog.V(5).Infof("Image ID %s is no longer present; removing from imageRecords", image) + klog.V(5).Infof("Image ID %s is no longer present; removing from imageRecords", image) delete(im.imageRecords, image) } } @@ -282,7 +282,7 @@ func (im *realImageGCManager) GarbageCollect() error { } if available > capacity { - glog.Warningf("available %d is larger than capacity %d", available, capacity) + klog.Warningf("available %d is larger than capacity %d", available, capacity) available = capacity } @@ -297,7 +297,7 @@ func (im *realImageGCManager) GarbageCollect() error { usagePercent := 100 - int(available*100/capacity) if usagePercent >= im.policy.HighThresholdPercent { amountToFree := capacity*int64(100-im.policy.LowThresholdPercent)/100 - available - glog.Infof("[imageGCManager]: Disk usage on image filesystem is at %d%% which is over the high threshold (%d%%). Trying to free %d bytes down to the low threshold (%d%%).", usagePercent, im.policy.HighThresholdPercent, amountToFree, im.policy.LowThresholdPercent) + klog.Infof("[imageGCManager]: Disk usage on image filesystem is at %d%% which is over the high threshold (%d%%). Trying to free %d bytes down to the low threshold (%d%%).", usagePercent, im.policy.HighThresholdPercent, amountToFree, im.policy.LowThresholdPercent) freed, err := im.freeSpace(amountToFree, time.Now()) if err != nil { return err @@ -314,7 +314,7 @@ func (im *realImageGCManager) GarbageCollect() error { } func (im *realImageGCManager) DeleteUnusedImages() error { - glog.Infof("attempting to delete unused images") + klog.Infof("attempting to delete unused images") _, err := im.freeSpace(math.MaxInt64, time.Now()) return err } @@ -338,7 +338,7 @@ func (im *realImageGCManager) freeSpace(bytesToFree int64, freeTime time.Time) ( images := make([]evictionInfo, 0, len(im.imageRecords)) for image, record := range im.imageRecords { if isImageUsed(image, imagesInUse) { - glog.V(5).Infof("Image ID %s is being used", image) + klog.V(5).Infof("Image ID %s is being used", image) continue } images = append(images, evictionInfo{ @@ -352,10 +352,10 @@ func (im *realImageGCManager) freeSpace(bytesToFree int64, freeTime time.Time) ( var deletionErrors []error spaceFreed := int64(0) for _, image := range images { - glog.V(5).Infof("Evaluating image ID %s for possible garbage collection", image.id) + klog.V(5).Infof("Evaluating image ID %s for possible garbage collection", image.id) // Images that are currently in used were given a newer lastUsed. if image.lastUsed.Equal(freeTime) || image.lastUsed.After(freeTime) { - glog.V(5).Infof("Image ID %s has lastUsed=%v which is >= freeTime=%v, not eligible for garbage collection", image.id, image.lastUsed, freeTime) + klog.V(5).Infof("Image ID %s has lastUsed=%v which is >= freeTime=%v, not eligible for garbage collection", image.id, image.lastUsed, freeTime) continue } @@ -363,12 +363,12 @@ func (im *realImageGCManager) freeSpace(bytesToFree int64, freeTime time.Time) ( // In such a case, the image may have just been pulled down, and will be used by a container right away. if freeTime.Sub(image.firstDetected) < im.policy.MinAge { - glog.V(5).Infof("Image ID %s has age %v which is less than the policy's minAge of %v, not eligible for garbage collection", image.id, freeTime.Sub(image.firstDetected), im.policy.MinAge) + klog.V(5).Infof("Image ID %s has age %v which is less than the policy's minAge of %v, not eligible for garbage collection", image.id, freeTime.Sub(image.firstDetected), im.policy.MinAge) continue } // Remove image. Continue despite errors. - glog.Infof("[imageGCManager]: Removing image %q to free %d bytes", image.id, image.size) + klog.Infof("[imageGCManager]: Removing image %q to free %d bytes", image.id, image.size) err := im.runtime.RemoveImage(container.ImageSpec{Image: image.id}) if err != nil { deletionErrors = append(deletionErrors, err) diff --git a/pkg/kubelet/images/image_manager.go b/pkg/kubelet/images/image_manager.go index 381c55e4084..36b36a9e991 100644 --- a/pkg/kubelet/images/image_manager.go +++ b/pkg/kubelet/images/image_manager.go @@ -20,10 +20,10 @@ import ( "fmt" dockerref "github.com/docker/distribution/reference" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/flowcontrol" + "k8s.io/klog" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/util/parsers" @@ -88,14 +88,14 @@ func (m *imageManager) EnsureImageExists(pod *v1.Pod, container *v1.Container, p logPrefix := fmt.Sprintf("%s/%s", pod.Name, container.Image) ref, err := kubecontainer.GenerateContainerRef(pod, container) if err != nil { - glog.Errorf("Couldn't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err) + klog.Errorf("Couldn't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err) } // If the image contains no tag or digest, a default tag should be applied. image, err := applyDefaultImageTag(container.Image) if err != nil { msg := fmt.Sprintf("Failed to apply default image tag %q: %v", container.Image, err) - m.logIt(ref, v1.EventTypeWarning, events.FailedToInspectImage, logPrefix, msg, glog.Warning) + m.logIt(ref, v1.EventTypeWarning, events.FailedToInspectImage, logPrefix, msg, klog.Warning) return "", msg, ErrInvalidImageName } @@ -103,7 +103,7 @@ func (m *imageManager) EnsureImageExists(pod *v1.Pod, container *v1.Container, p imageRef, err := m.imageService.GetImageRef(spec) if err != nil { msg := fmt.Sprintf("Failed to inspect image %q: %v", container.Image, err) - m.logIt(ref, v1.EventTypeWarning, events.FailedToInspectImage, logPrefix, msg, glog.Warning) + m.logIt(ref, v1.EventTypeWarning, events.FailedToInspectImage, logPrefix, msg, klog.Warning) return "", msg, ErrImageInspect } @@ -111,26 +111,26 @@ func (m *imageManager) EnsureImageExists(pod *v1.Pod, container *v1.Container, p if !shouldPullImage(container, present) { if present { msg := fmt.Sprintf("Container image %q already present on machine", container.Image) - m.logIt(ref, v1.EventTypeNormal, events.PulledImage, logPrefix, msg, glog.Info) + m.logIt(ref, v1.EventTypeNormal, events.PulledImage, logPrefix, msg, klog.Info) return imageRef, "", nil } msg := fmt.Sprintf("Container image %q is not present with pull policy of Never", container.Image) - m.logIt(ref, v1.EventTypeWarning, events.ErrImageNeverPullPolicy, logPrefix, msg, glog.Warning) + m.logIt(ref, v1.EventTypeWarning, events.ErrImageNeverPullPolicy, logPrefix, msg, klog.Warning) return "", msg, ErrImageNeverPull } backOffKey := fmt.Sprintf("%s_%s", pod.UID, container.Image) if m.backOff.IsInBackOffSinceUpdate(backOffKey, m.backOff.Clock.Now()) { msg := fmt.Sprintf("Back-off pulling image %q", container.Image) - m.logIt(ref, v1.EventTypeNormal, events.BackOffPullImage, logPrefix, msg, glog.Info) + m.logIt(ref, v1.EventTypeNormal, events.BackOffPullImage, logPrefix, msg, klog.Info) return "", msg, ErrImagePullBackOff } - m.logIt(ref, v1.EventTypeNormal, events.PullingImage, logPrefix, fmt.Sprintf("pulling image %q", container.Image), glog.Info) + m.logIt(ref, v1.EventTypeNormal, events.PullingImage, logPrefix, fmt.Sprintf("pulling image %q", container.Image), klog.Info) pullChan := make(chan pullResult) m.puller.pullImage(spec, pullSecrets, pullChan) imagePullResult := <-pullChan if imagePullResult.err != nil { - m.logIt(ref, v1.EventTypeWarning, events.FailedToPullImage, logPrefix, fmt.Sprintf("Failed to pull image %q: %v", container.Image, imagePullResult.err), glog.Warning) + m.logIt(ref, v1.EventTypeWarning, events.FailedToPullImage, logPrefix, fmt.Sprintf("Failed to pull image %q: %v", container.Image, imagePullResult.err), klog.Warning) m.backOff.Next(backOffKey, m.backOff.Clock.Now()) if imagePullResult.err == ErrRegistryUnavailable { msg := fmt.Sprintf("image pull failed for %s because the registry is unavailable.", container.Image) @@ -139,7 +139,7 @@ func (m *imageManager) EnsureImageExists(pod *v1.Pod, container *v1.Container, p return "", imagePullResult.err.Error(), ErrImagePull } - m.logIt(ref, v1.EventTypeNormal, events.PulledImage, logPrefix, fmt.Sprintf("Successfully pulled image %q", container.Image), glog.Info) + m.logIt(ref, v1.EventTypeNormal, events.PulledImage, logPrefix, fmt.Sprintf("Successfully pulled image %q", container.Image), klog.Info) m.backOff.GC() return imagePullResult.imageRef, "", nil } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 00adbec2696..0e065a1b629 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -31,7 +31,6 @@ import ( "sync/atomic" "time" - "github.com/golang/glog" cadvisorapi "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" "k8s.io/api/core/v1" @@ -55,11 +54,13 @@ import ( "k8s.io/client-go/util/integer" cloudprovider "k8s.io/cloud-provider" csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" + "k8s.io/klog" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/features" kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" - pluginwatcherapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1alpha1" + pluginwatcherapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1" + "k8s.io/kubernetes/pkg/kubelet/apis/podresources" "k8s.io/kubernetes/pkg/kubelet/cadvisor" kubeletcertificate "k8s.io/kubernetes/pkg/kubelet/certificate" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" @@ -97,6 +98,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/sysctl" "k8s.io/kubernetes/pkg/kubelet/token" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" + "k8s.io/kubernetes/pkg/kubelet/util" "k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/kubelet/util/manager" "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher" @@ -192,6 +194,7 @@ type Bootstrap interface { StartGarbageCollection() ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableDebuggingHandlers, enableContentionProfiling bool) ListenAndServeReadOnly(address net.IP, port uint) + ListenAndServePodResources() Run(<-chan kubetypes.PodUpdate) RunOnce(<-chan kubetypes.PodUpdate) ([]RunPodResult, error) } @@ -275,13 +278,13 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku // define file config source if kubeCfg.StaticPodPath != "" { - glog.Infof("Adding pod path: %v", kubeCfg.StaticPodPath) + klog.Infof("Adding pod path: %v", kubeCfg.StaticPodPath) config.NewSourceFile(kubeCfg.StaticPodPath, nodeName, kubeCfg.FileCheckFrequency.Duration, cfg.Channel(kubetypes.FileSource)) } // define url config source if kubeCfg.StaticPodURL != "" { - glog.Infof("Adding pod url %q with HTTP header %v", kubeCfg.StaticPodURL, manifestURLHeader) + klog.Infof("Adding pod url %q with HTTP header %v", kubeCfg.StaticPodURL, manifestURLHeader) config.NewSourceURL(kubeCfg.StaticPodURL, manifestURLHeader, nodeName, kubeCfg.HTTPCheckFrequency.Duration, cfg.Channel(kubetypes.HTTPSource)) } @@ -291,7 +294,7 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku var updatechannel chan<- interface{} if bootstrapCheckpointPath != "" { - glog.Infof("Adding checkpoint path: %v", bootstrapCheckpointPath) + klog.Infof("Adding checkpoint path: %v", bootstrapCheckpointPath) updatechannel = cfg.Channel(kubetypes.ApiserverSource) err := cfg.Restore(bootstrapCheckpointPath, updatechannel) if err != nil { @@ -300,7 +303,7 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku } if kubeDeps.KubeClient != nil { - glog.Infof("Watching apiserver") + klog.Infof("Watching apiserver") if updatechannel == nil { updatechannel = cfg.Channel(kubetypes.ApiserverSource) } @@ -391,7 +394,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, return nil, fmt.Errorf("error fetching current instance name from cloud provider: %v", err) } - glog.V(2).Infof("cloud provider determined current node name to be %s", nodeName) + klog.V(2).Infof("cloud provider determined current node name to be %s", nodeName) } if kubeDeps.PodConfig == nil { @@ -470,7 +473,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, for _, ipEntry := range kubeCfg.ClusterDNS { ip := net.ParseIP(ipEntry) if ip == nil { - glog.Warningf("Invalid clusterDNS ip '%q'", ipEntry) + klog.Warningf("Invalid clusterDNS ip '%q'", ipEntry) } else { clusterDNS = append(clusterDNS, ip) } @@ -479,7 +482,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, parsedNodeIP := net.ParseIP(nodeIP) protocol := utilipt.ProtocolIpv4 if parsedNodeIP != nil && parsedNodeIP.To4() == nil { - glog.V(0).Infof("IPv6 node IP (%s), assume IPv6 operation", nodeIP) + klog.V(0).Infof("IPv6 node IP (%s), assume IPv6 operation", nodeIP) protocol = utilipt.ProtocolIpv6 } @@ -510,6 +513,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, nodeRef: nodeRef, nodeLabels: nodeLabels, nodeStatusUpdateFrequency: kubeCfg.NodeStatusUpdateFrequency.Duration, + nodeStatusReportFrequency: kubeCfg.NodeStatusReportFrequency.Duration, os: kubeDeps.OSInterface, oomWatcher: oomWatcher, cgroupsPerQOS: kubeCfg.CgroupsPerQOS, @@ -562,7 +566,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, klet.configMapManager = configMapManager if klet.experimentalHostUserNamespaceDefaulting { - glog.Infof("Experimental host user namespace defaulting is enabled.") + klog.Infof("Experimental host user namespace defaulting is enabled.") } machineInfo, err := klet.cadvisor.MachineInfo() @@ -606,7 +610,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, klet.resourceAnalyzer = serverstats.NewResourceAnalyzer(klet, kubeCfg.VolumeStatsAggPeriod.Duration) if containerRuntime == "rkt" { - glog.Fatalln("rktnetes has been deprecated in favor of rktlet. Please see https://github.com/kubernetes-incubator/rktlet for more information.") + klog.Fatalln("rktnetes has been deprecated in favor of rktlet. Please see https://github.com/kubernetes-incubator/rktlet for more information.") } // if left at nil, that means it is unneeded @@ -626,10 +630,10 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, } // The unix socket for kubelet <-> dockershim communication. - glog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q", + klog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q", remoteRuntimeEndpoint, remoteImageEndpoint) - glog.V(2).Infof("Starting the GRPC server for the docker CRI shim.") + klog.V(2).Infof("Starting the GRPC server for the docker CRI shim.") server := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds) if err := server.Start(); err != nil { return nil, err @@ -716,8 +720,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, klet.pleg = pleg.NewGenericPLEG(klet.containerRuntime, plegChannelCapacity, plegRelistPeriod, klet.podCache, clock.RealClock{}) klet.runtimeState = newRuntimeState(maxWaitForContainerRuntime) klet.runtimeState.addHealthCheck("PLEG", klet.pleg.Healthy) - if err := klet.updatePodCIDR(kubeCfg.PodCIDR); err != nil { - glog.Errorf("Pod CIDR update failed %v", err) + if _, err := klet.updatePodCIDR(kubeCfg.PodCIDR); err != nil { + klog.Errorf("Pod CIDR update failed %v", err) } // setup containerGC @@ -1035,8 +1039,9 @@ type Kubelet struct { // used for generating ContainerStatus. reasonCache *ReasonCache - // nodeStatusUpdateFrequency specifies how often kubelet posts node status to master. - // Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod + // nodeStatusUpdateFrequency specifies how often kubelet computes node status. If node lease + // feature is not enabled, it is also the frequency that kubelet posts node status to master. + // In that case, be cautious when changing the constant, it must work with nodeMonitorGracePeriod // in nodecontroller. There are several constraints: // 1. nodeMonitorGracePeriod must be N times more than nodeStatusUpdateFrequency, where // N means number of retries allowed for kubelet to post node status. It is pointless @@ -1048,6 +1053,13 @@ type Kubelet struct { // as it takes time to gather all necessary node information. nodeStatusUpdateFrequency time.Duration + // nodeStatusUpdateFrequency is the frequency that kubelet posts node + // status to master. It is only used when node lease feature is enabled. + nodeStatusReportFrequency time.Duration + + // lastStatusReportTime is the time when node status was last reported. + lastStatusReportTime time.Time + // syncNodeStatusMux is a lock on updating the node status, because this path is not thread-safe. // This lock is used by Kublet.syncNodeStatus function and shouldn't be used anywhere else. syncNodeStatusMux sync.Mutex @@ -1233,6 +1245,7 @@ func allGlobalUnicastIPs() ([]net.IP, error) { // 1. the root directory // 2. the pods directory // 3. the plugins directory +// 4. the pod-resources directory func (kl *Kubelet) setupDataDirs() error { kl.rootDirectory = path.Clean(kl.rootDirectory) if err := os.MkdirAll(kl.getRootDir(), 0750); err != nil { @@ -1247,6 +1260,9 @@ func (kl *Kubelet) setupDataDirs() error { if err := os.MkdirAll(kl.getPluginsDir(), 0750); err != nil { return fmt.Errorf("error creating plugins directory: %v", err) } + if err := os.MkdirAll(kl.getPodResourcesDir(), 0750); err != nil { + return fmt.Errorf("error creating podresources directory: %v", err) + } return nil } @@ -1255,23 +1271,23 @@ func (kl *Kubelet) StartGarbageCollection() { loggedContainerGCFailure := false go wait.Until(func() { if err := kl.containerGC.GarbageCollect(); err != nil { - glog.Errorf("Container garbage collection failed: %v", err) + klog.Errorf("Container garbage collection failed: %v", err) kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.ContainerGCFailed, err.Error()) loggedContainerGCFailure = true } else { - var vLevel glog.Level = 4 + var vLevel klog.Level = 4 if loggedContainerGCFailure { vLevel = 1 loggedContainerGCFailure = false } - glog.V(vLevel).Infof("Container garbage collection succeeded") + klog.V(vLevel).Infof("Container garbage collection succeeded") } }, ContainerGCPeriod, wait.NeverStop) // when the high threshold is set to 100, stub the image GC manager if kl.kubeletConfiguration.ImageGCHighThresholdPercent == 100 { - glog.V(2).Infof("ImageGCHighThresholdPercent is set 100, Disable image GC") + klog.V(2).Infof("ImageGCHighThresholdPercent is set 100, Disable image GC") return } @@ -1279,21 +1295,21 @@ func (kl *Kubelet) StartGarbageCollection() { go wait.Until(func() { if err := kl.imageManager.GarbageCollect(); err != nil { if prevImageGCFailed { - glog.Errorf("Image garbage collection failed multiple times in a row: %v", err) + klog.Errorf("Image garbage collection failed multiple times in a row: %v", err) // Only create an event for repeated failures kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.ImageGCFailed, err.Error()) } else { - glog.Errorf("Image garbage collection failed once. Stats initialization may not have completed yet: %v", err) + klog.Errorf("Image garbage collection failed once. Stats initialization may not have completed yet: %v", err) } prevImageGCFailed = true } else { - var vLevel glog.Level = 4 + var vLevel klog.Level = 4 if prevImageGCFailed { vLevel = 1 prevImageGCFailed = false } - glog.V(vLevel).Infof("Image garbage collection succeeded") + klog.V(vLevel).Infof("Image garbage collection succeeded") } }, ImageGCPeriod, wait.NeverStop) } @@ -1302,7 +1318,11 @@ func (kl *Kubelet) StartGarbageCollection() { // Note that the modules here must not depend on modules that are not initialized here. func (kl *Kubelet) initializeModules() error { // Prometheus metrics. - metrics.Register(kl.runtimeCache, collectors.NewVolumeStatsCollector(kl)) + metrics.Register( + kl.runtimeCache, + collectors.NewVolumeStatsCollector(kl), + collectors.NewLogMetricsCollector(kl.StatsProvider.ListPodStats), + ) // Setup filesystem directories. if err := kl.setupDataDirs(); err != nil { @@ -1312,7 +1332,7 @@ func (kl *Kubelet) initializeModules() error { // If the container logs directory does not exist, create it. if _, err := os.Stat(ContainerLogsDir); err != nil { if err := kl.os.MkdirAll(ContainerLogsDir, 0755); err != nil { - glog.Errorf("Failed to create directory %q: %v", ContainerLogsDir, err) + klog.Errorf("Failed to create directory %q: %v", ContainerLogsDir, err) } } @@ -1340,7 +1360,7 @@ func (kl *Kubelet) initializeRuntimeDependentModules() { if err := kl.cadvisor.Start(); err != nil { // Fail kubelet and rely on the babysitter to retry starting kubelet. // TODO(random-liu): Add backoff logic in the babysitter - glog.Fatalf("Failed to start cAdvisor %v", err) + klog.Fatalf("Failed to start cAdvisor %v", err) } // trigger on-demand stats collection once so that we have capacity information for ephemeral storage. @@ -1350,12 +1370,12 @@ func (kl *Kubelet) initializeRuntimeDependentModules() { node, err := kl.getNodeAnyWay() if err != nil { // Fail kubelet and rely on the babysitter to retry starting kubelet. - glog.Fatalf("Kubelet failed to get node info: %v", err) + klog.Fatalf("Kubelet failed to get node info: %v", err) } // containerManager must start after cAdvisor because it needs filesystem capacity information if err := kl.containerManager.Start(node, kl.GetActivePods, kl.sourcesReady, kl.statusManager, kl.runtimeService); err != nil { // Fail kubelet and rely on the babysitter to retry starting kubelet. - glog.Fatalf("Failed to start ContainerManager %v", err) + klog.Fatalf("Failed to start ContainerManager %v", err) } // eviction manager must start after cadvisor because it needs to know if the container runtime has a dedicated imagefs kl.evictionManager.Start(kl.StatsProvider, kl.GetActivePods, kl.podResourcesAreReclaimed, evictionMonitoringPeriod) @@ -1365,14 +1385,14 @@ func (kl *Kubelet) initializeRuntimeDependentModules() { kl.containerLogManager.Start() if kl.enablePluginsWatcher { // Adding Registration Callback function for CSI Driver - kl.pluginWatcher.AddHandler("CSIPlugin", pluginwatcher.PluginHandler(csi.PluginHandler)) + kl.pluginWatcher.AddHandler(pluginwatcherapi.CSIPlugin, pluginwatcher.PluginHandler(csi.PluginHandler)) // Adding Registration Callback function for Device Manager kl.pluginWatcher.AddHandler(pluginwatcherapi.DevicePlugin, kl.containerManager.GetPluginRegistrationHandler()) // Start the plugin watcher - glog.V(4).Infof("starting watcher") + klog.V(4).Infof("starting watcher") if err := kl.pluginWatcher.Start(); err != nil { kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.KubeletSetupFailed, err.Error()) - glog.Fatalf("failed to start Plugin Watcher. err: %v", err) + klog.Fatalf("failed to start Plugin Watcher. err: %v", err) } } } @@ -1383,7 +1403,7 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) { kl.logServer = http.StripPrefix("/logs/", http.FileServer(http.Dir("/var/log/"))) } if kl.kubeClient == nil { - glog.Warning("No api server defined - no node status update will be sent.") + klog.Warning("No api server defined - no node status update will be sent.") } // Start the cloud provider sync manager @@ -1393,7 +1413,7 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) { if err := kl.initializeModules(); err != nil { kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.KubeletSetupFailed, err.Error()) - glog.Fatal(err) + klog.Fatal(err) } // Start volume manager @@ -1501,7 +1521,7 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error { // since kubelet first saw the pod if firstSeenTime is set. metrics.PodWorkerStartLatency.Observe(metrics.SinceInMicroseconds(firstSeenTime)) } else { - glog.V(3).Infof("First seen time not recorded for pod %q", pod.UID) + klog.V(3).Infof("First seen time not recorded for pod %q", pod.UID) } } @@ -1600,7 +1620,7 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error { if !(podKilled && pod.Spec.RestartPolicy == v1.RestartPolicyNever) { if !pcm.Exists(pod) { if err := kl.containerManager.UpdateQOSCgroups(); err != nil { - glog.V(2).Infof("Failed to update QoS cgroups while syncing pod: %v", err) + klog.V(2).Infof("Failed to update QoS cgroups while syncing pod: %v", err) } if err := pcm.EnsureExists(pod); err != nil { kl.recorder.Eventf(pod, v1.EventTypeWarning, events.FailedToCreatePodContainer, "unable to ensure pod container exists: %v", err) @@ -1618,9 +1638,9 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error { if mirrorPod.DeletionTimestamp != nil || !kl.podManager.IsMirrorPodOf(mirrorPod, pod) { // The mirror pod is semantically different from the static pod. Remove // it. The mirror pod will get recreated later. - glog.Warningf("Deleting mirror pod %q because it is outdated", format.Pod(mirrorPod)) + klog.Warningf("Deleting mirror pod %q because it is outdated", format.Pod(mirrorPod)) if err := kl.podManager.DeleteMirrorPod(podFullName); err != nil { - glog.Errorf("Failed deleting mirror pod %q: %v", format.Pod(mirrorPod), err) + klog.Errorf("Failed deleting mirror pod %q: %v", format.Pod(mirrorPod), err) } else { deleted = true } @@ -1629,11 +1649,11 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error { if mirrorPod == nil || deleted { node, err := kl.GetNode() if err != nil || node.DeletionTimestamp != nil { - glog.V(4).Infof("No need to create a mirror pod, since node %q has been removed from the cluster", kl.nodeName) + klog.V(4).Infof("No need to create a mirror pod, since node %q has been removed from the cluster", kl.nodeName) } else { - glog.V(4).Infof("Creating a mirror pod for static pod %q", format.Pod(pod)) + klog.V(4).Infof("Creating a mirror pod for static pod %q", format.Pod(pod)) if err := kl.podManager.CreateMirrorPod(pod); err != nil { - glog.Errorf("Failed creating a mirror pod for %q: %v", format.Pod(pod), err) + klog.Errorf("Failed creating a mirror pod for %q: %v", format.Pod(pod), err) } } } @@ -1642,7 +1662,7 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error { // Make data directories for the pod if err := kl.makePodDataDirs(pod); err != nil { kl.recorder.Eventf(pod, v1.EventTypeWarning, events.FailedToMakePodDataDirectories, "error making pod data directories: %v", err) - glog.Errorf("Unable to make pod data directories for pod %q: %v", format.Pod(pod), err) + klog.Errorf("Unable to make pod data directories for pod %q: %v", format.Pod(pod), err) return err } @@ -1651,7 +1671,7 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error { // Wait for volumes to attach/mount if err := kl.volumeManager.WaitForAttachAndMount(pod); err != nil { kl.recorder.Eventf(pod, v1.EventTypeWarning, events.FailedMountVolume, "Unable to mount volumes for pod %q: %v", format.Pod(pod), err) - glog.Errorf("Unable to mount volumes for pod %q: %v; skipping pod", format.Pod(pod), err) + klog.Errorf("Unable to mount volumes for pod %q: %v; skipping pod", format.Pod(pod), err) return err } } @@ -1800,8 +1820,8 @@ func (kl *Kubelet) canRunPod(pod *v1.Pod) lifecycle.PodAdmitResult { // no changes are seen to the configuration, will synchronize the last known desired // state every sync-frequency seconds. Never returns. func (kl *Kubelet) syncLoop(updates <-chan kubetypes.PodUpdate, handler SyncHandler) { - glog.Info("Starting kubelet main sync loop.") - // The resyncTicker wakes up kubelet to checks if there are any pod workers + klog.Info("Starting kubelet main sync loop.") + // The syncTicker wakes up kubelet to checks if there are any pod workers // that need to be sync'd. A one-second period is sufficient because the // sync interval is defaulted to 10s. syncTicker := time.NewTicker(time.Second) @@ -1817,7 +1837,7 @@ func (kl *Kubelet) syncLoop(updates <-chan kubetypes.PodUpdate, handler SyncHand duration := base for { if rs := kl.runtimeState.runtimeErrors(); len(rs) != 0 { - glog.Infof("skipping pod synchronization - %v", rs) + klog.Infof("skipping pod synchronization - %v", rs) // exponential backoff time.Sleep(duration) duration = time.Duration(math.Min(float64(max), factor*float64(duration))) @@ -1873,39 +1893,39 @@ func (kl *Kubelet) syncLoopIteration(configCh <-chan kubetypes.PodUpdate, handle // Update from a config source; dispatch it to the right handler // callback. if !open { - glog.Errorf("Update channel is closed. Exiting the sync loop.") + klog.Errorf("Update channel is closed. Exiting the sync loop.") return false } switch u.Op { case kubetypes.ADD: - glog.V(2).Infof("SyncLoop (ADD, %q): %q", u.Source, format.Pods(u.Pods)) + klog.V(2).Infof("SyncLoop (ADD, %q): %q", u.Source, format.Pods(u.Pods)) // After restarting, kubelet will get all existing pods through // ADD as if they are new pods. These pods will then go through the // admission process and *may* be rejected. This can be resolved // once we have checkpointing. handler.HandlePodAdditions(u.Pods) case kubetypes.UPDATE: - glog.V(2).Infof("SyncLoop (UPDATE, %q): %q", u.Source, format.PodsWithDeletionTimestamps(u.Pods)) + klog.V(2).Infof("SyncLoop (UPDATE, %q): %q", u.Source, format.PodsWithDeletionTimestamps(u.Pods)) handler.HandlePodUpdates(u.Pods) case kubetypes.REMOVE: - glog.V(2).Infof("SyncLoop (REMOVE, %q): %q", u.Source, format.Pods(u.Pods)) + klog.V(2).Infof("SyncLoop (REMOVE, %q): %q", u.Source, format.Pods(u.Pods)) handler.HandlePodRemoves(u.Pods) case kubetypes.RECONCILE: - glog.V(4).Infof("SyncLoop (RECONCILE, %q): %q", u.Source, format.Pods(u.Pods)) + klog.V(4).Infof("SyncLoop (RECONCILE, %q): %q", u.Source, format.Pods(u.Pods)) handler.HandlePodReconcile(u.Pods) case kubetypes.DELETE: - glog.V(2).Infof("SyncLoop (DELETE, %q): %q", u.Source, format.Pods(u.Pods)) + klog.V(2).Infof("SyncLoop (DELETE, %q): %q", u.Source, format.Pods(u.Pods)) // DELETE is treated as a UPDATE because of graceful deletion. handler.HandlePodUpdates(u.Pods) case kubetypes.RESTORE: - glog.V(2).Infof("SyncLoop (RESTORE, %q): %q", u.Source, format.Pods(u.Pods)) + klog.V(2).Infof("SyncLoop (RESTORE, %q): %q", u.Source, format.Pods(u.Pods)) // These are pods restored from the checkpoint. Treat them as new // pods. handler.HandlePodAdditions(u.Pods) case kubetypes.SET: // TODO: Do we want to support this? - glog.Errorf("Kubelet does not support snapshot update") + klog.Errorf("Kubelet does not support snapshot update") } if u.Op != kubetypes.RESTORE { @@ -1924,11 +1944,11 @@ func (kl *Kubelet) syncLoopIteration(configCh <-chan kubetypes.PodUpdate, handle if isSyncPodWorthy(e) { // PLEG event for a pod; sync it. if pod, ok := kl.podManager.GetPodByUID(e.ID); ok { - glog.V(2).Infof("SyncLoop (PLEG): %q, event: %#v", format.Pod(pod), e) + klog.V(2).Infof("SyncLoop (PLEG): %q, event: %#v", format.Pod(pod), e) handler.HandlePodSyncs([]*v1.Pod{pod}) } else { // If the pod no longer exists, ignore the event. - glog.V(4).Infof("SyncLoop (PLEG): ignore irrelevant event: %#v", e) + klog.V(4).Infof("SyncLoop (PLEG): ignore irrelevant event: %#v", e) } } @@ -1943,7 +1963,7 @@ func (kl *Kubelet) syncLoopIteration(configCh <-chan kubetypes.PodUpdate, handle if len(podsToSync) == 0 { break } - glog.V(4).Infof("SyncLoop (SYNC): %d pods; %s", len(podsToSync), format.Pods(podsToSync)) + klog.V(4).Infof("SyncLoop (SYNC): %d pods; %s", len(podsToSync), format.Pods(podsToSync)) handler.HandlePodSyncs(podsToSync) case update := <-kl.livenessManager.Updates(): if update.Result == proberesults.Failure { @@ -1954,21 +1974,21 @@ func (kl *Kubelet) syncLoopIteration(configCh <-chan kubetypes.PodUpdate, handle pod, ok := kl.podManager.GetPodByUID(update.PodUID) if !ok { // If the pod no longer exists, ignore the update. - glog.V(4).Infof("SyncLoop (container unhealthy): ignore irrelevant update: %#v", update) + klog.V(4).Infof("SyncLoop (container unhealthy): ignore irrelevant update: %#v", update) break } - glog.V(1).Infof("SyncLoop (container unhealthy): %q", format.Pod(pod)) + klog.V(1).Infof("SyncLoop (container unhealthy): %q", format.Pod(pod)) handler.HandlePodSyncs([]*v1.Pod{pod}) } case <-housekeepingCh: if !kl.sourcesReady.AllReady() { // If the sources aren't ready or volume manager has not yet synced the states, // skip housekeeping, as we may accidentally delete pods from unready sources. - glog.V(4).Infof("SyncLoop (housekeeping, skipped): sources aren't ready yet.") + klog.V(4).Infof("SyncLoop (housekeeping, skipped): sources aren't ready yet.") } else { - glog.V(4).Infof("SyncLoop (housekeeping)") + klog.V(4).Infof("SyncLoop (housekeeping)") if err := handler.HandlePodCleanups(); err != nil { - glog.Errorf("Failed cleaning pods: %v", err) + klog.Errorf("Failed cleaning pods: %v", err) } } } @@ -2091,7 +2111,7 @@ func (kl *Kubelet) HandlePodRemoves(pods []*v1.Pod) { // Deletion is allowed to fail because the periodic cleanup routine // will trigger deletion again. if err := kl.deletePod(pod); err != nil { - glog.V(2).Infof("Failed to delete pod %q, err: %v", format.Pod(pod), err) + klog.V(2).Infof("Failed to delete pod %q, err: %v", format.Pod(pod), err) } kl.probeManager.RemovePod(pod) } @@ -2150,20 +2170,20 @@ func (kl *Kubelet) updateRuntimeUp() { s, err := kl.containerRuntime.Status() if err != nil { - glog.Errorf("Container runtime sanity check failed: %v", err) + klog.Errorf("Container runtime sanity check failed: %v", err) return } if s == nil { - glog.Errorf("Container runtime status is nil") + klog.Errorf("Container runtime status is nil") return } // Periodically log the whole runtime status for debugging. // TODO(random-liu): Consider to send node event when optional // condition is unmet. - glog.V(4).Infof("Container runtime status: %v", s) + klog.V(4).Infof("Container runtime status: %v", s) networkReady := s.GetRuntimeCondition(kubecontainer.NetworkReady) if networkReady == nil || !networkReady.Status { - glog.Errorf("Container runtime network not ready: %v", networkReady) + klog.Errorf("Container runtime network not ready: %v", networkReady) kl.runtimeState.setNetworkState(fmt.Errorf("runtime network not ready: %v", networkReady)) } else { // Set nil if the container runtime network is ready. @@ -2175,7 +2195,7 @@ func (kl *Kubelet) updateRuntimeUp() { runtimeReady := s.GetRuntimeCondition(kubecontainer.RuntimeReady) // If RuntimeReady is not set or is false, report an error. if runtimeReady == nil || !runtimeReady.Status { - glog.Errorf("Container runtime not ready: %v", runtimeReady) + klog.Errorf("Container runtime not ready: %v", runtimeReady) return } kl.oneTimeInitializer.Do(kl.initializeRuntimeDependentModules) @@ -2208,6 +2228,11 @@ func (kl *Kubelet) ListenAndServeReadOnly(address net.IP, port uint) { server.ListenAndServeKubeletReadOnlyServer(kl, kl.resourceAnalyzer, address, port) } +// ListenAndServePodResources runs the kubelet podresources grpc service +func (kl *Kubelet) ListenAndServePodResources() { + server.ListenAndServePodResources(util.LocalEndpoint(kl.getPodResourcesDir(), podresources.Socket), kl.podManager, kl.containerManager) +} + // Delete the eligible dead container instances in a pod. Depending on the configuration, the latest dead containers may be kept around. func (kl *Kubelet) cleanUpContainersInPod(podID types.UID, exitedContainerID string) { if podStatus, err := kl.podCache.Get(podID); err == nil { @@ -2232,12 +2257,12 @@ func (kl *Kubelet) fastStatusUpdateOnce() { time.Sleep(100 * time.Millisecond) node, err := kl.GetNode() if err != nil { - glog.Errorf(err.Error()) + klog.Errorf(err.Error()) continue } if node.Spec.PodCIDR != "" { - if err := kl.updatePodCIDR(node.Spec.PodCIDR); err != nil { - glog.Errorf("Pod CIDR update failed %v", err) + if _, err := kl.updatePodCIDR(node.Spec.PodCIDR); err != nil { + klog.Errorf("Pod CIDR update failed %v", err) continue } kl.updateRuntimeUp() diff --git a/pkg/kubelet/kubelet_getters.go b/pkg/kubelet/kubelet_getters.go index 89077eb7f73..3e6bc43e9b3 100644 --- a/pkg/kubelet/kubelet_getters.go +++ b/pkg/kubelet/kubelet_getters.go @@ -22,8 +22,8 @@ import ( "net" "path/filepath" - "github.com/golang/glog" cadvisorapiv1 "github.com/google/cadvisor/info/v1" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -139,6 +139,11 @@ func (kl *Kubelet) getPodContainerDir(podUID types.UID, ctrName string) string { return filepath.Join(kl.getPodDir(podUID), config.DefaultKubeletContainersDirName, ctrName) } +// getPodResourcesSocket returns the full path to the directory containing the pod resources socket +func (kl *Kubelet) getPodResourcesDir() string { + return filepath.Join(kl.getRootDir(), config.DefaultKubeletPodResourcesDirName) +} + // GetPods returns all pods bound to the kubelet and their spec, and the mirror // pods. func (kl *Kubelet) GetPods() []*v1.Pod { @@ -261,13 +266,13 @@ func (kl *Kubelet) getPodVolumePathListFromDisk(podUID types.UID) ([]string, err if pathExists, pathErr := volumeutil.PathExists(podVolDir); pathErr != nil { return volumes, fmt.Errorf("Error checking if path %q exists: %v", podVolDir, pathErr) } else if !pathExists { - glog.Warningf("Path %q does not exist", podVolDir) + klog.Warningf("Path %q does not exist", podVolDir) return volumes, nil } volumePluginDirs, err := ioutil.ReadDir(podVolDir) if err != nil { - glog.Errorf("Could not read directory %s: %v", podVolDir, err) + klog.Errorf("Could not read directory %s: %v", podVolDir, err) return volumes, err } for _, volumePluginDir := range volumePluginDirs { diff --git a/pkg/kubelet/kubelet_network.go b/pkg/kubelet/kubelet_network.go index 0528d2c1e57..c604eddb4c1 100644 --- a/pkg/kubelet/kubelet_network.go +++ b/pkg/kubelet/kubelet_network.go @@ -19,8 +19,8 @@ package kubelet import ( "fmt" - "github.com/golang/glog" "k8s.io/api/core/v1" + "k8s.io/klog" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" utiliptables "k8s.io/kubernetes/pkg/util/iptables" ) @@ -55,26 +55,28 @@ func (kl *Kubelet) providerRequiresNetworkingConfiguration() bool { } // updatePodCIDR updates the pod CIDR in the runtime state if it is different -// from the current CIDR. -func (kl *Kubelet) updatePodCIDR(cidr string) error { +// from the current CIDR. Return true if pod CIDR is actually changed. +func (kl *Kubelet) updatePodCIDR(cidr string) (bool, error) { kl.updatePodCIDRMux.Lock() defer kl.updatePodCIDRMux.Unlock() podCIDR := kl.runtimeState.podCIDR() if podCIDR == cidr { - return nil + return false, nil } // kubelet -> generic runtime -> runtime shim -> network plugin // docker/non-cri implementations have a passthrough UpdatePodCIDR if err := kl.getRuntime().UpdatePodCIDR(cidr); err != nil { - return fmt.Errorf("failed to update pod CIDR: %v", err) + // If updatePodCIDR would fail, theoretically pod CIDR could not change. + // But it is better to be on the safe side to still return true here. + return true, fmt.Errorf("failed to update pod CIDR: %v", err) } - glog.Infof("Setting Pod CIDR: %v -> %v", podCIDR, cidr) + klog.Infof("Setting Pod CIDR: %v -> %v", podCIDR, cidr) kl.runtimeState.setPodCIDR(cidr) - return nil + return true, nil } // GetPodDNS returns DNS settings for the pod. diff --git a/pkg/kubelet/kubelet_network_linux.go b/pkg/kubelet/kubelet_network_linux.go index 002b226b190..ec7d41d9557 100644 --- a/pkg/kubelet/kubelet_network_linux.go +++ b/pkg/kubelet/kubelet_network_linux.go @@ -21,7 +21,7 @@ package kubelet import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" utiliptables "k8s.io/kubernetes/pkg/util/iptables" ) @@ -33,73 +33,73 @@ import ( // Marked connection will get SNAT on POSTROUTING Chain in nat table func (kl *Kubelet) syncNetworkUtil() { if kl.iptablesMasqueradeBit < 0 || kl.iptablesMasqueradeBit > 31 { - glog.Errorf("invalid iptables-masquerade-bit %v not in [0, 31]", kl.iptablesMasqueradeBit) + klog.Errorf("invalid iptables-masquerade-bit %v not in [0, 31]", kl.iptablesMasqueradeBit) return } if kl.iptablesDropBit < 0 || kl.iptablesDropBit > 31 { - glog.Errorf("invalid iptables-drop-bit %v not in [0, 31]", kl.iptablesDropBit) + klog.Errorf("invalid iptables-drop-bit %v not in [0, 31]", kl.iptablesDropBit) return } if kl.iptablesDropBit == kl.iptablesMasqueradeBit { - glog.Errorf("iptables-masquerade-bit %v and iptables-drop-bit %v must be different", kl.iptablesMasqueradeBit, kl.iptablesDropBit) + klog.Errorf("iptables-masquerade-bit %v and iptables-drop-bit %v must be different", kl.iptablesMasqueradeBit, kl.iptablesDropBit) return } // Setup KUBE-MARK-DROP rules dropMark := getIPTablesMark(kl.iptablesDropBit) if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkDropChain); err != nil { - glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkDropChain, err) + klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkDropChain, err) return } if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkDropChain, "-j", "MARK", "--set-xmark", dropMark); err != nil { - glog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkDropChain, err) + klog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkDropChain, err) return } if _, err := kl.iptClient.EnsureChain(utiliptables.TableFilter, KubeFirewallChain); err != nil { - glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableFilter, KubeFirewallChain, err) + klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableFilter, KubeFirewallChain, err) return } if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain, "-m", "comment", "--comment", "kubernetes firewall for dropping marked packets", "-m", "mark", "--mark", dropMark, "-j", "DROP"); err != nil { - glog.Errorf("Failed to ensure rule to drop packet marked by %v in %v chain %v: %v", KubeMarkDropChain, utiliptables.TableFilter, KubeFirewallChain, err) + klog.Errorf("Failed to ensure rule to drop packet marked by %v in %v chain %v: %v", KubeMarkDropChain, utiliptables.TableFilter, KubeFirewallChain, err) return } if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil { - glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainOutput, KubeFirewallChain, err) + klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainOutput, KubeFirewallChain, err) return } if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil { - glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainInput, KubeFirewallChain, err) + klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainInput, KubeFirewallChain, err) return } // Setup KUBE-MARK-MASQ rules masqueradeMark := getIPTablesMark(kl.iptablesMasqueradeBit) if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkMasqChain); err != nil { - glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkMasqChain, err) + klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkMasqChain, err) return } if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubePostroutingChain); err != nil { - glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubePostroutingChain, err) + klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubePostroutingChain, err) return } if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkMasqChain, "-j", "MARK", "--set-xmark", masqueradeMark); err != nil { - glog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkMasqChain, err) + klog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkMasqChain, err) return } if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableNAT, utiliptables.ChainPostrouting, "-m", "comment", "--comment", "kubernetes postrouting rules", "-j", string(KubePostroutingChain)); err != nil { - glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, KubePostroutingChain, err) + klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, KubePostroutingChain, err) return } if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, "-m", "comment", "--comment", "kubernetes service traffic requiring SNAT", "-m", "mark", "--mark", masqueradeMark, "-j", "MASQUERADE"); err != nil { - glog.Errorf("Failed to ensure SNAT rule for packets marked by %v in %v chain %v: %v", KubeMarkMasqChain, utiliptables.TableNAT, KubePostroutingChain, err) + klog.Errorf("Failed to ensure SNAT rule for packets marked by %v in %v chain %v: %v", KubeMarkMasqChain, utiliptables.TableNAT, KubePostroutingChain, err) return } } diff --git a/pkg/kubelet/kubelet_node_status.go b/pkg/kubelet/kubelet_node_status.go index 7e5dac7964d..d15e4167e38 100644 --- a/pkg/kubelet/kubelet_node_status.go +++ b/pkg/kubelet/kubelet_node_status.go @@ -21,11 +21,13 @@ import ( "fmt" "net" goruntime "runtime" + "sort" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -63,14 +65,14 @@ func (kl *Kubelet) registerWithAPIServer() { node, err := kl.initialNode() if err != nil { - glog.Errorf("Unable to construct v1.Node object for kubelet: %v", err) + klog.Errorf("Unable to construct v1.Node object for kubelet: %v", err) continue } - glog.Infof("Attempting to register node %s", node.Name) + klog.Infof("Attempting to register node %s", node.Name) registered := kl.tryRegisterWithAPIServer(node) if registered { - glog.Infof("Successfully registered node %s", node.Name) + klog.Infof("Successfully registered node %s", node.Name) kl.registrationCompleted = true return } @@ -89,27 +91,27 @@ func (kl *Kubelet) tryRegisterWithAPIServer(node *v1.Node) bool { } if !apierrors.IsAlreadyExists(err) { - glog.Errorf("Unable to register node %q with API server: %v", kl.nodeName, err) + klog.Errorf("Unable to register node %q with API server: %v", kl.nodeName, err) return false } existingNode, err := kl.kubeClient.CoreV1().Nodes().Get(string(kl.nodeName), metav1.GetOptions{}) if err != nil { - glog.Errorf("Unable to register node %q with API server: error getting existing node: %v", kl.nodeName, err) + klog.Errorf("Unable to register node %q with API server: error getting existing node: %v", kl.nodeName, err) return false } if existingNode == nil { - glog.Errorf("Unable to register node %q with API server: no node instance returned", kl.nodeName) + klog.Errorf("Unable to register node %q with API server: no node instance returned", kl.nodeName) return false } originalNode := existingNode.DeepCopy() if originalNode == nil { - glog.Errorf("Nil %q node object", kl.nodeName) + klog.Errorf("Nil %q node object", kl.nodeName) return false } - glog.Infof("Node %s was previously registered", kl.nodeName) + klog.Infof("Node %s was previously registered", kl.nodeName) // Edge case: the node was previously registered; reconcile // the value of the controller-managed attach-detach @@ -119,7 +121,7 @@ func (kl *Kubelet) tryRegisterWithAPIServer(node *v1.Node) bool { requiresUpdate = kl.reconcileExtendedResource(node, existingNode) || requiresUpdate if requiresUpdate { if _, _, err := nodeutil.PatchNodeStatus(kl.kubeClient.CoreV1(), types.NodeName(kl.nodeName), originalNode, existingNode); err != nil { - glog.Errorf("Unable to reconcile node %q with API server: error updating node: %v", kl.nodeName, err) + klog.Errorf("Unable to reconcile node %q with API server: error updating node: %v", kl.nodeName, err) return false } } @@ -191,10 +193,10 @@ func (kl *Kubelet) reconcileCMADAnnotationWithExistingNode(node, existingNode *v // not have the same value, update the existing node with // the correct value of the annotation. if !newSet { - glog.Info("Controller attach-detach setting changed to false; updating existing Node") + klog.Info("Controller attach-detach setting changed to false; updating existing Node") delete(existingNode.Annotations, volutil.ControllerManagedAttachAnnotation) } else { - glog.Info("Controller attach-detach setting changed to true; updating existing Node") + klog.Info("Controller attach-detach setting changed to true; updating existing Node") if existingNode.Annotations == nil { existingNode.Annotations = make(map[string]string) } @@ -273,24 +275,24 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) { node.Annotations = make(map[string]string) } - glog.Infof("Setting node annotation to enable volume controller attach/detach") + klog.Infof("Setting node annotation to enable volume controller attach/detach") node.Annotations[volutil.ControllerManagedAttachAnnotation] = "true" } else { - glog.Infof("Controller attach/detach is disabled for this node; Kubelet will attach and detach volumes") + klog.Infof("Controller attach/detach is disabled for this node; Kubelet will attach and detach volumes") } if kl.keepTerminatedPodVolumes { if node.Annotations == nil { node.Annotations = make(map[string]string) } - glog.Infof("Setting node annotation to keep pod volumes of terminated pods attached to the node") + klog.Infof("Setting node annotation to keep pod volumes of terminated pods attached to the node") node.Annotations[volutil.KeepTerminatedPodVolumesAnnotation] = "true" } // @question: should this be place after the call to the cloud provider? which also applies labels for k, v := range kl.nodeLabels { if cv, found := node.ObjectMeta.Labels[k]; found { - glog.Warningf("the node label %s=%s will overwrite default setting %s", k, v, cv) + klog.Warningf("the node label %s=%s will overwrite default setting %s", k, v, cv) } node.ObjectMeta.Labels[k] = v } @@ -321,7 +323,7 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) { return nil, err } if instanceType != "" { - glog.Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelInstanceType, instanceType) + klog.Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelInstanceType, instanceType) node.ObjectMeta.Labels[kubeletapis.LabelInstanceType] = instanceType } // If the cloud has zone information, label the node with the zone information @@ -332,11 +334,11 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) { return nil, fmt.Errorf("failed to get zone from cloud provider: %v", err) } if zone.FailureDomain != "" { - glog.Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelZoneFailureDomain, zone.FailureDomain) + klog.Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelZoneFailureDomain, zone.FailureDomain) node.ObjectMeta.Labels[kubeletapis.LabelZoneFailureDomain] = zone.FailureDomain } if zone.Region != "" { - glog.Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelZoneRegion, zone.Region) + klog.Infof("Adding node label from cloud provider: %s=%s", kubeletapis.LabelZoneRegion, zone.Region) node.ObjectMeta.Labels[kubeletapis.LabelZoneRegion] = zone.Region } } @@ -348,8 +350,8 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) { } // syncNodeStatus should be called periodically from a goroutine. -// It synchronizes node status to master, registering the kubelet first if -// necessary. +// It synchronizes node status to master if there is any change or enough time +// passed from the last sync, registering the kubelet first if necessary. func (kl *Kubelet) syncNodeStatus() { kl.syncNodeStatusMux.Lock() defer kl.syncNodeStatusMux.Unlock() @@ -362,19 +364,20 @@ func (kl *Kubelet) syncNodeStatus() { kl.registerWithAPIServer() } if err := kl.updateNodeStatus(); err != nil { - glog.Errorf("Unable to update node status: %v", err) + klog.Errorf("Unable to update node status: %v", err) } } -// updateNodeStatus updates node status to master with retries. +// updateNodeStatus updates node status to master with retries if there is any +// change or enough time passed from the last sync. func (kl *Kubelet) updateNodeStatus() error { - glog.V(5).Infof("Updating node status") + klog.V(5).Infof("Updating node status") for i := 0; i < nodeStatusUpdateRetry; i++ { if err := kl.tryUpdateNodeStatus(i); err != nil { if i > 0 && kl.onRepeatedHeartbeatFailure != nil { kl.onRepeatedHeartbeatFailure() } - glog.Errorf("Error updating node status, will retry: %v", err) + klog.Errorf("Error updating node status, will retry: %v", err) } else { return nil } @@ -382,7 +385,8 @@ func (kl *Kubelet) updateNodeStatus() error { return fmt.Errorf("update node status exceeds retry count") } -// tryUpdateNodeStatus tries to update node status to master. +// tryUpdateNodeStatus tries to update node status to master if there is any +// change or enough time passed from the last sync. func (kl *Kubelet) tryUpdateNodeStatus(tryNumber int) error { // In large clusters, GET and PUT operations on Node objects coming // from here are the majority of load on apiserver and etcd. @@ -404,18 +408,31 @@ func (kl *Kubelet) tryUpdateNodeStatus(tryNumber int) error { return fmt.Errorf("nil %q node object", kl.nodeName) } + podCIDRChanged := false if node.Spec.PodCIDR != "" { - if err := kl.updatePodCIDR(node.Spec.PodCIDR); err != nil { - glog.Errorf(err.Error()) + // Pod CIDR could have been updated before, so we cannot rely on + // node.Spec.PodCIDR being non-empty. We also need to know if pod CIDR is + // actually changed. + if podCIDRChanged, err = kl.updatePodCIDR(node.Spec.PodCIDR); err != nil { + klog.Errorf(err.Error()) } } kl.setNodeStatus(node) + + now := kl.clock.Now() + if utilfeature.DefaultFeatureGate.Enabled(features.NodeLease) && now.Before(kl.lastStatusReportTime.Add(kl.nodeStatusReportFrequency)) { + if !podCIDRChanged && !nodeStatusHasChanged(&originalNode.Status, &node.Status) { + return nil + } + } + // Patch the current status on the API server updatedNode, _, err := nodeutil.PatchNodeStatus(kl.heartbeatClient.CoreV1(), types.NodeName(kl.nodeName), originalNode, node) if err != nil { return err } + kl.lastStatusReportTime = now kl.setLastObservedNodeAddresses(updatedNode.Status.Addresses) // If update finishes successfully, mark the volumeInUse as reportedInUse to indicate // those volumes are already updated in the node's status @@ -426,7 +443,7 @@ func (kl *Kubelet) tryUpdateNodeStatus(tryNumber int) error { // recordNodeStatusEvent records an event of the given type with the given // message for the node. func (kl *Kubelet) recordNodeStatusEvent(eventType, event string) { - glog.V(2).Infof("Recording %s event message for node %s", event, kl.nodeName) + klog.V(2).Infof("Recording %s event message for node %s", event, kl.nodeName) // TODO: This requires a transaction, either both node status is updated // and event is recorded or neither should happen, see issue #6055. kl.recorder.Eventf(kl.nodeRef, eventType, event, "Node %s status is now: %s", kl.nodeName, event) @@ -458,9 +475,9 @@ func (kl *Kubelet) recordNodeSchedulableEvent(node *v1.Node) error { // refactor the node status condition code out to a different file. func (kl *Kubelet) setNodeStatus(node *v1.Node) { for i, f := range kl.setNodeStatusFuncs { - glog.V(5).Infof("Setting node status at position %v", i) + klog.V(5).Infof("Setting node status at position %v", i) if err := f(node); err != nil { - glog.Warningf("Failed to set some node status fields: %s", err) + klog.Warningf("Failed to set some node status fields: %s", err) } } } @@ -553,3 +570,53 @@ func validateNodeIP(nodeIP net.IP) error { } return fmt.Errorf("Node IP: %q not found in the host's network interfaces", nodeIP.String()) } + +// nodeStatusHasChanged compares the original node and current node's status and +// returns true if any change happens. The heartbeat timestamp is ignored. +func nodeStatusHasChanged(originalStatus *v1.NodeStatus, status *v1.NodeStatus) bool { + if originalStatus == nil && status == nil { + return false + } + if originalStatus == nil || status == nil { + return true + } + + // Compare node conditions here because we need to ignore the heartbeat timestamp. + if nodeConditionsHaveChanged(originalStatus.Conditions, status.Conditions) { + return true + } + + // Compare other fields of NodeStatus. + originalStatusCopy := originalStatus.DeepCopy() + statusCopy := status.DeepCopy() + originalStatusCopy.Conditions = nil + statusCopy.Conditions = nil + return !apiequality.Semantic.DeepEqual(originalStatusCopy, statusCopy) +} + +// nodeConditionsHaveChanged compares the original node and current node's +// conditions and returns true if any change happens. The heartbeat timestamp is +// ignored. +func nodeConditionsHaveChanged(originalConditions []v1.NodeCondition, conditions []v1.NodeCondition) bool { + if len(originalConditions) != len(conditions) { + return true + } + + originalConditionsCopy := make([]v1.NodeCondition, 0, len(originalConditions)) + originalConditionsCopy = append(originalConditionsCopy, originalConditions...) + conditionsCopy := make([]v1.NodeCondition, 0, len(conditions)) + conditionsCopy = append(conditionsCopy, conditions...) + + sort.SliceStable(originalConditionsCopy, func(i, j int) bool { return originalConditionsCopy[i].Type < originalConditionsCopy[j].Type }) + sort.SliceStable(conditionsCopy, func(i, j int) bool { return conditionsCopy[i].Type < conditionsCopy[j].Type }) + + replacedheartbeatTime := metav1.Time{} + for i := range conditionsCopy { + originalConditionsCopy[i].LastHeartbeatTime = replacedheartbeatTime + conditionsCopy[i].LastHeartbeatTime = replacedheartbeatTime + if !apiequality.Semantic.DeepEqual(&originalConditionsCopy[i], &conditionsCopy[i]) { + return true + } + } + return false +} diff --git a/pkg/kubelet/kubelet_node_status_test.go b/pkg/kubelet/kubelet_node_status_test.go index af73cd5f378..caeda4c0ff0 100644 --- a/pkg/kubelet/kubelet_node_status_test.go +++ b/pkg/kubelet/kubelet_node_status_test.go @@ -43,6 +43,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/rest" @@ -795,6 +796,239 @@ func TestUpdateNodeStatusError(t *testing.T) { assert.Len(t, testKubelet.fakeKubeClient.Actions(), nodeStatusUpdateRetry) } +func TestUpdateNodeStatusWithLease(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeLease, true)() + + testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) + defer testKubelet.Cleanup() + clock := testKubelet.fakeClock + kubelet := testKubelet.kubelet + kubelet.nodeStatusMaxImages = 5 // don't truncate the image list that gets constructed by hand for this test + kubelet.kubeClient = nil // ensure only the heartbeat client is used + kubelet.containerManager = &localCM{ + ContainerManager: cm.NewStubContainerManager(), + allocatableReservation: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(200, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(100E6, resource.BinarySI), + }, + capacity: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(20E9, resource.BinarySI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(5000, resource.BinarySI), + }, + } + // Since this test retroactively overrides the stub container manager, + // we have to regenerate default status setters. + kubelet.setNodeStatusFuncs = kubelet.defaultNodeStatusFuncs() + kubelet.nodeStatusReportFrequency = time.Minute + + kubeClient := testKubelet.fakeKubeClient + existingNode := &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}} + kubeClient.ReactionChain = fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{*existingNode}}).ReactionChain + machineInfo := &cadvisorapi.MachineInfo{ + MachineID: "123", + SystemUUID: "abc", + BootID: "1b3", + NumCores: 2, + MemoryCapacity: 20E9, + } + kubelet.machineInfo = machineInfo + + now := metav1.NewTime(clock.Now()).Rfc3339Copy() + expectedNode := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}, + Spec: v1.NodeSpec{}, + Status: v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + { + Type: v1.NodeMemoryPressure, + Status: v1.ConditionFalse, + Reason: "KubeletHasSufficientMemory", + Message: fmt.Sprintf("kubelet has sufficient memory available"), + LastHeartbeatTime: now, + LastTransitionTime: now, + }, + { + Type: v1.NodeDiskPressure, + Status: v1.ConditionFalse, + Reason: "KubeletHasNoDiskPressure", + Message: fmt.Sprintf("kubelet has no disk pressure"), + LastHeartbeatTime: now, + LastTransitionTime: now, + }, + { + Type: v1.NodePIDPressure, + Status: v1.ConditionFalse, + Reason: "KubeletHasSufficientPID", + Message: fmt.Sprintf("kubelet has sufficient PID available"), + LastHeartbeatTime: now, + LastTransitionTime: now, + }, + { + Type: v1.NodeReady, + Status: v1.ConditionTrue, + Reason: "KubeletReady", + Message: fmt.Sprintf("kubelet is posting ready status"), + LastHeartbeatTime: now, + LastTransitionTime: now, + }, + }, + NodeInfo: v1.NodeSystemInfo{ + MachineID: "123", + SystemUUID: "abc", + BootID: "1b3", + KernelVersion: cadvisortest.FakeKernelVersion, + OSImage: cadvisortest.FakeContainerOsVersion, + OperatingSystem: goruntime.GOOS, + Architecture: goruntime.GOARCH, + ContainerRuntimeVersion: "test://1.5.0", + KubeletVersion: version.Get().String(), + KubeProxyVersion: version.Get().String(), + }, + Capacity: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(20E9, resource.BinarySI), + v1.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(5000, resource.BinarySI), + }, + Allocatable: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(1800, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(19900E6, resource.BinarySI), + v1.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(5000, resource.BinarySI), + }, + Addresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "127.0.0.1"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + // images will be sorted from max to min in node status. + Images: []v1.ContainerImage{ + { + Names: []string{"k8s.gcr.io:v1", "k8s.gcr.io:v2"}, + SizeBytes: 123, + }, + { + Names: []string{"k8s.gcr.io:v3", "k8s.gcr.io:v4"}, + SizeBytes: 456, + }, + }, + }, + } + + // Update node status when node status is created. + // Report node status. + kubelet.updateRuntimeUp() + assert.NoError(t, kubelet.updateNodeStatus()) + + actions := kubeClient.Actions() + assert.Len(t, actions, 2) + assert.IsType(t, core.GetActionImpl{}, actions[0]) + assert.IsType(t, core.PatchActionImpl{}, actions[1]) + patchAction := actions[1].(core.PatchActionImpl) + + updatedNode, err := applyNodeStatusPatch(existingNode, patchAction.GetPatch()) + require.NoError(t, err) + for _, cond := range updatedNode.Status.Conditions { + cond.LastHeartbeatTime = cond.LastHeartbeatTime.Rfc3339Copy() + cond.LastTransitionTime = cond.LastTransitionTime.Rfc3339Copy() + } + assert.True(t, apiequality.Semantic.DeepEqual(expectedNode, updatedNode), "%s", diff.ObjectDiff(expectedNode, updatedNode)) + + // Version skew workaround. See: https://github.com/kubernetes/kubernetes/issues/16961 + assert.Equal(t, v1.NodeReady, updatedNode.Status.Conditions[len(updatedNode.Status.Conditions)-1].Type, + "NodeReady should be the last condition") + + // Update node status again when nothing is changed (except heatbeat time). + // Report node status if it has exceeded the duration of nodeStatusReportFrequency. + clock.Step(time.Minute) + assert.NoError(t, kubelet.updateNodeStatus()) + + // 2 more action (There were 2 actions before). + actions = kubeClient.Actions() + assert.Len(t, actions, 4) + assert.IsType(t, core.GetActionImpl{}, actions[2]) + assert.IsType(t, core.PatchActionImpl{}, actions[3]) + patchAction = actions[3].(core.PatchActionImpl) + + updatedNode, err = applyNodeStatusPatch(updatedNode, patchAction.GetPatch()) + require.NoError(t, err) + for _, cond := range updatedNode.Status.Conditions { + cond.LastHeartbeatTime = cond.LastHeartbeatTime.Rfc3339Copy() + cond.LastTransitionTime = cond.LastTransitionTime.Rfc3339Copy() + } + + // Expect LastHearbeat updated, other things unchanged. + for i, cond := range expectedNode.Status.Conditions { + expectedNode.Status.Conditions[i].LastHeartbeatTime = metav1.NewTime(cond.LastHeartbeatTime.Time.Add(time.Minute)).Rfc3339Copy() + } + assert.True(t, apiequality.Semantic.DeepEqual(expectedNode, updatedNode), "%s", diff.ObjectDiff(expectedNode, updatedNode)) + + // Update node status again when nothing is changed (except heatbeat time). + // Do not report node status if it is within the duration of nodeStatusReportFrequency. + clock.Step(10 * time.Second) + assert.NoError(t, kubelet.updateNodeStatus()) + + // Only 1 more action (There were 4 actions before). + actions = kubeClient.Actions() + assert.Len(t, actions, 5) + assert.IsType(t, core.GetActionImpl{}, actions[4]) + + // Update node status again when something is changed. + // Report node status even if it is still within the duration of nodeStatusReportFrequency. + clock.Step(10 * time.Second) + var newMemoryCapacity int64 = 40E9 + kubelet.machineInfo.MemoryCapacity = uint64(newMemoryCapacity) + assert.NoError(t, kubelet.updateNodeStatus()) + + // 2 more action (There were 5 actions before). + actions = kubeClient.Actions() + assert.Len(t, actions, 7) + assert.IsType(t, core.GetActionImpl{}, actions[5]) + assert.IsType(t, core.PatchActionImpl{}, actions[6]) + patchAction = actions[6].(core.PatchActionImpl) + + updatedNode, err = applyNodeStatusPatch(updatedNode, patchAction.GetPatch()) + require.NoError(t, err) + memCapacity, _ := updatedNode.Status.Capacity[v1.ResourceMemory] + updatedMemoryCapacity, _ := (&memCapacity).AsInt64() + assert.Equal(t, newMemoryCapacity, updatedMemoryCapacity, "Memory capacity") + + now = metav1.NewTime(clock.Now()).Rfc3339Copy() + for _, cond := range updatedNode.Status.Conditions { + // Expect LastHearbeat updated, while LastTransitionTime unchanged. + assert.Equal(t, now, cond.LastHeartbeatTime.Rfc3339Copy(), + "LastHeartbeatTime for condition %v", cond.Type) + assert.Equal(t, now, metav1.NewTime(cond.LastTransitionTime.Time.Add(time.Minute+20*time.Second)).Rfc3339Copy(), + "LastTransitionTime for condition %v", cond.Type) + } + + // Update node status when changing pod CIDR. + // Report node status if it is still within the duration of nodeStatusReportFrequency. + clock.Step(10 * time.Second) + assert.Equal(t, "", kubelet.runtimeState.podCIDR(), "Pod CIDR should be empty") + podCIDR := "10.0.0.0/24" + updatedNode.Spec.PodCIDR = podCIDR + kubeClient.ReactionChain = fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{*updatedNode}}).ReactionChain + assert.NoError(t, kubelet.updateNodeStatus()) + assert.Equal(t, podCIDR, kubelet.runtimeState.podCIDR(), "Pod CIDR should be updated now") + // 2 more action (There were 7 actions before). + actions = kubeClient.Actions() + assert.Len(t, actions, 9) + assert.IsType(t, core.GetActionImpl{}, actions[7]) + assert.IsType(t, core.PatchActionImpl{}, actions[8]) + patchAction = actions[8].(core.PatchActionImpl) + + // Update node status when keeping the pod CIDR. + // Do not report node status if it is within the duration of nodeStatusReportFrequency. + clock.Step(10 * time.Second) + assert.Equal(t, podCIDR, kubelet.runtimeState.podCIDR(), "Pod CIDR should already be updated") + assert.NoError(t, kubelet.updateNodeStatus()) + // Only 1 more action (There were 9 actions before). + actions = kubeClient.Actions() + assert.Len(t, actions, 10) + assert.IsType(t, core.GetActionImpl{}, actions[9]) +} + func TestRegisterWithApiServer(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() @@ -1529,3 +1763,159 @@ func TestRegisterWithApiServerWithTaint(t *testing.T) { return }) } + +func TestNodeStatusHasChanged(t *testing.T) { + fakeNow := metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC) + fakeFuture := metav1.Time{Time: fakeNow.Time.Add(time.Minute)} + readyCondition := v1.NodeCondition{ + Type: v1.NodeReady, + Status: v1.ConditionTrue, + LastHeartbeatTime: fakeNow, + LastTransitionTime: fakeNow, + } + readyConditionAtDiffHearbeatTime := v1.NodeCondition{ + Type: v1.NodeReady, + Status: v1.ConditionTrue, + LastHeartbeatTime: fakeFuture, + LastTransitionTime: fakeNow, + } + readyConditionAtDiffTransitionTime := v1.NodeCondition{ + Type: v1.NodeReady, + Status: v1.ConditionTrue, + LastHeartbeatTime: fakeFuture, + LastTransitionTime: fakeFuture, + } + notReadyCondition := v1.NodeCondition{ + Type: v1.NodeReady, + Status: v1.ConditionFalse, + LastHeartbeatTime: fakeNow, + LastTransitionTime: fakeNow, + } + memoryPressureCondition := v1.NodeCondition{ + Type: v1.NodeMemoryPressure, + Status: v1.ConditionFalse, + LastHeartbeatTime: fakeNow, + LastTransitionTime: fakeNow, + } + testcases := []struct { + name string + originalStatus *v1.NodeStatus + status *v1.NodeStatus + expectChange bool + }{ + { + name: "Node status does not change with nil status.", + originalStatus: nil, + status: nil, + expectChange: false, + }, + { + name: "Node status does not change with default status.", + originalStatus: &v1.NodeStatus{}, + status: &v1.NodeStatus{}, + expectChange: false, + }, + { + name: "Node status changes with nil and default status.", + originalStatus: nil, + status: &v1.NodeStatus{}, + expectChange: true, + }, + { + name: "Node status changes with nil and status.", + originalStatus: nil, + status: &v1.NodeStatus{ + Conditions: []v1.NodeCondition{readyCondition, memoryPressureCondition}, + }, + expectChange: true, + }, + { + name: "Node status does not change with empty conditions.", + originalStatus: &v1.NodeStatus{Conditions: []v1.NodeCondition{}}, + status: &v1.NodeStatus{Conditions: []v1.NodeCondition{}}, + expectChange: false, + }, + { + name: "Node status does not change", + originalStatus: &v1.NodeStatus{ + Conditions: []v1.NodeCondition{readyCondition, memoryPressureCondition}, + }, + status: &v1.NodeStatus{ + Conditions: []v1.NodeCondition{readyCondition, memoryPressureCondition}, + }, + expectChange: false, + }, + { + name: "Node status does not change even if heartbeat time changes.", + originalStatus: &v1.NodeStatus{ + Conditions: []v1.NodeCondition{readyCondition, memoryPressureCondition}, + }, + status: &v1.NodeStatus{ + Conditions: []v1.NodeCondition{readyConditionAtDiffHearbeatTime, memoryPressureCondition}, + }, + expectChange: false, + }, + { + name: "Node status does not change even if the orders of conditions are different.", + originalStatus: &v1.NodeStatus{ + Conditions: []v1.NodeCondition{readyCondition, memoryPressureCondition}, + }, + status: &v1.NodeStatus{ + Conditions: []v1.NodeCondition{memoryPressureCondition, readyConditionAtDiffHearbeatTime}, + }, + expectChange: false, + }, + { + name: "Node status changes if condition status differs.", + originalStatus: &v1.NodeStatus{ + Conditions: []v1.NodeCondition{readyCondition, memoryPressureCondition}, + }, + status: &v1.NodeStatus{ + Conditions: []v1.NodeCondition{notReadyCondition, memoryPressureCondition}, + }, + expectChange: true, + }, + { + name: "Node status changes if transition time changes.", + originalStatus: &v1.NodeStatus{ + Conditions: []v1.NodeCondition{readyCondition, memoryPressureCondition}, + }, + status: &v1.NodeStatus{ + Conditions: []v1.NodeCondition{readyConditionAtDiffTransitionTime, memoryPressureCondition}, + }, + expectChange: true, + }, + { + name: "Node status changes with different number of conditions.", + originalStatus: &v1.NodeStatus{ + Conditions: []v1.NodeCondition{readyCondition}, + }, + status: &v1.NodeStatus{ + Conditions: []v1.NodeCondition{readyCondition, memoryPressureCondition}, + }, + expectChange: true, + }, + { + name: "Node status changes with different phase.", + originalStatus: &v1.NodeStatus{ + Phase: v1.NodePending, + Conditions: []v1.NodeCondition{readyCondition}, + }, + status: &v1.NodeStatus{ + Phase: v1.NodeRunning, + Conditions: []v1.NodeCondition{readyCondition}, + }, + expectChange: true, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + originalStatusCopy := tc.originalStatus.DeepCopy() + statusCopy := tc.status.DeepCopy() + changed := nodeStatusHasChanged(tc.originalStatus, tc.status) + assert.Equal(t, tc.expectChange, changed, "Expect node status change to be %t, but got %t.", tc.expectChange, changed) + assert.True(t, apiequality.Semantic.DeepEqual(originalStatusCopy, tc.originalStatus), "%s", diff.ObjectDiff(originalStatusCopy, tc.originalStatus)) + assert.True(t, apiequality.Semantic.DeepEqual(statusCopy, tc.status), "%s", diff.ObjectDiff(statusCopy, tc.status)) + }) + } +} diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index 5216a7b2f42..59a2fd60bb4 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -32,7 +32,6 @@ import ( "strings" "sync" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -41,6 +40,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" utilvalidation "k8s.io/apimachinery/pkg/util/validation" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/api/v1/resource" podshelper "k8s.io/kubernetes/pkg/apis/core/pods" @@ -104,7 +104,7 @@ func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVol } vol, ok := podVolumes[device.Name] if !ok || vol.BlockVolumeMapper == nil { - glog.Errorf("Block volume cannot be satisfied for container %q, because the volume is missing or the volume mapper is nil: %+v", container.Name, device) + klog.Errorf("Block volume cannot be satisfied for container %q, because the volume is missing or the volume mapper is nil: %+v", container.Name, device) return nil, fmt.Errorf("cannot find volume %q to pass into container %q", device.Name, container.Name) } // Get a symbolic link associated to a block device under pod device path @@ -118,7 +118,7 @@ func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVol if vol.ReadOnly { permission = "r" } - glog.V(4).Infof("Device will be attached to container %q. Path on host: %v", container.Name, symlinkPath) + klog.V(4).Infof("Device will be attached to container %q. Path on host: %v", container.Name, symlinkPath) devices = append(devices, kubecontainer.DeviceInfo{PathOnHost: symlinkPath, PathInContainer: device.DevicePath, Permissions: permission}) } } @@ -135,7 +135,7 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h // Kubernetes will not mount /etc/hosts if: // - when the Pod sandbox is being created, its IP is still unknown. Hence, PodIP will not have been set. mountEtcHostsFile := len(podIP) > 0 && runtime.GOOS != "windows" - glog.V(3).Infof("container: %v/%v/%v podIP: %q creating hosts mount: %v", pod.Namespace, pod.Name, container.Name, podIP, mountEtcHostsFile) + klog.V(3).Infof("container: %v/%v/%v podIP: %q creating hosts mount: %v", pod.Namespace, pod.Name, container.Name, podIP, mountEtcHostsFile) mounts := []kubecontainer.Mount{} var cleanupAction func() for i, mount := range container.VolumeMounts { @@ -143,7 +143,7 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h mountEtcHostsFile = mountEtcHostsFile && (mount.MountPath != etcHostsPath) vol, ok := podVolumes[mount.Name] if !ok || vol.Mounter == nil { - glog.Errorf("Mount cannot be satisfied for container %q, because the volume is missing or the volume mounter is nil: %+v", container.Name, mount) + klog.Errorf("Mount cannot be satisfied for container %q, because the volume is missing or the volume mounter is nil: %+v", container.Name, mount) return nil, cleanupAction, fmt.Errorf("cannot find volume %q to mount into container %q", mount.Name, container.Name) } @@ -182,7 +182,7 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h hostPath = filepath.Join(volumePath, mount.SubPath) if subPathExists, err := mounter.ExistsPath(hostPath); err != nil { - glog.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", hostPath) + klog.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", hostPath) } else if !subPathExists { // Create the sub path now because if it's auto-created later when referenced, it may have an // incorrect ownership and mode. For example, the sub path directory must have at least g+rwx @@ -195,7 +195,7 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h } if err := mounter.SafeMakeDir(mount.SubPath, volumePath, perm); err != nil { // Don't pass detailed error back to the user because it could give information about host filesystem - glog.Errorf("failed to create subPath directory for volumeMount %q of container %q: %v", mount.Name, container.Name, err) + klog.Errorf("failed to create subPath directory for volumeMount %q of container %q: %v", mount.Name, container.Name, err) return nil, cleanupAction, fmt.Errorf("failed to create subPath directory for volumeMount %q of container %q", mount.Name, container.Name) } } @@ -209,7 +209,7 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h }) if err != nil { // Don't pass detailed error back to the user because it could give information about host filesystem - glog.Errorf("failed to prepare subPath for volumeMount %q of container %q: %v", mount.Name, container.Name, err) + klog.Errorf("failed to prepare subPath for volumeMount %q of container %q: %v", mount.Name, container.Name, err) return nil, cleanupAction, fmt.Errorf("failed to prepare subPath for volumeMount %q of container %q", mount.Name, container.Name) } } @@ -229,7 +229,7 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h if err != nil { return nil, cleanupAction, err } - glog.V(5).Infof("Pod %q container %q mount %q has propagation %q", format.Pod(pod), container.Name, mount.Name, propagation) + klog.V(5).Infof("Pod %q container %q mount %q has propagation %q", format.Pod(pod), container.Name, mount.Name, propagation) mustMountRO := vol.Mounter.GetAttributes().ReadOnly @@ -356,11 +356,9 @@ func hostsEntriesFromHostAliases(hostAliases []v1.HostAlias) []byte { var buffer bytes.Buffer buffer.WriteString("\n") buffer.WriteString("# Entries added by HostAliases.\n") - // write each IP/hostname pair as an entry into hosts file + // for each IP, write all aliases onto single line in hosts file for _, hostAlias := range hostAliases { - for _, hostname := range hostAlias.Hostnames { - buffer.WriteString(fmt.Sprintf("%s\t%s\n", hostAlias.IP, hostname)) - } + buffer.WriteString(fmt.Sprintf("%s\t%s\n", hostAlias.IP, strings.Join(hostAlias.Hostnames, "\t"))) } return buffer.Bytes() } @@ -373,7 +371,7 @@ func truncatePodHostnameIfNeeded(podName, hostname string) (string, error) { return hostname, nil } truncated := hostname[:hostnameMaxLen] - glog.Errorf("hostname for pod:%q was longer than %d. Truncated hostname to :%q", podName, hostnameMaxLen, truncated) + klog.Errorf("hostname for pod:%q was longer than %d. Truncated hostname to :%q", podName, hostnameMaxLen, truncated) // hostname should not end with '-' or '.' truncated = strings.TrimRight(truncated, "-.") if len(truncated) == 0 { @@ -465,7 +463,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai if len(container.TerminationMessagePath) != 0 && runtime.GOOS != "windows" { p := kl.getPodContainerDir(pod.UID, container.Name) if err := os.MkdirAll(p, 0750); err != nil { - glog.Errorf("Error on creating %q: %v", p, err) + klog.Errorf("Error on creating %q: %v", p, err) } else { opts.PodContainerDir = p } @@ -803,7 +801,7 @@ func (kl *Kubelet) killPod(pod *v1.Pod, runningPod *kubecontainer.Pod, status *k return err } if err := kl.containerManager.UpdateQOSCgroups(); err != nil { - glog.V(2).Infof("Failed to update QoS cgroups while killing pod: %v", err) + klog.V(2).Infof("Failed to update QoS cgroups while killing pod: %v", err) } return nil } @@ -831,7 +829,7 @@ func (kl *Kubelet) getPullSecretsForPod(pod *v1.Pod) []v1.Secret { for _, secretRef := range pod.Spec.ImagePullSecrets { secret, err := kl.secretManager.GetSecret(pod.Namespace, secretRef.Name) if err != nil { - glog.Warningf("Unable to retrieve pull secret %s/%s for %s/%s due to %v. The image pull may not succeed.", pod.Namespace, secretRef.Name, pod.Namespace, pod.Name, err) + klog.Warningf("Unable to retrieve pull secret %s/%s for %s/%s due to %v. The image pull may not succeed.", pod.Namespace, secretRef.Name, pod.Namespace, pod.Name, err) continue } @@ -885,13 +883,13 @@ func (kl *Kubelet) IsPodDeleted(uid types.UID) bool { func (kl *Kubelet) PodResourcesAreReclaimed(pod *v1.Pod, status v1.PodStatus) bool { if !notRunning(status.ContainerStatuses) { // We shouldnt delete pods that still have running containers - glog.V(3).Infof("Pod %q is terminated, but some containers are still running", format.Pod(pod)) + klog.V(3).Infof("Pod %q is terminated, but some containers are still running", format.Pod(pod)) return false } // pod's containers should be deleted runtimeStatus, err := kl.podCache.Get(pod.UID) if err != nil { - glog.V(3).Infof("Pod %q is terminated, Error getting runtimeStatus from the podCache: %s", format.Pod(pod), err) + klog.V(3).Infof("Pod %q is terminated, Error getting runtimeStatus from the podCache: %s", format.Pod(pod), err) return false } if len(runtimeStatus.ContainerStatuses) > 0 { @@ -899,18 +897,18 @@ func (kl *Kubelet) PodResourcesAreReclaimed(pod *v1.Pod, status v1.PodStatus) bo for _, status := range runtimeStatus.ContainerStatuses { statusStr += fmt.Sprintf("%+v ", *status) } - glog.V(3).Infof("Pod %q is terminated, but some containers have not been cleaned up: %s", format.Pod(pod), statusStr) + klog.V(3).Infof("Pod %q is terminated, but some containers have not been cleaned up: %s", format.Pod(pod), statusStr) return false } if kl.podVolumesExist(pod.UID) && !kl.keepTerminatedPodVolumes { // We shouldnt delete pods whose volumes have not been cleaned up if we are not keeping terminated pod volumes - glog.V(3).Infof("Pod %q is terminated, but some volumes have not been cleaned up", format.Pod(pod)) + klog.V(3).Infof("Pod %q is terminated, but some volumes have not been cleaned up", format.Pod(pod)) return false } if kl.kubeletConfiguration.CgroupsPerQOS { pcm := kl.containerManager.NewPodContainerManager() if pcm.Exists(pod) { - glog.V(3).Infof("Pod %q is terminated, but pod cgroup sandbox has not been cleaned up", format.Pod(pod)) + klog.V(3).Infof("Pod %q is terminated, but pod cgroup sandbox has not been cleaned up", format.Pod(pod)) return false } } @@ -1009,7 +1007,7 @@ func (kl *Kubelet) HandlePodCleanups() error { runningPods, err := kl.runtimeCache.GetPods() if err != nil { - glog.Errorf("Error listing containers: %#v", err) + klog.Errorf("Error listing containers: %#v", err) return err } for _, pod := range runningPods { @@ -1025,7 +1023,7 @@ func (kl *Kubelet) HandlePodCleanups() error { // TODO: Evaluate the performance impact of bypassing the runtime cache. runningPods, err = kl.containerRuntime.GetPods(false) if err != nil { - glog.Errorf("Error listing containers: %#v", err) + klog.Errorf("Error listing containers: %#v", err) return err } @@ -1038,7 +1036,7 @@ func (kl *Kubelet) HandlePodCleanups() error { // We want all cleanup tasks to be run even if one of them failed. So // we just log an error here and continue other cleanup tasks. // This also applies to the other clean up tasks. - glog.Errorf("Failed cleaning up orphaned pod directories: %v", err) + klog.Errorf("Failed cleaning up orphaned pod directories: %v", err) } // Remove any orphaned mirror pods. @@ -1072,10 +1070,10 @@ func (kl *Kubelet) podKiller() { if !exists { go func(apiPod *v1.Pod, runningPod *kubecontainer.Pod) { - glog.V(2).Infof("Killing unwanted pod %q", runningPod.Name) + klog.V(2).Infof("Killing unwanted pod %q", runningPod.Name) err := kl.killPod(apiPod, runningPod, nil, nil) if err != nil { - glog.Errorf("Failed killing the pod %q: %v", runningPod.Name, err) + klog.Errorf("Failed killing the pod %q: %v", runningPod.Name, err) } lock.Lock() killing.Delete(string(runningPod.ID)) @@ -1281,7 +1279,7 @@ func getPhase(spec *v1.PodSpec, info []v1.ContainerStatus) v1.PodPhase { case pendingInitialization > 0: fallthrough case waiting > 0: - glog.V(5).Infof("pod waiting > 0, pending") + klog.V(5).Infof("pod waiting > 0, pending") // One or more containers has not been started return v1.PodPending case running > 0 && unknown == 0: @@ -1308,7 +1306,7 @@ func getPhase(spec *v1.PodSpec, info []v1.ContainerStatus) v1.PodPhase { // and in the process of restarting return v1.PodRunning default: - glog.V(5).Infof("pod default case, pending") + klog.V(5).Infof("pod default case, pending") return v1.PodPending } } @@ -1316,7 +1314,7 @@ func getPhase(spec *v1.PodSpec, info []v1.ContainerStatus) v1.PodPhase { // generateAPIPodStatus creates the final API pod status for a pod, given the // internal pod status. func (kl *Kubelet) generateAPIPodStatus(pod *v1.Pod, podStatus *kubecontainer.PodStatus) v1.PodStatus { - glog.V(3).Infof("Generating status for %q", format.Pod(pod)) + klog.V(3).Infof("Generating status for %q", format.Pod(pod)) s := kl.convertStatusToAPIStatus(pod, podStatus) @@ -1338,7 +1336,7 @@ func (kl *Kubelet) generateAPIPodStatus(pod *v1.Pod, podStatus *kubecontainer.Po if pod.Status.Phase == v1.PodFailed || pod.Status.Phase == v1.PodSucceeded { // API server shows terminal phase; transitions are not allowed if s.Phase != pod.Status.Phase { - glog.Errorf("Pod attempted illegal phase transition from %s to %s: %v", pod.Status.Phase, s.Phase, s) + klog.Errorf("Pod attempted illegal phase transition from %s to %s: %v", pod.Status.Phase, s.Phase, s) // Force back to phase from the API server s.Phase = pod.Status.Phase } @@ -1358,7 +1356,7 @@ func (kl *Kubelet) generateAPIPodStatus(pod *v1.Pod, podStatus *kubecontainer.Po if kl.kubeClient != nil { hostIP, err := kl.getHostIPAnyWay() if err != nil { - glog.V(4).Infof("Cannot get host IP: %v", err) + klog.V(4).Infof("Cannot get host IP: %v", err) } else { s.HostIP = hostIP.String() if kubecontainer.IsHostNetworkPod(pod) && s.PodIP == "" { @@ -1665,13 +1663,13 @@ func (kl *Kubelet) cleanupOrphanedPodCgroups(cgroupPods map[types.UID]cm.CgroupN // process in the cgroup to the minimum value while we wait. if the kubelet // is configured to keep terminated volumes, we will delete the cgroup and not block. if podVolumesExist := kl.podVolumesExist(uid); podVolumesExist && !kl.keepTerminatedPodVolumes { - glog.V(3).Infof("Orphaned pod %q found, but volumes not yet removed. Reducing cpu to minimum", uid) + klog.V(3).Infof("Orphaned pod %q found, but volumes not yet removed. Reducing cpu to minimum", uid) if err := pcm.ReduceCPULimits(val); err != nil { - glog.Warningf("Failed to reduce cpu time for pod %q pending volume cleanup due to %v", uid, err) + klog.Warningf("Failed to reduce cpu time for pod %q pending volume cleanup due to %v", uid, err) } continue } - glog.V(3).Infof("Orphaned pod %q found, removing pod cgroups", uid) + klog.V(3).Infof("Orphaned pod %q found, removing pod cgroups", uid) // Destroy all cgroups of pod that should not be running, // by first killing all the attached processes to these cgroups. // We ignore errors thrown by the method, as the housekeeping loop would @@ -1734,13 +1732,13 @@ func (kl *Kubelet) hasHostMountPVC(pod *v1.Pod) bool { if volume.PersistentVolumeClaim != nil { pvc, err := kl.kubeClient.CoreV1().PersistentVolumeClaims(pod.Namespace).Get(volume.PersistentVolumeClaim.ClaimName, metav1.GetOptions{}) if err != nil { - glog.Warningf("unable to retrieve pvc %s:%s - %v", pod.Namespace, volume.PersistentVolumeClaim.ClaimName, err) + klog.Warningf("unable to retrieve pvc %s:%s - %v", pod.Namespace, volume.PersistentVolumeClaim.ClaimName, err) continue } if pvc != nil { referencedVolume, err := kl.kubeClient.CoreV1().PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{}) if err != nil { - glog.Warningf("unable to retrieve pv %s - %v", pvc.Spec.VolumeName, err) + klog.Warningf("unable to retrieve pv %s - %v", pvc.Spec.VolumeName, err) continue } if referencedVolume != nil && referencedVolume.Spec.HostPath != nil { diff --git a/pkg/kubelet/kubelet_pods_test.go b/pkg/kubelet/kubelet_pods_test.go index 3e34435adb8..0b4e0e02787 100644 --- a/pkg/kubelet/kubelet_pods_test.go +++ b/pkg/kubelet/kubelet_pods_test.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" core "k8s.io/client-go/testing" "k8s.io/client-go/tools/record" @@ -42,6 +43,7 @@ import ( // to "v1"? _ "k8s.io/kubernetes/pkg/apis/core/install" + "k8s.io/kubernetes/pkg/features" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" "k8s.io/kubernetes/pkg/kubelet/server/portforward" @@ -91,9 +93,7 @@ func TestDisabledSubpath(t *testing.T) { }, } - utilfeature.DefaultFeatureGate.Set("VolumeSubpath=false") - defer utilfeature.DefaultFeatureGate.Set("VolumeSubpath=true") - + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpath, false)() for name, test := range cases { _, _, err := makeMounts(&pod, "/pod", &test.container, "fakepodname", "", "", podVolumes, fm, nil) if err != nil && !test.expectError { @@ -183,9 +183,7 @@ fe00::2 ip6-allrouters 123.45.67.89 some.domain # Entries added by HostAliases. -123.45.67.89 foo -123.45.67.89 bar -123.45.67.89 baz +123.45.67.89 foo bar baz `, }, { @@ -214,12 +212,8 @@ fe00::2 ip6-allrouters 12.34.56.78 another.domain # Entries added by HostAliases. -123.45.67.89 foo -123.45.67.89 bar -123.45.67.89 baz -456.78.90.123 park -456.78.90.123 doo -456.78.90.123 boo +123.45.67.89 foo bar baz +456.78.90.123 park doo boo `, }, } @@ -300,9 +294,7 @@ fe00::2 ip6-allrouters 203.0.113.1 podFoo.domainFoo podFoo # Entries added by HostAliases. -123.45.67.89 foo -123.45.67.89 bar -123.45.67.89 baz +123.45.67.89 foo bar baz `, }, { @@ -323,12 +315,8 @@ fe00::2 ip6-allrouters 203.0.113.1 podFoo.domainFoo podFoo # Entries added by HostAliases. -123.45.67.89 foo -123.45.67.89 bar -123.45.67.89 baz -456.78.90.123 park -456.78.90.123 doo -456.78.90.123 boo +123.45.67.89 foo bar baz +456.78.90.123 park doo boo `, }, } diff --git a/pkg/kubelet/kubelet_resources.go b/pkg/kubelet/kubelet_resources.go index dc47a85880e..be6d29738ba 100644 --- a/pkg/kubelet/kubelet_resources.go +++ b/pkg/kubelet/kubelet_resources.go @@ -19,7 +19,7 @@ package kubelet import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/api/v1/resource" @@ -42,7 +42,7 @@ func (kl *Kubelet) defaultPodLimitsForDownwardAPI(pod *v1.Pod, container *v1.Con return nil, nil, fmt.Errorf("failed to find node object, expected a node") } allocatable := node.Status.Allocatable - glog.Infof("allocatable: %v", allocatable) + klog.Infof("allocatable: %v", allocatable) outputPod := pod.DeepCopy() for idx := range outputPod.Spec.Containers { resource.MergeContainerResourceLimits(&outputPod.Spec.Containers[idx], allocatable) diff --git a/pkg/kubelet/kubelet_volumes.go b/pkg/kubelet/kubelet_volumes.go index 09179fec804..7681ee6529e 100644 --- a/pkg/kubelet/kubelet_volumes.go +++ b/pkg/kubelet/kubelet_volumes.go @@ -19,11 +19,11 @@ package kubelet import ( "fmt" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/util/removeall" "k8s.io/kubernetes/pkg/volume" @@ -62,11 +62,11 @@ func (kl *Kubelet) podVolumesExist(podUID types.UID) bool { // There are some volume plugins such as flexvolume might not have mounts. See issue #61229 volumePaths, err := kl.getMountedVolumePathListFromDisk(podUID) if err != nil { - glog.Errorf("pod %q found, but error %v occurred during checking mounted volumes from disk", podUID, err) + klog.Errorf("pod %q found, but error %v occurred during checking mounted volumes from disk", podUID, err) return true } if len(volumePaths) > 0 { - glog.V(4).Infof("pod %q found, but volumes are still mounted on disk %v", podUID, volumePaths) + klog.V(4).Infof("pod %q found, but volumes are still mounted on disk %v", podUID, volumePaths) return true } @@ -85,7 +85,7 @@ func (kl *Kubelet) newVolumeMounterFromPlugins(spec *volume.Spec, pod *v1.Pod, o if err != nil { return nil, fmt.Errorf("failed to instantiate mounter for volume: %s using plugin: %s with a root cause: %v", spec.Name(), plugin.GetPluginName(), err) } - glog.V(10).Infof("Using volume plugin %q to mount %s", plugin.GetPluginName(), spec.Name()) + klog.V(10).Infof("Using volume plugin %q to mount %s", plugin.GetPluginName(), spec.Name()) return physicalMounter, nil } @@ -115,7 +115,7 @@ func (kl *Kubelet) cleanupOrphanedPodDirs(pods []*v1.Pod, runningPods []*kubecon // If volumes have not been unmounted/detached, do not delete directory. // Doing so may result in corruption of data. if podVolumesExist := kl.podVolumesExist(uid); podVolumesExist { - glog.V(3).Infof("Orphaned pod %q found, but volumes are not cleaned up", uid) + klog.V(3).Infof("Orphaned pod %q found, but volumes are not cleaned up", uid) continue } // If there are still volume directories, do not delete directory @@ -128,18 +128,18 @@ func (kl *Kubelet) cleanupOrphanedPodDirs(pods []*v1.Pod, runningPods []*kubecon orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("Orphaned pod %q found, but volume paths are still present on disk", uid)) continue } - glog.V(3).Infof("Orphaned pod %q found, removing", uid) + klog.V(3).Infof("Orphaned pod %q found, removing", uid) if err := removeall.RemoveAllOneFilesystem(kl.mounter, kl.getPodDir(uid)); err != nil { - glog.Errorf("Failed to remove orphaned pod %q dir; err: %v", uid, err) + klog.Errorf("Failed to remove orphaned pod %q dir; err: %v", uid, err) orphanRemovalErrors = append(orphanRemovalErrors, err) } } logSpew := func(errs []error) { if len(errs) > 0 { - glog.Errorf("%v : There were a total of %v errors similar to this. Turn up verbosity to see them.", errs[0], len(errs)) + klog.Errorf("%v : There were a total of %v errors similar to this. Turn up verbosity to see them.", errs[0], len(errs)) for _, err := range errs { - glog.V(5).Infof("Orphan pod: %v", err) + klog.V(5).Infof("Orphan pod: %v", err) } } } diff --git a/pkg/kubelet/kubeletconfig/BUILD b/pkg/kubelet/kubeletconfig/BUILD index 03e386f8086..55383942ff0 100644 --- a/pkg/kubelet/kubeletconfig/BUILD +++ b/pkg/kubelet/kubeletconfig/BUILD @@ -1,9 +1,6 @@ package(default_visibility = ["//visibility:public"]) -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -33,7 +30,7 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -60,3 +57,15 @@ filegroup( ], tags = ["automanaged"], ) + +go_test( + name = "go_default_test", + srcs = ["controller_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/kubelet/kubeletconfig/checkpoint:go_default_library", + "//pkg/kubelet/kubeletconfig/checkpoint/store:go_default_library", + "//pkg/kubelet/kubeletconfig/status:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + ], +) diff --git a/pkg/kubelet/kubeletconfig/checkpoint/store/fakestore.go b/pkg/kubelet/kubeletconfig/checkpoint/store/fakestore.go index 6e442cf2cc2..854294806b0 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/store/fakestore.go +++ b/pkg/kubelet/kubeletconfig/checkpoint/store/fakestore.go @@ -32,6 +32,11 @@ type fakeStore struct { var _ Store = (*fakeStore)(nil) +// NewFakeStore constructs a fake Store +func NewFakeStore() Store { + return &fakeStore{} +} + func (s *fakeStore) Initialize() error { return fmt.Errorf("Initialize method not supported") } diff --git a/pkg/kubelet/kubeletconfig/configsync.go b/pkg/kubelet/kubeletconfig/configsync.go index cb92fb7e654..7a897f3badf 100644 --- a/pkg/kubelet/kubeletconfig/configsync.go +++ b/pkg/kubelet/kubeletconfig/configsync.go @@ -21,7 +21,7 @@ import ( "os" "time" - "github.com/golang/glog" + "k8s.io/klog" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -197,7 +197,7 @@ func restartForNewConfig(eventClient v1core.EventsGetter, nodeName string, sourc // we directly log and send the event, instead of using the event recorder, // because the event recorder won't flush its queue before we exit (we'd lose the event) event := makeEvent(nodeName, apiv1.EventTypeNormal, KubeletConfigChangedEventReason, message) - glog.V(3).Infof("Event(%#v): type: '%v' reason: '%v' %v", event.InvolvedObject, event.Type, event.Reason, event.Message) + klog.V(3).Infof("Event(%#v): type: '%v' reason: '%v' %v", event.InvolvedObject, event.Type, event.Reason, event.Message) if _, err := eventClient.Events(apiv1.NamespaceDefault).Create(event); err != nil { utillog.Errorf("failed to send event, error: %v", err) } diff --git a/pkg/kubelet/kubeletconfig/controller.go b/pkg/kubelet/kubeletconfig/controller.go index 0dcc36c7523..50b65a67a3a 100644 --- a/pkg/kubelet/kubeletconfig/controller.go +++ b/pkg/kubelet/kubeletconfig/controller.go @@ -309,7 +309,7 @@ func (cc *Controller) graduateAssignedToLastKnownGood() error { } // if the sources are equal, no need to change if assigned == lastKnownGood || - assigned != nil && lastKnownGood != nil && apiequality.Semantic.DeepEqual(assigned, lastKnownGood) { + assigned != nil && lastKnownGood != nil && apiequality.Semantic.DeepEqual(assigned.NodeConfigSource(), lastKnownGood.NodeConfigSource()) { return nil } // update last-known-good diff --git a/pkg/kubelet/kubeletconfig/controller_test.go b/pkg/kubelet/kubeletconfig/controller_test.go new file mode 100644 index 00000000000..037a5163d1d --- /dev/null +++ b/pkg/kubelet/kubeletconfig/controller_test.go @@ -0,0 +1,81 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeletconfig + +import ( + "testing" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint" + "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store" + "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/status" +) + +func TestGraduateAssignedToLastKnownGood(t *testing.T) { + realSource1, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ + ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Namespace: "foo", + Name: "1", + KubeletConfigKey: "kubelet", + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + realSource2, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ + ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Namespace: "foo", + Name: "2", + KubeletConfigKey: "kubelet", + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + cases := []struct { + name string + assigned checkpoint.RemoteConfigSource + lkg checkpoint.RemoteConfigSource + }{ + { + name: "nil lkg to non-nil lkg", + assigned: realSource1, + lkg: nil, + }, + { + name: "non-nil lkg to non-nil lkg", + assigned: realSource2, + lkg: realSource1, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + controller := &Controller{ + configStatus: status.NewNodeConfigStatus(), + checkpointStore: store.NewFakeStore(), + } + controller.checkpointStore.SetLastKnownGood(tc.lkg) + controller.checkpointStore.SetAssigned(tc.assigned) + if err := controller.graduateAssignedToLastKnownGood(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} diff --git a/pkg/kubelet/kubeletconfig/util/log/BUILD b/pkg/kubelet/kubeletconfig/util/log/BUILD index 34b20a27e34..f47e2eead47 100644 --- a/pkg/kubelet/kubeletconfig/util/log/BUILD +++ b/pkg/kubelet/kubeletconfig/util/log/BUILD @@ -9,7 +9,7 @@ go_library( name = "go_default_library", srcs = ["log.go"], importpath = "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log", - deps = ["//vendor/github.com/golang/glog:go_default_library"], + deps = ["//vendor/k8s.io/klog:go_default_library"], ) filegroup( diff --git a/pkg/kubelet/kubeletconfig/util/log/log.go b/pkg/kubelet/kubeletconfig/util/log/log.go index b4ecfe4dc99..6e68b46a0fe 100644 --- a/pkg/kubelet/kubeletconfig/util/log/log.go +++ b/pkg/kubelet/kubeletconfig/util/log/log.go @@ -19,7 +19,7 @@ package log import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" ) const logFmt = "kubelet config controller: %s" @@ -33,7 +33,7 @@ func Errorf(format string, args ...interface{}) { } else { s = format } - glog.ErrorDepth(1, fmt.Sprintf(logFmt, s)) + klog.ErrorDepth(1, fmt.Sprintf(logFmt, s)) } // Infof shim that inserts "kubelet config controller" at the beginning of the log message, @@ -45,5 +45,5 @@ func Infof(format string, args ...interface{}) { } else { s = format } - glog.InfoDepth(1, fmt.Sprintf(logFmt, s)) + klog.InfoDepth(1, fmt.Sprintf(logFmt, s)) } diff --git a/pkg/kubelet/kuberuntime/BUILD b/pkg/kubelet/kuberuntime/BUILD index 67c04164816..361c3487115 100644 --- a/pkg/kubelet/kuberuntime/BUILD +++ b/pkg/kubelet/kuberuntime/BUILD @@ -66,9 +66,9 @@ go_library( "//staging/src/k8s.io/client-go/tools/reference:go_default_library", "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//vendor/github.com/armon/circbuf:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//pkg/kubelet/qos:go_default_library", @@ -95,6 +95,7 @@ go_test( "kuberuntime_sandbox_test.go", "labels_test.go", "legacy_test.go", + "main_test.go", "security_context_test.go", ], embed = [":go_default_library"], diff --git a/pkg/kubelet/kuberuntime/helpers.go b/pkg/kubelet/kuberuntime/helpers.go index fd977d5a16e..056e4d0ae75 100644 --- a/pkg/kubelet/kuberuntime/helpers.go +++ b/pkg/kubelet/kuberuntime/helpers.go @@ -22,10 +22,10 @@ import ( "strconv" "strings" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -83,7 +83,7 @@ func toRuntimeProtocol(protocol v1.Protocol) runtimeapi.Protocol { return runtimeapi.Protocol_SCTP } - glog.Warningf("Unknown protocol %q: defaulting to TCP", protocol) + klog.Warningf("Unknown protocol %q: defaulting to TCP", protocol) return runtimeapi.Protocol_TCP } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index 6e0b0ce045c..9407a03c63a 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -33,7 +33,7 @@ import ( "google.golang.org/grpc" "github.com/armon/circbuf" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -68,7 +68,7 @@ var ( func (m *kubeGenericRuntimeManager) recordContainerEvent(pod *v1.Pod, container *v1.Container, containerID, eventType, reason, message string, args ...interface{}) { ref, err := kubecontainer.GenerateContainerRef(pod, container) if err != nil { - glog.Errorf("Can't make a ref to pod %q, container %v: %v", format.Pod(pod), container.Name, err) + klog.Errorf("Can't make a ref to pod %q, container %v: %v", format.Pod(pod), container.Name, err) return } eventMessage := message @@ -101,9 +101,9 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb // Step 2: create the container. ref, err := kubecontainer.GenerateContainerRef(pod, container) if err != nil { - glog.Errorf("Can't make a ref to pod %q, container %v: %v", format.Pod(pod), container.Name, err) + klog.Errorf("Can't make a ref to pod %q, container %v: %v", format.Pod(pod), container.Name, err) } - glog.V(4).Infof("Generating ref for container %s: %#v", container.Name, ref) + klog.V(4).Infof("Generating ref for container %s: %#v", container.Name, ref) // For a new container, the RestartCount should be 0 restartCount := 0 @@ -162,7 +162,7 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb // to create it in the first place. it happens when journald logging driver is used with docker. if _, err := m.osInterface.Stat(containerLog); !os.IsNotExist(err) { if err := m.osInterface.Symlink(containerLog, legacySymlink); err != nil { - glog.Errorf("Failed to create legacy symbolic link %q to container %q log %q: %v", + klog.Errorf("Failed to create legacy symbolic link %q to container %q log %q: %v", legacySymlink, containerID, containerLog, err) } } @@ -177,7 +177,7 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb if handlerErr != nil { m.recordContainerEvent(pod, container, kubeContainerID.ID, v1.EventTypeWarning, events.FailedPostStartHook, msg) if err := m.killContainer(pod, kubeContainerID, container.Name, "FailedPostStartHook", nil); err != nil { - glog.Errorf("Failed to kill container %q(id=%q) in pod %q: %v, %v", + klog.Errorf("Failed to kill container %q(id=%q) in pod %q: %v, %v", container.Name, kubeContainerID.String(), format.Pod(pod), ErrPostStartHook, err) } return msg, fmt.Errorf("%s: %v", ErrPostStartHook, handlerErr) @@ -332,7 +332,7 @@ func (m *kubeGenericRuntimeManager) getKubeletContainers(allContainers bool) ([] containers, err := m.runtimeService.ListContainers(filter) if err != nil { - glog.Errorf("getKubeletContainers failed: %v", err) + klog.Errorf("getKubeletContainers failed: %v", err) return nil, err } @@ -385,7 +385,7 @@ func (m *kubeGenericRuntimeManager) getPodContainerStatuses(uid kubetypes.UID, n LabelSelector: map[string]string{types.KubernetesPodUIDLabel: string(uid)}, }) if err != nil { - glog.Errorf("ListContainers error: %v", err) + klog.Errorf("ListContainers error: %v", err) return nil, err } @@ -394,7 +394,7 @@ func (m *kubeGenericRuntimeManager) getPodContainerStatuses(uid kubetypes.UID, n for i, c := range containers { status, err := m.runtimeService.ContainerStatus(c.Id) if err != nil { - glog.Errorf("ContainerStatus for %s error: %v", c.Id, err) + klog.Errorf("ContainerStatus for %s error: %v", c.Id, err) return nil, err } cStatus := toKubeContainerStatus(status, m.runtimeName) @@ -461,7 +461,7 @@ func toKubeContainerStatus(status *runtimeapi.ContainerStatus, runtimeName strin // executePreStopHook runs the pre-stop lifecycle hooks if applicable and returns the duration it takes. func (m *kubeGenericRuntimeManager) executePreStopHook(pod *v1.Pod, containerID kubecontainer.ContainerID, containerSpec *v1.Container, gracePeriod int64) int64 { - glog.V(3).Infof("Running preStop hook for container %q", containerID.String()) + klog.V(3).Infof("Running preStop hook for container %q", containerID.String()) start := metav1.Now() done := make(chan struct{}) @@ -469,16 +469,16 @@ func (m *kubeGenericRuntimeManager) executePreStopHook(pod *v1.Pod, containerID defer close(done) defer utilruntime.HandleCrash() if msg, err := m.runner.Run(containerID, pod, containerSpec, containerSpec.Lifecycle.PreStop); err != nil { - glog.Errorf("preStop hook for container %q failed: %v", containerSpec.Name, err) + klog.Errorf("preStop hook for container %q failed: %v", containerSpec.Name, err) m.recordContainerEvent(pod, containerSpec, containerID.ID, v1.EventTypeWarning, events.FailedPreStopHook, msg) } }() select { case <-time.After(time.Duration(gracePeriod) * time.Second): - glog.V(2).Infof("preStop hook for container %q did not complete in %d seconds", containerID, gracePeriod) + klog.V(2).Infof("preStop hook for container %q did not complete in %d seconds", containerID, gracePeriod) case <-done: - glog.V(3).Infof("preStop hook for container %q completed", containerID) + klog.V(3).Infof("preStop hook for container %q completed", containerID) } return int64(metav1.Now().Sub(start.Time).Seconds()) @@ -556,7 +556,7 @@ func (m *kubeGenericRuntimeManager) killContainer(pod *v1.Pod, containerID kubec gracePeriod = *pod.Spec.TerminationGracePeriodSeconds } - glog.V(2).Infof("Killing container %q with %d second grace period", containerID.String(), gracePeriod) + klog.V(2).Infof("Killing container %q with %d second grace period", containerID.String(), gracePeriod) // Run internal pre-stop lifecycle hook if err := m.internalLifecycle.PreStopContainer(containerID.ID); err != nil { @@ -573,14 +573,14 @@ func (m *kubeGenericRuntimeManager) killContainer(pod *v1.Pod, containerID kubec } if gracePeriodOverride != nil { gracePeriod = *gracePeriodOverride - glog.V(3).Infof("Killing container %q, but using %d second grace period override", containerID, gracePeriod) + klog.V(3).Infof("Killing container %q, but using %d second grace period override", containerID, gracePeriod) } err := m.runtimeService.StopContainer(containerID.ID, gracePeriod) if err != nil { - glog.Errorf("Container %q termination failed with gracePeriod %d: %v", containerID.String(), gracePeriod, err) + klog.Errorf("Container %q termination failed with gracePeriod %d: %v", containerID.String(), gracePeriod, err) } else { - glog.V(3).Infof("Container %q exited normally", containerID.String()) + klog.V(3).Infof("Container %q exited normally", containerID.String()) } message := fmt.Sprintf("Killing container with id %s", containerID.String()) @@ -643,7 +643,7 @@ func (m *kubeGenericRuntimeManager) pruneInitContainersBeforeStart(pod *v1.Pod, continue } // prune all other init containers that match this container name - glog.V(4).Infof("Removing init container %q instance %q %d", status.Name, status.ID.ID, count) + klog.V(4).Infof("Removing init container %q instance %q %d", status.Name, status.ID.ID, count) if err := m.removeContainer(status.ID.ID); err != nil { utilruntime.HandleError(fmt.Errorf("failed to remove pod init container %q: %v; Skipping pod %q", status.Name, err, format.Pod(pod))) continue @@ -653,7 +653,7 @@ func (m *kubeGenericRuntimeManager) pruneInitContainersBeforeStart(pod *v1.Pod, if _, ok := m.containerRefManager.GetRef(status.ID); ok { m.containerRefManager.ClearRef(status.ID) } else { - glog.Warningf("No ref for container %q", status.ID) + klog.Warningf("No ref for container %q", status.ID) } } } @@ -675,7 +675,7 @@ func (m *kubeGenericRuntimeManager) purgeInitContainers(pod *v1.Pod, podStatus * } count++ // Purge all init containers that match this container name - glog.V(4).Infof("Removing init container %q instance %q %d", status.Name, status.ID.ID, count) + klog.V(4).Infof("Removing init container %q instance %q %d", status.Name, status.ID.ID, count) if err := m.removeContainer(status.ID.ID); err != nil { utilruntime.HandleError(fmt.Errorf("failed to remove pod init container %q: %v; Skipping pod %q", status.Name, err, format.Pod(pod))) continue @@ -684,7 +684,7 @@ func (m *kubeGenericRuntimeManager) purgeInitContainers(pod *v1.Pod, podStatus * if _, ok := m.containerRefManager.GetRef(status.ID); ok { m.containerRefManager.ClearRef(status.ID) } else { - glog.Warningf("No ref for container %q", status.ID) + klog.Warningf("No ref for container %q", status.ID) } } } @@ -739,7 +739,7 @@ func findNextInitContainerToRun(pod *v1.Pod, podStatus *kubecontainer.PodStatus) func (m *kubeGenericRuntimeManager) GetContainerLogs(ctx context.Context, pod *v1.Pod, containerID kubecontainer.ContainerID, logOptions *v1.PodLogOptions, stdout, stderr io.Writer) (err error) { status, err := m.runtimeService.ContainerStatus(containerID.ID) if err != nil { - glog.V(4).Infof("failed to get container status for %v: %v", containerID.String(), err) + klog.V(4).Infof("failed to get container status for %v: %v", containerID.String(), err) return fmt.Errorf("Unable to retrieve container logs for %v", containerID.String()) } return m.ReadLogs(ctx, status.GetLogPath(), containerID.ID, logOptions, stdout, stderr) @@ -795,7 +795,7 @@ func (m *kubeGenericRuntimeManager) RunInContainer(id kubecontainer.ContainerID, // Notice that we assume that the container should only be removed in non-running state, and // it will not write container logs anymore in that state. func (m *kubeGenericRuntimeManager) removeContainer(containerID string) error { - glog.V(4).Infof("Removing container %q", containerID) + klog.V(4).Infof("Removing container %q", containerID) // Call internal container post-stop lifecycle hook. if err := m.internalLifecycle.PostStopContainer(containerID); err != nil { return err diff --git a/pkg/kubelet/kuberuntime/kuberuntime_gc.go b/pkg/kubelet/kuberuntime/kuberuntime_gc.go index 3e0afbc45b6..e2ba00d0645 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_gc.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_gc.go @@ -23,9 +23,9 @@ import ( "sort" "time" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -123,7 +123,7 @@ func (cgc *containerGC) removeOldestN(containers []containerGCInfo, toRemove int numToKeep := len(containers) - toRemove for i := len(containers) - 1; i >= numToKeep; i-- { if err := cgc.manager.removeContainer(containers[i].id); err != nil { - glog.Errorf("Failed to remove container %q: %v", containers[i].id, err) + klog.Errorf("Failed to remove container %q: %v", containers[i].id, err) } } @@ -145,16 +145,16 @@ func (cgc *containerGC) removeOldestNSandboxes(sandboxes []sandboxGCInfo, toRemo // removeSandbox removes the sandbox by sandboxID. func (cgc *containerGC) removeSandbox(sandboxID string) { - glog.V(4).Infof("Removing sandbox %q", sandboxID) + klog.V(4).Infof("Removing sandbox %q", sandboxID) // In normal cases, kubelet should've already called StopPodSandbox before // GC kicks in. To guard against the rare cases where this is not true, try // stopping the sandbox before removing it. if err := cgc.client.StopPodSandbox(sandboxID); err != nil { - glog.Errorf("Failed to stop sandbox %q before removing: %v", sandboxID, err) + klog.Errorf("Failed to stop sandbox %q before removing: %v", sandboxID, err) return } if err := cgc.client.RemovePodSandbox(sandboxID); err != nil { - glog.Errorf("Failed to remove sandbox %q: %v", sandboxID, err) + klog.Errorf("Failed to remove sandbox %q: %v", sandboxID, err) } } @@ -328,7 +328,7 @@ func (cgc *containerGC) evictPodLogsDirectories(allSourcesReady bool) error { } err := osInterface.RemoveAll(filepath.Join(podLogsRootDirectory, name)) if err != nil { - glog.Errorf("Failed to remove pod logs directory %q: %v", name, err) + klog.Errorf("Failed to remove pod logs directory %q: %v", name, err) } } } @@ -340,7 +340,7 @@ func (cgc *containerGC) evictPodLogsDirectories(allSourcesReady bool) error { if _, err := osInterface.Stat(logSymlink); os.IsNotExist(err) { err := osInterface.Remove(logSymlink) if err != nil { - glog.Errorf("Failed to remove container log dead symlink %q: %v", logSymlink, err) + klog.Errorf("Failed to remove container log dead symlink %q: %v", logSymlink, err) } } } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_image.go b/pkg/kubelet/kuberuntime/kuberuntime_image.go index a8f2c1c060b..60fc6201c01 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_image.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_image.go @@ -17,9 +17,9 @@ limitations under the License. package kuberuntime import ( - "github.com/golang/glog" "k8s.io/api/core/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog" "k8s.io/kubernetes/pkg/credentialprovider" credentialprovidersecrets "k8s.io/kubernetes/pkg/credentialprovider/secrets" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" @@ -44,11 +44,11 @@ func (m *kubeGenericRuntimeManager) PullImage(image kubecontainer.ImageSpec, pul imgSpec := &runtimeapi.ImageSpec{Image: img} creds, withCredentials := keyring.Lookup(repoToPull) if !withCredentials { - glog.V(3).Infof("Pulling image %q without credentials", img) + klog.V(3).Infof("Pulling image %q without credentials", img) imageRef, err := m.imageService.PullImage(imgSpec, nil) if err != nil { - glog.Errorf("Pull image %q failed: %v", img, err) + klog.Errorf("Pull image %q failed: %v", img, err) return "", err } @@ -84,7 +84,7 @@ func (m *kubeGenericRuntimeManager) PullImage(image kubecontainer.ImageSpec, pul func (m *kubeGenericRuntimeManager) GetImageRef(image kubecontainer.ImageSpec) (string, error) { status, err := m.imageService.ImageStatus(&runtimeapi.ImageSpec{Image: image.Image}) if err != nil { - glog.Errorf("ImageStatus for image %q failed: %v", image, err) + klog.Errorf("ImageStatus for image %q failed: %v", image, err) return "", err } if status == nil { @@ -99,7 +99,7 @@ func (m *kubeGenericRuntimeManager) ListImages() ([]kubecontainer.Image, error) allImages, err := m.imageService.ListImages(nil) if err != nil { - glog.Errorf("ListImages failed: %v", err) + klog.Errorf("ListImages failed: %v", err) return nil, err } @@ -119,7 +119,7 @@ func (m *kubeGenericRuntimeManager) ListImages() ([]kubecontainer.Image, error) func (m *kubeGenericRuntimeManager) RemoveImage(image kubecontainer.ImageSpec) error { err := m.imageService.RemoveImage(&runtimeapi.ImageSpec{Image: image.Image}) if err != nil { - glog.Errorf("Remove image %q failed: %v", image.Image, err) + klog.Errorf("Remove image %q failed: %v", image.Image, err) return err } @@ -133,7 +133,7 @@ func (m *kubeGenericRuntimeManager) RemoveImage(image kubecontainer.ImageSpec) e func (m *kubeGenericRuntimeManager) ImageStats() (*kubecontainer.ImageStats, error) { allImages, err := m.imageService.ListImages(nil) if err != nil { - glog.Errorf("ListImages failed: %v", err) + klog.Errorf("ListImages failed: %v", err) return nil, err } stats := &kubecontainer.ImageStats{} diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index 42b5ff9f86b..90039beda09 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -22,8 +22,8 @@ import ( "os" "time" - "github.com/golang/glog" cadvisorapi "github.com/google/cadvisor/info/v1" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -181,21 +181,21 @@ func NewKubeGenericRuntimeManager( typedVersion, err := kubeRuntimeManager.runtimeService.Version(kubeRuntimeAPIVersion) if err != nil { - glog.Errorf("Get runtime version failed: %v", err) + klog.Errorf("Get runtime version failed: %v", err) return nil, err } // Only matching kubeRuntimeAPIVersion is supported now // TODO: Runtime API machinery is under discussion at https://github.com/kubernetes/kubernetes/issues/28642 if typedVersion.Version != kubeRuntimeAPIVersion { - glog.Errorf("Runtime api version %s is not supported, only %s is supported now", + klog.Errorf("Runtime api version %s is not supported, only %s is supported now", typedVersion.Version, kubeRuntimeAPIVersion) return nil, ErrVersionNotSupported } kubeRuntimeManager.runtimeName = typedVersion.RuntimeName - glog.Infof("Container runtime %s initialized, version: %s, apiVersion: %s", + klog.Infof("Container runtime %s initialized, version: %s, apiVersion: %s", typedVersion.RuntimeName, typedVersion.RuntimeVersion, typedVersion.RuntimeApiVersion) @@ -205,7 +205,7 @@ func NewKubeGenericRuntimeManager( // new runtime interface if _, err := osInterface.Stat(podLogsRootDirectory); os.IsNotExist(err) { if err := osInterface.MkdirAll(podLogsRootDirectory, 0755); err != nil { - glog.Errorf("Failed to create directory %q: %v", podLogsRootDirectory, err) + klog.Errorf("Failed to create directory %q: %v", podLogsRootDirectory, err) } } @@ -244,7 +244,7 @@ func newRuntimeVersion(version string) (*utilversion.Version, error) { func (m *kubeGenericRuntimeManager) getTypedVersion() (*runtimeapi.VersionResponse, error) { typedVersion, err := m.runtimeService.Version(kubeRuntimeAPIVersion) if err != nil { - glog.Errorf("Get remote runtime typed version failed: %v", err) + klog.Errorf("Get remote runtime typed version failed: %v", err) return nil, err } return typedVersion, nil @@ -254,7 +254,7 @@ func (m *kubeGenericRuntimeManager) getTypedVersion() (*runtimeapi.VersionRespon func (m *kubeGenericRuntimeManager) Version() (kubecontainer.Version, error) { typedVersion, err := m.runtimeService.Version(kubeRuntimeAPIVersion) if err != nil { - glog.Errorf("Get remote runtime version failed: %v", err) + klog.Errorf("Get remote runtime version failed: %v", err) return nil, err } @@ -296,7 +296,7 @@ func (m *kubeGenericRuntimeManager) GetPods(all bool) ([]*kubecontainer.Pod, err for i := range sandboxes { s := sandboxes[i] if s.Metadata == nil { - glog.V(4).Infof("Sandbox does not have metadata: %+v", s) + klog.V(4).Infof("Sandbox does not have metadata: %+v", s) continue } podUID := kubetypes.UID(s.Metadata.Uid) @@ -310,7 +310,7 @@ func (m *kubeGenericRuntimeManager) GetPods(all bool) ([]*kubecontainer.Pod, err p := pods[podUID] converted, err := m.sandboxToKubeContainer(s) if err != nil { - glog.V(4).Infof("Convert %q sandbox %v of pod %q failed: %v", m.runtimeName, s, podUID, err) + klog.V(4).Infof("Convert %q sandbox %v of pod %q failed: %v", m.runtimeName, s, podUID, err) continue } p.Sandboxes = append(p.Sandboxes, converted) @@ -323,7 +323,7 @@ func (m *kubeGenericRuntimeManager) GetPods(all bool) ([]*kubecontainer.Pod, err for i := range containers { c := containers[i] if c.Metadata == nil { - glog.V(4).Infof("Container does not have metadata: %+v", c) + klog.V(4).Infof("Container does not have metadata: %+v", c) continue } @@ -340,7 +340,7 @@ func (m *kubeGenericRuntimeManager) GetPods(all bool) ([]*kubecontainer.Pod, err converted, err := m.toKubeContainer(c) if err != nil { - glog.V(4).Infof("Convert %s container %v of pod %q failed: %v", m.runtimeName, c, labelledInfo.PodUID, err) + klog.V(4).Infof("Convert %s container %v of pod %q failed: %v", m.runtimeName, c, labelledInfo.PodUID, err) continue } @@ -394,7 +394,7 @@ type podActions struct { // (changed, new attempt, original sandboxID if exist). func (m *kubeGenericRuntimeManager) podSandboxChanged(pod *v1.Pod, podStatus *kubecontainer.PodStatus) (bool, uint32, string) { if len(podStatus.SandboxStatuses) == 0 { - glog.V(2).Infof("No sandbox for pod %q can be found. Need to start a new one", format.Pod(pod)) + klog.V(2).Infof("No sandbox for pod %q can be found. Need to start a new one", format.Pod(pod)) return true, 0, "" } @@ -408,23 +408,23 @@ func (m *kubeGenericRuntimeManager) podSandboxChanged(pod *v1.Pod, podStatus *ku // Needs to create a new sandbox when readySandboxCount > 1 or the ready sandbox is not the latest one. sandboxStatus := podStatus.SandboxStatuses[0] if readySandboxCount > 1 { - glog.V(2).Infof("More than 1 sandboxes for pod %q are ready. Need to reconcile them", format.Pod(pod)) + klog.V(2).Infof("More than 1 sandboxes for pod %q are ready. Need to reconcile them", format.Pod(pod)) return true, sandboxStatus.Metadata.Attempt + 1, sandboxStatus.Id } if sandboxStatus.State != runtimeapi.PodSandboxState_SANDBOX_READY { - glog.V(2).Infof("No ready sandbox for pod %q can be found. Need to start a new one", format.Pod(pod)) + klog.V(2).Infof("No ready sandbox for pod %q can be found. Need to start a new one", format.Pod(pod)) return true, sandboxStatus.Metadata.Attempt + 1, sandboxStatus.Id } // Needs to create a new sandbox when network namespace changed. if sandboxStatus.GetLinux().GetNamespaces().GetOptions().GetNetwork() != networkNamespaceForPod(pod) { - glog.V(2).Infof("Sandbox for pod %q has changed. Need to start a new one", format.Pod(pod)) + klog.V(2).Infof("Sandbox for pod %q has changed. Need to start a new one", format.Pod(pod)) return true, sandboxStatus.Metadata.Attempt + 1, "" } // Needs to create a new sandbox when the sandbox does not have an IP address. if !kubecontainer.IsHostNetworkPod(pod) && sandboxStatus.Network.Ip == "" { - glog.V(2).Infof("Sandbox for pod %q has no IP address. Need to start a new one", format.Pod(pod)) + klog.V(2).Infof("Sandbox for pod %q has no IP address. Need to start a new one", format.Pod(pod)) return true, sandboxStatus.Metadata.Attempt + 1, sandboxStatus.Id } @@ -450,7 +450,7 @@ func containerSucceeded(c *v1.Container, podStatus *kubecontainer.PodStatus) boo // computePodActions checks whether the pod spec has changed and returns the changes if true. func (m *kubeGenericRuntimeManager) computePodActions(pod *v1.Pod, podStatus *kubecontainer.PodStatus) podActions { - glog.V(5).Infof("Syncing Pod %q: %+v", format.Pod(pod), pod) + klog.V(5).Infof("Syncing Pod %q: %+v", format.Pod(pod), pod) createPodSandbox, attempt, sandboxID := m.podSandboxChanged(pod, podStatus) changes := podActions{ @@ -516,7 +516,7 @@ func (m *kubeGenericRuntimeManager) computePodActions(pod *v1.Pod, podStatus *ku // to it. if containerStatus != nil && containerStatus.State != kubecontainer.ContainerStateRunning { if err := m.internalLifecycle.PostStopContainer(containerStatus.ID.ID); err != nil { - glog.Errorf("internal container post-stop lifecycle hook failed for container %v in pod %v with error %v", + klog.Errorf("internal container post-stop lifecycle hook failed for container %v in pod %v with error %v", container.Name, pod.Name, err) } } @@ -526,7 +526,7 @@ func (m *kubeGenericRuntimeManager) computePodActions(pod *v1.Pod, podStatus *ku if containerStatus == nil || containerStatus.State != kubecontainer.ContainerStateRunning { if kubecontainer.ShouldContainerBeRestarted(&container, pod, podStatus) { message := fmt.Sprintf("Container %+v is dead, but RestartPolicy says that we should restart it.", container) - glog.V(3).Infof(message) + klog.V(3).Infof(message) changes.ContainersToStart = append(changes.ContainersToStart, idx) } continue @@ -562,7 +562,7 @@ func (m *kubeGenericRuntimeManager) computePodActions(pod *v1.Pod, podStatus *ku container: &pod.Spec.Containers[idx], message: message, } - glog.V(2).Infof("Container %q (%q) of pod %s: %s", container.Name, containerStatus.ID, format.Pod(pod), message) + klog.V(2).Infof("Container %q (%q) of pod %s: %s", container.Name, containerStatus.ID, format.Pod(pod), message) } if keepCount == 0 && len(changes.ContainersToStart) == 0 { @@ -583,31 +583,31 @@ func (m *kubeGenericRuntimeManager) computePodActions(pod *v1.Pod, podStatus *ku func (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, _ v1.PodStatus, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) { // Step 1: Compute sandbox and container changes. podContainerChanges := m.computePodActions(pod, podStatus) - glog.V(3).Infof("computePodActions got %+v for pod %q", podContainerChanges, format.Pod(pod)) + klog.V(3).Infof("computePodActions got %+v for pod %q", podContainerChanges, format.Pod(pod)) if podContainerChanges.CreateSandbox { ref, err := ref.GetReference(legacyscheme.Scheme, pod) if err != nil { - glog.Errorf("Couldn't make a ref to pod %q: '%v'", format.Pod(pod), err) + klog.Errorf("Couldn't make a ref to pod %q: '%v'", format.Pod(pod), err) } if podContainerChanges.SandboxID != "" { m.recorder.Eventf(ref, v1.EventTypeNormal, events.SandboxChanged, "Pod sandbox changed, it will be killed and re-created.") } else { - glog.V(4).Infof("SyncPod received new pod %q, will create a sandbox for it", format.Pod(pod)) + klog.V(4).Infof("SyncPod received new pod %q, will create a sandbox for it", format.Pod(pod)) } } // Step 2: Kill the pod if the sandbox has changed. if podContainerChanges.KillPod { if !podContainerChanges.CreateSandbox { - glog.V(4).Infof("Stopping PodSandbox for %q because all other containers are dead.", format.Pod(pod)) + klog.V(4).Infof("Stopping PodSandbox for %q because all other containers are dead.", format.Pod(pod)) } else { - glog.V(4).Infof("Stopping PodSandbox for %q, will start new one", format.Pod(pod)) + klog.V(4).Infof("Stopping PodSandbox for %q, will start new one", format.Pod(pod)) } killResult := m.killPodWithSyncResult(pod, kubecontainer.ConvertPodStatusToRunningPod(m.runtimeName, podStatus), nil) result.AddPodSyncResult(killResult) if killResult.Error() != nil { - glog.Errorf("killPodWithSyncResult failed: %v", killResult.Error()) + klog.Errorf("killPodWithSyncResult failed: %v", killResult.Error()) return } @@ -617,12 +617,12 @@ func (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, _ v1.PodStatus, podStat } else { // Step 3: kill any running containers in this pod which are not to keep. for containerID, containerInfo := range podContainerChanges.ContainersToKill { - glog.V(3).Infof("Killing unwanted container %q(id=%q) for pod %q", containerInfo.name, containerID, format.Pod(pod)) + klog.V(3).Infof("Killing unwanted container %q(id=%q) for pod %q", containerInfo.name, containerID, format.Pod(pod)) killContainerResult := kubecontainer.NewSyncResult(kubecontainer.KillContainer, containerInfo.name) result.AddSyncResult(killContainerResult) if err := m.killContainer(pod, containerID, containerInfo.name, containerInfo.message, nil); err != nil { killContainerResult.Fail(kubecontainer.ErrKillContainer, err.Error()) - glog.Errorf("killContainer %q(id=%q) for pod %q failed: %v", containerInfo.name, containerID, format.Pod(pod), err) + klog.Errorf("killContainer %q(id=%q) for pod %q failed: %v", containerInfo.name, containerID, format.Pod(pod), err) return } } @@ -653,30 +653,30 @@ func (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, _ v1.PodStatus, podStat var msg string var err error - glog.V(4).Infof("Creating sandbox for pod %q", format.Pod(pod)) + klog.V(4).Infof("Creating sandbox for pod %q", format.Pod(pod)) createSandboxResult := kubecontainer.NewSyncResult(kubecontainer.CreatePodSandbox, format.Pod(pod)) result.AddSyncResult(createSandboxResult) podSandboxID, msg, err = m.createPodSandbox(pod, podContainerChanges.Attempt) if err != nil { createSandboxResult.Fail(kubecontainer.ErrCreatePodSandbox, msg) - glog.Errorf("createPodSandbox for pod %q failed: %v", format.Pod(pod), err) + klog.Errorf("createPodSandbox for pod %q failed: %v", format.Pod(pod), err) ref, referr := ref.GetReference(legacyscheme.Scheme, pod) if referr != nil { - glog.Errorf("Couldn't make a ref to pod %q: '%v'", format.Pod(pod), referr) + klog.Errorf("Couldn't make a ref to pod %q: '%v'", format.Pod(pod), referr) } m.recorder.Eventf(ref, v1.EventTypeWarning, events.FailedCreatePodSandBox, "Failed create pod sandbox: %v", err) return } - glog.V(4).Infof("Created PodSandbox %q for pod %q", podSandboxID, format.Pod(pod)) + klog.V(4).Infof("Created PodSandbox %q for pod %q", podSandboxID, format.Pod(pod)) podSandboxStatus, err := m.runtimeService.PodSandboxStatus(podSandboxID) if err != nil { ref, referr := ref.GetReference(legacyscheme.Scheme, pod) if referr != nil { - glog.Errorf("Couldn't make a ref to pod %q: '%v'", format.Pod(pod), referr) + klog.Errorf("Couldn't make a ref to pod %q: '%v'", format.Pod(pod), referr) } m.recorder.Eventf(ref, v1.EventTypeWarning, events.FailedStatusPodSandBox, "Unable to get pod sandbox status: %v", err) - glog.Errorf("Failed to get pod sandbox status: %v; Skipping pod %q", err, format.Pod(pod)) + klog.Errorf("Failed to get pod sandbox status: %v; Skipping pod %q", err, format.Pod(pod)) result.Fail(err) return } @@ -686,7 +686,7 @@ func (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, _ v1.PodStatus, podStat if !kubecontainer.IsHostNetworkPod(pod) { // Overwrite the podIP passed in the pod status, since we just started the pod sandbox. podIP = m.determinePodSandboxIP(pod.Namespace, pod.Name, podSandboxStatus) - glog.V(4).Infof("Determined the ip %q for pod %q after sandbox changed", podIP, format.Pod(pod)) + klog.V(4).Infof("Determined the ip %q for pod %q after sandbox changed", podIP, format.Pod(pod)) } } @@ -696,7 +696,7 @@ func (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, _ v1.PodStatus, podStat podSandboxConfig, err := m.generatePodSandboxConfig(pod, podContainerChanges.Attempt) if err != nil { message := fmt.Sprintf("GeneratePodSandboxConfig for pod %q failed: %v", format.Pod(pod), err) - glog.Error(message) + klog.Error(message) configPodSandboxResult.Fail(kubecontainer.ErrConfigPodSandbox, message) return } @@ -709,11 +709,11 @@ func (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, _ v1.PodStatus, podStat isInBackOff, msg, err := m.doBackOff(pod, container, podStatus, backOff) if isInBackOff { startContainerResult.Fail(err, msg) - glog.V(4).Infof("Backing Off restarting init container %+v in pod %v", container, format.Pod(pod)) + klog.V(4).Infof("Backing Off restarting init container %+v in pod %v", container, format.Pod(pod)) return } - glog.V(4).Infof("Creating init container %+v in pod %v", container, format.Pod(pod)) + klog.V(4).Infof("Creating init container %+v in pod %v", container, format.Pod(pod)) if msg, err := m.startContainer(podSandboxID, podSandboxConfig, container, pod, podStatus, pullSecrets, podIP, kubecontainer.ContainerTypeInit); err != nil { startContainerResult.Fail(err, msg) utilruntime.HandleError(fmt.Errorf("init container start failed: %v: %s", err, msg)) @@ -721,7 +721,7 @@ func (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, _ v1.PodStatus, podStat } // Successfully started the container; clear the entry in the failure - glog.V(4).Infof("Completed init container %q for pod %q", container.Name, format.Pod(pod)) + klog.V(4).Infof("Completed init container %q for pod %q", container.Name, format.Pod(pod)) } // Step 6: start containers in podContainerChanges.ContainersToStart. @@ -733,18 +733,18 @@ func (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, _ v1.PodStatus, podStat isInBackOff, msg, err := m.doBackOff(pod, container, podStatus, backOff) if isInBackOff { startContainerResult.Fail(err, msg) - glog.V(4).Infof("Backing Off restarting container %+v in pod %v", container, format.Pod(pod)) + klog.V(4).Infof("Backing Off restarting container %+v in pod %v", container, format.Pod(pod)) continue } - glog.V(4).Infof("Creating container %+v in pod %v", container, format.Pod(pod)) + klog.V(4).Infof("Creating container %+v in pod %v", container, format.Pod(pod)) if msg, err := m.startContainer(podSandboxID, podSandboxConfig, container, pod, podStatus, pullSecrets, podIP, kubecontainer.ContainerTypeRegular); err != nil { startContainerResult.Fail(err, msg) // known errors that are logged in other places are logged at higher levels here to avoid // repetitive log spam switch { case err == images.ErrImagePullBackOff: - glog.V(3).Infof("container start failed: %v: %s", err, msg) + klog.V(3).Infof("container start failed: %v: %s", err, msg) default: utilruntime.HandleError(fmt.Errorf("container start failed: %v: %s", err, msg)) } @@ -770,7 +770,7 @@ func (m *kubeGenericRuntimeManager) doBackOff(pod *v1.Pod, container *v1.Contain return false, "", nil } - glog.V(3).Infof("checking backoff for container %q in pod %q", container.Name, format.Pod(pod)) + klog.V(3).Infof("checking backoff for container %q in pod %q", container.Name, format.Pod(pod)) // Use the finished time of the latest exited container as the start point to calculate whether to do back-off. ts := cStatus.FinishedAt // backOff requires a unique key to identify the container. @@ -780,7 +780,7 @@ func (m *kubeGenericRuntimeManager) doBackOff(pod *v1.Pod, container *v1.Contain m.recorder.Eventf(ref, v1.EventTypeWarning, events.BackOffStartContainer, "Back-off restarting failed container") } err := fmt.Errorf("Back-off %s restarting failed container=%s pod=%s", backOff.Get(key), container.Name, format.Pod(pod)) - glog.V(3).Infof("%s", err.Error()) + klog.V(3).Infof("%s", err.Error()) return true, err.Error(), kubecontainer.ErrCrashLoopBackOff } @@ -812,7 +812,7 @@ func (m *kubeGenericRuntimeManager) killPodWithSyncResult(pod *v1.Pod, runningPo for _, podSandbox := range runningPod.Sandboxes { if err := m.runtimeService.StopPodSandbox(podSandbox.ID.ID); err != nil { killSandboxResult.Fail(kubecontainer.ErrKillPodSandbox, err.Error()) - glog.Errorf("Failed to stop sandbox %q", podSandbox.ID) + klog.Errorf("Failed to stop sandbox %q", podSandbox.ID) } } @@ -847,14 +847,14 @@ func (m *kubeGenericRuntimeManager) GetPodStatus(uid kubetypes.UID, name, namesp UID: uid, }, }) - glog.V(4).Infof("getSandboxIDByPodUID got sandbox IDs %q for pod %q", podSandboxIDs, podFullName) + klog.V(4).Infof("getSandboxIDByPodUID got sandbox IDs %q for pod %q", podSandboxIDs, podFullName) sandboxStatuses := make([]*runtimeapi.PodSandboxStatus, len(podSandboxIDs)) podIP := "" for idx, podSandboxID := range podSandboxIDs { podSandboxStatus, err := m.runtimeService.PodSandboxStatus(podSandboxID) if err != nil { - glog.Errorf("PodSandboxStatus of sandbox %q for pod %q error: %v", podSandboxID, podFullName, err) + klog.Errorf("PodSandboxStatus of sandbox %q for pod %q error: %v", podSandboxID, podFullName, err) return nil, err } sandboxStatuses[idx] = podSandboxStatus @@ -868,7 +868,7 @@ func (m *kubeGenericRuntimeManager) GetPodStatus(uid kubetypes.UID, name, namesp // Get statuses of all containers visible in the pod. containerStatuses, err := m.getPodContainerStatuses(uid, name, namespace) if err != nil { - glog.Errorf("getPodContainerStatuses for pod %q failed: %v", podFullName, err) + klog.Errorf("getPodContainerStatuses for pod %q failed: %v", podFullName, err) return nil, err } @@ -899,7 +899,7 @@ func (m *kubeGenericRuntimeManager) GarbageCollect(gcPolicy kubecontainer.Contai func (m *kubeGenericRuntimeManager) GetPodContainerID(pod *kubecontainer.Pod) (kubecontainer.ContainerID, error) { formattedPod := kubecontainer.FormatPod(pod) if len(pod.Sandboxes) == 0 { - glog.Errorf("No sandboxes are found for pod %q", formattedPod) + klog.Errorf("No sandboxes are found for pod %q", formattedPod) return kubecontainer.ContainerID{}, fmt.Errorf("sandboxes for pod %q not found", formattedPod) } @@ -912,7 +912,7 @@ func (m *kubeGenericRuntimeManager) GetPodContainerID(pod *kubecontainer.Pod) (k func (m *kubeGenericRuntimeManager) UpdatePodCIDR(podCIDR string) error { // TODO(#35531): do we really want to write a method on this manager for each // field of the config? - glog.Infof("updating runtime config through cri with podcidr %v", podCIDR) + klog.Infof("updating runtime config through cri with podcidr %v", podCIDR) return m.runtimeService.UpdateRuntimeConfig( &runtimeapi.RuntimeConfig{ NetworkConfig: &runtimeapi.NetworkConfig{ diff --git a/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go b/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go index c9c8fd28970..dd51bda32ca 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go @@ -22,10 +22,10 @@ import ( "net/url" "sort" - "github.com/golang/glog" "k8s.io/api/core/v1" kubetypes "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -38,7 +38,7 @@ func (m *kubeGenericRuntimeManager) createPodSandbox(pod *v1.Pod, attempt uint32 podSandboxConfig, err := m.generatePodSandboxConfig(pod, attempt) if err != nil { message := fmt.Sprintf("GeneratePodSandboxConfig for pod %q failed: %v", format.Pod(pod), err) - glog.Error(message) + klog.Error(message) return "", message, err } @@ -46,7 +46,7 @@ func (m *kubeGenericRuntimeManager) createPodSandbox(pod *v1.Pod, attempt uint32 err = m.osInterface.MkdirAll(podSandboxConfig.LogDirectory, 0755) if err != nil { message := fmt.Sprintf("Create pod log directory for pod %q failed: %v", format.Pod(pod), err) - glog.Errorf(message) + klog.Errorf(message) return "", message, err } @@ -62,7 +62,7 @@ func (m *kubeGenericRuntimeManager) createPodSandbox(pod *v1.Pod, attempt uint32 podSandBoxID, err := m.runtimeService.RunPodSandbox(podSandboxConfig, runtimeHandler) if err != nil { message := fmt.Sprintf("CreatePodSandbox for pod %q failed: %v", format.Pod(pod), err) - glog.Error(message) + klog.Error(message) return "", message, err } @@ -204,7 +204,7 @@ func (m *kubeGenericRuntimeManager) getKubeletSandboxes(all bool) ([]*runtimeapi resp, err := m.runtimeService.ListPodSandbox(filter) if err != nil { - glog.Errorf("ListPodSandbox failed: %v", err) + klog.Errorf("ListPodSandbox failed: %v", err) return nil, err } @@ -214,14 +214,14 @@ func (m *kubeGenericRuntimeManager) getKubeletSandboxes(all bool) ([]*runtimeapi // determinePodSandboxIP determines the IP address of the given pod sandbox. func (m *kubeGenericRuntimeManager) determinePodSandboxIP(podNamespace, podName string, podSandbox *runtimeapi.PodSandboxStatus) string { if podSandbox.Network == nil { - glog.Warningf("Pod Sandbox status doesn't have network information, cannot report IP") + klog.Warningf("Pod Sandbox status doesn't have network information, cannot report IP") return "" } ip := podSandbox.Network.Ip if len(ip) != 0 && net.ParseIP(ip) == nil { // ip could be an empty string if runtime is not responsible for the // IP (e.g., host networking). - glog.Warningf("Pod Sandbox reported an unparseable IP %v", ip) + klog.Warningf("Pod Sandbox reported an unparseable IP %v", ip) return "" } return ip @@ -240,7 +240,7 @@ func (m *kubeGenericRuntimeManager) getSandboxIDByPodUID(podUID kubetypes.UID, s } sandboxes, err := m.runtimeService.ListPodSandbox(filter) if err != nil { - glog.Errorf("ListPodSandbox with pod UID %q failed: %v", podUID, err) + klog.Errorf("ListPodSandbox with pod UID %q failed: %v", podUID, err) return nil, err } diff --git a/pkg/kubelet/kuberuntime/labels.go b/pkg/kubelet/kuberuntime/labels.go index 9adc0205a26..5f36e0b5d98 100644 --- a/pkg/kubelet/kuberuntime/labels.go +++ b/pkg/kubelet/kuberuntime/labels.go @@ -20,10 +20,10 @@ import ( "encoding/json" "strconv" - "github.com/golang/glog" "k8s.io/api/core/v1" kubetypes "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/types" @@ -135,7 +135,7 @@ func newContainerAnnotations(container *v1.Container, pod *v1.Pod, restartCount // Using json encoding so that the PreStop handler object is readable after writing as a label rawPreStop, err := json.Marshal(container.Lifecycle.PreStop) if err != nil { - glog.Errorf("Unable to marshal lifecycle PreStop handler for container %q of pod %q: %v", container.Name, format.Pod(pod), err) + klog.Errorf("Unable to marshal lifecycle PreStop handler for container %q of pod %q: %v", container.Name, format.Pod(pod), err) } else { annotations[containerPreStopHandlerLabel] = string(rawPreStop) } @@ -144,7 +144,7 @@ func newContainerAnnotations(container *v1.Container, pod *v1.Pod, restartCount if len(container.Ports) > 0 { rawContainerPorts, err := json.Marshal(container.Ports) if err != nil { - glog.Errorf("Unable to marshal container ports for container %q for pod %q: %v", container.Name, format.Pod(pod), err) + klog.Errorf("Unable to marshal container ports for container %q for pod %q: %v", container.Name, format.Pod(pod), err) } else { annotations[containerPortsLabel] = string(rawContainerPorts) } @@ -203,28 +203,28 @@ func getContainerInfoFromAnnotations(annotations map[string]string) *annotatedCo } if containerInfo.Hash, err = getUint64ValueFromLabel(annotations, containerHashLabel); err != nil { - glog.Errorf("Unable to get %q from annotations %q: %v", containerHashLabel, annotations, err) + klog.Errorf("Unable to get %q from annotations %q: %v", containerHashLabel, annotations, err) } if containerInfo.RestartCount, err = getIntValueFromLabel(annotations, containerRestartCountLabel); err != nil { - glog.Errorf("Unable to get %q from annotations %q: %v", containerRestartCountLabel, annotations, err) + klog.Errorf("Unable to get %q from annotations %q: %v", containerRestartCountLabel, annotations, err) } if containerInfo.PodDeletionGracePeriod, err = getInt64PointerFromLabel(annotations, podDeletionGracePeriodLabel); err != nil { - glog.Errorf("Unable to get %q from annotations %q: %v", podDeletionGracePeriodLabel, annotations, err) + klog.Errorf("Unable to get %q from annotations %q: %v", podDeletionGracePeriodLabel, annotations, err) } if containerInfo.PodTerminationGracePeriod, err = getInt64PointerFromLabel(annotations, podTerminationGracePeriodLabel); err != nil { - glog.Errorf("Unable to get %q from annotations %q: %v", podTerminationGracePeriodLabel, annotations, err) + klog.Errorf("Unable to get %q from annotations %q: %v", podTerminationGracePeriodLabel, annotations, err) } preStopHandler := &v1.Handler{} if found, err := getJSONObjectFromLabel(annotations, containerPreStopHandlerLabel, preStopHandler); err != nil { - glog.Errorf("Unable to get %q from annotations %q: %v", containerPreStopHandlerLabel, annotations, err) + klog.Errorf("Unable to get %q from annotations %q: %v", containerPreStopHandlerLabel, annotations, err) } else if found { containerInfo.PreStopHandler = preStopHandler } containerPorts := []v1.ContainerPort{} if found, err := getJSONObjectFromLabel(annotations, containerPortsLabel, &containerPorts); err != nil { - glog.Errorf("Unable to get %q from annotations %q: %v", containerPortsLabel, annotations, err) + klog.Errorf("Unable to get %q from annotations %q: %v", containerPortsLabel, annotations, err) } else if found { containerInfo.ContainerPorts = containerPorts } @@ -237,7 +237,7 @@ func getStringValueFromLabel(labels map[string]string, label string) string { return value } // Do not report error, because there should be many old containers without label now. - glog.V(3).Infof("Container doesn't have label %s, it may be an old or invalid container", label) + klog.V(3).Infof("Container doesn't have label %s, it may be an old or invalid container", label) // Return empty string "" for these containers, the caller will get value by other ways. return "" } @@ -252,7 +252,7 @@ func getIntValueFromLabel(labels map[string]string, label string) (int, error) { return intValue, nil } // Do not report error, because there should be many old containers without label now. - glog.V(3).Infof("Container doesn't have label %s, it may be an old or invalid container", label) + klog.V(3).Infof("Container doesn't have label %s, it may be an old or invalid container", label) // Just set the value to 0 return 0, nil } @@ -267,7 +267,7 @@ func getUint64ValueFromLabel(labels map[string]string, label string) (uint64, er return intValue, nil } // Do not report error, because there should be many old containers without label now. - glog.V(3).Infof("Container doesn't have label %s, it may be an old or invalid container", label) + klog.V(3).Infof("Container doesn't have label %s, it may be an old or invalid container", label) // Just set the value to 0 return 0, nil } diff --git a/pkg/kubelet/kuberuntime/logs/BUILD b/pkg/kubelet/kuberuntime/logs/BUILD index ba0dc3f49e9..61eeead77f3 100644 --- a/pkg/kubelet/kuberuntime/logs/BUILD +++ b/pkg/kubelet/kuberuntime/logs/BUILD @@ -12,7 +12,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//vendor/github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog:go_default_library", "//vendor/github.com/fsnotify/fsnotify:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/kuberuntime/logs/logs.go b/pkg/kubelet/kuberuntime/logs/logs.go index c81194d1e63..e679cee84b6 100644 --- a/pkg/kubelet/kuberuntime/logs/logs.go +++ b/pkg/kubelet/kuberuntime/logs/logs.go @@ -30,7 +30,7 @@ import ( "github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog" "github.com/fsnotify/fsnotify" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" @@ -293,7 +293,7 @@ func ReadLogs(ctx context.Context, path, containerID string, opts *LogOptions, r msg := &logMessage{} for { if stop { - glog.V(2).Infof("Finish parsing log file %q", path) + klog.V(2).Infof("Finish parsing log file %q", path) return nil } l, err := r.ReadBytes(eol[0]) @@ -328,7 +328,7 @@ func ReadLogs(ctx context.Context, path, containerID string, opts *LogOptions, r if len(l) == 0 { continue } - glog.Warningf("Incomplete line in log file %q: %q", path, l) + klog.Warningf("Incomplete line in log file %q: %q", path, l) } if parse == nil { // Initialize the log parsing function. @@ -340,16 +340,16 @@ func ReadLogs(ctx context.Context, path, containerID string, opts *LogOptions, r // Parse the log line. msg.reset() if err := parse(l, msg); err != nil { - glog.Errorf("Failed with err %v when parsing log for log file %q: %q", err, path, l) + klog.Errorf("Failed with err %v when parsing log for log file %q: %q", err, path, l) continue } // Write the log line into the stream. if err := writer.write(msg); err != nil { if err == errMaximumWrite { - glog.V(2).Infof("Finish parsing log file %q, hit bytes limit %d(bytes)", path, opts.bytes) + klog.V(2).Infof("Finish parsing log file %q, hit bytes limit %d(bytes)", path, opts.bytes) return nil } - glog.Errorf("Failed with err %v when writing log for log file %q: %+v", err, path, msg) + klog.Errorf("Failed with err %v when writing log for log file %q: %+v", err, path, msg) return err } } @@ -362,7 +362,7 @@ func isContainerRunning(id string, r internalapi.RuntimeService) (bool, error) { } // Only keep following container log when it is running. if s.State != runtimeapi.ContainerState_CONTAINER_RUNNING { - glog.V(5).Infof("Container %q is not running (state=%q)", id, s.State) + klog.V(5).Infof("Container %q is not running (state=%q)", id, s.State) // Do not return error because it's normal that the container stops // during waiting. return false, nil @@ -387,10 +387,10 @@ func waitLogs(ctx context.Context, id string, w *fsnotify.Watcher, runtimeServic case fsnotify.Write: return true, nil default: - glog.Errorf("Unexpected fsnotify event: %v, retrying...", e) + klog.Errorf("Unexpected fsnotify event: %v, retrying...", e) } case err := <-w.Errors: - glog.Errorf("Fsnotify watch error: %v, %d error retries remaining", err, errRetry) + klog.Errorf("Fsnotify watch error: %v, %d error retries remaining", err, errRetry) if errRetry == 0 { return false, err } diff --git a/pkg/kubelet/kuberuntime/main_test.go b/pkg/kubelet/kuberuntime/main_test.go new file mode 100644 index 00000000000..8507d04f373 --- /dev/null +++ b/pkg/kubelet/kuberuntime/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kuberuntime + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/kubelet/lifecycle/BUILD b/pkg/kubelet/lifecycle/BUILD index e46f993837b..ff518ddfdf1 100644 --- a/pkg/kubelet/lifecycle/BUILD +++ b/pkg/kubelet/lifecycle/BUILD @@ -28,7 +28,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/lifecycle/handlers.go b/pkg/kubelet/lifecycle/handlers.go index c0630ebb2aa..8b5a8b8d406 100644 --- a/pkg/kubelet/lifecycle/handlers.go +++ b/pkg/kubelet/lifecycle/handlers.go @@ -23,10 +23,10 @@ import ( "net/http" "strconv" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/klog" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/kubelet/util/format" @@ -59,20 +59,20 @@ func (hr *HandlerRunner) Run(containerID kubecontainer.ContainerID, pod *v1.Pod, output, err := hr.commandRunner.RunInContainer(containerID, handler.Exec.Command, 0) if err != nil { msg = fmt.Sprintf("Exec lifecycle hook (%v) for Container %q in Pod %q failed - error: %v, message: %q", handler.Exec.Command, container.Name, format.Pod(pod), err, string(output)) - glog.V(1).Infof(msg) + klog.V(1).Infof(msg) } return msg, err case handler.HTTPGet != nil: msg, err := hr.runHTTPHandler(pod, container, handler) if err != nil { msg = fmt.Sprintf("Http lifecycle hook (%s) for Container %q in Pod %q failed - error: %v, message: %q", handler.HTTPGet.Path, container.Name, format.Pod(pod), err, msg) - glog.V(1).Infof(msg) + klog.V(1).Infof(msg) } return msg, err default: err := fmt.Errorf("Invalid handler: %v", handler) msg := fmt.Sprintf("Cannot run handler: %v", err) - glog.Errorf(msg) + klog.Errorf(msg) return msg, err } } @@ -105,7 +105,7 @@ func (hr *HandlerRunner) runHTTPHandler(pod *v1.Pod, container *v1.Container, ha if len(host) == 0 { status, err := hr.containerManager.GetPodStatus(pod.UID, pod.Name, pod.Namespace) if err != nil { - glog.Errorf("Unable to get pod info, event handlers may be invalid.") + klog.Errorf("Unable to get pod info, event handlers may be invalid.") return "", err } if status.IP == "" { diff --git a/pkg/kubelet/lifecycle/predicate.go b/pkg/kubelet/lifecycle/predicate.go index ba6d25b584c..df4a32d1add 100644 --- a/pkg/kubelet/lifecycle/predicate.go +++ b/pkg/kubelet/lifecycle/predicate.go @@ -19,7 +19,7 @@ package lifecycle import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" @@ -58,7 +58,7 @@ func NewPredicateAdmitHandler(getNodeAnyWayFunc getNodeAnyWayFuncType, admission func (w *predicateAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult { node, err := w.getNodeAnyWayFunc() if err != nil { - glog.Errorf("Cannot get Node info: %v", err) + klog.Errorf("Cannot get Node info: %v", err) return PodAdmitResult{ Admit: false, Reason: "InvalidNodeInfo", @@ -72,7 +72,7 @@ func (w *predicateAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult // ensure the node has enough plugin resources for that required in pods if err = w.pluginResourceUpdateFunc(nodeInfo, attrs); err != nil { message := fmt.Sprintf("Update plugin resources failed due to %v, which is unexpected.", err) - glog.Warningf("Failed to admit pod %v - %s", format.Pod(admitPod), message) + klog.Warningf("Failed to admit pod %v - %s", format.Pod(admitPod), message) return PodAdmitResult{ Admit: false, Reason: "UnexpectedAdmissionError", @@ -93,7 +93,7 @@ func (w *predicateAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult fit, reasons, err := predicates.GeneralPredicates(podWithoutMissingExtendedResources, nil, nodeInfo) if err != nil { message := fmt.Sprintf("GeneralPredicates failed due to %v, which is unexpected.", err) - glog.Warningf("Failed to admit pod %v - %s", format.Pod(admitPod), message) + klog.Warningf("Failed to admit pod %v - %s", format.Pod(admitPod), message) return PodAdmitResult{ Admit: fit, Reason: "UnexpectedAdmissionError", @@ -104,7 +104,7 @@ func (w *predicateAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult fit, reasons, err = w.admissionFailureHandler.HandleAdmissionFailure(admitPod, reasons) if err != nil { message := fmt.Sprintf("Unexpected error while attempting to recover from admission failure: %v", err) - glog.Warningf("Failed to admit pod %v - %s", format.Pod(admitPod), message) + klog.Warningf("Failed to admit pod %v - %s", format.Pod(admitPod), message) return PodAdmitResult{ Admit: fit, Reason: "UnexpectedAdmissionError", @@ -117,7 +117,7 @@ func (w *predicateAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult var message string if len(reasons) == 0 { message = fmt.Sprint("GeneralPredicates failed due to unknown reason, which is unexpected.") - glog.Warningf("Failed to admit pod %v - %s", format.Pod(admitPod), message) + klog.Warningf("Failed to admit pod %v - %s", format.Pod(admitPod), message) return PodAdmitResult{ Admit: fit, Reason: "UnknownReason", @@ -130,19 +130,19 @@ func (w *predicateAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult case *predicates.PredicateFailureError: reason = re.PredicateName message = re.Error() - glog.V(2).Infof("Predicate failed on Pod: %v, for reason: %v", format.Pod(admitPod), message) + klog.V(2).Infof("Predicate failed on Pod: %v, for reason: %v", format.Pod(admitPod), message) case *predicates.InsufficientResourceError: reason = fmt.Sprintf("OutOf%s", re.ResourceName) message = re.Error() - glog.V(2).Infof("Predicate failed on Pod: %v, for reason: %v", format.Pod(admitPod), message) + klog.V(2).Infof("Predicate failed on Pod: %v, for reason: %v", format.Pod(admitPod), message) case *predicates.FailureReason: reason = re.GetReason() message = fmt.Sprintf("Failure: %s", re.GetReason()) - glog.V(2).Infof("Predicate failed on Pod: %v, for reason: %v", format.Pod(admitPod), message) + klog.V(2).Infof("Predicate failed on Pod: %v, for reason: %v", format.Pod(admitPod), message) default: reason = "UnexpectedPredicateFailureType" message = fmt.Sprintf("GeneralPredicates failed due to %v, which is unexpected.", r) - glog.Warningf("Failed to admit pod %v - %s", format.Pod(admitPod), message) + klog.Warningf("Failed to admit pod %v - %s", format.Pod(admitPod), message) } return PodAdmitResult{ Admit: fit, diff --git a/pkg/kubelet/logs/BUILD b/pkg/kubelet/logs/BUILD index 41e4281dbaa..58810681434 100644 --- a/pkg/kubelet/logs/BUILD +++ b/pkg/kubelet/logs/BUILD @@ -14,7 +14,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/logs/container_log_manager.go b/pkg/kubelet/logs/container_log_manager.go index cae78993d1b..50cfcf49534 100644 --- a/pkg/kubelet/logs/container_log_manager.go +++ b/pkg/kubelet/logs/container_log_manager.go @@ -26,7 +26,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/clock" @@ -171,7 +171,7 @@ func (c *containerLogManager) Start() { // Start a goroutine periodically does container log rotation. go wait.Forever(func() { if err := c.rotateLogs(); err != nil { - glog.Errorf("Failed to rotate container logs: %v", err) + klog.Errorf("Failed to rotate container logs: %v", err) } }, logMonitorPeriod) } @@ -193,27 +193,27 @@ func (c *containerLogManager) rotateLogs() error { // Note that we should not block log rotate for an error of a single container. status, err := c.runtimeService.ContainerStatus(id) if err != nil { - glog.Errorf("Failed to get container status for %q: %v", id, err) + klog.Errorf("Failed to get container status for %q: %v", id, err) continue } path := status.GetLogPath() info, err := os.Stat(path) if err != nil { if !os.IsNotExist(err) { - glog.Errorf("Failed to stat container log %q: %v", path, err) + klog.Errorf("Failed to stat container log %q: %v", path, err) continue } // In rotateLatestLog, there are several cases that we may // lose original container log after ReopenContainerLog fails. // We try to recover it by reopening container log. if err := c.runtimeService.ReopenContainerLog(id); err != nil { - glog.Errorf("Container %q log %q doesn't exist, reopen container log failed: %v", id, path, err) + klog.Errorf("Container %q log %q doesn't exist, reopen container log failed: %v", id, path, err) continue } // The container log should be recovered. info, err = os.Stat(path) if err != nil { - glog.Errorf("Failed to stat container log %q after reopen: %v", path, err) + klog.Errorf("Failed to stat container log %q after reopen: %v", path, err) continue } } @@ -222,7 +222,7 @@ func (c *containerLogManager) rotateLogs() error { } // Perform log rotation. if err := c.rotateLog(id, path); err != nil { - glog.Errorf("Failed to rotate log %q for container %q: %v", path, id, err) + klog.Errorf("Failed to rotate log %q for container %q: %v", path, id, err) continue } } @@ -379,7 +379,7 @@ func (c *containerLogManager) rotateLatestLog(id, log string) error { // This shouldn't happen. // Report an error if this happens, because we will lose original // log. - glog.Errorf("Failed to rename rotated log %q back to %q: %v, reopen container log error: %v", rotated, log, renameErr, err) + klog.Errorf("Failed to rename rotated log %q back to %q: %v, reopen container log error: %v", rotated, log, renameErr, err) } return fmt.Errorf("failed to reopen container log %q: %v", id, err) } diff --git a/pkg/kubelet/main_test.go b/pkg/kubelet/main_test.go new file mode 100644 index 00000000000..f7ab8e1ca1f --- /dev/null +++ b/pkg/kubelet/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/kubelet/metrics/BUILD b/pkg/kubelet/metrics/BUILD index 4587b6b7900..e104574eba8 100644 --- a/pkg/kubelet/metrics/BUILD +++ b/pkg/kubelet/metrics/BUILD @@ -14,8 +14,8 @@ go_library( "//pkg/kubelet/container:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/metrics/collectors/BUILD b/pkg/kubelet/metrics/collectors/BUILD index 4b41a1a3227..64022f36676 100644 --- a/pkg/kubelet/metrics/collectors/BUILD +++ b/pkg/kubelet/metrics/collectors/BUILD @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["volume_stats.go"], + srcs = [ + "log_metrics.go", + "volume_stats.go", + ], importpath = "k8s.io/kubernetes/pkg/kubelet/metrics/collectors", visibility = ["//visibility:public"], deps = [ @@ -10,8 +13,8 @@ go_library( "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/server/stats:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -19,6 +22,7 @@ go_test( name = "go_default_test", srcs = [ "helper_test.go", + "log_metrics_test.go", "volume_stats_test.go", ], embed = [":go_default_library"], diff --git a/pkg/kubelet/metrics/collectors/log_metrics.go b/pkg/kubelet/metrics/collectors/log_metrics.go new file mode 100644 index 00000000000..c43915e0708 --- /dev/null +++ b/pkg/kubelet/metrics/collectors/log_metrics.go @@ -0,0 +1,77 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collectors + +import ( + "github.com/prometheus/client_golang/prometheus" + "k8s.io/klog" + + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" +) + +var ( + descLogSize = prometheus.NewDesc( + "kubelet_container_log_filesystem_used_bytes", + "Bytes used by the container's logs on the filesystem.", + []string{ + "namespace", + "pod", + "container", + }, nil, + ) +) + +type logMetricsCollector struct { + podStats func() ([]statsapi.PodStats, error) +} + +// NewLogMetricsCollector implements the prometheus.Collector interface and +// exposes metrics about container's log volume size. +func NewLogMetricsCollector(podStats func() ([]statsapi.PodStats, error)) prometheus.Collector { + return &logMetricsCollector{ + podStats: podStats, + } +} + +// Describe implements the prometheus.Collector interface. +func (c *logMetricsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- descLogSize +} + +// Collect implements the prometheus.Collector interface. +func (c *logMetricsCollector) Collect(ch chan<- prometheus.Metric) { + podStats, err := c.podStats() + if err != nil { + klog.Errorf("failed to get pod stats: %v", err) + return + } + + for _, ps := range podStats { + for _, c := range ps.Containers { + if c.Logs != nil && c.Logs.UsedBytes != nil { + ch <- prometheus.MustNewConstMetric( + descLogSize, + prometheus.GaugeValue, + float64(*c.Logs.UsedBytes), + ps.PodRef.Namespace, + ps.PodRef.Name, + c.Name, + ) + } + } + } +} diff --git a/pkg/kubelet/metrics/collectors/log_metrics_test.go b/pkg/kubelet/metrics/collectors/log_metrics_test.go new file mode 100644 index 00000000000..62cc22a1f77 --- /dev/null +++ b/pkg/kubelet/metrics/collectors/log_metrics_test.go @@ -0,0 +1,74 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collectors + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" +) + +func TestNoMetricsCollected(t *testing.T) { + ch := make(chan prometheus.Metric) + + collector := &logMetricsCollector{ + podStats: func() ([]statsapi.PodStats, error) { + return []statsapi.PodStats{}, nil + }, + } + collector.Collect(ch) + + num := len(ch) + if num != 0 { + t.Fatalf("Channel expected to be empty, but received %d", num) + } +} + +func TestMetricsCollected(t *testing.T) { + size := uint64(18) + collector := &logMetricsCollector{ + podStats: func() ([]statsapi.PodStats, error) { + return []statsapi.PodStats{ + { + PodRef: statsapi.PodReference{ + Namespace: "some-namespace", + Name: "podName1", + }, + Containers: []statsapi.ContainerStats{ + { + Name: "containerName1", + Logs: &statsapi.FsStats{ + UsedBytes: &size, + }, + }, + }, + }, + }, nil + }, + } + + err := gatherAndCompare(collector, ` + # HELP kubelet_container_log_filesystem_used_bytes Bytes used by the container's logs on the filesystem. + # TYPE kubelet_container_log_filesystem_used_bytes gauge + kubelet_container_log_filesystem_used_bytes{container="containerName1",namespace="some-namespace",pod="podName1"} 18 +`, nil) + if err != nil { + t.Fatal(err) + } +} diff --git a/pkg/kubelet/metrics/collectors/volume_stats.go b/pkg/kubelet/metrics/collectors/volume_stats.go index e6f1cf36da8..c281aaea57f 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats.go +++ b/pkg/kubelet/metrics/collectors/volume_stats.go @@ -17,9 +17,9 @@ limitations under the License. package collectors import ( - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/metrics" serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats" @@ -87,7 +87,7 @@ func (collector *volumeStatsCollector) Collect(ch chan<- prometheus.Metric) { lv = append([]string{pvcRef.Namespace, pvcRef.Name}, lv...) metric, err := prometheus.NewConstMetric(desc, prometheus.GaugeValue, v, lv...) if err != nil { - glog.Warningf("Failed to generate metric: %v", err) + klog.Warningf("Failed to generate metric: %v", err) return } ch <- metric diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go index 8a2e027a285..ec1a7166c30 100644 --- a/pkg/kubelet/metrics/metrics.go +++ b/pkg/kubelet/metrics/metrics.go @@ -21,10 +21,10 @@ import ( "sync" "time" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -270,7 +270,7 @@ func (pc *podAndContainerCollector) Describe(ch chan<- *prometheus.Desc) { func (pc *podAndContainerCollector) Collect(ch chan<- prometheus.Metric) { runningPods, err := pc.containerCache.GetPods() if err != nil { - glog.Warningf("Failed to get running container information while collecting metrics: %v", err) + klog.Warningf("Failed to get running container information while collecting metrics: %v", err) return } diff --git a/pkg/kubelet/mountpod/BUILD b/pkg/kubelet/mountpod/BUILD index f7d56e3b1f7..411da640f20 100644 --- a/pkg/kubelet/mountpod/BUILD +++ b/pkg/kubelet/mountpod/BUILD @@ -25,7 +25,7 @@ go_test( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/mountpod/mount_pod_test.go b/pkg/kubelet/mountpod/mount_pod_test.go index bf7a0ec5602..e248b5e3df5 100644 --- a/pkg/kubelet/mountpod/mount_pod_test.go +++ b/pkg/kubelet/mountpod/mount_pod_test.go @@ -22,7 +22,7 @@ import ( "path" "testing" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -137,7 +137,7 @@ func TestGetVolumeExec(t *testing.T) { } pod, container, err := mgr.GetMountPod("kubernetes.io/glusterfs") if err != nil { - glog.V(5).Infof("test %q returned error %s", test.name, err) + klog.V(5).Infof("test %q returned error %s", test.name, err) } if err == nil && test.expectError { t.Errorf("test %q: expected error, got none", test.name) diff --git a/pkg/kubelet/network/dns/BUILD b/pkg/kubelet/network/dns/BUILD index ae7bba5cf9e..f7d550391f3 100644 --- a/pkg/kubelet/network/dns/BUILD +++ b/pkg/kubelet/network/dns/BUILD @@ -15,21 +15,26 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) go_test( name = "go_default_test", - srcs = ["dns_test.go"], + srcs = [ + "dns_test.go", + "main_test.go", + ], embed = [":go_default_library"], deps = [ + "//pkg/features:go_default_library", "//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", diff --git a/pkg/kubelet/network/dns/dns.go b/pkg/kubelet/network/dns/dns.go index b77f349fc35..aa3a8e01a5f 100644 --- a/pkg/kubelet/network/dns/dns.go +++ b/pkg/kubelet/network/dns/dns.go @@ -35,7 +35,7 @@ import ( kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/util/format" - "github.com/golang/glog" + "k8s.io/klog" ) var ( @@ -119,7 +119,7 @@ func (c *Configurer) formDNSSearchFitsLimits(composedSearch []string, pod *v1.Po if limitsExceeded { log := fmt.Sprintf("Search Line limits were exceeded, some search paths have been omitted, the applied search line is: %s", strings.Join(composedSearch, " ")) c.recorder.Event(pod, v1.EventTypeWarning, "DNSConfigForming", log) - glog.Error(log) + klog.Error(log) } return composedSearch } @@ -129,7 +129,7 @@ func (c *Configurer) formDNSNameserversFitsLimits(nameservers []string, pod *v1. nameservers = nameservers[0:validation.MaxDNSNameservers] log := fmt.Sprintf("Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: %s", strings.Join(nameservers, " ")) c.recorder.Event(pod, v1.EventTypeWarning, "DNSConfigForming", log) - glog.Error(log) + klog.Error(log) } return nameservers } @@ -157,7 +157,7 @@ func (c *Configurer) CheckLimitsForResolvConf() { f, err := os.Open(c.ResolverConfig) if err != nil { c.recorder.Event(c.nodeRef, v1.EventTypeWarning, "CheckLimitsForResolvConf", err.Error()) - glog.V(4).Infof("CheckLimitsForResolvConf: " + err.Error()) + klog.V(4).Infof("CheckLimitsForResolvConf: " + err.Error()) return } defer f.Close() @@ -165,7 +165,7 @@ func (c *Configurer) CheckLimitsForResolvConf() { _, hostSearch, _, err := parseResolvConf(f) if err != nil { c.recorder.Event(c.nodeRef, v1.EventTypeWarning, "CheckLimitsForResolvConf", err.Error()) - glog.V(4).Infof("CheckLimitsForResolvConf: " + err.Error()) + klog.V(4).Infof("CheckLimitsForResolvConf: " + err.Error()) return } @@ -178,14 +178,14 @@ func (c *Configurer) CheckLimitsForResolvConf() { if len(hostSearch) > domainCountLimit { log := fmt.Sprintf("Resolv.conf file '%s' contains search line consisting of more than %d domains!", c.ResolverConfig, domainCountLimit) c.recorder.Event(c.nodeRef, v1.EventTypeWarning, "CheckLimitsForResolvConf", log) - glog.V(4).Infof("CheckLimitsForResolvConf: " + log) + klog.V(4).Infof("CheckLimitsForResolvConf: " + log) return } if len(strings.Join(hostSearch, " ")) > validation.MaxDNSSearchListChars { log := fmt.Sprintf("Resolv.conf file '%s' contains search line which length is more than allowed %d chars!", c.ResolverConfig, validation.MaxDNSSearchListChars) c.recorder.Event(c.nodeRef, v1.EventTypeWarning, "CheckLimitsForResolvConf", log) - glog.V(4).Infof("CheckLimitsForResolvConf: " + log) + klog.V(4).Infof("CheckLimitsForResolvConf: " + log) return } @@ -336,7 +336,7 @@ func (c *Configurer) GetPodDNS(pod *v1.Pod) (*runtimeapi.DNSConfig, error) { dnsType, err := getPodDNSType(pod) if err != nil { - glog.Errorf("Failed to get DNS type for pod %q: %v. Falling back to DNSClusterFirst policy.", format.Pod(pod), err) + klog.Errorf("Failed to get DNS type for pod %q: %v. Falling back to DNSClusterFirst policy.", format.Pod(pod), err) dnsType = podDNSCluster } switch dnsType { @@ -400,11 +400,11 @@ func (c *Configurer) SetupDNSinContainerizedMounter(mounterPath string) { f, err := os.Open(c.ResolverConfig) defer f.Close() if err != nil { - glog.Error("Could not open resolverConf file") + klog.Error("Could not open resolverConf file") } else { _, hostSearch, _, err := parseResolvConf(f) if err != nil { - glog.Errorf("Error for parsing the reslov.conf file: %v", err) + klog.Errorf("Error for parsing the reslov.conf file: %v", err) } else { dnsString = dnsString + "search" for _, search := range hostSearch { @@ -415,6 +415,6 @@ func (c *Configurer) SetupDNSinContainerizedMounter(mounterPath string) { } } if err := ioutil.WriteFile(resolvePath, []byte(dnsString), 0600); err != nil { - glog.Errorf("Could not write dns nameserver in file %s, with error %v", resolvePath, err) + klog.Errorf("Could not write dns nameserver in file %s, with error %v", resolvePath, err) } } diff --git a/pkg/kubelet/network/dns/main_test.go b/pkg/kubelet/network/dns/main_test.go new file mode 100644 index 00000000000..1ed5fa15436 --- /dev/null +++ b/pkg/kubelet/network/dns/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dns + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/kubelet/nodelease/BUILD b/pkg/kubelet/nodelease/BUILD index 2fe0520311b..579a2ddbf60 100644 --- a/pkg/kubelet/nodelease/BUILD +++ b/pkg/kubelet/nodelease/BUILD @@ -14,7 +14,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/coordination/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", ], ) @@ -25,10 +25,13 @@ go_test( embed = [":go_default_library"], deps = [ "//staging/src/k8s.io/api/coordination/v1beta1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/pkg/kubelet/nodelease/controller.go b/pkg/kubelet/nodelease/controller.go index 5e587f6c593..c614368517b 100644 --- a/pkg/kubelet/nodelease/controller.go +++ b/pkg/kubelet/nodelease/controller.go @@ -29,7 +29,7 @@ import ( coordclientset "k8s.io/client-go/kubernetes/typed/coordination/v1beta1" "k8s.io/utils/pointer" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -52,7 +52,8 @@ type Controller interface { } type controller struct { - client coordclientset.LeaseInterface + client clientset.Interface + leaseClient coordclientset.LeaseInterface holderIdentity string leaseDurationSeconds int32 renewInterval time.Duration @@ -67,7 +68,8 @@ func NewController(clock clock.Clock, client clientset.Interface, holderIdentity leaseClient = client.CoordinationV1beta1().Leases(corev1.NamespaceNodeLease) } return &controller{ - client: leaseClient, + client: client, + leaseClient: leaseClient, holderIdentity: holderIdentity, leaseDurationSeconds: leaseDurationSeconds, renewInterval: renewInterval, @@ -78,8 +80,8 @@ func NewController(clock clock.Clock, client clientset.Interface, holderIdentity // Run runs the controller func (c *controller) Run(stopCh <-chan struct{}) { - if c.client == nil { - glog.Infof("node lease controller has nil client, will not claim or renew leases") + if c.leaseClient == nil { + klog.Infof("node lease controller has nil lease client, will not claim or renew leases") return } wait.Until(c.sync, c.renewInterval, stopCh) @@ -110,7 +112,7 @@ func (c *controller) backoffEnsureLease() (*coordv1beta1.Lease, bool) { break } sleep = minDuration(2*sleep, maxBackoff) - glog.Errorf("failed to ensure node lease exists, will retry in %v, error: %v", sleep, err) + klog.Errorf("failed to ensure node lease exists, will retry in %v, error: %v", sleep, err) // backoff wait c.clock.Sleep(sleep) } @@ -120,10 +122,10 @@ func (c *controller) backoffEnsureLease() (*coordv1beta1.Lease, bool) { // ensureLease creates the lease if it does not exist. Returns the lease and // a bool (true if this call created the lease), or any error that occurs. func (c *controller) ensureLease() (*coordv1beta1.Lease, bool, error) { - lease, err := c.client.Get(c.holderIdentity, metav1.GetOptions{}) + lease, err := c.leaseClient.Get(c.holderIdentity, metav1.GetOptions{}) if apierrors.IsNotFound(err) { // lease does not exist, create it - lease, err := c.client.Create(c.newLease(nil)) + lease, err := c.leaseClient.Create(c.newLease(nil)) if err != nil { return nil, false, err } @@ -140,33 +142,59 @@ func (c *controller) ensureLease() (*coordv1beta1.Lease, bool, error) { // call this once you're sure the lease has been created func (c *controller) retryUpdateLease(base *coordv1beta1.Lease) { for i := 0; i < maxUpdateRetries; i++ { - _, err := c.client.Update(c.newLease(base)) + _, err := c.leaseClient.Update(c.newLease(base)) if err == nil { return } - glog.Errorf("failed to update node lease, error: %v", err) + klog.Errorf("failed to update node lease, error: %v", err) if i > 0 && c.onRepeatedHeartbeatFailure != nil { c.onRepeatedHeartbeatFailure() } } - glog.Errorf("failed %d attempts to update node lease, will retry after %v", maxUpdateRetries, c.renewInterval) + klog.Errorf("failed %d attempts to update node lease, will retry after %v", maxUpdateRetries, c.renewInterval) } // newLease constructs a new lease if base is nil, or returns a copy of base // with desired state asserted on the copy. func (c *controller) newLease(base *coordv1beta1.Lease) *coordv1beta1.Lease { + // Use the bare minimum set of fields; other fields exist for debugging/legacy, + // but we don't need to make node heartbeats more complicated by using them. var lease *coordv1beta1.Lease if base == nil { - lease = &coordv1beta1.Lease{} + lease = &coordv1beta1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.holderIdentity, + Namespace: corev1.NamespaceNodeLease, + }, + Spec: coordv1beta1.LeaseSpec{ + HolderIdentity: pointer.StringPtr(c.holderIdentity), + LeaseDurationSeconds: pointer.Int32Ptr(c.leaseDurationSeconds), + }, + } } else { lease = base.DeepCopy() } - // Use the bare minimum set of fields; other fields exist for debugging/legacy, - // but we don't need to make node heartbeats more complicated by using them. - lease.Name = c.holderIdentity - lease.Spec.HolderIdentity = pointer.StringPtr(c.holderIdentity) - lease.Spec.LeaseDurationSeconds = pointer.Int32Ptr(c.leaseDurationSeconds) lease.Spec.RenewTime = &metav1.MicroTime{Time: c.clock.Now()} + + // Setting owner reference needs node's UID. Note that it is different from + // kubelet.nodeRef.UID. When lease is initially created, it is possible that + // the connection between master and node is not ready yet. So try to set + // owner reference every time when renewing the lease, until successful. + if lease.OwnerReferences == nil || len(lease.OwnerReferences) == 0 { + if node, err := c.client.CoreV1().Nodes().Get(c.holderIdentity, metav1.GetOptions{}); err == nil { + lease.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: corev1.SchemeGroupVersion.WithKind("Node").Version, + Kind: corev1.SchemeGroupVersion.WithKind("Node").Kind, + Name: c.holderIdentity, + UID: node.UID, + }, + } + } else { + klog.Errorf("failed to get node %q when trying to set owner ref to the node lease: %v", c.holderIdentity, err) + } + } + return lease } diff --git a/pkg/kubelet/nodelease/controller_test.go b/pkg/kubelet/nodelease/controller_test.go index d48d3d7e7c5..651fdae781d 100644 --- a/pkg/kubelet/nodelease/controller_test.go +++ b/pkg/kubelet/nodelease/controller_test.go @@ -21,15 +21,24 @@ import ( "time" coordv1beta1 "k8s.io/api/coordination/v1beta1" + corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/client-go/kubernetes/fake" "k8s.io/utils/pointer" ) func TestNewLease(t *testing.T) { fakeClock := clock.NewFakeClock(time.Now()) + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + UID: types.UID("foo-uid"), + }, + } cases := []struct { desc string controller *controller @@ -37,47 +46,136 @@ func TestNewLease(t *testing.T) { expect *coordv1beta1.Lease }{ { - desc: "nil base", + desc: "nil base without node", controller: &controller{ - holderIdentity: "foo", + client: fake.NewSimpleClientset(), + holderIdentity: node.Name, leaseDurationSeconds: 10, clock: fakeClock, }, base: nil, expect: &coordv1beta1.Lease{ ObjectMeta: metav1.ObjectMeta{ - Name: "foo", + Name: node.Name, + Namespace: corev1.NamespaceNodeLease, }, Spec: coordv1beta1.LeaseSpec{ - HolderIdentity: pointer.StringPtr("foo"), + HolderIdentity: pointer.StringPtr(node.Name), LeaseDurationSeconds: pointer.Int32Ptr(10), RenewTime: &metav1.MicroTime{Time: fakeClock.Now()}, }, }, }, { - desc: "non-nil base renew time is updated", + desc: "nil base with node", controller: &controller{ - holderIdentity: "foo", + client: fake.NewSimpleClientset(node), + holderIdentity: node.Name, + leaseDurationSeconds: 10, + clock: fakeClock, + }, + base: nil, + expect: &coordv1beta1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + Namespace: corev1.NamespaceNodeLease, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: corev1.SchemeGroupVersion.WithKind("Node").Version, + Kind: corev1.SchemeGroupVersion.WithKind("Node").Kind, + Name: node.Name, + UID: node.UID, + }, + }, + }, + Spec: coordv1beta1.LeaseSpec{ + HolderIdentity: pointer.StringPtr(node.Name), + LeaseDurationSeconds: pointer.Int32Ptr(10), + RenewTime: &metav1.MicroTime{Time: fakeClock.Now()}, + }, + }, + }, + { + desc: "non-nil base without owner ref, renew time is updated", + controller: &controller{ + client: fake.NewSimpleClientset(node), + holderIdentity: node.Name, leaseDurationSeconds: 10, clock: fakeClock, }, base: &coordv1beta1.Lease{ ObjectMeta: metav1.ObjectMeta{ - Name: "foo", + Name: node.Name, + Namespace: corev1.NamespaceNodeLease, }, Spec: coordv1beta1.LeaseSpec{ - HolderIdentity: pointer.StringPtr("foo"), + HolderIdentity: pointer.StringPtr(node.Name), LeaseDurationSeconds: pointer.Int32Ptr(10), RenewTime: &metav1.MicroTime{Time: fakeClock.Now().Add(-10 * time.Second)}, }, }, expect: &coordv1beta1.Lease{ ObjectMeta: metav1.ObjectMeta{ - Name: "foo", + Name: node.Name, + Namespace: corev1.NamespaceNodeLease, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: corev1.SchemeGroupVersion.WithKind("Node").Version, + Kind: corev1.SchemeGroupVersion.WithKind("Node").Kind, + Name: node.Name, + UID: node.UID, + }, + }, }, Spec: coordv1beta1.LeaseSpec{ - HolderIdentity: pointer.StringPtr("foo"), + HolderIdentity: pointer.StringPtr(node.Name), + LeaseDurationSeconds: pointer.Int32Ptr(10), + RenewTime: &metav1.MicroTime{Time: fakeClock.Now()}, + }, + }, + }, + { + desc: "non-nil base with owner ref, renew time is updated", + controller: &controller{ + client: fake.NewSimpleClientset(node), + holderIdentity: node.Name, + leaseDurationSeconds: 10, + clock: fakeClock, + }, + base: &coordv1beta1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + Namespace: corev1.NamespaceNodeLease, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: corev1.SchemeGroupVersion.WithKind("Node").Version, + Kind: corev1.SchemeGroupVersion.WithKind("Node").Kind, + Name: node.Name, + UID: node.UID, + }, + }, + }, + Spec: coordv1beta1.LeaseSpec{ + HolderIdentity: pointer.StringPtr(node.Name), + LeaseDurationSeconds: pointer.Int32Ptr(10), + RenewTime: &metav1.MicroTime{Time: fakeClock.Now().Add(-10 * time.Second)}, + }, + }, + expect: &coordv1beta1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + Namespace: corev1.NamespaceNodeLease, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: corev1.SchemeGroupVersion.WithKind("Node").Version, + Kind: corev1.SchemeGroupVersion.WithKind("Node").Kind, + Name: node.Name, + UID: node.UID, + }, + }, + }, + Spec: coordv1beta1.LeaseSpec{ + HolderIdentity: pointer.StringPtr(node.Name), LeaseDurationSeconds: pointer.Int32Ptr(10), RenewTime: &metav1.MicroTime{Time: fakeClock.Now()}, }, diff --git a/pkg/kubelet/nodestatus/BUILD b/pkg/kubelet/nodestatus/BUILD index 9e38273fc54..a626ce19866 100644 --- a/pkg/kubelet/nodestatus/BUILD +++ b/pkg/kubelet/nodestatus/BUILD @@ -21,8 +21,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/nodestatus/setters.go b/pkg/kubelet/nodestatus/setters.go index bf049c4824b..d16bcec2949 100644 --- a/pkg/kubelet/nodestatus/setters.go +++ b/pkg/kubelet/nodestatus/setters.go @@ -42,7 +42,7 @@ import ( "k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/pkg/volume" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -69,7 +69,7 @@ func NodeAddress(nodeIP net.IP, // typically Kubelet.nodeIP if err := validateNodeIPFunc(nodeIP); err != nil { return fmt.Errorf("failed to validate nodeIP: %v", err) } - glog.V(2).Infof("Using node IP: %q", nodeIP.String()) + klog.V(2).Infof("Using node IP: %q", nodeIP.String()) } if externalCloudProvider { @@ -137,11 +137,11 @@ func NodeAddress(nodeIP net.IP, // typically Kubelet.nodeIP if existingHostnameAddress == nil { // no existing Hostname address found, add it - glog.Warningf("adding overridden hostname of %v to cloudprovider-reported addresses", hostname) + klog.Warningf("adding overridden hostname of %v to cloudprovider-reported addresses", hostname) nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeHostName, Address: hostname}) } else { // override the Hostname address reported by the cloud provider - glog.Warningf("replacing cloudprovider-reported hostname of %v with overridden hostname of %v", existingHostnameAddress.Address, hostname) + klog.Warningf("replacing cloudprovider-reported hostname of %v with overridden hostname of %v", existingHostnameAddress.Address, hostname) existingHostnameAddress.Address = hostname } } @@ -239,7 +239,7 @@ func MachineInfo(nodeName string, node.Status.Capacity[v1.ResourceCPU] = *resource.NewMilliQuantity(0, resource.DecimalSI) node.Status.Capacity[v1.ResourceMemory] = resource.MustParse("0Gi") node.Status.Capacity[v1.ResourcePods] = *resource.NewQuantity(int64(maxPods), resource.DecimalSI) - glog.Errorf("Error getting machine info: %v", err) + klog.Errorf("Error getting machine info: %v", err) } else { node.Status.NodeInfo.MachineID = info.MachineID node.Status.NodeInfo.SystemUUID = info.SystemUUID @@ -278,14 +278,14 @@ func MachineInfo(nodeName string, if devicePluginCapacity != nil { for k, v := range devicePluginCapacity { if old, ok := node.Status.Capacity[k]; !ok || old.Value() != v.Value() { - glog.V(2).Infof("Update capacity for %s to %d", k, v.Value()) + klog.V(2).Infof("Update capacity for %s to %d", k, v.Value()) } node.Status.Capacity[k] = v } } for _, removedResource := range removedDevicePlugins { - glog.V(2).Infof("Set capacity for %s to 0 on device removal", removedResource) + klog.V(2).Infof("Set capacity for %s to 0 on device removal", removedResource) // Set the capacity of the removed resource to 0 instead of // removing the resource from the node status. This is to indicate // that the resource is managed by device plugin and had been @@ -326,7 +326,7 @@ func MachineInfo(nodeName string, if devicePluginAllocatable != nil { for k, v := range devicePluginAllocatable { if old, ok := node.Status.Allocatable[k]; !ok || old.Value() != v.Value() { - glog.V(2).Infof("Update allocatable for %s to %d", k, v.Value()) + klog.V(2).Infof("Update allocatable for %s to %d", k, v.Value()) } node.Status.Allocatable[k] = v } @@ -357,7 +357,7 @@ func VersionInfo(versionInfoFunc func() (*cadvisorapiv1.VersionInfo, error), // verinfo, err := versionInfoFunc() if err != nil { // TODO(mtaufen): consider removing this log line, since returned error will be logged - glog.Errorf("Error getting version info: %v", err) + klog.Errorf("Error getting version info: %v", err) return fmt.Errorf("error getting version info: %v", err) } @@ -397,7 +397,7 @@ func Images(nodeStatusMaxImages int32, containerImages, err := imageListFunc() if err != nil { // TODO(mtaufen): consider removing this log line, since returned error will be logged - glog.Errorf("Error getting image list: %v", err) + klog.Errorf("Error getting image list: %v", err) node.Status.Images = imagesOnNode return fmt.Errorf("error getting image list: %v", err) } @@ -515,7 +515,7 @@ func ReadyCondition( recordEventFunc(v1.EventTypeNormal, events.NodeReady) } else { recordEventFunc(v1.EventTypeNormal, events.NodeNotReady) - glog.Infof("Node became not ready: %+v", newNodeReadyCondition) + klog.Infof("Node became not ready: %+v", newNodeReadyCondition) } } return nil @@ -733,7 +733,7 @@ func VolumeLimits(volumePluginListFunc func() []volume.VolumePluginWithAttachLim for _, volumePlugin := range pluginWithLimits { attachLimits, err := volumePlugin.GetVolumeLimits() if err != nil { - glog.V(4).Infof("Error getting volume limit for plugin %s", volumePlugin.GetPluginName()) + klog.V(4).Infof("Error getting volume limit for plugin %s", volumePlugin.GetPluginName()) continue } for limitKey, value := range attachLimits { diff --git a/pkg/kubelet/oom_watcher.go b/pkg/kubelet/oom_watcher.go index 448082b05f1..1ca014b4bab 100644 --- a/pkg/kubelet/oom_watcher.go +++ b/pkg/kubelet/oom_watcher.go @@ -17,13 +17,13 @@ limitations under the License. package kubelet import ( - "github.com/golang/glog" "github.com/google/cadvisor/events" cadvisorapi "github.com/google/cadvisor/info/v1" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/record" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/cadvisor" ) @@ -65,10 +65,10 @@ func (ow *realOOMWatcher) Start(ref *v1.ObjectReference) error { defer runtime.HandleCrash() for event := range eventChannel.GetChannel() { - glog.V(2).Infof("Got sys oom event from cadvisor: %v", event) + klog.V(2).Infof("Got sys oom event from cadvisor: %v", event) ow.recorder.PastEventf(ref, metav1.Time{Time: event.Timestamp}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered") } - glog.Errorf("Unexpectedly stopped receiving OOM notifications from cAdvisor") + klog.Errorf("Unexpectedly stopped receiving OOM notifications from cAdvisor") }() return nil } diff --git a/pkg/kubelet/pleg/BUILD b/pkg/kubelet/pleg/BUILD index f3d4073e340..2d54aee0886 100644 --- a/pkg/kubelet/pleg/BUILD +++ b/pkg/kubelet/pleg/BUILD @@ -22,7 +22,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/pleg/generic.go b/pkg/kubelet/pleg/generic.go index bbe9a91365e..4de9c721130 100644 --- a/pkg/kubelet/pleg/generic.go +++ b/pkg/kubelet/pleg/generic.go @@ -21,11 +21,11 @@ import ( "sync/atomic" "time" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/metrics" @@ -150,7 +150,7 @@ func generateEvents(podID types.UID, cid string, oldState, newState plegContaine return nil } - glog.V(4).Infof("GenericPLEG: %v/%v: %v -> %v", podID, cid, oldState, newState) + klog.V(4).Infof("GenericPLEG: %v/%v: %v -> %v", podID, cid, oldState, newState) switch newState { case plegContainerRunning: return []*PodLifecycleEvent{{ID: podID, Type: ContainerStarted, Data: cid}} @@ -186,7 +186,7 @@ func (g *GenericPLEG) updateRelistTime(timestamp time.Time) { // relist queries the container runtime for list of pods/containers, compare // with the internal pods/containers, and generates events accordingly. func (g *GenericPLEG) relist() { - glog.V(5).Infof("GenericPLEG: Relisting") + klog.V(5).Infof("GenericPLEG: Relisting") if lastRelistTime := g.getRelistTime(); !lastRelistTime.IsZero() { metrics.PLEGRelistInterval.Observe(metrics.SinceInMicroseconds(lastRelistTime)) @@ -200,7 +200,7 @@ func (g *GenericPLEG) relist() { // Get all the pods. podList, err := g.runtime.GetPods(true) if err != nil { - glog.Errorf("GenericPLEG: Unable to retrieve pods: %v", err) + klog.Errorf("GenericPLEG: Unable to retrieve pods: %v", err) return } @@ -244,7 +244,7 @@ func (g *GenericPLEG) relist() { // serially may take a while. We should be aware of this and // parallelize if needed. if err := g.updateCache(pod, pid); err != nil { - glog.Errorf("PLEG: Ignoring events for pod %s/%s: %v", pod.Name, pod.Namespace, err) + klog.Errorf("PLEG: Ignoring events for pod %s/%s: %v", pod.Name, pod.Namespace, err) // make sure we try to reinspect the pod during the next relisting needsReinspection[pid] = pod @@ -271,10 +271,10 @@ func (g *GenericPLEG) relist() { if g.cacheEnabled() { // reinspect any pods that failed inspection during the previous relist if len(g.podsToReinspect) > 0 { - glog.V(5).Infof("GenericPLEG: Reinspecting pods that previously failed inspection") + klog.V(5).Infof("GenericPLEG: Reinspecting pods that previously failed inspection") for pid, pod := range g.podsToReinspect { if err := g.updateCache(pod, pid); err != nil { - glog.Errorf("PLEG: pod %s/%s failed reinspection: %v", pod.Name, pod.Namespace, err) + klog.Errorf("PLEG: pod %s/%s failed reinspection: %v", pod.Name, pod.Namespace, err) needsReinspection[pid] = pod } } @@ -374,7 +374,7 @@ func (g *GenericPLEG) updateCache(pod *kubecontainer.Pod, pid types.UID) error { if pod == nil { // The pod is missing in the current relist. This means that // the pod has no visible (active or inactive) containers. - glog.V(4).Infof("PLEG: Delete status for pod %q", string(pid)) + klog.V(4).Infof("PLEG: Delete status for pod %q", string(pid)) g.cache.Delete(pid) return nil } @@ -383,7 +383,7 @@ func (g *GenericPLEG) updateCache(pod *kubecontainer.Pod, pid types.UID) error { // GetPodStatus(pod *kubecontainer.Pod) so that Docker can avoid listing // all containers again. status, err := g.runtime.GetPodStatus(pod.ID, pod.Name, pod.Namespace) - glog.V(4).Infof("PLEG: Write status for %s/%s: %#v (err: %v)", pod.Name, pod.Namespace, status, err) + klog.V(4).Infof("PLEG: Write status for %s/%s: %#v (err: %v)", pod.Name, pod.Namespace, status, err) if err == nil { // Preserve the pod IP across cache updates if the new IP is empty. // When a pod is torn down, kubelet may race with PLEG and retrieve diff --git a/pkg/kubelet/pod/BUILD b/pkg/kubelet/pod/BUILD index 4096c025864..8d4811ab459 100644 --- a/pkg/kubelet/pod/BUILD +++ b/pkg/kubelet/pod/BUILD @@ -25,7 +25,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/pod/mirror_client.go b/pkg/kubelet/pod/mirror_client.go index 9b8add5bc5a..b4b6abe61ce 100644 --- a/pkg/kubelet/pod/mirror_client.go +++ b/pkg/kubelet/pod/mirror_client.go @@ -17,11 +17,11 @@ limitations under the License. package pod import ( - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" ) @@ -79,13 +79,13 @@ func (mc *basicMirrorClient) DeleteMirrorPod(podFullName string) error { } name, namespace, err := kubecontainer.ParsePodFullName(podFullName) if err != nil { - glog.Errorf("Failed to parse a pod full name %q", podFullName) + klog.Errorf("Failed to parse a pod full name %q", podFullName) return err } - glog.V(2).Infof("Deleting a mirror pod %q", podFullName) + klog.V(2).Infof("Deleting a mirror pod %q", podFullName) // TODO(random-liu): Delete the mirror pod with uid precondition in mirror pod manager if err := mc.apiserverClient.CoreV1().Pods(namespace).Delete(name, metav1.NewDeleteOptions(0)); err != nil && !errors.IsNotFound(err) { - glog.Errorf("Failed deleting a mirror pod %q: %v", podFullName, err) + klog.Errorf("Failed deleting a mirror pod %q: %v", podFullName, err) } return nil } diff --git a/pkg/kubelet/pod/pod_manager.go b/pkg/kubelet/pod/pod_manager.go index 6033ae8d50a..ce5c1c30c62 100644 --- a/pkg/kubelet/pod/pod_manager.go +++ b/pkg/kubelet/pod/pod_manager.go @@ -19,7 +19,7 @@ package pod import ( "sync" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -163,7 +163,7 @@ func (pm *basicManager) UpdatePod(pod *v1.Pod) { pm.updatePodsInternal(pod) if pm.checkpointManager != nil { if err := checkpoint.WritePod(pm.checkpointManager, pod); err != nil { - glog.Errorf("Error writing checkpoint for pod: %v", pod.GetName()) + klog.Errorf("Error writing checkpoint for pod: %v", pod.GetName()) } } } @@ -226,7 +226,7 @@ func (pm *basicManager) DeletePod(pod *v1.Pod) { } if pm.checkpointManager != nil { if err := checkpoint.DeletePod(pm.checkpointManager, pod); err != nil { - glog.Errorf("Error deleting checkpoint for pod: %v", pod.GetName()) + klog.Errorf("Error deleting checkpoint for pod: %v", pod.GetName()) } } } diff --git a/pkg/kubelet/pod_container_deletor.go b/pkg/kubelet/pod_container_deletor.go index 48a85958db3..0a00ac90698 100644 --- a/pkg/kubelet/pod_container_deletor.go +++ b/pkg/kubelet/pod_container_deletor.go @@ -19,8 +19,8 @@ package kubelet import ( "sort" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -72,7 +72,7 @@ func getContainersToDeleteInPod(filterContainerID string, podStatus *kubecontain }(filterContainerID, podStatus) if filterContainerID != "" && matchedContainer == nil { - glog.Warningf("Container %q not found in pod's containers", filterContainerID) + klog.Warningf("Container %q not found in pod's containers", filterContainerID) return containerStatusbyCreatedList{} } @@ -106,7 +106,7 @@ func (p *podContainerDeletor) deleteContainersInPod(filterContainerID string, po select { case p.worker <- candidate.ID: default: - glog.Warningf("Failed to issue the request to remove container %v", candidate.ID) + klog.Warningf("Failed to issue the request to remove container %v", candidate.ID) } } } diff --git a/pkg/kubelet/pod_workers.go b/pkg/kubelet/pod_workers.go index a42589a99c5..6b7c2244f34 100644 --- a/pkg/kubelet/pod_workers.go +++ b/pkg/kubelet/pod_workers.go @@ -22,12 +22,12 @@ import ( "sync" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/record" + "k8s.io/klog" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/kubelet/eviction" @@ -187,7 +187,7 @@ func (p *podWorkers) managePodLoop(podUpdates <-chan UpdatePodOptions) { } if err != nil { // IMPORTANT: we do not log errors here, the syncPodFn is responsible for logging errors - glog.Errorf("Error syncing pod %s (%q), skipping: %v", update.Pod.UID, format.Pod(update.Pod), err) + klog.Errorf("Error syncing pod %s (%q), skipping: %v", update.Pod.UID, format.Pod(update.Pod), err) } p.wrapUp(update.Pod.UID, err) } diff --git a/pkg/kubelet/preemption/BUILD b/pkg/kubelet/preemption/BUILD index 9e8e44331b8..d1a06818df3 100644 --- a/pkg/kubelet/preemption/BUILD +++ b/pkg/kubelet/preemption/BUILD @@ -22,7 +22,7 @@ go_library( "//pkg/scheduler/algorithm/predicates:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -41,16 +41,21 @@ filegroup( go_test( name = "go_default_test", - srcs = ["preemption_test.go"], + srcs = [ + "main_test.go", + "preemption_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", "//pkg/apis/scheduling:go_default_library", + "//pkg/features:go_default_library", "//pkg/kubelet/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", ], ) diff --git a/pkg/kubelet/preemption/main_test.go b/pkg/kubelet/preemption/main_test.go new file mode 100644 index 00000000000..2878db15c9b --- /dev/null +++ b/pkg/kubelet/preemption/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package preemption + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/kubelet/preemption/preemption.go b/pkg/kubelet/preemption/preemption.go index c7236e2c5f7..94a3afad6b5 100644 --- a/pkg/kubelet/preemption/preemption.go +++ b/pkg/kubelet/preemption/preemption.go @@ -20,9 +20,9 @@ import ( "fmt" "math" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/client-go/tools/record" + "k8s.io/klog" "k8s.io/kubernetes/pkg/api/v1/resource" v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" "k8s.io/kubernetes/pkg/kubelet/events" @@ -96,7 +96,7 @@ func (c *CriticalPodAdmissionHandler) evictPodsToFreeRequests(admitPod *v1.Pod, if err != nil { return fmt.Errorf("preemption: error finding a set of pods to preempt: %v", err) } - glog.Infof("preemption: attempting to evict pods %v, in order to free up resources: %s", podsToPreempt, insufficientResources.toString()) + klog.Infof("preemption: attempting to evict pods %v, in order to free up resources: %s", podsToPreempt, insufficientResources.toString()) for _, pod := range podsToPreempt { status := v1.PodStatus{ Phase: v1.PodFailed, @@ -110,7 +110,7 @@ func (c *CriticalPodAdmissionHandler) evictPodsToFreeRequests(admitPod *v1.Pod, if err != nil { return fmt.Errorf("preemption: pod %s failed to evict %v", format.Pod(pod), err) } - glog.Infof("preemption: pod %s evicted successfully", format.Pod(pod)) + klog.Infof("preemption: pod %s evicted successfully", format.Pod(pod)) } return nil } diff --git a/pkg/kubelet/preemption/preemption_test.go b/pkg/kubelet/preemption/preemption_test.go index 3e85348078a..e0edae5776e 100644 --- a/pkg/kubelet/preemption/preemption_test.go +++ b/pkg/kubelet/preemption/preemption_test.go @@ -24,9 +24,11 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/tools/record" kubeapi "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/scheduling" + "k8s.io/kubernetes/pkg/features" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" ) @@ -89,9 +91,7 @@ func getTestCriticalPodAdmissionHandler(podProvider *fakePodProvider, podKiller } func TestEvictPodsToFreeRequests(t *testing.T) { - if err := utilfeature.DefaultFeatureGate.Set("ExperimentalCriticalPodAnnotation=true"); err != nil { - t.Errorf("failed to set ExperimentalCriticalPodAnnotation to true: %v", err) - } + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() type testRun struct { testName string inputPods []*v1.Pod @@ -159,6 +159,7 @@ func BenchmarkGetPodsToPreempt(t *testing.B) { } func TestGetPodsToPreempt(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() type testRun struct { testName string preemptor *v1.Pod diff --git a/pkg/kubelet/prober/BUILD b/pkg/kubelet/prober/BUILD index 78394bf7289..18922e59e33 100644 --- a/pkg/kubelet/prober/BUILD +++ b/pkg/kubelet/prober/BUILD @@ -32,8 +32,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) @@ -64,7 +64,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/kubelet/prober/prober.go b/pkg/kubelet/prober/prober.go index 77cea646031..efec60c98d2 100644 --- a/pkg/kubelet/prober/prober.go +++ b/pkg/kubelet/prober/prober.go @@ -39,7 +39,7 @@ import ( tcprobe "k8s.io/kubernetes/pkg/probe/tcp" "k8s.io/utils/exec" - "github.com/golang/glog" + "k8s.io/klog" ) const maxProbeRetries = 3 @@ -91,7 +91,7 @@ func (pb *prober) probe(probeType probeType, pod *v1.Pod, status v1.PodStatus, c ctrName := fmt.Sprintf("%s:%s", format.Pod(pod), container.Name) if probeSpec == nil { - glog.Warningf("%s probe for %s is nil", probeType, ctrName) + klog.Warningf("%s probe for %s is nil", probeType, ctrName) return results.Success, nil } @@ -100,22 +100,22 @@ func (pb *prober) probe(probeType probeType, pod *v1.Pod, status v1.PodStatus, c // Probe failed in one way or another. ref, hasRef := pb.refManager.GetRef(containerID) if !hasRef { - glog.Warningf("No ref for container %q (%s)", containerID.String(), ctrName) + klog.Warningf("No ref for container %q (%s)", containerID.String(), ctrName) } if err != nil { - glog.V(1).Infof("%s probe for %q errored: %v", probeType, ctrName, err) + klog.V(1).Infof("%s probe for %q errored: %v", probeType, ctrName, err) if hasRef { pb.recorder.Eventf(ref, v1.EventTypeWarning, events.ContainerUnhealthy, "%s probe errored: %v", probeType, err) } } else { // result != probe.Success - glog.V(1).Infof("%s probe for %q failed (%v): %s", probeType, ctrName, result, output) + klog.V(1).Infof("%s probe for %q failed (%v): %s", probeType, ctrName, result, output) if hasRef { pb.recorder.Eventf(ref, v1.EventTypeWarning, events.ContainerUnhealthy, "%s probe failed: %s", probeType, output) } } return results.Failure, err } - glog.V(3).Infof("%s probe for %q succeeded", probeType, ctrName) + klog.V(3).Infof("%s probe for %q succeeded", probeType, ctrName) return results.Success, nil } @@ -147,7 +147,7 @@ func buildHeader(headerList []v1.HTTPHeader) http.Header { func (pb *prober) runProbe(probeType probeType, p *v1.Probe, pod *v1.Pod, status v1.PodStatus, container v1.Container, containerID kubecontainer.ContainerID) (probe.Result, string, error) { timeout := time.Duration(p.TimeoutSeconds) * time.Second if p.Exec != nil { - glog.V(4).Infof("Exec-Probe Pod: %v, Container: %v, Command: %v", pod, container, p.Exec.Command) + klog.V(4).Infof("Exec-Probe Pod: %v, Container: %v, Command: %v", pod, container, p.Exec.Command) command := kubecontainer.ExpandContainerCommandOnlyStatic(p.Exec.Command, container.Env) return pb.exec.Probe(pb.newExecInContainer(container, containerID, command, timeout)) } @@ -162,10 +162,10 @@ func (pb *prober) runProbe(probeType probeType, p *v1.Probe, pod *v1.Pod, status return probe.Unknown, "", err } path := p.HTTPGet.Path - glog.V(4).Infof("HTTP-Probe Host: %v://%v, Port: %v, Path: %v", scheme, host, port, path) + klog.V(4).Infof("HTTP-Probe Host: %v://%v, Port: %v, Path: %v", scheme, host, port, path) url := formatURL(scheme, host, port, path) headers := buildHeader(p.HTTPGet.HTTPHeaders) - glog.V(4).Infof("HTTP-Probe Headers: %v", headers) + klog.V(4).Infof("HTTP-Probe Headers: %v", headers) if probeType == liveness { return pb.livenessHttp.Probe(url, headers, timeout) } else { // readiness @@ -181,10 +181,10 @@ func (pb *prober) runProbe(probeType probeType, p *v1.Probe, pod *v1.Pod, status if host == "" { host = status.PodIP } - glog.V(4).Infof("TCP-Probe Host: %v, Port: %v, Timeout: %v", host, port, timeout) + klog.V(4).Infof("TCP-Probe Host: %v, Port: %v, Timeout: %v", host, port, timeout) return pb.tcp.Probe(host, port, timeout) } - glog.Warningf("Failed to find probe builder for container: %v", container) + klog.Warningf("Failed to find probe builder for container: %v", container) return probe.Unknown, "", fmt.Errorf("Missing probe handler for %s:%s", format.Pod(pod), container.Name) } diff --git a/pkg/kubelet/prober/prober_manager.go b/pkg/kubelet/prober/prober_manager.go index 0e53e094076..a913598ef69 100644 --- a/pkg/kubelet/prober/prober_manager.go +++ b/pkg/kubelet/prober/prober_manager.go @@ -19,13 +19,13 @@ package prober import ( "sync" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/record" + "k8s.io/klog" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/prober/results" "k8s.io/kubernetes/pkg/kubelet/status" @@ -149,7 +149,7 @@ func (m *manager) AddPod(pod *v1.Pod) { if c.ReadinessProbe != nil { key.probeType = readiness if _, ok := m.workers[key]; ok { - glog.Errorf("Readiness probe already exists! %v - %v", + klog.Errorf("Readiness probe already exists! %v - %v", format.Pod(pod), c.Name) return } @@ -161,7 +161,7 @@ func (m *manager) AddPod(pod *v1.Pod) { if c.LivenessProbe != nil { key.probeType = liveness if _, ok := m.workers[key]; ok { - glog.Errorf("Liveness probe already exists! %v - %v", + klog.Errorf("Liveness probe already exists! %v - %v", format.Pod(pod), c.Name) return } diff --git a/pkg/kubelet/prober/prober_manager_test.go b/pkg/kubelet/prober/prober_manager_test.go index f684a4ff599..2ea5d63fbc9 100644 --- a/pkg/kubelet/prober/prober_manager_test.go +++ b/pkg/kubelet/prober/prober_manager_test.go @@ -22,12 +22,12 @@ import ( "testing" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/prober/results" "k8s.io/kubernetes/pkg/probe" @@ -359,7 +359,7 @@ func waitForWorkerExit(m *manager, workerPaths []probeKey) error { if exited, _ := condition(); exited { continue // Already exited, no need to poll. } - glog.Infof("Polling %v", w) + klog.Infof("Polling %v", w) if err := wait.Poll(interval, wait.ForeverTestTimeout, condition); err != nil { return err } @@ -384,7 +384,7 @@ func waitForReadyStatus(m *manager, ready bool) error { } return status.ContainerStatuses[0].Ready == ready, nil } - glog.Infof("Polling for ready state %v", ready) + klog.Infof("Polling for ready state %v", ready) if err := wait.Poll(interval, wait.ForeverTestTimeout, condition); err != nil { return err } @@ -399,7 +399,7 @@ func cleanup(t *testing.T, m *manager) { condition := func() (bool, error) { workerCount := m.workerCount() if workerCount > 0 { - glog.Infof("Waiting for %d workers to exit...", workerCount) + klog.Infof("Waiting for %d workers to exit...", workerCount) } return workerCount == 0, nil } diff --git a/pkg/kubelet/prober/worker.go b/pkg/kubelet/prober/worker.go index cdefc1da2c6..0602419d7b6 100644 --- a/pkg/kubelet/prober/worker.go +++ b/pkg/kubelet/prober/worker.go @@ -20,10 +20,10 @@ import ( "math/rand" "time" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog" podutil "k8s.io/kubernetes/pkg/api/v1/pod" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/prober/results" @@ -160,13 +160,13 @@ func (w *worker) doProbe() (keepGoing bool) { status, ok := w.probeManager.statusManager.GetPodStatus(w.pod.UID) if !ok { // Either the pod has not been created yet, or it was already deleted. - glog.V(3).Infof("No status for pod: %v", format.Pod(w.pod)) + klog.V(3).Infof("No status for pod: %v", format.Pod(w.pod)) return true } // Worker should terminate if pod is terminated. if status.Phase == v1.PodFailed || status.Phase == v1.PodSucceeded { - glog.V(3).Infof("Pod %v %v, exiting probe worker", + klog.V(3).Infof("Pod %v %v, exiting probe worker", format.Pod(w.pod), status.Phase) return false } @@ -174,7 +174,7 @@ func (w *worker) doProbe() (keepGoing bool) { c, ok := podutil.GetContainerStatus(status.ContainerStatuses, w.container.Name) if !ok || len(c.ContainerID) == 0 { // Either the container has not been created yet, or it was deleted. - glog.V(3).Infof("Probe target container not found: %v - %v", + klog.V(3).Infof("Probe target container not found: %v - %v", format.Pod(w.pod), w.container.Name) return true // Wait for more information. } @@ -195,7 +195,7 @@ func (w *worker) doProbe() (keepGoing bool) { } if c.State.Running == nil { - glog.V(3).Infof("Non-running container probed: %v - %v", + klog.V(3).Infof("Non-running container probed: %v - %v", format.Pod(w.pod), w.container.Name) if !w.containerID.IsEmpty() { w.resultsManager.Set(w.containerID, results.Failure, w.pod) diff --git a/pkg/kubelet/remote/BUILD b/pkg/kubelet/remote/BUILD index 05c6efd271a..cf19206ce2b 100644 --- a/pkg/kubelet/remote/BUILD +++ b/pkg/kubelet/remote/BUILD @@ -19,8 +19,8 @@ go_library( "//pkg/kubelet/apis/cri:go_default_library", "//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library", "//pkg/kubelet/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/kubelet/remote/remote_image.go b/pkg/kubelet/remote/remote_image.go index a6bc88f7cc4..17b90574dc0 100644 --- a/pkg/kubelet/remote/remote_image.go +++ b/pkg/kubelet/remote/remote_image.go @@ -22,8 +22,8 @@ import ( "fmt" "time" - "github.com/golang/glog" "google.golang.org/grpc" + "k8s.io/klog" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" @@ -38,7 +38,7 @@ type RemoteImageService struct { // NewRemoteImageService creates a new internalapi.ImageManagerService. func NewRemoteImageService(endpoint string, connectionTimeout time.Duration) (internalapi.ImageManagerService, error) { - glog.V(3).Infof("Connecting to image service %s", endpoint) + klog.V(3).Infof("Connecting to image service %s", endpoint) addr, dailer, err := util.GetAddressAndDialer(endpoint) if err != nil { return nil, err @@ -49,7 +49,7 @@ func NewRemoteImageService(endpoint string, connectionTimeout time.Duration) (in conn, err := grpc.DialContext(ctx, addr, grpc.WithInsecure(), grpc.WithDialer(dailer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize))) if err != nil { - glog.Errorf("Connect remote image service %s failed: %v", addr, err) + klog.Errorf("Connect remote image service %s failed: %v", addr, err) return nil, err } @@ -68,7 +68,7 @@ func (r *RemoteImageService) ListImages(filter *runtimeapi.ImageFilter) ([]*runt Filter: filter, }) if err != nil { - glog.Errorf("ListImages with filter %+v from image service failed: %v", filter, err) + klog.Errorf("ListImages with filter %+v from image service failed: %v", filter, err) return nil, err } @@ -84,14 +84,14 @@ func (r *RemoteImageService) ImageStatus(image *runtimeapi.ImageSpec) (*runtimea Image: image, }) if err != nil { - glog.Errorf("ImageStatus %q from image service failed: %v", image.Image, err) + klog.Errorf("ImageStatus %q from image service failed: %v", image.Image, err) return nil, err } if resp.Image != nil { if resp.Image.Id == "" || resp.Image.Size_ == 0 { errorMessage := fmt.Sprintf("Id or size of image %q is not set", image.Image) - glog.Errorf("ImageStatus failed: %s", errorMessage) + klog.Errorf("ImageStatus failed: %s", errorMessage) return nil, errors.New(errorMessage) } } @@ -109,13 +109,13 @@ func (r *RemoteImageService) PullImage(image *runtimeapi.ImageSpec, auth *runtim Auth: auth, }) if err != nil { - glog.Errorf("PullImage %q from image service failed: %v", image.Image, err) + klog.Errorf("PullImage %q from image service failed: %v", image.Image, err) return "", err } if resp.ImageRef == "" { errorMessage := fmt.Sprintf("imageRef of image %q is not set", image.Image) - glog.Errorf("PullImage failed: %s", errorMessage) + klog.Errorf("PullImage failed: %s", errorMessage) return "", errors.New(errorMessage) } @@ -131,7 +131,7 @@ func (r *RemoteImageService) RemoveImage(image *runtimeapi.ImageSpec) error { Image: image, }) if err != nil { - glog.Errorf("RemoveImage %q from image service failed: %v", image.Image, err) + klog.Errorf("RemoveImage %q from image service failed: %v", image.Image, err) return err } @@ -147,7 +147,7 @@ func (r *RemoteImageService) ImageFsInfo() ([]*runtimeapi.FilesystemUsage, error resp, err := r.imageClient.ImageFsInfo(ctx, &runtimeapi.ImageFsInfoRequest{}) if err != nil { - glog.Errorf("ImageFsInfo from image service failed: %v", err) + klog.Errorf("ImageFsInfo from image service failed: %v", err) return nil, err } return resp.GetImageFilesystems(), nil diff --git a/pkg/kubelet/remote/remote_runtime.go b/pkg/kubelet/remote/remote_runtime.go index f447b850e15..16e16daff84 100644 --- a/pkg/kubelet/remote/remote_runtime.go +++ b/pkg/kubelet/remote/remote_runtime.go @@ -23,8 +23,8 @@ import ( "strings" "time" - "github.com/golang/glog" "google.golang.org/grpc" + "k8s.io/klog" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" @@ -40,7 +40,7 @@ type RemoteRuntimeService struct { // NewRemoteRuntimeService creates a new internalapi.RuntimeService. func NewRemoteRuntimeService(endpoint string, connectionTimeout time.Duration) (internalapi.RuntimeService, error) { - glog.V(3).Infof("Connecting to runtime service %s", endpoint) + klog.V(3).Infof("Connecting to runtime service %s", endpoint) addr, dailer, err := util.GetAddressAndDialer(endpoint) if err != nil { return nil, err @@ -50,7 +50,7 @@ func NewRemoteRuntimeService(endpoint string, connectionTimeout time.Duration) ( conn, err := grpc.DialContext(ctx, addr, grpc.WithInsecure(), grpc.WithDialer(dailer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize))) if err != nil { - glog.Errorf("Connect remote runtime %s failed: %v", addr, err) + klog.Errorf("Connect remote runtime %s failed: %v", addr, err) return nil, err } @@ -69,7 +69,7 @@ func (r *RemoteRuntimeService) Version(apiVersion string) (*runtimeapi.VersionRe Version: apiVersion, }) if err != nil { - glog.Errorf("Version from runtime service failed: %v", err) + klog.Errorf("Version from runtime service failed: %v", err) return nil, err } @@ -93,13 +93,13 @@ func (r *RemoteRuntimeService) RunPodSandbox(config *runtimeapi.PodSandboxConfig RuntimeHandler: runtimeHandler, }) if err != nil { - glog.Errorf("RunPodSandbox from runtime service failed: %v", err) + klog.Errorf("RunPodSandbox from runtime service failed: %v", err) return "", err } if resp.PodSandboxId == "" { errorMessage := fmt.Sprintf("PodSandboxId is not set for sandbox %q", config.GetMetadata()) - glog.Errorf("RunPodSandbox failed: %s", errorMessage) + klog.Errorf("RunPodSandbox failed: %s", errorMessage) return "", errors.New(errorMessage) } @@ -116,7 +116,7 @@ func (r *RemoteRuntimeService) StopPodSandbox(podSandBoxID string) error { PodSandboxId: podSandBoxID, }) if err != nil { - glog.Errorf("StopPodSandbox %q from runtime service failed: %v", podSandBoxID, err) + klog.Errorf("StopPodSandbox %q from runtime service failed: %v", podSandBoxID, err) return err } @@ -133,7 +133,7 @@ func (r *RemoteRuntimeService) RemovePodSandbox(podSandBoxID string) error { PodSandboxId: podSandBoxID, }) if err != nil { - glog.Errorf("RemovePodSandbox %q from runtime service failed: %v", podSandBoxID, err) + klog.Errorf("RemovePodSandbox %q from runtime service failed: %v", podSandBoxID, err) return err } @@ -170,7 +170,7 @@ func (r *RemoteRuntimeService) ListPodSandbox(filter *runtimeapi.PodSandboxFilte Filter: filter, }) if err != nil { - glog.Errorf("ListPodSandbox with filter %+v from runtime service failed: %v", filter, err) + klog.Errorf("ListPodSandbox with filter %+v from runtime service failed: %v", filter, err) return nil, err } @@ -188,13 +188,13 @@ func (r *RemoteRuntimeService) CreateContainer(podSandBoxID string, config *runt SandboxConfig: sandboxConfig, }) if err != nil { - glog.Errorf("CreateContainer in sandbox %q from runtime service failed: %v", podSandBoxID, err) + klog.Errorf("CreateContainer in sandbox %q from runtime service failed: %v", podSandBoxID, err) return "", err } if resp.ContainerId == "" { errorMessage := fmt.Sprintf("ContainerId is not set for container %q", config.GetMetadata()) - glog.Errorf("CreateContainer failed: %s", errorMessage) + klog.Errorf("CreateContainer failed: %s", errorMessage) return "", errors.New(errorMessage) } @@ -210,7 +210,7 @@ func (r *RemoteRuntimeService) StartContainer(containerID string) error { ContainerId: containerID, }) if err != nil { - glog.Errorf("StartContainer %q from runtime service failed: %v", containerID, err) + klog.Errorf("StartContainer %q from runtime service failed: %v", containerID, err) return err } @@ -230,7 +230,7 @@ func (r *RemoteRuntimeService) StopContainer(containerID string, timeout int64) Timeout: timeout, }) if err != nil { - glog.Errorf("StopContainer %q from runtime service failed: %v", containerID, err) + klog.Errorf("StopContainer %q from runtime service failed: %v", containerID, err) return err } @@ -247,7 +247,7 @@ func (r *RemoteRuntimeService) RemoveContainer(containerID string) error { ContainerId: containerID, }) if err != nil { - glog.Errorf("RemoveContainer %q from runtime service failed: %v", containerID, err) + klog.Errorf("RemoveContainer %q from runtime service failed: %v", containerID, err) return err } @@ -263,7 +263,7 @@ func (r *RemoteRuntimeService) ListContainers(filter *runtimeapi.ContainerFilter Filter: filter, }) if err != nil { - glog.Errorf("ListContainers with filter %+v from runtime service failed: %v", filter, err) + klog.Errorf("ListContainers with filter %+v from runtime service failed: %v", filter, err) return nil, err } @@ -279,13 +279,13 @@ func (r *RemoteRuntimeService) ContainerStatus(containerID string) (*runtimeapi. ContainerId: containerID, }) if err != nil { - glog.Errorf("ContainerStatus %q from runtime service failed: %v", containerID, err) + klog.Errorf("ContainerStatus %q from runtime service failed: %v", containerID, err) return nil, err } if resp.Status != nil { if err := verifyContainerStatus(resp.Status); err != nil { - glog.Errorf("ContainerStatus of %q failed: %v", containerID, err) + klog.Errorf("ContainerStatus of %q failed: %v", containerID, err) return nil, err } } @@ -303,7 +303,7 @@ func (r *RemoteRuntimeService) UpdateContainerResources(containerID string, reso Linux: resources, }) if err != nil { - glog.Errorf("UpdateContainerResources %q from runtime service failed: %v", containerID, err) + klog.Errorf("UpdateContainerResources %q from runtime service failed: %v", containerID, err) return err } @@ -333,7 +333,7 @@ func (r *RemoteRuntimeService) ExecSync(containerID string, cmd []string, timeou } resp, err := r.runtimeClient.ExecSync(ctx, req) if err != nil { - glog.Errorf("ExecSync %s '%s' from runtime service failed: %v", containerID, strings.Join(cmd, " "), err) + klog.Errorf("ExecSync %s '%s' from runtime service failed: %v", containerID, strings.Join(cmd, " "), err) return nil, nil, err } @@ -355,13 +355,13 @@ func (r *RemoteRuntimeService) Exec(req *runtimeapi.ExecRequest) (*runtimeapi.Ex resp, err := r.runtimeClient.Exec(ctx, req) if err != nil { - glog.Errorf("Exec %s '%s' from runtime service failed: %v", req.ContainerId, strings.Join(req.Cmd, " "), err) + klog.Errorf("Exec %s '%s' from runtime service failed: %v", req.ContainerId, strings.Join(req.Cmd, " "), err) return nil, err } if resp.Url == "" { errorMessage := "URL is not set" - glog.Errorf("Exec failed: %s", errorMessage) + klog.Errorf("Exec failed: %s", errorMessage) return nil, errors.New(errorMessage) } @@ -375,13 +375,13 @@ func (r *RemoteRuntimeService) Attach(req *runtimeapi.AttachRequest) (*runtimeap resp, err := r.runtimeClient.Attach(ctx, req) if err != nil { - glog.Errorf("Attach %s from runtime service failed: %v", req.ContainerId, err) + klog.Errorf("Attach %s from runtime service failed: %v", req.ContainerId, err) return nil, err } if resp.Url == "" { errorMessage := "URL is not set" - glog.Errorf("Exec failed: %s", errorMessage) + klog.Errorf("Exec failed: %s", errorMessage) return nil, errors.New(errorMessage) } return resp, nil @@ -394,13 +394,13 @@ func (r *RemoteRuntimeService) PortForward(req *runtimeapi.PortForwardRequest) ( resp, err := r.runtimeClient.PortForward(ctx, req) if err != nil { - glog.Errorf("PortForward %s from runtime service failed: %v", req.PodSandboxId, err) + klog.Errorf("PortForward %s from runtime service failed: %v", req.PodSandboxId, err) return nil, err } if resp.Url == "" { errorMessage := "URL is not set" - glog.Errorf("Exec failed: %s", errorMessage) + klog.Errorf("Exec failed: %s", errorMessage) return nil, errors.New(errorMessage) } @@ -435,13 +435,13 @@ func (r *RemoteRuntimeService) Status() (*runtimeapi.RuntimeStatus, error) { resp, err := r.runtimeClient.Status(ctx, &runtimeapi.StatusRequest{}) if err != nil { - glog.Errorf("Status from runtime service failed: %v", err) + klog.Errorf("Status from runtime service failed: %v", err) return nil, err } if resp.Status == nil || len(resp.Status.Conditions) < 2 { errorMessage := "RuntimeReady or NetworkReady condition are not set" - glog.Errorf("Status failed: %s", errorMessage) + klog.Errorf("Status failed: %s", errorMessage) return nil, errors.New(errorMessage) } @@ -457,7 +457,7 @@ func (r *RemoteRuntimeService) ContainerStats(containerID string) (*runtimeapi.C ContainerId: containerID, }) if err != nil { - glog.Errorf("ContainerStatus %q from runtime service failed: %v", containerID, err) + klog.Errorf("ContainerStatus %q from runtime service failed: %v", containerID, err) return nil, err } @@ -474,7 +474,7 @@ func (r *RemoteRuntimeService) ListContainerStats(filter *runtimeapi.ContainerSt Filter: filter, }) if err != nil { - glog.Errorf("ListContainerStats with filter %+v from runtime service failed: %v", filter, err) + klog.Errorf("ListContainerStats with filter %+v from runtime service failed: %v", filter, err) return nil, err } @@ -487,7 +487,7 @@ func (r *RemoteRuntimeService) ReopenContainerLog(containerID string) error { _, err := r.runtimeClient.ReopenContainerLog(ctx, &runtimeapi.ReopenContainerLogRequest{ContainerId: containerID}) if err != nil { - glog.Errorf("ReopenContainerLog %q from runtime service failed: %v", containerID, err) + klog.Errorf("ReopenContainerLog %q from runtime service failed: %v", containerID, err) return err } return nil diff --git a/pkg/kubelet/runonce.go b/pkg/kubelet/runonce.go index 294580e7a09..86c1f36f0c7 100644 --- a/pkg/kubelet/runonce.go +++ b/pkg/kubelet/runonce.go @@ -21,8 +21,8 @@ import ( "os" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" + "k8s.io/klog" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/kubelet/util/format" @@ -51,15 +51,15 @@ func (kl *Kubelet) RunOnce(updates <-chan kubetypes.PodUpdate) ([]RunPodResult, // If the container logs directory does not exist, create it. if _, err := os.Stat(ContainerLogsDir); err != nil { if err := kl.os.MkdirAll(ContainerLogsDir, 0755); err != nil { - glog.Errorf("Failed to create directory %q: %v", ContainerLogsDir, err) + klog.Errorf("Failed to create directory %q: %v", ContainerLogsDir, err) } } select { case u := <-updates: - glog.Infof("processing manifest with %d pods", len(u.Pods)) + klog.Infof("processing manifest with %d pods", len(u.Pods)) result, err := kl.runOnce(u.Pods, runOnceRetryDelay) - glog.Infof("finished processing %d pods", len(u.Pods)) + klog.Infof("finished processing %d pods", len(u.Pods)) return result, err case <-time.After(runOnceManifestDelay): return nil, fmt.Errorf("no pod manifest update after %v", runOnceManifestDelay) @@ -85,7 +85,7 @@ func (kl *Kubelet) runOnce(pods []*v1.Pod, retryDelay time.Duration) (results [] }(pod) } - glog.Infof("Waiting for %d pods", len(admitted)) + klog.Infof("Waiting for %d pods", len(admitted)) failedPods := []string{} for i := 0; i < len(admitted); i++ { res := <-ch @@ -93,19 +93,19 @@ func (kl *Kubelet) runOnce(pods []*v1.Pod, retryDelay time.Duration) (results [] if res.Err != nil { faliedContainerName, err := kl.getFailedContainers(res.Pod) if err != nil { - glog.Infof("unable to get failed containers' names for pod %q, error:%v", format.Pod(res.Pod), err) + klog.Infof("unable to get failed containers' names for pod %q, error:%v", format.Pod(res.Pod), err) } else { - glog.Infof("unable to start pod %q because container:%v failed", format.Pod(res.Pod), faliedContainerName) + klog.Infof("unable to start pod %q because container:%v failed", format.Pod(res.Pod), faliedContainerName) } failedPods = append(failedPods, format.Pod(res.Pod)) } else { - glog.Infof("started pod %q", format.Pod(res.Pod)) + klog.Infof("started pod %q", format.Pod(res.Pod)) } } if len(failedPods) > 0 { return results, fmt.Errorf("error running pods: %v", failedPods) } - glog.Infof("%d pods started", len(pods)) + klog.Infof("%d pods started", len(pods)) return results, err } @@ -120,14 +120,14 @@ func (kl *Kubelet) runPod(pod *v1.Pod, retryDelay time.Duration) error { } if kl.isPodRunning(pod, status) { - glog.Infof("pod %q containers running", format.Pod(pod)) + klog.Infof("pod %q containers running", format.Pod(pod)) return nil } - glog.Infof("pod %q containers not running: syncing", format.Pod(pod)) + klog.Infof("pod %q containers not running: syncing", format.Pod(pod)) - glog.Infof("Creating a mirror pod for static pod %q", format.Pod(pod)) + klog.Infof("Creating a mirror pod for static pod %q", format.Pod(pod)) if err := kl.podManager.CreateMirrorPod(pod); err != nil { - glog.Errorf("Failed creating a mirror pod %q: %v", format.Pod(pod), err) + klog.Errorf("Failed creating a mirror pod %q: %v", format.Pod(pod), err) } mirrorPod, _ := kl.podManager.GetMirrorPodByPod(pod) if err = kl.syncPod(syncPodOptions{ @@ -142,7 +142,7 @@ func (kl *Kubelet) runPod(pod *v1.Pod, retryDelay time.Duration) error { return fmt.Errorf("timeout error: pod %q containers not running after %d retries", format.Pod(pod), runOnceMaxRetries) } // TODO(proppy): health checking would be better than waiting + checking the state at the next iteration. - glog.Infof("pod %q containers synced, waiting for %v", format.Pod(pod), delay) + klog.Infof("pod %q containers synced, waiting for %v", format.Pod(pod), delay) time.Sleep(delay) retry++ delay *= runOnceRetryDelayBackoff @@ -154,7 +154,7 @@ func (kl *Kubelet) isPodRunning(pod *v1.Pod, status *kubecontainer.PodStatus) bo for _, c := range pod.Spec.Containers { cs := status.FindContainerStatusByName(c.Name) if cs == nil || cs.State != kubecontainer.ContainerStateRunning { - glog.Infof("Container %q for pod %q not running", c.Name, format.Pod(pod)) + klog.Infof("Container %q for pod %q not running", c.Name, format.Pod(pod)) return false } } diff --git a/pkg/kubelet/server/BUILD b/pkg/kubelet/server/BUILD index 2a9fd9efee9..595bca6af35 100644 --- a/pkg/kubelet/server/BUILD +++ b/pkg/kubelet/server/BUILD @@ -18,6 +18,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/core:go_default_library", "//pkg/apis/core/v1/validation:go_default_library", + "//pkg/kubelet/apis/podresources:go_default_library", + "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/prober:go_default_library", "//pkg/kubelet/server/portforward:go_default_library", @@ -25,6 +27,7 @@ go_library( "//pkg/kubelet/server/stats:go_default_library", "//pkg/kubelet/server/streaming:go_default_library", "//pkg/kubelet/types:go_default_library", + "//pkg/kubelet/util:go_default_library", "//pkg/util/configz:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -42,12 +45,13 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/flushwriter:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/container:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/metrics:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus/promhttp:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/server/auth.go b/pkg/kubelet/server/auth.go index 68e483f6ce0..8b8805cf5cf 100644 --- a/pkg/kubelet/server/auth.go +++ b/pkg/kubelet/server/auth.go @@ -20,11 +20,11 @@ import ( "net/http" "strings" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/klog" ) // KubeletAuth implements AuthInterface @@ -108,7 +108,7 @@ func (n nodeAuthorizerAttributesGetter) GetRequestAttributes(u user.Info, r *htt attrs.Subresource = "spec" } - glog.V(5).Infof("Node request attributes: user=%#v attrs=%#v", attrs.GetUser(), attrs) + klog.V(5).Infof("Node request attributes: user=%#v attrs=%#v", attrs.GetUser(), attrs) return attrs } diff --git a/pkg/kubelet/server/portforward/BUILD b/pkg/kubelet/server/portforward/BUILD index 07541328a3a..46c57de81bf 100644 --- a/pkg/kubelet/server/portforward/BUILD +++ b/pkg/kubelet/server/portforward/BUILD @@ -23,7 +23,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/httplog:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/wsstream:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/server/portforward/httpstream.go b/pkg/kubelet/server/portforward/httpstream.go index 06ed961e0e2..43393bd57ae 100644 --- a/pkg/kubelet/server/portforward/httpstream.go +++ b/pkg/kubelet/server/portforward/httpstream.go @@ -30,7 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" api "k8s.io/kubernetes/pkg/apis/core" - "github.com/golang/glog" + "k8s.io/klog" ) func handleHttpStreams(req *http.Request, w http.ResponseWriter, portForwarder PortForwarder, podName string, uid types.UID, supportedPortForwardProtocols []string, idleTimeout, streamCreationTimeout time.Duration) error { @@ -42,7 +42,7 @@ func handleHttpStreams(req *http.Request, w http.ResponseWriter, portForwarder P } streamChan := make(chan httpstream.Stream, 1) - glog.V(5).Infof("Upgrading port forward response") + klog.V(5).Infof("Upgrading port forward response") upgrader := spdy.NewResponseUpgrader() conn := upgrader.UpgradeResponse(w, req, httpStreamReceived(streamChan)) if conn == nil { @@ -50,7 +50,7 @@ func handleHttpStreams(req *http.Request, w http.ResponseWriter, portForwarder P } defer conn.Close() - glog.V(5).Infof("(conn=%p) setting port forwarding streaming connection idle timeout to %v", conn, idleTimeout) + klog.V(5).Infof("(conn=%p) setting port forwarding streaming connection idle timeout to %v", conn, idleTimeout) conn.SetIdleTimeout(idleTimeout) h := &httpStreamHandler{ @@ -121,11 +121,11 @@ func (h *httpStreamHandler) getStreamPair(requestID string) (*httpStreamPair, bo defer h.streamPairsLock.Unlock() if p, ok := h.streamPairs[requestID]; ok { - glog.V(5).Infof("(conn=%p, request=%s) found existing stream pair", h.conn, requestID) + klog.V(5).Infof("(conn=%p, request=%s) found existing stream pair", h.conn, requestID) return p, false } - glog.V(5).Infof("(conn=%p, request=%s) creating new stream pair", h.conn, requestID) + klog.V(5).Infof("(conn=%p, request=%s) creating new stream pair", h.conn, requestID) p := newPortForwardPair(requestID) h.streamPairs[requestID] = p @@ -143,7 +143,7 @@ func (h *httpStreamHandler) monitorStreamPair(p *httpStreamPair, timeout <-chan utilruntime.HandleError(err) p.printError(err.Error()) case <-p.complete: - glog.V(5).Infof("(conn=%v, request=%s) successfully received error and data streams", h.conn, p.requestID) + klog.V(5).Infof("(conn=%v, request=%s) successfully received error and data streams", h.conn, p.requestID) } h.removeStreamPair(p.requestID) } @@ -170,7 +170,7 @@ func (h *httpStreamHandler) removeStreamPair(requestID string) { func (h *httpStreamHandler) requestID(stream httpstream.Stream) string { requestID := stream.Headers().Get(api.PortForwardRequestIDHeader) if len(requestID) == 0 { - glog.V(5).Infof("(conn=%p) stream received without %s header", h.conn, api.PortForwardRequestIDHeader) + klog.V(5).Infof("(conn=%p) stream received without %s header", h.conn, api.PortForwardRequestIDHeader) // If we get here, it's because the connection came from an older client // that isn't generating the request id header // (https://github.com/kubernetes/kubernetes/blob/843134885e7e0b360eb5441e85b1410a8b1a7a0c/pkg/client/unversioned/portforward/portforward.go#L258-L287) @@ -197,7 +197,7 @@ func (h *httpStreamHandler) requestID(stream httpstream.Stream) string { requestID = strconv.Itoa(int(stream.Identifier()) - 2) } - glog.V(5).Infof("(conn=%p) automatically assigning request ID=%q from stream type=%s, stream ID=%d", h.conn, requestID, streamType, stream.Identifier()) + klog.V(5).Infof("(conn=%p) automatically assigning request ID=%q from stream type=%s, stream ID=%d", h.conn, requestID, streamType, stream.Identifier()) } return requestID } @@ -206,17 +206,17 @@ func (h *httpStreamHandler) requestID(stream httpstream.Stream) string { // streams, invoking portForward for each complete stream pair. The loop exits // when the httpstream.Connection is closed. func (h *httpStreamHandler) run() { - glog.V(5).Infof("(conn=%p) waiting for port forward streams", h.conn) + klog.V(5).Infof("(conn=%p) waiting for port forward streams", h.conn) Loop: for { select { case <-h.conn.CloseChan(): - glog.V(5).Infof("(conn=%p) upgraded connection closed", h.conn) + klog.V(5).Infof("(conn=%p) upgraded connection closed", h.conn) break Loop case stream := <-h.streamChan: requestID := h.requestID(stream) streamType := stream.Headers().Get(api.StreamType) - glog.V(5).Infof("(conn=%p, request=%s) received new stream of type %s", h.conn, requestID, streamType) + klog.V(5).Infof("(conn=%p, request=%s) received new stream of type %s", h.conn, requestID, streamType) p, created := h.getStreamPair(requestID) if created { @@ -242,9 +242,9 @@ func (h *httpStreamHandler) portForward(p *httpStreamPair) { portString := p.dataStream.Headers().Get(api.PortHeader) port, _ := strconv.ParseInt(portString, 10, 32) - glog.V(5).Infof("(conn=%p, request=%s) invoking forwarder.PortForward for port %s", h.conn, p.requestID, portString) + klog.V(5).Infof("(conn=%p, request=%s) invoking forwarder.PortForward for port %s", h.conn, p.requestID, portString) err := h.forwarder.PortForward(h.pod, h.uid, int32(port), p.dataStream) - glog.V(5).Infof("(conn=%p, request=%s) done invoking forwarder.PortForward for port %s", h.conn, p.requestID, portString) + klog.V(5).Infof("(conn=%p, request=%s) done invoking forwarder.PortForward for port %s", h.conn, p.requestID, portString) if err != nil { msg := fmt.Errorf("error forwarding port %d to pod %s, uid %v: %v", port, h.pod, h.uid, err) diff --git a/pkg/kubelet/server/portforward/websocket.go b/pkg/kubelet/server/portforward/websocket.go index 8bd6eb70980..1b23d74b519 100644 --- a/pkg/kubelet/server/portforward/websocket.go +++ b/pkg/kubelet/server/portforward/websocket.go @@ -26,7 +26,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/runtime" @@ -185,9 +185,9 @@ func (h *websocketStreamHandler) portForward(p *websocketStreamPair) { defer p.dataStream.Close() defer p.errorStream.Close() - glog.V(5).Infof("(conn=%p) invoking forwarder.PortForward for port %d", h.conn, p.port) + klog.V(5).Infof("(conn=%p) invoking forwarder.PortForward for port %d", h.conn, p.port) err := h.forwarder.PortForward(h.pod, h.uid, p.port, p.dataStream) - glog.V(5).Infof("(conn=%p) done invoking forwarder.PortForward for port %d", h.conn, p.port) + klog.V(5).Infof("(conn=%p) done invoking forwarder.PortForward for port %d", h.conn, p.port) if err != nil { msg := fmt.Errorf("error forwarding port %d to pod %s, uid %v: %v", p.port, h.pod, h.uid, err) diff --git a/pkg/kubelet/server/remotecommand/BUILD b/pkg/kubelet/server/remotecommand/BUILD index 776abc764c9..fef6fec9c2c 100644 --- a/pkg/kubelet/server/remotecommand/BUILD +++ b/pkg/kubelet/server/remotecommand/BUILD @@ -27,7 +27,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/server/httplog:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/wsstream:go_default_library", "//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/kubelet/server/remotecommand/httpstream.go b/pkg/kubelet/server/remotecommand/httpstream.go index e504a32c522..8bff323ec7a 100644 --- a/pkg/kubelet/server/remotecommand/httpstream.go +++ b/pkg/kubelet/server/remotecommand/httpstream.go @@ -34,7 +34,7 @@ import ( "k8s.io/client-go/tools/remotecommand" api "k8s.io/kubernetes/pkg/apis/core" - "github.com/golang/glog" + "k8s.io/klog" ) // Options contains details about which streams are required for @@ -54,7 +54,7 @@ func NewOptions(req *http.Request) (*Options, error) { stderr := req.FormValue(api.ExecStderrParam) == "1" if tty && stderr { // TODO: make this an error before we reach this method - glog.V(4).Infof("Access to exec with tty and stderr is not supported, bypassing stderr") + klog.V(4).Infof("Access to exec with tty and stderr is not supported, bypassing stderr") stderr = false } @@ -155,7 +155,7 @@ func createHttpStreamStreams(req *http.Request, w http.ResponseWriter, opts *Opt case remotecommandconsts.StreamProtocolV2Name: handler = &v2ProtocolHandler{} case "": - glog.V(4).Infof("Client did not request protocol negotiation. Falling back to %q", remotecommandconsts.StreamProtocolV1Name) + klog.V(4).Infof("Client did not request protocol negotiation. Falling back to %q", remotecommandconsts.StreamProtocolV1Name) fallthrough case remotecommandconsts.StreamProtocolV1Name: handler = &v1ProtocolHandler{} diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index 4c8a3c3cdf7..66bc7f54d95 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -31,13 +31,14 @@ import ( "strings" "time" - restful "github.com/emicklei/go-restful" - "github.com/golang/glog" + "github.com/emicklei/go-restful" cadvisormetrics "github.com/google/cadvisor/container" cadvisorapi "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/metrics" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "google.golang.org/grpc" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -56,6 +57,8 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/core/v1/validation" + "k8s.io/kubernetes/pkg/kubelet/apis/podresources" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/prober" "k8s.io/kubernetes/pkg/kubelet/server/portforward" @@ -63,6 +66,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/server/stats" "k8s.io/kubernetes/pkg/kubelet/server/streaming" kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" + "k8s.io/kubernetes/pkg/kubelet/util" "k8s.io/kubernetes/pkg/util/configz" ) @@ -130,7 +134,7 @@ func ListenAndServeKubeletServer( enableContentionProfiling, redirectContainerStreaming bool, criHandler http.Handler) { - glog.Infof("Starting to listen on %s:%d", address, port) + klog.Infof("Starting to listen on %s:%d", address, port) handler := NewServer(host, resourceAnalyzer, auth, enableDebuggingHandlers, enableContentionProfiling, redirectContainerStreaming, criHandler) s := &http.Server{ Addr: net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)), @@ -142,15 +146,15 @@ func ListenAndServeKubeletServer( // Passing empty strings as the cert and key files means no // cert/keys are specified and GetCertificate in the TLSConfig // should be called instead. - glog.Fatal(s.ListenAndServeTLS(tlsOptions.CertFile, tlsOptions.KeyFile)) + klog.Fatal(s.ListenAndServeTLS(tlsOptions.CertFile, tlsOptions.KeyFile)) } else { - glog.Fatal(s.ListenAndServe()) + klog.Fatal(s.ListenAndServe()) } } // ListenAndServeKubeletReadOnlyServer initializes a server to respond to HTTP network requests on the Kubelet. func ListenAndServeKubeletReadOnlyServer(host HostInterface, resourceAnalyzer stats.ResourceAnalyzer, address net.IP, port uint) { - glog.V(1).Infof("Starting to listen read-only on %s:%d", address, port) + klog.V(1).Infof("Starting to listen read-only on %s:%d", address, port) s := NewServer(host, resourceAnalyzer, nil, false, false, false, nil) server := &http.Server{ @@ -158,7 +162,18 @@ func ListenAndServeKubeletReadOnlyServer(host HostInterface, resourceAnalyzer st Handler: &s, MaxHeaderBytes: 1 << 20, } - glog.Fatal(server.ListenAndServe()) + klog.Fatal(server.ListenAndServe()) +} + +// ListenAndServePodResources initializes a grpc server to serve the PodResources service +func ListenAndServePodResources(socket string, podsProvider podresources.PodsProvider, devicesProvider podresources.DevicesProvider) { + server := grpc.NewServer() + podresourcesapi.RegisterPodResourcesListerServer(server, podresources.NewPodResourcesServer(podsProvider, devicesProvider)) + l, err := util.CreateListener(socket) + if err != nil { + klog.Fatalf("Failed to create listener for podResources endpoint: %v", err) + } + klog.Fatal(server.Serve(l)) } // AuthInterface contains all methods required by the auth filters @@ -223,7 +238,7 @@ func (s *Server) InstallAuthFilter() { // Authenticate info, ok, err := s.auth.AuthenticateRequest(req.Request) if err != nil { - glog.Errorf("Unable to authenticate the request due to an error: %v", err) + klog.Errorf("Unable to authenticate the request due to an error: %v", err) resp.WriteErrorString(http.StatusUnauthorized, "Unauthorized") return } @@ -239,13 +254,13 @@ func (s *Server) InstallAuthFilter() { decision, _, err := s.auth.Authorize(attrs) if err != nil { msg := fmt.Sprintf("Authorization error (user=%s, verb=%s, resource=%s, subresource=%s)", attrs.GetUser().GetName(), attrs.GetVerb(), attrs.GetResource(), attrs.GetSubresource()) - glog.Errorf(msg, err) + klog.Errorf(msg, err) resp.WriteErrorString(http.StatusInternalServerError, msg) return } if decision != authorizer.DecisionAllow { msg := fmt.Sprintf("Forbidden (user=%s, verb=%s, resource=%s, subresource=%s)", attrs.GetUser().GetName(), attrs.GetVerb(), attrs.GetResource(), attrs.GetSubresource()) - glog.V(2).Info(msg) + klog.V(2).Info(msg) resp.WriteErrorString(http.StatusForbidden, msg) return } @@ -315,7 +330,7 @@ const pprofBasePath = "/debug/pprof/" // InstallDebuggingHandlers registers the HTTP request patterns that serve logs or run commands/containers func (s *Server) InstallDebuggingHandlers(criHandler http.Handler) { - glog.Infof("Adding debug handlers to kubelet server.") + klog.Infof("Adding debug handlers to kubelet server.") ws := new(restful.WebService) ws. @@ -648,7 +663,7 @@ type responder struct { } func (r *responder) Error(w http.ResponseWriter, req *http.Request, err error) { - glog.Errorf("Error while proxying request: %v", err) + klog.Errorf("Error while proxying request: %v", err) http.Error(w, err.Error(), http.StatusInternalServerError) } @@ -745,7 +760,7 @@ func writeJsonResponse(response *restful.Response, data []byte) { response.Header().Set(restful.HEADER_ContentType, restful.MIME_JSON) response.WriteHeader(http.StatusOK) if _, err := response.Write(data); err != nil { - glog.Errorf("Error writing response: %v", err) + klog.Errorf("Error writing response: %v", err) } } diff --git a/pkg/kubelet/server/stats/BUILD b/pkg/kubelet/server/stats/BUILD index c3623fd98d1..6fec6f2eb99 100644 --- a/pkg/kubelet/server/stats/BUILD +++ b/pkg/kubelet/server/stats/BUILD @@ -25,8 +25,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/server/stats/fs_resource_analyzer.go b/pkg/kubelet/server/stats/fs_resource_analyzer.go index 5f72134c4f5..331a0b27ba6 100644 --- a/pkg/kubelet/server/stats/fs_resource_analyzer.go +++ b/pkg/kubelet/server/stats/fs_resource_analyzer.go @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - "github.com/golang/glog" + "k8s.io/klog" ) // Map to PodVolumeStats pointers since the addresses for map values are not constant and can cause pain @@ -60,10 +60,10 @@ func newFsResourceAnalyzer(statsProvider StatsProvider, calcVolumePeriod time.Du func (s *fsResourceAnalyzer) Start() { s.startOnce.Do(func() { if s.calcPeriod <= 0 { - glog.Info("Volume stats collection disabled.") + klog.Info("Volume stats collection disabled.") return } - glog.Info("Starting FS ResourceAnalyzer") + klog.Info("Starting FS ResourceAnalyzer") go wait.Forever(func() { s.updateCachedPodVolumeStats() }, s.calcPeriod) }) } diff --git a/pkg/kubelet/server/stats/handler.go b/pkg/kubelet/server/stats/handler.go index 683f20c32f3..f87e5c820c5 100644 --- a/pkg/kubelet/server/stats/handler.go +++ b/pkg/kubelet/server/stats/handler.go @@ -25,8 +25,8 @@ import ( "time" restful "github.com/emicklei/go-restful" - "github.com/golang/glog" cadvisorapi "github.com/google/cadvisor/info/v1" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -236,7 +236,7 @@ func (h *handler) handleSystemContainer(request *restful.Request, response *rest if err != nil { if _, ok := stats[containerName]; ok { // If the failure is partial, log it and return a best-effort response. - glog.Errorf("Partial failure issuing GetRawContainerInfo(%v): %v", query, err) + klog.Errorf("Partial failure issuing GetRawContainerInfo(%v): %v", query, err) } else { handleError(response, fmt.Sprintf("/stats/container %v", query), err) return @@ -272,7 +272,7 @@ func (h *handler) handlePodContainer(request *restful.Request, response *restful pod, ok := h.provider.GetPodByName(params["namespace"], params["podName"]) if !ok { - glog.V(4).Infof("Container not found: %v", params) + klog.V(4).Infof("Container not found: %v", params) response.WriteError(http.StatusNotFound, kubecontainer.ErrContainerNotFound) return } @@ -291,7 +291,7 @@ func (h *handler) handlePodContainer(request *restful.Request, response *restful func writeResponse(response *restful.Response, stats interface{}) { if err := response.WriteAsJson(stats); err != nil { - glog.Errorf("Error writing response: %v", err) + klog.Errorf("Error writing response: %v", err) } } @@ -303,7 +303,7 @@ func handleError(response *restful.Response, request string, err error) { response.WriteError(http.StatusNotFound, err) default: msg := fmt.Sprintf("Internal Error: %v", err) - glog.Errorf("HTTP InternalServerError serving %s: %s", request, msg) + klog.Errorf("HTTP InternalServerError serving %s: %s", request, msg) response.WriteErrorString(http.StatusInternalServerError, msg) } } diff --git a/pkg/kubelet/server/stats/summary_sys_containers.go b/pkg/kubelet/server/stats/summary_sys_containers.go index 7179e828020..baaff0ab1bd 100644 --- a/pkg/kubelet/server/stats/summary_sys_containers.go +++ b/pkg/kubelet/server/stats/summary_sys_containers.go @@ -19,7 +19,7 @@ limitations under the License. package stats import ( - "github.com/golang/glog" + "k8s.io/klog" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cm" @@ -42,7 +42,7 @@ func (sp *summaryProviderImpl) GetSystemContainersStats(nodeConfig cm.NodeConfig } s, _, err := sp.provider.GetCgroupStats(cont.name, cont.forceStatsUpdate) if err != nil { - glog.Errorf("Failed to get system container stats for %q: %v", cont.name, err) + klog.Errorf("Failed to get system container stats for %q: %v", cont.name, err) continue } // System containers don't have a filesystem associated with them. @@ -71,7 +71,7 @@ func (sp *summaryProviderImpl) GetSystemContainersCPUAndMemoryStats(nodeConfig c } s, err := sp.provider.GetCgroupCPUAndMemoryStats(cont.name, cont.forceStatsUpdate) if err != nil { - glog.Errorf("Failed to get system container stats for %q: %v", cont.name, err) + klog.Errorf("Failed to get system container stats for %q: %v", cont.name, err) continue } s.Name = sys diff --git a/pkg/kubelet/server/stats/volume_stat_calculator.go b/pkg/kubelet/server/stats/volume_stat_calculator.go index 2c535f56241..220d8785f53 100644 --- a/pkg/kubelet/server/stats/volume_stat_calculator.go +++ b/pkg/kubelet/server/stats/volume_stat_calculator.go @@ -27,7 +27,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/volume" - "github.com/golang/glog" + "k8s.io/klog" ) // volumeStatCalculator calculates volume metrics for a given pod periodically in the background and caches the result @@ -109,7 +109,7 @@ func (s *volumeStatCalculator) calcAndStoreStats() { if err != nil { // Expected for Volumes that don't support Metrics if !volume.IsNotSupported(err) { - glog.V(4).Infof("Failed to calculate volume metrics for pod %s volume %s: %+v", format.Pod(s.pod), name, err) + klog.V(4).Infof("Failed to calculate volume metrics for pod %s volume %s: %+v", format.Pod(s.pod), name, err) } continue } diff --git a/pkg/kubelet/stats/BUILD b/pkg/kubelet/stats/BUILD index ddd6639ea26..07ca1b54540 100644 --- a/pkg/kubelet/stats/BUILD +++ b/pkg/kubelet/stats/BUILD @@ -28,10 +28,10 @@ go_library( "//pkg/volume:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/fs:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/stats/cadvisor_stats_provider.go b/pkg/kubelet/stats/cadvisor_stats_provider.go index 35820f7e94a..9549c12a1a6 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider.go @@ -22,8 +22,8 @@ import ( "sort" "strings" - "github.com/golang/glog" cadvisorapiv2 "github.com/google/cadvisor/info/v2" + "k8s.io/klog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -306,7 +306,7 @@ func isPodManagedContainer(cinfo *cadvisorapiv2.ContainerInfo) bool { podNamespace := kubetypes.GetPodNamespace(cinfo.Spec.Labels) managed := podName != "" && podNamespace != "" if !managed && podName != podNamespace { - glog.Warningf( + klog.Warningf( "Expect container to have either both podName (%s) and podNamespace (%s) labels, or neither.", podName, podNamespace) } @@ -429,7 +429,7 @@ func getCadvisorContainerInfo(ca cadvisor.Interface) (map[string]cadvisorapiv2.C if _, ok := infos["/"]; ok { // If the failure is partial, log it and return a best-effort // response. - glog.Errorf("Partial failure issuing cadvisor.ContainerInfoV2: %v", err) + klog.Errorf("Partial failure issuing cadvisor.ContainerInfoV2: %v", err) } else { return nil, fmt.Errorf("failed to get root cgroup stats: %v", err) } diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index 3ebf3a0fec2..bafd24d65c4 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -24,8 +24,8 @@ import ( "strings" "time" - "github.com/golang/glog" cadvisorfs "github.com/google/cadvisor/fs" + "k8s.io/klog" cadvisorapiv2 "github.com/google/cadvisor/info/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -154,7 +154,7 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { // container stats caStats, caFound := caInfos[containerID] if !caFound { - glog.V(4).Infof("Unable to find cadvisor stats for %q", containerID) + klog.V(4).Infof("Unable to find cadvisor stats for %q", containerID) } else { p.addCadvisorContainerStats(cs, &caStats) } @@ -236,7 +236,7 @@ func (p *criStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, erro // container stats caStats, caFound := caInfos[containerID] if !caFound { - glog.V(4).Infof("Unable to find cadvisor stats for %q", containerID) + klog.V(4).Infof("Unable to find cadvisor stats for %q", containerID) } else { p.addCadvisorContainerStats(cs, &caStats) } @@ -307,7 +307,7 @@ func (p *criStatsProvider) ImageFsDevice() (string, error) { // nil. func (p *criStatsProvider) getFsInfo(fsID *runtimeapi.FilesystemIdentifier) *cadvisorapiv2.FsInfo { if fsID == nil { - glog.V(2).Infof("Failed to get filesystem info: fsID is nil.") + klog.V(2).Infof("Failed to get filesystem info: fsID is nil.") return nil } mountpoint := fsID.GetMountpoint() @@ -315,9 +315,9 @@ func (p *criStatsProvider) getFsInfo(fsID *runtimeapi.FilesystemIdentifier) *cad if err != nil { msg := fmt.Sprintf("Failed to get the info of the filesystem with mountpoint %q: %v.", mountpoint, err) if err == cadvisorfs.ErrNoSuchDevice { - glog.V(2).Info(msg) + klog.V(2).Info(msg) } else { - glog.Error(msg) + klog.Error(msg) } return nil } @@ -362,7 +362,7 @@ func (p *criStatsProvider) addPodNetworkStats( } // TODO: sum Pod network stats from container stats. - glog.V(4).Infof("Unable to find cadvisor stats for sandbox %q", podSandboxID) + klog.V(4).Infof("Unable to find cadvisor stats for sandbox %q", podSandboxID) } func (p *criStatsProvider) addPodCPUMemoryStats( @@ -579,7 +579,7 @@ func (p *criStatsProvider) getContainerLogStats(path string, rootFsInfo *cadviso m := p.logMetricsService.createLogMetricsProvider(path) logMetrics, err := m.GetMetrics() if err != nil { - glog.Errorf("Unable to fetch container log stats for path %s: %v ", path, err) + klog.Errorf("Unable to fetch container log stats for path %s: %v ", path, err) return nil } result := &statsapi.FsStats{ diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go index 2bdda4314a6..54f3093e553 100644 --- a/pkg/kubelet/stats/helper.go +++ b/pkg/kubelet/stats/helper.go @@ -20,7 +20,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" cadvisorapiv1 "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" @@ -206,7 +206,7 @@ func cadvisorInfoToUserDefinedMetrics(info *cadvisorapiv2.ContainerInfo) []stats for name, values := range stat.CustomMetrics { specVal, ok := udmMap[name] if !ok { - glog.Warningf("spec for custom metric %q is missing from cAdvisor output. Spec: %+v, Metrics: %+v", name, info.Spec, stat.CustomMetrics) + klog.Warningf("spec for custom metric %q is missing from cAdvisor output. Spec: %+v, Metrics: %+v", name, info.Spec, stat.CustomMetrics) continue } for _, value := range values { diff --git a/pkg/kubelet/status/BUILD b/pkg/kubelet/status/BUILD index 6e341e9b498..0b590440e58 100644 --- a/pkg/kubelet/status/BUILD +++ b/pkg/kubelet/status/BUILD @@ -28,7 +28,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/status/status_manager.go b/pkg/kubelet/status/status_manager.go index f184003117c..37d5bfa0354 100644 --- a/pkg/kubelet/status/status_manager.go +++ b/pkg/kubelet/status/status_manager.go @@ -24,7 +24,6 @@ import ( clientset "k8s.io/client-go/kubernetes" - "github.com/golang/glog" "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" @@ -32,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" podutil "k8s.io/kubernetes/pkg/api/v1/pod" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" kubepod "k8s.io/kubernetes/pkg/kubelet/pod" @@ -145,17 +145,17 @@ func (m *manager) Start() { // on the master, where the kubelet is responsible for bootstrapping the pods // of the master components. if m.kubeClient == nil { - glog.Infof("Kubernetes client is nil, not starting status manager.") + klog.Infof("Kubernetes client is nil, not starting status manager.") return } - glog.Info("Starting to sync pod status with apiserver") + klog.Info("Starting to sync pod status with apiserver") syncTicker := time.Tick(syncPeriod) // syncPod and syncBatch share the same go routine to avoid sync races. go wait.Forever(func() { select { case syncRequest := <-m.podStatusChannel: - glog.V(5).Infof("Status Manager: syncing pod: %q, with status: (%d, %v) from podStatusChannel", + klog.V(5).Infof("Status Manager: syncing pod: %q, with status: (%d, %v) from podStatusChannel", syncRequest.podUID, syncRequest.status.version, syncRequest.status.status) m.syncPod(syncRequest.podUID, syncRequest.status) case <-syncTicker: @@ -177,7 +177,7 @@ func (m *manager) SetPodStatus(pod *v1.Pod, status v1.PodStatus) { for _, c := range pod.Status.Conditions { if !kubetypes.PodConditionByKubelet(c.Type) { - glog.Errorf("Kubelet is trying to update pod condition %q for pod %q. "+ + klog.Errorf("Kubelet is trying to update pod condition %q for pod %q. "+ "But it is not owned by kubelet.", string(c.Type), format.Pod(pod)) } } @@ -196,13 +196,13 @@ func (m *manager) SetContainerReadiness(podUID types.UID, containerID kubecontai pod, ok := m.podManager.GetPodByUID(podUID) if !ok { - glog.V(4).Infof("Pod %q has been deleted, no need to update readiness", string(podUID)) + klog.V(4).Infof("Pod %q has been deleted, no need to update readiness", string(podUID)) return } oldStatus, found := m.podStatuses[pod.UID] if !found { - glog.Warningf("Container readiness changed before pod has synced: %q - %q", + klog.Warningf("Container readiness changed before pod has synced: %q - %q", format.Pod(pod), containerID.String()) return } @@ -210,13 +210,13 @@ func (m *manager) SetContainerReadiness(podUID types.UID, containerID kubecontai // Find the container to update. containerStatus, _, ok := findContainerStatus(&oldStatus.status, containerID.String()) if !ok { - glog.Warningf("Container readiness changed for unknown container: %q - %q", + klog.Warningf("Container readiness changed for unknown container: %q - %q", format.Pod(pod), containerID.String()) return } if containerStatus.Ready == ready { - glog.V(4).Infof("Container readiness unchanged (%v): %q - %q", ready, + klog.V(4).Infof("Container readiness unchanged (%v): %q - %q", ready, format.Pod(pod), containerID.String()) return } @@ -238,7 +238,7 @@ func (m *manager) SetContainerReadiness(podUID types.UID, containerID kubecontai if conditionIndex != -1 { status.Conditions[conditionIndex] = condition } else { - glog.Warningf("PodStatus missing %s type condition: %+v", conditionType, status) + klog.Warningf("PodStatus missing %s type condition: %+v", conditionType, status) status.Conditions = append(status.Conditions, condition) } } @@ -328,11 +328,11 @@ func (m *manager) updateStatusInternal(pod *v1.Pod, status v1.PodStatus, forceUp // Check for illegal state transition in containers if err := checkContainerStateTransition(oldStatus.ContainerStatuses, status.ContainerStatuses, pod.Spec.RestartPolicy); err != nil { - glog.Errorf("Status update on pod %v/%v aborted: %v", pod.Namespace, pod.Name, err) + klog.Errorf("Status update on pod %v/%v aborted: %v", pod.Namespace, pod.Name, err) return false } if err := checkContainerStateTransition(oldStatus.InitContainerStatuses, status.InitContainerStatuses, pod.Spec.RestartPolicy); err != nil { - glog.Errorf("Status update on pod %v/%v aborted: %v", pod.Namespace, pod.Name, err) + klog.Errorf("Status update on pod %v/%v aborted: %v", pod.Namespace, pod.Name, err) return false } @@ -361,7 +361,7 @@ func (m *manager) updateStatusInternal(pod *v1.Pod, status v1.PodStatus, forceUp // The intent here is to prevent concurrent updates to a pod's status from // clobbering each other so the phase of a pod progresses monotonically. if isCached && isPodStatusByKubeletEqual(&cachedStatus.status, &status) && !forceUpdate { - glog.V(3).Infof("Ignoring same status for pod %q, status: %+v", format.Pod(pod), status) + klog.V(3).Infof("Ignoring same status for pod %q, status: %+v", format.Pod(pod), status) return false // No new status. } @@ -375,13 +375,13 @@ func (m *manager) updateStatusInternal(pod *v1.Pod, status v1.PodStatus, forceUp select { case m.podStatusChannel <- podStatusSyncRequest{pod.UID, newStatus}: - glog.V(5).Infof("Status Manager: adding pod: %q, with status: (%q, %v) to podStatusChannel", + klog.V(5).Infof("Status Manager: adding pod: %q, with status: (%q, %v) to podStatusChannel", pod.UID, newStatus.version, newStatus.status) return true default: // Let the periodic syncBatch handle the update if the channel is full. // We can't block, since we hold the mutex lock. - glog.V(4).Infof("Skipping the status update for pod %q for now because the channel is full; status: %+v", + klog.V(4).Infof("Skipping the status update for pod %q for now because the channel is full; status: %+v", format.Pod(pod), status) return false } @@ -415,7 +415,7 @@ func (m *manager) RemoveOrphanedStatuses(podUIDs map[types.UID]bool) { defer m.podStatusesLock.Unlock() for key := range m.podStatuses { if _, ok := podUIDs[key]; !ok { - glog.V(5).Infof("Removing %q from status map.", key) + klog.V(5).Infof("Removing %q from status map.", key) delete(m.podStatuses, key) } } @@ -442,7 +442,7 @@ func (m *manager) syncBatch() { syncedUID := kubetypes.MirrorPodUID(uid) if mirrorUID, ok := podToMirror[kubetypes.ResolvedPodUID(uid)]; ok { if mirrorUID == "" { - glog.V(5).Infof("Static pod %q (%s/%s) does not have a corresponding mirror pod; skipping", uid, status.podName, status.podNamespace) + klog.V(5).Infof("Static pod %q (%s/%s) does not have a corresponding mirror pod; skipping", uid, status.podName, status.podNamespace) continue } syncedUID = mirrorUID @@ -461,7 +461,7 @@ func (m *manager) syncBatch() { }() for _, update := range updatedStatuses { - glog.V(5).Infof("Status Manager: syncPod in syncbatch. pod UID: %q", update.podUID) + klog.V(5).Infof("Status Manager: syncPod in syncbatch. pod UID: %q", update.podUID) m.syncPod(update.podUID, update.status) } } @@ -469,41 +469,41 @@ func (m *manager) syncBatch() { // syncPod syncs the given status with the API server. The caller must not hold the lock. func (m *manager) syncPod(uid types.UID, status versionedPodStatus) { if !m.needsUpdate(uid, status) { - glog.V(1).Infof("Status for pod %q is up-to-date; skipping", uid) + klog.V(1).Infof("Status for pod %q is up-to-date; skipping", uid) return } // TODO: make me easier to express from client code pod, err := m.kubeClient.CoreV1().Pods(status.podNamespace).Get(status.podName, metav1.GetOptions{}) if errors.IsNotFound(err) { - glog.V(3).Infof("Pod %q (%s) does not exist on the server", status.podName, uid) + klog.V(3).Infof("Pod %q (%s) does not exist on the server", status.podName, uid) // If the Pod is deleted the status will be cleared in // RemoveOrphanedStatuses, so we just ignore the update here. return } if err != nil { - glog.Warningf("Failed to get status for pod %q: %v", format.PodDesc(status.podName, status.podNamespace, uid), err) + klog.Warningf("Failed to get status for pod %q: %v", format.PodDesc(status.podName, status.podNamespace, uid), err) return } translatedUID := m.podManager.TranslatePodUID(pod.UID) // Type convert original uid just for the purpose of comparison. if len(translatedUID) > 0 && translatedUID != kubetypes.ResolvedPodUID(uid) { - glog.V(2).Infof("Pod %q was deleted and then recreated, skipping status update; old UID %q, new UID %q", format.Pod(pod), uid, translatedUID) + klog.V(2).Infof("Pod %q was deleted and then recreated, skipping status update; old UID %q, new UID %q", format.Pod(pod), uid, translatedUID) m.deletePodStatus(uid) return } oldStatus := pod.Status.DeepCopy() newPod, patchBytes, err := statusutil.PatchPodStatus(m.kubeClient, pod.Namespace, pod.Name, *oldStatus, mergePodStatus(*oldStatus, status.status)) - glog.V(3).Infof("Patch status for pod %q with %q", format.Pod(pod), patchBytes) + klog.V(3).Infof("Patch status for pod %q with %q", format.Pod(pod), patchBytes) if err != nil { - glog.Warningf("Failed to update status for pod %q: %v", format.Pod(pod), err) + klog.Warningf("Failed to update status for pod %q: %v", format.Pod(pod), err) return } pod = newPod - glog.V(3).Infof("Status for pod %q updated successfully: (%d, %+v)", format.Pod(pod), status.version, status.status) + klog.V(3).Infof("Status for pod %q updated successfully: (%d, %+v)", format.Pod(pod), status.version, status.status) m.apiStatusVersions[kubetypes.MirrorPodUID(pod.UID)] = status.version // We don't handle graceful deletion of mirror pods. @@ -513,10 +513,10 @@ func (m *manager) syncPod(uid types.UID, status versionedPodStatus) { deleteOptions.Preconditions = metav1.NewUIDPreconditions(string(pod.UID)) err = m.kubeClient.CoreV1().Pods(pod.Namespace).Delete(pod.Name, deleteOptions) if err != nil { - glog.Warningf("Failed to delete status for pod %q: %v", format.Pod(pod), err) + klog.Warningf("Failed to delete status for pod %q: %v", format.Pod(pod), err) return } - glog.V(3).Infof("Pod %q fully terminated and removed from etcd", format.Pod(pod)) + klog.V(3).Infof("Pod %q fully terminated and removed from etcd", format.Pod(pod)) m.deletePodStatus(uid) } } @@ -555,14 +555,14 @@ func (m *manager) needsReconcile(uid types.UID, status v1.PodStatus) bool { // The pod could be a static pod, so we should translate first. pod, ok := m.podManager.GetPodByUID(uid) if !ok { - glog.V(4).Infof("Pod %q has been deleted, no need to reconcile", string(uid)) + klog.V(4).Infof("Pod %q has been deleted, no need to reconcile", string(uid)) return false } // If the pod is a static pod, we should check its mirror pod, because only status in mirror pod is meaningful to us. if kubepod.IsStaticPod(pod) { mirrorPod, ok := m.podManager.GetMirrorPodByPod(pod) if !ok { - glog.V(4).Infof("Static pod %q has no corresponding mirror pod, no need to reconcile", format.Pod(pod)) + klog.V(4).Infof("Static pod %q has no corresponding mirror pod, no need to reconcile", format.Pod(pod)) return false } pod = mirrorPod @@ -576,7 +576,7 @@ func (m *manager) needsReconcile(uid types.UID, status v1.PodStatus) bool { // reconcile is not needed. Just return. return false } - glog.V(3).Infof("Pod status is inconsistent with cached status for pod %q, a reconciliation should be triggered:\n %+v", format.Pod(pod), + klog.V(3).Infof("Pod status is inconsistent with cached status for pod %q, a reconciliation should be triggered:\n %+v", format.Pod(pod), diff.ObjectDiff(podStatus, status)) return true diff --git a/pkg/kubelet/token/BUILD b/pkg/kubelet/token/BUILD index 2591a4ec275..9f7d4db7caa 100644 --- a/pkg/kubelet/token/BUILD +++ b/pkg/kubelet/token/BUILD @@ -25,7 +25,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/token/OWNERS b/pkg/kubelet/token/OWNERS index 33904f77888..d914c0d7195 100644 --- a/pkg/kubelet/token/OWNERS +++ b/pkg/kubelet/token/OWNERS @@ -1,6 +1,7 @@ approvers: -- mikedanese +- sig-auth-serviceaccounts-approvers reviewers: -- mikedanese -- awly -- tallclair +- sig-auth-serviceaccounts-reviewers +labels: +- sig/auth + diff --git a/pkg/kubelet/token/token_manager.go b/pkg/kubelet/token/token_manager.go index 75086c86b97..76819ac72df 100644 --- a/pkg/kubelet/token/token_manager.go +++ b/pkg/kubelet/token/token_manager.go @@ -24,12 +24,12 @@ import ( "sync" "time" - "github.com/golang/glog" authenticationv1 "k8s.io/api/authentication/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" ) const ( @@ -90,7 +90,7 @@ func (m *Manager) GetServiceAccountToken(namespace, name string, tr *authenticat case m.expired(ctr): return nil, fmt.Errorf("token %s expired and refresh failed: %v", key, err) default: - glog.Errorf("couldn't update token %s: %v", key, err) + klog.Errorf("couldn't update token %s: %v", key, err) return ctr, nil } } @@ -142,7 +142,7 @@ func (m *Manager) expired(t *authenticationv1.TokenRequest) bool { // ttl, or if the token is older than 24 hours. func (m *Manager) requiresRefresh(tr *authenticationv1.TokenRequest) bool { if tr.Spec.ExpirationSeconds == nil { - glog.Errorf("expiration seconds was nil for tr: %#v", tr) + klog.Errorf("expiration seconds was nil for tr: %#v", tr) return false } now := m.clock.Now() diff --git a/pkg/kubelet/types/BUILD b/pkg/kubelet/types/BUILD index bc62f58d673..b0fceedbbbd 100644 --- a/pkg/kubelet/types/BUILD +++ b/pkg/kubelet/types/BUILD @@ -32,15 +32,18 @@ go_test( name = "go_default_test", srcs = [ "labels_test.go", + "main_test.go", "pod_status_test.go", "pod_update_test.go", "types_test.go", ], embed = [":go_default_library"], deps = [ + "//pkg/features:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", ], diff --git a/pkg/kubelet/types/main_test.go b/pkg/kubelet/types/main_test.go new file mode 100644 index 00000000000..928b1122930 --- /dev/null +++ b/pkg/kubelet/types/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/kubelet/types/pod_update_test.go b/pkg/kubelet/types/pod_update_test.go index 4efc451e1ae..73a93d7e178 100644 --- a/pkg/kubelet/types/pod_update_test.go +++ b/pkg/kubelet/types/pod_update_test.go @@ -24,6 +24,8 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + "k8s.io/kubernetes/pkg/features" ) func TestGetValidatedSources(t *testing.T) { @@ -116,9 +118,7 @@ func TestString(t *testing.T) { } func TestIsCriticalPod(t *testing.T) { - if err := utilfeature.DefaultFeatureGate.Set("ExperimentalCriticalPodAnnotation=true"); err != nil { - t.Errorf("failed to set ExperimentalCriticalPodAnnotation to true: %v", err) - } + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() cases := []struct { pod v1.Pod expected bool diff --git a/pkg/kubelet/util/BUILD b/pkg/kubelet/util/BUILD index 01847254a98..4165864e0a3 100644 --- a/pkg/kubelet/util/BUILD +++ b/pkg/kubelet/util/BUILD @@ -45,16 +45,16 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:darwin": [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:freebsd": [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:linux": [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:windows": [ "//vendor/github.com/Microsoft/go-winio:go_default_library", diff --git a/pkg/kubelet/util/pluginwatcher/BUILD b/pkg/kubelet/util/pluginwatcher/BUILD index 0301c6b95b3..5f4e61e5e54 100644 --- a/pkg/kubelet/util/pluginwatcher/BUILD +++ b/pkg/kubelet/util/pluginwatcher/BUILD @@ -11,15 +11,15 @@ go_library( importpath = "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher", visibility = ["//visibility:public"], deps = [ - "//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library", + "//pkg/kubelet/apis/pluginregistration/v1:go_default_library", "//pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1:go_default_library", "//pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta2:go_default_library", "//pkg/util/filesystem:go_default_library", "//vendor/github.com/fsnotify/fsnotify:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", "//vendor/golang.org/x/net/context:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -28,8 +28,9 @@ go_test( srcs = ["plugin_watcher_test.go"], embed = [":go_default_library"], deps = [ - "//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library", + "//pkg/kubelet/apis/pluginregistration/v1:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/util/pluginwatcher/example_handler.go b/pkg/kubelet/util/pluginwatcher/example_handler.go index 8f9cac5d9bd..fc14dfe9559 100644 --- a/pkg/kubelet/util/pluginwatcher/example_handler.go +++ b/pkg/kubelet/util/pluginwatcher/example_handler.go @@ -23,8 +23,8 @@ import ( "sync" "time" - "github.com/golang/glog" "golang.org/x/net/context" + "k8s.io/klog" v1beta1 "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1" v1beta2 "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta2" @@ -117,7 +117,7 @@ func (p *exampleHandler) EventChan(pluginName string) chan examplePluginEvent { } func (p *exampleHandler) SendEvent(pluginName string, event examplePluginEvent) { - glog.V(2).Infof("Sending %v for plugin %s over chan %v", event, pluginName, p.eventChans[pluginName]) + klog.V(2).Infof("Sending %v for plugin %s over chan %v", event, pluginName, p.eventChans[pluginName]) p.eventChans[pluginName] <- event } diff --git a/pkg/kubelet/util/pluginwatcher/example_plugin.go b/pkg/kubelet/util/pluginwatcher/example_plugin.go index 694b3661202..d55130ef0bc 100644 --- a/pkg/kubelet/util/pluginwatcher/example_plugin.go +++ b/pkg/kubelet/util/pluginwatcher/example_plugin.go @@ -24,11 +24,11 @@ import ( "sync" "time" - "github.com/golang/glog" "golang.org/x/net/context" "google.golang.org/grpc" + "k8s.io/klog" - registerapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1alpha1" + registerapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1" v1beta1 "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1" v1beta2 "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta2" ) @@ -49,7 +49,7 @@ type pluginServiceV1Beta1 struct { } func (s *pluginServiceV1Beta1) GetExampleInfo(ctx context.Context, rqt *v1beta1.ExampleRequest) (*v1beta1.ExampleResponse, error) { - glog.Infof("GetExampleInfo v1beta1field: %s", rqt.V1Beta1Field) + klog.Infof("GetExampleInfo v1beta1field: %s", rqt.V1Beta1Field) return &v1beta1.ExampleResponse{}, nil } @@ -62,7 +62,7 @@ type pluginServiceV1Beta2 struct { } func (s *pluginServiceV1Beta2) GetExampleInfo(ctx context.Context, rqt *v1beta2.ExampleRequest) (*v1beta2.ExampleResponse, error) { - glog.Infof("GetExampleInfo v1beta2_field: %s", rqt.V1Beta2Field) + klog.Infof("GetExampleInfo v1beta2_field: %s", rqt.V1Beta2Field) return &v1beta2.ExampleResponse{}, nil } @@ -97,7 +97,7 @@ func (e *examplePlugin) GetInfo(ctx context.Context, req *registerapi.InfoReques } func (e *examplePlugin) NotifyRegistrationStatus(ctx context.Context, status *registerapi.RegistrationStatus) (*registerapi.RegistrationStatusResponse, error) { - glog.Errorf("Registration is: %v\n", status) + klog.Errorf("Registration is: %v\n", status) if e.registrationStatus != nil { e.registrationStatus <- *status @@ -108,13 +108,13 @@ func (e *examplePlugin) NotifyRegistrationStatus(ctx context.Context, status *re // Serve starts a pluginwatcher server and one or more of the plugin services func (e *examplePlugin) Serve(services ...string) error { - glog.Infof("starting example server at: %s\n", e.endpoint) + klog.Infof("starting example server at: %s\n", e.endpoint) lis, err := net.Listen("unix", e.endpoint) if err != nil { return err } - glog.Infof("example server started at: %s\n", e.endpoint) + klog.Infof("example server started at: %s\n", e.endpoint) e.grpcServer = grpc.NewServer() // Registers kubelet plugin watcher api. @@ -141,7 +141,7 @@ func (e *examplePlugin) Serve(services ...string) error { defer e.wg.Done() // Blocking call to accept incoming connections. if err := e.grpcServer.Serve(lis); err != nil { - glog.Errorf("example server stopped serving: %v", err) + klog.Errorf("example server stopped serving: %v", err) } }() @@ -149,7 +149,7 @@ func (e *examplePlugin) Serve(services ...string) error { } func (e *examplePlugin) Stop() error { - glog.Infof("Stopping example server at: %s\n", e.endpoint) + klog.Infof("Stopping example server at: %s\n", e.endpoint) e.grpcServer.Stop() c := make(chan struct{}) diff --git a/pkg/kubelet/util/pluginwatcher/plugin_watcher.go b/pkg/kubelet/util/pluginwatcher/plugin_watcher.go index de2addd2846..03970f3cc08 100644 --- a/pkg/kubelet/util/pluginwatcher/plugin_watcher.go +++ b/pkg/kubelet/util/pluginwatcher/plugin_watcher.go @@ -25,12 +25,12 @@ import ( "time" "github.com/fsnotify/fsnotify" - "github.com/golang/glog" "github.com/pkg/errors" "golang.org/x/net/context" "google.golang.org/grpc" + "k8s.io/klog" - registerapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1alpha1" + registerapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1" utilfs "k8s.io/kubernetes/pkg/util/filesystem" ) @@ -82,7 +82,7 @@ func (w *Watcher) getHandler(pluginType string) (PluginHandler, bool) { // Start watches for the creation of plugin sockets at the path func (w *Watcher) Start() error { - glog.V(2).Infof("Plugin Watcher Start at %s", w.path) + klog.V(2).Infof("Plugin Watcher Start at %s", w.path) w.stopCh = make(chan interface{}) // Creating the directory to be watched if it doesn't exist yet, @@ -112,12 +112,12 @@ func (w *Watcher) Start() error { if event.Op&fsnotify.Create == fsnotify.Create { err := w.handleCreateEvent(event) if err != nil { - glog.Errorf("error %v when handling create event: %s", err, event) + klog.Errorf("error %v when handling create event: %s", err, event) } } else if event.Op&fsnotify.Remove == fsnotify.Remove { err := w.handleDeleteEvent(event) if err != nil { - glog.Errorf("error %v when handling delete event: %s", err, event) + klog.Errorf("error %v when handling delete event: %s", err, event) } } return @@ -125,7 +125,7 @@ func (w *Watcher) Start() error { continue case err := <-fsWatcher.Errors: if err != nil { - glog.Errorf("fsWatcher received error: %v", err) + klog.Errorf("fsWatcher received error: %v", err) } continue case <-w.stopCh: @@ -165,7 +165,7 @@ func (w *Watcher) Stop() error { } func (w *Watcher) init() error { - glog.V(4).Infof("Ensuring Plugin directory at %s ", w.path) + klog.V(4).Infof("Ensuring Plugin directory at %s ", w.path) if err := w.fs.MkdirAll(w.path, 0755); err != nil { return fmt.Errorf("error (re-)creating root %s: %v", w.path, err) @@ -175,10 +175,17 @@ func (w *Watcher) init() error { } // Walks through the plugin directory discover any existing plugin sockets. +// Goroutines started here will be waited for in Stop() before cleaning up. +// Ignore all errors except root dir not being walkable func (w *Watcher) traversePluginDir(dir string) error { return w.fs.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { - return fmt.Errorf("error accessing path: %s error: %v", path, err) + if path == dir { + return fmt.Errorf("error accessing path: %s error: %v", path, err) + } + + klog.Errorf("error accessing path: %s error: %v", path, err) + return nil } switch mode := info.Mode(); { @@ -187,14 +194,16 @@ func (w *Watcher) traversePluginDir(dir string) error { return fmt.Errorf("failed to watch %s, err: %v", path, err) } case mode&os.ModeSocket != 0: + w.wg.Add(1) go func() { + defer w.wg.Done() w.fsWatcher.Events <- fsnotify.Event{ Name: path, Op: fsnotify.Create, } }() default: - glog.V(5).Infof("Ignoring file %s with mode %v", path, mode) + klog.V(5).Infof("Ignoring file %s with mode %v", path, mode) } return nil @@ -203,7 +212,7 @@ func (w *Watcher) traversePluginDir(dir string) error { // Handle filesystem notify event. func (w *Watcher) handleCreateEvent(event fsnotify.Event) error { - glog.V(6).Infof("Handling create event: %v", event) + klog.V(6).Infof("Handling create event: %v", event) fi, err := os.Stat(event.Name) if err != nil { @@ -211,7 +220,7 @@ func (w *Watcher) handleCreateEvent(event fsnotify.Event) error { } if strings.HasPrefix(fi.Name(), ".") { - glog.Errorf("Ignoring file: %s", fi.Name()) + klog.Errorf("Ignoring file: %s", fi.Name()) return nil } @@ -277,7 +286,7 @@ func (w *Watcher) handlePluginRegistration(socketPath string) error { } func (w *Watcher) handleDeleteEvent(event fsnotify.Event) error { - glog.V(6).Infof("Handling delete event: %v", event) + klog.V(6).Infof("Handling delete event: %v", event) plugin, ok := w.getPlugin(event.Name) if !ok { @@ -295,7 +304,7 @@ func (w *Watcher) handleDeleteEvent(event fsnotify.Event) error { // When ReRegistering, the new plugin will have removed the current mapping (map[socketPath] = plugin) and replaced // it with it's own socketPath. if _, ok = w.getPlugin(event.Name); !ok { - glog.V(2).Infof("A newer plugin watcher has been registered for plugin %v, dropping DeRegister call", plugin) + klog.V(2).Infof("A newer plugin watcher has been registered for plugin %v, dropping DeRegister call", plugin) return nil } @@ -304,7 +313,7 @@ func (w *Watcher) handleDeleteEvent(event fsnotify.Event) error { return fmt.Errorf("could not find handler %s for plugin %s at path %s", plugin.pluginType, plugin.pluginName, event.Name) } - glog.V(2).Infof("DeRegistering plugin %v at path %s", plugin, event.Name) + klog.V(2).Infof("DeRegistering plugin %v at path %s", plugin, event.Name) w.deRegisterPlugin(event.Name, plugin.pluginType, plugin.pluginName) h.DeRegisterPlugin(plugin.pluginName) diff --git a/pkg/kubelet/util/pluginwatcher/plugin_watcher_test.go b/pkg/kubelet/util/pluginwatcher/plugin_watcher_test.go index fdcb8b705bc..05dac25ac80 100644 --- a/pkg/kubelet/util/pluginwatcher/plugin_watcher_test.go +++ b/pkg/kubelet/util/pluginwatcher/plugin_watcher_test.go @@ -27,7 +27,8 @@ import ( "github.com/stretchr/testify/require" - registerapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1alpha1" + "k8s.io/klog" + registerapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1" ) var ( @@ -39,6 +40,7 @@ var ( func init() { var logLevel string + klog.InitFlags(flag.CommandLine) flag.Set("alsologtostderr", fmt.Sprintf("%t", true)) flag.StringVar(&logLevel, "logLevel", "6", "test") flag.Lookup("v").Value.Set(logLevel) diff --git a/pkg/kubelet/util/util_unix.go b/pkg/kubelet/util/util_unix.go index d12d51a1aef..0ddc6384391 100644 --- a/pkg/kubelet/util/util_unix.go +++ b/pkg/kubelet/util/util_unix.go @@ -23,10 +23,11 @@ import ( "net" "net/url" "os" + "path/filepath" "time" - "github.com/golang/glog" "golang.org/x/sys/unix" + "k8s.io/klog" ) const ( @@ -73,7 +74,7 @@ func parseEndpointWithFallbackProtocol(endpoint string, fallbackProtocol string) fallbackEndpoint := fallbackProtocol + "://" + endpoint protocol, addr, err = parseEndpoint(fallbackEndpoint) if err == nil { - glog.Warningf("Using %q as endpoint is deprecated, please consider using full url format %q.", endpoint, fallbackEndpoint) + klog.Warningf("Using %q as endpoint is deprecated, please consider using full url format %q.", endpoint, fallbackEndpoint) } } return @@ -99,3 +100,12 @@ func parseEndpoint(endpoint string) (string, string, error) { return u.Scheme, "", fmt.Errorf("protocol %q not supported", u.Scheme) } } + +// LocalEndpoint returns the full path to a unix socket at the given endpoint +func LocalEndpoint(path, file string) string { + u := url.URL{ + Scheme: unixProtocol, + Path: path, + } + return filepath.Join(u.String(), file+".sock") +} diff --git a/pkg/kubelet/util/util_unsupported.go b/pkg/kubelet/util/util_unsupported.go index 77f14ea5255..6661678aced 100644 --- a/pkg/kubelet/util/util_unsupported.go +++ b/pkg/kubelet/util/util_unsupported.go @@ -40,3 +40,8 @@ func LockAndCheckSubPath(volumePath, subPath string) ([]uintptr, error) { // UnlockPath empty implementation func UnlockPath(fileHandles []uintptr) { } + +// LocalEndpoint empty implementation +func LocalEndpoint(path, file string) string { + return "" +} diff --git a/pkg/kubelet/util/util_windows.go b/pkg/kubelet/util/util_windows.go index cac80755a73..7123728ff94 100644 --- a/pkg/kubelet/util/util_windows.go +++ b/pkg/kubelet/util/util_windows.go @@ -103,3 +103,12 @@ func parseEndpoint(endpoint string) (string, string, error) { return u.Scheme, "", fmt.Errorf("protocol %q not supported", u.Scheme) } } + +// LocalEndpoint returns the full path to a windows named pipe +func LocalEndpoint(path, file string) string { + u := url.URL{ + Scheme: npipeProtocol, + Path: path, + } + return u.String() + "//./pipe/" + file +} diff --git a/pkg/kubelet/volume_host.go b/pkg/kubelet/volume_host.go index 35b7d784601..425afebdbf2 100644 --- a/pkg/kubelet/volume_host.go +++ b/pkg/kubelet/volume_host.go @@ -21,7 +21,7 @@ import ( "net" "runtime" - "github.com/golang/glog" + "k8s.io/klog" authenticationv1 "k8s.io/api/authentication/v1" "k8s.io/api/core/v1" @@ -162,7 +162,7 @@ func (kvh *kubeletVolumeHost) GetCloudProvider() cloudprovider.Interface { func (kvh *kubeletVolumeHost) GetMounter(pluginName string) mount.Interface { exec, err := kvh.getMountExec(pluginName) if err != nil { - glog.V(2).Infof("Error finding mount pod for plugin %s: %s", pluginName, err.Error()) + klog.V(2).Infof("Error finding mount pod for plugin %s: %s", pluginName, err.Error()) // Use the default mounter exec = nil } @@ -223,7 +223,7 @@ func (kvh *kubeletVolumeHost) GetEventRecorder() record.EventRecorder { func (kvh *kubeletVolumeHost) GetExec(pluginName string) mount.Exec { exec, err := kvh.getMountExec(pluginName) if err != nil { - glog.V(2).Infof("Error finding mount pod for plugin %s: %s", pluginName, err.Error()) + klog.V(2).Infof("Error finding mount pod for plugin %s: %s", pluginName, err.Error()) // Use the default exec exec = nil } @@ -238,7 +238,7 @@ func (kvh *kubeletVolumeHost) GetExec(pluginName string) mount.Exec { // os.Exec should be used. func (kvh *kubeletVolumeHost) getMountExec(pluginName string) (mount.Exec, error) { if !utilfeature.DefaultFeatureGate.Enabled(features.MountContainers) { - glog.V(5).Infof("using default mounter/exec for %s", pluginName) + klog.V(5).Infof("using default mounter/exec for %s", pluginName) return nil, nil } @@ -248,10 +248,10 @@ func (kvh *kubeletVolumeHost) getMountExec(pluginName string) (mount.Exec, error } if pod == nil { // Use default mounter/exec for this plugin - glog.V(5).Infof("using default mounter/exec for %s", pluginName) + klog.V(5).Infof("using default mounter/exec for %s", pluginName) return nil, nil } - glog.V(5).Infof("using container %s/%s/%s to execute mount utilities for %s", pod.Namespace, pod.Name, container, pluginName) + klog.V(5).Infof("using container %s/%s/%s to execute mount utilities for %s", pod.Namespace, pod.Name, container, pluginName) return &containerExec{ pod: pod, containerName: container, @@ -271,6 +271,6 @@ var _ mount.Exec = &containerExec{} func (e *containerExec) Run(cmd string, args ...string) ([]byte, error) { cmdline := append([]string{cmd}, args...) - glog.V(5).Infof("Exec mounter running in pod %s/%s/%s: %v", e.pod.Namespace, e.pod.Name, e.containerName, cmdline) + klog.V(5).Infof("Exec mounter running in pod %s/%s/%s: %v", e.pod.Namespace, e.pod.Name, e.containerName, cmdline) return e.kl.RunInContainer(container.GetPodFullName(e.pod), e.pod.UID, e.containerName, cmdline) } diff --git a/pkg/kubelet/volumemanager/BUILD b/pkg/kubelet/volumemanager/BUILD index 855d1b65700..269eb189192 100644 --- a/pkg/kubelet/volumemanager/BUILD +++ b/pkg/kubelet/volumemanager/BUILD @@ -33,7 +33,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/volumemanager/cache/BUILD b/pkg/kubelet/volumemanager/cache/BUILD index e9861638ab6..3bc1a3886bb 100644 --- a/pkg/kubelet/volumemanager/cache/BUILD +++ b/pkg/kubelet/volumemanager/cache/BUILD @@ -22,7 +22,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go index 82aef86c720..a7b06af94ea 100644 --- a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go @@ -24,11 +24,10 @@ import ( "fmt" "sync" - "github.com/golang/glog" - "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" @@ -407,7 +406,7 @@ func (asw *actualStateOfWorld) addVolume( } else { // If volume object already exists, update the fields such as device path volumeObj.devicePath = devicePath - glog.V(2).Infof("Volume %q is already added to attachedVolume list, update device path %q", + klog.V(2).Infof("Volume %q is already added to attachedVolume list, update device path %q", volumeName, devicePath) } @@ -476,7 +475,7 @@ func (asw *actualStateOfWorld) MarkVolumeAsResized( volumeName) } - glog.V(5).Infof("Volume %s(OuterVolumeSpecName %s) of pod %s has been resized", + klog.V(5).Infof("Volume %s(OuterVolumeSpecName %s) of pod %s has been resized", volumeName, podObj.outerVolumeSpecName, podName) podObj.fsResizeRequired = false asw.attachedVolumes[volumeName].mountedPods[podName] = podObj @@ -497,7 +496,7 @@ func (asw *actualStateOfWorld) MarkRemountRequired( asw.volumePluginMgr.FindPluginBySpec(podObj.volumeSpec) if err != nil || volumePlugin == nil { // Log and continue processing - glog.Errorf( + klog.Errorf( "MarkRemountRequired failed to FindPluginBySpec for pod %q (podUid %q) volume: %q (volSpecName: %q)", podObj.podName, podObj.podUID, @@ -521,13 +520,13 @@ func (asw *actualStateOfWorld) MarkFSResizeRequired( defer asw.Unlock() volumeObj, exist := asw.attachedVolumes[volumeName] if !exist { - glog.Warningf("MarkFSResizeRequired for volume %s failed as volume not exist", volumeName) + klog.Warningf("MarkFSResizeRequired for volume %s failed as volume not exist", volumeName) return } podObj, exist := volumeObj.mountedPods[podName] if !exist { - glog.Warningf("MarkFSResizeRequired for volume %s failed "+ + klog.Warningf("MarkFSResizeRequired for volume %s failed "+ "as pod(%s) not exist", volumeName, podName) return } @@ -536,7 +535,7 @@ func (asw *actualStateOfWorld) MarkFSResizeRequired( asw.volumePluginMgr.FindExpandablePluginBySpec(podObj.volumeSpec) if err != nil || volumePlugin == nil { // Log and continue processing - glog.Errorf( + klog.Errorf( "MarkFSResizeRequired failed to find expandable plugin for pod %q volume: %q (volSpecName: %q)", podObj.podName, volumeObj.volumeName, @@ -546,7 +545,7 @@ func (asw *actualStateOfWorld) MarkFSResizeRequired( if volumePlugin.RequiresFSResize() { if !podObj.fsResizeRequired { - glog.V(3).Infof("PVC volume %s(OuterVolumeSpecName %s) of pod %s requires file system resize", + klog.V(3).Infof("PVC volume %s(OuterVolumeSpecName %s) of pod %s requires file system resize", volumeName, podObj.outerVolumeSpecName, podName) podObj.fsResizeRequired = true } @@ -568,7 +567,9 @@ func (asw *actualStateOfWorld) SetVolumeGloballyMounted( volumeObj.globallyMounted = globallyMounted volumeObj.deviceMountPath = deviceMountPath - volumeObj.devicePath = devicePath + if devicePath != "" { + volumeObj.devicePath = devicePath + } asw.attachedVolumes[volumeName] = volumeObj return nil } diff --git a/pkg/kubelet/volumemanager/metrics/BUILD b/pkg/kubelet/volumemanager/metrics/BUILD index 6d44fabf88b..522eb010844 100644 --- a/pkg/kubelet/volumemanager/metrics/BUILD +++ b/pkg/kubelet/volumemanager/metrics/BUILD @@ -8,8 +8,9 @@ go_library( deps = [ "//pkg/kubelet/volumemanager/cache:go_default_library", "//pkg/volume:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//pkg/volume/util:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/volumemanager/metrics/metrics.go b/pkg/kubelet/volumemanager/metrics/metrics.go index c428401e08f..2536eb55917 100644 --- a/pkg/kubelet/volumemanager/metrics/metrics.go +++ b/pkg/kubelet/volumemanager/metrics/metrics.go @@ -19,10 +19,11 @@ package metrics import ( "sync" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/volumemanager/cache" "k8s.io/kubernetes/pkg/volume" + volumeutil "k8s.io/kubernetes/pkg/volume/util" ) const ( @@ -85,7 +86,7 @@ func (c *totalVolumesCollector) Collect(ch chan<- prometheus.Metric) { pluginName, stateName) if err != nil { - glog.Warningf("Failed to create metric : %v", err) + klog.Warningf("Failed to create metric : %v", err) } ch <- metric } @@ -95,7 +96,7 @@ func (c *totalVolumesCollector) Collect(ch chan<- prometheus.Metric) { func (c *totalVolumesCollector) getVolumeCount() volumeCount { counter := make(volumeCount) for _, mountedVolume := range c.asw.GetMountedVolumes() { - pluginName := mountedVolume.PluginName + pluginName := volumeutil.GetFullQualifiedPluginNameForVolume(mountedVolume.PluginName, mountedVolume.VolumeSpec) if pluginName == "" { pluginName = pluginNameNotAvailable } @@ -105,7 +106,7 @@ func (c *totalVolumesCollector) getVolumeCount() volumeCount { for _, volumeToMount := range c.dsw.GetVolumesToMount() { pluginName := pluginNameNotAvailable if plugin, err := c.pluginMgr.FindPluginBySpec(volumeToMount.VolumeSpec); err == nil { - pluginName = plugin.GetPluginName() + pluginName = volumeutil.GetFullQualifiedPluginNameForVolume(plugin.GetPluginName(), volumeToMount.VolumeSpec) } counter.add("desired_state_of_world", pluginName) } diff --git a/pkg/kubelet/volumemanager/populator/BUILD b/pkg/kubelet/volumemanager/populator/BUILD index 6a370b5124f..a41a69e0cf5 100644 --- a/pkg/kubelet/volumemanager/populator/BUILD +++ b/pkg/kubelet/volumemanager/populator/BUILD @@ -28,7 +28,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -47,7 +47,10 @@ filegroup( go_test( name = "go_default_test", - srcs = ["desired_state_of_world_populator_test.go"], + srcs = [ + "desired_state_of_world_populator_test.go", + "main_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/features:go_default_library", @@ -67,6 +70,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", ], diff --git a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go index 3f1b71a05ef..81bf6e60e91 100644 --- a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go +++ b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go @@ -25,7 +25,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -127,7 +127,7 @@ type processedPods struct { func (dswp *desiredStateOfWorldPopulator) Run(sourcesReady config.SourcesReady, stopCh <-chan struct{}) { // Wait for the completion of a loop that started after sources are all ready, then set hasAddedPods accordingly - glog.Infof("Desired state populator starts to run") + klog.Infof("Desired state populator starts to run") wait.PollUntil(dswp.loopSleepDuration, func() (bool, error) { done := sourcesReady.AllReady() dswp.populatorLoop() @@ -159,7 +159,7 @@ func (dswp *desiredStateOfWorldPopulator) populatorLoop() { // findAndRemoveDeletedPods() is called independently of the main // populator loop. if time.Since(dswp.timeOfLastGetPodStatus) < dswp.getPodStatusRetryDuration { - glog.V(5).Infof( + klog.V(5).Infof( "Skipping findAndRemoveDeletedPods(). Not permitted until %v (getPodStatusRetryDuration %v).", dswp.timeOfLastGetPodStatus.Add(dswp.getPodStatusRetryDuration), dswp.getPodStatusRetryDuration) @@ -230,7 +230,7 @@ func (dswp *desiredStateOfWorldPopulator) findAndRemoveDeletedPods() { var getPodsErr error runningPods, getPodsErr = dswp.kubeContainerRuntime.GetPods(false) if getPodsErr != nil { - glog.Errorf( + klog.Errorf( "kubeContainerRuntime.findAndRemoveDeletedPods returned error %v.", getPodsErr) continue @@ -252,17 +252,17 @@ func (dswp *desiredStateOfWorldPopulator) findAndRemoveDeletedPods() { } if runningContainers { - glog.V(4).Infof( + klog.V(4).Infof( "Pod %q has been removed from pod manager. However, it still has one or more containers in the non-exited state. Therefore, it will not be removed from volume manager.", format.Pod(volumeToMount.Pod)) continue } if !dswp.actualStateOfWorld.VolumeExists(volumeToMount.VolumeName) && podExists { - glog.V(4).Infof(volumeToMount.GenerateMsgDetailed("Actual state has not yet has this information skip removing volume from desired state", "")) + klog.V(4).Infof(volumeToMount.GenerateMsgDetailed("Actual state has not yet has this information skip removing volume from desired state", "")) continue } - glog.V(4).Infof(volumeToMount.GenerateMsgDetailed("Removing volume from desired state", "")) + klog.V(4).Infof(volumeToMount.GenerateMsgDetailed("Removing volume from desired state", "")) dswp.desiredStateOfWorld.DeletePodFromVolume( volumeToMount.PodName, volumeToMount.VolumeName) @@ -293,7 +293,7 @@ func (dswp *desiredStateOfWorldPopulator) processPodVolumes( pvc, volumeSpec, volumeGidValue, err := dswp.createVolumeSpec(podVolume, pod.Name, pod.Namespace, mountsMap, devicesMap) if err != nil { - glog.Errorf( + klog.Errorf( "Error processing volume %q for pod %q: %v", podVolume.Name, format.Pod(pod), @@ -306,7 +306,7 @@ func (dswp *desiredStateOfWorldPopulator) processPodVolumes( _, err = dswp.desiredStateOfWorld.AddPodToVolume( uniquePodName, pod, volumeSpec, podVolume.Name, volumeGidValue) if err != nil { - glog.Errorf( + klog.Errorf( "Failed to add volume %q (specName: %q) for pod %q to desiredStateOfWorld. err=%v", podVolume.Name, volumeSpec.Name(), @@ -315,7 +315,7 @@ func (dswp *desiredStateOfWorldPopulator) processPodVolumes( allVolumesAdded = false } - glog.V(4).Infof( + klog.V(4).Infof( "Added volume %q (volSpec=%q) for pod %q to desired state.", podVolume.Name, volumeSpec.Name(), @@ -365,12 +365,12 @@ func (dswp *desiredStateOfWorldPopulator) checkVolumeFSResize( } fsVolume, err := util.CheckVolumeModeFilesystem(volumeSpec) if err != nil { - glog.Errorf("Check volume mode failed for volume %s(OuterVolumeSpecName %s): %v", + klog.Errorf("Check volume mode failed for volume %s(OuterVolumeSpecName %s): %v", uniqueVolumeName, podVolume.Name, err) return } if !fsVolume { - glog.V(5).Infof("Block mode volume needn't to check file system resize request") + klog.V(5).Infof("Block mode volume needn't to check file system resize request") return } if processedVolumesForFSResize.Has(string(uniqueVolumeName)) { @@ -380,7 +380,7 @@ func (dswp *desiredStateOfWorldPopulator) checkVolumeFSResize( } if mountedReadOnlyByPod(podVolume, pod) { // This volume is used as read only by this pod, we don't perform resize for read only volumes. - glog.V(5).Infof("Skip file system resize check for volume %s in pod %s/%s "+ + klog.V(5).Infof("Skip file system resize check for volume %s in pod %s/%s "+ "as the volume is mounted as readonly", podVolume.Name, pod.Namespace, pod.Name) return } @@ -474,7 +474,7 @@ func (dswp *desiredStateOfWorldPopulator) createVolumeSpec( podVolume v1.Volume, podName string, podNamespace string, mountsMap map[string]bool, devicesMap map[string]bool) (*v1.PersistentVolumeClaim, *volume.Spec, string, error) { if pvcSource := podVolume.VolumeSource.PersistentVolumeClaim; pvcSource != nil { - glog.V(5).Infof( + klog.V(5).Infof( "Found PVC, ClaimName: %q/%q", podNamespace, pvcSource.ClaimName) @@ -491,7 +491,7 @@ func (dswp *desiredStateOfWorldPopulator) createVolumeSpec( } pvName, pvcUID := pvc.Spec.VolumeName, pvc.UID - glog.V(5).Infof( + klog.V(5).Infof( "Found bound PV for PVC (ClaimName %q/%q pvcUID %v): pvName=%q", podNamespace, pvcSource.ClaimName, @@ -509,7 +509,7 @@ func (dswp *desiredStateOfWorldPopulator) createVolumeSpec( err) } - glog.V(5).Infof( + klog.V(5).Infof( "Extracted volumeSpec (%v) from bound PV (pvName %q) and PVC (ClaimName %q/%q pvcUID %v)", volumeSpec.Name(), pvName, diff --git a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go index 969ac66911f..d2e90f031fa 100644 --- a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go +++ b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go @@ -27,6 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" "k8s.io/kubernetes/pkg/features" @@ -45,12 +46,14 @@ import ( func TestFindAndAddNewPods_FindAndRemoveDeletedPods(t *testing.T) { // create dswp + mode := v1.PersistentVolumeFilesystem pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "dswp-test-volume-name", }, Spec: v1.PersistentVolumeSpec{ - ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "file-bound"}, + ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "file-bound"}, + VolumeMode: &mode, }, } pvc := &v1.PersistentVolumeClaim{ @@ -151,7 +154,7 @@ func TestFindAndAddNewPods_FindAndRemoveDeletedPods(t *testing.T) { func TestFindAndAddNewPods_FindAndRemoveDeletedPods_Valid_Block_VolumeDevices(t *testing.T) { // Enable BlockVolume feature gate - utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() // create dswp mode := v1.PersistentVolumeBlock @@ -256,9 +259,6 @@ func TestFindAndAddNewPods_FindAndRemoveDeletedPods_Valid_Block_VolumeDevices(t expectedVolumeName) } } - - // Rollback feature gate to false. - utilfeature.DefaultFeatureGate.Set("BlockVolume=false") } func TestCreateVolumeSpec_Valid_File_VolumeMounts(t *testing.T) { @@ -309,7 +309,7 @@ func TestCreateVolumeSpec_Valid_File_VolumeMounts(t *testing.T) { func TestCreateVolumeSpec_Valid_Block_VolumeDevices(t *testing.T) { // Enable BlockVolume feature gate - utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() // create dswp mode := v1.PersistentVolumeBlock @@ -354,14 +354,11 @@ func TestCreateVolumeSpec_Valid_Block_VolumeDevices(t *testing.T) { if volumeSpec == nil || err != nil { t.Fatalf("Failed to create volumeSpec with combination of block mode and volumeDevices. err: %v", err) } - - // Rollback feature gate to false. - utilfeature.DefaultFeatureGate.Set("BlockVolume=false") } func TestCreateVolumeSpec_Invalid_File_VolumeDevices(t *testing.T) { // Enable BlockVolume feature gate - utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() // create dswp mode := v1.PersistentVolumeFilesystem @@ -406,14 +403,11 @@ func TestCreateVolumeSpec_Invalid_File_VolumeDevices(t *testing.T) { if volumeSpec != nil || err == nil { t.Fatalf("Unexpected volumeMode and volumeMounts/volumeDevices combination is accepted") } - - // Rollback feature gate to false. - utilfeature.DefaultFeatureGate.Set("BlockVolume=false") } func TestCreateVolumeSpec_Invalid_Block_VolumeMounts(t *testing.T) { // Enable BlockVolume feature gate - utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() // create dswp mode := v1.PersistentVolumeBlock @@ -458,12 +452,10 @@ func TestCreateVolumeSpec_Invalid_Block_VolumeMounts(t *testing.T) { if volumeSpec != nil || err == nil { t.Fatalf("Unexpected volumeMode and volumeMounts/volumeDevices combination is accepted") } - - // Rollback feature gate to false. - utilfeature.DefaultFeatureGate.Set("BlockVolume=false") } func TestCheckVolumeFSResize(t *testing.T) { + mode := v1.PersistentVolumeFilesystem pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "dswp-test-volume-name", @@ -472,6 +464,7 @@ func TestCheckVolumeFSResize(t *testing.T) { PersistentVolumeSource: v1.PersistentVolumeSource{RBD: &v1.RBDPersistentVolumeSource{}}, Capacity: volumeCapacity(1), ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "file-bound"}, + VolumeMode: &mode, }, } pvc := &v1.PersistentVolumeClaim{ @@ -510,7 +503,7 @@ func TestCheckVolumeFSResize(t *testing.T) { reconcileASW(fakeASW, fakeDSW, t) // No resize request for volume, volumes in ASW shouldn't be marked as fsResizeRequired. - setExpandOnlinePersistentVolumesFeatureGate("true", t) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExpandInUsePersistentVolumes, true)() resizeRequiredVolumes := reprocess(dswp, uniquePodName, fakeDSW, fakeASW) if len(resizeRequiredVolumes) > 0 { t.Fatalf("No resize request for any volumes, but found resize required volumes in ASW: %v", resizeRequiredVolumes) @@ -521,14 +514,14 @@ func TestCheckVolumeFSResize(t *testing.T) { pvc.Spec.Resources.Requests = volumeCapacity(2) // Disable the feature gate, so volume shouldn't be marked as fsResizeRequired. - setExpandOnlinePersistentVolumesFeatureGate("false", t) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExpandInUsePersistentVolumes, false)() resizeRequiredVolumes = reprocess(dswp, uniquePodName, fakeDSW, fakeASW) if len(resizeRequiredVolumes) > 0 { t.Fatalf("Feature gate disabled, but found resize required volumes in ASW: %v", resizeRequiredVolumes) } // Make volume used as ReadOnly, so volume shouldn't be marked as fsResizeRequired. - setExpandOnlinePersistentVolumesFeatureGate("true", t) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExpandInUsePersistentVolumes, true)() pod.Spec.Containers[0].VolumeMounts[0].ReadOnly = true resizeRequiredVolumes = reprocess(dswp, uniquePodName, fakeDSW, fakeASW) if len(resizeRequiredVolumes) > 0 { @@ -561,13 +554,6 @@ func volumeCapacity(size int) v1.ResourceList { return v1.ResourceList{v1.ResourceStorage: resource.MustParse(fmt.Sprintf("%dGi", size))} } -func setExpandOnlinePersistentVolumesFeatureGate(value string, t *testing.T) { - err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=%s", features.ExpandInUsePersistentVolumes, value)) - if err != nil { - t.Fatalf("Set ExpandInUsePersistentVolumes feature gate to %s failed: %v", value, err) - } -} - func reconcileASW(asw cache.ActualStateOfWorld, dsw cache.DesiredStateOfWorld, t *testing.T) { for _, volumeToMount := range dsw.GetVolumesToMount() { err := asw.MarkVolumeAsAttached(volumeToMount.VolumeName, volumeToMount.VolumeSpec, "", "") diff --git a/pkg/kubelet/volumemanager/populator/main_test.go b/pkg/kubelet/volumemanager/populator/main_test.go new file mode 100644 index 00000000000..de49504ed70 --- /dev/null +++ b/pkg/kubelet/volumemanager/populator/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package populator + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/kubelet/volumemanager/reconciler/BUILD b/pkg/kubelet/volumemanager/reconciler/BUILD index 586067da32e..53508804484 100644 --- a/pkg/kubelet/volumemanager/reconciler/BUILD +++ b/pkg/kubelet/volumemanager/reconciler/BUILD @@ -29,13 +29,16 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) go_test( name = "go_default_test", - srcs = ["reconciler_test.go"], + srcs = [ + "main_test.go", + "reconciler_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/features:go_default_library", @@ -52,10 +55,12 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/volumemanager/reconciler/main_test.go b/pkg/kubelet/volumemanager/reconciler/main_test.go new file mode 100644 index 00000000000..e89d5ca3ea0 --- /dev/null +++ b/pkg/kubelet/volumemanager/reconciler/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/kubelet/volumemanager/reconciler/reconciler.go b/pkg/kubelet/volumemanager/reconciler/reconciler.go index 2ffb5c99fa7..1b6bbbfbe77 100644 --- a/pkg/kubelet/volumemanager/reconciler/reconciler.go +++ b/pkg/kubelet/volumemanager/reconciler/reconciler.go @@ -26,13 +26,13 @@ import ( "path" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/config" "k8s.io/kubernetes/pkg/kubelet/volumemanager/cache" @@ -151,7 +151,7 @@ func (rc *reconciler) reconciliationLoopFunc() func() { // Otherwise, the reconstruct process may clean up pods' volumes that are still in use because // desired state of world does not contain a complete list of pods. if rc.populatorHasAddedPods() && !rc.StatesHasBeenSynced() { - glog.Infof("Reconciler: start to sync state") + klog.Infof("Reconciler: start to sync state") rc.sync() } } @@ -167,7 +167,7 @@ func (rc *reconciler) reconcile() { for _, mountedVolume := range rc.actualStateOfWorld.GetMountedVolumes() { if !rc.desiredStateOfWorld.PodExistsInVolume(mountedVolume.PodName, mountedVolume.VolumeName) { // Volume is mounted, unmount it - glog.V(5).Infof(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountVolume", "")) + klog.V(5).Infof(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountVolume", "")) err := rc.operationExecutor.UnmountVolume( mountedVolume.MountedVolume, rc.actualStateOfWorld, rc.kubeletPodsDir) if err != nil && @@ -175,10 +175,10 @@ func (rc *reconciler) reconcile() { !exponentialbackoff.IsExponentialBackoff(err) { // Ignore nestedpendingoperations.IsAlreadyExists and exponentialbackoff.IsExponentialBackoff errors, they are expected. // Log all other errors. - glog.Errorf(mountedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.UnmountVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error()) + klog.Errorf(mountedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.UnmountVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error()) } if err == nil { - glog.Infof(mountedVolume.GenerateMsgDetailed("operationExecutor.UnmountVolume started", "")) + klog.Infof(mountedVolume.GenerateMsgDetailed("operationExecutor.UnmountVolume started", "")) } } } @@ -191,7 +191,7 @@ func (rc *reconciler) reconcile() { if rc.controllerAttachDetachEnabled || !volumeToMount.PluginIsAttachable { // Volume is not attached (or doesn't implement attacher), kubelet attach is disabled, wait // for controller to finish attaching volume. - glog.V(5).Infof(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.VerifyControllerAttachedVolume", "")) + klog.V(5).Infof(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.VerifyControllerAttachedVolume", "")) err := rc.operationExecutor.VerifyControllerAttachedVolume( volumeToMount.VolumeToMount, rc.nodeName, @@ -201,10 +201,10 @@ func (rc *reconciler) reconcile() { !exponentialbackoff.IsExponentialBackoff(err) { // Ignore nestedpendingoperations.IsAlreadyExists and exponentialbackoff.IsExponentialBackoff errors, they are expected. // Log all other errors. - glog.Errorf(volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.VerifyControllerAttachedVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error()) + klog.Errorf(volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.VerifyControllerAttachedVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error()) } if err == nil { - glog.Infof(volumeToMount.GenerateMsgDetailed("operationExecutor.VerifyControllerAttachedVolume started", "")) + klog.Infof(volumeToMount.GenerateMsgDetailed("operationExecutor.VerifyControllerAttachedVolume started", "")) } } else { // Volume is not attached to node, kubelet attach is enabled, volume implements an attacher, @@ -214,17 +214,17 @@ func (rc *reconciler) reconcile() { VolumeSpec: volumeToMount.VolumeSpec, NodeName: rc.nodeName, } - glog.V(5).Infof(volumeToAttach.GenerateMsgDetailed("Starting operationExecutor.AttachVolume", "")) + klog.V(5).Infof(volumeToAttach.GenerateMsgDetailed("Starting operationExecutor.AttachVolume", "")) err := rc.operationExecutor.AttachVolume(volumeToAttach, rc.actualStateOfWorld) if err != nil && !nestedpendingoperations.IsAlreadyExists(err) && !exponentialbackoff.IsExponentialBackoff(err) { // Ignore nestedpendingoperations.IsAlreadyExists and exponentialbackoff.IsExponentialBackoff errors, they are expected. // Log all other errors. - glog.Errorf(volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.AttachVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error()) + klog.Errorf(volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.AttachVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error()) } if err == nil { - glog.Infof(volumeToMount.GenerateMsgDetailed("operationExecutor.AttachVolume started", "")) + klog.Infof(volumeToMount.GenerateMsgDetailed("operationExecutor.AttachVolume started", "")) } } } else if !volMounted || cache.IsRemountRequiredError(err) { @@ -234,7 +234,7 @@ func (rc *reconciler) reconcile() { if isRemount { remountingLogStr = "Volume is already mounted to pod, but remount was requested." } - glog.V(4).Infof(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.MountVolume", remountingLogStr)) + klog.V(4).Infof(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.MountVolume", remountingLogStr)) err := rc.operationExecutor.MountVolume( rc.waitForAttachTimeout, volumeToMount.VolumeToMount, @@ -245,18 +245,18 @@ func (rc *reconciler) reconcile() { !exponentialbackoff.IsExponentialBackoff(err) { // Ignore nestedpendingoperations.IsAlreadyExists and exponentialbackoff.IsExponentialBackoff errors, they are expected. // Log all other errors. - glog.Errorf(volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.MountVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error()) + klog.Errorf(volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.MountVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error()) } if err == nil { if remountingLogStr == "" { - glog.V(1).Infof(volumeToMount.GenerateMsgDetailed("operationExecutor.MountVolume started", remountingLogStr)) + klog.V(1).Infof(volumeToMount.GenerateMsgDetailed("operationExecutor.MountVolume started", remountingLogStr)) } else { - glog.V(5).Infof(volumeToMount.GenerateMsgDetailed("operationExecutor.MountVolume started", remountingLogStr)) + klog.V(5).Infof(volumeToMount.GenerateMsgDetailed("operationExecutor.MountVolume started", remountingLogStr)) } } } else if cache.IsFSResizeRequiredError(err) && utilfeature.DefaultFeatureGate.Enabled(features.ExpandInUsePersistentVolumes) { - glog.V(4).Infof(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.ExpandVolumeFSWithoutUnmounting", "")) + klog.V(4).Infof(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.ExpandVolumeFSWithoutUnmounting", "")) err := rc.operationExecutor.ExpandVolumeFSWithoutUnmounting( volumeToMount.VolumeToMount, rc.actualStateOfWorld) @@ -265,10 +265,10 @@ func (rc *reconciler) reconcile() { !exponentialbackoff.IsExponentialBackoff(err) { // Ignore nestedpendingoperations.IsAlreadyExists and exponentialbackoff.IsExponentialBackoff errors, they are expected. // Log all other errors. - glog.Errorf(volumeToMount.GenerateErrorDetailed("operationExecutor.ExpandVolumeFSWithoutUnmounting failed", err).Error()) + klog.Errorf(volumeToMount.GenerateErrorDetailed("operationExecutor.ExpandVolumeFSWithoutUnmounting failed", err).Error()) } if err == nil { - glog.V(4).Infof(volumeToMount.GenerateMsgDetailed("operationExecutor.ExpandVolumeFSWithoutUnmounting started", "")) + klog.V(4).Infof(volumeToMount.GenerateMsgDetailed("operationExecutor.ExpandVolumeFSWithoutUnmounting started", "")) } } } @@ -280,7 +280,7 @@ func (rc *reconciler) reconcile() { !rc.operationExecutor.IsOperationPending(attachedVolume.VolumeName, nestedpendingoperations.EmptyUniquePodName) { if attachedVolume.GloballyMounted { // Volume is globally mounted to device, unmount it - glog.V(5).Infof(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountDevice", "")) + klog.V(5).Infof(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountDevice", "")) err := rc.operationExecutor.UnmountDevice( attachedVolume.AttachedVolume, rc.actualStateOfWorld, rc.mounter) if err != nil && @@ -288,20 +288,20 @@ func (rc *reconciler) reconcile() { !exponentialbackoff.IsExponentialBackoff(err) { // Ignore nestedpendingoperations.IsAlreadyExists and exponentialbackoff.IsExponentialBackoff errors, they are expected. // Log all other errors. - glog.Errorf(attachedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.UnmountDevice failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error()) + klog.Errorf(attachedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.UnmountDevice failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error()) } if err == nil { - glog.Infof(attachedVolume.GenerateMsgDetailed("operationExecutor.UnmountDevice started", "")) + klog.Infof(attachedVolume.GenerateMsgDetailed("operationExecutor.UnmountDevice started", "")) } } else { // Volume is attached to node, detach it // Kubelet not responsible for detaching or this volume has a non-attachable volume plugin. if rc.controllerAttachDetachEnabled || !attachedVolume.PluginIsAttachable { rc.actualStateOfWorld.MarkVolumeAsDetached(attachedVolume.VolumeName, attachedVolume.NodeName) - glog.Infof(attachedVolume.GenerateMsgDetailed("Volume detached", fmt.Sprintf("DevicePath %q", attachedVolume.DevicePath))) + klog.Infof(attachedVolume.GenerateMsgDetailed("Volume detached", fmt.Sprintf("DevicePath %q", attachedVolume.DevicePath))) } else { // Only detach if kubelet detach is enabled - glog.V(5).Infof(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.DetachVolume", "")) + klog.V(5).Infof(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.DetachVolume", "")) err := rc.operationExecutor.DetachVolume( attachedVolume.AttachedVolume, false /* verifySafeToDetach */, rc.actualStateOfWorld) if err != nil && @@ -309,10 +309,10 @@ func (rc *reconciler) reconcile() { !exponentialbackoff.IsExponentialBackoff(err) { // Ignore nestedpendingoperations.IsAlreadyExists && exponentialbackoff.IsExponentialBackoff errors, they are expected. // Log all other errors. - glog.Errorf(attachedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.DetachVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error()) + klog.Errorf(attachedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.DetachVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error()) } if err == nil { - glog.Infof(attachedVolume.GenerateMsgDetailed("operationExecutor.DetachVolume started", "")) + klog.Infof(attachedVolume.GenerateMsgDetailed("operationExecutor.DetachVolume started", "")) } } } @@ -369,14 +369,14 @@ func (rc *reconciler) syncStates() { // Get volumes information by reading the pod's directory podVolumes, err := getVolumesFromPodDir(rc.kubeletPodsDir) if err != nil { - glog.Errorf("Cannot get volumes from disk %v", err) + klog.Errorf("Cannot get volumes from disk %v", err) return } volumesNeedUpdate := make(map[v1.UniqueVolumeName]*reconstructedVolume) volumeNeedReport := []v1.UniqueVolumeName{} for _, volume := range podVolumes { if rc.actualStateOfWorld.VolumeExistsWithSpecName(volume.podName, volume.volumeSpecName) { - glog.V(4).Infof("Volume exists in actual state (volume.SpecName %s, pod.UID %s), skip cleaning up mounts", volume.volumeSpecName, volume.podName) + klog.V(4).Infof("Volume exists in actual state (volume.SpecName %s, pod.UID %s), skip cleaning up mounts", volume.volumeSpecName, volume.podName) // There is nothing to reconstruct continue } @@ -387,11 +387,11 @@ func (rc *reconciler) syncStates() { if volumeInDSW { // Some pod needs the volume, don't clean it up and hope that // reconcile() calls SetUp and reconstructs the volume in ASW. - glog.V(4).Infof("Volume exists in desired state (volume.SpecName %s, pod.UID %s), skip cleaning up mounts", volume.volumeSpecName, volume.podName) + klog.V(4).Infof("Volume exists in desired state (volume.SpecName %s, pod.UID %s), skip cleaning up mounts", volume.volumeSpecName, volume.podName) continue } // No pod needs the volume. - glog.Warningf("Could not construct volume information, cleanup the mounts. (pod.UID %s, volume.SpecName %s): %v", volume.podName, volume.volumeSpecName, err) + klog.Warningf("Could not construct volume information, cleanup the mounts. (pod.UID %s, volume.SpecName %s): %v", volume.podName, volume.volumeSpecName, err) rc.cleanupMounts(volume) continue } @@ -402,14 +402,14 @@ func (rc *reconciler) syncStates() { // this new kubelet so reconcile() calls SetUp and re-mounts the // volume if it's necessary. volumeNeedReport = append(volumeNeedReport, reconstructedVolume.volumeName) - glog.V(4).Infof("Volume exists in desired state (volume.SpecName %s, pod.UID %s), marking as InUse", volume.volumeSpecName, volume.podName) + klog.V(4).Infof("Volume exists in desired state (volume.SpecName %s, pod.UID %s), marking as InUse", volume.volumeSpecName, volume.podName) continue } // There is no pod that uses the volume. if rc.operationExecutor.IsOperationPending(reconstructedVolume.volumeName, nestedpendingoperations.EmptyUniquePodName) { - glog.Warning("Volume is in pending operation, skip cleaning up mounts") + klog.Warning("Volume is in pending operation, skip cleaning up mounts") } - glog.V(2).Infof( + klog.V(2).Infof( "Reconciler sync states: could not find pod information in desired state, update it in actual state: %+v", reconstructedVolume) volumesNeedUpdate[reconstructedVolume.volumeName] = reconstructedVolume @@ -417,7 +417,7 @@ func (rc *reconciler) syncStates() { if len(volumesNeedUpdate) > 0 { if err = rc.updateStates(volumesNeedUpdate); err != nil { - glog.Errorf("Error occurred during reconstruct volume from disk: %v", err) + klog.Errorf("Error occurred during reconstruct volume from disk: %v", err) } } if len(volumeNeedReport) > 0 { @@ -426,7 +426,7 @@ func (rc *reconciler) syncStates() { } func (rc *reconciler) cleanupMounts(volume podVolume) { - glog.V(2).Infof("Reconciler sync states: could not find information (PID: %s) (Volume SpecName: %s) in desired state, clean up the mount points", + klog.V(2).Infof("Reconciler sync states: could not find information (PID: %s) (Volume SpecName: %s) in desired state, clean up the mount points", volume.podName, volume.volumeSpecName) mountedVolume := operationexecutor.MountedVolume{ PodName: volume.podName, @@ -439,7 +439,7 @@ func (rc *reconciler) cleanupMounts(volume podVolume) { // to unmount both volume and device in the same routine. err := rc.operationExecutor.UnmountVolume(mountedVolume, rc.actualStateOfWorld, rc.kubeletPodsDir) if err != nil { - glog.Errorf(mountedVolume.GenerateErrorDetailed(fmt.Sprintf("volumeHandler.UnmountVolumeHandler for UnmountVolume failed"), err).Error()) + klog.Errorf(mountedVolume.GenerateErrorDetailed(fmt.Sprintf("volumeHandler.UnmountVolumeHandler for UnmountVolume failed"), err).Error()) return } } @@ -557,13 +557,13 @@ func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume, func (rc *reconciler) updateDevicePath(volumesNeedUpdate map[v1.UniqueVolumeName]*reconstructedVolume) { node, fetchErr := rc.kubeClient.CoreV1().Nodes().Get(string(rc.nodeName), metav1.GetOptions{}) if fetchErr != nil { - glog.Errorf("updateStates in reconciler: could not get node status with error %v", fetchErr) + klog.Errorf("updateStates in reconciler: could not get node status with error %v", fetchErr) } else { for _, attachedVolume := range node.Status.VolumesAttached { if volume, exists := volumesNeedUpdate[attachedVolume.Name]; exists { volume.devicePath = attachedVolume.DevicePath volumesNeedUpdate[attachedVolume.Name] = volume - glog.V(4).Infof("Update devicePath from node status for volume (%q): %q", attachedVolume.Name, volume.devicePath) + klog.V(4).Infof("Update devicePath from node status for volume (%q): %q", attachedVolume.Name, volume.devicePath) } } } @@ -599,7 +599,7 @@ func (rc *reconciler) updateStates(volumesNeedUpdate map[v1.UniqueVolumeName]*re //TODO: the devicePath might not be correct for some volume plugins: see issue #54108 volume.volumeName, volume.volumeSpec, "" /* nodeName */, volume.devicePath) if err != nil { - glog.Errorf("Could not add volume information to actual state of world: %v", err) + klog.Errorf("Could not add volume information to actual state of world: %v", err) continue } err = rc.actualStateOfWorld.MarkVolumeAsMounted( @@ -612,22 +612,22 @@ func (rc *reconciler) updateStates(volumesNeedUpdate map[v1.UniqueVolumeName]*re volume.volumeGidValue, volume.volumeSpec) if err != nil { - glog.Errorf("Could not add pod to volume information to actual state of world: %v", err) + klog.Errorf("Could not add pod to volume information to actual state of world: %v", err) continue } - glog.V(4).Infof("Volume: %s (pod UID %s) is marked as mounted and added into the actual state", volume.volumeName, volume.podName) + klog.V(4).Infof("Volume: %s (pod UID %s) is marked as mounted and added into the actual state", volume.volumeName, volume.podName) if volume.attachablePlugin != nil { deviceMountPath, err := getDeviceMountPath(volume) if err != nil { - glog.Errorf("Could not find device mount path for volume %s", volume.volumeName) + klog.Errorf("Could not find device mount path for volume %s", volume.volumeName) continue } err = rc.actualStateOfWorld.MarkDeviceAsMounted(volume.volumeName, volume.devicePath, deviceMountPath) if err != nil { - glog.Errorf("Could not mark device is mounted to actual state of world: %v", err) + klog.Errorf("Could not mark device is mounted to actual state of world: %v", err) continue } - glog.V(4).Infof("Volume: %s (pod UID %s) is marked device as mounted and added into the actual state", volume.volumeName, volume.podName) + klog.V(4).Infof("Volume: %s (pod UID %s) is marked device as mounted and added into the actual state", volume.volumeName, volume.podName) } } return nil @@ -671,13 +671,13 @@ func getVolumesFromPodDir(podDir string) ([]podVolume, error) { volumePluginPath := path.Join(volumesDir, pluginName) volumePluginDirs, err := utilfile.ReadDirNoStat(volumePluginPath) if err != nil { - glog.Errorf("Could not read volume plugin directory %q: %v", volumePluginPath, err) + klog.Errorf("Could not read volume plugin directory %q: %v", volumePluginPath, err) continue } unescapePluginName := utilstrings.UnescapeQualifiedNameForDisk(pluginName) for _, volumeName := range volumePluginDirs { mountPath := path.Join(volumePluginPath, volumeName) - glog.V(5).Infof("podName: %v, mount path from volume plugin directory: %v, ", podName, mountPath) + klog.V(5).Infof("podName: %v, mount path from volume plugin directory: %v, ", podName, mountPath) volumes = append(volumes, podVolume{ podName: volumetypes.UniquePodName(podName), volumeSpecName: volumeName, @@ -689,6 +689,6 @@ func getVolumesFromPodDir(podDir string) ([]podVolume, error) { } } } - glog.V(4).Infof("Get volumes from pod directory %q %+v", podDir, volumes) + klog.V(4).Infof("Get volumes from pod directory %q %+v", podDir, volumes) return volumes, nil } diff --git a/pkg/kubelet/volumemanager/reconciler/reconciler_test.go b/pkg/kubelet/volumemanager/reconciler/reconciler_test.go index 38eaba2f03c..0bdd4078d6e 100644 --- a/pkg/kubelet/volumemanager/reconciler/reconciler_test.go +++ b/pkg/kubelet/volumemanager/reconciler/reconciler_test.go @@ -29,9 +29,11 @@ import ( k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" "k8s.io/client-go/tools/record" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/volumemanager/cache" "k8s.io/kubernetes/pkg/util/mount" @@ -439,7 +441,8 @@ func Test_Run_Positive_VolumeUnmountControllerAttachEnabled(t *testing.T) { // no detach/teardownDevice calls. func Test_Run_Positive_VolumeAttachAndMap(t *testing.T) { // Enable BlockVolume feature gate - utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() + // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t) dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) @@ -513,9 +516,6 @@ func Test_Run_Positive_VolumeAttachAndMap(t *testing.T) { 1 /* expectedGetMapDeviceCallCount */, fakePlugin)) assert.NoError(t, volumetesting.VerifyZeroTearDownDeviceCallCount(fakePlugin)) assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin)) - - // Rollback feature gate to false. - utilfeature.DefaultFeatureGate.Set("BlockVolume=false") } // Populates desiredStateOfWorld cache with one volume/pod. @@ -526,7 +526,8 @@ func Test_Run_Positive_VolumeAttachAndMap(t *testing.T) { // Verifies there are no attach/detach calls. func Test_Run_Positive_BlockVolumeMapControllerAttachEnabled(t *testing.T) { // Enable BlockVolume feature gate - utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() + // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t) dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) @@ -601,9 +602,6 @@ func Test_Run_Positive_BlockVolumeMapControllerAttachEnabled(t *testing.T) { 1 /* expectedGetMapDeviceCallCount */, fakePlugin)) assert.NoError(t, volumetesting.VerifyZeroTearDownDeviceCallCount(fakePlugin)) assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin)) - - // Rollback feature gate to false. - utilfeature.DefaultFeatureGate.Set("BlockVolume=false") } // Populates desiredStateOfWorld cache with one volume/pod. @@ -614,7 +612,8 @@ func Test_Run_Positive_BlockVolumeMapControllerAttachEnabled(t *testing.T) { // Verifies one detach/teardownDevice calls are issued. func Test_Run_Positive_BlockVolumeAttachMapUnmapDetach(t *testing.T) { // Enable BlockVolume feature gate - utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() + // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t) dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) @@ -698,9 +697,6 @@ func Test_Run_Positive_BlockVolumeAttachMapUnmapDetach(t *testing.T) { 1 /* expectedTearDownDeviceCallCount */, fakePlugin)) assert.NoError(t, volumetesting.VerifyDetachCallCount( 1 /* expectedDetachCallCount */, fakePlugin)) - - // Rollback feature gate to false. - utilfeature.DefaultFeatureGate.Set("BlockVolume=false") } // Populates desiredStateOfWorld cache with one volume/pod. @@ -712,7 +708,8 @@ func Test_Run_Positive_BlockVolumeAttachMapUnmapDetach(t *testing.T) { // Verifies there are no attach/detach calls made. func Test_Run_Positive_VolumeUnmapControllerAttachEnabled(t *testing.T) { // Enable BlockVolume feature gate - utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() + // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t) dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) @@ -797,9 +794,6 @@ func Test_Run_Positive_VolumeUnmapControllerAttachEnabled(t *testing.T) { assert.NoError(t, volumetesting.VerifyTearDownDeviceCallCount( 1 /* expectedTearDownDeviceCallCount */, fakePlugin)) assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin)) - - // Rollback feature gate to false. - utilfeature.DefaultFeatureGate.Set("BlockVolume=false") } func Test_GenerateMapVolumeFunc_Plugin_Not_Found(t *testing.T) { @@ -821,7 +815,8 @@ func Test_GenerateMapVolumeFunc_Plugin_Not_Found(t *testing.T) { } // Enable BlockVolume feature gate - utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() + for name, tc := range testCases { t.Run(name, func(t *testing.T) { volumePluginMgr := &volume.VolumePluginMgr{} @@ -853,8 +848,6 @@ func Test_GenerateMapVolumeFunc_Plugin_Not_Found(t *testing.T) { } }) } - // Rollback feature gate to false. - utilfeature.DefaultFeatureGate.Set("BlockVolume=false") } func Test_GenerateUnmapVolumeFunc_Plugin_Not_Found(t *testing.T) { @@ -876,7 +869,8 @@ func Test_GenerateUnmapVolumeFunc_Plugin_Not_Found(t *testing.T) { } // Enable BlockVolume feature gate - utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() + for name, tc := range testCases { t.Run(name, func(t *testing.T) { volumePluginMgr := &volume.VolumePluginMgr{} @@ -900,8 +894,6 @@ func Test_GenerateUnmapVolumeFunc_Plugin_Not_Found(t *testing.T) { } }) } - // Rollback feature gate to false. - utilfeature.DefaultFeatureGate.Set("BlockVolume=false") } func Test_GenerateUnmapDeviceFunc_Plugin_Not_Found(t *testing.T) { @@ -923,7 +915,8 @@ func Test_GenerateUnmapDeviceFunc_Plugin_Not_Found(t *testing.T) { } // Enable BlockVolume feature gate - utilfeature.DefaultFeatureGate.Set("BlockVolume=true") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, true)() + for name, tc := range testCases { t.Run(name, func(t *testing.T) { volumePluginMgr := &volume.VolumePluginMgr{} @@ -946,8 +939,6 @@ func Test_GenerateUnmapDeviceFunc_Plugin_Not_Found(t *testing.T) { } }) } - // Rollback feature gate to false. - utilfeature.DefaultFeatureGate.Set("BlockVolume=false") } // Populates desiredStateOfWorld cache with one volume/pod. @@ -957,14 +948,17 @@ func Test_GenerateUnmapDeviceFunc_Plugin_Not_Found(t *testing.T) { // Mark volume as fsResizeRequired in ASW. // Verifies volume's fsResizeRequired flag is cleared later. func Test_Run_Positive_VolumeFSResizeControllerAttachEnabled(t *testing.T) { - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.ExpandInUsePersistentVolumes)) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExpandInUsePersistentVolumes, true)() + + fs := v1.PersistentVolumeFilesystem pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pv", UID: "pvuid", }, Spec: v1.PersistentVolumeSpec{ - ClaimRef: &v1.ObjectReference{Name: "pvc"}, + ClaimRef: &v1.ObjectReference{Name: "pvc"}, + VolumeMode: &fs, }, } pvc := &v1.PersistentVolumeClaim{ @@ -974,6 +968,7 @@ func Test_Run_Positive_VolumeFSResizeControllerAttachEnabled(t *testing.T) { }, Spec: v1.PersistentVolumeClaimSpec{ VolumeName: "pv", + VolumeMode: &fs, }, } pod := &v1.Pod{ @@ -1129,7 +1124,7 @@ func createTestClient() *fake.Clientset { VolumesAttached: []v1.AttachedVolume{ { Name: "fake-plugin/fake-device1", - DevicePath: "fake/path", + DevicePath: "/fake/path", }, }}, }, nil @@ -1170,3 +1165,96 @@ func createtestClientWithPVPVC(pv *v1.PersistentVolume, pvc *v1.PersistentVolume }) return fakeClient } + +func Test_Run_Positive_VolumeMountControllerAttachEnabledRace(t *testing.T) { + // Arrange + volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t) + + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) + kubeClient := createTestClient() + fakeRecorder := &record.FakeRecorder{} + fakeHandler := volumetesting.NewBlockVolumePathHandler() + oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( + kubeClient, + volumePluginMgr, + fakeRecorder, + false, /* checkNodeCapabilitiesBeforeMount */ + fakeHandler)) + reconciler := NewReconciler( + kubeClient, + true, /* controllerAttachDetachEnabled */ + reconcilerLoopSleepDuration, + reconcilerSyncStatesSleepPeriod, + waitForAttachTimeout, + nodeName, + dsw, + asw, + hasAddedPods, + oex, + &mount.FakeMounter{}, + volumePluginMgr, + kubeletPodsDir) + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + UID: "pod1uid", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "volume-name", + VolumeSource: v1.VolumeSource{ + GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ + PDName: "fake-device1", + }, + }, + }, + }, + }, + } + + // Some steps are executes out of order in callbacks, follow the numbers. + + // 1. Add a volume to DSW and wait until it's mounted + volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]} + podName := util.GetUniquePodName(pod) + generatedVolumeName, err := dsw.AddPodToVolume( + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + dsw.MarkVolumesReportedInUse([]v1.UniqueVolumeName{generatedVolumeName}) + + if err != nil { + t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) + } + runReconciler(reconciler) + waitForMount(t, fakePlugin, generatedVolumeName, asw) + + finished := make(chan interface{}) + fakePlugin.UnmountDeviceHook = func(mountPath string) error { + // Act: + // 3. While a volume is being unmounted, add it back to the desired state of world + klog.Infof("UnmountDevice called") + generatedVolumeName, err = dsw.AddPodToVolume( + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + dsw.MarkVolumesReportedInUse([]v1.UniqueVolumeName{generatedVolumeName}) + return nil + } + + fakePlugin.WaitForAttachHook = func(spec *volume.Spec, devicePath string, pod *v1.Pod, spectimeout time.Duration) (string, error) { + // Assert + // 4. When the volume is mounted again, expect that UnmountDevice operation did not clear devicePath + if devicePath == "" { + t.Errorf("Expected WaitForAttach called with devicePath from Node.Status") + close(finished) + return "", fmt.Errorf("Expected devicePath from Node.Status") + } + close(finished) + return devicePath, nil + } + + // 2. Delete the volume from DSW (and wait for callbacks) + dsw.DeletePodFromVolume(podName, generatedVolumeName) + + <-finished + waitForMount(t, fakePlugin, generatedVolumeName, asw) +} diff --git a/pkg/kubelet/volumemanager/volume_manager.go b/pkg/kubelet/volumemanager/volume_manager.go index 06bdcb88425..ba3d99d64c8 100644 --- a/pkg/kubelet/volumemanager/volume_manager.go +++ b/pkg/kubelet/volumemanager/volume_manager.go @@ -22,7 +22,6 @@ import ( "strconv" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/runtime" @@ -30,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/record" + "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/config" "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/pod" @@ -243,15 +243,15 @@ func (vm *volumeManager) Run(sourcesReady config.SourcesReady, stopCh <-chan str defer runtime.HandleCrash() go vm.desiredStateOfWorldPopulator.Run(sourcesReady, stopCh) - glog.V(2).Infof("The desired_state_of_world populator starts") + klog.V(2).Infof("The desired_state_of_world populator starts") - glog.Infof("Starting Kubelet Volume Manager") + klog.Infof("Starting Kubelet Volume Manager") go vm.reconciler.Run(stopCh) metrics.Register(vm.actualStateOfWorld, vm.desiredStateOfWorld, vm.volumePluginMgr) <-stopCh - glog.Infof("Shutting down Kubelet Volume Manager") + klog.Infof("Shutting down Kubelet Volume Manager") } func (vm *volumeManager) GetMountedVolumesForPod(podName types.UniquePodName) container.VolumeMap { @@ -347,7 +347,7 @@ func (vm *volumeManager) WaitForAttachAndMount(pod *v1.Pod) error { return nil } - glog.V(3).Infof("Waiting for volumes to attach and mount for pod %q", format.Pod(pod)) + klog.V(3).Infof("Waiting for volumes to attach and mount for pod %q", format.Pod(pod)) uniquePodName := util.GetUniquePodName(pod) // Some pods expect to have Setup called over and over again to update. @@ -380,7 +380,7 @@ func (vm *volumeManager) WaitForAttachAndMount(pod *v1.Pod) error { unattachedVolumes) } - glog.V(3).Infof("All volumes are attached and mounted for pod %q", format.Pod(pod)) + klog.V(3).Infof("All volumes are attached and mounted for pod %q", format.Pod(pod)) return nil } diff --git a/pkg/kubelet/volumemanager/volume_manager_test.go b/pkg/kubelet/volumemanager/volume_manager_test.go index 74656d8054a..4d03a097ac3 100644 --- a/pkg/kubelet/volumemanager/volume_manager_test.go +++ b/pkg/kubelet/volumemanager/volume_manager_test.go @@ -167,6 +167,7 @@ func TestGetExtraSupplementalGroupsForPod(t *testing.T) { } for _, tc := range cases { + fs := v1.PersistentVolumeFilesystem pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pvA", @@ -183,6 +184,7 @@ func TestGetExtraSupplementalGroupsForPod(t *testing.T) { ClaimRef: &v1.ObjectReference{ Name: claim.ObjectMeta.Name, }, + VolumeMode: &fs, }, } kubeClient := fake.NewSimpleClientset(node, pod, pv, claim) @@ -273,6 +275,7 @@ func createObjects() (*v1.Node, *v1.Pod, *v1.PersistentVolume, *v1.PersistentVol }, }, } + fs := v1.PersistentVolumeFilesystem pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pvA", @@ -286,6 +289,7 @@ func createObjects() (*v1.Node, *v1.Pod, *v1.PersistentVolume, *v1.PersistentVol ClaimRef: &v1.ObjectReference{ Name: "claimA", }, + VolumeMode: &fs, }, } claim := &v1.PersistentVolumeClaim{ diff --git a/pkg/kubelet/winstats/BUILD b/pkg/kubelet/winstats/BUILD index 731b86303c2..25afdd4cfc0 100644 --- a/pkg/kubelet/winstats/BUILD +++ b/pkg/kubelet/winstats/BUILD @@ -15,10 +15,10 @@ go_library( "@io_bazel_rules_go//go/platform:windows": [ "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/github.com/JeffAshton/win_pdh:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library", "//vendor/golang.org/x/sys/windows:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "//conditions:default": [], }), diff --git a/pkg/kubelet/winstats/perfcounter_nodestats.go b/pkg/kubelet/winstats/perfcounter_nodestats.go index 9777d88745a..51fb15ae93d 100644 --- a/pkg/kubelet/winstats/perfcounter_nodestats.go +++ b/pkg/kubelet/winstats/perfcounter_nodestats.go @@ -26,10 +26,10 @@ import ( "time" "unsafe" - "github.com/golang/glog" cadvisorapi "github.com/google/cadvisor/info/v1" "golang.org/x/sys/windows" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" ) // MemoryStatusEx is the same as Windows structure MEMORYSTATUSEX @@ -141,19 +141,19 @@ func (p *perfCounterNodeStatsClient) getNodeInfo() nodeInfo { func (p *perfCounterNodeStatsClient) collectMetricsData(cpuCounter, memWorkingSetCounter, memCommittedBytesCounter *perfCounter) { cpuValue, err := cpuCounter.getData() if err != nil { - glog.Errorf("Unable to get cpu perf counter data; err: %v", err) + klog.Errorf("Unable to get cpu perf counter data; err: %v", err) return } memWorkingSetValue, err := memWorkingSetCounter.getData() if err != nil { - glog.Errorf("Unable to get memWorkingSet perf counter data; err: %v", err) + klog.Errorf("Unable to get memWorkingSet perf counter data; err: %v", err) return } memCommittedBytesValue, err := memCommittedBytesCounter.getData() if err != nil { - glog.Errorf("Unable to get memCommittedBytes perf counter data; err: %v", err) + klog.Errorf("Unable to get memCommittedBytes perf counter data; err: %v", err) return } diff --git a/pkg/kubemark/BUILD b/pkg/kubemark/BUILD index d00990bb6ba..0eea6adcfe6 100644 --- a/pkg/kubemark/BUILD +++ b/pkg/kubemark/BUILD @@ -49,7 +49,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//test/utils:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", ], diff --git a/pkg/kubemark/controller.go b/pkg/kubemark/controller.go index 42438bac315..3ae38b2f5ef 100644 --- a/pkg/kubemark/controller.go +++ b/pkg/kubemark/controller.go @@ -33,7 +33,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/kubernetes/pkg/controller" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -125,7 +125,7 @@ func (kubemarkController *KubemarkController) WaitForCacheSync(stopCh chan struc func (kubemarkController *KubemarkController) Run(stopCh chan struct{}) { nodeTemplate, err := kubemarkController.getNodeTemplate() if err != nil { - glog.Fatalf("failed to get node template: %s", err) + klog.Fatalf("failed to get node template: %s", err) } kubemarkController.nodeTemplate = nodeTemplate @@ -239,7 +239,7 @@ func (kubemarkController *KubemarkController) addNodeToNodeGroup(nodeGroup strin func (kubemarkController *KubemarkController) RemoveNodeFromNodeGroup(nodeGroup string, node string) error { pod := kubemarkController.getPodByName(node) if pod == nil { - glog.Warningf("Can't delete node %s from nodegroup %s. Node does not exist.", node, nodeGroup) + klog.Warningf("Can't delete node %s from nodegroup %s. Node does not exist.", node, nodeGroup) return nil } if pod.ObjectMeta.Labels[nodeGroupLabel] != nodeGroup { @@ -252,7 +252,7 @@ func (kubemarkController *KubemarkController) RemoveNodeFromNodeGroup(nodeGroup pod.ObjectMeta.Labels["name"], &metav1.DeleteOptions{PropagationPolicy: &policy}) if err == nil { - glog.Infof("marking node %s for deletion", node) + klog.Infof("marking node %s for deletion", node) // Mark node for deletion from kubemark cluster. // Once it becomes unready after replication controller // deletion has been noticed, we will delete it explicitly. @@ -340,7 +340,7 @@ func (kubemarkController *KubemarkController) runNodeCreation(stop <-chan struct kubemarkController.nodeGroupQueueSizeLock.Lock() err := kubemarkController.addNodeToNodeGroup(nodeGroup) if err != nil { - glog.Errorf("failed to add node to node group %s: %v", nodeGroup, err) + klog.Errorf("failed to add node to node group %s: %v", nodeGroup, err) } else { kubemarkController.nodeGroupQueueSize[nodeGroup]-- } @@ -376,7 +376,7 @@ func (kubemarkCluster *kubemarkCluster) removeUnneededNodes(oldObj interface{}, if kubemarkCluster.nodesToDelete[node.Name] { kubemarkCluster.nodesToDelete[node.Name] = false if err := kubemarkCluster.client.CoreV1().Nodes().Delete(node.Name, &metav1.DeleteOptions{}); err != nil { - glog.Errorf("failed to delete node %s from kubemark cluster, err: %v", node.Name, err) + klog.Errorf("failed to delete node %s from kubemark cluster, err: %v", node.Name, err) } } return diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go index 15bec8a9c92..875d84c3003 100644 --- a/pkg/kubemark/hollow_kubelet.go +++ b/pkg/kubemark/hollow_kubelet.go @@ -37,7 +37,7 @@ import ( "k8s.io/kubernetes/pkg/volume/secret" "k8s.io/kubernetes/test/utils" - "github.com/golang/glog" + "k8s.io/klog" ) type HollowKubelet struct { @@ -93,7 +93,7 @@ func (hk *HollowKubelet) Run() { KubeletFlags: *hk.KubeletFlags, KubeletConfiguration: *hk.KubeletConfiguration, }, hk.KubeletDeps, false); err != nil { - glog.Fatalf("Failed to run HollowKubelet: %v. Exiting.", err) + klog.Fatalf("Failed to run HollowKubelet: %v. Exiting.", err) } select {} } @@ -109,7 +109,7 @@ func GetHollowKubeletConfig( testRootDir := utils.MakeTempDirOrDie("hollow-kubelet.", "") podFilePath := utils.MakeTempDirOrDie("static-pods", testRootDir) - glog.Infof("Using %s as root dir for hollow-kubelet", testRootDir) + klog.Infof("Using %s as root dir for hollow-kubelet", testRootDir) // Flags struct f := options.NewKubeletFlags() diff --git a/pkg/kubemark/hollow_proxy.go b/pkg/kubemark/hollow_proxy.go index dcde82c576d..5e4a7ec8897 100644 --- a/pkg/kubemark/hollow_proxy.go +++ b/pkg/kubemark/hollow_proxy.go @@ -35,7 +35,7 @@ import ( utilexec "k8s.io/utils/exec" utilpointer "k8s.io/utils/pointer" - "github.com/golang/glog" + "k8s.io/klog" ) type HollowProxy struct { @@ -133,6 +133,6 @@ func NewHollowProxyOrDie( func (hp *HollowProxy) Run() { if err := hp.ProxyServer.Run(); err != nil { - glog.Fatalf("Error while running proxy: %v\n", err) + klog.Fatalf("Error while running proxy: %v\n", err) } } diff --git a/pkg/master/BUILD b/pkg/master/BUILD index 429f489c8ea..6c0ef9e50cf 100644 --- a/pkg/master/BUILD +++ b/pkg/master/BUILD @@ -35,7 +35,6 @@ go_library( "//pkg/apis/scheduling/install:go_default_library", "//pkg/apis/settings/install:go_default_library", "//pkg/apis/storage/install:go_default_library", - "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library", "//pkg/features:go_default_library", "//pkg/kubeapiserver/options:go_default_library", "//pkg/kubelet/client:go_default_library", @@ -50,7 +49,6 @@ go_library( "//pkg/registry/batch/rest:go_default_library", "//pkg/registry/certificates/rest:go_default_library", "//pkg/registry/coordination/rest:go_default_library", - "//pkg/registry/core/endpoint/storage:go_default_library", "//pkg/registry/core/rangeallocation:go_default_library", "//pkg/registry/core/rest:go_default_library", "//pkg/registry/core/service/ipallocator:go_default_library", @@ -106,7 +104,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", @@ -116,8 +113,8 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -145,7 +142,6 @@ go_test( "//pkg/apis/extensions:go_default_library", "//pkg/apis/rbac:go_default_library", "//pkg/apis/storage:go_default_library", - "//pkg/client/clientset_generated/internalclientset/fake:go_default_library", "//pkg/generated/openapi:go_default_library", "//pkg/kubelet/client:go_default_library", "//pkg/master/reconcilers:go_default_library", diff --git a/pkg/master/client_ca_hook.go b/pkg/master/client_ca_hook.go index 4e5d4ff5198..e8a52dd9951 100644 --- a/pkg/master/client_ca_hook.go +++ b/pkg/master/client_ca_hook.go @@ -22,13 +22,13 @@ import ( "reflect" "time" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" genericapiserver "k8s.io/apiserver/pkg/server" - api "k8s.io/kubernetes/pkg/apis/core" - coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" ) type ClientCARegistrationHook struct { @@ -49,7 +49,7 @@ func (h ClientCARegistrationHook) PostStartHook(hookContext genericapiserver.Pos // retry building the config since sometimes the server can be in an in-between state which caused // some kind of auto detection failure as I recall from other post start hooks. // TODO see if this is still true and fix the RBAC one too if it isn't. - client, err := coreclient.NewForConfig(hookContext.LoopbackClientConfig) + client, err := corev1client.NewForConfig(hookContext.LoopbackClientConfig) if err != nil { utilruntime.HandleError(err) return false, nil @@ -68,7 +68,7 @@ func (h ClientCARegistrationHook) PostStartHook(hookContext genericapiserver.Pos // tryToWriteClientCAs is here for unit testing with a fake client. This is a wait.ConditionFunc so the bool // indicates if the condition was met. True when its finished, false when it should retry. -func (h ClientCARegistrationHook) tryToWriteClientCAs(client coreclient.CoreInterface) (bool, error) { +func (h ClientCARegistrationHook) tryToWriteClientCAs(client corev1client.CoreV1Interface) (bool, error) { if err := createNamespaceIfNeeded(client, metav1.NamespaceSystem); err != nil { utilruntime.HandleError(err) return false, nil @@ -119,10 +119,10 @@ func jsonSerializeStringSlice(in []string) (string, error) { return string(out), err } -func writeConfigMap(client coreclient.ConfigMapsGetter, name string, data map[string]string) error { +func writeConfigMap(client corev1client.ConfigMapsGetter, name string, data map[string]string) error { existing, err := client.ConfigMaps(metav1.NamespaceSystem).Get(name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { - _, err := client.ConfigMaps(metav1.NamespaceSystem).Create(&api.ConfigMap{ + _, err := client.ConfigMaps(metav1.NamespaceSystem).Create(&corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: name}, Data: data, }) diff --git a/pkg/master/client_ca_hook_test.go b/pkg/master/client_ca_hook_test.go index fecfc685606..1d88dbe0c2c 100644 --- a/pkg/master/client_ca_hook_test.go +++ b/pkg/master/client_ca_hook_test.go @@ -20,12 +20,12 @@ import ( "reflect" "testing" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/client-go/kubernetes/fake" clienttesting "k8s.io/client-go/testing" - api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake" ) func TestWriteClientCAs(t *testing.T) { @@ -33,7 +33,7 @@ func TestWriteClientCAs(t *testing.T) { name string hook ClientCARegistrationHook preexistingObjs []runtime.Object - expectedConfigMaps map[string]*api.ConfigMap + expectedConfigMaps map[string]*corev1.ConfigMap expectUpdate bool }{ { @@ -46,7 +46,7 @@ func TestWriteClientCAs(t *testing.T) { RequestHeaderCA: []byte("bar"), RequestHeaderAllowedNames: []string{"first", "second"}, }, - expectedConfigMaps: map[string]*api.ConfigMap{ + expectedConfigMaps: map[string]*corev1.ConfigMap{ "extension-apiserver-authentication": { ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, Data: map[string]string{ @@ -66,7 +66,7 @@ func TestWriteClientCAs(t *testing.T) { RequestHeaderCA: []byte("bar"), RequestHeaderAllowedNames: []string{"first", "second"}, }, - expectedConfigMaps: map[string]*api.ConfigMap{ + expectedConfigMaps: map[string]*corev1.ConfigMap{ "extension-apiserver-authentication": { ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, Data: map[string]string{ @@ -84,7 +84,7 @@ func TestWriteClientCAs(t *testing.T) { hook: ClientCARegistrationHook{ ClientCA: []byte("foo"), }, - expectedConfigMaps: map[string]*api.ConfigMap{ + expectedConfigMaps: map[string]*corev1.ConfigMap{ "extension-apiserver-authentication": { ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, Data: map[string]string{ @@ -98,7 +98,7 @@ func TestWriteClientCAs(t *testing.T) { hook: ClientCARegistrationHook{ RequestHeaderCA: []byte("bar"), }, - expectedConfigMaps: map[string]*api.ConfigMap{ + expectedConfigMaps: map[string]*corev1.ConfigMap{ "extension-apiserver-authentication": { ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, Data: map[string]string{ @@ -117,14 +117,14 @@ func TestWriteClientCAs(t *testing.T) { ClientCA: []byte("foo"), }, preexistingObjs: []runtime.Object{ - &api.ConfigMap{ + &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, Data: map[string]string{ "client-ca-file": "other", }, }, }, - expectedConfigMaps: map[string]*api.ConfigMap{ + expectedConfigMaps: map[string]*corev1.ConfigMap{ "extension-apiserver-authentication": { ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, Data: map[string]string{ @@ -144,7 +144,7 @@ func TestWriteClientCAs(t *testing.T) { RequestHeaderAllowedNames: []string{}, }, preexistingObjs: []runtime.Object{ - &api.ConfigMap{ + &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, Data: map[string]string{ "requestheader-username-headers": `null`, @@ -155,7 +155,7 @@ func TestWriteClientCAs(t *testing.T) { }, }, }, - expectedConfigMaps: map[string]*api.ConfigMap{ + expectedConfigMaps: map[string]*corev1.ConfigMap{ "extension-apiserver-authentication": { ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, Data: map[string]string{ @@ -175,9 +175,9 @@ func TestWriteClientCAs(t *testing.T) { ClientCA: []byte("foo"), }, preexistingObjs: []runtime.Object{ - &api.Namespace{ObjectMeta: metav1.ObjectMeta{Name: metav1.NamespaceSystem}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: metav1.NamespaceSystem}}, }, - expectedConfigMaps: map[string]*api.ConfigMap{ + expectedConfigMaps: map[string]*corev1.ConfigMap{ "extension-apiserver-authentication": { ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, Data: map[string]string{ @@ -196,7 +196,7 @@ func TestWriteClientCAs(t *testing.T) { RequestHeaderAllowedNames: []string{}, }, preexistingObjs: []runtime.Object{ - &api.ConfigMap{ + &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, Data: map[string]string{ "requestheader-username-headers": `[]`, @@ -207,7 +207,7 @@ func TestWriteClientCAs(t *testing.T) { }, }, }, - expectedConfigMaps: map[string]*api.ConfigMap{}, + expectedConfigMaps: map[string]*corev1.ConfigMap{}, expectUpdate: false, }, } @@ -217,7 +217,7 @@ func TestWriteClientCAs(t *testing.T) { client := fake.NewSimpleClientset(test.preexistingObjs...) test.hook.tryToWriteClientCAs(client.Core()) - actualConfigMaps, updated := getFinalConfiMaps(client) + actualConfigMaps, updated := getFinalConfigMaps(client) if !reflect.DeepEqual(test.expectedConfigMaps, actualConfigMaps) { t.Fatalf("%s: %v", test.name, diff.ObjectReflectDiff(test.expectedConfigMaps, actualConfigMaps)) } @@ -228,18 +228,18 @@ func TestWriteClientCAs(t *testing.T) { } } -func getFinalConfiMaps(client *fake.Clientset) (map[string]*api.ConfigMap, bool) { - ret := map[string]*api.ConfigMap{} +func getFinalConfigMaps(client *fake.Clientset) (map[string]*corev1.ConfigMap, bool) { + ret := map[string]*corev1.ConfigMap{} updated := false for _, action := range client.Actions() { if action.Matches("create", "configmaps") { - obj := action.(clienttesting.CreateAction).GetObject().(*api.ConfigMap) + obj := action.(clienttesting.CreateAction).GetObject().(*corev1.ConfigMap) ret[obj.Name] = obj } if action.Matches("update", "configmaps") { updated = true - obj := action.(clienttesting.UpdateAction).GetObject().(*api.ConfigMap) + obj := action.(clienttesting.UpdateAction).GetObject().(*corev1.ConfigMap) ret[obj.Name] = obj } } diff --git a/pkg/master/client_util.go b/pkg/master/client_util.go index 3868fbae5c5..acfcb0e8994 100644 --- a/pkg/master/client_util.go +++ b/pkg/master/client_util.go @@ -17,18 +17,18 @@ limitations under the License. package master import ( + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "k8s.io/kubernetes/pkg/apis/core" - coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" ) -func createNamespaceIfNeeded(c coreclient.NamespacesGetter, ns string) error { +func createNamespaceIfNeeded(c corev1client.NamespacesGetter, ns string) error { if _, err := c.Namespaces().Get(ns, metav1.GetOptions{}); err == nil { // the namespace already exists return nil } - newNs := &api.Namespace{ + newNs := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: ns, Namespace: "", diff --git a/pkg/master/controller.go b/pkg/master/controller.go index f4cb74d8da4..29c9d2da752 100644 --- a/pkg/master/controller.go +++ b/pkg/master/controller.go @@ -21,7 +21,6 @@ import ( "net" "time" - "github.com/golang/glog" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,8 +30,8 @@ import ( "k8s.io/apimachinery/pkg/util/wait" genericapiserver "k8s.io/apiserver/pkg/server" utilfeature "k8s.io/apiserver/pkg/util/feature" - api "k8s.io/kubernetes/pkg/apis/core" - coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/master/reconcilers" "k8s.io/kubernetes/pkg/registry/core/rangeallocation" @@ -49,9 +48,9 @@ const kubernetesServiceName = "kubernetes" // "default", "kube-system" and "kube-public" namespaces, and provide the IP // repair check on service IPs type Controller struct { - ServiceClient coreclient.ServicesGetter - NamespaceClient coreclient.NamespacesGetter - EventClient coreclient.EventsGetter + ServiceClient corev1client.ServicesGetter + NamespaceClient corev1client.NamespacesGetter + EventClient corev1client.EventsGetter ServiceClusterIPRegistry rangeallocation.RangeRegistry ServiceClusterIPInterval time.Duration @@ -72,8 +71,8 @@ type Controller struct { // ServiceIP indicates where the kubernetes service will live. It may not be nil. ServiceIP net.IP ServicePort int - ExtraServicePorts []api.ServicePort - ExtraEndpointPorts []api.EndpointPort + ExtraServicePorts []corev1.ServicePort + ExtraEndpointPorts []corev1.EndpointPort PublicServicePort int KubernetesServiceNodePort int @@ -81,10 +80,10 @@ type Controller struct { } // NewBootstrapController returns a controller for watching the core capabilities of the master -func (c *completedConfig) NewBootstrapController(legacyRESTStorage corerest.LegacyRESTStorage, serviceClient coreclient.ServicesGetter, nsClient coreclient.NamespacesGetter, eventClient coreclient.EventsGetter) *Controller { +func (c *completedConfig) NewBootstrapController(legacyRESTStorage corerest.LegacyRESTStorage, serviceClient corev1client.ServicesGetter, nsClient corev1client.NamespacesGetter, eventClient corev1client.EventsGetter) *Controller { _, publicServicePort, err := c.GenericConfig.SecureServing.HostPort() if err != nil { - glog.Fatalf("failed to get listener address: %v", err) + klog.Fatalf("failed to get listener address: %v", err) } systemNamespaces := []string{metav1.NamespaceSystem, metav1.NamespacePublic} @@ -145,15 +144,15 @@ func (c *Controller) Start() { // run all of the controllers once prior to returning from Start. if err := repairClusterIPs.RunOnce(); err != nil { // If we fail to repair cluster IPs apiserver is useless. We should restart and retry. - glog.Fatalf("Unable to perform initial IP allocation check: %v", err) + klog.Fatalf("Unable to perform initial IP allocation check: %v", err) } if err := repairNodePorts.RunOnce(); err != nil { // If we fail to repair node ports apiserver is useless. We should restart and retry. - glog.Fatalf("Unable to perform initial service nodePort check: %v", err) + klog.Fatalf("Unable to perform initial service nodePort check: %v", err) } // Service definition is reconciled during first run to correct port and type per expectations. if err := c.UpdateKubernetesService(true); err != nil { - glog.Errorf("Unable to perform initial Kubernetes service initialization: %v", err) + klog.Errorf("Unable to perform initial Kubernetes service initialization: %v", err) } c.runner = async.NewRunner(c.RunKubernetesNamespaces, c.RunKubernetesService, repairClusterIPs.RunUntil, repairNodePorts.RunUntil) @@ -168,9 +167,9 @@ func (c *Controller) Stop() { finishedReconciling := make(chan struct{}) go func() { defer close(finishedReconciling) - glog.Infof("Shutting down kubernetes service endpoint reconciler") + klog.Infof("Shutting down kubernetes service endpoint reconciler") if err := c.EndpointReconciler.StopReconciling("kubernetes", c.PublicIP, endpointPorts); err != nil { - glog.Error(err) + klog.Error(err) } }() @@ -179,7 +178,7 @@ func (c *Controller) Stop() { // done case <-time.After(2 * c.EndpointInterval): // don't block server shutdown forever if we can't reach etcd to remove ourselves - glog.Warning("StopReconciling() timed out") + klog.Warning("StopReconciling() timed out") } } @@ -230,17 +229,17 @@ func (c *Controller) UpdateKubernetesService(reconcile bool) error { // createPortAndServiceSpec creates an array of service ports. // If the NodePort value is 0, just the servicePort is used, otherwise, a node port is exposed. -func createPortAndServiceSpec(servicePort int, targetServicePort int, nodePort int, servicePortName string, extraServicePorts []api.ServicePort) ([]api.ServicePort, api.ServiceType) { +func createPortAndServiceSpec(servicePort int, targetServicePort int, nodePort int, servicePortName string, extraServicePorts []corev1.ServicePort) ([]corev1.ServicePort, corev1.ServiceType) { //Use the Cluster IP type for the service port if NodePort isn't provided. //Otherwise, we will be binding the master service to a NodePort. - servicePorts := []api.ServicePort{{Protocol: api.ProtocolTCP, + servicePorts := []corev1.ServicePort{{Protocol: corev1.ProtocolTCP, Port: int32(servicePort), Name: servicePortName, TargetPort: intstr.FromInt(targetServicePort)}} - serviceType := api.ServiceTypeClusterIP + serviceType := corev1.ServiceTypeClusterIP if nodePort > 0 { servicePorts[0].NodePort = int32(nodePort) - serviceType = api.ServiceTypeNodePort + serviceType = corev1.ServiceTypeNodePort } if extraServicePorts != nil { servicePorts = append(servicePorts, extraServicePorts...) @@ -249,8 +248,8 @@ func createPortAndServiceSpec(servicePort int, targetServicePort int, nodePort i } // createEndpointPortSpec creates an array of endpoint ports -func createEndpointPortSpec(endpointPort int, endpointPortName string, extraEndpointPorts []api.EndpointPort) []api.EndpointPort { - endpointPorts := []api.EndpointPort{{Protocol: api.ProtocolTCP, +func createEndpointPortSpec(endpointPort int, endpointPortName string, extraEndpointPorts []corev1.EndpointPort) []corev1.EndpointPort { + endpointPorts := []corev1.EndpointPort{{Protocol: corev1.ProtocolTCP, Port: int32(endpointPort), Name: endpointPortName, }} @@ -262,30 +261,30 @@ func createEndpointPortSpec(endpointPort int, endpointPortName string, extraEndp // CreateMasterServiceIfNeeded will create the specified service if it // doesn't already exist. -func (c *Controller) CreateOrUpdateMasterServiceIfNeeded(serviceName string, serviceIP net.IP, servicePorts []api.ServicePort, serviceType api.ServiceType, reconcile bool) error { +func (c *Controller) CreateOrUpdateMasterServiceIfNeeded(serviceName string, serviceIP net.IP, servicePorts []corev1.ServicePort, serviceType corev1.ServiceType, reconcile bool) error { if s, err := c.ServiceClient.Services(metav1.NamespaceDefault).Get(serviceName, metav1.GetOptions{}); err == nil { // The service already exists. if reconcile { if svc, updated := reconcilers.GetMasterServiceUpdateIfNeeded(s, servicePorts, serviceType); updated { - glog.Warningf("Resetting master service %q to %#v", serviceName, svc) + klog.Warningf("Resetting master service %q to %#v", serviceName, svc) _, err := c.ServiceClient.Services(metav1.NamespaceDefault).Update(svc) return err } } return nil } - svc := &api.Service{ + svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName, Namespace: metav1.NamespaceDefault, Labels: map[string]string{"provider": "kubernetes", "component": "apiserver"}, }, - Spec: api.ServiceSpec{ + Spec: corev1.ServiceSpec{ Ports: servicePorts, // maintained by this code, not by the pod selector Selector: nil, ClusterIP: serviceIP.String(), - SessionAffinity: api.ServiceAffinityNone, + SessionAffinity: corev1.ServiceAffinityNone, Type: serviceType, }, } diff --git a/pkg/master/controller/crdregistration/BUILD b/pkg/master/controller/crdregistration/BUILD index 0e605162d0d..fa592c60605 100644 --- a/pkg/master/controller/crdregistration/BUILD +++ b/pkg/master/controller/crdregistration/BUILD @@ -23,7 +23,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/master/controller/crdregistration/crdregistration_controller.go b/pkg/master/controller/crdregistration/crdregistration_controller.go index 49e3b1edc20..1670a5b0780 100644 --- a/pkg/master/controller/crdregistration/crdregistration_controller.go +++ b/pkg/master/controller/crdregistration/crdregistration_controller.go @@ -20,7 +20,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" crdinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion/apiextensions/internalversion" @@ -88,12 +88,12 @@ func NewAutoRegistrationController(crdinformer crdinformers.CustomResourceDefini if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.V(2).Infof("Couldn't get object from tombstone %#v", obj) + klog.V(2).Infof("Couldn't get object from tombstone %#v", obj) return } cast, ok = tombstone.Obj.(*apiextensions.CustomResourceDefinition) if !ok { - glog.V(2).Infof("Tombstone contained unexpected object: %#v", obj) + klog.V(2).Infof("Tombstone contained unexpected object: %#v", obj) return } } @@ -109,8 +109,8 @@ func (c *crdRegistrationController) Run(threadiness int, stopCh <-chan struct{}) // make sure the work queue is shutdown which will trigger workers to end defer c.queue.ShutDown() - glog.Infof("Starting crd-autoregister controller") - defer glog.Infof("Shutting down crd-autoregister controller") + klog.Infof("Starting crd-autoregister controller") + defer klog.Infof("Shutting down crd-autoregister controller") // wait for your secondary caches to fill before starting your work if !controller.WaitForCacheSync("crd-autoregister", stopCh, c.crdSynced) { diff --git a/pkg/master/controller_test.go b/pkg/master/controller_test.go index 8e31a71ccfc..cfab7074ca2 100644 --- a/pkg/master/controller_test.go +++ b/pkg/master/controller_test.go @@ -21,11 +21,11 @@ import ( "reflect" "testing" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" - api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake" "k8s.io/kubernetes/pkg/master/reconcilers" ) @@ -38,23 +38,23 @@ func TestReconcileEndpoints(t *testing.T) { testName string serviceName string ip string - endpointPorts []api.EndpointPort + endpointPorts []corev1.EndpointPort additionalMasters int - endpoints *api.EndpointsList - expectUpdate *api.Endpoints // nil means none expected - expectCreate *api.Endpoints // nil means none expected + endpoints *corev1.EndpointsList + expectUpdate *corev1.Endpoints // nil means none expected + expectCreate *corev1.Endpoints // nil means none expected }{ { testName: "no existing endpoints", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, endpoints: nil, - expectCreate: &api.Endpoints{ + expectCreate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -62,13 +62,13 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints satisfy", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, @@ -77,21 +77,21 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints satisfy but too many", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}, {IP: "4.3.2.1"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}, {IP: "4.3.2.1"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -99,33 +99,33 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints satisfy but too many + extra masters", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, additionalMasters: 3, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "1.2.3.4"}, {IP: "4.3.2.1"}, {IP: "4.3.2.2"}, {IP: "4.3.2.3"}, {IP: "4.3.2.4"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "1.2.3.4"}, {IP: "4.3.2.2"}, {IP: "4.3.2.3"}, {IP: "4.3.2.4"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -133,33 +133,33 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints satisfy but too many + extra masters + delete first", serviceName: "foo", ip: "4.3.2.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, additionalMasters: 3, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "1.2.3.4"}, {IP: "4.3.2.1"}, {IP: "4.3.2.2"}, {IP: "4.3.2.3"}, {IP: "4.3.2.4"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "4.3.2.1"}, {IP: "4.3.2.2"}, {IP: "4.3.2.3"}, {IP: "4.3.2.4"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -167,17 +167,17 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints satisfy and endpoint addresses length less than master count", serviceName: "foo", ip: "4.3.2.2", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, additionalMasters: 3, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "4.3.2.1"}, {IP: "4.3.2.2"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, @@ -187,27 +187,27 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints current IP missing and address length less than master count", serviceName: "foo", ip: "4.3.2.2", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, additionalMasters: 3, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "4.3.2.1"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "4.3.2.1"}, {IP: "4.3.2.2"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -215,21 +215,21 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints wrong name", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("bar"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectCreate: &api.Endpoints{ + expectCreate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -237,21 +237,21 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints wrong IP", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "4.3.2.1"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "4.3.2.1"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -259,21 +259,21 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints wrong port", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 9090, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 9090, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -281,21 +281,21 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints wrong protocol", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "UDP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "UDP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -303,21 +303,21 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints wrong port name", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "baz", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "baz", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "baz", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "baz", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -325,17 +325,17 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints extra service ports satisfy", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{ + endpointPorts: []corev1.EndpointPort{ {Name: "foo", Port: 8080, Protocol: "TCP"}, {Name: "bar", Port: 1000, Protocol: "TCP"}, {Name: "baz", Port: 1010, Protocol: "TCP"}, }, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{ {Name: "foo", Port: 8080, Protocol: "TCP"}, {Name: "bar", Port: 1000, Protocol: "TCP"}, {Name: "baz", Port: 1010, Protocol: "TCP"}, @@ -348,24 +348,24 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints extra service ports missing port", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{ + endpointPorts: []corev1.EndpointPort{ {Name: "foo", Port: 8080, Protocol: "TCP"}, {Name: "bar", Port: 1000, Protocol: "TCP"}, }, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{ {Name: "foo", Port: 8080, Protocol: "TCP"}, {Name: "bar", Port: 1000, Protocol: "TCP"}, }, @@ -376,13 +376,13 @@ func TestReconcileEndpoints(t *testing.T) { testName: "no existing sctp endpoints", serviceName: "boo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "boo", Port: 7777, Protocol: "SCTP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "boo", Port: 7777, Protocol: "SCTP"}}, endpoints: nil, - expectCreate: &api.Endpoints{ + expectCreate: &corev1.Endpoints{ ObjectMeta: om("boo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "boo", Port: 7777, Protocol: "SCTP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "boo", Port: 7777, Protocol: "SCTP"}}, }}, }, }, @@ -440,26 +440,26 @@ func TestReconcileEndpoints(t *testing.T) { testName string serviceName string ip string - endpointPorts []api.EndpointPort + endpointPorts []corev1.EndpointPort additionalMasters int - endpoints *api.EndpointsList - expectUpdate *api.Endpoints // nil means none expected - expectCreate *api.Endpoints // nil means none expected + endpoints *corev1.EndpointsList + expectUpdate *corev1.Endpoints // nil means none expected + expectCreate *corev1.Endpoints // nil means none expected }{ { testName: "existing endpoints extra service ports missing port no update", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{ + endpointPorts: []corev1.EndpointPort{ {Name: "foo", Port: 8080, Protocol: "TCP"}, {Name: "bar", Port: 1000, Protocol: "TCP"}, }, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, @@ -469,24 +469,24 @@ func TestReconcileEndpoints(t *testing.T) { testName: "existing endpoints extra service ports, wrong ports, wrong IP", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{ + endpointPorts: []corev1.EndpointPort{ {Name: "foo", Port: 8080, Protocol: "TCP"}, {Name: "bar", Port: 1000, Protocol: "TCP"}, }, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "4.3.2.1"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "4.3.2.1"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -494,13 +494,13 @@ func TestReconcileEndpoints(t *testing.T) { testName: "no existing endpoints", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, endpoints: nil, - expectCreate: &api.Endpoints{ + expectCreate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -565,27 +565,27 @@ func TestCreateOrUpdateMasterService(t *testing.T) { create_tests := []struct { testName string serviceName string - servicePorts []api.ServicePort - serviceType api.ServiceType - expectCreate *api.Service // nil means none expected + servicePorts []corev1.ServicePort + serviceType corev1.ServiceType + expectCreate *corev1.Service // nil means none expected }{ { testName: "service does not exist", serviceName: "foo", - servicePorts: []api.ServicePort{ + servicePorts: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, - serviceType: api.ServiceTypeClusterIP, - expectCreate: &api.Service{ + serviceType: corev1.ServiceTypeClusterIP, + expectCreate: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, }, @@ -606,7 +606,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { t.Errorf("case %q: unexpected creations: %v", test.testName, creates) } else { obj := creates[0].GetObject() - if e, a := test.expectCreate.Spec, obj.(*api.Service).Spec; !reflect.DeepEqual(e, a) { + if e, a := test.expectCreate.Spec, obj.(*corev1.Service).Spec; !reflect.DeepEqual(e, a) { t.Errorf("case %q: expected create:\n%#v\ngot:\n%#v\n", test.testName, e, a) } } @@ -619,254 +619,254 @@ func TestCreateOrUpdateMasterService(t *testing.T) { reconcile_tests := []struct { testName string serviceName string - servicePorts []api.ServicePort - serviceType api.ServiceType - service *api.Service - expectUpdate *api.Service // nil means none expected + servicePorts []corev1.ServicePort + serviceType corev1.ServiceType + service *corev1.Service + expectUpdate *corev1.Service // nil means none expected }{ { testName: "service definition wrong port", serviceName: "foo", - servicePorts: []api.ServicePort{ + servicePorts: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, - serviceType: api.ServiceTypeClusterIP, - service: &api.Service{ + serviceType: corev1.ServiceTypeClusterIP, + service: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8000, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, - expectUpdate: &api.Service{ + expectUpdate: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, }, { testName: "service definition missing port", serviceName: "foo", - servicePorts: []api.ServicePort{ + servicePorts: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, {Name: "baz", Port: 1000, Protocol: "TCP", TargetPort: intstr.FromInt(1000)}, }, - serviceType: api.ServiceTypeClusterIP, - service: &api.Service{ + serviceType: corev1.ServiceTypeClusterIP, + service: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, - expectUpdate: &api.Service{ + expectUpdate: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, {Name: "baz", Port: 1000, Protocol: "TCP", TargetPort: intstr.FromInt(1000)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, }, { testName: "service definition incorrect port", serviceName: "foo", - servicePorts: []api.ServicePort{ + servicePorts: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, - serviceType: api.ServiceTypeClusterIP, - service: &api.Service{ + serviceType: corev1.ServiceTypeClusterIP, + service: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "bar", Port: 1000, Protocol: "UDP", TargetPort: intstr.FromInt(1000)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, - expectUpdate: &api.Service{ + expectUpdate: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, }, { testName: "service definition incorrect port name", serviceName: "foo", - servicePorts: []api.ServicePort{ + servicePorts: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, - serviceType: api.ServiceTypeClusterIP, - service: &api.Service{ + serviceType: corev1.ServiceTypeClusterIP, + service: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 1000, Protocol: "UDP", TargetPort: intstr.FromInt(1000)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, - expectUpdate: &api.Service{ + expectUpdate: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, }, { testName: "service definition incorrect target port", serviceName: "foo", - servicePorts: []api.ServicePort{ + servicePorts: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, - serviceType: api.ServiceTypeClusterIP, - service: &api.Service{ + serviceType: corev1.ServiceTypeClusterIP, + service: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(1000)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, - expectUpdate: &api.Service{ + expectUpdate: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, }, { testName: "service definition incorrect protocol", serviceName: "foo", - servicePorts: []api.ServicePort{ + servicePorts: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, - serviceType: api.ServiceTypeClusterIP, - service: &api.Service{ + serviceType: corev1.ServiceTypeClusterIP, + service: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "UDP", TargetPort: intstr.FromInt(8080)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, - expectUpdate: &api.Service{ + expectUpdate: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, }, { testName: "service definition has incorrect type", serviceName: "foo", - servicePorts: []api.ServicePort{ + servicePorts: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, - serviceType: api.ServiceTypeClusterIP, - service: &api.Service{ + serviceType: corev1.ServiceTypeClusterIP, + service: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeNodePort, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeNodePort, }, }, - expectUpdate: &api.Service{ + expectUpdate: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, }, { testName: "service definition satisfies", serviceName: "foo", - servicePorts: []api.ServicePort{ + servicePorts: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, - serviceType: api.ServiceTypeClusterIP, - service: &api.Service{ + serviceType: corev1.ServiceTypeClusterIP, + service: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, expectUpdate: nil, @@ -891,7 +891,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { t.Errorf("case %q: unexpected updates: %v", test.testName, updates) } else { obj := updates[0].GetObject() - if e, a := test.expectUpdate.Spec, obj.(*api.Service).Spec; !reflect.DeepEqual(e, a) { + if e, a := test.expectUpdate.Spec, obj.(*corev1.Service).Spec; !reflect.DeepEqual(e, a) { t.Errorf("case %q: expected update:\n%#v\ngot:\n%#v\n", test.testName, e, a) } } @@ -904,28 +904,28 @@ func TestCreateOrUpdateMasterService(t *testing.T) { non_reconcile_tests := []struct { testName string serviceName string - servicePorts []api.ServicePort - serviceType api.ServiceType - service *api.Service - expectUpdate *api.Service // nil means none expected + servicePorts []corev1.ServicePort + serviceType corev1.ServiceType + service *corev1.Service + expectUpdate *corev1.Service // nil means none expected }{ { testName: "service definition wrong port, no expected update", serviceName: "foo", - servicePorts: []api.ServicePort{ + servicePorts: []corev1.ServicePort{ {Name: "foo", Port: 8080, Protocol: "TCP", TargetPort: intstr.FromInt(8080)}, }, - serviceType: api.ServiceTypeClusterIP, - service: &api.Service{ + serviceType: corev1.ServiceTypeClusterIP, + service: &corev1.Service{ ObjectMeta: om("foo"), - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ {Name: "foo", Port: 1000, Protocol: "TCP", TargetPort: intstr.FromInt(1000)}, }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeClusterIP, }, }, expectUpdate: nil, @@ -950,7 +950,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { t.Errorf("case %q: unexpected updates: %v", test.testName, updates) } else { obj := updates[0].GetObject() - if e, a := test.expectUpdate.Spec, obj.(*api.Service).Spec; !reflect.DeepEqual(e, a) { + if e, a := test.expectUpdate.Spec, obj.(*corev1.Service).Spec; !reflect.DeepEqual(e, a) { t.Errorf("case %q: expected update:\n%#v\ngot:\n%#v\n", test.testName, e, a) } } diff --git a/pkg/master/master.go b/pkg/master/master.go index a7107ffc5ff..b2d25a12f70 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -58,7 +58,6 @@ import ( storageapiv1beta1 "k8s.io/api/storage/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilnet "k8s.io/apimachinery/pkg/util/net" - "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/endpoints/discovery" "k8s.io/apiserver/pkg/registry/generic" genericapiserver "k8s.io/apiserver/pkg/server" @@ -68,18 +67,16 @@ import ( "k8s.io/client-go/informers" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" api "k8s.io/kubernetes/pkg/apis/core" - coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" "k8s.io/kubernetes/pkg/master/reconcilers" "k8s.io/kubernetes/pkg/master/tunneler" - endpointsstorage "k8s.io/kubernetes/pkg/registry/core/endpoint/storage" "k8s.io/kubernetes/pkg/routes" "k8s.io/kubernetes/pkg/serviceaccount" nodeutil "k8s.io/kubernetes/pkg/util/node" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" + "k8s.io/klog" // RESTStorage installers admissionregistrationrest "k8s.io/kubernetes/pkg/registry/admissionregistration/rest" @@ -144,10 +141,10 @@ type ExtraConfig struct { // service because this pkg is linked by out-of-tree projects // like openshift which want to use the GenericAPIServer but also do // more stuff. - ExtraServicePorts []api.ServicePort + ExtraServicePorts []apiv1.ServicePort // Additional ports to be exposed on the GenericAPIServer endpoints // Port names should align with ports defined in ExtraServicePorts - ExtraEndpointPorts []api.EndpointPort + ExtraEndpointPorts []apiv1.EndpointPort // If non-zero, the "kubernetes" services uses this port as NodePort. KubernetesServiceNodePort int @@ -171,8 +168,6 @@ type ExtraConfig struct { ServiceAccountIssuer serviceaccount.TokenGenerator ServiceAccountMaxExpiration time.Duration - APIAudiences authenticator.Audiences - VersionedInformers informers.SharedInformerFactory } @@ -206,7 +201,7 @@ type Master struct { } func (c *Config) createMasterCountReconciler() reconcilers.EndpointReconciler { - endpointClient := coreclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) + endpointClient := corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) return reconcilers.NewMasterCountEndpointReconciler(c.ExtraConfig.MasterCount, endpointClient) } @@ -215,31 +210,22 @@ func (c *Config) createNoneReconciler() reconcilers.EndpointReconciler { } func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler { + endpointClient := corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) ttl := c.ExtraConfig.MasterEndpointReconcileTTL config, err := c.ExtraConfig.StorageFactory.NewConfig(api.Resource("apiServerIPInfo")) if err != nil { - glog.Fatalf("Error determining service IP ranges: %v", err) + klog.Fatalf("Error determining service IP ranges: %v", err) } leaseStorage, _, err := storagefactory.Create(*config) if err != nil { - glog.Fatalf("Error creating storage factory: %v", err) + klog.Fatalf("Error creating storage factory: %v", err) } - endpointConfig, err := c.ExtraConfig.StorageFactory.NewConfig(api.Resource("endpoints")) - if err != nil { - glog.Fatalf("Error getting storage config: %v", err) - } - endpointsStorage := endpointsstorage.NewREST(generic.RESTOptions{ - StorageConfig: endpointConfig, - Decorator: generic.UndecoratedStorage, - DeleteCollectionWorkers: 0, - ResourcePrefix: c.ExtraConfig.StorageFactory.ResourcePrefix(api.Resource("endpoints")), - }) masterLeases := reconcilers.NewLeases(leaseStorage, "/masterleases/", ttl) - return reconcilers.NewLeaseEndpointReconciler(endpointsStorage.Store, masterLeases) + return reconcilers.NewLeaseEndpointReconciler(endpointClient, masterLeases) } func (c *Config) createEndpointReconciler() reconcilers.EndpointReconciler { - glog.Infof("Using reconciler: %v", c.ExtraConfig.EndpointReconcilerType) + klog.Infof("Using reconciler: %v", c.ExtraConfig.EndpointReconcilerType) switch c.ExtraConfig.EndpointReconcilerType { // there are numerous test dependencies that depend on a default controller case "", reconcilers.MasterCountReconcilerType: @@ -249,7 +235,7 @@ func (c *Config) createEndpointReconciler() reconcilers.EndpointReconciler { case reconcilers.NoneEndpointReconcilerType: return c.createNoneReconciler() default: - glog.Fatalf("Reconciler not implemented: %v", c.ExtraConfig.EndpointReconcilerType) + klog.Fatalf("Reconciler not implemented: %v", c.ExtraConfig.EndpointReconcilerType) } return nil } @@ -263,7 +249,7 @@ func (cfg *Config) Complete() CompletedConfig { serviceIPRange, apiServerServiceIP, err := DefaultServiceIPRange(c.ExtraConfig.ServiceIPRange) if err != nil { - glog.Fatalf("Error determining service IP ranges: %v", err) + klog.Fatalf("Error determining service IP ranges: %v", err) } if c.ExtraConfig.ServiceIPRange.IP == nil { c.ExtraConfig.ServiceIPRange = serviceIPRange @@ -283,7 +269,7 @@ func (cfg *Config) Complete() CompletedConfig { // but then that breaks the strict nestedness of ServiceType. // Review post-v1 c.ExtraConfig.ServiceNodePortRange = kubeoptions.DefaultServiceNodePortRange - glog.Infof("Node port range unspecified. Defaulting to %v.", c.ExtraConfig.ServiceNodePortRange) + klog.Infof("Node port range unspecified. Defaulting to %v.", c.ExtraConfig.ServiceNodePortRange) } if c.ExtraConfig.EndpointReconcilerConfig.Interval == 0 { @@ -335,7 +321,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) LoopbackClientConfig: c.GenericConfig.LoopbackClientConfig, ServiceAccountIssuer: c.ExtraConfig.ServiceAccountIssuer, ServiceAccountMaxExpiration: c.ExtraConfig.ServiceAccountMaxExpiration, - APIAudiences: c.ExtraConfig.APIAudiences, + APIAudiences: c.GenericConfig.Authentication.APIAudiences, } m.InstallLegacyAPI(&c, c.GenericConfig.RESTOptionsGetter, legacyRESTStorageProvider) } @@ -349,7 +335,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) // handlers that we have. restStorageProviders := []RESTStorageProvider{ auditregistrationrest.RESTStorageProvider{}, - authenticationrest.RESTStorageProvider{Authenticator: c.GenericConfig.Authentication.Authenticator}, + authenticationrest.RESTStorageProvider{Authenticator: c.GenericConfig.Authentication.Authenticator, APIAudiences: c.GenericConfig.Authentication.APIAudiences}, authorizationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer, RuleResolver: c.GenericConfig.RuleResolver}, autoscalingrest.RESTStorageProvider{}, batchrest.RESTStorageProvider{}, @@ -382,17 +368,17 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) func (m *Master) InstallLegacyAPI(c *completedConfig, restOptionsGetter generic.RESTOptionsGetter, legacyRESTStorageProvider corerest.LegacyRESTStorageProvider) { legacyRESTStorage, apiGroupInfo, err := legacyRESTStorageProvider.NewLegacyRESTStorage(restOptionsGetter) if err != nil { - glog.Fatalf("Error building core storage: %v", err) + klog.Fatalf("Error building core storage: %v", err) } controllerName := "bootstrap-controller" - coreClient := coreclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) + coreClient := corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) bootstrapController := c.NewBootstrapController(legacyRESTStorage, coreClient, coreClient, coreClient) m.GenericAPIServer.AddPostStartHookOrDie(controllerName, bootstrapController.PostStartHook) m.GenericAPIServer.AddPreShutdownHookOrDie(controllerName, bootstrapController.PreShutdownHook) if err := m.GenericAPIServer.InstallLegacyAPIGroup(genericapiserver.DefaultLegacyAPIPrefix, &apiGroupInfo); err != nil { - glog.Fatalf("Error in registering group versions: %v", err) + klog.Fatalf("Error in registering group versions: %v", err) } } @@ -418,20 +404,20 @@ func (m *Master) InstallAPIs(apiResourceConfigSource serverstorage.APIResourceCo for _, restStorageBuilder := range restStorageProviders { groupName := restStorageBuilder.GroupName() if !apiResourceConfigSource.AnyVersionForGroupEnabled(groupName) { - glog.V(1).Infof("Skipping disabled API group %q.", groupName) + klog.V(1).Infof("Skipping disabled API group %q.", groupName) continue } apiGroupInfo, enabled := restStorageBuilder.NewRESTStorage(apiResourceConfigSource, restOptionsGetter) if !enabled { - glog.Warningf("Problem initializing API group %q, skipping.", groupName) + klog.Warningf("Problem initializing API group %q, skipping.", groupName) continue } - glog.V(1).Infof("Enabling API group %q.", groupName) + klog.V(1).Infof("Enabling API group %q.", groupName) if postHookProvider, ok := restStorageBuilder.(genericapiserver.PostStartHookProvider); ok { name, hook, err := postHookProvider.PostStartHook() if err != nil { - glog.Fatalf("Error building PostStartHook: %v", err) + klog.Fatalf("Error building PostStartHook: %v", err) } m.GenericAPIServer.AddPostStartHookOrDie(name, hook) } @@ -441,7 +427,7 @@ func (m *Master) InstallAPIs(apiResourceConfigSource serverstorage.APIResourceCo for i := range apiGroupsInfo { if err := m.GenericAPIServer.InstallAPIGroup(&apiGroupsInfo[i]); err != nil { - glog.Fatalf("Error in registering group versions: %v", err) + klog.Fatalf("Error in registering group versions: %v", err) } } } diff --git a/pkg/master/ports/ports.go b/pkg/master/ports/ports.go index 19207a1012b..23faba1d3ec 100644 --- a/pkg/master/ports/ports.go +++ b/pkg/master/ports/ports.go @@ -23,9 +23,10 @@ const ( // KubeletPort is the default port for the kubelet server on each host machine. // May be overridden by a flag at startup. KubeletPort = 10250 - // SchedulerPort is the default port for the scheduler status server. + // InsecureSchedulerPort is the default port for the scheduler status server. // May be overridden by a flag at startup. - SchedulerPort = 10251 + // Deprecated: use the secure KubeSchedulerPort instead. + InsecureSchedulerPort = 10251 // InsecureKubeControllerManagerPort is the default port for the controller manager status server. // May be overridden by a flag at startup. // Deprecated: use the secure KubeControllerManagerPort instead. @@ -49,4 +50,8 @@ const ( // CloudControllerManagerPort is the default port for the cloud controller manager server. // This value may be overridden by a flag at startup. CloudControllerManagerPort = 10258 + + // KubeSchedulerPort is the default port for the scheduler status server. + // May be overridden by a flag at startup. + KubeSchedulerPort = 10259 ) diff --git a/pkg/master/reconcilers/BUILD b/pkg/master/reconcilers/BUILD index 09d274bea82..4ad16ac8b74 100644 --- a/pkg/master/reconcilers/BUILD +++ b/pkg/master/reconcilers/BUILD @@ -12,17 +12,16 @@ go_library( importpath = "k8s.io/kubernetes/pkg/master/reconcilers", visibility = ["//visibility:public"], deps = [ - "//pkg/api/endpoints:go_default_library", - "//pkg/apis/core:go_default_library", - "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library", + "//pkg/api/v1/endpoints:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -31,9 +30,9 @@ go_test( srcs = ["lease_test.go"], embed = [":go_default_library"], deps = [ - "//pkg/apis/core:go_default_library", - "//pkg/registry/registrytest:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], ) diff --git a/pkg/master/reconcilers/lease.go b/pkg/master/reconcilers/lease.go index 65a397db0b3..d7ea00b0096 100644 --- a/pkg/master/reconcilers/lease.go +++ b/pkg/master/reconcilers/lease.go @@ -28,16 +28,16 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kruntime "k8s.io/apimachinery/pkg/runtime" apirequest "k8s.io/apiserver/pkg/endpoints/request" - "k8s.io/apiserver/pkg/registry/rest" "k8s.io/apiserver/pkg/storage" - "k8s.io/kubernetes/pkg/api/endpoints" - api "k8s.io/kubernetes/pkg/apis/core" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + endpointsv1 "k8s.io/kubernetes/pkg/api/v1/endpoints" ) // Leases is an interface which assists in managing the set of active masters @@ -62,7 +62,7 @@ var _ Leases = &storageLeases{} // ListLeases retrieves a list of the current master IPs from storage func (s *storageLeases) ListLeases() ([]string, error) { - ipInfoList := &api.EndpointsList{} + ipInfoList := &corev1.EndpointsList{} if err := s.storage.List(apirequest.NewDefaultContext(), s.baseKey, "0", storage.Everything, ipInfoList); err != nil { return nil, err } @@ -72,7 +72,7 @@ func (s *storageLeases) ListLeases() ([]string, error) { ipList[i] = ip.Subsets[0].Addresses[0].IP } - glog.V(6).Infof("Current master IPs listed in storage are %v", ipList) + klog.V(6).Infof("Current master IPs listed in storage are %v", ipList) return ipList, nil } @@ -80,12 +80,12 @@ func (s *storageLeases) ListLeases() ([]string, error) { // UpdateLease resets the TTL on a master IP in storage func (s *storageLeases) UpdateLease(ip string) error { key := path.Join(s.baseKey, ip) - return s.storage.GuaranteedUpdate(apirequest.NewDefaultContext(), key, &api.Endpoints{}, true, nil, func(input kruntime.Object, respMeta storage.ResponseMeta) (kruntime.Object, *uint64, error) { + return s.storage.GuaranteedUpdate(apirequest.NewDefaultContext(), key, &corev1.Endpoints{}, true, nil, func(input kruntime.Object, respMeta storage.ResponseMeta) (kruntime.Object, *uint64, error) { // just make sure we've got the right IP set, and then refresh the TTL - existing := input.(*api.Endpoints) - existing.Subsets = []api.EndpointSubset{ + existing := input.(*corev1.Endpoints) + existing.Subsets = []corev1.EndpointSubset{ { - Addresses: []api.EndpointAddress{{IP: ip}}, + Addresses: []corev1.EndpointAddress{{IP: ip}}, }, } @@ -98,7 +98,7 @@ func (s *storageLeases) UpdateLease(ip string) error { // changing a field. existing.Generation++ - glog.V(6).Infof("Resetting TTL on master IP %q listed in storage to %v", ip, leaseTime) + klog.V(6).Infof("Resetting TTL on master IP %q listed in storage to %v", ip, leaseTime) return existing, &leaseTime, nil }) @@ -106,7 +106,7 @@ func (s *storageLeases) UpdateLease(ip string) error { // RemoveLease removes the lease on a master IP in storage func (s *storageLeases) RemoveLease(ip string) error { - return s.storage.Delete(apirequest.NewDefaultContext(), s.baseKey+"/"+ip, &api.Endpoints{}, nil) + return s.storage.Delete(apirequest.NewDefaultContext(), s.baseKey+"/"+ip, &corev1.Endpoints{}, nil) } // NewLeases creates a new etcd-based Leases implementation. @@ -119,16 +119,16 @@ func NewLeases(storage storage.Interface, baseKey string, leaseTime time.Duratio } type leaseEndpointReconciler struct { - endpointStorage rest.StandardStorage + endpointClient corev1client.EndpointsGetter masterLeases Leases stopReconcilingCalled bool reconcilingLock sync.Mutex } // NewLeaseEndpointReconciler creates a new LeaseEndpoint reconciler -func NewLeaseEndpointReconciler(endpointStorage rest.StandardStorage, masterLeases Leases) EndpointReconciler { +func NewLeaseEndpointReconciler(endpointClient corev1client.EndpointsGetter, masterLeases Leases) EndpointReconciler { return &leaseEndpointReconciler{ - endpointStorage: endpointStorage, + endpointClient: endpointClient, masterLeases: masterLeases, stopReconcilingCalled: false, } @@ -141,7 +141,7 @@ func NewLeaseEndpointReconciler(endpointStorage rest.StandardStorage, masterLeas // expire. ReconcileEndpoints will notice that the endpoints object is // different from the directory listing, and update the endpoints object // accordingly. -func (r *leaseEndpointReconciler) ReconcileEndpoints(serviceName string, ip net.IP, endpointPorts []api.EndpointPort, reconcilePorts bool) error { +func (r *leaseEndpointReconciler) ReconcileEndpoints(serviceName string, ip net.IP, endpointPorts []corev1.EndpointPort, reconcilePorts bool) error { r.reconcilingLock.Lock() defer r.reconcilingLock.Unlock() @@ -159,25 +159,21 @@ func (r *leaseEndpointReconciler) ReconcileEndpoints(serviceName string, ip net. return r.doReconcile(serviceName, endpointPorts, reconcilePorts) } -func (r *leaseEndpointReconciler) doReconcile(serviceName string, endpointPorts []api.EndpointPort, reconcilePorts bool) error { - ctx := apirequest.NewDefaultContext() - - // Retrieve the current list of endpoints... - var e *api.Endpoints - obj, err := r.endpointStorage.Get(ctx, serviceName, &metav1.GetOptions{}) +func (r *leaseEndpointReconciler) doReconcile(serviceName string, endpointPorts []corev1.EndpointPort, reconcilePorts bool) error { + e, err := r.endpointClient.Endpoints(corev1.NamespaceDefault).Get(serviceName, metav1.GetOptions{}) + shouldCreate := false if err != nil { if !errors.IsNotFound(err) { return err } - e = &api.Endpoints{ + shouldCreate = true + e = &corev1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName, - Namespace: api.NamespaceDefault, + Namespace: corev1.NamespaceDefault, }, } - } else { - e = obj.(*api.Endpoints) } // ... and the list of master IP keys from etcd @@ -201,21 +197,21 @@ func (r *leaseEndpointReconciler) doReconcile(serviceName string, endpointPorts if !formatCorrect { // Something is egregiously wrong, just re-make the endpoints record. - e.Subsets = []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{}, + e.Subsets = []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{}, Ports: endpointPorts, }} } if !formatCorrect || !ipCorrect { // repopulate the addresses according to the expected IPs from etcd - e.Subsets[0].Addresses = make([]api.EndpointAddress, len(masterIPs)) + e.Subsets[0].Addresses = make([]corev1.EndpointAddress, len(masterIPs)) for ind, ip := range masterIPs { - e.Subsets[0].Addresses[ind] = api.EndpointAddress{IP: ip} + e.Subsets[0].Addresses[ind] = corev1.EndpointAddress{IP: ip} } // Lexicographic order is retained by this step. - e.Subsets = endpoints.RepackSubsets(e.Subsets) + e.Subsets = endpointsv1.RepackSubsets(e.Subsets) } if !portsCorrect { @@ -223,8 +219,14 @@ func (r *leaseEndpointReconciler) doReconcile(serviceName string, endpointPorts e.Subsets[0].Ports = endpointPorts } - glog.Warningf("Resetting endpoints for master service %q to %v", serviceName, masterIPs) - _, _, err = r.endpointStorage.Update(ctx, e.Name, rest.DefaultUpdatedObjectInfo(e), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}) + klog.Warningf("Resetting endpoints for master service %q to %v", serviceName, masterIPs) + if shouldCreate { + if _, err = r.endpointClient.Endpoints(corev1.NamespaceDefault).Create(e); errors.IsAlreadyExists(err) { + err = nil + } + } else { + _, err = r.endpointClient.Endpoints(corev1.NamespaceDefault).Update(e) + } return err } @@ -236,7 +238,7 @@ func (r *leaseEndpointReconciler) doReconcile(serviceName string, endpointPorts // * ipsCorrect when the addresses in the endpoints match the expected addresses list // * portsCorrect is true when endpoint ports exactly match provided ports. // portsCorrect is only evaluated when reconcilePorts is set to true. -func checkEndpointSubsetFormatWithLease(e *api.Endpoints, expectedIPs []string, ports []api.EndpointPort, reconcilePorts bool) (formatCorrect bool, ipsCorrect bool, portsCorrect bool) { +func checkEndpointSubsetFormatWithLease(e *corev1.Endpoints, expectedIPs []string, ports []corev1.EndpointPort, reconcilePorts bool) (formatCorrect bool, ipsCorrect bool, portsCorrect bool) { if len(e.Subsets) != 1 { return false, false, false } @@ -281,7 +283,7 @@ func checkEndpointSubsetFormatWithLease(e *api.Endpoints, expectedIPs []string, return true, ipsCorrect, portsCorrect } -func (r *leaseEndpointReconciler) StopReconciling(serviceName string, ip net.IP, endpointPorts []api.EndpointPort) error { +func (r *leaseEndpointReconciler) StopReconciling(serviceName string, ip net.IP, endpointPorts []corev1.EndpointPort) error { r.reconcilingLock.Lock() defer r.reconcilingLock.Unlock() r.stopReconcilingCalled = true diff --git a/pkg/master/reconcilers/lease_test.go b/pkg/master/reconcilers/lease_test.go index 97000d39fba..3c8402a7867 100644 --- a/pkg/master/reconcilers/lease_test.go +++ b/pkg/master/reconcilers/lease_test.go @@ -26,9 +26,9 @@ import ( "reflect" "testing" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/registry/registrytest" + "k8s.io/client-go/kubernetes/fake" ) type fakeLeases struct { @@ -76,7 +76,7 @@ func (f *fakeLeases) GetUpdatedKeys() []string { } func TestLeaseEndpointReconciler(t *testing.T) { - ns := api.NamespaceDefault + ns := corev1.NamespaceDefault om := func(name string) metav1.ObjectMeta { return metav1.ObjectMeta{Namespace: ns, Name: name} } @@ -84,22 +84,22 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName string serviceName string ip string - endpointPorts []api.EndpointPort + endpointPorts []corev1.EndpointPort endpointKeys []string - endpoints *api.EndpointsList - expectUpdate *api.Endpoints // nil means none expected + endpoints *corev1.EndpointsList + expectUpdate *corev1.Endpoints // nil means none expected }{ { testName: "no existing endpoints", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, endpoints: nil, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -107,13 +107,13 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints satisfy", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, @@ -122,14 +122,14 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints satisfy + refresh existing key", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, endpointKeys: []string{"1.2.3.4"}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, @@ -138,21 +138,21 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints satisfy but too many", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}, {IP: "4.3.2.1"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}, {IP: "4.3.2.1"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -160,33 +160,33 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints satisfy but too many + extra masters", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, endpointKeys: []string{"1.2.3.4", "4.3.2.2", "4.3.2.3", "4.3.2.4"}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "1.2.3.4"}, {IP: "4.3.2.1"}, {IP: "4.3.2.2"}, {IP: "4.3.2.3"}, {IP: "4.3.2.4"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "1.2.3.4"}, {IP: "4.3.2.2"}, {IP: "4.3.2.3"}, {IP: "4.3.2.4"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -194,33 +194,33 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints satisfy but too many + extra masters + delete first", serviceName: "foo", ip: "4.3.2.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, endpointKeys: []string{"4.3.2.1", "4.3.2.2", "4.3.2.3", "4.3.2.4"}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "1.2.3.4"}, {IP: "4.3.2.1"}, {IP: "4.3.2.2"}, {IP: "4.3.2.3"}, {IP: "4.3.2.4"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "4.3.2.1"}, {IP: "4.3.2.2"}, {IP: "4.3.2.3"}, {IP: "4.3.2.4"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -228,27 +228,27 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints current IP missing", serviceName: "foo", ip: "4.3.2.2", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, endpointKeys: []string{"4.3.2.1"}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "4.3.2.1"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "4.3.2.1"}, {IP: "4.3.2.2"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -256,21 +256,21 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints wrong name", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("bar"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -278,21 +278,21 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints wrong IP", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "4.3.2.1"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "4.3.2.1"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -300,21 +300,21 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints wrong port", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 9090, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 9090, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -322,21 +322,21 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints wrong protocol", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "UDP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "UDP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -344,21 +344,21 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints wrong port name", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "baz", Port: 8080, Protocol: "TCP"}}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpointPorts: []corev1.EndpointPort{{Name: "baz", Port: 8080, Protocol: "TCP"}}, + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "baz", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "baz", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -366,17 +366,17 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints extra service ports satisfy", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{ + endpointPorts: []corev1.EndpointPort{ {Name: "foo", Port: 8080, Protocol: "TCP"}, {Name: "bar", Port: 1000, Protocol: "TCP"}, {Name: "baz", Port: 1010, Protocol: "TCP"}, }, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{ {Name: "foo", Port: 8080, Protocol: "TCP"}, {Name: "bar", Port: 1000, Protocol: "TCP"}, {Name: "baz", Port: 1010, Protocol: "TCP"}, @@ -389,24 +389,24 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints extra service ports missing port", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{ + endpointPorts: []corev1.EndpointPort{ {Name: "foo", Port: 8080, Protocol: "TCP"}, {Name: "bar", Port: 1000, Protocol: "TCP"}, }, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{ {Name: "foo", Port: 8080, Protocol: "TCP"}, {Name: "bar", Port: 1000, Protocol: "TCP"}, }, @@ -417,24 +417,29 @@ func TestLeaseEndpointReconciler(t *testing.T) { for _, test := range reconcileTests { fakeLeases := newFakeLeases() fakeLeases.SetKeys(test.endpointKeys) - registry := ®istrytest.EndpointRegistry{ - Endpoints: test.endpoints, + clientset := fake.NewSimpleClientset() + if test.endpoints != nil { + for _, ep := range test.endpoints.Items { + if _, err := clientset.CoreV1().Endpoints(ep.Namespace).Create(&ep); err != nil { + t.Errorf("case %q: unexpected error: %v", test.testName, err) + continue + } + } } - r := NewLeaseEndpointReconciler(registry, fakeLeases) + r := NewLeaseEndpointReconciler(clientset.CoreV1(), fakeLeases) err := r.ReconcileEndpoints(test.serviceName, net.ParseIP(test.ip), test.endpointPorts, true) if err != nil { t.Errorf("case %q: unexpected error: %v", test.testName, err) } + actualEndpoints, err := clientset.CoreV1().Endpoints(corev1.NamespaceDefault).Get(test.serviceName, metav1.GetOptions{}) + if err != nil { + t.Errorf("case %q: unexpected error: %v", test.testName, err) + } if test.expectUpdate != nil { - if len(registry.Updates) != 1 { - t.Errorf("case %q: unexpected updates: %v", test.testName, registry.Updates) - } else if e, a := test.expectUpdate, ®istry.Updates[0]; !reflect.DeepEqual(e, a) { + if e, a := test.expectUpdate, actualEndpoints; !reflect.DeepEqual(e, a) { t.Errorf("case %q: expected update:\n%#v\ngot:\n%#v\n", test.testName, e, a) } } - if test.expectUpdate == nil && len(registry.Updates) > 0 { - t.Errorf("case %q: no update expected, yet saw: %v", test.testName, registry.Updates) - } if updatedKeys := fakeLeases.GetUpdatedKeys(); len(updatedKeys) != 1 || updatedKeys[0] != test.ip { t.Errorf("case %q: expected the master's IP to be refreshed, but the following IPs were refreshed instead: %v", test.testName, updatedKeys) } @@ -444,25 +449,25 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName string serviceName string ip string - endpointPorts []api.EndpointPort + endpointPorts []corev1.EndpointPort endpointKeys []string - endpoints *api.EndpointsList - expectUpdate *api.Endpoints // nil means none expected + endpoints *corev1.EndpointsList + expectUpdate *corev1.Endpoints // nil means none expected }{ { testName: "existing endpoints extra service ports missing port no update", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{ + endpointPorts: []corev1.EndpointPort{ {Name: "foo", Port: 8080, Protocol: "TCP"}, {Name: "bar", Port: 1000, Protocol: "TCP"}, }, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, @@ -472,24 +477,24 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "existing endpoints extra service ports, wrong ports, wrong IP", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{ + endpointPorts: []corev1.EndpointPort{ {Name: "foo", Port: 8080, Protocol: "TCP"}, {Name: "bar", Port: 1000, Protocol: "TCP"}, }, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "4.3.2.1"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "4.3.2.1"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -497,13 +502,13 @@ func TestLeaseEndpointReconciler(t *testing.T) { testName: "no existing endpoints", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, endpoints: nil, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}}, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: "1.2.3.4"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -512,24 +517,29 @@ func TestLeaseEndpointReconciler(t *testing.T) { t.Run(test.testName, func(t *testing.T) { fakeLeases := newFakeLeases() fakeLeases.SetKeys(test.endpointKeys) - registry := ®istrytest.EndpointRegistry{ - Endpoints: test.endpoints, + clientset := fake.NewSimpleClientset() + if test.endpoints != nil { + for _, ep := range test.endpoints.Items { + if _, err := clientset.CoreV1().Endpoints(ep.Namespace).Create(&ep); err != nil { + t.Errorf("case %q: unexpected error: %v", test.testName, err) + continue + } + } } - r := NewLeaseEndpointReconciler(registry, fakeLeases) + r := NewLeaseEndpointReconciler(clientset.CoreV1(), fakeLeases) err := r.ReconcileEndpoints(test.serviceName, net.ParseIP(test.ip), test.endpointPorts, false) if err != nil { t.Errorf("case %q: unexpected error: %v", test.testName, err) } + actualEndpoints, err := clientset.CoreV1().Endpoints(corev1.NamespaceDefault).Get(test.serviceName, metav1.GetOptions{}) + if err != nil { + t.Errorf("case %q: unexpected error: %v", test.testName, err) + } if test.expectUpdate != nil { - if len(registry.Updates) != 1 { - t.Errorf("case %q: unexpected updates: %v", test.testName, registry.Updates) - } else if e, a := test.expectUpdate, ®istry.Updates[0]; !reflect.DeepEqual(e, a) { + if e, a := test.expectUpdate, actualEndpoints; !reflect.DeepEqual(e, a) { t.Errorf("case %q: expected update:\n%#v\ngot:\n%#v\n", test.testName, e, a) } } - if test.expectUpdate == nil && len(registry.Updates) > 0 { - t.Errorf("case %q: no update expected, yet saw: %v", test.testName, registry.Updates) - } if updatedKeys := fakeLeases.GetUpdatedKeys(); len(updatedKeys) != 1 || updatedKeys[0] != test.ip { t.Errorf("case %q: expected the master's IP to be refreshed, but the following IPs were refreshed instead: %v", test.testName, updatedKeys) } @@ -538,7 +548,7 @@ func TestLeaseEndpointReconciler(t *testing.T) { } func TestLeaseStopReconciling(t *testing.T) { - ns := api.NamespaceDefault + ns := corev1.NamespaceDefault om := func(name string) metav1.ObjectMeta { return metav1.ObjectMeta{Namespace: ns, Name: name} } @@ -546,40 +556,40 @@ func TestLeaseStopReconciling(t *testing.T) { testName string serviceName string ip string - endpointPorts []api.EndpointPort + endpointPorts []corev1.EndpointPort endpointKeys []string - endpoints *api.EndpointsList - expectUpdate *api.Endpoints // nil means none expected + endpoints *corev1.EndpointsList + expectUpdate *corev1.Endpoints // nil means none expected }{ { testName: "successful stop reconciling", serviceName: "foo", ip: "1.2.3.4", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, endpointKeys: []string{"1.2.3.4", "4.3.2.2", "4.3.2.3", "4.3.2.4"}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "1.2.3.4"}, {IP: "4.3.2.2"}, {IP: "4.3.2.3"}, {IP: "4.3.2.4"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, - expectUpdate: &api.Endpoints{ + expectUpdate: &corev1.Endpoints{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "4.3.2.2"}, {IP: "4.3.2.3"}, {IP: "4.3.2.4"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }, }, @@ -587,19 +597,19 @@ func TestLeaseStopReconciling(t *testing.T) { testName: "stop reconciling with ip not in endpoint ip list", serviceName: "foo", ip: "5.6.7.8", - endpointPorts: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + endpointPorts: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, endpointKeys: []string{"1.2.3.4", "4.3.2.2", "4.3.2.3", "4.3.2.4"}, - endpoints: &api.EndpointsList{ - Items: []api.Endpoints{{ + endpoints: &corev1.EndpointsList{ + Items: []corev1.Endpoints{{ ObjectMeta: om("foo"), - Subsets: []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{ + Subsets: []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{ {IP: "1.2.3.4"}, {IP: "4.3.2.2"}, {IP: "4.3.2.3"}, {IP: "4.3.2.4"}, }, - Ports: []api.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, + Ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, }}, }}, }, @@ -609,24 +619,27 @@ func TestLeaseStopReconciling(t *testing.T) { t.Run(test.testName, func(t *testing.T) { fakeLeases := newFakeLeases() fakeLeases.SetKeys(test.endpointKeys) - registry := ®istrytest.EndpointRegistry{ - Endpoints: test.endpoints, + clientset := fake.NewSimpleClientset() + for _, ep := range test.endpoints.Items { + if _, err := clientset.CoreV1().Endpoints(ep.Namespace).Create(&ep); err != nil { + t.Errorf("case %q: unexpected error: %v", test.testName, err) + continue + } } - r := NewLeaseEndpointReconciler(registry, fakeLeases) + r := NewLeaseEndpointReconciler(clientset.CoreV1(), fakeLeases) err := r.StopReconciling(test.serviceName, net.ParseIP(test.ip), test.endpointPorts) if err != nil { t.Errorf("case %q: unexpected error: %v", test.testName, err) } + actualEndpoints, err := clientset.CoreV1().Endpoints(corev1.NamespaceDefault).Get(test.serviceName, metav1.GetOptions{}) + if err != nil { + t.Errorf("case %q: unexpected error: %v", test.testName, err) + } if test.expectUpdate != nil { - if len(registry.Updates) != 1 { - t.Errorf("case %q: unexpected updates: %v", test.testName, registry.Updates) - } else if e, a := test.expectUpdate, ®istry.Updates[0]; !reflect.DeepEqual(e, a) { + if e, a := test.expectUpdate, actualEndpoints; !reflect.DeepEqual(e, a) { t.Errorf("case %q: expected update:\n%#v\ngot:\n%#v\n", test.testName, e, a) } } - if test.expectUpdate == nil && len(registry.Updates) > 0 { - t.Errorf("case %q: no update expected, yet saw: %v", test.testName, registry.Updates) - } for _, key := range fakeLeases.GetUpdatedKeys() { if key == test.ip { t.Errorf("case %q: Found ip %s in leases but shouldn't be there", test.testName, key) diff --git a/pkg/master/reconcilers/mastercount.go b/pkg/master/reconcilers/mastercount.go index d7aa4c77185..18a635b6a74 100644 --- a/pkg/master/reconcilers/mastercount.go +++ b/pkg/master/reconcilers/mastercount.go @@ -21,27 +21,27 @@ import ( "net" "sync" - "github.com/golang/glog" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/util/retry" - "k8s.io/kubernetes/pkg/api/endpoints" - api "k8s.io/kubernetes/pkg/apis/core" - coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" + "k8s.io/klog" + endpointsv1 "k8s.io/kubernetes/pkg/api/v1/endpoints" ) // masterCountEndpointReconciler reconciles endpoints based on a specified expected number of // masters. masterCountEndpointReconciler implements EndpointReconciler. type masterCountEndpointReconciler struct { masterCount int - endpointClient coreclient.EndpointsGetter + endpointClient corev1client.EndpointsGetter stopReconcilingCalled bool reconcilingLock sync.Mutex } // NewMasterCountEndpointReconciler creates a new EndpointReconciler that reconciles based on a // specified expected number of masters. -func NewMasterCountEndpointReconciler(masterCount int, endpointClient coreclient.EndpointsGetter) EndpointReconciler { +func NewMasterCountEndpointReconciler(masterCount int, endpointClient corev1client.EndpointsGetter) EndpointReconciler { return &masterCountEndpointReconciler{ masterCount: masterCount, endpointClient: endpointClient, @@ -60,7 +60,7 @@ func NewMasterCountEndpointReconciler(masterCount int, endpointClient coreclient // * All apiservers MUST know and agree on the number of apiservers expected // to be running (c.masterCount). // * ReconcileEndpoints is called periodically from all apiservers. -func (r *masterCountEndpointReconciler) ReconcileEndpoints(serviceName string, ip net.IP, endpointPorts []api.EndpointPort, reconcilePorts bool) error { +func (r *masterCountEndpointReconciler) ReconcileEndpoints(serviceName string, ip net.IP, endpointPorts []corev1.EndpointPort, reconcilePorts bool) error { r.reconcilingLock.Lock() defer r.reconcilingLock.Unlock() @@ -70,7 +70,7 @@ func (r *masterCountEndpointReconciler) ReconcileEndpoints(serviceName string, i e, err := r.endpointClient.Endpoints(metav1.NamespaceDefault).Get(serviceName, metav1.GetOptions{}) if err != nil { - e = &api.Endpoints{ + e = &corev1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName, Namespace: metav1.NamespaceDefault, @@ -79,8 +79,8 @@ func (r *masterCountEndpointReconciler) ReconcileEndpoints(serviceName string, i } if errors.IsNotFound(err) { // Simply create non-existing endpoints for the service. - e.Subsets = []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: ip.String()}}, + e.Subsets = []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: ip.String()}}, Ports: endpointPorts, }} _, err = r.endpointClient.Endpoints(metav1.NamespaceDefault).Create(e) @@ -92,11 +92,11 @@ func (r *masterCountEndpointReconciler) ReconcileEndpoints(serviceName string, i formatCorrect, ipCorrect, portsCorrect := checkEndpointSubsetFormat(e, ip.String(), endpointPorts, r.masterCount, reconcilePorts) if !formatCorrect { // Something is egregiously wrong, just re-make the endpoints record. - e.Subsets = []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{IP: ip.String()}}, + e.Subsets = []corev1.EndpointSubset{{ + Addresses: []corev1.EndpointAddress{{IP: ip.String()}}, Ports: endpointPorts, }} - glog.Warningf("Resetting endpoints for master service %q to %#v", serviceName, e) + klog.Warningf("Resetting endpoints for master service %q to %#v", serviceName, e) _, err = r.endpointClient.Endpoints(metav1.NamespaceDefault).Update(e) return err } @@ -105,10 +105,10 @@ func (r *masterCountEndpointReconciler) ReconcileEndpoints(serviceName string, i } if !ipCorrect { // We *always* add our own IP address. - e.Subsets[0].Addresses = append(e.Subsets[0].Addresses, api.EndpointAddress{IP: ip.String()}) + e.Subsets[0].Addresses = append(e.Subsets[0].Addresses, corev1.EndpointAddress{IP: ip.String()}) // Lexicographic order is retained by this step. - e.Subsets = endpoints.RepackSubsets(e.Subsets) + e.Subsets = endpointsv1.RepackSubsets(e.Subsets) // If too many IP addresses, remove the ones lexicographically after our // own IP address. Given the requirements stated at the top of @@ -132,12 +132,12 @@ func (r *masterCountEndpointReconciler) ReconcileEndpoints(serviceName string, i // Reset ports. e.Subsets[0].Ports = endpointPorts } - glog.Warningf("Resetting endpoints for master service %q to %v", serviceName, e) + klog.Warningf("Resetting endpoints for master service %q to %v", serviceName, e) _, err = r.endpointClient.Endpoints(metav1.NamespaceDefault).Update(e) return err } -func (r *masterCountEndpointReconciler) StopReconciling(serviceName string, ip net.IP, endpointPorts []api.EndpointPort) error { +func (r *masterCountEndpointReconciler) StopReconciling(serviceName string, ip net.IP, endpointPorts []corev1.EndpointPort) error { r.reconcilingLock.Lock() defer r.reconcilingLock.Unlock() r.stopReconcilingCalled = true @@ -152,14 +152,14 @@ func (r *masterCountEndpointReconciler) StopReconciling(serviceName string, ip n } // Remove our IP from the list of addresses - new := []api.EndpointAddress{} + new := []corev1.EndpointAddress{} for _, addr := range e.Subsets[0].Addresses { if addr.IP != ip.String() { new = append(new, addr) } } e.Subsets[0].Addresses = new - e.Subsets = endpoints.RepackSubsets(e.Subsets) + e.Subsets = endpointsv1.RepackSubsets(e.Subsets) err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { _, err := r.endpointClient.Endpoints(metav1.NamespaceDefault).Update(e) return err @@ -175,7 +175,7 @@ func (r *masterCountEndpointReconciler) StopReconciling(serviceName string, ip n // of addresses is less than or equal to the master count. // * portsCorrect is true when endpoint ports exactly match provided ports. // portsCorrect is only evaluated when reconcilePorts is set to true. -func checkEndpointSubsetFormat(e *api.Endpoints, ip string, ports []api.EndpointPort, count int, reconcilePorts bool) (formatCorrect bool, ipCorrect bool, portsCorrect bool) { +func checkEndpointSubsetFormat(e *corev1.Endpoints, ip string, ports []corev1.EndpointPort, count int, reconcilePorts bool) (formatCorrect bool, ipCorrect bool, portsCorrect bool) { if len(e.Subsets) != 1 { return false, false, false } @@ -214,7 +214,7 @@ func checkEndpointSubsetFormat(e *api.Endpoints, ip string, ports []api.Endpoint // * All apiservers MUST use GetMasterServiceUpdateIfNeeded and only // GetMasterServiceUpdateIfNeeded to manage service attributes // * updateMasterService is called periodically from all apiservers. -func GetMasterServiceUpdateIfNeeded(svc *api.Service, servicePorts []api.ServicePort, serviceType api.ServiceType) (s *api.Service, updated bool) { +func GetMasterServiceUpdateIfNeeded(svc *corev1.Service, servicePorts []corev1.ServicePort, serviceType corev1.ServiceType) (s *corev1.Service, updated bool) { // Determine if the service is in the format we expect // (servicePorts are present and service type matches) formatCorrect := checkServiceFormat(svc, servicePorts, serviceType) @@ -229,7 +229,7 @@ func GetMasterServiceUpdateIfNeeded(svc *api.Service, servicePorts []api.Service // Determine if the service is in the correct format // GetMasterServiceUpdateIfNeeded expects (servicePorts are correct // and service type matches). -func checkServiceFormat(s *api.Service, ports []api.ServicePort, serviceType api.ServiceType) (formatCorrect bool) { +func checkServiceFormat(s *corev1.Service, ports []corev1.ServicePort, serviceType corev1.ServiceType) (formatCorrect bool) { if s.Spec.Type != serviceType { return false } diff --git a/pkg/master/reconcilers/none.go b/pkg/master/reconcilers/none.go index dce38c2a66a..9bd4ee5ad7f 100644 --- a/pkg/master/reconcilers/none.go +++ b/pkg/master/reconcilers/none.go @@ -18,7 +18,7 @@ limitations under the License. package reconcilers import ( - api "k8s.io/kubernetes/pkg/apis/core" + corev1 "k8s.io/api/core/v1" "net" ) @@ -32,11 +32,11 @@ func NewNoneEndpointReconciler() EndpointReconciler { } // ReconcileEndpoints noop reconcile -func (r *noneEndpointReconciler) ReconcileEndpoints(serviceName string, ip net.IP, endpointPorts []api.EndpointPort, reconcilePorts bool) error { +func (r *noneEndpointReconciler) ReconcileEndpoints(serviceName string, ip net.IP, endpointPorts []corev1.EndpointPort, reconcilePorts bool) error { return nil } // StopReconciling noop reconcile -func (r *noneEndpointReconciler) StopReconciling(serviceName string, ip net.IP, endpointPorts []api.EndpointPort) error { +func (r *noneEndpointReconciler) StopReconciling(serviceName string, ip net.IP, endpointPorts []corev1.EndpointPort) error { return nil } diff --git a/pkg/master/reconcilers/reconcilers.go b/pkg/master/reconcilers/reconcilers.go index 8bbabc65901..0cfb9a0aaf8 100644 --- a/pkg/master/reconcilers/reconcilers.go +++ b/pkg/master/reconcilers/reconcilers.go @@ -18,7 +18,7 @@ limitations under the License. package reconcilers import ( - api "k8s.io/kubernetes/pkg/apis/core" + corev1 "k8s.io/api/core/v1" "net" ) @@ -34,8 +34,8 @@ type EndpointReconciler interface { // * All apiservers MUST use ReconcileEndpoints and only ReconcileEndpoints to manage the // endpoints for their {rw, ro} services. // * ReconcileEndpoints is called periodically from all apiservers. - ReconcileEndpoints(serviceName string, ip net.IP, endpointPorts []api.EndpointPort, reconcilePorts bool) error - StopReconciling(serviceName string, ip net.IP, endpointPorts []api.EndpointPort) error + ReconcileEndpoints(serviceName string, ip net.IP, endpointPorts []corev1.EndpointPort, reconcilePorts bool) error + StopReconciling(serviceName string, ip net.IP, endpointPorts []corev1.EndpointPort) error } // Type the reconciler type diff --git a/pkg/master/services.go b/pkg/master/services.go index 44bb15edffc..dbd06191023 100644 --- a/pkg/master/services.go +++ b/pkg/master/services.go @@ -20,7 +20,7 @@ import ( "fmt" "net" - "github.com/golang/glog" + "k8s.io/klog" kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" ) @@ -30,7 +30,7 @@ import ( func DefaultServiceIPRange(passedServiceClusterIPRange net.IPNet) (net.IPNet, net.IP, error) { serviceClusterIPRange := passedServiceClusterIPRange if passedServiceClusterIPRange.IP == nil { - glog.Infof("Network range for service cluster IPs is unspecified. Defaulting to %v.", kubeoptions.DefaultServiceIPCIDR) + klog.Infof("Network range for service cluster IPs is unspecified. Defaulting to %v.", kubeoptions.DefaultServiceIPCIDR) serviceClusterIPRange = kubeoptions.DefaultServiceIPCIDR } if size := ipallocator.RangeSize(&serviceClusterIPRange); size < 8 { @@ -42,7 +42,7 @@ func DefaultServiceIPRange(passedServiceClusterIPRange net.IPNet) (net.IPNet, ne if err != nil { return net.IPNet{}, net.IP{}, err } - glog.V(4).Infof("Setting service IP to %q (read-write).", apiServerServiceIP) + klog.V(4).Infof("Setting service IP to %q (read-write).", apiServerServiceIP) return serviceClusterIPRange, apiServerServiceIP, nil } diff --git a/pkg/master/tunneler/BUILD b/pkg/master/tunneler/BUILD index 1bcdc497688..fd4bf629dd9 100644 --- a/pkg/master/tunneler/BUILD +++ b/pkg/master/tunneler/BUILD @@ -25,8 +25,8 @@ go_library( "//pkg/util/file:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/master/tunneler/ssh.go b/pkg/master/tunneler/ssh.go index 6a55015d5c1..88217d170ed 100644 --- a/pkg/master/tunneler/ssh.go +++ b/pkg/master/tunneler/ssh.go @@ -32,8 +32,8 @@ import ( "k8s.io/kubernetes/pkg/ssh" utilfile "k8s.io/kubernetes/pkg/util/file" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" + "k8s.io/klog" ) type InstallSSHKey func(ctx context.Context, user string, data []byte) error @@ -115,20 +115,20 @@ func (c *SSHTunneler) Run(getAddresses AddressFunc) { // Usernames are capped @ 32 if len(c.SSHUser) > 32 { - glog.Warning("SSH User is too long, truncating to 32 chars") + klog.Warning("SSH User is too long, truncating to 32 chars") c.SSHUser = c.SSHUser[0:32] } - glog.Infof("Setting up proxy: %s %s", c.SSHUser, c.SSHKeyfile) + klog.Infof("Setting up proxy: %s %s", c.SSHUser, c.SSHKeyfile) // public keyfile is written last, so check for that. publicKeyFile := c.SSHKeyfile + ".pub" exists, err := utilfile.FileExists(publicKeyFile) if err != nil { - glog.Errorf("Error detecting if key exists: %v", err) + klog.Errorf("Error detecting if key exists: %v", err) } else if !exists { - glog.Infof("Key doesn't exist, attempting to create") + klog.Infof("Key doesn't exist, attempting to create") if err := generateSSHKey(c.SSHKeyfile, publicKeyFile); err != nil { - glog.Errorf("Failed to create key pair: %v", err) + klog.Errorf("Failed to create key pair: %v", err) } } @@ -168,21 +168,21 @@ func (c *SSHTunneler) SecondsSinceSSHKeySync() int64 { func (c *SSHTunneler) installSSHKeySyncLoop(user, publicKeyfile string) { go wait.Until(func() { if c.InstallSSHKey == nil { - glog.Error("Won't attempt to install ssh key: InstallSSHKey function is nil") + klog.Error("Won't attempt to install ssh key: InstallSSHKey function is nil") return } key, err := ssh.ParsePublicKeyFromFile(publicKeyfile) if err != nil { - glog.Errorf("Failed to load public key: %v", err) + klog.Errorf("Failed to load public key: %v", err) return } keyData, err := ssh.EncodeSSHKey(key) if err != nil { - glog.Errorf("Failed to encode public key: %v", err) + klog.Errorf("Failed to encode public key: %v", err) return } if err := c.InstallSSHKey(context.TODO(), user, keyData); err != nil { - glog.Errorf("Failed to install ssh key: %v", err) + klog.Errorf("Failed to install ssh key: %v", err) return } atomic.StoreInt64(&c.lastSSHKeySync, c.clock.Now().Unix()) @@ -195,9 +195,9 @@ func (c *SSHTunneler) nodesSyncLoop() { // TODO (cjcullen) make this watch. go wait.Until(func() { addrs, err := c.getAddresses() - glog.V(4).Infof("Calling update w/ addrs: %v", addrs) + klog.V(4).Infof("Calling update w/ addrs: %v", addrs) if err != nil { - glog.Errorf("Failed to getAddresses: %v", err) + klog.Errorf("Failed to getAddresses: %v", err) } c.tunnels.Update(addrs) atomic.StoreInt64(&c.lastSync, c.clock.Now().Unix()) @@ -213,11 +213,11 @@ func generateSSHKey(privateKeyfile, publicKeyfile string) error { // through last time, so delete it. exists, err := utilfile.FileExists(privateKeyfile) if err != nil { - glog.Errorf("Error detecting if private key exists: %v", err) + klog.Errorf("Error detecting if private key exists: %v", err) } else if exists { - glog.Infof("Private key exists, but public key does not") + klog.Infof("Private key exists, but public key does not") if err := os.Remove(privateKeyfile); err != nil { - glog.Errorf("Failed to remove stale private key: %v", err) + klog.Errorf("Failed to remove stale private key: %v", err) } } if err := ioutil.WriteFile(privateKeyfile, ssh.EncodePrivateKey(private), 0600); err != nil { diff --git a/pkg/printers/BUILD b/pkg/printers/BUILD index 4d4ab061e8e..4c9f9e647ab 100644 --- a/pkg/printers/BUILD +++ b/pkg/printers/BUILD @@ -9,15 +9,12 @@ load( go_library( name = "go_default_library", srcs = [ - "customcolumn.go", - "customcolumn_flags.go", "humanreadable.go", "interface.go", "tabwriter.go", ], importpath = "k8s.io/kubernetes/pkg/printers", deps = [ - "//pkg/kubectl/scheme:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", @@ -26,10 +23,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", - "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", - "//staging/src/k8s.io/client-go/util/jsonpath:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", ], ) @@ -52,19 +45,12 @@ filegroup( go_test( name = "go_default_test", - srcs = [ - "customcolumn_flags_test.go", - "customcolumn_test.go", - "humanreadable_test.go", - ], + srcs = ["humanreadable_test.go"], embed = [":go_default_library"], deps = [ - "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/core:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", ], ) diff --git a/pkg/printers/interface.go b/pkg/printers/interface.go index f528de5caa6..6e14f61c5be 100644 --- a/pkg/printers/interface.go +++ b/pkg/printers/interface.go @@ -17,7 +17,6 @@ limitations under the License. package printers import ( - "fmt" "io" "k8s.io/apimachinery/pkg/runtime" @@ -58,36 +57,3 @@ type PrintOptions struct { // indicates if it is OK to ignore missing keys for rendering an output template. AllowMissingKeys bool } - -// Describer generates output for the named resource or an error -// if the output could not be generated. Implementers typically -// abstract the retrieval of the named object from a remote server. -type Describer interface { - Describe(namespace, name string, describerSettings DescriberSettings) (output string, err error) -} - -// DescriberSettings holds display configuration for each object -// describer to control what is printed. -type DescriberSettings struct { - ShowEvents bool -} - -// ObjectDescriber is an interface for displaying arbitrary objects with extra -// information. Use when an object is in hand (on disk, or already retrieved). -// Implementers may ignore the additional information passed on extra, or use it -// by default. ObjectDescribers may return ErrNoDescriber if no suitable describer -// is found. -type ObjectDescriber interface { - DescribeObject(object interface{}, extra ...interface{}) (output string, err error) -} - -// ErrNoDescriber is a structured error indicating the provided object or objects -// cannot be described. -type ErrNoDescriber struct { - Types []string -} - -// Error implements the error interface. -func (e ErrNoDescriber) Error() string { - return fmt.Sprintf("no describer has been defined for %v", e.Types) -} diff --git a/pkg/printers/internalversion/BUILD b/pkg/printers/internalversion/BUILD index 8f04a12db04..147e1e197b1 100644 --- a/pkg/printers/internalversion/BUILD +++ b/pkg/printers/internalversion/BUILD @@ -9,7 +9,6 @@ load( go_test( name = "go_default_test", srcs = [ - "describe_test.go", "printers_test.go", "sorted_resource_name_list_test.go", ], @@ -23,16 +22,11 @@ go_test( "//pkg/apis/coordination:go_default_library", "//pkg/apis/core:go_default_library", "//pkg/apis/extensions:go_default_library", - "//pkg/apis/networking:go_default_library", "//pkg/apis/policy:go_default_library", "//pkg/apis/scheduling:go_default_library", "//pkg/apis/storage:go_default_library", - "//pkg/client/clientset_generated/internalclientset:go_default_library", - "//pkg/client/clientset_generated/internalclientset/fake:go_default_library", "//pkg/printers:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", @@ -45,49 +39,49 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) go_library( name = "go_default_library", srcs = [ - "describe.go", + "import_known_versions.go", "printers.go", ], importpath = "k8s.io/kubernetes/pkg/printers/internalversion", deps = [ - "//pkg/api/events:go_default_library", - "//pkg/api/legacyscheme:go_default_library", - "//pkg/api/ref:go_default_library", - "//pkg/api/resource:go_default_library", "//pkg/apis/apps:go_default_library", + "//pkg/apis/apps/install:go_default_library", + "//pkg/apis/authentication/install:go_default_library", + "//pkg/apis/authorization/install:go_default_library", "//pkg/apis/autoscaling:go_default_library", + "//pkg/apis/autoscaling/install:go_default_library", "//pkg/apis/batch:go_default_library", + "//pkg/apis/batch/install:go_default_library", "//pkg/apis/certificates:go_default_library", + "//pkg/apis/certificates/install:go_default_library", "//pkg/apis/coordination:go_default_library", + "//pkg/apis/coordination/install:go_default_library", "//pkg/apis/core:go_default_library", "//pkg/apis/core/helper:go_default_library", - "//pkg/apis/core/helper/qos:go_default_library", + "//pkg/apis/core/install:go_default_library", + "//pkg/apis/events/install:go_default_library", "//pkg/apis/extensions:go_default_library", + "//pkg/apis/extensions/install:go_default_library", "//pkg/apis/networking:go_default_library", "//pkg/apis/policy:go_default_library", + "//pkg/apis/policy/install:go_default_library", "//pkg/apis/rbac:go_default_library", - "//pkg/apis/rbac/v1:go_default_library", + "//pkg/apis/rbac/install:go_default_library", "//pkg/apis/scheduling:go_default_library", + "//pkg/apis/scheduling/install:go_default_library", + "//pkg/apis/settings/install:go_default_library", "//pkg/apis/storage:go_default_library", + "//pkg/apis/storage/install:go_default_library", "//pkg/apis/storage/util:go_default_library", - "//pkg/client/clientset_generated/internalclientset:go_default_library", - "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library", - "//pkg/controller/deployment/util:go_default_library", - "//pkg/fieldpath:go_default_library", "//pkg/printers:go_default_library", - "//pkg/registry/rbac/validation:go_default_library", "//pkg/util/node:go_default_library", - "//pkg/util/slice:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/apps/v1beta1:go_default_library", "//staging/src/k8s.io/api/autoscaling/v2beta1:go_default_library", "//staging/src/k8s.io/api/batch/v1:go_default_library", @@ -97,28 +91,17 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library", "//staging/src/k8s.io/api/policy/v1beta1:go_default_library", - "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library", "//staging/src/k8s.io/api/scheduling/v1beta1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1beta1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/duration:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/client-go/dynamic:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//vendor/github.com/fatih/camelcase:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", ], ) diff --git a/pkg/printers/internalversion/describe.go b/pkg/printers/internalversion/describe.go deleted file mode 100644 index ef79f5e05cb..00000000000 --- a/pkg/printers/internalversion/describe.go +++ /dev/null @@ -1,4261 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package internalversion - -import ( - "bytes" - "crypto/x509" - "fmt" - "io" - "net" - "net/url" - "reflect" - "sort" - "strconv" - "strings" - "text/tabwriter" - "time" - - "github.com/golang/glog" - - "github.com/fatih/camelcase" - - appsv1 "k8s.io/api/apps/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/duration" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/dynamic" - externalclient "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/kubernetes/pkg/api/events" - "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/api/ref" - resourcehelper "k8s.io/kubernetes/pkg/api/resource" - "k8s.io/kubernetes/pkg/apis/apps" - "k8s.io/kubernetes/pkg/apis/autoscaling" - "k8s.io/kubernetes/pkg/apis/batch" - "k8s.io/kubernetes/pkg/apis/certificates" - api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/apis/core/helper" - "k8s.io/kubernetes/pkg/apis/core/helper/qos" - "k8s.io/kubernetes/pkg/apis/extensions" - "k8s.io/kubernetes/pkg/apis/networking" - "k8s.io/kubernetes/pkg/apis/policy" - "k8s.io/kubernetes/pkg/apis/rbac" - rbacv1helpers "k8s.io/kubernetes/pkg/apis/rbac/v1" - "k8s.io/kubernetes/pkg/apis/scheduling" - "k8s.io/kubernetes/pkg/apis/storage" - storageutil "k8s.io/kubernetes/pkg/apis/storage/util" - clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" - deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util" - "k8s.io/kubernetes/pkg/fieldpath" - "k8s.io/kubernetes/pkg/printers" - "k8s.io/kubernetes/pkg/registry/rbac/validation" - "k8s.io/kubernetes/pkg/util/slice" -) - -// Each level has 2 spaces for PrefixWriter -const ( - LEVEL_0 = iota - LEVEL_1 - LEVEL_2 - LEVEL_3 -) - -// PrefixWriter can write text at various indentation levels. -type PrefixWriter interface { - // Write writes text with the specified indentation level. - Write(level int, format string, a ...interface{}) - // WriteLine writes an entire line with no indentation level. - WriteLine(a ...interface{}) - // Flush forces indentation to be reset. - Flush() -} - -// prefixWriter implements PrefixWriter -type prefixWriter struct { - out io.Writer -} - -var _ PrefixWriter = &prefixWriter{} - -// NewPrefixWriter creates a new PrefixWriter. -func NewPrefixWriter(out io.Writer) PrefixWriter { - return &prefixWriter{out: out} -} - -func (pw *prefixWriter) Write(level int, format string, a ...interface{}) { - levelSpace := " " - prefix := "" - for i := 0; i < level; i++ { - prefix += levelSpace - } - fmt.Fprintf(pw.out, prefix+format, a...) -} - -func (pw *prefixWriter) WriteLine(a ...interface{}) { - fmt.Fprintln(pw.out, a...) -} - -func (pw *prefixWriter) Flush() { - if f, ok := pw.out.(flusher); ok { - f.Flush() - } -} - -func describerMap(clientConfig *rest.Config) (map[schema.GroupKind]printers.Describer, error) { - c, err := clientset.NewForConfig(clientConfig) - if err != nil { - return nil, err - } - externalclient, err := externalclient.NewForConfig(clientConfig) - if err != nil { - return nil, err - } - - m := map[schema.GroupKind]printers.Describer{ - api.Kind("Pod"): &PodDescriber{c}, - api.Kind("ReplicationController"): &ReplicationControllerDescriber{c}, - api.Kind("Secret"): &SecretDescriber{c}, - api.Kind("Service"): &ServiceDescriber{c}, - api.Kind("ServiceAccount"): &ServiceAccountDescriber{c}, - api.Kind("Node"): &NodeDescriber{c}, - api.Kind("LimitRange"): &LimitRangeDescriber{c}, - api.Kind("ResourceQuota"): &ResourceQuotaDescriber{c}, - api.Kind("PersistentVolume"): &PersistentVolumeDescriber{c}, - api.Kind("PersistentVolumeClaim"): &PersistentVolumeClaimDescriber{c}, - api.Kind("Namespace"): &NamespaceDescriber{c}, - api.Kind("Endpoints"): &EndpointsDescriber{c}, - api.Kind("ConfigMap"): &ConfigMapDescriber{c}, - api.Kind("PriorityClass"): &PriorityClassDescriber{c}, - - extensions.Kind("ReplicaSet"): &ReplicaSetDescriber{c}, - extensions.Kind("NetworkPolicy"): &NetworkPolicyDescriber{c}, - extensions.Kind("PodSecurityPolicy"): &PodSecurityPolicyDescriber{c}, - autoscaling.Kind("HorizontalPodAutoscaler"): &HorizontalPodAutoscalerDescriber{c}, - extensions.Kind("DaemonSet"): &DaemonSetDescriber{c}, - extensions.Kind("Deployment"): &DeploymentDescriber{c, externalclient}, - extensions.Kind("Ingress"): &IngressDescriber{c}, - batch.Kind("Job"): &JobDescriber{c}, - batch.Kind("CronJob"): &CronJobDescriber{c, externalclient}, - apps.Kind("StatefulSet"): &StatefulSetDescriber{c}, - apps.Kind("Deployment"): &DeploymentDescriber{c, externalclient}, - apps.Kind("DaemonSet"): &DaemonSetDescriber{c}, - apps.Kind("ReplicaSet"): &ReplicaSetDescriber{c}, - certificates.Kind("CertificateSigningRequest"): &CertificateSigningRequestDescriber{c}, - storage.Kind("StorageClass"): &StorageClassDescriber{c}, - policy.Kind("PodDisruptionBudget"): &PodDisruptionBudgetDescriber{c}, - rbac.Kind("Role"): &RoleDescriber{externalclient}, - rbac.Kind("ClusterRole"): &ClusterRoleDescriber{externalclient}, - rbac.Kind("RoleBinding"): &RoleBindingDescriber{externalclient}, - rbac.Kind("ClusterRoleBinding"): &ClusterRoleBindingDescriber{externalclient}, - networking.Kind("NetworkPolicy"): &NetworkPolicyDescriber{c}, - scheduling.Kind("PriorityClass"): &PriorityClassDescriber{c}, - } - - return m, nil -} - -// DescriberFor returns the default describe functions for each of the standard -// Kubernetes types. -func DescriberFor(kind schema.GroupKind, clientConfig *rest.Config) (printers.Describer, bool) { - describers, err := describerMap(clientConfig) - if err != nil { - glog.V(1).Info(err) - return nil, false - } - - f, ok := describers[kind] - return f, ok -} - -// GenericDescriberFor returns a generic describer for the specified mapping -// that uses only information available from runtime.Unstructured -func GenericDescriberFor(mapping *meta.RESTMapping, clientConfig *rest.Config) (printers.Describer, bool) { - // used to fetch the resource - dynamicClient, err := dynamic.NewForConfig(clientConfig) - if err != nil { - return nil, false - } - - // used to get events for the resource - clientSet, err := clientset.NewForConfig(clientConfig) - if err != nil { - return nil, false - } - eventsClient := clientSet.Core() - - return &genericDescriber{mapping, dynamicClient, eventsClient}, true -} - -type genericDescriber struct { - mapping *meta.RESTMapping - dynamic dynamic.Interface - events coreclient.EventsGetter -} - -func (g *genericDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (output string, err error) { - obj, err := g.dynamic.Resource(g.mapping.Resource).Namespace(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = g.events.Events(namespace).Search(legacyscheme.Scheme, obj) - } - - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", obj.GetName()) - w.Write(LEVEL_0, "Namespace:\t%s\n", obj.GetNamespace()) - printLabelsMultiline(w, "Labels", obj.GetLabels()) - printAnnotationsMultiline(w, "Annotations", obj.GetAnnotations()) - printUnstructuredContent(w, LEVEL_0, obj.UnstructuredContent(), "", ".metadata.name", ".metadata.namespace", ".metadata.labels", ".metadata.annotations") - if events != nil { - DescribeEvents(events, w) - } - return nil - }) -} - -func printUnstructuredContent(w PrefixWriter, level int, content map[string]interface{}, skipPrefix string, skip ...string) { - fields := []string{} - for field := range content { - fields = append(fields, field) - } - sort.Strings(fields) - - for _, field := range fields { - value := content[field] - switch typedValue := value.(type) { - case map[string]interface{}: - skipExpr := fmt.Sprintf("%s.%s", skipPrefix, field) - if slice.ContainsString(skip, skipExpr, nil) { - continue - } - w.Write(level, "%s:\n", smartLabelFor(field)) - printUnstructuredContent(w, level+1, typedValue, skipExpr, skip...) - - case []interface{}: - skipExpr := fmt.Sprintf("%s.%s", skipPrefix, field) - if slice.ContainsString(skip, skipExpr, nil) { - continue - } - w.Write(level, "%s:\n", smartLabelFor(field)) - for _, child := range typedValue { - switch typedChild := child.(type) { - case map[string]interface{}: - printUnstructuredContent(w, level+1, typedChild, skipExpr, skip...) - default: - w.Write(level+1, "%v\n", typedChild) - } - } - - default: - skipExpr := fmt.Sprintf("%s.%s", skipPrefix, field) - if slice.ContainsString(skip, skipExpr, nil) { - continue - } - w.Write(level, "%s:\t%v\n", smartLabelFor(field), typedValue) - } - } -} - -func smartLabelFor(field string) string { - commonAcronyms := []string{"API", "URL", "UID", "OSB", "GUID"} - - parts := camelcase.Split(field) - result := make([]string, 0, len(parts)) - for _, part := range parts { - if part == "_" { - continue - } - - if slice.ContainsString(commonAcronyms, strings.ToUpper(part), nil) { - part = strings.ToUpper(part) - } else { - part = strings.Title(part) - } - result = append(result, part) - } - - return strings.Join(result, " ") -} - -// DefaultObjectDescriber can describe the default Kubernetes objects. -var DefaultObjectDescriber printers.ObjectDescriber - -func init() { - d := &Describers{} - err := d.Add( - describeLimitRange, - describeQuota, - describePod, - describeService, - describeReplicationController, - describeDaemonSet, - describeNode, - describeNamespace, - ) - if err != nil { - glog.Fatalf("Cannot register describers: %v", err) - } - DefaultObjectDescriber = d -} - -// NamespaceDescriber generates information about a namespace -type NamespaceDescriber struct { - clientset.Interface -} - -func (d *NamespaceDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - ns, err := d.Core().Namespaces().Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - resourceQuotaList, err := d.Core().ResourceQuotas(name).List(metav1.ListOptions{}) - if err != nil { - if errors.IsNotFound(err) { - // Server does not support resource quotas. - // Not an error, will not show resource quotas information. - resourceQuotaList = nil - } else { - return "", err - } - } - limitRangeList, err := d.Core().LimitRanges(name).List(metav1.ListOptions{}) - if err != nil { - if errors.IsNotFound(err) { - // Server does not support limit ranges. - // Not an error, will not show limit ranges information. - limitRangeList = nil - } else { - return "", err - } - } - return describeNamespace(ns, resourceQuotaList, limitRangeList) -} - -func describeNamespace(namespace *api.Namespace, resourceQuotaList *api.ResourceQuotaList, limitRangeList *api.LimitRangeList) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", namespace.Name) - printLabelsMultiline(w, "Labels", namespace.Labels) - printAnnotationsMultiline(w, "Annotations", namespace.Annotations) - w.Write(LEVEL_0, "Status:\t%s\n", string(namespace.Status.Phase)) - if resourceQuotaList != nil { - w.Write(LEVEL_0, "\n") - DescribeResourceQuotas(resourceQuotaList, w) - } - if limitRangeList != nil { - w.Write(LEVEL_0, "\n") - DescribeLimitRanges(limitRangeList, w) - } - return nil - }) -} - -func describeLimitRangeSpec(spec api.LimitRangeSpec, prefix string, w PrefixWriter) { - for i := range spec.Limits { - item := spec.Limits[i] - maxResources := item.Max - minResources := item.Min - defaultLimitResources := item.Default - defaultRequestResources := item.DefaultRequest - ratio := item.MaxLimitRequestRatio - - set := map[api.ResourceName]bool{} - for k := range maxResources { - set[k] = true - } - for k := range minResources { - set[k] = true - } - for k := range defaultLimitResources { - set[k] = true - } - for k := range defaultRequestResources { - set[k] = true - } - for k := range ratio { - set[k] = true - } - - for k := range set { - // if no value is set, we output - - maxValue := "-" - minValue := "-" - defaultLimitValue := "-" - defaultRequestValue := "-" - ratioValue := "-" - - maxQuantity, maxQuantityFound := maxResources[k] - if maxQuantityFound { - maxValue = maxQuantity.String() - } - - minQuantity, minQuantityFound := minResources[k] - if minQuantityFound { - minValue = minQuantity.String() - } - - defaultLimitQuantity, defaultLimitQuantityFound := defaultLimitResources[k] - if defaultLimitQuantityFound { - defaultLimitValue = defaultLimitQuantity.String() - } - - defaultRequestQuantity, defaultRequestQuantityFound := defaultRequestResources[k] - if defaultRequestQuantityFound { - defaultRequestValue = defaultRequestQuantity.String() - } - - ratioQuantity, ratioQuantityFound := ratio[k] - if ratioQuantityFound { - ratioValue = ratioQuantity.String() - } - - msg := "%s%s\t%v\t%v\t%v\t%v\t%v\t%v\n" - w.Write(LEVEL_0, msg, prefix, item.Type, k, minValue, maxValue, defaultRequestValue, defaultLimitValue, ratioValue) - } - } -} - -// DescribeLimitRanges merges a set of limit range items into a single tabular description -func DescribeLimitRanges(limitRanges *api.LimitRangeList, w PrefixWriter) { - if len(limitRanges.Items) == 0 { - w.Write(LEVEL_0, "No resource limits.\n") - return - } - w.Write(LEVEL_0, "Resource Limits\n Type\tResource\tMin\tMax\tDefault Request\tDefault Limit\tMax Limit/Request Ratio\n") - w.Write(LEVEL_0, " ----\t--------\t---\t---\t---------------\t-------------\t-----------------------\n") - for _, limitRange := range limitRanges.Items { - describeLimitRangeSpec(limitRange.Spec, " ", w) - } -} - -// DescribeResourceQuotas merges a set of quota items into a single tabular description of all quotas -func DescribeResourceQuotas(quotas *api.ResourceQuotaList, w PrefixWriter) { - if len(quotas.Items) == 0 { - w.Write(LEVEL_0, "No resource quota.\n") - return - } - sort.Sort(SortableResourceQuotas(quotas.Items)) - - w.Write(LEVEL_0, "Resource Quotas") - for _, q := range quotas.Items { - w.Write(LEVEL_0, "\n Name:\t%s\n", q.Name) - if len(q.Spec.Scopes) > 0 { - scopes := make([]string, 0, len(q.Spec.Scopes)) - for _, scope := range q.Spec.Scopes { - scopes = append(scopes, string(scope)) - } - sort.Strings(scopes) - w.Write(LEVEL_0, " Scopes:\t%s\n", strings.Join(scopes, ", ")) - for _, scope := range scopes { - helpText := helpTextForResourceQuotaScope(api.ResourceQuotaScope(scope)) - if len(helpText) > 0 { - w.Write(LEVEL_0, " * %s\n", helpText) - } - } - } - - w.Write(LEVEL_0, " Resource\tUsed\tHard\n") - w.Write(LEVEL_0, " --------\t---\t---\n") - - resources := make([]api.ResourceName, 0, len(q.Status.Hard)) - for resource := range q.Status.Hard { - resources = append(resources, resource) - } - sort.Sort(SortableResourceNames(resources)) - - for _, resource := range resources { - hardQuantity := q.Status.Hard[resource] - usedQuantity := q.Status.Used[resource] - w.Write(LEVEL_0, " %s\t%s\t%s\n", string(resource), usedQuantity.String(), hardQuantity.String()) - } - } -} - -// LimitRangeDescriber generates information about a limit range -type LimitRangeDescriber struct { - clientset.Interface -} - -func (d *LimitRangeDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - lr := d.Core().LimitRanges(namespace) - - limitRange, err := lr.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - return describeLimitRange(limitRange) -} - -func describeLimitRange(limitRange *api.LimitRange) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", limitRange.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", limitRange.Namespace) - w.Write(LEVEL_0, "Type\tResource\tMin\tMax\tDefault Request\tDefault Limit\tMax Limit/Request Ratio\n") - w.Write(LEVEL_0, "----\t--------\t---\t---\t---------------\t-------------\t-----------------------\n") - describeLimitRangeSpec(limitRange.Spec, "", w) - return nil - }) -} - -// ResourceQuotaDescriber generates information about a resource quota -type ResourceQuotaDescriber struct { - clientset.Interface -} - -func (d *ResourceQuotaDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - rq := d.Core().ResourceQuotas(namespace) - - resourceQuota, err := rq.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - return describeQuota(resourceQuota) -} - -func helpTextForResourceQuotaScope(scope api.ResourceQuotaScope) string { - switch scope { - case api.ResourceQuotaScopeTerminating: - return "Matches all pods that have an active deadline. These pods have a limited lifespan on a node before being actively terminated by the system." - case api.ResourceQuotaScopeNotTerminating: - return "Matches all pods that do not have an active deadline. These pods usually include long running pods whose container command is not expected to terminate." - case api.ResourceQuotaScopeBestEffort: - return "Matches all pods that do not have resource requirements set. These pods have a best effort quality of service." - case api.ResourceQuotaScopeNotBestEffort: - return "Matches all pods that have at least one resource requirement set. These pods have a burstable or guaranteed quality of service." - default: - return "" - } -} -func describeQuota(resourceQuota *api.ResourceQuota) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", resourceQuota.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", resourceQuota.Namespace) - if len(resourceQuota.Spec.Scopes) > 0 { - scopes := make([]string, 0, len(resourceQuota.Spec.Scopes)) - for _, scope := range resourceQuota.Spec.Scopes { - scopes = append(scopes, string(scope)) - } - sort.Strings(scopes) - w.Write(LEVEL_0, "Scopes:\t%s\n", strings.Join(scopes, ", ")) - for _, scope := range scopes { - helpText := helpTextForResourceQuotaScope(api.ResourceQuotaScope(scope)) - if len(helpText) > 0 { - w.Write(LEVEL_0, " * %s\n", helpText) - } - } - } - w.Write(LEVEL_0, "Resource\tUsed\tHard\n") - w.Write(LEVEL_0, "--------\t----\t----\n") - - resources := make([]api.ResourceName, 0, len(resourceQuota.Status.Hard)) - for resource := range resourceQuota.Status.Hard { - resources = append(resources, resource) - } - sort.Sort(SortableResourceNames(resources)) - - msg := "%v\t%v\t%v\n" - for i := range resources { - resource := resources[i] - hardQuantity := resourceQuota.Status.Hard[resource] - usedQuantity := resourceQuota.Status.Used[resource] - w.Write(LEVEL_0, msg, resource, usedQuantity.String(), hardQuantity.String()) - } - return nil - }) -} - -// PodDescriber generates information about a pod and the replication controllers that -// create it. -type PodDescriber struct { - clientset.Interface -} - -func (d *PodDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - pod, err := d.Core().Pods(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - if describerSettings.ShowEvents { - eventsInterface := d.Core().Events(namespace) - selector := eventsInterface.GetFieldSelector(&name, &namespace, nil, nil) - options := metav1.ListOptions{FieldSelector: selector.String()} - events, err2 := eventsInterface.List(options) - if describerSettings.ShowEvents && err2 == nil && len(events.Items) > 0 { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Pod '%v': error '%v', but found events.\n", name, err) - DescribeEvents(events, w) - return nil - }) - } - } - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - if ref, err := ref.GetReference(legacyscheme.Scheme, pod); err != nil { - glog.Errorf("Unable to construct reference to '%#v': %v", pod, err) - } else { - ref.Kind = "" - events, _ = d.Core().Events(namespace).Search(legacyscheme.Scheme, ref) - } - } - - return describePod(pod, events) -} - -func describePod(pod *api.Pod, events *api.EventList) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", pod.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", pod.Namespace) - if pod.Spec.Priority != nil { - w.Write(LEVEL_0, "Priority:\t%d\n", *pod.Spec.Priority) - w.Write(LEVEL_0, "PriorityClassName:\t%s\n", stringOrNone(pod.Spec.PriorityClassName)) - } - if pod.Spec.NodeName == "" { - w.Write(LEVEL_0, "Node:\t\n") - } else { - w.Write(LEVEL_0, "Node:\t%s\n", pod.Spec.NodeName+"/"+pod.Status.HostIP) - } - if pod.Status.StartTime != nil { - w.Write(LEVEL_0, "Start Time:\t%s\n", pod.Status.StartTime.Time.Format(time.RFC1123Z)) - } - printLabelsMultiline(w, "Labels", pod.Labels) - printAnnotationsMultiline(w, "Annotations", pod.Annotations) - if pod.DeletionTimestamp != nil { - w.Write(LEVEL_0, "Status:\tTerminating (lasts %s)\n", translateTimestampUntil(*pod.DeletionTimestamp)) - w.Write(LEVEL_0, "Termination Grace Period:\t%ds\n", *pod.DeletionGracePeriodSeconds) - } else { - w.Write(LEVEL_0, "Status:\t%s\n", string(pod.Status.Phase)) - } - if len(pod.Status.Reason) > 0 { - w.Write(LEVEL_0, "Reason:\t%s\n", pod.Status.Reason) - } - if len(pod.Status.Message) > 0 { - w.Write(LEVEL_0, "Message:\t%s\n", pod.Status.Message) - } - w.Write(LEVEL_0, "IP:\t%s\n", pod.Status.PodIP) - if controlledBy := printController(pod); len(controlledBy) > 0 { - w.Write(LEVEL_0, "Controlled By:\t%s\n", controlledBy) - } - if len(pod.Status.NominatedNodeName) > 0 { - w.Write(LEVEL_0, "NominatedNodeName:\t%s\n", pod.Status.NominatedNodeName) - } - - if len(pod.Spec.InitContainers) > 0 { - describeContainers("Init Containers", pod.Spec.InitContainers, pod.Status.InitContainerStatuses, EnvValueRetriever(pod), w, "") - } - describeContainers("Containers", pod.Spec.Containers, pod.Status.ContainerStatuses, EnvValueRetriever(pod), w, "") - if len(pod.Status.Conditions) > 0 { - w.Write(LEVEL_0, "Conditions:\n Type\tStatus\n") - for _, c := range pod.Status.Conditions { - w.Write(LEVEL_1, "%v \t%v \n", - c.Type, - c.Status) - } - } - describeVolumes(pod.Spec.Volumes, w, "") - if pod.Status.QOSClass != "" { - w.Write(LEVEL_0, "QoS Class:\t%s\n", pod.Status.QOSClass) - } else { - w.Write(LEVEL_0, "QoS Class:\t%s\n", qos.GetPodQOS(pod)) - } - printLabelsMultiline(w, "Node-Selectors", pod.Spec.NodeSelector) - printPodTolerationsMultiline(w, "Tolerations", pod.Spec.Tolerations) - if events != nil { - DescribeEvents(events, w) - } - return nil - }) -} - -func printController(controllee metav1.Object) string { - if controllerRef := metav1.GetControllerOf(controllee); controllerRef != nil { - return fmt.Sprintf("%s/%s", controllerRef.Kind, controllerRef.Name) - } - return "" -} - -func describeVolumes(volumes []api.Volume, w PrefixWriter, space string) { - if volumes == nil || len(volumes) == 0 { - w.Write(LEVEL_0, "%sVolumes:\t\n", space) - return - } - w.Write(LEVEL_0, "%sVolumes:\n", space) - for _, volume := range volumes { - nameIndent := "" - if len(space) > 0 { - nameIndent = " " - } - w.Write(LEVEL_1, "%s%v:\n", nameIndent, volume.Name) - switch { - case volume.VolumeSource.HostPath != nil: - printHostPathVolumeSource(volume.VolumeSource.HostPath, w) - case volume.VolumeSource.EmptyDir != nil: - printEmptyDirVolumeSource(volume.VolumeSource.EmptyDir, w) - case volume.VolumeSource.GCEPersistentDisk != nil: - printGCEPersistentDiskVolumeSource(volume.VolumeSource.GCEPersistentDisk, w) - case volume.VolumeSource.AWSElasticBlockStore != nil: - printAWSElasticBlockStoreVolumeSource(volume.VolumeSource.AWSElasticBlockStore, w) - case volume.VolumeSource.GitRepo != nil: - printGitRepoVolumeSource(volume.VolumeSource.GitRepo, w) - case volume.VolumeSource.Secret != nil: - printSecretVolumeSource(volume.VolumeSource.Secret, w) - case volume.VolumeSource.ConfigMap != nil: - printConfigMapVolumeSource(volume.VolumeSource.ConfigMap, w) - case volume.VolumeSource.NFS != nil: - printNFSVolumeSource(volume.VolumeSource.NFS, w) - case volume.VolumeSource.ISCSI != nil: - printISCSIVolumeSource(volume.VolumeSource.ISCSI, w) - case volume.VolumeSource.Glusterfs != nil: - printGlusterfsVolumeSource(volume.VolumeSource.Glusterfs, w) - case volume.VolumeSource.PersistentVolumeClaim != nil: - printPersistentVolumeClaimVolumeSource(volume.VolumeSource.PersistentVolumeClaim, w) - case volume.VolumeSource.RBD != nil: - printRBDVolumeSource(volume.VolumeSource.RBD, w) - case volume.VolumeSource.Quobyte != nil: - printQuobyteVolumeSource(volume.VolumeSource.Quobyte, w) - case volume.VolumeSource.DownwardAPI != nil: - printDownwardAPIVolumeSource(volume.VolumeSource.DownwardAPI, w) - case volume.VolumeSource.AzureDisk != nil: - printAzureDiskVolumeSource(volume.VolumeSource.AzureDisk, w) - case volume.VolumeSource.VsphereVolume != nil: - printVsphereVolumeSource(volume.VolumeSource.VsphereVolume, w) - case volume.VolumeSource.Cinder != nil: - printCinderVolumeSource(volume.VolumeSource.Cinder, w) - case volume.VolumeSource.PhotonPersistentDisk != nil: - printPhotonPersistentDiskVolumeSource(volume.VolumeSource.PhotonPersistentDisk, w) - case volume.VolumeSource.PortworxVolume != nil: - printPortworxVolumeSource(volume.VolumeSource.PortworxVolume, w) - case volume.VolumeSource.ScaleIO != nil: - printScaleIOVolumeSource(volume.VolumeSource.ScaleIO, w) - case volume.VolumeSource.CephFS != nil: - printCephFSVolumeSource(volume.VolumeSource.CephFS, w) - case volume.VolumeSource.StorageOS != nil: - printStorageOSVolumeSource(volume.VolumeSource.StorageOS, w) - case volume.VolumeSource.FC != nil: - printFCVolumeSource(volume.VolumeSource.FC, w) - case volume.VolumeSource.AzureFile != nil: - printAzureFileVolumeSource(volume.VolumeSource.AzureFile, w) - case volume.VolumeSource.FlexVolume != nil: - printFlexVolumeSource(volume.VolumeSource.FlexVolume, w) - case volume.VolumeSource.Flocker != nil: - printFlockerVolumeSource(volume.VolumeSource.Flocker, w) - case volume.VolumeSource.Projected != nil: - printProjectedVolumeSource(volume.VolumeSource.Projected, w) - default: - w.Write(LEVEL_1, "\n") - } - } -} - -func printHostPathVolumeSource(hostPath *api.HostPathVolumeSource, w PrefixWriter) { - hostPathType := "" - if hostPath.Type != nil { - hostPathType = string(*hostPath.Type) - } - w.Write(LEVEL_2, "Type:\tHostPath (bare host directory volume)\n"+ - " Path:\t%v\n"+ - " HostPathType:\t%v\n", - hostPath.Path, hostPathType) -} - -func printEmptyDirVolumeSource(emptyDir *api.EmptyDirVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tEmptyDir (a temporary directory that shares a pod's lifetime)\n"+ - " Medium:\t%v\n", emptyDir.Medium) -} - -func printGCEPersistentDiskVolumeSource(gce *api.GCEPersistentDiskVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tGCEPersistentDisk (a Persistent Disk resource in Google Compute Engine)\n"+ - " PDName:\t%v\n"+ - " FSType:\t%v\n"+ - " Partition:\t%v\n"+ - " ReadOnly:\t%v\n", - gce.PDName, gce.FSType, gce.Partition, gce.ReadOnly) -} - -func printAWSElasticBlockStoreVolumeSource(aws *api.AWSElasticBlockStoreVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tAWSElasticBlockStore (a Persistent Disk resource in AWS)\n"+ - " VolumeID:\t%v\n"+ - " FSType:\t%v\n"+ - " Partition:\t%v\n"+ - " ReadOnly:\t%v\n", - aws.VolumeID, aws.FSType, aws.Partition, aws.ReadOnly) -} - -func printGitRepoVolumeSource(git *api.GitRepoVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tGitRepo (a volume that is pulled from git when the pod is created)\n"+ - " Repository:\t%v\n"+ - " Revision:\t%v\n", - git.Repository, git.Revision) -} - -func printSecretVolumeSource(secret *api.SecretVolumeSource, w PrefixWriter) { - optional := secret.Optional != nil && *secret.Optional - w.Write(LEVEL_2, "Type:\tSecret (a volume populated by a Secret)\n"+ - " SecretName:\t%v\n"+ - " Optional:\t%v\n", - secret.SecretName, optional) -} - -func printConfigMapVolumeSource(configMap *api.ConfigMapVolumeSource, w PrefixWriter) { - optional := configMap.Optional != nil && *configMap.Optional - w.Write(LEVEL_2, "Type:\tConfigMap (a volume populated by a ConfigMap)\n"+ - " Name:\t%v\n"+ - " Optional:\t%v\n", - configMap.Name, optional) -} - -func printProjectedVolumeSource(projected *api.ProjectedVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tProjected (a volume that contains injected data from multiple sources)\n") - for _, source := range projected.Sources { - if source.Secret != nil { - w.Write(LEVEL_2, "SecretName:\t%v\n"+ - " SecretOptionalName:\t%v\n", - source.Secret.Name, source.Secret.Optional) - } else if source.DownwardAPI != nil { - w.Write(LEVEL_2, "DownwardAPI:\ttrue\n") - } else if source.ConfigMap != nil { - w.Write(LEVEL_2, "ConfigMapName:\t%v\n"+ - " ConfigMapOptional:\t%v\n", - source.ConfigMap.Name, source.ConfigMap.Optional) - } else if source.ServiceAccountToken != nil { - w.Write(LEVEL_2, "TokenExpirationSeconds:\t%v\n", - source.ServiceAccountToken.ExpirationSeconds) - } - } -} - -func printNFSVolumeSource(nfs *api.NFSVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tNFS (an NFS mount that lasts the lifetime of a pod)\n"+ - " Server:\t%v\n"+ - " Path:\t%v\n"+ - " ReadOnly:\t%v\n", - nfs.Server, nfs.Path, nfs.ReadOnly) -} - -func printQuobyteVolumeSource(quobyte *api.QuobyteVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tQuobyte (a Quobyte mount on the host that shares a pod's lifetime)\n"+ - " Registry:\t%v\n"+ - " Volume:\t%v\n"+ - " ReadOnly:\t%v\n", - quobyte.Registry, quobyte.Volume, quobyte.ReadOnly) -} - -func printPortworxVolumeSource(pwxVolume *api.PortworxVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tPortworxVolume (a Portworx Volume resource)\n"+ - " VolumeID:\t%v\n", - pwxVolume.VolumeID) -} - -func printISCSIVolumeSource(iscsi *api.ISCSIVolumeSource, w PrefixWriter) { - initiator := "" - if iscsi.InitiatorName != nil { - initiator = *iscsi.InitiatorName - } - w.Write(LEVEL_2, "Type:\tISCSI (an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod)\n"+ - " TargetPortal:\t%v\n"+ - " IQN:\t%v\n"+ - " Lun:\t%v\n"+ - " ISCSIInterface\t%v\n"+ - " FSType:\t%v\n"+ - " ReadOnly:\t%v\n"+ - " Portals:\t%v\n"+ - " DiscoveryCHAPAuth:\t%v\n"+ - " SessionCHAPAuth:\t%v\n"+ - " SecretRef:\t%v\n"+ - " InitiatorName:\t%v\n", - iscsi.TargetPortal, iscsi.IQN, iscsi.Lun, iscsi.ISCSIInterface, iscsi.FSType, iscsi.ReadOnly, iscsi.Portals, iscsi.DiscoveryCHAPAuth, iscsi.SessionCHAPAuth, iscsi.SecretRef, initiator) -} - -func printISCSIPersistentVolumeSource(iscsi *api.ISCSIPersistentVolumeSource, w PrefixWriter) { - initiatorName := "" - if iscsi.InitiatorName != nil { - initiatorName = *iscsi.InitiatorName - } - w.Write(LEVEL_2, "Type:\tISCSI (an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod)\n"+ - " TargetPortal:\t%v\n"+ - " IQN:\t%v\n"+ - " Lun:\t%v\n"+ - " ISCSIInterface\t%v\n"+ - " FSType:\t%v\n"+ - " ReadOnly:\t%v\n"+ - " Portals:\t%v\n"+ - " DiscoveryCHAPAuth:\t%v\n"+ - " SessionCHAPAuth:\t%v\n"+ - " SecretRef:\t%v\n"+ - " InitiatorName:\t%v\n", - iscsi.TargetPortal, iscsi.IQN, iscsi.Lun, iscsi.ISCSIInterface, iscsi.FSType, iscsi.ReadOnly, iscsi.Portals, iscsi.DiscoveryCHAPAuth, iscsi.SessionCHAPAuth, iscsi.SecretRef, initiatorName) -} - -func printGlusterfsVolumeSource(glusterfs *api.GlusterfsVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tGlusterfs (a Glusterfs mount on the host that shares a pod's lifetime)\n"+ - " EndpointsName:\t%v\n"+ - " Path:\t%v\n"+ - " ReadOnly:\t%v\n", - glusterfs.EndpointsName, glusterfs.Path, glusterfs.ReadOnly) -} - -func printPersistentVolumeClaimVolumeSource(claim *api.PersistentVolumeClaimVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tPersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)\n"+ - " ClaimName:\t%v\n"+ - " ReadOnly:\t%v\n", - claim.ClaimName, claim.ReadOnly) -} - -func printRBDVolumeSource(rbd *api.RBDVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tRBD (a Rados Block Device mount on the host that shares a pod's lifetime)\n"+ - " CephMonitors:\t%v\n"+ - " RBDImage:\t%v\n"+ - " FSType:\t%v\n"+ - " RBDPool:\t%v\n"+ - " RadosUser:\t%v\n"+ - " Keyring:\t%v\n"+ - " SecretRef:\t%v\n"+ - " ReadOnly:\t%v\n", - rbd.CephMonitors, rbd.RBDImage, rbd.FSType, rbd.RBDPool, rbd.RadosUser, rbd.Keyring, rbd.SecretRef, rbd.ReadOnly) -} - -func printRBDPersistentVolumeSource(rbd *api.RBDPersistentVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tRBD (a Rados Block Device mount on the host that shares a pod's lifetime)\n"+ - " CephMonitors:\t%v\n"+ - " RBDImage:\t%v\n"+ - " FSType:\t%v\n"+ - " RBDPool:\t%v\n"+ - " RadosUser:\t%v\n"+ - " Keyring:\t%v\n"+ - " SecretRef:\t%v\n"+ - " ReadOnly:\t%v\n", - rbd.CephMonitors, rbd.RBDImage, rbd.FSType, rbd.RBDPool, rbd.RadosUser, rbd.Keyring, rbd.SecretRef, rbd.ReadOnly) -} - -func printDownwardAPIVolumeSource(d *api.DownwardAPIVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tDownwardAPI (a volume populated by information about the pod)\n Items:\n") - for _, mapping := range d.Items { - if mapping.FieldRef != nil { - w.Write(LEVEL_3, "%v -> %v\n", mapping.FieldRef.FieldPath, mapping.Path) - } - if mapping.ResourceFieldRef != nil { - w.Write(LEVEL_3, "%v -> %v\n", mapping.ResourceFieldRef.Resource, mapping.Path) - } - } -} - -func printAzureDiskVolumeSource(d *api.AzureDiskVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tAzureDisk (an Azure Data Disk mount on the host and bind mount to the pod)\n"+ - " DiskName:\t%v\n"+ - " DiskURI:\t%v\n"+ - " Kind: \t%v\n"+ - " FSType:\t%v\n"+ - " CachingMode:\t%v\n"+ - " ReadOnly:\t%v\n", - d.DiskName, d.DataDiskURI, *d.Kind, *d.FSType, *d.CachingMode, *d.ReadOnly) -} - -func printVsphereVolumeSource(vsphere *api.VsphereVirtualDiskVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tvSphereVolume (a Persistent Disk resource in vSphere)\n"+ - " VolumePath:\t%v\n"+ - " FSType:\t%v\n"+ - " StoragePolicyName:\t%v\n", - vsphere.VolumePath, vsphere.FSType, vsphere.StoragePolicyName) -} - -func printPhotonPersistentDiskVolumeSource(photon *api.PhotonPersistentDiskVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tPhotonPersistentDisk (a Persistent Disk resource in photon platform)\n"+ - " PdID:\t%v\n"+ - " FSType:\t%v\n", - photon.PdID, photon.FSType) -} - -func printCinderVolumeSource(cinder *api.CinderVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tCinder (a Persistent Disk resource in OpenStack)\n"+ - " VolumeID:\t%v\n"+ - " FSType:\t%v\n"+ - " ReadOnly:\t%v\n", - " SecretRef:\t%v\n"+ - cinder.VolumeID, cinder.FSType, cinder.ReadOnly, cinder.SecretRef) -} - -func printCinderPersistentVolumeSource(cinder *api.CinderPersistentVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tCinder (a Persistent Disk resource in OpenStack)\n"+ - " VolumeID:\t%v\n"+ - " FSType:\t%v\n"+ - " ReadOnly:\t%v\n", - " SecretRef:\t%v\n"+ - cinder.VolumeID, cinder.SecretRef, cinder.FSType, cinder.ReadOnly, cinder.SecretRef) -} - -func printScaleIOVolumeSource(sio *api.ScaleIOVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tScaleIO (a persistent volume backed by a block device in ScaleIO)\n"+ - " Gateway:\t%v\n"+ - " System:\t%v\n"+ - " Protection Domain:\t%v\n"+ - " Storage Pool:\t%v\n"+ - " Storage Mode:\t%v\n"+ - " VolumeName:\t%v\n"+ - " FSType:\t%v\n"+ - " ReadOnly:\t%v\n", - sio.Gateway, sio.System, sio.ProtectionDomain, sio.StoragePool, sio.StorageMode, sio.VolumeName, sio.FSType, sio.ReadOnly) -} - -func printScaleIOPersistentVolumeSource(sio *api.ScaleIOPersistentVolumeSource, w PrefixWriter) { - var secretNS, secretName string - if sio.SecretRef != nil { - secretName = sio.SecretRef.Name - secretNS = sio.SecretRef.Namespace - } - w.Write(LEVEL_2, "Type:\tScaleIO (a persistent volume backed by a block device in ScaleIO)\n"+ - " Gateway:\t%v\n"+ - " System:\t%v\n"+ - " Protection Domain:\t%v\n"+ - " Storage Pool:\t%v\n"+ - " Storage Mode:\t%v\n"+ - " VolumeName:\t%v\n"+ - " SecretName:\t%v\n"+ - " SecretNamespace:\t%v\n"+ - " FSType:\t%v\n"+ - " ReadOnly:\t%v\n", - sio.Gateway, sio.System, sio.ProtectionDomain, sio.StoragePool, sio.StorageMode, sio.VolumeName, secretName, secretNS, sio.FSType, sio.ReadOnly) -} - -func printLocalVolumeSource(ls *api.LocalVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tLocalVolume (a persistent volume backed by local storage on a node)\n"+ - " Path:\t%v\n", - ls.Path) -} - -func printCephFSVolumeSource(cephfs *api.CephFSVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tCephFS (a CephFS mount on the host that shares a pod's lifetime)\n"+ - " Monitors:\t%v\n"+ - " Path:\t%v\n"+ - " User:\t%v\n"+ - " SecretFile:\t%v\n"+ - " SecretRef:\t%v\n"+ - " ReadOnly:\t%v\n", - cephfs.Monitors, cephfs.Path, cephfs.User, cephfs.SecretFile, cephfs.SecretRef, cephfs.ReadOnly) -} - -func printCephFSPersistentVolumeSource(cephfs *api.CephFSPersistentVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tCephFS (a CephFS mount on the host that shares a pod's lifetime)\n"+ - " Monitors:\t%v\n"+ - " Path:\t%v\n"+ - " User:\t%v\n"+ - " SecretFile:\t%v\n"+ - " SecretRef:\t%v\n"+ - " ReadOnly:\t%v\n", - cephfs.Monitors, cephfs.Path, cephfs.User, cephfs.SecretFile, cephfs.SecretRef, cephfs.ReadOnly) -} - -func printStorageOSVolumeSource(storageos *api.StorageOSVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tStorageOS (a StorageOS Persistent Disk resource)\n"+ - " VolumeName:\t%v\n"+ - " VolumeNamespace:\t%v\n"+ - " FSType:\t%v\n"+ - " ReadOnly:\t%v\n", - storageos.VolumeName, storageos.VolumeNamespace, storageos.FSType, storageos.ReadOnly) -} - -func printStorageOSPersistentVolumeSource(storageos *api.StorageOSPersistentVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tStorageOS (a StorageOS Persistent Disk resource)\n"+ - " VolumeName:\t%v\n"+ - " VolumeNamespace:\t%v\n"+ - " FSType:\t%v\n"+ - " ReadOnly:\t%v\n", - storageos.VolumeName, storageos.VolumeNamespace, storageos.FSType, storageos.ReadOnly) -} - -func printFCVolumeSource(fc *api.FCVolumeSource, w PrefixWriter) { - lun := "" - if fc.Lun != nil { - lun = strconv.Itoa(int(*fc.Lun)) - } - w.Write(LEVEL_2, "Type:\tFC (a Fibre Channel disk)\n"+ - " TargetWWNs:\t%v\n"+ - " LUN:\t%v\n"+ - " FSType:\t%v\n"+ - " ReadOnly:\t%v\n", - strings.Join(fc.TargetWWNs, ", "), lun, fc.FSType, fc.ReadOnly) -} - -func printAzureFileVolumeSource(azureFile *api.AzureFileVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tAzureFile (an Azure File Service mount on the host and bind mount to the pod)\n"+ - " SecretName:\t%v\n"+ - " ShareName:\t%v\n"+ - " ReadOnly:\t%v\n", - azureFile.SecretName, azureFile.ShareName, azureFile.ReadOnly) -} - -func printAzureFilePersistentVolumeSource(azureFile *api.AzureFilePersistentVolumeSource, w PrefixWriter) { - ns := "" - if azureFile.SecretNamespace != nil { - ns = *azureFile.SecretNamespace - } - w.Write(LEVEL_2, "Type:\tAzureFile (an Azure File Service mount on the host and bind mount to the pod)\n"+ - " SecretName:\t%v\n"+ - " SecretNamespace:\t%v\n"+ - " ShareName:\t%v\n"+ - " ReadOnly:\t%v\n", - azureFile.SecretName, ns, azureFile.ShareName, azureFile.ReadOnly) -} - -func printFlexPersistentVolumeSource(flex *api.FlexPersistentVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tFlexVolume (a generic volume resource that is provisioned/attached using an exec based plugin)\n"+ - " Driver:\t%v\n"+ - " FSType:\t%v\n"+ - " SecretRef:\t%v\n"+ - " ReadOnly:\t%v\n"+ - " Options:\t%v\n", - flex.Driver, flex.FSType, flex.SecretRef, flex.ReadOnly, flex.Options) -} - -func printFlexVolumeSource(flex *api.FlexVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tFlexVolume (a generic volume resource that is provisioned/attached using an exec based plugin)\n"+ - " Driver:\t%v\n"+ - " FSType:\t%v\n"+ - " SecretRef:\t%v\n"+ - " ReadOnly:\t%v\n"+ - " Options:\t%v\n", - flex.Driver, flex.FSType, flex.SecretRef, flex.ReadOnly, flex.Options) -} - -func printFlockerVolumeSource(flocker *api.FlockerVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tFlocker (a Flocker volume mounted by the Flocker agent)\n"+ - " DatasetName:\t%v\n"+ - " DatasetUUID:\t%v\n", - flocker.DatasetName, flocker.DatasetUUID) -} - -func printCSIPersistentVolumeSource(csi *api.CSIPersistentVolumeSource, w PrefixWriter) { - w.Write(LEVEL_2, "Type:\tCSI (a Container Storage Interface (CSI) volume source)\n"+ - " Driver:\t%v\n"+ - " VolumeHandle:\t%v\n"+ - " ReadOnly:\t%v\n", - csi.Driver, csi.VolumeHandle, csi.ReadOnly) - printCSIPersistentVolumeAttributesMultiline(w, "VolumeAttributes", csi.VolumeAttributes) -} - -func printCSIPersistentVolumeAttributesMultiline(w PrefixWriter, title string, annotations map[string]string) { - printCSIPersistentVolumeAttributesMultilineIndent(w, "", title, "\t", annotations, sets.NewString()) -} - -func printCSIPersistentVolumeAttributesMultilineIndent(w PrefixWriter, initialIndent, title, innerIndent string, attributes map[string]string, skip sets.String) { - w.Write(LEVEL_2, "%s%s:%s", initialIndent, title, innerIndent) - - if len(attributes) == 0 { - w.WriteLine("") - return - } - - // to print labels in the sorted order - keys := make([]string, 0, len(attributes)) - for key := range attributes { - if skip.Has(key) { - continue - } - keys = append(keys, key) - } - if len(attributes) == 0 { - w.WriteLine("") - return - } - sort.Strings(keys) - - for i, key := range keys { - if i != 0 { - w.Write(LEVEL_2, initialIndent) - w.Write(LEVEL_2, innerIndent) - } - line := fmt.Sprintf("%s=%s", key, attributes[key]) - if len(line) > maxAnnotationLen { - w.Write(LEVEL_2, "%s...\n", line[:maxAnnotationLen]) - } else { - w.Write(LEVEL_2, "%s\n", line) - } - i++ - } -} - -type PersistentVolumeDescriber struct { - clientset.Interface -} - -func (d *PersistentVolumeDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - c := d.Core().PersistentVolumes() - - pv, err := c.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = d.Core().Events(namespace).Search(legacyscheme.Scheme, pv) - } - - return describePersistentVolume(pv, events) -} - -func printVolumeNodeAffinity(w PrefixWriter, affinity *api.VolumeNodeAffinity) { - w.Write(LEVEL_0, "Node Affinity:\t") - if affinity == nil || affinity.Required == nil { - w.WriteLine("") - return - } - w.WriteLine("") - - if affinity.Required != nil { - w.Write(LEVEL_1, "Required Terms:\t") - if len(affinity.Required.NodeSelectorTerms) == 0 { - w.WriteLine("") - } else { - w.WriteLine("") - for i, term := range affinity.Required.NodeSelectorTerms { - printNodeSelectorTermsMultilineWithIndent(w, LEVEL_2, fmt.Sprintf("Term %v", i), "\t", term.MatchExpressions) - } - } - } -} - -// printLabelsMultiline prints multiple labels with a user-defined alignment. -func printNodeSelectorTermsMultilineWithIndent(w PrefixWriter, indentLevel int, title, innerIndent string, reqs []api.NodeSelectorRequirement) { - w.Write(indentLevel, "%s:%s", title, innerIndent) - - if len(reqs) == 0 { - w.WriteLine("") - return - } - - for i, req := range reqs { - if i != 0 { - w.Write(indentLevel, "%s", innerIndent) - } - exprStr := fmt.Sprintf("%s %s", req.Key, strings.ToLower(string(req.Operator))) - if len(req.Values) > 0 { - exprStr = fmt.Sprintf("%s [%s]", exprStr, strings.Join(req.Values, ", ")) - } - w.Write(LEVEL_0, "%s\n", exprStr) - } -} - -func describePersistentVolume(pv *api.PersistentVolume, events *api.EventList) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", pv.Name) - printLabelsMultiline(w, "Labels", pv.ObjectMeta.Labels) - printAnnotationsMultiline(w, "Annotations", pv.ObjectMeta.Annotations) - w.Write(LEVEL_0, "Finalizers:\t%v\n", pv.ObjectMeta.Finalizers) - w.Write(LEVEL_0, "StorageClass:\t%s\n", helper.GetPersistentVolumeClass(pv)) - if pv.ObjectMeta.DeletionTimestamp != nil { - w.Write(LEVEL_0, "Status:\tTerminating (lasts %s)\n", translateTimestampUntil(*pv.ObjectMeta.DeletionTimestamp)) - } else { - w.Write(LEVEL_0, "Status:\t%v\n", pv.Status.Phase) - } - if pv.Spec.ClaimRef != nil { - w.Write(LEVEL_0, "Claim:\t%s\n", pv.Spec.ClaimRef.Namespace+"/"+pv.Spec.ClaimRef.Name) - } else { - w.Write(LEVEL_0, "Claim:\t%s\n", "") - } - w.Write(LEVEL_0, "Reclaim Policy:\t%v\n", pv.Spec.PersistentVolumeReclaimPolicy) - w.Write(LEVEL_0, "Access Modes:\t%s\n", helper.GetAccessModesAsString(pv.Spec.AccessModes)) - if pv.Spec.VolumeMode != nil { - w.Write(LEVEL_0, "VolumeMode:\t%v\n", *pv.Spec.VolumeMode) - } - storage := pv.Spec.Capacity[api.ResourceStorage] - w.Write(LEVEL_0, "Capacity:\t%s\n", storage.String()) - printVolumeNodeAffinity(w, pv.Spec.NodeAffinity) - w.Write(LEVEL_0, "Message:\t%s\n", pv.Status.Message) - w.Write(LEVEL_0, "Source:\n") - - switch { - case pv.Spec.HostPath != nil: - printHostPathVolumeSource(pv.Spec.HostPath, w) - case pv.Spec.GCEPersistentDisk != nil: - printGCEPersistentDiskVolumeSource(pv.Spec.GCEPersistentDisk, w) - case pv.Spec.AWSElasticBlockStore != nil: - printAWSElasticBlockStoreVolumeSource(pv.Spec.AWSElasticBlockStore, w) - case pv.Spec.NFS != nil: - printNFSVolumeSource(pv.Spec.NFS, w) - case pv.Spec.ISCSI != nil: - printISCSIPersistentVolumeSource(pv.Spec.ISCSI, w) - case pv.Spec.Glusterfs != nil: - printGlusterfsVolumeSource(pv.Spec.Glusterfs, w) - case pv.Spec.RBD != nil: - printRBDPersistentVolumeSource(pv.Spec.RBD, w) - case pv.Spec.Quobyte != nil: - printQuobyteVolumeSource(pv.Spec.Quobyte, w) - case pv.Spec.VsphereVolume != nil: - printVsphereVolumeSource(pv.Spec.VsphereVolume, w) - case pv.Spec.Cinder != nil: - printCinderPersistentVolumeSource(pv.Spec.Cinder, w) - case pv.Spec.AzureDisk != nil: - printAzureDiskVolumeSource(pv.Spec.AzureDisk, w) - case pv.Spec.PhotonPersistentDisk != nil: - printPhotonPersistentDiskVolumeSource(pv.Spec.PhotonPersistentDisk, w) - case pv.Spec.PortworxVolume != nil: - printPortworxVolumeSource(pv.Spec.PortworxVolume, w) - case pv.Spec.ScaleIO != nil: - printScaleIOPersistentVolumeSource(pv.Spec.ScaleIO, w) - case pv.Spec.Local != nil: - printLocalVolumeSource(pv.Spec.Local, w) - case pv.Spec.CephFS != nil: - printCephFSPersistentVolumeSource(pv.Spec.CephFS, w) - case pv.Spec.StorageOS != nil: - printStorageOSPersistentVolumeSource(pv.Spec.StorageOS, w) - case pv.Spec.FC != nil: - printFCVolumeSource(pv.Spec.FC, w) - case pv.Spec.AzureFile != nil: - printAzureFilePersistentVolumeSource(pv.Spec.AzureFile, w) - case pv.Spec.FlexVolume != nil: - printFlexPersistentVolumeSource(pv.Spec.FlexVolume, w) - case pv.Spec.Flocker != nil: - printFlockerVolumeSource(pv.Spec.Flocker, w) - case pv.Spec.CSI != nil: - printCSIPersistentVolumeSource(pv.Spec.CSI, w) - default: - w.Write(LEVEL_1, "\n") - } - - if events != nil { - DescribeEvents(events, w) - } - - return nil - }) -} - -type PersistentVolumeClaimDescriber struct { - clientset.Interface -} - -func (d *PersistentVolumeClaimDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - c := d.Core().PersistentVolumeClaims(namespace) - - pvc, err := c.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - pc := d.Core().Pods(namespace) - - mountPods, err := getMountPods(pc, pvc.Name) - if err != nil { - return "", err - } - - events, _ := d.Core().Events(namespace).Search(legacyscheme.Scheme, pvc) - - return describePersistentVolumeClaim(pvc, events, mountPods) -} - -func getMountPods(c coreclient.PodInterface, pvcName string) ([]api.Pod, error) { - nsPods, err := c.List(metav1.ListOptions{}) - if err != nil { - return []api.Pod{}, err - } - - var pods []api.Pod - - for _, pod := range nsPods.Items { - pvcs := getPvcs(pod.Spec.Volumes) - - for _, pvc := range pvcs { - if pvc.PersistentVolumeClaim.ClaimName == pvcName { - pods = append(pods, pod) - } - } - } - - return pods, nil -} - -func getPvcs(volumes []api.Volume) []api.Volume { - var pvcs []api.Volume - - for _, volume := range volumes { - if volume.VolumeSource.PersistentVolumeClaim != nil { - pvcs = append(pvcs, volume) - } - } - - return pvcs -} - -func describePersistentVolumeClaim(pvc *api.PersistentVolumeClaim, events *api.EventList, mountPods []api.Pod) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", pvc.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", pvc.Namespace) - w.Write(LEVEL_0, "StorageClass:\t%s\n", helper.GetPersistentVolumeClaimClass(pvc)) - if pvc.ObjectMeta.DeletionTimestamp != nil { - w.Write(LEVEL_0, "Status:\tTerminating (lasts %s)\n", translateTimestampUntil(*pvc.ObjectMeta.DeletionTimestamp)) - } else { - w.Write(LEVEL_0, "Status:\t%v\n", pvc.Status.Phase) - } - w.Write(LEVEL_0, "Volume:\t%s\n", pvc.Spec.VolumeName) - printLabelsMultiline(w, "Labels", pvc.Labels) - printAnnotationsMultiline(w, "Annotations", pvc.Annotations) - w.Write(LEVEL_0, "Finalizers:\t%v\n", pvc.ObjectMeta.Finalizers) - storage := pvc.Spec.Resources.Requests[api.ResourceStorage] - capacity := "" - accessModes := "" - if pvc.Spec.VolumeName != "" { - accessModes = helper.GetAccessModesAsString(pvc.Status.AccessModes) - storage = pvc.Status.Capacity[api.ResourceStorage] - capacity = storage.String() - } - w.Write(LEVEL_0, "Capacity:\t%s\n", capacity) - w.Write(LEVEL_0, "Access Modes:\t%s\n", accessModes) - if pvc.Spec.VolumeMode != nil { - w.Write(LEVEL_0, "VolumeMode:\t%v\n", *pvc.Spec.VolumeMode) - } - if len(pvc.Status.Conditions) > 0 { - w.Write(LEVEL_0, "Conditions:\n") - w.Write(LEVEL_1, "Type\tStatus\tLastProbeTime\tLastTransitionTime\tReason\tMessage\n") - w.Write(LEVEL_1, "----\t------\t-----------------\t------------------\t------\t-------\n") - for _, c := range pvc.Status.Conditions { - w.Write(LEVEL_1, "%v \t%v \t%s \t%s \t%v \t%v\n", - c.Type, - c.Status, - c.LastProbeTime.Time.Format(time.RFC1123Z), - c.LastTransitionTime.Time.Format(time.RFC1123Z), - c.Reason, - c.Message) - } - } - if events != nil { - DescribeEvents(events, w) - } - - printPodsMultiline(w, "Mounted By", mountPods) - - return nil - }) -} - -func describeContainers(label string, containers []api.Container, containerStatuses []api.ContainerStatus, - resolverFn EnvVarResolverFunc, w PrefixWriter, space string) { - statuses := map[string]api.ContainerStatus{} - for _, status := range containerStatuses { - statuses[status.Name] = status - } - - describeContainersLabel(containers, label, space, w) - - for _, container := range containers { - status, ok := statuses[container.Name] - describeContainerBasicInfo(container, status, ok, space, w) - describeContainerCommand(container, w) - if ok { - describeContainerState(status, w) - } - describeContainerResource(container, w) - describeContainerProbe(container, w) - if len(container.EnvFrom) > 0 { - describeContainerEnvFrom(container, resolverFn, w) - } - describeContainerEnvVars(container, resolverFn, w) - describeContainerVolumes(container, w) - } -} - -func describeContainersLabel(containers []api.Container, label, space string, w PrefixWriter) { - none := "" - if len(containers) == 0 { - none = " " - } - w.Write(LEVEL_0, "%s%s:%s\n", space, label, none) -} - -func describeContainerBasicInfo(container api.Container, status api.ContainerStatus, ok bool, space string, w PrefixWriter) { - nameIndent := "" - if len(space) > 0 { - nameIndent = " " - } - w.Write(LEVEL_1, "%s%v:\n", nameIndent, container.Name) - if ok { - w.Write(LEVEL_2, "Container ID:\t%s\n", status.ContainerID) - } - w.Write(LEVEL_2, "Image:\t%s\n", container.Image) - if ok { - w.Write(LEVEL_2, "Image ID:\t%s\n", status.ImageID) - } - portString := describeContainerPorts(container.Ports) - if strings.Contains(portString, ",") { - w.Write(LEVEL_2, "Ports:\t%s\n", portString) - } else { - w.Write(LEVEL_2, "Port:\t%s\n", stringOrNone(portString)) - } - hostPortString := describeContainerHostPorts(container.Ports) - if strings.Contains(hostPortString, ",") { - w.Write(LEVEL_2, "Host Ports:\t%s\n", hostPortString) - } else { - w.Write(LEVEL_2, "Host Port:\t%s\n", stringOrNone(hostPortString)) - } -} - -func describeContainerPorts(cPorts []api.ContainerPort) string { - ports := make([]string, 0, len(cPorts)) - for _, cPort := range cPorts { - ports = append(ports, fmt.Sprintf("%d/%s", cPort.ContainerPort, cPort.Protocol)) - } - return strings.Join(ports, ", ") -} - -func describeContainerHostPorts(cPorts []api.ContainerPort) string { - ports := make([]string, 0, len(cPorts)) - for _, cPort := range cPorts { - ports = append(ports, fmt.Sprintf("%d/%s", cPort.HostPort, cPort.Protocol)) - } - return strings.Join(ports, ", ") -} - -func describeContainerCommand(container api.Container, w PrefixWriter) { - if len(container.Command) > 0 { - w.Write(LEVEL_2, "Command:\n") - for _, c := range container.Command { - for _, s := range strings.Split(c, "\n") { - w.Write(LEVEL_3, "%s\n", s) - } - } - } - if len(container.Args) > 0 { - w.Write(LEVEL_2, "Args:\n") - for _, arg := range container.Args { - for _, s := range strings.Split(arg, "\n") { - w.Write(LEVEL_3, "%s\n", s) - } - } - } -} - -func describeContainerResource(container api.Container, w PrefixWriter) { - resources := container.Resources - if len(resources.Limits) > 0 { - w.Write(LEVEL_2, "Limits:\n") - } - for _, name := range SortedResourceNames(resources.Limits) { - quantity := resources.Limits[name] - w.Write(LEVEL_3, "%s:\t%s\n", name, quantity.String()) - } - - if len(resources.Requests) > 0 { - w.Write(LEVEL_2, "Requests:\n") - } - for _, name := range SortedResourceNames(resources.Requests) { - quantity := resources.Requests[name] - w.Write(LEVEL_3, "%s:\t%s\n", name, quantity.String()) - } -} - -func describeContainerState(status api.ContainerStatus, w PrefixWriter) { - describeStatus("State", status.State, w) - if status.LastTerminationState.Terminated != nil { - describeStatus("Last State", status.LastTerminationState, w) - } - w.Write(LEVEL_2, "Ready:\t%v\n", printBool(status.Ready)) - w.Write(LEVEL_2, "Restart Count:\t%d\n", status.RestartCount) -} - -func describeContainerProbe(container api.Container, w PrefixWriter) { - if container.LivenessProbe != nil { - probe := DescribeProbe(container.LivenessProbe) - w.Write(LEVEL_2, "Liveness:\t%s\n", probe) - } - if container.ReadinessProbe != nil { - probe := DescribeProbe(container.ReadinessProbe) - w.Write(LEVEL_2, "Readiness:\t%s\n", probe) - } -} - -func describeContainerVolumes(container api.Container, w PrefixWriter) { - // Show volumeMounts - none := "" - if len(container.VolumeMounts) == 0 { - none = "\t" - } - w.Write(LEVEL_2, "Mounts:%s\n", none) - sort.Sort(SortableVolumeMounts(container.VolumeMounts)) - for _, mount := range container.VolumeMounts { - flags := []string{} - switch { - case mount.ReadOnly: - flags = append(flags, "ro") - case !mount.ReadOnly: - flags = append(flags, "rw") - case len(mount.SubPath) > 0: - flags = append(flags, fmt.Sprintf("path=%q", mount.SubPath)) - } - w.Write(LEVEL_3, "%s from %s (%s)\n", mount.MountPath, mount.Name, strings.Join(flags, ",")) - } - // Show volumeDevices if exists - if len(container.VolumeDevices) > 0 { - w.Write(LEVEL_2, "Devices:%s\n", none) - sort.Sort(SortableVolumeDevices(container.VolumeDevices)) - for _, device := range container.VolumeDevices { - w.Write(LEVEL_3, "%s from %s\n", device.DevicePath, device.Name) - } - } -} - -func describeContainerEnvVars(container api.Container, resolverFn EnvVarResolverFunc, w PrefixWriter) { - none := "" - if len(container.Env) == 0 { - none = "\t" - } - w.Write(LEVEL_2, "Environment:%s\n", none) - - for _, e := range container.Env { - if e.ValueFrom == nil { - for i, s := range strings.Split(e.Value, "\n") { - if i == 0 { - w.Write(LEVEL_3, "%s:\t%s\n", e.Name, s) - } else { - w.Write(LEVEL_3, "\t%s\n", s) - } - } - continue - } - - switch { - case e.ValueFrom.FieldRef != nil: - var valueFrom string - if resolverFn != nil { - valueFrom = resolverFn(e) - } - w.Write(LEVEL_3, "%s:\t%s (%s:%s)\n", e.Name, valueFrom, e.ValueFrom.FieldRef.APIVersion, e.ValueFrom.FieldRef.FieldPath) - case e.ValueFrom.ResourceFieldRef != nil: - valueFrom, err := resourcehelper.ExtractContainerResourceValue(e.ValueFrom.ResourceFieldRef, &container) - if err != nil { - valueFrom = "" - } - resource := e.ValueFrom.ResourceFieldRef.Resource - if valueFrom == "0" && (resource == "limits.cpu" || resource == "limits.memory") { - valueFrom = "node allocatable" - } - w.Write(LEVEL_3, "%s:\t%s (%s)\n", e.Name, valueFrom, resource) - case e.ValueFrom.SecretKeyRef != nil: - optional := e.ValueFrom.SecretKeyRef.Optional != nil && *e.ValueFrom.SecretKeyRef.Optional - w.Write(LEVEL_3, "%s:\t\tOptional: %t\n", e.Name, e.ValueFrom.SecretKeyRef.Key, e.ValueFrom.SecretKeyRef.Name, optional) - case e.ValueFrom.ConfigMapKeyRef != nil: - optional := e.ValueFrom.ConfigMapKeyRef.Optional != nil && *e.ValueFrom.ConfigMapKeyRef.Optional - w.Write(LEVEL_3, "%s:\t\tOptional: %t\n", e.Name, e.ValueFrom.ConfigMapKeyRef.Key, e.ValueFrom.ConfigMapKeyRef.Name, optional) - } - } -} - -func describeContainerEnvFrom(container api.Container, resolverFn EnvVarResolverFunc, w PrefixWriter) { - none := "" - if len(container.EnvFrom) == 0 { - none = "\t" - } - w.Write(LEVEL_2, "Environment Variables from:%s\n", none) - - for _, e := range container.EnvFrom { - from := "" - name := "" - optional := false - if e.ConfigMapRef != nil { - from = "ConfigMap" - name = e.ConfigMapRef.Name - optional = e.ConfigMapRef.Optional != nil && *e.ConfigMapRef.Optional - } else if e.SecretRef != nil { - from = "Secret" - name = e.SecretRef.Name - optional = e.SecretRef.Optional != nil && *e.SecretRef.Optional - } - if len(e.Prefix) == 0 { - w.Write(LEVEL_3, "%s\t%s\tOptional: %t\n", name, from, optional) - } else { - w.Write(LEVEL_3, "%s\t%s with prefix '%s'\tOptional: %t\n", name, from, e.Prefix, optional) - } - } -} - -// DescribeProbe is exported for consumers in other API groups that have probes -func DescribeProbe(probe *api.Probe) string { - attrs := fmt.Sprintf("delay=%ds timeout=%ds period=%ds #success=%d #failure=%d", probe.InitialDelaySeconds, probe.TimeoutSeconds, probe.PeriodSeconds, probe.SuccessThreshold, probe.FailureThreshold) - switch { - case probe.Exec != nil: - return fmt.Sprintf("exec %v %s", probe.Exec.Command, attrs) - case probe.HTTPGet != nil: - url := &url.URL{} - url.Scheme = strings.ToLower(string(probe.HTTPGet.Scheme)) - if len(probe.HTTPGet.Port.String()) > 0 { - url.Host = net.JoinHostPort(probe.HTTPGet.Host, probe.HTTPGet.Port.String()) - } else { - url.Host = probe.HTTPGet.Host - } - url.Path = probe.HTTPGet.Path - return fmt.Sprintf("http-get %s %s", url.String(), attrs) - case probe.TCPSocket != nil: - return fmt.Sprintf("tcp-socket %s:%s %s", probe.TCPSocket.Host, probe.TCPSocket.Port.String(), attrs) - } - return fmt.Sprintf("unknown %s", attrs) -} - -type EnvVarResolverFunc func(e api.EnvVar) string - -// EnvValueFrom is exported for use by describers in other packages -func EnvValueRetriever(pod *api.Pod) EnvVarResolverFunc { - return func(e api.EnvVar) string { - gv, err := schema.ParseGroupVersion(e.ValueFrom.FieldRef.APIVersion) - if err != nil { - return "" - } - gvk := gv.WithKind("Pod") - internalFieldPath, _, err := legacyscheme.Scheme.ConvertFieldLabel(gvk, e.ValueFrom.FieldRef.FieldPath, "") - if err != nil { - return "" // pod validation should catch this on create - } - - valueFrom, err := fieldpath.ExtractFieldPathAsString(pod, internalFieldPath) - if err != nil { - return "" // pod validation should catch this on create - } - - return valueFrom - } -} - -func describeStatus(stateName string, state api.ContainerState, w PrefixWriter) { - switch { - case state.Running != nil: - w.Write(LEVEL_2, "%s:\tRunning\n", stateName) - w.Write(LEVEL_3, "Started:\t%v\n", state.Running.StartedAt.Time.Format(time.RFC1123Z)) - case state.Waiting != nil: - w.Write(LEVEL_2, "%s:\tWaiting\n", stateName) - if state.Waiting.Reason != "" { - w.Write(LEVEL_3, "Reason:\t%s\n", state.Waiting.Reason) - } - case state.Terminated != nil: - w.Write(LEVEL_2, "%s:\tTerminated\n", stateName) - if state.Terminated.Reason != "" { - w.Write(LEVEL_3, "Reason:\t%s\n", state.Terminated.Reason) - } - if state.Terminated.Message != "" { - w.Write(LEVEL_3, "Message:\t%s\n", state.Terminated.Message) - } - w.Write(LEVEL_3, "Exit Code:\t%d\n", state.Terminated.ExitCode) - if state.Terminated.Signal > 0 { - w.Write(LEVEL_3, "Signal:\t%d\n", state.Terminated.Signal) - } - w.Write(LEVEL_3, "Started:\t%s\n", state.Terminated.StartedAt.Time.Format(time.RFC1123Z)) - w.Write(LEVEL_3, "Finished:\t%s\n", state.Terminated.FinishedAt.Time.Format(time.RFC1123Z)) - default: - w.Write(LEVEL_2, "%s:\tWaiting\n", stateName) - } -} - -func describeVolumeClaimTemplates(templates []api.PersistentVolumeClaim, w PrefixWriter) { - if len(templates) == 0 { - w.Write(LEVEL_0, "Volume Claims:\t\n") - return - } - w.Write(LEVEL_0, "Volume Claims:\n") - for _, pvc := range templates { - w.Write(LEVEL_1, "Name:\t%s\n", pvc.Name) - w.Write(LEVEL_1, "StorageClass:\t%s\n", helper.GetPersistentVolumeClaimClass(&pvc)) - printLabelsMultilineWithIndent(w, " ", "Labels", "\t", pvc.Labels, sets.NewString()) - printLabelsMultilineWithIndent(w, " ", "Annotations", "\t", pvc.Annotations, sets.NewString()) - if capacity, ok := pvc.Spec.Resources.Requests[api.ResourceStorage]; ok { - w.Write(LEVEL_1, "Capacity:\t%s\n", capacity.String()) - } else { - w.Write(LEVEL_1, "Capacity:\t%s\n", "") - } - w.Write(LEVEL_1, "Access Modes:\t%s\n", pvc.Spec.AccessModes) - } -} - -func printBoolPtr(value *bool) string { - if value != nil { - return printBool(*value) - } - - return "" -} - -func printBool(value bool) string { - if value { - return "True" - } - - return "False" -} - -// ReplicationControllerDescriber generates information about a replication controller -// and the pods it has created. -type ReplicationControllerDescriber struct { - clientset.Interface -} - -func (d *ReplicationControllerDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - rc := d.Core().ReplicationControllers(namespace) - pc := d.Core().Pods(namespace) - - controller, err := rc.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - running, waiting, succeeded, failed, err := getPodStatusForController(pc, labels.SelectorFromSet(controller.Spec.Selector), controller.UID) - if err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = d.Core().Events(namespace).Search(legacyscheme.Scheme, controller) - } - - return describeReplicationController(controller, events, running, waiting, succeeded, failed) -} - -func describeReplicationController(controller *api.ReplicationController, events *api.EventList, running, waiting, succeeded, failed int) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", controller.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", controller.Namespace) - w.Write(LEVEL_0, "Selector:\t%s\n", labels.FormatLabels(controller.Spec.Selector)) - printLabelsMultiline(w, "Labels", controller.Labels) - printAnnotationsMultiline(w, "Annotations", controller.Annotations) - w.Write(LEVEL_0, "Replicas:\t%d current / %d desired\n", controller.Status.Replicas, controller.Spec.Replicas) - w.Write(LEVEL_0, "Pods Status:\t%d Running / %d Waiting / %d Succeeded / %d Failed\n", running, waiting, succeeded, failed) - DescribePodTemplate(controller.Spec.Template, w) - if len(controller.Status.Conditions) > 0 { - w.Write(LEVEL_0, "Conditions:\n Type\tStatus\tReason\n") - w.Write(LEVEL_1, "----\t------\t------\n") - for _, c := range controller.Status.Conditions { - w.Write(LEVEL_1, "%v \t%v\t%v\n", c.Type, c.Status, c.Reason) - } - } - if events != nil { - DescribeEvents(events, w) - } - return nil - }) -} - -func DescribePodTemplate(template *api.PodTemplateSpec, w PrefixWriter) { - w.Write(LEVEL_0, "Pod Template:\n") - if template == nil { - w.Write(LEVEL_1, "") - return - } - printLabelsMultiline(w, " Labels", template.Labels) - if len(template.Annotations) > 0 { - printAnnotationsMultiline(w, " Annotations", template.Annotations) - } - if len(template.Spec.ServiceAccountName) > 0 { - w.Write(LEVEL_1, "Service Account:\t%s\n", template.Spec.ServiceAccountName) - } - if len(template.Spec.InitContainers) > 0 { - describeContainers("Init Containers", template.Spec.InitContainers, nil, nil, w, " ") - } - describeContainers("Containers", template.Spec.Containers, nil, nil, w, " ") - describeVolumes(template.Spec.Volumes, w, " ") -} - -// ReplicaSetDescriber generates information about a ReplicaSet and the pods it has created. -type ReplicaSetDescriber struct { - clientset.Interface -} - -func (d *ReplicaSetDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - rsc := d.Apps().ReplicaSets(namespace) - pc := d.Core().Pods(namespace) - - rs, err := rsc.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector) - if err != nil { - return "", err - } - - running, waiting, succeeded, failed, getPodErr := getPodStatusForController(pc, selector, rs.UID) - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = d.Core().Events(namespace).Search(legacyscheme.Scheme, rs) - } - - return describeReplicaSet(rs, events, running, waiting, succeeded, failed, getPodErr) -} - -func describeReplicaSet(rs *apps.ReplicaSet, events *api.EventList, running, waiting, succeeded, failed int, getPodErr error) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", rs.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", rs.Namespace) - w.Write(LEVEL_0, "Selector:\t%s\n", metav1.FormatLabelSelector(rs.Spec.Selector)) - printLabelsMultiline(w, "Labels", rs.Labels) - printAnnotationsMultiline(w, "Annotations", rs.Annotations) - if controlledBy := printController(rs); len(controlledBy) > 0 { - w.Write(LEVEL_0, "Controlled By:\t%s\n", controlledBy) - } - w.Write(LEVEL_0, "Replicas:\t%d current / %d desired\n", rs.Status.Replicas, rs.Spec.Replicas) - w.Write(LEVEL_0, "Pods Status:\t") - if getPodErr != nil { - w.Write(LEVEL_0, "error in fetching pods: %s\n", getPodErr) - } else { - w.Write(LEVEL_0, "%d Running / %d Waiting / %d Succeeded / %d Failed\n", running, waiting, succeeded, failed) - } - DescribePodTemplate(&rs.Spec.Template, w) - if len(rs.Status.Conditions) > 0 { - w.Write(LEVEL_0, "Conditions:\n Type\tStatus\tReason\n") - w.Write(LEVEL_1, "----\t------\t------\n") - for _, c := range rs.Status.Conditions { - w.Write(LEVEL_1, "%v \t%v\t%v\n", c.Type, c.Status, c.Reason) - } - } - if events != nil { - DescribeEvents(events, w) - } - return nil - }) -} - -// JobDescriber generates information about a job and the pods it has created. -type JobDescriber struct { - clientset.Interface -} - -func (d *JobDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - job, err := d.Batch().Jobs(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = d.Core().Events(namespace).Search(legacyscheme.Scheme, job) - } - - return describeJob(job, events) -} - -func describeJob(job *batch.Job, events *api.EventList) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", job.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", job.Namespace) - selector, _ := metav1.LabelSelectorAsSelector(job.Spec.Selector) - w.Write(LEVEL_0, "Selector:\t%s\n", selector) - printLabelsMultiline(w, "Labels", job.Labels) - printAnnotationsMultiline(w, "Annotations", job.Annotations) - if controlledBy := printController(job); len(controlledBy) > 0 { - w.Write(LEVEL_0, "Controlled By:\t%s\n", controlledBy) - } - w.Write(LEVEL_0, "Parallelism:\t%d\n", *job.Spec.Parallelism) - if job.Spec.Completions != nil { - w.Write(LEVEL_0, "Completions:\t%d\n", *job.Spec.Completions) - } else { - w.Write(LEVEL_0, "Completions:\t\n") - } - if job.Status.StartTime != nil { - w.Write(LEVEL_0, "Start Time:\t%s\n", job.Status.StartTime.Time.Format(time.RFC1123Z)) - } - if job.Status.CompletionTime != nil { - w.Write(LEVEL_0, "Completed At:\t%s\n", job.Status.CompletionTime.Time.Format(time.RFC1123Z)) - } - if job.Status.StartTime != nil && job.Status.CompletionTime != nil { - w.Write(LEVEL_0, "Duration:\t%s\n", duration.HumanDuration(job.Status.CompletionTime.Sub(job.Status.StartTime.Time))) - } - if job.Spec.ActiveDeadlineSeconds != nil { - w.Write(LEVEL_0, "Active Deadline Seconds:\t%ds\n", *job.Spec.ActiveDeadlineSeconds) - } - w.Write(LEVEL_0, "Pods Statuses:\t%d Running / %d Succeeded / %d Failed\n", job.Status.Active, job.Status.Succeeded, job.Status.Failed) - DescribePodTemplate(&job.Spec.Template, w) - if events != nil { - DescribeEvents(events, w) - } - return nil - }) -} - -// CronJobDescriber generates information about a cron job and the jobs it has created. -type CronJobDescriber struct { - clientset.Interface - external externalclient.Interface -} - -func (d *CronJobDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - cronJob, err := d.external.BatchV1beta1().CronJobs(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = d.Core().Events(namespace).Search(legacyscheme.Scheme, cronJob) - } - - internalCronJob := &batch.CronJob{} - if err := legacyscheme.Scheme.Convert(cronJob, internalCronJob, nil); err != nil { - return "", err - } - - return describeCronJob(internalCronJob, events) -} - -func describeCronJob(cronJob *batch.CronJob, events *api.EventList) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", cronJob.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", cronJob.Namespace) - printLabelsMultiline(w, "Labels", cronJob.Labels) - printAnnotationsMultiline(w, "Annotations", cronJob.Annotations) - w.Write(LEVEL_0, "Schedule:\t%s\n", cronJob.Spec.Schedule) - w.Write(LEVEL_0, "Concurrency Policy:\t%s\n", cronJob.Spec.ConcurrencyPolicy) - w.Write(LEVEL_0, "Suspend:\t%s\n", printBoolPtr(cronJob.Spec.Suspend)) - if cronJob.Spec.StartingDeadlineSeconds != nil { - w.Write(LEVEL_0, "Starting Deadline Seconds:\t%ds\n", *cronJob.Spec.StartingDeadlineSeconds) - } else { - w.Write(LEVEL_0, "Starting Deadline Seconds:\t\n") - } - describeJobTemplate(cronJob.Spec.JobTemplate, w) - if cronJob.Status.LastScheduleTime != nil { - w.Write(LEVEL_0, "Last Schedule Time:\t%s\n", cronJob.Status.LastScheduleTime.Time.Format(time.RFC1123Z)) - } else { - w.Write(LEVEL_0, "Last Schedule Time:\t\n") - } - printActiveJobs(w, "Active Jobs", cronJob.Status.Active) - if events != nil { - DescribeEvents(events, w) - } - return nil - }) -} - -func describeJobTemplate(jobTemplate batch.JobTemplateSpec, w PrefixWriter) { - if jobTemplate.Spec.Selector != nil { - selector, _ := metav1.LabelSelectorAsSelector(jobTemplate.Spec.Selector) - w.Write(LEVEL_0, "Selector:\t%s\n", selector) - } else { - w.Write(LEVEL_0, "Selector:\t\n") - } - if jobTemplate.Spec.Parallelism != nil { - w.Write(LEVEL_0, "Parallelism:\t%d\n", *jobTemplate.Spec.Parallelism) - } else { - w.Write(LEVEL_0, "Parallelism:\t\n") - } - if jobTemplate.Spec.Completions != nil { - w.Write(LEVEL_0, "Completions:\t%d\n", *jobTemplate.Spec.Completions) - } else { - w.Write(LEVEL_0, "Completions:\t\n") - } - if jobTemplate.Spec.ActiveDeadlineSeconds != nil { - w.Write(LEVEL_0, "Active Deadline Seconds:\t%ds\n", *jobTemplate.Spec.ActiveDeadlineSeconds) - } - DescribePodTemplate(&jobTemplate.Spec.Template, w) -} - -func printActiveJobs(w PrefixWriter, title string, jobs []api.ObjectReference) { - w.Write(LEVEL_0, "%s:\t", title) - if len(jobs) == 0 { - w.WriteLine("") - return - } - - for i, job := range jobs { - if i != 0 { - w.Write(LEVEL_0, ", ") - } - w.Write(LEVEL_0, "%s", job.Name) - } - w.WriteLine("") -} - -// DaemonSetDescriber generates information about a daemon set and the pods it has created. -type DaemonSetDescriber struct { - clientset.Interface -} - -func (d *DaemonSetDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - dc := d.Apps().DaemonSets(namespace) - pc := d.Core().Pods(namespace) - - daemon, err := dc.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - selector, err := metav1.LabelSelectorAsSelector(daemon.Spec.Selector) - if err != nil { - return "", err - } - running, waiting, succeeded, failed, err := getPodStatusForController(pc, selector, daemon.UID) - if err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = d.Core().Events(namespace).Search(legacyscheme.Scheme, daemon) - } - - return describeDaemonSet(daemon, events, running, waiting, succeeded, failed) -} - -func describeDaemonSet(daemon *apps.DaemonSet, events *api.EventList, running, waiting, succeeded, failed int) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", daemon.Name) - selector, err := metav1.LabelSelectorAsSelector(daemon.Spec.Selector) - if err != nil { - // this shouldn't happen if LabelSelector passed validation - return err - } - w.Write(LEVEL_0, "Selector:\t%s\n", selector) - w.Write(LEVEL_0, "Node-Selector:\t%s\n", labels.FormatLabels(daemon.Spec.Template.Spec.NodeSelector)) - printLabelsMultiline(w, "Labels", daemon.Labels) - printAnnotationsMultiline(w, "Annotations", daemon.Annotations) - w.Write(LEVEL_0, "Desired Number of Nodes Scheduled: %d\n", daemon.Status.DesiredNumberScheduled) - w.Write(LEVEL_0, "Current Number of Nodes Scheduled: %d\n", daemon.Status.CurrentNumberScheduled) - w.Write(LEVEL_0, "Number of Nodes Scheduled with Up-to-date Pods: %d\n", daemon.Status.UpdatedNumberScheduled) - w.Write(LEVEL_0, "Number of Nodes Scheduled with Available Pods: %d\n", daemon.Status.NumberAvailable) - w.Write(LEVEL_0, "Number of Nodes Misscheduled: %d\n", daemon.Status.NumberMisscheduled) - w.Write(LEVEL_0, "Pods Status:\t%d Running / %d Waiting / %d Succeeded / %d Failed\n", running, waiting, succeeded, failed) - DescribePodTemplate(&daemon.Spec.Template, w) - if events != nil { - DescribeEvents(events, w) - } - return nil - }) -} - -// SecretDescriber generates information about a secret -type SecretDescriber struct { - clientset.Interface -} - -func (d *SecretDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - c := d.Core().Secrets(namespace) - - secret, err := c.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - return describeSecret(secret) -} - -func describeSecret(secret *api.Secret) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", secret.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", secret.Namespace) - printLabelsMultiline(w, "Labels", secret.Labels) - skipAnnotations := sets.NewString(api.LastAppliedConfigAnnotation) - printAnnotationsMultilineWithFilter(w, "Annotations", secret.Annotations, skipAnnotations) - - w.Write(LEVEL_0, "\nType:\t%s\n", secret.Type) - - w.Write(LEVEL_0, "\nData\n====\n") - for k, v := range secret.Data { - switch { - case k == api.ServiceAccountTokenKey && secret.Type == api.SecretTypeServiceAccountToken: - w.Write(LEVEL_0, "%s:\t%s\n", k, string(v)) - default: - w.Write(LEVEL_0, "%s:\t%d bytes\n", k, len(v)) - } - } - - return nil - }) -} - -type IngressDescriber struct { - clientset.Interface -} - -func (i *IngressDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - c := i.Extensions().Ingresses(namespace) - ing, err := c.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - return i.describeIngress(ing, describerSettings) -} - -func (i *IngressDescriber) describeBackend(ns string, backend *extensions.IngressBackend) string { - endpoints, _ := i.Core().Endpoints(ns).Get(backend.ServiceName, metav1.GetOptions{}) - service, _ := i.Core().Services(ns).Get(backend.ServiceName, metav1.GetOptions{}) - spName := "" - for i := range service.Spec.Ports { - sp := &service.Spec.Ports[i] - switch backend.ServicePort.Type { - case intstr.String: - if backend.ServicePort.StrVal == sp.Name { - spName = sp.Name - } - case intstr.Int: - if int32(backend.ServicePort.IntVal) == sp.Port { - spName = sp.Name - } - } - } - return formatEndpoints(endpoints, sets.NewString(spName)) -} - -func (i *IngressDescriber) describeIngress(ing *extensions.Ingress, describerSettings printers.DescriberSettings) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%v\n", ing.Name) - w.Write(LEVEL_0, "Namespace:\t%v\n", ing.Namespace) - w.Write(LEVEL_0, "Address:\t%v\n", loadBalancerStatusStringer(ing.Status.LoadBalancer, true)) - def := ing.Spec.Backend - ns := ing.Namespace - if def == nil { - // Ingresses that don't specify a default backend inherit the - // default backend in the kube-system namespace. - def = &extensions.IngressBackend{ - ServiceName: "default-http-backend", - ServicePort: intstr.IntOrString{Type: intstr.Int, IntVal: 80}, - } - ns = metav1.NamespaceSystem - } - w.Write(LEVEL_0, "Default backend:\t%s (%s)\n", backendStringer(def), i.describeBackend(ns, def)) - if len(ing.Spec.TLS) != 0 { - describeIngressTLS(w, ing.Spec.TLS) - } - w.Write(LEVEL_0, "Rules:\n Host\tPath\tBackends\n") - w.Write(LEVEL_1, "----\t----\t--------\n") - count := 0 - for _, rules := range ing.Spec.Rules { - if rules.HTTP == nil { - continue - } - count++ - host := rules.Host - if len(host) == 0 { - host = "*" - } - w.Write(LEVEL_1, "%s\t\n", host) - for _, path := range rules.HTTP.Paths { - w.Write(LEVEL_2, "\t%s \t%s (%s)\n", path.Path, backendStringer(&path.Backend), i.describeBackend(ns, &path.Backend)) - } - } - if count == 0 { - w.Write(LEVEL_1, "%s\t%s \t%s (%s)\n", "*", "*", backendStringer(def), i.describeBackend(ns, def)) - } - describeIngressAnnotations(w, ing.Annotations) - - if describerSettings.ShowEvents { - events, _ := i.Core().Events(ing.Namespace).Search(legacyscheme.Scheme, ing) - if events != nil { - DescribeEvents(events, w) - } - } - return nil - }) -} - -func describeIngressTLS(w PrefixWriter, ingTLS []extensions.IngressTLS) { - w.Write(LEVEL_0, "TLS:\n") - for _, t := range ingTLS { - if t.SecretName == "" { - w.Write(LEVEL_1, "SNI routes %v\n", strings.Join(t.Hosts, ",")) - } else { - w.Write(LEVEL_1, "%v terminates %v\n", t.SecretName, strings.Join(t.Hosts, ",")) - } - } - return -} - -// TODO: Move from annotations into Ingress status. -func describeIngressAnnotations(w PrefixWriter, annotations map[string]string) { - w.Write(LEVEL_0, "Annotations:\n") - for k, v := range annotations { - w.Write(LEVEL_1, "%v:\t%s\n", k, v) - } - return -} - -// ServiceDescriber generates information about a service. -type ServiceDescriber struct { - clientset.Interface -} - -func (d *ServiceDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - c := d.Core().Services(namespace) - - service, err := c.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - endpoints, _ := d.Core().Endpoints(namespace).Get(name, metav1.GetOptions{}) - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = d.Core().Events(namespace).Search(legacyscheme.Scheme, service) - } - return describeService(service, endpoints, events) -} - -func buildIngressString(ingress []api.LoadBalancerIngress) string { - var buffer bytes.Buffer - - for i := range ingress { - if i != 0 { - buffer.WriteString(", ") - } - if ingress[i].IP != "" { - buffer.WriteString(ingress[i].IP) - } else { - buffer.WriteString(ingress[i].Hostname) - } - } - return buffer.String() -} - -func describeService(service *api.Service, endpoints *api.Endpoints, events *api.EventList) (string, error) { - if endpoints == nil { - endpoints = &api.Endpoints{} - } - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", service.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", service.Namespace) - printLabelsMultiline(w, "Labels", service.Labels) - printAnnotationsMultiline(w, "Annotations", service.Annotations) - w.Write(LEVEL_0, "Selector:\t%s\n", labels.FormatLabels(service.Spec.Selector)) - w.Write(LEVEL_0, "Type:\t%s\n", service.Spec.Type) - w.Write(LEVEL_0, "IP:\t%s\n", service.Spec.ClusterIP) - if len(service.Spec.ExternalIPs) > 0 { - w.Write(LEVEL_0, "External IPs:\t%v\n", strings.Join(service.Spec.ExternalIPs, ",")) - } - if service.Spec.LoadBalancerIP != "" { - w.Write(LEVEL_0, "IP:\t%s\n", service.Spec.LoadBalancerIP) - } - if service.Spec.ExternalName != "" { - w.Write(LEVEL_0, "External Name:\t%s\n", service.Spec.ExternalName) - } - if len(service.Status.LoadBalancer.Ingress) > 0 { - list := buildIngressString(service.Status.LoadBalancer.Ingress) - w.Write(LEVEL_0, "LoadBalancer Ingress:\t%s\n", list) - } - for i := range service.Spec.Ports { - sp := &service.Spec.Ports[i] - - name := sp.Name - if name == "" { - name = "" - } - w.Write(LEVEL_0, "Port:\t%s\t%d/%s\n", name, sp.Port, sp.Protocol) - if sp.TargetPort.Type == intstr.Type(intstr.Int) { - w.Write(LEVEL_0, "TargetPort:\t%d/%s\n", sp.TargetPort.IntVal, sp.Protocol) - } else { - w.Write(LEVEL_0, "TargetPort:\t%s/%s\n", sp.TargetPort.StrVal, sp.Protocol) - } - if sp.NodePort != 0 { - w.Write(LEVEL_0, "NodePort:\t%s\t%d/%s\n", name, sp.NodePort, sp.Protocol) - } - w.Write(LEVEL_0, "Endpoints:\t%s\n", formatEndpoints(endpoints, sets.NewString(sp.Name))) - } - w.Write(LEVEL_0, "Session Affinity:\t%s\n", service.Spec.SessionAffinity) - if service.Spec.ExternalTrafficPolicy != "" { - w.Write(LEVEL_0, "External Traffic Policy:\t%s\n", service.Spec.ExternalTrafficPolicy) - } - if service.Spec.HealthCheckNodePort != 0 { - w.Write(LEVEL_0, "HealthCheck NodePort:\t%d\n", service.Spec.HealthCheckNodePort) - } - if len(service.Spec.LoadBalancerSourceRanges) > 0 { - w.Write(LEVEL_0, "LoadBalancer Source Ranges:\t%v\n", strings.Join(service.Spec.LoadBalancerSourceRanges, ",")) - } - if events != nil { - DescribeEvents(events, w) - } - return nil - }) -} - -// EndpointsDescriber generates information about an Endpoint. -type EndpointsDescriber struct { - clientset.Interface -} - -func (d *EndpointsDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - c := d.Core().Endpoints(namespace) - - ep, err := c.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = d.Core().Events(namespace).Search(legacyscheme.Scheme, ep) - } - - return describeEndpoints(ep, events) -} - -func describeEndpoints(ep *api.Endpoints, events *api.EventList) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", ep.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", ep.Namespace) - printLabelsMultiline(w, "Labels", ep.Labels) - printAnnotationsMultiline(w, "Annotations", ep.Annotations) - - w.Write(LEVEL_0, "Subsets:\n") - for i := range ep.Subsets { - subset := &ep.Subsets[i] - - addresses := make([]string, 0, len(subset.Addresses)) - for _, addr := range subset.Addresses { - addresses = append(addresses, addr.IP) - } - addressesString := strings.Join(addresses, ",") - if len(addressesString) == 0 { - addressesString = "" - } - w.Write(LEVEL_1, "Addresses:\t%s\n", addressesString) - - notReadyAddresses := make([]string, 0, len(subset.NotReadyAddresses)) - for _, addr := range subset.NotReadyAddresses { - notReadyAddresses = append(notReadyAddresses, addr.IP) - } - notReadyAddressesString := strings.Join(notReadyAddresses, ",") - if len(notReadyAddressesString) == 0 { - notReadyAddressesString = "" - } - w.Write(LEVEL_1, "NotReadyAddresses:\t%s\n", notReadyAddressesString) - - if len(subset.Ports) > 0 { - w.Write(LEVEL_1, "Ports:\n") - w.Write(LEVEL_2, "Name\tPort\tProtocol\n") - w.Write(LEVEL_2, "----\t----\t--------\n") - for _, port := range subset.Ports { - name := port.Name - if len(name) == 0 { - name = "" - } - w.Write(LEVEL_2, "%s\t%d\t%s\n", name, port.Port, port.Protocol) - } - } - w.Write(LEVEL_0, "\n") - } - - if events != nil { - DescribeEvents(events, w) - } - return nil - }) -} - -// ServiceAccountDescriber generates information about a service. -type ServiceAccountDescriber struct { - clientset.Interface -} - -func (d *ServiceAccountDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - c := d.Core().ServiceAccounts(namespace) - - serviceAccount, err := c.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - tokens := []api.Secret{} - - // missingSecrets is the set of all secrets present in the - // serviceAccount but not present in the set of existing secrets. - missingSecrets := sets.NewString() - secrets, err := d.Core().Secrets(namespace).List(metav1.ListOptions{}) - - // errors are tolerated here in order to describe the serviceAccount with all - // of the secrets that it references, even if those secrets cannot be fetched. - if err == nil { - // existingSecrets is the set of all secrets remaining on a - // service account that are not present in the "tokens" slice. - existingSecrets := sets.NewString() - - for _, s := range secrets.Items { - if s.Type == api.SecretTypeServiceAccountToken { - name, _ := s.Annotations[api.ServiceAccountNameKey] - uid, _ := s.Annotations[api.ServiceAccountUIDKey] - if name == serviceAccount.Name && uid == string(serviceAccount.UID) { - tokens = append(tokens, s) - } - } - existingSecrets.Insert(s.Name) - } - - for _, s := range serviceAccount.Secrets { - if !existingSecrets.Has(s.Name) { - missingSecrets.Insert(s.Name) - } - } - for _, s := range serviceAccount.ImagePullSecrets { - if !existingSecrets.Has(s.Name) { - missingSecrets.Insert(s.Name) - } - } - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = d.Core().Events(namespace).Search(legacyscheme.Scheme, serviceAccount) - } - - return describeServiceAccount(serviceAccount, tokens, missingSecrets, events) -} - -func describeServiceAccount(serviceAccount *api.ServiceAccount, tokens []api.Secret, missingSecrets sets.String, events *api.EventList) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", serviceAccount.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", serviceAccount.Namespace) - printLabelsMultiline(w, "Labels", serviceAccount.Labels) - printAnnotationsMultiline(w, "Annotations", serviceAccount.Annotations) - - var ( - emptyHeader = " " - pullHeader = "Image pull secrets:" - mountHeader = "Mountable secrets: " - tokenHeader = "Tokens: " - - pullSecretNames = []string{} - mountSecretNames = []string{} - tokenSecretNames = []string{} - ) - - for _, s := range serviceAccount.ImagePullSecrets { - pullSecretNames = append(pullSecretNames, s.Name) - } - for _, s := range serviceAccount.Secrets { - mountSecretNames = append(mountSecretNames, s.Name) - } - for _, s := range tokens { - tokenSecretNames = append(tokenSecretNames, s.Name) - } - - types := map[string][]string{ - pullHeader: pullSecretNames, - mountHeader: mountSecretNames, - tokenHeader: tokenSecretNames, - } - for _, header := range sets.StringKeySet(types).List() { - names := types[header] - if len(names) == 0 { - w.Write(LEVEL_0, "%s\t\n", header) - } else { - prefix := header - for _, name := range names { - if missingSecrets.Has(name) { - w.Write(LEVEL_0, "%s\t%s (not found)\n", prefix, name) - } else { - w.Write(LEVEL_0, "%s\t%s\n", prefix, name) - } - prefix = emptyHeader - } - } - } - - if events != nil { - DescribeEvents(events, w) - } - - return nil - }) -} - -// RoleDescriber generates information about a node. -type RoleDescriber struct { - externalclient.Interface -} - -func (d *RoleDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - role, err := d.Rbac().Roles(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - breakdownRules := []rbacv1.PolicyRule{} - for _, rule := range role.Rules { - breakdownRules = append(breakdownRules, validation.BreakdownRule(rule)...) - } - - compactRules, err := validation.CompactRules(breakdownRules) - if err != nil { - return "", err - } - sort.Stable(rbacv1helpers.SortableRuleSlice(compactRules)) - - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", role.Name) - printLabelsMultiline(w, "Labels", role.Labels) - printAnnotationsMultiline(w, "Annotations", role.Annotations) - - w.Write(LEVEL_0, "PolicyRule:\n") - w.Write(LEVEL_1, "Resources\tNon-Resource URLs\tResource Names\tVerbs\n") - w.Write(LEVEL_1, "---------\t-----------------\t--------------\t-----\n") - for _, r := range compactRules { - w.Write(LEVEL_1, "%s\t%v\t%v\t%v\n", combineResourceGroup(r.Resources, r.APIGroups), r.NonResourceURLs, r.ResourceNames, r.Verbs) - } - - return nil - }) -} - -// ClusterRoleDescriber generates information about a node. -type ClusterRoleDescriber struct { - externalclient.Interface -} - -func (d *ClusterRoleDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - role, err := d.Rbac().ClusterRoles().Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - breakdownRules := []rbacv1.PolicyRule{} - for _, rule := range role.Rules { - breakdownRules = append(breakdownRules, validation.BreakdownRule(rule)...) - } - - compactRules, err := validation.CompactRules(breakdownRules) - if err != nil { - return "", err - } - sort.Stable(rbacv1helpers.SortableRuleSlice(compactRules)) - - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", role.Name) - printLabelsMultiline(w, "Labels", role.Labels) - printAnnotationsMultiline(w, "Annotations", role.Annotations) - - w.Write(LEVEL_0, "PolicyRule:\n") - w.Write(LEVEL_1, "Resources\tNon-Resource URLs\tResource Names\tVerbs\n") - w.Write(LEVEL_1, "---------\t-----------------\t--------------\t-----\n") - for _, r := range compactRules { - w.Write(LEVEL_1, "%s\t%v\t%v\t%v\n", combineResourceGroup(r.Resources, r.APIGroups), r.NonResourceURLs, r.ResourceNames, r.Verbs) - } - - return nil - }) -} - -func combineResourceGroup(resource, group []string) string { - if len(resource) == 0 { - return "" - } - parts := strings.SplitN(resource[0], "/", 2) - combine := parts[0] - - if len(group) > 0 && group[0] != "" { - combine = combine + "." + group[0] - } - - if len(parts) == 2 { - combine = combine + "/" + parts[1] - } - return combine -} - -// RoleBindingDescriber generates information about a node. -type RoleBindingDescriber struct { - externalclient.Interface -} - -func (d *RoleBindingDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - binding, err := d.Rbac().RoleBindings(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", binding.Name) - printLabelsMultiline(w, "Labels", binding.Labels) - printAnnotationsMultiline(w, "Annotations", binding.Annotations) - - w.Write(LEVEL_0, "Role:\n") - w.Write(LEVEL_1, "Kind:\t%s\n", binding.RoleRef.Kind) - w.Write(LEVEL_1, "Name:\t%s\n", binding.RoleRef.Name) - - w.Write(LEVEL_0, "Subjects:\n") - w.Write(LEVEL_1, "Kind\tName\tNamespace\n") - w.Write(LEVEL_1, "----\t----\t---------\n") - for _, s := range binding.Subjects { - w.Write(LEVEL_1, "%s\t%s\t%s\n", s.Kind, s.Name, s.Namespace) - } - - return nil - }) -} - -// ClusterRoleBindingDescriber generates information about a node. -type ClusterRoleBindingDescriber struct { - externalclient.Interface -} - -func (d *ClusterRoleBindingDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - binding, err := d.Rbac().ClusterRoleBindings().Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", binding.Name) - printLabelsMultiline(w, "Labels", binding.Labels) - printAnnotationsMultiline(w, "Annotations", binding.Annotations) - - w.Write(LEVEL_0, "Role:\n") - w.Write(LEVEL_1, "Kind:\t%s\n", binding.RoleRef.Kind) - w.Write(LEVEL_1, "Name:\t%s\n", binding.RoleRef.Name) - - w.Write(LEVEL_0, "Subjects:\n") - w.Write(LEVEL_1, "Kind\tName\tNamespace\n") - w.Write(LEVEL_1, "----\t----\t---------\n") - for _, s := range binding.Subjects { - w.Write(LEVEL_1, "%s\t%s\t%s\n", s.Kind, s.Name, s.Namespace) - } - - return nil - }) -} - -// NodeDescriber generates information about a node. -type NodeDescriber struct { - clientset.Interface -} - -func (d *NodeDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - mc := d.Core().Nodes() - node, err := mc.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - fieldSelector, err := fields.ParseSelector("spec.nodeName=" + name + ",status.phase!=" + string(api.PodSucceeded) + ",status.phase!=" + string(api.PodFailed)) - if err != nil { - return "", err - } - // in a policy aware setting, users may have access to a node, but not all pods - // in that case, we note that the user does not have access to the pods - canViewPods := true - nodeNonTerminatedPodsList, err := d.Core().Pods(namespace).List(metav1.ListOptions{FieldSelector: fieldSelector.String()}) - if err != nil { - if !errors.IsForbidden(err) { - return "", err - } - canViewPods = false - } - - var events *api.EventList - if describerSettings.ShowEvents { - if ref, err := ref.GetReference(legacyscheme.Scheme, node); err != nil { - glog.Errorf("Unable to construct reference to '%#v': %v", node, err) - } else { - // TODO: We haven't decided the namespace for Node object yet. - ref.UID = types.UID(ref.Name) - events, _ = d.Core().Events("").Search(legacyscheme.Scheme, ref) - } - } - - return describeNode(node, nodeNonTerminatedPodsList, events, canViewPods) -} - -func describeNode(node *api.Node, nodeNonTerminatedPodsList *api.PodList, events *api.EventList, canViewPods bool) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", node.Name) - if roles := findNodeRoles(node); len(roles) > 0 { - w.Write(LEVEL_0, "Roles:\t%s\n", strings.Join(roles, ",")) - } else { - w.Write(LEVEL_0, "Roles:\t%s\n", "") - } - printLabelsMultiline(w, "Labels", node.Labels) - printAnnotationsMultiline(w, "Annotations", node.Annotations) - w.Write(LEVEL_0, "CreationTimestamp:\t%s\n", node.CreationTimestamp.Time.Format(time.RFC1123Z)) - printNodeTaintsMultiline(w, "Taints", node.Spec.Taints) - w.Write(LEVEL_0, "Unschedulable:\t%v\n", node.Spec.Unschedulable) - if len(node.Status.Conditions) > 0 { - w.Write(LEVEL_0, "Conditions:\n Type\tStatus\tLastHeartbeatTime\tLastTransitionTime\tReason\tMessage\n") - w.Write(LEVEL_1, "----\t------\t-----------------\t------------------\t------\t-------\n") - for _, c := range node.Status.Conditions { - w.Write(LEVEL_1, "%v \t%v \t%s \t%s \t%v \t%v\n", - c.Type, - c.Status, - c.LastHeartbeatTime.Time.Format(time.RFC1123Z), - c.LastTransitionTime.Time.Format(time.RFC1123Z), - c.Reason, - c.Message) - } - } - - w.Write(LEVEL_0, "Addresses:\n") - for _, address := range node.Status.Addresses { - w.Write(LEVEL_1, "%s:\t%s\n", address.Type, address.Address) - } - - printResourceList := func(resourceList api.ResourceList) { - resources := make([]api.ResourceName, 0, len(resourceList)) - for resource := range resourceList { - resources = append(resources, resource) - } - sort.Sort(SortableResourceNames(resources)) - for _, resource := range resources { - value := resourceList[resource] - w.Write(LEVEL_0, " %s:\t%s\n", resource, value.String()) - } - } - - if len(node.Status.Capacity) > 0 { - w.Write(LEVEL_0, "Capacity:\n") - printResourceList(node.Status.Capacity) - } - if len(node.Status.Allocatable) > 0 { - w.Write(LEVEL_0, "Allocatable:\n") - printResourceList(node.Status.Allocatable) - } - - w.Write(LEVEL_0, "System Info:\n") - w.Write(LEVEL_0, " Machine ID:\t%s\n", node.Status.NodeInfo.MachineID) - w.Write(LEVEL_0, " System UUID:\t%s\n", node.Status.NodeInfo.SystemUUID) - w.Write(LEVEL_0, " Boot ID:\t%s\n", node.Status.NodeInfo.BootID) - w.Write(LEVEL_0, " Kernel Version:\t%s\n", node.Status.NodeInfo.KernelVersion) - w.Write(LEVEL_0, " OS Image:\t%s\n", node.Status.NodeInfo.OSImage) - w.Write(LEVEL_0, " Operating System:\t%s\n", node.Status.NodeInfo.OperatingSystem) - w.Write(LEVEL_0, " Architecture:\t%s\n", node.Status.NodeInfo.Architecture) - w.Write(LEVEL_0, " Container Runtime Version:\t%s\n", node.Status.NodeInfo.ContainerRuntimeVersion) - w.Write(LEVEL_0, " Kubelet Version:\t%s\n", node.Status.NodeInfo.KubeletVersion) - w.Write(LEVEL_0, " Kube-Proxy Version:\t%s\n", node.Status.NodeInfo.KubeProxyVersion) - - if len(node.Spec.PodCIDR) > 0 { - w.Write(LEVEL_0, "PodCIDR:\t%s\n", node.Spec.PodCIDR) - } - if len(node.Spec.ProviderID) > 0 { - w.Write(LEVEL_0, "ProviderID:\t%s\n", node.Spec.ProviderID) - } - if canViewPods && nodeNonTerminatedPodsList != nil { - describeNodeResource(nodeNonTerminatedPodsList, node, w) - } else { - w.Write(LEVEL_0, "Pods:\tnot authorized\n") - } - if events != nil { - DescribeEvents(events, w) - } - return nil - }) -} - -type StatefulSetDescriber struct { - client clientset.Interface -} - -func (p *StatefulSetDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - ps, err := p.client.Apps().StatefulSets(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - pc := p.client.Core().Pods(namespace) - - selector, err := metav1.LabelSelectorAsSelector(ps.Spec.Selector) - if err != nil { - return "", err - } - - running, waiting, succeeded, failed, err := getPodStatusForController(pc, selector, ps.UID) - if err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = p.client.Core().Events(namespace).Search(legacyscheme.Scheme, ps) - } - - return describeStatefulSet(ps, selector, events, running, waiting, succeeded, failed) -} - -func describeStatefulSet(ps *apps.StatefulSet, selector labels.Selector, events *api.EventList, running, waiting, succeeded, failed int) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", ps.ObjectMeta.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", ps.ObjectMeta.Namespace) - w.Write(LEVEL_0, "CreationTimestamp:\t%s\n", ps.CreationTimestamp.Time.Format(time.RFC1123Z)) - w.Write(LEVEL_0, "Selector:\t%s\n", selector) - printLabelsMultiline(w, "Labels", ps.Labels) - printAnnotationsMultiline(w, "Annotations", ps.Annotations) - w.Write(LEVEL_0, "Replicas:\t%d desired | %d total\n", ps.Spec.Replicas, ps.Status.Replicas) - w.Write(LEVEL_0, "Update Strategy:\t%s\n", ps.Spec.UpdateStrategy.Type) - if ps.Spec.UpdateStrategy.RollingUpdate != nil { - ru := ps.Spec.UpdateStrategy.RollingUpdate - if ru.Partition != 0 { - w.Write(LEVEL_1, "Partition:\t%d\n", ru.Partition) - } - } - - w.Write(LEVEL_0, "Pods Status:\t%d Running / %d Waiting / %d Succeeded / %d Failed\n", running, waiting, succeeded, failed) - DescribePodTemplate(&ps.Spec.Template, w) - describeVolumeClaimTemplates(ps.Spec.VolumeClaimTemplates, w) - if events != nil { - DescribeEvents(events, w) - } - - return nil - }) -} - -type CertificateSigningRequestDescriber struct { - client clientset.Interface -} - -func (p *CertificateSigningRequestDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - csr, err := p.client.Certificates().CertificateSigningRequests().Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - cr, err := certificates.ParseCSR(csr) - if err != nil { - return "", fmt.Errorf("Error parsing CSR: %v", err) - } - status, err := extractCSRStatus(csr) - if err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = p.client.Core().Events(namespace).Search(legacyscheme.Scheme, csr) - } - - return describeCertificateSigningRequest(csr, cr, status, events) -} - -func describeCertificateSigningRequest(csr *certificates.CertificateSigningRequest, cr *x509.CertificateRequest, status string, events *api.EventList) (string, error) { - printListHelper := func(w PrefixWriter, prefix, name string, values []string) { - if len(values) == 0 { - return - } - w.Write(LEVEL_0, prefix+name+":\t") - w.Write(LEVEL_0, strings.Join(values, "\n"+prefix+"\t")) - w.Write(LEVEL_0, "\n") - } - - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", csr.Name) - w.Write(LEVEL_0, "Labels:\t%s\n", labels.FormatLabels(csr.Labels)) - w.Write(LEVEL_0, "Annotations:\t%s\n", labels.FormatLabels(csr.Annotations)) - w.Write(LEVEL_0, "CreationTimestamp:\t%s\n", csr.CreationTimestamp.Time.Format(time.RFC1123Z)) - w.Write(LEVEL_0, "Requesting User:\t%s\n", csr.Spec.Username) - w.Write(LEVEL_0, "Status:\t%s\n", status) - - w.Write(LEVEL_0, "Subject:\n") - w.Write(LEVEL_0, "\tCommon Name:\t%s\n", cr.Subject.CommonName) - w.Write(LEVEL_0, "\tSerial Number:\t%s\n", cr.Subject.SerialNumber) - printListHelper(w, "\t", "Organization", cr.Subject.Organization) - printListHelper(w, "\t", "Organizational Unit", cr.Subject.OrganizationalUnit) - printListHelper(w, "\t", "Country", cr.Subject.Country) - printListHelper(w, "\t", "Locality", cr.Subject.Locality) - printListHelper(w, "\t", "Province", cr.Subject.Province) - printListHelper(w, "\t", "StreetAddress", cr.Subject.StreetAddress) - printListHelper(w, "\t", "PostalCode", cr.Subject.PostalCode) - - if len(cr.DNSNames)+len(cr.EmailAddresses)+len(cr.IPAddresses) > 0 { - w.Write(LEVEL_0, "Subject Alternative Names:\n") - printListHelper(w, "\t", "DNS Names", cr.DNSNames) - printListHelper(w, "\t", "Email Addresses", cr.EmailAddresses) - var ipaddrs []string - for _, ipaddr := range cr.IPAddresses { - ipaddrs = append(ipaddrs, ipaddr.String()) - } - printListHelper(w, "\t", "IP Addresses", ipaddrs) - } - - if events != nil { - DescribeEvents(events, w) - } - - return nil - }) -} - -// HorizontalPodAutoscalerDescriber generates information about a horizontal pod autoscaler. -type HorizontalPodAutoscalerDescriber struct { - client clientset.Interface -} - -func (d *HorizontalPodAutoscalerDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - hpa, err := d.client.Autoscaling().HorizontalPodAutoscalers(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = d.client.Core().Events(namespace).Search(legacyscheme.Scheme, hpa) - } - - return describeHorizontalPodAutoscaler(hpa, events, d) -} - -func describeHorizontalPodAutoscaler(hpa *autoscaling.HorizontalPodAutoscaler, events *api.EventList, d *HorizontalPodAutoscalerDescriber) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", hpa.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", hpa.Namespace) - printLabelsMultiline(w, "Labels", hpa.Labels) - printAnnotationsMultiline(w, "Annotations", hpa.Annotations) - w.Write(LEVEL_0, "CreationTimestamp:\t%s\n", hpa.CreationTimestamp.Time.Format(time.RFC1123Z)) - w.Write(LEVEL_0, "Reference:\t%s/%s\n", - hpa.Spec.ScaleTargetRef.Kind, - hpa.Spec.ScaleTargetRef.Name) - w.Write(LEVEL_0, "Metrics:\t( current / target )\n") - for i, metric := range hpa.Spec.Metrics { - switch metric.Type { - case autoscaling.ExternalMetricSourceType: - if metric.External.Target.AverageValue != nil { - current := "" - if len(hpa.Status.CurrentMetrics) > i && hpa.Status.CurrentMetrics[i].External != nil && - &hpa.Status.CurrentMetrics[i].External.Current.AverageValue != nil { - current = hpa.Status.CurrentMetrics[i].External.Current.AverageValue.String() - } - w.Write(LEVEL_1, "%q (target average value):\t%s / %s\n", metric.External.Metric.Name, current, metric.External.Target.AverageValue.String()) - } else { - current := "" - if len(hpa.Status.CurrentMetrics) > i && hpa.Status.CurrentMetrics[i].External != nil { - current = hpa.Status.CurrentMetrics[i].External.Current.Value.String() - } - w.Write(LEVEL_1, "%q (target value):\t%s / %s\n", metric.External.Metric.Name, current, metric.External.Target.Value.String()) - - } - case autoscaling.PodsMetricSourceType: - current := "" - if len(hpa.Status.CurrentMetrics) > i && hpa.Status.CurrentMetrics[i].Pods != nil { - current = hpa.Status.CurrentMetrics[i].Pods.Current.AverageValue.String() - } - w.Write(LEVEL_1, "%q on pods:\t%s / %s\n", metric.Pods.Metric.Name, current, metric.Pods.Target.AverageValue.String()) - case autoscaling.ObjectMetricSourceType: - current := "" - if len(hpa.Status.CurrentMetrics) > i && hpa.Status.CurrentMetrics[i].Object != nil { - current = hpa.Status.CurrentMetrics[i].Object.Current.Value.String() - } - w.Write(LEVEL_1, "%q on %s/%s:\t%s / %s\n", metric.Object.Metric.Name, metric.Object.DescribedObject.Kind, metric.Object.DescribedObject.Name, current, metric.Object.Target.Value.String()) - case autoscaling.ResourceMetricSourceType: - w.Write(LEVEL_1, "resource %s on pods", string(metric.Resource.Name)) - if metric.Resource.Target.AverageValue != nil { - current := "" - if len(hpa.Status.CurrentMetrics) > i && hpa.Status.CurrentMetrics[i].Resource != nil { - current = hpa.Status.CurrentMetrics[i].Resource.Current.AverageValue.String() - } - w.Write(LEVEL_0, ":\t%s / %s\n", current, metric.Resource.Target.AverageValue.String()) - } else { - current := "" - if len(hpa.Status.CurrentMetrics) > i && hpa.Status.CurrentMetrics[i].Resource != nil && hpa.Status.CurrentMetrics[i].Resource.Current.AverageUtilization != nil { - current = fmt.Sprintf("%d%% (%s)", *hpa.Status.CurrentMetrics[i].Resource.Current.AverageUtilization, hpa.Status.CurrentMetrics[i].Resource.Current.AverageValue.String()) - } - - target := "" - if metric.Resource.Target.AverageUtilization != nil { - target = fmt.Sprintf("%d%%", *metric.Resource.Target.AverageUtilization) - } - w.Write(LEVEL_1, "(as a percentage of request):\t%s / %s\n", current, target) - } - default: - w.Write(LEVEL_1, "", string(metric.Type)) - } - } - minReplicas := "" - if hpa.Spec.MinReplicas != nil { - minReplicas = fmt.Sprintf("%d", *hpa.Spec.MinReplicas) - } - w.Write(LEVEL_0, "Min replicas:\t%s\n", minReplicas) - w.Write(LEVEL_0, "Max replicas:\t%d\n", hpa.Spec.MaxReplicas) - w.Write(LEVEL_0, "%s pods:\t", hpa.Spec.ScaleTargetRef.Kind) - w.Write(LEVEL_0, "%d current / %d desired\n", hpa.Status.CurrentReplicas, hpa.Status.DesiredReplicas) - - if len(hpa.Status.Conditions) > 0 { - w.Write(LEVEL_0, "Conditions:\n") - w.Write(LEVEL_1, "Type\tStatus\tReason\tMessage\n") - w.Write(LEVEL_1, "----\t------\t------\t-------\n") - for _, c := range hpa.Status.Conditions { - w.Write(LEVEL_1, "%v\t%v\t%v\t%v\n", c.Type, c.Status, c.Reason, c.Message) - } - } - - if events != nil { - DescribeEvents(events, w) - } - - return nil - }) -} - -func describeNodeResource(nodeNonTerminatedPodsList *api.PodList, node *api.Node, w PrefixWriter) { - w.Write(LEVEL_0, "Non-terminated Pods:\t(%d in total)\n", len(nodeNonTerminatedPodsList.Items)) - w.Write(LEVEL_1, "Namespace\tName\t\tCPU Requests\tCPU Limits\tMemory Requests\tMemory Limits\tAGE\n") - w.Write(LEVEL_1, "---------\t----\t\t------------\t----------\t---------------\t-------------\t---\n") - allocatable := node.Status.Capacity - if len(node.Status.Allocatable) > 0 { - allocatable = node.Status.Allocatable - } - - for _, pod := range nodeNonTerminatedPodsList.Items { - req, limit := resourcehelper.PodRequestsAndLimits(&pod) - cpuReq, cpuLimit, memoryReq, memoryLimit := req[api.ResourceCPU], limit[api.ResourceCPU], req[api.ResourceMemory], limit[api.ResourceMemory] - fractionCpuReq := float64(cpuReq.MilliValue()) / float64(allocatable.Cpu().MilliValue()) * 100 - fractionCpuLimit := float64(cpuLimit.MilliValue()) / float64(allocatable.Cpu().MilliValue()) * 100 - fractionMemoryReq := float64(memoryReq.Value()) / float64(allocatable.Memory().Value()) * 100 - fractionMemoryLimit := float64(memoryLimit.Value()) / float64(allocatable.Memory().Value()) * 100 - w.Write(LEVEL_1, "%s\t%s\t\t%s (%d%%)\t%s (%d%%)\t%s (%d%%)\t%s (%d%%)\t%s\n", pod.Namespace, pod.Name, - cpuReq.String(), int64(fractionCpuReq), cpuLimit.String(), int64(fractionCpuLimit), - memoryReq.String(), int64(fractionMemoryReq), memoryLimit.String(), int64(fractionMemoryLimit), translateTimestampSince(pod.CreationTimestamp)) - } - - w.Write(LEVEL_0, "Allocated resources:\n (Total limits may be over 100 percent, i.e., overcommitted.)\n") - w.Write(LEVEL_1, "Resource\tRequests\tLimits\n") - w.Write(LEVEL_1, "--------\t--------\t------\n") - reqs, limits := getPodsTotalRequestsAndLimits(nodeNonTerminatedPodsList) - cpuReqs, cpuLimits, memoryReqs, memoryLimits, ephemeralstorageReqs, ephemeralstorageLimits := - reqs[api.ResourceCPU], limits[api.ResourceCPU], reqs[api.ResourceMemory], limits[api.ResourceMemory], reqs[api.ResourceEphemeralStorage], limits[api.ResourceEphemeralStorage] - fractionCpuReqs := float64(0) - fractionCpuLimits := float64(0) - if allocatable.Cpu().MilliValue() != 0 { - fractionCpuReqs = float64(cpuReqs.MilliValue()) / float64(allocatable.Cpu().MilliValue()) * 100 - fractionCpuLimits = float64(cpuLimits.MilliValue()) / float64(allocatable.Cpu().MilliValue()) * 100 - } - fractionMemoryReqs := float64(0) - fractionMemoryLimits := float64(0) - if allocatable.Memory().Value() != 0 { - fractionMemoryReqs = float64(memoryReqs.Value()) / float64(allocatable.Memory().Value()) * 100 - fractionMemoryLimits = float64(memoryLimits.Value()) / float64(allocatable.Memory().Value()) * 100 - } - fractionEphemeralStorageReqs := float64(0) - fractionEphemeralStorageLimits := float64(0) - if allocatable.StorageEphemeral().Value() != 0 { - fractionEphemeralStorageReqs = float64(ephemeralstorageReqs.Value()) / float64(allocatable.StorageEphemeral().Value()) * 100 - fractionEphemeralStorageLimits = float64(ephemeralstorageLimits.Value()) / float64(allocatable.StorageEphemeral().Value()) * 100 - } - w.Write(LEVEL_1, "%s\t%s (%d%%)\t%s (%d%%)\n", - api.ResourceCPU, cpuReqs.String(), int64(fractionCpuReqs), cpuLimits.String(), int64(fractionCpuLimits)) - w.Write(LEVEL_1, "%s\t%s (%d%%)\t%s (%d%%)\n", - api.ResourceMemory, memoryReqs.String(), int64(fractionMemoryReqs), memoryLimits.String(), int64(fractionMemoryLimits)) - w.Write(LEVEL_1, "%s\t%s (%d%%)\t%s (%d%%)\n", - api.ResourceEphemeralStorage, ephemeralstorageReqs.String(), int64(fractionEphemeralStorageReqs), ephemeralstorageLimits.String(), int64(fractionEphemeralStorageLimits)) - extResources := make([]string, 0, len(allocatable)) - for resource := range allocatable { - if !helper.IsStandardContainerResourceName(string(resource)) && resource != api.ResourcePods { - extResources = append(extResources, string(resource)) - } - } - sort.Strings(extResources) - for _, ext := range extResources { - extRequests, extLimits := reqs[api.ResourceName(ext)], limits[api.ResourceName(ext)] - w.Write(LEVEL_1, "%s\t%s\t%s\n", ext, extRequests.String(), extLimits.String()) - } -} - -func getPodsTotalRequestsAndLimits(podList *api.PodList) (reqs map[api.ResourceName]resource.Quantity, limits map[api.ResourceName]resource.Quantity) { - reqs, limits = map[api.ResourceName]resource.Quantity{}, map[api.ResourceName]resource.Quantity{} - for _, pod := range podList.Items { - podReqs, podLimits := resourcehelper.PodRequestsAndLimits(&pod) - for podReqName, podReqValue := range podReqs { - if value, ok := reqs[podReqName]; !ok { - reqs[podReqName] = *podReqValue.Copy() - } else { - value.Add(podReqValue) - reqs[podReqName] = value - } - } - for podLimitName, podLimitValue := range podLimits { - if value, ok := limits[podLimitName]; !ok { - limits[podLimitName] = *podLimitValue.Copy() - } else { - value.Add(podLimitValue) - limits[podLimitName] = value - } - } - } - return -} - -func DescribeEvents(el *api.EventList, w PrefixWriter) { - if len(el.Items) == 0 { - w.Write(LEVEL_0, "Events:\t\n") - return - } - w.Flush() - sort.Sort(events.SortableEvents(el.Items)) - w.Write(LEVEL_0, "Events:\n Type\tReason\tAge\tFrom\tMessage\n") - w.Write(LEVEL_1, "----\t------\t----\t----\t-------\n") - for _, e := range el.Items { - var interval string - if e.Count > 1 { - interval = fmt.Sprintf("%s (x%d over %s)", translateTimestampSince(e.LastTimestamp), e.Count, translateTimestampSince(e.FirstTimestamp)) - } else { - interval = translateTimestampSince(e.FirstTimestamp) - } - w.Write(LEVEL_1, "%v\t%v\t%s\t%v\t%v\n", - e.Type, - e.Reason, - interval, - formatEventSource(e.Source), - strings.TrimSpace(e.Message), - ) - } -} - -// DeploymentDescriber generates information about a deployment. -type DeploymentDescriber struct { - clientset.Interface - external externalclient.Interface -} - -func (dd *DeploymentDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - d, err := dd.external.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - selector, err := metav1.LabelSelectorAsSelector(d.Spec.Selector) - if err != nil { - return "", err - } - internalDeployment := &apps.Deployment{} - if err := legacyscheme.Scheme.Convert(d, internalDeployment, apps.SchemeGroupVersion); err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = dd.Core().Events(namespace).Search(legacyscheme.Scheme, d) - } - - return describeDeployment(d, selector, internalDeployment, events, dd) -} - -func describeDeployment(d *appsv1.Deployment, selector labels.Selector, internalDeployment *apps.Deployment, events *api.EventList, dd *DeploymentDescriber) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", d.ObjectMeta.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", d.ObjectMeta.Namespace) - w.Write(LEVEL_0, "CreationTimestamp:\t%s\n", d.CreationTimestamp.Time.Format(time.RFC1123Z)) - printLabelsMultiline(w, "Labels", d.Labels) - printAnnotationsMultiline(w, "Annotations", d.Annotations) - w.Write(LEVEL_0, "Selector:\t%s\n", selector) - w.Write(LEVEL_0, "Replicas:\t%d desired | %d updated | %d total | %d available | %d unavailable\n", *(d.Spec.Replicas), d.Status.UpdatedReplicas, d.Status.Replicas, d.Status.AvailableReplicas, d.Status.UnavailableReplicas) - w.Write(LEVEL_0, "StrategyType:\t%s\n", d.Spec.Strategy.Type) - w.Write(LEVEL_0, "MinReadySeconds:\t%d\n", d.Spec.MinReadySeconds) - if d.Spec.Strategy.RollingUpdate != nil { - ru := d.Spec.Strategy.RollingUpdate - w.Write(LEVEL_0, "RollingUpdateStrategy:\t%s max unavailable, %s max surge\n", ru.MaxUnavailable.String(), ru.MaxSurge.String()) - } - DescribePodTemplate(&internalDeployment.Spec.Template, w) - if len(d.Status.Conditions) > 0 { - w.Write(LEVEL_0, "Conditions:\n Type\tStatus\tReason\n") - w.Write(LEVEL_1, "----\t------\t------\n") - for _, c := range d.Status.Conditions { - w.Write(LEVEL_1, "%v \t%v\t%v\n", c.Type, c.Status, c.Reason) - } - } - oldRSs, _, newRS, err := deploymentutil.GetAllReplicaSets(d, dd.external.AppsV1()) - if err == nil { - w.Write(LEVEL_0, "OldReplicaSets:\t%s\n", printReplicaSetsByLabels(oldRSs)) - var newRSs []*appsv1.ReplicaSet - if newRS != nil { - newRSs = append(newRSs, newRS) - } - w.Write(LEVEL_0, "NewReplicaSet:\t%s\n", printReplicaSetsByLabels(newRSs)) - } - if events != nil { - DescribeEvents(events, w) - } - - return nil - }) -} - -func printReplicaSetsByLabels(matchingRSs []*appsv1.ReplicaSet) string { - // Format the matching ReplicaSets into strings. - rsStrings := make([]string, 0, len(matchingRSs)) - for _, rs := range matchingRSs { - rsStrings = append(rsStrings, fmt.Sprintf("%s (%d/%d replicas created)", rs.Name, rs.Status.Replicas, *rs.Spec.Replicas)) - } - - list := strings.Join(rsStrings, ", ") - if list == "" { - return "" - } - return list -} - -func getPodStatusForController(c coreclient.PodInterface, selector labels.Selector, uid types.UID) (running, waiting, succeeded, failed int, err error) { - options := metav1.ListOptions{LabelSelector: selector.String()} - rcPods, err := c.List(options) - if err != nil { - return - } - for _, pod := range rcPods.Items { - controllerRef := metav1.GetControllerOf(&pod) - // Skip pods that are orphans or owned by other controllers. - if controllerRef == nil || controllerRef.UID != uid { - continue - } - switch pod.Status.Phase { - case api.PodRunning: - running++ - case api.PodPending: - waiting++ - case api.PodSucceeded: - succeeded++ - case api.PodFailed: - failed++ - } - } - return -} - -// ConfigMapDescriber generates information about a ConfigMap -type ConfigMapDescriber struct { - clientset.Interface -} - -func (d *ConfigMapDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - c := d.Core().ConfigMaps(namespace) - - configMap, err := c.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", configMap.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", configMap.Namespace) - printLabelsMultiline(w, "Labels", configMap.Labels) - printAnnotationsMultiline(w, "Annotations", configMap.Annotations) - - w.Write(LEVEL_0, "\nData\n====\n") - for k, v := range configMap.Data { - w.Write(LEVEL_0, "%s:\n----\n", k) - w.Write(LEVEL_0, "%s\n", string(v)) - } - if describerSettings.ShowEvents { - events, err := d.Core().Events(namespace).Search(legacyscheme.Scheme, configMap) - if err != nil { - return err - } - if events != nil { - DescribeEvents(events, w) - } - } - return nil - }) -} - -// NetworkPolicyDescriber generates information about a networking.NetworkPolicy -type NetworkPolicyDescriber struct { - clientset.Interface -} - -func (d *NetworkPolicyDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - c := d.Networking().NetworkPolicies(namespace) - - networkPolicy, err := c.Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - return describeNetworkPolicy(networkPolicy) -} - -func describeNetworkPolicy(networkPolicy *networking.NetworkPolicy) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", networkPolicy.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", networkPolicy.Namespace) - w.Write(LEVEL_0, "Created on:\t%s\n", networkPolicy.CreationTimestamp) - printLabelsMultiline(w, "Labels", networkPolicy.Labels) - printAnnotationsMultiline(w, "Annotations", networkPolicy.Annotations) - describeNetworkPolicySpec(networkPolicy.Spec, w) - return nil - }) -} - -func describeNetworkPolicySpec(nps networking.NetworkPolicySpec, w PrefixWriter) { - w.Write(LEVEL_0, "Spec:\n") - w.Write(LEVEL_1, "PodSelector: ") - if len(nps.PodSelector.MatchLabels) == 0 && len(nps.PodSelector.MatchExpressions) == 0 { - w.Write(LEVEL_2, " (Allowing the specific traffic to all pods in this namespace)\n") - } else { - w.Write(LEVEL_2, "%s\n", metav1.FormatLabelSelector(&nps.PodSelector)) - } - w.Write(LEVEL_1, "Allowing ingress traffic:\n") - printNetworkPolicySpecIngressFrom(nps.Ingress, " ", w) - w.Write(LEVEL_1, "Allowing egress traffic:\n") - printNetworkPolicySpecEgressTo(nps.Egress, " ", w) - w.Write(LEVEL_1, "Policy Types: %v\n", policyTypesToString(nps.PolicyTypes)) -} - -func printNetworkPolicySpecIngressFrom(npirs []networking.NetworkPolicyIngressRule, initialIndent string, w PrefixWriter) { - if len(npirs) == 0 { - w.Write(LEVEL_0, "%s%s\n", initialIndent, " (Selected pods are isolated for ingress connectivity)") - return - } - for i, npir := range npirs { - if len(npir.Ports) == 0 { - w.Write(LEVEL_0, "%s%s\n", initialIndent, "To Port: (traffic allowed to all ports)") - } else { - for _, port := range npir.Ports { - var proto api.Protocol - if port.Protocol != nil { - proto = *port.Protocol - } else { - proto = api.ProtocolTCP - } - w.Write(LEVEL_0, "%s%s: %s/%s\n", initialIndent, "To Port", port.Port, proto) - } - } - if len(npir.From) == 0 { - w.Write(LEVEL_0, "%s%s\n", initialIndent, "From: (traffic not restricted by source)") - } else { - for _, from := range npir.From { - w.Write(LEVEL_0, "%s%s\n", initialIndent, "From:") - if from.PodSelector != nil && from.NamespaceSelector != nil { - w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "NamespaceSelector", metav1.FormatLabelSelector(from.NamespaceSelector)) - w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "PodSelector", metav1.FormatLabelSelector(from.PodSelector)) - } else if from.PodSelector != nil { - w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "PodSelector", metav1.FormatLabelSelector(from.PodSelector)) - } else if from.NamespaceSelector != nil { - w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "NamespaceSelector", metav1.FormatLabelSelector(from.NamespaceSelector)) - } else if from.IPBlock != nil { - w.Write(LEVEL_1, "%sIPBlock:\n", initialIndent) - w.Write(LEVEL_2, "%sCIDR: %s\n", initialIndent, from.IPBlock.CIDR) - w.Write(LEVEL_2, "%sExcept: %v\n", initialIndent, strings.Join(from.IPBlock.Except, ", ")) - } - } - } - if i != len(npirs)-1 { - w.Write(LEVEL_0, "%s%s\n", initialIndent, "----------") - } - } -} - -func printNetworkPolicySpecEgressTo(npers []networking.NetworkPolicyEgressRule, initialIndent string, w PrefixWriter) { - if len(npers) == 0 { - w.Write(LEVEL_0, "%s%s\n", initialIndent, " (Selected pods are isolated for egress connectivity)") - return - } - for i, nper := range npers { - if len(nper.Ports) == 0 { - w.Write(LEVEL_0, "%s%s\n", initialIndent, "To Port: (traffic allowed to all ports)") - } else { - for _, port := range nper.Ports { - var proto api.Protocol - if port.Protocol != nil { - proto = *port.Protocol - } else { - proto = api.ProtocolTCP - } - w.Write(LEVEL_0, "%s%s: %s/%s\n", initialIndent, "To Port", port.Port, proto) - } - } - if len(nper.To) == 0 { - w.Write(LEVEL_0, "%s%s\n", initialIndent, "To: (traffic not restricted by source)") - } else { - for _, to := range nper.To { - w.Write(LEVEL_0, "%s%s\n", initialIndent, "To:") - if to.PodSelector != nil && to.NamespaceSelector != nil { - w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "NamespaceSelector", metav1.FormatLabelSelector(to.NamespaceSelector)) - w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "PodSelector", metav1.FormatLabelSelector(to.PodSelector)) - } else if to.PodSelector != nil { - w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "PodSelector", metav1.FormatLabelSelector(to.PodSelector)) - } else if to.NamespaceSelector != nil { - w.Write(LEVEL_1, "%s%s: %s\n", initialIndent, "NamespaceSelector", metav1.FormatLabelSelector(to.NamespaceSelector)) - } else if to.IPBlock != nil { - w.Write(LEVEL_1, "%sIPBlock:\n", initialIndent) - w.Write(LEVEL_2, "%sCIDR: %s\n", initialIndent, to.IPBlock.CIDR) - w.Write(LEVEL_2, "%sExcept: %v\n", initialIndent, strings.Join(to.IPBlock.Except, ", ")) - } - } - } - if i != len(npers)-1 { - w.Write(LEVEL_0, "%s%s\n", initialIndent, "----------") - } - } -} - -type StorageClassDescriber struct { - clientset.Interface -} - -func (s *StorageClassDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - sc, err := s.Storage().StorageClasses().Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = s.Core().Events(namespace).Search(legacyscheme.Scheme, sc) - } - - return describeStorageClass(sc, events) -} - -func describeStorageClass(sc *storage.StorageClass, events *api.EventList) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", sc.Name) - w.Write(LEVEL_0, "IsDefaultClass:\t%s\n", storageutil.IsDefaultAnnotationText(sc.ObjectMeta)) - w.Write(LEVEL_0, "Annotations:\t%s\n", labels.FormatLabels(sc.Annotations)) - w.Write(LEVEL_0, "Provisioner:\t%s\n", sc.Provisioner) - w.Write(LEVEL_0, "Parameters:\t%s\n", labels.FormatLabels(sc.Parameters)) - w.Write(LEVEL_0, "AllowVolumeExpansion:\t%s\n", printBoolPtr(sc.AllowVolumeExpansion)) - if len(sc.MountOptions) == 0 { - w.Write(LEVEL_0, "MountOptions:\t\n") - } else { - w.Write(LEVEL_0, "MountOptions:\n") - for _, option := range sc.MountOptions { - w.Write(LEVEL_1, "%s\n", option) - } - } - if sc.ReclaimPolicy != nil { - w.Write(LEVEL_0, "ReclaimPolicy:\t%s\n", *sc.ReclaimPolicy) - } - if sc.VolumeBindingMode != nil { - w.Write(LEVEL_0, "VolumeBindingMode:\t%s\n", *sc.VolumeBindingMode) - } - if sc.AllowedTopologies != nil { - printAllowedTopologies(w, sc.AllowedTopologies) - } - if events != nil { - DescribeEvents(events, w) - } - - return nil - }) -} - -func printAllowedTopologies(w PrefixWriter, topologies []api.TopologySelectorTerm) { - w.Write(LEVEL_0, "AllowedTopologies:\t") - if len(topologies) == 0 { - w.WriteLine("") - return - } - w.WriteLine("") - for i, term := range topologies { - printTopologySelectorTermsMultilineWithIndent(w, LEVEL_1, fmt.Sprintf("Term %d", i), "\t", term.MatchLabelExpressions) - } -} - -func printTopologySelectorTermsMultilineWithIndent(w PrefixWriter, indentLevel int, title, innerIndent string, reqs []api.TopologySelectorLabelRequirement) { - w.Write(indentLevel, "%s:%s", title, innerIndent) - - if len(reqs) == 0 { - w.WriteLine("") - return - } - - for i, req := range reqs { - if i != 0 { - w.Write(indentLevel, "%s", innerIndent) - } - exprStr := fmt.Sprintf("%s %s", req.Key, "in") - if len(req.Values) > 0 { - exprStr = fmt.Sprintf("%s [%s]", exprStr, strings.Join(req.Values, ", ")) - } - w.Write(LEVEL_0, "%s\n", exprStr) - } -} - -type PodDisruptionBudgetDescriber struct { - clientset.Interface -} - -func (p *PodDisruptionBudgetDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - pdb, err := p.Policy().PodDisruptionBudgets(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = p.Core().Events(namespace).Search(legacyscheme.Scheme, pdb) - } - - return describePodDisruptionBudget(pdb, events) -} - -func describePodDisruptionBudget(pdb *policy.PodDisruptionBudget, events *api.EventList) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", pdb.Name) - w.Write(LEVEL_0, "Namespace:\t%s\n", pdb.Namespace) - - if pdb.Spec.MinAvailable != nil { - w.Write(LEVEL_0, "Min available:\t%s\n", pdb.Spec.MinAvailable.String()) - } else if pdb.Spec.MaxUnavailable != nil { - w.Write(LEVEL_0, "Max unavailable:\t%s\n", pdb.Spec.MaxUnavailable.String()) - } - - if pdb.Spec.Selector != nil { - w.Write(LEVEL_0, "Selector:\t%s\n", metav1.FormatLabelSelector(pdb.Spec.Selector)) - } else { - w.Write(LEVEL_0, "Selector:\t\n") - } - w.Write(LEVEL_0, "Status:\n") - w.Write(LEVEL_2, "Allowed disruptions:\t%d\n", pdb.Status.PodDisruptionsAllowed) - w.Write(LEVEL_2, "Current:\t%d\n", pdb.Status.CurrentHealthy) - w.Write(LEVEL_2, "Desired:\t%d\n", pdb.Status.DesiredHealthy) - w.Write(LEVEL_2, "Total:\t%d\n", pdb.Status.ExpectedPods) - if events != nil { - DescribeEvents(events, w) - } - - return nil - }) -} - -// PriorityClassDescriber generates information about a PriorityClass. -type PriorityClassDescriber struct { - clientset.Interface -} - -func (s *PriorityClassDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - pc, err := s.Scheduling().PriorityClasses().Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - var events *api.EventList - if describerSettings.ShowEvents { - events, _ = s.Core().Events(namespace).Search(legacyscheme.Scheme, pc) - } - - return describePriorityClass(pc, events) -} - -func describePriorityClass(pc *scheduling.PriorityClass, events *api.EventList) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", pc.Name) - w.Write(LEVEL_0, "Value:\t%v\n", pc.Value) - w.Write(LEVEL_0, "GlobalDefault:\t%v\n", pc.GlobalDefault) - w.Write(LEVEL_0, "Description:\t%s\n", pc.Description) - - w.Write(LEVEL_0, "Annotations:\t%s\n", labels.FormatLabels(pc.Annotations)) - if events != nil { - DescribeEvents(events, w) - } - - return nil - }) -} - -// PodSecurityPolicyDescriber generates information about a PodSecurityPolicy. -type PodSecurityPolicyDescriber struct { - clientset.Interface -} - -func (d *PodSecurityPolicyDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { - psp, err := d.Policy().PodSecurityPolicies().Get(name, metav1.GetOptions{}) - if err != nil { - return "", err - } - - return describePodSecurityPolicy(psp) -} - -func describePodSecurityPolicy(psp *policy.PodSecurityPolicy) (string, error) { - return tabbedString(func(out io.Writer) error { - w := NewPrefixWriter(out) - w.Write(LEVEL_0, "Name:\t%s\n", psp.Name) - - w.Write(LEVEL_0, "\nSettings:\n") - - w.Write(LEVEL_1, "Allow Privileged:\t%t\n", psp.Spec.Privileged) - w.Write(LEVEL_1, "Allow Privilege Escalation:\t%v\n", psp.Spec.AllowPrivilegeEscalation) - w.Write(LEVEL_1, "Default Add Capabilities:\t%v\n", capsToString(psp.Spec.DefaultAddCapabilities)) - w.Write(LEVEL_1, "Required Drop Capabilities:\t%s\n", capsToString(psp.Spec.RequiredDropCapabilities)) - w.Write(LEVEL_1, "Allowed Capabilities:\t%s\n", capsToString(psp.Spec.AllowedCapabilities)) - w.Write(LEVEL_1, "Allowed Volume Types:\t%s\n", fsTypeToString(psp.Spec.Volumes)) - - if len(psp.Spec.AllowedFlexVolumes) > 0 { - w.Write(LEVEL_1, "Allowed FlexVolume Types:\t%s\n", flexVolumesToString(psp.Spec.AllowedFlexVolumes)) - } - if len(psp.Spec.AllowedUnsafeSysctls) > 0 { - w.Write(LEVEL_1, "Allowed Unsafe Sysctls:\t%s\n", sysctlsToString(psp.Spec.AllowedUnsafeSysctls)) - } - if len(psp.Spec.ForbiddenSysctls) > 0 { - w.Write(LEVEL_1, "Forbidden Sysctls:\t%s\n", sysctlsToString(psp.Spec.ForbiddenSysctls)) - } - w.Write(LEVEL_1, "Allow Host Network:\t%t\n", psp.Spec.HostNetwork) - w.Write(LEVEL_1, "Allow Host Ports:\t%s\n", hostPortRangeToString(psp.Spec.HostPorts)) - w.Write(LEVEL_1, "Allow Host PID:\t%t\n", psp.Spec.HostPID) - w.Write(LEVEL_1, "Allow Host IPC:\t%t\n", psp.Spec.HostIPC) - w.Write(LEVEL_1, "Read Only Root Filesystem:\t%v\n", psp.Spec.ReadOnlyRootFilesystem) - - w.Write(LEVEL_1, "SELinux Context Strategy: %s\t\n", string(psp.Spec.SELinux.Rule)) - var user, role, seLinuxType, level string - if psp.Spec.SELinux.SELinuxOptions != nil { - user = psp.Spec.SELinux.SELinuxOptions.User - role = psp.Spec.SELinux.SELinuxOptions.Role - seLinuxType = psp.Spec.SELinux.SELinuxOptions.Type - level = psp.Spec.SELinux.SELinuxOptions.Level - } - w.Write(LEVEL_2, "User:\t%s\n", stringOrNone(user)) - w.Write(LEVEL_2, "Role:\t%s\n", stringOrNone(role)) - w.Write(LEVEL_2, "Type:\t%s\n", stringOrNone(seLinuxType)) - w.Write(LEVEL_2, "Level:\t%s\n", stringOrNone(level)) - - w.Write(LEVEL_1, "Run As User Strategy: %s\t\n", string(psp.Spec.RunAsUser.Rule)) - w.Write(LEVEL_2, "Ranges:\t%s\n", idRangeToString(psp.Spec.RunAsUser.Ranges)) - - w.Write(LEVEL_1, "FSGroup Strategy: %s\t\n", string(psp.Spec.FSGroup.Rule)) - w.Write(LEVEL_2, "Ranges:\t%s\n", idRangeToString(psp.Spec.FSGroup.Ranges)) - - w.Write(LEVEL_1, "Supplemental Groups Strategy: %s\t\n", string(psp.Spec.SupplementalGroups.Rule)) - w.Write(LEVEL_2, "Ranges:\t%s\n", idRangeToString(psp.Spec.SupplementalGroups.Ranges)) - - return nil - }) -} - -func stringOrNone(s string) string { - return stringOrDefaultValue(s, "") -} - -func stringOrDefaultValue(s, defaultValue string) string { - if len(s) > 0 { - return s - } - return defaultValue -} - -func fsTypeToString(volumes []policy.FSType) string { - strVolumes := []string{} - for _, v := range volumes { - strVolumes = append(strVolumes, string(v)) - } - return stringOrNone(strings.Join(strVolumes, ",")) -} - -func flexVolumesToString(flexVolumes []policy.AllowedFlexVolume) string { - volumes := []string{} - for _, flexVolume := range flexVolumes { - volumes = append(volumes, "driver="+flexVolume.Driver) - } - return stringOrDefaultValue(strings.Join(volumes, ","), "") -} - -func sysctlsToString(sysctls []string) string { - return stringOrNone(strings.Join(sysctls, ",")) -} - -func hostPortRangeToString(ranges []policy.HostPortRange) string { - formattedString := "" - if ranges != nil { - strRanges := []string{} - for _, r := range ranges { - strRanges = append(strRanges, fmt.Sprintf("%d-%d", r.Min, r.Max)) - } - formattedString = strings.Join(strRanges, ",") - } - return stringOrNone(formattedString) -} - -func idRangeToString(ranges []policy.IDRange) string { - formattedString := "" - if ranges != nil { - strRanges := []string{} - for _, r := range ranges { - strRanges = append(strRanges, fmt.Sprintf("%d-%d", r.Min, r.Max)) - } - formattedString = strings.Join(strRanges, ",") - } - return stringOrNone(formattedString) -} - -func capsToString(caps []api.Capability) string { - formattedString := "" - if caps != nil { - strCaps := []string{} - for _, c := range caps { - strCaps = append(strCaps, string(c)) - } - formattedString = strings.Join(strCaps, ",") - } - return stringOrNone(formattedString) -} - -func policyTypesToString(pts []networking.PolicyType) string { - formattedString := "" - if pts != nil { - strPts := []string{} - for _, p := range pts { - strPts = append(strPts, string(p)) - } - formattedString = strings.Join(strPts, ", ") - } - return stringOrNone(formattedString) -} - -// newErrNoDescriber creates a new ErrNoDescriber with the names of the provided types. -func newErrNoDescriber(types ...reflect.Type) error { - names := make([]string, 0, len(types)) - for _, t := range types { - names = append(names, t.String()) - } - return printers.ErrNoDescriber{Types: names} -} - -// Describers implements ObjectDescriber against functions registered via Add. Those functions can -// be strongly typed. Types are exactly matched (no conversion or assignable checks). -type Describers struct { - searchFns map[reflect.Type][]typeFunc -} - -// DescribeObject implements ObjectDescriber and will attempt to print the provided object to a string, -// if at least one describer function has been registered with the exact types passed, or if any -// describer can print the exact object in its first argument (the remainder will be provided empty -// values). If no function registered with Add can satisfy the passed objects, an ErrNoDescriber will -// be returned -// TODO: reorder and partial match extra. -func (d *Describers) DescribeObject(exact interface{}, extra ...interface{}) (string, error) { - exactType := reflect.TypeOf(exact) - fns, ok := d.searchFns[exactType] - if !ok { - return "", newErrNoDescriber(exactType) - } - if len(extra) == 0 { - for _, typeFn := range fns { - if len(typeFn.Extra) == 0 { - return typeFn.Describe(exact, extra...) - } - } - typeFn := fns[0] - for _, t := range typeFn.Extra { - v := reflect.New(t).Elem() - extra = append(extra, v.Interface()) - } - return fns[0].Describe(exact, extra...) - } - - types := make([]reflect.Type, 0, len(extra)) - for _, obj := range extra { - types = append(types, reflect.TypeOf(obj)) - } - for _, typeFn := range fns { - if typeFn.Matches(types) { - return typeFn.Describe(exact, extra...) - } - } - return "", newErrNoDescriber(append([]reflect.Type{exactType}, types...)...) -} - -// Add adds one or more describer functions to the printers.Describer. The passed function must -// match the signature: -// -// func(...) (string, error) -// -// Any number of arguments may be provided. -func (d *Describers) Add(fns ...interface{}) error { - for _, fn := range fns { - fv := reflect.ValueOf(fn) - ft := fv.Type() - if ft.Kind() != reflect.Func { - return fmt.Errorf("expected func, got: %v", ft) - } - numIn := ft.NumIn() - if numIn == 0 { - return fmt.Errorf("expected at least one 'in' params, got: %v", ft) - } - if ft.NumOut() != 2 { - return fmt.Errorf("expected two 'out' params - (string, error), got: %v", ft) - } - types := make([]reflect.Type, 0, numIn) - for i := 0; i < numIn; i++ { - types = append(types, ft.In(i)) - } - if ft.Out(0) != reflect.TypeOf(string("")) { - return fmt.Errorf("expected string return, got: %v", ft) - } - var forErrorType error - // This convolution is necessary, otherwise TypeOf picks up on the fact - // that forErrorType is nil. - errorType := reflect.TypeOf(&forErrorType).Elem() - if ft.Out(1) != errorType { - return fmt.Errorf("expected error return, got: %v", ft) - } - - exact := types[0] - extra := types[1:] - if d.searchFns == nil { - d.searchFns = make(map[reflect.Type][]typeFunc) - } - fns := d.searchFns[exact] - fn := typeFunc{Extra: extra, Fn: fv} - fns = append(fns, fn) - d.searchFns[exact] = fns - } - return nil -} - -// typeFunc holds information about a describer function and the types it accepts -type typeFunc struct { - Extra []reflect.Type - Fn reflect.Value -} - -// Matches returns true when the passed types exactly match the Extra list. -func (fn typeFunc) Matches(types []reflect.Type) bool { - if len(fn.Extra) != len(types) { - return false - } - // reorder the items in array types and fn.Extra - // convert the type into string and sort them, check if they are matched - varMap := make(map[reflect.Type]bool) - for i := range fn.Extra { - varMap[fn.Extra[i]] = true - } - for i := range types { - if _, found := varMap[types[i]]; !found { - return false - } - } - return true -} - -// Describe invokes the nested function with the exact number of arguments. -func (fn typeFunc) Describe(exact interface{}, extra ...interface{}) (string, error) { - values := []reflect.Value{reflect.ValueOf(exact)} - for _, obj := range extra { - values = append(values, reflect.ValueOf(obj)) - } - out := fn.Fn.Call(values) - s := out[0].Interface().(string) - var err error - if !out[1].IsNil() { - err = out[1].Interface().(error) - } - return s, err -} - -// printLabelsMultiline prints multiple labels with a proper alignment. -func printLabelsMultiline(w PrefixWriter, title string, labels map[string]string) { - printLabelsMultilineWithIndent(w, "", title, "\t", labels, sets.NewString()) -} - -// printLabelsMultiline prints multiple labels with a user-defined alignment. -func printLabelsMultilineWithIndent(w PrefixWriter, initialIndent, title, innerIndent string, labels map[string]string, skip sets.String) { - w.Write(LEVEL_0, "%s%s:%s", initialIndent, title, innerIndent) - - if labels == nil || len(labels) == 0 { - w.WriteLine("") - return - } - - // to print labels in the sorted order - keys := make([]string, 0, len(labels)) - for key := range labels { - if skip.Has(key) { - continue - } - keys = append(keys, key) - } - if len(keys) == 0 { - w.WriteLine("") - return - } - sort.Strings(keys) - - for i, key := range keys { - if i != 0 { - w.Write(LEVEL_0, "%s", initialIndent) - w.Write(LEVEL_0, "%s", innerIndent) - } - w.Write(LEVEL_0, "%s=%s\n", key, labels[key]) - i++ - } -} - -// printTaintsMultiline prints multiple taints with a proper alignment. -func printNodeTaintsMultiline(w PrefixWriter, title string, taints []api.Taint) { - printTaintsMultilineWithIndent(w, "", title, "\t", taints) -} - -// printTaintsMultilineWithIndent prints multiple taints with a user-defined alignment. -func printTaintsMultilineWithIndent(w PrefixWriter, initialIndent, title, innerIndent string, taints []api.Taint) { - w.Write(LEVEL_0, "%s%s:%s", initialIndent, title, innerIndent) - - if taints == nil || len(taints) == 0 { - w.WriteLine("") - return - } - - // to print taints in the sorted order - sort.Slice(taints, func(i, j int) bool { - cmpKey := func(taint api.Taint) string { - return string(taint.Effect) + "," + taint.Key - } - return cmpKey(taints[i]) < cmpKey(taints[j]) - }) - - for i, taint := range taints { - if i != 0 { - w.Write(LEVEL_0, "%s", initialIndent) - w.Write(LEVEL_0, "%s", innerIndent) - } - w.Write(LEVEL_0, "%s\n", taint.ToString()) - } -} - -// printPodsMultiline prints multiple pods with a proper alignment. -func printPodsMultiline(w PrefixWriter, title string, pods []api.Pod) { - printPodsMultilineWithIndent(w, "", title, "\t", pods) -} - -// printPodsMultilineWithIndent prints multiple pods with a user-defined alignment. -func printPodsMultilineWithIndent(w PrefixWriter, initialIndent, title, innerIndent string, pods []api.Pod) { - w.Write(LEVEL_0, "%s%s:%s", initialIndent, title, innerIndent) - - if pods == nil || len(pods) == 0 { - w.WriteLine("") - return - } - - // to print pods in the sorted order - sort.Slice(pods, func(i, j int) bool { - cmpKey := func(pod api.Pod) string { - return pod.Name - } - return cmpKey(pods[i]) < cmpKey(pods[j]) - }) - - for i, pod := range pods { - if i != 0 { - w.Write(LEVEL_0, "%s", initialIndent) - w.Write(LEVEL_0, "%s", innerIndent) - } - w.Write(LEVEL_0, "%s\n", pod.Name) - } -} - -// printPodTolerationsMultiline prints multiple tolerations with a proper alignment. -func printPodTolerationsMultiline(w PrefixWriter, title string, tolerations []api.Toleration) { - printTolerationsMultilineWithIndent(w, "", title, "\t", tolerations) -} - -// printTolerationsMultilineWithIndent prints multiple tolerations with a user-defined alignment. -func printTolerationsMultilineWithIndent(w PrefixWriter, initialIndent, title, innerIndent string, tolerations []api.Toleration) { - w.Write(LEVEL_0, "%s%s:%s", initialIndent, title, innerIndent) - - if tolerations == nil || len(tolerations) == 0 { - w.WriteLine("") - return - } - - // to print tolerations in the sorted order - sort.Slice(tolerations, func(i, j int) bool { - return tolerations[i].Key < tolerations[j].Key - }) - - for i, toleration := range tolerations { - if i != 0 { - w.Write(LEVEL_0, "%s", initialIndent) - w.Write(LEVEL_0, "%s", innerIndent) - } - w.Write(LEVEL_0, "%s", toleration.Key) - if len(toleration.Value) != 0 { - w.Write(LEVEL_0, "=%s", toleration.Value) - } - if len(toleration.Effect) != 0 { - w.Write(LEVEL_0, ":%s", toleration.Effect) - } - if toleration.TolerationSeconds != nil { - w.Write(LEVEL_0, " for %ds", *toleration.TolerationSeconds) - } - w.Write(LEVEL_0, "\n") - } -} - -type flusher interface { - Flush() -} - -func tabbedString(f func(io.Writer) error) (string, error) { - out := new(tabwriter.Writer) - buf := &bytes.Buffer{} - out.Init(buf, 0, 8, 2, ' ', 0) - - err := f(out) - if err != nil { - return "", err - } - - out.Flush() - str := string(buf.String()) - return str, nil -} - -type SortableResourceNames []api.ResourceName - -func (list SortableResourceNames) Len() int { - return len(list) -} - -func (list SortableResourceNames) Swap(i, j int) { - list[i], list[j] = list[j], list[i] -} - -func (list SortableResourceNames) Less(i, j int) bool { - return list[i] < list[j] -} - -// SortedResourceNames returns the sorted resource names of a resource list. -func SortedResourceNames(list api.ResourceList) []api.ResourceName { - resources := make([]api.ResourceName, 0, len(list)) - for res := range list { - resources = append(resources, res) - } - sort.Sort(SortableResourceNames(resources)) - return resources -} - -type SortableResourceQuotas []api.ResourceQuota - -func (list SortableResourceQuotas) Len() int { - return len(list) -} - -func (list SortableResourceQuotas) Swap(i, j int) { - list[i], list[j] = list[j], list[i] -} - -func (list SortableResourceQuotas) Less(i, j int) bool { - return list[i].Name < list[j].Name -} - -type SortableVolumeMounts []api.VolumeMount - -func (list SortableVolumeMounts) Len() int { - return len(list) -} - -func (list SortableVolumeMounts) Swap(i, j int) { - list[i], list[j] = list[j], list[i] -} - -func (list SortableVolumeMounts) Less(i, j int) bool { - return list[i].MountPath < list[j].MountPath -} - -type SortableVolumeDevices []api.VolumeDevice - -func (list SortableVolumeDevices) Len() int { - return len(list) -} - -func (list SortableVolumeDevices) Swap(i, j int) { - list[i], list[j] = list[j], list[i] -} - -func (list SortableVolumeDevices) Less(i, j int) bool { - return list[i].DevicePath < list[j].DevicePath -} - -var maxAnnotationLen = 140 - -// printAnnotationsMultilineWithFilter prints filtered multiple annotations with a proper alignment. -func printAnnotationsMultilineWithFilter(w PrefixWriter, title string, annotations map[string]string, skip sets.String) { - printAnnotationsMultilineWithIndent(w, "", title, "\t", annotations, skip) -} - -// printAnnotationsMultiline prints multiple annotations with a proper alignment. -func printAnnotationsMultiline(w PrefixWriter, title string, annotations map[string]string) { - printAnnotationsMultilineWithIndent(w, "", title, "\t", annotations, sets.NewString()) -} - -// printAnnotationsMultilineWithIndent prints multiple annotations with a user-defined alignment. -// If annotation string is too long, we omit chars more than 200 length. -func printAnnotationsMultilineWithIndent(w PrefixWriter, initialIndent, title, innerIndent string, annotations map[string]string, skip sets.String) { - - w.Write(LEVEL_0, "%s%s:%s", initialIndent, title, innerIndent) - - if len(annotations) == 0 { - w.WriteLine("") - return - } - - // to print labels in the sorted order - keys := make([]string, 0, len(annotations)) - for key := range annotations { - if skip.Has(key) { - continue - } - keys = append(keys, key) - } - if len(annotations) == 0 { - w.WriteLine("") - return - } - sort.Strings(keys) - indent := initialIndent + innerIndent - for i, key := range keys { - if i != 0 { - w.Write(LEVEL_0, indent) - } - value := strings.TrimSuffix(annotations[key], "\n") - if (len(value)+len(key)+2) > maxAnnotationLen || strings.Contains(value, "\n") { - w.Write(LEVEL_0, "%s:\n", key) - for _, s := range strings.Split(value, "\n") { - w.Write(LEVEL_0, "%s %s\n", indent, shorten(s, maxAnnotationLen-2)) - } - } else { - w.Write(LEVEL_0, "%s: %s\n", key, value) - } - i++ - } -} - -func shorten(s string, maxLength int) string { - if len(s) > maxLength { - return s[:maxLength] + "..." - } - return s -} diff --git a/pkg/printers/internalversion/import_known_versions.go b/pkg/printers/internalversion/import_known_versions.go new file mode 100644 index 00000000000..154e6c76a6c --- /dev/null +++ b/pkg/printers/internalversion/import_known_versions.go @@ -0,0 +1,37 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internalversion + +// These imports are the API groups the client will support. +// TODO: Remove these manual install once we don't need legacy scheme in get comman +import ( + _ "k8s.io/kubernetes/pkg/apis/apps/install" + _ "k8s.io/kubernetes/pkg/apis/authentication/install" + _ "k8s.io/kubernetes/pkg/apis/authorization/install" + _ "k8s.io/kubernetes/pkg/apis/autoscaling/install" + _ "k8s.io/kubernetes/pkg/apis/batch/install" + _ "k8s.io/kubernetes/pkg/apis/certificates/install" + _ "k8s.io/kubernetes/pkg/apis/coordination/install" + _ "k8s.io/kubernetes/pkg/apis/core/install" + _ "k8s.io/kubernetes/pkg/apis/events/install" + _ "k8s.io/kubernetes/pkg/apis/extensions/install" + _ "k8s.io/kubernetes/pkg/apis/policy/install" + _ "k8s.io/kubernetes/pkg/apis/rbac/install" + _ "k8s.io/kubernetes/pkg/apis/scheduling/install" + _ "k8s.io/kubernetes/pkg/apis/settings/install" + _ "k8s.io/kubernetes/pkg/apis/storage/install" +) diff --git a/pkg/printers/internalversion/printers.go b/pkg/printers/internalversion/printers.go index 749d4a2da06..70dadf902cc 100644 --- a/pkg/printers/internalversion/printers.go +++ b/pkg/printers/internalversion/printers.go @@ -86,6 +86,7 @@ func AddHandlers(h printers.PrintHandler) { {Name: "IP", Type: "string", Priority: 1, Description: apiv1.PodStatus{}.SwaggerDoc()["podIP"]}, {Name: "Node", Type: "string", Priority: 1, Description: apiv1.PodSpec{}.SwaggerDoc()["nodeName"]}, {Name: "Nominated Node", Type: "string", Priority: 1, Description: apiv1.PodStatus{}.SwaggerDoc()["nominatedNodeName"]}, + {Name: "Readiness Gates", Type: "string", Priority: 1, Description: apiv1.PodSpec{}.SwaggerDoc()["readinessGates"]}, } h.TableHandler(podColumnDefinitions, printPodList) h.TableHandler(podColumnDefinitions, printPod) @@ -202,8 +203,7 @@ func AddHandlers(h printers.PrintHandler) { statefulSetColumnDefinitions := []metav1beta1.TableColumnDefinition{ {Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]}, - {Name: "Desired", Type: "string", Description: appsv1beta1.StatefulSetSpec{}.SwaggerDoc()["replicas"]}, - {Name: "Current", Type: "string", Description: appsv1beta1.StatefulSetStatus{}.SwaggerDoc()["replicas"]}, + {Name: "Ready", Type: "string", Description: "Number of the pod with ready state"}, {Name: "Age", Type: "string", Description: metav1.ObjectMeta{}.SwaggerDoc()["creationTimestamp"]}, {Name: "Containers", Type: "string", Priority: 1, Description: "Names of each container in the template."}, {Name: "Images", Type: "string", Priority: 1, Description: "Images referenced by each container in the template."}, @@ -312,8 +312,7 @@ func AddHandlers(h printers.PrintHandler) { deploymentColumnDefinitions := []metav1beta1.TableColumnDefinition{ {Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]}, - {Name: "Desired", Type: "string", Description: extensionsv1beta1.DeploymentSpec{}.SwaggerDoc()["replicas"]}, - {Name: "Current", Type: "string", Description: extensionsv1beta1.DeploymentStatus{}.SwaggerDoc()["replicas"]}, + {Name: "Ready", Type: "string", Description: "Number of the pod with ready state"}, {Name: "Up-to-date", Type: "string", Description: extensionsv1beta1.DeploymentStatus{}.SwaggerDoc()["updatedReplicas"]}, {Name: "Available", Type: "string", Description: extensionsv1beta1.DeploymentStatus{}.SwaggerDoc()["availableReplicas"]}, {Name: "Age", Type: "string", Description: metav1.ObjectMeta{}.SwaggerDoc()["creationTimestamp"]}, @@ -663,11 +662,11 @@ func printPod(pod *api.Pod, options printers.PrintOptions) ([]metav1beta1.TableR } row.Cells = append(row.Cells, pod.Name, fmt.Sprintf("%d/%d", readyContainers, totalContainers), reason, int64(restarts), translateTimestampSince(pod.CreationTimestamp)) - if options.Wide { nodeName := pod.Spec.NodeName nominatedNodeName := pod.Status.NominatedNodeName podIP := pod.Status.PodIP + if podIP == "" { podIP = "" } @@ -677,7 +676,24 @@ func printPod(pod *api.Pod, options printers.PrintOptions) ([]metav1beta1.TableR if nominatedNodeName == "" { nominatedNodeName = "" } - row.Cells = append(row.Cells, podIP, nodeName, nominatedNodeName) + + readinessGates := "" + if len(pod.Spec.ReadinessGates) > 0 { + trueConditions := 0 + for _, readinessGate := range pod.Spec.ReadinessGates { + conditionType := readinessGate.ConditionType + for _, condition := range pod.Status.Conditions { + if condition.Type == conditionType { + if condition.Status == api.ConditionTrue { + trueConditions += 1 + } + break + } + } + } + readinessGates = fmt.Sprintf("%d/%d", trueConditions, len(pod.Spec.ReadinessGates)) + } + row.Cells = append(row.Cells, podIP, nodeName, nominatedNodeName, readinessGates) } return []metav1beta1.TableRow{row}, nil @@ -1041,9 +1057,9 @@ func printStatefulSet(obj *apps.StatefulSet, options printers.PrintOptions) ([]m Object: runtime.RawExtension{Object: obj}, } desiredReplicas := obj.Spec.Replicas - currentReplicas := obj.Status.Replicas + readyReplicas := obj.Status.ReadyReplicas createTime := translateTimestampSince(obj.CreationTimestamp) - row.Cells = append(row.Cells, obj.Name, int64(desiredReplicas), int64(currentReplicas), createTime) + row.Cells = append(row.Cells, obj.Name, fmt.Sprintf("%d/%d", int64(readyReplicas), int64(desiredReplicas)), createTime) if options.Wide { names, images := layoutContainerCells(obj.Spec.Template.Spec.Containers) row.Cells = append(row.Cells, names, images) @@ -1553,8 +1569,8 @@ func printDeployment(obj *apps.Deployment, options printers.PrintOptions) ([]met Object: runtime.RawExtension{Object: obj}, } desiredReplicas := obj.Spec.Replicas - currentReplicas := obj.Status.Replicas updatedReplicas := obj.Status.UpdatedReplicas + readyReplicas := obj.Status.ReadyReplicas availableReplicas := obj.Status.AvailableReplicas age := translateTimestampSince(obj.CreationTimestamp) containers := obj.Spec.Template.Spec.Containers @@ -1563,7 +1579,7 @@ func printDeployment(obj *apps.Deployment, options printers.PrintOptions) ([]met // this shouldn't happen if LabelSelector passed validation return nil, err } - row.Cells = append(row.Cells, obj.Name, int64(desiredReplicas), int64(currentReplicas), int64(updatedReplicas), int64(availableReplicas), age) + row.Cells = append(row.Cells, obj.Name, fmt.Sprintf("%d/%d", int64(readyReplicas), int64(desiredReplicas)), int64(updatedReplicas), int64(availableReplicas), age) if options.Wide { containers, images := layoutContainerCells(containers) row.Cells = append(row.Cells, containers, images, selector.String()) @@ -1949,3 +1965,33 @@ func printPriorityClassList(list *scheduling.PriorityClassList, options printers } return rows, nil } + +func printBoolPtr(value *bool) string { + if value != nil { + return printBool(*value) + } + + return "" +} + +func printBool(value bool) string { + if value { + return "True" + } + + return "False" +} + +type SortableResourceNames []api.ResourceName + +func (list SortableResourceNames) Len() int { + return len(list) +} + +func (list SortableResourceNames) Swap(i, j int) { + list[i], list[j] = list[j], list[i] +} + +func (list SortableResourceNames) Less(i, j int) bool { + return list[i] < list[j] +} diff --git a/pkg/printers/internalversion/printers_test.go b/pkg/printers/internalversion/printers_test.go index 01afac8198a..bdc1757d82f 100644 --- a/pkg/printers/internalversion/printers_test.go +++ b/pkg/printers/internalversion/printers_test.go @@ -28,7 +28,7 @@ import ( "testing" "time" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -1649,6 +1649,9 @@ func TestPrintPod(t *testing.T) { } func TestPrintPodwide(t *testing.T) { + condition1 := "condition1" + condition2 := "condition2" + condition3 := "condition3" tests := []struct { pod api.Pod expect []metav1beta1.TableRow @@ -1660,8 +1663,29 @@ func TestPrintPodwide(t *testing.T) { Spec: api.PodSpec{ Containers: make([]api.Container, 2), NodeName: "test1", + ReadinessGates: []api.PodReadinessGate{ + { + ConditionType: api.PodConditionType(condition1), + }, + { + ConditionType: api.PodConditionType(condition2), + }, + { + ConditionType: api.PodConditionType(condition3), + }, + }, }, Status: api.PodStatus{ + Conditions: []api.PodCondition{ + { + Type: api.PodConditionType(condition1), + Status: api.ConditionFalse, + }, + { + Type: api.PodConditionType(condition2), + Status: api.ConditionTrue, + }, + }, Phase: "podPhase", PodIP: "1.1.1.1", ContainerStatuses: []api.ContainerStatus{ @@ -1671,7 +1695,7 @@ func TestPrintPodwide(t *testing.T) { NominatedNodeName: "node1", }, }, - []metav1beta1.TableRow{{Cells: []interface{}{"test1", "1/2", "podPhase", int64(6), "", "1.1.1.1", "test1", "node1"}}}, + []metav1beta1.TableRow{{Cells: []interface{}{"test1", "1/2", "podPhase", int64(6), "", "1.1.1.1", "test1", "node1", "1/3"}}}, }, { // Test when the NodeName and PodIP are none @@ -1690,7 +1714,7 @@ func TestPrintPodwide(t *testing.T) { }, }, }, - []metav1beta1.TableRow{{Cells: []interface{}{"test2", "1/2", "ContainerWaitingReason", int64(6), "", "", "", ""}}}, + []metav1beta1.TableRow{{Cells: []interface{}{"test2", "1/2", "ContainerWaitingReason", int64(6), "", "", "", "", ""}}}, }, } @@ -2006,8 +2030,8 @@ func TestPrintDeployment(t *testing.T) { UnavailableReplicas: 4, }, }, - "test1\t5\t10\t2\t1\t0s\n", - "test1\t5\t10\t2\t1\t0s\tfake-container1,fake-container2\tfake-image1,fake-image2\tfoo=bar\n", + "test1\t0/5\t2\t1\t0s\n", + "test1\t0/5\t2\t1\t0s\tfake-container1,fake-container2\tfake-image1,fake-image2\tfoo=bar\n", }, } @@ -3545,3 +3569,27 @@ func verifyTable(t *testing.T, table *metav1beta1.Table) { t.Errorf("unexpected panic during deepcopy of table %#v: %v", table, panicErr) } } + +// VerifyDatesInOrder checks the start of each line for a RFC1123Z date +// and posts error if all subsequent dates are not equal or increasing +func VerifyDatesInOrder( + resultToTest, rowDelimiter, columnDelimiter string, t *testing.T) { + lines := strings.Split(resultToTest, rowDelimiter) + var previousTime time.Time + for _, str := range lines { + columns := strings.Split(str, columnDelimiter) + if len(columns) > 0 { + currentTime, err := time.Parse(time.RFC1123Z, columns[0]) + if err == nil { + if previousTime.After(currentTime) { + t.Errorf( + "Output is not sorted by time. %s should be listed after %s. Complete output: %s", + previousTime.Format(time.RFC1123Z), + currentTime.Format(time.RFC1123Z), + resultToTest) + } + previousTime = currentTime + } + } + } +} diff --git a/pkg/probe/exec/BUILD b/pkg/probe/exec/BUILD index 42b68b510b6..317dc0fc0cd 100644 --- a/pkg/probe/exec/BUILD +++ b/pkg/probe/exec/BUILD @@ -12,7 +12,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/probe/exec", deps = [ "//pkg/probe:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/probe/exec/exec.go b/pkg/probe/exec/exec.go index 5901e35a6a1..a6ae523aa62 100644 --- a/pkg/probe/exec/exec.go +++ b/pkg/probe/exec/exec.go @@ -20,7 +20,7 @@ import ( "k8s.io/kubernetes/pkg/probe" "k8s.io/utils/exec" - "github.com/golang/glog" + "k8s.io/klog" ) // New creates a Prober. @@ -40,7 +40,7 @@ type execProber struct{} // errors if any. func (pr execProber) Probe(e exec.Cmd) (probe.Result, string, error) { data, err := e.CombinedOutput() - glog.V(4).Infof("Exec probe response: %q", string(data)) + klog.V(4).Infof("Exec probe response: %q", string(data)) if err != nil { exit, ok := err.(exec.ExitError) if ok { diff --git a/pkg/probe/http/BUILD b/pkg/probe/http/BUILD index 5c788fc3ab9..98387df305b 100644 --- a/pkg/probe/http/BUILD +++ b/pkg/probe/http/BUILD @@ -14,7 +14,7 @@ go_library( "//pkg/probe:go_default_library", "//pkg/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/probe/http/http.go b/pkg/probe/http/http.go index 11bbddfa8be..e9bcc0f3e1b 100644 --- a/pkg/probe/http/http.go +++ b/pkg/probe/http/http.go @@ -28,7 +28,7 @@ import ( "k8s.io/kubernetes/pkg/probe" "k8s.io/kubernetes/pkg/version" - "github.com/golang/glog" + "k8s.io/klog" ) // New creates Prober that will skip TLS verification while probing. @@ -96,9 +96,9 @@ func DoHTTPProbe(url *url.URL, headers http.Header, client GetHTTPInterface) (pr } body := string(b) if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest { - glog.V(4).Infof("Probe succeeded for %s, Response: %v", url.String(), *res) + klog.V(4).Infof("Probe succeeded for %s, Response: %v", url.String(), *res) return probe.Success, body, nil } - glog.V(4).Infof("Probe failed for %s with request headers %v, response body: %v", url.String(), headers, body) + klog.V(4).Infof("Probe failed for %s with request headers %v, response body: %v", url.String(), headers, body) return probe.Failure, fmt.Sprintf("HTTP probe failed with statuscode: %d", res.StatusCode), nil } diff --git a/pkg/probe/tcp/BUILD b/pkg/probe/tcp/BUILD index b14c197e255..929f1e4fad3 100644 --- a/pkg/probe/tcp/BUILD +++ b/pkg/probe/tcp/BUILD @@ -12,7 +12,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/probe/tcp", deps = [ "//pkg/probe:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/probe/tcp/tcp.go b/pkg/probe/tcp/tcp.go index cf18eb1a5b9..7b183e0b45a 100644 --- a/pkg/probe/tcp/tcp.go +++ b/pkg/probe/tcp/tcp.go @@ -23,7 +23,7 @@ import ( "k8s.io/kubernetes/pkg/probe" - "github.com/golang/glog" + "k8s.io/klog" ) // New creates Prober. @@ -55,7 +55,7 @@ func DoTCPProbe(addr string, timeout time.Duration) (probe.Result, string, error } err = conn.Close() if err != nil { - glog.Errorf("Unexpected error closing TCP probe socket: %v (%#v)", err, err) + klog.Errorf("Unexpected error closing TCP probe socket: %v (%#v)", err, err) } return probe.Success, "", nil } diff --git a/pkg/proxy/BUILD b/pkg/proxy/BUILD index 9920c511765..bb290126fea 100644 --- a/pkg/proxy/BUILD +++ b/pkg/proxy/BUILD @@ -23,7 +23,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/proxy/apis/config/types.go b/pkg/proxy/apis/config/types.go index e309317b0a2..dacee32bb3a 100644 --- a/pkg/proxy/apis/config/types.go +++ b/pkg/proxy/apis/config/types.go @@ -172,7 +172,7 @@ type IPVSSchedulerMethod string const ( // RoundRobin distributes jobs equally amongst the available real servers. RoundRobin IPVSSchedulerMethod = "rr" - // WeightedRoundRobin assigns jobs to real servers proportionally to there real servers' weight. + // WeightedRoundRobin assigns jobs to real servers proportionally to their real servers' weight. // Servers with higher weights receive new jobs first and get more jobs than servers with lower weights. // Servers with equal weights get an equal distribution of new jobs. WeightedRoundRobin IPVSSchedulerMethod = "wrr" diff --git a/pkg/proxy/config/BUILD b/pkg/proxy/config/BUILD index e45009ff4a8..b68b6f036e2 100644 --- a/pkg/proxy/config/BUILD +++ b/pkg/proxy/config/BUILD @@ -20,7 +20,7 @@ go_library( "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/proxy/config/config.go b/pkg/proxy/config/config.go index 03256f040d9..2b22e643b97 100644 --- a/pkg/proxy/config/config.go +++ b/pkg/proxy/config/config.go @@ -20,12 +20,12 @@ import ( "fmt" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" coreinformers "k8s.io/client-go/informers/core/v1" listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller" ) @@ -99,15 +99,15 @@ func (c *EndpointsConfig) RegisterEventHandler(handler EndpointsHandler) { func (c *EndpointsConfig) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() - glog.Info("Starting endpoints config controller") - defer glog.Info("Shutting down endpoints config controller") + klog.Info("Starting endpoints config controller") + defer klog.Info("Shutting down endpoints config controller") if !controller.WaitForCacheSync("endpoints config", stopCh, c.listerSynced) { return } for i := range c.eventHandlers { - glog.V(3).Infof("Calling handler.OnEndpointsSynced()") + klog.V(3).Infof("Calling handler.OnEndpointsSynced()") c.eventHandlers[i].OnEndpointsSynced() } @@ -121,7 +121,7 @@ func (c *EndpointsConfig) handleAddEndpoints(obj interface{}) { return } for i := range c.eventHandlers { - glog.V(4).Infof("Calling handler.OnEndpointsAdd") + klog.V(4).Infof("Calling handler.OnEndpointsAdd") c.eventHandlers[i].OnEndpointsAdd(endpoints) } } @@ -138,7 +138,7 @@ func (c *EndpointsConfig) handleUpdateEndpoints(oldObj, newObj interface{}) { return } for i := range c.eventHandlers { - glog.V(4).Infof("Calling handler.OnEndpointsUpdate") + klog.V(4).Infof("Calling handler.OnEndpointsUpdate") c.eventHandlers[i].OnEndpointsUpdate(oldEndpoints, endpoints) } } @@ -157,7 +157,7 @@ func (c *EndpointsConfig) handleDeleteEndpoints(obj interface{}) { } } for i := range c.eventHandlers { - glog.V(4).Infof("Calling handler.OnEndpointsDelete") + klog.V(4).Infof("Calling handler.OnEndpointsDelete") c.eventHandlers[i].OnEndpointsDelete(endpoints) } } @@ -199,15 +199,15 @@ func (c *ServiceConfig) RegisterEventHandler(handler ServiceHandler) { func (c *ServiceConfig) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() - glog.Info("Starting service config controller") - defer glog.Info("Shutting down service config controller") + klog.Info("Starting service config controller") + defer klog.Info("Shutting down service config controller") if !controller.WaitForCacheSync("service config", stopCh, c.listerSynced) { return } for i := range c.eventHandlers { - glog.V(3).Info("Calling handler.OnServiceSynced()") + klog.V(3).Info("Calling handler.OnServiceSynced()") c.eventHandlers[i].OnServiceSynced() } @@ -221,7 +221,7 @@ func (c *ServiceConfig) handleAddService(obj interface{}) { return } for i := range c.eventHandlers { - glog.V(4).Info("Calling handler.OnServiceAdd") + klog.V(4).Info("Calling handler.OnServiceAdd") c.eventHandlers[i].OnServiceAdd(service) } } @@ -238,7 +238,7 @@ func (c *ServiceConfig) handleUpdateService(oldObj, newObj interface{}) { return } for i := range c.eventHandlers { - glog.V(4).Info("Calling handler.OnServiceUpdate") + klog.V(4).Info("Calling handler.OnServiceUpdate") c.eventHandlers[i].OnServiceUpdate(oldService, service) } } @@ -257,7 +257,7 @@ func (c *ServiceConfig) handleDeleteService(obj interface{}) { } } for i := range c.eventHandlers { - glog.V(4).Info("Calling handler.OnServiceDelete") + klog.V(4).Info("Calling handler.OnServiceDelete") c.eventHandlers[i].OnServiceDelete(service) } } diff --git a/pkg/proxy/endpoints.go b/pkg/proxy/endpoints.go index 06c7d3ad5ab..067be71d79b 100644 --- a/pkg/proxy/endpoints.go +++ b/pkg/proxy/endpoints.go @@ -22,7 +22,7 @@ import ( "strconv" "sync" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -197,7 +197,7 @@ func (ect *EndpointChangeTracker) endpointsToEndpointsMap(endpoints *v1.Endpoint for i := range ss.Ports { port := &ss.Ports[i] if port.Port == 0 { - glog.Warningf("ignoring invalid endpoint port %s", port.Name) + klog.Warningf("ignoring invalid endpoint port %s", port.Name) continue } svcPortName := ServicePortName{ @@ -207,7 +207,7 @@ func (ect *EndpointChangeTracker) endpointsToEndpointsMap(endpoints *v1.Endpoint for i := range ss.Addresses { addr := &ss.Addresses[i] if addr.IP == "" { - glog.Warningf("ignoring invalid endpoint port %s with empty host", port.Name) + klog.Warningf("ignoring invalid endpoint port %s with empty host", port.Name) continue } // Filter out the incorrect IP version case. @@ -226,12 +226,12 @@ func (ect *EndpointChangeTracker) endpointsToEndpointsMap(endpoints *v1.Endpoint endpointsMap[svcPortName] = append(endpointsMap[svcPortName], baseEndpointInfo) } } - if glog.V(3) { + if klog.V(3) { newEPList := []string{} for _, ep := range endpointsMap[svcPortName] { newEPList = append(newEPList, ep.String()) } - glog.Infof("Setting endpoints for %q to %+v", svcPortName, newEPList) + klog.Infof("Setting endpoints for %q to %+v", svcPortName, newEPList) } } } @@ -299,7 +299,7 @@ func detectStaleConnections(oldEndpointsMap, newEndpointsMap EndpointsMap, stale } } if stale { - glog.V(4).Infof("Stale endpoint %v -> %v", svcPortName, ep.String()) + klog.V(4).Infof("Stale endpoint %v -> %v", svcPortName, ep.String()) *staleEndpoints = append(*staleEndpoints, ServiceEndpoint{Endpoint: ep.String(), ServicePortName: svcPortName}) } } diff --git a/pkg/proxy/healthcheck/BUILD b/pkg/proxy/healthcheck/BUILD index b711fcdb07d..f94de593ce0 100644 --- a/pkg/proxy/healthcheck/BUILD +++ b/pkg/proxy/healthcheck/BUILD @@ -20,8 +20,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/renstrom/dedent:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/proxy/healthcheck/healthcheck.go b/pkg/proxy/healthcheck/healthcheck.go index 0c59a38cf79..5dc3f009ecb 100644 --- a/pkg/proxy/healthcheck/healthcheck.go +++ b/pkg/proxy/healthcheck/healthcheck.go @@ -25,8 +25,8 @@ import ( "sync/atomic" "time" - "github.com/golang/glog" "github.com/renstrom/dedent" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -133,9 +133,9 @@ func (hcs *server) SyncServices(newServices map[types.NamespacedName]uint16) err // Remove any that are not needed any more. for nsn, svc := range hcs.services { if port, found := newServices[nsn]; !found || port != svc.port { - glog.V(2).Infof("Closing healthcheck %q on port %d", nsn.String(), svc.port) + klog.V(2).Infof("Closing healthcheck %q on port %d", nsn.String(), svc.port) if err := svc.listener.Close(); err != nil { - glog.Errorf("Close(%v): %v", svc.listener.Addr(), err) + klog.Errorf("Close(%v): %v", svc.listener.Addr(), err) } delete(hcs.services, nsn) } @@ -144,11 +144,11 @@ func (hcs *server) SyncServices(newServices map[types.NamespacedName]uint16) err // Add any that are needed. for nsn, port := range newServices { if hcs.services[nsn] != nil { - glog.V(3).Infof("Existing healthcheck %q on port %d", nsn.String(), port) + klog.V(3).Infof("Existing healthcheck %q on port %d", nsn.String(), port) continue } - glog.V(2).Infof("Opening healthcheck %q on port %d", nsn.String(), port) + klog.V(2).Infof("Opening healthcheck %q on port %d", nsn.String(), port) svc := &hcInstance{port: port} addr := fmt.Sprintf(":%d", port) svc.server = hcs.httpFactory.New(addr, hcHandler{name: nsn, hcs: hcs}) @@ -166,19 +166,19 @@ func (hcs *server) SyncServices(newServices map[types.NamespacedName]uint16) err UID: types.UID(nsn.String()), }, api.EventTypeWarning, "FailedToStartServiceHealthcheck", msg) } - glog.Error(msg) + klog.Error(msg) continue } hcs.services[nsn] = svc go func(nsn types.NamespacedName, svc *hcInstance) { // Serve() will exit when the listener is closed. - glog.V(3).Infof("Starting goroutine for healthcheck %q on port %d", nsn.String(), svc.port) + klog.V(3).Infof("Starting goroutine for healthcheck %q on port %d", nsn.String(), svc.port) if err := svc.server.Serve(svc.listener); err != nil { - glog.V(3).Infof("Healthcheck %q closed: %v", nsn.String(), err) + klog.V(3).Infof("Healthcheck %q closed: %v", nsn.String(), err) return } - glog.V(3).Infof("Healthcheck %q closed", nsn.String()) + klog.V(3).Infof("Healthcheck %q closed", nsn.String()) }(nsn, svc) } return nil @@ -203,7 +203,7 @@ func (h hcHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { svc, ok := h.hcs.services[h.name] if !ok || svc == nil { h.hcs.lock.Unlock() - glog.Errorf("Received request for closed healthcheck %q", h.name.String()) + klog.Errorf("Received request for closed healthcheck %q", h.name.String()) return } count := svc.endpoints @@ -232,10 +232,10 @@ func (hcs *server) SyncEndpoints(newEndpoints map[types.NamespacedName]int) erro for nsn, count := range newEndpoints { if hcs.services[nsn] == nil { - glog.V(3).Infof("Not saving endpoints for unknown healthcheck %q", nsn.String()) + klog.V(3).Infof("Not saving endpoints for unknown healthcheck %q", nsn.String()) continue } - glog.V(3).Infof("Reporting %d endpoints for healthcheck %q", count, nsn.String()) + klog.V(3).Infof("Reporting %d endpoints for healthcheck %q", count, nsn.String()) hcs.services[nsn].endpoints = count } for nsn, hci := range hcs.services { @@ -306,7 +306,7 @@ func (hs *HealthzServer) Run() { server := hs.httpFactory.New(hs.addr, serveMux) go wait.Until(func() { - glog.V(3).Infof("Starting goroutine for healthz on %s", hs.addr) + klog.V(3).Infof("Starting goroutine for healthz on %s", hs.addr) listener, err := hs.listener.Listen(hs.addr) if err != nil { @@ -314,15 +314,15 @@ func (hs *HealthzServer) Run() { if hs.recorder != nil { hs.recorder.Eventf(hs.nodeRef, api.EventTypeWarning, "FailedToStartNodeHealthcheck", msg) } - glog.Error(msg) + klog.Error(msg) return } if err := server.Serve(listener); err != nil { - glog.Errorf("Healthz closed with error: %v", err) + klog.Errorf("Healthz closed with error: %v", err) return } - glog.Error("Unexpected healthz closed.") + klog.Error("Unexpected healthz closed.") }, nodeHealthzRetryInterval, wait.NeverStop) } diff --git a/pkg/proxy/iptables/BUILD b/pkg/proxy/iptables/BUILD index 84154ed565f..2d1794529cd 100644 --- a/pkg/proxy/iptables/BUILD +++ b/pkg/proxy/iptables/BUILD @@ -25,7 +25,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) @@ -46,7 +46,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec/testing:go_default_library", ], diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index db76a7770a8..1fb994be32f 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -32,7 +32,7 @@ import ( "sync/atomic" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -182,7 +182,7 @@ func newEndpointInfo(baseInfo *proxy.BaseEndpointInfo) proxy.Endpoint { func (e *endpointsInfo) Equal(other proxy.Endpoint) bool { o, ok := other.(*endpointsInfo) if !ok { - glog.Error("Failed to cast endpointsInfo") + klog.Error("Failed to cast endpointsInfo") return false } return e.Endpoint == o.Endpoint && @@ -303,7 +303,7 @@ func NewProxier(ipt utiliptables.Interface, // are connected to a Linux bridge (but not SDN bridges). Until most // plugins handle this, log when config is missing if val, err := sysctl.GetSysctl(sysctlBridgeCallIPTables); err == nil && val != 1 { - glog.Warning("missing br-netfilter module or unset sysctl br-nf-call-iptables; proxy may not work as intended") + klog.Warning("missing br-netfilter module or unset sysctl br-nf-call-iptables; proxy may not work as intended") } // Generate the masquerade mark to use for SNAT rules. @@ -311,12 +311,12 @@ func NewProxier(ipt utiliptables.Interface, masqueradeMark := fmt.Sprintf("%#08x/%#08x", masqueradeValue, masqueradeValue) if nodeIP == nil { - glog.Warning("invalid nodeIP, initializing kube-proxy with 127.0.0.1 as nodeIP") + klog.Warning("invalid nodeIP, initializing kube-proxy with 127.0.0.1 as nodeIP") nodeIP = net.ParseIP("127.0.0.1") } if len(clusterCIDR) == 0 { - glog.Warning("clusterCIDR not specified, unable to distinguish between internal and external traffic") + klog.Warning("clusterCIDR not specified, unable to distinguish between internal and external traffic") } else if utilnet.IsIPv6CIDR(clusterCIDR) != ipt.IsIpv6() { return nil, fmt.Errorf("clusterCIDR %s has incorrect IP version: expect isIPv6=%t", clusterCIDR, ipt.IsIpv6()) } @@ -352,7 +352,7 @@ func NewProxier(ipt utiliptables.Interface, networkInterfacer: utilproxy.RealNetwork{}, } burstSyncs := 2 - glog.V(3).Infof("minSyncPeriod: %v, syncPeriod: %v, burstSyncs: %d", minSyncPeriod, syncPeriod, burstSyncs) + klog.V(3).Infof("minSyncPeriod: %v, syncPeriod: %v, burstSyncs: %d", minSyncPeriod, syncPeriod, burstSyncs) proxier.syncRunner = async.NewBoundedFrequencyRunner("sync-runner", proxier.syncProxyRules, minSyncPeriod, syncPeriod, burstSyncs) return proxier, nil } @@ -392,7 +392,7 @@ func CleanupLeftovers(ipt utiliptables.Interface) (encounteredError bool) { ) if err := ipt.DeleteRule(chain.table, chain.sourceChain, args...); err != nil { if !utiliptables.IsNotFoundError(err) { - glog.Errorf("Error removing pure-iptables proxy rule: %v", err) + klog.Errorf("Error removing pure-iptables proxy rule: %v", err) encounteredError = true } } @@ -401,7 +401,7 @@ func CleanupLeftovers(ipt utiliptables.Interface) (encounteredError bool) { // Flush and remove all of our "-t nat" chains. iptablesData := bytes.NewBuffer(nil) if err := ipt.SaveInto(utiliptables.TableNAT, iptablesData); err != nil { - glog.Errorf("Failed to execute iptables-save for %s: %v", utiliptables.TableNAT, err) + klog.Errorf("Failed to execute iptables-save for %s: %v", utiliptables.TableNAT, err) encounteredError = true } else { existingNATChains := utiliptables.GetChainLines(utiliptables.TableNAT, iptablesData.Bytes()) @@ -429,7 +429,7 @@ func CleanupLeftovers(ipt utiliptables.Interface) (encounteredError bool) { // Write it. err = ipt.Restore(utiliptables.TableNAT, natLines, utiliptables.NoFlushTables, utiliptables.RestoreCounters) if err != nil { - glog.Errorf("Failed to execute iptables-restore for %s: %v", utiliptables.TableNAT, err) + klog.Errorf("Failed to execute iptables-restore for %s: %v", utiliptables.TableNAT, err) encounteredError = true } } @@ -437,7 +437,7 @@ func CleanupLeftovers(ipt utiliptables.Interface) (encounteredError bool) { // Flush and remove all of our "-t filter" chains. iptablesData.Reset() if err := ipt.SaveInto(utiliptables.TableFilter, iptablesData); err != nil { - glog.Errorf("Failed to execute iptables-save for %s: %v", utiliptables.TableFilter, err) + klog.Errorf("Failed to execute iptables-save for %s: %v", utiliptables.TableFilter, err) encounteredError = true } else { existingFilterChains := utiliptables.GetChainLines(utiliptables.TableFilter, iptablesData.Bytes()) @@ -455,7 +455,7 @@ func CleanupLeftovers(ipt utiliptables.Interface) (encounteredError bool) { filterLines := append(filterChains.Bytes(), filterRules.Bytes()...) // Write it. if err := ipt.Restore(utiliptables.TableFilter, filterLines, utiliptables.NoFlushTables, utiliptables.RestoreCounters); err != nil { - glog.Errorf("Failed to execute iptables-restore for %s: %v", utiliptables.TableFilter, err) + klog.Errorf("Failed to execute iptables-restore for %s: %v", utiliptables.TableFilter, err) encounteredError = true } } @@ -609,7 +609,7 @@ func (proxier *Proxier) deleteEndpointConnections(connectionMap []proxy.ServiceE endpointIP := utilproxy.IPPart(epSvcPair.Endpoint) err := conntrack.ClearEntriesForNAT(proxier.exec, svcInfo.ClusterIPString(), endpointIP, v1.ProtocolUDP) if err != nil { - glog.Errorf("Failed to delete %s endpoint connections, error: %v", epSvcPair.ServicePortName.String(), err) + klog.Errorf("Failed to delete %s endpoint connections, error: %v", epSvcPair.ServicePortName.String(), err) } } } @@ -638,11 +638,11 @@ func (proxier *Proxier) syncProxyRules() { start := time.Now() defer func() { metrics.SyncProxyRulesLatency.Observe(metrics.SinceInMicroseconds(start)) - glog.V(4).Infof("syncProxyRules took %v", time.Since(start)) + klog.V(4).Infof("syncProxyRules took %v", time.Since(start)) }() // don't sync rules till we've received services and endpoints if !proxier.endpointsSynced || !proxier.servicesSynced { - glog.V(2).Info("Not syncing iptables until Services and Endpoints have been received from master") + klog.V(2).Info("Not syncing iptables until Services and Endpoints have been received from master") return } @@ -656,17 +656,17 @@ func (proxier *Proxier) syncProxyRules() { // merge stale services gathered from updateEndpointsMap for _, svcPortName := range endpointUpdateResult.StaleServiceNames { if svcInfo, ok := proxier.serviceMap[svcPortName]; ok && svcInfo != nil && svcInfo.GetProtocol() == v1.ProtocolUDP { - glog.V(2).Infof("Stale udp service %v -> %s", svcPortName, svcInfo.ClusterIPString()) + klog.V(2).Infof("Stale udp service %v -> %s", svcPortName, svcInfo.ClusterIPString()) staleServices.Insert(svcInfo.ClusterIPString()) } } - glog.V(3).Info("Syncing iptables rules") + klog.V(3).Info("Syncing iptables rules") // Create and link the kube chains. for _, chain := range iptablesJumpChains { if _, err := proxier.iptables.EnsureChain(chain.table, chain.chain); err != nil { - glog.Errorf("Failed to ensure that %s chain %s exists: %v", chain.table, kubeServicesChain, err) + klog.Errorf("Failed to ensure that %s chain %s exists: %v", chain.table, kubeServicesChain, err) return } args := append(chain.extraArgs, @@ -674,7 +674,7 @@ func (proxier *Proxier) syncProxyRules() { "-j", string(chain.chain), ) if _, err := proxier.iptables.EnsureRule(utiliptables.Prepend, chain.table, chain.sourceChain, args...); err != nil { - glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", chain.table, chain.sourceChain, chain.chain, err) + klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", chain.table, chain.sourceChain, chain.chain, err) return } } @@ -689,7 +689,7 @@ func (proxier *Proxier) syncProxyRules() { proxier.existingFilterChainsData.Reset() err := proxier.iptables.SaveInto(utiliptables.TableFilter, proxier.existingFilterChainsData) if err != nil { // if we failed to get any rules - glog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err) + klog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err) } else { // otherwise parse the output existingFilterChains = utiliptables.GetChainLines(utiliptables.TableFilter, proxier.existingFilterChainsData.Bytes()) } @@ -699,7 +699,7 @@ func (proxier *Proxier) syncProxyRules() { proxier.iptablesData.Reset() err = proxier.iptables.SaveInto(utiliptables.TableNAT, proxier.iptablesData) if err != nil { // if we failed to get any rules - glog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err) + klog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err) } else { // otherwise parse the output existingNATChains = utiliptables.GetChainLines(utiliptables.TableNAT, proxier.iptablesData.Bytes()) } @@ -780,7 +780,7 @@ func (proxier *Proxier) syncProxyRules() { for svcName, svc := range proxier.serviceMap { svcInfo, ok := svc.(*serviceInfo) if !ok { - glog.Errorf("Failed to cast serviceInfo %q", svcName.String()) + klog.Errorf("Failed to cast serviceInfo %q", svcName.String()) continue } isIPv6 := utilnet.IsIPv6(svcInfo.ClusterIP) @@ -848,7 +848,7 @@ func (proxier *Proxier) syncProxyRules() { // machine, hold the local port open so no other process can open it // (because the socket might open but it would never work). if local, err := utilproxy.IsLocalIP(externalIP); err != nil { - glog.Errorf("can't determine if IP is local, assuming not: %v", err) + klog.Errorf("can't determine if IP is local, assuming not: %v", err) } else if local && (svcInfo.GetProtocol() != v1.ProtocolSCTP) { lp := utilproxy.LocalPort{ Description: "externalIP for " + svcNameString, @@ -857,7 +857,7 @@ func (proxier *Proxier) syncProxyRules() { Protocol: protocol, } if proxier.portsMap[lp] != nil { - glog.V(4).Infof("Port %s was open before and is still needed", lp.String()) + klog.V(4).Infof("Port %s was open before and is still needed", lp.String()) replacementPortsMap[lp] = proxier.portsMap[lp] } else { socket, err := proxier.portMapper.OpenLocalPort(&lp) @@ -871,7 +871,7 @@ func (proxier *Proxier) syncProxyRules() { UID: types.UID(proxier.hostname), Namespace: "", }, v1.EventTypeWarning, err.Error(), msg) - glog.Error(msg) + klog.Error(msg) continue } replacementPortsMap[lp] = socket @@ -991,7 +991,7 @@ func (proxier *Proxier) syncProxyRules() { // (because the socket might open but it would never work). addresses, err := utilproxy.GetNodeAddresses(proxier.nodePortAddresses, proxier.networkInterfacer) if err != nil { - glog.Errorf("Failed to get node ip address matching nodeport cidr: %v", err) + klog.Errorf("Failed to get node ip address matching nodeport cidr: %v", err) continue } @@ -1016,12 +1016,12 @@ func (proxier *Proxier) syncProxyRules() { // For ports on node IPs, open the actual port and hold it. for _, lp := range lps { if proxier.portsMap[lp] != nil { - glog.V(4).Infof("Port %s was open before and is still needed", lp.String()) + klog.V(4).Infof("Port %s was open before and is still needed", lp.String()) replacementPortsMap[lp] = proxier.portsMap[lp] } else if svcInfo.GetProtocol() != v1.ProtocolSCTP { socket, err := proxier.portMapper.OpenLocalPort(&lp) if err != nil { - glog.Errorf("can't open %s, skipping this nodePort: %v", lp.String(), err) + klog.Errorf("can't open %s, skipping this nodePort: %v", lp.String(), err) continue } if lp.Protocol == "udp" { @@ -1031,7 +1031,7 @@ func (proxier *Proxier) syncProxyRules() { // See issue: https://github.com/kubernetes/kubernetes/issues/49881 err := conntrack.ClearEntriesForPort(proxier.exec, lp.Port, isIPv6, v1.ProtocolUDP) if err != nil { - glog.Errorf("Failed to clear udp conntrack for port %d, error: %v", lp.Port, err) + klog.Errorf("Failed to clear udp conntrack for port %d, error: %v", lp.Port, err) } } replacementPortsMap[lp] = socket @@ -1087,7 +1087,7 @@ func (proxier *Proxier) syncProxyRules() { for _, ep := range proxier.endpointsMap[svcName] { epInfo, ok := ep.(*endpointsInfo) if !ok { - glog.Errorf("Failed to cast endpointsInfo %q", ep.String()) + klog.Errorf("Failed to cast endpointsInfo %q", ep.String()) continue } endpoints = append(endpoints, epInfo) @@ -1253,7 +1253,7 @@ func (proxier *Proxier) syncProxyRules() { // other service portal rules. addresses, err := utilproxy.GetNodeAddresses(proxier.nodePortAddresses, proxier.networkInterfacer) if err != nil { - glog.Errorf("Failed to get node ip address matching nodeport cidr") + klog.Errorf("Failed to get node ip address matching nodeport cidr") } else { isIPv6 := proxier.iptables.IsIpv6() for address := range addresses { @@ -1270,7 +1270,7 @@ func (proxier *Proxier) syncProxyRules() { } // Ignore IP addresses with incorrect version if isIPv6 && !utilnet.IsIPv6String(address) || !isIPv6 && utilnet.IsIPv6String(address) { - glog.Errorf("IP address %s has incorrect IP version", address) + klog.Errorf("IP address %s has incorrect IP version", address) continue } // create nodeport rules for each IP one by one @@ -1329,12 +1329,12 @@ func (proxier *Proxier) syncProxyRules() { proxier.iptablesData.Write(proxier.natChains.Bytes()) proxier.iptablesData.Write(proxier.natRules.Bytes()) - glog.V(5).Infof("Restoring iptables rules: %s", proxier.iptablesData.Bytes()) + klog.V(5).Infof("Restoring iptables rules: %s", proxier.iptablesData.Bytes()) err = proxier.iptables.RestoreAll(proxier.iptablesData.Bytes(), utiliptables.NoFlushTables, utiliptables.RestoreCounters) if err != nil { - glog.Errorf("Failed to execute iptables-restore: %v", err) + klog.Errorf("Failed to execute iptables-restore: %v", err) // Revert new local ports. - glog.V(2).Infof("Closing local ports after iptables-restore failure") + klog.V(2).Infof("Closing local ports after iptables-restore failure") utilproxy.RevertPorts(replacementPortsMap, proxier.portsMap) return } @@ -1356,17 +1356,17 @@ func (proxier *Proxier) syncProxyRules() { // not "OnlyLocal", but the services list will not, and the healthChecker // will just drop those endpoints. if err := proxier.healthChecker.SyncServices(serviceUpdateResult.HCServiceNodePorts); err != nil { - glog.Errorf("Error syncing healtcheck services: %v", err) + klog.Errorf("Error syncing healthcheck services: %v", err) } if err := proxier.healthChecker.SyncEndpoints(endpointUpdateResult.HCEndpointsLocalIPSize); err != nil { - glog.Errorf("Error syncing healthcheck endpoints: %v", err) + klog.Errorf("Error syncing healthcheck endpoints: %v", err) } // Finish housekeeping. // TODO: these could be made more consistent. for _, svcIP := range staleServices.UnsortedList() { if err := conntrack.ClearEntriesForIP(proxier.exec, svcIP, v1.ProtocolUDP); err != nil { - glog.Errorf("Failed to delete stale service IP %s connections, error: %v", svcIP, err) + klog.Errorf("Failed to delete stale service IP %s connections, error: %v", svcIP, err) } } proxier.deleteEndpointConnections(endpointUpdateResult.StaleEndpoints) @@ -1424,6 +1424,6 @@ func openLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) { default: return nil, fmt.Errorf("unknown protocol %q", lp.Protocol) } - glog.V(2).Infof("Opened local port %s", lp.String()) + klog.V(2).Infof("Opened local port %s", lp.String()) return socket, nil } diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index b14d0f416fc..94ae8802a22 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -281,7 +281,7 @@ func TestDeleteEndpointConnections(t *testing.T) { // Run the test cases for _, tc := range testCases { priorExecs := fexec.CommandCalls - priorGlogErrs := glog.Stats.Error.Lines() + priorGlogErrs := klog.Stats.Error.Lines() input := []proxy.ServiceEndpoint{tc.epSvcPair} fakeProxier.deleteEndpointConnections(input) @@ -319,7 +319,7 @@ func TestDeleteEndpointConnections(t *testing.T) { if tc.simulatedErr != "" && tc.simulatedErr != conntrack.NoConnectionToDelete { expGlogErrs = 1 } - glogErrs := glog.Stats.Error.Lines() - priorGlogErrs + glogErrs := klog.Stats.Error.Lines() - priorGlogErrs if glogErrs != expGlogErrs { t.Errorf("%s: Expected %d glogged errors, but got %d", tc.description, expGlogErrs, glogErrs) } diff --git a/pkg/proxy/ipvs/BUILD b/pkg/proxy/ipvs/BUILD index e7b13f3cac6..002b616f3f8 100644 --- a/pkg/proxy/ipvs/BUILD +++ b/pkg/proxy/ipvs/BUILD @@ -63,7 +63,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ diff --git a/pkg/proxy/ipvs/README.md b/pkg/proxy/ipvs/README.md index 9e8fa493fa4..eb0f456f690 100644 --- a/pkg/proxy/ipvs/README.md +++ b/pkg/proxy/ipvs/README.md @@ -38,7 +38,7 @@ Differences between IPVS mode and IPTABLES mode are as follows: ### When ipvs falls back to iptables IPVS proxier will employ iptables in doing packet filtering, SNAT or masquerade. -Specifically, ipvs proxier will use ipset to store source or destination address of traffics that need DROP or do masquared, to make sure the number of iptables rules be constant, no metter how many services we have. +Specifically, ipvs proxier will use ipset to store source or destination address of traffics that need DROP or do masquerade, to make sure the number of iptables rules be constant, no metter how many services we have. Here is the table of ipset sets that ipvs proxier used. diff --git a/pkg/proxy/ipvs/graceful_termination.go b/pkg/proxy/ipvs/graceful_termination.go index 0d8c4ebc567..d9357d2c6d8 100644 --- a/pkg/proxy/ipvs/graceful_termination.go +++ b/pkg/proxy/ipvs/graceful_termination.go @@ -21,8 +21,8 @@ import ( "time" "fmt" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" utilipvs "k8s.io/kubernetes/pkg/util/ipvs" ) @@ -63,7 +63,7 @@ func (q *graceTerminateRSList) add(rs *listItem) bool { return false } - glog.V(5).Infof("Adding rs %v to graceful delete rsList", rs) + klog.V(5).Infof("Adding rs %v to graceful delete rsList", rs) q.list[uniqueRS] = rs return true } @@ -86,11 +86,11 @@ func (q *graceTerminateRSList) flushList(handler func(rsToDelete *listItem) (boo for name, rs := range q.list { deleted, err := handler(rs) if err != nil { - glog.Errorf("Try delete rs %q err: %v", name, err) + klog.Errorf("Try delete rs %q err: %v", name, err) success = false } if deleted { - glog.Infof("lw: remote out of the list: %s", name) + klog.Infof("lw: remote out of the list: %s", name) q.remove(rs) } } @@ -141,7 +141,7 @@ func (m *GracefulTerminationManager) GracefulDeleteRS(vs *utilipvs.VirtualServer } deleted, err := m.deleteRsFunc(ele) if err != nil { - glog.Errorf("Delete rs %q err: %v", ele.String(), err) + klog.Errorf("Delete rs %q err: %v", ele.String(), err) } if deleted { return nil @@ -151,13 +151,13 @@ func (m *GracefulTerminationManager) GracefulDeleteRS(vs *utilipvs.VirtualServer if err != nil { return err } - glog.V(5).Infof("Adding an element to graceful delete rsList: %+v", ele) + klog.V(5).Infof("Adding an element to graceful delete rsList: %+v", ele) m.rsList.add(ele) return nil } func (m *GracefulTerminationManager) deleteRsFunc(rsToDelete *listItem) (bool, error) { - glog.Infof("Trying to delete rs: %s", rsToDelete.String()) + klog.Infof("Trying to delete rs: %s", rsToDelete.String()) rss, err := m.ipvs.GetRealServers(rsToDelete.VirtualServer) if err != nil { return false, err @@ -167,7 +167,7 @@ func (m *GracefulTerminationManager) deleteRsFunc(rsToDelete *listItem) (bool, e if rs.ActiveConn != 0 { return false, nil } - glog.Infof("Deleting rs: %s", rsToDelete.String()) + klog.Infof("Deleting rs: %s", rsToDelete.String()) err := m.ipvs.DeleteRealServer(rsToDelete.VirtualServer, rs) if err != nil { return false, fmt.Errorf("Delete destination %q err: %v", rs.String(), err) @@ -180,7 +180,7 @@ func (m *GracefulTerminationManager) deleteRsFunc(rsToDelete *listItem) (bool, e func (m *GracefulTerminationManager) tryDeleteRs() { if !m.rsList.flushList(m.deleteRsFunc) { - glog.Errorf("Try flush graceful termination list err") + klog.Errorf("Try flush graceful termination list err") } } @@ -203,12 +203,12 @@ func (m *GracefulTerminationManager) Run() { // before start, add leftover in delete rs to graceful delete rsList vss, err := m.ipvs.GetVirtualServers() if err != nil { - glog.Errorf("IPVS graceful delete manager failed to get IPVS virtualserver") + klog.Errorf("IPVS graceful delete manager failed to get IPVS virtualserver") } for _, vs := range vss { rss, err := m.ipvs.GetRealServers(vs) if err != nil { - glog.Errorf("IPVS graceful delete manager failed to get %v realserver", vs) + klog.Errorf("IPVS graceful delete manager failed to get %v realserver", vs) continue } for _, rs := range rss { diff --git a/pkg/proxy/ipvs/ipset.go b/pkg/proxy/ipvs/ipset.go index e25ee009853..e449cc1ee3f 100644 --- a/pkg/proxy/ipvs/ipset.go +++ b/pkg/proxy/ipvs/ipset.go @@ -22,7 +22,7 @@ import ( utilipset "k8s.io/kubernetes/pkg/util/ipset" "fmt" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -125,7 +125,7 @@ func (set *IPSet) resetEntries() { func (set *IPSet) syncIPSetEntries() { appliedEntries, err := set.handle.ListEntries(set.Name) if err != nil { - glog.Errorf("Failed to list ip set entries, error: %v", err) + klog.Errorf("Failed to list ip set entries, error: %v", err) return } @@ -140,18 +140,18 @@ func (set *IPSet) syncIPSetEntries() { for _, entry := range currentIPSetEntries.Difference(set.activeEntries).List() { if err := set.handle.DelEntry(entry, set.Name); err != nil { if !utilipset.IsNotFoundError(err) { - glog.Errorf("Failed to delete ip set entry: %s from ip set: %s, error: %v", entry, set.Name, err) + klog.Errorf("Failed to delete ip set entry: %s from ip set: %s, error: %v", entry, set.Name, err) } } else { - glog.V(3).Infof("Successfully delete legacy ip set entry: %s from ip set: %s", entry, set.Name) + klog.V(3).Infof("Successfully delete legacy ip set entry: %s from ip set: %s", entry, set.Name) } } // Create active entries for _, entry := range set.activeEntries.Difference(currentIPSetEntries).List() { if err := set.handle.AddEntry(entry, &set.IPSet, true); err != nil { - glog.Errorf("Failed to add entry: %v to ip set: %s, error: %v", entry, set.Name, err) + klog.Errorf("Failed to add entry: %v to ip set: %s, error: %v", entry, set.Name, err) } else { - glog.V(3).Infof("Successfully add entry: %v to ip set: %s", entry, set.Name) + klog.V(3).Infof("Successfully add entry: %v to ip set: %s", entry, set.Name) } } } @@ -159,7 +159,7 @@ func (set *IPSet) syncIPSetEntries() { func ensureIPSet(set *IPSet) error { if err := set.handle.CreateSet(&set.IPSet, true); err != nil { - glog.Errorf("Failed to make sure ip set: %v exist, error: %v", set, err) + klog.Errorf("Failed to make sure ip set: %v exist, error: %v", set, err) return err } return nil @@ -169,13 +169,13 @@ func ensureIPSet(set *IPSet) error { func checkMinVersion(vstring string) bool { version, err := utilversion.ParseGeneric(vstring) if err != nil { - glog.Errorf("vstring (%s) is not a valid version string: %v", vstring, err) + klog.Errorf("vstring (%s) is not a valid version string: %v", vstring, err) return false } minVersion, err := utilversion.ParseGeneric(MinIPSetCheckVersion) if err != nil { - glog.Errorf("MinCheckVersion (%s) is not a valid version string: %v", MinIPSetCheckVersion, err) + klog.Errorf("MinCheckVersion (%s) is not a valid version string: %v", MinIPSetCheckVersion, err) return false } return !version.LessThan(minVersion) diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index cfce9bb5085..242a6025ca6 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -28,7 +28,7 @@ import ( "sync/atomic" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -157,19 +157,11 @@ var ipsetWithIptablesChain = []struct { {kubeNodePortLocalSetSCTP, string(KubeNodePortChain), "RETURN", "dst", "sctp"}, } -var ipvsModules = []string{ - "ip_vs", - "ip_vs_rr", - "ip_vs_wrr", - "ip_vs_sh", - "nf_conntrack_ipv4", - "nf_conntrack", -} - // In IPVS proxy mode, the following flags need to be set const sysctlRouteLocalnet = "net/ipv4/conf/all/route_localnet" const sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables" const sysctlVSConnTrack = "net/ipv4/vs/conntrack" +const sysctlConnReuse = "net/ipv4/vs/conn_reuse_mode" const sysctlForward = "net/ipv4/ip_forward" const sysctlArpIgnore = "net/ipv4/conf/all/arp_ignore" const sysctlArpAnnounce = "net/ipv4/conf/all/arp_announce" @@ -312,7 +304,7 @@ func NewProxier(ipt utiliptables.Interface, // are connected to a Linux bridge (but not SDN bridges). Until most // plugins handle this, log when config is missing if val, err := sysctl.GetSysctl(sysctlBridgeCallIPTables); err == nil && val != 1 { - glog.Infof("missing br-netfilter module or unset sysctl br-nf-call-iptables; proxy may not work as intended") + klog.Infof("missing br-netfilter module or unset sysctl br-nf-call-iptables; proxy may not work as intended") } // Set the conntrack sysctl we need for @@ -322,6 +314,13 @@ func NewProxier(ipt utiliptables.Interface, } } + // Set the connection reuse mode + if val, _ := sysctl.GetSysctl(sysctlConnReuse); val != 0 { + if err := sysctl.SetSysctl(sysctlConnReuse, 0); err != nil { + return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlConnReuse, err) + } + } + // Set the ip_forward sysctl we need for if val, _ := sysctl.GetSysctl(sysctlForward); val != 1 { if err := sysctl.SetSysctl(sysctlForward, 1); err != nil { @@ -348,22 +347,22 @@ func NewProxier(ipt utiliptables.Interface, masqueradeMark := fmt.Sprintf("%#08x/%#08x", masqueradeValue, masqueradeValue) if nodeIP == nil { - glog.Warningf("invalid nodeIP, initializing kube-proxy with 127.0.0.1 as nodeIP") + klog.Warningf("invalid nodeIP, initializing kube-proxy with 127.0.0.1 as nodeIP") nodeIP = net.ParseIP("127.0.0.1") } isIPv6 := utilnet.IsIPv6(nodeIP) - glog.V(2).Infof("nodeIP: %v, isIPv6: %v", nodeIP, isIPv6) + klog.V(2).Infof("nodeIP: %v, isIPv6: %v", nodeIP, isIPv6) if len(clusterCIDR) == 0 { - glog.Warningf("clusterCIDR not specified, unable to distinguish between internal and external traffic") + klog.Warningf("clusterCIDR not specified, unable to distinguish between internal and external traffic") } else if utilnet.IsIPv6CIDR(clusterCIDR) != isIPv6 { return nil, fmt.Errorf("clusterCIDR %s has incorrect IP version: expect isIPv6=%t", clusterCIDR, isIPv6) } if len(scheduler) == 0 { - glog.Warningf("IPVS scheduler not specified, use %s by default", DefaultScheduler) + klog.Warningf("IPVS scheduler not specified, use %s by default", DefaultScheduler) scheduler = DefaultScheduler } @@ -410,7 +409,7 @@ func NewProxier(ipt utiliptables.Interface, proxier.ipsetList[is.name] = NewIPSet(ipset, is.name, is.setType, isIPv6, is.comment) } burstSyncs := 2 - glog.V(3).Infof("minSyncPeriod: %v, syncPeriod: %v, burstSyncs: %d", minSyncPeriod, syncPeriod, burstSyncs) + klog.V(3).Infof("minSyncPeriod: %v, syncPeriod: %v, burstSyncs: %d", minSyncPeriod, syncPeriod, burstSyncs) proxier.syncRunner = async.NewBoundedFrequencyRunner("sync-runner", proxier.syncProxyRules, minSyncPeriod, syncPeriod, burstSyncs) proxier.gracefuldeleteManager.Run() return proxier, nil @@ -455,16 +454,14 @@ func NewLinuxKernelHandler() *LinuxKernelHandler { // GetModules returns all installed kernel modules. func (handle *LinuxKernelHandler) GetModules() ([]string, error) { // Check whether IPVS required kernel modules are built-in - kernelVersionFile := "/proc/sys/kernel/osrelease" - b, err := ioutil.ReadFile(kernelVersionFile) + kernelVersion, ipvsModules, err := utilipvs.GetKernelVersionAndIPVSMods(handle.executor) if err != nil { - glog.Errorf("Failed to read file %s with error %v", kernelVersionFile, err) + return nil, err } - kernelVersion := strings.TrimSpace(string(b)) builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersion) - b, err = ioutil.ReadFile(builtinModsFilePath) + b, err := ioutil.ReadFile(builtinModsFilePath) if err != nil { - glog.Warningf("Failed to read file %s with error %v. You can ignore this message when kube-proxy is running inside container without mounting /lib/modules", builtinModsFilePath, err) + klog.Warningf("Failed to read file %s with error %v. You can ignore this message when kube-proxy is running inside container without mounting /lib/modules", builtinModsFilePath, err) } var bmods []string for _, module := range ipvsModules { @@ -477,7 +474,7 @@ func (handle *LinuxKernelHandler) GetModules() ([]string, error) { for _, kmod := range ipvsModules { err := handle.executor.Command("modprobe", "--", kmod).Run() if err != nil { - glog.Warningf("Failed to load kernel module %v with modprobe. "+ + klog.Warningf("Failed to load kernel module %v with modprobe. "+ "You can ignore this message when kube-proxy is running inside container without mounting /lib/modules", kmod) } } @@ -503,6 +500,8 @@ func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, err } wantModules := sets.NewString() loadModules := sets.NewString() + linuxKernelHandler := NewLinuxKernelHandler() + _, ipvsModules, _ := utilipvs.GetKernelVersionAndIPVSMods(linuxKernelHandler.executor) wantModules.Insert(ipvsModules...) loadModules.Insert(mods...) modules := wantModules.Difference(loadModules).UnsortedList() @@ -545,7 +544,7 @@ func cleanupIptablesLeftovers(ipt utiliptables.Interface) (encounteredError bool } if err := ipt.DeleteRule(jc.table, jc.from, args...); err != nil { if !utiliptables.IsNotFoundError(err) { - glog.Errorf("Error removing iptables rules in ipvs proxier: %v", err) + klog.Errorf("Error removing iptables rules in ipvs proxier: %v", err) encounteredError = true } } @@ -555,13 +554,13 @@ func cleanupIptablesLeftovers(ipt utiliptables.Interface) (encounteredError bool for _, ch := range iptablesChains { if err := ipt.FlushChain(ch.table, ch.chain); err != nil { if !utiliptables.IsNotFoundError(err) { - glog.Errorf("Error removing iptables rules in ipvs proxier: %v", err) + klog.Errorf("Error removing iptables rules in ipvs proxier: %v", err) encounteredError = true } } if err := ipt.DeleteChain(ch.table, ch.chain); err != nil { if !utiliptables.IsNotFoundError(err) { - glog.Errorf("Error removing iptables rules in ipvs proxier: %v", err) + klog.Errorf("Error removing iptables rules in ipvs proxier: %v", err) encounteredError = true } } @@ -580,7 +579,7 @@ func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset encounteredError = false err := ipvs.Flush() if err != nil { - glog.Errorf("Error flushing IPVS rules: %v", err) + klog.Errorf("Error flushing IPVS rules: %v", err) encounteredError = true } } @@ -588,7 +587,7 @@ func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset nl := NewNetLinkHandle() err := nl.DeleteDummyDevice(DefaultDummyDevice) if err != nil { - glog.Errorf("Error deleting dummy device %s created by IPVS proxier: %v", DefaultDummyDevice, err) + klog.Errorf("Error deleting dummy device %s created by IPVS proxier: %v", DefaultDummyDevice, err) encounteredError = true } // Clear iptables created by ipvs Proxier. @@ -599,7 +598,7 @@ func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset err = ipset.DestroySet(set.name) if err != nil { if !utilipset.IsNotFoundError(err) { - glog.Errorf("Error removing ipset %s, error: %v", set.name, err) + klog.Errorf("Error removing ipset %s, error: %v", set.name, err) encounteredError = true } } @@ -701,11 +700,11 @@ func (proxier *Proxier) syncProxyRules() { start := time.Now() defer func() { metrics.SyncProxyRulesLatency.Observe(metrics.SinceInMicroseconds(start)) - glog.V(4).Infof("syncProxyRules took %v", time.Since(start)) + klog.V(4).Infof("syncProxyRules took %v", time.Since(start)) }() // don't sync rules till we've received services and endpoints if !proxier.endpointsSynced || !proxier.servicesSynced { - glog.V(2).Info("Not syncing ipvs rules until Services and Endpoints have been received from master") + klog.V(2).Info("Not syncing ipvs rules until Services and Endpoints have been received from master") return } @@ -719,12 +718,12 @@ func (proxier *Proxier) syncProxyRules() { // merge stale services gathered from updateEndpointsMap for _, svcPortName := range endpointUpdateResult.StaleServiceNames { if svcInfo, ok := proxier.serviceMap[svcPortName]; ok && svcInfo != nil && svcInfo.GetProtocol() == v1.ProtocolUDP { - glog.V(2).Infof("Stale udp service %v -> %s", svcPortName, svcInfo.ClusterIPString()) + klog.V(2).Infof("Stale udp service %v -> %s", svcPortName, svcInfo.ClusterIPString()) staleServices.Insert(svcInfo.ClusterIPString()) } } - glog.V(3).Infof("Syncing ipvs Proxier rules") + klog.V(3).Infof("Syncing ipvs Proxier rules") // Begin install iptables @@ -744,7 +743,7 @@ func (proxier *Proxier) syncProxyRules() { // make sure dummy interface exists in the system where ipvs Proxier will bind service address on it _, err := proxier.netlinkHandle.EnsureDummyDevice(DefaultDummyDevice) if err != nil { - glog.Errorf("Failed to create dummy interface: %s, error: %v", DefaultDummyDevice, err) + klog.Errorf("Failed to create dummy interface: %s, error: %v", DefaultDummyDevice, err) return } @@ -769,7 +768,7 @@ func (proxier *Proxier) syncProxyRules() { for svcName, svc := range proxier.serviceMap { svcInfo, ok := svc.(*serviceInfo) if !ok { - glog.Errorf("Failed to cast serviceInfo %q", svcName.String()) + klog.Errorf("Failed to cast serviceInfo %q", svcName.String()) continue } protocol := strings.ToLower(string(svcInfo.Protocol)) @@ -781,7 +780,10 @@ func (proxier *Proxier) syncProxyRules() { for _, e := range proxier.endpointsMap[svcName] { ep, ok := e.(*proxy.BaseEndpointInfo) if !ok { - glog.Errorf("Failed to cast BaseEndpointInfo %q", e.String()) + klog.Errorf("Failed to cast BaseEndpointInfo %q", e.String()) + continue + } + if !ep.IsLocal { continue } epIP := ep.IP() @@ -798,7 +800,7 @@ func (proxier *Proxier) syncProxyRules() { SetType: utilipset.HashIPPortIP, } if valid := proxier.ipsetList[kubeLoopBackIPSet].validateEntry(entry); !valid { - glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeLoopBackIPSet].Name)) + klog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeLoopBackIPSet].Name)) continue } proxier.ipsetList[kubeLoopBackIPSet].activeEntries.Insert(entry.String()) @@ -815,7 +817,7 @@ func (proxier *Proxier) syncProxyRules() { // add service Cluster IP:Port to kubeServiceAccess ip set for the purpose of solving hairpin. // proxier.kubeServiceAccessSet.activeEntries.Insert(entry.String()) if valid := proxier.ipsetList[kubeClusterIPSet].validateEntry(entry); !valid { - glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeClusterIPSet].Name)) + klog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeClusterIPSet].Name)) continue } proxier.ipsetList[kubeClusterIPSet].activeEntries.Insert(entry.String()) @@ -838,16 +840,16 @@ func (proxier *Proxier) syncProxyRules() { // ExternalTrafficPolicy only works for NodePort and external LB traffic, does not affect ClusterIP // So we still need clusterIP rules in onlyNodeLocalEndpoints mode. if err := proxier.syncEndpoint(svcName, false, serv); err != nil { - glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err) + klog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err) } } else { - glog.Errorf("Failed to sync service: %v, err: %v", serv, err) + klog.Errorf("Failed to sync service: %v, err: %v", serv, err) } // Capture externalIPs. for _, externalIP := range svcInfo.ExternalIPs { if local, err := utilproxy.IsLocalIP(externalIP); err != nil { - glog.Errorf("can't determine if IP is local, assuming not: %v", err) + klog.Errorf("can't determine if IP is local, assuming not: %v", err) // We do not start listening on SCTP ports, according to our agreement in the // SCTP support KEP } else if local && (svcInfo.GetProtocol() != v1.ProtocolSCTP) { @@ -858,7 +860,7 @@ func (proxier *Proxier) syncProxyRules() { Protocol: protocol, } if proxier.portsMap[lp] != nil { - glog.V(4).Infof("Port %s was open before and is still needed", lp.String()) + klog.V(4).Infof("Port %s was open before and is still needed", lp.String()) replacementPortsMap[lp] = proxier.portsMap[lp] } else { socket, err := proxier.portMapper.OpenLocalPort(&lp) @@ -872,7 +874,7 @@ func (proxier *Proxier) syncProxyRules() { UID: types.UID(proxier.hostname), Namespace: "", }, v1.EventTypeWarning, err.Error(), msg) - glog.Error(msg) + klog.Error(msg) continue } replacementPortsMap[lp] = socket @@ -888,7 +890,7 @@ func (proxier *Proxier) syncProxyRules() { } // We have to SNAT packets to external IPs. if valid := proxier.ipsetList[kubeExternalIPSet].validateEntry(entry); !valid { - glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeExternalIPSet].Name)) + klog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeExternalIPSet].Name)) continue } proxier.ipsetList[kubeExternalIPSet].activeEntries.Insert(entry.String()) @@ -908,10 +910,10 @@ func (proxier *Proxier) syncProxyRules() { activeIPVSServices[serv.String()] = true activeBindAddrs[serv.Address.String()] = true if err := proxier.syncEndpoint(svcName, false, serv); err != nil { - glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err) + klog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err) } } else { - glog.Errorf("Failed to sync service: %v, err: %v", serv, err) + klog.Errorf("Failed to sync service: %v, err: %v", serv, err) } } @@ -930,14 +932,14 @@ func (proxier *Proxier) syncProxyRules() { // If we are proxying globally, we need to masquerade in case we cross nodes. // If we are proxying only locally, we can retain the source IP. if valid := proxier.ipsetList[kubeLoadBalancerSet].validateEntry(entry); !valid { - glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeLoadBalancerSet].Name)) + klog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeLoadBalancerSet].Name)) continue } proxier.ipsetList[kubeLoadBalancerSet].activeEntries.Insert(entry.String()) // insert loadbalancer entry to lbIngressLocalSet if service externaltrafficpolicy=local if svcInfo.OnlyNodeLocalEndpoints { if valid := proxier.ipsetList[kubeLoadBalancerLocalSet].validateEntry(entry); !valid { - glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeLoadBalancerLocalSet].Name)) + klog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeLoadBalancerLocalSet].Name)) continue } proxier.ipsetList[kubeLoadBalancerLocalSet].activeEntries.Insert(entry.String()) @@ -947,7 +949,7 @@ func (proxier *Proxier) syncProxyRules() { // This currently works for loadbalancers that preserves source ips. // For loadbalancers which direct traffic to service NodePort, the firewall rules will not apply. if valid := proxier.ipsetList[kubeLoadbalancerFWSet].validateEntry(entry); !valid { - glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeLoadbalancerFWSet].Name)) + klog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeLoadbalancerFWSet].Name)) continue } proxier.ipsetList[kubeLoadbalancerFWSet].activeEntries.Insert(entry.String()) @@ -963,7 +965,7 @@ func (proxier *Proxier) syncProxyRules() { } // enumerate all white list source cidr if valid := proxier.ipsetList[kubeLoadBalancerSourceCIDRSet].validateEntry(entry); !valid { - glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeLoadBalancerSourceCIDRSet].Name)) + klog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeLoadBalancerSourceCIDRSet].Name)) continue } proxier.ipsetList[kubeLoadBalancerSourceCIDRSet].activeEntries.Insert(entry.String()) @@ -987,7 +989,7 @@ func (proxier *Proxier) syncProxyRules() { } // enumerate all white list source ip if valid := proxier.ipsetList[kubeLoadBalancerSourceIPSet].validateEntry(entry); !valid { - glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeLoadBalancerSourceIPSet].Name)) + klog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeLoadBalancerSourceIPSet].Name)) continue } proxier.ipsetList[kubeLoadBalancerSourceIPSet].activeEntries.Insert(entry.String()) @@ -1011,10 +1013,10 @@ func (proxier *Proxier) syncProxyRules() { activeIPVSServices[serv.String()] = true activeBindAddrs[serv.Address.String()] = true if err := proxier.syncEndpoint(svcName, onlyLocal, serv); err != nil { - glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err) + klog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err) } } else { - glog.Errorf("Failed to sync service: %v, err: %v", serv, err) + klog.Errorf("Failed to sync service: %v, err: %v", serv, err) } } } @@ -1022,7 +1024,7 @@ func (proxier *Proxier) syncProxyRules() { if svcInfo.NodePort != 0 { addresses, err := utilproxy.GetNodeAddresses(proxier.nodePortAddresses, proxier.networkInterfacer) if err != nil { - glog.Errorf("Failed to get node ip address matching nodeport cidr: %v", err) + klog.Errorf("Failed to get node ip address matching nodeport cidr: %v", err) continue } @@ -1047,14 +1049,14 @@ func (proxier *Proxier) syncProxyRules() { // For ports on node IPs, open the actual port and hold it. for _, lp := range lps { if proxier.portsMap[lp] != nil { - glog.V(4).Infof("Port %s was open before and is still needed", lp.String()) + klog.V(4).Infof("Port %s was open before and is still needed", lp.String()) replacementPortsMap[lp] = proxier.portsMap[lp] // We do not start listening on SCTP ports, according to our agreement in the // SCTP support KEP } else if svcInfo.GetProtocol() != v1.ProtocolSCTP { socket, err := proxier.portMapper.OpenLocalPort(&lp) if err != nil { - glog.Errorf("can't open %s, skipping this nodePort: %v", lp.String(), err) + klog.Errorf("can't open %s, skipping this nodePort: %v", lp.String(), err) continue } if lp.Protocol == "udp" { @@ -1083,11 +1085,11 @@ func (proxier *Proxier) syncProxyRules() { nodePortSet = proxier.ipsetList[kubeNodePortSetSCTP] default: // It should never hit - glog.Errorf("Unsupported protocol type: %s", protocol) + klog.Errorf("Unsupported protocol type: %s", protocol) } if nodePortSet != nil { if valid := nodePortSet.validateEntry(entry); !valid { - glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, nodePortSet.Name)) + klog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, nodePortSet.Name)) continue } nodePortSet.activeEntries.Insert(entry.String()) @@ -1105,11 +1107,11 @@ func (proxier *Proxier) syncProxyRules() { nodePortLocalSet = proxier.ipsetList[kubeNodePortLocalSetSCTP] default: // It should never hit - glog.Errorf("Unsupported protocol type: %s", protocol) + klog.Errorf("Unsupported protocol type: %s", protocol) } if nodePortLocalSet != nil { if valid := nodePortLocalSet.validateEntry(entry); !valid { - glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, nodePortLocalSet.Name)) + klog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, nodePortLocalSet.Name)) continue } nodePortLocalSet.activeEntries.Insert(entry.String()) @@ -1126,7 +1128,7 @@ func (proxier *Proxier) syncProxyRules() { // zero cidr nodeIPs, err = proxier.ipGetter.NodeIPs() if err != nil { - glog.Errorf("Failed to list all node IPs from host, err: %v", err) + klog.Errorf("Failed to list all node IPs from host, err: %v", err) } } for _, nodeIP := range nodeIPs { @@ -1145,10 +1147,10 @@ func (proxier *Proxier) syncProxyRules() { if err := proxier.syncService(svcNameString, serv, false); err == nil { activeIPVSServices[serv.String()] = true if err := proxier.syncEndpoint(svcName, svcInfo.OnlyNodeLocalEndpoints, serv); err != nil { - glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err) + klog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err) } } else { - glog.Errorf("Failed to sync service: %v, err: %v", serv, err) + klog.Errorf("Failed to sync service: %v, err: %v", serv, err) } } } @@ -1171,10 +1173,10 @@ func (proxier *Proxier) syncProxyRules() { proxier.iptablesData.Write(proxier.filterChains.Bytes()) proxier.iptablesData.Write(proxier.filterRules.Bytes()) - glog.V(5).Infof("Restoring iptables rules: %s", proxier.iptablesData.Bytes()) + klog.V(5).Infof("Restoring iptables rules: %s", proxier.iptablesData.Bytes()) err = proxier.iptables.RestoreAll(proxier.iptablesData.Bytes(), utiliptables.NoFlushTables, utiliptables.RestoreCounters) if err != nil { - glog.Errorf("Failed to execute iptables-restore: %v\nRules:\n%s", err, proxier.iptablesData.Bytes()) + klog.Errorf("Failed to execute iptables-restore: %v\nRules:\n%s", err, proxier.iptablesData.Bytes()) // Revert new local ports. utilproxy.RevertPorts(replacementPortsMap, proxier.portsMap) return @@ -1195,7 +1197,7 @@ func (proxier *Proxier) syncProxyRules() { currentIPVSServices[appliedSvc.String()] = appliedSvc } } else { - glog.Errorf("Failed to get ipvs service, err: %v", err) + klog.Errorf("Failed to get ipvs service, err: %v", err) } proxier.cleanLegacyService(activeIPVSServices, currentIPVSServices) @@ -1203,7 +1205,7 @@ func (proxier *Proxier) syncProxyRules() { // currentBindAddrs represents ip addresses bind to DefaultDummyDevice from the system currentBindAddrs, err := proxier.netlinkHandle.ListBindAddress(DefaultDummyDevice) if err != nil { - glog.Errorf("Failed to get bind address, err: %v", err) + klog.Errorf("Failed to get bind address, err: %v", err) } proxier.cleanLegacyBindAddr(activeBindAddrs, currentBindAddrs) @@ -1216,17 +1218,17 @@ func (proxier *Proxier) syncProxyRules() { // not "OnlyLocal", but the services list will not, and the healthChecker // will just drop those endpoints. if err := proxier.healthChecker.SyncServices(serviceUpdateResult.HCServiceNodePorts); err != nil { - glog.Errorf("Error syncing healtcheck services: %v", err) + klog.Errorf("Error syncing healthcheck services: %v", err) } if err := proxier.healthChecker.SyncEndpoints(endpointUpdateResult.HCEndpointsLocalIPSize); err != nil { - glog.Errorf("Error syncing healthcheck endpoints: %v", err) + klog.Errorf("Error syncing healthcheck endpoints: %v", err) } // Finish housekeeping. // TODO: these could be made more consistent. for _, svcIP := range staleServices.UnsortedList() { if err := conntrack.ClearEntriesForIP(proxier.exec, svcIP, v1.ProtocolUDP); err != nil { - glog.Errorf("Failed to delete stale service IP %s connections, error: %v", svcIP, err) + klog.Errorf("Failed to delete stale service IP %s connections, error: %v", svcIP, err) } } proxier.deleteEndpointConnections(endpointUpdateResult.StaleEndpoints) @@ -1404,7 +1406,7 @@ func (proxier *Proxier) createAndLinkeKubeChain() { // Make sure we keep stats for the top-level chains for _, ch := range iptablesChains { if _, err := proxier.iptables.EnsureChain(ch.table, ch.chain); err != nil { - glog.Errorf("Failed to ensure that %s chain %s exists: %v", ch.table, ch.chain, err) + klog.Errorf("Failed to ensure that %s chain %s exists: %v", ch.table, ch.chain, err) return } if ch.table == utiliptables.TableNAT { @@ -1425,7 +1427,7 @@ func (proxier *Proxier) createAndLinkeKubeChain() { for _, jc := range iptablesJumpChain { args := []string{"-m", "comment", "--comment", jc.comment, "-j", string(jc.to)} if _, err := proxier.iptables.EnsureRule(utiliptables.Prepend, jc.table, jc.from, args...); err != nil { - glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", jc.table, jc.from, jc.to, err) + klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", jc.table, jc.from, jc.to, err) } } @@ -1455,7 +1457,7 @@ func (proxier *Proxier) getExistingChains(buffer *bytes.Buffer, table utiliptabl buffer.Reset() err := proxier.iptables.SaveInto(table, buffer) if err != nil { // if we failed to get any rules - glog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err) + klog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err) } else { // otherwise parse the output return utiliptables.GetChainLines(table, buffer.Bytes()) } @@ -1471,7 +1473,7 @@ func (proxier *Proxier) deleteEndpointConnections(connectionMap []proxy.ServiceE endpointIP := utilproxy.IPPart(epSvcPair.Endpoint) err := conntrack.ClearEntriesForNAT(proxier.exec, svcInfo.ClusterIPString(), endpointIP, v1.ProtocolUDP) if err != nil { - glog.Errorf("Failed to delete %s endpoint connections, error: %v", epSvcPair.ServicePortName.String(), err) + klog.Errorf("Failed to delete %s endpoint connections, error: %v", epSvcPair.ServicePortName.String(), err) } } } @@ -1482,17 +1484,17 @@ func (proxier *Proxier) syncService(svcName string, vs *utilipvs.VirtualServer, if appliedVirtualServer == nil || !appliedVirtualServer.Equal(vs) { if appliedVirtualServer == nil { // IPVS service is not found, create a new service - glog.V(3).Infof("Adding new service %q %s:%d/%s", svcName, vs.Address, vs.Port, vs.Protocol) + klog.V(3).Infof("Adding new service %q %s:%d/%s", svcName, vs.Address, vs.Port, vs.Protocol) if err := proxier.ipvs.AddVirtualServer(vs); err != nil { - glog.Errorf("Failed to add IPVS service %q: %v", svcName, err) + klog.Errorf("Failed to add IPVS service %q: %v", svcName, err) return err } } else { // IPVS service was changed, update the existing one // During updates, service VIP will not go down - glog.V(3).Infof("IPVS service %s was changed", svcName) + klog.V(3).Infof("IPVS service %s was changed", svcName) if err := proxier.ipvs.UpdateVirtualServer(vs); err != nil { - glog.Errorf("Failed to update IPVS service, err:%v", err) + klog.Errorf("Failed to update IPVS service, err:%v", err) return err } } @@ -1501,10 +1503,10 @@ func (proxier *Proxier) syncService(svcName string, vs *utilipvs.VirtualServer, // bind service address to dummy interface even if service not changed, // in case that service IP was removed by other processes if bindAddr { - glog.V(4).Infof("Bind addr %s", vs.Address.String()) + klog.V(4).Infof("Bind addr %s", vs.Address.String()) _, err := proxier.netlinkHandle.EnsureAddressBind(vs.Address.String(), DefaultDummyDevice) if err != nil { - glog.Errorf("Failed to bind service address to dummy device %q: %v", svcName, err) + klog.Errorf("Failed to bind service address to dummy device %q: %v", svcName, err) return err } } @@ -1514,7 +1516,7 @@ func (proxier *Proxier) syncService(svcName string, vs *utilipvs.VirtualServer, func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNodeLocalEndpoints bool, vs *utilipvs.VirtualServer) error { appliedVirtualServer, err := proxier.ipvs.GetVirtualServer(vs) if err != nil || appliedVirtualServer == nil { - glog.Errorf("Failed to get IPVS service, error: %v", err) + klog.Errorf("Failed to get IPVS service, error: %v", err) return err } @@ -1525,7 +1527,7 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode curDests, err := proxier.ipvs.GetRealServers(appliedVirtualServer) if err != nil { - glog.Errorf("Failed to list IPVS destinations, error: %v", err) + klog.Errorf("Failed to list IPVS destinations, error: %v", err) return err } for _, des := range curDests { @@ -1543,12 +1545,12 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode for _, ep := range newEndpoints.List() { ip, port, err := net.SplitHostPort(ep) if err != nil { - glog.Errorf("Failed to parse endpoint: %v, error: %v", ep, err) + klog.Errorf("Failed to parse endpoint: %v, error: %v", ep, err) continue } portNum, err := strconv.Atoi(port) if err != nil { - glog.Errorf("Failed to parse endpoint port %s, error: %v", port, err) + klog.Errorf("Failed to parse endpoint port %s, error: %v", port, err) continue } @@ -1564,16 +1566,16 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode if !proxier.gracefuldeleteManager.InTerminationList(uniqueRS) { continue } - glog.V(5).Infof("new ep %q is in graceful delete list", uniqueRS) + klog.V(5).Infof("new ep %q is in graceful delete list", uniqueRS) err := proxier.gracefuldeleteManager.MoveRSOutofGracefulDeleteList(uniqueRS) if err != nil { - glog.Errorf("Failed to delete endpoint: %v in gracefulDeleteQueue, error: %v", ep, err) + klog.Errorf("Failed to delete endpoint: %v in gracefulDeleteQueue, error: %v", ep, err) continue } } err = proxier.ipvs.AddRealServer(appliedVirtualServer, newDest) if err != nil { - glog.Errorf("Failed to add destination: %v, error: %v", newDest, err) + klog.Errorf("Failed to add destination: %v, error: %v", newDest, err) continue } } @@ -1586,12 +1588,12 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode } ip, port, err := net.SplitHostPort(ep) if err != nil { - glog.Errorf("Failed to parse endpoint: %v, error: %v", ep, err) + klog.Errorf("Failed to parse endpoint: %v, error: %v", ep, err) continue } portNum, err := strconv.Atoi(port) if err != nil { - glog.Errorf("Failed to parse endpoint port %s, error: %v", port, err) + klog.Errorf("Failed to parse endpoint port %s, error: %v", port, err) continue } @@ -1600,10 +1602,10 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode Port: uint16(portNum), } - glog.V(5).Infof("Using graceful delete to delete: %v", delDest) + klog.V(5).Infof("Using graceful delete to delete: %v", delDest) err = proxier.gracefuldeleteManager.GracefulDeleteRS(appliedVirtualServer, delDest) if err != nil { - glog.Errorf("Failed to delete destination: %v, error: %v", delDest, err) + klog.Errorf("Failed to delete destination: %v, error: %v", delDest, err) continue } } @@ -1636,7 +1638,7 @@ func (proxier *Proxier) cleanLegacyService(activeServices map[string]bool, curre } if okayToDelete { if err := proxier.ipvs.DeleteVirtualServer(svc); err != nil { - glog.Errorf("Failed to delete service, error: %v", err) + klog.Errorf("Failed to delete service, error: %v", err) } } } @@ -1647,11 +1649,11 @@ func (proxier *Proxier) cleanLegacyBindAddr(activeBindAddrs map[string]bool, cur for _, addr := range currentBindAddrs { if _, ok := activeBindAddrs[addr]; !ok { // This address was not processed in the latest sync loop - glog.V(4).Infof("Unbind addr %s", addr) + klog.V(4).Infof("Unbind addr %s", addr) err := proxier.netlinkHandle.UnbindAddress(addr, DefaultDummyDevice) // Ignore no such address error when try to unbind address if err != nil { - glog.Errorf("Failed to unbind service addr %s from dummy interface %s: %v", addr, DefaultDummyDevice, err) + klog.Errorf("Failed to unbind service addr %s from dummy interface %s: %v", addr, DefaultDummyDevice, err) } } } @@ -1717,7 +1719,7 @@ func openLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) { default: return nil, fmt.Errorf("unknown protocol %q", lp.Protocol) } - glog.V(2).Infof("Opened local port %s", lp.String()) + klog.V(2).Infof("Opened local port %s", lp.String()) return socket, nil } diff --git a/pkg/proxy/service.go b/pkg/proxy/service.go index 5853ef9ca00..8386e62c0e2 100644 --- a/pkg/proxy/service.go +++ b/pkg/proxy/service.go @@ -23,7 +23,7 @@ import ( "strings" "sync" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -119,7 +119,7 @@ func (sct *ServiceChangeTracker) newBaseServiceInfo(port *v1.ServicePort, servic if apiservice.NeedsHealthCheck(service) { p := service.Spec.HealthCheckNodePort if p == 0 { - glog.Errorf("Service %s/%s has no healthcheck nodeport", service.Namespace, service.Name) + klog.Errorf("Service %s/%s has no healthcheck nodeport", service.Namespace, service.Name) } else { info.HealthCheckNodePort = int(p) } @@ -306,9 +306,9 @@ func (sm *ServiceMap) merge(other ServiceMap) sets.String { existingPorts.Insert(svcPortName.String()) _, exists := (*sm)[svcPortName] if !exists { - glog.V(1).Infof("Adding new service port %q at %s", svcPortName, info.String()) + klog.V(1).Infof("Adding new service port %q at %s", svcPortName, info.String()) } else { - glog.V(1).Infof("Updating existing service port %q at %s", svcPortName, info.String()) + klog.V(1).Infof("Updating existing service port %q at %s", svcPortName, info.String()) } (*sm)[svcPortName] = info } @@ -331,13 +331,13 @@ func (sm *ServiceMap) unmerge(other ServiceMap, UDPStaleClusterIP sets.String) { for svcPortName := range other { info, exists := (*sm)[svcPortName] if exists { - glog.V(1).Infof("Removing service port %q", svcPortName) + klog.V(1).Infof("Removing service port %q", svcPortName) if info.GetProtocol() == v1.ProtocolUDP { UDPStaleClusterIP.Insert(info.ClusterIPString()) } delete(*sm, svcPortName) } else { - glog.Errorf("Service port %q doesn't exists", svcPortName) + klog.Errorf("Service port %q doesn't exists", svcPortName) } } } diff --git a/pkg/proxy/userspace/BUILD b/pkg/proxy/userspace/BUILD index 0d800fc66d3..b7887c85b5d 100644 --- a/pkg/proxy/userspace/BUILD +++ b/pkg/proxy/userspace/BUILD @@ -32,7 +32,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:android": [ diff --git a/pkg/proxy/userspace/proxier.go b/pkg/proxy/userspace/proxier.go index 38ee0c2ce6a..661092b1b2b 100644 --- a/pkg/proxy/userspace/proxier.go +++ b/pkg/proxy/userspace/proxier.go @@ -25,13 +25,13 @@ import ( "sync/atomic" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/proxy" utilproxy "k8s.io/kubernetes/pkg/proxy/util" @@ -81,7 +81,7 @@ func (info *ServiceInfo) IsAlive() bool { func logTimeout(err error) bool { if e, ok := err.(net.Error); ok { if e.Timeout() { - glog.V(3).Infof("connection to endpoint closed due to inactivity") + klog.V(3).Infof("connection to endpoint closed due to inactivity") return true } } @@ -184,7 +184,7 @@ func NewCustomProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptab proxyPorts := newPortAllocator(pr) - glog.V(2).Infof("Setting proxy IP to %v and initializing iptables", hostIP) + klog.V(2).Infof("Setting proxy IP to %v and initializing iptables", hostIP) return createProxier(loadBalancer, listenIP, iptables, exec, hostIP, proxyPorts, syncPeriod, minSyncPeriod, udpIdleTimeout, makeProxySocket) } @@ -229,13 +229,13 @@ func CleanupLeftovers(ipt iptables.Interface) (encounteredError bool) { args := []string{"-m", "comment", "--comment", "handle ClusterIPs; NOTE: this must be before the NodePort rules"} if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainOutput, append(args, "-j", string(iptablesHostPortalChain))...); err != nil { if !iptables.IsNotFoundError(err) { - glog.Errorf("Error removing userspace rule: %v", err) + klog.Errorf("Error removing userspace rule: %v", err) encounteredError = true } } if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainPrerouting, append(args, "-j", string(iptablesContainerPortalChain))...); err != nil { if !iptables.IsNotFoundError(err) { - glog.Errorf("Error removing userspace rule: %v", err) + klog.Errorf("Error removing userspace rule: %v", err) encounteredError = true } } @@ -243,20 +243,20 @@ func CleanupLeftovers(ipt iptables.Interface) (encounteredError bool) { args = append(args, "-m", "comment", "--comment", "handle service NodePorts; NOTE: this must be the last rule in the chain") if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainOutput, append(args, "-j", string(iptablesHostNodePortChain))...); err != nil { if !iptables.IsNotFoundError(err) { - glog.Errorf("Error removing userspace rule: %v", err) + klog.Errorf("Error removing userspace rule: %v", err) encounteredError = true } } if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainPrerouting, append(args, "-j", string(iptablesContainerNodePortChain))...); err != nil { if !iptables.IsNotFoundError(err) { - glog.Errorf("Error removing userspace rule: %v", err) + klog.Errorf("Error removing userspace rule: %v", err) encounteredError = true } } args = []string{"-m", "comment", "--comment", "Ensure that non-local NodePort traffic can flow"} if err := ipt.DeleteRule(iptables.TableFilter, iptables.ChainInput, append(args, "-j", string(iptablesNonLocalNodePortChain))...); err != nil { if !iptables.IsNotFoundError(err) { - glog.Errorf("Error removing userspace rule: %v", err) + klog.Errorf("Error removing userspace rule: %v", err) encounteredError = true } } @@ -271,13 +271,13 @@ func CleanupLeftovers(ipt iptables.Interface) (encounteredError bool) { // flush chain, then if successful delete, delete will fail if flush fails. if err := ipt.FlushChain(table, c); err != nil { if !iptables.IsNotFoundError(err) { - glog.Errorf("Error flushing userspace chain: %v", err) + klog.Errorf("Error flushing userspace chain: %v", err) encounteredError = true } } else { if err = ipt.DeleteChain(table, c); err != nil { if !iptables.IsNotFoundError(err) { - glog.Errorf("Error deleting userspace chain: %v", err) + klog.Errorf("Error deleting userspace chain: %v", err) encounteredError = true } } @@ -290,7 +290,7 @@ func CleanupLeftovers(ipt iptables.Interface) (encounteredError bool) { // Sync is called to immediately synchronize the proxier state to iptables func (proxier *Proxier) Sync() { if err := iptablesInit(proxier.iptables); err != nil { - glog.Errorf("Failed to ensure iptables: %v", err) + klog.Errorf("Failed to ensure iptables: %v", err) } proxier.ensurePortals() proxier.cleanupStaleStickySessions() @@ -302,7 +302,7 @@ func (proxier *Proxier) SyncLoop() { defer t.Stop() for { <-t.C - glog.V(6).Infof("Periodic sync") + klog.V(6).Infof("Periodic sync") proxier.Sync() } } @@ -315,7 +315,7 @@ func (proxier *Proxier) ensurePortals() { for name, info := range proxier.serviceMap { err := proxier.openPortal(name, info) if err != nil { - glog.Errorf("Failed to ensure portal for %q: %v", name, err) + klog.Errorf("Failed to ensure portal for %q: %v", name, err) } } } @@ -388,7 +388,7 @@ func (proxier *Proxier) addServiceOnPort(service proxy.ServicePortName, protocol } proxier.setServiceInfo(service, si) - glog.V(2).Infof("Proxying for service %q on %s port %d", service, protocol, portNum) + klog.V(2).Infof("Proxying for service %q on %s port %d", service, protocol, portNum) go func(service proxy.ServicePortName, proxier *Proxier) { defer runtime.HandleCrash() atomic.AddInt32(&proxier.numProxyLoops, 1) @@ -405,7 +405,7 @@ func (proxier *Proxier) mergeService(service *v1.Service) sets.String { } svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} if !helper.IsServiceIPSet(service) { - glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) + klog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) return nil } existingPorts := sets.NewString() @@ -420,25 +420,25 @@ func (proxier *Proxier) mergeService(service *v1.Service) sets.String { continue } if exists { - glog.V(4).Infof("Something changed for service %q: stopping it", serviceName) + klog.V(4).Infof("Something changed for service %q: stopping it", serviceName) if err := proxier.closePortal(serviceName, info); err != nil { - glog.Errorf("Failed to close portal for %q: %v", serviceName, err) + klog.Errorf("Failed to close portal for %q: %v", serviceName, err) } if err := proxier.stopProxy(serviceName, info); err != nil { - glog.Errorf("Failed to stop service %q: %v", serviceName, err) + klog.Errorf("Failed to stop service %q: %v", serviceName, err) } } proxyPort, err := proxier.proxyPorts.AllocateNext() if err != nil { - glog.Errorf("failed to allocate proxy port for service %q: %v", serviceName, err) + klog.Errorf("failed to allocate proxy port for service %q: %v", serviceName, err) continue } serviceIP := net.ParseIP(service.Spec.ClusterIP) - glog.V(1).Infof("Adding new service %q at %s/%s", serviceName, net.JoinHostPort(serviceIP.String(), strconv.Itoa(int(servicePort.Port))), servicePort.Protocol) + klog.V(1).Infof("Adding new service %q at %s/%s", serviceName, net.JoinHostPort(serviceIP.String(), strconv.Itoa(int(servicePort.Port))), servicePort.Protocol) info, err = proxier.addServiceOnPort(serviceName, servicePort.Protocol, proxyPort, proxier.udpIdleTimeout) if err != nil { - glog.Errorf("Failed to start proxy for %q: %v", serviceName, err) + klog.Errorf("Failed to start proxy for %q: %v", serviceName, err) continue } info.portal.ip = serviceIP @@ -453,10 +453,10 @@ func (proxier *Proxier) mergeService(service *v1.Service) sets.String { info.stickyMaxAgeSeconds = int(*service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds) } - glog.V(4).Infof("info: %#v", info) + klog.V(4).Infof("info: %#v", info) if err := proxier.openPortal(serviceName, info); err != nil { - glog.Errorf("Failed to open portal for %q: %v", serviceName, err) + klog.Errorf("Failed to open portal for %q: %v", serviceName, err) } proxier.loadBalancer.NewService(serviceName, info.sessionAffinityType, info.stickyMaxAgeSeconds) } @@ -470,7 +470,7 @@ func (proxier *Proxier) unmergeService(service *v1.Service, existingPorts sets.S } svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} if !helper.IsServiceIPSet(service) { - glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) + klog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) return } @@ -484,10 +484,10 @@ func (proxier *Proxier) unmergeService(service *v1.Service, existingPorts sets.S } serviceName := proxy.ServicePortName{NamespacedName: svcName, Port: servicePort.Name} - glog.V(1).Infof("Stopping service %q", serviceName) + klog.V(1).Infof("Stopping service %q", serviceName) info, exists := proxier.serviceMap[serviceName] if !exists { - glog.Errorf("Service %q is being removed but doesn't exist", serviceName) + klog.Errorf("Service %q is being removed but doesn't exist", serviceName) continue } @@ -496,16 +496,16 @@ func (proxier *Proxier) unmergeService(service *v1.Service, existingPorts sets.S } if err := proxier.closePortal(serviceName, info); err != nil { - glog.Errorf("Failed to close portal for %q: %v", serviceName, err) + klog.Errorf("Failed to close portal for %q: %v", serviceName, err) } if err := proxier.stopProxyInternal(serviceName, info); err != nil { - glog.Errorf("Failed to stop service %q: %v", serviceName, err) + klog.Errorf("Failed to stop service %q: %v", serviceName, err) } proxier.loadBalancer.DeleteService(serviceName) } for _, svcIP := range staleUDPServices.UnsortedList() { if err := conntrack.ClearEntriesForIP(proxier.exec, svcIP, v1.ProtocolUDP); err != nil { - glog.Errorf("Failed to delete stale service IP %s connections, error: %v", svcIP, err) + klog.Errorf("Failed to delete stale service IP %s connections, error: %v", svcIP, err) } } } @@ -600,31 +600,31 @@ func (proxier *Proxier) openOnePortal(portal portal, protocol v1.Protocol, proxy portalAddress := net.JoinHostPort(portal.ip.String(), strconv.Itoa(portal.port)) existed, err := proxier.iptables.EnsureRule(iptables.Append, iptables.TableNAT, iptablesContainerPortalChain, args...) if err != nil { - glog.Errorf("Failed to install iptables %s rule for service %q, args:%v", iptablesContainerPortalChain, name, args) + klog.Errorf("Failed to install iptables %s rule for service %q, args:%v", iptablesContainerPortalChain, name, args) return err } if !existed { - glog.V(3).Infof("Opened iptables from-containers portal for service %q on %s %s", name, protocol, portalAddress) + klog.V(3).Infof("Opened iptables from-containers portal for service %q on %s %s", name, protocol, portalAddress) } if portal.isExternal { args := proxier.iptablesContainerPortalArgs(portal.ip, false, true, portal.port, protocol, proxyIP, proxyPort, name) existed, err := proxier.iptables.EnsureRule(iptables.Append, iptables.TableNAT, iptablesContainerPortalChain, args...) if err != nil { - glog.Errorf("Failed to install iptables %s rule that opens service %q for local traffic, args:%v", iptablesContainerPortalChain, name, args) + klog.Errorf("Failed to install iptables %s rule that opens service %q for local traffic, args:%v", iptablesContainerPortalChain, name, args) return err } if !existed { - glog.V(3).Infof("Opened iptables from-containers portal for service %q on %s %s for local traffic", name, protocol, portalAddress) + klog.V(3).Infof("Opened iptables from-containers portal for service %q on %s %s for local traffic", name, protocol, portalAddress) } args = proxier.iptablesHostPortalArgs(portal.ip, true, portal.port, protocol, proxyIP, proxyPort, name) existed, err = proxier.iptables.EnsureRule(iptables.Append, iptables.TableNAT, iptablesHostPortalChain, args...) if err != nil { - glog.Errorf("Failed to install iptables %s rule for service %q for dst-local traffic", iptablesHostPortalChain, name) + klog.Errorf("Failed to install iptables %s rule for service %q for dst-local traffic", iptablesHostPortalChain, name) return err } if !existed { - glog.V(3).Infof("Opened iptables from-host portal for service %q on %s %s for dst-local traffic", name, protocol, portalAddress) + klog.V(3).Infof("Opened iptables from-host portal for service %q on %s %s for dst-local traffic", name, protocol, portalAddress) } return nil } @@ -633,11 +633,11 @@ func (proxier *Proxier) openOnePortal(portal portal, protocol v1.Protocol, proxy args = proxier.iptablesHostPortalArgs(portal.ip, false, portal.port, protocol, proxyIP, proxyPort, name) existed, err = proxier.iptables.EnsureRule(iptables.Append, iptables.TableNAT, iptablesHostPortalChain, args...) if err != nil { - glog.Errorf("Failed to install iptables %s rule for service %q", iptablesHostPortalChain, name) + klog.Errorf("Failed to install iptables %s rule for service %q", iptablesHostPortalChain, name) return err } if !existed { - glog.V(3).Infof("Opened iptables from-host portal for service %q on %s %s", name, protocol, portalAddress) + klog.V(3).Infof("Opened iptables from-host portal for service %q on %s %s", name, protocol, portalAddress) } return nil } @@ -665,7 +665,7 @@ func (proxier *Proxier) claimNodePort(ip net.IP, port int, protocol v1.Protocol, return fmt.Errorf("can't open node port for %s: %v", key.String(), err) } proxier.portMap[key] = &portMapValue{owner: owner, socket: socket} - glog.V(2).Infof("Claimed local port %s", key.String()) + klog.V(2).Infof("Claimed local port %s", key.String()) return nil } if existing.owner == owner { @@ -685,7 +685,7 @@ func (proxier *Proxier) releaseNodePort(ip net.IP, port int, protocol v1.Protoco existing, found := proxier.portMap[key] if !found { // We tolerate this, it happens if we are cleaning up a failed allocation - glog.Infof("Ignoring release on unowned port: %v", key) + klog.Infof("Ignoring release on unowned port: %v", key) return nil } if existing.owner != owner { @@ -709,32 +709,32 @@ func (proxier *Proxier) openNodePort(nodePort int, protocol v1.Protocol, proxyIP args := proxier.iptablesContainerPortalArgs(nil, false, false, nodePort, protocol, proxyIP, proxyPort, name) existed, err := proxier.iptables.EnsureRule(iptables.Append, iptables.TableNAT, iptablesContainerNodePortChain, args...) if err != nil { - glog.Errorf("Failed to install iptables %s rule for service %q", iptablesContainerNodePortChain, name) + klog.Errorf("Failed to install iptables %s rule for service %q", iptablesContainerNodePortChain, name) return err } if !existed { - glog.Infof("Opened iptables from-containers public port for service %q on %s port %d", name, protocol, nodePort) + klog.Infof("Opened iptables from-containers public port for service %q on %s port %d", name, protocol, nodePort) } // Handle traffic from the host. args = proxier.iptablesHostNodePortArgs(nodePort, protocol, proxyIP, proxyPort, name) existed, err = proxier.iptables.EnsureRule(iptables.Append, iptables.TableNAT, iptablesHostNodePortChain, args...) if err != nil { - glog.Errorf("Failed to install iptables %s rule for service %q", iptablesHostNodePortChain, name) + klog.Errorf("Failed to install iptables %s rule for service %q", iptablesHostNodePortChain, name) return err } if !existed { - glog.Infof("Opened iptables from-host public port for service %q on %s port %d", name, protocol, nodePort) + klog.Infof("Opened iptables from-host public port for service %q on %s port %d", name, protocol, nodePort) } args = proxier.iptablesNonLocalNodePortArgs(nodePort, protocol, proxyIP, proxyPort, name) existed, err = proxier.iptables.EnsureRule(iptables.Append, iptables.TableFilter, iptablesNonLocalNodePortChain, args...) if err != nil { - glog.Errorf("Failed to install iptables %s rule for service %q", iptablesNonLocalNodePortChain, name) + klog.Errorf("Failed to install iptables %s rule for service %q", iptablesNonLocalNodePortChain, name) return err } if !existed { - glog.Infof("Opened iptables from-non-local public port for service %q on %s port %d", name, protocol, nodePort) + klog.Infof("Opened iptables from-non-local public port for service %q on %s port %d", name, protocol, nodePort) } return nil @@ -755,9 +755,9 @@ func (proxier *Proxier) closePortal(service proxy.ServicePortName, info *Service el = append(el, proxier.closeNodePort(info.nodePort, info.protocol, proxier.listenIP, info.proxyPort, service)...) } if len(el) == 0 { - glog.V(3).Infof("Closed iptables portals for service %q", service) + klog.V(3).Infof("Closed iptables portals for service %q", service) } else { - glog.Errorf("Some errors closing iptables portals for service %q", service) + klog.Errorf("Some errors closing iptables portals for service %q", service) } return utilerrors.NewAggregate(el) } @@ -776,20 +776,20 @@ func (proxier *Proxier) closeOnePortal(portal portal, protocol v1.Protocol, prox // Handle traffic from containers. args := proxier.iptablesContainerPortalArgs(portal.ip, portal.isExternal, false, portal.port, protocol, proxyIP, proxyPort, name) if err := proxier.iptables.DeleteRule(iptables.TableNAT, iptablesContainerPortalChain, args...); err != nil { - glog.Errorf("Failed to delete iptables %s rule for service %q", iptablesContainerPortalChain, name) + klog.Errorf("Failed to delete iptables %s rule for service %q", iptablesContainerPortalChain, name) el = append(el, err) } if portal.isExternal { args := proxier.iptablesContainerPortalArgs(portal.ip, false, true, portal.port, protocol, proxyIP, proxyPort, name) if err := proxier.iptables.DeleteRule(iptables.TableNAT, iptablesContainerPortalChain, args...); err != nil { - glog.Errorf("Failed to delete iptables %s rule for service %q", iptablesContainerPortalChain, name) + klog.Errorf("Failed to delete iptables %s rule for service %q", iptablesContainerPortalChain, name) el = append(el, err) } args = proxier.iptablesHostPortalArgs(portal.ip, true, portal.port, protocol, proxyIP, proxyPort, name) if err := proxier.iptables.DeleteRule(iptables.TableNAT, iptablesHostPortalChain, args...); err != nil { - glog.Errorf("Failed to delete iptables %s rule for service %q", iptablesHostPortalChain, name) + klog.Errorf("Failed to delete iptables %s rule for service %q", iptablesHostPortalChain, name) el = append(el, err) } return el @@ -798,7 +798,7 @@ func (proxier *Proxier) closeOnePortal(portal portal, protocol v1.Protocol, prox // Handle traffic from the host (portalIP is not external). args = proxier.iptablesHostPortalArgs(portal.ip, false, portal.port, protocol, proxyIP, proxyPort, name) if err := proxier.iptables.DeleteRule(iptables.TableNAT, iptablesHostPortalChain, args...); err != nil { - glog.Errorf("Failed to delete iptables %s rule for service %q", iptablesHostPortalChain, name) + klog.Errorf("Failed to delete iptables %s rule for service %q", iptablesHostPortalChain, name) el = append(el, err) } @@ -811,21 +811,21 @@ func (proxier *Proxier) closeNodePort(nodePort int, protocol v1.Protocol, proxyI // Handle traffic from containers. args := proxier.iptablesContainerPortalArgs(nil, false, false, nodePort, protocol, proxyIP, proxyPort, name) if err := proxier.iptables.DeleteRule(iptables.TableNAT, iptablesContainerNodePortChain, args...); err != nil { - glog.Errorf("Failed to delete iptables %s rule for service %q", iptablesContainerNodePortChain, name) + klog.Errorf("Failed to delete iptables %s rule for service %q", iptablesContainerNodePortChain, name) el = append(el, err) } // Handle traffic from the host. args = proxier.iptablesHostNodePortArgs(nodePort, protocol, proxyIP, proxyPort, name) if err := proxier.iptables.DeleteRule(iptables.TableNAT, iptablesHostNodePortChain, args...); err != nil { - glog.Errorf("Failed to delete iptables %s rule for service %q", iptablesHostNodePortChain, name) + klog.Errorf("Failed to delete iptables %s rule for service %q", iptablesHostNodePortChain, name) el = append(el, err) } // Handle traffic not local to the host args = proxier.iptablesNonLocalNodePortArgs(nodePort, protocol, proxyIP, proxyPort, name) if err := proxier.iptables.DeleteRule(iptables.TableFilter, iptablesNonLocalNodePortChain, args...); err != nil { - glog.Errorf("Failed to delete iptables %s rule for service %q", iptablesNonLocalNodePortChain, name) + klog.Errorf("Failed to delete iptables %s rule for service %q", iptablesNonLocalNodePortChain, name) el = append(el, err) } @@ -934,7 +934,7 @@ func iptablesFlush(ipt iptables.Interface) error { el = append(el, err) } if len(el) != 0 { - glog.Errorf("Some errors flushing old iptables portals: %v", el) + klog.Errorf("Some errors flushing old iptables portals: %v", el) } return utilerrors.NewAggregate(el) } diff --git a/pkg/proxy/userspace/proxysocket.go b/pkg/proxy/userspace/proxysocket.go index 098f68c15aa..2ff49805465 100644 --- a/pkg/proxy/userspace/proxysocket.go +++ b/pkg/proxy/userspace/proxysocket.go @@ -25,9 +25,9 @@ import ( "sync" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog" "k8s.io/kubernetes/pkg/proxy" ) @@ -95,10 +95,10 @@ func TryConnectEndpoints(service proxy.ServicePortName, srcAddr net.Addr, protoc for _, dialTimeout := range EndpointDialTimeouts { endpoint, err := loadBalancer.NextEndpoint(service, srcAddr, sessionAffinityReset) if err != nil { - glog.Errorf("Couldn't find an endpoint for %s: %v", service, err) + klog.Errorf("Couldn't find an endpoint for %s: %v", service, err) return nil, err } - glog.V(3).Infof("Mapped service %q to endpoint %s", service, endpoint) + klog.V(3).Infof("Mapped service %q to endpoint %s", service, endpoint) // TODO: This could spin up a new goroutine to make the outbound connection, // and keep accepting inbound traffic. outConn, err := net.DialTimeout(protocol, endpoint, dialTimeout) @@ -106,7 +106,7 @@ func TryConnectEndpoints(service proxy.ServicePortName, srcAddr net.Addr, protoc if isTooManyFDsError(err) { panic("Dial failed: " + err.Error()) } - glog.Errorf("Dial failed: %v", err) + klog.Errorf("Dial failed: %v", err) sessionAffinityReset = true continue } @@ -135,13 +135,13 @@ func (tcp *tcpProxySocket) ProxyLoop(service proxy.ServicePortName, myInfo *Serv // Then the service port was just closed so the accept failure is to be expected. return } - glog.Errorf("Accept failed: %v", err) + klog.Errorf("Accept failed: %v", err) continue } - glog.V(3).Infof("Accepted TCP connection from %v to %v", inConn.RemoteAddr(), inConn.LocalAddr()) + klog.V(3).Infof("Accepted TCP connection from %v to %v", inConn.RemoteAddr(), inConn.LocalAddr()) outConn, err := TryConnectEndpoints(service, inConn.(*net.TCPConn).RemoteAddr(), "tcp", loadBalancer) if err != nil { - glog.Errorf("Failed to connect to balancer: %v", err) + klog.Errorf("Failed to connect to balancer: %v", err) inConn.Close() continue } @@ -154,7 +154,7 @@ func (tcp *tcpProxySocket) ProxyLoop(service proxy.ServicePortName, myInfo *Serv func ProxyTCP(in, out *net.TCPConn) { var wg sync.WaitGroup wg.Add(2) - glog.V(4).Infof("Creating proxy between %v <-> %v <-> %v <-> %v", + klog.V(4).Infof("Creating proxy between %v <-> %v <-> %v <-> %v", in.RemoteAddr(), in.LocalAddr(), out.LocalAddr(), out.RemoteAddr()) go copyBytes("from backend", in, out, &wg) go copyBytes("to backend", out, in, &wg) @@ -163,14 +163,14 @@ func ProxyTCP(in, out *net.TCPConn) { func copyBytes(direction string, dest, src *net.TCPConn, wg *sync.WaitGroup) { defer wg.Done() - glog.V(4).Infof("Copying %s: %s -> %s", direction, src.RemoteAddr(), dest.RemoteAddr()) + klog.V(4).Infof("Copying %s: %s -> %s", direction, src.RemoteAddr(), dest.RemoteAddr()) n, err := io.Copy(dest, src) if err != nil { if !isClosedError(err) { - glog.Errorf("I/O error: %v", err) + klog.Errorf("I/O error: %v", err) } } - glog.V(4).Infof("Copied %d bytes %s: %s -> %s", n, direction, src.RemoteAddr(), dest.RemoteAddr()) + klog.V(4).Infof("Copied %d bytes %s: %s -> %s", n, direction, src.RemoteAddr(), dest.RemoteAddr()) dest.Close() src.Close() } @@ -215,11 +215,11 @@ func (udp *udpProxySocket) ProxyLoop(service proxy.ServicePortName, myInfo *Serv if err != nil { if e, ok := err.(net.Error); ok { if e.Temporary() { - glog.V(1).Infof("ReadFrom had a temporary failure: %v", err) + klog.V(1).Infof("ReadFrom had a temporary failure: %v", err) continue } } - glog.Errorf("ReadFrom failed, exiting ProxyLoop: %v", err) + klog.Errorf("ReadFrom failed, exiting ProxyLoop: %v", err) break } // If this is a client we know already, reuse the connection and goroutine. @@ -232,14 +232,14 @@ func (udp *udpProxySocket) ProxyLoop(service proxy.ServicePortName, myInfo *Serv _, err = svrConn.Write(buffer[0:n]) if err != nil { if !logTimeout(err) { - glog.Errorf("Write failed: %v", err) + klog.Errorf("Write failed: %v", err) // TODO: Maybe tear down the goroutine for this client/server pair? } continue } err = svrConn.SetDeadline(time.Now().Add(myInfo.Timeout)) if err != nil { - glog.Errorf("SetDeadline failed: %v", err) + klog.Errorf("SetDeadline failed: %v", err) continue } } @@ -253,14 +253,14 @@ func (udp *udpProxySocket) getBackendConn(activeClients *ClientCache, cliAddr ne if !found { // TODO: This could spin up a new goroutine to make the outbound connection, // and keep accepting inbound traffic. - glog.V(3).Infof("New UDP connection from %s", cliAddr) + klog.V(3).Infof("New UDP connection from %s", cliAddr) var err error svrConn, err = TryConnectEndpoints(service, cliAddr, "udp", loadBalancer) if err != nil { return nil, err } if err = svrConn.SetDeadline(time.Now().Add(timeout)); err != nil { - glog.Errorf("SetDeadline failed: %v", err) + klog.Errorf("SetDeadline failed: %v", err) return nil, err } activeClients.Clients[cliAddr.String()] = svrConn @@ -281,19 +281,19 @@ func (udp *udpProxySocket) proxyClient(cliAddr net.Addr, svrConn net.Conn, activ n, err := svrConn.Read(buffer[0:]) if err != nil { if !logTimeout(err) { - glog.Errorf("Read failed: %v", err) + klog.Errorf("Read failed: %v", err) } break } err = svrConn.SetDeadline(time.Now().Add(timeout)) if err != nil { - glog.Errorf("SetDeadline failed: %v", err) + klog.Errorf("SetDeadline failed: %v", err) break } n, err = udp.WriteTo(buffer[0:n], cliAddr) if err != nil { if !logTimeout(err) { - glog.Errorf("WriteTo failed: %v", err) + klog.Errorf("WriteTo failed: %v", err) } break } diff --git a/pkg/proxy/userspace/roundrobin.go b/pkg/proxy/userspace/roundrobin.go index 5e84e8c27c9..6bbc558eae8 100644 --- a/pkg/proxy/userspace/roundrobin.go +++ b/pkg/proxy/userspace/roundrobin.go @@ -25,9 +25,9 @@ import ( "sync" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/proxy" "k8s.io/kubernetes/pkg/util/slice" ) @@ -82,7 +82,7 @@ func NewLoadBalancerRR() *LoadBalancerRR { } func (lb *LoadBalancerRR) NewService(svcPort proxy.ServicePortName, affinityType v1.ServiceAffinity, ttlSeconds int) error { - glog.V(4).Infof("LoadBalancerRR NewService %q", svcPort) + klog.V(4).Infof("LoadBalancerRR NewService %q", svcPort) lb.lock.Lock() defer lb.lock.Unlock() lb.newServiceInternal(svcPort, affinityType, ttlSeconds) @@ -97,7 +97,7 @@ func (lb *LoadBalancerRR) newServiceInternal(svcPort proxy.ServicePortName, affi if _, exists := lb.services[svcPort]; !exists { lb.services[svcPort] = &balancerState{affinity: *newAffinityPolicy(affinityType, ttlSeconds)} - glog.V(4).Infof("LoadBalancerRR service %q did not exist, created", svcPort) + klog.V(4).Infof("LoadBalancerRR service %q did not exist, created", svcPort) } else if affinityType != "" { lb.services[svcPort].affinity.affinityType = affinityType } @@ -105,7 +105,7 @@ func (lb *LoadBalancerRR) newServiceInternal(svcPort proxy.ServicePortName, affi } func (lb *LoadBalancerRR) DeleteService(svcPort proxy.ServicePortName) { - glog.V(4).Infof("LoadBalancerRR DeleteService %q", svcPort) + klog.V(4).Infof("LoadBalancerRR DeleteService %q", svcPort) lb.lock.Lock() defer lb.lock.Unlock() delete(lb.services, svcPort) @@ -145,7 +145,7 @@ func (lb *LoadBalancerRR) NextEndpoint(svcPort proxy.ServicePortName, srcAddr ne if len(state.endpoints) == 0 { return "", ErrMissingEndpoints } - glog.V(4).Infof("NextEndpoint for service %q, srcAddr=%v: endpoints: %+v", svcPort, srcAddr, state.endpoints) + klog.V(4).Infof("NextEndpoint for service %q, srcAddr=%v: endpoints: %+v", svcPort, srcAddr, state.endpoints) sessionAffinityEnabled := isSessionAffinity(&state.affinity) @@ -163,7 +163,7 @@ func (lb *LoadBalancerRR) NextEndpoint(svcPort proxy.ServicePortName, srcAddr ne // Affinity wins. endpoint := sessionAffinity.endpoint sessionAffinity.lastUsed = time.Now() - glog.V(4).Infof("NextEndpoint for service %q from IP %s with sessionAffinity %#v: %s", svcPort, ipaddr, sessionAffinity, endpoint) + klog.V(4).Infof("NextEndpoint for service %q from IP %s with sessionAffinity %#v: %s", svcPort, ipaddr, sessionAffinity, endpoint) return endpoint, nil } } @@ -182,7 +182,7 @@ func (lb *LoadBalancerRR) NextEndpoint(svcPort proxy.ServicePortName, srcAddr ne affinity.lastUsed = time.Now() affinity.endpoint = endpoint affinity.clientIP = ipaddr - glog.V(4).Infof("Updated affinity key %s: %#v", ipaddr, state.affinity.affinityMap[ipaddr]) + klog.V(4).Infof("Updated affinity key %s: %#v", ipaddr, state.affinity.affinityMap[ipaddr]) } return endpoint, nil @@ -214,7 +214,7 @@ func flattenValidEndpoints(endpoints []hostPortPair) []string { func removeSessionAffinityByEndpoint(state *balancerState, svcPort proxy.ServicePortName, endpoint string) { for _, affinity := range state.affinity.affinityMap { if affinity.endpoint == endpoint { - glog.V(4).Infof("Removing client: %s from affinityMap for service %q", affinity.endpoint, svcPort) + klog.V(4).Infof("Removing client: %s from affinityMap for service %q", affinity.endpoint, svcPort) delete(state.affinity.affinityMap, affinity.clientIP) } } @@ -237,7 +237,7 @@ func (lb *LoadBalancerRR) updateAffinityMap(svcPort proxy.ServicePortName, newEn } for mKey, mVal := range allEndpoints { if mVal == 1 { - glog.V(2).Infof("Delete endpoint %s for service %q", mKey, svcPort) + klog.V(2).Infof("Delete endpoint %s for service %q", mKey, svcPort) removeSessionAffinityByEndpoint(state, svcPort, mKey) } } @@ -273,7 +273,7 @@ func (lb *LoadBalancerRR) OnEndpointsAdd(endpoints *v1.Endpoints) { state, exists := lb.services[svcPort] if !exists || state == nil || len(newEndpoints) > 0 { - glog.V(1).Infof("LoadBalancerRR: Setting endpoints for %s to %+v", svcPort, newEndpoints) + klog.V(1).Infof("LoadBalancerRR: Setting endpoints for %s to %+v", svcPort, newEndpoints) lb.updateAffinityMap(svcPort, newEndpoints) // OnEndpointsAdd can be called without NewService being called externally. // To be safe we will call it here. A new service will only be created @@ -307,7 +307,7 @@ func (lb *LoadBalancerRR) OnEndpointsUpdate(oldEndpoints, endpoints *v1.Endpoint } if !exists || state == nil || len(curEndpoints) != len(newEndpoints) || !slicesEquiv(slice.CopyStrings(curEndpoints), newEndpoints) { - glog.V(1).Infof("LoadBalancerRR: Setting endpoints for %s to %+v", svcPort, newEndpoints) + klog.V(1).Infof("LoadBalancerRR: Setting endpoints for %s to %+v", svcPort, newEndpoints) lb.updateAffinityMap(svcPort, newEndpoints) // OnEndpointsUpdate can be called without NewService being called externally. // To be safe we will call it here. A new service will only be created @@ -335,7 +335,7 @@ func (lb *LoadBalancerRR) resetService(svcPort proxy.ServicePortName) { // If the service is still around, reset but don't delete. if state, ok := lb.services[svcPort]; ok { if len(state.endpoints) > 0 { - glog.V(2).Infof("LoadBalancerRR: Removing endpoints for %s", svcPort) + klog.V(2).Infof("LoadBalancerRR: Removing endpoints for %s", svcPort) state.endpoints = []string{} } state.index = 0 @@ -379,7 +379,7 @@ func (lb *LoadBalancerRR) CleanupStaleStickySessions(svcPort proxy.ServicePortNa } for ip, affinity := range state.affinity.affinityMap { if int(time.Since(affinity.lastUsed).Seconds()) >= state.affinity.ttlSeconds { - glog.V(4).Infof("Removing client %s from affinityMap for service %q", affinity.clientIP, svcPort) + klog.V(4).Infof("Removing client %s from affinityMap for service %q", affinity.clientIP, svcPort) delete(state.affinity.affinityMap, ip) } } diff --git a/pkg/proxy/util/BUILD b/pkg/proxy/util/BUILD index 999dcf88dae..fbbb01b18e7 100644 --- a/pkg/proxy/util/BUILD +++ b/pkg/proxy/util/BUILD @@ -17,7 +17,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/proxy/util/endpoints.go b/pkg/proxy/util/endpoints.go index 7c91b9cd6a1..716491cd25c 100644 --- a/pkg/proxy/util/endpoints.go +++ b/pkg/proxy/util/endpoints.go @@ -21,7 +21,7 @@ import ( "net" "strconv" - "github.com/golang/glog" + "k8s.io/klog" ) // IPPart returns just the IP part of an IP or IP:port or endpoint string. If the IP @@ -35,14 +35,14 @@ func IPPart(s string) string { // Must be IP:port host, _, err := net.SplitHostPort(s) if err != nil { - glog.Errorf("Error parsing '%s': %v", s, err) + klog.Errorf("Error parsing '%s': %v", s, err) return "" } // Check if host string is a valid IP address if ip := net.ParseIP(host); ip != nil { return ip.String() } else { - glog.Errorf("invalid IP part '%s'", host) + klog.Errorf("invalid IP part '%s'", host) } return "" } @@ -52,12 +52,12 @@ func PortPart(s string) (int, error) { // Must be IP:port _, port, err := net.SplitHostPort(s) if err != nil { - glog.Errorf("Error parsing '%s': %v", s, err) + klog.Errorf("Error parsing '%s': %v", s, err) return -1, err } portNumber, err := strconv.Atoi(port) if err != nil { - glog.Errorf("Error parsing '%s': %v", port, err) + klog.Errorf("Error parsing '%s': %v", port, err) return -1, err } return portNumber, nil diff --git a/pkg/proxy/util/port.go b/pkg/proxy/util/port.go index 96317b1dc82..35924e05e87 100644 --- a/pkg/proxy/util/port.go +++ b/pkg/proxy/util/port.go @@ -21,7 +21,7 @@ import ( "net" "strconv" - "github.com/golang/glog" + "k8s.io/klog" ) // LocalPort describes a port on specific IP address and protocol @@ -60,7 +60,7 @@ func RevertPorts(replacementPortsMap, originalPortsMap map[LocalPort]Closeable) for k, v := range replacementPortsMap { // Only close newly opened local ports - leave ones that were open before this update if originalPortsMap[k] == nil { - glog.V(2).Infof("Closing local port %s", k.String()) + klog.V(2).Infof("Closing local port %s", k.String()) v.Close() } } diff --git a/pkg/proxy/util/utils.go b/pkg/proxy/util/utils.go index ca1e6c8fc99..f1db309a941 100644 --- a/pkg/proxy/util/utils.go +++ b/pkg/proxy/util/utils.go @@ -27,7 +27,7 @@ import ( helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" utilnet "k8s.io/kubernetes/pkg/util/net" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -62,12 +62,12 @@ func IsLocalIP(ip string) (bool, error) { func ShouldSkipService(svcName types.NamespacedName, service *v1.Service) bool { // if ClusterIP is "None" or empty, skip proxying if !helper.IsServiceIPSet(service) { - glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) + klog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) return true } // Even if ClusterIP is set, ServiceTypeExternalName services don't get proxied if service.Spec.Type == v1.ServiceTypeExternalName { - glog.V(3).Infof("Skipping service %s due to Type=ExternalName", svcName) + klog.V(3).Infof("Skipping service %s due to Type=ExternalName", svcName) return true } return false @@ -134,7 +134,7 @@ func GetNodeAddresses(cidrs []string, nw NetworkInterfacer) (sets.String, error) // LogAndEmitIncorrectIPVersionEvent logs and emits incorrect IP version event. func LogAndEmitIncorrectIPVersionEvent(recorder record.EventRecorder, fieldName, fieldValue, svcNamespace, svcName string, svcUID types.UID) { errMsg := fmt.Sprintf("%s in %s has incorrect IP version", fieldValue, fieldName) - glog.Errorf("%s (service %s/%s).", errMsg, svcNamespace, svcName) + klog.Errorf("%s (service %s/%s).", errMsg, svcNamespace, svcName) if recorder != nil { recorder.Eventf( &v1.ObjectReference{ diff --git a/pkg/proxy/winkernel/BUILD b/pkg/proxy/winkernel/BUILD index 0e3636f1990..4d6b5e0f881 100644 --- a/pkg/proxy/winkernel/BUILD +++ b/pkg/proxy/winkernel/BUILD @@ -24,7 +24,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//vendor/github.com/Microsoft/hcsshim:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "//conditions:default": [], }), diff --git a/pkg/proxy/winkernel/proxier.go b/pkg/proxy/winkernel/proxier.go index a1343d24b30..b0289ffe4d7 100644 --- a/pkg/proxy/winkernel/proxier.go +++ b/pkg/proxy/winkernel/proxier.go @@ -30,7 +30,7 @@ import ( "github.com/Microsoft/hcsshim" "github.com/davecgh/go-spew/spew" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -107,14 +107,14 @@ type hnsNetworkInfo struct { id string } -func Log(v interface{}, message string, level glog.Level) { - glog.V(level).Infof("%s, %s", message, spew.Sdump(v)) +func Log(v interface{}, message string, level klog.Level) { + klog.V(level).Infof("%s, %s", message, spew.Sdump(v)) } -func LogJson(v interface{}, message string, level glog.Level) { +func LogJson(v interface{}, message string, level klog.Level) { jsonString, err := json.Marshal(v) if err == nil { - glog.V(level).Infof("%s, %s", message, string(jsonString)) + klog.V(level).Infof("%s, %s", message, string(jsonString)) } } @@ -159,7 +159,7 @@ func (ep *endpointsInfo) Cleanup() { // Never delete a Local Endpoint. Local Endpoints are already created by other entities. // Remove only remote endpoints created by this service if ep.refCount <= 0 && !ep.isLocal { - glog.V(4).Infof("Removing endpoints for %v, since no one is referencing it", ep) + klog.V(4).Infof("Removing endpoints for %v, since no one is referencing it", ep) deleteHnsEndpoint(ep.hnsID) ep.hnsID = "" } @@ -206,7 +206,7 @@ func newServiceInfo(svcPortName proxy.ServicePortName, port *v1.ServicePort, ser if apiservice.NeedsHealthCheck(service) { p := service.Spec.HealthCheckNodePort if p == 0 { - glog.Errorf("Service %q has no healthcheck nodeport", svcPortName.NamespacedName.String()) + klog.Errorf("Service %q has no healthcheck nodeport", svcPortName.NamespacedName.String()) } else { info.healthCheckNodePort = int(p) } @@ -303,9 +303,9 @@ func (sm *proxyServiceMap) merge(other proxyServiceMap, curEndpoints proxyEndpoi existingPorts.Insert(svcPortName.Port) svcInfo, exists := (*sm)[svcPortName] if !exists { - glog.V(1).Infof("Adding new service port %q at %s:%d/%s", svcPortName, info.clusterIP, info.port, info.protocol) + klog.V(1).Infof("Adding new service port %q at %s:%d/%s", svcPortName, info.clusterIP, info.port, info.protocol) } else { - glog.V(1).Infof("Updating existing service port %q at %s:%d/%s", svcPortName, info.clusterIP, info.port, info.protocol) + klog.V(1).Infof("Updating existing service port %q at %s:%d/%s", svcPortName, info.clusterIP, info.port, info.protocol) svcInfo.cleanupAllPolicies(curEndpoints[svcPortName]) delete(*sm, svcPortName) } @@ -321,14 +321,14 @@ func (sm *proxyServiceMap) unmerge(other proxyServiceMap, existingPorts, staleSe } info, exists := (*sm)[svcPortName] if exists { - glog.V(1).Infof("Removing service port %q", svcPortName) + klog.V(1).Infof("Removing service port %q", svcPortName) if info.protocol == v1.ProtocolUDP { staleServices.Insert(info.clusterIP.String()) } info.cleanupAllPolicies(curEndpoints[svcPortName]) delete(*sm, svcPortName) } else { - glog.Errorf("Service port %q removed, but doesn't exists", svcPortName) + klog.Errorf("Service port %q removed, but doesn't exists", svcPortName) } } } @@ -340,13 +340,13 @@ func (em proxyEndpointsMap) merge(other proxyEndpointsMap, curServices proxyServ if exists { // info, exists := curServices[svcPortName] - glog.V(1).Infof("Updating existing service port %q at %s:%d/%s", svcPortName, info.clusterIP, info.port, info.protocol) + klog.V(1).Infof("Updating existing service port %q at %s:%d/%s", svcPortName, info.clusterIP, info.port, info.protocol) if exists { - glog.V(2).Infof("Endpoints are modified. Service [%v] is stale", svcPortName) + klog.V(2).Infof("Endpoints are modified. Service [%v] is stale", svcPortName) info.cleanupAllPolicies(epInfos) } else { // If no service exists, just cleanup the remote endpoints - glog.V(2).Infof("Endpoints are orphaned. Cleaning up") + klog.V(2).Infof("Endpoints are orphaned. Cleaning up") // Cleanup Endpoints references for _, ep := range epInfos { ep.Cleanup() @@ -365,11 +365,11 @@ func (em proxyEndpointsMap) unmerge(other proxyEndpointsMap, curServices proxySe for svcPortName := range other { info, exists := curServices[svcPortName] if exists { - glog.V(2).Infof("Service [%v] is stale", info) + klog.V(2).Infof("Service [%v] is stale", info) info.cleanupAllPolicies(em[svcPortName]) } else { // If no service exists, just cleanup the remote endpoints - glog.V(2).Infof("Endpoints are orphaned. Cleaning up") + klog.V(2).Infof("Endpoints are orphaned. Cleaning up") // Cleanup Endpoints references epInfos, exists := em[svcPortName] if exists { @@ -470,12 +470,12 @@ func NewProxier( masqueradeMark := fmt.Sprintf("%#08x/%#08x", masqueradeValue, masqueradeValue) if nodeIP == nil { - glog.Warningf("invalid nodeIP, initializing kube-proxy with 127.0.0.1 as nodeIP") + klog.Warningf("invalid nodeIP, initializing kube-proxy with 127.0.0.1 as nodeIP") nodeIP = net.ParseIP("127.0.0.1") } if len(clusterCIDR) == 0 { - glog.Warningf("clusterCIDR not specified, unable to distinguish between internal and external traffic") + klog.Warningf("clusterCIDR not specified, unable to distinguish between internal and external traffic") } healthChecker := healthcheck.NewServer(hostname, recorder, nil, nil) // use default implementations of deps @@ -487,11 +487,11 @@ func NewProxier( } hnsNetwork, err := getHnsNetworkInfo(hnsNetworkName) if err != nil { - glog.Fatalf("Unable to find Hns Network specified by %s. Please check environment variable KUBE_NETWORK", hnsNetworkName) + klog.Fatalf("Unable to find Hns Network specified by %s. Please check environment variable KUBE_NETWORK", hnsNetworkName) return nil, err } - glog.V(1).Infof("Hns Network loaded with info = %v", hnsNetwork) + klog.V(1).Infof("Hns Network loaded with info = %v", hnsNetwork) proxier := &Proxier{ portsMap: make(map[localPort]closeable), @@ -511,7 +511,7 @@ func NewProxier( } burstSyncs := 2 - glog.V(3).Infof("minSyncPeriod: %v, syncPeriod: %v, burstSyncs: %d", minSyncPeriod, syncPeriod, burstSyncs) + klog.V(3).Infof("minSyncPeriod: %v, syncPeriod: %v, burstSyncs: %d", minSyncPeriod, syncPeriod, burstSyncs) proxier.syncRunner = async.NewBoundedFrequencyRunner("sync-runner", proxier.syncProxyRules, minSyncPeriod, syncPeriod, burstSyncs) return proxier, nil @@ -568,7 +568,7 @@ func deleteAllHnsLoadBalancerPolicy() { LogJson(plist, "Remove Policy", 3) _, err = plist.Delete() if err != nil { - glog.Errorf("%v", err) + klog.Errorf("%v", err) } } @@ -629,36 +629,36 @@ func deleteHnsLoadBalancerPolicy(hnsID string) { // Cleanup HNS policies hnsloadBalancer, err := hcsshim.GetPolicyListByID(hnsID) if err != nil { - glog.Errorf("%v", err) + klog.Errorf("%v", err) return } LogJson(hnsloadBalancer, "Removing Policy", 2) _, err = hnsloadBalancer.Delete() if err != nil { - glog.Errorf("%v", err) + klog.Errorf("%v", err) } } func deleteHnsEndpoint(hnsID string) { hnsendpoint, err := hcsshim.GetHNSEndpointByID(hnsID) if err != nil { - glog.Errorf("%v", err) + klog.Errorf("%v", err) return } _, err = hnsendpoint.Delete() if err != nil { - glog.Errorf("%v", err) + klog.Errorf("%v", err) } - glog.V(3).Infof("Remote endpoint resource deleted id %s", hnsID) + klog.V(3).Infof("Remote endpoint resource deleted id %s", hnsID) } func getHnsNetworkInfo(hnsNetworkName string) (*hnsNetworkInfo, error) { hnsnetwork, err := hcsshim.GetHNSNetworkByName(hnsNetworkName) if err != nil { - glog.Errorf("%v", err) + klog.Errorf("%v", err) return nil, err } @@ -671,7 +671,7 @@ func getHnsNetworkInfo(hnsNetworkName string) (*hnsNetworkInfo, error) { func getHnsEndpointByIpAddress(ip net.IP, networkName string) (*hcsshim.HNSEndpoint, error) { hnsnetwork, err := hcsshim.GetHNSNetworkByName(networkName) if err != nil { - glog.Errorf("%v", err) + klog.Errorf("%v", err) return nil, err } @@ -746,12 +746,12 @@ func (proxier *Proxier) OnServiceSynced() { func shouldSkipService(svcName types.NamespacedName, service *v1.Service) bool { // if ClusterIP is "None" or empty, skip proxying if !helper.IsServiceIPSet(service) { - glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) + klog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) return true } // Even if ClusterIP is set, ServiceTypeExternalName services don't get proxied if service.Spec.Type == v1.ServiceTypeExternalName { - glog.V(3).Infof("Skipping service %s due to Type=ExternalName", svcName) + klog.V(3).Infof("Skipping service %s due to Type=ExternalName", svcName) return true } return false @@ -880,7 +880,7 @@ func endpointsToEndpointsMap(endpoints *v1.Endpoints, hostname string) proxyEndp for i := range ss.Ports { port := &ss.Ports[i] if port.Port == 0 { - glog.Warningf("ignoring invalid endpoint port %s", port.Name) + klog.Warningf("ignoring invalid endpoint port %s", port.Name) continue } svcPortName := proxy.ServicePortName{ @@ -890,19 +890,19 @@ func endpointsToEndpointsMap(endpoints *v1.Endpoints, hostname string) proxyEndp for i := range ss.Addresses { addr := &ss.Addresses[i] if addr.IP == "" { - glog.Warningf("ignoring invalid endpoint port %s with empty host", port.Name) + klog.Warningf("ignoring invalid endpoint port %s with empty host", port.Name) continue } isLocal := addr.NodeName != nil && *addr.NodeName == hostname epInfo := newEndpointInfo(addr.IP, uint16(port.Port), isLocal) endpointsMap[svcPortName] = append(endpointsMap[svcPortName], epInfo) } - if glog.V(3) { + if klog.V(3) { newEPList := []*endpointsInfo{} for _, ep := range endpointsMap[svcPortName] { newEPList = append(newEPList, ep) } - glog.Infof("Setting endpoints for %q to %+v", svcPortName, newEPList) + klog.Infof("Setting endpoints for %q to %+v", svcPortName, newEPList) } } } @@ -939,11 +939,11 @@ func (proxier *Proxier) syncProxyRules() { start := time.Now() defer func() { SyncProxyRulesLatency.Observe(sinceInMicroseconds(start)) - glog.V(4).Infof("syncProxyRules took %v", time.Since(start)) + klog.V(4).Infof("syncProxyRules took %v", time.Since(start)) }() // don't sync rules till we've received services and endpoints if !proxier.endpointsSynced || !proxier.servicesSynced { - glog.V(2).Info("Not syncing hns until Services and Endpoints have been received from master") + klog.V(2).Info("Not syncing hns until Services and Endpoints have been received from master") return } @@ -957,22 +957,22 @@ func (proxier *Proxier) syncProxyRules() { // merge stale services gathered from updateEndpointsMap for svcPortName := range endpointUpdateResult.staleServiceNames { if svcInfo, ok := proxier.serviceMap[svcPortName]; ok && svcInfo != nil && svcInfo.protocol == v1.ProtocolUDP { - glog.V(2).Infof("Stale udp service %v -> %s", svcPortName, svcInfo.clusterIP.String()) + klog.V(2).Infof("Stale udp service %v -> %s", svcPortName, svcInfo.clusterIP.String()) staleServices.Insert(svcInfo.clusterIP.String()) } } - glog.V(3).Infof("Syncing Policies") + klog.V(3).Infof("Syncing Policies") // Program HNS by adding corresponding policies for each service. for svcName, svcInfo := range proxier.serviceMap { if svcInfo.policyApplied { - glog.V(4).Infof("Policy already applied for %s", spew.Sdump(svcInfo)) + klog.V(4).Infof("Policy already applied for %s", spew.Sdump(svcInfo)) continue } var hnsEndpoints []hcsshim.HNSEndpoint - glog.V(4).Infof("====Applying Policy for %s====", svcName) + klog.V(4).Infof("====Applying Policy for %s====", svcName) // Create Remote endpoints for every endpoint, corresponding to the service for _, ep := range proxier.endpointsMap[svcName] { @@ -1000,13 +1000,13 @@ func (proxier *Proxier) syncProxyRules() { if newHnsEndpoint == nil { if ep.isLocal { - glog.Errorf("Local endpoint not found for %v: err: %v on network %s", ep.ip, err, hnsNetworkName) + klog.Errorf("Local endpoint not found for %v: err: %v on network %s", ep.ip, err, hnsNetworkName) continue } // hns Endpoint resource was not found, create one hnsnetwork, err := hcsshim.GetHNSNetworkByName(hnsNetworkName) if err != nil { - glog.Errorf("%v", err) + klog.Errorf("%v", err) continue } @@ -1017,7 +1017,7 @@ func (proxier *Proxier) syncProxyRules() { newHnsEndpoint, err = hnsnetwork.CreateRemoteEndpoint(hnsEndpoint) if err != nil { - glog.Errorf("Remote endpoint creation failed: %v", err) + klog.Errorf("Remote endpoint creation failed: %v", err) continue } } @@ -1030,19 +1030,19 @@ func (proxier *Proxier) syncProxyRules() { Log(ep, "Endpoint resource found", 3) } - glog.V(3).Infof("Associated endpoints [%s] for service [%s]", spew.Sdump(hnsEndpoints), svcName) + klog.V(3).Infof("Associated endpoints [%s] for service [%s]", spew.Sdump(hnsEndpoints), svcName) if len(svcInfo.hnsID) > 0 { // This should not happen - glog.Warningf("Load Balancer already exists %s -- Debug ", svcInfo.hnsID) + klog.Warningf("Load Balancer already exists %s -- Debug ", svcInfo.hnsID) } if len(hnsEndpoints) == 0 { - glog.Errorf("Endpoint information not available for service %s. Not applying any policy", svcName) + klog.Errorf("Endpoint information not available for service %s. Not applying any policy", svcName) continue } - glog.V(4).Infof("Trying to Apply Policies for service %s", spew.Sdump(svcInfo)) + klog.V(4).Infof("Trying to Apply Policies for service %s", spew.Sdump(svcInfo)) var hnsLoadBalancer *hcsshim.PolicyList hnsLoadBalancer, err := getHnsLoadBalancer( @@ -1054,12 +1054,12 @@ func (proxier *Proxier) syncProxyRules() { uint16(svcInfo.port), ) if err != nil { - glog.Errorf("Policy creation failed: %v", err) + klog.Errorf("Policy creation failed: %v", err) continue } svcInfo.hnsID = hnsLoadBalancer.ID - glog.V(3).Infof("Hns LoadBalancer resource created for cluster ip resources %v, Id [%s]", svcInfo.clusterIP, hnsLoadBalancer.ID) + klog.V(3).Infof("Hns LoadBalancer resource created for cluster ip resources %v, Id [%s]", svcInfo.clusterIP, hnsLoadBalancer.ID) // If nodePort is specified, user should be able to use nodeIP:nodePort to reach the backend endpoints if svcInfo.nodePort > 0 { @@ -1072,12 +1072,12 @@ func (proxier *Proxier) syncProxyRules() { uint16(svcInfo.nodePort), ) if err != nil { - glog.Errorf("Policy creation failed: %v", err) + klog.Errorf("Policy creation failed: %v", err) continue } svcInfo.nodePorthnsID = hnsLoadBalancer.ID - glog.V(3).Infof("Hns LoadBalancer resource created for nodePort resources %v, Id [%s]", svcInfo.clusterIP, hnsLoadBalancer.ID) + klog.V(3).Infof("Hns LoadBalancer resource created for nodePort resources %v, Id [%s]", svcInfo.clusterIP, hnsLoadBalancer.ID) } // Create a Load Balancer Policy for each external IP @@ -1092,11 +1092,11 @@ func (proxier *Proxier) syncProxyRules() { uint16(svcInfo.port), ) if err != nil { - glog.Errorf("Policy creation failed: %v", err) + klog.Errorf("Policy creation failed: %v", err) continue } externalIp.hnsID = hnsLoadBalancer.ID - glog.V(3).Infof("Hns LoadBalancer resource created for externalIp resources %v, Id[%s]", externalIp, hnsLoadBalancer.ID) + klog.V(3).Infof("Hns LoadBalancer resource created for externalIp resources %v, Id[%s]", externalIp, hnsLoadBalancer.ID) } // Create a Load Balancer Policy for each loadbalancer ingress for _, lbIngressIp := range svcInfo.loadBalancerIngressIPs { @@ -1110,11 +1110,11 @@ func (proxier *Proxier) syncProxyRules() { uint16(svcInfo.port), ) if err != nil { - glog.Errorf("Policy creation failed: %v", err) + klog.Errorf("Policy creation failed: %v", err) continue } lbIngressIp.hnsID = hnsLoadBalancer.ID - glog.V(3).Infof("Hns LoadBalancer resource created for loadBalancer Ingress resources %v", lbIngressIp) + klog.V(3).Infof("Hns LoadBalancer resource created for loadBalancer Ingress resources %v", lbIngressIp) } svcInfo.policyApplied = true Log(svcInfo, "+++Policy Successfully applied for service +++", 2) @@ -1129,17 +1129,17 @@ func (proxier *Proxier) syncProxyRules() { // not "OnlyLocal", but the services list will not, and the healthChecker // will just drop those endpoints. if err := proxier.healthChecker.SyncServices(serviceUpdateResult.hcServices); err != nil { - glog.Errorf("Error syncing healtcheck services: %v", err) + klog.Errorf("Error syncing healthcheck services: %v", err) } if err := proxier.healthChecker.SyncEndpoints(endpointUpdateResult.hcEndpoints); err != nil { - glog.Errorf("Error syncing healthcheck endpoints: %v", err) + klog.Errorf("Error syncing healthcheck endpoints: %v", err) } // Finish housekeeping. // TODO: these could be made more consistent. for _, svcIP := range staleServices.UnsortedList() { // TODO : Check if this is required to cleanup stale services here - glog.V(5).Infof("Pending delete stale service IP %s connections", svcIP) + klog.V(5).Infof("Pending delete stale service IP %s connections", svcIP) } } diff --git a/pkg/proxy/winuserspace/BUILD b/pkg/proxy/winuserspace/BUILD index 2c2e4d4e5d9..3925dc96d72 100644 --- a/pkg/proxy/winuserspace/BUILD +++ b/pkg/proxy/winuserspace/BUILD @@ -26,8 +26,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/miekg/dns:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/proxy/winuserspace/proxier.go b/pkg/proxy/winuserspace/proxier.go index 9770451e7b8..4b5a218cfc0 100644 --- a/pkg/proxy/winuserspace/proxier.go +++ b/pkg/proxy/winuserspace/proxier.go @@ -25,7 +25,7 @@ import ( "sync/atomic" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -70,7 +70,7 @@ func (info *serviceInfo) isAlive() bool { func logTimeout(err error) bool { if e, ok := err.(net.Error); ok { if e.Timeout() { - glog.V(3).Infof("connection to endpoint closed due to inactivity") + klog.V(3).Infof("connection to endpoint closed due to inactivity") return true } } @@ -140,7 +140,7 @@ func NewProxier(loadBalancer LoadBalancer, listenIP net.IP, netsh netsh.Interfac return nil, fmt.Errorf("failed to select a host interface: %v", err) } - glog.V(2).Infof("Setting proxy IP to %v", hostIP) + klog.V(2).Infof("Setting proxy IP to %v", hostIP) return createProxier(loadBalancer, listenIP, netsh, hostIP, syncPeriod, udpIdleTimeout) } @@ -167,7 +167,7 @@ func (proxier *Proxier) SyncLoop() { defer t.Stop() for { <-t.C - glog.V(6).Infof("Periodic sync") + klog.V(6).Infof("Periodic sync") proxier.Sync() } } @@ -234,7 +234,7 @@ func (proxier *Proxier) addServicePortPortal(servicePortPortalName ServicePortPo if existed, err := proxier.netsh.EnsureIPAddress(args, serviceIP); err != nil { return nil, err } else if !existed { - glog.V(3).Infof("Added ip address to fowarder interface for service %q at %s/%s", servicePortPortalName, net.JoinHostPort(listenIP, strconv.Itoa(port)), protocol) + klog.V(3).Infof("Added ip address to fowarder interface for service %q at %s/%s", servicePortPortalName, net.JoinHostPort(listenIP, strconv.Itoa(port)), protocol) } } @@ -259,7 +259,7 @@ func (proxier *Proxier) addServicePortPortal(servicePortPortalName ServicePortPo } proxier.setServiceInfo(servicePortPortalName, si) - glog.V(2).Infof("Proxying for service %q at %s/%s", servicePortPortalName, net.JoinHostPort(listenIP, strconv.Itoa(port)), protocol) + klog.V(2).Infof("Proxying for service %q at %s/%s", servicePortPortalName, net.JoinHostPort(listenIP, strconv.Itoa(port)), protocol) go func(service ServicePortPortalName, proxier *Proxier) { defer runtime.HandleCrash() atomic.AddInt32(&proxier.numProxyLoops, 1) @@ -313,7 +313,7 @@ func (proxier *Proxier) mergeService(service *v1.Service) map[ServicePortPortalN } svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} if !helper.IsServiceIPSet(service) { - glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) + klog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) return nil } existingPortPortals := make(map[ServicePortPortalName]bool) @@ -337,19 +337,19 @@ func (proxier *Proxier) mergeService(service *v1.Service) map[ServicePortPortalN continue } if exists { - glog.V(4).Infof("Something changed for service %q: stopping it", servicePortPortalName) + klog.V(4).Infof("Something changed for service %q: stopping it", servicePortPortalName) if err := proxier.closeServicePortPortal(servicePortPortalName, info); err != nil { - glog.Errorf("Failed to close service port portal %q: %v", servicePortPortalName, err) + klog.Errorf("Failed to close service port portal %q: %v", servicePortPortalName, err) } } - glog.V(1).Infof("Adding new service %q at %s/%s", servicePortPortalName, net.JoinHostPort(listenIP, strconv.Itoa(listenPort)), protocol) + klog.V(1).Infof("Adding new service %q at %s/%s", servicePortPortalName, net.JoinHostPort(listenIP, strconv.Itoa(listenPort)), protocol) info, err := proxier.addServicePortPortal(servicePortPortalName, protocol, listenIP, listenPort, proxier.udpIdleTimeout) if err != nil { - glog.Errorf("Failed to start proxy for %q: %v", servicePortPortalName, err) + klog.Errorf("Failed to start proxy for %q: %v", servicePortPortalName, err) continue } info.sessionAffinityType = service.Spec.SessionAffinity - glog.V(10).Infof("info: %#v", info) + klog.V(10).Infof("info: %#v", info) } if len(listenIPPortMap) > 0 { // only one loadbalancer per service port portal @@ -377,7 +377,7 @@ func (proxier *Proxier) unmergeService(service *v1.Service, existingPortPortals } svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} if !helper.IsServiceIPSet(service) { - glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) + klog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) return } @@ -409,15 +409,15 @@ func (proxier *Proxier) unmergeService(service *v1.Service, existingPortPortals continue } - glog.V(1).Infof("Stopping service %q", servicePortPortalName) + klog.V(1).Infof("Stopping service %q", servicePortPortalName) info, exists := proxier.getServiceInfo(servicePortPortalName) if !exists { - glog.Errorf("Service %q is being removed but doesn't exist", servicePortPortalName) + klog.Errorf("Service %q is being removed but doesn't exist", servicePortPortalName) continue } if err := proxier.closeServicePortPortal(servicePortPortalName, info); err != nil { - glog.Errorf("Failed to close service port portal %q: %v", servicePortPortalName, err) + klog.Errorf("Failed to close service port portal %q: %v", servicePortPortalName, err) } } diff --git a/pkg/proxy/winuserspace/proxysocket.go b/pkg/proxy/winuserspace/proxysocket.go index 23782d6209b..c7c5691e3b3 100644 --- a/pkg/proxy/winuserspace/proxysocket.go +++ b/pkg/proxy/winuserspace/proxysocket.go @@ -26,11 +26,11 @@ import ( "sync/atomic" "time" - "github.com/golang/glog" "github.com/miekg/dns" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog" "k8s.io/kubernetes/pkg/proxy" "k8s.io/kubernetes/pkg/util/ipconfig" "k8s.io/utils/exec" @@ -133,10 +133,10 @@ func tryConnect(service ServicePortPortalName, srcAddr net.Addr, protocol string } endpoint, err := proxier.loadBalancer.NextEndpoint(servicePortName, srcAddr, sessionAffinityReset) if err != nil { - glog.Errorf("Couldn't find an endpoint for %s: %v", service, err) + klog.Errorf("Couldn't find an endpoint for %s: %v", service, err) return nil, err } - glog.V(3).Infof("Mapped service %q to endpoint %s", service, endpoint) + klog.V(3).Infof("Mapped service %q to endpoint %s", service, endpoint) // TODO: This could spin up a new goroutine to make the outbound connection, // and keep accepting inbound traffic. outConn, err := net.DialTimeout(protocol, endpoint, dialTimeout) @@ -144,7 +144,7 @@ func tryConnect(service ServicePortPortalName, srcAddr net.Addr, protocol string if isTooManyFDsError(err) { panic("Dial failed: " + err.Error()) } - glog.Errorf("Dial failed: %v", err) + klog.Errorf("Dial failed: %v", err) sessionAffinityReset = true continue } @@ -173,13 +173,13 @@ func (tcp *tcpProxySocket) ProxyLoop(service ServicePortPortalName, myInfo *serv // Then the service port was just closed so the accept failure is to be expected. return } - glog.Errorf("Accept failed: %v", err) + klog.Errorf("Accept failed: %v", err) continue } - glog.V(3).Infof("Accepted TCP connection from %v to %v", inConn.RemoteAddr(), inConn.LocalAddr()) + klog.V(3).Infof("Accepted TCP connection from %v to %v", inConn.RemoteAddr(), inConn.LocalAddr()) outConn, err := tryConnect(service, inConn.(*net.TCPConn).RemoteAddr(), "tcp", proxier) if err != nil { - glog.Errorf("Failed to connect to balancer: %v", err) + klog.Errorf("Failed to connect to balancer: %v", err) inConn.Close() continue } @@ -192,7 +192,7 @@ func (tcp *tcpProxySocket) ProxyLoop(service ServicePortPortalName, myInfo *serv func proxyTCP(in, out *net.TCPConn) { var wg sync.WaitGroup wg.Add(2) - glog.V(4).Infof("Creating proxy between %v <-> %v <-> %v <-> %v", + klog.V(4).Infof("Creating proxy between %v <-> %v <-> %v <-> %v", in.RemoteAddr(), in.LocalAddr(), out.LocalAddr(), out.RemoteAddr()) go copyBytes("from backend", in, out, &wg) go copyBytes("to backend", out, in, &wg) @@ -201,14 +201,14 @@ func proxyTCP(in, out *net.TCPConn) { func copyBytes(direction string, dest, src *net.TCPConn, wg *sync.WaitGroup) { defer wg.Done() - glog.V(4).Infof("Copying %s: %s -> %s", direction, src.RemoteAddr(), dest.RemoteAddr()) + klog.V(4).Infof("Copying %s: %s -> %s", direction, src.RemoteAddr(), dest.RemoteAddr()) n, err := io.Copy(dest, src) if err != nil { if !isClosedError(err) { - glog.Errorf("I/O error: %v", err) + klog.Errorf("I/O error: %v", err) } } - glog.V(4).Infof("Copied %d bytes %s: %s -> %s", n, direction, src.RemoteAddr(), dest.RemoteAddr()) + klog.V(4).Infof("Copied %d bytes %s: %s -> %s", n, direction, src.RemoteAddr(), dest.RemoteAddr()) dest.Close() src.Close() } @@ -283,7 +283,7 @@ func appendDNSSuffix(msg *dns.Msg, buffer []byte, length int, dnsSuffix string) msg.Question[0].Name = origName if err != nil { - glog.Warningf("Unable to pack DNS packet. Error is: %v", err) + klog.Warningf("Unable to pack DNS packet. Error is: %v", err) return length, err } @@ -310,7 +310,7 @@ func recoverDNSQuestion(origName string, msg *dns.Msg, buffer []byte, length int mbuf, err := msg.PackBuffer(buffer) if err != nil { - glog.Warningf("Unable to pack DNS packet. Error is: %v", err) + klog.Warningf("Unable to pack DNS packet. Error is: %v", err) return length, err } @@ -330,7 +330,7 @@ func processUnpackedDNSQueryPacket( length int, dnsSearch []string) int { if dnsSearch == nil || len(dnsSearch) == 0 { - glog.V(1).Infof("DNS search list is not initialized and is empty.") + klog.V(1).Infof("DNS search list is not initialized and is empty.") return length } @@ -348,13 +348,13 @@ func processUnpackedDNSQueryPacket( state.msg.MsgHdr.Id = msg.MsgHdr.Id if index < 0 || index >= int32(len(dnsSearch)) { - glog.V(1).Infof("Search index %d is out of range.", index) + klog.V(1).Infof("Search index %d is out of range.", index) return length } length, err := appendDNSSuffix(msg, buffer, length, dnsSearch[index]) if err != nil { - glog.Errorf("Append DNS suffix failed: %v", err) + klog.Errorf("Append DNS suffix failed: %v", err) } return length @@ -373,7 +373,7 @@ func processUnpackedDNSResponsePacket( var drop bool var err error if dnsSearch == nil || len(dnsSearch) == 0 { - glog.V(1).Infof("DNS search list is not initialized and is empty.") + klog.V(1).Infof("DNS search list is not initialized and is empty.") return drop, length } @@ -389,19 +389,19 @@ func processUnpackedDNSResponsePacket( drop = true length, err = appendDNSSuffix(state.msg, buffer, length, dnsSearch[index]) if err != nil { - glog.Errorf("Append DNS suffix failed: %v", err) + klog.Errorf("Append DNS suffix failed: %v", err) } _, err = svrConn.Write(buffer[0:length]) if err != nil { if !logTimeout(err) { - glog.Errorf("Write failed: %v", err) + klog.Errorf("Write failed: %v", err) } } } else { length, err = recoverDNSQuestion(state.msg.Question[0].Name, msg, buffer, length) if err != nil { - glog.Errorf("Recover DNS question failed: %v", err) + klog.Errorf("Recover DNS question failed: %v", err) } dnsClients.mu.Lock() @@ -421,7 +421,7 @@ func processDNSQueryPacket( dnsSearch []string) (int, error) { msg := &dns.Msg{} if err := msg.Unpack(buffer[:length]); err != nil { - glog.Warningf("Unable to unpack DNS packet. Error is: %v", err) + klog.Warningf("Unable to unpack DNS packet. Error is: %v", err) return length, err } @@ -432,14 +432,14 @@ func processDNSQueryPacket( // QDCOUNT if len(msg.Question) != 1 { - glog.V(1).Infof("Number of entries in the question section of the DNS packet is: %d", len(msg.Question)) - glog.V(1).Infof("DNS suffix appending does not support more than one question.") + klog.V(1).Infof("Number of entries in the question section of the DNS packet is: %d", len(msg.Question)) + klog.V(1).Infof("DNS suffix appending does not support more than one question.") return length, nil } // ANCOUNT, NSCOUNT, ARCOUNT if len(msg.Answer) != 0 || len(msg.Ns) != 0 || len(msg.Extra) != 0 { - glog.V(1).Infof("DNS packet contains more than question section.") + klog.V(1).Infof("DNS packet contains more than question section.") return length, nil } @@ -448,7 +448,7 @@ func processDNSQueryPacket( if packetRequiresDNSSuffix(dnsQType, dnsQClass) { host, _, err := net.SplitHostPort(cliAddr.String()) if err != nil { - glog.V(1).Infof("Failed to get host from client address: %v", err) + klog.V(1).Infof("Failed to get host from client address: %v", err) host = cliAddr.String() } @@ -468,7 +468,7 @@ func processDNSResponsePacket( var drop bool msg := &dns.Msg{} if err := msg.Unpack(buffer[:length]); err != nil { - glog.Warningf("Unable to unpack DNS packet. Error is: %v", err) + klog.Warningf("Unable to unpack DNS packet. Error is: %v", err) return drop, length, err } @@ -479,7 +479,7 @@ func processDNSResponsePacket( // QDCOUNT if len(msg.Question) != 1 { - glog.V(1).Infof("Number of entries in the response section of the DNS packet is: %d", len(msg.Answer)) + klog.V(1).Infof("Number of entries in the response section of the DNS packet is: %d", len(msg.Answer)) return drop, length, nil } @@ -488,7 +488,7 @@ func processDNSResponsePacket( if packetRequiresDNSSuffix(dnsQType, dnsQClass) { host, _, err := net.SplitHostPort(cliAddr.String()) if err != nil { - glog.V(1).Infof("Failed to get host from client address: %v", err) + klog.V(1).Infof("Failed to get host from client address: %v", err) host = cliAddr.String() } @@ -525,11 +525,11 @@ func (udp *udpProxySocket) ProxyLoop(service ServicePortPortalName, myInfo *serv if err != nil { if e, ok := err.(net.Error); ok { if e.Temporary() { - glog.V(1).Infof("ReadFrom had a temporary failure: %v", err) + klog.V(1).Infof("ReadFrom had a temporary failure: %v", err) continue } } - glog.Errorf("ReadFrom failed, exiting ProxyLoop: %v", err) + klog.Errorf("ReadFrom failed, exiting ProxyLoop: %v", err) break } @@ -537,7 +537,7 @@ func (udp *udpProxySocket) ProxyLoop(service ServicePortPortalName, myInfo *serv if isDNSService(service.Port) { n, err = processDNSQueryPacket(myInfo.dnsClients, cliAddr, buffer[:], n, dnsSearch) if err != nil { - glog.Errorf("Process DNS query packet failed: %v", err) + klog.Errorf("Process DNS query packet failed: %v", err) } } @@ -551,14 +551,14 @@ func (udp *udpProxySocket) ProxyLoop(service ServicePortPortalName, myInfo *serv _, err = svrConn.Write(buffer[0:n]) if err != nil { if !logTimeout(err) { - glog.Errorf("Write failed: %v", err) + klog.Errorf("Write failed: %v", err) // TODO: Maybe tear down the goroutine for this client/server pair? } continue } err = svrConn.SetDeadline(time.Now().Add(myInfo.timeout)) if err != nil { - glog.Errorf("SetDeadline failed: %v", err) + klog.Errorf("SetDeadline failed: %v", err) continue } } @@ -572,14 +572,14 @@ func (udp *udpProxySocket) getBackendConn(activeClients *clientCache, dnsClients if !found { // TODO: This could spin up a new goroutine to make the outbound connection, // and keep accepting inbound traffic. - glog.V(3).Infof("New UDP connection from %s", cliAddr) + klog.V(3).Infof("New UDP connection from %s", cliAddr) var err error svrConn, err = tryConnect(service, cliAddr, "udp", proxier) if err != nil { return nil, err } if err = svrConn.SetDeadline(time.Now().Add(timeout)); err != nil { - glog.Errorf("SetDeadline failed: %v", err) + klog.Errorf("SetDeadline failed: %v", err) return nil, err } activeClients.clients[cliAddr.String()] = svrConn @@ -600,7 +600,7 @@ func (udp *udpProxySocket) proxyClient(cliAddr net.Addr, svrConn net.Conn, activ n, err := svrConn.Read(buffer[0:]) if err != nil { if !logTimeout(err) { - glog.Errorf("Read failed: %v", err) + klog.Errorf("Read failed: %v", err) } break } @@ -609,20 +609,20 @@ func (udp *udpProxySocket) proxyClient(cliAddr net.Addr, svrConn net.Conn, activ if isDNSService(service.Port) { drop, n, err = processDNSResponsePacket(svrConn, dnsClients, cliAddr, buffer[:], n, dnsSearch) if err != nil { - glog.Errorf("Process DNS response packet failed: %v", err) + klog.Errorf("Process DNS response packet failed: %v", err) } } if !drop { err = svrConn.SetDeadline(time.Now().Add(timeout)) if err != nil { - glog.Errorf("SetDeadline failed: %v", err) + klog.Errorf("SetDeadline failed: %v", err) break } n, err = udp.WriteTo(buffer[0:n], cliAddr) if err != nil { if !logTimeout(err) { - glog.Errorf("WriteTo failed: %v", err) + klog.Errorf("WriteTo failed: %v", err) } break } diff --git a/pkg/proxy/winuserspace/roundrobin.go b/pkg/proxy/winuserspace/roundrobin.go index d3135a038af..a712ed60bd1 100644 --- a/pkg/proxy/winuserspace/roundrobin.go +++ b/pkg/proxy/winuserspace/roundrobin.go @@ -25,9 +25,9 @@ import ( "sync" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/proxy" "k8s.io/kubernetes/pkg/util/slice" ) @@ -82,7 +82,7 @@ func NewLoadBalancerRR() *LoadBalancerRR { } func (lb *LoadBalancerRR) NewService(svcPort proxy.ServicePortName, affinityType v1.ServiceAffinity, ttlSeconds int) error { - glog.V(4).Infof("LoadBalancerRR NewService %q", svcPort) + klog.V(4).Infof("LoadBalancerRR NewService %q", svcPort) lb.lock.Lock() defer lb.lock.Unlock() lb.newServiceInternal(svcPort, affinityType, ttlSeconds) @@ -97,7 +97,7 @@ func (lb *LoadBalancerRR) newServiceInternal(svcPort proxy.ServicePortName, affi if _, exists := lb.services[svcPort]; !exists { lb.services[svcPort] = &balancerState{affinity: *newAffinityPolicy(affinityType, ttlSeconds)} - glog.V(4).Infof("LoadBalancerRR service %q did not exist, created", svcPort) + klog.V(4).Infof("LoadBalancerRR service %q did not exist, created", svcPort) } else if affinityType != "" { lb.services[svcPort].affinity.affinityType = affinityType } @@ -105,7 +105,7 @@ func (lb *LoadBalancerRR) newServiceInternal(svcPort proxy.ServicePortName, affi } func (lb *LoadBalancerRR) DeleteService(svcPort proxy.ServicePortName) { - glog.V(4).Infof("LoadBalancerRR DeleteService %q", svcPort) + klog.V(4).Infof("LoadBalancerRR DeleteService %q", svcPort) lb.lock.Lock() defer lb.lock.Unlock() delete(lb.services, svcPort) @@ -135,7 +135,7 @@ func (lb *LoadBalancerRR) NextEndpoint(svcPort proxy.ServicePortName, srcAddr ne if len(state.endpoints) == 0 { return "", ErrMissingEndpoints } - glog.V(4).Infof("NextEndpoint for service %q, srcAddr=%v: endpoints: %+v", svcPort, srcAddr, state.endpoints) + klog.V(4).Infof("NextEndpoint for service %q, srcAddr=%v: endpoints: %+v", svcPort, srcAddr, state.endpoints) sessionAffinityEnabled := isSessionAffinity(&state.affinity) @@ -153,7 +153,7 @@ func (lb *LoadBalancerRR) NextEndpoint(svcPort proxy.ServicePortName, srcAddr ne // Affinity wins. endpoint := sessionAffinity.endpoint sessionAffinity.lastUsed = time.Now() - glog.V(4).Infof("NextEndpoint for service %q from IP %s with sessionAffinity %#v: %s", svcPort, ipaddr, sessionAffinity, endpoint) + klog.V(4).Infof("NextEndpoint for service %q from IP %s with sessionAffinity %#v: %s", svcPort, ipaddr, sessionAffinity, endpoint) return endpoint, nil } } @@ -172,7 +172,7 @@ func (lb *LoadBalancerRR) NextEndpoint(svcPort proxy.ServicePortName, srcAddr ne affinity.lastUsed = time.Now() affinity.endpoint = endpoint affinity.clientIP = ipaddr - glog.V(4).Infof("Updated affinity key %s: %#v", ipaddr, state.affinity.affinityMap[ipaddr]) + klog.V(4).Infof("Updated affinity key %s: %#v", ipaddr, state.affinity.affinityMap[ipaddr]) } return endpoint, nil @@ -204,7 +204,7 @@ func flattenValidEndpoints(endpoints []hostPortPair) []string { func removeSessionAffinityByEndpoint(state *balancerState, svcPort proxy.ServicePortName, endpoint string) { for _, affinity := range state.affinity.affinityMap { if affinity.endpoint == endpoint { - glog.V(4).Infof("Removing client: %s from affinityMap for service %q", affinity.endpoint, svcPort) + klog.V(4).Infof("Removing client: %s from affinityMap for service %q", affinity.endpoint, svcPort) delete(state.affinity.affinityMap, affinity.clientIP) } } @@ -227,7 +227,7 @@ func (lb *LoadBalancerRR) updateAffinityMap(svcPort proxy.ServicePortName, newEn } for mKey, mVal := range allEndpoints { if mVal == 1 { - glog.V(2).Infof("Delete endpoint %s for service %q", mKey, svcPort) + klog.V(2).Infof("Delete endpoint %s for service %q", mKey, svcPort) removeSessionAffinityByEndpoint(state, svcPort, mKey) } } @@ -263,7 +263,7 @@ func (lb *LoadBalancerRR) OnEndpointsAdd(endpoints *v1.Endpoints) { state, exists := lb.services[svcPort] if !exists || state == nil || len(newEndpoints) > 0 { - glog.V(1).Infof("LoadBalancerRR: Setting endpoints for %s to %+v", svcPort, newEndpoints) + klog.V(1).Infof("LoadBalancerRR: Setting endpoints for %s to %+v", svcPort, newEndpoints) lb.updateAffinityMap(svcPort, newEndpoints) // OnEndpointsAdd can be called without NewService being called externally. // To be safe we will call it here. A new service will only be created @@ -297,7 +297,7 @@ func (lb *LoadBalancerRR) OnEndpointsUpdate(oldEndpoints, endpoints *v1.Endpoint } if !exists || state == nil || len(curEndpoints) != len(newEndpoints) || !slicesEquiv(slice.CopyStrings(curEndpoints), newEndpoints) { - glog.V(1).Infof("LoadBalancerRR: Setting endpoints for %s to %+v", svcPort, newEndpoints) + klog.V(1).Infof("LoadBalancerRR: Setting endpoints for %s to %+v", svcPort, newEndpoints) lb.updateAffinityMap(svcPort, newEndpoints) // OnEndpointsUpdate can be called without NewService being called externally. // To be safe we will call it here. A new service will only be created @@ -315,7 +315,7 @@ func (lb *LoadBalancerRR) OnEndpointsUpdate(oldEndpoints, endpoints *v1.Endpoint for portname := range oldPortsToEndpoints { svcPort := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: endpoints.Namespace, Name: endpoints.Name}, Port: portname} if _, exists := registeredEndpoints[svcPort]; !exists { - glog.V(2).Infof("LoadBalancerRR: Removing endpoints for %s", svcPort) + klog.V(2).Infof("LoadBalancerRR: Removing endpoints for %s", svcPort) // Reset but don't delete. state := lb.services[svcPort] state.endpoints = []string{} @@ -333,7 +333,7 @@ func (lb *LoadBalancerRR) OnEndpointsDelete(endpoints *v1.Endpoints) { for portname := range portsToEndpoints { svcPort := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: endpoints.Namespace, Name: endpoints.Name}, Port: portname} - glog.V(2).Infof("LoadBalancerRR: Removing endpoints for %s", svcPort) + klog.V(2).Infof("LoadBalancerRR: Removing endpoints for %s", svcPort) // If the service is still around, reset but don't delete. if state, ok := lb.services[svcPort]; ok { state.endpoints = []string{} @@ -367,7 +367,7 @@ func (lb *LoadBalancerRR) CleanupStaleStickySessions(svcPort proxy.ServicePortNa } for ip, affinity := range state.affinity.affinityMap { if int(time.Since(affinity.lastUsed).Seconds()) >= state.affinity.ttlSeconds { - glog.V(4).Infof("Removing client %s from affinityMap for service %q", affinity.clientIP, svcPort) + klog.V(4).Infof("Removing client %s from affinityMap for service %q", affinity.clientIP, svcPort) delete(state.affinity.affinityMap, ip) } } diff --git a/pkg/registry/authentication/OWNERS b/pkg/registry/authentication/OWNERS index 81919881ff3..c607d2aa8c5 100755 --- a/pkg/registry/authentication/OWNERS +++ b/pkg/registry/authentication/OWNERS @@ -1,2 +1,7 @@ +approvers: +- sig-auth-authenticators-approvers reviewers: -- deads2k +- sig-auth-authenticators-reviewers +labels: +- sig/auth + diff --git a/pkg/registry/authentication/rest/storage_authentication.go b/pkg/registry/authentication/rest/storage_authentication.go index 5dec4aeb077..9dc3cf1a8e4 100644 --- a/pkg/registry/authentication/rest/storage_authentication.go +++ b/pkg/registry/authentication/rest/storage_authentication.go @@ -31,6 +31,7 @@ import ( type RESTStorageProvider struct { Authenticator authenticator.Request + APIAudiences authenticator.Audiences } func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { @@ -56,7 +57,7 @@ func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorag func (p RESTStorageProvider) v1beta1Storage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) map[string]rest.Storage { storage := map[string]rest.Storage{} // tokenreviews - tokenReviewStorage := tokenreview.NewREST(p.Authenticator) + tokenReviewStorage := tokenreview.NewREST(p.Authenticator, p.APIAudiences) storage["tokenreviews"] = tokenReviewStorage return storage @@ -65,7 +66,7 @@ func (p RESTStorageProvider) v1beta1Storage(apiResourceConfigSource serverstorag func (p RESTStorageProvider) v1Storage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) map[string]rest.Storage { storage := map[string]rest.Storage{} // tokenreviews - tokenReviewStorage := tokenreview.NewREST(p.Authenticator) + tokenReviewStorage := tokenreview.NewREST(p.Authenticator, p.APIAudiences) storage["tokenreviews"] = tokenReviewStorage return storage diff --git a/pkg/registry/authentication/tokenreview/BUILD b/pkg/registry/authentication/tokenreview/BUILD index 7ef5901f8f3..4533e49e8b7 100644 --- a/pkg/registry/authentication/tokenreview/BUILD +++ b/pkg/registry/authentication/tokenreview/BUILD @@ -17,6 +17,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/registry/authentication/tokenreview/storage.go b/pkg/registry/authentication/tokenreview/storage.go index 3150b2afd1b..d14ee1a90f7 100644 --- a/pkg/registry/authentication/tokenreview/storage.go +++ b/pkg/registry/authentication/tokenreview/storage.go @@ -18,6 +18,7 @@ package tokenreview import ( "context" + "errors" "fmt" "net/http" @@ -27,15 +28,22 @@ import ( "k8s.io/apiserver/pkg/authentication/authenticator" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" + "k8s.io/klog" "k8s.io/kubernetes/pkg/apis/authentication" ) +var badAuthenticatorAuds = apierrors.NewInternalError(errors.New("error validating audiences")) + type REST struct { tokenAuthenticator authenticator.Request + apiAudiences []string } -func NewREST(tokenAuthenticator authenticator.Request) *REST { - return &REST{tokenAuthenticator: tokenAuthenticator} +func NewREST(tokenAuthenticator authenticator.Request, apiAudiences []string) *REST { + return &REST{ + tokenAuthenticator: tokenAuthenticator, + apiAudiences: apiAudiences, + } } func (r *REST) NamespaceScoped() bool { @@ -68,14 +76,24 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation fakeReq := &http.Request{Header: http.Header{}} fakeReq.Header.Add("Authorization", "Bearer "+tokenReview.Spec.Token) + auds := tokenReview.Spec.Audiences + if len(auds) == 0 { + auds = r.apiAudiences + } + if len(auds) > 0 { + fakeReq = fakeReq.WithContext(authenticator.WithAudiences(fakeReq.Context(), auds)) + } + resp, ok, err := r.tokenAuthenticator.AuthenticateRequest(fakeReq) tokenReview.Status.Authenticated = ok if err != nil { tokenReview.Status.Error = err.Error() } - // TODO(mikedanese): verify the response audience matches one of apiAuds if - // non-empty + if len(auds) > 0 && resp != nil && len(authenticator.Audiences(auds).Intersect(resp.Audiences)) == 0 { + klog.Errorf("error validating audience. want=%q got=%q", auds, resp.Audiences) + return nil, badAuthenticatorAuds + } if resp != nil && resp.User != nil { tokenReview.Status.User = authentication.UserInfo{ @@ -87,6 +105,7 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation for k, v := range resp.User.GetExtra() { tokenReview.Status.User.Extra[k] = authentication.ExtraValue(v) } + tokenReview.Status.Audiences = resp.Audiences } return tokenReview, nil diff --git a/pkg/registry/authorization/OWNERS b/pkg/registry/authorization/OWNERS index d21ad8662fd..cd0d70a0f8f 100755 --- a/pkg/registry/authorization/OWNERS +++ b/pkg/registry/authorization/OWNERS @@ -1,4 +1,7 @@ +approvers: +- sig-auth-authorizers-approvers reviewers: -- deads2k -- liggitt -- enj +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/pkg/registry/certificates/OWNERS b/pkg/registry/certificates/OWNERS index ce89a734755..470b7a1c92d 100755 --- a/pkg/registry/certificates/OWNERS +++ b/pkg/registry/certificates/OWNERS @@ -1,11 +1,7 @@ +approvers: +- sig-auth-certificates-approvers reviewers: -- smarterclayton -- wojtek-t -- deads2k -- mikedanese -- liggitt -- timothysc -- dims -- hongchaodeng -- david-mcmahon -- enj +- sig-auth-certificates-reviewers +labels: +- sig/auth + diff --git a/pkg/registry/core/persistentvolume/strategy.go b/pkg/registry/core/persistentvolume/strategy.go index eb172251a8d..312fa8cfb4b 100644 --- a/pkg/registry/core/persistentvolume/strategy.go +++ b/pkg/registry/core/persistentvolume/strategy.go @@ -53,7 +53,7 @@ func (persistentvolumeStrategy) PrepareForCreate(ctx context.Context, obj runtim pv := obj.(*api.PersistentVolume) pv.Status = api.PersistentVolumeStatus{} - pvutil.DropDisabledAlphaFields(&pv.Spec) + pvutil.DropDisabledFields(&pv.Spec, nil) } func (persistentvolumeStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { @@ -76,8 +76,7 @@ func (persistentvolumeStrategy) PrepareForUpdate(ctx context.Context, obj, old r oldPv := old.(*api.PersistentVolume) newPv.Status = oldPv.Status - pvutil.DropDisabledAlphaFields(&newPv.Spec) - pvutil.DropDisabledAlphaFields(&oldPv.Spec) + pvutil.DropDisabledFields(&newPv.Spec, &oldPv.Spec) } func (persistentvolumeStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { diff --git a/pkg/registry/core/pod/storage/storage_test.go b/pkg/registry/core/pod/storage/storage_test.go index c806fe6441c..5a084bc1311 100644 --- a/pkg/registry/core/pod/storage/storage_test.go +++ b/pkg/registry/core/pod/storage/storage_test.go @@ -421,8 +421,11 @@ func TestConvertToTableList(t *testing.T) { {Name: "IP", Type: "string", Priority: 1, Description: v1.PodStatus{}.SwaggerDoc()["podIP"]}, {Name: "Node", Type: "string", Priority: 1, Description: v1.PodSpec{}.SwaggerDoc()["nodeName"]}, {Name: "Nominated Node", Type: "string", Priority: 1, Description: v1.PodStatus{}.SwaggerDoc()["nominatedNodeName"]}, + {Name: "Readiness Gates", Type: "string", Priority: 1, Description: v1.PodSpec{}.SwaggerDoc()["readinessGates"]}, } + condition1 := "condition1" + condition2 := "condition2" pod1 := &api.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "foo", CreationTimestamp: metav1.NewTime(time.Now().Add(-370 * 24 * time.Hour))}, Spec: api.PodSpec{ @@ -431,8 +434,26 @@ func TestConvertToTableList(t *testing.T) { {Name: "ctr2", Ports: []api.ContainerPort{{ContainerPort: 9376}}}, }, NodeName: "test-node", + ReadinessGates: []api.PodReadinessGate{ + { + ConditionType: api.PodConditionType(condition1), + }, + { + ConditionType: api.PodConditionType(condition2), + }, + }, }, Status: api.PodStatus{ + Conditions: []api.PodCondition{ + { + Type: api.PodConditionType(condition1), + Status: api.ConditionFalse, + }, + { + Type: api.PodConditionType(condition2), + Status: api.ConditionTrue, + }, + }, PodIP: "10.1.2.3", Phase: api.PodPending, ContainerStatuses: []api.ContainerStatus{ @@ -457,7 +478,7 @@ func TestConvertToTableList(t *testing.T) { out: &metav1beta1.Table{ ColumnDefinitions: columns, Rows: []metav1beta1.TableRow{ - {Cells: []interface{}{"", "0/0", "", int64(0), "", "", "", ""}, Object: runtime.RawExtension{Object: &api.Pod{}}}, + {Cells: []interface{}{"", "0/0", "", int64(0), "", "", "", "", ""}, Object: runtime.RawExtension{Object: &api.Pod{}}}, }, }, }, @@ -466,7 +487,7 @@ func TestConvertToTableList(t *testing.T) { out: &metav1beta1.Table{ ColumnDefinitions: columns, Rows: []metav1beta1.TableRow{ - {Cells: []interface{}{"foo", "1/2", "Pending", int64(10), "370d", "10.1.2.3", "test-node", "nominated-node"}, Object: runtime.RawExtension{Object: pod1}}, + {Cells: []interface{}{"foo", "1/2", "Pending", int64(10), "370d", "10.1.2.3", "test-node", "nominated-node", "1/2"}, Object: runtime.RawExtension{Object: pod1}}, }, }, }, diff --git a/pkg/registry/core/rest/BUILD b/pkg/registry/core/rest/BUILD index 1931f2e9159..f03a3434662 100644 --- a/pkg/registry/core/rest/BUILD +++ b/pkg/registry/core/rest/BUILD @@ -60,7 +60,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/storage/etcd/util:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go index b9994db1618..f188d9e49a4 100644 --- a/pkg/registry/core/rest/storage_core.go +++ b/pkg/registry/core/rest/storage_core.go @@ -25,7 +25,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/runtime/schema" utilnet "k8s.io/apimachinery/pkg/util/net" @@ -254,13 +254,13 @@ type componentStatusStorage struct { func (s componentStatusStorage) serversToValidate() map[string]*componentstatus.Server { serversToValidate := map[string]*componentstatus.Server{ "controller-manager": {Addr: "127.0.0.1", Port: ports.InsecureKubeControllerManagerPort, Path: "/healthz"}, - "scheduler": {Addr: "127.0.0.1", Port: ports.SchedulerPort, Path: "/healthz"}, + "scheduler": {Addr: "127.0.0.1", Port: ports.InsecureSchedulerPort, Path: "/healthz"}, } for ix, machine := range s.storageFactory.Backends() { etcdUrl, err := url.Parse(machine.Server) if err != nil { - glog.Errorf("Failed to parse etcd url for validation: %v", err) + klog.Errorf("Failed to parse etcd url for validation: %v", err) continue } var port int @@ -269,7 +269,7 @@ func (s componentStatusStorage) serversToValidate() map[string]*componentstatus. var portString string addr, portString, err = net.SplitHostPort(etcdUrl.Host) if err != nil { - glog.Errorf("Failed to split host/port: %s (%v)", etcdUrl.Host, err) + klog.Errorf("Failed to split host/port: %s (%v)", etcdUrl.Host, err) continue } port, _ = strconv.Atoi(portString) diff --git a/pkg/registry/core/service/ipallocator/controller/BUILD b/pkg/registry/core/service/ipallocator/controller/BUILD index c10229edbb7..8921bbc23ad 100644 --- a/pkg/registry/core/service/ipallocator/controller/BUILD +++ b/pkg/registry/core/service/ipallocator/controller/BUILD @@ -13,8 +13,7 @@ go_library( deps = [ "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/core:go_default_library", - "//pkg/apis/core/helper:go_default_library", - "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library", + "//pkg/apis/core/v1/helper:go_default_library", "//pkg/registry/core/rangeallocation:go_default_library", "//pkg/registry/core/service/ipallocator:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", @@ -22,6 +21,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", ], @@ -33,9 +33,10 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", - "//pkg/client/clientset_generated/internalclientset/fake:go_default_library", "//pkg/registry/core/service/ipallocator:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], ) diff --git a/pkg/registry/core/service/ipallocator/controller/repair.go b/pkg/registry/core/service/ipallocator/controller/repair.go index b4aaf1c289c..11bb117d743 100644 --- a/pkg/registry/core/service/ipallocator/controller/repair.go +++ b/pkg/registry/core/service/ipallocator/controller/repair.go @@ -26,12 +26,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/apis/core/helper" - coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" + "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/registry/core/rangeallocation" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" ) @@ -53,7 +53,7 @@ import ( // TODO: perform repair? type Repair struct { interval time.Duration - serviceClient coreclient.ServicesGetter + serviceClient corev1client.ServicesGetter network *net.IPNet alloc rangeallocation.RangeRegistry leaks map[string]int // counter per leaked IP @@ -66,9 +66,9 @@ const numRepairsBeforeLeakCleanup = 3 // NewRepair creates a controller that periodically ensures that all clusterIPs are uniquely allocated across the cluster // and generates informational warnings for a cluster that is not in sync. -func NewRepair(interval time.Duration, serviceClient coreclient.ServicesGetter, eventClient coreclient.EventsGetter, network *net.IPNet, alloc rangeallocation.RangeRegistry) *Repair { +func NewRepair(interval time.Duration, serviceClient corev1client.ServicesGetter, eventClient corev1client.EventsGetter, network *net.IPNet, alloc rangeallocation.RangeRegistry) *Repair { eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartRecordingToSink(&coreclient.EventSinkImpl{Interface: eventClient.Events("")}) + eventBroadcaster.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: eventClient.Events("")}) recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "ipallocator-repair-controller"}) return &Repair{ diff --git a/pkg/registry/core/service/ipallocator/controller/repair_test.go b/pkg/registry/core/service/ipallocator/controller/repair_test.go index 0bbd6f5959c..af2e59b22f4 100644 --- a/pkg/registry/core/service/ipallocator/controller/repair_test.go +++ b/pkg/registry/core/service/ipallocator/controller/repair_test.go @@ -22,9 +22,10 @@ import ( "strings" "testing" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" ) @@ -134,29 +135,29 @@ func TestRepairWithExisting(t *testing.T) { } fakeClient := fake.NewSimpleClientset( - &api.Service{ + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Namespace: "one", Name: "one"}, - Spec: api.ServiceSpec{ClusterIP: "192.168.1.1"}, + Spec: corev1.ServiceSpec{ClusterIP: "192.168.1.1"}, }, - &api.Service{ + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Namespace: "two", Name: "two"}, - Spec: api.ServiceSpec{ClusterIP: "192.168.1.100"}, + Spec: corev1.ServiceSpec{ClusterIP: "192.168.1.100"}, }, - &api.Service{ // outside CIDR, will be dropped + &corev1.Service{ // outside CIDR, will be dropped ObjectMeta: metav1.ObjectMeta{Namespace: "three", Name: "three"}, - Spec: api.ServiceSpec{ClusterIP: "192.168.0.1"}, + Spec: corev1.ServiceSpec{ClusterIP: "192.168.0.1"}, }, - &api.Service{ // empty, ignored + &corev1.Service{ // empty, ignored ObjectMeta: metav1.ObjectMeta{Namespace: "four", Name: "four"}, - Spec: api.ServiceSpec{ClusterIP: ""}, + Spec: corev1.ServiceSpec{ClusterIP: ""}, }, - &api.Service{ // duplicate, dropped + &corev1.Service{ // duplicate, dropped ObjectMeta: metav1.ObjectMeta{Namespace: "five", Name: "five"}, - Spec: api.ServiceSpec{ClusterIP: "192.168.1.1"}, + Spec: corev1.ServiceSpec{ClusterIP: "192.168.1.1"}, }, - &api.Service{ // headless + &corev1.Service{ // headless ObjectMeta: metav1.ObjectMeta{Namespace: "six", Name: "six"}, - Spec: api.ServiceSpec{ClusterIP: "None"}, + Spec: corev1.ServiceSpec{ClusterIP: "None"}, }, ) diff --git a/pkg/registry/core/service/portallocator/BUILD b/pkg/registry/core/service/portallocator/BUILD index 6da73adec1b..f2251bf9c83 100644 --- a/pkg/registry/core/service/portallocator/BUILD +++ b/pkg/registry/core/service/portallocator/BUILD @@ -17,7 +17,7 @@ go_library( "//pkg/apis/core:go_default_library", "//pkg/registry/core/service/allocator:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/registry/core/service/portallocator/allocator.go b/pkg/registry/core/service/portallocator/allocator.go index a9db71cd328..f33a2ce79e7 100644 --- a/pkg/registry/core/service/portallocator/allocator.go +++ b/pkg/registry/core/service/portallocator/allocator.go @@ -24,7 +24,7 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/registry/core/service/allocator" - "github.com/golang/glog" + "k8s.io/klog" ) // Interface manages the allocation of ports out of a range. Interface @@ -152,7 +152,7 @@ func (r *PortAllocator) ForEach(fn func(int)) { func (r *PortAllocator) Release(port int) error { ok, offset := r.contains(port) if !ok { - glog.Warningf("port is not in the range when release it. port: %v", port) + klog.Warningf("port is not in the range when release it. port: %v", port) return nil } diff --git a/pkg/registry/core/service/portallocator/controller/BUILD b/pkg/registry/core/service/portallocator/controller/BUILD index 8a6b511eda6..c4ea3f284d0 100644 --- a/pkg/registry/core/service/portallocator/controller/BUILD +++ b/pkg/registry/core/service/portallocator/controller/BUILD @@ -13,7 +13,6 @@ go_library( deps = [ "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/core:go_default_library", - "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library", "//pkg/registry/core/rangeallocation:go_default_library", "//pkg/registry/core/service/portallocator:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", @@ -22,6 +21,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", ], @@ -33,10 +33,11 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", - "//pkg/client/clientset_generated/internalclientset/fake:go_default_library", "//pkg/registry/core/service/portallocator:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], ) diff --git a/pkg/registry/core/service/portallocator/controller/repair.go b/pkg/registry/core/service/portallocator/controller/repair.go index c1a4f0dadbd..20485cc0442 100644 --- a/pkg/registry/core/service/portallocator/controller/repair.go +++ b/pkg/registry/core/service/portallocator/controller/repair.go @@ -21,16 +21,17 @@ import ( "time" "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" - coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" "k8s.io/kubernetes/pkg/registry/core/rangeallocation" "k8s.io/kubernetes/pkg/registry/core/service/portallocator" ) @@ -38,7 +39,7 @@ import ( // See ipallocator/controller/repair.go; this is a copy for ports. type Repair struct { interval time.Duration - serviceClient coreclient.ServicesGetter + serviceClient corev1client.ServicesGetter portRange net.PortRange alloc rangeallocation.RangeRegistry leaks map[int]int // counter per leaked port @@ -51,9 +52,9 @@ const numRepairsBeforeLeakCleanup = 3 // NewRepair creates a controller that periodically ensures that all ports are uniquely allocated across the cluster // and generates informational warnings for a cluster that is not in sync. -func NewRepair(interval time.Duration, serviceClient coreclient.ServicesGetter, eventClient coreclient.EventsGetter, portRange net.PortRange, alloc rangeallocation.RangeRegistry) *Repair { +func NewRepair(interval time.Duration, serviceClient corev1client.ServicesGetter, eventClient corev1client.EventsGetter, portRange net.PortRange, alloc rangeallocation.RangeRegistry) *Repair { eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartRecordingToSink(&coreclient.EventSinkImpl{Interface: eventClient.Events("")}) + eventBroadcaster.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: eventClient.Events("")}) recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "portallocator-repair-controller"}) return &Repair{ @@ -196,7 +197,7 @@ func (c *Repair) runOnce() error { return nil } -func collectServiceNodePorts(service *api.Service) []int { +func collectServiceNodePorts(service *corev1.Service) []int { servicePorts := []int{} for i := range service.Spec.Ports { servicePort := &service.Spec.Ports[i] diff --git a/pkg/registry/core/service/portallocator/controller/repair_test.go b/pkg/registry/core/service/portallocator/controller/repair_test.go index 48c41aa989a..0df94f898d7 100644 --- a/pkg/registry/core/service/portallocator/controller/repair_test.go +++ b/pkg/registry/core/service/portallocator/controller/repair_test.go @@ -21,10 +21,11 @@ import ( "strings" "testing" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/net" + "k8s.io/client-go/kubernetes/fake" api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake" "k8s.io/kubernetes/pkg/registry/core/service/portallocator" ) @@ -134,39 +135,39 @@ func TestRepairWithExisting(t *testing.T) { } fakeClient := fake.NewSimpleClientset( - &api.Service{ + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Namespace: "one", Name: "one"}, - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{{NodePort: 111}}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{NodePort: 111}}, }, }, - &api.Service{ + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Namespace: "two", Name: "two"}, - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{{NodePort: 122}, {NodePort: 133}}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{NodePort: 122}, {NodePort: 133}}, }, }, - &api.Service{ // outside range, will be dropped + &corev1.Service{ // outside range, will be dropped ObjectMeta: metav1.ObjectMeta{Namespace: "three", Name: "three"}, - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{{NodePort: 201}}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{NodePort: 201}}, }, }, - &api.Service{ // empty, ignored + &corev1.Service{ // empty, ignored ObjectMeta: metav1.ObjectMeta{Namespace: "four", Name: "four"}, - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{{}}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{}}, }, }, - &api.Service{ // duplicate, dropped + &corev1.Service{ // duplicate, dropped ObjectMeta: metav1.ObjectMeta{Namespace: "five", Name: "five"}, - Spec: api.ServiceSpec{ - Ports: []api.ServicePort{{NodePort: 111}}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{NodePort: 111}}, }, }, - &api.Service{ + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Namespace: "six", Name: "six"}, - Spec: api.ServiceSpec{ + Spec: corev1.ServiceSpec{ HealthCheckNodePort: 144, }, }, diff --git a/pkg/registry/core/service/storage/BUILD b/pkg/registry/core/service/storage/BUILD index 779dd91710e..40c4719a256 100644 --- a/pkg/registry/core/service/storage/BUILD +++ b/pkg/registry/core/service/storage/BUILD @@ -75,7 +75,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/dryrun:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go index 15862cdb99f..b0a54a21951 100644 --- a/pkg/registry/core/service/storage/rest.go +++ b/pkg/registry/core/service/storage/rest.go @@ -25,7 +25,6 @@ import ( "net/url" "strconv" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/api/errors" metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -38,6 +37,7 @@ import ( genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" "k8s.io/apiserver/pkg/util/dryrun" + "k8s.io/klog" apiservice "k8s.io/kubernetes/pkg/api/service" api "k8s.io/kubernetes/pkg/apis/core" @@ -306,7 +306,7 @@ func (rs *REST) healthCheckNodePortUpdate(oldService, service *api.Service, node // Allocate a health check node port or attempt to reserve the user-specified one if provided. // Insert health check node port into the service's HealthCheckNodePort field if needed. case !neededHealthCheckNodePort && needsHealthCheckNodePort: - glog.Infof("Transition to LoadBalancer type service with ExternalTrafficPolicy=Local") + klog.Infof("Transition to LoadBalancer type service with ExternalTrafficPolicy=Local") if err := allocateHealthCheckNodePort(service, nodePortOp); err != nil { return false, errors.NewInternalError(err) } @@ -314,8 +314,8 @@ func (rs *REST) healthCheckNodePortUpdate(oldService, service *api.Service, node // Case 2: Transition from needs HealthCheckNodePort to don't need HealthCheckNodePort. // Free the existing healthCheckNodePort and clear the HealthCheckNodePort field. case neededHealthCheckNodePort && !needsHealthCheckNodePort: - glog.Infof("Transition to non LoadBalancer type service or LoadBalancer type service with ExternalTrafficPolicy=Global") - glog.V(4).Infof("Releasing healthCheckNodePort: %d", oldHealthCheckNodePort) + klog.Infof("Transition to non LoadBalancer type service or LoadBalancer type service with ExternalTrafficPolicy=Global") + klog.V(4).Infof("Releasing healthCheckNodePort: %d", oldHealthCheckNodePort) nodePortOp.ReleaseDeferred(int(oldHealthCheckNodePort)) // Clear the HealthCheckNodePort field. service.Spec.HealthCheckNodePort = 0 @@ -324,7 +324,7 @@ func (rs *REST) healthCheckNodePortUpdate(oldService, service *api.Service, node // Reject changing the value of the HealthCheckNodePort field. case neededHealthCheckNodePort && needsHealthCheckNodePort: if oldHealthCheckNodePort != newHealthCheckNodePort { - glog.Warningf("Attempt to change value of health check node port DENIED") + klog.Warningf("Attempt to change value of health check node port DENIED") fldPath := field.NewPath("spec", "healthCheckNodePort") el := field.ErrorList{field.Invalid(fldPath, newHealthCheckNodePort, "cannot change healthCheckNodePort on loadBalancer service with externalTraffic=Local during update")} @@ -571,7 +571,7 @@ func allocateHealthCheckNodePort(service *api.Service, nodePortOp *portallocator return fmt.Errorf("failed to allocate requested HealthCheck NodePort %v: %v", healthCheckNodePort, err) } - glog.V(4).Infof("Reserved user requested healthCheckNodePort: %d", healthCheckNodePort) + klog.V(4).Infof("Reserved user requested healthCheckNodePort: %d", healthCheckNodePort) } else { // If the request has no health check nodePort specified, allocate any. healthCheckNodePort, err := nodePortOp.AllocateNext() @@ -579,7 +579,7 @@ func allocateHealthCheckNodePort(service *api.Service, nodePortOp *portallocator return fmt.Errorf("failed to allocate a HealthCheck NodePort %v: %v", healthCheckNodePort, err) } service.Spec.HealthCheckNodePort = int32(healthCheckNodePort) - glog.V(4).Infof("Reserved allocated healthCheckNodePort: %d", healthCheckNodePort) + klog.V(4).Infof("Reserved allocated healthCheckNodePort: %d", healthCheckNodePort) } return nil } diff --git a/pkg/registry/policy/OWNERS b/pkg/registry/policy/OWNERS index f65afe9ed9a..787630abd2f 100755 --- a/pkg/registry/policy/OWNERS +++ b/pkg/registry/policy/OWNERS @@ -1,3 +1,7 @@ +approvers: +- sig-auth-policy-approvers reviewers: -- deads2k -- hongchaodeng +- sig-auth-policy-reviewers +labels: +- sig/auth + diff --git a/pkg/registry/rbac/OWNERS b/pkg/registry/rbac/OWNERS index d21ad8662fd..cd0d70a0f8f 100755 --- a/pkg/registry/rbac/OWNERS +++ b/pkg/registry/rbac/OWNERS @@ -1,4 +1,7 @@ +approvers: +- sig-auth-authorizers-approvers reviewers: -- deads2k -- liggitt -- enj +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/pkg/registry/rbac/rest/BUILD b/pkg/registry/rbac/rest/BUILD index 7686c89b4d7..0a6c7efa245 100644 --- a/pkg/registry/rbac/rest/BUILD +++ b/pkg/registry/rbac/rest/BUILD @@ -43,7 +43,7 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/registry/rbac/rest/storage_rbac.go b/pkg/registry/rbac/rest/storage_rbac.go index 6568688e94a..1c0b500f9d8 100644 --- a/pkg/registry/rbac/rest/storage_rbac.go +++ b/pkg/registry/rbac/rest/storage_rbac.go @@ -20,7 +20,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" rbacapiv1 "k8s.io/api/rbac/v1" rbacapiv1alpha1 "k8s.io/api/rbac/v1alpha1" @@ -180,11 +180,11 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc { } switch { case result.Protected && result.Operation != reconciliation.ReconcileNone: - glog.Warningf("skipped reconcile-protected clusterrole.%s/%s with missing permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules) + klog.Warningf("skipped reconcile-protected clusterrole.%s/%s with missing permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules) case result.Operation == reconciliation.ReconcileUpdate: - glog.Infof("updated clusterrole.%s/%s with additional permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules) + klog.Infof("updated clusterrole.%s/%s with additional permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules) case result.Operation == reconciliation.ReconcileCreate: - glog.Infof("created clusterrole.%s/%s", rbac.GroupName, clusterRole.Name) + klog.Infof("created clusterrole.%s/%s", rbac.GroupName, clusterRole.Name) } return nil }) @@ -208,13 +208,13 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc { } switch { case result.Protected && result.Operation != reconciliation.ReconcileNone: - glog.Warningf("skipped reconcile-protected clusterrolebinding.%s/%s with missing subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects) + klog.Warningf("skipped reconcile-protected clusterrolebinding.%s/%s with missing subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects) case result.Operation == reconciliation.ReconcileUpdate: - glog.Infof("updated clusterrolebinding.%s/%s with additional subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects) + klog.Infof("updated clusterrolebinding.%s/%s with additional subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects) case result.Operation == reconciliation.ReconcileCreate: - glog.Infof("created clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name) + klog.Infof("created clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name) case result.Operation == reconciliation.ReconcileRecreate: - glog.Infof("recreated clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name) + klog.Infof("recreated clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name) } return nil }) @@ -239,11 +239,11 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc { } switch { case result.Protected && result.Operation != reconciliation.ReconcileNone: - glog.Warningf("skipped reconcile-protected role.%s/%s in %v with missing permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules) + klog.Warningf("skipped reconcile-protected role.%s/%s in %v with missing permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules) case result.Operation == reconciliation.ReconcileUpdate: - glog.Infof("updated role.%s/%s in %v with additional permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules) + klog.Infof("updated role.%s/%s in %v with additional permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules) case result.Operation == reconciliation.ReconcileCreate: - glog.Infof("created role.%s/%s in %v", rbac.GroupName, role.Name, namespace) + klog.Infof("created role.%s/%s in %v", rbac.GroupName, role.Name, namespace) } return nil }) @@ -269,13 +269,13 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc { } switch { case result.Protected && result.Operation != reconciliation.ReconcileNone: - glog.Warningf("skipped reconcile-protected rolebinding.%s/%s in %v with missing subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects) + klog.Warningf("skipped reconcile-protected rolebinding.%s/%s in %v with missing subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects) case result.Operation == reconciliation.ReconcileUpdate: - glog.Infof("updated rolebinding.%s/%s in %v with additional subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects) + klog.Infof("updated rolebinding.%s/%s in %v with additional subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects) case result.Operation == reconciliation.ReconcileCreate: - glog.Infof("created rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace) + klog.Infof("created rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace) case result.Operation == reconciliation.ReconcileRecreate: - glog.Infof("recreated rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace) + klog.Infof("recreated rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace) } return nil }) @@ -324,7 +324,7 @@ func primeAggregatedClusterRoles(clusterRolesToAggregate map[string]string, clus // the old role already moved to an aggregated role, so there are no custom rules to migrate at this point return nil } - glog.V(1).Infof("migrating %v to %v", existingRole.Name, newName) + klog.V(1).Infof("migrating %v to %v", existingRole.Name, newName) existingRole.Name = newName existingRole.ResourceVersion = "" // clear this so the object can be created. if _, err := clusterRoleClient.ClusterRoles().Create(existingRole); err != nil && !apierrors.IsAlreadyExists(err) { diff --git a/pkg/registry/rbac/validation/BUILD b/pkg/registry/rbac/validation/BUILD index ad664075455..00095ac3612 100644 --- a/pkg/registry/rbac/validation/BUILD +++ b/pkg/registry/rbac/validation/BUILD @@ -41,7 +41,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/registry/rbac/validation/rule.go b/pkg/registry/rbac/validation/rule.go index 833ffc1e6c1..6c88791f8be 100644 --- a/pkg/registry/rbac/validation/rule.go +++ b/pkg/registry/rbac/validation/rule.go @@ -22,7 +22,7 @@ import ( "fmt" "strings" - "github.com/golang/glog" + "k8s.io/klog" rbacv1 "k8s.io/api/rbac/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" @@ -61,7 +61,7 @@ func ConfirmNoEscalation(ctx context.Context, ruleResolver AuthorizationRuleReso ownerRules, err := ruleResolver.RulesFor(user, namespace) if err != nil { // As per AuthorizationRuleResolver contract, this may return a non fatal error with an incomplete list of policies. Log the error and continue. - glog.V(1).Infof("non-fatal error getting local rules for %v: %v", user, err) + klog.V(1).Infof("non-fatal error getting local rules for %v: %v", user, err) ruleResolutionErrors = append(ruleResolutionErrors, err) } diff --git a/pkg/registry/scheduling/rest/BUILD b/pkg/registry/scheduling/rest/BUILD index bb9ed054c63..25f8f79d437 100644 --- a/pkg/registry/scheduling/rest/BUILD +++ b/pkg/registry/scheduling/rest/BUILD @@ -24,7 +24,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/registry/scheduling/rest/storage_scheduling.go b/pkg/registry/scheduling/rest/storage_scheduling.go index aad6e09ec04..ed291b32c65 100644 --- a/pkg/registry/scheduling/rest/storage_scheduling.go +++ b/pkg/registry/scheduling/rest/storage_scheduling.go @@ -20,7 +20,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -88,16 +88,16 @@ func AddSystemPriorityClasses() genericapiserver.PostStartHookFunc { if err != nil && !apierrors.IsAlreadyExists(err) { return false, err } else { - glog.Infof("created PriorityClass %s with value %v", pc.Name, pc.Value) + klog.Infof("created PriorityClass %s with value %v", pc.Name, pc.Value) } } else { // Unable to get the priority class for reasons other than "not found". - glog.Warningf("unable to get PriorityClass %v: %v. Retrying...", pc.Name, err) + klog.Warningf("unable to get PriorityClass %v: %v. Retrying...", pc.Name, err) return false, err } } } - glog.Infof("all system priority classes are created successfully or already exist.") + klog.Infof("all system priority classes are created successfully or already exist.") return true, nil }) // if we're never able to make it through initialization, kill the API server. diff --git a/pkg/registry/storage/rest/storage_storage.go b/pkg/registry/storage/rest/storage_storage.go index 5e8b2529a5f..f7992260012 100644 --- a/pkg/registry/storage/rest/storage_storage.go +++ b/pkg/registry/storage/rest/storage_storage.go @@ -54,8 +54,8 @@ func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorag func (p RESTStorageProvider) v1alpha1Storage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) map[string]rest.Storage { storage := map[string]rest.Storage{} // volumeattachments - volumeAttachmentStorage := volumeattachmentstore.NewREST(restOptionsGetter) - storage["volumeattachments"] = volumeAttachmentStorage + volumeAttachmentStorage := volumeattachmentstore.NewStorage(restOptionsGetter) + storage["volumeattachments"] = volumeAttachmentStorage.VolumeAttachment return storage } @@ -67,17 +67,24 @@ func (p RESTStorageProvider) v1beta1Storage(apiResourceConfigSource serverstorag storage["storageclasses"] = storageClassStorage // volumeattachments - volumeAttachmentStorage := volumeattachmentstore.NewREST(restOptionsGetter) - storage["volumeattachments"] = volumeAttachmentStorage + volumeAttachmentStorage := volumeattachmentstore.NewStorage(restOptionsGetter) + storage["volumeattachments"] = volumeAttachmentStorage.VolumeAttachment return storage } func (p RESTStorageProvider) v1Storage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) map[string]rest.Storage { - storage := map[string]rest.Storage{} - // storageclasses storageClassStorage := storageclassstore.NewREST(restOptionsGetter) - storage["storageclasses"] = storageClassStorage + volumeAttachmentStorage := volumeattachmentstore.NewStorage(restOptionsGetter) + + storage := map[string]rest.Storage{ + // storageclasses + "storageclasses": storageClassStorage, + + // volumeattachments + "volumeattachments": volumeAttachmentStorage.VolumeAttachment, + "volumeattachments/status": volumeAttachmentStorage.Status, + } return storage } diff --git a/pkg/registry/storage/volumeattachment/BUILD b/pkg/registry/storage/volumeattachment/BUILD index 5fa8cbd44b8..7cb2a4bb38d 100644 --- a/pkg/registry/storage/volumeattachment/BUILD +++ b/pkg/registry/storage/volumeattachment/BUILD @@ -12,8 +12,11 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/storage:go_default_library", "//pkg/apis/storage/validation:go_default_library", + "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library", ], ) @@ -24,7 +27,10 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/storage:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", ], ) diff --git a/pkg/registry/storage/volumeattachment/storage/BUILD b/pkg/registry/storage/volumeattachment/storage/BUILD index 0ed148659ec..9d1389020c7 100644 --- a/pkg/registry/storage/volumeattachment/storage/BUILD +++ b/pkg/registry/storage/volumeattachment/storage/BUILD @@ -8,9 +8,11 @@ go_library( deps = [ "//pkg/apis/storage:go_default_library", "//pkg/registry/storage/volumeattachment:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", ], ) @@ -19,17 +21,18 @@ go_test( srcs = ["storage_test.go"], embed = [":go_default_library"], deps = [ - "//pkg/api/testapi:go_default_library", "//pkg/apis/storage:go_default_library", "//pkg/registry/registrytest:go_default_library", - "//staging/src/k8s.io/api/storage/v1alpha1:go_default_library", - "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic/testing:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library", ], ) diff --git a/pkg/registry/storage/volumeattachment/storage/storage.go b/pkg/registry/storage/volumeattachment/storage/storage.go index 28a3208e413..df8c470c895 100644 --- a/pkg/registry/storage/volumeattachment/storage/storage.go +++ b/pkg/registry/storage/volumeattachment/storage/storage.go @@ -17,20 +17,30 @@ limitations under the License. package storage import ( + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" + "k8s.io/apiserver/pkg/registry/rest" storageapi "k8s.io/kubernetes/pkg/apis/storage" "k8s.io/kubernetes/pkg/registry/storage/volumeattachment" ) -// REST object that will work against persistent volumes. +// VolumeAttachmentStorage includes storage for VolumeAttachments and all subresources +type VolumeAttachmentStorage struct { + VolumeAttachment *REST + Status *StatusREST +} + +// REST object that will work for VolumeAttachments type REST struct { *genericregistry.Store } -// NewREST returns a RESTStorage object that will work against persistent volumes. -func NewREST(optsGetter generic.RESTOptionsGetter) *REST { +// NewStorage returns a RESTStorage object that will work against VolumeAttachments +func NewStorage(optsGetter generic.RESTOptionsGetter) *VolumeAttachmentStorage { store := &genericregistry.Store{ NewFunc: func() runtime.Object { return &storageapi.VolumeAttachment{} }, NewListFunc: func() runtime.Object { return &storageapi.VolumeAttachmentList{} }, @@ -46,5 +56,33 @@ func NewREST(optsGetter generic.RESTOptionsGetter) *REST { panic(err) // TODO: Propagate error up } - return &REST{store} + statusStore := *store + statusStore.UpdateStrategy = volumeattachment.StatusStrategy + + return &VolumeAttachmentStorage{ + VolumeAttachment: &REST{store}, + Status: &StatusREST{store: &statusStore}, + } +} + +// StatusREST implements the REST endpoint for changing the status of a VolumeAttachment +type StatusREST struct { + store *genericregistry.Store +} + +// New creates a new VolumeAttachment resource +func (r *StatusREST) New() runtime.Object { + return &storageapi.VolumeAttachment{} +} + +// Get retrieves the object from the storage. It is required to support Patch. +func (r *StatusREST) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) { + return r.store.Get(ctx, name, options) +} + +// Update alters the status subset of an object. +func (r *StatusREST) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) { + // We are explicitly setting forceAllowCreate to false in the call to the underlying storage because + // subresources should never allow create on update. + return r.store.Update(ctx, name, objInfo, createValidation, updateValidation, false, options) } diff --git a/pkg/registry/storage/volumeattachment/storage/storage_test.go b/pkg/registry/storage/volumeattachment/storage/storage_test.go index 7e2d389af6a..10b61ba597e 100644 --- a/pkg/registry/storage/volumeattachment/storage/storage_test.go +++ b/pkg/registry/storage/volumeattachment/storage/storage_test.go @@ -19,21 +19,22 @@ package storage import ( "testing" - storageapiv1alpha1 "k8s.io/api/storage/v1alpha1" - storageapiv1beta1 "k8s.io/api/storage/v1beta1" + apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/diff" + genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic" genericregistrytest "k8s.io/apiserver/pkg/registry/generic/testing" + "k8s.io/apiserver/pkg/registry/rest" etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing" - "k8s.io/kubernetes/pkg/api/testapi" storageapi "k8s.io/kubernetes/pkg/apis/storage" "k8s.io/kubernetes/pkg/registry/registrytest" ) -func newStorage(t *testing.T) (*REST, *etcdtesting.EtcdTestServer) { +func newStorage(t *testing.T) (*REST, *StatusREST, *etcdtesting.EtcdTestServer) { etcdStorage, server := registrytest.NewEtcdStorage(t, storageapi.GroupName) restOptions := generic.RESTOptions{ StorageConfig: etcdStorage, @@ -41,8 +42,8 @@ func newStorage(t *testing.T) (*REST, *etcdtesting.EtcdTestServer) { DeleteCollectionWorkers: 1, ResourcePrefix: "volumeattachments", } - volumeAttachmentStorage := NewREST(restOptions) - return volumeAttachmentStorage, server + volumeAttachmentStorage := NewStorage(restOptions) + return volumeAttachmentStorage.VolumeAttachment, volumeAttachmentStorage.Status, server } func validNewVolumeAttachment(name string) *storageapi.VolumeAttachment { @@ -62,13 +63,7 @@ func validNewVolumeAttachment(name string) *storageapi.VolumeAttachment { } func TestCreate(t *testing.T) { - if *testapi.Storage.GroupVersion() != storageapiv1alpha1.SchemeGroupVersion && - *testapi.Storage.GroupVersion() != storageapiv1beta1.SchemeGroupVersion { - // skip the test for all versions exception v1alpha1 and v1beta1 - return - } - - storage, server := newStorage(t) + storage, _, server := newStorage(t) defer server.Terminate(t) defer storage.Store.DestroyFunc() test := genericregistrytest.New(t, storage.Store).ClusterScope() @@ -93,20 +88,16 @@ func TestCreate(t *testing.T) { } func TestUpdate(t *testing.T) { - if *testapi.Storage.GroupVersion() != storageapiv1alpha1.SchemeGroupVersion && - *testapi.Storage.GroupVersion() != storageapiv1beta1.SchemeGroupVersion { - // skip the test for all versions exception v1alpha1 and v1beta1 - return - } - - storage, server := newStorage(t) + storage, _, server := newStorage(t) defer server.Terminate(t) defer storage.Store.DestroyFunc() test := genericregistrytest.New(t, storage.Store).ClusterScope() + test.TestUpdate( // valid validNewVolumeAttachment("foo"), - // updateFunc + // we still allow status field to be set in both v1 and v1beta1 + // it is just that in v1 the new value does not take effect. func(obj runtime.Object) runtime.Object { object := obj.(*storageapi.VolumeAttachment) object.Status.Attached = true @@ -122,13 +113,7 @@ func TestUpdate(t *testing.T) { } func TestDelete(t *testing.T) { - if *testapi.Storage.GroupVersion() != storageapiv1alpha1.SchemeGroupVersion && - *testapi.Storage.GroupVersion() != storageapiv1beta1.SchemeGroupVersion { - // skip the test for all versions exception v1alpha1 and v1beta1 - return - } - - storage, server := newStorage(t) + storage, _, server := newStorage(t) defer server.Terminate(t) defer storage.Store.DestroyFunc() test := genericregistrytest.New(t, storage.Store).ClusterScope().ReturnDeletedObject() @@ -136,13 +121,7 @@ func TestDelete(t *testing.T) { } func TestGet(t *testing.T) { - if *testapi.Storage.GroupVersion() != storageapiv1alpha1.SchemeGroupVersion && - *testapi.Storage.GroupVersion() != storageapiv1beta1.SchemeGroupVersion { - // skip the test for all versions exception v1alpha1 and v1beta1 - return - } - - storage, server := newStorage(t) + storage, _, server := newStorage(t) defer server.Terminate(t) defer storage.Store.DestroyFunc() test := genericregistrytest.New(t, storage.Store).ClusterScope() @@ -150,13 +129,7 @@ func TestGet(t *testing.T) { } func TestList(t *testing.T) { - if *testapi.Storage.GroupVersion() != storageapiv1alpha1.SchemeGroupVersion && - *testapi.Storage.GroupVersion() != storageapiv1beta1.SchemeGroupVersion { - // skip the test for all versions exception v1alpha1 and v1beta1 - return - } - - storage, server := newStorage(t) + storage, _, server := newStorage(t) defer server.Terminate(t) defer storage.Store.DestroyFunc() test := genericregistrytest.New(t, storage.Store).ClusterScope() @@ -164,13 +137,7 @@ func TestList(t *testing.T) { } func TestWatch(t *testing.T) { - if *testapi.Storage.GroupVersion() != storageapiv1alpha1.SchemeGroupVersion && - *testapi.Storage.GroupVersion() != storageapiv1beta1.SchemeGroupVersion { - // skip the test for all versions exception v1alpha1 and v1beta1 - return - } - - storage, server := newStorage(t) + storage, _, server := newStorage(t) defer server.Terminate(t) defer storage.Store.DestroyFunc() test := genericregistrytest.New(t, storage.Store).ClusterScope() @@ -192,3 +159,41 @@ func TestWatch(t *testing.T) { }, ) } + +func TestEtcdStatusUpdate(t *testing.T) { + storage, statusStorage, server := newStorage(t) + defer server.Terminate(t) + defer storage.Store.DestroyFunc() + ctx := genericapirequest.NewDefaultContext() + + attachment := validNewVolumeAttachment("foo") + if _, err := storage.Create(ctx, attachment, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + obj, err := storage.Get(ctx, attachment.ObjectMeta.Name, &metav1.GetOptions{}) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + // update status + attachmentIn := obj.(*storageapi.VolumeAttachment).DeepCopy() + attachmentIn.Status.Attached = true + + _, _, err = statusStorage.Update(ctx, attachmentIn.Name, rest.DefaultUpdatedObjectInfo(attachmentIn), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("Failed to update status: %v", err) + } + + // validate object got updated + obj, err = storage.Get(ctx, attachmentIn.ObjectMeta.Name, &metav1.GetOptions{}) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + attachmentOut := obj.(*storageapi.VolumeAttachment) + if !apiequality.Semantic.DeepEqual(attachmentIn.Spec, attachmentOut.Spec) { + t.Errorf("objects differ: %v", diff.ObjectDiff(attachmentOut.Spec, attachmentIn.Spec)) + } + if !apiequality.Semantic.DeepEqual(attachmentIn.Status, attachmentOut.Status) { + t.Errorf("objects differ: %v", diff.ObjectDiff(attachmentOut.Status, attachmentIn.Status)) + } +} diff --git a/pkg/registry/storage/volumeattachment/strategy.go b/pkg/registry/storage/volumeattachment/strategy.go index 9ee76f1874a..6d37f695c44 100644 --- a/pkg/registry/storage/volumeattachment/strategy.go +++ b/pkg/registry/storage/volumeattachment/strategy.go @@ -19,8 +19,11 @@ package volumeattachment import ( "context" + storageapiv1beta1 "k8s.io/api/storage/v1beta1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation/field" + genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/storage/names" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/storage" @@ -43,11 +46,40 @@ func (volumeAttachmentStrategy) NamespaceScoped() bool { // ResetBeforeCreate clears the Status field which is not allowed to be set by end users on creation. func (volumeAttachmentStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) { + var groupVersion schema.GroupVersion + + if requestInfo, found := genericapirequest.RequestInfoFrom(ctx); found { + groupVersion = schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion} + } + + switch groupVersion { + case storageapiv1beta1.SchemeGroupVersion: + // allow modification of status for v1beta1 + default: + volumeAttachment := obj.(*storage.VolumeAttachment) + volumeAttachment.Status = storage.VolumeAttachmentStatus{} + } } func (volumeAttachmentStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { volumeAttachment := obj.(*storage.VolumeAttachment) - return validation.ValidateVolumeAttachment(volumeAttachment) + + errs := validation.ValidateVolumeAttachment(volumeAttachment) + + var groupVersion schema.GroupVersion + + if requestInfo, found := genericapirequest.RequestInfoFrom(ctx); found { + groupVersion = schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion} + } + + switch groupVersion { + case storageapiv1beta1.SchemeGroupVersion: + // no extra validation + default: + // tighten up validation of newly created v1 attachments + errs = append(errs, validation.ValidateVolumeAttachmentV1(volumeAttachment)...) + } + return errs } // Canonicalize normalizes the object after validation. @@ -58,8 +90,22 @@ func (volumeAttachmentStrategy) AllowCreateOnUpdate() bool { return false } -// PrepareForUpdate sets the Status fields which is not allowed to be set by an end user updating a PV +// PrepareForUpdate sets the Status fields which is not allowed to be set by an end user updating a VolumeAttachment func (volumeAttachmentStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { + var groupVersion schema.GroupVersion + + if requestInfo, found := genericapirequest.RequestInfoFrom(ctx); found { + groupVersion = schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion} + } + switch groupVersion { + case storageapiv1beta1.SchemeGroupVersion: + // allow modification of Status via main resource for v1beta1 + default: + newVolumeAttachment := obj.(*storage.VolumeAttachment) + oldVolumeAttachment := old.(*storage.VolumeAttachment) + newVolumeAttachment.Status = oldVolumeAttachment.Status + // No need to increment Generation because we don't allow updates to spec + } } func (volumeAttachmentStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { @@ -72,3 +118,30 @@ func (volumeAttachmentStrategy) ValidateUpdate(ctx context.Context, obj, old run func (volumeAttachmentStrategy) AllowUnconditionalUpdate() bool { return false } + +// volumeAttachmentStatusStrategy implements behavior for VolumeAttachmentStatus subresource +type volumeAttachmentStatusStrategy struct { + volumeAttachmentStrategy +} + +// StatusStrategy is the default logic that applies when creating and updating +// VolumeAttachmentStatus subresource via the REST API. +var StatusStrategy = volumeAttachmentStatusStrategy{Strategy} + +// PrepareForUpdate sets the Status fields which is not allowed to be set by an end user updating a VolumeAttachment +func (volumeAttachmentStatusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { + newVolumeAttachment := obj.(*storage.VolumeAttachment) + oldVolumeAttachment := old.(*storage.VolumeAttachment) + + newVolumeAttachment.Spec = oldVolumeAttachment.Spec + + oldMeta := oldVolumeAttachment.ObjectMeta + newMeta := &newVolumeAttachment.ObjectMeta + newMeta.SetDeletionTimestamp(oldMeta.GetDeletionTimestamp()) + newMeta.SetGeneration(oldMeta.GetGeneration()) + newMeta.SetSelfLink(oldMeta.GetSelfLink()) + newMeta.SetLabels(oldMeta.GetLabels()) + newMeta.SetAnnotations(oldMeta.GetAnnotations()) + newMeta.SetFinalizers(oldMeta.GetFinalizers()) + newMeta.SetOwnerReferences(oldMeta.GetOwnerReferences()) +} diff --git a/pkg/registry/storage/volumeattachment/strategy_test.go b/pkg/registry/storage/volumeattachment/strategy_test.go index 33e8985e79a..e2ec6b4b708 100644 --- a/pkg/registry/storage/volumeattachment/strategy_test.go +++ b/pkg/registry/storage/volumeattachment/strategy_test.go @@ -19,13 +19,35 @@ package volumeattachment import ( "testing" + apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/validation/field" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/kubernetes/pkg/apis/storage" ) +func getValidVolumeAttachment(name string) *storage.VolumeAttachment { + return &storage.VolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: storage.VolumeAttachmentSpec{ + Attacher: "valid-attacher", + Source: storage.VolumeAttachmentSource{ + PersistentVolumeName: &name, + }, + NodeName: "valid-node", + }, + } +} + func TestVolumeAttachmentStrategy(t *testing.T) { - ctx := genericapirequest.NewDefaultContext() + ctx := genericapirequest.WithRequestInfo(genericapirequest.NewContext(), &genericapirequest.RequestInfo{ + APIGroup: "storage.k8s.io", + APIVersion: "v1", + Resource: "volumeattachments", + }) if Strategy.NamespaceScoped() { t.Errorf("VolumeAttachment must not be namespace scoped") } @@ -33,19 +55,7 @@ func TestVolumeAttachmentStrategy(t *testing.T) { t.Errorf("VolumeAttachment should not allow create on update") } - pvName := "name" - volumeAttachment := &storage.VolumeAttachment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "valid-attachment", - }, - Spec: storage.VolumeAttachmentSpec{ - Attacher: "valid-attacher", - Source: storage.VolumeAttachmentSource{ - PersistentVolumeName: &pvName, - }, - NodeName: "valid-node", - }, - } + volumeAttachment := getValidVolumeAttachment("valid-attachment") Strategy.PrepareForCreate(ctx, volumeAttachment) @@ -54,19 +64,18 @@ func TestVolumeAttachmentStrategy(t *testing.T) { t.Errorf("unexpected error validating %v", errs) } - newVolumeAttachment := &storage.VolumeAttachment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "valid-attachment-2", - }, - Spec: storage.VolumeAttachmentSpec{ - Attacher: "valid-attacher-2", - Source: storage.VolumeAttachmentSource{ - PersistentVolumeName: &pvName, - }, - NodeName: "valid-node-2", - }, + // Create with status should drop status + statusVolumeAttachment := volumeAttachment.DeepCopy() + statusVolumeAttachment.Status = storage.VolumeAttachmentStatus{Attached: true} + Strategy.PrepareForCreate(ctx, statusVolumeAttachment) + if !apiequality.Semantic.DeepEqual(statusVolumeAttachment, volumeAttachment) { + t.Errorf("unexpected objects difference after creating with status: %v", diff.ObjectDiff(statusVolumeAttachment, volumeAttachment)) } + // Update of spec is disallowed + newVolumeAttachment := volumeAttachment.DeepCopy() + newVolumeAttachment.Spec.NodeName = "valid-node-2" + Strategy.PrepareForUpdate(ctx, newVolumeAttachment, volumeAttachment) errs = Strategy.ValidateUpdate(ctx, newVolumeAttachment, volumeAttachment) @@ -74,4 +83,215 @@ func TestVolumeAttachmentStrategy(t *testing.T) { t.Errorf("Expected a validation error") } + // modifying status should be dropped + statusVolumeAttachment = volumeAttachment.DeepCopy() + statusVolumeAttachment.Status = storage.VolumeAttachmentStatus{Attached: true} + + Strategy.PrepareForUpdate(ctx, statusVolumeAttachment, volumeAttachment) + + if !apiequality.Semantic.DeepEqual(statusVolumeAttachment, volumeAttachment) { + t.Errorf("unexpected objects difference after modfying status: %v", diff.ObjectDiff(statusVolumeAttachment, volumeAttachment)) + } +} + +func TestVolumeAttachmentStatusStrategy(t *testing.T) { + ctx := genericapirequest.WithRequestInfo(genericapirequest.NewContext(), &genericapirequest.RequestInfo{ + APIGroup: "storage.k8s.io", + APIVersion: "v1", + Resource: "volumeattachments", + }) + + volumeAttachment := getValidVolumeAttachment("valid-attachment") + + // modifying status should be allowed + statusVolumeAttachment := volumeAttachment.DeepCopy() + statusVolumeAttachment.Status = storage.VolumeAttachmentStatus{Attached: true} + + expectedVolumeAttachment := statusVolumeAttachment.DeepCopy() + StatusStrategy.PrepareForUpdate(ctx, statusVolumeAttachment, volumeAttachment) + if !apiequality.Semantic.DeepEqual(statusVolumeAttachment, expectedVolumeAttachment) { + t.Errorf("unexpected objects differerence after modifying status: %v", diff.ObjectDiff(statusVolumeAttachment, expectedVolumeAttachment)) + } + + // modifying spec should be dropped + newVolumeAttachment := volumeAttachment.DeepCopy() + newVolumeAttachment.Spec.NodeName = "valid-node-2" + + StatusStrategy.PrepareForUpdate(ctx, newVolumeAttachment, volumeAttachment) + if !apiequality.Semantic.DeepEqual(newVolumeAttachment, volumeAttachment) { + t.Errorf("unexpected objects differerence after modifying spec: %v", diff.ObjectDiff(newVolumeAttachment, volumeAttachment)) + } +} + +func TestBetaAndV1StatusUpdate(t *testing.T) { + tests := []struct { + requestInfo genericapirequest.RequestInfo + newStatus bool + expectedStatus bool + }{ + { + genericapirequest.RequestInfo{ + APIGroup: "storage.k8s.io", + APIVersion: "v1", + Resource: "volumeattachments", + }, + true, + false, + }, + { + genericapirequest.RequestInfo{ + APIGroup: "storage.k8s.io", + APIVersion: "v1beta1", + Resource: "volumeattachments", + }, + true, + true, + }, + } + for _, test := range tests { + va := getValidVolumeAttachment("valid-attachment") + newAttachment := va.DeepCopy() + newAttachment.Status.Attached = test.newStatus + context := genericapirequest.WithRequestInfo(genericapirequest.NewContext(), &test.requestInfo) + Strategy.PrepareForUpdate(context, newAttachment, va) + if newAttachment.Status.Attached != test.expectedStatus { + t.Errorf("expected status to be %v got %v", test.expectedStatus, newAttachment.Status.Attached) + } + } + +} + +func TestBetaAndV1StatusCreate(t *testing.T) { + tests := []struct { + requestInfo genericapirequest.RequestInfo + newStatus bool + expectedStatus bool + }{ + { + genericapirequest.RequestInfo{ + APIGroup: "storage.k8s.io", + APIVersion: "v1", + Resource: "volumeattachments", + }, + true, + false, + }, + { + genericapirequest.RequestInfo{ + APIGroup: "storage.k8s.io", + APIVersion: "v1beta1", + Resource: "volumeattachments", + }, + true, + true, + }, + } + for _, test := range tests { + va := getValidVolumeAttachment("valid-attachment") + va.Status.Attached = test.newStatus + context := genericapirequest.WithRequestInfo(genericapirequest.NewContext(), &test.requestInfo) + Strategy.PrepareForCreate(context, va) + if va.Status.Attached != test.expectedStatus { + t.Errorf("expected status to be %v got %v", test.expectedStatus, va.Status.Attached) + } + } +} + +func TestVolumeAttachmentValidation(t *testing.T) { + invalidPVName := "invalid-!@#$%^&*()" + validPVName := "valid-volume-name" + tests := []struct { + name string + volumeAttachment *storage.VolumeAttachment + expectBetaError bool + expectV1Error bool + }{ + { + "valid attachment", + getValidVolumeAttachment("foo"), + false, + false, + }, + { + "invalid PV name", + &storage.VolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: storage.VolumeAttachmentSpec{ + Attacher: "valid-attacher", + Source: storage.VolumeAttachmentSource{ + PersistentVolumeName: &invalidPVName, + }, + NodeName: "valid-node", + }, + }, + false, + true, + }, + { + "invalid attacher name", + &storage.VolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: storage.VolumeAttachmentSpec{ + Attacher: "invalid!@#$%^&*()", + Source: storage.VolumeAttachmentSource{ + PersistentVolumeName: &validPVName, + }, + NodeName: "valid-node", + }, + }, + false, + true, + }, + { + "invalid volume attachment", + &storage.VolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: storage.VolumeAttachmentSpec{ + Attacher: "invalid!@#$%^&*()", + Source: storage.VolumeAttachmentSource{ + PersistentVolumeName: nil, + }, + NodeName: "valid-node", + }, + }, + true, + true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + testValidation := func(va *storage.VolumeAttachment, apiVersion string) field.ErrorList { + ctx := genericapirequest.WithRequestInfo(genericapirequest.NewContext(), &genericapirequest.RequestInfo{ + APIGroup: "storage.k8s.io", + APIVersion: apiVersion, + Resource: "volumeattachments", + }) + return Strategy.Validate(ctx, va) + } + + v1Err := testValidation(test.volumeAttachment, "v1") + if len(v1Err) > 0 && !test.expectV1Error { + t.Errorf("Validation of v1 object failed: %+v", v1Err) + } + if len(v1Err) == 0 && test.expectV1Error { + t.Errorf("Validation of v1 object unexpectedly succeeded") + } + + betaErr := testValidation(test.volumeAttachment, "v1beta1") + if len(betaErr) > 0 && !test.expectBetaError { + t.Errorf("Validation of v1beta1 object failed: %+v", betaErr) + } + if len(betaErr) == 0 && test.expectBetaError { + t.Errorf("Validation of v1beta1 object unexpectedly succeeded") + } + }) + } } diff --git a/pkg/scheduler/BUILD b/pkg/scheduler/BUILD index 22585cbad75..c3dc753d20d 100644 --- a/pkg/scheduler/BUILD +++ b/pkg/scheduler/BUILD @@ -34,17 +34,21 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) go_test( name = "go_default_test", - srcs = ["scheduler_test.go"], + srcs = [ + "main_test.go", + "scheduler_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/api/legacyscheme:go_default_library", "//pkg/controller/volume/persistentvolume:go_default_library", + "//pkg/features:go_default_library", "//pkg/scheduler/algorithm:go_default_library", "//pkg/scheduler/algorithm/predicates:go_default_library", "//pkg/scheduler/api:go_default_library", @@ -65,6 +69,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", diff --git a/pkg/scheduler/algorithm/predicates/BUILD b/pkg/scheduler/algorithm/predicates/BUILD index fa735524e35..58ee7b487cc 100644 --- a/pkg/scheduler/algorithm/predicates/BUILD +++ b/pkg/scheduler/algorithm/predicates/BUILD @@ -41,7 +41,7 @@ go_library( "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/storage/v1:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -49,6 +49,7 @@ go_test( name = "go_default_test", srcs = [ "csi_volume_predicate_test.go", + "main_test.go", "max_attachable_volume_predicate_test.go", "metadata_test.go", "predicates_test.go", diff --git a/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go b/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go index 8e155ea2f18..ff2215eb28d 100644 --- a/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go +++ b/pkg/scheduler/algorithm/predicates/csi_volume_predicate.go @@ -19,9 +19,9 @@ package predicates import ( "fmt" - "github.com/golang/glog" "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/algorithm" schedulercache "k8s.io/kubernetes/pkg/scheduler/cache" @@ -126,26 +126,26 @@ func (c *CSIMaxVolumeLimitChecker) filterAttachableVolumes( pvc, err := c.pvcInfo.GetPersistentVolumeClaimInfo(namespace, pvcName) if err != nil { - glog.V(4).Infof("Unable to look up PVC info for %s/%s", namespace, pvcName) + klog.V(4).Infof("Unable to look up PVC info for %s/%s", namespace, pvcName) continue } pvName := pvc.Spec.VolumeName // TODO - the actual handling of unbound PVCs will be fixed by late binding design. if pvName == "" { - glog.V(4).Infof("Persistent volume had no name for claim %s/%s", namespace, pvcName) + klog.V(4).Infof("Persistent volume had no name for claim %s/%s", namespace, pvcName) continue } pv, err := c.pvInfo.GetPersistentVolumeInfo(pvName) if err != nil { - glog.V(4).Infof("Unable to look up PV info for PVC %s/%s and PV %s", namespace, pvcName, pvName) + klog.V(4).Infof("Unable to look up PV info for PVC %s/%s and PV %s", namespace, pvcName, pvName) continue } csiSource := pv.Spec.PersistentVolumeSource.CSI if csiSource == nil { - glog.V(4).Infof("Not considering non-CSI volume %s/%s", namespace, pvcName) + klog.V(4).Infof("Not considering non-CSI volume %s/%s", namespace, pvcName) continue } driverName := csiSource.Driver diff --git a/pkg/scheduler/algorithm/predicates/main_test.go b/pkg/scheduler/algorithm/predicates/main_test.go new file mode 100644 index 00000000000..11bc537373b --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package predicates + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/scheduler/algorithm/predicates/metadata.go b/pkg/scheduler/algorithm/predicates/metadata.go index 3feb23e4937..9284cda2381 100644 --- a/pkg/scheduler/algorithm/predicates/metadata.go +++ b/pkg/scheduler/algorithm/predicates/metadata.go @@ -21,7 +21,7 @@ import ( "fmt" "sync" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -140,7 +140,7 @@ func (pfactory *PredicateMetadataFactory) GetMetadata(pod *v1.Pod, nodeNameToInf // incomingPodAntiAffinityMap will be used later for efficient check on incoming pod's anti-affinity incomingPodAffinityMap, incomingPodAntiAffinityMap, err := getTPMapMatchingIncomingAffinityAntiAffinity(pod, nodeNameToInfoMap) if err != nil { - glog.Errorf("[predicate meta data generation] error finding pods that match affinity terms: %v", err) + klog.Errorf("[predicate meta data generation] error finding pods that match affinity terms: %v", err) return nil } predicateMetadata := &predicateMetadata{ @@ -153,7 +153,7 @@ func (pfactory *PredicateMetadataFactory) GetMetadata(pod *v1.Pod, nodeNameToInf topologyPairsAntiAffinityPodsMap: existingPodAntiAffinityMap, } for predicateName, precomputeFunc := range predicateMetadataProducers { - glog.V(10).Infof("Precompute: %v", predicateName) + klog.V(10).Infof("Precompute: %v", predicateName) precomputeFunc(predicateMetadata) } return predicateMetadata @@ -502,7 +502,7 @@ func targetPodMatchesAffinityOfPod(pod, targetPod *v1.Pod) bool { } affinityProperties, err := getAffinityTermProperties(pod, GetPodAffinityTerms(affinity.PodAffinity)) if err != nil { - glog.Errorf("error in getting affinity properties of Pod %v", pod.Name) + klog.Errorf("error in getting affinity properties of Pod %v", pod.Name) return false } return podMatchesAllAffinityTermProperties(targetPod, affinityProperties) @@ -519,7 +519,7 @@ func targetPodMatchesAntiAffinityOfPod(pod, targetPod *v1.Pod) bool { } properties, err := getAffinityTermProperties(pod, GetPodAntiAffinityTerms(affinity.PodAntiAffinity)) if err != nil { - glog.Errorf("error in getting anti-affinity properties of Pod %v", pod.Name) + klog.Errorf("error in getting anti-affinity properties of Pod %v", pod.Name) return false } return podMatchesAnyAffinityTermProperties(targetPod, properties) diff --git a/pkg/scheduler/algorithm/predicates/predicates.go b/pkg/scheduler/algorithm/predicates/predicates.go index 03df6d57519..7594c76405d 100644 --- a/pkg/scheduler/algorithm/predicates/predicates.go +++ b/pkg/scheduler/algorithm/predicates/predicates.go @@ -23,7 +23,7 @@ import ( "regexp" "strconv" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -329,7 +329,7 @@ func NewMaxPDVolumeCountPredicate( filter = AzureDiskVolumeFilter volumeLimitKey = v1.ResourceName(volumeutil.AzureVolumeLimitKey) default: - glog.Fatalf("Wrong filterName, Only Support %v %v %v ", EBSVolumeFilterType, + klog.Fatalf("Wrong filterName, Only Support %v %v %v ", EBSVolumeFilterType, GCEPDVolumeFilterType, AzureDiskVolumeFilterType) return nil @@ -383,9 +383,9 @@ func getMaxEBSVolume(nodeInstanceType string) int { func getMaxVolLimitFromEnv() int { if rawMaxVols := os.Getenv(KubeMaxPDVols); rawMaxVols != "" { if parsedMaxVols, err := strconv.Atoi(rawMaxVols); err != nil { - glog.Errorf("Unable to parse maximum PD volumes value, using default: %v", err) + klog.Errorf("Unable to parse maximum PD volumes value, using default: %v", err) } else if parsedMaxVols <= 0 { - glog.Errorf("Maximum PD volumes must be a positive value, using default ") + klog.Errorf("Maximum PD volumes must be a positive value, using default ") } else { return parsedMaxVols } @@ -413,7 +413,7 @@ func (c *MaxPDVolumeCountChecker) filterVolumes(volumes []v1.Volume, namespace s pvc, err := c.pvcInfo.GetPersistentVolumeClaimInfo(namespace, pvcName) if err != nil || pvc == nil { // if the PVC is not found, log the error and count the PV towards the PV limit - glog.V(4).Infof("Unable to look up PVC info for %s/%s, assuming PVC matches predicate when counting limits: %v", namespace, pvcName, err) + klog.V(4).Infof("Unable to look up PVC info for %s/%s, assuming PVC matches predicate when counting limits: %v", namespace, pvcName, err) filteredVolumes[pvID] = true continue } @@ -424,7 +424,7 @@ func (c *MaxPDVolumeCountChecker) filterVolumes(volumes []v1.Volume, namespace s // it was forcefully unbound by admin. The pod can still use the // original PV where it was bound to -> log the error and count // the PV towards the PV limit - glog.V(4).Infof("PVC %s/%s is not bound, assuming PVC matches predicate when counting limits", namespace, pvcName) + klog.V(4).Infof("PVC %s/%s is not bound, assuming PVC matches predicate when counting limits", namespace, pvcName) filteredVolumes[pvID] = true continue } @@ -433,7 +433,7 @@ func (c *MaxPDVolumeCountChecker) filterVolumes(volumes []v1.Volume, namespace s if err != nil || pv == nil { // if the PV is not found, log the error // and count the PV towards the PV limit - glog.V(4).Infof("Unable to look up PV info for %s/%s/%s, assuming PV matches predicate when counting limits: %v", namespace, pvcName, pvName, err) + klog.V(4).Infof("Unable to look up PV info for %s/%s/%s, assuming PV matches predicate when counting limits: %v", namespace, pvcName, pvName, err) filteredVolumes[pvID] = true continue } @@ -665,12 +665,12 @@ func (c *VolumeZoneChecker) predicate(pod *v1.Pod, meta algorithm.PredicateMetad nodeV, _ := nodeConstraints[k] volumeVSet, err := volumeutil.LabelZonesToSet(v) if err != nil { - glog.Warningf("Failed to parse label for %q: %q. Ignoring the label. err=%v. ", k, v, err) + klog.Warningf("Failed to parse label for %q: %q. Ignoring the label. err=%v. ", k, v, err) continue } if !volumeVSet.Has(nodeV) { - glog.V(10).Infof("Won't schedule pod %q onto node %q due to volume %q (mismatch on %q)", pod.Name, node.Name, pvName, k) + klog.V(10).Infof("Won't schedule pod %q onto node %q due to volume %q (mismatch on %q)", pod.Name, node.Name, pvName, k) return false, []algorithm.PredicateFailureReason{ErrVolumeZoneConflict}, nil } } @@ -781,11 +781,11 @@ func PodFitsResources(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *s } } - if glog.V(10) { + if klog.V(10) { if len(predicateFails) == 0 { - // We explicitly don't do glog.V(10).Infof() to avoid computing all the parameters if this is + // We explicitly don't do klog.V(10).Infof() to avoid computing all the parameters if this is // not logged. There is visible performance gain from it. - glog.Infof("Schedule Pod %+v on Node %+v is allowed, Node is running only %v out of %v Pods.", + klog.Infof("Schedule Pod %+v on Node %+v is allowed, Node is running only %v out of %v Pods.", podName(pod), node.Name, len(nodeInfo.Pods()), allowedPodNumber) } } @@ -834,14 +834,14 @@ func podMatchesNodeSelectorAndAffinityTerms(pod *v1.Pod, node *v1.Node) bool { // TODO: Uncomment this block when implement RequiredDuringSchedulingRequiredDuringExecution. // if nodeAffinity.RequiredDuringSchedulingRequiredDuringExecution != nil { // nodeSelectorTerms := nodeAffinity.RequiredDuringSchedulingRequiredDuringExecution.NodeSelectorTerms - // glog.V(10).Infof("Match for RequiredDuringSchedulingRequiredDuringExecution node selector terms %+v", nodeSelectorTerms) + // klog.V(10).Infof("Match for RequiredDuringSchedulingRequiredDuringExecution node selector terms %+v", nodeSelectorTerms) // nodeAffinityMatches = nodeMatchesNodeSelectorTerms(node, nodeSelectorTerms) // } // Match node selector for requiredDuringSchedulingIgnoredDuringExecution. if nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { nodeSelectorTerms := nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms - glog.V(10).Infof("Match for RequiredDuringSchedulingIgnoredDuringExecution node selector terms %+v", nodeSelectorTerms) + klog.V(10).Infof("Match for RequiredDuringSchedulingIgnoredDuringExecution node selector terms %+v", nodeSelectorTerms) nodeAffinityMatches = nodeAffinityMatches && nodeMatchesNodeSelectorTerms(node, nodeSelectorTerms) } @@ -933,7 +933,7 @@ type ServiceAffinity struct { // only should be referenced by NewServiceAffinityPredicate. func (s *ServiceAffinity) serviceAffinityMetadataProducer(pm *predicateMetadata) { if pm.pod == nil { - glog.Errorf("Cannot precompute service affinity, a pod is required to calculate service affinity.") + klog.Errorf("Cannot precompute service affinity, a pod is required to calculate service affinity.") return } pm.serviceAffinityInUse = true @@ -945,7 +945,7 @@ func (s *ServiceAffinity) serviceAffinityMetadataProducer(pm *predicateMetadata) // In the future maybe we will return them as part of the function. if errSvc != nil || errList != nil { - glog.Errorf("Some Error were found while precomputing svc affinity: \nservices:%v , \npods:%v", errSvc, errList) + klog.Errorf("Some Error were found while precomputing svc affinity: \nservices:%v , \npods:%v", errSvc, errList) } // consider only the pods that belong to the same namespace pm.serviceAffinityMatchingPodList = FilterPodsByNamespace(allMatches, pm.pod.Namespace) @@ -1172,10 +1172,10 @@ func (c *PodAffinityChecker) InterPodAffinityMatches(pod *v1.Pod, meta algorithm return false, failedPredicates, error } - if glog.V(10) { - // We explicitly don't do glog.V(10).Infof() to avoid computing all the parameters if this is + if klog.V(10) { + // We explicitly don't do klog.V(10).Infof() to avoid computing all the parameters if this is // not logged. There is visible performance gain from it. - glog.Infof("Schedule Pod %+v on Node %+v is allowed, pod (anti)affinity constraints satisfied", + klog.Infof("Schedule Pod %+v on Node %+v is allowed, pod (anti)affinity constraints satisfied", podName(pod), node.Name) } return true, nil, nil @@ -1274,7 +1274,7 @@ func (c *PodAffinityChecker) getMatchingAntiAffinityTopologyPairsOfPods(pod *v1. existingPodNode, err := c.info.GetNodeInfo(existingPod.Spec.NodeName) if err != nil { if apierrors.IsNotFound(err) { - glog.Errorf("Node not found, %v", existingPod.Spec.NodeName) + klog.Errorf("Node not found, %v", existingPod.Spec.NodeName) continue } return nil, err @@ -1304,12 +1304,12 @@ func (c *PodAffinityChecker) satisfiesExistingPodsAntiAffinity(pod *v1.Pod, meta filteredPods, err := c.podLister.FilteredList(nodeInfo.Filter, labels.Everything()) if err != nil { errMessage := fmt.Sprintf("Failed to get all pods, %+v", err) - glog.Error(errMessage) + klog.Error(errMessage) return ErrExistingPodsAntiAffinityRulesNotMatch, errors.New(errMessage) } if topologyMaps, err = c.getMatchingAntiAffinityTopologyPairsOfPods(pod, filteredPods); err != nil { errMessage := fmt.Sprintf("Failed to get all terms that pod %+v matches, err: %+v", podName(pod), err) - glog.Error(errMessage) + klog.Error(errMessage) return ErrExistingPodsAntiAffinityRulesNotMatch, errors.New(errMessage) } } @@ -1318,14 +1318,14 @@ func (c *PodAffinityChecker) satisfiesExistingPodsAntiAffinity(pod *v1.Pod, meta // the scheduled pod anti-affinity terms for topologyKey, topologyValue := range node.Labels { if topologyMaps.topologyPairToPods[topologyPair{key: topologyKey, value: topologyValue}] != nil { - glog.V(10).Infof("Cannot schedule pod %+v onto node %v", podName(pod), node.Name) + klog.V(10).Infof("Cannot schedule pod %+v onto node %v", podName(pod), node.Name) return ErrExistingPodsAntiAffinityRulesNotMatch, nil } } - if glog.V(10) { - // We explicitly don't do glog.V(10).Infof() to avoid computing all the parameters if this is + if klog.V(10) { + // We explicitly don't do klog.V(10).Infof() to avoid computing all the parameters if this is // not logged. There is visible performance gain from it. - glog.Infof("Schedule Pod %+v on Node %+v is allowed, existing pods anti-affinity terms satisfied.", + klog.Infof("Schedule Pod %+v on Node %+v is allowed, existing pods anti-affinity terms satisfied.", podName(pod), node.Name) } return nil, nil @@ -1382,7 +1382,7 @@ func (c *PodAffinityChecker) satisfiesPodsAffinityAntiAffinity(pod *v1.Pod, // in the cluster matches the namespace and selector of this pod and the pod matches // its own terms, then we allow the pod to pass the affinity check. if !(len(topologyPairsPotentialAffinityPods.topologyPairToPods) == 0 && targetPodMatchesAffinityOfPod(pod, pod)) { - glog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAffinity", + klog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAffinity", podName(pod), node.Name) return ErrPodAffinityRulesNotMatch, nil } @@ -1394,7 +1394,7 @@ func (c *PodAffinityChecker) satisfiesPodsAffinityAntiAffinity(pod *v1.Pod, if antiAffinityTerms := GetPodAntiAffinityTerms(affinity.PodAntiAffinity); len(antiAffinityTerms) > 0 { matchExists := c.nodeMatchesAnyTopologyTerm(pod, topologyPairsPotentialAntiAffinityPods, nodeInfo, antiAffinityTerms) if matchExists { - glog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAntiAffinity", + klog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAntiAffinity", podName(pod), node.Name) return ErrPodAntiAffinityRulesNotMatch, nil } @@ -1414,7 +1414,7 @@ func (c *PodAffinityChecker) satisfiesPodsAffinityAntiAffinity(pod *v1.Pod, affTermsMatch, termsSelectorMatch, err := c.podMatchesPodAffinityTerms(pod, targetPod, nodeInfo, affinityTerms) if err != nil { errMessage := fmt.Sprintf("Cannot schedule pod %+v onto node %v, because of PodAffinity, err: %v", podName(pod), node.Name, err) - glog.Error(errMessage) + klog.Error(errMessage) return ErrPodAffinityRulesNotMatch, errors.New(errMessage) } if termsSelectorMatch { @@ -1429,7 +1429,7 @@ func (c *PodAffinityChecker) satisfiesPodsAffinityAntiAffinity(pod *v1.Pod, if len(antiAffinityTerms) > 0 { antiAffTermsMatch, _, err := c.podMatchesPodAffinityTerms(pod, targetPod, nodeInfo, antiAffinityTerms) if err != nil || antiAffTermsMatch { - glog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAntiAffinityTerm, err: %v", + klog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAntiAffinityTerm, err: %v", podName(pod), node.Name, err) return ErrPodAntiAffinityRulesNotMatch, nil } @@ -1443,23 +1443,23 @@ func (c *PodAffinityChecker) satisfiesPodsAffinityAntiAffinity(pod *v1.Pod, // in the cluster matches the namespace and selector of this pod and the pod matches // its own terms, then we allow the pod to pass the affinity check. if termsSelectorMatchFound { - glog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAffinity", + klog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAffinity", podName(pod), node.Name) return ErrPodAffinityRulesNotMatch, nil } // Check if pod matches its own affinity properties (namespace and label selector). if !targetPodMatchesAffinityOfPod(pod, pod) { - glog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAffinity", + klog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAffinity", podName(pod), node.Name) return ErrPodAffinityRulesNotMatch, nil } } } - if glog.V(10) { - // We explicitly don't do glog.V(10).Infof() to avoid computing all the parameters if this is + if klog.V(10) { + // We explicitly don't do klog.V(10).Infof() to avoid computing all the parameters if this is // not logged. There is visible performance gain from it. - glog.Infof("Schedule Pod %+v on Node %+v is allowed, pod affinity/anti-affinity constraints satisfied.", + klog.Infof("Schedule Pod %+v on Node %+v is allowed, pod affinity/anti-affinity constraints satisfied.", podName(pod), node.Name) } return nil, nil @@ -1634,12 +1634,12 @@ func (c *VolumeBindingChecker) predicate(pod *v1.Pod, meta algorithm.PredicateMe failReasons := []algorithm.PredicateFailureReason{} if !boundSatisfied { - glog.V(5).Infof("Bound PVs not satisfied for pod %v/%v, node %q", pod.Namespace, pod.Name, node.Name) + klog.V(5).Infof("Bound PVs not satisfied for pod %v/%v, node %q", pod.Namespace, pod.Name, node.Name) failReasons = append(failReasons, ErrVolumeNodeConflict) } if !unboundSatisfied { - glog.V(5).Infof("Couldn't find matching PVs for pod %v/%v, node %q", pod.Namespace, pod.Name, node.Name) + klog.V(5).Infof("Couldn't find matching PVs for pod %v/%v, node %q", pod.Namespace, pod.Name, node.Name) failReasons = append(failReasons, ErrVolumeBindConflict) } @@ -1648,6 +1648,6 @@ func (c *VolumeBindingChecker) predicate(pod *v1.Pod, meta algorithm.PredicateMe } // All volumes bound or matching PVs found for all unbound PVCs - glog.V(5).Infof("All PVCs found matches for pod %v/%v, node %q", pod.Namespace, pod.Name, node.Name) + klog.V(5).Infof("All PVCs found matches for pod %v/%v, node %q", pod.Namespace, pod.Name, node.Name) return true, nil, nil } diff --git a/pkg/scheduler/algorithm/predicates/predicates_test.go b/pkg/scheduler/algorithm/predicates/predicates_test.go index d147ec2d716..5a752901060 100644 --- a/pkg/scheduler/algorithm/predicates/predicates_test.go +++ b/pkg/scheduler/algorithm/predicates/predicates_test.go @@ -30,7 +30,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" + "k8s.io/kubernetes/pkg/features" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/pkg/scheduler/algorithm" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" @@ -4913,10 +4915,7 @@ func TestVolumeZonePredicateWithVolumeBinding(t *testing.T) { }, } - err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - if err != nil { - t.Fatalf("Failed to enable feature gate for VolumeScheduling: %v", err) - } + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -4937,10 +4936,6 @@ func TestVolumeZonePredicateWithVolumeBinding(t *testing.T) { }) } - err = utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - if err != nil { - t.Fatalf("Failed to disable feature gate for VolumeScheduling: %v", err) - } } func TestGetMaxVols(t *testing.T) { diff --git a/pkg/scheduler/algorithm/priorities/BUILD b/pkg/scheduler/algorithm/priorities/BUILD index 2dfe92abdce..e8a08011f35 100644 --- a/pkg/scheduler/algorithm/priorities/BUILD +++ b/pkg/scheduler/algorithm/priorities/BUILD @@ -44,7 +44,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -55,6 +55,7 @@ go_test( "image_locality_test.go", "interpod_affinity_test.go", "least_requested_test.go", + "main_test.go", "metadata_test.go", "most_requested_test.go", "node_affinity_test.go", @@ -79,6 +80,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", ], ) diff --git a/pkg/scheduler/algorithm/priorities/balanced_resource_allocation_test.go b/pkg/scheduler/algorithm/priorities/balanced_resource_allocation_test.go index ae7b601b84a..ccbfe45959b 100644 --- a/pkg/scheduler/algorithm/priorities/balanced_resource_allocation_test.go +++ b/pkg/scheduler/algorithm/priorities/balanced_resource_allocation_test.go @@ -17,7 +17,6 @@ limitations under the License. package priorities import ( - "fmt" "reflect" "testing" @@ -25,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/kubernetes/pkg/features" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" schedulercache "k8s.io/kubernetes/pkg/scheduler/cache" @@ -44,7 +44,7 @@ func getExistingVolumeCountForNode(pods []*v1.Pod, maxVolumes int) int { func TestBalancedResourceAllocation(t *testing.T) { // Enable volumesOnNodeForBalancing to do balanced resource allocation - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.BalanceAttachedNodeVolumes)) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BalanceAttachedNodeVolumes, true)() podwithVol1 := v1.PodSpec{ Containers: []v1.Container{ { diff --git a/pkg/scheduler/algorithm/priorities/interpod_affinity.go b/pkg/scheduler/algorithm/priorities/interpod_affinity.go index 7e640566c82..32cf27c83bf 100644 --- a/pkg/scheduler/algorithm/priorities/interpod_affinity.go +++ b/pkg/scheduler/algorithm/priorities/interpod_affinity.go @@ -30,7 +30,7 @@ import ( schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" schedulercache "k8s.io/kubernetes/pkg/scheduler/cache" - "github.com/golang/glog" + "k8s.io/klog" ) // InterPodAffinity contains information to calculate inter pod affinity. @@ -137,7 +137,7 @@ func (ipa *InterPodAffinity) CalculateInterPodAffinityPriority(pod *v1.Pod, node existingPodNode, err := ipa.info.GetNodeInfo(existingPod.Spec.NodeName) if err != nil { if apierrors.IsNotFound(err) { - glog.Errorf("Node not found, %v", existingPod.Spec.NodeName) + klog.Errorf("Node not found, %v", existingPod.Spec.NodeName) return nil } return err @@ -233,8 +233,8 @@ func (ipa *InterPodAffinity) CalculateInterPodAffinityPriority(pod *v1.Pod, node fScore = float64(schedulerapi.MaxPriority) * ((pm.counts[node.Name] - minCount) / (maxCount - minCount)) } result = append(result, schedulerapi.HostPriority{Host: node.Name, Score: int(fScore)}) - if glog.V(10) { - glog.Infof("%v -> %v: InterPodAffinityPriority, Score: (%d)", pod.Name, node.Name, int(fScore)) + if klog.V(10) { + klog.Infof("%v -> %v: InterPodAffinityPriority, Score: (%d)", pod.Name, node.Name, int(fScore)) } } return result, nil diff --git a/pkg/scheduler/algorithm/priorities/main_test.go b/pkg/scheduler/algorithm/priorities/main_test.go new file mode 100644 index 00000000000..8ca621ecd52 --- /dev/null +++ b/pkg/scheduler/algorithm/priorities/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package priorities + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/scheduler/algorithm/priorities/metadata_test.go b/pkg/scheduler/algorithm/priorities/metadata_test.go index c6a5f20f9e6..a778df6f649 100644 --- a/pkg/scheduler/algorithm/priorities/metadata_test.go +++ b/pkg/scheduler/algorithm/priorities/metadata_test.go @@ -152,14 +152,14 @@ func TestPriorityMetadata(t *testing.T) { name: "Produce a priorityMetadata with specified requests", }, } - mataDataProducer := NewPriorityMetadataFactory( + metaDataProducer := NewPriorityMetadataFactory( schedulertesting.FakeServiceLister([]*v1.Service{}), schedulertesting.FakeControllerLister([]*v1.ReplicationController{}), schedulertesting.FakeReplicaSetLister([]*apps.ReplicaSet{}), schedulertesting.FakeStatefulSetLister([]*apps.StatefulSet{})) for _, test := range tests { t.Run(test.name, func(t *testing.T) { - ptData := mataDataProducer(test.pod, nil) + ptData := metaDataProducer(test.pod, nil) if !reflect.DeepEqual(test.expected, ptData) { t.Errorf("expected %#v, got %#v", test.expected, ptData) } diff --git a/pkg/scheduler/algorithm/priorities/resource_allocation.go b/pkg/scheduler/algorithm/priorities/resource_allocation.go index 8d709dcb44e..027eabae5ff 100644 --- a/pkg/scheduler/algorithm/priorities/resource_allocation.go +++ b/pkg/scheduler/algorithm/priorities/resource_allocation.go @@ -19,9 +19,9 @@ package priorities import ( "fmt" - "github.com/golang/glog" "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" priorityutil "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/util" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" @@ -64,9 +64,9 @@ func (r *ResourceAllocationPriority) PriorityMap( score = r.scorer(&requested, &allocatable, false, 0, 0) } - if glog.V(10) { + if klog.V(10) { if len(pod.Spec.Volumes) >= 0 && utilfeature.DefaultFeatureGate.Enabled(features.BalanceAttachedNodeVolumes) && nodeInfo.TransientInfo != nil { - glog.Infof( + klog.Infof( "%v -> %v: %v, capacity %d millicores %d memory bytes, %d volumes, total request %d millicores %d memory bytes %d volumes, score %d", pod.Name, node.Name, r.Name, allocatable.MilliCPU, allocatable.Memory, nodeInfo.TransientInfo.TransNodeInfo.AllocatableVolumesCount, @@ -75,7 +75,7 @@ func (r *ResourceAllocationPriority) PriorityMap( score, ) } else { - glog.Infof( + klog.Infof( "%v -> %v: %v, capacity %d millicores %d memory bytes, total request %d millicores %d memory bytes, score %d", pod.Name, node.Name, r.Name, allocatable.MilliCPU, allocatable.Memory, diff --git a/pkg/scheduler/algorithm/priorities/resource_limits.go b/pkg/scheduler/algorithm/priorities/resource_limits.go index 816b423f588..82b803cbf72 100644 --- a/pkg/scheduler/algorithm/priorities/resource_limits.go +++ b/pkg/scheduler/algorithm/priorities/resource_limits.go @@ -23,7 +23,7 @@ import ( schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" schedulercache "k8s.io/kubernetes/pkg/scheduler/cache" - "github.com/golang/glog" + "k8s.io/klog" ) // ResourceLimitsPriorityMap is a priority function that increases score of input node by 1 if the node satisfies @@ -52,10 +52,10 @@ func ResourceLimitsPriorityMap(pod *v1.Pod, meta interface{}, nodeInfo *schedule score = 1 } - if glog.V(10) { - // We explicitly don't do glog.V(10).Infof() to avoid computing all the parameters if this is + if klog.V(10) { + // We explicitly don't do klog.V(10).Infof() to avoid computing all the parameters if this is // not logged. There is visible performance gain from it. - glog.Infof( + klog.Infof( "%v -> %v: Resource Limits Priority, allocatable %d millicores %d memory bytes, pod limits %d millicores %d memory bytes, score %d", pod.Name, node.Name, allocatableResources.MilliCPU, allocatableResources.Memory, diff --git a/pkg/scheduler/algorithm/priorities/selector_spreading.go b/pkg/scheduler/algorithm/priorities/selector_spreading.go index 52bb9804412..1371d765a53 100644 --- a/pkg/scheduler/algorithm/priorities/selector_spreading.go +++ b/pkg/scheduler/algorithm/priorities/selector_spreading.go @@ -26,7 +26,7 @@ import ( schedulercache "k8s.io/kubernetes/pkg/scheduler/cache" utilnode "k8s.io/kubernetes/pkg/util/node" - "github.com/golang/glog" + "k8s.io/klog" ) // When zone information is present, give 2/3 of the weighting to zone spreading, 1/3 to node spreading @@ -94,7 +94,7 @@ func (s *SelectorSpread) CalculateSpreadPriorityMap(pod *v1.Pod, meta interface{ // Ignore the previous deleted version for spreading purposes // (it can still be considered for resource restrictions etc.) if nodePod.DeletionTimestamp != nil { - glog.V(4).Infof("skipping pending-deleted pod: %s/%s", nodePod.Namespace, nodePod.Name) + klog.V(4).Infof("skipping pending-deleted pod: %s/%s", nodePod.Namespace, nodePod.Name) continue } for _, selector := range selectors { @@ -160,8 +160,8 @@ func (s *SelectorSpread) CalculateSpreadPriorityReduce(pod *v1.Pod, meta interfa } } result[i].Score = int(fScore) - if glog.V(10) { - glog.Infof( + if klog.V(10) { + klog.Infof( "%v -> %v: SelectorSpreadPriority, Score: (%d)", pod.Name, result[i].Host, int(fScore), ) } diff --git a/pkg/scheduler/algorithm/priorities/selector_spreading_test.go b/pkg/scheduler/algorithm/priorities/selector_spreading_test.go index c0872f42723..954f7900683 100644 --- a/pkg/scheduler/algorithm/priorities/selector_spreading_test.go +++ b/pkg/scheduler/algorithm/priorities/selector_spreading_test.go @@ -347,14 +347,14 @@ func TestSelectorSpreadPriority(t *testing.T) { statefulSetLister: schedulertesting.FakeStatefulSetLister(test.sss), } - mataDataProducer := NewPriorityMetadataFactory( + metaDataProducer := NewPriorityMetadataFactory( schedulertesting.FakeServiceLister(test.services), schedulertesting.FakeControllerLister(test.rcs), schedulertesting.FakeReplicaSetLister(test.rss), schedulertesting.FakeStatefulSetLister(test.sss)) - mataData := mataDataProducer(test.pod, nodeNameToInfo) + metaData := metaDataProducer(test.pod, nodeNameToInfo) - ttp := priorityFunction(selectorSpread.CalculateSpreadPriorityMap, selectorSpread.CalculateSpreadPriorityReduce, mataData) + ttp := priorityFunction(selectorSpread.CalculateSpreadPriorityMap, selectorSpread.CalculateSpreadPriorityReduce, metaData) list, err := ttp(test.pod, nodeNameToInfo, makeNodeList(test.nodes)) if err != nil { t.Errorf("unexpected error: %v \n", err) @@ -583,13 +583,13 @@ func TestZoneSelectorSpreadPriority(t *testing.T) { statefulSetLister: schedulertesting.FakeStatefulSetLister(test.sss), } - mataDataProducer := NewPriorityMetadataFactory( + metaDataProducer := NewPriorityMetadataFactory( schedulertesting.FakeServiceLister(test.services), schedulertesting.FakeControllerLister(test.rcs), schedulertesting.FakeReplicaSetLister(test.rss), schedulertesting.FakeStatefulSetLister(test.sss)) - mataData := mataDataProducer(test.pod, nodeNameToInfo) - ttp := priorityFunction(selectorSpread.CalculateSpreadPriorityMap, selectorSpread.CalculateSpreadPriorityReduce, mataData) + metaData := metaDataProducer(test.pod, nodeNameToInfo) + ttp := priorityFunction(selectorSpread.CalculateSpreadPriorityMap, selectorSpread.CalculateSpreadPriorityReduce, metaData) list, err := ttp(test.pod, nodeNameToInfo, makeLabeledNodeList(labeledNodes)) if err != nil { t.Errorf("unexpected error: %v", err) @@ -760,7 +760,7 @@ func TestZoneSpreadPriority(t *testing.T) { }, } // these local variables just make sure controllerLister\replicaSetLister\statefulSetLister not nil - // when construct mataDataProducer + // when construct metaDataProducer sss := []*apps.StatefulSet{{Spec: apps.StatefulSetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}}}}} rcs := []*v1.ReplicationController{{Spec: v1.ReplicationControllerSpec{Selector: map[string]string{"foo": "bar"}}}} rss := []*apps.ReplicaSet{{Spec: apps.ReplicaSetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}}}}} @@ -770,13 +770,13 @@ func TestZoneSpreadPriority(t *testing.T) { nodeNameToInfo := schedulercache.CreateNodeNameToInfoMap(test.pods, makeLabeledNodeList(test.nodes)) zoneSpread := ServiceAntiAffinity{podLister: schedulertesting.FakePodLister(test.pods), serviceLister: schedulertesting.FakeServiceLister(test.services), label: "zone"} - mataDataProducer := NewPriorityMetadataFactory( + metaDataProducer := NewPriorityMetadataFactory( schedulertesting.FakeServiceLister(test.services), schedulertesting.FakeControllerLister(rcs), schedulertesting.FakeReplicaSetLister(rss), schedulertesting.FakeStatefulSetLister(sss)) - mataData := mataDataProducer(test.pod, nodeNameToInfo) - ttp := priorityFunction(zoneSpread.CalculateAntiAffinityPriorityMap, zoneSpread.CalculateAntiAffinityPriorityReduce, mataData) + metaData := metaDataProducer(test.pod, nodeNameToInfo) + ttp := priorityFunction(zoneSpread.CalculateAntiAffinityPriorityMap, zoneSpread.CalculateAntiAffinityPriorityReduce, metaData) list, err := ttp(test.pod, nodeNameToInfo, makeLabeledNodeList(test.nodes)) if err != nil { t.Errorf("unexpected error: %v", err) diff --git a/pkg/scheduler/algorithm/priorities/test_util.go b/pkg/scheduler/algorithm/priorities/test_util.go index 74ffef34cfb..da85c6b391b 100644 --- a/pkg/scheduler/algorithm/priorities/test_util.go +++ b/pkg/scheduler/algorithm/priorities/test_util.go @@ -41,18 +41,18 @@ func makeNode(node string, milliCPU, memory int64) *v1.Node { } } -func priorityFunction(mapFn algorithm.PriorityMapFunction, reduceFn algorithm.PriorityReduceFunction, mataData interface{}) algorithm.PriorityFunction { +func priorityFunction(mapFn algorithm.PriorityMapFunction, reduceFn algorithm.PriorityReduceFunction, metaData interface{}) algorithm.PriorityFunction { return func(pod *v1.Pod, nodeNameToInfo map[string]*schedulercache.NodeInfo, nodes []*v1.Node) (schedulerapi.HostPriorityList, error) { result := make(schedulerapi.HostPriorityList, 0, len(nodes)) for i := range nodes { - hostResult, err := mapFn(pod, mataData, nodeNameToInfo[nodes[i].Name]) + hostResult, err := mapFn(pod, metaData, nodeNameToInfo[nodes[i].Name]) if err != nil { return nil, err } result = append(result, hostResult) } if reduceFn != nil { - if err := reduceFn(pod, mataData, nodeNameToInfo, result); err != nil { + if err := reduceFn(pod, metaData, nodeNameToInfo, result); err != nil { return nil, err } } diff --git a/pkg/scheduler/algorithm/scheduler_interface.go b/pkg/scheduler/algorithm/scheduler_interface.go index ffa275d45f0..d74af089d35 100644 --- a/pkg/scheduler/algorithm/scheduler_interface.go +++ b/pkg/scheduler/algorithm/scheduler_interface.go @@ -26,6 +26,9 @@ import ( // decisions made by Kubernetes. This is typically needed for resources not directly // managed by Kubernetes. type SchedulerExtender interface { + // Name returns a unique name that identifies the extender. + Name() string + // Filter based on extender-implemented predicate functions. The filtered list is // expected to be a subset of the supplied list. failedNodesMap optionally contains // the list of failed nodes and failure reasons. diff --git a/pkg/scheduler/algorithmprovider/BUILD b/pkg/scheduler/algorithmprovider/BUILD index a4afc8fa475..1e3a118c9f2 100644 --- a/pkg/scheduler/algorithmprovider/BUILD +++ b/pkg/scheduler/algorithmprovider/BUILD @@ -15,11 +15,16 @@ go_library( go_test( name = "go_default_test", - srcs = ["plugins_test.go"], + srcs = [ + "main_test.go", + "plugins_test.go", + ], embed = [":go_default_library"], deps = [ + "//pkg/features:go_default_library", "//pkg/scheduler/factory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/pkg/scheduler/algorithmprovider/defaults/BUILD b/pkg/scheduler/algorithmprovider/defaults/BUILD index 7e624f3737f..ae77ea8609c 100644 --- a/pkg/scheduler/algorithmprovider/defaults/BUILD +++ b/pkg/scheduler/algorithmprovider/defaults/BUILD @@ -19,7 +19,7 @@ go_library( "//pkg/scheduler/factory:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/scheduler/algorithmprovider/defaults/defaults.go b/pkg/scheduler/algorithmprovider/defaults/defaults.go index 6976e659d5a..74467fe17a5 100644 --- a/pkg/scheduler/algorithmprovider/defaults/defaults.go +++ b/pkg/scheduler/algorithmprovider/defaults/defaults.go @@ -17,7 +17,7 @@ limitations under the License. package defaults import ( - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -207,12 +207,12 @@ func ApplyFeatureGates() { factory.InsertPredicateKeyToAlgorithmProviderMap(predicates.PodToleratesNodeTaintsPred) factory.InsertPredicateKeyToAlgorithmProviderMap(predicates.CheckNodeUnschedulablePred) - glog.Infof("TaintNodesByCondition is enabled, PodToleratesNodeTaints predicate is mandatory") + klog.Infof("TaintNodesByCondition is enabled, PodToleratesNodeTaints predicate is mandatory") } // Prioritizes nodes that satisfy pod's resource limits if utilfeature.DefaultFeatureGate.Enabled(features.ResourceLimitsPriorityFunction) { - glog.Infof("Registering resourcelimits priority function") + klog.Infof("Registering resourcelimits priority function") factory.RegisterPriorityFunction2("ResourceLimitsPriority", priorities.ResourceLimitsPriorityMap, nil, 1) // Register the priority function to specific provider too. factory.InsertPriorityKeyToAlgorithmProviderMap(factory.RegisterPriorityFunction2("ResourceLimitsPriority", priorities.ResourceLimitsPriorityMap, nil, 1)) diff --git a/pkg/scheduler/algorithmprovider/main_test.go b/pkg/scheduler/algorithmprovider/main_test.go new file mode 100644 index 00000000000..73e322ffdac --- /dev/null +++ b/pkg/scheduler/algorithmprovider/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package algorithmprovider + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/scheduler/algorithmprovider/plugins_test.go b/pkg/scheduler/algorithmprovider/plugins_test.go index c2f993ea177..958ce7cb60c 100644 --- a/pkg/scheduler/algorithmprovider/plugins_test.go +++ b/pkg/scheduler/algorithmprovider/plugins_test.go @@ -21,6 +21,8 @@ import ( "testing" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/factory" ) @@ -90,7 +92,7 @@ func TestApplyFeatureGates(t *testing.T) { } // Apply features for algorithm providers. - utilfeature.DefaultFeatureGate.Set("TaintNodesByCondition=True") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.TaintNodesByCondition, true)() ApplyFeatureGates() diff --git a/pkg/scheduler/apis/config/v1alpha1/defaults.go b/pkg/scheduler/apis/config/v1alpha1/defaults.go index ea608b6b37f..e212dcf4370 100644 --- a/pkg/scheduler/apis/config/v1alpha1/defaults.go +++ b/pkg/scheduler/apis/config/v1alpha1/defaults.go @@ -62,7 +62,7 @@ func SetDefaults_KubeSchedulerConfiguration(obj *kubescedulerconfigv1alpha1.Kube } obj.HealthzBindAddress = net.JoinHostPort(host, port) } else { - obj.HealthzBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.SchedulerPort)) + obj.HealthzBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.InsecureSchedulerPort)) } if host, port, err := net.SplitHostPort(obj.MetricsBindAddress); err == nil { @@ -71,7 +71,7 @@ func SetDefaults_KubeSchedulerConfiguration(obj *kubescedulerconfigv1alpha1.Kube } obj.MetricsBindAddress = net.JoinHostPort(host, port) } else { - obj.MetricsBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.SchedulerPort)) + obj.MetricsBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.InsecureSchedulerPort)) } if len(obj.LeaderElection.LockObjectNamespace) == 0 { diff --git a/pkg/scheduler/cache/BUILD b/pkg/scheduler/cache/BUILD index 861cfbdbbe0..ba0ac4da9a7 100644 --- a/pkg/scheduler/cache/BUILD +++ b/pkg/scheduler/cache/BUILD @@ -15,7 +15,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/scheduler/cache/node_info.go b/pkg/scheduler/cache/node_info.go index 1af4f647b1f..8b623c72ca3 100644 --- a/pkg/scheduler/cache/node_info.go +++ b/pkg/scheduler/cache/node_info.go @@ -22,7 +22,7 @@ import ( "sync" "sync/atomic" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -529,7 +529,7 @@ func (n *NodeInfo) RemovePod(pod *v1.Pod) error { for i := range n.podsWithAffinity { k2, err := GetPodKey(n.podsWithAffinity[i]) if err != nil { - glog.Errorf("Cannot get pod key, err: %v", err) + klog.Errorf("Cannot get pod key, err: %v", err) continue } if k1 == k2 { @@ -542,7 +542,7 @@ func (n *NodeInfo) RemovePod(pod *v1.Pod) error { for i := range n.pods { k2, err := GetPodKey(n.pods[i]) if err != nil { - glog.Errorf("Cannot get pod key, err: %v", err) + klog.Errorf("Cannot get pod key, err: %v", err) continue } if k1 == k2 { diff --git a/pkg/scheduler/core/BUILD b/pkg/scheduler/core/BUILD index 1415c430274..7df943d3f7e 100644 --- a/pkg/scheduler/core/BUILD +++ b/pkg/scheduler/core/BUILD @@ -30,7 +30,7 @@ go_library( "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/scheduler/core/equivalence/BUILD b/pkg/scheduler/core/equivalence/BUILD index b4595519b80..a6ad12ff724 100644 --- a/pkg/scheduler/core/equivalence/BUILD +++ b/pkg/scheduler/core/equivalence/BUILD @@ -15,7 +15,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/scheduler/core/equivalence/eqivalence.go b/pkg/scheduler/core/equivalence/eqivalence.go index db94c738580..d776981999b 100644 --- a/pkg/scheduler/core/equivalence/eqivalence.go +++ b/pkg/scheduler/core/equivalence/eqivalence.go @@ -23,10 +23,10 @@ import ( "hash/fnv" "sync" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/algorithm" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" @@ -143,7 +143,7 @@ func (c *Cache) predicateKeysToIDs(predicateKeys sets.String) []int { if id, ok := c.predicateIDMap[predicateKey]; ok { predicateIDs = append(predicateIDs, id) } else { - glog.Errorf("predicate key %q not found", predicateKey) + klog.Errorf("predicate key %q not found", predicateKey) } } return predicateIDs @@ -160,7 +160,7 @@ func (c *Cache) InvalidatePredicates(predicateKeys sets.String) { for _, n := range c.nodeToCache { n.invalidatePreds(predicateIDs) } - glog.V(5).Infof("Cache invalidation: node=*,predicates=%v", predicateKeys) + klog.V(5).Infof("Cache invalidation: node=*,predicates=%v", predicateKeys) } @@ -175,7 +175,7 @@ func (c *Cache) InvalidatePredicatesOnNode(nodeName string, predicateKeys sets.S if n, ok := c.nodeToCache[nodeName]; ok { n.invalidatePreds(predicateIDs) } - glog.V(5).Infof("Cache invalidation: node=%s,predicates=%v", nodeName, predicateKeys) + klog.V(5).Infof("Cache invalidation: node=%s,predicates=%v", nodeName, predicateKeys) } // InvalidateAllPredicatesOnNode clears all cached results for one node. @@ -185,7 +185,7 @@ func (c *Cache) InvalidateAllPredicatesOnNode(nodeName string) { if node, ok := c.nodeToCache[nodeName]; ok { node.invalidate() } - glog.V(5).Infof("Cache invalidation: node=%s,predicates=*", nodeName) + klog.V(5).Infof("Cache invalidation: node=%s,predicates=*", nodeName) } // InvalidateCachedPredicateItemForPodAdd is a wrapper of @@ -344,7 +344,7 @@ func (n *NodeCache) updateResult( } n.predicateGenerations[predicateID]++ - glog.V(5).Infof("Cache update: node=%s, predicate=%s,pod=%s,value=%v", + klog.V(5).Infof("Cache update: node=%s, predicate=%s,pod=%s,value=%v", nodeInfo.Node().Name, predicateKey, podName, predicateItem) } diff --git a/pkg/scheduler/core/extender.go b/pkg/scheduler/core/extender.go index 6235f33074e..9053a2e4ea2 100644 --- a/pkg/scheduler/core/extender.go +++ b/pkg/scheduler/core/extender.go @@ -107,6 +107,11 @@ func NewHTTPExtender(config *schedulerapi.ExtenderConfig) (algorithm.SchedulerEx }, nil } +// Name returns extenderURL to identifies the extender. +func (h *HTTPExtender) Name() string { + return h.extenderURL +} + // IsIgnorable returns true indicates scheduling should not fail when this extender // is unavailable func (h *HTTPExtender) IsIgnorable() bool { diff --git a/pkg/scheduler/core/extender_test.go b/pkg/scheduler/core/extender_test.go index c2b93eafe57..e71f8805d67 100644 --- a/pkg/scheduler/core/extender_test.go +++ b/pkg/scheduler/core/extender_test.go @@ -120,6 +120,10 @@ type FakeExtender struct { cachedNodeNameToInfo map[string]*schedulercache.NodeInfo } +func (f *FakeExtender) Name() string { + return "FakeExtender" +} + func (f *FakeExtender) IsIgnorable() bool { return f.ignorable } diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go index 1860098b85f..5f163889c48 100644 --- a/pkg/scheduler/core/generic_scheduler.go +++ b/pkg/scheduler/core/generic_scheduler.go @@ -26,7 +26,7 @@ import ( "sync/atomic" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" @@ -253,7 +253,7 @@ func (g *genericScheduler) Preempt(pod *v1.Pod, nodeLister algorithm.NodeLister, return nil, nil, nil, err } if !podEligibleToPreemptOthers(pod, g.cachedNodeInfoMap) { - glog.V(5).Infof("Pod %v/%v is not eligible for more preemption.", pod.Namespace, pod.Name) + klog.V(5).Infof("Pod %v/%v is not eligible for more preemption.", pod.Namespace, pod.Name) return nil, nil, nil, nil } allNodes, err := nodeLister.List() @@ -265,7 +265,7 @@ func (g *genericScheduler) Preempt(pod *v1.Pod, nodeLister algorithm.NodeLister, } potentialNodes := nodesWherePreemptionMightHelp(allNodes, fitError.FailedPredicates) if len(potentialNodes) == 0 { - glog.V(3).Infof("Preemption will not help schedule pod %v/%v on any node.", pod.Namespace, pod.Name) + klog.V(3).Infof("Preemption will not help schedule pod %v/%v on any node.", pod.Namespace, pod.Name) // In this case, we should clean-up any existing nominated node name of the pod. return nil, nil, []*v1.Pod{pod}, nil } @@ -321,7 +321,7 @@ func (g *genericScheduler) processPreemptionWithExtenders( ) if err != nil { if extender.IsIgnorable() { - glog.Warningf("Skipping extender %v as it returned error %v and has ignorable flag set", + klog.Warningf("Skipping extender %v as it returned error %v and has ignorable flag set", extender, err) continue } @@ -468,7 +468,7 @@ func (g *genericScheduler) findNodesThatFit(pod *v1.Pod, nodes []*v1.Node) ([]*v filteredList, failedMap, err := extender.Filter(pod, filtered, g.cachedNodeInfoMap) if err != nil { if extender.IsIgnorable() { - glog.Warningf("Skipping extender %v as it returned error %v and has ignorable flag set", + klog.Warningf("Skipping extender %v as it returned error %v and has ignorable flag set", extender, err) continue } else { @@ -494,7 +494,7 @@ func (g *genericScheduler) findNodesThatFit(pod *v1.Pod, nodes []*v1.Node) ([]*v // addNominatedPods adds pods with equal or greater priority which are nominated // to run on the node given in nodeInfo to meta and nodeInfo. It returns 1) whether // any pod was found, 2) augmented meta data, 3) augmented nodeInfo. -func addNominatedPods(podPriority int32, meta algorithm.PredicateMetadata, +func addNominatedPods(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *schedulercache.NodeInfo, queue internalqueue.SchedulingQueue) (bool, algorithm.PredicateMetadata, *schedulercache.NodeInfo) { if queue == nil || nodeInfo == nil || nodeInfo.Node() == nil { @@ -511,7 +511,7 @@ func addNominatedPods(podPriority int32, meta algorithm.PredicateMetadata, } nodeInfoOut := nodeInfo.Clone() for _, p := range nominatedPods { - if util.GetPodPriority(p) >= podPriority { + if util.GetPodPriority(p) >= util.GetPodPriority(pod) && p.UID != pod.UID { nodeInfoOut.AddPod(p) if metaOut != nil { metaOut.AddPod(p, nodeInfoOut) @@ -569,7 +569,7 @@ func podFitsOnNode( metaToUse := meta nodeInfoToUse := info if i == 0 { - podsAdded, metaToUse, nodeInfoToUse = addNominatedPods(util.GetPodPriority(pod), meta, info, queue) + podsAdded, metaToUse, nodeInfoToUse = addNominatedPods(pod, meta, info, queue) } else if !podsAdded || len(failedPredicates) != 0 { break } @@ -599,7 +599,7 @@ func podFitsOnNode( failedPredicates = append(failedPredicates, reasons...) // if alwaysCheckAllPredicates is false, short circuit all predicates when one predicate fails. if !alwaysCheckAllPredicates { - glog.V(5).Infoln("since alwaysCheckAllPredicates has not been set, the predicate " + + klog.V(5).Infoln("since alwaysCheckAllPredicates has not been set, the predicate " + "evaluation is short circuited and there are chances " + "of other predicates failing as well.") break @@ -653,38 +653,38 @@ func PrioritizeNodes( results := make([]schedulerapi.HostPriorityList, len(priorityConfigs), len(priorityConfigs)) - for i, priorityConfig := range priorityConfigs { - if priorityConfig.Function != nil { - // DEPRECATED - wg.Add(1) - go func(index int, config algorithm.PriorityConfig) { - defer wg.Done() - var err error - results[index], err = config.Function(pod, nodeNameToInfo, nodes) - if err != nil { - appendError(err) - } - }(i, priorityConfig) - } else { + // DEPRECATED: we can remove this when all priorityConfigs implement the + // Map-Reduce pattern. + workqueue.ParallelizeUntil(context.TODO(), 16, len(priorityConfigs), func(i int) { + priorityConfig := priorityConfigs[i] + if priorityConfig.Function == nil { results[i] = make(schedulerapi.HostPriorityList, len(nodes)) + return } - } - processNode := func(index int) { - nodeInfo := nodeNameToInfo[nodes[index].Name] var err error - for i := range priorityConfigs { - if priorityConfigs[i].Function != nil { + results[i], err = priorityConfig.Function(pod, nodeNameToInfo, nodes) + if err != nil { + appendError(err) + } + }) + + workqueue.ParallelizeUntil(context.TODO(), 16, len(nodes), func(index int) { + nodeInfo := nodeNameToInfo[nodes[index].Name] + for i, priorityConfig := range priorityConfigs { + if priorityConfig.Function != nil { continue } + + var err error results[i][index], err = priorityConfigs[i].Map(pod, meta, nodeInfo) if err != nil { appendError(err) results[i][index].Host = nodes[index].Name } } - } - workqueue.ParallelizeUntil(context.TODO(), 16, len(nodes), processNode) + }) + for i, priorityConfig := range priorityConfigs { if priorityConfig.Reduce == nil { continue @@ -695,9 +695,9 @@ func PrioritizeNodes( if err := config.Reduce(pod, meta, nodeNameToInfo, results[index]); err != nil { appendError(err) } - if glog.V(10) { + if klog.V(10) { for _, hostPriority := range results[index] { - glog.Infof("%v -> %v: %v, Score: (%d)", pod.Name, hostPriority.Host, config.Name, hostPriority.Score) + klog.Infof("%v -> %v: %v, Score: (%d)", util.GetPodFullName(pod), hostPriority.Host, config.Name, hostPriority.Score) } } }(i, priorityConfig) @@ -735,6 +735,9 @@ func PrioritizeNodes( mu.Lock() for i := range *prioritizedList { host, score := (*prioritizedList)[i].Host, (*prioritizedList)[i].Score + if klog.V(10) { + klog.Infof("%v -> %v: %v, Score: (%d)", util.GetPodFullName(pod), host, ext.Name(), score) + } combinedScores[host] += score * weight } mu.Unlock() @@ -747,9 +750,9 @@ func PrioritizeNodes( } } - if glog.V(10) { + if klog.V(10) { for i := range result { - glog.Infof("Host %s => Score %d", result[i].Host, result[i].Score) + klog.Infof("Host %s => Score %d", result[i].Host, result[i].Score) } } return result, nil @@ -878,7 +881,7 @@ func pickOneNodeForPreemption(nodesToVictims map[*v1.Node]*schedulerapi.Victims) if lenNodes2 > 0 { return minNodes2[0] } - glog.Errorf("Error in logic of node scoring for preemption. We should never reach here!") + klog.Errorf("Error in logic of node scoring for preemption. We should never reach here!") return nil } @@ -892,7 +895,6 @@ func selectNodesForPreemption(pod *v1.Pod, queue internalqueue.SchedulingQueue, pdbs []*policy.PodDisruptionBudget, ) (map[*v1.Node]*schedulerapi.Victims, error) { - nodeToVictims := map[*v1.Node]*schedulerapi.Victims{} var resultLock sync.Mutex @@ -981,6 +983,9 @@ func selectVictimsOnNode( queue internalqueue.SchedulingQueue, pdbs []*policy.PodDisruptionBudget, ) ([]*v1.Pod, int, bool) { + if nodeInfo == nil { + return nil, 0, false + } potentialVictims := util.SortableList{CompFunc: util.HigherPriorityPod} nodeInfoCopy := nodeInfo.Clone() @@ -1013,7 +1018,7 @@ func selectVictimsOnNode( // TODO(bsalamat): Consider checking affinity to lower priority pods if feasible with reasonable performance. if fits, _, err := podFitsOnNode(pod, meta, nodeInfoCopy, fitPredicates, nil, queue, false, nil); !fits { if err != nil { - glog.Warningf("Encountered error while selecting victims on node %v: %v", nodeInfo.Node().Name, err) + klog.Warningf("Encountered error while selecting victims on node %v: %v", nodeInfo.Node().Name, err) } return nil, 0, false } @@ -1029,7 +1034,7 @@ func selectVictimsOnNode( if !fits { removePod(p) victims = append(victims, p) - glog.V(5).Infof("Pod %v is a potential preemption victim on node %v.", p.Name, nodeInfo.Node().Name) + klog.V(5).Infof("Pod %v is a potential preemption victim on node %v.", p.Name, nodeInfo.Node().Name) } return fits } @@ -1084,7 +1089,7 @@ func nodesWherePreemptionMightHelp(nodes []*v1.Node, failedPredicatesMap FailedP } } if !found || !unresolvableReasonExist { - glog.V(3).Infof("Node %v is a potential node for preemption.", node.Name) + klog.V(3).Infof("Node %v is a potential node for preemption.", node.Name) potentialNodes = append(potentialNodes, node) } } diff --git a/pkg/scheduler/core/generic_scheduler_test.go b/pkg/scheduler/core/generic_scheduler_test.go index eae28ac6e95..d0152a9fbb6 100644 --- a/pkg/scheduler/core/generic_scheduler_test.go +++ b/pkg/scheduler/core/generic_scheduler_test.go @@ -706,15 +706,15 @@ func TestZeroRequest(t *testing.T) { nodeNameToInfo := schedulercache.CreateNodeNameToInfoMap(test.pods, test.nodes) - mataDataProducer := algorithmpriorities.NewPriorityMetadataFactory( + metaDataProducer := algorithmpriorities.NewPriorityMetadataFactory( schedulertesting.FakeServiceLister([]*v1.Service{}), schedulertesting.FakeControllerLister([]*v1.ReplicationController{}), schedulertesting.FakeReplicaSetLister([]*apps.ReplicaSet{}), schedulertesting.FakeStatefulSetLister([]*apps.StatefulSet{})) - mataData := mataDataProducer(test.pod, nodeNameToInfo) + metaData := metaDataProducer(test.pod, nodeNameToInfo) list, err := PrioritizeNodes( - test.pod, nodeNameToInfo, mataData, priorityConfigs, + test.pod, nodeNameToInfo, metaData, priorityConfigs, schedulertesting.FakeNodeLister(test.nodes), []algorithm.SchedulerExtender{}) if err != nil { t.Errorf("unexpected error: %v", err) @@ -960,6 +960,11 @@ func TestSelectNodesForPreemption(t *testing.T) { test.predicates[algorithmpredicates.MatchInterPodAffinityPred] = algorithmpredicates.NewPodAffinityPredicate(FakeNodeInfo(*nodes[0]), schedulertesting.FakePodLister(test.pods)) } nodeNameToInfo := schedulercache.CreateNodeNameToInfoMap(test.pods, nodes) + // newnode simulate a case that a new node is added to the cluster, but nodeNameToInfo + // doesn't have it yet. + newnode := makeNode("newnode", 1000*5, priorityutil.DefaultMemoryRequest*5) + newnode.ObjectMeta.Labels = map[string]string{"hostname": "newnode"} + nodes = append(nodes, newnode) nodeToPods, err := selectNodesForPreemption(test.pod, nodeNameToInfo, nodes, test.predicates, PredicateMetadata, nil, nil) if err != nil { t.Error(err) diff --git a/pkg/scheduler/factory/BUILD b/pkg/scheduler/factory/BUILD index dfade874b3d..7be893b0606 100644 --- a/pkg/scheduler/factory/BUILD +++ b/pkg/scheduler/factory/BUILD @@ -23,7 +23,7 @@ go_library( "//pkg/scheduler/core:go_default_library", "//pkg/scheduler/core/equivalence:go_default_library", "//pkg/scheduler/internal/cache:go_default_library", - "//pkg/scheduler/internal/cache/comparer:go_default_library", + "//pkg/scheduler/internal/cache/debugger:go_default_library", "//pkg/scheduler/internal/queue:go_default_library", "//pkg/scheduler/util:go_default_library", "//pkg/scheduler/volumebinder:go_default_library", @@ -37,6 +37,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers/apps/v1:go_default_library", "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", @@ -49,7 +50,7 @@ go_library( "//staging/src/k8s.io/client-go/listers/storage/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -62,7 +63,6 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/api/testing:go_default_library", - "//pkg/apis/core:go_default_library", "//pkg/scheduler/algorithm:go_default_library", "//pkg/scheduler/algorithm/priorities:go_default_library", "//pkg/scheduler/api:go_default_library", @@ -73,7 +73,6 @@ go_test( "//pkg/scheduler/testing:go_default_library", "//pkg/scheduler/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/pkg/scheduler/factory/factory.go b/pkg/scheduler/factory/factory.go index 2d0801480f9..aa420bcb173 100644 --- a/pkg/scheduler/factory/factory.go +++ b/pkg/scheduler/factory/factory.go @@ -25,7 +25,7 @@ import ( "reflect" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -37,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" appsinformers "k8s.io/client-go/informers/apps/v1" coreinformers "k8s.io/client-go/informers/core/v1" @@ -60,7 +61,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/core" "k8s.io/kubernetes/pkg/scheduler/core/equivalence" schedulerinternalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" - cachecomparer "k8s.io/kubernetes/pkg/scheduler/internal/cache/comparer" + cachedebugger "k8s.io/kubernetes/pkg/scheduler/internal/cache/debugger" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" "k8s.io/kubernetes/pkg/scheduler/util" "k8s.io/kubernetes/pkg/scheduler/volumebinder" @@ -127,13 +128,16 @@ type Config struct { Recorder record.EventRecorder // Close this to shut down the scheduler. - StopEverything chan struct{} + StopEverything <-chan struct{} // VolumeBinder handles PVC/PV binding for the pod. VolumeBinder *volumebinder.VolumeBinder // Disable pod preemption or not. DisablePreemption bool + + // SchedulingQueue holds pods to be scheduled + SchedulingQueue internalqueue.SchedulingQueue } // PodPreemptor has methods needed to delete a pod and to update @@ -200,7 +204,7 @@ type configFactory struct { storageClassLister storagelisters.StorageClassLister // Close this to stop all reflectors - StopEverything chan struct{} + StopEverything <-chan struct{} scheduledPodsHasSynced cache.InformerSynced @@ -253,12 +257,16 @@ type ConfigFactoryArgs struct { DisablePreemption bool PercentageOfNodesToScore int32 BindTimeoutSeconds int64 + StopCh <-chan struct{} } // NewConfigFactory initializes the default implementation of a Configurator. To encourage eventual privatization of the struct type, we only // return the interface. func NewConfigFactory(args *ConfigFactoryArgs) Configurator { - stopEverything := make(chan struct{}) + stopEverything := args.StopCh + if stopEverything == nil { + stopEverything = wait.NeverStop + } schedulerCache := schedulerinternalcache.New(30*time.Second, stopEverything) // storageClassInformer is only enabled through VolumeScheduling feature gate @@ -398,7 +406,7 @@ func NewConfigFactory(args *ConfigFactoryArgs) Configurator { } // Setup cache comparer - comparer := cachecomparer.New( + debugger := cachedebugger.New( args.NodeInformer.Lister(), args.PodInformer.Lister(), c.schedulerCache, @@ -415,7 +423,8 @@ func NewConfigFactory(args *ConfigFactoryArgs) Configurator { c.podQueue.Close() return case <-ch: - comparer.Compare() + debugger.Comparer.Compare() + debugger.Dumper.DumpAll() } } }() @@ -466,7 +475,7 @@ func (c *configFactory) skipPodUpdate(pod *v1.Pod) bool { if !reflect.DeepEqual(assumedPodCopy, podCopy) { return false } - glog.V(3).Infof("Skipping pod %s/%s update", pod.Namespace, pod.Name) + klog.V(3).Infof("Skipping pod %s/%s update", pod.Namespace, pod.Name) return true } @@ -474,7 +483,7 @@ func (c *configFactory) onPvAdd(obj interface{}) { if c.enableEquivalenceClassCache { pv, ok := obj.(*v1.PersistentVolume) if !ok { - glog.Errorf("cannot convert to *v1.PersistentVolume: %v", obj) + klog.Errorf("cannot convert to *v1.PersistentVolume: %v", obj) return } c.invalidatePredicatesForPv(pv) @@ -492,12 +501,12 @@ func (c *configFactory) onPvUpdate(old, new interface{}) { if c.enableEquivalenceClassCache { newPV, ok := new.(*v1.PersistentVolume) if !ok { - glog.Errorf("cannot convert to *v1.PersistentVolume: %v", new) + klog.Errorf("cannot convert to *v1.PersistentVolume: %v", new) return } oldPV, ok := old.(*v1.PersistentVolume) if !ok { - glog.Errorf("cannot convert to *v1.PersistentVolume: %v", old) + klog.Errorf("cannot convert to *v1.PersistentVolume: %v", old) return } c.invalidatePredicatesForPvUpdate(oldPV, newPV) @@ -543,11 +552,11 @@ func (c *configFactory) onPvDelete(obj interface{}) { var ok bool pv, ok = t.Obj.(*v1.PersistentVolume) if !ok { - glog.Errorf("cannot convert to *v1.PersistentVolume: %v", t.Obj) + klog.Errorf("cannot convert to *v1.PersistentVolume: %v", t.Obj) return } default: - glog.Errorf("cannot convert to *v1.PersistentVolume: %v", t) + klog.Errorf("cannot convert to *v1.PersistentVolume: %v", t) return } c.invalidatePredicatesForPv(pv) @@ -594,7 +603,7 @@ func (c *configFactory) onPvcAdd(obj interface{}) { if c.enableEquivalenceClassCache { pvc, ok := obj.(*v1.PersistentVolumeClaim) if !ok { - glog.Errorf("cannot convert to *v1.PersistentVolumeClaim: %v", obj) + klog.Errorf("cannot convert to *v1.PersistentVolumeClaim: %v", obj) return } c.invalidatePredicatesForPvc(pvc) @@ -610,12 +619,12 @@ func (c *configFactory) onPvcUpdate(old, new interface{}) { if c.enableEquivalenceClassCache { newPVC, ok := new.(*v1.PersistentVolumeClaim) if !ok { - glog.Errorf("cannot convert to *v1.PersistentVolumeClaim: %v", new) + klog.Errorf("cannot convert to *v1.PersistentVolumeClaim: %v", new) return } oldPVC, ok := old.(*v1.PersistentVolumeClaim) if !ok { - glog.Errorf("cannot convert to *v1.PersistentVolumeClaim: %v", old) + klog.Errorf("cannot convert to *v1.PersistentVolumeClaim: %v", old) return } c.invalidatePredicatesForPvcUpdate(oldPVC, newPVC) @@ -633,11 +642,11 @@ func (c *configFactory) onPvcDelete(obj interface{}) { var ok bool pvc, ok = t.Obj.(*v1.PersistentVolumeClaim) if !ok { - glog.Errorf("cannot convert to *v1.PersistentVolumeClaim: %v", t.Obj) + klog.Errorf("cannot convert to *v1.PersistentVolumeClaim: %v", t.Obj) return } default: - glog.Errorf("cannot convert to *v1.PersistentVolumeClaim: %v", t) + klog.Errorf("cannot convert to *v1.PersistentVolumeClaim: %v", t) return } c.invalidatePredicatesForPvc(pvc) @@ -686,7 +695,7 @@ func (c *configFactory) invalidatePredicatesForPvcUpdate(old, new *v1.Persistent func (c *configFactory) onStorageClassAdd(obj interface{}) { sc, ok := obj.(*storagev1.StorageClass) if !ok { - glog.Errorf("cannot convert to *storagev1.StorageClass: %v", obj) + klog.Errorf("cannot convert to *storagev1.StorageClass: %v", obj) return } @@ -711,11 +720,11 @@ func (c *configFactory) onStorageClassDelete(obj interface{}) { var ok bool sc, ok = t.Obj.(*storagev1.StorageClass) if !ok { - glog.Errorf("cannot convert to *storagev1.StorageClass: %v", t.Obj) + klog.Errorf("cannot convert to *storagev1.StorageClass: %v", t.Obj) return } default: - glog.Errorf("cannot convert to *storagev1.StorageClass: %v", t) + klog.Errorf("cannot convert to *storagev1.StorageClass: %v", t) return } c.invalidatePredicatesForStorageClass(sc) @@ -788,12 +797,12 @@ func (c *configFactory) GetScheduledPodLister() corelisters.PodLister { func (c *configFactory) addPodToCache(obj interface{}) { pod, ok := obj.(*v1.Pod) if !ok { - glog.Errorf("cannot convert to *v1.Pod: %v", obj) + klog.Errorf("cannot convert to *v1.Pod: %v", obj) return } if err := c.schedulerCache.AddPod(pod); err != nil { - glog.Errorf("scheduler cache AddPod failed: %v", err) + klog.Errorf("scheduler cache AddPod failed: %v", err) } c.podQueue.AssignedPodAdded(pod) @@ -805,12 +814,12 @@ func (c *configFactory) addPodToCache(obj interface{}) { func (c *configFactory) updatePodInCache(oldObj, newObj interface{}) { oldPod, ok := oldObj.(*v1.Pod) if !ok { - glog.Errorf("cannot convert oldObj to *v1.Pod: %v", oldObj) + klog.Errorf("cannot convert oldObj to *v1.Pod: %v", oldObj) return } newPod, ok := newObj.(*v1.Pod) if !ok { - glog.Errorf("cannot convert newObj to *v1.Pod: %v", newObj) + klog.Errorf("cannot convert newObj to *v1.Pod: %v", newObj) return } @@ -820,7 +829,7 @@ func (c *configFactory) updatePodInCache(oldObj, newObj interface{}) { // snapshotted before updates are written, we would update equivalence // cache with stale information which is based on snapshot of old cache. if err := c.schedulerCache.UpdatePod(oldPod, newPod); err != nil { - glog.Errorf("scheduler cache UpdatePod failed: %v", err) + klog.Errorf("scheduler cache UpdatePod failed: %v", err) } c.invalidateCachedPredicatesOnUpdatePod(newPod, oldPod) @@ -898,11 +907,11 @@ func (c *configFactory) deletePodFromCache(obj interface{}) { var ok bool pod, ok = t.Obj.(*v1.Pod) if !ok { - glog.Errorf("cannot convert to *v1.Pod: %v", t.Obj) + klog.Errorf("cannot convert to *v1.Pod: %v", t.Obj) return } default: - glog.Errorf("cannot convert to *v1.Pod: %v", t) + klog.Errorf("cannot convert to *v1.Pod: %v", t) return } // NOTE: Updates must be written to scheduler cache before invalidating @@ -911,7 +920,7 @@ func (c *configFactory) deletePodFromCache(obj interface{}) { // snapshotted before updates are written, we would update equivalence // cache with stale information which is based on snapshot of old cache. if err := c.schedulerCache.RemovePod(pod); err != nil { - glog.Errorf("scheduler cache RemovePod failed: %v", err) + klog.Errorf("scheduler cache RemovePod failed: %v", err) } c.invalidateCachedPredicatesOnDeletePod(pod) @@ -942,7 +951,7 @@ func (c *configFactory) invalidateCachedPredicatesOnDeletePod(pod *v1.Pod) { func (c *configFactory) addNodeToCache(obj interface{}) { node, ok := obj.(*v1.Node) if !ok { - glog.Errorf("cannot convert to *v1.Node: %v", obj) + klog.Errorf("cannot convert to *v1.Node: %v", obj) return } @@ -954,7 +963,7 @@ func (c *configFactory) addNodeToCache(obj interface{}) { } if err := c.schedulerCache.AddNode(node); err != nil { - glog.Errorf("scheduler cache AddNode failed: %v", err) + klog.Errorf("scheduler cache AddNode failed: %v", err) } c.podQueue.MoveAllToActiveQueue() @@ -964,12 +973,12 @@ func (c *configFactory) addNodeToCache(obj interface{}) { func (c *configFactory) updateNodeInCache(oldObj, newObj interface{}) { oldNode, ok := oldObj.(*v1.Node) if !ok { - glog.Errorf("cannot convert oldObj to *v1.Node: %v", oldObj) + klog.Errorf("cannot convert oldObj to *v1.Node: %v", oldObj) return } newNode, ok := newObj.(*v1.Node) if !ok { - glog.Errorf("cannot convert newObj to *v1.Node: %v", newObj) + klog.Errorf("cannot convert newObj to *v1.Node: %v", newObj) return } @@ -979,14 +988,11 @@ func (c *configFactory) updateNodeInCache(oldObj, newObj interface{}) { // snapshotted before updates are written, we would update equivalence // cache with stale information which is based on snapshot of old cache. if err := c.schedulerCache.UpdateNode(oldNode, newNode); err != nil { - glog.Errorf("scheduler cache UpdateNode failed: %v", err) + klog.Errorf("scheduler cache UpdateNode failed: %v", err) } c.invalidateCachedPredicatesOnNodeUpdate(newNode, oldNode) - // Only activate unschedulable pods if the node became more schedulable. - if nodeSchedulingPropertiesChanged(newNode, oldNode) { - c.podQueue.MoveAllToActiveQueue() - } + c.podQueue.MoveAllToActiveQueue() } func (c *configFactory) invalidateCachedPredicatesOnNodeUpdate(newNode *v1.Node, oldNode *v1.Node) { @@ -1016,11 +1022,11 @@ func (c *configFactory) invalidateCachedPredicatesOnNodeUpdate(newNode *v1.Node, oldTaints, oldErr := helper.GetTaintsFromNodeAnnotations(oldNode.GetAnnotations()) if oldErr != nil { - glog.Errorf("Failed to get taints from old node annotation for equivalence cache") + klog.Errorf("Failed to get taints from old node annotation for equivalence cache") } newTaints, newErr := helper.GetTaintsFromNodeAnnotations(newNode.GetAnnotations()) if newErr != nil { - glog.Errorf("Failed to get taints from new node annotation for equivalence cache") + klog.Errorf("Failed to get taints from new node annotation for equivalence cache") } if !reflect.DeepEqual(oldTaints, newTaints) || !reflect.DeepEqual(oldNode.Spec.Taints, newNode.Spec.Taints) { @@ -1058,64 +1064,6 @@ func (c *configFactory) invalidateCachedPredicatesOnNodeUpdate(newNode *v1.Node, } } -func nodeSchedulingPropertiesChanged(newNode *v1.Node, oldNode *v1.Node) bool { - if nodeAllocatableChanged(newNode, oldNode) { - glog.V(4).Infof("Allocatable resource of node %s changed", newNode.Name) - return true - } - if nodeLabelsChanged(newNode, oldNode) { - glog.V(4).Infof("Labels of node %s changed", newNode.Name) - return true - } - if nodeTaintsChanged(newNode, oldNode) { - glog.V(4).Infof("Taints of node %s changed", newNode.Name) - return true - } - if nodeConditionsChanged(newNode, oldNode) { - glog.V(4).Infof("Conditions of node %s changed", newNode.Name) - return true - } - if newNode.Spec.Unschedulable != oldNode.Spec.Unschedulable && newNode.Spec.Unschedulable == false { - glog.V(4).Infof("Node %s changed to schedulable", newNode.Name) - return true - } - return false -} - -func nodeAllocatableChanged(newNode *v1.Node, oldNode *v1.Node) bool { - return !reflect.DeepEqual(oldNode.Status.Allocatable, newNode.Status.Allocatable) -} - -func nodeLabelsChanged(newNode *v1.Node, oldNode *v1.Node) bool { - return !reflect.DeepEqual(oldNode.GetLabels(), newNode.GetLabels()) -} - -func nodeTaintsChanged(newNode *v1.Node, oldNode *v1.Node) bool { - if !reflect.DeepEqual(newNode.Spec.Taints, oldNode.Spec.Taints) { - return true - } - oldTaints, oldErr := helper.GetTaintsFromNodeAnnotations(oldNode.GetAnnotations()) - if oldErr != nil { - // If parse old node's taint annotation failed, we assume node's taint changed. - glog.Errorf("Failed to get taints from annotation of old node %s: %v", oldNode.Name, oldErr) - return true - } - newTaints, newErr := helper.GetTaintsFromNodeAnnotations(newNode.GetAnnotations()) - if newErr != nil { - // If parse new node's taint annotation failed, we assume node's taint changed. - glog.Errorf("Failed to get taints from annotation of new node %s: %v", newNode.Name, newErr) - return true - } - if !reflect.DeepEqual(oldTaints, newTaints) { - return true - } - return false -} - -func nodeConditionsChanged(newNode *v1.Node, oldNode *v1.Node) bool { - return !reflect.DeepEqual(oldNode.Status.Conditions, newNode.Status.Conditions) -} - func (c *configFactory) deleteNodeFromCache(obj interface{}) { var node *v1.Node switch t := obj.(type) { @@ -1125,11 +1073,11 @@ func (c *configFactory) deleteNodeFromCache(obj interface{}) { var ok bool node, ok = t.Obj.(*v1.Node) if !ok { - glog.Errorf("cannot convert to *v1.Node: %v", t.Obj) + klog.Errorf("cannot convert to *v1.Node: %v", t.Obj) return } default: - glog.Errorf("cannot convert to *v1.Node: %v", t) + klog.Errorf("cannot convert to *v1.Node: %v", t) return } // NOTE: Updates must be written to scheduler cache before invalidating @@ -1138,7 +1086,7 @@ func (c *configFactory) deleteNodeFromCache(obj interface{}) { // snapshotted before updates are written, we would update equivalence // cache with stale information which is based on snapshot of old cache. if err := c.schedulerCache.RemoveNode(node); err != nil { - glog.Errorf("scheduler cache RemoveNode failed: %v", err) + klog.Errorf("scheduler cache RemoveNode failed: %v", err) } if c.enableEquivalenceClassCache { c.equivalencePodCache.InvalidateAllPredicatesOnNode(node.GetName()) @@ -1152,7 +1100,7 @@ func (c *configFactory) Create() (*Config, error) { // Creates a scheduler from the name of a registered algorithm provider. func (c *configFactory) CreateFromProvider(providerName string) (*Config, error) { - glog.V(2).Infof("Creating scheduler from algorithm provider '%v'", providerName) + klog.V(2).Infof("Creating scheduler from algorithm provider '%v'", providerName) provider, err := GetAlgorithmProvider(providerName) if err != nil { return nil, err @@ -1162,7 +1110,7 @@ func (c *configFactory) CreateFromProvider(providerName string) (*Config, error) // Creates a scheduler from the configuration file func (c *configFactory) CreateFromConfig(policy schedulerapi.Policy) (*Config, error) { - glog.V(2).Infof("Creating scheduler from configuration: %v", policy) + klog.V(2).Infof("Creating scheduler from configuration: %v", policy) // validate the policy configuration if err := validation.ValidatePolicy(policy); err != nil { @@ -1171,7 +1119,7 @@ func (c *configFactory) CreateFromConfig(policy schedulerapi.Policy) (*Config, e predicateKeys := sets.NewString() if policy.Predicates == nil { - glog.V(2).Infof("Using predicates from algorithm provider '%v'", DefaultProvider) + klog.V(2).Infof("Using predicates from algorithm provider '%v'", DefaultProvider) provider, err := GetAlgorithmProvider(DefaultProvider) if err != nil { return nil, err @@ -1179,14 +1127,14 @@ func (c *configFactory) CreateFromConfig(policy schedulerapi.Policy) (*Config, e predicateKeys = provider.FitPredicateKeys } else { for _, predicate := range policy.Predicates { - glog.V(2).Infof("Registering predicate: %s", predicate.Name) + klog.V(2).Infof("Registering predicate: %s", predicate.Name) predicateKeys.Insert(RegisterCustomFitPredicate(predicate)) } } priorityKeys := sets.NewString() if policy.Priorities == nil { - glog.V(2).Infof("Using priorities from algorithm provider '%v'", DefaultProvider) + klog.V(2).Infof("Using priorities from algorithm provider '%v'", DefaultProvider) provider, err := GetAlgorithmProvider(DefaultProvider) if err != nil { return nil, err @@ -1194,7 +1142,7 @@ func (c *configFactory) CreateFromConfig(policy schedulerapi.Policy) (*Config, e priorityKeys = provider.PriorityFunctionKeys } else { for _, priority := range policy.Priorities { - glog.V(2).Infof("Registering priority: %s", priority.Name) + klog.V(2).Infof("Registering priority: %s", priority.Name) priorityKeys.Insert(RegisterCustomPriorityFunction(priority)) } } @@ -1203,7 +1151,7 @@ func (c *configFactory) CreateFromConfig(policy schedulerapi.Policy) (*Config, e if len(policy.ExtenderConfigs) != 0 { ignoredExtendedResources := sets.NewString() for ii := range policy.ExtenderConfigs { - glog.V(2).Infof("Creating extender with config %+v", policy.ExtenderConfigs[ii]) + klog.V(2).Infof("Creating extender with config %+v", policy.ExtenderConfigs[ii]) extender, err := core.NewHTTPExtender(&policy.ExtenderConfigs[ii]) if err != nil { return nil, err @@ -1251,7 +1199,7 @@ func (c *configFactory) getBinderFunc(extenders []algorithm.SchedulerExtender) f // Creates a scheduler from a set of registered fit predicate keys and priority keys. func (c *configFactory) CreateFromKeys(predicateKeys, priorityKeys sets.String, extenders []algorithm.SchedulerExtender) (*Config, error) { - glog.V(2).Infof("Creating scheduler with fit predicates '%v' and priority functions '%v'", predicateKeys, priorityKeys) + klog.V(2).Infof("Creating scheduler with fit predicates '%v' and priority functions '%v'", predicateKeys, priorityKeys) if c.GetHardPodAffinitySymmetricWeight() < 1 || c.GetHardPodAffinitySymmetricWeight() > 100 { return nil, fmt.Errorf("invalid hardPodAffinitySymmetricWeight: %d, must be in the range 1-100", c.GetHardPodAffinitySymmetricWeight()) @@ -1280,7 +1228,7 @@ func (c *configFactory) CreateFromKeys(predicateKeys, priorityKeys sets.String, // Init equivalence class cache if c.enableEquivalenceClassCache { c.equivalencePodCache = equivalence.NewCache(predicates.Ordering()) - glog.Info("Created equivalence class cache") + klog.Info("Created equivalence class cache") } algo := core.NewGenericScheduler( @@ -1316,9 +1264,10 @@ func (c *configFactory) CreateFromKeys(predicateKeys, priorityKeys sets.String, NextPod: func() *v1.Pod { return c.getNextPod() }, - Error: c.MakeDefaultErrorFunc(podBackoff, c.podQueue), - StopEverything: c.StopEverything, - VolumeBinder: c.volumeBinder, + Error: c.MakeDefaultErrorFunc(podBackoff, c.podQueue), + StopEverything: c.StopEverything, + VolumeBinder: c.volumeBinder, + SchedulingQueue: c.podQueue, }, nil } @@ -1386,10 +1335,10 @@ func (c *configFactory) getPluginArgs() (*PluginFactoryArgs, error) { func (c *configFactory) getNextPod() *v1.Pod { pod, err := c.podQueue.Pop() if err == nil { - glog.V(4).Infof("About to try and schedule pod %v/%v", pod.Namespace, pod.Name) + klog.V(4).Infof("About to try and schedule pod %v/%v", pod.Namespace, pod.Name) return pod } - glog.Errorf("Error while retrieving next pod from scheduling queue: %v", err) + klog.Errorf("Error while retrieving next pod from scheduling queue: %v", err) return nil } @@ -1488,10 +1437,10 @@ func NewPodInformer(client clientset.Interface, resyncPeriod time.Duration) core func (c *configFactory) MakeDefaultErrorFunc(backoff *util.PodBackoff, podQueue internalqueue.SchedulingQueue) func(pod *v1.Pod, err error) { return func(pod *v1.Pod, err error) { if err == core.ErrNoNodesAvailable { - glog.V(4).Infof("Unable to schedule %v/%v: no nodes are registered to the cluster; waiting", pod.Namespace, pod.Name) + klog.V(4).Infof("Unable to schedule %v/%v: no nodes are registered to the cluster; waiting", pod.Namespace, pod.Name) } else { if _, ok := err.(*core.FitError); ok { - glog.V(4).Infof("Unable to schedule %v/%v: no fit: %v; waiting", pod.Namespace, pod.Name, err) + klog.V(4).Infof("Unable to schedule %v/%v: no fit: %v; waiting", pod.Namespace, pod.Name, err) } else if errors.IsNotFound(err) { if errStatus, ok := err.(errors.APIStatus); ok && errStatus.Status().Details.Kind == "node" { nodeName := errStatus.Status().Details.Name @@ -1513,7 +1462,7 @@ func (c *configFactory) MakeDefaultErrorFunc(backoff *util.PodBackoff, podQueue } } } else { - glog.Errorf("Error scheduling %v/%v: %v; retrying", pod.Namespace, pod.Name, err) + klog.Errorf("Error scheduling %v/%v: %v; retrying", pod.Namespace, pod.Name, err) } } @@ -1535,7 +1484,7 @@ func (c *configFactory) MakeDefaultErrorFunc(backoff *util.PodBackoff, podQueue if !util.PodPriorityEnabled() { entry := backoff.GetEntry(podID) if !entry.TryWait(backoff.MaxDuration()) { - glog.Warningf("Request for pod %v already in flight, abandoning", podID) + klog.Warningf("Request for pod %v already in flight, abandoning", podID) return } } @@ -1555,7 +1504,7 @@ func (c *configFactory) MakeDefaultErrorFunc(backoff *util.PodBackoff, podQueue break } if errors.IsNotFound(err) { - glog.Warningf("A pod %v no longer exists", podID) + klog.Warningf("A pod %v no longer exists", podID) if c.volumeBinder != nil { // Volume binder only wants to keep unassigned pods @@ -1563,7 +1512,7 @@ func (c *configFactory) MakeDefaultErrorFunc(backoff *util.PodBackoff, podQueue } return } - glog.Errorf("Error getting pod %v for retry: %v; retrying...", podID, err) + klog.Errorf("Error getting pod %v for retry: %v; retrying...", podID, err) if getBackoff = getBackoff * 2; getBackoff > maximalGetBackoff { getBackoff = maximalGetBackoff } @@ -1597,7 +1546,7 @@ type binder struct { // Bind just does a POST binding RPC. func (b *binder) Bind(binding *v1.Binding) error { - glog.V(3).Infof("Attempting to bind %v to %v", binding.Name, binding.Target.Name) + klog.V(3).Infof("Attempting to bind %v to %v", binding.Name, binding.Target.Name) return b.Client.CoreV1().Pods(binding.Namespace).Bind(binding) } @@ -1606,7 +1555,7 @@ type podConditionUpdater struct { } func (p *podConditionUpdater) Update(pod *v1.Pod, condition *v1.PodCondition) error { - glog.V(3).Infof("Updating pod condition for %s/%s to (%s==%s)", pod.Namespace, pod.Name, condition.Type, condition.Status) + klog.V(3).Infof("Updating pod condition for %s/%s to (%s==%s)", pod.Namespace, pod.Name, condition.Type, condition.Status) if podutil.UpdatePodCondition(&pod.Status, condition) { _, err := p.Client.CoreV1().Pods(pod.Namespace).UpdateStatus(pod) return err diff --git a/pkg/scheduler/factory/factory_test.go b/pkg/scheduler/factory/factory_test.go index 2e7d4256e78..aa68c716fc0 100644 --- a/pkg/scheduler/factory/factory_test.go +++ b/pkg/scheduler/factory/factory_test.go @@ -24,7 +24,6 @@ import ( "time" "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -35,7 +34,6 @@ import ( clienttesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" apitesting "k8s.io/kubernetes/pkg/api/testing" - "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/scheduler/algorithm" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" latestschedulerapi "k8s.io/kubernetes/pkg/scheduler/api/latest" @@ -54,7 +52,9 @@ const ( func TestCreate(t *testing.T) { client := fake.NewSimpleClientset() - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) factory.Create() } @@ -65,7 +65,9 @@ func TestCreateFromConfig(t *testing.T) { var policy schedulerapi.Policy client := fake.NewSimpleClientset() - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) // Pre-register some predicate and priority functions RegisterFitPredicate("PredicateOne", PredicateOne) @@ -103,7 +105,9 @@ func TestCreateFromConfigWithHardPodAffinitySymmetricWeight(t *testing.T) { var policy schedulerapi.Policy client := fake.NewSimpleClientset() - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) // Pre-register some predicate and priority functions RegisterFitPredicate("PredicateOne", PredicateOne) @@ -142,7 +146,9 @@ func TestCreateFromEmptyConfig(t *testing.T) { var policy schedulerapi.Policy client := fake.NewSimpleClientset() - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) configData = []byte(`{}`) if err := runtime.DecodeInto(latestschedulerapi.Codec, configData, &policy); err != nil { @@ -157,7 +163,9 @@ func TestCreateFromEmptyConfig(t *testing.T) { // The predicate/priority from DefaultProvider will be used. func TestCreateFromConfigWithUnspecifiedPredicatesOrPriorities(t *testing.T) { client := fake.NewSimpleClientset() - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) RegisterFitPredicate("PredicateOne", PredicateOne) RegisterPriorityFunction("PriorityOne", PriorityOne, 1) @@ -190,7 +198,9 @@ func TestCreateFromConfigWithUnspecifiedPredicatesOrPriorities(t *testing.T) { // Empty predicate/priority sets will be used. func TestCreateFromConfigWithEmptyPredicatesOrPriorities(t *testing.T) { client := fake.NewSimpleClientset() - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) RegisterFitPredicate("PredicateOne", PredicateOne) RegisterPriorityFunction("PriorityOne", PriorityOne, 1) @@ -242,7 +252,9 @@ func TestDefaultErrorFunc(t *testing.T) { Spec: apitesting.V1DeepEqualSafePodSpec(), } client := fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*testPod}}) - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) queue := &internalqueue.FIFO{FIFO: cache.NewFIFO(cache.MetaNamespaceKeyFunc)} podBackoff := util.CreatePodBackoff(1*time.Millisecond, 1*time.Second) errFunc := factory.MakeDefaultErrorFunc(podBackoff, queue) @@ -372,7 +384,9 @@ func testBind(binding *v1.Binding, t *testing.T) { func TestInvalidHardPodAffinitySymmetricWeight(t *testing.T) { client := fake.NewSimpleClientset() // factory of "default-scheduler" - factory := newConfigFactory(client, -1) + stopCh := make(chan struct{}) + factory := newConfigFactory(client, -1, stopCh) + defer close(stopCh) _, err := factory.Create() if err == nil { t.Errorf("expected err: invalid hardPodAffinitySymmetricWeight, got nothing") @@ -401,7 +415,9 @@ func TestInvalidFactoryArgs(t *testing.T) { for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - factory := newConfigFactory(client, test.hardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + factory := newConfigFactory(client, test.hardPodAffinitySymmetricWeight, stopCh) + defer close(stopCh) _, err := factory.Create() if err == nil { t.Errorf("expected err: %s, got nothing", test.expectErr) @@ -503,7 +519,7 @@ func TestSkipPodUpdate(t *testing.T) { } } -func newConfigFactory(client clientset.Interface, hardPodAffinitySymmetricWeight int32) Configurator { +func newConfigFactory(client clientset.Interface, hardPodAffinitySymmetricWeight int32, stopCh <-chan struct{}) Configurator { informerFactory := informers.NewSharedInformerFactory(client, 0) return NewConfigFactory(&ConfigFactoryArgs{ v1.DefaultSchedulerName, @@ -523,6 +539,7 @@ func newConfigFactory(client clientset.Interface, hardPodAffinitySymmetricWeight disablePodPreemption, schedulerapi.DefaultPercentageOfNodesToScore, bindTimeoutSeconds, + stopCh, }) } @@ -532,6 +549,10 @@ type fakeExtender struct { ignorable bool } +func (f *fakeExtender) Name() string { + return "fakeExtender" +} + func (f *fakeExtender) IsIgnorable() bool { return f.ignorable } @@ -636,130 +657,3 @@ func testGetBinderFunc(expectedBinderType, podName string, extenders []algorithm t.Errorf("Expected binder %q but got %q", expectedBinderType, binderType) } } - -func TestNodeAllocatableChanged(t *testing.T) { - newQuantity := func(value int64) resource.Quantity { - return *resource.NewQuantity(value, resource.BinarySI) - } - for _, c := range []struct { - Changed bool - OldAllocatable v1.ResourceList - NewAllocatable v1.ResourceList - }{ - // No allocatable resources changed. - { - Changed: false, - OldAllocatable: v1.ResourceList{v1.ResourceMemory: newQuantity(1024)}, - NewAllocatable: v1.ResourceList{v1.ResourceMemory: newQuantity(1024)}, - }, - // New node has more allocatable resources. - { - Changed: true, - OldAllocatable: v1.ResourceList{v1.ResourceMemory: newQuantity(1024)}, - NewAllocatable: v1.ResourceList{v1.ResourceMemory: newQuantity(1024), v1.ResourceStorage: newQuantity(1024)}, - }, - } { - oldNode := &v1.Node{Status: v1.NodeStatus{Allocatable: c.OldAllocatable}} - newNode := &v1.Node{Status: v1.NodeStatus{Allocatable: c.NewAllocatable}} - changed := nodeAllocatableChanged(newNode, oldNode) - if changed != c.Changed { - t.Errorf("nodeAllocatableChanged should be %t, got %t", c.Changed, changed) - } - } -} - -func TestNodeLabelsChanged(t *testing.T) { - for _, c := range []struct { - Changed bool - OldLabels map[string]string - NewLabels map[string]string - }{ - // No labels changed. - {Changed: false, OldLabels: map[string]string{"foo": "bar"}, NewLabels: map[string]string{"foo": "bar"}}, - // Labels changed. - {Changed: true, OldLabels: map[string]string{"foo": "bar"}, NewLabels: map[string]string{"foo": "bar", "test": "value"}}, - } { - oldNode := &v1.Node{ObjectMeta: metav1.ObjectMeta{Labels: c.OldLabels}} - newNode := &v1.Node{ObjectMeta: metav1.ObjectMeta{Labels: c.NewLabels}} - changed := nodeLabelsChanged(newNode, oldNode) - if changed != c.Changed { - t.Errorf("nodeLabelsChanged should be %t, got %t", c.Changed, changed) - } - } -} - -func TestNodeTaintsChanged(t *testing.T) { - for _, c := range []struct { - Changed bool - OldTaints []v1.Taint - NewTaints []v1.Taint - OldAnnotations map[string]string - NewAnnotations map[string]string - }{ - // Taints use annotation and no change. - { - Changed: false, - OldAnnotations: map[string]string{core.TaintsAnnotationKey: `[{"key":"value"}]`}, - NewAnnotations: map[string]string{core.TaintsAnnotationKey: `[{"key":"value"}]`}, - }, - // Taints use annotation and changed. - { - Changed: true, - OldAnnotations: map[string]string{core.TaintsAnnotationKey: `[{"key":"value1"}]`}, - NewAnnotations: map[string]string{core.TaintsAnnotationKey: `[{"key":"value2"}]`}, - }, - // Taints use Spec.Taints and no change. - { - Changed: false, - OldTaints: []v1.Taint{{Key: "key", Value: "value"}}, - NewTaints: []v1.Taint{{Key: "key", Value: "value"}}, - }, - // Taints use Spec.Taints and changed. - { - Changed: true, - OldTaints: []v1.Taint{{Key: "key", Value: "value1"}}, - NewTaints: []v1.Taint{{Key: "key", Value: "value2"}}, - }, - } { - oldNode := &v1.Node{ObjectMeta: metav1.ObjectMeta{Annotations: c.OldAnnotations}, Spec: v1.NodeSpec{Taints: c.OldTaints}} - newNode := &v1.Node{ObjectMeta: metav1.ObjectMeta{Annotations: c.NewAnnotations}, Spec: v1.NodeSpec{Taints: c.NewTaints}} - changed := nodeTaintsChanged(newNode, oldNode) - if changed != c.Changed { - t.Errorf("nodeTaintsChanged should be %t, not %t", c.Changed, changed) - } - } -} - -func TestNodeConditionsChanged(t *testing.T) { - for _, c := range []struct { - Changed bool - OldConditions []v1.NodeCondition - NewConditions []v1.NodeCondition - }{ - // No conditions changed. - { - Changed: false, - OldConditions: []v1.NodeCondition{{Type: v1.NodeOutOfDisk, Status: v1.ConditionTrue}}, - NewConditions: []v1.NodeCondition{{Type: v1.NodeOutOfDisk, Status: v1.ConditionTrue}}, - }, - // New node has more healthy conditions. - { - Changed: true, - OldConditions: []v1.NodeCondition{}, - NewConditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}, - }, - // NodeReady False -> True - { - Changed: true, - OldConditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionFalse}}, - NewConditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}, - }, - } { - oldNode := &v1.Node{Status: v1.NodeStatus{Conditions: c.OldConditions}} - newNode := &v1.Node{Status: v1.NodeStatus{Conditions: c.NewConditions}} - changed := nodeConditionsChanged(newNode, oldNode) - if changed != c.Changed { - t.Errorf("nodeConditionsChanged should be %t, got %t", c.Changed, changed) - } - } -} diff --git a/pkg/scheduler/factory/plugins.go b/pkg/scheduler/factory/plugins.go index a83cf78e5ec..216a93f7e60 100644 --- a/pkg/scheduler/factory/plugins.go +++ b/pkg/scheduler/factory/plugins.go @@ -30,7 +30,7 @@ import ( schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" "k8s.io/kubernetes/pkg/scheduler/volumebinder" - "github.com/golang/glog" + "k8s.io/klog" ) // PluginFactoryArgs are passed to all plugin factory functions. @@ -233,12 +233,12 @@ func RegisterCustomFitPredicate(policy schedulerapi.PredicatePolicy) string { } } else if predicateFactory, ok = fitPredicateMap[policy.Name]; ok { // checking to see if a pre-defined predicate is requested - glog.V(2).Infof("Predicate type %s already registered, reusing.", policy.Name) + klog.V(2).Infof("Predicate type %s already registered, reusing.", policy.Name) return policy.Name } if predicateFactory == nil { - glog.Fatalf("Invalid configuration: Predicate type not found for %s", policy.Name) + klog.Fatalf("Invalid configuration: Predicate type not found for %s", policy.Name) } return RegisterFitPredicateFactory(policy.Name, predicateFactory) @@ -345,7 +345,7 @@ func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy) string { } } } else if existingPcf, ok := priorityFunctionMap[policy.Name]; ok { - glog.V(2).Infof("Priority type %s already registered, reusing.", policy.Name) + klog.V(2).Infof("Priority type %s already registered, reusing.", policy.Name) // set/update the weight based on the policy pcf = &PriorityConfigFactory{ Function: existingPcf.Function, @@ -355,7 +355,7 @@ func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy) string { } if pcf == nil { - glog.Fatalf("Invalid configuration: Priority type not found for %s", policy.Name) + klog.Fatalf("Invalid configuration: Priority type not found for %s", policy.Name) } return RegisterPriorityConfigFactory(policy.Name, *pcf) @@ -369,7 +369,7 @@ func buildScoringFunctionShapeFromRequestedToCapacityRatioArguments(arguments *s } shape, err := priorities.NewFunctionShape(points) if err != nil { - glog.Fatalf("invalid RequestedToCapacityRatioPriority arguments: %s", err.Error()) + klog.Fatalf("invalid RequestedToCapacityRatioPriority arguments: %s", err.Error()) } return shape } @@ -500,7 +500,7 @@ var validName = regexp.MustCompile("^[a-zA-Z0-9]([-a-zA-Z0-9]*[a-zA-Z0-9])$") func validateAlgorithmNameOrDie(name string) { if !validName.MatchString(name) { - glog.Fatalf("Algorithm name %v does not match the name validation regexp \"%v\".", name, validName) + klog.Fatalf("Algorithm name %v does not match the name validation regexp \"%v\".", name, validName) } } @@ -514,7 +514,7 @@ func validatePredicateOrDie(predicate schedulerapi.PredicatePolicy) { numArgs++ } if numArgs != 1 { - glog.Fatalf("Exactly 1 predicate argument is required, numArgs: %v, Predicate: %s", numArgs, predicate.Name) + klog.Fatalf("Exactly 1 predicate argument is required, numArgs: %v, Predicate: %s", numArgs, predicate.Name) } } } @@ -532,7 +532,7 @@ func validatePriorityOrDie(priority schedulerapi.PriorityPolicy) { numArgs++ } if numArgs != 1 { - glog.Fatalf("Exactly 1 priority argument is required, numArgs: %v, Priority: %s", numArgs, priority.Name) + klog.Fatalf("Exactly 1 priority argument is required, numArgs: %v, Priority: %s", numArgs, priority.Name) } } } diff --git a/pkg/scheduler/internal/cache/BUILD b/pkg/scheduler/internal/cache/BUILD index cce14ec53bc..61202c8c286 100644 --- a/pkg/scheduler/internal/cache/BUILD +++ b/pkg/scheduler/internal/cache/BUILD @@ -18,7 +18,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -26,6 +26,7 @@ go_test( name = "go_default_test", srcs = [ "cache_test.go", + "main_test.go", "node_tree_test.go", ], embed = [":go_default_library"], @@ -40,6 +41,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) @@ -54,7 +56,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", - "//pkg/scheduler/internal/cache/comparer:all-srcs", + "//pkg/scheduler/internal/cache/debugger:all-srcs", "//pkg/scheduler/internal/cache/fake:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/scheduler/internal/cache/cache.go b/pkg/scheduler/internal/cache/cache.go index 16f7b25799d..535236e5c1f 100644 --- a/pkg/scheduler/internal/cache/cache.go +++ b/pkg/scheduler/internal/cache/cache.go @@ -29,7 +29,7 @@ import ( "k8s.io/kubernetes/pkg/features" schedulercache "k8s.io/kubernetes/pkg/scheduler/cache" - "github.com/golang/glog" + "k8s.io/klog" ) var ( @@ -205,7 +205,7 @@ func (cache *schedulerCache) finishBinding(pod *v1.Pod, now time.Time) error { cache.mu.RLock() defer cache.mu.RUnlock() - glog.V(5).Infof("Finished binding for pod %v. Can be expired.", key) + klog.V(5).Infof("Finished binding for pod %v. Can be expired.", key) currState, ok := cache.podStates[key] if ok && cache.assumedPods[key] { dl := now.Add(cache.ttl) @@ -289,7 +289,7 @@ func (cache *schedulerCache) AddPod(pod *v1.Pod) error { case ok && cache.assumedPods[key]: if currState.pod.Spec.NodeName != pod.Spec.NodeName { // The pod was added to a different node than it was assumed to. - glog.Warningf("Pod %v was assumed to be on %v but got added to %v", key, pod.Spec.NodeName, currState.pod.Spec.NodeName) + klog.Warningf("Pod %v was assumed to be on %v but got added to %v", key, pod.Spec.NodeName, currState.pod.Spec.NodeName) // Clean this up. cache.removePod(currState.pod) cache.addPod(pod) @@ -325,8 +325,8 @@ func (cache *schedulerCache) UpdatePod(oldPod, newPod *v1.Pod) error { // before Update event, in which case the state would change from Assumed to Added. case ok && !cache.assumedPods[key]: if currState.pod.Spec.NodeName != newPod.Spec.NodeName { - glog.Errorf("Pod %v updated on a different node than previously added to.", key) - glog.Fatalf("Schedulercache is corrupted and can badly affect scheduling decisions") + klog.Errorf("Pod %v updated on a different node than previously added to.", key) + klog.Fatalf("Schedulercache is corrupted and can badly affect scheduling decisions") } if err := cache.updatePod(oldPod, newPod); err != nil { return err @@ -353,8 +353,8 @@ func (cache *schedulerCache) RemovePod(pod *v1.Pod) error { // before Remove event, in which case the state would change from Assumed to Added. case ok && !cache.assumedPods[key]: if currState.pod.Spec.NodeName != pod.Spec.NodeName { - glog.Errorf("Pod %v was assumed to be on %v but got added to %v", key, pod.Spec.NodeName, currState.pod.Spec.NodeName) - glog.Fatalf("Schedulercache is corrupted and can badly affect scheduling decisions") + klog.Errorf("Pod %v was assumed to be on %v but got added to %v", key, pod.Spec.NodeName, currState.pod.Spec.NodeName) + klog.Fatalf("Schedulercache is corrupted and can badly affect scheduling decisions") } err := cache.removePod(currState.pod) if err != nil { @@ -526,14 +526,14 @@ func (cache *schedulerCache) cleanupAssumedPods(now time.Time) { panic("Key found in assumed set but not in podStates. Potentially a logical error.") } if !ps.bindingFinished { - glog.V(3).Infof("Couldn't expire cache for pod %v/%v. Binding is still in progress.", + klog.V(3).Infof("Couldn't expire cache for pod %v/%v. Binding is still in progress.", ps.pod.Namespace, ps.pod.Name) continue } if now.After(*ps.deadline) { - glog.Warningf("Pod %s/%s expired", ps.pod.Namespace, ps.pod.Name) + klog.Warningf("Pod %s/%s expired", ps.pod.Namespace, ps.pod.Name) if err := cache.expirePod(key, ps); err != nil { - glog.Errorf("ExpirePod failed for %s: %v", key, err) + klog.Errorf("ExpirePod failed for %s: %v", key, err) } } } diff --git a/pkg/scheduler/internal/cache/cache_test.go b/pkg/scheduler/internal/cache/cache_test.go index 8186784968a..97301725151 100644 --- a/pkg/scheduler/internal/cache/cache_test.go +++ b/pkg/scheduler/internal/cache/cache_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/kubernetes/pkg/features" priorityutil "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/util" schedulercache "k8s.io/kubernetes/pkg/scheduler/cache" @@ -92,7 +93,7 @@ func newNodeInfo(requestedResource *schedulercache.Resource, // on node level. func TestAssumePodScheduled(t *testing.T) { // Enable volumesOnNodeForBalancing to do balanced resource allocation - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.BalanceAttachedNodeVolumes)) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BalanceAttachedNodeVolumes, true)() nodeName := "node" testPods := []*v1.Pod{ makeBasePod(t, nodeName, "test", "100m", "500", "", []v1.ContainerPort{{HostIP: "127.0.0.1", HostPort: 80, Protocol: "TCP"}}), @@ -240,7 +241,7 @@ func assumeAndFinishBinding(cache *schedulerCache, pod *v1.Pod, assumedTime time // The removal will be reflected in node info. func TestExpirePod(t *testing.T) { // Enable volumesOnNodeForBalancing to do balanced resource allocation - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.BalanceAttachedNodeVolumes)) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BalanceAttachedNodeVolumes, true)() nodeName := "node" testPods := []*v1.Pod{ makeBasePod(t, nodeName, "test-1", "100m", "500", "", []v1.ContainerPort{{HostIP: "127.0.0.1", HostPort: 80, Protocol: "TCP"}}), @@ -299,7 +300,7 @@ func TestExpirePod(t *testing.T) { // The pod info should still exist after manually expiring unconfirmed pods. func TestAddPodWillConfirm(t *testing.T) { // Enable volumesOnNodeForBalancing to do balanced resource allocation - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.BalanceAttachedNodeVolumes)) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BalanceAttachedNodeVolumes, true)() nodeName := "node" now := time.Now() ttl := 10 * time.Second @@ -455,7 +456,7 @@ func TestAddPodWillReplaceAssumed(t *testing.T) { // TestAddPodAfterExpiration tests that a pod being Add()ed will be added back if expired. func TestAddPodAfterExpiration(t *testing.T) { // Enable volumesOnNodeForBalancing to do balanced resource allocation - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.BalanceAttachedNodeVolumes)) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BalanceAttachedNodeVolumes, true)() nodeName := "node" ttl := 10 * time.Second basePod := makeBasePod(t, nodeName, "test", "100m", "500", "", []v1.ContainerPort{{HostIP: "127.0.0.1", HostPort: 80, Protocol: "TCP"}}) @@ -504,7 +505,7 @@ func TestAddPodAfterExpiration(t *testing.T) { // TestUpdatePod tests that a pod will be updated if added before. func TestUpdatePod(t *testing.T) { // Enable volumesOnNodeForBalancing to do balanced resource allocation - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.BalanceAttachedNodeVolumes)) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BalanceAttachedNodeVolumes, true)() nodeName := "node" ttl := 10 * time.Second testPods := []*v1.Pod{ @@ -630,7 +631,7 @@ func TestUpdatePodAndGet(t *testing.T) { // TestExpireAddUpdatePod test the sequence that a pod is expired, added, then updated func TestExpireAddUpdatePod(t *testing.T) { // Enable volumesOnNodeForBalancing to do balanced resource allocation - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.BalanceAttachedNodeVolumes)) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BalanceAttachedNodeVolumes, true)() nodeName := "node" ttl := 10 * time.Second testPods := []*v1.Pod{ @@ -727,7 +728,7 @@ func makePodWithEphemeralStorage(nodeName, ephemeralStorage string) *v1.Pod { func TestEphemeralStorageResource(t *testing.T) { // Enable volumesOnNodeForBalancing to do balanced resource allocation - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.BalanceAttachedNodeVolumes)) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BalanceAttachedNodeVolumes, true)() nodeName := "node" podE := makePodWithEphemeralStorage(nodeName, "500") tests := []struct { @@ -772,7 +773,7 @@ func TestEphemeralStorageResource(t *testing.T) { // TestRemovePod tests after added pod is removed, its information should also be subtracted. func TestRemovePod(t *testing.T) { // Enable volumesOnNodeForBalancing to do balanced resource allocation - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.BalanceAttachedNodeVolumes)) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BalanceAttachedNodeVolumes, true)() nodeName := "node" basePod := makeBasePod(t, nodeName, "test", "100m", "500", "", []v1.ContainerPort{{HostIP: "127.0.0.1", HostPort: 80, Protocol: "TCP"}}) tests := []struct { @@ -1126,7 +1127,7 @@ func BenchmarkList1kNodes30kPods(b *testing.B) { func BenchmarkUpdate1kNodes30kPods(b *testing.B) { // Enable volumesOnNodeForBalancing to do balanced resource allocation - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.BalanceAttachedNodeVolumes)) + defer utilfeaturetesting.SetFeatureGateDuringTest(nil, utilfeature.DefaultFeatureGate, features.BalanceAttachedNodeVolumes, true)() cache := setupCacheOf1kNodes30kPods(b) b.ResetTimer() for n := 0; n < b.N; n++ { diff --git a/pkg/scheduler/internal/cache/comparer/BUILD b/pkg/scheduler/internal/cache/debugger/BUILD similarity index 89% rename from pkg/scheduler/internal/cache/comparer/BUILD rename to pkg/scheduler/internal/cache/debugger/BUILD index c35831fbb83..320c9734fba 100644 --- a/pkg/scheduler/internal/cache/comparer/BUILD +++ b/pkg/scheduler/internal/cache/debugger/BUILD @@ -2,8 +2,12 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["comparer.go"], - importpath = "k8s.io/kubernetes/pkg/scheduler/internal/cache/comparer", + srcs = [ + "comparer.go", + "debugger.go", + "dumper.go", + ], + importpath = "k8s.io/kubernetes/pkg/scheduler/internal/cache/debugger", visibility = ["//pkg/scheduler:__subpackages__"], deps = [ "//pkg/scheduler/cache:go_default_library", @@ -12,7 +16,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/scheduler/internal/cache/comparer/comparer.go b/pkg/scheduler/internal/cache/debugger/comparer.go similarity index 84% rename from pkg/scheduler/internal/cache/comparer/comparer.go rename to pkg/scheduler/internal/cache/debugger/comparer.go index bf043fa3f92..e78df11184a 100644 --- a/pkg/scheduler/internal/cache/comparer/comparer.go +++ b/pkg/scheduler/internal/cache/debugger/comparer.go @@ -14,16 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -package comparer +package debugger import ( "sort" "strings" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/klog" schedulercache "k8s.io/kubernetes/pkg/scheduler/cache" schedulerinternalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" @@ -37,25 +37,10 @@ type CacheComparer struct { PodQueue internalqueue.SchedulingQueue } -// New creates a CacheComparer. -func New( - nodeLister corelisters.NodeLister, - podLister corelisters.PodLister, - cache schedulerinternalcache.Cache, - podQueue internalqueue.SchedulingQueue, -) *CacheComparer { - return &CacheComparer{ - NodeLister: nodeLister, - PodLister: podLister, - Cache: cache, - PodQueue: podQueue, - } -} - // Compare compares the nodes and pods of NodeLister with Cache.Snapshot. func (c *CacheComparer) Compare() error { - glog.V(3).Info("cache comparer started") - defer glog.V(3).Info("cache comparer finished") + klog.V(3).Info("cache comparer started") + defer klog.V(3).Info("cache comparer finished") nodes, err := c.NodeLister.List(labels.Everything()) if err != nil { @@ -72,11 +57,11 @@ func (c *CacheComparer) Compare() error { waitingPods := c.PodQueue.WaitingPods() if missed, redundant := c.CompareNodes(nodes, snapshot.Nodes); len(missed)+len(redundant) != 0 { - glog.Warningf("cache mismatch: missed nodes: %s; redundant nodes: %s", missed, redundant) + klog.Warningf("cache mismatch: missed nodes: %s; redundant nodes: %s", missed, redundant) } if missed, redundant := c.ComparePods(pods, waitingPods, snapshot.Nodes); len(missed)+len(redundant) != 0 { - glog.Warningf("cache mismatch: missed pods: %s; redundant pods: %s", missed, redundant) + klog.Warningf("cache mismatch: missed pods: %s; redundant pods: %s", missed, redundant) } return nil diff --git a/pkg/scheduler/internal/cache/comparer/comparer_test.go b/pkg/scheduler/internal/cache/debugger/comparer_test.go similarity index 99% rename from pkg/scheduler/internal/cache/comparer/comparer_test.go rename to pkg/scheduler/internal/cache/debugger/comparer_test.go index 701bd87ada1..967b4027b5d 100644 --- a/pkg/scheduler/internal/cache/comparer/comparer_test.go +++ b/pkg/scheduler/internal/cache/debugger/comparer_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package comparer +package debugger import ( "reflect" diff --git a/pkg/scheduler/internal/cache/debugger/debugger.go b/pkg/scheduler/internal/cache/debugger/debugger.go new file mode 100644 index 00000000000..64428d5693e --- /dev/null +++ b/pkg/scheduler/internal/cache/debugger/debugger.go @@ -0,0 +1,50 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package debugger + +import ( + corelisters "k8s.io/client-go/listers/core/v1" + internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" + internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" +) + +// CacheDebugger provides ways to check and write cache information for debugging. +type CacheDebugger struct { + Comparer CacheComparer + Dumper CacheDumper +} + +// New creates a CacheDebugger. +func New( + nodeLister corelisters.NodeLister, + podLister corelisters.PodLister, + cache internalcache.Cache, + podQueue internalqueue.SchedulingQueue, +) *CacheDebugger { + return &CacheDebugger{ + Comparer: CacheComparer{ + NodeLister: nodeLister, + PodLister: podLister, + Cache: cache, + PodQueue: podQueue, + }, + Dumper: CacheDumper{ + cache: cache, + podQueue: podQueue, + }, + } +} diff --git a/pkg/scheduler/internal/cache/debugger/dumper.go b/pkg/scheduler/internal/cache/debugger/dumper.go new file mode 100644 index 00000000000..b9084d377d9 --- /dev/null +++ b/pkg/scheduler/internal/cache/debugger/dumper.go @@ -0,0 +1,78 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package debugger + +import ( + "fmt" + "strings" + + "k8s.io/klog" + + "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/scheduler/cache" + internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" + "k8s.io/kubernetes/pkg/scheduler/internal/queue" +) + +// CacheDumper writes some information from the scheduler cache and the scheduling queue to the +// scheduler logs for debugging purposes. +type CacheDumper struct { + cache internalcache.Cache + podQueue queue.SchedulingQueue +} + +// DumpAll writes cached nodes and scheduling queue information to the scheduler logs. +func (d *CacheDumper) DumpAll() { + d.dumpNodes() + d.dumpSchedulingQueue() +} + +// dumpNodes writes NodeInfo to the scheduler logs. +func (d *CacheDumper) dumpNodes() { + snapshot := d.cache.Snapshot() + klog.Info("Dump of cached NodeInfo") + for _, nodeInfo := range snapshot.Nodes { + klog.Info(printNodeInfo(nodeInfo)) + } +} + +// dumpSchedulingQueue writes pods in the scheduling queue to the scheduler logs. +func (d *CacheDumper) dumpSchedulingQueue() { + waitingPods := d.podQueue.WaitingPods() + var podData strings.Builder + for _, p := range waitingPods { + podData.WriteString(printPod(p)) + } + klog.Infof("Dump of scheduling queue:\n%s", podData.String()) +} + +// printNodeInfo writes parts of NodeInfo to a string. +func printNodeInfo(n *cache.NodeInfo) string { + var nodeData strings.Builder + nodeData.WriteString(fmt.Sprintf("\nNode name: %+v\nRequested Resources: %+v\nAllocatable Resources:%+v\nNumber of Pods: %v\nPods:\n", + n.Node().Name, n.RequestedResource(), n.AllocatableResource(), len(n.Pods()))) + // Dumping Pod Info + for _, p := range n.Pods() { + nodeData.WriteString(printPod(p)) + } + return nodeData.String() +} + +// printPod writes parts of a Pod object to a string. +func printPod(p *v1.Pod) string { + return fmt.Sprintf("name: %v, namespace: %v, uid: %v, phase: %v, nominated node: %v\n", p.Name, p.Namespace, p.UID, p.Status.Phase, p.Status.NominatedNodeName) +} diff --git a/pkg/scheduler/internal/cache/main_test.go b/pkg/scheduler/internal/cache/main_test.go new file mode 100644 index 00000000000..e29bc63f43a --- /dev/null +++ b/pkg/scheduler/internal/cache/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/scheduler/internal/cache/node_tree.go b/pkg/scheduler/internal/cache/node_tree.go index 8e8b4f0a6a1..80ce6d195fa 100644 --- a/pkg/scheduler/internal/cache/node_tree.go +++ b/pkg/scheduler/internal/cache/node_tree.go @@ -23,7 +23,7 @@ import ( "k8s.io/api/core/v1" utilnode "k8s.io/kubernetes/pkg/util/node" - "github.com/golang/glog" + "k8s.io/klog" ) // NodeTree is a tree-like data structure that holds node names in each zone. Zone names are @@ -46,7 +46,7 @@ type nodeArray struct { func (na *nodeArray) next() (nodeName string, exhausted bool) { if len(na.nodes) == 0 { - glog.Error("The nodeArray is empty. It should have been deleted from NodeTree.") + klog.Error("The nodeArray is empty. It should have been deleted from NodeTree.") return "", false } if na.lastIndex >= len(na.nodes) { @@ -81,7 +81,7 @@ func (nt *NodeTree) addNode(n *v1.Node) { if na, ok := nt.tree[zone]; ok { for _, nodeName := range na.nodes { if nodeName == n.Name { - glog.Warningf("node %v already exist in the NodeTree", n.Name) + klog.Warningf("node %v already exist in the NodeTree", n.Name) return } } @@ -90,7 +90,7 @@ func (nt *NodeTree) addNode(n *v1.Node) { nt.zones = append(nt.zones, zone) nt.tree[zone] = &nodeArray{nodes: []string{n.Name}, lastIndex: 0} } - glog.V(5).Infof("Added node %v in group %v to NodeTree", n.Name, zone) + klog.V(5).Infof("Added node %v in group %v to NodeTree", n.Name, zone) nt.NumNodes++ } @@ -110,13 +110,13 @@ func (nt *NodeTree) removeNode(n *v1.Node) error { if len(na.nodes) == 0 { nt.removeZone(zone) } - glog.V(5).Infof("Removed node %v in group %v from NodeTree", n.Name, zone) + klog.V(5).Infof("Removed node %v in group %v from NodeTree", n.Name, zone) nt.NumNodes-- return nil } } } - glog.Errorf("Node %v in group %v was not found", n.Name, zone) + klog.Errorf("Node %v in group %v was not found", n.Name, zone) return fmt.Errorf("node %v in group %v was not found", n.Name, zone) } diff --git a/pkg/scheduler/internal/queue/BUILD b/pkg/scheduler/internal/queue/BUILD index 3be73a6295d..c675f859248 100644 --- a/pkg/scheduler/internal/queue/BUILD +++ b/pkg/scheduler/internal/queue/BUILD @@ -13,7 +13,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/scheduler/internal/queue/scheduling_queue.go b/pkg/scheduler/internal/queue/scheduling_queue.go index 271264e0653..6f5aa682c98 100644 --- a/pkg/scheduler/internal/queue/scheduling_queue.go +++ b/pkg/scheduler/internal/queue/scheduling_queue.go @@ -32,7 +32,7 @@ import ( "reflect" "sync" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -67,6 +67,8 @@ type SchedulingQueue interface { // Close closes the SchedulingQueue so that the goroutine which is // waiting to pop items can exit gracefully. Close() + // DeleteNominatedPodIfExists deletes nominatedPod from internal cache + DeleteNominatedPodIfExists(pod *v1.Pod) } // NewSchedulingQueue initializes a new scheduling queue. If pod priority is @@ -157,6 +159,9 @@ func (f *FIFO) Close() { f.FIFO.Close() } +// DeleteNominatedPodIfExists does nothing in FIFO. +func (f *FIFO) DeleteNominatedPodIfExists(pod *v1.Pod) {} + // NewFIFO creates a FIFO object. func NewFIFO() *FIFO { return &FIFO{FIFO: cache.NewFIFO(cache.MetaNamespaceKeyFunc)} @@ -219,7 +224,7 @@ func (p *PriorityQueue) addNominatedPodIfNeeded(pod *v1.Pod) { if len(nnn) > 0 { for _, np := range p.nominatedPods[nnn] { if np.UID == pod.UID { - glog.Errorf("Pod %v/%v already exists in the nominated map!", pod.Namespace, pod.Name) + klog.V(4).Infof("Pod %v/%v already exists in the nominated map!", pod.Namespace, pod.Name) return } } @@ -228,6 +233,7 @@ func (p *PriorityQueue) addNominatedPodIfNeeded(pod *v1.Pod) { } // deleteNominatedPodIfExists deletes a pod from the nominatedPods. +// NOTE: this function assumes lock has been acquired in caller. func (p *PriorityQueue) deleteNominatedPodIfExists(pod *v1.Pod) { nnn := NominatedNodeName(pod) if len(nnn) > 0 { @@ -258,10 +264,10 @@ func (p *PriorityQueue) Add(pod *v1.Pod) error { defer p.lock.Unlock() err := p.activeQ.Add(pod) if err != nil { - glog.Errorf("Error adding pod %v/%v to the scheduling queue: %v", pod.Namespace, pod.Name, err) + klog.Errorf("Error adding pod %v/%v to the scheduling queue: %v", pod.Namespace, pod.Name, err) } else { if p.unschedulableQ.get(pod) != nil { - glog.Errorf("Error: pod %v/%v is already in the unschedulable queue.", pod.Namespace, pod.Name) + klog.Errorf("Error: pod %v/%v is already in the unschedulable queue.", pod.Namespace, pod.Name) p.deleteNominatedPodIfExists(pod) p.unschedulableQ.delete(pod) } @@ -284,7 +290,7 @@ func (p *PriorityQueue) AddIfNotPresent(pod *v1.Pod) error { } err := p.activeQ.Add(pod) if err != nil { - glog.Errorf("Error adding pod %v/%v to the scheduling queue: %v", pod.Namespace, pod.Name, err) + klog.Errorf("Error adding pod %v/%v to the scheduling queue: %v", pod.Namespace, pod.Name, err) } else { p.addNominatedPodIfNeeded(pod) p.cond.Broadcast() @@ -342,7 +348,6 @@ func (p *PriorityQueue) Pop() (*v1.Pod, error) { return nil, err } pod := obj.(*v1.Pod) - p.deleteNominatedPodIfExists(pod) p.receivedMoveRequest = false return pod, err } @@ -411,13 +416,17 @@ func (p *PriorityQueue) Delete(pod *v1.Pod) error { // AssignedPodAdded is called when a bound pod is added. Creation of this pod // may make pending pods with matching affinity terms schedulable. func (p *PriorityQueue) AssignedPodAdded(pod *v1.Pod) { + p.lock.Lock() p.movePodsToActiveQueue(p.getUnschedulablePodsWithMatchingAffinityTerm(pod)) + p.lock.Unlock() } // AssignedPodUpdated is called when a bound pod is updated. Change of labels // may make pending pods with matching affinity terms schedulable. func (p *PriorityQueue) AssignedPodUpdated(pod *v1.Pod) { + p.lock.Lock() p.movePodsToActiveQueue(p.getUnschedulablePodsWithMatchingAffinityTerm(pod)) + p.lock.Unlock() } // MoveAllToActiveQueue moves all pods from unschedulableQ to activeQ. This @@ -433,7 +442,7 @@ func (p *PriorityQueue) MoveAllToActiveQueue() { defer p.lock.Unlock() for _, pod := range p.unschedulableQ.pods { if err := p.activeQ.Add(pod); err != nil { - glog.Errorf("Error adding pod %v/%v to the scheduling queue: %v", pod.Namespace, pod.Name, err) + klog.Errorf("Error adding pod %v/%v to the scheduling queue: %v", pod.Namespace, pod.Name, err) } } p.unschedulableQ.clear() @@ -441,14 +450,13 @@ func (p *PriorityQueue) MoveAllToActiveQueue() { p.cond.Broadcast() } +// NOTE: this function assumes lock has been acquired in caller func (p *PriorityQueue) movePodsToActiveQueue(pods []*v1.Pod) { - p.lock.Lock() - defer p.lock.Unlock() for _, pod := range pods { if err := p.activeQ.Add(pod); err == nil { p.unschedulableQ.delete(pod) } else { - glog.Errorf("Error adding pod %v/%v to the scheduling queue: %v", pod.Namespace, pod.Name, err) + klog.Errorf("Error adding pod %v/%v to the scheduling queue: %v", pod.Namespace, pod.Name, err) } } p.receivedMoveRequest = true @@ -457,9 +465,8 @@ func (p *PriorityQueue) movePodsToActiveQueue(pods []*v1.Pod) { // getUnschedulablePodsWithMatchingAffinityTerm returns unschedulable pods which have // any affinity term that matches "pod". +// NOTE: this function assumes lock has been acquired in caller. func (p *PriorityQueue) getUnschedulablePodsWithMatchingAffinityTerm(pod *v1.Pod) []*v1.Pod { - p.lock.RLock() - defer p.lock.RUnlock() var podsToMove []*v1.Pod for _, up := range p.unschedulableQ.pods { affinity := up.Spec.Affinity @@ -469,7 +476,7 @@ func (p *PriorityQueue) getUnschedulablePodsWithMatchingAffinityTerm(pod *v1.Pod namespaces := priorityutil.GetNamespacesFromPodAffinityTerm(up, &term) selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector) if err != nil { - glog.Errorf("Error getting label selectors for pod: %v.", up.Name) + klog.Errorf("Error getting label selectors for pod: %v.", up.Name) } if priorityutil.PodMatchesTermsNamespaceAndSelector(pod, namespaces, selector) { podsToMove = append(podsToMove, up) @@ -516,6 +523,13 @@ func (p *PriorityQueue) Close() { p.cond.Broadcast() } +// DeleteNominatedPodIfExists deletes pod from internal cache if it's a nominatedPod +func (p *PriorityQueue) DeleteNominatedPodIfExists(pod *v1.Pod) { + p.lock.Lock() + p.deleteNominatedPodIfExists(pod) + p.lock.Unlock() +} + // UnschedulablePodsMap holds pods that cannot be scheduled. This data structure // is used to implement unschedulableQ. type UnschedulablePodsMap struct { diff --git a/pkg/scheduler/internal/queue/scheduling_queue_test.go b/pkg/scheduler/internal/queue/scheduling_queue_test.go index cdce43adc6a..ca0e3369656 100644 --- a/pkg/scheduler/internal/queue/scheduling_queue_test.go +++ b/pkg/scheduler/internal/queue/scheduling_queue_test.go @@ -112,8 +112,8 @@ func TestPriorityQueue_Add(t *testing.T) { if p, err := q.Pop(); err != nil || p != &unschedulablePod { t.Errorf("Expected: %v after Pop, but got: %v", unschedulablePod.Name, p.Name) } - if len(q.nominatedPods) != 0 { - t.Errorf("Expected nomindatePods to be empty: %v", q.nominatedPods) + if len(q.nominatedPods["node1"]) != 2 { + t.Errorf("Expected medPriorityPod and unschedulablePod to be still present in nomindatePods: %v", q.nominatedPods["node1"]) } } @@ -135,8 +135,8 @@ func TestPriorityQueue_AddIfNotPresent(t *testing.T) { if p, err := q.Pop(); err != nil || p != &unschedulablePod { t.Errorf("Expected: %v after Pop, but got: %v", unschedulablePod.Name, p.Name) } - if len(q.nominatedPods) != 0 { - t.Errorf("Expected nomindatePods to be empty: %v", q.nominatedPods) + if len(q.nominatedPods["node1"]) != 2 { + t.Errorf("Expected medPriorityPod and unschedulablePod to be still present in nomindatePods: %v", q.nominatedPods["node1"]) } if q.unschedulableQ.get(&highPriNominatedPod) != &highPriNominatedPod { t.Errorf("Pod %v was not found in the unschedulableQ.", highPriNominatedPod.Name) @@ -178,8 +178,8 @@ func TestPriorityQueue_Pop(t *testing.T) { if p, err := q.Pop(); err != nil || p != &medPriorityPod { t.Errorf("Expected: %v after Pop, but got: %v", medPriorityPod.Name, p.Name) } - if len(q.nominatedPods) != 0 { - t.Errorf("Expected nomindatePods to be empty: %v", q.nominatedPods) + if len(q.nominatedPods["node1"]) != 1 { + t.Errorf("Expected medPriorityPod to be present in nomindatePods: %v", q.nominatedPods["node1"]) } }() q.Add(&medPriorityPod) diff --git a/pkg/scheduler/main_test.go b/pkg/scheduler/main_test.go new file mode 100644 index 00000000000..7644879e952 --- /dev/null +++ b/pkg/scheduler/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduler + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index a30d5b50dd5..05aa16b6ab5 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -45,7 +45,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/metrics" "k8s.io/kubernetes/pkg/scheduler/util" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -59,12 +59,6 @@ type Scheduler struct { config *factory.Config } -// StopEverything closes the scheduler config's StopEverything channel, to shut -// down the Scheduler. -func (sched *Scheduler) StopEverything() { - close(sched.config.StopEverything) -} - // Cache returns the cache in scheduler for test to check the data in scheduler. func (sched *Scheduler) Cache() schedulerinternalcache.Cache { return sched.config.SchedulerCache @@ -147,6 +141,7 @@ func New(client clientset.Interface, storageClassInformer storageinformers.StorageClassInformer, recorder record.EventRecorder, schedulerAlgorithmSource kubeschedulerconfig.SchedulerAlgorithmSource, + stopCh <-chan struct{}, opts ...func(o *schedulerOptions)) (*Scheduler, error) { options := defaultSchedulerOptions @@ -189,34 +184,12 @@ func New(client clientset.Interface, policy := &schedulerapi.Policy{} switch { case source.Policy.File != nil: - // Use a policy serialized in a file. - policyFile := source.Policy.File.Path - _, err := os.Stat(policyFile) - if err != nil { - return nil, fmt.Errorf("missing policy config file %s", policyFile) - } - data, err := ioutil.ReadFile(policyFile) - if err != nil { - return nil, fmt.Errorf("couldn't read policy config: %v", err) - } - err = runtime.DecodeInto(latestschedulerapi.Codec, []byte(data), policy) - if err != nil { - return nil, fmt.Errorf("invalid policy: %v", err) + if err := initPolicyFromFile(source.Policy.File.Path, policy); err != nil { + return nil, err } case source.Policy.ConfigMap != nil: - // Use a policy serialized in a config map value. - policyRef := source.Policy.ConfigMap - policyConfigMap, err := client.CoreV1().ConfigMaps(policyRef.Namespace).Get(policyRef.Name, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("couldn't get policy config map %s/%s: %v", policyRef.Namespace, policyRef.Name, err) - } - data, found := policyConfigMap.Data[kubeschedulerconfig.SchedulerPolicyConfigMapKey] - if !found { - return nil, fmt.Errorf("missing policy config map value at key %q", kubeschedulerconfig.SchedulerPolicyConfigMapKey) - } - err = runtime.DecodeInto(latestschedulerapi.Codec, []byte(data), policy) - if err != nil { - return nil, fmt.Errorf("invalid policy: %v", err) + if err := initPolicyFromConfigMap(client, source.Policy.ConfigMap, policy); err != nil { + return nil, err } } sc, err := configurator.CreateFromConfig(*policy) @@ -230,11 +203,48 @@ func New(client clientset.Interface, // Additional tweaks to the config produced by the configurator. config.Recorder = recorder config.DisablePreemption = options.disablePreemption + config.StopEverything = stopCh // Create the scheduler. sched := NewFromConfig(config) return sched, nil } +// initPolicyFromFile initialize policy from file +func initPolicyFromFile(policyFile string, policy *schedulerapi.Policy) error { + // Use a policy serialized in a file. + _, err := os.Stat(policyFile) + if err != nil { + return fmt.Errorf("missing policy config file %s", policyFile) + } + data, err := ioutil.ReadFile(policyFile) + if err != nil { + return fmt.Errorf("couldn't read policy config: %v", err) + } + err = runtime.DecodeInto(latestschedulerapi.Codec, []byte(data), policy) + if err != nil { + return fmt.Errorf("invalid policy: %v", err) + } + return nil +} + +// initPolicyFromConfigMap initialize policy from configMap +func initPolicyFromConfigMap(client clientset.Interface, policyRef *kubeschedulerconfig.SchedulerPolicyConfigMapSource, policy *schedulerapi.Policy) error { + // Use a policy serialized in a config map value. + policyConfigMap, err := client.CoreV1().ConfigMaps(policyRef.Namespace).Get(policyRef.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("couldn't get policy config map %s/%s: %v", policyRef.Namespace, policyRef.Name, err) + } + data, found := policyConfigMap.Data[kubeschedulerconfig.SchedulerPolicyConfigMapKey] + if !found { + return fmt.Errorf("missing policy config map value at key %q", kubeschedulerconfig.SchedulerPolicyConfigMapKey) + } + err = runtime.DecodeInto(latestschedulerapi.Codec, []byte(data), policy) + if err != nil { + return fmt.Errorf("invalid policy: %v", err) + } + return nil +} + // NewFromConfigurator returns a new scheduler that is created entirely by the Configurator. Assumes Create() is implemented. // Supports intermediate Config mutation for now if you provide modifier functions which will run after Config is created. func NewFromConfigurator(c factory.Configurator, modifiers ...func(c *factory.Config)) (*Scheduler, error) { @@ -299,20 +309,20 @@ func (sched *Scheduler) schedule(pod *v1.Pod) (string, error) { // It returns the node name and an error if any. func (sched *Scheduler) preempt(preemptor *v1.Pod, scheduleErr error) (string, error) { if !util.PodPriorityEnabled() || sched.config.DisablePreemption { - glog.V(3).Infof("Pod priority feature is not enabled or preemption is disabled by scheduler configuration." + + klog.V(3).Infof("Pod priority feature is not enabled or preemption is disabled by scheduler configuration." + " No preemption is performed.") return "", nil } preemptor, err := sched.config.PodPreemptor.GetUpdatedPod(preemptor) if err != nil { - glog.Errorf("Error getting the updated preemptor pod object: %v", err) + klog.Errorf("Error getting the updated preemptor pod object: %v", err) return "", err } node, victims, nominatedPodsToClear, err := sched.config.Algorithm.Preempt(preemptor, sched.config.NodeLister, scheduleErr) metrics.PreemptionVictims.Set(float64(len(victims))) if err != nil { - glog.Errorf("Error preempting victims to make room for %v/%v.", preemptor.Namespace, preemptor.Name) + klog.Errorf("Error preempting victims to make room for %v/%v.", preemptor.Namespace, preemptor.Name) return "", err } var nodeName = "" @@ -320,12 +330,12 @@ func (sched *Scheduler) preempt(preemptor *v1.Pod, scheduleErr error) (string, e nodeName = node.Name err = sched.config.PodPreemptor.SetNominatedNodeName(preemptor, nodeName) if err != nil { - glog.Errorf("Error in preemption process. Cannot update pod %v/%v annotations: %v", preemptor.Namespace, preemptor.Name, err) + klog.Errorf("Error in preemption process. Cannot update pod %v/%v annotations: %v", preemptor.Namespace, preemptor.Name, err) return "", err } for _, victim := range victims { if err := sched.config.PodPreemptor.DeletePod(victim); err != nil { - glog.Errorf("Error preempting pod %v/%v: %v", victim.Namespace, victim.Name, err) + klog.Errorf("Error preempting pod %v/%v: %v", victim.Namespace, victim.Name, err) return "", err } sched.config.Recorder.Eventf(victim, v1.EventTypeNormal, "Preempted", "by %v/%v on node %v", preemptor.Namespace, preemptor.Name, nodeName) @@ -338,7 +348,7 @@ func (sched *Scheduler) preempt(preemptor *v1.Pod, scheduleErr error) (string, e for _, p := range nominatedPodsToClear { rErr := sched.config.PodPreemptor.RemoveNominatedNodeName(p) if rErr != nil { - glog.Errorf("Cannot remove nominated node annotation of pod: %v", rErr) + klog.Errorf("Cannot remove nominated node annotation of pod: %v", rErr) // We do not return as this error is not critical. } } @@ -380,14 +390,14 @@ func (sched *Scheduler) bindVolumes(assumed *v1.Pod) error { var reason string var eventType string - glog.V(5).Infof("Trying to bind volumes for pod \"%v/%v\"", assumed.Namespace, assumed.Name) + klog.V(5).Infof("Trying to bind volumes for pod \"%v/%v\"", assumed.Namespace, assumed.Name) err := sched.config.VolumeBinder.Binder.BindPodVolumes(assumed) if err != nil { - glog.V(1).Infof("Failed to bind volumes for pod \"%v/%v\": %v", assumed.Namespace, assumed.Name, err) + klog.V(1).Infof("Failed to bind volumes for pod \"%v/%v\": %v", assumed.Namespace, assumed.Name, err) // Unassume the Pod and retry scheduling if forgetErr := sched.config.SchedulerCache.ForgetPod(assumed); forgetErr != nil { - glog.Errorf("scheduler cache ForgetPod failed: %v", forgetErr) + klog.Errorf("scheduler cache ForgetPod failed: %v", forgetErr) } reason = "VolumeBindingFailed" @@ -402,7 +412,7 @@ func (sched *Scheduler) bindVolumes(assumed *v1.Pod) error { return err } - glog.V(5).Infof("Success binding volumes for pod \"%v/%v\"", assumed.Namespace, assumed.Name) + klog.V(5).Infof("Success binding volumes for pod \"%v/%v\"", assumed.Namespace, assumed.Name) return nil } @@ -420,7 +430,7 @@ func (sched *Scheduler) assume(assumed *v1.Pod, host string) error { // snapshotted before updates are written, we would update equivalence // cache with stale information which is based on snapshot of old cache. if err := sched.config.SchedulerCache.AssumePod(assumed); err != nil { - glog.Errorf("scheduler cache AssumePod failed: %v", err) + klog.Errorf("scheduler cache AssumePod failed: %v", err) // This is most probably result of a BUG in retrying logic. // We report an error here so that pod scheduling can be retried. @@ -437,6 +447,10 @@ func (sched *Scheduler) assume(assumed *v1.Pod, host string) error { }) return err } + // if "assumed" is a nominated pod, we should remove it from internal cache + if sched.config.SchedulingQueue != nil { + sched.config.SchedulingQueue.DeleteNominatedPodIfExists(assumed) + } // Optimistically assume that the binding will succeed, so we need to invalidate affected // predicates in equivalence cache. @@ -455,12 +469,12 @@ func (sched *Scheduler) bind(assumed *v1.Pod, b *v1.Binding) error { // it's atomic with setting host. err := sched.config.GetBinder(assumed).Bind(b) if finErr := sched.config.SchedulerCache.FinishBinding(assumed); finErr != nil { - glog.Errorf("scheduler cache FinishBinding failed: %v", finErr) + klog.Errorf("scheduler cache FinishBinding failed: %v", finErr) } if err != nil { - glog.V(1).Infof("Failed to bind pod: %v/%v", assumed.Namespace, assumed.Name) + klog.V(1).Infof("Failed to bind pod: %v/%v", assumed.Namespace, assumed.Name) if err := sched.config.SchedulerCache.ForgetPod(assumed); err != nil { - glog.Errorf("scheduler cache ForgetPod failed: %v", err) + klog.Errorf("scheduler cache ForgetPod failed: %v", err) } sched.config.Error(assumed, err) sched.config.Recorder.Eventf(assumed, v1.EventTypeWarning, "FailedScheduling", "Binding rejected: %v", err) @@ -487,11 +501,11 @@ func (sched *Scheduler) scheduleOne() { } if pod.DeletionTimestamp != nil { sched.config.Recorder.Eventf(pod, v1.EventTypeWarning, "FailedScheduling", "skip schedule deleting pod: %v/%v", pod.Namespace, pod.Name) - glog.V(3).Infof("Skip schedule deleting pod: %v/%v", pod.Namespace, pod.Name) + klog.V(3).Infof("Skip schedule deleting pod: %v/%v", pod.Namespace, pod.Name) return } - glog.V(3).Infof("Attempting to schedule pod: %v/%v", pod.Namespace, pod.Name) + klog.V(3).Infof("Attempting to schedule pod: %v/%v", pod.Namespace, pod.Name) // Synchronously attempt to find a fit for the pod. start := time.Now() @@ -512,7 +526,7 @@ func (sched *Scheduler) scheduleOne() { // schedule it. (hopefully) metrics.PodScheduleFailures.Inc() } else { - glog.Errorf("error selecting node for pod: %v", err) + klog.Errorf("error selecting node for pod: %v", err) metrics.PodScheduleErrors.Inc() } return @@ -531,7 +545,7 @@ func (sched *Scheduler) scheduleOne() { // This function modifies 'assumedPod' if volume binding is required. allBound, err := sched.assumeVolumes(assumedPod, suggestedHost) if err != nil { - glog.Errorf("error assuming volumes: %v", err) + klog.Errorf("error assuming volumes: %v", err) metrics.PodScheduleErrors.Inc() return } @@ -539,7 +553,7 @@ func (sched *Scheduler) scheduleOne() { // assume modifies `assumedPod` by setting NodeName=suggestedHost err = sched.assume(assumedPod, suggestedHost) if err != nil { - glog.Errorf("error assuming pod: %v", err) + klog.Errorf("error assuming pod: %v", err) metrics.PodScheduleErrors.Inc() return } @@ -549,7 +563,7 @@ func (sched *Scheduler) scheduleOne() { if !allBound { err := sched.bindVolumes(assumedPod) if err != nil { - glog.Errorf("error binding volumes: %v", err) + klog.Errorf("error binding volumes: %v", err) metrics.PodScheduleErrors.Inc() return } @@ -564,7 +578,7 @@ func (sched *Scheduler) scheduleOne() { }) metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start)) if err != nil { - glog.Errorf("error binding pod: %v", err) + klog.Errorf("error binding pod: %v", err) metrics.PodScheduleErrors.Inc() } else { metrics.PodScheduleSuccesses.Inc() diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 699418ac840..4dac1a7bd2d 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/informers" clientsetfake "k8s.io/client-go/kubernetes/fake" corelister "k8s.io/client-go/listers/core/v1" @@ -40,6 +41,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/controller/volume/persistentvolume" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/algorithm" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" "k8s.io/kubernetes/pkg/scheduler/api" @@ -175,6 +177,8 @@ func TestSchedulerCreation(t *testing.T) { factory.RegisterPriorityFunction("PriorityOne", PriorityOne, 1) factory.RegisterAlgorithmProvider(testSource, sets.NewString("PredicateOne"), sets.NewString("PriorityOne")) + stopCh := make(chan struct{}) + defer close(stopCh) _, err := New(client, informerFactory.Core().V1().Nodes(), factory.NewPodInformer(client, 0), @@ -188,6 +192,7 @@ func TestSchedulerCreation(t *testing.T) { informerFactory.Storage().V1().StorageClasses(), eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "scheduler"}), kubeschedulerconfig.SchedulerAlgorithmSource{Provider: &testSource}, + stopCh, WithBindTimeoutSeconds(defaultBindTimeout)) if err != nil { @@ -771,8 +776,7 @@ func TestSchedulerWithVolumeBinding(t *testing.T) { // This can be small because we wait for pod to finish scheduling first chanTimeout := 2 * time.Second - utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - defer utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() table := []struct { name string diff --git a/pkg/scheduler/util/BUILD b/pkg/scheduler/util/BUILD index 57839b6708e..810d2c5cb0a 100644 --- a/pkg/scheduler/util/BUILD +++ b/pkg/scheduler/util/BUILD @@ -34,7 +34,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/scheduler/util/backoff_utils.go b/pkg/scheduler/util/backoff_utils.go index 50920ae86c8..506cd1270ac 100644 --- a/pkg/scheduler/util/backoff_utils.go +++ b/pkg/scheduler/util/backoff_utils.go @@ -24,7 +24,7 @@ import ( ktypes "k8s.io/apimachinery/pkg/types" - "github.com/golang/glog" + "k8s.io/klog" ) type clock interface { @@ -76,7 +76,7 @@ func (b *BackoffEntry) getBackoff(maxDuration time.Duration) time.Duration { newDuration = maxDuration } b.backoff = newDuration - glog.V(4).Infof("Backing off %s", duration.String()) + klog.V(4).Infof("Backing off %s", duration.String()) return duration } diff --git a/pkg/security/podsecuritypolicy/OWNERS b/pkg/security/podsecuritypolicy/OWNERS index 10c4b132f5e..787630abd2f 100644 --- a/pkg/security/podsecuritypolicy/OWNERS +++ b/pkg/security/podsecuritypolicy/OWNERS @@ -1,10 +1,7 @@ approvers: - - deads2k - - liggitt - - pweil- - - tallclair +- sig-auth-policy-approvers reviewers: - - deads2k - - liggitt - - pweil- - - tallclair +- sig-auth-policy-reviewers +labels: +- sig/auth + diff --git a/pkg/serviceaccount/BUILD b/pkg/serviceaccount/BUILD index 71a9836dce4..d0ad133b626 100644 --- a/pkg/serviceaccount/BUILD +++ b/pkg/serviceaccount/BUILD @@ -22,9 +22,9 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/gopkg.in/square/go-jose.v2:go_default_library", "//vendor/gopkg.in/square/go-jose.v2/jwt:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -54,6 +54,7 @@ go_test( "//pkg/controller/serviceaccount:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", diff --git a/pkg/serviceaccount/OWNERS b/pkg/serviceaccount/OWNERS index af89d3e6d65..d914c0d7195 100644 --- a/pkg/serviceaccount/OWNERS +++ b/pkg/serviceaccount/OWNERS @@ -1,9 +1,7 @@ approvers: -- liggitt -- deads2k -- mikedanese +- sig-auth-serviceaccounts-approvers reviewers: -- liggitt -- deads2k -- mikedanese -- enj +- sig-auth-serviceaccounts-reviewers +labels: +- sig/auth + diff --git a/pkg/serviceaccount/claims.go b/pkg/serviceaccount/claims.go index 7e36d39cc29..3d48b6f2dce 100644 --- a/pkg/serviceaccount/claims.go +++ b/pkg/serviceaccount/claims.go @@ -21,8 +21,8 @@ import ( "fmt" "time" - "github.com/golang/glog" "gopkg.in/square/go-jose.v2/jwt" + "k8s.io/klog" apiserverserviceaccount "k8s.io/apiserver/pkg/authentication/serviceaccount" "k8s.io/kubernetes/pkg/apis/core" @@ -80,15 +80,13 @@ func Claims(sa core.ServiceAccount, pod *core.Pod, secret *core.Secret, expirati return sc, pc } -func NewValidator(audiences []string, getter ServiceAccountTokenGetter) Validator { +func NewValidator(getter ServiceAccountTokenGetter) Validator { return &validator{ - auds: audiences, getter: getter, } } type validator struct { - auds []string getter ServiceAccountTokenGetter } @@ -97,7 +95,7 @@ var _ = Validator(&validator{}) func (v *validator) Validate(_ string, public *jwt.Claims, privateObj interface{}) (*ServiceAccountInfo, error) { private, ok := privateObj.(*privateClaims) if !ok { - glog.Errorf("jwt validator expected private claim of type *privateClaims but got: %T", privateObj) + klog.Errorf("jwt validator expected private claim of type *privateClaims but got: %T", privateObj) return nil, errors.New("Token could not be validated.") } err := public.Validate(jwt.Expected{ @@ -108,23 +106,10 @@ func (v *validator) Validate(_ string, public *jwt.Claims, privateObj interface{ case err == jwt.ErrExpired: return nil, errors.New("Token has expired.") default: - glog.Errorf("unexpected validation error: %T", err) + klog.Errorf("unexpected validation error: %T", err) return nil, errors.New("Token could not be validated.") } - var audValid bool - - for _, aud := range v.auds { - audValid = public.Audience.Contains(aud) - if audValid { - break - } - } - - if !audValid { - return nil, errors.New("Token is invalid for this audience.") - } - namespace := private.Kubernetes.Namespace saref := private.Kubernetes.Svcacct podref := private.Kubernetes.Pod @@ -132,15 +117,15 @@ func (v *validator) Validate(_ string, public *jwt.Claims, privateObj interface{ // Make sure service account still exists (name and UID) serviceAccount, err := v.getter.GetServiceAccount(namespace, saref.Name) if err != nil { - glog.V(4).Infof("Could not retrieve service account %s/%s: %v", namespace, saref.Name, err) + klog.V(4).Infof("Could not retrieve service account %s/%s: %v", namespace, saref.Name, err) return nil, err } if serviceAccount.DeletionTimestamp != nil { - glog.V(4).Infof("Service account has been deleted %s/%s", namespace, saref.Name) + klog.V(4).Infof("Service account has been deleted %s/%s", namespace, saref.Name) return nil, fmt.Errorf("ServiceAccount %s/%s has been deleted", namespace, saref.Name) } if string(serviceAccount.UID) != saref.UID { - glog.V(4).Infof("Service account UID no longer matches %s/%s: %q != %q", namespace, saref.Name, string(serviceAccount.UID), saref.UID) + klog.V(4).Infof("Service account UID no longer matches %s/%s: %q != %q", namespace, saref.Name, string(serviceAccount.UID), saref.UID) return nil, fmt.Errorf("ServiceAccount UID (%s) does not match claim (%s)", serviceAccount.UID, saref.UID) } @@ -148,15 +133,15 @@ func (v *validator) Validate(_ string, public *jwt.Claims, privateObj interface{ // Make sure token hasn't been invalidated by deletion of the secret secret, err := v.getter.GetSecret(namespace, secref.Name) if err != nil { - glog.V(4).Infof("Could not retrieve bound secret %s/%s for service account %s/%s: %v", namespace, secref.Name, namespace, saref.Name, err) + klog.V(4).Infof("Could not retrieve bound secret %s/%s for service account %s/%s: %v", namespace, secref.Name, namespace, saref.Name, err) return nil, errors.New("Token has been invalidated") } if secret.DeletionTimestamp != nil { - glog.V(4).Infof("Bound secret is deleted and awaiting removal: %s/%s for service account %s/%s", namespace, secref.Name, namespace, saref.Name) + klog.V(4).Infof("Bound secret is deleted and awaiting removal: %s/%s for service account %s/%s", namespace, secref.Name, namespace, saref.Name) return nil, errors.New("Token has been invalidated") } if secref.UID != string(secret.UID) { - glog.V(4).Infof("Secret UID no longer matches %s/%s: %q != %q", namespace, secref.Name, string(secret.UID), secref.UID) + klog.V(4).Infof("Secret UID no longer matches %s/%s: %q != %q", namespace, secref.Name, string(secret.UID), secref.UID) return nil, fmt.Errorf("Secret UID (%s) does not match claim (%s)", secret.UID, secref.UID) } } @@ -166,15 +151,15 @@ func (v *validator) Validate(_ string, public *jwt.Claims, privateObj interface{ // Make sure token hasn't been invalidated by deletion of the pod pod, err := v.getter.GetPod(namespace, podref.Name) if err != nil { - glog.V(4).Infof("Could not retrieve bound pod %s/%s for service account %s/%s: %v", namespace, podref.Name, namespace, saref.Name, err) + klog.V(4).Infof("Could not retrieve bound pod %s/%s for service account %s/%s: %v", namespace, podref.Name, namespace, saref.Name, err) return nil, errors.New("Token has been invalidated") } if pod.DeletionTimestamp != nil { - glog.V(4).Infof("Bound pod is deleted and awaiting removal: %s/%s for service account %s/%s", namespace, podref.Name, namespace, saref.Name) + klog.V(4).Infof("Bound pod is deleted and awaiting removal: %s/%s for service account %s/%s", namespace, podref.Name, namespace, saref.Name) return nil, errors.New("Token has been invalidated") } if podref.UID != string(pod.UID) { - glog.V(4).Infof("Pod UID no longer matches %s/%s: %q != %q", namespace, podref.Name, string(pod.UID), podref.UID) + klog.V(4).Infof("Pod UID no longer matches %s/%s: %q != %q", namespace, podref.Name, string(pod.UID), podref.UID) return nil, fmt.Errorf("Pod UID (%s) does not match claim (%s)", pod.UID, podref.UID) } podName = podref.Name diff --git a/pkg/serviceaccount/jwt.go b/pkg/serviceaccount/jwt.go index 23c8934fbf3..233fdee2d68 100644 --- a/pkg/serviceaccount/jwt.go +++ b/pkg/serviceaccount/jwt.go @@ -111,18 +111,20 @@ func (j *jwtTokenGenerator) GenerateToken(claims *jwt.Claims, privateClaims inte // JWTTokenAuthenticator authenticates tokens as JWT tokens produced by JWTTokenGenerator // Token signatures are verified using each of the given public keys until one works (allowing key rotation) // If lookup is true, the service account and secret referenced as claims inside the token are retrieved and verified with the provided ServiceAccountTokenGetter -func JWTTokenAuthenticator(iss string, keys []interface{}, validator Validator) authenticator.Token { +func JWTTokenAuthenticator(iss string, keys []interface{}, implicitAuds authenticator.Audiences, validator Validator) authenticator.Token { return &jwtTokenAuthenticator{ - iss: iss, - keys: keys, - validator: validator, + iss: iss, + keys: keys, + implicitAuds: implicitAuds, + validator: validator, } } type jwtTokenAuthenticator struct { - iss string - keys []interface{} - validator Validator + iss string + keys []interface{} + validator Validator + implicitAuds authenticator.Audiences } // Validator is called by the JWT token authenticator to apply domain specific @@ -170,6 +172,23 @@ func (j *jwtTokenAuthenticator) AuthenticateToken(ctx context.Context, tokenData return nil, false, utilerrors.NewAggregate(errlist) } + tokenAudiences := authenticator.Audiences(public.Audience) + if len(tokenAudiences) == 0 { + // only apiserver audiences are allowed for legacy tokens + tokenAudiences = j.implicitAuds + } + + requestedAudiences, ok := authenticator.AudiencesFrom(ctx) + if !ok { + // default to apiserver audiences + requestedAudiences = j.implicitAuds + } + + auds := authenticator.Audiences(tokenAudiences).Intersect(requestedAudiences) + if len(auds) == 0 && len(j.implicitAuds) != 0 { + return nil, false, fmt.Errorf("token audiences %q is invalid for the target audiences %q", tokenAudiences, requestedAudiences) + } + // If we get here, we have a token with a recognized signature and // issuer string. sa, err := j.validator.Validate(tokenData, public, private) @@ -177,7 +196,10 @@ func (j *jwtTokenAuthenticator) AuthenticateToken(ctx context.Context, tokenData return nil, false, err } - return &authenticator.Response{User: sa.UserInfo()}, true, nil + return &authenticator.Response{ + User: sa.UserInfo(), + Audiences: auds, + }, true, nil } // hasCorrectIssuer returns true if tokenData is a valid JWT in compact diff --git a/pkg/serviceaccount/jwt_test.go b/pkg/serviceaccount/jwt_test.go index 61cca23e12a..36be027c09d 100644 --- a/pkg/serviceaccount/jwt_test.go +++ b/pkg/serviceaccount/jwt_test.go @@ -23,6 +23,7 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/authentication/authenticator" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" certutil "k8s.io/client-go/util/cert" @@ -275,16 +276,18 @@ func TestTokenGenerateAndValidate(t *testing.T) { } for k, tc := range testCases { + auds := authenticator.Audiences{"api"} getter := serviceaccountcontroller.NewGetterFromClient(tc.Client) - authenticator := serviceaccount.JWTTokenAuthenticator(serviceaccount.LegacyIssuer, tc.Keys, serviceaccount.NewLegacyValidator(tc.Client != nil, getter)) + authn := serviceaccount.JWTTokenAuthenticator(serviceaccount.LegacyIssuer, tc.Keys, auds, serviceaccount.NewLegacyValidator(tc.Client != nil, getter)) // An invalid, non-JWT token should always fail - if _, ok, err := authenticator.AuthenticateToken(context.Background(), "invalid token"); err != nil || ok { + ctx := authenticator.WithAudiences(context.Background(), auds) + if _, ok, err := authn.AuthenticateToken(ctx, "invalid token"); err != nil || ok { t.Errorf("%s: Expected err=nil, ok=false for non-JWT token", k) continue } - resp, ok, err := authenticator.AuthenticateToken(context.Background(), tc.Token) + resp, ok, err := authn.AuthenticateToken(ctx, tc.Token) if (err != nil) != tc.ExpectedErr { t.Errorf("%s: Expected error=%v, got %v", k, tc.ExpectedErr, err) continue diff --git a/pkg/serviceaccount/legacy.go b/pkg/serviceaccount/legacy.go index 4d0a32c2cda..57c482f0ba6 100644 --- a/pkg/serviceaccount/legacy.go +++ b/pkg/serviceaccount/legacy.go @@ -21,8 +21,8 @@ import ( "errors" "fmt" - "github.com/golang/glog" "gopkg.in/square/go-jose.v2/jwt" + "k8s.io/klog" "k8s.io/api/core/v1" apiserverserviceaccount "k8s.io/apiserver/pkg/authentication/serviceaccount" @@ -65,7 +65,7 @@ var _ = Validator(&legacyValidator{}) func (v *legacyValidator) Validate(tokenData string, public *jwt.Claims, privateObj interface{}) (*ServiceAccountInfo, error) { private, ok := privateObj.(*legacyPrivateClaims) if !ok { - glog.Errorf("jwt validator expected private claim of type *legacyPrivateClaims but got: %T", privateObj) + klog.Errorf("jwt validator expected private claim of type *legacyPrivateClaims but got: %T", privateObj) return nil, errors.New("Token could not be validated.") } @@ -99,30 +99,30 @@ func (v *legacyValidator) Validate(tokenData string, public *jwt.Claims, private // Make sure token hasn't been invalidated by deletion of the secret secret, err := v.getter.GetSecret(namespace, secretName) if err != nil { - glog.V(4).Infof("Could not retrieve token %s/%s for service account %s/%s: %v", namespace, secretName, namespace, serviceAccountName, err) + klog.V(4).Infof("Could not retrieve token %s/%s for service account %s/%s: %v", namespace, secretName, namespace, serviceAccountName, err) return nil, errors.New("Token has been invalidated") } if secret.DeletionTimestamp != nil { - glog.V(4).Infof("Token is deleted and awaiting removal: %s/%s for service account %s/%s", namespace, secretName, namespace, serviceAccountName) + klog.V(4).Infof("Token is deleted and awaiting removal: %s/%s for service account %s/%s", namespace, secretName, namespace, serviceAccountName) return nil, errors.New("Token has been invalidated") } if bytes.Compare(secret.Data[v1.ServiceAccountTokenKey], []byte(tokenData)) != 0 { - glog.V(4).Infof("Token contents no longer matches %s/%s for service account %s/%s", namespace, secretName, namespace, serviceAccountName) + klog.V(4).Infof("Token contents no longer matches %s/%s for service account %s/%s", namespace, secretName, namespace, serviceAccountName) return nil, errors.New("Token does not match server's copy") } // Make sure service account still exists (name and UID) serviceAccount, err := v.getter.GetServiceAccount(namespace, serviceAccountName) if err != nil { - glog.V(4).Infof("Could not retrieve service account %s/%s: %v", namespace, serviceAccountName, err) + klog.V(4).Infof("Could not retrieve service account %s/%s: %v", namespace, serviceAccountName, err) return nil, err } if serviceAccount.DeletionTimestamp != nil { - glog.V(4).Infof("Service account has been deleted %s/%s", namespace, serviceAccountName) + klog.V(4).Infof("Service account has been deleted %s/%s", namespace, serviceAccountName) return nil, fmt.Errorf("ServiceAccount %s/%s has been deleted", namespace, serviceAccountName) } if string(serviceAccount.UID) != serviceAccountUID { - glog.V(4).Infof("Service account UID no longer matches %s/%s: %q != %q", namespace, serviceAccountName, string(serviceAccount.UID), serviceAccountUID) + klog.V(4).Infof("Service account UID no longer matches %s/%s: %q != %q", namespace, serviceAccountName, string(serviceAccount.UID), serviceAccountUID) return nil, fmt.Errorf("ServiceAccount UID (%s) does not match claim (%s)", serviceAccount.UID, serviceAccountUID) } } diff --git a/pkg/ssh/BUILD b/pkg/ssh/BUILD index ea52417688d..2cc03ea717c 100644 --- a/pkg/ssh/BUILD +++ b/pkg/ssh/BUILD @@ -12,8 +12,8 @@ go_test( embed = [":go_default_library"], deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/crypto/ssh:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -25,9 +25,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/golang.org/x/crypto/ssh:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/ssh/ssh.go b/pkg/ssh/ssh.go index dc8aa3acc1e..bee21f6ee5a 100644 --- a/pkg/ssh/ssh.go +++ b/pkg/ssh/ssh.go @@ -37,9 +37,9 @@ import ( "sync" "time" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" "golang.org/x/crypto/ssh" + "k8s.io/klog" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/runtime" @@ -79,7 +79,7 @@ type SSHTunnel struct { func (s *SSHTunnel) copyBytes(out io.Writer, in io.Reader) { if _, err := io.Copy(out, in); err != nil { - glog.Errorf("Error in SSH tunnel: %v", err) + klog.Errorf("Error in SSH tunnel: %v", err) } } @@ -353,8 +353,8 @@ func (l *SSHTunnelList) delayedHealthCheck(e sshTunnelEntry, delay time.Duration defer runtime.HandleCrash() time.Sleep(delay) if err := l.healthCheck(e); err != nil { - glog.Errorf("Healthcheck failed for tunnel to %q: %v", e.Address, err) - glog.Infof("Attempting once to re-establish tunnel to %q", e.Address) + klog.Errorf("Healthcheck failed for tunnel to %q: %v", e.Address, err) + klog.Infof("Attempting once to re-establish tunnel to %q", e.Address) l.removeAndReAdd(e) } }() @@ -391,7 +391,7 @@ func (l *SSHTunnelList) removeAndReAdd(e sshTunnelEntry) { } l.tunnelsLock.Unlock() if err := e.Tunnel.Close(); err != nil { - glog.Infof("Failed to close removed tunnel: %v", err) + klog.Infof("Failed to close removed tunnel: %v", err) } go l.createAndAddTunnel(e.Address) } @@ -399,9 +399,9 @@ func (l *SSHTunnelList) removeAndReAdd(e sshTunnelEntry) { func (l *SSHTunnelList) Dial(ctx context.Context, net, addr string) (net.Conn, error) { start := time.Now() id := mathrand.Int63() // So you can match begins/ends in the log. - glog.Infof("[%x: %v] Dialing...", id, addr) + klog.Infof("[%x: %v] Dialing...", id, addr) defer func() { - glog.Infof("[%x: %v] Dialed in %v.", id, addr, time.Since(start)) + klog.Infof("[%x: %v] Dialed in %v.", id, addr, time.Since(start)) }() tunnel, err := l.pickTunnel(strings.Split(addr, ":")[0]) if err != nil { @@ -423,7 +423,7 @@ func (l *SSHTunnelList) pickTunnel(addr string) (tunnel, error) { return entry.Tunnel, nil } } - glog.Warningf("SSH tunnel not found for address %q, picking random node", addr) + klog.Warningf("SSH tunnel not found for address %q, picking random node", addr) n := mathrand.Intn(len(l.entries)) return l.entries[n].Tunnel, nil } @@ -464,11 +464,11 @@ func (l *SSHTunnelList) Update(addrs []string) { for i := range l.entries { if _, ok := wantAddrsMap[l.entries[i].Address]; !ok { tunnelEntry := l.entries[i] - glog.Infof("Removing tunnel to deleted node at %q", tunnelEntry.Address) + klog.Infof("Removing tunnel to deleted node at %q", tunnelEntry.Address) go func() { defer runtime.HandleCrash() if err := tunnelEntry.Tunnel.Close(); err != nil { - glog.Errorf("Failed to close tunnel to %q: %v", tunnelEntry.Address, err) + klog.Errorf("Failed to close tunnel to %q: %v", tunnelEntry.Address, err) } }() } else { @@ -480,14 +480,14 @@ func (l *SSHTunnelList) Update(addrs []string) { } func (l *SSHTunnelList) createAndAddTunnel(addr string) { - glog.Infof("Trying to add tunnel to %q", addr) + klog.Infof("Trying to add tunnel to %q", addr) tunnel, err := l.tunnelCreator.NewSSHTunnel(l.user, l.keyfile, addr) if err != nil { - glog.Errorf("Failed to create tunnel for %q: %v", addr, err) + klog.Errorf("Failed to create tunnel for %q: %v", addr, err) return } if err := tunnel.Open(); err != nil { - glog.Errorf("Failed to open tunnel to %q: %v", addr, err) + klog.Errorf("Failed to open tunnel to %q: %v", addr, err) l.tunnelsLock.Lock() delete(l.adding, addr) l.tunnelsLock.Unlock() @@ -497,7 +497,7 @@ func (l *SSHTunnelList) createAndAddTunnel(addr string) { l.entries = append(l.entries, sshTunnelEntry{addr, tunnel}) delete(l.adding, addr) l.tunnelsLock.Unlock() - glog.Infof("Successfully added tunnel for %q", addr) + klog.Infof("Successfully added tunnel for %q", addr) } func EncodePrivateKey(private *rsa.PrivateKey) []byte { diff --git a/pkg/ssh/ssh_test.go b/pkg/ssh/ssh_test.go index a8098973d28..6148233f7ec 100644 --- a/pkg/ssh/ssh_test.go +++ b/pkg/ssh/ssh_test.go @@ -29,8 +29,8 @@ import ( "k8s.io/apimachinery/pkg/util/wait" - "github.com/golang/glog" "golang.org/x/crypto/ssh" + "k8s.io/klog" ) type testSSHServer struct { @@ -94,11 +94,11 @@ func runTestSSHServer(user, password string) (*testSSHServer, error) { conn, err := listener.Accept() if err != nil { - glog.Errorf("Failed to accept: %v", err) + klog.Errorf("Failed to accept: %v", err) } _, chans, reqs, err := ssh.NewServerConn(conn, config) if err != nil { - glog.Errorf("Failed handshake: %v", err) + klog.Errorf("Failed handshake: %v", err) } go ssh.DiscardRequests(reqs) for newChannel := range chans { @@ -108,11 +108,11 @@ func runTestSSHServer(user, password string) (*testSSHServer, error) { } channel, requests, err := newChannel.Accept() if err != nil { - glog.Errorf("Failed to accept channel: %v", err) + klog.Errorf("Failed to accept channel: %v", err) } for req := range requests { - glog.Infof("Got request: %v", req) + klog.Infof("Got request: %v", req) } channel.Close() diff --git a/pkg/util/async/BUILD b/pkg/util/async/BUILD index 57830ca887a..c201c307be9 100644 --- a/pkg/util/async/BUILD +++ b/pkg/util/async/BUILD @@ -15,7 +15,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/util/async", deps = [ "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/util/async/bounded_frequency_runner.go b/pkg/util/async/bounded_frequency_runner.go index f03a7ec2d5d..20a06ad668a 100644 --- a/pkg/util/async/bounded_frequency_runner.go +++ b/pkg/util/async/bounded_frequency_runner.go @@ -23,7 +23,7 @@ import ( "k8s.io/client-go/util/flowcontrol" - "github.com/golang/glog" + "k8s.io/klog" ) // BoundedFrequencyRunner manages runs of a user-provided function. @@ -167,13 +167,13 @@ func construct(name string, fn func(), minInterval, maxInterval time.Duration, b // Loop handles the periodic timer and run requests. This is expected to be // called as a goroutine. func (bfr *BoundedFrequencyRunner) Loop(stop <-chan struct{}) { - glog.V(3).Infof("%s Loop running", bfr.name) + klog.V(3).Infof("%s Loop running", bfr.name) bfr.timer.Reset(bfr.maxInterval) for { select { case <-stop: bfr.stop() - glog.V(3).Infof("%s Loop stopping", bfr.name) + klog.V(3).Infof("%s Loop stopping", bfr.name) return case <-bfr.timer.C(): bfr.tryRun() @@ -218,7 +218,7 @@ func (bfr *BoundedFrequencyRunner) tryRun() { bfr.lastRun = bfr.timer.Now() bfr.timer.Stop() bfr.timer.Reset(bfr.maxInterval) - glog.V(3).Infof("%s: ran, next possible in %v, periodic in %v", bfr.name, bfr.minInterval, bfr.maxInterval) + klog.V(3).Infof("%s: ran, next possible in %v, periodic in %v", bfr.name, bfr.minInterval, bfr.maxInterval) return } @@ -227,13 +227,13 @@ func (bfr *BoundedFrequencyRunner) tryRun() { elapsed := bfr.timer.Since(bfr.lastRun) // how long since last run nextPossible := bfr.minInterval - elapsed // time to next possible run nextScheduled := bfr.maxInterval - elapsed // time to next periodic run - glog.V(4).Infof("%s: %v since last run, possible in %v, scheduled in %v", bfr.name, elapsed, nextPossible, nextScheduled) + klog.V(4).Infof("%s: %v since last run, possible in %v, scheduled in %v", bfr.name, elapsed, nextPossible, nextScheduled) if nextPossible < nextScheduled { // Set the timer for ASAP, but don't drain here. Assuming Loop is running, // it might get a delivery in the mean time, but that is OK. bfr.timer.Stop() bfr.timer.Reset(nextPossible) - glog.V(3).Infof("%s: throttled, scheduling run in %v", bfr.name, nextPossible) + klog.V(3).Infof("%s: throttled, scheduling run in %v", bfr.name, nextPossible) } } diff --git a/pkg/util/bandwidth/BUILD b/pkg/util/bandwidth/BUILD index 2854892376d..bc590bbfa7c 100644 --- a/pkg/util/bandwidth/BUILD +++ b/pkg/util/bandwidth/BUILD @@ -22,7 +22,7 @@ go_library( ] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], "//conditions:default": [], diff --git a/pkg/util/bandwidth/linux.go b/pkg/util/bandwidth/linux.go index b8936a34f01..7050b4f763c 100644 --- a/pkg/util/bandwidth/linux.go +++ b/pkg/util/bandwidth/linux.go @@ -30,7 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/exec" - "github.com/golang/glog" + "k8s.io/klog" ) // tcShaper provides an implementation of the BandwidthShaper interface on Linux using the 'tc' tool. @@ -53,10 +53,10 @@ func NewTCShaper(iface string) BandwidthShaper { } func (t *tcShaper) execAndLog(cmdStr string, args ...string) error { - glog.V(6).Infof("Running: %s %s", cmdStr, strings.Join(args, " ")) + klog.V(6).Infof("Running: %s %s", cmdStr, strings.Join(args, " ")) cmd := t.e.Command(cmdStr, args...) out, err := cmd.CombinedOutput() - glog.V(6).Infof("Output from tc: %s", string(out)) + klog.V(6).Infof("Output from tc: %s", string(out)) return err } @@ -259,7 +259,7 @@ func (t *tcShaper) ReconcileInterface() error { return err } if !exists { - glog.V(4).Info("Didn't find bandwidth interface, creating") + klog.V(4).Info("Didn't find bandwidth interface, creating") return t.initializeInterface() } fields := strings.Split(output, " ") diff --git a/pkg/util/coverage/coverage.go b/pkg/util/coverage/coverage.go index a6cdb2e73d4..2a36558e335 100644 --- a/pkg/util/coverage/coverage.go +++ b/pkg/util/coverage/coverage.go @@ -23,8 +23,8 @@ package coverage import ( "flag" "fmt" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" "os" "testing" "time" @@ -86,6 +86,6 @@ func FlushCoverage() { // This gets us atomic updates from the perspective of another process trying to access // the file. if err := os.Rename(tempCoveragePath(), coverageFile); err != nil { - glog.Errorf("Couldn't move coverage file from %s to %s", coverageFile, tempCoveragePath()) + klog.Errorf("Couldn't move coverage file from %s to %s", coverageFile, tempCoveragePath()) } } diff --git a/pkg/util/flag/BUILD b/pkg/util/flag/BUILD index ac88251b33c..976dd85000b 100644 --- a/pkg/util/flag/BUILD +++ b/pkg/util/flag/BUILD @@ -12,8 +12,8 @@ go_library( importpath = "k8s.io/kubernetes/pkg/util/flag", deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/util/flag/flags.go b/pkg/util/flag/flags.go index b58a52cc0e7..1d57c3e8d6b 100644 --- a/pkg/util/flag/flags.go +++ b/pkg/util/flag/flags.go @@ -21,8 +21,8 @@ import ( "net" "strconv" - "github.com/golang/glog" "github.com/spf13/pflag" + "k8s.io/klog" utilnet "k8s.io/apimachinery/pkg/util/net" ) @@ -30,7 +30,7 @@ import ( // PrintFlags logs the flags in the flagset func PrintFlags(flags *pflag.FlagSet) { flags.VisitAll(func(flag *pflag.Flag) { - glog.V(1).Infof("FLAG: --%s=%q", flag.Name, flag.Value) + klog.V(1).Infof("FLAG: --%s=%q", flag.Name, flag.Value) }) } diff --git a/pkg/util/goroutinemap/BUILD b/pkg/util/goroutinemap/BUILD index f1d77a03354..92688fde686 100644 --- a/pkg/util/goroutinemap/BUILD +++ b/pkg/util/goroutinemap/BUILD @@ -13,7 +13,7 @@ go_library( deps = [ "//pkg/util/goroutinemap/exponentialbackoff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/util/goroutinemap/goroutinemap.go b/pkg/util/goroutinemap/goroutinemap.go index 474e1215cc3..9b6eb731927 100644 --- a/pkg/util/goroutinemap/goroutinemap.go +++ b/pkg/util/goroutinemap/goroutinemap.go @@ -25,8 +25,8 @@ import ( "fmt" "sync" - "github.com/golang/glog" k8sRuntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff" ) @@ -135,7 +135,7 @@ func (grm *goRoutineMap) operationComplete( delete(grm.operations, operationName) if *err != nil { // Log error - glog.Errorf("operation for %q failed with: %v", + klog.Errorf("operation for %q failed with: %v", operationName, *err) } @@ -147,7 +147,7 @@ func (grm *goRoutineMap) operationComplete( grm.operations[operationName] = existingOp // Log error - glog.Errorf("%v", + klog.Errorf("%v", existingOp.expBackoff.GenerateNoRetriesPermittedMsg(operationName)) } } diff --git a/pkg/util/ipconfig/BUILD b/pkg/util/ipconfig/BUILD index ffcecee1031..39504f89bbb 100644 --- a/pkg/util/ipconfig/BUILD +++ b/pkg/util/ipconfig/BUILD @@ -14,7 +14,7 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/util/ipconfig", deps = [ - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/util/ipconfig/ipconfig.go b/pkg/util/ipconfig/ipconfig.go index 924c20b94a6..5764511602b 100644 --- a/pkg/util/ipconfig/ipconfig.go +++ b/pkg/util/ipconfig/ipconfig.go @@ -21,7 +21,7 @@ import ( "strings" "sync" - "github.com/golang/glog" + "k8s.io/klog" utilexec "k8s.io/utils/exec" ) @@ -66,7 +66,7 @@ func (runner *runner) GetDNSSuffixSearchList() ([]string, error) { // TODO: this does not work when the label is localized suffixList := []string{} if runtime.GOOS != "windows" { - glog.V(1).Infof("ipconfig not supported on GOOS=%s", runtime.GOOS) + klog.V(1).Infof("ipconfig not supported on GOOS=%s", runtime.GOOS) return suffixList, nil } @@ -92,7 +92,7 @@ func (runner *runner) GetDNSSuffixSearchList() ([]string, error) { } } } else { - glog.V(1).Infof("Running %s %s failed: %v", cmdIpconfig, cmdDefaultArgs, err) + klog.V(1).Infof("Running %s %s failed: %v", cmdIpconfig, cmdDefaultArgs, err) } return suffixList, err diff --git a/pkg/util/ipset/BUILD b/pkg/util/ipset/BUILD index 8be0d8d63d9..186fa432043 100644 --- a/pkg/util/ipset/BUILD +++ b/pkg/util/ipset/BUILD @@ -9,7 +9,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/util/ipset", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/util/ipset/ipset.go b/pkg/util/ipset/ipset.go index fd367f28121..2d400a6244e 100644 --- a/pkg/util/ipset/ipset.go +++ b/pkg/util/ipset/ipset.go @@ -24,7 +24,7 @@ import ( "strconv" "strings" - "github.com/golang/glog" + "k8s.io/klog" utilexec "k8s.io/utils/exec" ) @@ -111,12 +111,12 @@ func (set *IPSet) Validate() bool { } // check hash size value of ipset if set.HashSize <= 0 { - glog.Errorf("Invalid hashsize value %d, should be >0", set.HashSize) + klog.Errorf("Invalid hashsize value %d, should be >0", set.HashSize) return false } // check max elem value of ipset if set.MaxElem <= 0 { - glog.Errorf("Invalid maxelem value %d, should be >0", set.MaxElem) + klog.Errorf("Invalid maxelem value %d, should be >0", set.MaxElem) return false } @@ -167,7 +167,7 @@ type Entry struct { // Validate checks if a given ipset entry is valid or not. The set parameter is the ipset that entry belongs to. func (e *Entry) Validate(set *IPSet) bool { if e.Port < 0 { - glog.Errorf("Entry %v port number %d should be >=0 for ipset %v", e, e.Port, set) + klog.Errorf("Entry %v port number %d should be >=0 for ipset %v", e, e.Port, set) return false } switch e.SetType { @@ -184,7 +184,7 @@ func (e *Entry) Validate(set *IPSet) bool { // IP2 can not be empty for `hash:ip,port,ip` type ip set if net.ParseIP(e.IP2) == nil { - glog.Errorf("Error parsing entry %v second ip address %v for ipset %v", e, e.IP2, set) + klog.Errorf("Error parsing entry %v second ip address %v for ipset %v", e, e.IP2, set) return false } case HashIPPortNet: @@ -195,22 +195,22 @@ func (e *Entry) Validate(set *IPSet) bool { // Net can not be empty for `hash:ip,port,net` type ip set if _, ipNet, err := net.ParseCIDR(e.Net); ipNet == nil { - glog.Errorf("Error parsing entry %v ip net %v for ipset %v, error: %v", e, e.Net, set, err) + klog.Errorf("Error parsing entry %v ip net %v for ipset %v, error: %v", e, e.Net, set, err) return false } case BitmapPort: // check if port number satisfies its ipset's requirement of port range if set == nil { - glog.Errorf("Unable to reference ip set where the entry %v exists", e) + klog.Errorf("Unable to reference ip set where the entry %v exists", e) return false } begin, end, err := parsePortRange(set.PortRange) if err != nil { - glog.Errorf("Failed to parse set %v port range %s for ipset %v, error: %v", set, set.PortRange, set, err) + klog.Errorf("Failed to parse set %v port range %s for ipset %v, error: %v", set, set.PortRange, set, err) return false } if e.Port < begin || e.Port > end { - glog.Errorf("Entry %v port number %d is not in the port range %s of its ipset %v", e, e.Port, set.PortRange, set) + klog.Errorf("Entry %v port number %d is not in the port range %s of its ipset %v", e, e.Port, set.PortRange, set) return false } } @@ -251,7 +251,7 @@ func (e *Entry) checkIPandProtocol(set *IPSet) bool { } if net.ParseIP(e.IP) == nil { - glog.Errorf("Error parsing entry %v ip address %v for ipset %v", e, e.IP, set) + klog.Errorf("Error parsing entry %v ip address %v for ipset %v", e, e.IP, set) return false } @@ -424,17 +424,17 @@ func getIPSetVersionString(exec utilexec.Interface) (string, error) { func validatePortRange(portRange string) bool { strs := strings.Split(portRange, "-") if len(strs) != 2 { - glog.Errorf("port range should be in the format of `a-b`") + klog.Errorf("port range should be in the format of `a-b`") return false } for i := range strs { num, err := strconv.Atoi(strs[i]) if err != nil { - glog.Errorf("Failed to parse %s, error: %v", strs[i], err) + klog.Errorf("Failed to parse %s, error: %v", strs[i], err) return false } if num < 0 { - glog.Errorf("port number %d should be >=0", num) + klog.Errorf("port number %d should be >=0", num) return false } } @@ -448,7 +448,7 @@ func validateIPSetType(set Type) bool { return true } } - glog.Errorf("Currently supported ipset types are: %v, %s is not supported", ValidIPSetTypes, set) + klog.Errorf("Currently supported ipset types are: %v, %s is not supported", ValidIPSetTypes, set) return false } @@ -457,7 +457,7 @@ func validateHashFamily(family string) bool { if family == ProtocolFamilyIPV4 || family == ProtocolFamilyIPV6 { return true } - glog.Errorf("Currently supported ip set hash families are: [%s, %s], %s is not supported", ProtocolFamilyIPV4, ProtocolFamilyIPV6, family) + klog.Errorf("Currently supported ip set hash families are: [%s, %s], %s is not supported", ProtocolFamilyIPV4, ProtocolFamilyIPV6, family) return false } @@ -485,7 +485,7 @@ func validateProtocol(protocol string) bool { if protocol == ProtocolTCP || protocol == ProtocolUDP || protocol == ProtocolSCTP { return true } - glog.Errorf("Invalid entry's protocol: %s, supported protocols are [%s, %s, %s]", protocol, ProtocolTCP, ProtocolUDP, ProtocolSCTP) + klog.Errorf("Invalid entry's protocol: %s, supported protocols are [%s, %s, %s]", protocol, ProtocolTCP, ProtocolUDP, ProtocolSCTP) return false } diff --git a/pkg/util/iptables/BUILD b/pkg/util/iptables/BUILD index 40529906a58..9b289951f52 100644 --- a/pkg/util/iptables/BUILD +++ b/pkg/util/iptables/BUILD @@ -22,7 +22,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/trace:go_default_library", "//vendor/github.com/godbus/dbus:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ diff --git a/pkg/util/iptables/iptables.go b/pkg/util/iptables/iptables.go index e81c21497ba..c9f3d9860d1 100644 --- a/pkg/util/iptables/iptables.go +++ b/pkg/util/iptables/iptables.go @@ -26,10 +26,10 @@ import ( "time" godbus "github.com/godbus/dbus" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/sets" utilversion "k8s.io/apimachinery/pkg/util/version" utiltrace "k8s.io/apiserver/pkg/util/trace" + "k8s.io/klog" utildbus "k8s.io/kubernetes/pkg/util/dbus" utilexec "k8s.io/utils/exec" ) @@ -152,7 +152,7 @@ type runner struct { func newInternal(exec utilexec.Interface, dbus utildbus.Interface, protocol Protocol, lockfilePath string) Interface { vstring, err := getIPTablesVersionString(exec, protocol) if err != nil { - glog.Warningf("Error checking iptables version, assuming version at least %s: %v", MinCheckVersion, err) + klog.Warningf("Error checking iptables version, assuming version at least %s: %v", MinCheckVersion, err) vstring = MinCheckVersion } @@ -197,7 +197,7 @@ const ( func (runner *runner) connectToFirewallD() { bus, err := runner.dbus.SystemBus() if err != nil { - glog.V(1).Infof("Could not connect to D-Bus system bus: %s", err) + klog.V(1).Infof("Could not connect to D-Bus system bus: %s", err) return } runner.hasListener = true @@ -324,7 +324,7 @@ func (runner *runner) SaveInto(table Table, buffer *bytes.Buffer) error { // run and return iptablesSaveCmd := iptablesSaveCommand(runner.protocol) args := []string{"-t", string(table)} - glog.V(4).Infof("running %s %v", iptablesSaveCmd, args) + klog.V(4).Infof("running %s %v", iptablesSaveCmd, args) cmd := runner.exec.Command(iptablesSaveCmd, args...) // Since CombinedOutput() doesn't support redirecting it to a buffer, // we need to workaround it by redirecting stdout and stderr to buffer @@ -380,7 +380,7 @@ func (runner *runner) restoreInternal(args []string, data []byte, flush FlushFla trace.Step("Locks grabbed") defer func(locker iptablesLocker) { if err := locker.Close(); err != nil { - glog.Errorf("Failed to close iptables locks: %v", err) + klog.Errorf("Failed to close iptables locks: %v", err) } }(locker) } @@ -388,7 +388,7 @@ func (runner *runner) restoreInternal(args []string, data []byte, flush FlushFla // run the command and return the output or an error including the output and error fullArgs := append(runner.restoreWaitFlag, args...) iptablesRestoreCmd := iptablesRestoreCommand(runner.protocol) - glog.V(4).Infof("running %s %v", iptablesRestoreCmd, fullArgs) + klog.V(4).Infof("running %s %v", iptablesRestoreCmd, fullArgs) cmd := runner.exec.Command(iptablesRestoreCmd, fullArgs...) cmd.SetStdin(bytes.NewBuffer(data)) b, err := cmd.CombinedOutput() @@ -430,7 +430,7 @@ func (runner *runner) runContext(ctx context.Context, op operation, args []strin iptablesCmd := iptablesCommand(runner.protocol) fullArgs := append(runner.waitFlag, string(op)) fullArgs = append(fullArgs, args...) - glog.V(5).Infof("running iptables %s %v", string(op), args) + klog.V(5).Infof("running iptables %s %v", string(op), args) if ctx == nil { return runner.exec.Command(iptablesCmd, fullArgs...).CombinedOutput() } @@ -458,7 +458,7 @@ func trimhex(s string) string { // of hack and half-measures. We should nix this ASAP. func (runner *runner) checkRuleWithoutCheck(table Table, chain Chain, args ...string) (bool, error) { iptablesSaveCmd := iptablesSaveCommand(runner.protocol) - glog.V(1).Infof("running %s -t %s", iptablesSaveCmd, string(table)) + klog.V(1).Infof("running %s -t %s", iptablesSaveCmd, string(table)) out, err := runner.exec.Command(iptablesSaveCmd, "-t", string(table)).CombinedOutput() if err != nil { return false, fmt.Errorf("error checking rule: %v", err) @@ -497,7 +497,7 @@ func (runner *runner) checkRuleWithoutCheck(table Table, chain Chain, args ...st if sets.NewString(fields...).IsSuperset(argset) { return true, nil } - glog.V(5).Infof("DBG: fields is not a superset of args: fields=%v args=%v", fields, args) + klog.V(5).Infof("DBG: fields is not a superset of args: fields=%v args=%v", fields, args) } return false, nil @@ -544,12 +544,12 @@ func makeFullArgs(table Table, chain Chain, args ...string) []string { func getIPTablesHasCheckCommand(vstring string) bool { minVersion, err := utilversion.ParseGeneric(MinCheckVersion) if err != nil { - glog.Errorf("MinCheckVersion (%s) is not a valid version string: %v", MinCheckVersion, err) + klog.Errorf("MinCheckVersion (%s) is not a valid version string: %v", MinCheckVersion, err) return true } version, err := utilversion.ParseGeneric(vstring) if err != nil { - glog.Errorf("vstring (%s) is not a valid version string: %v", vstring, err) + klog.Errorf("vstring (%s) is not a valid version string: %v", vstring, err) return true } return version.AtLeast(minVersion) @@ -559,13 +559,13 @@ func getIPTablesHasCheckCommand(vstring string) bool { func getIPTablesWaitFlag(vstring string) []string { version, err := utilversion.ParseGeneric(vstring) if err != nil { - glog.Errorf("vstring (%s) is not a valid version string: %v", vstring, err) + klog.Errorf("vstring (%s) is not a valid version string: %v", vstring, err) return nil } minVersion, err := utilversion.ParseGeneric(WaitMinVersion) if err != nil { - glog.Errorf("WaitMinVersion (%s) is not a valid version string: %v", WaitMinVersion, err) + klog.Errorf("WaitMinVersion (%s) is not a valid version string: %v", WaitMinVersion, err) return nil } if version.LessThan(minVersion) { @@ -574,7 +574,7 @@ func getIPTablesWaitFlag(vstring string) []string { minVersion, err = utilversion.ParseGeneric(WaitSecondsMinVersion) if err != nil { - glog.Errorf("WaitSecondsMinVersion (%s) is not a valid version string: %v", WaitSecondsMinVersion, err) + klog.Errorf("WaitSecondsMinVersion (%s) is not a valid version string: %v", WaitSecondsMinVersion, err) return nil } if version.LessThan(minVersion) { @@ -608,11 +608,11 @@ func getIPTablesVersionString(exec utilexec.Interface, protocol Protocol) (strin func getIPTablesRestoreWaitFlag(exec utilexec.Interface, protocol Protocol) []string { vstring, err := getIPTablesRestoreVersionString(exec, protocol) if err != nil || vstring == "" { - glog.V(3).Infof("couldn't get iptables-restore version; assuming it doesn't support --wait") + klog.V(3).Infof("couldn't get iptables-restore version; assuming it doesn't support --wait") return nil } if _, err := utilversion.ParseGeneric(vstring); err != nil { - glog.V(3).Infof("couldn't parse iptables-restore version; assuming it doesn't support --wait") + klog.V(3).Infof("couldn't parse iptables-restore version; assuming it doesn't support --wait") return nil } @@ -691,7 +691,7 @@ func (runner *runner) AddReloadFunc(reloadFunc func()) { // runs all reload funcs to re-sync iptables rules func (runner *runner) reload() { - glog.V(1).Infof("reloading iptables rules") + klog.V(1).Infof("reloading iptables rules") for _, f := range runner.reloadFuncs { f() diff --git a/pkg/util/ipvs/BUILD b/pkg/util/ipvs/BUILD index 9e487e4968a..784a343e37d 100644 --- a/pkg/util/ipvs/BUILD +++ b/pkg/util/ipvs/BUILD @@ -34,42 +34,14 @@ go_library( "kernelcheck_unsupported.go", ], importpath = "k8s.io/kubernetes/pkg/util/ipvs", - deps = select({ - "@io_bazel_rules_go//go/platform:android": [ - "//vendor/k8s.io/utils/exec:go_default_library", - ], - "@io_bazel_rules_go//go/platform:darwin": [ - "//vendor/k8s.io/utils/exec:go_default_library", - ], - "@io_bazel_rules_go//go/platform:dragonfly": [ - "//vendor/k8s.io/utils/exec:go_default_library", - ], - "@io_bazel_rules_go//go/platform:freebsd": [ - "//vendor/k8s.io/utils/exec:go_default_library", - ], + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + ] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/github.com/docker/libnetwork/ipvs:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - ], - "@io_bazel_rules_go//go/platform:nacl": [ - "//vendor/k8s.io/utils/exec:go_default_library", - ], - "@io_bazel_rules_go//go/platform:netbsd": [ - "//vendor/k8s.io/utils/exec:go_default_library", - ], - "@io_bazel_rules_go//go/platform:openbsd": [ - "//vendor/k8s.io/utils/exec:go_default_library", - ], - "@io_bazel_rules_go//go/platform:plan9": [ - "//vendor/k8s.io/utils/exec:go_default_library", - ], - "@io_bazel_rules_go//go/platform:solaris": [ - "//vendor/k8s.io/utils/exec:go_default_library", - ], - "@io_bazel_rules_go//go/platform:windows": [ - "//vendor/k8s.io/utils/exec:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "//conditions:default": [], }), diff --git a/pkg/util/ipvs/ipvs.go b/pkg/util/ipvs/ipvs.go index 5bfa3b1a218..6cd081851ce 100644 --- a/pkg/util/ipvs/ipvs.go +++ b/pkg/util/ipvs/ipvs.go @@ -19,6 +19,11 @@ package ipvs import ( "net" "strconv" + "strings" + + "fmt" + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/utils/exec" ) // Interface is an injectable interface for running ipvs commands. Implementations must be goroutine-safe. @@ -67,14 +72,21 @@ const ( IPVSProxyMode = "ipvs" ) -// Sets of IPVS required kernel modules. -var ipvsModules = []string{ - "ip_vs", - "ip_vs_rr", - "ip_vs_wrr", - "ip_vs_sh", - "nf_conntrack_ipv4", -} +// IPVS required kernel modules. +const ( + // ModIPVS is the kernel module "ip_vs" + ModIPVS string = "ip_vs" + // ModIPVSRR is the kernel module "ip_vs_rr" + ModIPVSRR string = "ip_vs_rr" + // ModIPVSWRR is the kernel module "ip_vs_wrr" + ModIPVSWRR string = "ip_vs_wrr" + // ModIPVSSH is the kernel module "ip_vs_sh" + ModIPVSSH string = "ip_vs_sh" + // ModNfConntrackIPV4 is the module "nf_conntrack_ipv4" + ModNfConntrackIPV4 string = "nf_conntrack_ipv4" + // ModNfConntrack is the kernel module "nf_conntrack" + ModNfConntrack string = "nf_conntrack" +) // Equal check the equality of virtual server. // We don't use struct == since it doesn't work because of slice. @@ -110,3 +122,29 @@ func (rs *RealServer) Equal(other *RealServer) bool { return rs.Address.Equal(other.Address) && rs.Port == other.Port } + +// GetKernelVersionAndIPVSMods returns the linux kernel version and the required ipvs modules +func GetKernelVersionAndIPVSMods(Executor exec.Interface) (kernelVersion string, ipvsModules []string, err error) { + kernelVersionFile := "/proc/sys/kernel/osrelease" + out, err := Executor.Command("cut", "-f1", "-d", " ", kernelVersionFile).CombinedOutput() + if err != nil { + return "", nil, fmt.Errorf("error getting os release kernel version: %v(%s)", err, out) + } + kernelVersion = strings.TrimSpace(string(out)) + // parse kernel version + ver1, err := version.ParseGeneric(kernelVersion) + if err != nil { + return kernelVersion, nil, fmt.Errorf("error parsing kernel version: %v(%s)", err, kernelVersion) + } + // "nf_conntrack_ipv4" has been removed since v4.19 + // see https://github.com/torvalds/linux/commit/a0ae2562c6c4b2721d9fddba63b7286c13517d9f + ver2, _ := version.ParseGeneric("4.19") + // get required ipvs modules + if ver1.LessThan(ver2) { + ipvsModules = append(ipvsModules, ModIPVS, ModIPVSRR, ModIPVSWRR, ModIPVSSH, ModNfConntrackIPV4) + } else { + ipvsModules = append(ipvsModules, ModIPVS, ModIPVSRR, ModIPVSWRR, ModIPVSSH, ModNfConntrack) + } + + return kernelVersion, ipvsModules, nil +} diff --git a/pkg/util/ipvs/ipvs_linux.go b/pkg/util/ipvs/ipvs_linux.go index b7fcca90531..b4f53be6fdb 100644 --- a/pkg/util/ipvs/ipvs_linux.go +++ b/pkg/util/ipvs/ipvs_linux.go @@ -26,7 +26,7 @@ import ( "syscall" libipvs "github.com/docker/libnetwork/ipvs" - "github.com/golang/glog" + "k8s.io/klog" utilexec "k8s.io/utils/exec" ) @@ -43,7 +43,7 @@ type Protocol uint16 func New(exec utilexec.Interface) Interface { handle, err := libipvs.New("") if err != nil { - glog.Errorf("IPVS interface can't be initialized, error: %v", err) + klog.Errorf("IPVS interface can't be initialized, error: %v", err) return nil } return &runner{ diff --git a/pkg/util/ipvs/kernelcheck_linux.go b/pkg/util/ipvs/kernelcheck_linux.go index 83c3e6b0fe9..286a3098c0c 100644 --- a/pkg/util/ipvs/kernelcheck_linux.go +++ b/pkg/util/ipvs/kernelcheck_linux.go @@ -26,7 +26,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" utilsexec "k8s.io/utils/exec" - "github.com/golang/glog" + "k8s.io/klog" ) // RequiredIPVSKernelModulesAvailableCheck tests IPVS required kernel modules. @@ -42,7 +42,12 @@ func (r RequiredIPVSKernelModulesAvailableCheck) Name() string { // Check try to validates IPVS required kernel modules exists or not. // The name of function can not be changed. func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []error) { - glog.V(1).Infoln("validating the kernel module IPVS required exists in machine or not") + klog.V(1).Infoln("validating the kernel module IPVS required exists in machine or not") + + kernelVersion, ipvsModules, err := GetKernelVersionAndIPVSMods(r.Executor) + if err != nil { + errors = append(errors, err) + } // Find out loaded kernel modules out, err := r.Executor.Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput() @@ -60,14 +65,6 @@ func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []err // Check builtin modules exist or not if len(modules) != 0 { - kernelVersionFile := "/proc/sys/kernel/osrelease" - b, err := r.Executor.Command("cut", "-f1", "-d", " ", kernelVersionFile).CombinedOutput() - if err != nil { - errors = append(errors, fmt.Errorf("error getting os release kernel version: %v(%s)", err, out)) - return nil, errors - } - - kernelVersion := strings.TrimSpace(string(b)) builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersion) out, err := r.Executor.Command("cut", "-f1", "-d", " ", builtinModsFilePath).CombinedOutput() if err != nil { diff --git a/pkg/util/ipvs/kernelcheck_linux_test.go b/pkg/util/ipvs/kernelcheck_linux_test.go index 1f87a923901..8d2eb76465b 100644 --- a/pkg/util/ipvs/kernelcheck_linux_test.go +++ b/pkg/util/ipvs/kernelcheck_linux_test.go @@ -97,8 +97,8 @@ func TestRequiredIPVSKernelModulesAvailableCheck(t *testing.T) { for i, tc := range cases { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ - func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil }, func() ([]byte, error) { return []byte(cases[i].kernelVersion), nil }, + func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil }, func() ([]byte, error) { return []byte(cases[i].builtinKernel), nil }, }, } diff --git a/pkg/util/keymutex/BUILD b/pkg/util/keymutex/BUILD index 256ed34181e..267a4b1f633 100644 --- a/pkg/util/keymutex/BUILD +++ b/pkg/util/keymutex/BUILD @@ -13,7 +13,7 @@ go_library( "keymutex.go", ], importpath = "k8s.io/kubernetes/pkg/util/keymutex", - deps = ["//vendor/github.com/golang/glog:go_default_library"], + deps = ["//vendor/k8s.io/klog:go_default_library"], ) go_test( diff --git a/pkg/util/keymutex/hashed.go b/pkg/util/keymutex/hashed.go index 5fe9a025c24..5176ae916c2 100644 --- a/pkg/util/keymutex/hashed.go +++ b/pkg/util/keymutex/hashed.go @@ -21,7 +21,7 @@ import ( "runtime" "sync" - "github.com/golang/glog" + "k8s.io/klog" ) // NewHashed returns a new instance of KeyMutex which hashes arbitrary keys to @@ -44,16 +44,16 @@ type hashedKeyMutex struct { // Acquires a lock associated with the specified ID. func (km *hashedKeyMutex) LockKey(id string) { - glog.V(5).Infof("hashedKeyMutex.LockKey(...) called for id %q\r\n", id) + klog.V(5).Infof("hashedKeyMutex.LockKey(...) called for id %q\r\n", id) km.mutexes[km.hash(id)%len(km.mutexes)].Lock() - glog.V(5).Infof("hashedKeyMutex.LockKey(...) for id %q completed.\r\n", id) + klog.V(5).Infof("hashedKeyMutex.LockKey(...) for id %q completed.\r\n", id) } // Releases the lock associated with the specified ID. func (km *hashedKeyMutex) UnlockKey(id string) error { - glog.V(5).Infof("hashedKeyMutex.UnlockKey(...) called for id %q\r\n", id) + klog.V(5).Infof("hashedKeyMutex.UnlockKey(...) called for id %q\r\n", id) km.mutexes[km.hash(id)%len(km.mutexes)].Unlock() - glog.V(5).Infof("hashedKeyMutex.UnlockKey(...) for id %q completed.\r\n", id) + klog.V(5).Infof("hashedKeyMutex.UnlockKey(...) for id %q completed.\r\n", id) return nil } diff --git a/pkg/util/mount/BUILD b/pkg/util/mount/BUILD index 8a7211ec4af..221afb7a9c6 100644 --- a/pkg/util/mount/BUILD +++ b/pkg/util/mount/BUILD @@ -18,7 +18,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/util/mount", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:android": [ @@ -79,8 +79,8 @@ go_test( ] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//pkg/util/nsenter:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], "@io_bazel_rules_go//go/platform:windows": [ diff --git a/pkg/util/mount/exec_mount.go b/pkg/util/mount/exec_mount.go index 226f02704cc..634189dea9b 100644 --- a/pkg/util/mount/exec_mount.go +++ b/pkg/util/mount/exec_mount.go @@ -22,7 +22,7 @@ import ( "fmt" "os" - "github.com/golang/glog" + "k8s.io/klog" ) // ExecMounter is a mounter that uses provided Exec interface to mount and @@ -59,10 +59,10 @@ func (m *execMounter) Mount(source string, target string, fstype string, options // doExecMount calls exec(mount ) using given exec interface. func (m *execMounter) doExecMount(source, target, fstype string, options []string) error { - glog.V(5).Infof("Exec Mounting %s %s %s %v", source, target, fstype, options) + klog.V(5).Infof("Exec Mounting %s %s %s %v", source, target, fstype, options) mountArgs := makeMountArgs(source, target, fstype, options) output, err := m.exec.Run("mount", mountArgs...) - glog.V(5).Infof("Exec mounted %v: %v: %s", mountArgs, err, string(output)) + klog.V(5).Infof("Exec mounted %v: %v: %s", mountArgs, err, string(output)) if err != nil { return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s %s %s %v\nOutput: %s\n", err, "mount", source, target, fstype, options, string(output)) @@ -75,9 +75,9 @@ func (m *execMounter) doExecMount(source, target, fstype string, options []strin func (m *execMounter) Unmount(target string) error { outputBytes, err := m.exec.Run("umount", target) if err == nil { - glog.V(5).Infof("Exec unmounted %s: %s", target, string(outputBytes)) + klog.V(5).Infof("Exec unmounted %s: %s", target, string(outputBytes)) } else { - glog.V(5).Infof("Failed to exec unmount %s: err: %q, umount output: %s", target, err, string(outputBytes)) + klog.V(5).Infof("Failed to exec unmount %s: err: %q, umount output: %s", target, err, string(outputBytes)) } return err diff --git a/pkg/util/mount/fake.go b/pkg/util/mount/fake.go index 27279630c2f..06e0fcccdc1 100644 --- a/pkg/util/mount/fake.go +++ b/pkg/util/mount/fake.go @@ -22,7 +22,7 @@ import ( "path/filepath" "sync" - "github.com/golang/glog" + "k8s.io/klog" ) // FakeMounter implements mount.Interface for tests. @@ -93,7 +93,7 @@ func (f *FakeMounter) Mount(source string, target string, fstype string, options absTarget = target } f.MountPoints = append(f.MountPoints, MountPoint{Device: source, Path: absTarget, Type: fstype, Opts: opts}) - glog.V(5).Infof("Fake mounter: mounted %s to %s", source, absTarget) + klog.V(5).Infof("Fake mounter: mounted %s to %s", source, absTarget) f.Log = append(f.Log, FakeAction{Action: FakeActionMount, Target: absTarget, Source: source, FSType: fstype}) return nil } @@ -111,7 +111,7 @@ func (f *FakeMounter) Unmount(target string) error { newMountpoints := []MountPoint{} for _, mp := range f.MountPoints { if mp.Path == absTarget { - glog.V(5).Infof("Fake mounter: unmounted %s from %s", mp.Device, absTarget) + klog.V(5).Infof("Fake mounter: unmounted %s from %s", mp.Device, absTarget) // Don't copy it to newMountpoints continue } @@ -154,11 +154,11 @@ func (f *FakeMounter) IsLikelyNotMountPoint(file string) (bool, error) { for _, mp := range f.MountPoints { if mp.Path == absFile { - glog.V(5).Infof("isLikelyNotMountPoint for %s: mounted %s, false", file, mp.Path) + klog.V(5).Infof("isLikelyNotMountPoint for %s: mounted %s, false", file, mp.Path) return false, nil } } - glog.V(5).Infof("isLikelyNotMountPoint for %s: true", file) + klog.V(5).Infof("isLikelyNotMountPoint for %s: true", file) return true, nil } diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index c3f3a226afb..6ebeff053b5 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -30,9 +30,9 @@ import ( "strings" "syscall" - "github.com/golang/glog" "golang.org/x/sys/unix" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" utilfile "k8s.io/kubernetes/pkg/util/file" utilio "k8s.io/kubernetes/pkg/util/io" utilexec "k8s.io/utils/exec" @@ -143,12 +143,12 @@ func (m *Mounter) doMount(mounterPath string, mountCmd string, source string, ta // No code here, mountCmd and mountArgs are already populated. } - glog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgs) + klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgs) command := exec.Command(mountCmd, mountArgs...) output, err := command.CombinedOutput() if err != nil { args := strings.Join(mountArgs, " ") - glog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, args, string(output)) + klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, args, string(output)) return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, args, string(output)) } @@ -161,7 +161,7 @@ func (m *Mounter) doMount(mounterPath string, mountCmd string, source string, ta // systemd-runs (needed by Mount()) works. func detectSystemd() bool { if _, err := exec.LookPath("systemd-run"); err != nil { - glog.V(2).Infof("Detected OS without systemd") + klog.V(2).Infof("Detected OS without systemd") return false } // Try to run systemd-run --scope /bin/true, that should be enough @@ -171,12 +171,12 @@ func detectSystemd() bool { cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true") output, err := cmd.CombinedOutput() if err != nil { - glog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS") - glog.V(4).Infof("systemd-run failed with: %v", err) - glog.V(4).Infof("systemd-run output: %s", string(output)) + klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS") + klog.V(4).Infof("systemd-run failed with: %v", err) + klog.V(4).Infof("systemd-run output: %s", string(output)) return false } - glog.V(2).Infof("Detected OS with systemd") + klog.V(2).Infof("Detected OS with systemd") return true } @@ -208,7 +208,7 @@ func addSystemdScope(systemdRunPath, mountName, command string, args []string) ( // Unmount unmounts the target. func (mounter *Mounter) Unmount(target string) error { - glog.V(4).Infof("Unmounting %s", target) + klog.V(4).Infof("Unmounting %s", target) command := exec.Command("umount", target) output, err := command.CombinedOutput() if err != nil { @@ -290,7 +290,7 @@ func exclusiveOpenFailsOnDevice(pathname string) (bool, error) { } if !isDevice { - glog.Errorf("Path %q is not referring to a device.", pathname) + klog.Errorf("Path %q is not referring to a device.", pathname) return false, nil } fd, errno := unix.Open(pathname, unix.O_RDONLY|unix.O_EXCL, 0) @@ -319,11 +319,11 @@ func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (str func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { refs, err := mounter.GetMountRefs(mountPath) if err != nil { - glog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) + klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) return "", err } if len(refs) == 0 { - glog.V(4).Infof("Directory %s is not mounted", mountPath) + klog.V(4).Infof("Directory %s is not mounted", mountPath) return "", fmt.Errorf("directory %s is not mounted", mountPath) } basemountPath := path.Join(pluginDir, MountsInGlobalPDPath) @@ -331,7 +331,7 @@ func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (str if strings.HasPrefix(ref, basemountPath) { volumeID, err := filepath.Rel(basemountPath, ref) if err != nil { - glog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) + klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) return "", err } return volumeID, nil @@ -437,26 +437,26 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, if !readOnly { // Run fsck on the disk to fix repairable issues, only do this for volumes requested as rw. - glog.V(4).Infof("Checking for issues with fsck on disk: %s", source) + klog.V(4).Infof("Checking for issues with fsck on disk: %s", source) args := []string{"-a", source} out, err := mounter.Exec.Run("fsck", args...) if err != nil { ee, isExitError := err.(utilexec.ExitError) switch { case err == utilexec.ErrExecutableNotFound: - glog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.") + klog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.") case isExitError && ee.ExitStatus() == fsckErrorsCorrected: - glog.Infof("Device %s has errors which were corrected by fsck.", source) + klog.Infof("Device %s has errors which were corrected by fsck.", source) case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s.", source, string(out)) case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: - glog.Infof("`fsck` error %s", string(out)) + klog.Infof("`fsck` error %s", string(out)) } } } // Try to mount the disk - glog.V(4).Infof("Attempting to mount disk: %s %s %s", fstype, source, target) + klog.V(4).Infof("Attempting to mount disk: %s %s %s", fstype, source, target) mountErr := mounter.Interface.Mount(source, target, fstype, options) if mountErr != nil { // Mount failed. This indicates either that the disk is unformatted or @@ -485,14 +485,14 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, source, } } - glog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) + klog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) _, err := mounter.Exec.Run("mkfs."+fstype, args...) if err == nil { // the disk has been formatted successfully try to mount it again. - glog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target) + klog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target) return mounter.Interface.Mount(source, target, fstype, options) } - glog.Errorf("format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", source, fstype, target, options, err) + klog.Errorf("format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", source, fstype, target, options, err) return err } else { // Disk is already formatted and failed to mount @@ -511,10 +511,10 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, // GetDiskFormat uses 'blkid' to see if the given disk is unformated func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk} - glog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args) + klog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args) dataOut, err := mounter.Exec.Run("blkid", args...) output := string(dataOut) - glog.V(4).Infof("Output: %q, err: %v", output, err) + klog.V(4).Infof("Output: %q, err: %v", output, err) if err != nil { if exit, ok := err.(utilexec.ExitError); ok { @@ -526,7 +526,7 @@ func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { return "", nil } } - glog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) + klog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) return "", err } @@ -552,7 +552,7 @@ func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { } if len(pttype) > 0 { - glog.V(4).Infof("Disk %s detected partition table type: %s", disk, pttype) + klog.V(4).Infof("Disk %s detected partition table type: %s", disk, pttype) // Returns a special non-empty string as filesystem type, then kubelet // will not format it. return "unknown data, probably partitions", nil @@ -686,11 +686,11 @@ func doMakeRShared(path string, mountInfoFilename string) error { return err } if shared { - glog.V(4).Infof("Directory %s is already on a shared mount", path) + klog.V(4).Infof("Directory %s is already on a shared mount", path) return nil } - glog.V(2).Infof("Bind-mounting %q with shared mount propagation", path) + klog.V(2).Infof("Bind-mounting %q with shared mount propagation", path) // mount --bind /var/lib/kubelet /var/lib/kubelet if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_BIND, "" /*data*/); err != nil { return fmt.Errorf("failed to bind-mount %s: %v", path, err) @@ -766,7 +766,7 @@ func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, err } if !notMount { // It's already mounted - glog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget) + klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget) return true, bindPathTarget, nil } @@ -819,7 +819,7 @@ func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err err if err != nil { return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err) } - glog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath) + klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath) subpath.VolumePath = newVolumePath subpath.Path = newPath @@ -841,9 +841,9 @@ func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err err defer func() { // Cleanup subpath on error if !success { - glog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) + klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil { - glog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) + klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) } } }() @@ -853,13 +853,13 @@ func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err err // Do the bind mount options := []string{"bind"} - glog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget) + klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget) if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil { return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err) } success = true - glog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) + klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) return bindPathTarget, nil } @@ -871,7 +871,7 @@ func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error { // scan /var/lib/kubelet/pods//volume-subpaths//* subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName) - glog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir) + klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir) containerDirs, err := ioutil.ReadDir(subPathDir) if err != nil { @@ -883,10 +883,10 @@ func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error for _, containerDir := range containerDirs { if !containerDir.IsDir() { - glog.V(4).Infof("Container file is not a directory: %s", containerDir.Name()) + klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name()) continue } - glog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name()) + klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name()) // scan /var/lib/kubelet/pods//volume-subpaths///* fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name()) @@ -903,27 +903,27 @@ func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error if err := os.Remove(fullContainerDirPath); err != nil { return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err) } - glog.V(5).Infof("Removed %s", fullContainerDirPath) + klog.V(5).Infof("Removed %s", fullContainerDirPath) } // Whole pod volume subpaths have been cleaned up, remove its subpath directory. if err := os.Remove(subPathDir); err != nil { return fmt.Errorf("error deleting %s: %s", subPathDir, err) } - glog.V(5).Infof("Removed %s", subPathDir) + klog.V(5).Infof("Removed %s", subPathDir) // Remove entire subpath directory if it's the last one podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName) if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) { return fmt.Errorf("error deleting %s: %s", podSubPathDir, err) } - glog.V(5).Infof("Removed %s", podSubPathDir) + klog.V(5).Infof("Removed %s", podSubPathDir) return nil } // doCleanSubPath tears down the single subpath bind mount func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error { // process /var/lib/kubelet/pods//volume-subpaths/// - glog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex) + klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex) fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex) notMnt, err := IsNotMountPoint(mounter, fullSubPath) if err != nil { @@ -934,13 +934,13 @@ func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string if err = mounter.Unmount(fullSubPath); err != nil { return fmt.Errorf("error unmounting %s: %s", fullSubPath, err) } - glog.V(5).Infof("Unmounted %s", fullSubPath) + klog.V(5).Infof("Unmounted %s", fullSubPath) } // Remove it *non*-recursively, just in case there were some hiccups. if err = os.Remove(fullSubPath); err != nil { return fmt.Errorf("error deleting %s: %s", fullSubPath, err) } - glog.V(5).Infof("Removed %s", fullSubPath) + klog.V(5).Infof("Removed %s", fullSubPath) return nil } @@ -972,7 +972,7 @@ func removeEmptyDirs(baseDir, endDir string) error { s, err := os.Stat(curDir) if err != nil { if os.IsNotExist(err) { - glog.V(5).Infof("curDir %q doesn't exist, skipping", curDir) + klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir) continue } return fmt.Errorf("error stat %q: %v", curDir, err) @@ -983,12 +983,12 @@ func removeEmptyDirs(baseDir, endDir string) error { err = os.Remove(curDir) if os.IsExist(err) { - glog.V(5).Infof("Directory %q not empty, not removing", curDir) + klog.V(5).Infof("Directory %q not empty, not removing", curDir) break } else if err != nil { return fmt.Errorf("error removing directory %q: %v", curDir, err) } - glog.V(5).Infof("Removed directory %q", curDir) + klog.V(5).Infof("Removed directory %q", curDir) } return nil } @@ -1055,7 +1055,7 @@ func getMode(pathname string) (os.FileMode, error) { // and base must be either already resolved symlinks or thet will be resolved in // kubelet's mount namespace (in case it runs containerized). func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { - glog.V(4).Infof("Creating directory %q within base %q", pathname, base) + klog.V(4).Infof("Creating directory %q within base %q", pathname, base) if !PathWithinBase(pathname, base) { return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) @@ -1068,7 +1068,7 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { if s.IsDir() { // The directory already exists. It can be outside of the parent, // but there is no race-proof check. - glog.V(4).Infof("Directory %s already exists", pathname) + klog.V(4).Infof("Directory %s already exists", pathname) return nil } return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR} @@ -1088,7 +1088,7 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) } - glog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) + klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) parentFD, err := doSafeOpen(fullExistingPath, base) if err != nil { return fmt.Errorf("cannot open directory %s: %s", existingPath, err) @@ -1097,12 +1097,12 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { defer func() { if parentFD != -1 { if err = syscall.Close(parentFD); err != nil { - glog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) } } if childFD != -1 { if err = syscall.Close(childFD); err != nil { - glog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err) + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err) } } }() @@ -1112,7 +1112,7 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { // created directory into symlink. for _, dir := range toCreate { currentPath = filepath.Join(currentPath, dir) - glog.V(4).Infof("Creating %s", dir) + klog.V(4).Infof("Creating %s", dir) err = syscall.Mkdirat(parentFD, currentPath, uint32(perm)) if err != nil { return fmt.Errorf("cannot create directory %s: %s", currentPath, err) @@ -1131,7 +1131,7 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { // and user either gets error or the file that it can already access. if err = syscall.Close(parentFD); err != nil { - glog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) } parentFD = childFD childFD = -1 @@ -1180,7 +1180,7 @@ func findExistingPrefix(base, pathname string) (string, []string, error) { } defer func() { if err = syscall.Close(fd); err != nil { - glog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) + klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) } }() for i, dir := range dirs { @@ -1194,7 +1194,7 @@ func findExistingPrefix(base, pathname string) (string, []string, error) { return base, nil, err } if err = syscall.Close(fd); err != nil { - glog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) + klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) } fd = childFD currentPath = filepath.Join(currentPath, dir) @@ -1226,7 +1226,7 @@ func doSafeOpen(pathname string, base string) (int, error) { defer func() { if parentFD != -1 { if err = syscall.Close(parentFD); err != nil { - glog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err) + klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err) } } }() @@ -1235,7 +1235,7 @@ func doSafeOpen(pathname string, base string) (int, error) { defer func() { if childFD != -1 { if err = syscall.Close(childFD); err != nil { - glog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err) + klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err) } } }() @@ -1250,7 +1250,7 @@ func doSafeOpen(pathname string, base string) (int, error) { return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base) } - glog.V(5).Infof("Opening path %s", currentPath) + klog.V(5).Infof("Opening path %s", currentPath) childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0) if err != nil { return -1, fmt.Errorf("cannot open %s: %s", currentPath, err) diff --git a/pkg/util/mount/mount_linux_test.go b/pkg/util/mount/mount_linux_test.go index 417592602da..89dbfdf041c 100644 --- a/pkg/util/mount/mount_linux_test.go +++ b/pkg/util/mount/mount_linux_test.go @@ -32,7 +32,7 @@ import ( "k8s.io/utils/exec" - "github.com/golang/glog" + "k8s.io/klog" ) func TestReadProcMountsFrom(t *testing.T) { @@ -634,7 +634,7 @@ func TestSafeMakeDir(t *testing.T) { } for _, test := range tests { - glog.V(4).Infof("test %q", test.name) + klog.V(4).Infof("test %q", test.name) base, err := ioutil.TempDir("", "safe-make-dir-"+test.name+"-") if err != nil { t.Fatalf(err.Error()) @@ -646,7 +646,7 @@ func TestSafeMakeDir(t *testing.T) { t.Errorf("test %q: %s", test.name, err) } if err != nil { - glog.Infof("got error: %s", err) + klog.Infof("got error: %s", err) } if err == nil && test.expectError { t.Errorf("test %q: expected error, got none", test.name) @@ -792,7 +792,7 @@ func TestRemoveEmptyDirs(t *testing.T) { } for _, test := range tests { - glog.V(4).Infof("test %q", test.name) + klog.V(4).Infof("test %q", test.name) base, err := ioutil.TempDir("", "remove-empty-dirs-"+test.name+"-") if err != nil { t.Fatalf(err.Error()) @@ -963,7 +963,7 @@ func TestCleanSubPaths(t *testing.T) { } for _, test := range tests { - glog.V(4).Infof("test %q", test.name) + klog.V(4).Infof("test %q", test.name) base, err := ioutil.TempDir("", "clean-subpaths-"+test.name+"-") if err != nil { t.Fatalf(err.Error()) @@ -1219,7 +1219,7 @@ func TestBindSubPath(t *testing.T) { } for _, test := range tests { - glog.V(4).Infof("test %q", test.name) + klog.V(4).Infof("test %q", test.name) base, err := ioutil.TempDir("", "bind-subpath-"+test.name+"-") if err != nil { t.Fatalf(err.Error()) @@ -1651,7 +1651,7 @@ func TestSafeOpen(t *testing.T) { } for _, test := range tests { - glog.V(4).Infof("test %q", test.name) + klog.V(4).Infof("test %q", test.name) base, err := ioutil.TempDir("", "safe-open-"+test.name+"-") if err != nil { t.Fatalf(err.Error()) @@ -1664,7 +1664,7 @@ func TestSafeOpen(t *testing.T) { t.Errorf("test %q: %s", test.name, err) } if err != nil { - glog.Infof("got error: %s", err) + klog.Infof("got error: %s", err) } if err == nil && test.expectError { t.Errorf("test %q: expected error, got none", test.name) @@ -1798,7 +1798,7 @@ func TestFindExistingPrefix(t *testing.T) { } for _, test := range tests { - glog.V(4).Infof("test %q", test.name) + klog.V(4).Infof("test %q", test.name) base, err := ioutil.TempDir("", "find-prefix-"+test.name+"-") if err != nil { t.Fatalf(err.Error()) @@ -1810,7 +1810,7 @@ func TestFindExistingPrefix(t *testing.T) { t.Errorf("test %q: %s", test.name, err) } if err != nil { - glog.Infof("got error: %s", err) + klog.Infof("got error: %s", err) } if err == nil && test.expectError { t.Errorf("test %q: expected error, got none", test.name) diff --git a/pkg/util/mount/mount_windows.go b/pkg/util/mount/mount_windows.go index 535803abf50..dfdcdfc3377 100644 --- a/pkg/util/mount/mount_windows.go +++ b/pkg/util/mount/mount_windows.go @@ -28,7 +28,7 @@ import ( "strings" "syscall" - "github.com/golang/glog" + "k8s.io/klog" utilfile "k8s.io/kubernetes/pkg/util/file" ) @@ -54,7 +54,7 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio target = normalizeWindowsPath(target) if source == "tmpfs" { - glog.V(3).Infof("azureMount: mounting source (%q), target (%q), with options (%q)", source, target, options) + klog.V(3).Infof("azureMount: mounting source (%q), target (%q), with options (%q)", source, target, options) return os.MkdirAll(target, 0755) } @@ -63,7 +63,7 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio return err } - glog.V(4).Infof("azureMount: mount options(%q) source:%q, target:%q, fstype:%q, begin to mount", + klog.V(4).Infof("azureMount: mount options(%q) source:%q, target:%q, fstype:%q, begin to mount", options, source, target, fstype) bindSource := "" @@ -73,7 +73,7 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio bindSource = normalizeWindowsPath(source) } else { if len(options) < 2 { - glog.Warningf("azureMount: mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting", + klog.Warningf("azureMount: mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting", options, len(options), source, target) return nil } @@ -102,7 +102,7 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio } if output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput(); err != nil { - glog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output)) + klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output)) return err } @@ -111,10 +111,10 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio // Unmount unmounts the target. func (mounter *Mounter) Unmount(target string) error { - glog.V(4).Infof("azureMount: Unmount target (%q)", target) + klog.V(4).Infof("azureMount: Unmount target (%q)", target) target = normalizeWindowsPath(target) if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil { - glog.Errorf("rmdir failed: %v, output: %q", err, string(output)) + klog.Errorf("rmdir failed: %v, output: %q", err, string(output)) return err } return nil @@ -168,7 +168,7 @@ func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (str func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { refs, err := mounter.GetMountRefs(mountPath) if err != nil { - glog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) + klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) return "", err } if len(refs) == 0 { @@ -179,7 +179,7 @@ func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (str if strings.Contains(ref, basemountPath) { volumeID, err := filepath.Rel(normalizeWindowsPath(basemountPath), ref) if err != nil { - glog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) + klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) return "", err } return volumeID, nil @@ -362,10 +362,10 @@ func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { // Try to mount the disk - glog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target) + klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target) if err := ValidateDiskNumber(source); err != nil { - glog.Errorf("diskMount: formatAndMount failed, err: %v", err) + klog.Errorf("diskMount: formatAndMount failed, err: %v", err) return err } @@ -380,7 +380,7 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, if output, err := mounter.Exec.Run("powershell", "/c", cmd); err != nil { return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output)) } - glog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype) + klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype) driveLetter, err := getDriveLetterByDiskNumber(source, mounter.Exec) if err != nil { @@ -388,9 +388,9 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, } driverPath := driveLetter + ":" target = normalizeWindowsPath(target) - glog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target) + klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target) if output, err := mounter.Exec.Run("cmd", "/c", "mklink", "/D", target, driverPath); err != nil { - glog.Errorf("mklink failed: %v, output: %q", err, string(output)) + klog.Errorf("mklink failed: %v, output: %q", err, string(output)) return err } return nil @@ -499,7 +499,7 @@ func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode } func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { - glog.V(4).Infof("Creating directory %q within base %q", pathname, base) + klog.V(4).Infof("Creating directory %q within base %q", pathname, base) if !PathWithinBase(pathname, base) { return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) @@ -512,7 +512,7 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { if s.IsDir() { // The directory already exists. It can be outside of the parent, // but there is no race-proof check. - glog.V(4).Infof("Directory %s already exists", pathname) + klog.V(4).Infof("Directory %s already exists", pathname) return nil } return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR} @@ -547,13 +547,13 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { return err } - glog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) + klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) currentPath := fullExistingPath // create the directories one by one, making sure nobody can change // created directory into symlink by lock that directory immediately for _, dir := range toCreate { currentPath = filepath.Join(currentPath, dir) - glog.V(4).Infof("Creating %s", dir) + klog.V(4).Infof("Creating %s", dir) if err := os.Mkdir(currentPath, perm); err != nil { return fmt.Errorf("cannot create directory %s: %s", currentPath, err) } diff --git a/pkg/util/mount/nsenter_mount.go b/pkg/util/mount/nsenter_mount.go index 330883d400e..ff2eceaf7c3 100644 --- a/pkg/util/mount/nsenter_mount.go +++ b/pkg/util/mount/nsenter_mount.go @@ -25,8 +25,8 @@ import ( "strings" "syscall" - "github.com/golang/glog" "golang.org/x/sys/unix" + "k8s.io/klog" utilfile "k8s.io/kubernetes/pkg/util/file" "k8s.io/kubernetes/pkg/util/nsenter" ) @@ -77,11 +77,11 @@ func (n *NsenterMounter) Mount(source string, target string, fstype string, opti // doNsenterMount nsenters the host's mount namespace and performs the // requested mount. func (n *NsenterMounter) doNsenterMount(source, target, fstype string, options []string) error { - glog.V(5).Infof("nsenter mount %s %s %s %v", source, target, fstype, options) + klog.V(5).Infof("nsenter mount %s %s %s %v", source, target, fstype, options) cmd, args := n.makeNsenterArgs(source, target, fstype, options) outputBytes, err := n.ne.Exec(cmd, args).CombinedOutput() if len(outputBytes) != 0 { - glog.V(5).Infof("Output of mounting %s to %s: %v", source, target, string(outputBytes)) + klog.V(5).Infof("Output of mounting %s to %s: %v", source, target, string(outputBytes)) } return err } @@ -131,10 +131,10 @@ func (n *NsenterMounter) Unmount(target string) error { // No need to execute systemd-run here, it's enough that unmount is executed // in the host's mount namespace. It will finish appropriate fuse daemon(s) // running in any scope. - glog.V(5).Infof("nsenter unmount args: %v", args) + klog.V(5).Infof("nsenter unmount args: %v", args) outputBytes, err := n.ne.Exec("umount", args).CombinedOutput() if len(outputBytes) != 0 { - glog.V(5).Infof("Output of unmounting %s: %v", target, string(outputBytes)) + klog.V(5).Infof("Output of unmounting %s: %v", target, string(outputBytes)) } return err } @@ -163,7 +163,7 @@ func (n *NsenterMounter) IsLikelyNotMountPoint(file string) (bool, error) { // Check the directory exists if _, err = os.Stat(file); os.IsNotExist(err) { - glog.V(5).Infof("findmnt: directory %s does not exist", file) + klog.V(5).Infof("findmnt: directory %s does not exist", file) return true, err } @@ -178,10 +178,10 @@ func (n *NsenterMounter) IsLikelyNotMountPoint(file string) (bool, error) { // Also add fstype output to make sure that the output of target file will give the full path // TODO: Need more refactoring for this function. Track the solution with issue #26996 args := []string{"-o", "target,fstype", "--noheadings", "--first-only", "--target", resolvedFile} - glog.V(5).Infof("nsenter findmnt args: %v", args) + klog.V(5).Infof("nsenter findmnt args: %v", args) out, err := n.ne.Exec("findmnt", args).CombinedOutput() if err != nil { - glog.V(2).Infof("Failed findmnt command for path %s: %s %v", resolvedFile, out, err) + klog.V(2).Infof("Failed findmnt command for path %s: %s %v", resolvedFile, out, err) // Different operating systems behave differently for paths which are not mount points. // On older versions (e.g. 2.20.1) we'd get error, on newer ones (e.g. 2.26.2) we'd get "/". // It's safer to assume that it's not a mount point. @@ -192,13 +192,13 @@ func (n *NsenterMounter) IsLikelyNotMountPoint(file string) (bool, error) { return false, err } - glog.V(5).Infof("IsLikelyNotMountPoint findmnt output for path %s: %v:", resolvedFile, mountTarget) + klog.V(5).Infof("IsLikelyNotMountPoint findmnt output for path %s: %v:", resolvedFile, mountTarget) if mountTarget == resolvedFile { - glog.V(5).Infof("IsLikelyNotMountPoint: %s is a mount point", resolvedFile) + klog.V(5).Infof("IsLikelyNotMountPoint: %s is a mount point", resolvedFile) return false, nil } - glog.V(5).Infof("IsLikelyNotMountPoint: %s is not a mount point", resolvedFile) + klog.V(5).Infof("IsLikelyNotMountPoint: %s is not a mount point", resolvedFile) return true, nil } @@ -375,7 +375,7 @@ func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath st if err != nil { return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err) } - glog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath) + klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath) subpath.VolumePath = mounter.ne.KubeletPath(evaluatedHostVolumePath) subpath.Path = mounter.ne.KubeletPath(evaluatedHostSubpath) @@ -398,9 +398,9 @@ func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath st defer func() { // Cleanup subpath on error if !success { - glog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) + klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil { - glog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) + klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) } } }() @@ -408,7 +408,7 @@ func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath st // Leap of faith: optimistically expect that nobody has modified previously // expanded evalSubPath with evil symlinks and bind-mount it. // Mount is done on the host! don't use kubelet path! - glog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget) + klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget) if err = mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil { return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err) } @@ -421,7 +421,7 @@ func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath st } success = true - glog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) + klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) return bindPathTarget, nil } diff --git a/pkg/util/netsh/BUILD b/pkg/util/netsh/BUILD index 980f16ba9ad..a356f094a9e 100644 --- a/pkg/util/netsh/BUILD +++ b/pkg/util/netsh/BUILD @@ -14,7 +14,7 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/util/netsh", deps = [ - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/util/netsh/netsh.go b/pkg/util/netsh/netsh.go index 30b66536b1d..a9f1731b45d 100644 --- a/pkg/util/netsh/netsh.go +++ b/pkg/util/netsh/netsh.go @@ -24,7 +24,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" utilexec "k8s.io/utils/exec" ) @@ -68,7 +68,7 @@ func New(exec utilexec.Interface) Interface { // EnsurePortProxyRule checks if the specified redirect exists, if not creates it. func (runner *runner) EnsurePortProxyRule(args []string) (bool, error) { - glog.V(4).Infof("running netsh interface portproxy add v4tov4 %v", args) + klog.V(4).Infof("running netsh interface portproxy add v4tov4 %v", args) out, err := runner.exec.Command(cmdNetsh, args...).CombinedOutput() if err == nil { @@ -87,7 +87,7 @@ func (runner *runner) EnsurePortProxyRule(args []string) (bool, error) { // DeletePortProxyRule deletes the specified portproxy rule. If the rule did not exist, return error. func (runner *runner) DeletePortProxyRule(args []string) error { - glog.V(4).Infof("running netsh interface portproxy delete v4tov4 %v", args) + klog.V(4).Infof("running netsh interface portproxy delete v4tov4 %v", args) out, err := runner.exec.Command(cmdNetsh, args...).CombinedOutput() if err == nil { @@ -116,12 +116,12 @@ func (runner *runner) EnsureIPAddress(args []string, ip net.IP) (bool, error) { exists, _ := checkIPExists(ipToCheck, argsShowAddress, runner) if exists == true { - glog.V(4).Infof("not adding IP address %q as it already exists", ipToCheck) + klog.V(4).Infof("not adding IP address %q as it already exists", ipToCheck) return true, nil } // IP Address is not already added, add it now - glog.V(4).Infof("running netsh interface ipv4 add address %v", args) + klog.V(4).Infof("running netsh interface ipv4 add address %v", args) out, err := runner.exec.Command(cmdNetsh, args...).CombinedOutput() if err == nil { @@ -129,7 +129,7 @@ func (runner *runner) EnsureIPAddress(args []string, ip net.IP) (bool, error) { // Query all the IP addresses and see if the one we added is present // PS: We are using netsh interface ipv4 show address here to query all the IP addresses, instead of // querying net.InterfaceAddrs() as it returns the IP address as soon as it is added even though it is uninitialized - glog.V(3).Infof("Waiting until IP: %v is added to the network adapter", ipToCheck) + klog.V(3).Infof("Waiting until IP: %v is added to the network adapter", ipToCheck) for { if exists, _ := checkIPExists(ipToCheck, argsShowAddress, runner); exists { return true, nil @@ -149,7 +149,7 @@ func (runner *runner) EnsureIPAddress(args []string, ip net.IP) (bool, error) { // DeleteIPAddress checks if the specified IP address is present and, if so, deletes it. func (runner *runner) DeleteIPAddress(args []string) error { - glog.V(4).Infof("running netsh interface ipv4 delete address %v", args) + klog.V(4).Infof("running netsh interface ipv4 delete address %v", args) out, err := runner.exec.Command(cmdNetsh, args...).CombinedOutput() if err == nil { @@ -187,7 +187,7 @@ func checkIPExists(ipToCheck string, args []string, runner *runner) (bool, error return false, err } ipAddressString := string(ipAddress[:]) - glog.V(3).Infof("Searching for IP: %v in IP dump: %v", ipToCheck, ipAddressString) + klog.V(3).Infof("Searching for IP: %v in IP dump: %v", ipToCheck, ipAddressString) showAddressArray := strings.Split(ipAddressString, "\n") for _, showAddress := range showAddressArray { if strings.Contains(showAddress, "IP") { diff --git a/pkg/util/node/BUILD b/pkg/util/node/BUILD index 5a6dfba5f2b..7e1696cefea 100644 --- a/pkg/util/node/BUILD +++ b/pkg/util/node/BUILD @@ -18,7 +18,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/util/node/node.go b/pkg/util/node/node.go index 76bf808ccd6..ff503855150 100644 --- a/pkg/util/node/node.go +++ b/pkg/util/node/node.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -100,12 +100,12 @@ func GetNodeIP(client clientset.Interface, hostname string) net.IP { var nodeIP net.IP node, err := client.CoreV1().Nodes().Get(hostname, metav1.GetOptions{}) if err != nil { - glog.Warningf("Failed to retrieve node info: %v", err) + klog.Warningf("Failed to retrieve node info: %v", err) return nil } nodeIP, err = GetNodeHostIP(node) if err != nil { - glog.Warningf("Failed to retrieve node IP: %v", err) + klog.Warningf("Failed to retrieve node IP: %v", err) return nil } return nodeIP diff --git a/pkg/util/nsenter/BUILD b/pkg/util/nsenter/BUILD index 05765dcc9be..9a94fae1015 100644 --- a/pkg/util/nsenter/BUILD +++ b/pkg/util/nsenter/BUILD @@ -24,7 +24,7 @@ go_library( "//vendor/k8s.io/utils/exec:go_default_library", ], "@io_bazel_rules_go//go/platform:linux": [ - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ diff --git a/pkg/util/nsenter/exec.go b/pkg/util/nsenter/exec.go index 201f1270c77..134497f0a75 100644 --- a/pkg/util/nsenter/exec.go +++ b/pkg/util/nsenter/exec.go @@ -23,7 +23,7 @@ import ( "fmt" "path/filepath" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/utils/exec" ) @@ -49,7 +49,7 @@ func NewNsenterExecutor(hostRootFsPath string, executor exec.Interface) *Executo func (nsExecutor *Executor) Command(cmd string, args ...string) exec.Cmd { fullArgs := append([]string{fmt.Sprintf("--mount=%s", nsExecutor.hostProcMountNsPath), "--"}, append([]string{cmd}, args...)...) - glog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs) + klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs) return nsExecutor.executor.Command(nsenterPath, fullArgs...) } @@ -57,7 +57,7 @@ func (nsExecutor *Executor) Command(cmd string, args ...string) exec.Cmd { func (nsExecutor *Executor) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd { fullArgs := append([]string{fmt.Sprintf("--mount=%s", nsExecutor.hostProcMountNsPath), "--"}, append([]string{cmd}, args...)...) - glog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs) + klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs) return nsExecutor.executor.CommandContext(ctx, nsenterPath, fullArgs...) } diff --git a/pkg/util/nsenter/nsenter.go b/pkg/util/nsenter/nsenter.go index e928a57ac9f..56361e7846e 100644 --- a/pkg/util/nsenter/nsenter.go +++ b/pkg/util/nsenter/nsenter.go @@ -28,7 +28,7 @@ import ( "k8s.io/utils/exec" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -127,7 +127,7 @@ func (ne *Nsenter) Exec(cmd string, args []string) exec.Cmd { hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath) fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"}, append([]string{ne.AbsHostPath(cmd)}, args...)...) - glog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs) + klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs) return ne.executor.Command(nsenterPath, fullArgs...) } @@ -170,7 +170,7 @@ func (ne *Nsenter) EvalSymlinks(pathname string, mustExist bool) (string, error) } outBytes, err := ne.Exec("realpath", args).CombinedOutput() if err != nil { - glog.Infof("failed to resolve symbolic links on %s: %v", pathname, err) + klog.Infof("failed to resolve symbolic links on %s: %v", pathname, err) return "", err } return strings.TrimSpace(string(outBytes)), nil diff --git a/pkg/util/oom/BUILD b/pkg/util/oom/BUILD index 712fc9061a6..b27ec22804e 100644 --- a/pkg/util/oom/BUILD +++ b/pkg/util/oom/BUILD @@ -19,7 +19,7 @@ go_library( deps = select({ "@io_bazel_rules_go//go/platform:linux": [ "//pkg/kubelet/cm/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "//conditions:default": [], }), diff --git a/pkg/util/oom/oom_linux.go b/pkg/util/oom/oom_linux.go index ad6d5c264b5..243c30d0cc4 100644 --- a/pkg/util/oom/oom_linux.go +++ b/pkg/util/oom/oom_linux.go @@ -29,7 +29,7 @@ import ( cmutil "k8s.io/kubernetes/pkg/kubelet/cm/util" - "github.com/golang/glog" + "k8s.io/klog" ) func NewOOMAdjuster() *OOMAdjuster { @@ -62,24 +62,24 @@ func applyOOMScoreAdj(pid int, oomScoreAdj int) error { maxTries := 2 oomScoreAdjPath := path.Join("/proc", pidStr, "oom_score_adj") value := strconv.Itoa(oomScoreAdj) - glog.V(4).Infof("attempting to set %q to %q", oomScoreAdjPath, value) + klog.V(4).Infof("attempting to set %q to %q", oomScoreAdjPath, value) var err error for i := 0; i < maxTries; i++ { err = ioutil.WriteFile(oomScoreAdjPath, []byte(value), 0700) if err != nil { if os.IsNotExist(err) { - glog.V(2).Infof("%q does not exist", oomScoreAdjPath) + klog.V(2).Infof("%q does not exist", oomScoreAdjPath) return os.ErrNotExist } - glog.V(3).Info(err) + klog.V(3).Info(err) time.Sleep(100 * time.Millisecond) continue } return nil } if err != nil { - glog.V(2).Infof("failed to set %q to %q: %v", oomScoreAdjPath, value, err) + klog.V(2).Infof("failed to set %q to %q: %v", oomScoreAdjPath, value, err) } return err } @@ -97,20 +97,20 @@ func (oomAdjuster *OOMAdjuster) applyOOMScoreAdjContainer(cgroupName string, oom return os.ErrNotExist } continueAdjusting = true - glog.V(10).Infof("Error getting process list for cgroup %s: %+v", cgroupName, err) + klog.V(10).Infof("Error getting process list for cgroup %s: %+v", cgroupName, err) } else if len(pidList) == 0 { - glog.V(10).Infof("Pid list is empty") + klog.V(10).Infof("Pid list is empty") continueAdjusting = true } else { for _, pid := range pidList { if !adjustedProcessSet[pid] { - glog.V(10).Infof("pid %d needs to be set", pid) + klog.V(10).Infof("pid %d needs to be set", pid) if err = oomAdjuster.ApplyOOMScoreAdj(pid, oomScoreAdj); err == nil { adjustedProcessSet[pid] = true } else if err == os.ErrNotExist { continue } else { - glog.V(10).Infof("cannot adjust oom score for pid %d - %v", pid, err) + klog.V(10).Infof("cannot adjust oom score for pid %d - %v", pid, err) continueAdjusting = true } // Processes can come and go while we try to apply oom score adjust value. So ignore errors here. diff --git a/pkg/util/procfs/BUILD b/pkg/util/procfs/BUILD index 1c02e429701..223f2265fb6 100644 --- a/pkg/util/procfs/BUILD +++ b/pkg/util/procfs/BUILD @@ -19,7 +19,7 @@ go_library( deps = select({ "@io_bazel_rules_go//go/platform:linux": [ "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "//conditions:default": [], }), diff --git a/pkg/util/procfs/procfs_linux.go b/pkg/util/procfs/procfs_linux.go index 0741b617efe..46059c33e77 100644 --- a/pkg/util/procfs/procfs_linux.go +++ b/pkg/util/procfs/procfs_linux.go @@ -32,8 +32,8 @@ import ( "syscall" "unicode" - "github.com/golang/glog" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog" ) type ProcFS struct{} @@ -137,7 +137,7 @@ func getPids(re *regexp.Regexp) []int { cmdline, err := ioutil.ReadFile(filepath.Join("/proc", entry.Name(), "cmdline")) if err != nil { - glog.V(4).Infof("Error reading file %s: %+v", filepath.Join("/proc", entry.Name(), "cmdline"), err) + klog.V(4).Infof("Error reading file %s: %+v", filepath.Join("/proc", entry.Name(), "cmdline"), err) continue } diff --git a/pkg/util/resizefs/BUILD b/pkg/util/resizefs/BUILD index 9d1cf01487f..04f11e1fc0c 100644 --- a/pkg/util/resizefs/BUILD +++ b/pkg/util/resizefs/BUILD @@ -23,7 +23,7 @@ go_library( ], "@io_bazel_rules_go//go/platform:linux": [ "//pkg/util/mount:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ "//pkg/util/mount:go_default_library", diff --git a/pkg/util/resizefs/resizefs_linux.go b/pkg/util/resizefs/resizefs_linux.go index 518eba2bb60..4eabdb1ddc0 100644 --- a/pkg/util/resizefs/resizefs_linux.go +++ b/pkg/util/resizefs/resizefs_linux.go @@ -21,7 +21,7 @@ package resizefs import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" ) @@ -50,7 +50,7 @@ func (resizefs *ResizeFs) Resize(devicePath string, deviceMountPath string) (boo return false, nil } - glog.V(3).Infof("ResizeFS.Resize - Expanding mounted volume %s", devicePath) + klog.V(3).Infof("ResizeFS.Resize - Expanding mounted volume %s", devicePath) switch format { case "ext3", "ext4": return resizefs.extResize(devicePath) @@ -63,7 +63,7 @@ func (resizefs *ResizeFs) Resize(devicePath string, deviceMountPath string) (boo func (resizefs *ResizeFs) extResize(devicePath string) (bool, error) { output, err := resizefs.mounter.Exec.Run("resize2fs", devicePath) if err == nil { - glog.V(2).Infof("Device %s resized successfully", devicePath) + klog.V(2).Infof("Device %s resized successfully", devicePath) return true, nil } @@ -77,7 +77,7 @@ func (resizefs *ResizeFs) xfsResize(deviceMountPath string) (bool, error) { output, err := resizefs.mounter.Exec.Run("xfs_growfs", args...) if err == nil { - glog.V(2).Infof("Device %s resized successfully", deviceMountPath) + klog.V(2).Infof("Device %s resized successfully", deviceMountPath) return true, nil } diff --git a/pkg/util/taints/taints.go b/pkg/util/taints/taints.go index 0cf6212eb8f..0777f6d6922 100644 --- a/pkg/util/taints/taints.go +++ b/pkg/util/taints/taints.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// package taints implements utilites for working with taints +// package taints implements utilities for working with taints package taints import ( diff --git a/pkg/util/workqueue/prometheus/prometheus.go b/pkg/util/workqueue/prometheus/prometheus.go index 81b2cf4acfd..6f5cf38f09d 100644 --- a/pkg/util/workqueue/prometheus/prometheus.go +++ b/pkg/util/workqueue/prometheus/prometheus.go @@ -71,6 +71,30 @@ func (prometheusMetricsProvider) NewWorkDurationMetric(name string) workqueue.Su return workDuration } +func (prometheusMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric { + unfinished := prometheus.NewGauge(prometheus.GaugeOpts{ + Subsystem: name, + Name: "unfinished_work_seconds", + Help: "How many seconds of work " + name + " has done that " + + "is in progress and hasn't been observed by work_duration. Large " + + "values indicate stuck threads. One can deduce the number of stuck " + + "threads by observing the rate at which this increases.", + }) + prometheus.Register(unfinished) + return unfinished +} + +func (prometheusMetricsProvider) NewLongestRunningProcessorMicrosecondsMetric(name string) workqueue.SettableGaugeMetric { + unfinished := prometheus.NewGauge(prometheus.GaugeOpts{ + Subsystem: name, + Name: "longest_running_processor_microseconds", + Help: "How many microseconds has the longest running " + + "processor for " + name + " been running.", + }) + prometheus.Register(unfinished) + return unfinished +} + func (prometheusMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric { retries := prometheus.NewCounter(prometheus.CounterOpts{ Subsystem: name, diff --git a/pkg/volume/BUILD b/pkg/volume/BUILD index 4d77d607326..ba5d04ba078 100644 --- a/pkg/volume/BUILD +++ b/pkg/volume/BUILD @@ -32,7 +32,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/awsebs/BUILD b/pkg/volume/awsebs/BUILD index e1c1fa595fa..27213a3b622 100644 --- a/pkg/volume/awsebs/BUILD +++ b/pkg/volume/awsebs/BUILD @@ -31,7 +31,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -54,7 +54,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/awsebs/attacher.go b/pkg/volume/awsebs/attacher.go index 6d325b978cf..a4a320633c2 100644 --- a/pkg/volume/awsebs/attacher.go +++ b/pkg/volume/awsebs/attacher.go @@ -23,7 +23,7 @@ import ( "strconv" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -79,7 +79,7 @@ func (attacher *awsElasticBlockStoreAttacher) Attach(spec *volume.Spec, nodeName // succeeds in that case, so no need to do that separately. devicePath, err := attacher.awsVolumes.AttachDisk(volumeID, nodeName) if err != nil { - glog.Errorf("Error attaching volume %q to node %q: %+v", volumeID, nodeName, err) + klog.Errorf("Error attaching volume %q to node %q: %+v", volumeID, nodeName, err) return "", err } @@ -88,14 +88,14 @@ func (attacher *awsElasticBlockStoreAttacher) Attach(spec *volume.Spec, nodeName func (attacher *awsElasticBlockStoreAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) { - glog.Warningf("Attacher.VolumesAreAttached called for node %q - Please use BulkVerifyVolumes for AWS", nodeName) + klog.Warningf("Attacher.VolumesAreAttached called for node %q - Please use BulkVerifyVolumes for AWS", nodeName) volumeNodeMap := map[types.NodeName][]*volume.Spec{ nodeName: specs, } nodeVolumesResult := make(map[*volume.Spec]bool) nodesVerificationMap, err := attacher.BulkVerifyVolumes(volumeNodeMap) if err != nil { - glog.Errorf("Attacher.VolumesAreAttached - error checking volumes for node %q with %v", nodeName, err) + klog.Errorf("Attacher.VolumesAreAttached - error checking volumes for node %q with %v", nodeName, err) return nodeVolumesResult, err } @@ -115,7 +115,7 @@ func (attacher *awsElasticBlockStoreAttacher) BulkVerifyVolumes(volumesByNode ma volumeSource, _, err := getVolumeSource(volumeSpec) if err != nil { - glog.Errorf("Error getting volume (%q) source : %v", volumeSpec.Name(), err) + klog.Errorf("Error getting volume (%q) source : %v", volumeSpec.Name(), err) continue } @@ -135,7 +135,7 @@ func (attacher *awsElasticBlockStoreAttacher) BulkVerifyVolumes(volumesByNode ma attachedResult, err := attacher.awsVolumes.DisksAreAttached(diskNamesByNode) if err != nil { - glog.Errorf("Error checking if volumes are attached to nodes err = %v", err) + klog.Errorf("Error checking if volumes are attached to nodes err = %v", err) return volumesAttachedCheck, err } @@ -175,15 +175,15 @@ func (attacher *awsElasticBlockStoreAttacher) WaitForAttach(spec *volume.Spec, d for { select { case <-ticker.C: - glog.V(5).Infof("Checking AWS Volume %q is attached.", volumeID) + klog.V(5).Infof("Checking AWS Volume %q is attached.", volumeID) devicePaths := getDiskByIDPaths(aws.KubernetesVolumeID(volumeSource.VolumeID), partition, devicePath) path, err := verifyDevicePath(devicePaths) if err != nil { // Log error, if any, and continue checking periodically. See issue #11321 - glog.Errorf("Error verifying AWS Volume (%q) is attached: %v", volumeID, err) + klog.Errorf("Error verifying AWS Volume (%q) is attached: %v", volumeID, err) } else if path != "" { // A device path has successfully been created for the PD - glog.Infof("Successfully found attached AWS Volume %q.", volumeID) + klog.Infof("Successfully found attached AWS Volume %q.", volumeID) return path, nil } case <-timer.C: @@ -267,7 +267,7 @@ func (detacher *awsElasticBlockStoreDetacher) Detach(volumeName string, nodeName volumeID := aws.KubernetesVolumeID(path.Base(volumeName)) if _, err := detacher.awsVolumes.DetachDisk(volumeID, nodeName); err != nil { - glog.Errorf("Error detaching volumeID %q: %v", volumeID, err) + klog.Errorf("Error detaching volumeID %q: %v", volumeID, err) return err } return nil diff --git a/pkg/volume/awsebs/attacher_test.go b/pkg/volume/awsebs/attacher_test.go index ae5fc9c8527..df92328f03e 100644 --- a/pkg/volume/awsebs/attacher_test.go +++ b/pkg/volume/awsebs/attacher_test.go @@ -20,7 +20,7 @@ import ( "errors" "testing" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -232,7 +232,7 @@ func (testcase *testcase) AttachDisk(diskName aws.KubernetesVolumeID, nodeName t return "", errors.New("Unexpected AttachDisk call: wrong nodeName") } - glog.V(4).Infof("AttachDisk call: %s, %s, returning %q, %v", diskName, nodeName, expected.retDeviceName, expected.ret) + klog.V(4).Infof("AttachDisk call: %s, %s, returning %q, %v", diskName, nodeName, expected.retDeviceName, expected.ret) return expected.retDeviceName, expected.ret } @@ -257,7 +257,7 @@ func (testcase *testcase) DetachDisk(diskName aws.KubernetesVolumeID, nodeName t return "", errors.New("Unexpected DetachDisk call: wrong nodeName") } - glog.V(4).Infof("DetachDisk call: %s, %s, returning %q, %v", diskName, nodeName, expected.retDeviceName, expected.ret) + klog.V(4).Infof("DetachDisk call: %s, %s, returning %q, %v", diskName, nodeName, expected.retDeviceName, expected.ret) return expected.retDeviceName, expected.ret } diff --git a/pkg/volume/awsebs/aws_ebs.go b/pkg/volume/awsebs/aws_ebs.go index 31e61eb3264..c6270c1752a 100644 --- a/pkg/volume/awsebs/aws_ebs.go +++ b/pkg/volume/awsebs/aws_ebs.go @@ -25,7 +25,7 @@ import ( "strconv" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -118,13 +118,13 @@ func (plugin *awsElasticBlockStorePlugin) GetVolumeLimits() (map[string]int64, e instances, ok := cloud.Instances() if !ok { - glog.V(3).Infof("Failed to get instances from cloud provider") + klog.V(3).Infof("Failed to get instances from cloud provider") return volumeLimits, nil } instanceType, err := instances.InstanceType(context.TODO(), plugin.host.GetNodeName()) if err != nil { - glog.Errorf("Failed to get instance type from AWS cloud provider") + klog.Errorf("Failed to get instance type from AWS cloud provider") return volumeLimits, nil } @@ -205,7 +205,7 @@ func (plugin *awsElasticBlockStorePlugin) NewDeleter(spec *volume.Spec) (volume. func (plugin *awsElasticBlockStorePlugin) newDeleterInternal(spec *volume.Spec, manager ebsManager) (volume.Deleter, error) { if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.AWSElasticBlockStore == nil { - glog.Errorf("spec.PersistentVolumeSource.AWSElasticBlockStore is nil") + klog.Errorf("spec.PersistentVolumeSource.AWSElasticBlockStore is nil") return nil, fmt.Errorf("spec.PersistentVolumeSource.AWSElasticBlockStore is nil") } return &awsElasticBlockStoreDeleter{ @@ -273,7 +273,7 @@ func (plugin *awsElasticBlockStorePlugin) ConstructVolumeSpec(volName, mountPath if length == 3 { sourceName = awsURLNamePrefix + names[1] + "/" + volName // names[1] is the zone label } - glog.V(4).Infof("Convert aws volume name from %q to %q ", volumeID, sourceName) + klog.V(4).Infof("Convert aws volume name from %q to %q ", volumeID, sourceName) } awsVolume := &v1.Volume{ @@ -382,9 +382,9 @@ func (b *awsElasticBlockStoreMounter) SetUp(fsGroup *int64) error { func (b *awsElasticBlockStoreMounter) SetUpAt(dir string, fsGroup *int64) error { // TODO: handle failed mounts here. notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) - glog.V(4).Infof("PersistentDisk set up: %s %v %v", dir, !notMnt, err) + klog.V(4).Infof("PersistentDisk set up: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { - glog.Errorf("cannot validate mount point: %s %v", dir, err) + klog.Errorf("cannot validate mount point: %s %v", dir, err) return err } if !notMnt { @@ -407,27 +407,27 @@ func (b *awsElasticBlockStoreMounter) SetUpAt(dir string, fsGroup *int64) error if err != nil { notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed for %s: %v", dir, mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed for %s: %v", dir, mntErr) return err } if !notMnt { if mntErr = b.mounter.Unmount(dir); mntErr != nil { - glog.Errorf("failed to unmount %s: %v", dir, mntErr) + klog.Errorf("failed to unmount %s: %v", dir, mntErr) return err } notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed for %s: %v", dir, mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed for %s: %v", dir, mntErr) return err } if !notMnt { // This is very odd, we don't expect it. We'll try again next sync loop. - glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) + klog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) return err } } os.Remove(dir) - glog.Errorf("Mount of disk %s failed: %v", dir, err) + klog.Errorf("Mount of disk %s failed: %v", dir, err) return err } @@ -435,7 +435,7 @@ func (b *awsElasticBlockStoreMounter) SetUpAt(dir string, fsGroup *int64) error volume.SetVolumeOwnership(b, fsGroup) } - glog.V(4).Infof("Successfully mounted %s", dir) + klog.V(4).Infof("Successfully mounted %s", dir) return nil } @@ -451,11 +451,11 @@ func getVolumeIDFromGlobalMount(host volume.VolumeHost, globalPath string) (stri basePath := filepath.Join(host.GetPluginDir(awsElasticBlockStorePluginName), mount.MountsInGlobalPDPath) rel, err := filepath.Rel(basePath, globalPath) if err != nil { - glog.Errorf("Failed to get volume id from global mount %s - %v", globalPath, err) + klog.Errorf("Failed to get volume id from global mount %s - %v", globalPath, err) return "", err } if strings.Contains(rel, "../") { - glog.Errorf("Unexpected mount path: %s", globalPath) + klog.Errorf("Unexpected mount path: %s", globalPath) return "", fmt.Errorf("unexpected mount path: " + globalPath) } // Reverse the :// replacement done in makeGlobalPDPath @@ -463,7 +463,7 @@ func getVolumeIDFromGlobalMount(host volume.VolumeHost, globalPath string) (stri if strings.HasPrefix(volumeID, "aws/") { volumeID = strings.Replace(volumeID, "aws/", "aws://", 1) } - glog.V(2).Info("Mapping mount dir ", globalPath, " to volumeID ", volumeID) + klog.V(2).Info("Mapping mount dir ", globalPath, " to volumeID ", volumeID) return volumeID, nil } @@ -517,7 +517,7 @@ func (c *awsElasticBlockStoreProvisioner) Provision(selectedNode *v1.Node, allow volumeID, sizeGB, labels, fstype, err := c.manager.CreateVolume(c, selectedNode, allowedTopologies) if err != nil { - glog.Errorf("Provision failed: %v", err) + klog.Errorf("Provision failed: %v", err) return nil, err } diff --git a/pkg/volume/awsebs/aws_ebs_block.go b/pkg/volume/awsebs/aws_ebs_block.go index efe52a470b8..a6d64b5c495 100644 --- a/pkg/volume/awsebs/aws_ebs_block.go +++ b/pkg/volume/awsebs/aws_ebs_block.go @@ -22,7 +22,7 @@ import ( "strconv" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -43,7 +43,7 @@ func (plugin *awsElasticBlockStorePlugin) ConstructBlockVolumeSpec(podUID types. if err != nil { return nil, err } - glog.V(5).Infof("globalMapPathUUID: %s", globalMapPathUUID) + klog.V(5).Infof("globalMapPathUUID: %s", globalMapPathUUID) globalMapPath := filepath.Dir(globalMapPathUUID) if len(globalMapPath) <= 1 { diff --git a/pkg/volume/awsebs/aws_util.go b/pkg/volume/awsebs/aws_util.go index ce5b0b1bc41..e1c85aa8b70 100644 --- a/pkg/volume/awsebs/aws_util.go +++ b/pkg/volume/awsebs/aws_util.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -60,13 +60,13 @@ func (util *AWSDiskUtil) DeleteVolume(d *awsElasticBlockStoreDeleter) error { if err != nil { // AWS cloud provider returns volume.deletedVolumeInUseError when // necessary, no handling needed here. - glog.V(2).Infof("Error deleting EBS Disk volume %s: %v", d.volumeID, err) + klog.V(2).Infof("Error deleting EBS Disk volume %s: %v", d.volumeID, err) return err } if deleted { - glog.V(2).Infof("Successfully deleted EBS Disk volume %s", d.volumeID) + klog.V(2).Infof("Successfully deleted EBS Disk volume %s", d.volumeID) } else { - glog.V(2).Infof("Successfully deleted EBS Disk volume %s (actually already deleted)", d.volumeID) + klog.V(2).Infof("Successfully deleted EBS Disk volume %s (actually already deleted)", d.volumeID) } return nil } @@ -97,7 +97,7 @@ func (util *AWSDiskUtil) CreateVolume(c *awsElasticBlockStoreProvisioner, node * volumeOptions, err := populateVolumeOptions(c.plugin.GetPluginName(), c.options.PVC.Name, capacity, tags, c.options.Parameters, node, allowedTopologies, zonesWithNodes) if err != nil { - glog.V(2).Infof("Error populating EBS options: %v", err) + klog.V(2).Infof("Error populating EBS options: %v", err) return "", 0, nil, "", err } @@ -108,15 +108,15 @@ func (util *AWSDiskUtil) CreateVolume(c *awsElasticBlockStoreProvisioner, node * name, err := cloud.CreateDisk(volumeOptions) if err != nil { - glog.V(2).Infof("Error creating EBS Disk volume: %v", err) + klog.V(2).Infof("Error creating EBS Disk volume: %v", err) return "", 0, nil, "", err } - glog.V(2).Infof("Successfully created EBS Disk volume %s", name) + klog.V(2).Infof("Successfully created EBS Disk volume %s", name) labels, err := cloud.GetVolumeLabels(name) if err != nil { // We don't really want to leak the volume here... - glog.Errorf("error building labels for new EBS volume %q: %v", name, err) + klog.Errorf("error building labels for new EBS volume %q: %v", name, err) } fstype := "" @@ -230,14 +230,14 @@ func getDiskByIDPaths(volumeID aws.KubernetesVolumeID, partition string, deviceP // and we have to get the volume id from the nvme interface awsVolumeID, err := volumeID.MapToAWSVolumeID() if err != nil { - glog.Warningf("error mapping volume %q to AWS volume: %v", volumeID, err) + klog.Warningf("error mapping volume %q to AWS volume: %v", volumeID, err) } else { // This is the magic name on which AWS presents NVME devices under /dev/disk/by-id/ // For example, vol-0fab1d5e3f72a5e23 creates a symlink at /dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol0fab1d5e3f72a5e23 nvmeName := "nvme-Amazon_Elastic_Block_Store_" + strings.Replace(string(awsVolumeID), "-", "", -1) nvmePath, err := findNvmeVolume(nvmeName) if err != nil { - glog.Warningf("error looking for nvme volume %q: %v", volumeID, err) + klog.Warningf("error looking for nvme volume %q: %v", volumeID, err) } else if nvmePath != "" { devicePaths = append(devicePaths, nvmePath) } @@ -263,14 +263,14 @@ func findNvmeVolume(findName string) (device string, err error) { stat, err := os.Lstat(p) if err != nil { if os.IsNotExist(err) { - glog.V(6).Infof("nvme path not found %q", p) + klog.V(6).Infof("nvme path not found %q", p) return "", nil } return "", fmt.Errorf("error getting stat of %q: %v", p, err) } if stat.Mode()&os.ModeSymlink != os.ModeSymlink { - glog.Warningf("nvme file %q found, but was not a symlink", p) + klog.Warningf("nvme file %q found, but was not a symlink", p) return "", nil } diff --git a/pkg/volume/azure_dd/BUILD b/pkg/volume/azure_dd/BUILD index 8330a7ab740..fd28f977d79 100644 --- a/pkg/volume/azure_dd/BUILD +++ b/pkg/volume/azure_dd/BUILD @@ -41,7 +41,7 @@ go_library( "//staging/src/k8s.io/cloud-provider:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/azure_dd/attacher.go b/pkg/volume/azure_dd/attacher.go index 5fd911f3724..5787518aa52 100644 --- a/pkg/volume/azure_dd/attacher.go +++ b/pkg/volume/azure_dd/attacher.go @@ -26,7 +26,7 @@ import ( "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -62,13 +62,13 @@ var getLunMutex = keymutex.NewHashed(0) func (a *azureDiskAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) { volumeSource, _, err := getVolumeSource(spec) if err != nil { - glog.Warningf("failed to get azure disk spec (%v)", err) + klog.Warningf("failed to get azure disk spec (%v)", err) return "", err } instanceid, err := a.cloud.InstanceID(context.TODO(), nodeName) if err != nil { - glog.Warningf("failed to get azure instance id (%v)", err) + klog.Warningf("failed to get azure instance id (%v)", err) return "", fmt.Errorf("failed to get azure instance id for node %q (%v)", nodeName, err) } @@ -80,31 +80,31 @@ func (a *azureDiskAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) ( lun, err := diskController.GetDiskLun(volumeSource.DiskName, volumeSource.DataDiskURI, nodeName) if err == cloudprovider.InstanceNotFound { // Log error and continue with attach - glog.Warningf( + klog.Warningf( "Error checking if volume is already attached to current node (%q). Will continue and try attach anyway. err=%v", instanceid, err) } if err == nil { // Volume is already attached to node. - glog.V(2).Infof("Attach operation is successful. volume %q is already attached to node %q at lun %d.", volumeSource.DiskName, instanceid, lun) + klog.V(2).Infof("Attach operation is successful. volume %q is already attached to node %q at lun %d.", volumeSource.DiskName, instanceid, lun) } else { - glog.V(2).Infof("GetDiskLun returned: %v. Initiating attaching volume %q to node %q.", err, volumeSource.DataDiskURI, nodeName) + klog.V(2).Infof("GetDiskLun returned: %v. Initiating attaching volume %q to node %q.", err, volumeSource.DataDiskURI, nodeName) getLunMutex.LockKey(instanceid) defer getLunMutex.UnlockKey(instanceid) lun, err = diskController.GetNextDiskLun(nodeName) if err != nil { - glog.Warningf("no LUN available for instance %q (%v)", nodeName, err) + klog.Warningf("no LUN available for instance %q (%v)", nodeName, err) return "", fmt.Errorf("all LUNs are used, cannot attach volume %q to instance %q (%v)", volumeSource.DiskName, instanceid, err) } - glog.V(2).Infof("Trying to attach volume %q lun %d to node %q.", volumeSource.DataDiskURI, lun, nodeName) + klog.V(2).Infof("Trying to attach volume %q lun %d to node %q.", volumeSource.DataDiskURI, lun, nodeName) isManagedDisk := (*volumeSource.Kind == v1.AzureManagedDisk) err = diskController.AttachDisk(isManagedDisk, volumeSource.DiskName, volumeSource.DataDiskURI, nodeName, lun, compute.CachingTypes(*volumeSource.CachingMode)) if err == nil { - glog.V(2).Infof("Attach operation successful: volume %q attached to node %q.", volumeSource.DataDiskURI, nodeName) + klog.V(2).Infof("Attach operation successful: volume %q attached to node %q.", volumeSource.DataDiskURI, nodeName) } else { - glog.V(2).Infof("Attach volume %q to instance %q failed with %v", volumeSource.DataDiskURI, instanceid, err) + klog.V(2).Infof("Attach volume %q to instance %q failed with %v", volumeSource.DataDiskURI, instanceid, err) return "", fmt.Errorf("Attach volume %q to instance %q failed with %v", volumeSource.DiskName, instanceid, err) } } @@ -119,7 +119,7 @@ func (a *azureDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName ty for _, spec := range specs { volumeSource, _, err := getVolumeSource(spec) if err != nil { - glog.Errorf("azureDisk - Error getting volume (%q) source : %v", spec.Name(), err) + klog.Errorf("azureDisk - Error getting volume (%q) source : %v", spec.Name(), err) continue } @@ -135,7 +135,7 @@ func (a *azureDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName ty attachedResult, err := diskController.DisksAreAttached(volumeIDList, nodeName) if err != nil { // Log error and continue with attach - glog.Errorf( + klog.Errorf( "azureDisk - Error checking if volumes (%v) are attached to current node (%q). err=%v", volumeIDList, nodeName, err) return volumesAttachedCheck, err @@ -145,7 +145,7 @@ func (a *azureDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName ty if !attached { spec := volumeSpecMap[volumeID] volumesAttachedCheck[spec] = false - glog.V(2).Infof("azureDisk - VolumesAreAttached: check volume %q (specName: %q) is no longer attached", volumeID, spec.Name()) + klog.V(2).Infof("azureDisk - VolumesAreAttached: check volume %q (specName: %q) is no longer attached", volumeID, spec.Name()) } } return volumesAttachedCheck, nil @@ -167,13 +167,13 @@ func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, var lun int32 if runtime.GOOS == "windows" { - glog.V(2).Infof("azureDisk - WaitForAttach: begin to GetDiskLun by diskName(%s), DataDiskURI(%s), nodeName(%s), devicePath(%s)", + klog.V(2).Infof("azureDisk - WaitForAttach: begin to GetDiskLun by diskName(%s), DataDiskURI(%s), nodeName(%s), devicePath(%s)", diskName, volumeSource.DataDiskURI, nodeName, devicePath) lun, err = diskController.GetDiskLun(diskName, volumeSource.DataDiskURI, nodeName) if err != nil { return "", err } - glog.V(2).Infof("azureDisk - WaitForAttach: GetDiskLun succeeded, got lun(%v)", lun) + klog.V(2).Infof("azureDisk - WaitForAttach: GetDiskLun succeeded, got lun(%v)", lun) } else { lun, err = getDiskLUN(devicePath) if err != nil { @@ -247,9 +247,9 @@ func (attacher *azureDiskAttacher) MountDevice(spec *volume.Spec, devicePath str // testing original mount point, make sure the mount link is valid if _, err := (&osIOHandler{}).ReadDir(deviceMountPath); err != nil { // mount link is invalid, now unmount and remount later - glog.Warningf("azureDisk - ReadDir %s failed with %v, unmount this directory", deviceMountPath, err) + klog.Warningf("azureDisk - ReadDir %s failed with %v, unmount this directory", deviceMountPath, err) if err := mounter.Unmount(deviceMountPath); err != nil { - glog.Errorf("azureDisk - Unmount deviceMountPath %s failed with %v", deviceMountPath, err) + klog.Errorf("azureDisk - Unmount deviceMountPath %s failed with %v", deviceMountPath, err) return err } notMnt = true @@ -284,11 +284,11 @@ func (d *azureDiskDetacher) Detach(diskURI string, nodeName types.NodeName) erro instanceid, err := d.cloud.InstanceID(context.TODO(), nodeName) if err != nil { - glog.Warningf("no instance id for node %q, skip detaching (%v)", nodeName, err) + klog.Warningf("no instance id for node %q, skip detaching (%v)", nodeName, err) return nil } - glog.V(2).Infof("detach %v from node %q", diskURI, nodeName) + klog.V(2).Infof("detach %v from node %q", diskURI, nodeName) diskController, err := getDiskController(d.plugin.host) if err != nil { @@ -300,10 +300,10 @@ func (d *azureDiskDetacher) Detach(diskURI string, nodeName types.NodeName) erro err = diskController.DetachDiskByName("", diskURI, nodeName) if err != nil { - glog.Errorf("failed to detach azure disk %q, err %v", diskURI, err) + klog.Errorf("failed to detach azure disk %q, err %v", diskURI, err) } - glog.V(2).Infof("azureDisk - disk:%s was detached from node:%v", diskURI, nodeName) + klog.V(2).Infof("azureDisk - disk:%s was detached from node:%v", diskURI, nodeName) return err } @@ -311,9 +311,9 @@ func (d *azureDiskDetacher) Detach(diskURI string, nodeName types.NodeName) erro func (detacher *azureDiskDetacher) UnmountDevice(deviceMountPath string) error { err := util.UnmountPath(deviceMountPath, detacher.plugin.host.GetMounter(detacher.plugin.GetPluginName())) if err == nil { - glog.V(2).Infof("azureDisk - Device %s was unmounted", deviceMountPath) + klog.V(2).Infof("azureDisk - Device %s was unmounted", deviceMountPath) } else { - glog.Warningf("azureDisk - Device %s failed to unmount with error: %s", deviceMountPath, err.Error()) + klog.Warningf("azureDisk - Device %s failed to unmount with error: %s", deviceMountPath, err.Error()) } return err } diff --git a/pkg/volume/azure_dd/azure_common_linux.go b/pkg/volume/azure_dd/azure_common_linux.go index 383b46f4687..6b693b17e2b 100644 --- a/pkg/volume/azure_dd/azure_common_linux.go +++ b/pkg/volume/azure_dd/azure_common_linux.go @@ -24,7 +24,7 @@ import ( "strconv" libstrings "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" ) @@ -42,21 +42,21 @@ func listAzureDiskPath(io ioHandler) []string { } } } - glog.V(12).Infof("Azure sys disks paths: %v", azureDiskList) + klog.V(12).Infof("Azure sys disks paths: %v", azureDiskList) return azureDiskList } // getDiskLinkByDevName get disk link by device name from devLinkPath, e.g. /dev/disk/azure/, /dev/disk/by-id/ func getDiskLinkByDevName(io ioHandler, devLinkPath, devName string) (string, error) { dirs, err := io.ReadDir(devLinkPath) - glog.V(12).Infof("azureDisk - begin to find %s from %s", devName, devLinkPath) + klog.V(12).Infof("azureDisk - begin to find %s from %s", devName, devLinkPath) if err == nil { for _, f := range dirs { diskPath := devLinkPath + f.Name() - glog.V(12).Infof("azureDisk - begin to Readlink: %s", diskPath) + klog.V(12).Infof("azureDisk - begin to Readlink: %s", diskPath) link, linkErr := io.Readlink(diskPath) if linkErr != nil { - glog.Warningf("azureDisk - read link (%s) error: %v", diskPath, linkErr) + klog.Warningf("azureDisk - read link (%s) error: %v", diskPath, linkErr) continue } if libstrings.HasSuffix(link, devName) { @@ -75,11 +75,11 @@ func scsiHostRescan(io ioHandler, exec mount.Exec) { name := scsi_path + f.Name() + "/scan" data := []byte("- - -") if err = io.WriteFile(name, data, 0666); err != nil { - glog.Warningf("failed to rescan scsi host %s", name) + klog.Warningf("failed to rescan scsi host %s", name) } } } else { - glog.Warningf("failed to read %s, err %v", scsi_path, err) + klog.Warningf("failed to read %s, err %v", scsi_path, err) } } @@ -101,10 +101,10 @@ func findDiskByLunWithConstraint(lun int, io ioHandler, azureDisks []string) (st continue } if len(azureDisks) == 0 { - glog.V(4).Infof("/dev/disk/azure is not populated, now try to parse %v directly", name) + klog.V(4).Infof("/dev/disk/azure is not populated, now try to parse %v directly", name) target, err := strconv.Atoi(arr[0]) if err != nil { - glog.Errorf("failed to parse target from %v (%v), err %v", arr[0], name, err) + klog.Errorf("failed to parse target from %v (%v), err %v", arr[0], name, err) continue } // as observed, targets 0-3 are used by OS disks. Skip them @@ -118,7 +118,7 @@ func findDiskByLunWithConstraint(lun int, io ioHandler, azureDisks []string) (st l, err := strconv.Atoi(arr[3]) if err != nil { // unknown path format, continue to read the next one - glog.V(4).Infof("azure disk - failed to parse lun from %v (%v), err %v", arr[3], name, err) + klog.V(4).Infof("azure disk - failed to parse lun from %v (%v), err %v", arr[3], name, err) continue } if lun == l { @@ -127,24 +127,24 @@ func findDiskByLunWithConstraint(lun int, io ioHandler, azureDisks []string) (st vendorPath := filepath.Join(sys_path, name, "vendor") vendorBytes, err := io.ReadFile(vendorPath) if err != nil { - glog.Errorf("failed to read device vendor, err: %v", err) + klog.Errorf("failed to read device vendor, err: %v", err) continue } vendor := libstrings.TrimSpace(string(vendorBytes)) if libstrings.ToUpper(vendor) != "MSFT" { - glog.V(4).Infof("vendor doesn't match VHD, got %s", vendor) + klog.V(4).Infof("vendor doesn't match VHD, got %s", vendor) continue } modelPath := filepath.Join(sys_path, name, "model") modelBytes, err := io.ReadFile(modelPath) if err != nil { - glog.Errorf("failed to read device model, err: %v", err) + klog.Errorf("failed to read device model, err: %v", err) continue } model := libstrings.TrimSpace(string(modelBytes)) if libstrings.ToUpper(model) != "VIRTUAL DISK" { - glog.V(4).Infof("model doesn't match VHD, got %s", model) + klog.V(4).Infof("model doesn't match VHD, got %s", model) continue } @@ -154,7 +154,7 @@ func findDiskByLunWithConstraint(lun int, io ioHandler, azureDisks []string) (st found := false devName := dev[0].Name() for _, diskName := range azureDisks { - glog.V(12).Infof("azureDisk - validating disk %q with sys disk %q", devName, diskName) + klog.V(12).Infof("azureDisk - validating disk %q with sys disk %q", devName, diskName) if devName == diskName { found = true break @@ -165,10 +165,10 @@ func findDiskByLunWithConstraint(lun int, io ioHandler, azureDisks []string) (st for _, devLinkPath := range devLinkPaths { diskPath, err := getDiskLinkByDevName(io, devLinkPath, devName) if err == nil { - glog.V(4).Infof("azureDisk - found %s by %s under %s", diskPath, devName, devLinkPath) + klog.V(4).Infof("azureDisk - found %s by %s under %s", diskPath, devName, devLinkPath) return diskPath, nil } - glog.Warningf("azureDisk - getDiskLinkByDevName by %s under %s failed, error: %v", devName, devLinkPath, err) + klog.Warningf("azureDisk - getDiskLinkByDevName by %s under %s failed, error: %v", devName, devLinkPath, err) } return "/dev/" + devName, nil } diff --git a/pkg/volume/azure_dd/azure_common_windows.go b/pkg/volume/azure_dd/azure_common_windows.go index 6ee1a6d0558..c48f191f309 100644 --- a/pkg/volume/azure_dd/azure_common_windows.go +++ b/pkg/volume/azure_dd/azure_common_windows.go @@ -24,7 +24,7 @@ import ( "strconv" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" ) @@ -33,7 +33,7 @@ func scsiHostRescan(io ioHandler, exec mount.Exec) { cmd := "Update-HostStorageCache" output, err := exec.Run("powershell", "/c", cmd) if err != nil { - glog.Errorf("Update-HostStorageCache failed in scsiHostRescan, error: %v, output: %q", err, string(output)) + klog.Errorf("Update-HostStorageCache failed in scsiHostRescan, error: %v, output: %q", err, string(output)) } } @@ -42,7 +42,7 @@ func findDiskByLun(lun int, iohandler ioHandler, exec mount.Exec) (string, error cmd := `Get-Disk | select number, location | ConvertTo-Json` output, err := exec.Run("powershell", "/c", cmd) if err != nil { - glog.Errorf("Get-Disk failed in findDiskByLun, error: %v, output: %q", err, string(output)) + klog.Errorf("Get-Disk failed in findDiskByLun, error: %v, output: %q", err, string(output)) return "", err } @@ -52,7 +52,7 @@ func findDiskByLun(lun int, iohandler ioHandler, exec mount.Exec) (string, error var data []map[string]interface{} if err = json.Unmarshal(output, &data); err != nil { - glog.Errorf("Get-Disk output is not a json array, output: %q", string(output)) + klog.Errorf("Get-Disk output is not a json array, output: %q", string(output)) return "", err } @@ -66,27 +66,27 @@ func findDiskByLun(lun int, iohandler ioHandler, exec mount.Exec) (string, error arr := strings.Split(location, " ") arrLen := len(arr) if arrLen < 3 { - glog.Warningf("unexpected json structure from Get-Disk, location: %q", jsonLocation) + klog.Warningf("unexpected json structure from Get-Disk, location: %q", jsonLocation) continue } - glog.V(4).Infof("found a disk, locatin: %q, lun: %q", location, arr[arrLen-1]) + klog.V(4).Infof("found a disk, locatin: %q, lun: %q", location, arr[arrLen-1]) //last element of location field is LUN number, e.g. // "location": "Integrated : Adapter 3 : Port 0 : Target 0 : LUN 1" l, err := strconv.Atoi(arr[arrLen-1]) if err != nil { - glog.Warningf("cannot parse element from data structure, location: %q, element: %q", location, arr[arrLen-1]) + klog.Warningf("cannot parse element from data structure, location: %q, element: %q", location, arr[arrLen-1]) continue } if l == lun { - glog.V(4).Infof("found a disk and lun, locatin: %q, lun: %d", location, lun) + klog.V(4).Infof("found a disk and lun, locatin: %q, lun: %d", location, lun) if d, ok := v["number"]; ok { if diskNum, ok := d.(float64); ok { - glog.V(2).Infof("azureDisk Mount: got disk number(%d) by LUN(%d)", int(diskNum), lun) + klog.V(2).Infof("azureDisk Mount: got disk number(%d) by LUN(%d)", int(diskNum), lun) return strconv.Itoa(int(diskNum)), nil } - glog.Warningf("LUN(%d) found, but could not get disk number(%q), location: %q", lun, d, location) + klog.Warningf("LUN(%d) found, but could not get disk number(%q), location: %q", lun, d, location) } return "", fmt.Errorf("LUN(%d) found, but could not get disk number, location: %q", lun, location) } @@ -99,7 +99,7 @@ func findDiskByLun(lun int, iohandler ioHandler, exec mount.Exec) (string, error func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) { if err := mount.ValidateDiskNumber(disk); err != nil { - glog.Errorf("azureDisk Mount: formatIfNotFormatted failed, err: %v\n", err) + klog.Errorf("azureDisk Mount: formatIfNotFormatted failed, err: %v\n", err) return } @@ -111,8 +111,8 @@ func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) { cmd += fmt.Sprintf(" | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", fstype) output, err := exec.Run("powershell", "/c", cmd) if err != nil { - glog.Errorf("azureDisk Mount: Get-Disk failed, error: %v, output: %q", err, string(output)) + klog.Errorf("azureDisk Mount: Get-Disk failed, error: %v, output: %q", err, string(output)) } else { - glog.Infof("azureDisk Mount: Disk successfully formatted, disk: %q, fstype: %q\n", disk, fstype) + klog.Infof("azureDisk Mount: Disk successfully formatted, disk: %q, fstype: %q\n", disk, fstype) } } diff --git a/pkg/volume/azure_dd/azure_dd.go b/pkg/volume/azure_dd/azure_dd.go index f5e4c206d63..961d5a35714 100644 --- a/pkg/volume/azure_dd/azure_dd.go +++ b/pkg/volume/azure_dd/azure_dd.go @@ -23,7 +23,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -149,20 +149,20 @@ func (plugin *azureDataDiskPlugin) GetVolumeLimits() (map[string]int64, error) { instances, ok := az.Instances() if !ok { - glog.Warningf("Failed to get instances from cloud provider") + klog.Warningf("Failed to get instances from cloud provider") return volumeLimits, nil } instanceType, err := instances.InstanceType(context.TODO(), plugin.host.GetNodeName()) if err != nil { - glog.Errorf("Failed to get instance type from Azure cloud provider, nodeName: %s", plugin.host.GetNodeName()) + klog.Errorf("Failed to get instance type from Azure cloud provider, nodeName: %s", plugin.host.GetNodeName()) return volumeLimits, nil } if vmSizeList == nil { result, err := az.VirtualMachineSizesClient.List(context.TODO(), az.Location) if err != nil || result.Value == nil { - glog.Errorf("failed to list vm sizes in GetVolumeLimits, plugin.host: %s, location: %s", plugin.host.GetHostName(), az.Location) + klog.Errorf("failed to list vm sizes in GetVolumeLimits, plugin.host: %s, location: %s", plugin.host.GetHostName(), az.Location) return volumeLimits, nil } vmSizeList = result.Value @@ -183,11 +183,11 @@ func getMaxDataDiskCount(instanceType string, sizeList *[]compute.VirtualMachine vmsize := strings.ToUpper(instanceType) for _, size := range *sizeList { if size.Name == nil || size.MaxDataDiskCount == nil { - glog.Errorf("failed to get vm size in getMaxDataDiskCount") + klog.Errorf("failed to get vm size in getMaxDataDiskCount") continue } if strings.ToUpper(*size.Name) == vmsize { - glog.V(2).Infof("got a matching size in getMaxDataDiskCount, Name: %s, MaxDataDiskCount: %d", *size.Name, *size.MaxDataDiskCount) + klog.V(2).Infof("got a matching size in getMaxDataDiskCount, Name: %s, MaxDataDiskCount: %d", *size.Name, *size.MaxDataDiskCount) return int64(*size.MaxDataDiskCount) } } @@ -208,7 +208,7 @@ func (plugin *azureDataDiskPlugin) GetAccessModes() []v1.PersistentVolumeAccessM func (plugin *azureDataDiskPlugin) NewAttacher() (volume.Attacher, error) { azure, err := getCloud(plugin.host) if err != nil { - glog.Errorf("failed to get azure cloud in NewAttacher, plugin.host : %s, err:%v", plugin.host.GetHostName(), err) + klog.Errorf("failed to get azure cloud in NewAttacher, plugin.host : %s, err:%v", plugin.host.GetHostName(), err) return nil, err } @@ -221,7 +221,7 @@ func (plugin *azureDataDiskPlugin) NewAttacher() (volume.Attacher, error) { func (plugin *azureDataDiskPlugin) NewDetacher() (volume.Detacher, error) { azure, err := getCloud(plugin.host) if err != nil { - glog.V(4).Infof("failed to get azure cloud in NewDetacher, plugin.host : %s", plugin.host.GetHostName()) + klog.V(4).Infof("failed to get azure cloud in NewDetacher, plugin.host : %s", plugin.host.GetHostName()) return nil, err } diff --git a/pkg/volume/azure_dd/azure_dd_block.go b/pkg/volume/azure_dd/azure_dd_block.go index af1d195047e..70430499463 100644 --- a/pkg/volume/azure_dd/azure_dd_block.go +++ b/pkg/volume/azure_dd/azure_dd_block.go @@ -20,10 +20,10 @@ import ( "fmt" "path/filepath" - "github.com/golang/glog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" kstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" @@ -44,7 +44,7 @@ func (plugin *azureDataDiskPlugin) ConstructBlockVolumeSpec(podUID types.UID, vo if err != nil { return nil, err } - glog.V(5).Infof("constructing block volume spec from globalMapPathUUID: %s", globalMapPathUUID) + klog.V(5).Infof("constructing block volume spec from globalMapPathUUID: %s", globalMapPathUUID) globalMapPath := filepath.Dir(globalMapPathUUID) if len(globalMapPath) <= 1 { @@ -63,7 +63,7 @@ func getVolumeSpecFromGlobalMapPath(globalMapPath, volumeName string) (*volume.S if len(diskName) <= 1 { return nil, fmt.Errorf("failed to get diskName from global path=%s", globalMapPath) } - glog.V(5).Infof("got diskName(%s) from globalMapPath: %s", globalMapPath, diskName) + klog.V(5).Infof("got diskName(%s) from globalMapPath: %s", globalMapPath, diskName) block := v1.PersistentVolumeBlock pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/volume/azure_dd/azure_mounter.go b/pkg/volume/azure_dd/azure_mounter.go index d8b7ae50df6..66c0e12910b 100644 --- a/pkg/volume/azure_dd/azure_mounter.go +++ b/pkg/volume/azure_dd/azure_mounter.go @@ -21,8 +21,8 @@ import ( "os" "runtime" - "github.com/golang/glog" "k8s.io/api/core/v1" + "k8s.io/klog" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" ) @@ -46,7 +46,7 @@ func (m *azureDiskMounter) GetAttributes() volume.Attributes { readOnly := false volumeSource, _, err := getVolumeSource(m.spec) if err != nil { - glog.Infof("azureDisk - mounter failed to get volume source for spec %s %v", m.spec.Name(), err) + klog.Infof("azureDisk - mounter failed to get volume source for spec %s %v", m.spec.Name(), err) } else if volumeSource.ReadOnly != nil { readOnly = *volumeSource.ReadOnly } @@ -74,7 +74,7 @@ func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error { volumeSource, _, err := getVolumeSource(m.spec) if err != nil { - glog.Infof("azureDisk - mounter failed to get volume source for spec %s", m.spec.Name()) + klog.Infof("azureDisk - mounter failed to get volume source for spec %s", m.spec.Name()) return err } @@ -82,20 +82,20 @@ func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error { mountPoint, err := mounter.IsLikelyNotMountPoint(dir) if err != nil && !os.IsNotExist(err) { - glog.Infof("azureDisk - cannot validate mount point for disk %s on %s %v", diskName, dir, err) + klog.Infof("azureDisk - cannot validate mount point for disk %s on %s %v", diskName, dir, err) return err } if !mountPoint { // testing original mount point, make sure the mount link is valid _, err := (&osIOHandler{}).ReadDir(dir) if err == nil { - glog.V(4).Infof("azureDisk - already mounted to target %s", dir) + klog.V(4).Infof("azureDisk - already mounted to target %s", dir) return nil } // mount link is invalid, now unmount and remount later - glog.Warningf("azureDisk - ReadDir %s failed with %v, unmount this directory", dir, err) + klog.Warningf("azureDisk - ReadDir %s failed with %v, unmount this directory", dir, err) if err := mounter.Unmount(dir); err != nil { - glog.Errorf("azureDisk - Unmount directory %s failed with %v", dir, err) + klog.Errorf("azureDisk - Unmount directory %s failed with %v", dir, err) return err } mountPoint = true @@ -104,7 +104,7 @@ func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error { if runtime.GOOS != "windows" { // in windows, we will use mklink to mount, will MkdirAll in Mount func if err := os.MkdirAll(dir, 0750); err != nil { - glog.Errorf("azureDisk - mkdir failed on disk %s on dir: %s (%v)", diskName, dir, err) + klog.Errorf("azureDisk - mkdir failed on disk %s on dir: %s (%v)", diskName, dir, err) return err } } @@ -119,7 +119,7 @@ func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error { options = util.JoinMountOptions(m.options.MountOptions, options) } - glog.V(4).Infof("azureDisk - Attempting to mount %s on %s", diskName, dir) + klog.V(4).Infof("azureDisk - Attempting to mount %s on %s", diskName, dir) isManagedDisk := (*volumeSource.Kind == v1.AzureManagedDisk) globalPDPath, err := makeGlobalPDPath(m.plugin.host, volumeSource.DataDiskURI, isManagedDisk) @@ -131,7 +131,7 @@ func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error { // Everything in the following control flow is meant as an // attempt cleanup a failed setupAt (bind mount) if mountErr != nil { - glog.Infof("azureDisk - SetupAt:Mount disk:%s at dir:%s failed during mounting with error:%v, will attempt to clean up", diskName, dir, mountErr) + klog.Infof("azureDisk - SetupAt:Mount disk:%s at dir:%s failed during mounting with error:%v, will attempt to clean up", diskName, dir, mountErr) mountPoint, err := mounter.IsLikelyNotMountPoint(dir) if err != nil { return fmt.Errorf("azureDisk - SetupAt:Mount:Failure:cleanup IsLikelyNotMountPoint check failed for disk:%s on dir:%s with error %v original-mountErr:%v", diskName, dir, err, mountErr) @@ -155,7 +155,7 @@ func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error { return fmt.Errorf("azureDisk - SetupAt:Mount:Failure error cleaning up (removing dir:%s) with error:%v original-mountErr:%v", dir, err, mountErr) } - glog.V(2).Infof("azureDisk - Mount of disk:%s on dir:%s failed with mount error:%v post failure clean up was completed", diskName, dir, mountErr) + klog.V(2).Infof("azureDisk - Mount of disk:%s on dir:%s failed with mount error:%v post failure clean up was completed", diskName, dir, mountErr) return mountErr } @@ -163,7 +163,7 @@ func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error { volume.SetVolumeOwnership(m, fsGroup) } - glog.V(2).Infof("azureDisk - successfully mounted disk %s on %s", diskName, dir) + klog.V(2).Infof("azureDisk - successfully mounted disk %s on %s", diskName, dir) return nil } @@ -175,11 +175,11 @@ func (u *azureDiskUnmounter) TearDownAt(dir string) error { if pathExists, pathErr := util.PathExists(dir); pathErr != nil { return fmt.Errorf("Error checking if path exists: %v", pathErr) } else if !pathExists { - glog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir) + klog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir) return nil } - glog.V(4).Infof("azureDisk - TearDownAt: %s", dir) + klog.V(4).Infof("azureDisk - TearDownAt: %s", dir) mounter := u.plugin.host.GetMounter(u.plugin.GetPluginName()) mountPoint, err := mounter.IsLikelyNotMountPoint(dir) if err != nil { diff --git a/pkg/volume/azure_file/BUILD b/pkg/volume/azure_file/BUILD index 3f278c7b227..b4fa68effea 100644 --- a/pkg/volume/azure_file/BUILD +++ b/pkg/volume/azure_file/BUILD @@ -28,7 +28,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/azure_file/azure_file.go b/pkg/volume/azure_file/azure_file.go index bc2c0deecc1..e0234ce780b 100644 --- a/pkg/volume/azure_file/azure_file.go +++ b/pkg/volume/azure_file/azure_file.go @@ -22,12 +22,12 @@ import ( "os" "runtime" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/azure" "k8s.io/kubernetes/pkg/util/mount" kstrings "k8s.io/kubernetes/pkg/util/strings" @@ -237,20 +237,20 @@ func (b *azureFileMounter) SetUp(fsGroup *int64) error { func (b *azureFileMounter) SetUpAt(dir string, fsGroup *int64) error { notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) - glog.V(4).Infof("AzureFile mount set up: %s %v %v", dir, !notMnt, err) + klog.V(4).Infof("AzureFile mount set up: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { return err } if !notMnt { // testing original mount point, make sure the mount link is valid if _, err := ioutil.ReadDir(dir); err == nil { - glog.V(4).Infof("azureFile - already mounted to target %s", dir) + klog.V(4).Infof("azureFile - already mounted to target %s", dir) return nil } // mount link is invalid, now unmount and remount later - glog.Warningf("azureFile - ReadDir %s failed with %v, unmount this directory", dir, err) + klog.Warningf("azureFile - ReadDir %s failed with %v, unmount this directory", dir, err) if err := b.mounter.Unmount(dir); err != nil { - glog.Errorf("azureFile - Unmount directory %s failed with %v", dir, err) + klog.Errorf("azureFile - Unmount directory %s failed with %v", dir, err) return err } notMnt = true @@ -285,22 +285,22 @@ func (b *azureFileMounter) SetUpAt(dir string, fsGroup *int64) error { if err != nil { notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !notMnt { if mntErr = b.mounter.Unmount(dir); mntErr != nil { - glog.Errorf("Failed to unmount: %v", mntErr) + klog.Errorf("Failed to unmount: %v", mntErr) return err } notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !notMnt { // This is very odd, we don't expect it. We'll try again next sync loop. - glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) + klog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) return err } } @@ -376,7 +376,7 @@ func getStorageEndpointSuffix(cloudprovider cloudprovider.Interface) string { const publicCloudStorageEndpointSuffix = "core.windows.net" azure, err := getAzureCloud(cloudprovider) if err != nil { - glog.Warningf("No Azure cloud provider found. Using the Azure public cloud endpoint: %s", publicCloudStorageEndpointSuffix) + klog.Warningf("No Azure cloud provider found. Using the Azure public cloud endpoint: %s", publicCloudStorageEndpointSuffix) return publicCloudStorageEndpointSuffix } return azure.Environment.StorageEndpointSuffix diff --git a/pkg/volume/azure_file/azure_provision.go b/pkg/volume/azure_file/azure_provision.go index 96ba0b10cc9..60466cedab6 100644 --- a/pkg/volume/azure_file/azure_provision.go +++ b/pkg/volume/azure_file/azure_provision.go @@ -21,7 +21,7 @@ import ( "strings" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -56,7 +56,7 @@ type azureFileDeleter struct { func (plugin *azureFilePlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) { azure, err := getAzureCloudProvider(plugin.host.GetCloudProvider()) if err != nil { - glog.V(4).Infof("failed to get azure provider") + klog.V(4).Infof("failed to get azure provider") return nil, err } @@ -92,7 +92,7 @@ func (plugin *azureFilePlugin) newDeleterInternal(spec *volume.Spec, util azureU func (plugin *azureFilePlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) { azure, err := getAzureCloudProvider(plugin.host.GetCloudProvider()) if err != nil { - glog.V(4).Infof("failed to get azure provider") + klog.V(4).Infof("failed to get azure provider") return nil, err } if len(options.PVC.Spec.AccessModes) == 0 { @@ -120,7 +120,7 @@ func (f *azureFileDeleter) GetPath() string { } func (f *azureFileDeleter) Delete() error { - glog.V(4).Infof("deleting volume %s", f.shareName) + klog.V(4).Infof("deleting volume %s", f.shareName) return f.azureProvider.DeleteFileShare(f.accountName, f.accountKey, f.shareName) } diff --git a/pkg/volume/cephfs/BUILD b/pkg/volume/cephfs/BUILD index 63d03a82c05..eeb4ee0dce5 100644 --- a/pkg/volume/cephfs/BUILD +++ b/pkg/volume/cephfs/BUILD @@ -21,7 +21,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/cephfs/cephfs.go b/pkg/volume/cephfs/cephfs.go index f9da83a4405..748fe11a475 100644 --- a/pkg/volume/cephfs/cephfs.go +++ b/pkg/volume/cephfs/cephfs.go @@ -24,17 +24,17 @@ import ( "runtime" "strings" - "github.com/golang/glog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" utilstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" ) -// This is the primary entrypoint for volume plugins. +// ProbeVolumePlugins is the primary entrypoint for volume plugins. func ProbeVolumePlugins() []volume.VolumePlugin { return []volume.VolumePlugin{&cephfsPlugin{nil}} } @@ -110,7 +110,7 @@ func (plugin *cephfsPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume. } for name, data := range secrets.Data { secret = string(data) - glog.V(4).Infof("found ceph secret info: %s", name) + klog.V(4).Infof("found ceph secret info: %s", name) } } return plugin.newMounterInternal(spec, pod.UID, plugin.host.GetMounter(plugin.GetPluginName()), secret) @@ -213,7 +213,7 @@ func (cephfsVolume *cephfsMounter) GetAttributes() volume.Attributes { // Checks prior to mount operations to verify that the required components (binaries, etc.) // to mount the volume are available on the underlying node. // If not, it returns an error -func (cephfsMounter *cephfsMounter) CanMount() error { +func (cephfsVolume *cephfsMounter) CanMount() error { return nil } @@ -225,7 +225,7 @@ func (cephfsVolume *cephfsMounter) SetUp(fsGroup *int64) error { // SetUpAt attaches the disk and bind mounts to the volume path. func (cephfsVolume *cephfsMounter) SetUpAt(dir string, fsGroup *int64) error { notMnt, err := cephfsVolume.mounter.IsLikelyNotMountPoint(dir) - glog.V(4).Infof("CephFS mount set up: %s %v %v", dir, !notMnt, err) + klog.V(4).Infof("CephFS mount set up: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { return err } @@ -239,7 +239,7 @@ func (cephfsVolume *cephfsMounter) SetUpAt(dir string, fsGroup *int64) error { // check whether it belongs to fuse, if not, default to use kernel mount. if cephfsVolume.checkFuseMount() { - glog.V(4).Info("CephFS fuse mount.") + klog.V(4).Info("CephFS fuse mount.") err = cephfsVolume.execFuseMount(dir) // cleanup no matter if fuse mount fail. keyringPath := cephfsVolume.GetKeyringPath() @@ -252,9 +252,10 @@ func (cephfsVolume *cephfsMounter) SetUpAt(dir string, fsGroup *int64) error { return nil } // if cephfs fuse mount failed, fallback to kernel mount. - glog.V(2).Infof("CephFS fuse mount failed: %v, fallback to kernel mount.", err) + klog.V(2).Infof("CephFS fuse mount failed: %v, fallback to kernel mount.", err) + } - glog.V(4).Info("CephFS kernel mount.") + klog.V(4).Info("CephFS kernel mount.") err = cephfsVolume.execMount(dir) if err != nil { @@ -330,12 +331,12 @@ func (cephfsVolume *cephfs) execMount(mountpoint string) error { return nil } -func (cephfsMounter *cephfsMounter) checkFuseMount() bool { - execute := cephfsMounter.plugin.host.GetExec(cephfsMounter.plugin.GetPluginName()) +func (cephfsVolume *cephfsMounter) checkFuseMount() bool { + execute := cephfsVolume.plugin.host.GetExec(cephfsVolume.plugin.GetPluginName()) switch runtime.GOOS { case "linux": if _, err := execute.Run("/usr/bin/test", "-x", "/sbin/mount.fuse.ceph"); err == nil { - glog.V(4).Info("/sbin/mount.fuse.ceph exists, it should be fuse mount.") + klog.V(4).Info("/sbin/mount.fuse.ceph exists, it should be fuse mount.") return true } return false @@ -350,7 +351,7 @@ func (cephfsVolume *cephfs) execFuseMount(mountpoint string) error { if cephfsVolume.secret != "" { // TODO: cephfs fuse currently doesn't support secret option, // remove keyring file create once secret option is supported. - glog.V(4).Info("cephfs mount begin using fuse.") + klog.V(4).Info("cephfs mount begin using fuse.") keyringPath := cephfsVolume.GetKeyringPath() os.MkdirAll(keyringPath, 0750) @@ -369,13 +370,13 @@ func (cephfsVolume *cephfs) execFuseMount(mountpoint string) error { writerContext := fmt.Sprintf("cephfuse:%v.keyring", cephfsVolume.id) writer, err := util.NewAtomicWriter(keyringPath, writerContext) if err != nil { - glog.Errorf("failed to create atomic writer: %v", err) + klog.Errorf("failed to create atomic writer: %v", err) return err } err = writer.Write(payload) if err != nil { - glog.Errorf("failed to write payload to dir: %v", err) + klog.Errorf("failed to write payload to dir: %v", err) return err } @@ -418,11 +419,11 @@ func (cephfsVolume *cephfs) execFuseMount(mountpoint string) error { mountArgs = append(mountArgs, strings.Join(opt, ",")) } - glog.V(4).Infof("Mounting cmd ceph-fuse with arguments (%s)", mountArgs) + klog.V(4).Infof("Mounting cmd ceph-fuse with arguments (%s)", mountArgs) command := exec.Command("ceph-fuse", mountArgs...) output, err := command.CombinedOutput() if err != nil || !(strings.Contains(string(output), "starting fuse")) { - return fmt.Errorf("ceph-fuse failed: %v\narguments: %s\nOutput: %s", err, mountArgs, string(output)) + return fmt.Errorf("Ceph-fuse failed: %v\narguments: %s\nOutput: %s", err, mountArgs, string(output)) } return nil diff --git a/pkg/volume/cinder/BUILD b/pkg/volume/cinder/BUILD index efd173464cc..9435be1ec42 100644 --- a/pkg/volume/cinder/BUILD +++ b/pkg/volume/cinder/BUILD @@ -35,7 +35,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) @@ -59,7 +59,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/cinder/attacher.go b/pkg/volume/cinder/attacher.go index 2cdc6da7a58..0db22a27386 100644 --- a/pkg/volume/cinder/attacher.go +++ b/pkg/volume/cinder/attacher.go @@ -24,10 +24,10 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" @@ -145,31 +145,31 @@ func (attacher *cinderDiskAttacher) Attach(spec *volume.Spec, nodeName types.Nod attached, err := attacher.cinderProvider.DiskIsAttached(instanceID, volumeID) if err != nil { // Log error and continue with attach - glog.Warningf( + klog.Warningf( "Error checking if volume (%q) is already attached to current instance (%q). Will continue and try attach anyway. err=%v", volumeID, instanceID, err) } if err == nil && attached { // Volume is already attached to instance. - glog.Infof("Attach operation is successful. volume %q is already attached to instance %q.", volumeID, instanceID) + klog.Infof("Attach operation is successful. volume %q is already attached to instance %q.", volumeID, instanceID) } else { _, err = attacher.cinderProvider.AttachDisk(instanceID, volumeID) if err == nil { if err = attacher.waitDiskAttached(instanceID, volumeID); err != nil { - glog.Errorf("Error waiting for volume %q to be attached from node %q: %v", volumeID, nodeName, err) + klog.Errorf("Error waiting for volume %q to be attached from node %q: %v", volumeID, nodeName, err) return "", err } - glog.Infof("Attach operation successful: volume %q attached to instance %q.", volumeID, instanceID) + klog.Infof("Attach operation successful: volume %q attached to instance %q.", volumeID, instanceID) } else { - glog.Infof("Attach volume %q to instance %q failed with: %v", volumeID, instanceID, err) + klog.Infof("Attach volume %q to instance %q failed with: %v", volumeID, instanceID, err) return "", err } } devicePath, err := attacher.cinderProvider.GetAttachmentDiskPath(instanceID, volumeID) if err != nil { - glog.Infof("Can not get device path of volume %q which be attached to instance %q, failed with: %v", volumeID, instanceID, err) + klog.Infof("Can not get device path of volume %q which be attached to instance %q, failed with: %v", volumeID, instanceID, err) return "", err } @@ -183,7 +183,7 @@ func (attacher *cinderDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nod for _, spec := range specs { volumeID, _, _, err := getVolumeInfo(spec) if err != nil { - glog.Errorf("Error getting volume (%q) source : %v", spec.Name(), err) + klog.Errorf("Error getting volume (%q) source : %v", spec.Name(), err) continue } @@ -195,7 +195,7 @@ func (attacher *cinderDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nod attachedResult, err := attacher.cinderProvider.DisksAreAttachedByName(nodeName, volumeIDList) if err != nil { // Log error and continue with attach - glog.Errorf( + klog.Errorf( "Error checking if Volumes (%v) are already attached to current node (%q). Will continue and try attach anyway. err=%v", volumeIDList, nodeName, err) return volumesAttachedCheck, err @@ -205,7 +205,7 @@ func (attacher *cinderDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nod if !attached { spec := volumeSpecMap[volumeID] volumesAttachedCheck[spec] = false - glog.V(2).Infof("VolumesAreAttached: check volume %q (specName: %q) is no longer attached", volumeID, spec.Name()) + klog.V(2).Infof("VolumesAreAttached: check volume %q (specName: %q) is no longer attached", volumeID, spec.Name()) } } return volumesAttachedCheck, nil @@ -231,7 +231,7 @@ func (attacher *cinderDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath for { select { case <-ticker.C: - glog.V(5).Infof("Checking Cinder disk %q is attached.", volumeID) + klog.V(5).Infof("Checking Cinder disk %q is attached.", volumeID) probeAttachedVolume() if !attacher.cinderProvider.ShouldTrustDevicePath() { // Using the Cinder volume ID, find the real device path (See Issue #33128) @@ -239,11 +239,11 @@ func (attacher *cinderDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath } exists, err := volumeutil.PathExists(devicePath) if exists && err == nil { - glog.Infof("Successfully found attached Cinder disk %q at %v.", volumeID, devicePath) + klog.Infof("Successfully found attached Cinder disk %q at %v.", volumeID, devicePath) return devicePath, nil } // Log an error, and continue checking periodically - glog.Errorf("Error: could not find attached Cinder disk %q (path: %q): %v", volumeID, devicePath, err) + klog.Errorf("Error: could not find attached Cinder disk %q (path: %q): %v", volumeID, devicePath, err) // Using exponential backoff instead of linear ticker.Stop() duration = time.Duration(float64(duration) * probeVolumeFactor) @@ -379,26 +379,26 @@ func (detacher *cinderDiskDetacher) Detach(volumeName string, nodeName types.Nod attached, instanceID, err := detacher.cinderProvider.DiskIsAttachedByName(nodeName, volumeID) if err != nil { // Log error and continue with detach - glog.Errorf( + klog.Errorf( "Error checking if volume (%q) is already attached to current node (%q). Will continue and try detach anyway. err=%v", volumeID, nodeName, err) } if err == nil && !attached { // Volume is already detached from node. - glog.Infof("detach operation was successful. volume %q is already detached from node %q.", volumeID, nodeName) + klog.Infof("detach operation was successful. volume %q is already detached from node %q.", volumeID, nodeName) return nil } if err = detacher.cinderProvider.DetachDisk(instanceID, volumeID); err != nil { - glog.Errorf("Error detaching volume %q from node %q: %v", volumeID, nodeName, err) + klog.Errorf("Error detaching volume %q from node %q: %v", volumeID, nodeName, err) return err } if err = detacher.waitDiskDetached(instanceID, volumeID); err != nil { - glog.Errorf("Error waiting for volume %q to detach from node %q: %v", volumeID, nodeName, err) + klog.Errorf("Error waiting for volume %q to detach from node %q: %v", volumeID, nodeName, err) return err } - glog.Infof("detached volume %q from node %q", volumeID, nodeName) + klog.Infof("detached volume %q from node %q", volumeID, nodeName) return nil } diff --git a/pkg/volume/cinder/attacher_test.go b/pkg/volume/cinder/attacher_test.go index 5902caf8fc9..7b444d84667 100644 --- a/pkg/volume/cinder/attacher_test.go +++ b/pkg/volume/cinder/attacher_test.go @@ -31,8 +31,8 @@ import ( "fmt" "sort" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" ) const ( @@ -468,7 +468,7 @@ func (testcase *testcase) AttachDisk(instanceID, volumeID string) (string, error return "", errors.New("unexpected AttachDisk call: wrong instanceID") } - glog.V(4).Infof("AttachDisk call: %s, %s, returning %q, %v", volumeID, instanceID, expected.retDeviceName, expected.ret) + klog.V(4).Infof("AttachDisk call: %s, %s, returning %q, %v", volumeID, instanceID, expected.retDeviceName, expected.ret) testcase.attachOrDetach = &attachStatus return expected.retDeviceName, expected.ret @@ -494,7 +494,7 @@ func (testcase *testcase) DetachDisk(instanceID, volumeID string) error { return errors.New("unexpected DetachDisk call: wrong instanceID") } - glog.V(4).Infof("DetachDisk call: %s, %s, returning %v", volumeID, instanceID, expected.ret) + klog.V(4).Infof("DetachDisk call: %s, %s, returning %v", volumeID, instanceID, expected.ret) testcase.attachOrDetach = &detachStatus return expected.ret @@ -504,11 +504,11 @@ func (testcase *testcase) OperationPending(diskName string) (bool, string, error expected := &testcase.operationPending if expected.volumeStatus == VolumeStatusPending { - glog.V(4).Infof("OperationPending call: %s, returning %v, %v, %v", diskName, expected.pending, expected.volumeStatus, expected.ret) + klog.V(4).Infof("OperationPending call: %s, returning %v, %v, %v", diskName, expected.pending, expected.volumeStatus, expected.ret) return true, expected.volumeStatus, expected.ret } - glog.V(4).Infof("OperationPending call: %s, returning %v, %v, %v", diskName, expected.pending, expected.volumeStatus, expected.ret) + klog.V(4).Infof("OperationPending call: %s, returning %v, %v, %v", diskName, expected.pending, expected.volumeStatus, expected.ret) return false, expected.volumeStatus, expected.ret } @@ -542,7 +542,7 @@ func (testcase *testcase) DiskIsAttached(instanceID, volumeID string) (bool, err return false, errors.New("unexpected DiskIsAttached call: wrong instanceID") } - glog.V(4).Infof("DiskIsAttached call: %s, %s, returning %v, %v", volumeID, instanceID, expected.isAttached, expected.ret) + klog.V(4).Infof("DiskIsAttached call: %s, %s, returning %v, %v", volumeID, instanceID, expected.isAttached, expected.ret) return expected.isAttached, expected.ret } @@ -566,7 +566,7 @@ func (testcase *testcase) GetAttachmentDiskPath(instanceID, volumeID string) (st return "", errors.New("unexpected GetAttachmentDiskPath call: wrong instanceID") } - glog.V(4).Infof("GetAttachmentDiskPath call: %s, %s, returning %v, %v", volumeID, instanceID, expected.retPath, expected.ret) + klog.V(4).Infof("GetAttachmentDiskPath call: %s, %s, returning %v, %v", volumeID, instanceID, expected.retPath, expected.ret) return expected.retPath, expected.ret } @@ -610,7 +610,7 @@ func (testcase *testcase) DiskIsAttachedByName(nodeName types.NodeName, volumeID return false, instanceID, errors.New("unexpected DiskIsAttachedByName call: wrong instanceID") } - glog.V(4).Infof("DiskIsAttachedByName call: %s, %s, returning %v, %v, %v", volumeID, nodeName, expected.isAttached, expected.instanceID, expected.ret) + klog.V(4).Infof("DiskIsAttachedByName call: %s, %s, returning %v, %v, %v", volumeID, nodeName, expected.isAttached, expected.instanceID, expected.ret) return expected.isAttached, expected.instanceID, expected.ret } @@ -664,7 +664,7 @@ func (testcase *testcase) DisksAreAttached(instanceID string, volumeIDs []string return areAttached, errors.New("Unexpected DisksAreAttached call: wrong instanceID") } - glog.V(4).Infof("DisksAreAttached call: %v, %s, returning %v, %v", volumeIDs, instanceID, expected.areAttached, expected.ret) + klog.V(4).Infof("DisksAreAttached call: %v, %s, returning %v, %v", volumeIDs, instanceID, expected.areAttached, expected.ret) return expected.areAttached, expected.ret } @@ -694,7 +694,7 @@ func (testcase *testcase) DisksAreAttachedByName(nodeName types.NodeName, volume return areAttached, errors.New("Unexpected DisksAreAttachedByName call: wrong instanceID") } - glog.V(4).Infof("DisksAreAttachedByName call: %v, %s, returning %v, %v", volumeIDs, nodeName, expected.areAttached, expected.ret) + klog.V(4).Infof("DisksAreAttachedByName call: %v, %s, returning %v, %v", volumeIDs, nodeName, expected.areAttached, expected.ret) return expected.areAttached, expected.ret } diff --git a/pkg/volume/cinder/cinder.go b/pkg/volume/cinder/cinder.go index 8e585338f02..953c4660f45 100644 --- a/pkg/volume/cinder/cinder.go +++ b/pkg/volume/cinder/cinder.go @@ -22,13 +22,13 @@ import ( "os" "path" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/keymutex" @@ -234,7 +234,7 @@ func (plugin *cinderPlugin) ConstructVolumeSpec(volumeName, mountPath string) (* if err != nil { return nil, err } - glog.V(4).Infof("Found volume %s mounted to %s", sourceName, mountPath) + klog.V(4).Infof("Found volume %s mounted to %s", sourceName, mountPath) cinderVolume := &v1.Volume{ Name: volumeName, VolumeSource: v1.VolumeSource{ @@ -263,7 +263,7 @@ func (plugin *cinderPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resour return oldSize, err } - glog.V(2).Infof("volume %s expanded to new size %d successfully", volumeID, int(newSize.Value())) + klog.V(2).Infof("volume %s expanded to new size %d successfully", volumeID, int(newSize.Value())) return expandedSize, nil } @@ -342,18 +342,18 @@ func (b *cinderVolumeMounter) SetUp(fsGroup *int64) error { // SetUp bind mounts to the volume path. func (b *cinderVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { - glog.V(5).Infof("Cinder SetUp %s to %s", b.pdName, dir) + klog.V(5).Infof("Cinder SetUp %s to %s", b.pdName, dir) b.plugin.volumeLocks.LockKey(b.pdName) defer b.plugin.volumeLocks.UnlockKey(b.pdName) notmnt, err := b.mounter.IsLikelyNotMountPoint(dir) if err != nil && !os.IsNotExist(err) { - glog.Errorf("Cannot validate mount point: %s %v", dir, err) + klog.Errorf("Cannot validate mount point: %s %v", dir, err) return err } if !notmnt { - glog.V(4).Infof("Something is already mounted to target %s", dir) + klog.V(4).Infof("Something is already mounted to target %s", dir) return nil } globalPDPath := makeGlobalPDName(b.plugin.host, b.pdName) @@ -364,46 +364,46 @@ func (b *cinderVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } if err := os.MkdirAll(dir, 0750); err != nil { - glog.V(4).Infof("Could not create directory %s: %v", dir, err) + klog.V(4).Infof("Could not create directory %s: %v", dir, err) return err } mountOptions := util.JoinMountOptions(options, b.mountOptions) // Perform a bind mount to the full path to allow duplicate mounts of the same PD. - glog.V(4).Infof("Attempting to mount cinder volume %s to %s with options %v", b.pdName, dir, mountOptions) + klog.V(4).Infof("Attempting to mount cinder volume %s to %s with options %v", b.pdName, dir, mountOptions) err = b.mounter.Mount(globalPDPath, dir, "", options) if err != nil { - glog.V(4).Infof("Mount failed: %v", err) + klog.V(4).Infof("Mount failed: %v", err) notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !notmnt { if mntErr = b.mounter.Unmount(dir); mntErr != nil { - glog.Errorf("Failed to unmount: %v", mntErr) + klog.Errorf("Failed to unmount: %v", mntErr) return err } notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !notmnt { // This is very odd, we don't expect it. We'll try again next sync loop. - glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", b.GetPath()) + klog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", b.GetPath()) return err } } os.Remove(dir) - glog.Errorf("Failed to mount %s: %v", dir, err) + klog.Errorf("Failed to mount %s: %v", dir, err) return err } if !b.readOnly { volume.SetVolumeOwnership(b, fsGroup) } - glog.V(3).Infof("Cinder volume %s mounted to %s", b.pdName, dir) + klog.V(3).Infof("Cinder volume %s mounted to %s", b.pdName, dir) return nil } @@ -432,18 +432,18 @@ func (c *cinderVolumeUnmounter) TearDownAt(dir string) error { if pathExists, pathErr := util.PathExists(dir); pathErr != nil { return fmt.Errorf("Error checking if path exists: %v", pathErr) } else if !pathExists { - glog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir) + klog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir) return nil } - glog.V(5).Infof("Cinder TearDown of %s", dir) + klog.V(5).Infof("Cinder TearDown of %s", dir) notmnt, err := c.mounter.IsLikelyNotMountPoint(dir) if err != nil { - glog.V(4).Infof("IsLikelyNotMountPoint check failed: %v", err) + klog.V(4).Infof("IsLikelyNotMountPoint check failed: %v", err) return err } if notmnt { - glog.V(4).Infof("Nothing is mounted to %s, ignoring", dir) + klog.V(4).Infof("Nothing is mounted to %s, ignoring", dir) return os.Remove(dir) } @@ -452,15 +452,15 @@ func (c *cinderVolumeUnmounter) TearDownAt(dir string) error { // NewMounter. We could then find volumeID there without probing MountRefs. refs, err := c.mounter.GetMountRefs(dir) if err != nil { - glog.V(4).Infof("GetMountRefs failed: %v", err) + klog.V(4).Infof("GetMountRefs failed: %v", err) return err } if len(refs) == 0 { - glog.V(4).Infof("Directory %s is not mounted", dir) + klog.V(4).Infof("Directory %s is not mounted", dir) return fmt.Errorf("directory %s is not mounted", dir) } c.pdName = path.Base(refs[0]) - glog.V(4).Infof("Found volume %s mounted to %s", c.pdName, dir) + klog.V(4).Infof("Found volume %s mounted to %s", c.pdName, dir) // lock the volume (and thus wait for any concurrrent SetUpAt to finish) c.plugin.volumeLocks.LockKey(c.pdName) @@ -469,23 +469,23 @@ func (c *cinderVolumeUnmounter) TearDownAt(dir string) error { // Reload list of references, there might be SetUpAt finished in the meantime refs, err = c.mounter.GetMountRefs(dir) if err != nil { - glog.V(4).Infof("GetMountRefs failed: %v", err) + klog.V(4).Infof("GetMountRefs failed: %v", err) return err } if err := c.mounter.Unmount(dir); err != nil { - glog.V(4).Infof("Unmount failed: %v", err) + klog.V(4).Infof("Unmount failed: %v", err) return err } - glog.V(3).Infof("Successfully unmounted: %s\n", dir) + klog.V(3).Infof("Successfully unmounted: %s\n", dir) notmnt, mntErr := c.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if notmnt { if err := os.Remove(dir); err != nil { - glog.V(4).Infof("Failed to remove directory after unmount: %v", err) + klog.V(4).Infof("Failed to remove directory after unmount: %v", err) return err } } diff --git a/pkg/volume/cinder/cinder_block.go b/pkg/volume/cinder/cinder_block.go index 02a5d744500..90e2056e049 100644 --- a/pkg/volume/cinder/cinder_block.go +++ b/pkg/volume/cinder/cinder_block.go @@ -20,9 +20,9 @@ import ( "fmt" "path/filepath" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" kstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" @@ -44,7 +44,7 @@ func (plugin *cinderPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeNam if err != nil { return nil, err } - glog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err) + klog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err) globalMapPath := filepath.Dir(globalMapPathUUID) if len(globalMapPath) <= 1 { diff --git a/pkg/volume/cinder/cinder_util.go b/pkg/volume/cinder/cinder_util.go index accc40a2a6e..57c5254e80b 100644 --- a/pkg/volume/cinder/cinder_util.go +++ b/pkg/volume/cinder/cinder_util.go @@ -24,8 +24,8 @@ import ( "strings" "time" - "github.com/golang/glog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -95,7 +95,7 @@ func (util *DiskUtil) AttachDisk(b *cinderVolumeMounter, globalPDPath string) er os.Remove(globalPDPath) return err } - glog.V(2).Infof("Safe mount successful: %q\n", devicePath) + klog.V(2).Infof("Safe mount successful: %q\n", devicePath) } return nil } @@ -109,7 +109,7 @@ func (util *DiskUtil) DetachDisk(cd *cinderVolumeUnmounter) error { if err := os.Remove(globalPDPath); err != nil { return err } - glog.V(2).Infof("Successfully unmounted main device: %s\n", globalPDPath) + klog.V(2).Infof("Successfully unmounted main device: %s\n", globalPDPath) cloud, err := cd.plugin.getCloudProvider() if err != nil { @@ -122,7 +122,7 @@ func (util *DiskUtil) DetachDisk(cd *cinderVolumeUnmounter) error { if err = cloud.DetachDisk(instanceid, cd.pdName); err != nil { return err } - glog.V(2).Infof("Successfully detached cinder volume %s", cd.pdName) + klog.V(2).Infof("Successfully detached cinder volume %s", cd.pdName) return nil } @@ -136,10 +136,10 @@ func (util *DiskUtil) DeleteVolume(cd *cinderVolumeDeleter) error { if err = cloud.DeleteVolume(cd.pdName); err != nil { // OpenStack cloud provider returns volume.tryAgainError when necessary, // no handling needed here. - glog.V(2).Infof("Error deleting cinder volume %s: %v", cd.pdName, err) + klog.V(2).Infof("Error deleting cinder volume %s: %v", cd.pdName, err) return err } - glog.V(2).Infof("Successfully deleted cinder volume %s", cd.pdName) + klog.V(2).Infof("Successfully deleted cinder volume %s", cd.pdName) return nil } @@ -149,7 +149,7 @@ func getZonesFromNodes(kubeClient clientset.Interface) (sets.String, error) { zones := make(sets.String) nodes, err := kubeClient.CoreV1().Nodes().List(metav1.ListOptions{}) if err != nil { - glog.V(2).Infof("Error listing nodes") + klog.V(2).Infof("Error listing nodes") return zones, err } for _, node := range nodes.Items { @@ -157,7 +157,7 @@ func getZonesFromNodes(kubeClient clientset.Interface) (sets.String, error) { zones.Insert(zone) } } - glog.V(4).Infof("zones found: %v", zones) + klog.V(4).Infof("zones found: %v", zones) return zones, nil } @@ -201,7 +201,7 @@ func (util *DiskUtil) CreateVolume(c *cinderVolumeProvisioner, node *v1.Node, al // No zone specified, choose one randomly in the same region zones, err := getZonesFromNodes(c.plugin.host.GetKubeClient()) if err != nil { - glog.V(2).Infof("error getting zone information: %v", err) + klog.V(2).Infof("error getting zone information: %v", err) return "", 0, nil, "", err } // if we did not get any zones, lets leave it blank and gophercloud will @@ -209,7 +209,7 @@ func (util *DiskUtil) CreateVolume(c *cinderVolumeProvisioner, node *v1.Node, al if len(zones) > 0 { availability, err = volutil.SelectZoneForVolume(false, false, "", nil, zones, node, allowedTopologies, c.options.PVC.Name) if err != nil { - glog.V(2).Infof("error selecting zone for volume: %v", err) + klog.V(2).Infof("error selecting zone for volume: %v", err) return "", 0, nil, "", err } } @@ -217,10 +217,10 @@ func (util *DiskUtil) CreateVolume(c *cinderVolumeProvisioner, node *v1.Node, al volumeID, volumeAZ, volumeRegion, IgnoreVolumeAZ, err := cloud.CreateVolume(name, volSizeGiB, vtype, availability, c.options.CloudTags) if err != nil { - glog.V(2).Infof("Error creating cinder volume: %v", err) + klog.V(2).Infof("Error creating cinder volume: %v", err) return "", 0, nil, "", err } - glog.V(2).Infof("Successfully created cinder volume %s", volumeID) + klog.V(2).Infof("Successfully created cinder volume %s", volumeID) // these are needed that pod is spawning to same AZ volumeLabels = make(map[string]string) @@ -248,17 +248,17 @@ func probeAttachedVolume() error { cmdSettle := executor.Command("udevadm", argsSettle...) _, errSettle := cmdSettle.CombinedOutput() if errSettle != nil { - glog.Errorf("error running udevadm settle %v\n", errSettle) + klog.Errorf("error running udevadm settle %v\n", errSettle) } args := []string{"trigger"} cmd := executor.Command("udevadm", args...) _, err := cmd.CombinedOutput() if err != nil { - glog.Errorf("error running udevadm trigger %v\n", err) + klog.Errorf("error running udevadm trigger %v\n", err) return err } - glog.V(4).Infof("Successfully probed all attachments") + klog.V(4).Infof("Successfully probed all attachments") return nil } diff --git a/pkg/volume/configmap/BUILD b/pkg/volume/configmap/BUILD index 64bfe582e60..1047a1b090f 100644 --- a/pkg/volume/configmap/BUILD +++ b/pkg/volume/configmap/BUILD @@ -22,7 +22,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/configmap/configmap.go b/pkg/volume/configmap/configmap.go index a1474d24595..823ce53d4b3 100644 --- a/pkg/volume/configmap/configmap.go +++ b/pkg/volume/configmap/configmap.go @@ -19,18 +19,18 @@ package configmap import ( "fmt" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" ) -// ProbeVolumePlugin is the entry point for plugin detection in a package. +// ProbeVolumePlugins is the entry point for plugin detection in a package. func ProbeVolumePlugins() []volume.VolumePlugin { return []volume.VolumePlugin{&configMapPlugin{}} } @@ -180,7 +180,7 @@ func (b *configMapVolumeMounter) SetUp(fsGroup *int64) error { } func (b *configMapVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { - glog.V(3).Infof("Setting up volume %v for pod %v at %v", b.volName, b.pod.UID, dir) + klog.V(3).Infof("Setting up volume %v for pod %v at %v", b.volName, b.pod.UID, dir) // Wrap EmptyDir, let it do the setup. wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec(), &b.pod, *b.opts) @@ -192,7 +192,7 @@ func (b *configMapVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { configMap, err := b.getConfigMap(b.pod.Namespace, b.source.Name) if err != nil { if !(errors.IsNotFound(err) && optional) { - glog.Errorf("Couldn't get configMap %v/%v: %v", b.pod.Namespace, b.source.Name, err) + klog.Errorf("Couldn't get configMap %v/%v: %v", b.pod.Namespace, b.source.Name, err) return err } configMap = &v1.ConfigMap{ @@ -204,7 +204,7 @@ func (b *configMapVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } totalBytes := totalBytes(configMap) - glog.V(3).Infof("Received configMap %v/%v containing (%v) pieces of data, %v total bytes", + klog.V(3).Infof("Received configMap %v/%v containing (%v) pieces of data, %v total bytes", b.pod.Namespace, b.source.Name, len(configMap.Data)+len(configMap.BinaryData), @@ -228,12 +228,12 @@ func (b *configMapVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if !setupSuccess { unmounter, unmountCreateErr := b.plugin.NewUnmounter(b.volName, b.podUID) if unmountCreateErr != nil { - glog.Errorf("error cleaning up mount %s after failure. Create unmounter failed with %v", b.volName, unmountCreateErr) + klog.Errorf("error cleaning up mount %s after failure. Create unmounter failed with %v", b.volName, unmountCreateErr) return } tearDownErr := unmounter.TearDown() if tearDownErr != nil { - glog.Errorf("Error tearing down volume %s with : %v", b.volName, tearDownErr) + klog.Errorf("Error tearing down volume %s with : %v", b.volName, tearDownErr) } } }() @@ -241,26 +241,26 @@ func (b *configMapVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { writerContext := fmt.Sprintf("pod %v/%v volume %v", b.pod.Namespace, b.pod.Name, b.volName) writer, err := volumeutil.NewAtomicWriter(dir, writerContext) if err != nil { - glog.Errorf("Error creating atomic writer: %v", err) + klog.Errorf("Error creating atomic writer: %v", err) return err } err = writer.Write(payload) if err != nil { - glog.Errorf("Error writing payload to dir: %v", err) + klog.Errorf("Error writing payload to dir: %v", err) return err } err = volume.SetVolumeOwnership(b, fsGroup) if err != nil { - glog.Errorf("Error applying volume ownership settings for group: %v", fsGroup) + klog.Errorf("Error applying volume ownership settings for group: %v", fsGroup) return err } setupSuccess = true return nil } -// Note: this function is exported so that it can be called from the projection volume driver +// MakePayload function is exported so that it can be called from the projection volume driver func MakePayload(mappings []v1.KeyToPath, configMap *v1.ConfigMap, defaultMode *int32, optional bool) (map[string]volumeutil.FileProjection, error) { if defaultMode == nil { return nil, fmt.Errorf("No defaultMode used, not even the default value for it") diff --git a/pkg/volume/csi/BUILD b/pkg/volume/csi/BUILD index c9adb17cc5a..2bc3929fe0f 100644 --- a/pkg/volume/csi/BUILD +++ b/pkg/volume/csi/BUILD @@ -30,9 +30,9 @@ go_library( "//staging/src/k8s.io/csi-api/pkg/client/informers/externalversions:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/informers/externalversions/csi/v1alpha1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/listers/csi/v1alpha1:go_default_library", - "//vendor/github.com/container-storage-interface/spec/lib/go/csi/v0:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/container-storage-interface/spec/lib/go/csi:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -44,6 +44,7 @@ go_test( "csi_client_test.go", "csi_mounter_test.go", "csi_plugin_test.go", + "main_test.go", ], embed = [":go_default_library"], deps = [ @@ -69,8 +70,8 @@ go_test( "//staging/src/k8s.io/client-go/util/testing:go_default_library", "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/fake:go_default_library", - "//vendor/github.com/container-storage-interface/spec/lib/go/csi/v0:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/container-storage-interface/spec/lib/go/csi:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/csi/csi_attacher.go b/pkg/volume/csi/csi_attacher.go index 34f75e9a65f..c66792af0f6 100644 --- a/pkg/volume/csi/csi_attacher.go +++ b/pkg/volume/csi/csi_attacher.go @@ -27,9 +27,9 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" - csipb "github.com/container-storage-interface/spec/lib/go/csi/v0" + csipb "github.com/container-storage-interface/spec/lib/go/csi" "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1beta1" apierrs "k8s.io/apimachinery/pkg/api/errors" @@ -60,23 +60,23 @@ var _ volume.DeviceMounter = &csiAttacher{} func (c *csiAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) { if spec == nil { - glog.Error(log("attacher.Attach missing volume.Spec")) + klog.Error(log("attacher.Attach missing volume.Spec")) return "", errors.New("missing spec") } csiSource, err := getCSISourceFromSpec(spec) if err != nil { - glog.Error(log("attacher.Attach failed to get CSI persistent source: %v", err)) + klog.Error(log("attacher.Attach failed to get CSI persistent source: %v", err)) return "", err } skip, err := c.plugin.skipAttach(csiSource.Driver) if err != nil { - glog.Error(log("attacher.Attach failed to find if driver is attachable: %v", err)) + klog.Error(log("attacher.Attach failed to find if driver is attachable: %v", err)) return "", err } if skip { - glog.V(4).Infof(log("skipping attach for driver %s", csiSource.Driver)) + klog.V(4).Infof(log("skipping attach for driver %s", csiSource.Driver)) return "", nil } @@ -95,48 +95,50 @@ func (c *csiAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string PersistentVolumeName: &pvName, }, }, - Status: storage.VolumeAttachmentStatus{Attached: false}, } _, err = c.k8s.StorageV1beta1().VolumeAttachments().Create(attachment) alreadyExist := false if err != nil { if !apierrs.IsAlreadyExists(err) { - glog.Error(log("attacher.Attach failed: %v", err)) + klog.Error(log("attacher.Attach failed: %v", err)) return "", err } alreadyExist = true } if alreadyExist { - glog.V(4).Info(log("attachment [%v] for volume [%v] already exists (will not be recreated)", attachID, csiSource.VolumeHandle)) + klog.V(4).Info(log("attachment [%v] for volume [%v] already exists (will not be recreated)", attachID, csiSource.VolumeHandle)) } else { - glog.V(4).Info(log("attachment [%v] for volume [%v] created successfully", attachID, csiSource.VolumeHandle)) + klog.V(4).Info(log("attachment [%v] for volume [%v] created successfully", attachID, csiSource.VolumeHandle)) } if _, err := c.waitForVolumeAttachment(csiSource.VolumeHandle, attachID, csiTimeout); err != nil { return "", err } - glog.V(4).Info(log("attacher.Attach finished OK with VolumeAttachment object [%s]", attachID)) + klog.V(4).Info(log("attacher.Attach finished OK with VolumeAttachment object [%s]", attachID)) + // TODO(71164): In 1.15, return empty devicePath return attachID, nil } -func (c *csiAttacher) WaitForAttach(spec *volume.Spec, attachID string, pod *v1.Pod, timeout time.Duration) (string, error) { +func (c *csiAttacher) WaitForAttach(spec *volume.Spec, _ string, pod *v1.Pod, timeout time.Duration) (string, error) { source, err := getCSISourceFromSpec(spec) if err != nil { - glog.Error(log("attacher.WaitForAttach failed to extract CSI volume source: %v", err)) + klog.Error(log("attacher.WaitForAttach failed to extract CSI volume source: %v", err)) return "", err } + attachID := getAttachmentName(source.VolumeHandle, source.Driver, string(c.plugin.host.GetNodeName())) + skip, err := c.plugin.skipAttach(source.Driver) if err != nil { - glog.Error(log("attacher.Attach failed to find if driver is attachable: %v", err)) + klog.Error(log("attacher.Attach failed to find if driver is attachable: %v", err)) return "", err } if skip { - glog.V(4).Infof(log("Driver is not attachable, skip waiting for attach")) + klog.V(4).Infof(log("Driver is not attachable, skip waiting for attach")) return "", nil } @@ -144,7 +146,7 @@ func (c *csiAttacher) WaitForAttach(spec *volume.Spec, attachID string, pod *v1. } func (c *csiAttacher) waitForVolumeAttachment(volumeHandle, attachID string, timeout time.Duration) (string, error) { - glog.V(4).Info(log("probing for updates from CSI driver for [attachment.ID=%v]", attachID)) + klog.V(4).Info(log("probing for updates from CSI driver for [attachment.ID=%v]", attachID)) timer := time.NewTimer(timeout) // TODO (vladimirvivien) investigate making this configurable defer timer.Stop() @@ -153,10 +155,10 @@ func (c *csiAttacher) waitForVolumeAttachment(volumeHandle, attachID string, tim } func (c *csiAttacher) waitForVolumeAttachmentInternal(volumeHandle, attachID string, timer *time.Timer, timeout time.Duration) (string, error) { - glog.V(4).Info(log("probing VolumeAttachment [id=%v]", attachID)) + klog.V(4).Info(log("probing VolumeAttachment [id=%v]", attachID)) attach, err := c.k8s.StorageV1beta1().VolumeAttachments().Get(attachID, meta.GetOptions{}) if err != nil { - glog.Error(log("attacher.WaitForAttach failed for volume [%s] (will continue to try): %v", volumeHandle, err)) + klog.Error(log("attacher.WaitForAttach failed for volume [%s] (will continue to try): %v", volumeHandle, err)) return "", fmt.Errorf("volume %v has GET error for volume attachment %v: %v", volumeHandle, attachID, err) } successful, err := verifyAttachmentStatus(attach, volumeHandle) @@ -179,7 +181,7 @@ func (c *csiAttacher) waitForVolumeAttachmentInternal(volumeHandle, attachID str select { case event, ok := <-ch: if !ok { - glog.Errorf("[attachment.ID=%v] watch channel had been closed", attachID) + klog.Errorf("[attachment.ID=%v] watch channel had been closed", attachID) return "", errors.New("volume attachment watch channel had been closed") } @@ -195,7 +197,7 @@ func (c *csiAttacher) waitForVolumeAttachmentInternal(volumeHandle, attachID str } case watch.Deleted: // if deleted, fail fast - glog.Error(log("VolumeAttachment [%s] has been deleted, will not continue to wait for attachment", attachID)) + klog.Error(log("VolumeAttachment [%s] has been deleted, will not continue to wait for attachment", attachID)) return "", errors.New("volume attachment has been deleted") case watch.Error: @@ -204,7 +206,7 @@ func (c *csiAttacher) waitForVolumeAttachmentInternal(volumeHandle, attachID str } case <-timer.C: - glog.Error(log("attacher.WaitForAttach timeout after %v [volume=%v; attachment.ID=%v]", timeout, volumeHandle, attachID)) + klog.Error(log("attacher.WaitForAttach timeout after %v [volume=%v; attachment.ID=%v]", timeout, volumeHandle, attachID)) return "", fmt.Errorf("attachment timeout for volume %v", volumeHandle) } } @@ -213,7 +215,7 @@ func (c *csiAttacher) waitForVolumeAttachmentInternal(volumeHandle, attachID str func verifyAttachmentStatus(attachment *storage.VolumeAttachment, volumeHandle string) (bool, error) { // if being deleted, fail fast if attachment.GetDeletionTimestamp() != nil { - glog.Error(log("VolumeAttachment [%s] has deletion timestamp, will not continue to wait for attachment", attachment.Name)) + klog.Error(log("VolumeAttachment [%s] has deletion timestamp, will not continue to wait for attachment", attachment.Name)) return false, errors.New("volume attachment is being deleted") } // attachment OK @@ -223,30 +225,30 @@ func verifyAttachmentStatus(attachment *storage.VolumeAttachment, volumeHandle s // driver reports attach error attachErr := attachment.Status.AttachError if attachErr != nil { - glog.Error(log("attachment for %v failed: %v", volumeHandle, attachErr.Message)) + klog.Error(log("attachment for %v failed: %v", volumeHandle, attachErr.Message)) return false, errors.New(attachErr.Message) } return false, nil } func (c *csiAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) { - glog.V(4).Info(log("probing attachment status for %d volume(s) ", len(specs))) + klog.V(4).Info(log("probing attachment status for %d volume(s) ", len(specs))) attached := make(map[*volume.Spec]bool) for _, spec := range specs { if spec == nil { - glog.Error(log("attacher.VolumesAreAttached missing volume.Spec")) + klog.Error(log("attacher.VolumesAreAttached missing volume.Spec")) return nil, errors.New("missing spec") } source, err := getCSISourceFromSpec(spec) if err != nil { - glog.Error(log("attacher.VolumesAreAttached failed: %v", err)) + klog.Error(log("attacher.VolumesAreAttached failed: %v", err)) continue } skip, err := c.plugin.skipAttach(source.Driver) if err != nil { - glog.Error(log("Failed to check CSIDriver for %s: %s", source.Driver, err)) + klog.Error(log("Failed to check CSIDriver for %s: %s", source.Driver, err)) } else { if skip { // This volume is not attachable, pretend it's attached @@ -256,14 +258,14 @@ func (c *csiAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.No } attachID := getAttachmentName(source.VolumeHandle, source.Driver, string(nodeName)) - glog.V(4).Info(log("probing attachment status for VolumeAttachment %v", attachID)) + klog.V(4).Info(log("probing attachment status for VolumeAttachment %v", attachID)) attach, err := c.k8s.StorageV1beta1().VolumeAttachments().Get(attachID, meta.GetOptions{}) if err != nil { attached[spec] = false - glog.Error(log("attacher.VolumesAreAttached failed for attach.ID=%v: %v", attachID, err)) + klog.Error(log("attacher.VolumesAreAttached failed for attach.ID=%v: %v", attachID, err)) continue } - glog.V(4).Info(log("attacher.VolumesAreAttached attachment [%v] has status.attached=%t", attachID, attach.Status.Attached)) + klog.V(4).Info(log("attacher.VolumesAreAttached attachment [%v] has status.attached=%t", attachID, attach.Status.Attached)) attached[spec] = attach.Status.Attached } @@ -271,18 +273,18 @@ func (c *csiAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.No } func (c *csiAttacher) GetDeviceMountPath(spec *volume.Spec) (string, error) { - glog.V(4).Info(log("attacher.GetDeviceMountPath(%v)", spec)) + klog.V(4).Info(log("attacher.GetDeviceMountPath(%v)", spec)) deviceMountPath, err := makeDeviceMountPath(c.plugin, spec) if err != nil { - glog.Error(log("attacher.GetDeviceMountPath failed to make device mount path: %v", err)) + klog.Error(log("attacher.GetDeviceMountPath failed to make device mount path: %v", err)) return "", err } - glog.V(4).Infof("attacher.GetDeviceMountPath succeeded, deviceMountPath: %s", deviceMountPath) + klog.V(4).Infof("attacher.GetDeviceMountPath succeeded, deviceMountPath: %s", deviceMountPath) return deviceMountPath, nil } func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) (err error) { - glog.V(4).Infof(log("attacher.MountDevice(%s, %s)", devicePath, deviceMountPath)) + klog.V(4).Infof(log("attacher.MountDevice(%s, %s)", devicePath, deviceMountPath)) if deviceMountPath == "" { err = fmt.Errorf("attacher.MountDevice failed, deviceMountPath is empty") @@ -291,12 +293,12 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo mounted, err := isDirMounted(c.plugin, deviceMountPath) if err != nil { - glog.Error(log("attacher.MountDevice failed while checking mount status for dir [%s]", deviceMountPath)) + klog.Error(log("attacher.MountDevice failed while checking mount status for dir [%s]", deviceMountPath)) return err } if mounted { - glog.V(4).Info(log("attacher.MountDevice skipping mount, dir already mounted [%s]", deviceMountPath)) + klog.V(4).Info(log("attacher.MountDevice skipping mount, dir already mounted [%s]", deviceMountPath)) return nil } @@ -306,35 +308,35 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo } csiSource, err := getCSISourceFromSpec(spec) if err != nil { - glog.Error(log("attacher.MountDevice failed to get CSI persistent source: %v", err)) + klog.Error(log("attacher.MountDevice failed to get CSI persistent source: %v", err)) return err } // Store volume metadata for UnmountDevice. Keep it around even if the // driver does not support NodeStage, UnmountDevice still needs it. if err = os.MkdirAll(deviceMountPath, 0750); err != nil { - glog.Error(log("attacher.MountDevice failed to create dir %#v: %v", deviceMountPath, err)) + klog.Error(log("attacher.MountDevice failed to create dir %#v: %v", deviceMountPath, err)) return err } - glog.V(4).Info(log("created target path successfully [%s]", deviceMountPath)) + klog.V(4).Info(log("created target path successfully [%s]", deviceMountPath)) dataDir := filepath.Dir(deviceMountPath) data := map[string]string{ volDataKey.volHandle: csiSource.VolumeHandle, volDataKey.driverName: csiSource.Driver, } if err = saveVolumeData(dataDir, volDataFileName, data); err != nil { - glog.Error(log("failed to save volume info data: %v", err)) + klog.Error(log("failed to save volume info data: %v", err)) if cleanerr := os.RemoveAll(dataDir); err != nil { - glog.Error(log("failed to remove dir after error [%s]: %v", dataDir, cleanerr)) + klog.Error(log("failed to remove dir after error [%s]: %v", dataDir, cleanerr)) } return err } defer func() { if err != nil { // clean up metadata - glog.Errorf(log("attacher.MountDevice failed: %v", err)) + klog.Errorf(log("attacher.MountDevice failed: %v", err)) if err := removeMountDir(c.plugin, deviceMountPath); err != nil { - glog.Error(log("attacher.MountDevice failed to remove mount dir after errir [%s]: %v", deviceMountPath, err)) + klog.Error(log("attacher.MountDevice failed to remove mount dir after errir [%s]: %v", deviceMountPath, err)) } } }() @@ -352,7 +354,7 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo return err } if !stageUnstageSet { - glog.Infof(log("attacher.MountDevice STAGE_UNSTAGE_VOLUME capability not set. Skipping MountDevice...")) + klog.Infof(log("attacher.MountDevice STAGE_UNSTAGE_VOLUME capability not set. Skipping MountDevice...")) // defer does *not* remove the metadata file and it's correct - UnmountDevice needs it there. return nil } @@ -391,7 +393,7 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo return err } - glog.V(4).Infof(log("attacher.MountDevice successfully requested NodeStageVolume [%s]", deviceMountPath)) + klog.V(4).Infof(log("attacher.MountDevice successfully requested NodeStageVolume [%s]", deviceMountPath)) return nil } @@ -402,12 +404,12 @@ var _ volume.DeviceUnmounter = &csiAttacher{} func (c *csiAttacher) Detach(volumeName string, nodeName types.NodeName) error { // volumeName in format driverNamevolumeHandle generated by plugin.GetVolumeName() if volumeName == "" { - glog.Error(log("detacher.Detach missing value for parameter volumeName")) + klog.Error(log("detacher.Detach missing value for parameter volumeName")) return errors.New("missing expected parameter volumeName") } parts := strings.Split(volumeName, volNameSep) if len(parts) != 2 { - glog.Error(log("detacher.Detach insufficient info encoded in volumeName")) + klog.Error(log("detacher.Detach insufficient info encoded in volumeName")) return errors.New("volumeName missing expected data") } @@ -417,19 +419,19 @@ func (c *csiAttacher) Detach(volumeName string, nodeName types.NodeName) error { if err := c.k8s.StorageV1beta1().VolumeAttachments().Delete(attachID, nil); err != nil { if apierrs.IsNotFound(err) { // object deleted or never existed, done - glog.V(4).Info(log("VolumeAttachment object [%v] for volume [%v] not found, object deleted", attachID, volID)) + klog.V(4).Info(log("VolumeAttachment object [%v] for volume [%v] not found, object deleted", attachID, volID)) return nil } - glog.Error(log("detacher.Detach failed to delete VolumeAttachment [%s]: %v", attachID, err)) + klog.Error(log("detacher.Detach failed to delete VolumeAttachment [%s]: %v", attachID, err)) return err } - glog.V(4).Info(log("detacher deleted ok VolumeAttachment.ID=%s", attachID)) + klog.V(4).Info(log("detacher deleted ok VolumeAttachment.ID=%s", attachID)) return c.waitForVolumeDetachment(volID, attachID) } func (c *csiAttacher) waitForVolumeDetachment(volumeHandle, attachID string) error { - glog.V(4).Info(log("probing for updates from CSI driver for [attachment.ID=%v]", attachID)) + klog.V(4).Info(log("probing for updates from CSI driver for [attachment.ID=%v]", attachID)) timeout := c.waitSleepTime * 10 timer := time.NewTimer(timeout) // TODO (vladimirvivien) investigate making this configurable @@ -439,21 +441,21 @@ func (c *csiAttacher) waitForVolumeDetachment(volumeHandle, attachID string) err } func (c *csiAttacher) waitForVolumeDetachmentInternal(volumeHandle, attachID string, timer *time.Timer, timeout time.Duration) error { - glog.V(4).Info(log("probing VolumeAttachment [id=%v]", attachID)) + klog.V(4).Info(log("probing VolumeAttachment [id=%v]", attachID)) attach, err := c.k8s.StorageV1beta1().VolumeAttachments().Get(attachID, meta.GetOptions{}) if err != nil { if apierrs.IsNotFound(err) { //object deleted or never existed, done - glog.V(4).Info(log("VolumeAttachment object [%v] for volume [%v] not found, object deleted", attachID, volumeHandle)) + klog.V(4).Info(log("VolumeAttachment object [%v] for volume [%v] not found, object deleted", attachID, volumeHandle)) return nil } - glog.Error(log("detacher.WaitForDetach failed for volume [%s] (will continue to try): %v", volumeHandle, err)) + klog.Error(log("detacher.WaitForDetach failed for volume [%s] (will continue to try): %v", volumeHandle, err)) return err } // driver reports attach error detachErr := attach.Status.DetachError if detachErr != nil { - glog.Error(log("detachment for VolumeAttachment [%v] for volume [%s] failed: %v", attachID, volumeHandle, detachErr.Message)) + klog.Error(log("detachment for VolumeAttachment [%v] for volume [%s] failed: %v", attachID, volumeHandle, detachErr.Message)) return errors.New(detachErr.Message) } @@ -468,7 +470,7 @@ func (c *csiAttacher) waitForVolumeDetachmentInternal(volumeHandle, attachID str select { case event, ok := <-ch: if !ok { - glog.Errorf("[attachment.ID=%v] watch channel had been closed", attachID) + klog.Errorf("[attachment.ID=%v] watch channel had been closed", attachID) return errors.New("volume attachment watch channel had been closed") } @@ -478,12 +480,12 @@ func (c *csiAttacher) waitForVolumeDetachmentInternal(volumeHandle, attachID str // driver reports attach error detachErr := attach.Status.DetachError if detachErr != nil { - glog.Error(log("detachment for VolumeAttachment [%v] for volume [%s] failed: %v", attachID, volumeHandle, detachErr.Message)) + klog.Error(log("detachment for VolumeAttachment [%v] for volume [%s] failed: %v", attachID, volumeHandle, detachErr.Message)) return errors.New(detachErr.Message) } case watch.Deleted: //object deleted - glog.V(4).Info(log("VolumeAttachment object [%v] for volume [%v] has been deleted", attachID, volumeHandle)) + klog.V(4).Info(log("VolumeAttachment object [%v] for volume [%v] has been deleted", attachID, volumeHandle)) return nil case watch.Error: @@ -492,14 +494,14 @@ func (c *csiAttacher) waitForVolumeDetachmentInternal(volumeHandle, attachID str } case <-timer.C: - glog.Error(log("detacher.WaitForDetach timeout after %v [volume=%v; attachment.ID=%v]", timeout, volumeHandle, attachID)) + klog.Error(log("detacher.WaitForDetach timeout after %v [volume=%v; attachment.ID=%v]", timeout, volumeHandle, attachID)) return fmt.Errorf("detachment timeout for volume %v", volumeHandle) } } } func (c *csiAttacher) UnmountDevice(deviceMountPath string) error { - glog.V(4).Info(log("attacher.UnmountDevice(%s)", deviceMountPath)) + klog.V(4).Info(log("attacher.UnmountDevice(%s)", deviceMountPath)) // Setup var driverName, volID string @@ -509,12 +511,12 @@ func (c *csiAttacher) UnmountDevice(deviceMountPath string) error { driverName = data[volDataKey.driverName] volID = data[volDataKey.volHandle] } else { - glog.Error(log("UnmountDevice failed to load volume data file [%s]: %v", dataDir, err)) + klog.Error(log("UnmountDevice failed to load volume data file [%s]: %v", dataDir, err)) // The volume might have been mounted by old CSI volume plugin. Fall back to the old behavior: read PV from API server driverName, volID, err = getDriverAndVolNameFromDeviceMountPath(c.k8s, deviceMountPath) if err != nil { - glog.Errorf(log("attacher.UnmountDevice failed to get driver and volume name from device mount path: %v", err)) + klog.Errorf(log("attacher.UnmountDevice failed to get driver and volume name from device mount path: %v", err)) return err } } @@ -529,11 +531,11 @@ func (c *csiAttacher) UnmountDevice(deviceMountPath string) error { // Check whether "STAGE_UNSTAGE_VOLUME" is set stageUnstageSet, err := hasStageUnstageCapability(ctx, csi) if err != nil { - glog.Errorf(log("attacher.UnmountDevice failed to check whether STAGE_UNSTAGE_VOLUME set: %v", err)) + klog.Errorf(log("attacher.UnmountDevice failed to check whether STAGE_UNSTAGE_VOLUME set: %v", err)) return err } if !stageUnstageSet { - glog.Infof(log("attacher.UnmountDevice STAGE_UNSTAGE_VOLUME capability not set. Skipping UnmountDevice...")) + klog.Infof(log("attacher.UnmountDevice STAGE_UNSTAGE_VOLUME capability not set. Skipping UnmountDevice...")) // Just delete the global directory + json file if err := removeMountDir(c.plugin, deviceMountPath); err != nil { return fmt.Errorf("failed to clean up gloubal mount %s: %s", dataDir, err) @@ -548,7 +550,7 @@ func (c *csiAttacher) UnmountDevice(deviceMountPath string) error { deviceMountPath) if err != nil { - glog.Errorf(log("attacher.UnmountDevice failed: %v", err)) + klog.Errorf(log("attacher.UnmountDevice failed: %v", err)) return err } @@ -557,7 +559,7 @@ func (c *csiAttacher) UnmountDevice(deviceMountPath string) error { return fmt.Errorf("failed to clean up gloubal mount %s: %s", dataDir, err) } - glog.V(4).Infof(log("attacher.UnmountDevice successfully requested NodeStageVolume [%s]", deviceMountPath)) + klog.V(4).Infof(log("attacher.UnmountDevice successfully requested NodeStageVolume [%s]", deviceMountPath)) return nil } diff --git a/pkg/volume/csi/csi_attacher_test.go b/pkg/volume/csi/csi_attacher_test.go index 52ef4f38def..c34008b6244 100644 --- a/pkg/volume/csi/csi_attacher_test.go +++ b/pkg/volume/csi/csi_attacher_test.go @@ -24,7 +24,6 @@ import ( "testing" "time" - "github.com/golang/glog" storage "k8s.io/api/storage/v1beta1" apierrs "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,6 +38,7 @@ import ( core "k8s.io/client-go/testing" utiltesting "k8s.io/client-go/util/testing" fakecsi "k8s.io/csi-api/pkg/client/clientset/versioned/fake" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" @@ -85,11 +85,11 @@ func markVolumeAttached(t *testing.T, client clientset.Interface, watch *watch.R t.Error(err) } if attach != nil { - glog.Infof("stopping wait") + klog.Infof("stopping wait") break } } - glog.Infof("stopped wait") + klog.Infof("stopped wait") if attach == nil { t.Logf("attachment not found for id:%v", attachID) @@ -332,6 +332,73 @@ func TestAttacherWaitForVolumeAttachmentWithCSIDriver(t *testing.T) { } } +func TestAttacherWaitForAttach(t *testing.T) { + tests := []struct { + name string + driver string + makeAttachment func() *storage.VolumeAttachment + expectedAttachID string + expectError bool + }{ + { + name: "successful attach", + driver: "attachable", + makeAttachment: func() *storage.VolumeAttachment { + + testAttachID := getAttachmentName("test-vol", "attachable", "node") + successfulAttachment := makeTestAttachment(testAttachID, "node", "test-pv") + successfulAttachment.Status.Attached = true + return successfulAttachment + }, + expectedAttachID: getAttachmentName("test-vol", "attachable", "node"), + expectError: false, + }, + { + name: "failed attach", + driver: "attachable", + expectError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + plug, _, tmpDir, _ := newTestWatchPlugin(t, nil) + defer os.RemoveAll(tmpDir) + + attacher, err := plug.NewAttacher() + if err != nil { + t.Fatalf("failed to create new attacher: %v", err) + } + csiAttacher := attacher.(*csiAttacher) + spec := volume.NewSpecFromPersistentVolume(makeTestPV("test-pv", 10, test.driver, "test-vol"), false) + + if test.makeAttachment != nil { + attachment := test.makeAttachment() + _, err = csiAttacher.k8s.StorageV1beta1().VolumeAttachments().Create(attachment) + if err != nil { + t.Fatalf("failed to create VolumeAttachment: %v", err) + } + gotAttachment, err := csiAttacher.k8s.StorageV1beta1().VolumeAttachments().Get(attachment.Name, meta.GetOptions{}) + if err != nil { + t.Fatalf("failed to get created VolumeAttachment: %v", err) + } + t.Logf("created test VolumeAttachment %+v", gotAttachment) + } + + attachID, err := csiAttacher.WaitForAttach(spec, "", nil, time.Second) + if err != nil && !test.expectError { + t.Errorf("Unexpected error: %s", err) + } + if err == nil && test.expectError { + t.Errorf("Expected error, got none") + } + if attachID != test.expectedAttachID { + t.Errorf("Expected attachID %q, got %q", test.expectedAttachID, attachID) + } + }) + } +} + func TestAttacherWaitForVolumeAttachment(t *testing.T) { nodeName := "test-node" testCases := []struct { diff --git a/pkg/volume/csi/csi_block.go b/pkg/volume/csi/csi_block.go index 389b80a443d..6a536cda1d2 100644 --- a/pkg/volume/csi/csi_block.go +++ b/pkg/volume/csi/csi_block.go @@ -21,15 +21,19 @@ import ( "errors" "fmt" "os" + "path" "path/filepath" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" + storage "k8s.io/api/storage/v1beta1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + kstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" + ioutil "k8s.io/kubernetes/pkg/volume/util" ) type csiBlockMapper struct { @@ -47,80 +51,61 @@ type csiBlockMapper struct { var _ volume.BlockVolumeMapper = &csiBlockMapper{} -// GetGlobalMapPath returns a path (on the node) to a device file which will be symlinked to -// Example: plugins/kubernetes.io/csi/volumeDevices/{volumeID}/dev +// GetGlobalMapPath returns a global map path (on the node) to a device file which will be symlinked to +// Example: plugins/kubernetes.io/csi/volumeDevices/{pvname}/dev func (m *csiBlockMapper) GetGlobalMapPath(spec *volume.Spec) (string, error) { dir := getVolumeDevicePluginDir(spec.Name(), m.plugin.host) - glog.V(4).Infof(log("blockMapper.GetGlobalMapPath = %s", dir)) + klog.V(4).Infof(log("blockMapper.GetGlobalMapPath = %s", dir)) return dir, nil } +// getStagingPath returns a staging path for a directory (on the node) that should be used on NodeStageVolume/NodeUnstageVolume +// Example: plugins/kubernetes.io/csi/volumeDevices/staging/{pvname} +func (m *csiBlockMapper) getStagingPath() string { + sanitizedSpecVolID := kstrings.EscapeQualifiedNameForDisk(m.specName) + return path.Join(m.plugin.host.GetVolumeDevicePluginDir(csiPluginName), "staging", sanitizedSpecVolID) +} + +// getPublishPath returns a publish path for a file (on the node) that should be used on NodePublishVolume/NodeUnpublishVolume +// Example: plugins/kubernetes.io/csi/volumeDevices/publish/{pvname} +func (m *csiBlockMapper) getPublishPath() string { + sanitizedSpecVolID := kstrings.EscapeQualifiedNameForDisk(m.specName) + return path.Join(m.plugin.host.GetVolumeDevicePluginDir(csiPluginName), "publish", sanitizedSpecVolID) +} + // GetPodDeviceMapPath returns pod's device file which will be mapped to a volume -// returns: pods/{podUid}/volumeDevices/kubernetes.io~csi/{volumeID}/dev, {volumeID} +// returns: pods/{podUid}/volumeDevices/kubernetes.io~csi, {pvname} func (m *csiBlockMapper) GetPodDeviceMapPath() (string, string) { - path := filepath.Join(m.plugin.host.GetPodVolumeDeviceDir(m.podUID, csiPluginName), m.specName, "dev") + path := m.plugin.host.GetPodVolumeDeviceDir(m.podUID, kstrings.EscapeQualifiedNameForDisk(csiPluginName)) specName := m.specName - glog.V(4).Infof(log("blockMapper.GetPodDeviceMapPath [path=%s; name=%s]", path, specName)) + klog.V(4).Infof(log("blockMapper.GetPodDeviceMapPath [path=%s; name=%s]", path, specName)) return path, specName } -// SetUpDevice ensures the device is attached returns path where the device is located. -func (m *csiBlockMapper) SetUpDevice() (string, error) { - if !m.plugin.blockEnabled { - return "", errors.New("CSIBlockVolume feature not enabled") - } +// stageVolumeForBlock stages a block volume to stagingPath +func (m *csiBlockMapper) stageVolumeForBlock( + ctx context.Context, + csi csiClient, + accessMode v1.PersistentVolumeAccessMode, + csiSource *v1.CSIPersistentVolumeSource, + attachment *storage.VolumeAttachment, +) (string, error) { + klog.V(4).Infof(log("blockMapper.stageVolumeForBlock called")) - glog.V(4).Infof(log("blockMapper.SetupDevice called")) - - if m.spec == nil { - glog.Error(log("blockMapper.Map spec is nil")) - return "", fmt.Errorf("spec is nil") - } - csiSource, err := getCSISourceFromSpec(m.spec) - if err != nil { - glog.Error(log("blockMapper.SetupDevice failed to get CSI persistent source: %v", err)) - return "", err - } - - globalMapPath, err := m.GetGlobalMapPath(m.spec) - if err != nil { - glog.Error(log("blockMapper.SetupDevice failed to get global map path: %v", err)) - return "", err - } - - globalMapPathBlockFile := filepath.Join(globalMapPath, "file") - glog.V(4).Infof(log("blockMapper.SetupDevice global device map path file set [%s]", globalMapPathBlockFile)) - - csi := m.csiClient - ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) - defer cancel() + stagingPath := m.getStagingPath() + klog.V(4).Infof(log("blockMapper.stageVolumeForBlock stagingPath set [%s]", stagingPath)) // Check whether "STAGE_UNSTAGE_VOLUME" is set stageUnstageSet, err := hasStageUnstageCapability(ctx, csi) if err != nil { - glog.Error(log("blockMapper.SetupDevice failed to check STAGE_UNSTAGE_VOLUME capability: %v", err)) + klog.Error(log("blockMapper.stageVolumeForBlock failed to check STAGE_UNSTAGE_VOLUME capability: %v", err)) return "", err } if !stageUnstageSet { - glog.Infof(log("blockMapper.SetupDevice STAGE_UNSTAGE_VOLUME capability not set. Skipping MountDevice...")) + klog.Infof(log("blockMapper.stageVolumeForBlock STAGE_UNSTAGE_VOLUME capability not set. Skipping MountDevice...")) return "", nil } - // Start MountDevice - nodeName := string(m.plugin.host.GetNodeName()) - attachID := getAttachmentName(csiSource.VolumeHandle, csiSource.Driver, nodeName) - - // search for attachment by VolumeAttachment.Spec.Source.PersistentVolumeName - attachment, err := m.k8s.StorageV1beta1().VolumeAttachments().Get(attachID, meta.GetOptions{}) - if err != nil { - glog.Error(log("blockMapper.SetupDevice failed to get volume attachment [id=%v]: %v", attachID, err)) - return "", err - } - - if attachment == nil { - glog.Error(log("blockMapper.SetupDevice unable to find VolumeAttachment [id=%s]", attachID)) - return "", errors.New("no existing VolumeAttachment found") - } publishVolumeInfo := attachment.Status.AttachmentMetadata nodeStageSecrets := map[string]string{} @@ -132,145 +117,78 @@ func (m *csiBlockMapper) SetUpDevice() (string, error) { } } - // setup path globalMapPath and block file before call to NodeStageVolume - if err := os.MkdirAll(globalMapPath, 0750); err != nil { - glog.Error(log("blockMapper.SetupDevice failed to create dir %s: %v", globalMapPath, err)) + // Creating a stagingPath directory before call to NodeStageVolume + if err := os.MkdirAll(stagingPath, 0750); err != nil { + klog.Error(log("blockMapper.stageVolumeForBlock failed to create dir %s: %v", stagingPath, err)) return "", err } - glog.V(4).Info(log("blockMapper.SetupDevice created global device map path successfully [%s]", globalMapPath)) - - // create block device file - blockFile, err := os.OpenFile(globalMapPathBlockFile, os.O_CREATE|os.O_RDWR, 0750) - if err != nil { - glog.Error(log("blockMapper.SetupDevice failed to create dir %s: %v", globalMapPathBlockFile, err)) - return "", err - } - if err := blockFile.Close(); err != nil { - glog.Error(log("blockMapper.SetupDevice failed to close file %s: %v", globalMapPathBlockFile, err)) - return "", err - } - glog.V(4).Info(log("blockMapper.SetupDevice created global map path block device file successfully [%s]", globalMapPathBlockFile)) - - //TODO (vladimirvivien) implement better AccessModes mapping between k8s and CSI - accessMode := v1.ReadWriteOnce - if m.spec.PersistentVolume.Spec.AccessModes != nil { - accessMode = m.spec.PersistentVolume.Spec.AccessModes[0] - } + klog.V(4).Info(log("blockMapper.stageVolumeForBlock created stagingPath directory successfully [%s]", stagingPath)) + // Request to stage a block volume to stagingPath. + // Expected implementation for driver is creating driver specific resource on stagingPath and + // attaching the block volume to the node. err = csi.NodeStageVolume(ctx, csiSource.VolumeHandle, publishVolumeInfo, - globalMapPathBlockFile, + stagingPath, fsTypeBlockName, accessMode, nodeStageSecrets, csiSource.VolumeAttributes) if err != nil { - glog.Error(log("blockMapper.SetupDevice failed: %v", err)) - if err := os.RemoveAll(globalMapPath); err != nil { - glog.Error(log("blockMapper.SetupDevice failed to remove dir after a NodeStageVolume() error [%s]: %v", globalMapPath, err)) - } + klog.Error(log("blockMapper.stageVolumeForBlock failed: %v", err)) return "", err } - glog.V(4).Infof(log("blockMapper.SetupDevice successfully requested NodeStageVolume [%s]", globalMapPathBlockFile)) - return globalMapPathBlockFile, nil + klog.V(4).Infof(log("blockMapper.stageVolumeForBlock successfully requested NodeStageVolume [%s]", stagingPath)) + return stagingPath, nil } -func (m *csiBlockMapper) MapDevice(devicePath, globalMapPath, volumeMapPath, volumeMapName string, podUID types.UID) error { - if !m.plugin.blockEnabled { - return errors.New("CSIBlockVolume feature not enabled") - } +// publishVolumeForBlock publishes a block volume to publishPath +func (m *csiBlockMapper) publishVolumeForBlock( + ctx context.Context, + csi csiClient, + accessMode v1.PersistentVolumeAccessMode, + csiSource *v1.CSIPersistentVolumeSource, + attachment *storage.VolumeAttachment, + stagingPath string, +) (string, error) { + klog.V(4).Infof(log("blockMapper.publishVolumeForBlock called")) - glog.V(4).Infof(log("blockMapper.MapDevice mapping block device %s", devicePath)) - - if m.spec == nil { - glog.Error(log("blockMapper.MapDevice spec is nil")) - return fmt.Errorf("spec is nil") - } - - csiSource, err := getCSISourceFromSpec(m.spec) - if err != nil { - glog.Error(log("blockMapper.MapDevice failed to get CSI persistent source: %v", err)) - return err - } - - csi := m.csiClient - ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) - defer cancel() - - globalMapPathBlockFile := devicePath - dir, _ := m.GetPodDeviceMapPath() - targetBlockFilePath := filepath.Join(dir, "file") - glog.V(4).Infof(log("blockMapper.MapDevice target volume map file path %s", targetBlockFilePath)) - - stageCapable, err := hasStageUnstageCapability(ctx, csi) - if err != nil { - glog.Error(log("blockMapper.MapDevice failed to check for STAGE_UNSTAGE_VOLUME capabilty: %v", err)) - return err - } - - if !stageCapable { - globalMapPathBlockFile = "" - } - - nodeName := string(m.plugin.host.GetNodeName()) - attachID := getAttachmentName(csiSource.VolumeHandle, csiSource.Driver, nodeName) - - // search for attachment by VolumeAttachment.Spec.Source.PersistentVolumeName - attachment, err := m.k8s.StorageV1beta1().VolumeAttachments().Get(attachID, meta.GetOptions{}) - if err != nil { - glog.Error(log("blockMapper.MapDevice failed to get volume attachment [id=%v]: %v", attachID, err)) - return err - } - - if attachment == nil { - glog.Error(log("blockMapper.MapDevice unable to find VolumeAttachment [id=%s]", attachID)) - return errors.New("no existing VolumeAttachment found") - } publishVolumeInfo := attachment.Status.AttachmentMetadata nodePublishSecrets := map[string]string{} + var err error if csiSource.NodePublishSecretRef != nil { nodePublishSecrets, err = getCredentialsFromSecret(m.k8s, csiSource.NodePublishSecretRef) if err != nil { - glog.Errorf("blockMapper.MapDevice failed to get NodePublishSecretRef %s/%s: %v", + klog.Errorf("blockMapper.publishVolumeForBlock failed to get NodePublishSecretRef %s/%s: %v", csiSource.NodePublishSecretRef.Namespace, csiSource.NodePublishSecretRef.Name, err) - return err + return "", err } } - if err := os.MkdirAll(dir, 0750); err != nil { - glog.Error(log("blockMapper.MapDevice failed to create dir %s: %v", dir, err)) - return err - } - glog.V(4).Info(log("blockMapper.MapDevice created target volume map path successfully [%s]", dir)) - - // create target map volume block file - targetBlockFile, err := os.OpenFile(targetBlockFilePath, os.O_CREATE|os.O_RDWR, 0750) - if err != nil { - glog.Error(log("blockMapper.MapDevice failed to create file %s: %v", targetBlockFilePath, err)) - return err - } - if err := targetBlockFile.Close(); err != nil { - glog.Error(log("blockMapper.MapDevice failed to close file %s: %v", targetBlockFilePath, err)) - return err - } - glog.V(4).Info(log("blockMapper.MapDevice created target volume map file successfully [%s]", targetBlockFilePath)) - - //TODO (vladimirvivien) implement better AccessModes mapping between k8s and CSI - accessMode := v1.ReadWriteOnce - if m.spec.PersistentVolume.Spec.AccessModes != nil { - accessMode = m.spec.PersistentVolume.Spec.AccessModes[0] + publishPath := m.getPublishPath() + // Setup a parent directory for publishPath before call to NodePublishVolume + publishDir := filepath.Dir(publishPath) + if err := os.MkdirAll(publishDir, 0750); err != nil { + klog.Error(log("blockMapper.publishVolumeForBlock failed to create dir %s: %v", publishDir, err)) + return "", err } + klog.V(4).Info(log("blockMapper.publishVolumeForBlock created directory for publishPath successfully [%s]", publishDir)) + // Request to publish a block volume to publishPath. + // Expectation for driver is to place a block volume on the publishPath, by bind-mounting the device file on the publishPath or + // creating device file on the publishPath. + // Parent directory for publishPath is created by k8s, but driver is responsible for creating publishPath itself. + // If driver doesn't implement NodeStageVolume, attaching the block volume to the node may be done, here. err = csi.NodePublishVolume( ctx, m.volumeID, m.readOnly, - globalMapPathBlockFile, - targetBlockFilePath, + stagingPath, + publishPath, accessMode, publishVolumeInfo, csiSource.VolumeAttributes, @@ -280,46 +198,163 @@ func (m *csiBlockMapper) MapDevice(devicePath, globalMapPath, volumeMapPath, vol ) if err != nil { - glog.Errorf(log("blockMapper.MapDevice failed: %v", err)) - if err := os.RemoveAll(dir); err != nil { - glog.Error(log("blockMapper.MapDevice failed to remove mapped dir after a NodePublish() error [%s]: %v", dir, err)) - } + klog.Errorf(log("blockMapper.publishVolumeForBlock failed: %v", err)) + return "", err + } + + return publishPath, nil +} + +// SetUpDevice ensures the device is attached returns path where the device is located. +func (m *csiBlockMapper) SetUpDevice() (string, error) { + if !m.plugin.blockEnabled { + return "", errors.New("CSIBlockVolume feature not enabled") + } + klog.V(4).Infof(log("blockMapper.SetUpDevice called")) + + // Get csiSource from spec + if m.spec == nil { + klog.Error(log("blockMapper.SetUpDevice spec is nil")) + return "", fmt.Errorf("spec is nil") + } + + csiSource, err := getCSISourceFromSpec(m.spec) + if err != nil { + klog.Error(log("blockMapper.SetUpDevice failed to get CSI persistent source: %v", err)) + return "", err + } + + // Search for attachment by VolumeAttachment.Spec.Source.PersistentVolumeName + nodeName := string(m.plugin.host.GetNodeName()) + attachID := getAttachmentName(csiSource.VolumeHandle, csiSource.Driver, nodeName) + attachment, err := m.k8s.StorageV1beta1().VolumeAttachments().Get(attachID, meta.GetOptions{}) + if err != nil { + klog.Error(log("blockMapper.SetupDevice failed to get volume attachment [id=%v]: %v", attachID, err)) + return "", err + } + + if attachment == nil { + klog.Error(log("blockMapper.SetupDevice unable to find VolumeAttachment [id=%s]", attachID)) + return "", errors.New("no existing VolumeAttachment found") + } + + //TODO (vladimirvivien) implement better AccessModes mapping between k8s and CSI + accessMode := v1.ReadWriteOnce + if m.spec.PersistentVolume.Spec.AccessModes != nil { + accessMode = m.spec.PersistentVolume.Spec.AccessModes[0] + } + + ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) + defer cancel() + + // Call NodeStageVolume + stagingPath, err := m.stageVolumeForBlock(ctx, m.csiClient, accessMode, csiSource, attachment) + if err != nil { + return "", err + } + + // Call NodePublishVolume + publishPath, err := m.publishVolumeForBlock(ctx, m.csiClient, accessMode, csiSource, attachment, stagingPath) + if err != nil { + return "", err + } + + return publishPath, nil +} + +func (m *csiBlockMapper) MapDevice(devicePath, globalMapPath, volumeMapPath, volumeMapName string, podUID types.UID) error { + return ioutil.MapBlockVolume(devicePath, globalMapPath, volumeMapPath, volumeMapName, podUID) +} + +var _ volume.BlockVolumeUnmapper = &csiBlockMapper{} + +// unpublishVolumeForBlock unpublishes a block volume from publishPath +func (m *csiBlockMapper) unpublishVolumeForBlock(ctx context.Context, csi csiClient, publishPath string) error { + // Request to unpublish a block volume from publishPath. + // Expectation for driver is to remove block volume from the publishPath, by unmounting bind-mounted device file + // or deleting device file. + // Driver is responsible for deleting publishPath itself. + // If driver doesn't implement NodeUnstageVolume, detaching the block volume from the node may be done, here. + if err := csi.NodeUnpublishVolume(ctx, m.volumeID, publishPath); err != nil { + klog.Error(log("blockMapper.unpublishVolumeForBlock failed: %v", err)) + return err + } + klog.V(4).Infof(log("blockMapper.unpublishVolumeForBlock NodeUnpublished successfully [%s]", publishPath)) + + return nil +} + +// unstageVolumeForBlock unstages a block volume from stagingPath +func (m *csiBlockMapper) unstageVolumeForBlock(ctx context.Context, csi csiClient, stagingPath string) error { + // Check whether "STAGE_UNSTAGE_VOLUME" is set + stageUnstageSet, err := hasStageUnstageCapability(ctx, csi) + if err != nil { + klog.Error(log("blockMapper.unstageVolumeForBlock failed to check STAGE_UNSTAGE_VOLUME capability: %v", err)) + return err + } + if !stageUnstageSet { + klog.Infof(log("blockMapper.unstageVolumeForBlock STAGE_UNSTAGE_VOLUME capability not set. Skipping unstageVolumeForBlock ...")) + return nil + } + + // Request to unstage a block volume from stagingPath. + // Expected implementation for driver is removing driver specific resource in stagingPath and + // detaching the block volume from the node. + if err := csi.NodeUnstageVolume(ctx, m.volumeID, stagingPath); err != nil { + klog.Errorf(log("blockMapper.unstageVolumeForBlock failed: %v", err)) + return err + } + klog.V(4).Infof(log("blockMapper.unstageVolumeForBlock NodeUnstageVolume successfully [%s]", stagingPath)) + + // Remove stagingPath directory and its contents + if err := os.RemoveAll(stagingPath); err != nil { + klog.Error(log("blockMapper.unstageVolumeForBlock failed to remove staging path after NodeUnstageVolume() error [%s]: %v", stagingPath, err)) return err } return nil } -var _ volume.BlockVolumeUnmapper = &csiBlockMapper{} - // TearDownDevice removes traces of the SetUpDevice. func (m *csiBlockMapper) TearDownDevice(globalMapPath, devicePath string) error { if !m.plugin.blockEnabled { return errors.New("CSIBlockVolume feature not enabled") } - glog.V(4).Infof(log("unmapper.TearDownDevice(globalMapPath=%s; devicePath=%s)", globalMapPath, devicePath)) + klog.V(4).Infof(log("unmapper.TearDownDevice(globalMapPath=%s; devicePath=%s)", globalMapPath, devicePath)) - csi := m.csiClient ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) defer cancel() - // unmap global device map path - if err := csi.NodeUnstageVolume(ctx, m.volumeID, globalMapPath); err != nil { - glog.Errorf(log("blockMapper.TearDownDevice failed: %v", err)) - return err - } - glog.V(4).Infof(log("blockMapper.TearDownDevice NodeUnstageVolume successfully [%s]", globalMapPath)) - - // request to remove pod volume map path also - podVolumePath, volumeName := m.GetPodDeviceMapPath() - podVolumeMapPath := filepath.Join(podVolumePath, volumeName) - if err := csi.NodeUnpublishVolume(ctx, m.volumeID, podVolumeMapPath); err != nil { - glog.Error(log("blockMapper.TearDownDevice failed: %v", err)) - return err + // Call NodeUnpublishVolume + publishPath := m.getPublishPath() + if _, err := os.Stat(publishPath); err != nil { + if os.IsNotExist(err) { + klog.V(4).Infof(log("blockMapper.TearDownDevice publishPath(%s) has already been deleted, skip calling NodeUnpublishVolume", publishPath)) + } else { + return err + } + } else { + err := m.unpublishVolumeForBlock(ctx, m.csiClient, publishPath) + if err != nil { + return err + } } - glog.V(4).Infof(log("blockMapper.TearDownDevice NodeUnpublished successfully [%s]", podVolumeMapPath)) + // Call NodeUnstageVolume + stagingPath := m.getStagingPath() + if _, err := os.Stat(stagingPath); err != nil { + if os.IsNotExist(err) { + klog.V(4).Infof(log("blockMapper.TearDownDevice stagingPath(%s) has already been deleted, skip calling NodeUnstageVolume", stagingPath)) + } else { + return err + } + } else { + err := m.unstageVolumeForBlock(ctx, m.csiClient, stagingPath) + if err != nil { + return err + } + } return nil } diff --git a/pkg/volume/csi/csi_block_test.go b/pkg/volume/csi/csi_block_test.go index b9de55609a1..b19c4867555 100644 --- a/pkg/volume/csi/csi_block_test.go +++ b/pkg/volume/csi/csi_block_test.go @@ -25,12 +25,32 @@ import ( api "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" fakeclient "k8s.io/client-go/kubernetes/fake" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" ) +func prepareBlockMapperTest(plug *csiPlugin, specVolumeName string) (*csiBlockMapper, *volume.Spec, *api.PersistentVolume, error) { + pv := makeTestPV(specVolumeName, 10, testDriver, testVol) + spec := volume.NewSpecFromPersistentVolume(pv, pv.Spec.PersistentVolumeSource.CSI.ReadOnly) + mapper, err := plug.NewBlockVolumeMapper( + spec, + &api.Pod{ObjectMeta: meta.ObjectMeta{UID: testPodUID, Namespace: testns}}, + volume.VolumeOptions{}, + ) + if err != nil { + return nil, nil, nil, fmt.Errorf("Failed to make a new Mapper: %v", err) + } + csiMapper := mapper.(*csiBlockMapper) + return csiMapper, spec, pv, nil +} + func TestBlockMapperGetGlobalMapPath(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) @@ -53,17 +73,10 @@ func TestBlockMapperGetGlobalMapPath(t *testing.T) { } for _, tc := range testCases { t.Logf("test case: %s", tc.name) - pv := makeTestPV(tc.specVolumeName, 10, testDriver, testVol) - spec := volume.NewSpecFromPersistentVolume(pv, pv.Spec.PersistentVolumeSource.CSI.ReadOnly) - mapper, err := plug.NewBlockVolumeMapper( - spec, - &api.Pod{ObjectMeta: meta.ObjectMeta{UID: testPodUID, Namespace: testns}}, - volume.VolumeOptions{}, - ) + csiMapper, spec, _, err := prepareBlockMapperTest(plug, tc.specVolumeName) if err != nil { t.Fatalf("Failed to make a new Mapper: %v", err) } - csiMapper := mapper.(*csiBlockMapper) path, err := csiMapper.GetGlobalMapPath(spec) if err != nil { @@ -76,7 +89,124 @@ func TestBlockMapperGetGlobalMapPath(t *testing.T) { } } +func TestBlockMapperGetStagingPath(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + + plug, tmpDir := newTestPlugin(t, nil, nil) + defer os.RemoveAll(tmpDir) + + testCases := []struct { + name string + specVolumeName string + path string + }{ + { + name: "simple specName", + specVolumeName: "spec-0", + path: path.Join(tmpDir, fmt.Sprintf("plugins/kubernetes.io/csi/volumeDevices/staging/%s", "spec-0")), + }, + { + name: "specName with dots", + specVolumeName: "test.spec.1", + path: path.Join(tmpDir, fmt.Sprintf("plugins/kubernetes.io/csi/volumeDevices/staging/%s", "test.spec.1")), + }, + } + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + csiMapper, _, _, err := prepareBlockMapperTest(plug, tc.specVolumeName) + if err != nil { + t.Fatalf("Failed to make a new Mapper: %v", err) + } + + path := csiMapper.getStagingPath() + + if tc.path != path { + t.Errorf("expecting path %s, got %s", tc.path, path) + } + } +} + +func TestBlockMapperGetPublishPath(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + + plug, tmpDir := newTestPlugin(t, nil, nil) + defer os.RemoveAll(tmpDir) + + testCases := []struct { + name string + specVolumeName string + path string + }{ + { + name: "simple specName", + specVolumeName: "spec-0", + path: path.Join(tmpDir, fmt.Sprintf("plugins/kubernetes.io/csi/volumeDevices/publish/%s", "spec-0")), + }, + { + name: "specName with dots", + specVolumeName: "test.spec.1", + path: path.Join(tmpDir, fmt.Sprintf("plugins/kubernetes.io/csi/volumeDevices/publish/%s", "test.spec.1")), + }, + } + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + csiMapper, _, _, err := prepareBlockMapperTest(plug, tc.specVolumeName) + if err != nil { + t.Fatalf("Failed to make a new Mapper: %v", err) + } + + path := csiMapper.getPublishPath() + + if tc.path != path { + t.Errorf("expecting path %s, got %s", tc.path, path) + } + } +} + +func TestBlockMapperGetDeviceMapPath(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + + plug, tmpDir := newTestPlugin(t, nil, nil) + defer os.RemoveAll(tmpDir) + + testCases := []struct { + name string + specVolumeName string + path string + }{ + { + name: "simple specName", + specVolumeName: "spec-0", + path: path.Join(tmpDir, fmt.Sprintf("pods/%s/volumeDevices/kubernetes.io~csi", testPodUID)), + }, + { + name: "specName with dots", + specVolumeName: "test.spec.1", + path: path.Join(tmpDir, fmt.Sprintf("pods/%s/volumeDevices/kubernetes.io~csi", testPodUID)), + }, + } + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + csiMapper, _, _, err := prepareBlockMapperTest(plug, tc.specVolumeName) + if err != nil { + t.Fatalf("Failed to make a new Mapper: %v", err) + } + + path, volName := csiMapper.GetPodDeviceMapPath() + + if tc.path != path { + t.Errorf("expecting path %s, got %s", tc.path, path) + } + + if tc.specVolumeName != volName { + t.Errorf("expecting volName %s, got %s", tc.specVolumeName, volName) + } + } +} + func TestBlockMapperSetupDevice(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) fakeClient := fakeclient.NewSimpleClientset() @@ -88,21 +218,15 @@ func TestBlockMapperSetupDevice(t *testing.T) { "fakeNode", ) plug.host = host - pv := makeTestPV("test-pv", 10, testDriver, testVol) + + csiMapper, _, pv, err := prepareBlockMapperTest(plug, "test-pv") + if err != nil { + t.Fatalf("Failed to make a new Mapper: %v", err) + } + pvName := pv.GetName() nodeName := string(plug.host.GetNodeName()) - spec := volume.NewSpecFromPersistentVolume(pv, pv.Spec.PersistentVolumeSource.CSI.ReadOnly) - // MapDevice - mapper, err := plug.NewBlockVolumeMapper( - spec, - &api.Pod{ObjectMeta: meta.ObjectMeta{UID: testPodUID, Namespace: testns}}, - volume.VolumeOptions{}, - ) - if err != nil { - t.Fatalf("failed to create new mapper: %v", err) - } - csiMapper := mapper.(*csiBlockMapper) csiMapper.csiClient = setupClient(t, true) attachID := getAttachmentName(csiMapper.volumeID, csiMapper.driverName, string(nodeName)) @@ -119,26 +243,37 @@ func TestBlockMapperSetupDevice(t *testing.T) { t.Fatalf("mapper failed to SetupDevice: %v", err) } - globalMapPath, err := csiMapper.GetGlobalMapPath(spec) - if err != nil { - t.Fatalf("mapper failed to GetGlobalMapPath: %v", err) + // Check if SetUpDevice returns the right path + publishPath := csiMapper.getPublishPath() + if devicePath != publishPath { + t.Fatalf("mapper.SetupDevice returned unexpected path %s instead of %v", devicePath, publishPath) } - if devicePath != filepath.Join(globalMapPath, "file") { - t.Fatalf("mapper.SetupDevice returned unexpected path %s instead of %v", devicePath, globalMapPath) + // Check if NodeStageVolume staged to the right path + stagingPath := csiMapper.getStagingPath() + svols := csiMapper.csiClient.(*fakeCsiDriverClient).nodeClient.GetNodeStagedVolumes() + svol, ok := svols[csiMapper.volumeID] + if !ok { + t.Error("csi server may not have received NodeStageVolume call") + } + if svol.Path != stagingPath { + t.Errorf("csi server expected device path %s, got %s", stagingPath, svol.Path) } - vols := csiMapper.csiClient.(*fakeCsiDriverClient).nodeClient.GetNodeStagedVolumes() - vol, ok := vols[csiMapper.volumeID] + // Check if NodePublishVolume published to the right path + pvols := csiMapper.csiClient.(*fakeCsiDriverClient).nodeClient.GetNodePublishedVolumes() + pvol, ok := pvols[csiMapper.volumeID] if !ok { t.Error("csi server may not have received NodePublishVolume call") } - if vol.Path != devicePath { - t.Errorf("csi server expected device path %s, got %s", devicePath, vol.Path) + if pvol.Path != publishPath { + t.Errorf("csi server expected path %s, got %s", publishPath, pvol.Path) } } func TestBlockMapperMapDevice(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) fakeClient := fakeclient.NewSimpleClientset() @@ -150,21 +285,15 @@ func TestBlockMapperMapDevice(t *testing.T) { "fakeNode", ) plug.host = host - pv := makeTestPV("test-pv", 10, testDriver, testVol) + + csiMapper, _, pv, err := prepareBlockMapperTest(plug, "test-pv") + if err != nil { + t.Fatalf("Failed to make a new Mapper: %v", err) + } + pvName := pv.GetName() nodeName := string(plug.host.GetNodeName()) - spec := volume.NewSpecFromPersistentVolume(pv, pv.Spec.PersistentVolumeSource.CSI.ReadOnly) - // MapDevice - mapper, err := plug.NewBlockVolumeMapper( - spec, - &api.Pod{ObjectMeta: meta.ObjectMeta{UID: testPodUID, Namespace: testns}}, - volume.VolumeOptions{}, - ) - if err != nil { - t.Fatalf("failed to create new mapper: %v", err) - } - csiMapper := mapper.(*csiBlockMapper) csiMapper.csiClient = setupClient(t, true) attachID := getAttachmentName(csiMapper.volumeID, csiMapper.driverName, string(nodeName)) @@ -185,6 +314,16 @@ func TestBlockMapperMapDevice(t *testing.T) { t.Fatalf("mapper failed to GetGlobalMapPath: %v", err) } + // Actual SetupDevice should create a symlink to or a bind mout of device in devicePath. + // Create dummy file there before calling MapDevice to test it properly. + fd, err := os.Create(devicePath) + if err != nil { + t.Fatalf("mapper failed to create dummy file in devicePath: %v", err) + } + if err := fd.Close(); err != nil { + t.Fatalf("mapper failed to close dummy file in devicePath: %v", err) + } + // Map device to global and pod device map path volumeMapPath, volName := csiMapper.GetPodDeviceMapPath() err = csiMapper.MapDevice(devicePath, globalMapPath, volumeMapPath, volName, csiMapper.podUID) @@ -192,26 +331,32 @@ func TestBlockMapperMapDevice(t *testing.T) { t.Fatalf("mapper failed to GetGlobalMapPath: %v", err) } - podVolumeBlockFilePath := filepath.Join(volumeMapPath, "file") - if _, err := os.Stat(podVolumeBlockFilePath); err != nil { + // Check if symlink {globalMapPath}/{podUID} exists + globalMapFilePath := filepath.Join(globalMapPath, string(csiMapper.podUID)) + if _, err := os.Stat(globalMapFilePath); err != nil { if os.IsNotExist(err) { - t.Errorf("mapper.MapDevice failed, volume path not created: %v", err) + t.Errorf("mapper.MapDevice failed, symlink in globalMapPath not created: %v", err) + t.Errorf("mapper.MapDevice devicePath:%v, globalMapPath: %v, globalMapFilePath: %v", + devicePath, globalMapPath, globalMapFilePath) } else { t.Errorf("mapper.MapDevice failed: %v", err) } } - pubs := csiMapper.csiClient.(*fakeCsiDriverClient).nodeClient.GetNodePublishedVolumes() - vol, ok := pubs[csiMapper.volumeID] - if !ok { - t.Error("csi server may not have received NodePublishVolume call") - } - if vol.Path != podVolumeBlockFilePath { - t.Errorf("csi server expected path %s, got %s", podVolumeBlockFilePath, vol.Path) + // Check if symlink {volumeMapPath}/{volName} exists + volumeMapFilePath := filepath.Join(volumeMapPath, volName) + if _, err := os.Stat(volumeMapFilePath); err != nil { + if os.IsNotExist(err) { + t.Errorf("mapper.MapDevice failed, symlink in volumeMapPath not created: %v", err) + } else { + t.Errorf("mapper.MapDevice failed: %v", err) + } } } func TestBlockMapperTearDownDevice(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) fakeClient := fakeclient.NewSimpleClientset() @@ -223,8 +368,11 @@ func TestBlockMapperTearDownDevice(t *testing.T) { "fakeNode", ) plug.host = host - pv := makeTestPV("test-pv", 10, testDriver, testVol) - spec := volume.NewSpecFromPersistentVolume(pv, pv.Spec.PersistentVolumeSource.CSI.ReadOnly) + + _, spec, pv, err := prepareBlockMapperTest(plug, "test-pv") + if err != nil { + t.Fatalf("Failed to make a new Mapper: %v", err) + } // save volume data dir := getVolumeDeviceDataDir(pv.ObjectMeta.Name, plug.host) @@ -262,15 +410,15 @@ func TestBlockMapperTearDownDevice(t *testing.T) { t.Fatal(err) } - // ensure csi client call and node unstaged - vols := csiUnmapper.csiClient.(*fakeCsiDriverClient).nodeClient.GetNodeStagedVolumes() - if _, ok := vols[csiUnmapper.volumeID]; ok { - t.Error("csi server may not have received NodeUnstageVolume call") - } - // ensure csi client call and node unpblished pubs := csiUnmapper.csiClient.(*fakeCsiDriverClient).nodeClient.GetNodePublishedVolumes() if _, ok := pubs[csiUnmapper.volumeID]; ok { t.Error("csi server may not have received NodeUnpublishVolume call") } + + // ensure csi client call and node unstaged + vols := csiUnmapper.csiClient.(*fakeCsiDriverClient).nodeClient.GetNodeStagedVolumes() + if _, ok := vols[csiUnmapper.volumeID]; ok { + t.Error("csi server may not have received NodeUnstageVolume call") + } } diff --git a/pkg/volume/csi/csi_client.go b/pkg/volume/csi/csi_client.go index 1d41400d115..ea6ba190d8e 100644 --- a/pkg/volume/csi/csi_client.go +++ b/pkg/volume/csi/csi_client.go @@ -24,11 +24,11 @@ import ( "net" "time" - csipb "github.com/container-storage-interface/spec/lib/go/csi/v0" - "github.com/golang/glog" + csipb "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc" api "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" ) @@ -46,8 +46,8 @@ type csiClient interface { targetPath string, accessMode api.PersistentVolumeAccessMode, volumeInfo map[string]string, - volumeAttribs map[string]string, - nodePublishSecrets map[string]string, + volumeContext map[string]string, + secrets map[string]string, fsType string, mountOptions []string, ) error @@ -62,8 +62,8 @@ type csiClient interface { stagingTargetPath string, fsType string, accessMode api.PersistentVolumeAccessMode, - nodeStageSecrets map[string]string, - volumeAttribs map[string]string, + secrets map[string]string, + volumeContext map[string]string, ) error NodeUnstageVolume(ctx context.Context, volID, stagingTargetPath string) error NodeGetCapabilities(ctx context.Context) ([]*csipb.NodeServiceCapability, error) @@ -112,7 +112,7 @@ func (c *csiDriverClient) NodeGetInfo(ctx context.Context) ( maxVolumePerNode int64, accessibleTopology *csipb.Topology, err error) { - glog.V(4).Info(log("calling NodeGetInfo rpc")) + klog.V(4).Info(log("calling NodeGetInfo rpc")) nodeClient, closer, err := c.nodeClientCreator(c.driverName) if err != nil { @@ -136,12 +136,12 @@ func (c *csiDriverClient) NodePublishVolume( targetPath string, accessMode api.PersistentVolumeAccessMode, volumeInfo map[string]string, - volumeAttribs map[string]string, - nodePublishSecrets map[string]string, + volumeContext map[string]string, + secrets map[string]string, fsType string, mountOptions []string, ) error { - glog.V(4).Info(log("calling NodePublishVolume rpc [volid=%s,target_path=%s]", volID, targetPath)) + klog.V(4).Info(log("calling NodePublishVolume rpc [volid=%s,target_path=%s]", volID, targetPath)) if volID == "" { return errors.New("missing volume id") } @@ -156,12 +156,12 @@ func (c *csiDriverClient) NodePublishVolume( defer closer.Close() req := &csipb.NodePublishVolumeRequest{ - VolumeId: volID, - TargetPath: targetPath, - Readonly: readOnly, - PublishInfo: volumeInfo, - VolumeAttributes: volumeAttribs, - NodePublishSecrets: nodePublishSecrets, + VolumeId: volID, + TargetPath: targetPath, + Readonly: readOnly, + PublishContext: volumeInfo, + VolumeContext: volumeContext, + Secrets: secrets, VolumeCapability: &csipb.VolumeCapability{ AccessMode: &csipb.VolumeCapability_AccessMode{ Mode: asCSIAccessMode(accessMode), @@ -190,7 +190,7 @@ func (c *csiDriverClient) NodePublishVolume( } func (c *csiDriverClient) NodeUnpublishVolume(ctx context.Context, volID string, targetPath string) error { - glog.V(4).Info(log("calling NodeUnpublishVolume rpc: [volid=%s, target_path=%s", volID, targetPath)) + klog.V(4).Info(log("calling NodeUnpublishVolume rpc: [volid=%s, target_path=%s", volID, targetPath)) if volID == "" { return errors.New("missing volume id") } @@ -215,14 +215,14 @@ func (c *csiDriverClient) NodeUnpublishVolume(ctx context.Context, volID string, func (c *csiDriverClient) NodeStageVolume(ctx context.Context, volID string, - publishInfo map[string]string, + publishContext map[string]string, stagingTargetPath string, fsType string, accessMode api.PersistentVolumeAccessMode, - nodeStageSecrets map[string]string, - volumeAttribs map[string]string, + secrets map[string]string, + volumeContext map[string]string, ) error { - glog.V(4).Info(log("calling NodeStageVolume rpc [volid=%s,staging_target_path=%s]", volID, stagingTargetPath)) + klog.V(4).Info(log("calling NodeStageVolume rpc [volid=%s,staging_target_path=%s]", volID, stagingTargetPath)) if volID == "" { return errors.New("missing volume id") } @@ -238,15 +238,15 @@ func (c *csiDriverClient) NodeStageVolume(ctx context.Context, req := &csipb.NodeStageVolumeRequest{ VolumeId: volID, - PublishInfo: publishInfo, + PublishContext: publishContext, StagingTargetPath: stagingTargetPath, VolumeCapability: &csipb.VolumeCapability{ AccessMode: &csipb.VolumeCapability_AccessMode{ Mode: asCSIAccessMode(accessMode), }, }, - NodeStageSecrets: nodeStageSecrets, - VolumeAttributes: volumeAttribs, + Secrets: secrets, + VolumeContext: volumeContext, } if fsType == fsTypeBlockName { @@ -266,7 +266,7 @@ func (c *csiDriverClient) NodeStageVolume(ctx context.Context, } func (c *csiDriverClient) NodeUnstageVolume(ctx context.Context, volID, stagingTargetPath string) error { - glog.V(4).Info(log("calling NodeUnstageVolume rpc [volid=%s,staging_target_path=%s]", volID, stagingTargetPath)) + klog.V(4).Info(log("calling NodeUnstageVolume rpc [volid=%s,staging_target_path=%s]", volID, stagingTargetPath)) if volID == "" { return errors.New("missing volume id") } @@ -289,7 +289,7 @@ func (c *csiDriverClient) NodeUnstageVolume(ctx context.Context, volID, stagingT } func (c *csiDriverClient) NodeGetCapabilities(ctx context.Context) ([]*csipb.NodeServiceCapability, error) { - glog.V(4).Info(log("calling NodeGetCapabilities rpc")) + klog.V(4).Info(log("calling NodeGetCapabilities rpc")) nodeClient, closer, err := c.nodeClientCreator(c.driverName) if err != nil { @@ -334,7 +334,7 @@ func newGrpcConn(driverName string) (*grpc.ClientConn, error) { addr = driver.driverEndpoint } network := "unix" - glog.V(4).Infof(log("creating new gRPC connection for [%s://%s]", network, addr)) + klog.V(4).Infof(log("creating new gRPC connection for [%s://%s]", network, addr)) return grpc.Dial( addr, diff --git a/pkg/volume/csi/csi_client_test.go b/pkg/volume/csi/csi_client_test.go index 83ae82de993..777be98b2de 100644 --- a/pkg/volume/csi/csi_client_test.go +++ b/pkg/volume/csi/csi_client_test.go @@ -23,7 +23,7 @@ import ( "reflect" "testing" - csipb "github.com/container-storage-interface/spec/lib/go/csi/v0" + csipb "github.com/container-storage-interface/spec/lib/go/csi" api "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/volume/csi/fake" ) @@ -57,19 +57,19 @@ func (c *fakeCsiDriverClient) NodePublishVolume( targetPath string, accessMode api.PersistentVolumeAccessMode, volumeInfo map[string]string, - volumeAttribs map[string]string, - nodePublishSecrets map[string]string, + volumeContext map[string]string, + secrets map[string]string, fsType string, mountOptions []string, ) error { c.t.Log("calling fake.NodePublishVolume...") req := &csipb.NodePublishVolumeRequest{ - VolumeId: volID, - TargetPath: targetPath, - Readonly: readOnly, - PublishInfo: volumeInfo, - VolumeAttributes: volumeAttribs, - NodePublishSecrets: nodePublishSecrets, + VolumeId: volID, + TargetPath: targetPath, + Readonly: readOnly, + PublishContext: volumeInfo, + VolumeContext: volumeContext, + Secrets: secrets, VolumeCapability: &csipb.VolumeCapability{ AccessMode: &csipb.VolumeCapability_AccessMode{ Mode: asCSIAccessMode(accessMode), @@ -100,17 +100,17 @@ func (c *fakeCsiDriverClient) NodeUnpublishVolume(ctx context.Context, volID str func (c *fakeCsiDriverClient) NodeStageVolume(ctx context.Context, volID string, - publishInfo map[string]string, + publishContext map[string]string, stagingTargetPath string, fsType string, accessMode api.PersistentVolumeAccessMode, - nodeStageSecrets map[string]string, - volumeAttribs map[string]string, + secrets map[string]string, + volumeContext map[string]string, ) error { c.t.Log("calling fake.NodeStageVolume...") req := &csipb.NodeStageVolumeRequest{ VolumeId: volID, - PublishInfo: publishInfo, + PublishContext: publishContext, StagingTargetPath: stagingTargetPath, VolumeCapability: &csipb.VolumeCapability{ AccessMode: &csipb.VolumeCapability_AccessMode{ @@ -122,8 +122,8 @@ func (c *fakeCsiDriverClient) NodeStageVolume(ctx context.Context, }, }, }, - NodeStageSecrets: nodeStageSecrets, - VolumeAttributes: volumeAttribs, + Secrets: secrets, + VolumeContext: volumeContext, } _, err := c.nodeClient.NodeStageVolume(ctx, req) @@ -321,7 +321,7 @@ func TestClientNodeStageVolume(t *testing.T) { volID string stagingTargetPath string fsType string - secret map[string]string + secrets map[string]string mustFail bool err error }{ @@ -351,7 +351,7 @@ func TestClientNodeStageVolume(t *testing.T) { tc.stagingTargetPath, tc.fsType, api.ReadWriteOnce, - tc.secret, + tc.secrets, map[string]string{"attr0": "val0"}, ) checkErr(t, tc.mustFail, err) diff --git a/pkg/volume/csi/csi_mounter.go b/pkg/volume/csi/csi_mounter.go index dfc9cd4a86d..b0884112774 100644 --- a/pkg/volume/csi/csi_mounter.go +++ b/pkg/volume/csi/csi_mounter.go @@ -23,7 +23,7 @@ import ( "os" "path" - "github.com/golang/glog" + "k8s.io/klog" api "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" @@ -75,7 +75,7 @@ var _ volume.Volume = &csiMountMgr{} func (c *csiMountMgr) GetPath() string { dir := path.Join(getTargetPath(c.podUID, c.specVolumeID, c.plugin.host), "/mount") - glog.V(4).Info(log("mounter.GetPath generated [%s]", dir)) + klog.V(4).Info(log("mounter.GetPath generated [%s]", dir)) return dir } @@ -96,22 +96,22 @@ func (c *csiMountMgr) SetUp(fsGroup *int64) error { } func (c *csiMountMgr) SetUpAt(dir string, fsGroup *int64) error { - glog.V(4).Infof(log("Mounter.SetUpAt(%s)", dir)) + klog.V(4).Infof(log("Mounter.SetUpAt(%s)", dir)) mounted, err := isDirMounted(c.plugin, dir) if err != nil { - glog.Error(log("mounter.SetUpAt failed while checking mount status for dir [%s]", dir)) + klog.Error(log("mounter.SetUpAt failed while checking mount status for dir [%s]", dir)) return err } if mounted { - glog.V(4).Info(log("mounter.SetUpAt skipping mount, dir already mounted [%s]", dir)) + klog.V(4).Info(log("mounter.SetUpAt skipping mount, dir already mounted [%s]", dir)) return nil } csiSource, err := getCSISourceFromSpec(c.spec) if err != nil { - glog.Error(log("mounter.SetupAt failed to get CSI persistent source: %v", err)) + klog.Error(log("mounter.SetupAt failed to get CSI persistent source: %v", err)) return err } @@ -123,14 +123,14 @@ func (c *csiMountMgr) SetUpAt(dir string, fsGroup *int64) error { deviceMountPath := "" stageUnstageSet, err := hasStageUnstageCapability(ctx, csi) if err != nil { - glog.Error(log("mounter.SetUpAt failed to check for STAGE_UNSTAGE_VOLUME capabilty: %v", err)) + klog.Error(log("mounter.SetUpAt failed to check for STAGE_UNSTAGE_VOLUME capabilty: %v", err)) return err } if stageUnstageSet { deviceMountPath, err = makeDeviceMountPath(c.plugin, c.spec) if err != nil { - glog.Error(log("mounter.SetUpAt failed to make device mount path: %v", err)) + klog.Error(log("mounter.SetUpAt failed to make device mount path: %v", err)) return err } } @@ -156,10 +156,10 @@ func (c *csiMountMgr) SetUpAt(dir string, fsGroup *int64) error { // create target_dir before call to NodePublish if err := os.MkdirAll(dir, 0750); err != nil { - glog.Error(log("mouter.SetUpAt failed to create dir %#v: %v", dir, err)) + klog.Error(log("mouter.SetUpAt failed to create dir %#v: %v", dir, err)) return err } - glog.V(4).Info(log("created target path successfully [%s]", dir)) + klog.V(4).Info(log("created target path successfully [%s]", dir)) //TODO (vladimirvivien) implement better AccessModes mapping between k8s and CSI accessMode := api.ReadWriteOnce @@ -170,7 +170,7 @@ func (c *csiMountMgr) SetUpAt(dir string, fsGroup *int64) error { // Inject pod information into volume_attributes podAttrs, err := c.podAttributes() if err != nil { - glog.Error(log("mouter.SetUpAt failed to assemble volume attributes: %v", err)) + klog.Error(log("mouter.SetUpAt failed to assemble volume attributes: %v", err)) return err } if podAttrs != nil { @@ -199,9 +199,9 @@ func (c *csiMountMgr) SetUpAt(dir string, fsGroup *int64) error { ) if err != nil { - glog.Errorf(log("mounter.SetupAt failed: %v", err)) + klog.Errorf(log("mounter.SetupAt failed: %v", err)) if removeMountDirErr := removeMountDir(c.plugin, dir); removeMountDirErr != nil { - glog.Error(log("mounter.SetupAt failed to remove mount dir after a NodePublish() error [%s]: %v", dir, removeMountDirErr)) + klog.Error(log("mounter.SetupAt failed to remove mount dir after a NodePublish() error [%s]: %v", dir, removeMountDirErr)) } return err } @@ -216,18 +216,18 @@ func (c *csiMountMgr) SetUpAt(dir string, fsGroup *int64) error { // attempt to rollback mount. fsGrpErr := fmt.Errorf("applyFSGroup failed for vol %s: %v", c.volumeID, err) if unpubErr := csi.NodeUnpublishVolume(ctx, c.volumeID, dir); unpubErr != nil { - glog.Error(log("NodeUnpublishVolume failed for [%s]: %v", c.volumeID, unpubErr)) + klog.Error(log("NodeUnpublishVolume failed for [%s]: %v", c.volumeID, unpubErr)) return fsGrpErr } if unmountErr := removeMountDir(c.plugin, dir); unmountErr != nil { - glog.Error(log("removeMountDir failed for [%s]: %v", dir, unmountErr)) + klog.Error(log("removeMountDir failed for [%s]: %v", dir, unmountErr)) return fsGrpErr } return fsGrpErr } - glog.V(4).Infof(log("mounter.SetUp successfully requested NodePublish [%s]", dir)) + klog.V(4).Infof(log("mounter.SetUp successfully requested NodePublish [%s]", dir)) return nil } @@ -242,7 +242,7 @@ func (c *csiMountMgr) podAttributes() (map[string]string, error) { csiDriver, err := c.plugin.csiDriverLister.Get(c.driverName) if err != nil { if apierrs.IsNotFound(err) { - glog.V(4).Infof(log("CSIDriver %q not found, not adding pod information", c.driverName)) + klog.V(4).Infof(log("CSIDriver %q not found, not adding pod information", c.driverName)) return nil, nil } return nil, err @@ -250,7 +250,7 @@ func (c *csiMountMgr) podAttributes() (map[string]string, error) { // if PodInfoOnMountVersion is not set or not v1 we do not set pod attributes if csiDriver.Spec.PodInfoOnMountVersion == nil || *csiDriver.Spec.PodInfoOnMountVersion != currentPodInfoMountVersion { - glog.V(4).Infof(log("CSIDriver %q does not require pod information", c.driverName)) + klog.V(4).Infof(log("CSIDriver %q does not require pod information", c.driverName)) return nil, nil } @@ -260,7 +260,7 @@ func (c *csiMountMgr) podAttributes() (map[string]string, error) { "csi.storage.k8s.io/pod.uid": string(c.pod.UID), "csi.storage.k8s.io/serviceAccount.name": c.pod.Spec.ServiceAccountName, } - glog.V(4).Infof(log("CSIDriver %q requires pod information", c.driverName)) + klog.V(4).Infof(log("CSIDriver %q requires pod information", c.driverName)) return attrs, nil } @@ -269,7 +269,7 @@ func (c *csiMountMgr) GetAttributes() volume.Attributes { path := c.GetPath() supportSelinux, err := mounter.GetSELinuxSupport(path) if err != nil { - glog.V(2).Info(log("error checking for SELinux support: %s", err)) + klog.V(2).Info(log("error checking for SELinux support: %s", err)) // Best guess supportSelinux = false } @@ -287,19 +287,19 @@ func (c *csiMountMgr) TearDown() error { return c.TearDownAt(c.GetPath()) } func (c *csiMountMgr) TearDownAt(dir string) error { - glog.V(4).Infof(log("Unmounter.TearDown(%s)", dir)) + klog.V(4).Infof(log("Unmounter.TearDown(%s)", dir)) // is dir even mounted ? // TODO (vladimirvivien) this check may not work for an emptyDir or local storage // see https://github.com/kubernetes/kubernetes/pull/56836#discussion_r155834524 mounted, err := isDirMounted(c.plugin, dir) if err != nil { - glog.Error(log("unmounter.Teardown failed while checking mount status for dir [%s]: %v", dir, err)) + klog.Error(log("unmounter.Teardown failed while checking mount status for dir [%s]: %v", dir, err)) return err } if !mounted { - glog.V(4).Info(log("unmounter.Teardown skipping unmount, dir not mounted [%s]", dir)) + klog.V(4).Info(log("unmounter.Teardown skipping unmount, dir not mounted [%s]", dir)) return nil } @@ -310,16 +310,16 @@ func (c *csiMountMgr) TearDownAt(dir string) error { defer cancel() if err := csi.NodeUnpublishVolume(ctx, volID, dir); err != nil { - glog.Errorf(log("mounter.TearDownAt failed: %v", err)) + klog.Errorf(log("mounter.TearDownAt failed: %v", err)) return err } // clean mount point dir if err := removeMountDir(c.plugin, dir); err != nil { - glog.Error(log("mounter.TearDownAt failed to clean mount dir [%s]: %v", dir, err)) + klog.Error(log("mounter.TearDownAt failed to clean mount dir [%s]: %v", dir, err)) return err } - glog.V(4).Infof(log("mounte.TearDownAt successfully unmounted dir [%s]", dir)) + klog.V(4).Infof(log("mounte.TearDownAt successfully unmounted dir [%s]", dir)) return nil } @@ -331,22 +331,22 @@ func (c *csiMountMgr) TearDownAt(dir string) error { func (c *csiMountMgr) applyFSGroup(fsType string, fsGroup *int64) error { if fsGroup != nil { if fsType == "" { - glog.V(4).Info(log("mounter.SetupAt WARNING: skipping fsGroup, fsType not provided")) + klog.V(4).Info(log("mounter.SetupAt WARNING: skipping fsGroup, fsType not provided")) return nil } accessModes := c.spec.PersistentVolume.Spec.AccessModes if c.spec.PersistentVolume.Spec.AccessModes == nil { - glog.V(4).Info(log("mounter.SetupAt WARNING: skipping fsGroup, access modes not provided")) + klog.V(4).Info(log("mounter.SetupAt WARNING: skipping fsGroup, access modes not provided")) return nil } if !hasReadWriteOnce(accessModes) { - glog.V(4).Info(log("mounter.SetupAt WARNING: skipping fsGroup, only support ReadWriteOnce access mode")) + klog.V(4).Info(log("mounter.SetupAt WARNING: skipping fsGroup, only support ReadWriteOnce access mode")) return nil } if c.readOnly { - glog.V(4).Info(log("mounter.SetupAt WARNING: skipping fsGroup, volume is readOnly")) + klog.V(4).Info(log("mounter.SetupAt WARNING: skipping fsGroup, volume is readOnly")) return nil } @@ -355,7 +355,7 @@ func (c *csiMountMgr) applyFSGroup(fsType string, fsGroup *int64) error { return err } - glog.V(4).Info(log("mounter.SetupAt fsGroup [%d] applied successfully to %s", *fsGroup, c.volumeID)) + klog.V(4).Info(log("mounter.SetupAt fsGroup [%d] applied successfully to %s", *fsGroup, c.volumeID)) } return nil @@ -366,7 +366,7 @@ func isDirMounted(plug *csiPlugin, dir string) (bool, error) { mounter := plug.host.GetMounter(plug.GetPluginName()) notMnt, err := mounter.IsLikelyNotMountPoint(dir) if err != nil && !os.IsNotExist(err) { - glog.Error(log("isDirMounted IsLikelyNotMountPoint test failed for dir [%v]", dir)) + klog.Error(log("isDirMounted IsLikelyNotMountPoint test failed for dir [%v]", dir)) return false, err } return !notMnt, nil @@ -374,39 +374,39 @@ func isDirMounted(plug *csiPlugin, dir string) (bool, error) { // removeMountDir cleans the mount dir when dir is not mounted and removed the volume data file in dir func removeMountDir(plug *csiPlugin, mountPath string) error { - glog.V(4).Info(log("removing mount path [%s]", mountPath)) + klog.V(4).Info(log("removing mount path [%s]", mountPath)) if pathExists, pathErr := util.PathExists(mountPath); pathErr != nil { - glog.Error(log("failed while checking mount path stat [%s]", pathErr)) + klog.Error(log("failed while checking mount path stat [%s]", pathErr)) return pathErr } else if !pathExists { - glog.Warning(log("skipping mount dir removal, path does not exist [%v]", mountPath)) + klog.Warning(log("skipping mount dir removal, path does not exist [%v]", mountPath)) return nil } mounter := plug.host.GetMounter(plug.GetPluginName()) notMnt, err := mounter.IsLikelyNotMountPoint(mountPath) if err != nil { - glog.Error(log("mount dir removal failed [%s]: %v", mountPath, err)) + klog.Error(log("mount dir removal failed [%s]: %v", mountPath, err)) return err } if notMnt { - glog.V(4).Info(log("dir not mounted, deleting it [%s]", mountPath)) + klog.V(4).Info(log("dir not mounted, deleting it [%s]", mountPath)) if err := os.Remove(mountPath); err != nil && !os.IsNotExist(err) { - glog.Error(log("failed to remove dir [%s]: %v", mountPath, err)) + klog.Error(log("failed to remove dir [%s]: %v", mountPath, err)) return err } // remove volume data file as well volPath := path.Dir(mountPath) dataFile := path.Join(volPath, volDataFileName) - glog.V(4).Info(log("also deleting volume info data file [%s]", dataFile)) + klog.V(4).Info(log("also deleting volume info data file [%s]", dataFile)) if err := os.Remove(dataFile); err != nil && !os.IsNotExist(err) { - glog.Error(log("failed to delete volume data file [%s]: %v", dataFile, err)) + klog.Error(log("failed to delete volume data file [%s]: %v", dataFile, err)) return err } // remove volume path - glog.V(4).Info(log("deleting volume path [%s]", volPath)) + klog.V(4).Info(log("deleting volume path [%s]", volPath)) if err := os.Remove(volPath); err != nil && !os.IsNotExist(err) { - glog.Error(log("failed to delete volume path [%s]: %v", volPath, err)) + klog.Error(log("failed to delete volume path [%s]: %v", volPath, err)) return err } } diff --git a/pkg/volume/csi/csi_mounter_test.go b/pkg/volume/csi/csi_mounter_test.go index 4aa1a44d797..1e86a210998 100644 --- a/pkg/volume/csi/csi_mounter_test.go +++ b/pkg/volume/csi/csi_mounter_test.go @@ -27,7 +27,6 @@ import ( "reflect" - "github.com/golang/glog" api "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1beta1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -38,6 +37,7 @@ import ( fakeclient "k8s.io/client-go/kubernetes/fake" csiapi "k8s.io/csi-api/pkg/apis/csi/v1alpha1" fakecsi "k8s.io/csi-api/pkg/client/clientset/versioned/fake" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" @@ -98,54 +98,53 @@ func TestMounterGetPath(t *testing.T) { func MounterSetUpTests(t *testing.T, podInfoEnabled bool) { defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIDriverRegistry, podInfoEnabled)() tests := []struct { - name string - driver string - attributes map[string]string - - expectedAttributes map[string]string + name string + driver string + volumeContext map[string]string + expectedVolumeContext map[string]string }{ { - name: "no pod info", - driver: "no-info", - attributes: nil, - expectedAttributes: nil, + name: "no pod info", + driver: "no-info", + volumeContext: nil, + expectedVolumeContext: nil, }, { - name: "no CSIDriver -> no pod info", - driver: "unknown-driver", - attributes: nil, - expectedAttributes: nil, + name: "no CSIDriver -> no pod info", + driver: "unknown-driver", + volumeContext: nil, + expectedVolumeContext: nil, }, { - name: "CSIDriver with PodInfoRequiredOnMount=nil -> no pod info", - driver: "nil", - attributes: nil, - expectedAttributes: nil, + name: "CSIDriver with PodInfoRequiredOnMount=nil -> no pod info", + driver: "nil", + volumeContext: nil, + expectedVolumeContext: nil, }, { - name: "no pod info -> keep existing attributes", - driver: "no-info", - attributes: map[string]string{"foo": "bar"}, - expectedAttributes: map[string]string{"foo": "bar"}, + name: "no pod info -> keep existing volumeContext", + driver: "no-info", + volumeContext: map[string]string{"foo": "bar"}, + expectedVolumeContext: map[string]string{"foo": "bar"}, }, { - name: "add pod info", - driver: "info", - attributes: nil, - expectedAttributes: map[string]string{"csi.storage.k8s.io/pod.uid": "test-pod", "csi.storage.k8s.io/serviceAccount.name": "test-service-account", "csi.storage.k8s.io/pod.name": "test-pod", "csi.storage.k8s.io/pod.namespace": "test-ns"}, + name: "add pod info", + driver: "info", + volumeContext: nil, + expectedVolumeContext: map[string]string{"csi.storage.k8s.io/pod.uid": "test-pod", "csi.storage.k8s.io/serviceAccount.name": "test-service-account", "csi.storage.k8s.io/pod.name": "test-pod", "csi.storage.k8s.io/pod.namespace": "test-ns"}, }, { - name: "add pod info -> keep existing attributes", - driver: "info", - attributes: map[string]string{"foo": "bar"}, - expectedAttributes: map[string]string{"foo": "bar", "csi.storage.k8s.io/pod.uid": "test-pod", "csi.storage.k8s.io/serviceAccount.name": "test-service-account", "csi.storage.k8s.io/pod.name": "test-pod", "csi.storage.k8s.io/pod.namespace": "test-ns"}, + name: "add pod info -> keep existing volumeContext", + driver: "info", + volumeContext: map[string]string{"foo": "bar"}, + expectedVolumeContext: map[string]string{"foo": "bar", "csi.storage.k8s.io/pod.uid": "test-pod", "csi.storage.k8s.io/serviceAccount.name": "test-service-account", "csi.storage.k8s.io/pod.name": "test-pod", "csi.storage.k8s.io/pod.namespace": "test-ns"}, }, } emptyPodMountInfoVersion := "" for _, test := range tests { t.Run(test.name, func(t *testing.T) { - glog.Infof("Starting test %s", test.name) + klog.Infof("Starting test %s", test.name) fakeClient := fakeclient.NewSimpleClientset() fakeCSIClient := fakecsi.NewSimpleClientset( getCSIDriver("no-info", &emptyPodMountInfoVersion, nil), @@ -163,7 +162,7 @@ func MounterSetUpTests(t *testing.T, podInfoEnabled bool) { } pv := makeTestPV("test-pv", 10, test.driver, testVol) - pv.Spec.CSI.VolumeAttributes = test.attributes + pv.Spec.CSI.VolumeAttributes = test.volumeContext pv.Spec.MountOptions = []string{"foo=bar", "baz=qux"} pvName := pv.GetName() @@ -245,13 +244,13 @@ func MounterSetUpTests(t *testing.T, podInfoEnabled bool) { t.Errorf("csi server expected mount options %v, got %v", pv.Spec.MountOptions, vol.MountFlags) } if podInfoEnabled { - if !reflect.DeepEqual(vol.Attributes, test.expectedAttributes) { - t.Errorf("csi server expected attributes %+v, got %+v", test.expectedAttributes, vol.Attributes) + if !reflect.DeepEqual(vol.VolumeContext, test.expectedVolumeContext) { + t.Errorf("csi server expected volumeContext %+v, got %+v", test.expectedVolumeContext, vol.VolumeContext) } } else { - // CSIPodInfo feature is disabled, we expect no modifications to attributes. - if !reflect.DeepEqual(vol.Attributes, test.attributes) { - t.Errorf("csi server expected attributes %+v, got %+v", test.attributes, vol.Attributes) + // CSIPodInfo feature is disabled, we expect no modifications to volumeContext. + if !reflect.DeepEqual(vol.VolumeContext, test.volumeContext) { + t.Errorf("csi server expected volumeContext %+v, got %+v", test.volumeContext, vol.VolumeContext) } } }) diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index 4b8f97c3d6d..271ca573aed 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -27,7 +27,7 @@ import ( "context" - "github.com/golang/glog" + "k8s.io/klog" api "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" @@ -108,7 +108,7 @@ var PluginHandler = &RegistrationHandler{} // ValidatePlugin is called by kubelet's plugin watcher upon detection // of a new registration socket opened by CSI Driver registrar side car. func (h *RegistrationHandler) ValidatePlugin(pluginName string, endpoint string, versions []string) error { - glog.Infof(log("Trying to register a new plugin with name: %s endpoint: %s versions: %s", + klog.Infof(log("Trying to register a new plugin with name: %s endpoint: %s versions: %s", pluginName, endpoint, strings.Join(versions, ","))) return nil @@ -116,7 +116,7 @@ func (h *RegistrationHandler) ValidatePlugin(pluginName string, endpoint string, // RegisterPlugin is called when a plugin can be registered func (h *RegistrationHandler) RegisterPlugin(pluginName string, endpoint string) error { - glog.Infof(log("Register new plugin with name: %s at endpoint: %s", pluginName, endpoint)) + klog.Infof(log("Register new plugin with name: %s at endpoint: %s", pluginName, endpoint)) func() { // Storing endpoint of newly registered CSI driver into the map, where CSI driver name will be the key @@ -138,19 +138,19 @@ func (h *RegistrationHandler) RegisterPlugin(pluginName string, endpoint string) driverNodeID, maxVolumePerNode, accessibleTopology, err := csi.NodeGetInfo(ctx) if err != nil { - glog.Error(log("registrationHandler.RegisterPlugin failed at CSI.NodeGetInfo: %v", err)) + klog.Error(log("registrationHandler.RegisterPlugin failed at CSI.NodeGetInfo: %v", err)) if unregErr := unregisterDriver(pluginName); unregErr != nil { - glog.Error(log("registrationHandler.RegisterPlugin failed to unregister plugin due to previous: %v", unregErr)) + klog.Error(log("registrationHandler.RegisterPlugin failed to unregister plugin due to previous: %v", unregErr)) return unregErr } return err } - err = nim.AddNodeInfo(pluginName, driverNodeID, maxVolumePerNode, accessibleTopology) + err = nim.InstallCSIDriver(pluginName, driverNodeID, maxVolumePerNode, accessibleTopology) if err != nil { - glog.Error(log("registrationHandler.RegisterPlugin failed at AddNodeInfo: %v", err)) + klog.Error(log("registrationHandler.RegisterPlugin failed at AddNodeInfo: %v", err)) if unregErr := unregisterDriver(pluginName); unregErr != nil { - glog.Error(log("registrationHandler.RegisterPlugin failed to unregister plugin due to previous error: %v", unregErr)) + klog.Error(log("registrationHandler.RegisterPlugin failed to unregister plugin due to previous error: %v", unregErr)) return unregErr } return err @@ -162,9 +162,9 @@ func (h *RegistrationHandler) RegisterPlugin(pluginName string, endpoint string) // DeRegisterPlugin is called when a plugin removed its socket, signaling // it is no longer available func (h *RegistrationHandler) DeRegisterPlugin(pluginName string) { - glog.V(4).Info(log("registrationHandler.DeRegisterPlugin request for plugin %s", pluginName)) + klog.V(4).Info(log("registrationHandler.DeRegisterPlugin request for plugin %s", pluginName)) if err := unregisterDriver(pluginName); err != nil { - glog.Error(log("registrationHandler.DeRegisterPlugin failed: %v", err)) + klog.Error(log("registrationHandler.DeRegisterPlugin failed: %v", err)) } } @@ -174,7 +174,7 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { if utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) { csiClient := host.GetCSIClient() if csiClient == nil { - glog.Warning("The client for CSI Custom Resources is not available, skipping informer initialization") + klog.Warning("The client for CSI Custom Resources is not available, skipping informer initialization") } else { // Start informer for CSIDrivers. factory := csiapiinformer.NewSharedInformerFactory(csiClient, csiResyncPeriod) @@ -188,6 +188,9 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { csiDrivers = csiDriversStore{driversMap: map[string]csiDriver{}} nim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host) + // TODO(#70514) Init CSINodeInfo object if the CRD exists and create Driver + // objects for migrated drivers. + return nil } @@ -200,7 +203,7 @@ func (p *csiPlugin) GetPluginName() string { func (p *csiPlugin) GetVolumeName(spec *volume.Spec) (string, error) { csi, err := getCSISourceFromSpec(spec) if err != nil { - glog.Error(log("plugin.GetVolumeName failed to extract volume source from spec: %v", err)) + klog.Error(log("plugin.GetVolumeName failed to extract volume source from spec: %v", err)) return "", err } @@ -233,7 +236,7 @@ func (p *csiPlugin) NewMounter( k8s := p.host.GetKubeClient() if k8s == nil { - glog.Error(log("failed to get a kubernetes client")) + klog.Error(log("failed to get a kubernetes client")) return nil, errors.New("failed to get a Kubernetes client") } @@ -257,10 +260,10 @@ func (p *csiPlugin) NewMounter( dataDir := path.Dir(dir) // dropoff /mount at end if err := os.MkdirAll(dataDir, 0750); err != nil { - glog.Error(log("failed to create dir %#v: %v", dataDir, err)) + klog.Error(log("failed to create dir %#v: %v", dataDir, err)) return nil, err } - glog.V(4).Info(log("created path successfully [%s]", dataDir)) + klog.V(4).Info(log("created path successfully [%s]", dataDir)) // persist volume info data for teardown node := string(p.host.GetNodeName()) @@ -274,21 +277,21 @@ func (p *csiPlugin) NewMounter( } if err := saveVolumeData(dataDir, volDataFileName, volData); err != nil { - glog.Error(log("failed to save volume info data: %v", err)) + klog.Error(log("failed to save volume info data: %v", err)) if err := os.RemoveAll(dataDir); err != nil { - glog.Error(log("failed to remove dir after error [%s]: %v", dataDir, err)) + klog.Error(log("failed to remove dir after error [%s]: %v", dataDir, err)) return nil, err } return nil, err } - glog.V(4).Info(log("mounter created successfully")) + klog.V(4).Info(log("mounter created successfully")) return mounter, nil } func (p *csiPlugin) NewUnmounter(specName string, podUID types.UID) (volume.Unmounter, error) { - glog.V(4).Infof(log("setting up unmounter for [name=%v, podUID=%v]", specName, podUID)) + klog.V(4).Infof(log("setting up unmounter for [name=%v, podUID=%v]", specName, podUID)) unmounter := &csiMountMgr{ plugin: p, @@ -301,7 +304,7 @@ func (p *csiPlugin) NewUnmounter(specName string, podUID types.UID) (volume.Unmo dataDir := path.Dir(dir) // dropoff /mount at end data, err := loadVolumeData(dataDir, volDataFileName) if err != nil { - glog.Error(log("unmounter failed to load volume data file [%s]: %v", dir, err)) + klog.Error(log("unmounter failed to load volume data file [%s]: %v", dir, err)) return nil, err } unmounter.driverName = data[volDataKey.driverName] @@ -312,15 +315,15 @@ func (p *csiPlugin) NewUnmounter(specName string, podUID types.UID) (volume.Unmo } func (p *csiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { - glog.V(4).Info(log("plugin.ConstructVolumeSpec [pv.Name=%v, path=%v]", volumeName, mountPath)) + klog.V(4).Info(log("plugin.ConstructVolumeSpec [pv.Name=%v, path=%v]", volumeName, mountPath)) volData, err := loadVolumeData(mountPath, volDataFileName) if err != nil { - glog.Error(log("plugin.ConstructVolumeSpec failed loading volume data using [%s]: %v", mountPath, err)) + klog.Error(log("plugin.ConstructVolumeSpec failed loading volume data using [%s]: %v", mountPath, err)) return nil, err } - glog.V(4).Info(log("plugin.ConstructVolumeSpec extracted [%#v]", volData)) + klog.V(4).Info(log("plugin.ConstructVolumeSpec extracted [%#v]", volData)) fsMode := api.PersistentVolumeFilesystem pv := &api.PersistentVolume{ @@ -362,7 +365,7 @@ var _ volume.DeviceMountableVolumePlugin = &csiPlugin{} func (p *csiPlugin) NewAttacher() (volume.Attacher, error) { k8s := p.host.GetKubeClient() if k8s == nil { - glog.Error(log("unable to get kubernetes client from host")) + klog.Error(log("unable to get kubernetes client from host")) return nil, errors.New("unable to get Kubernetes client") } @@ -380,7 +383,7 @@ func (p *csiPlugin) NewDeviceMounter() (volume.DeviceMounter, error) { func (p *csiPlugin) NewDetacher() (volume.Detacher, error) { k8s := p.host.GetKubeClient() if k8s == nil { - glog.Error(log("unable to get kubernetes client from host")) + klog.Error(log("unable to get kubernetes client from host")) return nil, errors.New("unable to get Kubernetes client") } @@ -417,12 +420,12 @@ func (p *csiPlugin) NewBlockVolumeMapper(spec *volume.Spec, podRef *api.Pod, opt return nil, err } - glog.V(4).Info(log("setting up block mapper for [volume=%v,driver=%v]", pvSource.VolumeHandle, pvSource.Driver)) + klog.V(4).Info(log("setting up block mapper for [volume=%v,driver=%v]", pvSource.VolumeHandle, pvSource.Driver)) client := newCsiDriverClient(pvSource.Driver) k8s := p.host.GetKubeClient() if k8s == nil { - glog.Error(log("failed to get a kubernetes client")) + klog.Error(log("failed to get a kubernetes client")) return nil, errors.New("failed to get a Kubernetes client") } @@ -442,10 +445,10 @@ func (p *csiPlugin) NewBlockVolumeMapper(spec *volume.Spec, podRef *api.Pod, opt dataDir := getVolumeDeviceDataDir(spec.Name(), p.host) if err := os.MkdirAll(dataDir, 0750); err != nil { - glog.Error(log("failed to create data dir %s: %v", dataDir, err)) + klog.Error(log("failed to create data dir %s: %v", dataDir, err)) return nil, err } - glog.V(4).Info(log("created path successfully [%s]", dataDir)) + klog.V(4).Info(log("created path successfully [%s]", dataDir)) // persist volume info data for teardown node := string(p.host.GetNodeName()) @@ -459,9 +462,9 @@ func (p *csiPlugin) NewBlockVolumeMapper(spec *volume.Spec, podRef *api.Pod, opt } if err := saveVolumeData(dataDir, volDataFileName, volData); err != nil { - glog.Error(log("failed to save volume info data: %v", err)) + klog.Error(log("failed to save volume info data: %v", err)) if err := os.RemoveAll(dataDir); err != nil { - glog.Error(log("failed to remove dir after error [%s]: %v", dataDir, err)) + klog.Error(log("failed to remove dir after error [%s]: %v", dataDir, err)) return nil, err } return nil, err @@ -475,7 +478,7 @@ func (p *csiPlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (vo return nil, errors.New("CSIBlockVolume feature not enabled") } - glog.V(4).Infof(log("setting up block unmapper for [Spec=%v, podUID=%v]", volName, podUID)) + klog.V(4).Infof(log("setting up block unmapper for [Spec=%v, podUID=%v]", volName, podUID)) unmapper := &csiBlockMapper{ plugin: p, podUID: podUID, @@ -486,7 +489,7 @@ func (p *csiPlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (vo dataDir := getVolumeDeviceDataDir(unmapper.specName, p.host) data, err := loadVolumeData(dataDir, volDataFileName) if err != nil { - glog.Error(log("unmapper failed to load volume data file [%s]: %v", dataDir, err)) + klog.Error(log("unmapper failed to load volume data file [%s]: %v", dataDir, err)) return nil, err } unmapper.driverName = data[volDataKey.driverName] @@ -501,16 +504,16 @@ func (p *csiPlugin) ConstructBlockVolumeSpec(podUID types.UID, specVolName, mapP return nil, errors.New("CSIBlockVolume feature not enabled") } - glog.V(4).Infof("plugin.ConstructBlockVolumeSpec [podUID=%s, specVolName=%s, path=%s]", string(podUID), specVolName, mapPath) + klog.V(4).Infof("plugin.ConstructBlockVolumeSpec [podUID=%s, specVolName=%s, path=%s]", string(podUID), specVolName, mapPath) dataDir := getVolumeDeviceDataDir(specVolName, p.host) volData, err := loadVolumeData(dataDir, volDataFileName) if err != nil { - glog.Error(log("plugin.ConstructBlockVolumeSpec failed loading volume data using [%s]: %v", mapPath, err)) + klog.Error(log("plugin.ConstructBlockVolumeSpec failed loading volume data using [%s]: %v", mapPath, err)) return nil, err } - glog.V(4).Info(log("plugin.ConstructBlockVolumeSpec extracted [%#v]", volData)) + klog.V(4).Info(log("plugin.ConstructBlockVolumeSpec extracted [%#v]", volData)) blockMode := api.PersistentVolumeBlock pv := &api.PersistentVolume{ @@ -583,8 +586,8 @@ func unregisterDriver(driverName string) error { delete(csiDrivers.driversMap, driverName) }() - if err := nim.RemoveNodeInfo(driverName); err != nil { - glog.Errorf("Error unregistering CSI driver: %v", err) + if err := nim.UninstallCSIDriver(driverName); err != nil { + klog.Errorf("Error uninstalling CSI driver: %v", err) return err } diff --git a/pkg/volume/csi/csi_plugin_test.go b/pkg/volume/csi/csi_plugin_test.go index a4d9e9b4040..c2fbafd95af 100644 --- a/pkg/volume/csi/csi_plugin_test.go +++ b/pkg/volume/csi/csi_plugin_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" fakeclient "k8s.io/client-go/kubernetes/fake" utiltesting "k8s.io/client-go/util/testing" fakecsi "k8s.io/csi-api/pkg/client/clientset/versioned/fake" @@ -39,11 +40,6 @@ import ( // create a plugin mgr to load plugins and setup a fake client func newTestPlugin(t *testing.T, client *fakeclient.Clientset, csiClient *fakecsi.Clientset) (*csiPlugin, string) { - err := utilfeature.DefaultFeatureGate.Set("CSIBlockVolume=true") - if err != nil { - t.Fatalf("Failed to enable feature gate for CSIBlockVolume: %v", err) - } - tmpDir, err := utiltesting.MkTmpdir("csi-test") if err != nil { t.Fatalf("can't create temp dir: %v", err) @@ -109,6 +105,8 @@ func makeTestPV(name string, sizeGig int, driverName, volID string) *api.Persist } func TestPluginGetPluginName(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) if plug.GetPluginName() != "kubernetes.io/csi" { @@ -117,6 +115,8 @@ func TestPluginGetPluginName(t *testing.T) { } func TestPluginGetVolumeName(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) testCases := []struct { @@ -146,6 +146,8 @@ func TestPluginGetVolumeName(t *testing.T) { } func TestPluginCanSupport(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) @@ -158,6 +160,8 @@ func TestPluginCanSupport(t *testing.T) { } func TestPluginConstructVolumeSpec(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) @@ -218,6 +222,8 @@ func TestPluginConstructVolumeSpec(t *testing.T) { } func TestPluginNewMounter(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) @@ -266,6 +272,8 @@ func TestPluginNewMounter(t *testing.T) { } func TestPluginNewUnmounter(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) @@ -311,6 +319,8 @@ func TestPluginNewUnmounter(t *testing.T) { } func TestPluginNewAttacher(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) @@ -329,6 +339,8 @@ func TestPluginNewAttacher(t *testing.T) { } func TestPluginNewDetacher(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) @@ -347,6 +359,8 @@ func TestPluginNewDetacher(t *testing.T) { } func TestPluginNewBlockMapper(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) @@ -392,6 +406,8 @@ func TestPluginNewBlockMapper(t *testing.T) { } func TestPluginNewUnmapper(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) @@ -449,6 +465,8 @@ func TestPluginNewUnmapper(t *testing.T) { } func TestPluginConstructBlockVolumeSpec(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() + plug, tmpDir := newTestPlugin(t, nil, nil) defer os.RemoveAll(tmpDir) diff --git a/pkg/volume/csi/csi_util.go b/pkg/volume/csi/csi_util.go index 00d40fef39e..3fe103584c4 100644 --- a/pkg/volume/csi/csi_util.go +++ b/pkg/volume/csi/csi_util.go @@ -22,10 +22,10 @@ import ( "os" "path" - "github.com/golang/glog" api "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/klog" kstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" "time" @@ -40,7 +40,7 @@ func getCredentialsFromSecret(k8s kubernetes.Interface, secretRef *api.SecretRef credentials := map[string]string{} secret, err := k8s.CoreV1().Secrets(secretRef.Namespace).Get(secretRef.Name, meta.GetOptions{}) if err != nil { - glog.Errorf("failed to find the secret %s in the namespace %s with error: %v\n", secretRef.Name, secretRef.Namespace, err) + klog.Errorf("failed to find the secret %s in the namespace %s with error: %v\n", secretRef.Name, secretRef.Namespace, err) return credentials, err } for key, value := range secret.Data { @@ -53,18 +53,18 @@ func getCredentialsFromSecret(k8s kubernetes.Interface, secretRef *api.SecretRef // saveVolumeData persists parameter data as json file at the provided location func saveVolumeData(dir string, fileName string, data map[string]string) error { dataFilePath := path.Join(dir, fileName) - glog.V(4).Info(log("saving volume data file [%s]", dataFilePath)) + klog.V(4).Info(log("saving volume data file [%s]", dataFilePath)) file, err := os.Create(dataFilePath) if err != nil { - glog.Error(log("failed to save volume data file %s: %v", dataFilePath, err)) + klog.Error(log("failed to save volume data file %s: %v", dataFilePath, err)) return err } defer file.Close() if err := json.NewEncoder(file).Encode(data); err != nil { - glog.Error(log("failed to save volume data file %s: %v", dataFilePath, err)) + klog.Error(log("failed to save volume data file %s: %v", dataFilePath, err)) return err } - glog.V(4).Info(log("volume data file saved successfully [%s]", dataFilePath)) + klog.V(4).Info(log("volume data file saved successfully [%s]", dataFilePath)) return nil } @@ -72,17 +72,17 @@ func saveVolumeData(dir string, fileName string, data map[string]string) error { func loadVolumeData(dir string, fileName string) (map[string]string, error) { // remove /mount at the end dataFileName := path.Join(dir, fileName) - glog.V(4).Info(log("loading volume data file [%s]", dataFileName)) + klog.V(4).Info(log("loading volume data file [%s]", dataFileName)) file, err := os.Open(dataFileName) if err != nil { - glog.Error(log("failed to open volume data file [%s]: %v", dataFileName, err)) + klog.Error(log("failed to open volume data file [%s]: %v", dataFileName, err)) return nil, err } defer file.Close() data := map[string]string{} if err := json.NewDecoder(file).Decode(&data); err != nil { - glog.Error(log("failed to parse volume data file [%s]: %v", dataFileName, err)) + klog.Error(log("failed to parse volume data file [%s]: %v", dataFileName, err)) return nil, err } diff --git a/pkg/volume/csi/fake/BUILD b/pkg/volume/csi/fake/BUILD index 679150a5b51..c2c6b44d4f5 100644 --- a/pkg/volume/csi/fake/BUILD +++ b/pkg/volume/csi/fake/BUILD @@ -9,7 +9,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/volume/csi/fake", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/container-storage-interface/spec/lib/go/csi/v0:go_default_library", + "//vendor/github.com/container-storage-interface/spec/lib/go/csi:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", ], ) diff --git a/pkg/volume/csi/fake/fake_client.go b/pkg/volume/csi/fake/fake_client.go index 775ff7ddfbf..c9ed5e6b9ab 100644 --- a/pkg/volume/csi/fake/fake_client.go +++ b/pkg/volume/csi/fake/fake_client.go @@ -23,7 +23,7 @@ import ( "google.golang.org/grpc" - csipb "github.com/container-storage-interface/spec/lib/go/csi/v0" + csipb "github.com/container-storage-interface/spec/lib/go/csi" ) // IdentityClient is a CSI identity client used for testing @@ -57,9 +57,9 @@ func (f *IdentityClient) Probe(ctx context.Context, in *csipb.ProbeRequest, opts } type CSIVolume struct { - Attributes map[string]string - Path string - MountFlags []string + VolumeContext map[string]string + Path string + MountFlags []string } // NodeClient returns CSI node client @@ -99,10 +99,10 @@ func (f *NodeClient) GetNodeStagedVolumes() map[string]CSIVolume { return f.nodeStagedVolumes } -func (f *NodeClient) AddNodeStagedVolume(volID, deviceMountPath string, attributes map[string]string) { +func (f *NodeClient) AddNodeStagedVolume(volID, deviceMountPath string, volumeContext map[string]string) { f.nodeStagedVolumes[volID] = CSIVolume{ - Path: deviceMountPath, - Attributes: attributes, + Path: deviceMountPath, + VolumeContext: volumeContext, } } @@ -125,9 +125,9 @@ func (f *NodeClient) NodePublishVolume(ctx context.Context, req *csipb.NodePubli return nil, errors.New("invalid fstype") } f.nodePublishedVolumes[req.GetVolumeId()] = CSIVolume{ - Path: req.GetTargetPath(), - Attributes: req.GetVolumeAttributes(), - MountFlags: req.GetVolumeCapability().GetMount().MountFlags, + Path: req.GetTargetPath(), + VolumeContext: req.GetVolumeContext(), + MountFlags: req.GetVolumeCapability().GetMount().MountFlags, } return &csipb.NodePublishVolumeResponse{}, nil } @@ -172,8 +172,8 @@ func (f *NodeClient) NodeStageVolume(ctx context.Context, req *csipb.NodeStageVo } f.nodeStagedVolumes[req.GetVolumeId()] = CSIVolume{ - Path: req.GetStagingTargetPath(), - Attributes: req.GetVolumeAttributes(), + Path: req.GetStagingTargetPath(), + VolumeContext: req.GetVolumeContext(), } return &csipb.NodeStageVolumeResponse{}, nil } @@ -195,11 +195,6 @@ func (f *NodeClient) NodeUnstageVolume(ctx context.Context, req *csipb.NodeUnsta return &csipb.NodeUnstageVolumeResponse{}, nil } -// NodeGetId implements method -func (f *NodeClient) NodeGetId(ctx context.Context, in *csipb.NodeGetIdRequest, opts ...grpc.CallOption) (*csipb.NodeGetIdResponse, error) { - return nil, nil -} - // NodeGetId implements csi method func (f *NodeClient) NodeGetInfo(ctx context.Context, in *csipb.NodeGetInfoRequest, opts ...grpc.CallOption) (*csipb.NodeGetInfoResponse, error) { if f.nextErr != nil { @@ -227,6 +222,11 @@ func (f *NodeClient) NodeGetCapabilities(ctx context.Context, in *csipb.NodeGetC return nil, nil } +// NodeGetVolumeStats implements csi method +func (f *NodeClient) NodeGetVolumeStats(ctx context.Context, in *csipb.NodeGetVolumeStatsRequest, opts ...grpc.CallOption) (*csipb.NodeGetVolumeStatsResponse, error) { + return nil, nil +} + // ControllerClient represents a CSI Controller client type ControllerClient struct { nextCapabilities []*csipb.ControllerServiceCapability diff --git a/pkg/volume/csi/main_test.go b/pkg/volume/csi/main_test.go new file mode 100644 index 00000000000..5322fcd7edc --- /dev/null +++ b/pkg/volume/csi/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/volume/csi/nodeinfomanager/BUILD b/pkg/volume/csi/nodeinfomanager/BUILD index 3eadf06129b..e949893feba 100644 --- a/pkg/volume/csi/nodeinfomanager/BUILD +++ b/pkg/volume/csi/nodeinfomanager/BUILD @@ -15,12 +15,14 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/client-go/util/retry:go_default_library", "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", - "//vendor/github.com/container-storage-interface/spec/lib/go/csi/v0:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", + "//vendor/github.com/container-storage-interface/spec/lib/go/csi:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -49,7 +51,6 @@ go_test( "//pkg/volume/testing:go_default_library", "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", @@ -62,7 +63,7 @@ go_test( "//staging/src/k8s.io/client-go/util/testing:go_default_library", "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/fake:go_default_library", - "//vendor/github.com/container-storage-interface/spec/lib/go/csi/v0:go_default_library", + "//vendor/github.com/container-storage-interface/spec/lib/go/csi:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", ], ) diff --git a/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go b/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go index 0a28d9c4ba9..cd832e169ff 100644 --- a/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go +++ b/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go @@ -21,22 +21,26 @@ package nodeinfomanager // import "k8s.io/kubernetes/pkg/volume/csi/nodeinfomana import ( "encoding/json" "fmt" + "strings" - csipb "github.com/container-storage-interface/spec/lib/go/csi/v0" - "github.com/golang/glog" + csipb "github.com/container-storage-interface/spec/lib/go/csi" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/client-go/util/retry" csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" + csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" nodeutil "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" + "time" ) const ( @@ -44,7 +48,15 @@ const ( annotationKeyNodeID = "csi.volume.kubernetes.io/nodeid" ) -var nodeKind = v1.SchemeGroupVersion.WithKind("Node") +var ( + nodeKind = v1.SchemeGroupVersion.WithKind("Node") + updateBackoff = wait.Backoff{ + Steps: 4, + Duration: 10 * time.Millisecond, + Factor: 5.0, + Jitter: 0.1, + } +) // nodeInfoManager contains necessary common dependencies to update node info on both // the Node and CSINodeInfo objects. @@ -58,15 +70,17 @@ type nodeUpdateFunc func(*v1.Node) (newNode *v1.Node, updated bool, err error) // Interface implements an interface for managing labels of a node type Interface interface { + CreateCSINodeInfo() (*csiv1alpha1.CSINodeInfo, error) + // Record in the cluster the given node information from the CSI driver with the given name. - // Concurrent calls to AddNodeInfo() is allowed, but they should not be intertwined with calls + // Concurrent calls to InstallCSIDriver() is allowed, but they should not be intertwined with calls // to other methods in this interface. - AddNodeInfo(driverName string, driverNodeID string, maxVolumeLimit int64, topology *csipb.Topology) error + InstallCSIDriver(driverName string, driverNodeID string, maxVolumeLimit int64, topology *csipb.Topology) error // Remove in the cluster node information from the CSI driver with the given name. - // Concurrent calls to RemoveNodeInfo() is allowed, but they should not be intertwined with calls + // Concurrent calls to UninstallCSIDriver() is allowed, but they should not be intertwined with calls // to other methods in this interface. - RemoveNodeInfo(driverName string) error + UninstallCSIDriver(driverName string) error } // NewNodeInfoManager initializes nodeInfoManager @@ -79,11 +93,11 @@ func NewNodeInfoManager( } } -// AddNodeInfo updates the node ID annotation in the Node object and CSIDrivers field in the +// InstallCSIDriver updates the node ID annotation in the Node object and CSIDrivers field in the // CSINodeInfo object. If the CSINodeInfo object doesn't yet exist, it will be created. -// If multiple calls to AddNodeInfo() are made in parallel, some calls might receive Node or +// If multiple calls to InstallCSIDriver() are made in parallel, some calls might receive Node or // CSINodeInfo update conflicts, which causes the function to retry the corresponding update. -func (nim *nodeInfoManager) AddNodeInfo(driverName string, driverNodeID string, maxAttachLimit int64, topology *csipb.Topology) error { +func (nim *nodeInfoManager) InstallCSIDriver(driverName string, driverNodeID string, maxAttachLimit int64, topology *csipb.Topology) error { if driverNodeID == "" { return fmt.Errorf("error adding CSI driver node info: driverNodeID must not be empty") } @@ -114,14 +128,14 @@ func (nim *nodeInfoManager) AddNodeInfo(driverName string, driverNodeID string, return nil } -// RemoveNodeInfo removes the node ID annotation from the Node object and CSIDrivers field from the +// UninstallCSIDriver removes the node ID annotation from the Node object and CSIDrivers field from the // CSINodeInfo object. If the CSINOdeInfo object contains no CSIDrivers, it will be deleted. -// If multiple calls to RemoveNodeInfo() are made in parallel, some calls might receive Node or +// If multiple calls to UninstallCSIDriver() are made in parallel, some calls might receive Node or // CSINodeInfo update conflicts, which causes the function to retry the corresponding update. -func (nim *nodeInfoManager) RemoveNodeInfo(driverName string) error { - err := nim.removeCSINodeInfo(driverName) +func (nim *nodeInfoManager) UninstallCSIDriver(driverName string) error { + err := nim.uninstallDriverFromCSINodeInfo(driverName) if err != nil { - return fmt.Errorf("error removing CSI driver node info from CSINodeInfo object %v", err) + return fmt.Errorf("error uninstalling CSI driver from CSINodeInfo object %v", err) } err = nim.updateNode( @@ -134,51 +148,59 @@ func (nim *nodeInfoManager) RemoveNodeInfo(driverName string) error { return nil } +func (nim *nodeInfoManager) updateNode(updateFuncs ...nodeUpdateFunc) error { + var updateErrs []error + err := wait.ExponentialBackoff(updateBackoff, func() (bool, error) { + if err := nim.tryUpdateNode(updateFuncs...); err != nil { + updateErrs = append(updateErrs, err) + return false, nil + } + return true, nil + }) + if err != nil { + return fmt.Errorf("error updating node: %v; caused by: %v", err, utilerrors.NewAggregate(updateErrs)) + } + return nil +} + // updateNode repeatedly attempts to update the corresponding node object // which is modified by applying the given update functions sequentially. // Because updateFuncs are applied sequentially, later updateFuncs should take into account // the effects of previous updateFuncs to avoid potential conflicts. For example, if multiple // functions update the same field, updates in the last function are persisted. -func (nim *nodeInfoManager) updateNode(updateFuncs ...nodeUpdateFunc) error { - retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - // Retrieve the latest version of Node before attempting update, so that - // existing changes are not overwritten. RetryOnConflict uses - // exponential backoff to avoid exhausting the apiserver. +func (nim *nodeInfoManager) tryUpdateNode(updateFuncs ...nodeUpdateFunc) error { + // Retrieve the latest version of Node before attempting update, so that + // existing changes are not overwritten. - kubeClient := nim.volumeHost.GetKubeClient() - if kubeClient == nil { - return fmt.Errorf("error getting kube client") - } - - nodeClient := kubeClient.CoreV1().Nodes() - originalNode, err := nodeClient.Get(string(nim.nodeName), metav1.GetOptions{}) - node := originalNode.DeepCopy() - if err != nil { - return err // do not wrap error - } - - needUpdate := false - for _, update := range updateFuncs { - newNode, updated, err := update(node) - if err != nil { - return err - } - node = newNode - needUpdate = needUpdate || updated - } - - if needUpdate { - // PatchNodeStatus can update both node's status and labels or annotations - // Updating status by directly updating node does not work - _, _, updateErr := nodeutil.PatchNodeStatus(kubeClient.CoreV1(), types.NodeName(node.Name), originalNode, node) - return updateErr // do not wrap error - } - - return nil - }) - if retryErr != nil { - return fmt.Errorf("node update failed: %v", retryErr) + kubeClient := nim.volumeHost.GetKubeClient() + if kubeClient == nil { + return fmt.Errorf("error getting kube client") } + + nodeClient := kubeClient.CoreV1().Nodes() + originalNode, err := nodeClient.Get(string(nim.nodeName), metav1.GetOptions{}) + if err != nil { + return err + } + node := originalNode.DeepCopy() + + needUpdate := false + for _, update := range updateFuncs { + newNode, updated, err := update(node) + if err != nil { + return err + } + node = newNode + needUpdate = needUpdate || updated + } + + if needUpdate { + // PatchNodeStatus can update both node's status and labels or annotations + // Updating status by directly updating node does not work + _, _, updateErr := nodeutil.PatchNodeStatus(kubeClient.CoreV1(), types.NodeName(node.Name), originalNode, node) + return updateErr + } + return nil } @@ -330,48 +352,52 @@ func (nim *nodeInfoManager) updateCSINodeInfo( return fmt.Errorf("error getting CSI client") } - retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - nodeInfo, err := csiKubeClient.CsiV1alpha1().CSINodeInfos().Get(string(nim.nodeName), metav1.GetOptions{}) - if nodeInfo == nil || errors.IsNotFound(err) { - return nim.createNodeInfoObject(driverName, driverNodeID, topology) + var updateErrs []error + err := wait.ExponentialBackoff(updateBackoff, func() (bool, error) { + if err := nim.tryUpdateCSINodeInfo(csiKubeClient, driverName, driverNodeID, topology); err != nil { + updateErrs = append(updateErrs, err) + return false, nil } - if err != nil { - return err // do not wrap error - } - - return nim.updateNodeInfoObject(nodeInfo, driverName, driverNodeID, topology) + return true, nil }) - if retryErr != nil { - return fmt.Errorf("CSINodeInfo update failed: %v", retryErr) + if err != nil { + return fmt.Errorf("error updating CSINodeInfo: %v; caused by: %v", err, utilerrors.NewAggregate(updateErrs)) } return nil } -func (nim *nodeInfoManager) createNodeInfoObject( +func (nim *nodeInfoManager) tryUpdateCSINodeInfo( + csiKubeClient csiclientset.Interface, driverName string, driverNodeID string, topology *csipb.Topology) error { + nodeInfo, err := csiKubeClient.CsiV1alpha1().CSINodeInfos().Get(string(nim.nodeName), metav1.GetOptions{}) + if nodeInfo == nil || errors.IsNotFound(err) { + nodeInfo, err = nim.CreateCSINodeInfo() + } + if err != nil { + return err + } + + return nim.installDriverToCSINodeInfo(nodeInfo, driverName, driverNodeID, topology) +} + +func (nim *nodeInfoManager) CreateCSINodeInfo() (*csiv1alpha1.CSINodeInfo, error) { + kubeClient := nim.volumeHost.GetKubeClient() if kubeClient == nil { - return fmt.Errorf("error getting kube client") + return nil, fmt.Errorf("error getting kube client") } csiKubeClient := nim.volumeHost.GetCSIClient() if csiKubeClient == nil { - return fmt.Errorf("error getting CSI client") - } - - topologyKeys := []string{} // must be an empty slice instead of nil to satisfy CRD OpenAPI Schema validation - if topology != nil { - for k := range topology.Segments { - topologyKeys = append(topologyKeys, k) - } + return nil, fmt.Errorf("error getting CSI client") } node, err := kubeClient.CoreV1().Nodes().Get(string(nim.nodeName), metav1.GetOptions{}) if err != nil { - return err // do not wrap error + return nil, err } nodeInfo := &csiv1alpha1.CSINodeInfo{ @@ -386,20 +412,18 @@ func (nim *nodeInfoManager) createNodeInfoObject( }, }, }, - CSIDrivers: []csiv1alpha1.CSIDriverInfo{ - { - Driver: driverName, - NodeID: driverNodeID, - TopologyKeys: topologyKeys, - }, + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: []csiv1alpha1.CSIDriverInfoSpec{}, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: []csiv1alpha1.CSIDriverInfoStatus{}, }, } - _, err = csiKubeClient.CsiV1alpha1().CSINodeInfos().Create(nodeInfo) - return err // do not wrap error + return csiKubeClient.CsiV1alpha1().CSINodeInfos().Create(nodeInfo) } -func (nim *nodeInfoManager) updateNodeInfoObject( +func (nim *nodeInfoManager) installDriverToCSINodeInfo( nodeInfo *csiv1alpha1.CSINodeInfo, driverName string, driverNodeID string, @@ -417,86 +441,128 @@ func (nim *nodeInfoManager) updateNodeInfoObject( } } - // Clone driver list, omitting the driver that matches the given driverName, - // unless the driver is identical to information provided, in which case no update is necessary. - var newDriverInfos []csiv1alpha1.CSIDriverInfo - for _, driverInfo := range nodeInfo.CSIDrivers { - if driverInfo.Driver == driverName { - prevTopologyKeys := sets.NewString(driverInfo.TopologyKeys...) - if driverInfo.NodeID == driverNodeID && prevTopologyKeys.Equal(topologyKeys) { - // No update needed - return nil + specModified := true + statusModified := true + // Clone driver list, omitting the driver that matches the given driverName + newDriverSpecs := []csiv1alpha1.CSIDriverInfoSpec{} + for _, driverInfoSpec := range nodeInfo.Spec.Drivers { + if driverInfoSpec.Name == driverName { + if driverInfoSpec.NodeID == driverNodeID && + sets.NewString(driverInfoSpec.TopologyKeys...).Equal(topologyKeys) { + specModified = false } } else { - // Omit driverInfo matching given driverName - newDriverInfos = append(newDriverInfos, driverInfo) + // Omit driverInfoSpec matching given driverName + newDriverSpecs = append(newDriverSpecs, driverInfoSpec) } } + newDriverStatuses := []csiv1alpha1.CSIDriverInfoStatus{} + for _, driverInfoStatus := range nodeInfo.Status.Drivers { + if driverInfoStatus.Name == driverName { + if driverInfoStatus.Available && + /* TODO(https://github.com/kubernetes/enhancements/issues/625): Add actual migration status */ + driverInfoStatus.VolumePluginMechanism == csiv1alpha1.VolumePluginMechanismInTree { + statusModified = false + } + } else { + // Omit driverInfoSpec matching given driverName + newDriverStatuses = append(newDriverStatuses, driverInfoStatus) + } + } + + if !specModified && !statusModified { + return nil + } // Append new driver - driverInfo := csiv1alpha1.CSIDriverInfo{ - Driver: driverName, + driverSpec := csiv1alpha1.CSIDriverInfoSpec{ + Name: driverName, NodeID: driverNodeID, TopologyKeys: topologyKeys.List(), } - newDriverInfos = append(newDriverInfos, driverInfo) - nodeInfo.CSIDrivers = newDriverInfos + driverStatus := csiv1alpha1.CSIDriverInfoStatus{ + Name: driverName, + Available: true, + // TODO(https://github.com/kubernetes/enhancements/issues/625): Add actual migration status + VolumePluginMechanism: csiv1alpha1.VolumePluginMechanismInTree, + } - _, err := csiKubeClient.CsiV1alpha1().CSINodeInfos().Update(nodeInfo) - return err // do not wrap error + newDriverSpecs = append(newDriverSpecs, driverSpec) + newDriverStatuses = append(newDriverStatuses, driverStatus) + nodeInfo.Spec.Drivers = newDriverSpecs + nodeInfo.Status.Drivers = newDriverStatuses + + err := validateCSINodeInfo(nodeInfo) + if err != nil { + return err + } + _, err = csiKubeClient.CsiV1alpha1().CSINodeInfos().Update(nodeInfo) + return err } -func (nim *nodeInfoManager) removeCSINodeInfo(csiDriverName string) error { - retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { +func (nim *nodeInfoManager) uninstallDriverFromCSINodeInfo( + csiDriverName string) error { - csiKubeClient := nim.volumeHost.GetCSIClient() - if csiKubeClient == nil { - return fmt.Errorf("error getting CSI client") + csiKubeClient := nim.volumeHost.GetCSIClient() + if csiKubeClient == nil { + return fmt.Errorf("error getting CSI client") + } + + var updateErrs []error + err := wait.ExponentialBackoff(updateBackoff, func() (bool, error) { + if err := nim.tryUninstallDriverFromCSINodeInfo(csiKubeClient, csiDriverName); err != nil { + updateErrs = append(updateErrs, err) + return false, nil } - - nodeInfoClient := csiKubeClient.CsiV1alpha1().CSINodeInfos() - nodeInfo, err := nodeInfoClient.Get(string(nim.nodeName), metav1.GetOptions{}) - if nodeInfo == nil || errors.IsNotFound(err) { - // do nothing - return nil - } - if err != nil { - return err // do not wrap error - } - - // Remove matching driver from driver list - var newDriverInfos []csiv1alpha1.CSIDriverInfo - for _, driverInfo := range nodeInfo.CSIDrivers { - if driverInfo.Driver != csiDriverName { - newDriverInfos = append(newDriverInfos, driverInfo) - } - } - - if len(newDriverInfos) == len(nodeInfo.CSIDrivers) { - // No changes, don't update - return nil - } - - if len(newDriverInfos) == 0 { - // No drivers left, delete CSINodeInfo object - return nodeInfoClient.Delete(string(nim.nodeName), &metav1.DeleteOptions{}) - } - - // TODO (verult) make sure CSINodeInfo has validation logic to prevent duplicate driver names - _, updateErr := nodeInfoClient.Update(nodeInfo) - return updateErr // do not wrap error - + return true, nil }) - if retryErr != nil { - return fmt.Errorf("CSINodeInfo update failed: %v", retryErr) + if err != nil { + return fmt.Errorf("error updating CSINodeInfo: %v; caused by: %v", err, utilerrors.NewAggregate(updateErrs)) } return nil } +func (nim *nodeInfoManager) tryUninstallDriverFromCSINodeInfo( + csiKubeClient csiclientset.Interface, + csiDriverName string) error { + + nodeInfoClient := csiKubeClient.CsiV1alpha1().CSINodeInfos() + nodeInfo, err := nodeInfoClient.Get(string(nim.nodeName), metav1.GetOptions{}) + if err != nil { + return err // do not wrap error + } + + hasModified := false + newDriverStatuses := []csiv1alpha1.CSIDriverInfoStatus{} + for _, driverStatus := range nodeInfo.Status.Drivers { + if driverStatus.Name == csiDriverName { + // Uninstall the driver if we find it + hasModified = driverStatus.Available + driverStatus.Available = false + } + newDriverStatuses = append(newDriverStatuses, driverStatus) + } + + nodeInfo.Status.Drivers = newDriverStatuses + + if !hasModified { + // No changes, don't update + return nil + } + + err = validateCSINodeInfo(nodeInfo) + if err != nil { + return err + } + _, updateErr := nodeInfoClient.Update(nodeInfo) + return updateErr // do not wrap error + +} + func updateMaxAttachLimit(driverName string, maxLimit int64) nodeUpdateFunc { return func(node *v1.Node) (*v1.Node, bool, error) { if maxLimit <= 0 { - glog.V(4).Infof("skipping adding attach limit for %s", driverName) + klog.V(4).Infof("skipping adding attach limit for %s", driverName) return node, false, nil } @@ -545,3 +611,50 @@ func removeMaxAttachLimit(driverName string) nodeUpdateFunc { return node, true, nil } } + +// validateCSINodeInfo ensures members of CSINodeInfo object satisfies map and set semantics. +// Before calling CSINodeInfoInterface.Update(), validateCSINodeInfo() should be invoked to +// make sure the CSINodeInfo is compliant +func validateCSINodeInfo(nodeInfo *csiv1alpha1.CSINodeInfo) error { + if len(nodeInfo.Status.Drivers) < 1 { + return fmt.Errorf("at least one Driver entry is required in driver statuses") + } + if len(nodeInfo.Spec.Drivers) < 1 { + return fmt.Errorf("at least one Driver entry is required in driver specs") + } + if len(nodeInfo.Status.Drivers) != len(nodeInfo.Spec.Drivers) { + return fmt.Errorf("") + } + // check for duplicate entries for the same driver in statuses + var errors []string + driverNamesInStatuses := make(sets.String) + for _, driverInfo := range nodeInfo.Status.Drivers { + if driverNamesInStatuses.Has(driverInfo.Name) { + errors = append(errors, fmt.Sprintf("duplicate entries found for driver: %s in driver statuses", driverInfo.Name)) + } + driverNamesInStatuses.Insert(driverInfo.Name) + } + // check for duplicate entries for the same driver in specs + driverNamesInSpecs := make(sets.String) + for _, driverInfo := range nodeInfo.Spec.Drivers { + if driverNamesInSpecs.Has(driverInfo.Name) { + errors = append(errors, fmt.Sprintf("duplicate entries found for driver: %s in driver specs", driverInfo.Name)) + } + driverNamesInSpecs.Insert(driverInfo.Name) + topoKeys := make(sets.String) + for _, key := range driverInfo.TopologyKeys { + if topoKeys.Has(key) { + errors = append(errors, fmt.Sprintf("duplicate topology keys %s found for driver %s in driver specs", key, driverInfo.Name)) + } + topoKeys.Insert(key) + } + } + // check all entries in specs and status match + if !driverNamesInSpecs.Equal(driverNamesInStatuses) { + errors = append(errors, fmt.Sprintf("list of drivers in specs: %v does not match list of drivers in statuses: %v", driverNamesInSpecs.List(), driverNamesInStatuses.List())) + } + if len(errors) == 0 { + return nil + } + return fmt.Errorf(strings.Join(errors, ", ")) +} diff --git a/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go b/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go index 8e42f77869a..39ee2fd0cdf 100644 --- a/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go +++ b/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go @@ -21,10 +21,9 @@ import ( "fmt" "testing" - "github.com/container-storage-interface/spec/lib/go/csi/v0" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -55,7 +54,6 @@ type testcase struct { expectedNodeIDMap map[string]string expectedTopologyMap map[string]sets.String expectedLabels map[string]string - expectNoNodeInfo bool expectedVolumeLimit int64 expectFail bool } @@ -64,9 +62,9 @@ type nodeIDMap map[string]string type topologyKeyMap map[string][]string type labelMap map[string]string -// TestAddNodeInfo tests AddNodeInfo with various existing Node and/or CSINodeInfo objects. +// TestInstallCSIDriver tests InstallCSIDriver with various existing Node and/or CSINodeInfo objects. // The node IDs in all test cases below are the same between the Node annotation and CSINodeInfo. -func TestAddNodeInfo(t *testing.T) { +func TestInstallCSIDriver(t *testing.T) { testcases := []testcase{ { name: "empty node", @@ -375,9 +373,9 @@ func TestAddNodeInfo(t *testing.T) { test(t, true /* addNodeInfo */, true /* csiNodeInfoEnabled */, testcases) } -// TestAddNodeInfo_CSINodeInfoDisabled tests AddNodeInfo with various existing Node annotations +// TestInstallCSIDriver_CSINodeInfoDisabled tests InstallCSIDriver with various existing Node annotations // and CSINodeInfo feature gate disabled. -func TestAddNodeInfo_CSINodeInfoDisabled(t *testing.T) { +func TestInstallCSIDriver_CSINodeInfoDisabled(t *testing.T) { testcases := []testcase{ { name: "empty node", @@ -420,16 +418,15 @@ func TestAddNodeInfo_CSINodeInfoDisabled(t *testing.T) { test(t, true /* addNodeInfo */, false /* csiNodeInfoEnabled */, testcases) } -// TestRemoveNodeInfo tests RemoveNodeInfo with various existing Node and/or CSINodeInfo objects. -func TestRemoveNodeInfo(t *testing.T) { +// TestUninstallCSIDriver tests UninstallCSIDriver with various existing Node and/or CSINodeInfo objects. +func TestUninstallCSIDriver(t *testing.T) { testcases := []testcase{ { - name: "empty node and no CSINodeInfo", + name: "empty node and empty CSINodeInfo", driverName: "com.example.csi/driver1", existingNode: generateNode(nil /* nodeIDs */, nil /* labels */, nil /*capacity*/), expectedNodeIDMap: nil, expectedLabels: nil, - expectNoNodeInfo: true, }, { name: "pre-existing node info from the same driver", @@ -451,7 +448,6 @@ func TestRemoveNodeInfo(t *testing.T) { ), expectedNodeIDMap: nil, expectedLabels: map[string]string{"com.example.csi/zone": "zoneA"}, - expectNoNodeInfo: true, }, { name: "pre-existing node info from different driver", @@ -480,7 +476,7 @@ func TestRemoveNodeInfo(t *testing.T) { expectedLabels: map[string]string{"net.example.storage/zone": "zoneA"}, }, { - name: "pre-existing info about the same driver in node, but no CSINodeInfo", + name: "pre-existing info about the same driver in node, but empty CSINodeInfo", driverName: "com.example.csi/driver1", existingNode: generateNode( nodeIDMap{ @@ -489,10 +485,9 @@ func TestRemoveNodeInfo(t *testing.T) { nil /* labels */, nil /*capacity*/), expectedNodeIDMap: nil, expectedLabels: nil, - expectNoNodeInfo: true, }, { - name: "pre-existing info about a different driver in node, but no CSINodeInfo", + name: "pre-existing info about a different driver in node, but empty CSINodeInfo", existingNode: generateNode( nodeIDMap{ "net.example.storage/other-driver": "net.example.storage/csi-node1", @@ -501,8 +496,7 @@ func TestRemoveNodeInfo(t *testing.T) { expectedNodeIDMap: map[string]string{ "net.example.storage/other-driver": "net.example.storage/csi-node1", }, - expectedLabels: nil, - expectNoNodeInfo: true, + expectedLabels: nil, }, { name: "new node with valid max limit", @@ -517,7 +511,6 @@ func TestRemoveNodeInfo(t *testing.T) { ), inputTopology: nil, inputNodeID: "com.example.csi/csi-node1", - expectNoNodeInfo: true, expectedVolumeLimit: 0, }, } @@ -525,9 +518,9 @@ func TestRemoveNodeInfo(t *testing.T) { test(t, false /* addNodeInfo */, true /* csiNodeInfoEnabled */, testcases) } -// TestRemoveNodeInfo tests RemoveNodeInfo with various existing Node objects and CSINodeInfo +// TestUninstallCSIDriver tests UninstallCSIDriver with various existing Node objects and CSINodeInfo // feature disabled. -func TestRemoveNodeInfo_CSINodeInfoDisabled(t *testing.T) { +func TestUninstallCSIDriver_CSINodeInfoDisabled(t *testing.T) { testcases := []testcase{ { name: "empty node", @@ -562,7 +555,7 @@ func TestRemoveNodeInfo_CSINodeInfoDisabled(t *testing.T) { test(t, false /* addNodeInfo */, false /* csiNodeInfoEnabled */, testcases) } -func TestAddNodeInfoExistingAnnotation(t *testing.T) { +func TestInstallCSIDriverExistingAnnotation(t *testing.T) { defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSINodeInfo, true)() driverName := "com.example.csi/driver1" @@ -573,7 +566,7 @@ func TestAddNodeInfoExistingAnnotation(t *testing.T) { existingNode *v1.Node }{ { - name: "pre-existing info about the same driver in node, but no CSINodeInfo", + name: "pre-existing info about the same driver in node, but empty CSINodeInfo", existingNode: generateNode( nodeIDMap{ "com.example.csi/driver1": "com.example.csi/csi-node1", @@ -581,7 +574,7 @@ func TestAddNodeInfoExistingAnnotation(t *testing.T) { nil /* labels */, nil /*capacity*/), }, { - name: "pre-existing info about a different driver in node, but no CSINodeInfo", + name: "pre-existing info about a different driver in node, but empty CSINodeInfo", existingNode: generateNode( nodeIDMap{ "net.example.storage/other-driver": "net.example.storage/test-node", @@ -613,9 +606,14 @@ func TestAddNodeInfoExistingAnnotation(t *testing.T) { nim := NewNodeInfoManager(types.NodeName(nodeName), host) // Act - err = nim.AddNodeInfo(driverName, nodeID, 0 /* maxVolumeLimit */, nil) // TODO test maxVolumeLimit + _, err = nim.CreateCSINodeInfo() if err != nil { - t.Errorf("expected no error from AddNodeInfo call but got: %v", err) + t.Errorf("expected no error from creating CSINodeinfo but got: %v", err) + continue + } + err = nim.InstallCSIDriver(driverName, nodeID, 0 /* maxVolumeLimit */, nil) // TODO test maxVolumeLimit + if err != nil { + t.Errorf("expected no error from InstallCSIDriver call but got: %v", err) continue } @@ -626,14 +624,305 @@ func TestAddNodeInfoExistingAnnotation(t *testing.T) { continue } - if len(nodeInfo.CSIDrivers) != 1 { - t.Errorf("expected 1 CSIDriverInfo entry but got: %d", len(nodeInfo.CSIDrivers)) + if len(nodeInfo.Spec.Drivers) != 1 || len(nodeInfo.Status.Drivers) != 1 { + t.Errorf("expected 1 CSIDriverInfoSpec and 1 CSIDriverInfoStatus entry but got: %d, %d", + len(nodeInfo.Spec.Drivers), len(nodeInfo.Status.Drivers)) continue } - driver := nodeInfo.CSIDrivers[0] - if driver.Driver != driverName || driver.NodeID != nodeID { - t.Errorf("expected Driver to be %q and NodeID to be %q, but got: %q:%q", driverName, nodeID, driver.Driver, driver.NodeID) + driver := nodeInfo.Spec.Drivers[0] + if driver.Name != driverName || driver.NodeID != nodeID { + t.Errorf("expected Driver to be %q and NodeID to be %q, but got: %q:%q", driverName, nodeID, driver.Name, driver.NodeID) + } + } +} + +func TestValidateCSINodeInfo(t *testing.T) { + testcases := []struct { + name string + nodeInfo *csiv1alpha1.CSINodeInfo + expectErr bool + }{ + { + name: "multiple drivers with different node IDs, topology keys and status", + nodeInfo: &csiv1alpha1.CSINodeInfo{ + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: []csiv1alpha1.CSIDriverInfoSpec{ + { + Name: "driver1", + NodeID: "node1", + TopologyKeys: []string{"key1, key2"}, + }, + { + Name: "driverB", + NodeID: "nodeA", + TopologyKeys: []string{"keyA", "keyB"}, + }, + }, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: []csiv1alpha1.CSIDriverInfoStatus{ + { + Name: "driver1", + Available: true, + VolumePluginMechanism: "in-tree", + }, + { + Name: "driverB", + Available: false, + VolumePluginMechanism: "csi", + }, + }, + }, + }, + expectErr: false, + }, + { + name: "multiple drivers with same node IDs, topology keys and status", + nodeInfo: &csiv1alpha1.CSINodeInfo{ + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: []csiv1alpha1.CSIDriverInfoSpec{ + { + Name: "driver1", + NodeID: "node1", + TopologyKeys: []string{"key1"}, + }, + { + Name: "driver2", + NodeID: "node1", + TopologyKeys: []string{"key1"}, + }, + }, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: []csiv1alpha1.CSIDriverInfoStatus{ + { + Name: "driver1", + Available: true, + VolumePluginMechanism: "csi", + }, + { + Name: "driver2", + Available: true, + VolumePluginMechanism: "csi", + }, + }, + }, + }, + expectErr: false, + }, + { + name: "duplicate drivers in driver specs", + nodeInfo: &csiv1alpha1.CSINodeInfo{ + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: []csiv1alpha1.CSIDriverInfoSpec{ + { + Name: "driver1", + NodeID: "node1", + TopologyKeys: []string{"key1", "key2"}, + }, + { + Name: "driver1", + NodeID: "nodeX", + TopologyKeys: []string{"keyA", "keyB"}, + }, + }, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: []csiv1alpha1.CSIDriverInfoStatus{ + { + Name: "driver1", + Available: true, + VolumePluginMechanism: "csi", + }, + }, + }, + }, + expectErr: true, + }, + { + name: "duplicate drivers in driver statuses", + nodeInfo: &csiv1alpha1.CSINodeInfo{ + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: []csiv1alpha1.CSIDriverInfoSpec{ + { + Name: "driver1", + NodeID: "node1", + TopologyKeys: []string{"key1", "key2"}, + }, + }, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: []csiv1alpha1.CSIDriverInfoStatus{ + { + Name: "driver1", + Available: true, + VolumePluginMechanism: "in-tree", + }, + { + Name: "driver1", + Available: false, + VolumePluginMechanism: "csi", + }, + }, + }, + }, + expectErr: true, + }, + { + name: "single driver with duplicate topology keys in driver specs", + nodeInfo: &csiv1alpha1.CSINodeInfo{ + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: []csiv1alpha1.CSIDriverInfoSpec{ + { + Name: "driver1", + NodeID: "node1", + TopologyKeys: []string{"key1", "key1"}, + }, + }, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: []csiv1alpha1.CSIDriverInfoStatus{ + { + Name: "driver1", + Available: true, + VolumePluginMechanism: "csi", + }, + }, + }, + }, + expectErr: true, + }, + { + name: "multiple drivers with one set of duplicate topology keys in driver specs", + nodeInfo: &csiv1alpha1.CSINodeInfo{ + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: []csiv1alpha1.CSIDriverInfoSpec{ + { + Name: "driver1", + NodeID: "node1", + TopologyKeys: []string{"key1"}, + }, + { + Name: "driver2", + NodeID: "nodeX", + TopologyKeys: []string{"keyA", "keyA"}, + }, + }, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: []csiv1alpha1.CSIDriverInfoStatus{ + { + Name: "driver1", + Available: true, + VolumePluginMechanism: "csi", + }, + { + Name: "driver2", + Available: true, + VolumePluginMechanism: "csi", + }, + }, + }, + }, + expectErr: true, + }, + { + name: "mismatch between drivers in specs and status (null intersection)", + nodeInfo: &csiv1alpha1.CSINodeInfo{ + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: []csiv1alpha1.CSIDriverInfoSpec{ + { + Name: "driver1", + NodeID: "node1", + TopologyKeys: []string{"key1"}, + }, + { + Name: "driver2", + NodeID: "nodeX", + TopologyKeys: []string{"keyA", "keyA"}, + }, + }, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: []csiv1alpha1.CSIDriverInfoStatus{ + { + Name: "driver3", + Available: true, + VolumePluginMechanism: "csi", + }, + }, + }, + }, + expectErr: true, + }, + { + name: "mismatch between drivers in specs and status (specs superset of status)", + nodeInfo: &csiv1alpha1.CSINodeInfo{ + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: []csiv1alpha1.CSIDriverInfoSpec{ + { + Name: "driver1", + NodeID: "node1", + TopologyKeys: []string{"key1"}, + }, + { + Name: "driver2", + NodeID: "nodeX", + TopologyKeys: []string{"keyA", "keyA"}, + }, + }, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: []csiv1alpha1.CSIDriverInfoStatus{ + { + Name: "driver1", + Available: true, + VolumePluginMechanism: "csi", + }, + }, + }, + }, + expectErr: true, + }, + { + name: "mismatch between drivers in specs and status (specs subset of status)", + nodeInfo: &csiv1alpha1.CSINodeInfo{ + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: []csiv1alpha1.CSIDriverInfoSpec{ + { + Name: "driver1", + NodeID: "node1", + TopologyKeys: []string{"key1"}, + }, + }, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: []csiv1alpha1.CSIDriverInfoStatus{ + { + Name: "driver1", + Available: true, + VolumePluginMechanism: "csi", + }, + { + Name: "driver2", + Available: true, + VolumePluginMechanism: "csi", + }, + }, + }, + }, + expectErr: true, + }, + } + for _, tc := range testcases { + t.Logf("test case: %q", tc.name) + err := validateCSINodeInfo(tc.nodeInfo) + if err != nil && !tc.expectErr { + t.Errorf("expected no errors from validateCSINodeInfo but got error %v", err) + } + if err == nil && tc.expectErr { + t.Errorf("expected error from validateCSINodeInfo but got no errors") } } } @@ -669,28 +958,29 @@ func test(t *testing.T, addNodeInfo bool, csiNodeInfoEnabled bool, testcases []t nim := NewNodeInfoManager(types.NodeName(nodeName), host) //// Act + nim.CreateCSINodeInfo() if addNodeInfo { - err = nim.AddNodeInfo(tc.driverName, tc.inputNodeID, tc.inputVolumeLimit, tc.inputTopology) + err = nim.InstallCSIDriver(tc.driverName, tc.inputNodeID, tc.inputVolumeLimit, tc.inputTopology) } else { - err = nim.RemoveNodeInfo(tc.driverName) + err = nim.UninstallCSIDriver(tc.driverName) } //// Assert if tc.expectFail { if err == nil { - t.Errorf("expected an error from AddNodeInfo call but got none") + t.Errorf("expected an error from InstallCSIDriver call but got none") } continue } else if err != nil { - t.Errorf("expected no error from AddNodeInfo call but got: %v", err) + t.Errorf("expected no error from InstallCSIDriver call but got: %v", err) continue } actions := client.Actions() var node *v1.Node - if hasPatchAction(actions) { - node, err = applyNodeStatusPatch(tc.existingNode, actions[1].(clienttesting.PatchActionImpl).GetPatch()) + if action := hasPatchAction(actions); action != nil { + node, err = applyNodeStatusPatch(tc.existingNode, action.(clienttesting.PatchActionImpl).GetPatch()) assert.NoError(t, err) } else { node, err = client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) @@ -710,6 +1000,7 @@ func test(t *testing.T, addNodeInfo bool, csiNodeInfoEnabled bool, testcases []t } // Node ID annotation + foundInNode := false annNodeID, ok := node.Annotations[annotationKeyNodeID] if ok { if tc.expectedNodeIDMap == nil { @@ -723,6 +1014,8 @@ func test(t *testing.T, addNodeInfo bool, csiNodeInfoEnabled bool, testcases []t if !helper.Semantic.DeepEqual(actualNodeIDs, tc.expectedNodeIDMap) { t.Errorf("expected annotation %v; got: %v", tc.expectedNodeIDMap, actualNodeIDs) + } else { + foundInNode = true } } } else { @@ -739,24 +1032,35 @@ func test(t *testing.T, addNodeInfo bool, csiNodeInfoEnabled bool, testcases []t /* CSINodeInfo validation */ nodeInfo, err := csiClient.Csi().CSINodeInfos().Get(nodeName, metav1.GetOptions{}) - if tc.expectNoNodeInfo && errors.IsNotFound(err) { - continue - } else if err != nil { + if err != nil { t.Errorf("error getting CSINodeInfo: %v", err) continue } // Extract node IDs and topology keys + + availableDrivers := sets.String{} actualNodeIDs := make(map[string]string) actualTopologyKeys := make(map[string]sets.String) - for _, driver := range nodeInfo.CSIDrivers { - actualNodeIDs[driver.Driver] = driver.NodeID - actualTopologyKeys[driver.Driver] = sets.NewString(driver.TopologyKeys...) + for _, driver := range nodeInfo.Status.Drivers { + if driver.Available { + availableDrivers.Insert(driver.Name) + } + + } + for _, driver := range nodeInfo.Spec.Drivers { + if availableDrivers.Has(driver.Name) { + actualNodeIDs[driver.Name] = driver.NodeID + actualTopologyKeys[driver.Name] = sets.NewString(driver.TopologyKeys...) + } } // Node IDs - if !helper.Semantic.DeepEqual(actualNodeIDs, tc.expectedNodeIDMap) { - t.Errorf("expected node IDs %v from CSINodeInfo; got: %v", tc.expectedNodeIDMap, actualNodeIDs) + // No need to check if Node ID found in NodeInfo if it was present in the NodeID + if !foundInNode { + if !helper.Semantic.DeepEqual(actualNodeIDs, tc.expectedNodeIDMap) { + t.Errorf("expected node IDs %v from CSINodeInfo; got: %v", tc.expectedNodeIDMap, actualNodeIDs) + } } // Topology keys @@ -802,22 +1106,34 @@ func generateNode(nodeIDs, labels map[string]string, capacity map[v1.ResourceNam } func generateNodeInfo(nodeIDs map[string]string, topologyKeys map[string][]string) *csiv1alpha1.CSINodeInfo { - var drivers []csiv1alpha1.CSIDriverInfo + driverInfoSpecs := []csiv1alpha1.CSIDriverInfoSpec{} + driverInfoStatuses := []csiv1alpha1.CSIDriverInfoStatus{} for k, nodeID := range nodeIDs { - d := csiv1alpha1.CSIDriverInfo{ - Driver: k, + dspec := csiv1alpha1.CSIDriverInfoSpec{ + Name: k, NodeID: nodeID, } - if top, exists := topologyKeys[k]; exists { - d.TopologyKeys = top + dstatus := csiv1alpha1.CSIDriverInfoStatus{ + Name: k, + Available: true, + VolumePluginMechanism: csiv1alpha1.VolumePluginMechanismInTree, } - drivers = append(drivers, d) + if top, exists := topologyKeys[k]; exists { + dspec.TopologyKeys = top + } + driverInfoSpecs = append(driverInfoSpecs, dspec) + driverInfoStatuses = append(driverInfoStatuses, dstatus) } return &csiv1alpha1.CSINodeInfo{ ObjectMeta: metav1.ObjectMeta{ Name: "node1", }, - CSIDrivers: drivers, + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: driverInfoSpecs, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: driverInfoStatuses, + }, } } @@ -838,11 +1154,11 @@ func applyNodeStatusPatch(originalNode *v1.Node, patch []byte) (*v1.Node, error) return updatedNode, nil } -func hasPatchAction(actions []clienttesting.Action) bool { +func hasPatchAction(actions []clienttesting.Action) clienttesting.Action { for _, action := range actions { if action.GetVerb() == "patch" { - return true + return action } } - return false + return nil } diff --git a/pkg/volume/downwardapi/BUILD b/pkg/volume/downwardapi/BUILD index deb9cf9b261..d223844116f 100644 --- a/pkg/volume/downwardapi/BUILD +++ b/pkg/volume/downwardapi/BUILD @@ -19,7 +19,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/downwardapi/downwardapi.go b/pkg/volume/downwardapi/downwardapi.go index 11fe62fc777..391198ad9dd 100644 --- a/pkg/volume/downwardapi/downwardapi.go +++ b/pkg/volume/downwardapi/downwardapi.go @@ -18,7 +18,6 @@ package downwardapi import ( "fmt" - "path" "path/filepath" "k8s.io/api/core/v1" @@ -30,7 +29,7 @@ import ( "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" - "github.com/golang/glog" + "k8s.io/klog" ) // ProbeVolumePlugins is the entry point for plugin detection in a package. @@ -171,23 +170,23 @@ func (b *downwardAPIVolumeMounter) SetUp(fsGroup *int64) error { } func (b *downwardAPIVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { - glog.V(3).Infof("Setting up a downwardAPI volume %v for pod %v/%v at %v", b.volName, b.pod.Namespace, b.pod.Name, dir) + klog.V(3).Infof("Setting up a downwardAPI volume %v for pod %v/%v at %v", b.volName, b.pod.Namespace, b.pod.Name, dir) // Wrap EmptyDir. Here we rely on the idempotency of the wrapped plugin to avoid repeatedly mounting wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec(), b.pod, *b.opts) if err != nil { - glog.Errorf("Couldn't setup downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error()) + klog.Errorf("Couldn't setup downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error()) return err } data, err := CollectData(b.source.Items, b.pod, b.plugin.host, b.source.DefaultMode) if err != nil { - glog.Errorf("Error preparing data for downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error()) + klog.Errorf("Error preparing data for downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error()) return err } setupSuccess := false if err := wrapped.SetUpAt(dir, fsGroup); err != nil { - glog.Errorf("Unable to setup downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error()) + klog.Errorf("Unable to setup downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error()) return err } @@ -200,12 +199,12 @@ func (b *downwardAPIVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if !setupSuccess { unmounter, unmountCreateErr := b.plugin.NewUnmounter(b.volName, b.podUID) if unmountCreateErr != nil { - glog.Errorf("error cleaning up mount %s after failure. Create unmounter failed with %v", b.volName, unmountCreateErr) + klog.Errorf("error cleaning up mount %s after failure. Create unmounter failed with %v", b.volName, unmountCreateErr) return } tearDownErr := unmounter.TearDown() if tearDownErr != nil { - glog.Errorf("error tearing down volume %s with : %v", b.volName, tearDownErr) + klog.Errorf("error tearing down volume %s with : %v", b.volName, tearDownErr) } } }() @@ -213,19 +212,19 @@ func (b *downwardAPIVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { writerContext := fmt.Sprintf("pod %v/%v volume %v", b.pod.Namespace, b.pod.Name, b.volName) writer, err := volumeutil.NewAtomicWriter(dir, writerContext) if err != nil { - glog.Errorf("Error creating atomic writer: %v", err) + klog.Errorf("Error creating atomic writer: %v", err) return err } err = writer.Write(data) if err != nil { - glog.Errorf("Error writing payload to dir: %v", err) + klog.Errorf("Error writing payload to dir: %v", err) return err } err = volume.SetVolumeOwnership(b, fsGroup) if err != nil { - glog.Errorf("Error applying volume ownership settings for group: %v", fsGroup) + klog.Errorf("Error applying volume ownership settings for group: %v", fsGroup) return err } @@ -256,7 +255,7 @@ func CollectData(items []v1.DownwardAPIVolumeFile, pod *v1.Pod, host volume.Volu if fileInfo.FieldRef != nil { // TODO: unify with Kubelet.podFieldSelectorRuntimeValue if values, err := fieldpath.ExtractFieldPathAsString(pod, fileInfo.FieldRef.FieldPath); err != nil { - glog.Errorf("Unable to extract field %s: %s", fileInfo.FieldRef.FieldPath, err.Error()) + klog.Errorf("Unable to extract field %s: %s", fileInfo.FieldRef.FieldPath, err.Error()) errlist = append(errlist, err) } else { fileProjection.Data = []byte(values) @@ -267,7 +266,7 @@ func CollectData(items []v1.DownwardAPIVolumeFile, pod *v1.Pod, host volume.Volu if err != nil { errlist = append(errlist, err) } else if values, err := resource.ExtractResourceValueByContainerNameAndNodeAllocatable(fileInfo.ResourceFieldRef, pod, containerName, nodeAllocatable); err != nil { - glog.Errorf("Unable to extract field %s: %s", fileInfo.ResourceFieldRef.Resource, err.Error()) + klog.Errorf("Unable to extract field %s: %s", fileInfo.ResourceFieldRef.Resource, err.Error()) errlist = append(errlist, err) } else { fileProjection.Data = []byte(values) @@ -299,10 +298,6 @@ func (c *downwardAPIVolumeUnmounter) TearDownAt(dir string) error { return volumeutil.UnmountViaEmptyDir(dir, c.plugin.host, c.volName, wrappedVolumeSpec(), c.podUID) } -func (b *downwardAPIVolumeMounter) getMetaDir() string { - return path.Join(b.plugin.host.GetPodPluginDir(b.podUID, utilstrings.EscapeQualifiedNameForDisk(downwardAPIPluginName)), b.volName) -} - func getVolumeSource(spec *volume.Spec) (*v1.DownwardAPIVolumeSource, bool) { var readOnly bool var volumeSource *v1.DownwardAPIVolumeSource diff --git a/pkg/volume/emptydir/BUILD b/pkg/volume/emptydir/BUILD index ceeeb1031fd..fd4eaff0332 100644 --- a/pkg/volume/emptydir/BUILD +++ b/pkg/volume/emptydir/BUILD @@ -25,7 +25,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//vendor/golang.org/x/sys/unix:go_default_library", diff --git a/pkg/volume/emptydir/empty_dir.go b/pkg/volume/emptydir/empty_dir.go index d01ac4edd55..0dd6c6508a0 100644 --- a/pkg/volume/emptydir/empty_dir.go +++ b/pkg/volume/emptydir/empty_dir.go @@ -21,11 +21,11 @@ import ( "os" "path" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/util/mount" stringsutil "k8s.io/kubernetes/pkg/util/strings" @@ -253,7 +253,7 @@ func (ed *emptyDir) setupTmpfs(dir string) error { return nil } - glog.V(3).Infof("pod %v: mounting tmpfs for volume %v", ed.pod.UID, ed.volName) + klog.V(3).Infof("pod %v: mounting tmpfs for volume %v", ed.pod.UID, ed.volName) return ed.mounter.Mount("tmpfs", dir, "tmpfs", nil /* options */) } @@ -281,7 +281,7 @@ func (ed *emptyDir) setupHugepages(dir string) error { return err } - glog.V(3).Infof("pod %v: mounting hugepages for volume %v", ed.pod.UID, ed.volName) + klog.V(3).Infof("pod %v: mounting hugepages for volume %v", ed.pod.UID, ed.volName) return ed.mounter.Mount("nodev", dir, "hugetlbfs", []string{pageSizeMountOption}) } @@ -349,7 +349,7 @@ func (ed *emptyDir) setupDir(dir string) error { } if fileinfo.Mode().Perm() != perm.Perm() { - glog.Errorf("Expected directory %q permissions to be: %s; got: %s", dir, perm.Perm(), fileinfo.Mode().Perm()) + klog.Errorf("Expected directory %q permissions to be: %s; got: %s", dir, perm.Perm(), fileinfo.Mode().Perm()) } } @@ -370,7 +370,7 @@ func (ed *emptyDir) TearDownAt(dir string) error { if pathExists, pathErr := volumeutil.PathExists(dir); pathErr != nil { return fmt.Errorf("Error checking if path exists: %v", pathErr) } else if !pathExists { - glog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir) + klog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir) return nil } diff --git a/pkg/volume/emptydir/empty_dir_linux.go b/pkg/volume/emptydir/empty_dir_linux.go index 880c2b37c30..3f8b1628274 100644 --- a/pkg/volume/emptydir/empty_dir_linux.go +++ b/pkg/volume/emptydir/empty_dir_linux.go @@ -21,8 +21,8 @@ package emptydir import ( "fmt" - "github.com/golang/glog" "golang.org/x/sys/unix" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/util/mount" @@ -40,7 +40,7 @@ type realMountDetector struct { } func (m *realMountDetector) GetMountMedium(path string) (v1.StorageMedium, bool, error) { - glog.V(5).Infof("Determining mount medium of %v", path) + klog.V(5).Infof("Determining mount medium of %v", path) notMnt, err := m.mounter.IsLikelyNotMountPoint(path) if err != nil { return v1.StorageMediumDefault, false, fmt.Errorf("IsLikelyNotMountPoint(%q): %v", path, err) @@ -50,7 +50,7 @@ func (m *realMountDetector) GetMountMedium(path string) (v1.StorageMedium, bool, return v1.StorageMediumDefault, false, fmt.Errorf("statfs(%q): %v", path, err) } - glog.V(5).Infof("Statfs_t of %v: %+v", path, buf) + klog.V(5).Infof("Statfs_t of %v: %+v", path, buf) if buf.Type == linuxTmpfsMagic { return v1.StorageMediumMemory, !notMnt, nil } else if int64(buf.Type) == linuxHugetlbfsMagic { diff --git a/pkg/volume/fc/BUILD b/pkg/volume/fc/BUILD index 9b8a102b84c..8c7328601db 100644 --- a/pkg/volume/fc/BUILD +++ b/pkg/volume/fc/BUILD @@ -27,7 +27,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/fc/attacher.go b/pkg/volume/fc/attacher.go index 3ef5dd20f98..748c14f38e1 100644 --- a/pkg/volume/fc/attacher.go +++ b/pkg/volume/fc/attacher.go @@ -23,10 +23,10 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" @@ -78,7 +78,7 @@ func (attacher *fcAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName ty func (attacher *fcAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) { mounter, err := volumeSpecToMounter(spec, attacher.host) if err != nil { - glog.Warningf("failed to get fc mounter: %v", err) + klog.Warningf("failed to get fc mounter: %v", err) return "", err } return attacher.manager.AttachDisk(*mounter) @@ -88,7 +88,7 @@ func (attacher *fcAttacher) GetDeviceMountPath( spec *volume.Spec) (string, error) { mounter, err := volumeSpecToMounter(spec, attacher.host) if err != nil { - glog.Warningf("failed to get fc mounter: %v", err) + klog.Warningf("failed to get fc mounter: %v", err) return "", err } @@ -158,7 +158,7 @@ func (detacher *fcDetacher) UnmountDevice(deviceMountPath string) error { // Specify device name for DetachDisk later devName, _, err := mount.GetDeviceNameFromMount(detacher.mounter, deviceMountPath) if err != nil { - glog.Errorf("fc: failed to get device from mnt: %s\nError: %v", deviceMountPath, err) + klog.Errorf("fc: failed to get device from mnt: %s\nError: %v", deviceMountPath, err) return err } // Unmount for deviceMountPath(=globalPDPath) @@ -171,7 +171,7 @@ func (detacher *fcDetacher) UnmountDevice(deviceMountPath string) error { if err != nil { return fmt.Errorf("fc: failed to detach disk: %s\nError: %v", devName, err) } - glog.V(4).Infof("fc: successfully detached disk: %s", devName) + klog.V(4).Infof("fc: successfully detached disk: %s", devName) return nil } @@ -206,7 +206,7 @@ func volumeSpecToMounter(spec *volume.Spec, host volume.VolumeHost) (*fcDiskMoun if err != nil { return nil, err } - glog.V(5).Infof("fc: volumeSpecToMounter volumeMode %s", volumeMode) + klog.V(5).Infof("fc: volumeSpecToMounter volumeMode %s", volumeMode) return &fcDiskMounter{ fcDisk: fcDisk, fsType: fc.FSType, diff --git a/pkg/volume/fc/disk_manager.go b/pkg/volume/fc/disk_manager.go index c6cc815ac2d..602b7aa0bdb 100644 --- a/pkg/volume/fc/disk_manager.go +++ b/pkg/volume/fc/disk_manager.go @@ -19,7 +19,7 @@ package fc import ( "os" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" @@ -43,14 +43,14 @@ func diskSetUp(manager diskManager, b fcDiskMounter, volPath string, mounter mou noMnt, err := mounter.IsLikelyNotMountPoint(volPath) if err != nil && !os.IsNotExist(err) { - glog.Errorf("cannot validate mountpoint: %s", volPath) + klog.Errorf("cannot validate mountpoint: %s", volPath) return err } if !noMnt { return nil } if err := os.MkdirAll(volPath, 0750); err != nil { - glog.Errorf("failed to mkdir:%s", volPath) + klog.Errorf("failed to mkdir:%s", volPath) return err } // Perform a bind mount to the full path to allow duplicate mounts of the same disk. @@ -61,25 +61,25 @@ func diskSetUp(manager diskManager, b fcDiskMounter, volPath string, mounter mou mountOptions := util.JoinMountOptions(options, b.mountOptions) err = mounter.Mount(globalPDPath, volPath, "", mountOptions) if err != nil { - glog.Errorf("Failed to bind mount: source:%s, target:%s, err:%v", globalPDPath, volPath, err) + klog.Errorf("Failed to bind mount: source:%s, target:%s, err:%v", globalPDPath, volPath, err) noMnt, mntErr := b.mounter.IsLikelyNotMountPoint(volPath) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !noMnt { if mntErr = b.mounter.Unmount(volPath); mntErr != nil { - glog.Errorf("Failed to unmount: %v", mntErr) + klog.Errorf("Failed to unmount: %v", mntErr) return err } noMnt, mntErr = b.mounter.IsLikelyNotMountPoint(volPath) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !noMnt { // will most likely retry on next sync loop. - glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", volPath) + klog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", volPath) return err } } diff --git a/pkg/volume/fc/fc.go b/pkg/volume/fc/fc.go index d413e02dbf4..cbd247aebca 100644 --- a/pkg/volume/fc/fc.go +++ b/pkg/volume/fc/fc.go @@ -22,11 +22,11 @@ import ( "strconv" "strings" - "github.com/golang/glog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" utilstrings "k8s.io/kubernetes/pkg/util/strings" @@ -137,7 +137,7 @@ func (plugin *fcPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, if err != nil { return nil, err } - glog.V(5).Infof("fc: newMounterInternal volumeMode %s", volumeMode) + klog.V(5).Infof("fc: newMounterInternal volumeMode %s", volumeMode) return &fcDiskMounter{ fcDisk: fcDisk, fsType: fc.FSType, @@ -276,7 +276,7 @@ func (plugin *fcPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volu FC: &v1.FCVolumeSource{TargetWWNs: []string{wwnLun[0]}, Lun: &lun32}, }, } - glog.V(5).Infof("ConstructVolumeSpec: TargetWWNs: %v, Lun: %v", + klog.V(5).Infof("ConstructVolumeSpec: TargetWWNs: %v, Lun: %v", fcVolume.VolumeSource.FC.TargetWWNs, *fcVolume.VolumeSource.FC.Lun) } else { fcVolume = &v1.Volume{ @@ -285,7 +285,7 @@ func (plugin *fcPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volu FC: &v1.FCVolumeSource{WWIDs: []string{volumeInfo}}, }, } - glog.V(5).Infof("ConstructVolumeSpec: WWIDs: %v", fcVolume.VolumeSource.FC.WWIDs) + klog.V(5).Infof("ConstructVolumeSpec: WWIDs: %v", fcVolume.VolumeSource.FC.WWIDs) } return volume.NewSpecFromVolume(fcVolume), nil } @@ -304,7 +304,7 @@ func (plugin *fcPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, m if err != nil { return nil, err } - glog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err) + klog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err) // Retrieve volumePluginDependentPath from globalMapPathUUID // globalMapPathUUID examples: @@ -328,13 +328,13 @@ func (plugin *fcPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, m lun32 := int32(lun) fcPV = createPersistentVolumeFromFCVolumeSource(volumeName, v1.FCVolumeSource{TargetWWNs: []string{wwnLun[0]}, Lun: &lun32}) - glog.V(5).Infof("ConstructBlockVolumeSpec: TargetWWNs: %v, Lun: %v", + klog.V(5).Infof("ConstructBlockVolumeSpec: TargetWWNs: %v, Lun: %v", fcPV.Spec.PersistentVolumeSource.FC.TargetWWNs, *fcPV.Spec.PersistentVolumeSource.FC.Lun) } else { fcPV = createPersistentVolumeFromFCVolumeSource(volumeName, v1.FCVolumeSource{WWIDs: []string{volumeInfo}}) - glog.V(5).Infof("ConstructBlockVolumeSpec: WWIDs: %v", fcPV.Spec.PersistentVolumeSource.FC.WWIDs) + klog.V(5).Infof("ConstructBlockVolumeSpec: WWIDs: %v", fcPV.Spec.PersistentVolumeSource.FC.WWIDs) } return volume.NewSpecFromPersistentVolume(fcPV, false), nil } @@ -363,7 +363,7 @@ func (fc *fcDisk) GetPath() string { func (fc *fcDisk) fcGlobalMapPath(spec *volume.Spec) (string, error) { mounter, err := volumeSpecToMounter(spec, fc.plugin.host) if err != nil { - glog.Warningf("failed to get fc mounter: %v", err) + klog.Warningf("failed to get fc mounter: %v", err) return "", err } return fc.manager.MakeGlobalVDPDName(*mounter.fcDisk), nil @@ -409,7 +409,7 @@ func (b *fcDiskMounter) SetUpAt(dir string, fsGroup *int64) error { // diskSetUp checks mountpoints and prevent repeated calls err := diskSetUp(b.manager, *b, dir, b.mounter, fsGroup) if err != nil { - glog.Errorf("fc: failed to setup") + klog.Errorf("fc: failed to setup") } return err } @@ -462,12 +462,12 @@ func (c *fcDiskUnmapper) TearDownDevice(mapPath, devicePath string) error { if err != nil { return fmt.Errorf("fc: failed to detach disk: %s\nError: %v", mapPath, err) } - glog.V(4).Infof("fc: %q is unmounted, deleting the directory", mapPath) + klog.V(4).Infof("fc: %q is unmounted, deleting the directory", mapPath) err = os.RemoveAll(mapPath) if err != nil { return fmt.Errorf("fc: failed to delete the directory: %s\nError: %v", mapPath, err) } - glog.V(4).Infof("fc: successfully detached disk: %s", mapPath) + klog.V(4).Infof("fc: successfully detached disk: %s", mapPath) return nil } diff --git a/pkg/volume/fc/fc_test.go b/pkg/volume/fc/fc_test.go index 7fd81669a8e..632b05c2f1e 100644 --- a/pkg/volume/fc/fc_test.go +++ b/pkg/volume/fc/fc_test.go @@ -254,6 +254,7 @@ func TestPluginVolume(t *testing.T) { func TestPluginPersistentVolume(t *testing.T) { lun := int32(0) + fs := v1.PersistentVolumeFilesystem vol := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "vol1", @@ -266,6 +267,7 @@ func TestPluginPersistentVolume(t *testing.T) { Lun: &lun, }, }, + VolumeMode: &fs, }, } doTestPlugin(t, volume.NewSpecFromPersistentVolume(vol, false)) @@ -285,6 +287,7 @@ func TestPluginVolumeWWIDs(t *testing.T) { } func TestPluginPersistentVolumeWWIDs(t *testing.T) { + fs := v1.PersistentVolumeFilesystem vol := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "vol1", @@ -296,6 +299,7 @@ func TestPluginPersistentVolumeWWIDs(t *testing.T) { FSType: "ext4", }, }, + VolumeMode: &fs, }, } doTestPlugin(t, volume.NewSpecFromPersistentVolume(vol, false)) @@ -314,6 +318,7 @@ func TestPluginVolumeNoDiskInfo(t *testing.T) { } func TestPluginPersistentVolumeNoDiskInfo(t *testing.T) { + fs := v1.PersistentVolumeFilesystem vol := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "vol1", @@ -324,6 +329,7 @@ func TestPluginPersistentVolumeNoDiskInfo(t *testing.T) { FSType: "ext4", }, }, + VolumeMode: &fs, }, } doTestPluginNilMounter(t, volume.NewSpecFromPersistentVolume(vol, false)) @@ -337,6 +343,7 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { defer os.RemoveAll(tmpDir) lun := int32(0) + fs := v1.PersistentVolumeFilesystem pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pvA", @@ -352,6 +359,7 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { ClaimRef: &v1.ObjectReference{ Name: "claimA", }, + VolumeMode: &fs, }, } @@ -362,6 +370,7 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { }, Spec: v1.PersistentVolumeClaimSpec{ VolumeName: "pvA", + VolumeMode: &fs, }, Status: v1.PersistentVolumeClaimStatus{ Phase: v1.ClaimBound, diff --git a/pkg/volume/fc/fc_util.go b/pkg/volume/fc/fc_util.go index 9d71a00a23d..2beafee94bf 100644 --- a/pkg/volume/fc/fc_util.go +++ b/pkg/volume/fc/fc_util.go @@ -24,9 +24,9 @@ import ( "path/filepath" "strings" - "github.com/golang/glog" "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" @@ -69,7 +69,7 @@ func findDisk(wwn, lun string, io ioHandler, deviceUtil volumeutil.DeviceUtil) ( if strings.Contains(name, fcPath) { if disk, err1 := io.EvalSymlinks(devPath + name); err1 == nil { dm := deviceUtil.FindMultipathDeviceForDevice(disk) - glog.Infof("fc: find disk: %v, dm: %v", disk, dm) + klog.Infof("fc: find disk: %v, dm: %v", disk, dm) return disk, dm } } @@ -97,23 +97,23 @@ func findDiskWWIDs(wwid string, io ioHandler, deviceUtil volumeutil.DeviceUtil) if name == fcPath { disk, err := io.EvalSymlinks(devID + name) if err != nil { - glog.V(2).Infof("fc: failed to find a corresponding disk from symlink[%s], error %v", devID+name, err) + klog.V(2).Infof("fc: failed to find a corresponding disk from symlink[%s], error %v", devID+name, err) return "", "" } dm := deviceUtil.FindMultipathDeviceForDevice(disk) - glog.Infof("fc: find disk: %v, dm: %v", disk, dm) + klog.Infof("fc: find disk: %v, dm: %v", disk, dm) return disk, dm } } } - glog.V(2).Infof("fc: failed to find a disk [%s]", devID+fcPath) + klog.V(2).Infof("fc: failed to find a disk [%s]", devID+fcPath) return "", "" } // Removes a scsi device based upon /dev/sdX name func removeFromScsiSubsystem(deviceName string, io ioHandler) { fileName := "/sys/block/" + deviceName + "/device/delete" - glog.V(4).Infof("fc: remove device from scsi-subsystem: path: %s", fileName) + klog.V(4).Infof("fc: remove device from scsi-subsystem: path: %s", fileName) data := []byte("1") io.WriteFile(fileName, data, 0666) } @@ -218,7 +218,7 @@ func (util *fcUtil) AttachDisk(b fcDiskMounter) (string, error) { if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { // If the volumeMode is 'Block', plugin don't have to format the volume. // The globalPDPath will be created by operationexecutor. Just return devicePath here. - glog.V(5).Infof("fc: AttachDisk volumeMode: %s, devicePath: %s", b.volumeMode, devicePath) + klog.V(5).Infof("fc: AttachDisk volumeMode: %s, devicePath: %s", b.volumeMode, devicePath) if b.volumeMode == v1.PersistentVolumeBlock { return devicePath, nil } @@ -235,7 +235,7 @@ func (util *fcUtil) AttachDisk(b fcDiskMounter) (string, error) { return devicePath, fmt.Errorf("Heuristic determination of mount point failed:%v", err) } if !noMnt { - glog.Infof("fc: %s already mounted", globalPDPath) + klog.Infof("fc: %s already mounted", globalPDPath) return devicePath, nil } @@ -262,17 +262,17 @@ func (util *fcUtil) DetachDisk(c fcDiskUnmounter, devicePath string) error { // Add single devicepath to devices devices = append(devices, dstPath) } - glog.V(4).Infof("fc: DetachDisk devicePath: %v, dstPath: %v, devices: %v", devicePath, dstPath, devices) + klog.V(4).Infof("fc: DetachDisk devicePath: %v, dstPath: %v, devices: %v", devicePath, dstPath, devices) var lastErr error for _, device := range devices { err := util.detachFCDisk(c.io, device) if err != nil { - glog.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err) + klog.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err) lastErr = fmt.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err) } } if lastErr != nil { - glog.Errorf("fc: last error occurred during detach disk:\n%v", lastErr) + klog.Errorf("fc: last error occurred during detach disk:\n%v", lastErr) return lastErr } return nil @@ -301,7 +301,7 @@ func (util *fcUtil) DetachBlockFCDisk(c fcDiskUnmapper, mapPath, devicePath stri } else { // TODO: FC plugin can't obtain the devicePath from kubelet because devicePath // in volume object isn't updated when volume is attached to kubelet node. - glog.Infof("fc: devicePath is empty. Try to retrieve FC configuration from global map path: %v", mapPath) + klog.Infof("fc: devicePath is empty. Try to retrieve FC configuration from global map path: %v", mapPath) } // Check if global map path is valid @@ -332,7 +332,7 @@ func (util *fcUtil) DetachBlockFCDisk(c fcDiskUnmapper, mapPath, devicePath stri for _, fi := range fis { if strings.Contains(fi.Name(), volumeInfo) { devicePath = path.Join(searchPath, fi.Name()) - glog.V(5).Infof("fc: updated devicePath: %s", devicePath) + klog.V(5).Infof("fc: updated devicePath: %s", devicePath) break } } @@ -343,7 +343,7 @@ func (util *fcUtil) DetachBlockFCDisk(c fcDiskUnmapper, mapPath, devicePath stri if err != nil { return err } - glog.V(4).Infof("fc: find destination device path from symlink: %v", dstPath) + klog.V(4).Infof("fc: find destination device path from symlink: %v", dstPath) var devices []string dm := c.deviceUtil.FindMultipathDeviceForDevice(dstPath) @@ -363,12 +363,12 @@ func (util *fcUtil) DetachBlockFCDisk(c fcDiskUnmapper, mapPath, devicePath stri for _, device := range devices { err = util.detachFCDisk(c.io, device) if err != nil { - glog.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err) + klog.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err) lastErr = fmt.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err) } } if lastErr != nil { - glog.Errorf("fc: last error occurred during detach disk:\n%v", lastErr) + klog.Errorf("fc: last error occurred during detach disk:\n%v", lastErr) return lastErr } return nil @@ -378,7 +378,7 @@ func checkPathExists(path string) (bool, error) { if pathExists, pathErr := volumeutil.PathExists(path); pathErr != nil { return pathExists, fmt.Errorf("Error checking if path exists: %v", pathErr) } else if !pathExists { - glog.Warningf("Warning: Unmap skipped because path does not exist: %v", path) + klog.Warningf("Warning: Unmap skipped because path does not exist: %v", path) return pathExists, nil } return true, nil diff --git a/pkg/volume/flexvolume/BUILD b/pkg/volume/flexvolume/BUILD index 38ff345cb35..f337fc42bb3 100644 --- a/pkg/volume/flexvolume/BUILD +++ b/pkg/volume/flexvolume/BUILD @@ -39,7 +39,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/github.com/fsnotify/fsnotify:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/volume/flexvolume/attacher-defaults.go b/pkg/volume/flexvolume/attacher-defaults.go index 368ac1be0bd..3c82f264b91 100644 --- a/pkg/volume/flexvolume/attacher-defaults.go +++ b/pkg/volume/flexvolume/attacher-defaults.go @@ -19,7 +19,7 @@ package flexvolume import ( "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/util/mount" @@ -30,13 +30,13 @@ type attacherDefaults flexVolumeAttacher // Attach is part of the volume.Attacher interface func (a *attacherDefaults) Attach(spec *volume.Spec, hostName types.NodeName) (string, error) { - glog.Warning(logPrefix(a.plugin.flexVolumePlugin), "using default Attach for volume ", spec.Name(), ", host ", hostName) + klog.Warning(logPrefix(a.plugin.flexVolumePlugin), "using default Attach for volume ", spec.Name(), ", host ", hostName) return "", nil } // WaitForAttach is part of the volume.Attacher interface func (a *attacherDefaults) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) { - glog.Warning(logPrefix(a.plugin.flexVolumePlugin), "using default WaitForAttach for volume ", spec.Name(), ", device ", devicePath) + klog.Warning(logPrefix(a.plugin.flexVolumePlugin), "using default WaitForAttach for volume ", spec.Name(), ", device ", devicePath) return devicePath, nil } @@ -47,7 +47,7 @@ func (a *attacherDefaults) GetDeviceMountPath(spec *volume.Spec, mountsDir strin // MountDevice is part of the volume.Attacher interface func (a *attacherDefaults) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, mounter mount.Interface) error { - glog.Warning(logPrefix(a.plugin.flexVolumePlugin), "using default MountDevice for volume ", spec.Name(), ", device ", devicePath, ", deviceMountPath ", deviceMountPath) + klog.Warning(logPrefix(a.plugin.flexVolumePlugin), "using default MountDevice for volume ", spec.Name(), ", device ", devicePath, ", deviceMountPath ", deviceMountPath) volSourceFSType, err := getFSType(spec) if err != nil { diff --git a/pkg/volume/flexvolume/attacher.go b/pkg/volume/flexvolume/attacher.go index 4d2169f36fc..3b98eefa079 100644 --- a/pkg/volume/flexvolume/attacher.go +++ b/pkg/volume/flexvolume/attacher.go @@ -19,9 +19,9 @@ package flexvolume import ( "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/volume" ) @@ -112,7 +112,7 @@ func (a *flexVolumeAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName t } else if err == nil { if !status.Attached { volumesAttachedCheck[spec] = false - glog.V(2).Infof("VolumesAreAttached: check volume (%q) is no longer attached", spec.Name()) + klog.V(2).Infof("VolumesAreAttached: check volume (%q) is no longer attached", spec.Name()) } } else { return nil, err diff --git a/pkg/volume/flexvolume/detacher-defaults.go b/pkg/volume/flexvolume/detacher-defaults.go index 181f87bde48..f0f43262394 100644 --- a/pkg/volume/flexvolume/detacher-defaults.go +++ b/pkg/volume/flexvolume/detacher-defaults.go @@ -19,8 +19,8 @@ package flexvolume import ( "time" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/volume/util" ) @@ -28,18 +28,18 @@ type detacherDefaults flexVolumeDetacher // Detach is part of the volume.Detacher interface. func (d *detacherDefaults) Detach(volumeName string, hostName types.NodeName) error { - glog.Warning(logPrefix(d.plugin.flexVolumePlugin), "using default Detach for volume ", volumeName, ", host ", hostName) + klog.Warning(logPrefix(d.plugin.flexVolumePlugin), "using default Detach for volume ", volumeName, ", host ", hostName) return nil } // WaitForDetach is part of the volume.Detacher interface. func (d *detacherDefaults) WaitForDetach(devicePath string, timeout time.Duration) error { - glog.Warning(logPrefix(d.plugin.flexVolumePlugin), "using default WaitForDetach for device ", devicePath) + klog.Warning(logPrefix(d.plugin.flexVolumePlugin), "using default WaitForDetach for device ", devicePath) return nil } // UnmountDevice is part of the volume.Detacher interface. func (d *detacherDefaults) UnmountDevice(deviceMountPath string) error { - glog.Warning(logPrefix(d.plugin.flexVolumePlugin), "using default UnmountDevice for device mount path ", deviceMountPath) + klog.Warning(logPrefix(d.plugin.flexVolumePlugin), "using default UnmountDevice for device mount path ", deviceMountPath) return util.UnmountPath(deviceMountPath, d.plugin.host.GetMounter(d.plugin.GetPluginName())) } diff --git a/pkg/volume/flexvolume/detacher.go b/pkg/volume/flexvolume/detacher.go index 6b576025846..d98791f3d1b 100644 --- a/pkg/volume/flexvolume/detacher.go +++ b/pkg/volume/flexvolume/detacher.go @@ -20,8 +20,8 @@ import ( "fmt" "os" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" ) @@ -53,7 +53,7 @@ func (d *flexVolumeDetacher) UnmountDevice(deviceMountPath string) error { pathExists, pathErr := util.PathExists(deviceMountPath) if !pathExists { - glog.Warningf("Warning: Unmount skipped because path does not exist: %v", deviceMountPath) + klog.Warningf("Warning: Unmount skipped because path does not exist: %v", deviceMountPath) return nil } if pathErr != nil && !util.IsCorruptedMnt(pathErr) { @@ -70,7 +70,7 @@ func (d *flexVolumeDetacher) UnmountDevice(deviceMountPath string) error { } if notmnt { - glog.Warningf("Warning: Path: %v already unmounted", deviceMountPath) + klog.Warningf("Warning: Path: %v already unmounted", deviceMountPath) } else { call := d.plugin.NewDriverCall(unmountDeviceCmd) call.Append(deviceMountPath) diff --git a/pkg/volume/flexvolume/driver-call.go b/pkg/volume/flexvolume/driver-call.go index 0a42a914247..c25b5999191 100644 --- a/pkg/volume/flexvolume/driver-call.go +++ b/pkg/volume/flexvolume/driver-call.go @@ -22,7 +22,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/volume" ) @@ -141,13 +141,13 @@ func (dc *DriverCall) Run() (*DriverStatus, error) { } _, err := handleCmdResponse(dc.Command, output) if err == nil { - glog.Errorf("FlexVolume: driver bug: %s: exec error (%s) but no error in response.", execPath, execErr) + klog.Errorf("FlexVolume: driver bug: %s: exec error (%s) but no error in response.", execPath, execErr) return nil, execErr } if isCmdNotSupportedErr(err) { dc.plugin.unsupported(dc.Command) } else { - glog.Warningf("FlexVolume: driver call failed: executable: %s, args: %s, error: %s, output: %q", execPath, dc.args, execErr.Error(), output) + klog.Warningf("FlexVolume: driver call failed: executable: %s, args: %s, error: %s, output: %q", execPath, dc.args, execErr.Error(), output) } return nil, err } @@ -264,14 +264,14 @@ func handleCmdResponse(cmd string, output []byte) (*DriverStatus, error) { Capabilities: defaultCapabilities(), } if err := json.Unmarshal(output, &status); err != nil { - glog.Errorf("Failed to unmarshal output for command: %s, output: %q, error: %s", cmd, string(output), err.Error()) + klog.Errorf("Failed to unmarshal output for command: %s, output: %q, error: %s", cmd, string(output), err.Error()) return nil, err } else if status.Status == StatusNotSupported { - glog.V(5).Infof("%s command is not supported by the driver", cmd) + klog.V(5).Infof("%s command is not supported by the driver", cmd) return nil, errors.New(status.Status) } else if status.Status != StatusSuccess { errMsg := fmt.Sprintf("%s command failed, status: %s, reason: %s", cmd, status.Status, status.Message) - glog.Errorf(errMsg) + klog.Errorf(errMsg) return nil, fmt.Errorf("%s", errMsg) } diff --git a/pkg/volume/flexvolume/expander-defaults.go b/pkg/volume/flexvolume/expander-defaults.go index e578cb32c63..4a33e184e38 100644 --- a/pkg/volume/flexvolume/expander-defaults.go +++ b/pkg/volume/flexvolume/expander-defaults.go @@ -17,8 +17,8 @@ limitations under the License. package flexvolume import ( - "github.com/golang/glog" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/klog" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" ) @@ -32,14 +32,14 @@ func newExpanderDefaults(plugin *flexVolumePlugin) *expanderDefaults { } func (e *expanderDefaults) ExpandVolumeDevice(spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) { - glog.Warning(logPrefix(e.plugin), "using default expand for volume ", spec.Name(), ", to size ", newSize, " from ", oldSize) + klog.Warning(logPrefix(e.plugin), "using default expand for volume ", spec.Name(), ", to size ", newSize, " from ", oldSize) return newSize, nil } // the defaults for ExpandFS return a generic resize indicator that will trigger the operation executor to go ahead with // generic filesystem resize func (e *expanderDefaults) ExpandFS(spec *volume.Spec, devicePath, deviceMountPath string, _, _ resource.Quantity) error { - glog.Warning(logPrefix(e.plugin), "using default filesystem resize for volume ", spec.Name(), ", at ", devicePath) + klog.Warning(logPrefix(e.plugin), "using default filesystem resize for volume ", spec.Name(), ", at ", devicePath) _, err := util.GenericResizeFS(e.plugin.host, e.plugin.GetPluginName(), devicePath, deviceMountPath) return err } diff --git a/pkg/volume/flexvolume/mounter-defaults.go b/pkg/volume/flexvolume/mounter-defaults.go index b3cd9430ff5..93080611897 100644 --- a/pkg/volume/flexvolume/mounter-defaults.go +++ b/pkg/volume/flexvolume/mounter-defaults.go @@ -17,7 +17,7 @@ limitations under the License. package flexvolume import ( - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/volume" ) @@ -27,7 +27,7 @@ type mounterDefaults flexVolumeMounter // SetUpAt is part of the volume.Mounter interface. // This implementation relies on the attacher's device mount path and does a bind mount to dir. func (f *mounterDefaults) SetUpAt(dir string, fsGroup *int64) error { - glog.Warning(logPrefix(f.plugin), "using default SetUpAt to ", dir) + klog.Warning(logPrefix(f.plugin), "using default SetUpAt to ", dir) src, err := f.plugin.getDeviceMountPath(f.spec) if err != nil { @@ -43,7 +43,7 @@ func (f *mounterDefaults) SetUpAt(dir string, fsGroup *int64) error { // Returns the default volume attributes. func (f *mounterDefaults) GetAttributes() volume.Attributes { - glog.V(5).Infof(logPrefix(f.plugin), "using default GetAttributes") + klog.V(5).Infof(logPrefix(f.plugin), "using default GetAttributes") return volume.Attributes{ ReadOnly: f.readOnly, Managed: !f.readOnly, diff --git a/pkg/volume/flexvolume/plugin-defaults.go b/pkg/volume/flexvolume/plugin-defaults.go index 6f090b4e188..9f9e1587b0a 100644 --- a/pkg/volume/flexvolume/plugin-defaults.go +++ b/pkg/volume/flexvolume/plugin-defaults.go @@ -17,7 +17,7 @@ limitations under the License. package flexvolume import ( - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/volume" ) @@ -29,6 +29,6 @@ func logPrefix(plugin *flexVolumePlugin) string { } func (plugin *pluginDefaults) GetVolumeName(spec *volume.Spec) (string, error) { - glog.Warning(logPrefix((*flexVolumePlugin)(plugin)), "using default GetVolumeName for volume ", spec.Name()) + klog.Warning(logPrefix((*flexVolumePlugin)(plugin)), "using default GetVolumeName for volume ", spec.Name()) return spec.Name(), nil } diff --git a/pkg/volume/flexvolume/plugin.go b/pkg/volume/flexvolume/plugin.go index 8859bcb1999..2e2a9f7e532 100644 --- a/pkg/volume/flexvolume/plugin.go +++ b/pkg/volume/flexvolume/plugin.go @@ -23,7 +23,7 @@ import ( "strings" "sync" - "github.com/golang/glog" + "k8s.io/klog" api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -135,7 +135,7 @@ func (plugin *flexVolumePlugin) GetVolumeName(spec *volume.Spec) (string, error) return "", err } - glog.Warning(logPrefix(plugin), "GetVolumeName is not supported yet. Defaulting to PV or volume name: ", name) + klog.Warning(logPrefix(plugin), "GetVolumeName is not supported yet. Defaulting to PV or volume name: ", name) return name, nil } diff --git a/pkg/volume/flexvolume/probe.go b/pkg/volume/flexvolume/probe.go index 2a792c6f409..7a929e5ce99 100644 --- a/pkg/volume/flexvolume/probe.go +++ b/pkg/volume/flexvolume/probe.go @@ -17,7 +17,7 @@ limitations under the License. package flexvolume import ( - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/volume" "k8s.io/utils/exec" @@ -234,7 +234,7 @@ func (prober *flexVolumeProber) addWatchRecursive(filename string) error { addWatch := func(path string, info os.FileInfo, err error) error { if err == nil && info.IsDir() { if err := prober.watcher.AddWatch(path); err != nil { - glog.Errorf("Error recursively adding watch: %v", err) + klog.Errorf("Error recursively adding watch: %v", err) } } return nil @@ -247,10 +247,10 @@ func (prober *flexVolumeProber) addWatchRecursive(filename string) error { func (prober *flexVolumeProber) initWatcher() error { err := prober.watcher.Init(func(event fsnotify.Event) { if err := prober.handleWatchEvent(event); err != nil { - glog.Errorf("Flexvolume prober watch: %s", err) + klog.Errorf("Flexvolume prober watch: %s", err) } }, func(err error) { - glog.Errorf("Received an error from watcher: %s", err) + klog.Errorf("Received an error from watcher: %s", err) }) if err != nil { return fmt.Errorf("Error initializing watcher: %s", err) @@ -268,7 +268,7 @@ func (prober *flexVolumeProber) initWatcher() error { // Creates the plugin directory, if it doesn't already exist. func (prober *flexVolumeProber) createPluginDir() error { if _, err := prober.fs.Stat(prober.pluginDir); os.IsNotExist(err) { - glog.Warningf("Flexvolume plugin directory at %s does not exist. Recreating.", prober.pluginDir) + klog.Warningf("Flexvolume plugin directory at %s does not exist. Recreating.", prober.pluginDir) err := prober.fs.MkdirAll(prober.pluginDir, 0755) if err != nil { return fmt.Errorf("Error (re-)creating driver directory: %s", err) diff --git a/pkg/volume/flexvolume/unmounter-defaults.go b/pkg/volume/flexvolume/unmounter-defaults.go index 67d9facf79d..919a6be890e 100644 --- a/pkg/volume/flexvolume/unmounter-defaults.go +++ b/pkg/volume/flexvolume/unmounter-defaults.go @@ -17,13 +17,13 @@ limitations under the License. package flexvolume import ( - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/volume/util" ) type unmounterDefaults flexVolumeUnmounter func (f *unmounterDefaults) TearDownAt(dir string) error { - glog.Warning(logPrefix(f.plugin), "using default TearDownAt for ", dir) + klog.Warning(logPrefix(f.plugin), "using default TearDownAt for ", dir) return util.UnmountPath(dir, f.mounter) } diff --git a/pkg/volume/flexvolume/unmounter.go b/pkg/volume/flexvolume/unmounter.go index 406c7f84f04..a62636aba40 100644 --- a/pkg/volume/flexvolume/unmounter.go +++ b/pkg/volume/flexvolume/unmounter.go @@ -20,7 +20,7 @@ import ( "fmt" "os" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/utils/exec" @@ -45,7 +45,7 @@ func (f *flexVolumeUnmounter) TearDownAt(dir string) error { pathExists, pathErr := util.PathExists(dir) if !pathExists { - glog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir) + klog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir) return nil } diff --git a/pkg/volume/flexvolume/util.go b/pkg/volume/flexvolume/util.go index 1efdb92112e..4a3019e25e3 100644 --- a/pkg/volume/flexvolume/util.go +++ b/pkg/volume/flexvolume/util.go @@ -21,7 +21,7 @@ import ( "fmt" "os" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" @@ -49,7 +49,7 @@ func addSecretsToOptions(options map[string]string, spec *volume.Spec, namespace } for name, data := range secrets { options[optionKeySecret+"/"+name] = base64.StdEncoding.EncodeToString([]byte(data)) - glog.V(1).Infof("found flex volume secret info: %s", name) + klog.V(1).Infof("found flex volume secret info: %s", name) } return nil @@ -141,7 +141,7 @@ func prepareForMount(mounter mount.Interface, deviceMountPath string) (bool, err func doMount(mounter mount.Interface, devicePath, deviceMountPath, fsType string, options []string) error { err := mounter.Mount(devicePath, deviceMountPath, fsType, options) if err != nil { - glog.Errorf("Failed to mount the volume at %s, device: %s, error: %s", deviceMountPath, devicePath, err.Error()) + klog.Errorf("Failed to mount the volume at %s, device: %s, error: %s", deviceMountPath, devicePath, err.Error()) return err } return nil @@ -150,7 +150,7 @@ func doMount(mounter mount.Interface, devicePath, deviceMountPath, fsType string func isNotMounted(mounter mount.Interface, deviceMountPath string) (bool, error) { notmnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath) if err != nil { - glog.Errorf("Error checking mount point %s, error: %v", deviceMountPath, err) + klog.Errorf("Error checking mount point %s, error: %v", deviceMountPath, err) return false, err } return notmnt, nil diff --git a/pkg/volume/flocker/BUILD b/pkg/volume/flocker/BUILD index d3c41b0fade..0067622c19d 100644 --- a/pkg/volume/flocker/BUILD +++ b/pkg/volume/flocker/BUILD @@ -27,7 +27,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library", "//vendor/github.com/clusterhq/flocker-go:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/flocker/flocker.go b/pkg/volume/flocker/flocker.go index 8a250ace67a..8651ba49083 100644 --- a/pkg/volume/flocker/flocker.go +++ b/pkg/volume/flocker/flocker.go @@ -22,9 +22,9 @@ import ( "path" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/env" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/strings" @@ -310,9 +310,9 @@ func (b *flockerVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { // TODO: handle failed mounts here. notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) - glog.V(4).Infof("flockerVolume set up: %s %v %v, datasetUUID %v readOnly %v", dir, !notMnt, err, datasetUUID, b.readOnly) + klog.V(4).Infof("flockerVolume set up: %s %v %v, datasetUUID %v readOnly %v", dir, !notMnt, err, datasetUUID, b.readOnly) if err != nil && !os.IsNotExist(err) { - glog.Errorf("cannot validate mount point: %s %v", dir, err) + klog.Errorf("cannot validate mount point: %s %v", dir, err) return err } if !notMnt { @@ -320,7 +320,7 @@ func (b *flockerVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } if err := os.MkdirAll(dir, 0750); err != nil { - glog.Errorf("mkdir failed on disk %s (%v)", dir, err) + klog.Errorf("mkdir failed on disk %s (%v)", dir, err) return err } @@ -331,33 +331,33 @@ func (b *flockerVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } globalFlockerPath := makeGlobalFlockerPath(datasetUUID) - glog.V(4).Infof("attempting to mount %s", dir) + klog.V(4).Infof("attempting to mount %s", dir) err = b.mounter.Mount(globalFlockerPath, dir, "", options) if err != nil { notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("isLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("isLikelyNotMountPoint check failed: %v", mntErr) return err } if !notMnt { if mntErr = b.mounter.Unmount(dir); mntErr != nil { - glog.Errorf("failed to unmount: %v", mntErr) + klog.Errorf("failed to unmount: %v", mntErr) return err } notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("isLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("isLikelyNotMountPoint check failed: %v", mntErr) return err } if !notMnt { // This is very odd, we don't expect it. We'll try again next sync loop. - glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) + klog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) return err } } os.Remove(dir) - glog.Errorf("mount of disk %s failed: %v", dir, err) + klog.Errorf("mount of disk %s failed: %v", dir, err) return err } @@ -365,7 +365,7 @@ func (b *flockerVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { volume.SetVolumeOwnership(b, fsGroup) } - glog.V(4).Infof("successfully mounted %s", dir) + klog.V(4).Infof("successfully mounted %s", dir) return nil } diff --git a/pkg/volume/flocker/flocker_util.go b/pkg/volume/flocker/flocker_util.go index 85b8263ab0b..e753da9acd0 100644 --- a/pkg/volume/flocker/flocker_util.go +++ b/pkg/volume/flocker/flocker_util.go @@ -25,7 +25,7 @@ import ( volutil "k8s.io/kubernetes/pkg/volume/util" flockerapi "github.com/clusterhq/flocker-go" - "github.com/golang/glog" + "k8s.io/klog" ) type flockerUtil struct{} @@ -68,7 +68,7 @@ func (util *flockerUtil) CreateVolume(c *flockerVolumeProvisioner) (datasetUUID // select random node node := nodes[rand.Intn(len(nodes))] - glog.V(2).Infof("selected flocker node with UUID '%s' to provision dataset", node.UUID) + klog.V(2).Infof("selected flocker node with UUID '%s' to provision dataset", node.UUID) capacity := c.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] requestBytes := capacity.Value() @@ -92,7 +92,7 @@ func (util *flockerUtil) CreateVolume(c *flockerVolumeProvisioner) (datasetUUID } datasetUUID = datasetState.DatasetID - glog.V(2).Infof("successfully created Flocker dataset with UUID '%s'", datasetUUID) + klog.V(2).Infof("successfully created Flocker dataset with UUID '%s'", datasetUUID) return } diff --git a/pkg/volume/gcepd/BUILD b/pkg/volume/gcepd/BUILD index 513e4bce706..938b5757330 100644 --- a/pkg/volume/gcepd/BUILD +++ b/pkg/volume/gcepd/BUILD @@ -33,7 +33,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) @@ -60,7 +60,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/gcepd/attacher.go b/pkg/volume/gcepd/attacher.go index 6ce2c8928a8..5567e29f28f 100644 --- a/pkg/volume/gcepd/attacher.go +++ b/pkg/volume/gcepd/attacher.go @@ -24,10 +24,10 @@ import ( "strconv" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" @@ -85,17 +85,17 @@ func (attacher *gcePersistentDiskAttacher) Attach(spec *volume.Spec, nodeName ty attached, err := attacher.gceDisks.DiskIsAttached(pdName, nodeName) if err != nil { // Log error and continue with attach - glog.Errorf( + klog.Errorf( "Error checking if PD (%q) is already attached to current node (%q). Will continue and try attach anyway. err=%v", pdName, nodeName, err) } if err == nil && attached { // Volume is already attached to node. - glog.Infof("Attach operation is successful. PD %q is already attached to node %q.", pdName, nodeName) + klog.Infof("Attach operation is successful. PD %q is already attached to node %q.", pdName, nodeName) } else { if err := attacher.gceDisks.AttachDisk(pdName, nodeName, readOnly, isRegionalPD(spec)); err != nil { - glog.Errorf("Error attaching PD %q to node %q: %+v", pdName, nodeName, err) + klog.Errorf("Error attaching PD %q to node %q: %+v", pdName, nodeName, err) return "", err } } @@ -111,7 +111,7 @@ func (attacher *gcePersistentDiskAttacher) VolumesAreAttached(specs []*volume.Sp volumeSource, _, err := getVolumeSource(spec) // If error is occurred, skip this volume and move to the next one if err != nil { - glog.Errorf("Error getting volume (%q) source : %v", spec.Name(), err) + klog.Errorf("Error getting volume (%q) source : %v", spec.Name(), err) continue } pdNameList = append(pdNameList, volumeSource.PDName) @@ -121,7 +121,7 @@ func (attacher *gcePersistentDiskAttacher) VolumesAreAttached(specs []*volume.Sp attachedResult, err := attacher.gceDisks.DisksAreAttached(pdNameList, nodeName) if err != nil { // Log error and continue with attach - glog.Errorf( + klog.Errorf( "Error checking if PDs (%v) are already attached to current node (%q). err=%v", pdNameList, nodeName, err) return volumesAttachedCheck, err @@ -131,7 +131,7 @@ func (attacher *gcePersistentDiskAttacher) VolumesAreAttached(specs []*volume.Sp if !attached { spec := volumePdNameMap[pdName] volumesAttachedCheck[spec] = false - glog.V(2).Infof("VolumesAreAttached: check volume %q (specName: %q) is no longer attached", pdName, spec.Name()) + klog.V(2).Infof("VolumesAreAttached: check volume %q (specName: %q) is no longer attached", pdName, spec.Name()) } } return volumesAttachedCheck, nil @@ -156,7 +156,7 @@ func (attacher *gcePersistentDiskAttacher) WaitForAttach(spec *volume.Spec, devi sdBefore, err := filepath.Glob(diskSDPattern) if err != nil { - glog.Errorf("Error filepath.Glob(\"%s\"): %v\r\n", diskSDPattern, err) + klog.Errorf("Error filepath.Glob(\"%s\"): %v\r\n", diskSDPattern, err) } sdBeforeSet := sets.NewString(sdBefore...) @@ -164,14 +164,14 @@ func (attacher *gcePersistentDiskAttacher) WaitForAttach(spec *volume.Spec, devi for { select { case <-ticker.C: - glog.V(5).Infof("Checking GCE PD %q is attached.", pdName) + klog.V(5).Infof("Checking GCE PD %q is attached.", pdName) path, err := verifyDevicePath(devicePaths, sdBeforeSet, pdName) if err != nil { // Log error, if any, and continue checking periodically. See issue #11321 - glog.Errorf("Error verifying GCE PD (%q) is attached: %v", pdName, err) + klog.Errorf("Error verifying GCE PD (%q) is attached: %v", pdName, err) } else if path != "" { // A device path has successfully been created for the PD - glog.Infof("Successfully found attached GCE PD %q.", pdName) + klog.Infof("Successfully found attached GCE PD %q.", pdName) return path, nil } case <-timer.C: @@ -222,7 +222,7 @@ func (attacher *gcePersistentDiskAttacher) MountDevice(spec *volume.Spec, device os.Remove(deviceMountPath) return err } - glog.V(4).Infof("formatting spec %v devicePath %v deviceMountPath %v fs %v with options %+v", spec.Name(), devicePath, deviceMountPath, volumeSource.FSType, options) + klog.V(4).Infof("formatting spec %v devicePath %v deviceMountPath %v fs %v with options %+v", spec.Name(), devicePath, deviceMountPath, volumeSource.FSType, options) } return nil } @@ -265,19 +265,19 @@ func (detacher *gcePersistentDiskDetacher) Detach(volumeName string, nodeName ty attached, err := detacher.gceDisks.DiskIsAttached(pdName, nodeName) if err != nil { // Log error and continue with detach - glog.Errorf( + klog.Errorf( "Error checking if PD (%q) is already attached to current node (%q). Will continue and try detach anyway. err=%v", pdName, nodeName, err) } if err == nil && !attached { // Volume is not attached to node. Success! - glog.Infof("Detach operation is successful. PD %q was not attached to node %q.", pdName, nodeName) + klog.Infof("Detach operation is successful. PD %q was not attached to node %q.", pdName, nodeName) return nil } if err = detacher.gceDisks.DetachDisk(pdName, nodeName); err != nil { - glog.Errorf("Error detaching PD %q from node %q: %v", pdName, nodeName, err) + klog.Errorf("Error detaching PD %q from node %q: %v", pdName, nodeName, err) return err } diff --git a/pkg/volume/gcepd/attacher_test.go b/pkg/volume/gcepd/attacher_test.go index df5490f2b97..6f9ab41c829 100644 --- a/pkg/volume/gcepd/attacher_test.go +++ b/pkg/volume/gcepd/attacher_test.go @@ -27,8 +27,8 @@ import ( "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "strings" ) @@ -346,7 +346,7 @@ func (testcase *testcase) AttachDisk(diskName string, nodeName types.NodeName, r return errors.New("Unexpected AttachDisk call: wrong regional") } - glog.V(4).Infof("AttachDisk call: %s, %s, %v, returning %v", diskName, nodeName, readOnly, expected.ret) + klog.V(4).Infof("AttachDisk call: %s, %s, %v, returning %v", diskName, nodeName, readOnly, expected.ret) return expected.ret } @@ -371,7 +371,7 @@ func (testcase *testcase) DetachDisk(devicePath string, nodeName types.NodeName) return errors.New("Unexpected DetachDisk call: wrong nodeName") } - glog.V(4).Infof("DetachDisk call: %s, %s, returning %v", devicePath, nodeName, expected.ret) + klog.V(4).Infof("DetachDisk call: %s, %s, returning %v", devicePath, nodeName, expected.ret) return expected.ret } @@ -396,7 +396,7 @@ func (testcase *testcase) DiskIsAttached(diskName string, nodeName types.NodeNam return false, errors.New("Unexpected DiskIsAttached call: wrong nodeName") } - glog.V(4).Infof("DiskIsAttached call: %s, %s, returning %v, %v", diskName, nodeName, expected.isAttached, expected.ret) + klog.V(4).Infof("DiskIsAttached call: %s, %s, returning %v, %v", diskName, nodeName, expected.isAttached, expected.ret) return expected.isAttached, expected.ret } diff --git a/pkg/volume/gcepd/gce_pd.go b/pkg/volume/gcepd/gce_pd.go index 4cbe76d4d06..45572203320 100644 --- a/pkg/volume/gcepd/gce_pd.go +++ b/pkg/volume/gcepd/gce_pd.go @@ -24,12 +24,12 @@ import ( "strconv" "strings" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" "k8s.io/kubernetes/pkg/features" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" @@ -137,13 +137,13 @@ func (plugin *gcePersistentDiskPlugin) GetVolumeLimits() (map[string]int64, erro instances, ok := cloud.Instances() if !ok { - glog.Warning("Failed to get instances from cloud provider") + klog.Warning("Failed to get instances from cloud provider") return volumeLimits, nil } instanceType, err := instances.InstanceType(context.TODO(), plugin.host.GetNodeName()) if err != nil { - glog.Errorf("Failed to get instance type from GCE cloud provider") + klog.Errorf("Failed to get instance type from GCE cloud provider") return volumeLimits, nil } if strings.HasPrefix(instanceType, "n1-") { @@ -367,9 +367,9 @@ func (b *gcePersistentDiskMounter) SetUp(fsGroup *int64) error { func (b *gcePersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error { // TODO: handle failed mounts here. notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) - glog.V(4).Infof("GCE PersistentDisk set up: Dir (%s) PD name (%q) Mounted (%t) Error (%v), ReadOnly (%t)", dir, b.pdName, !notMnt, err, b.readOnly) + klog.V(4).Infof("GCE PersistentDisk set up: Dir (%s) PD name (%q) Mounted (%t) Error (%v), ReadOnly (%t)", dir, b.pdName, !notMnt, err, b.readOnly) if err != nil && !os.IsNotExist(err) { - glog.Errorf("cannot validate mount point: %s %v", dir, err) + klog.Errorf("cannot validate mount point: %s %v", dir, err) return err } if !notMnt { @@ -377,7 +377,7 @@ func (b *gcePersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error { } if err := os.MkdirAll(dir, 0750); err != nil { - glog.Errorf("mkdir failed on disk %s (%v)", dir, err) + klog.Errorf("mkdir failed on disk %s (%v)", dir, err) return err } @@ -388,7 +388,7 @@ func (b *gcePersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error { } globalPDPath := makeGlobalPDName(b.plugin.host, b.pdName) - glog.V(4).Infof("attempting to mount %s", dir) + klog.V(4).Infof("attempting to mount %s", dir) mountOptions := util.JoinMountOptions(b.mountOptions, options) @@ -396,27 +396,27 @@ func (b *gcePersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error { if err != nil { notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !notMnt { if mntErr = b.mounter.Unmount(dir); mntErr != nil { - glog.Errorf("Failed to unmount: %v", mntErr) + klog.Errorf("Failed to unmount: %v", mntErr) return err } notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !notMnt { // This is very odd, we don't expect it. We'll try again next sync loop. - glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) + klog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) return err } } os.Remove(dir) - glog.Errorf("Mount of disk %s failed: %v", dir, err) + klog.Errorf("Mount of disk %s failed: %v", dir, err) return err } @@ -424,7 +424,7 @@ func (b *gcePersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error { volume.SetVolumeOwnership(b, fsGroup) } - glog.V(4).Infof("Successfully mounted %s", dir) + klog.V(4).Infof("Successfully mounted %s", dir) return nil } diff --git a/pkg/volume/gcepd/gce_pd_block.go b/pkg/volume/gcepd/gce_pd_block.go index 417a5c9652a..f4e7e2c7252 100644 --- a/pkg/volume/gcepd/gce_pd_block.go +++ b/pkg/volume/gcepd/gce_pd_block.go @@ -22,9 +22,9 @@ import ( "path/filepath" "strconv" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" kstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" @@ -46,7 +46,7 @@ func (plugin *gcePersistentDiskPlugin) ConstructBlockVolumeSpec(podUID types.UID if err != nil { return nil, err } - glog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err) + klog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err) globalMapPath := filepath.Dir(globalMapPathUUID) if len(globalMapPath) <= 1 { diff --git a/pkg/volume/gcepd/gce_util.go b/pkg/volume/gcepd/gce_util.go index 947cc13fe14..153bab8716d 100644 --- a/pkg/volume/gcepd/gce_util.go +++ b/pkg/volume/gcepd/gce_util.go @@ -24,11 +24,11 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" "k8s.io/kubernetes/pkg/features" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" @@ -79,12 +79,12 @@ func (util *GCEDiskUtil) DeleteVolume(d *gcePersistentDiskDeleter) error { } if err = cloud.DeleteDisk(d.pdName); err != nil { - glog.V(2).Infof("Error deleting GCE PD volume %s: %v", d.pdName, err) + klog.V(2).Infof("Error deleting GCE PD volume %s: %v", d.pdName, err) // GCE cloud provider returns volume.deletedVolumeInUseError when // necessary, no handling needed here. return err } - glog.V(2).Infof("Successfully deleted GCE PD volume %s", d.pdName) + klog.V(2).Infof("Successfully deleted GCE PD volume %s", d.pdName) return nil } @@ -154,7 +154,7 @@ func (util *GCEDiskUtil) CreateVolume(c *gcePersistentDiskProvisioner, node *v1. case replicationTypeRegionalPD: selectedZones, err := volumeutil.SelectZonesForVolume(zonePresent, zonesPresent, configuredZone, configuredZones, activezones, node, allowedTopologies, c.options.PVC.Name, maxRegionalPDZones) if err != nil { - glog.V(2).Infof("Error selecting zones for regional GCE PD volume: %v", err) + klog.V(2).Infof("Error selecting zones for regional GCE PD volume: %v", err) return "", 0, nil, "", err } if err = cloud.CreateRegionalDisk( @@ -163,10 +163,10 @@ func (util *GCEDiskUtil) CreateVolume(c *gcePersistentDiskProvisioner, node *v1. selectedZones, int64(requestGB), *c.options.CloudTags); err != nil { - glog.V(2).Infof("Error creating regional GCE PD volume: %v", err) + klog.V(2).Infof("Error creating regional GCE PD volume: %v", err) return "", 0, nil, "", err } - glog.V(2).Infof("Successfully created Regional GCE PD volume %s", name) + klog.V(2).Infof("Successfully created Regional GCE PD volume %s", name) case replicationTypeNone: selectedZone, err := volumeutil.SelectZoneForVolume(zonePresent, zonesPresent, configuredZone, configuredZones, activezones, node, allowedTopologies, c.options.PVC.Name) @@ -179,10 +179,10 @@ func (util *GCEDiskUtil) CreateVolume(c *gcePersistentDiskProvisioner, node *v1. selectedZone, int64(requestGB), *c.options.CloudTags); err != nil { - glog.V(2).Infof("Error creating single-zone GCE PD volume: %v", err) + klog.V(2).Infof("Error creating single-zone GCE PD volume: %v", err) return "", 0, nil, "", err } - glog.V(2).Infof("Successfully created single-zone GCE PD volume %s", name) + klog.V(2).Infof("Successfully created single-zone GCE PD volume %s", name) default: return "", 0, nil, "", fmt.Errorf("replication-type of '%s' is not supported", replicationType) @@ -191,7 +191,7 @@ func (util *GCEDiskUtil) CreateVolume(c *gcePersistentDiskProvisioner, node *v1. labels, err := cloud.GetAutoLabelsForPD(name, "" /* zone */) if err != nil { // We don't really want to leak the volume here... - glog.Errorf("error getting labels for volume %q: %v", name, err) + klog.Errorf("error getting labels for volume %q: %v", name, err) } return name, int(requestGB), labels, fstype, nil @@ -203,7 +203,7 @@ func verifyDevicePath(devicePaths []string, sdBeforeSet sets.String, diskName st // It's possible udevadm was called on other disks so it should not block this // call. If it did fail on this disk, then the devicePath will either // not exist or be wrong. If it's wrong, then the scsi_id check below will fail. - glog.Errorf("udevadmChangeToNewDrives failed with: %v", err) + klog.Errorf("udevadmChangeToNewDrives failed with: %v", err) } for _, path := range devicePaths { @@ -219,7 +219,7 @@ func verifyDevicePath(devicePaths []string, sdBeforeSet sets.String, diskName st // The device link is not pointing to the correct device // Trigger udev on this device to try to fix the link if udevErr := udevadmChangeToDrive(path); udevErr != nil { - glog.Errorf("udevadmChangeToDrive %q failed with: %v", path, err) + klog.Errorf("udevadmChangeToDrive %q failed with: %v", path, err) } // Return error to retry WaitForAttach and verifyDevicePath @@ -241,7 +241,7 @@ func getScsiSerial(devicePath, diskName string) (string, error) { } if !exists { - glog.V(6).Infof("scsi_id doesn't exist; skipping check for %v", devicePath) + klog.V(6).Infof("scsi_id doesn't exist; skipping check for %v", devicePath) return diskName, nil } @@ -290,7 +290,7 @@ func getCloudProvider(cloudProvider cloudprovider.Interface) (*gcecloud.Cloud, e gceCloudProvider, ok := cloudProvider.(*gcecloud.Cloud) if !ok || gceCloudProvider == nil { // Retry on error. See issue #11321 - glog.Errorf("Failed to get GCE Cloud Provider. plugin.host.GetCloudProvider returned %v instead", cloudProvider) + klog.Errorf("Failed to get GCE Cloud Provider. plugin.host.GetCloudProvider returned %v instead", cloudProvider) time.Sleep(errorSleepDuration) continue } @@ -324,14 +324,14 @@ func udevadmChangeToNewDrives(sdBeforeSet sets.String) error { // drivePath must be the block device path to trigger on, in the format "/dev/sd*", or a symlink to it. // This is workaround for Issue #7972. Once the underlying issue has been resolved, this may be removed. func udevadmChangeToDrive(drivePath string) error { - glog.V(5).Infof("udevadmChangeToDrive: drive=%q", drivePath) + klog.V(5).Infof("udevadmChangeToDrive: drive=%q", drivePath) // Evaluate symlink, if any drive, err := filepath.EvalSymlinks(drivePath) if err != nil { return fmt.Errorf("udevadmChangeToDrive: filepath.EvalSymlinks(%q) failed with %v", drivePath, err) } - glog.V(5).Infof("udevadmChangeToDrive: symlink path is %q", drive) + klog.V(5).Infof("udevadmChangeToDrive: symlink path is %q", drive) // Check to make sure input is "/dev/sd*" if !strings.Contains(drive, diskSDPath) { diff --git a/pkg/volume/glusterfs/BUILD b/pkg/volume/glusterfs/BUILD index bd7d5e0cc28..52e33003414 100644 --- a/pkg/volume/glusterfs/BUILD +++ b/pkg/volume/glusterfs/BUILD @@ -30,9 +30,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/heketi/heketi/client/api/go-client:go_default_library", "//vendor/github.com/heketi/heketi/pkg/glusterfs/api:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/glusterfs/glusterfs.go b/pkg/volume/glusterfs/glusterfs.go index 3223f58c2f1..5e1811263d6 100644 --- a/pkg/volume/glusterfs/glusterfs.go +++ b/pkg/volume/glusterfs/glusterfs.go @@ -26,7 +26,6 @@ import ( dstrings "strings" "sync" - "github.com/golang/glog" gcli "github.com/heketi/heketi/client/api/go-client" gapi "github.com/heketi/heketi/pkg/glusterfs/api" "k8s.io/api/core/v1" @@ -38,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/strings" @@ -67,7 +67,7 @@ var _ volume.Deleter = &glusterfsVolumeDeleter{} const ( glusterfsPluginName = "kubernetes.io/glusterfs" volPrefix = "vol_" - dynamicEpSvcPrefix = "glusterfs-dynamic-" + dynamicEpSvcPrefix = "glusterfs-dynamic" replicaCount = 3 durabilityType = "replicate" secretKeyName = "key" // key name used in secret @@ -75,6 +75,14 @@ const ( defaultGidMin = 2000 defaultGidMax = math.MaxInt32 + // maxCustomEpNamePrefix is the maximum number of chars. + // which can be used as ep/svc name prefix. This number is carved + // out from below formula. + // max length of name of an ep - length of pvc uuid + // where max length of name of an ep is 63 and length of uuid is 37 + + maxCustomEpNamePrefixLen = 26 + // absoluteGidMin/Max are currently the same as the // default values, but they play a different role and // could take a different value. Only thing we need is: @@ -98,15 +106,30 @@ func (plugin *glusterfsPlugin) GetPluginName() string { } func (plugin *glusterfsPlugin) GetVolumeName(spec *volume.Spec) (string, error) { - volumeSource, _, err := getVolumeSource(spec) + var endpointName string + var endpointsNsPtr *string + + volPath, _, err := getVolumeInfo(spec) if err != nil { return "", err } - return fmt.Sprintf( - "%v:%v", - volumeSource.EndpointsName, - volumeSource.Path), nil + if spec.Volume != nil && spec.Volume.Glusterfs != nil { + endpointName = spec.Volume.Glusterfs.EndpointsName + } else if spec.PersistentVolume != nil && + spec.PersistentVolume.Spec.Glusterfs != nil { + endpointName = spec.PersistentVolume.Spec.Glusterfs.EndpointsName + endpointsNsPtr = spec.PersistentVolume.Spec.Glusterfs.EndpointsNamespace + if endpointsNsPtr != nil && *endpointsNsPtr != "" { + return fmt.Sprintf("%v:%v:%v", endpointName, *endpointsNsPtr, volPath), nil + } + return "", fmt.Errorf("invalid endpointsnamespace in provided glusterfs PV spec") + + } else { + return "", fmt.Errorf("unable to fetch required parameters from provided glusterfs spec") + } + + return fmt.Sprintf("%v:%v", endpointName, volPath), nil } func (plugin *glusterfsPlugin) CanSupport(spec *volume.Spec) bool { @@ -139,29 +162,57 @@ func (plugin *glusterfsPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode } func (plugin *glusterfsPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { - source, _, err := getVolumeSource(spec) + epName, epNamespace, err := plugin.getEndpointNameAndNamespace(spec, pod.Namespace) if err != nil { - glog.Errorf("failed to get gluster volumesource: %v", err) return nil, err } - epName := source.EndpointsName - // PVC/POD is in same namespace. - podNs := pod.Namespace + kubeClient := plugin.host.GetKubeClient() if kubeClient == nil { return nil, fmt.Errorf("failed to get kube client to initialize mounter") } - ep, err := kubeClient.CoreV1().Endpoints(podNs).Get(epName, metav1.GetOptions{}) + ep, err := kubeClient.Core().Endpoints(epNamespace).Get(epName, metav1.GetOptions{}) + if err != nil { - glog.Errorf("failed to get endpoint %s: %v", epName, err) + klog.Errorf("failed to get endpoint %s: %v", epName, err) return nil, err } - glog.V(4).Infof("glusterfs pv endpoint %v", ep) + klog.V(4).Infof("glusterfs pv endpoint %v", ep) return plugin.newMounterInternal(spec, ep, pod, plugin.host.GetMounter(plugin.GetPluginName())) } +func (plugin *glusterfsPlugin) getEndpointNameAndNamespace(spec *volume.Spec, defaultNamespace string) (string, string, error) { + if spec.Volume != nil && spec.Volume.Glusterfs != nil { + endpoints := spec.Volume.Glusterfs.EndpointsName + if endpoints == "" { + return "", "", fmt.Errorf("no glusterFS endpoint specified") + } + return endpoints, defaultNamespace, nil + + } else if spec.PersistentVolume != nil && + spec.PersistentVolume.Spec.Glusterfs != nil { + endpoints := spec.PersistentVolume.Spec.Glusterfs.EndpointsName + endpointsNs := defaultNamespace + + overriddenNs := spec.PersistentVolume.Spec.Glusterfs.EndpointsNamespace + if overriddenNs != nil { + if len(*overriddenNs) > 0 { + endpointsNs = *overriddenNs + } else { + return "", "", fmt.Errorf("endpointnamespace field set, but no endpointnamespace specified") + } + } + return endpoints, endpointsNs, nil + } + return "", "", fmt.Errorf("Spec does not reference a GlusterFS volume type") + +} func (plugin *glusterfsPlugin) newMounterInternal(spec *volume.Spec, ep *v1.Endpoints, pod *v1.Pod, mounter mount.Interface) (volume.Mounter, error) { - source, readOnly, _ := getVolumeSource(spec) + volPath, readOnly, err := getVolumeInfo(spec) + if err != nil { + klog.Errorf("failed to get volumesource : %v", err) + return nil, err + } return &glusterfsMounter{ glusterfs: &glusterfs{ volName: spec.Name(), @@ -171,7 +222,7 @@ func (plugin *glusterfsPlugin) newMounterInternal(spec *volume.Spec, ep *v1.Endp MetricsProvider: volume.NewMetricsStatFS(plugin.host.GetPodVolumeDir(pod.UID, strings.EscapeQualifiedNameForDisk(glusterfsPluginName), spec.Name())), }, hosts: ep, - path: source.Path, + path: volPath, readOnly: readOnly, mountOptions: volutil.MountOptionFromSpec(spec), }, nil @@ -247,7 +298,7 @@ func (b *glusterfsMounter) SetUp(fsGroup *int64) error { func (b *glusterfsMounter) SetUpAt(dir string, fsGroup *int64) error { notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) - glog.V(4).Infof("mount setup: %s %v %v", dir, !notMnt, err) + klog.V(4).Infof("mount setup: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { return err } @@ -303,11 +354,11 @@ func (b *glusterfsMounter) setUpAtInternal(dir string) error { switch { case dstrings.HasPrefix(userOpt, "log-file"): - glog.V(4).Infof("log-file mount option has provided") + klog.V(4).Infof("log-file mount option has provided") hasLogFile = true case dstrings.HasPrefix(userOpt, "log-level"): - glog.V(4).Infof("log-level mount option has provided") + klog.V(4).Infof("log-level mount option has provided") hasLogLevel = true } @@ -367,7 +418,7 @@ func (b *glusterfsMounter) setUpAtInternal(dir string) error { ip := addrlist[0] errs = b.mounter.Mount(ip+":"+b.path, dir, "glusterfs", mountOptions) if errs == nil { - glog.Infof("successfully mounted directory %s", dir) + klog.Infof("successfully mounted directory %s", dir) return nil } @@ -384,7 +435,7 @@ func (b *glusterfsMounter) setUpAtInternal(dir string) error { } errs = b.mounter.Mount(ip+":"+b.path, dir, "glusterfs", noAutoMountOptions) if errs == nil { - glog.Infof("successfully mounted %s", dir) + klog.Infof("successfully mounted %s", dir) return nil } } @@ -403,15 +454,16 @@ func (b *glusterfsMounter) setUpAtInternal(dir string) error { } -func getVolumeSource(spec *volume.Spec) (*v1.GlusterfsVolumeSource, bool, error) { +//getVolumeInfo returns 'path' and 'readonly' field values from the provided glusterfs spec. +func getVolumeInfo(spec *volume.Spec) (string, bool, error) { if spec.Volume != nil && spec.Volume.Glusterfs != nil { - return spec.Volume.Glusterfs, spec.Volume.Glusterfs.ReadOnly, nil + return spec.Volume.Glusterfs.Path, spec.Volume.Glusterfs.ReadOnly, nil + } else if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Glusterfs != nil { - return spec.PersistentVolume.Spec.Glusterfs, spec.ReadOnly, nil + return spec.PersistentVolume.Spec.Glusterfs.Path, spec.ReadOnly, nil } - - return nil, false, fmt.Errorf("Spec does not reference a Glusterfs volume type") + return "", false, fmt.Errorf("Spec does not reference a Glusterfs volume type") } func (plugin *glusterfsPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) { @@ -443,6 +495,7 @@ type provisionerConfig struct { volumeOptions []string volumeNamePrefix string thinPoolSnapFactor float32 + customEpNamePrefix string } type glusterfsVolumeProvisioner struct { @@ -520,7 +573,7 @@ func (plugin *glusterfsPlugin) collectGids(className string, gidTable *MinMaxAll } pvList, err := kubeClient.CoreV1().PersistentVolumes().List(metav1.ListOptions{LabelSelector: labels.Everything().String()}) if err != nil { - glog.Error("failed to get existing persistent volumes") + klog.Error("failed to get existing persistent volumes") return err } @@ -534,21 +587,21 @@ func (plugin *glusterfsPlugin) collectGids(className string, gidTable *MinMaxAll gidStr, ok := pv.Annotations[volutil.VolumeGidAnnotationKey] if !ok { - glog.Warningf("no GID found in pv %v", pvName) + klog.Warningf("no GID found in pv %v", pvName) continue } gid, err := convertGid(gidStr) if err != nil { - glog.Errorf("failed to parse gid %s: %v", gidStr, err) + klog.Errorf("failed to parse gid %s: %v", gidStr, err) continue } _, err = gidTable.Allocate(gid) if err == ErrConflict { - glog.Warningf("GID %v found in pv %v was already allocated", gid, pvName) + klog.Warningf("GID %v found in pv %v was already allocated", gid, pvName) } else if err != nil { - glog.Errorf("failed to store gid %v found in pv %v: %v", gid, pvName, err) + klog.Errorf("failed to store gid %v found in pv %v: %v", gid, pvName, err) return err } } @@ -624,7 +677,7 @@ func (d *glusterfsVolumeDeleter) getGid() (int, bool, error) { } func (d *glusterfsVolumeDeleter) Delete() error { - glog.V(2).Infof("delete volume %s", d.glusterfsMounter.path) + klog.V(2).Infof("delete volume %s", d.glusterfsMounter.path) volumeName := d.glusterfsMounter.path volumeID, err := getVolumeID(d.spec, volumeName) @@ -643,11 +696,11 @@ func (d *glusterfsVolumeDeleter) Delete() error { } d.provisionerConfig = *cfg - glog.V(4).Infof("deleting volume %q", volumeID) + klog.V(4).Infof("deleting volume %q", volumeID) gid, exists, err := d.getGid() if err != nil { - glog.Error(err) + klog.Error(err) } else if exists { gidTable, err := d.plugin.getGidTable(class.Name, cfg.gidMin, cfg.gidMax) if err != nil { @@ -662,37 +715,37 @@ func (d *glusterfsVolumeDeleter) Delete() error { cli := gcli.NewClient(d.url, d.user, d.secretValue) if cli == nil { - glog.Errorf("failed to create glusterfs REST client") + klog.Errorf("failed to create glusterfs REST client") return fmt.Errorf("failed to create glusterfs REST client, REST server authentication failed") } err = cli.VolumeDelete(volumeID) if err != nil { - glog.Errorf("failed to delete volume %s: %v", volumeName, err) + klog.Errorf("failed to delete volume %s: %v", volumeName, err) return err } - glog.V(2).Infof("volume %s deleted successfully", volumeName) + klog.V(2).Infof("volume %s deleted successfully", volumeName) //Deleter takes endpoint and namespace from pv spec. pvSpec := d.spec.Spec var dynamicEndpoint, dynamicNamespace string if pvSpec.ClaimRef == nil { - glog.Errorf("ClaimRef is nil") + klog.Errorf("ClaimRef is nil") return fmt.Errorf("ClaimRef is nil") } if pvSpec.ClaimRef.Namespace == "" { - glog.Errorf("namespace is nil") + klog.Errorf("namespace is nil") return fmt.Errorf("namespace is nil") } dynamicNamespace = pvSpec.ClaimRef.Namespace if pvSpec.Glusterfs.EndpointsName != "" { dynamicEndpoint = pvSpec.Glusterfs.EndpointsName } - glog.V(3).Infof("dynamic namespace and endpoint %v/%v", dynamicNamespace, dynamicEndpoint) + klog.V(3).Infof("dynamic namespace and endpoint %v/%v", dynamicNamespace, dynamicEndpoint) err = d.deleteEndpointService(dynamicNamespace, dynamicEndpoint) if err != nil { - glog.Errorf("failed to delete endpoint/service %v/%v: %v", dynamicNamespace, dynamicEndpoint, err) + klog.Errorf("failed to delete endpoint/service %v/%v: %v", dynamicNamespace, dynamicEndpoint, err) } else { - glog.V(1).Infof("endpoint %v/%v is deleted successfully ", dynamicNamespace, dynamicEndpoint) + klog.V(1).Infof("endpoint %v/%v is deleted successfully ", dynamicNamespace, dynamicEndpoint) } return nil } @@ -703,7 +756,7 @@ func (p *glusterfsVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTop } if p.options.PVC.Spec.Selector != nil { - glog.V(4).Infof("not able to parse your claim Selector") + klog.V(4).Infof("not able to parse your claim Selector") return nil, fmt.Errorf("not able to parse your claim Selector") } @@ -711,7 +764,7 @@ func (p *glusterfsVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTop return nil, fmt.Errorf("%s does not support block volume provisioning", p.plugin.GetPluginName()) } - glog.V(4).Infof("Provision VolumeOptions %v", p.options) + klog.V(4).Infof("Provision VolumeOptions %v", p.options) scName := v1helper.GetPersistentVolumeClaimClass(p.options.PVC) cfg, err := parseClassParameters(p.options.Parameters, p.plugin.host.GetKubeClient()) if err != nil { @@ -726,19 +779,19 @@ func (p *glusterfsVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTop gid, _, err := gidTable.AllocateNext() if err != nil { - glog.Errorf("failed to reserve GID from table: %v", err) + klog.Errorf("failed to reserve GID from table: %v", err) return nil, fmt.Errorf("failed to reserve GID from table: %v", err) } - glog.V(2).Infof("Allocated GID %d for PVC %s", gid, p.options.PVC.Name) + klog.V(2).Infof("Allocated GID %d for PVC %s", gid, p.options.PVC.Name) glusterfs, sizeGiB, volID, err := p.CreateVolume(gid) if err != nil { if releaseErr := gidTable.Release(gid); releaseErr != nil { - glog.Errorf("error when releasing GID in storageclass %s: %v", scName, releaseErr) + klog.Errorf("error when releasing GID in storageclass %s: %v", scName, releaseErr) } - glog.Errorf("failed to create volume: %v", err) + klog.Errorf("failed to create volume: %v", err) return nil, fmt.Errorf("failed to create volume: %v", err) } mode := v1.PersistentVolumeFilesystem @@ -769,9 +822,11 @@ func (p *glusterfsVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTop return pv, nil } -func (p *glusterfsVolumeProvisioner) CreateVolume(gid int) (r *v1.GlusterfsVolumeSource, size int, volID string, err error) { +func (p *glusterfsVolumeProvisioner) CreateVolume(gid int) (r *v1.GlusterfsPersistentVolumeSource, size int, volID string, err error) { var clusterIDs []string customVolumeName := "" + epServiceName := "" + capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] // GlusterFS/heketi creates volumes in units of GiB. @@ -779,20 +834,20 @@ func (p *glusterfsVolumeProvisioner) CreateVolume(gid int) (r *v1.GlusterfsVolum if err != nil { return nil, 0, "", err } - glog.V(2).Infof("create volume of size %dGiB", sz) + klog.V(2).Infof("create volume of size %dGiB", sz) if p.url == "" { - glog.Errorf("REST server endpoint is empty") + klog.Errorf("REST server endpoint is empty") return nil, 0, "", fmt.Errorf("failed to create glusterfs REST client, REST URL is empty") } cli := gcli.NewClient(p.url, p.user, p.secretValue) if cli == nil { - glog.Errorf("failed to create glusterfs REST client") + klog.Errorf("failed to create glusterfs REST client") return nil, 0, "", fmt.Errorf("failed to create glusterfs REST client, REST server authentication failed") } if p.provisionerConfig.clusterID != "" { clusterIDs = dstrings.Split(p.clusterID, ",") - glog.V(4).Infof("provided clusterIDs %v", clusterIDs) + klog.V(4).Infof("provided clusterIDs %v", clusterIDs) } if p.provisionerConfig.volumeNamePrefix != "" { @@ -811,33 +866,38 @@ func (p *glusterfsVolumeProvisioner) CreateVolume(gid int) (r *v1.GlusterfsVolum volumeReq := &gapi.VolumeCreateRequest{Size: sz, Name: customVolumeName, Clusters: clusterIDs, Gid: gid64, Durability: p.volumeType, GlusterVolumeOptions: p.volumeOptions, Snapshot: snaps} volume, err := cli.VolumeCreate(volumeReq) if err != nil { - glog.Errorf("failed to create volume: %v", err) + klog.Errorf("failed to create volume: %v", err) return nil, 0, "", fmt.Errorf("failed to create volume: %v", err) } - glog.V(1).Infof("volume with size %d and name %s created", volume.Size, volume.Name) + klog.V(1).Infof("volume with size %d and name %s created", volume.Size, volume.Name) volID = volume.Id dynamicHostIps, err := getClusterNodes(cli, volume.Cluster) if err != nil { - glog.Errorf("failed to get cluster nodes for volume %s: %v", volume, err) + klog.Errorf("failed to get cluster nodes for volume %s: %v", volume, err) return nil, 0, "", fmt.Errorf("failed to get cluster nodes for volume %s: %v", volume, err) } - epServiceName := dynamicEpSvcPrefix + string(p.options.PVC.UID) + if len(p.provisionerConfig.customEpNamePrefix) == 0 { + epServiceName = string(p.options.PVC.UID) + } else { + epServiceName = p.provisionerConfig.customEpNamePrefix + "-" + string(p.options.PVC.UID) + } epNamespace := p.options.PVC.Namespace endpoint, service, err := p.createEndpointService(epNamespace, epServiceName, dynamicHostIps, p.options.PVC.Name) if err != nil { - glog.Errorf("failed to create endpoint/service %v/%v: %v", epNamespace, epServiceName, err) + klog.Errorf("failed to create endpoint/service %v/%v: %v", epNamespace, epServiceName, err) deleteErr := cli.VolumeDelete(volume.Id) if deleteErr != nil { - glog.Errorf("failed to delete volume: %v, manual deletion of the volume required", deleteErr) + klog.Errorf("failed to delete volume: %v, manual deletion of the volume required", deleteErr) } return nil, 0, "", fmt.Errorf("failed to create endpoint/service %v/%v: %v", epNamespace, epServiceName, err) } - glog.V(3).Infof("dynamic endpoint %v and service %v ", endpoint, service) - return &v1.GlusterfsVolumeSource{ - EndpointsName: endpoint.Name, - Path: volume.Name, - ReadOnly: false, + klog.V(3).Infof("dynamic endpoint %v and service %v ", endpoint, service) + return &v1.GlusterfsPersistentVolumeSource{ + EndpointsName: endpoint.Name, + EndpointsNamespace: &epNamespace, + Path: volume.Name, + ReadOnly: false, }, sz, volID, nil } @@ -870,11 +930,11 @@ func (p *glusterfsVolumeProvisioner) createEndpointService(namespace string, epS } _, err = kubeClient.CoreV1().Endpoints(namespace).Create(endpoint) if err != nil && errors.IsAlreadyExists(err) { - glog.V(1).Infof("endpoint %s already exist in namespace %s", endpoint, namespace) + klog.V(1).Infof("endpoint %s already exist in namespace %s", endpoint, namespace) err = nil } if err != nil { - glog.Errorf("failed to create endpoint: %v", err) + klog.Errorf("failed to create endpoint: %v", err) return nil, nil, fmt.Errorf("failed to create endpoint: %v", err) } service = &v1.Service{ @@ -890,11 +950,11 @@ func (p *glusterfsVolumeProvisioner) createEndpointService(namespace string, epS {Protocol: "TCP", Port: 1}}}} _, err = kubeClient.CoreV1().Services(namespace).Create(service) if err != nil && errors.IsAlreadyExists(err) { - glog.V(1).Infof("service %s already exist in namespace %s", service, namespace) + klog.V(1).Infof("service %s already exist in namespace %s", service, namespace) err = nil } if err != nil { - glog.Errorf("failed to create service: %v", err) + klog.Errorf("failed to create service: %v", err) return nil, nil, fmt.Errorf("error creating service: %v", err) } return endpoint, service, nil @@ -907,10 +967,10 @@ func (d *glusterfsVolumeDeleter) deleteEndpointService(namespace string, epServi } err = kubeClient.CoreV1().Services(namespace).Delete(epServiceName, nil) if err != nil { - glog.Errorf("failed to delete service %s/%s: %v", namespace, epServiceName, err) + klog.Errorf("failed to delete service %s/%s: %v", namespace, epServiceName, err) return fmt.Errorf("failed to delete service %s/%s: %v", namespace, epServiceName, err) } - glog.V(1).Infof("service/endpoint: %s/%s deleted successfully", namespace, epServiceName) + klog.V(1).Infof("service/endpoint: %s/%s deleted successfully", namespace, epServiceName) return nil } @@ -918,7 +978,7 @@ func (d *glusterfsVolumeDeleter) deleteEndpointService(namespace string, epServi func parseSecret(namespace, secretName string, kubeClient clientset.Interface) (string, error) { secretMap, err := volutil.GetSecretForPV(namespace, secretName, glusterfsPluginName, kubeClient) if err != nil { - glog.Errorf("failed to get secret: %s/%s: %v", namespace, secretName, err) + klog.Errorf("failed to get secret: %s/%s: %v", namespace, secretName, err) return "", fmt.Errorf("failed to get secret %s/%s: %v", namespace, secretName, err) } if len(secretMap) == 0 { @@ -940,7 +1000,7 @@ func parseSecret(namespace, secretName string, kubeClient clientset.Interface) ( func getClusterNodes(cli *gcli.Client, cluster string) (dynamicHostIps []string, err error) { clusterinfo, err := cli.ClusterInfo(cluster) if err != nil { - glog.Errorf("failed to get cluster details: %v", err) + klog.Errorf("failed to get cluster details: %v", err) return nil, fmt.Errorf("failed to get cluster details: %v", err) } @@ -950,15 +1010,15 @@ func getClusterNodes(cli *gcli.Client, cluster string) (dynamicHostIps []string, for _, node := range clusterinfo.Nodes { nodeInfo, err := cli.NodeInfo(string(node)) if err != nil { - glog.Errorf("failed to get host ipaddress: %v", err) + klog.Errorf("failed to get host ipaddress: %v", err) return nil, fmt.Errorf("failed to get host ipaddress: %v", err) } ipaddr := dstrings.Join(nodeInfo.NodeAddRequest.Hostnames.Storage, "") dynamicHostIps = append(dynamicHostIps, ipaddr) } - glog.V(3).Infof("host list :%v", dynamicHostIps) + klog.V(3).Infof("host list :%v", dynamicHostIps) if len(dynamicHostIps) == 0 { - glog.Errorf("no hosts found: %v", err) + klog.Errorf("no hosts found: %v", err) return nil, fmt.Errorf("no hosts found: %v", err) } return dynamicHostIps, nil @@ -970,6 +1030,7 @@ func parseClassParameters(params map[string]string, kubeClient clientset.Interfa var err error cfg.gidMin = defaultGidMin cfg.gidMax = defaultGidMax + cfg.customEpNamePrefix = dynamicEpSvcPrefix authEnabled := true parseVolumeType := "" @@ -1037,7 +1098,16 @@ func parseClassParameters(params map[string]string, kubeClient clientset.Interfa if len(v) != 0 { parseThinPoolSnapFactor = v } - + case "customepnameprefix": + // If the string has > 'maxCustomEpNamePrefixLen' chars, the final endpoint name will + // exceed the limitation of 63 chars, so fail if prefix is > 'maxCustomEpNamePrefixLen' + // characters. This is only applicable for 'customepnameprefix' string and default ep name + // string will always pass. + if len(v) <= maxCustomEpNamePrefixLen { + cfg.customEpNamePrefix = v + } else { + return nil, fmt.Errorf("'customepnameprefix' value should be < %d characters", maxCustomEpNamePrefixLen) + } default: return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, glusterfsPluginName) } @@ -1162,7 +1232,7 @@ func getVolumeID(pv *v1.PersistentVolume, volumeName string) (string, error) { func (plugin *glusterfsPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) { pvSpec := spec.PersistentVolume.Spec volumeName := pvSpec.Glusterfs.Path - glog.V(2).Infof("Received request to expand volume %s", volumeName) + klog.V(2).Infof("Received request to expand volume %s", volumeName) volumeID, err := getVolumeID(spec.PersistentVolume, volumeName) if err != nil { @@ -1179,12 +1249,12 @@ func (plugin *glusterfsPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize res return oldSize, err } - glog.V(4).Infof("expanding volume: %q", volumeID) + klog.V(4).Infof("expanding volume: %q", volumeID) //Create REST server connection cli := gcli.NewClient(cfg.url, cfg.user, cfg.secretValue) if cli == nil { - glog.Errorf("failed to create glusterfs REST client") + klog.Errorf("failed to create glusterfs REST client") return oldSize, fmt.Errorf("failed to create glusterfs REST client, REST server authentication failed") } @@ -1198,7 +1268,7 @@ func (plugin *glusterfsPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize res //Check the existing volume size currentVolumeInfo, err := cli.VolumeInfo(volumeID) if err != nil { - glog.Errorf("error when fetching details of volume %s: %v", volumeName, err) + klog.Errorf("error when fetching details of volume %s: %v", volumeName, err) return oldSize, err } @@ -1212,11 +1282,11 @@ func (plugin *glusterfsPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize res // Expand the volume volumeInfoRes, err := cli.VolumeExpand(volumeID, volumeExpandReq) if err != nil { - glog.Errorf("failed to expand volume %s: %v", volumeName, err) + klog.Errorf("failed to expand volume %s: %v", volumeName, err) return oldSize, err } - glog.V(2).Infof("volume %s expanded to new size %d successfully", volumeName, volumeInfoRes.Size) + klog.V(2).Infof("volume %s expanded to new size %d successfully", volumeName, volumeInfoRes.Size) newVolumeSize := resource.MustParse(fmt.Sprintf("%dGi", volumeInfoRes.Size)) return newVolumeSize, nil } diff --git a/pkg/volume/glusterfs/glusterfs_test.go b/pkg/volume/glusterfs/glusterfs_test.go index 2e8d0af83dc..0a8a505a0aa 100644 --- a/pkg/volume/glusterfs/glusterfs_test.go +++ b/pkg/volume/glusterfs/glusterfs_test.go @@ -66,7 +66,7 @@ func TestCanSupport(t *testing.T) { if plug.CanSupport(&volume.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{PersistentVolumeSource: v1.PersistentVolumeSource{}}}}) { t.Errorf("Expected false") } - if !plug.CanSupport(&volume.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{PersistentVolumeSource: v1.PersistentVolumeSource{Glusterfs: &v1.GlusterfsVolumeSource{}}}}}) { + if !plug.CanSupport(&volume.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{PersistentVolumeSource: v1.PersistentVolumeSource{Glusterfs: &v1.GlusterfsPersistentVolumeSource{}}}}}) { t.Errorf("Expected true") } } @@ -160,7 +160,7 @@ func TestPluginPersistentVolume(t *testing.T) { }, Spec: v1.PersistentVolumeSpec{ PersistentVolumeSource: v1.PersistentVolumeSource{ - Glusterfs: &v1.GlusterfsVolumeSource{EndpointsName: "ep", Path: "vol", ReadOnly: false}, + Glusterfs: &v1.GlusterfsPersistentVolumeSource{EndpointsName: "ep", Path: "vol", ReadOnly: false}, }, }, } @@ -181,7 +181,7 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { }, Spec: v1.PersistentVolumeSpec{ PersistentVolumeSource: v1.PersistentVolumeSource{ - Glusterfs: &v1.GlusterfsVolumeSource{EndpointsName: "ep", Path: "vol", ReadOnly: false}, + Glusterfs: &v1.GlusterfsPersistentVolumeSource{EndpointsName: "ep", Path: "vol", ReadOnly: false}, }, ClaimRef: &v1.ObjectReference{ Name: "claimA", @@ -261,6 +261,7 @@ func TestParseClassParameters(t *testing.T) { gidMax: 2147483647, volumeType: gapi.VolumeDurabilityInfo{Type: "replicate", Replicate: gapi.ReplicaDurability{Replica: 3}, Disperse: gapi.DisperseDurability{Data: 0, Redundancy: 0}}, thinPoolSnapFactor: float32(1.0), + customEpNamePrefix: "glusterfs-dynamic", }, }, { @@ -283,6 +284,7 @@ func TestParseClassParameters(t *testing.T) { gidMax: 2147483647, volumeType: gapi.VolumeDurabilityInfo{Type: "replicate", Replicate: gapi.ReplicaDurability{Replica: 3}, Disperse: gapi.DisperseDurability{Data: 0, Redundancy: 0}}, thinPoolSnapFactor: float32(1.0), + customEpNamePrefix: "glusterfs-dynamic", }, }, { @@ -299,6 +301,7 @@ func TestParseClassParameters(t *testing.T) { gidMax: 2147483647, volumeType: gapi.VolumeDurabilityInfo{Type: "replicate", Replicate: gapi.ReplicaDurability{Replica: 3}, Disperse: gapi.DisperseDurability{Data: 0, Redundancy: 0}}, thinPoolSnapFactor: float32(1.0), + customEpNamePrefix: "glusterfs-dynamic", }, }, { @@ -437,6 +440,7 @@ func TestParseClassParameters(t *testing.T) { gidMax: 2147483647, volumeType: gapi.VolumeDurabilityInfo{Type: "replicate", Replicate: gapi.ReplicaDurability{Replica: 3}, Disperse: gapi.DisperseDurability{Data: 0, Redundancy: 0}}, thinPoolSnapFactor: float32(1.0), + customEpNamePrefix: "glusterfs-dynamic", }, }, { @@ -454,6 +458,7 @@ func TestParseClassParameters(t *testing.T) { gidMax: 5000, volumeType: gapi.VolumeDurabilityInfo{Type: "replicate", Replicate: gapi.ReplicaDurability{Replica: 3}, Disperse: gapi.DisperseDurability{Data: 0, Redundancy: 0}}, thinPoolSnapFactor: float32(1.0), + customEpNamePrefix: "glusterfs-dynamic", }, }, { @@ -472,6 +477,7 @@ func TestParseClassParameters(t *testing.T) { gidMax: 5000, volumeType: gapi.VolumeDurabilityInfo{Type: "replicate", Replicate: gapi.ReplicaDurability{Replica: 3}, Disperse: gapi.DisperseDurability{Data: 0, Redundancy: 0}}, thinPoolSnapFactor: float32(1.0), + customEpNamePrefix: "glusterfs-dynamic", }, }, @@ -492,6 +498,7 @@ func TestParseClassParameters(t *testing.T) { gidMax: 5000, volumeType: gapi.VolumeDurabilityInfo{Type: "replicate", Replicate: gapi.ReplicaDurability{Replica: 4}, Disperse: gapi.DisperseDurability{Data: 0, Redundancy: 0}}, thinPoolSnapFactor: float32(1.0), + customEpNamePrefix: "glusterfs-dynamic", }, }, @@ -512,6 +519,7 @@ func TestParseClassParameters(t *testing.T) { gidMax: 5000, volumeType: gapi.VolumeDurabilityInfo{Type: "disperse", Replicate: gapi.ReplicaDurability{Replica: 0}, Disperse: gapi.DisperseDurability{Data: 4, Redundancy: 2}}, thinPoolSnapFactor: float32(1.0), + customEpNamePrefix: "glusterfs-dynamic", }, }, { @@ -532,6 +540,7 @@ func TestParseClassParameters(t *testing.T) { gidMax: 5000, volumeType: gapi.VolumeDurabilityInfo{Type: "disperse", Replicate: gapi.ReplicaDurability{Replica: 0}, Disperse: gapi.DisperseDurability{Data: 4, Redundancy: 2}}, thinPoolSnapFactor: float32(50), + customEpNamePrefix: "glusterfs-dynamic", }, }, @@ -555,6 +564,7 @@ func TestParseClassParameters(t *testing.T) { volumeType: gapi.VolumeDurabilityInfo{Type: "disperse", Replicate: gapi.ReplicaDurability{Replica: 0}, Disperse: gapi.DisperseDurability{Data: 4, Redundancy: 2}}, thinPoolSnapFactor: float32(50), volumeNamePrefix: "dept-dev", + customEpNamePrefix: "glusterfs-dynamic", }, }, { @@ -645,6 +655,84 @@ func TestParseClassParameters(t *testing.T) { true, // expect error nil, }, + + { + "enable custom ep/svc name: customEpNamePrefix: myprefix", + map[string]string{ + "resturl": "https://localhost:8080", + "restauthenabled": "false", + "gidMin": "4000", + "gidMax": "5000", + "volumetype": "replicate:4", + "customEpNamePrefix": "myprefix", + }, + &secret, + false, // expect error + &provisionerConfig{ + url: "https://localhost:8080", + gidMin: 4000, + gidMax: 5000, + volumeType: gapi.VolumeDurabilityInfo{Type: "replicate", Replicate: gapi.ReplicaDurability{Replica: 4}, Disperse: gapi.DisperseDurability{Data: 0, Redundancy: 0}}, + thinPoolSnapFactor: float32(1.0), + customEpNamePrefix: "myprefix", + }, + }, + { + "empty custom ep/svc name: customEpNamePrefix:''", + map[string]string{ + "resturl": "https://localhost:8080", + "restauthenabled": "false", + "gidMin": "4000", + "gidMax": "5000", + "volumetype": "replicate:4", + "customEpNamePrefix": "", + }, + &secret, + false, // expect error + &provisionerConfig{ + url: "https://localhost:8080", + gidMin: 4000, + gidMax: 5000, + volumeType: gapi.VolumeDurabilityInfo{Type: "replicate", Replicate: gapi.ReplicaDurability{Replica: 4}, Disperse: gapi.DisperseDurability{Data: 0, Redundancy: 0}}, + thinPoolSnapFactor: float32(1.0), + customEpNamePrefix: "", + }, + }, + { + "custom ep/svc name with 26 chars: customEpNamePrefix:'charstringhastwentysixchar'", + map[string]string{ + "resturl": "https://localhost:8080", + "restauthenabled": "false", + "gidMin": "4000", + "gidMax": "5000", + "volumetype": "replicate:4", + "customEpNamePrefix": "charstringhastwentysixchar", + }, + &secret, + false, // expect error + &provisionerConfig{ + url: "https://localhost:8080", + gidMin: 4000, + gidMax: 5000, + volumeType: gapi.VolumeDurabilityInfo{Type: "replicate", Replicate: gapi.ReplicaDurability{Replica: 4}, Disperse: gapi.DisperseDurability{Data: 0, Redundancy: 0}}, + thinPoolSnapFactor: float32(1.0), + customEpNamePrefix: "charstringhastwentysixchar", + }, + }, + { + "invalid customepnameprefix ( ie >26 chars) parameter", + map[string]string{ + "resturl": "https://localhost:8080", + "restauthenabled": "false", + "gidMin": "4000", + "gidMax": "5000", + "volumetype": "replicate:4", + "customEpNamePrefix": "myprefixhasmorethan26characters", + }, + &secret, + true, // expect error + nil, + }, } for _, test := range tests { diff --git a/pkg/volume/glusterfs/glusterfs_util.go b/pkg/volume/glusterfs/glusterfs_util.go index 83d8a13c769..2b19bf709f8 100644 --- a/pkg/volume/glusterfs/glusterfs_util.go +++ b/pkg/volume/glusterfs/glusterfs_util.go @@ -21,7 +21,7 @@ import ( "fmt" "os" - "github.com/golang/glog" + "k8s.io/klog" ) // readGlusterLog will take the last 2 lines of the log file @@ -34,7 +34,7 @@ func readGlusterLog(path string, podName string) error { var line2 string linecount := 0 - glog.Infof("failure, now attempting to read the gluster log for pod %s", podName) + klog.Infof("failure, now attempting to read the gluster log for pod %s", podName) // Check and make sure path exists if len(path) == 0 { diff --git a/pkg/volume/iscsi/BUILD b/pkg/volume/iscsi/BUILD index 1fb3b31bf4e..1143c892e86 100644 --- a/pkg/volume/iscsi/BUILD +++ b/pkg/volume/iscsi/BUILD @@ -28,7 +28,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/iscsi/attacher.go b/pkg/volume/iscsi/attacher.go index 26e9e52808b..922bc31de6c 100644 --- a/pkg/volume/iscsi/attacher.go +++ b/pkg/volume/iscsi/attacher.go @@ -21,10 +21,10 @@ import ( "os" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/keymutex" "k8s.io/kubernetes/pkg/util/mount" @@ -79,7 +79,7 @@ func (attacher *iscsiAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName func (attacher *iscsiAttacher) WaitForAttach(spec *volume.Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) { mounter, err := volumeSpecToMounter(spec, attacher.host, attacher.targetLocks, pod) if err != nil { - glog.Warningf("failed to get iscsi mounter: %v", err) + klog.Warningf("failed to get iscsi mounter: %v", err) return "", err } return attacher.manager.AttachDisk(*mounter) @@ -89,7 +89,7 @@ func (attacher *iscsiAttacher) GetDeviceMountPath( spec *volume.Spec) (string, error) { mounter, err := volumeSpecToMounter(spec, attacher.host, attacher.targetLocks, nil) if err != nil { - glog.Warningf("failed to get iscsi mounter: %v", err) + klog.Warningf("failed to get iscsi mounter: %v", err) return "", err } if mounter.InitiatorName != "" { @@ -167,12 +167,12 @@ func (detacher *iscsiDetacher) UnmountDevice(deviceMountPath string) error { if err != nil { return fmt.Errorf("iscsi: failed to detach disk: %s\nError: %v", deviceMountPath, err) } - glog.V(4).Infof("iscsi: %q is unmounted, deleting the directory", deviceMountPath) + klog.V(4).Infof("iscsi: %q is unmounted, deleting the directory", deviceMountPath) err = os.RemoveAll(deviceMountPath) if err != nil { return fmt.Errorf("iscsi: failed to delete the directory: %s\nError: %v", deviceMountPath, err) } - glog.V(4).Infof("iscsi: successfully detached disk: %s", deviceMountPath) + klog.V(4).Infof("iscsi: successfully detached disk: %s", deviceMountPath) return nil } @@ -206,7 +206,7 @@ func volumeSpecToMounter(spec *volume.Spec, host volume.VolumeHost, targetLocks if err != nil { return nil, err } - glog.V(5).Infof("iscsi: VolumeSpecToMounter volumeMode %s", volumeMode) + klog.V(5).Infof("iscsi: VolumeSpecToMounter volumeMode %s", volumeMode) return &iscsiDiskMounter{ iscsiDisk: iscsiDisk, fsType: fsType, diff --git a/pkg/volume/iscsi/disk_manager.go b/pkg/volume/iscsi/disk_manager.go index aa1caeaf99c..aff2045d7ff 100644 --- a/pkg/volume/iscsi/disk_manager.go +++ b/pkg/volume/iscsi/disk_manager.go @@ -19,7 +19,7 @@ package iscsi import ( "os" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" @@ -43,7 +43,7 @@ type diskManager interface { func diskSetUp(manager diskManager, b iscsiDiskMounter, volPath string, mounter mount.Interface, fsGroup *int64) error { notMnt, err := mounter.IsLikelyNotMountPoint(volPath) if err != nil && !os.IsNotExist(err) { - glog.Errorf("cannot validate mountpoint: %s", volPath) + klog.Errorf("cannot validate mountpoint: %s", volPath) return err } if !notMnt { @@ -51,7 +51,7 @@ func diskSetUp(manager diskManager, b iscsiDiskMounter, volPath string, mounter } if err := os.MkdirAll(volPath, 0750); err != nil { - glog.Errorf("failed to mkdir:%s", volPath) + klog.Errorf("failed to mkdir:%s", volPath) return err } // Perform a bind mount to the full path to allow duplicate mounts of the same disk. @@ -67,25 +67,25 @@ func diskSetUp(manager diskManager, b iscsiDiskMounter, volPath string, mounter mountOptions := util.JoinMountOptions(b.mountOptions, options) err = mounter.Mount(globalPDPath, volPath, "", mountOptions) if err != nil { - glog.Errorf("Failed to bind mount: source:%s, target:%s, err:%v", globalPDPath, volPath, err) + klog.Errorf("Failed to bind mount: source:%s, target:%s, err:%v", globalPDPath, volPath, err) noMnt, mntErr := b.mounter.IsLikelyNotMountPoint(volPath) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !noMnt { if mntErr = b.mounter.Unmount(volPath); mntErr != nil { - glog.Errorf("Failed to unmount: %v", mntErr) + klog.Errorf("Failed to unmount: %v", mntErr) return err } noMnt, mntErr = b.mounter.IsLikelyNotMountPoint(volPath) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !noMnt { // will most likely retry on next sync loop. - glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", volPath) + klog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", volPath) return err } } diff --git a/pkg/volume/iscsi/iscsi.go b/pkg/volume/iscsi/iscsi.go index 96dba1f03dc..54ad75febfe 100644 --- a/pkg/volume/iscsi/iscsi.go +++ b/pkg/volume/iscsi/iscsi.go @@ -23,10 +23,10 @@ import ( "strconv" "strings" - "github.com/golang/glog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/keymutex" "k8s.io/kubernetes/pkg/util/mount" utilstrings "k8s.io/kubernetes/pkg/util/strings" @@ -252,7 +252,7 @@ func (plugin *iscsiPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName if err != nil { return nil, err } - glog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err) + klog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err) // Retrieve volume information from globalMapPathUUID // globalMapPathUUID example: // plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid} @@ -287,7 +287,7 @@ func (iscsi *iscsiDisk) GetPath() string { func (iscsi *iscsiDisk) iscsiGlobalMapPath(spec *volume.Spec) (string, error) { mounter, err := volumeSpecToMounter(spec, iscsi.plugin.host, iscsi.plugin.targetLocks, nil /* pod */) if err != nil { - glog.Warningf("failed to get iscsi mounter: %v", err) + klog.Warningf("failed to get iscsi mounter: %v", err) return "", err } return iscsi.manager.MakeGlobalVDPDName(*mounter.iscsiDisk), nil @@ -334,7 +334,7 @@ func (b *iscsiDiskMounter) SetUpAt(dir string, fsGroup *int64) error { // diskSetUp checks mountpoints and prevent repeated calls err := diskSetUp(b.manager, *b, dir, b.mounter, fsGroup) if err != nil { - glog.Errorf("iscsi: failed to setup") + klog.Errorf("iscsi: failed to setup") } return err } @@ -392,12 +392,12 @@ func (c *iscsiDiskUnmapper) TearDownDevice(mapPath, _ string) error { if err != nil { return fmt.Errorf("iscsi: failed to detach disk: %s\nError: %v", mapPath, err) } - glog.V(4).Infof("iscsi: %q is unmounted, deleting the directory", mapPath) + klog.V(4).Infof("iscsi: %q is unmounted, deleting the directory", mapPath) err = os.RemoveAll(mapPath) if err != nil { return fmt.Errorf("iscsi: failed to delete the directory: %s\nError: %v", mapPath, err) } - glog.V(4).Infof("iscsi: successfully detached disk: %s", mapPath) + klog.V(4).Infof("iscsi: successfully detached disk: %s", mapPath) return nil } @@ -582,7 +582,7 @@ func createSecretMap(spec *volume.Spec, plugin *iscsiPlugin, namespace string) ( } secret = make(map[string]string) for name, data := range secretObj.Data { - glog.V(4).Infof("retrieving CHAP secret name: %s", name) + klog.V(4).Infof("retrieving CHAP secret name: %s", name) secret[name] = string(data) } } @@ -649,7 +649,7 @@ func getVolumeSpecFromGlobalMapPath(volumeName, globalMapPath string) (*volume.S ISCSIInterface: iface, }, ) - glog.V(5).Infof("ConstructBlockVolumeSpec: TargetPortal: %v, IQN: %v, Lun: %v, ISCSIInterface: %v", + klog.V(5).Infof("ConstructBlockVolumeSpec: TargetPortal: %v, IQN: %v, Lun: %v, ISCSIInterface: %v", iscsiPV.Spec.PersistentVolumeSource.ISCSI.TargetPortal, iscsiPV.Spec.PersistentVolumeSource.ISCSI.IQN, iscsiPV.Spec.PersistentVolumeSource.ISCSI.Lun, diff --git a/pkg/volume/iscsi/iscsi_util.go b/pkg/volume/iscsi/iscsi_util.go index 71cace74e20..e5e04c518fd 100644 --- a/pkg/volume/iscsi/iscsi_util.go +++ b/pkg/volume/iscsi/iscsi_util.go @@ -27,9 +27,9 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" @@ -253,7 +253,7 @@ func scanOneLun(hostNumber int, lunNumber int) error { return fmt.Errorf("No data written to file: %s", filename) } - glog.V(3).Infof("Scanned SCSI host %d LUN %d", hostNumber, lunNumber) + klog.V(3).Infof("Scanned SCSI host %d LUN %d", hostNumber, lunNumber) return nil } @@ -291,7 +291,7 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) { out, err := b.exec.Run("iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "show") if err != nil { - glog.Errorf("iscsi: could not read iface %s error: %s", b.Iface, string(out)) + klog.Errorf("iscsi: could not read iface %s error: %s", b.Iface, string(out)) return "", err } @@ -305,7 +305,7 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) { newIface := bkpPortal[0] + ":" + b.VolName err = cloneIface(b, newIface) if err != nil { - glog.Errorf("iscsi: failed to clone iface: %s error: %v", b.Iface, err) + klog.Errorf("iscsi: failed to clone iface: %s error: %v", b.Iface, err) return "", err } // update iface name @@ -323,18 +323,18 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) { if err != nil { return "", err } - glog.V(4).Infof("AttachDisk portal->host map for %s is %v", b.Iqn, portalHostMap) + klog.V(4).Infof("AttachDisk portal->host map for %s is %v", b.Iqn, portalHostMap) for i := 1; i <= maxAttachAttempts; i++ { for _, tp := range bkpPortal { if _, found := devicePaths[tp]; found { - glog.V(4).Infof("Device for portal %q already known", tp) + klog.V(4).Infof("Device for portal %q already known", tp) continue } hostNumber, loggedIn := portalHostMap[tp] if !loggedIn { - glog.V(4).Infof("Could not get SCSI host number for portal %s, will attempt login", tp) + klog.V(4).Infof("Could not get SCSI host number for portal %s, will attempt login", tp) // build discoverydb and discover iscsi target b.exec.Run("iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "new") @@ -374,7 +374,7 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) { out, err = b.exec.Run("iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-o", "update", "-n", "node.startup", "-v", "manual") if err != nil { // don't fail if we can't set startup mode, but log warning so there is a clue - glog.Warningf("Warning: Failed to set iSCSI login mode to manual. Error: %v", err) + klog.Warningf("Warning: Failed to set iSCSI login mode to manual. Error: %v", err) } // Rebuild the host map after logging in @@ -382,16 +382,16 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) { if err != nil { return "", err } - glog.V(6).Infof("AttachDisk portal->host map for %s is %v", b.Iqn, portalHostMap) + klog.V(6).Infof("AttachDisk portal->host map for %s is %v", b.Iqn, portalHostMap) hostNumber, loggedIn = portalHostMap[tp] if !loggedIn { - glog.Warningf("Could not get SCSI host number for portal %s after logging in", tp) + klog.Warningf("Could not get SCSI host number for portal %s after logging in", tp) continue } } - glog.V(5).Infof("AttachDisk: scanning SCSI host %d LUN %s", hostNumber, b.Lun) + klog.V(5).Infof("AttachDisk: scanning SCSI host %d LUN %s", hostNumber, b.Lun) lunNumber, err := strconv.Atoi(b.Lun) if err != nil { return "", fmt.Errorf("AttachDisk: lun is not a number: %s\nError: %v", b.Lun, err) @@ -404,7 +404,7 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) { } if iscsiTransport == "" { - glog.Errorf("iscsi: could not find transport name in iface %s", b.Iface) + klog.Errorf("iscsi: could not find transport name in iface %s", b.Iface) return "", fmt.Errorf("Could not parse iface file for %s", b.Iface) } if iscsiTransport == "tcp" { @@ -414,7 +414,7 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) { } if exist := waitForPathToExist(&devicePath, multipathDeviceTimeout, iscsiTransport); !exist { - glog.Errorf("Could not attach disk: Timeout after 10s") + klog.Errorf("Could not attach disk: Timeout after 10s") // update last error lastErr = fmt.Errorf("Could not attach disk: Timeout after 10s") continue @@ -422,28 +422,28 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) { devicePaths[tp] = devicePath } } - glog.V(4).Infof("iscsi: tried all devices for %q %d times, %d paths found", b.Iqn, i, len(devicePaths)) + klog.V(4).Infof("iscsi: tried all devices for %q %d times, %d paths found", b.Iqn, i, len(devicePaths)) if len(devicePaths) == 0 { // No path attached, report error and stop trying. kubelet will try again in a short while // delete cloned iface b.exec.Run("iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "delete") - glog.Errorf("iscsi: failed to get any path for iscsi disk, last err seen:\n%v", lastErr) + klog.Errorf("iscsi: failed to get any path for iscsi disk, last err seen:\n%v", lastErr) return "", fmt.Errorf("failed to get any path for iscsi disk, last err seen:\n%v", lastErr) } if len(devicePaths) == len(bkpPortal) { // We have all paths - glog.V(4).Infof("iscsi: all devices for %q found", b.Iqn) + klog.V(4).Infof("iscsi: all devices for %q found", b.Iqn) break } if len(devicePaths) >= minMultipathCount && i >= minAttachAttempts { // We have at least two paths for multipath and we tried the other paths long enough - glog.V(4).Infof("%d devices found for %q", len(devicePaths), b.Iqn) + klog.V(4).Infof("%d devices found for %q", len(devicePaths), b.Iqn) break } } if lastErr != nil { - glog.Errorf("iscsi: last error occurred during iscsi init:\n%v", lastErr) + klog.Errorf("iscsi: last error occurred during iscsi init:\n%v", lastErr) } devicePathList := []string{} @@ -466,7 +466,7 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) { devicePath = devicePathList[0] } - glog.V(5).Infof("iscsi: AttachDisk devicePath: %s", devicePath) + klog.V(5).Infof("iscsi: AttachDisk devicePath: %s", devicePath) // run global mount path related operations based on volumeMode return globalPDPathOperation(b)(b, devicePath, util) } @@ -479,14 +479,14 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) { func globalPDPathOperation(b iscsiDiskMounter) func(iscsiDiskMounter, string, *ISCSIUtil) (string, error) { // TODO: remove feature gate check after no longer needed if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { - glog.V(5).Infof("iscsi: AttachDisk volumeMode: %s", b.volumeMode) + klog.V(5).Infof("iscsi: AttachDisk volumeMode: %s", b.volumeMode) if b.volumeMode == v1.PersistentVolumeBlock { // If the volumeMode is 'Block', plugin don't need to format the volume. return func(b iscsiDiskMounter, devicePath string, util *ISCSIUtil) (string, error) { globalPDPath := b.manager.MakeGlobalVDPDName(*b.iscsiDisk) // Create dir like /var/lib/kubelet/plugins/kubernetes.io/iscsi/volumeDevices/{ifaceName}/{portal-some_iqn-lun-lun_id} if err := os.MkdirAll(globalPDPath, 0750); err != nil { - glog.Errorf("iscsi: failed to mkdir %s, error", globalPDPath) + klog.Errorf("iscsi: failed to mkdir %s, error", globalPDPath) return "", err } // Persist iscsi disk config to json file for DetachDisk path @@ -506,12 +506,12 @@ func globalPDPathOperation(b iscsiDiskMounter) func(iscsiDiskMounter, string, *I } // Return confirmed devicePath to caller if !notMnt { - glog.Infof("iscsi: %s already mounted", globalPDPath) + klog.Infof("iscsi: %s already mounted", globalPDPath) return devicePath, nil } // Create dir like /var/lib/kubelet/plugins/kubernetes.io/iscsi/{ifaceName}/{portal-some_iqn-lun-lun_id} if err := os.MkdirAll(globalPDPath, 0750); err != nil { - glog.Errorf("iscsi: failed to mkdir %s, error", globalPDPath) + klog.Errorf("iscsi: failed to mkdir %s, error", globalPDPath) return "", err } // Persist iscsi disk config to json file for DetachDisk path @@ -519,7 +519,7 @@ func globalPDPathOperation(b iscsiDiskMounter) func(iscsiDiskMounter, string, *I err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil) if err != nil { - glog.Errorf("iscsi: failed to mount iscsi volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err) + klog.Errorf("iscsi: failed to mount iscsi volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err) } return devicePath, nil @@ -541,7 +541,7 @@ func deleteDevice(deviceName string) error { } else if 0 == written { return fmt.Errorf("No data written to file: %s", filename) } - glog.V(4).Infof("Deleted block device: %s", deviceName) + klog.V(4).Infof("Deleted block device: %s", deviceName) return nil } @@ -550,13 +550,13 @@ func deleteDevice(deviceName string) error { func deleteDevices(c iscsiDiskUnmounter) error { lunNumber, err := strconv.Atoi(c.iscsiDisk.Lun) if err != nil { - glog.Errorf("iscsi delete devices: lun is not a number: %s\nError: %v", c.iscsiDisk.Lun, err) + klog.Errorf("iscsi delete devices: lun is not a number: %s\nError: %v", c.iscsiDisk.Lun, err) return err } // Enumerate the devices so we can delete them deviceNames, err := c.deviceUtil.FindDevicesForISCSILun(c.iscsiDisk.Iqn, lunNumber) if err != nil { - glog.Errorf("iscsi delete devices: could not get devices associated with LUN %d on target %s\nError: %v", + klog.Errorf("iscsi delete devices: could not get devices associated with LUN %d on target %s\nError: %v", lunNumber, c.iscsiDisk.Iqn, err) return err } @@ -573,15 +573,15 @@ func deleteDevices(c iscsiDiskUnmounter) error { for mpathDevice := range mpathDevices { _, err = c.exec.Run("multipath", "-f", mpathDevice) if err != nil { - glog.Warningf("Warning: Failed to flush multipath device map: %s\nError: %v", mpathDevice, err) + klog.Warningf("Warning: Failed to flush multipath device map: %s\nError: %v", mpathDevice, err) // Fall through -- keep deleting the block devices } - glog.V(4).Infof("Flushed multipath device: %s", mpathDevice) + klog.V(4).Infof("Flushed multipath device: %s", mpathDevice) } for _, deviceName := range deviceNames { err = deleteDevice(deviceName) if err != nil { - glog.Warningf("Warning: Failed to delete block device: %s\nError: %v", deviceName, err) + klog.Warningf("Warning: Failed to delete block device: %s\nError: %v", deviceName, err) // Fall through -- keep deleting other block devices } } @@ -593,7 +593,7 @@ func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error { if pathExists, pathErr := volumeutil.PathExists(mntPath); pathErr != nil { return fmt.Errorf("Error checking if path exists: %v", pathErr) } else if !pathExists { - glog.Warningf("Warning: Unmount skipped because path does not exist: %v", mntPath) + klog.Warningf("Warning: Unmount skipped because path does not exist: %v", mntPath) return nil } @@ -603,7 +603,7 @@ func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error { } if !notMnt { if err := c.mounter.Unmount(mntPath); err != nil { - glog.Errorf("iscsi detach disk: failed to unmount: %s\nError: %v", mntPath, err) + klog.Errorf("iscsi detach disk: failed to unmount: %s\nError: %v", mntPath, err) return err } } @@ -639,7 +639,7 @@ func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error { // Delete all the scsi devices and any multipath devices after unmounting if err = deleteDevices(c); err != nil { - glog.Warningf("iscsi detach disk: failed to delete devices\nError: %v", err) + klog.Warningf("iscsi detach disk: failed to delete devices\nError: %v", err) // Fall through -- even if deleting fails, a logout may fix problems } @@ -670,7 +670,7 @@ func (util *ISCSIUtil) DetachBlockISCSIDisk(c iscsiDiskUnmapper, mapPath string) if pathExists, pathErr := volumeutil.PathExists(mapPath); pathErr != nil { return fmt.Errorf("Error checking if path exists: %v", pathErr) } else if !pathExists { - glog.Warningf("Warning: Unmap skipped because path does not exist: %v", mapPath) + klog.Warningf("Warning: Unmap skipped because path does not exist: %v", mapPath) return nil } // If we arrive here, device is no longer used, see if need to logout the target @@ -711,7 +711,7 @@ func (util *ISCSIUtil) DetachBlockISCSIDisk(c iscsiDiskUnmapper, mapPath string) } devicePath := getDevByPath(portals[0], iqn, lun) - glog.V(5).Infof("iscsi: devicePath: %s", devicePath) + klog.V(5).Infof("iscsi: devicePath: %s", devicePath) if _, err = os.Stat(devicePath); err != nil { return fmt.Errorf("failed to validate devicePath: %s", devicePath) } @@ -735,16 +735,16 @@ func (util *ISCSIUtil) detachISCSIDisk(exec mount.Exec, portals []string, iqn, i logoutArgs = append(logoutArgs, []string{"-I", iface}...) deleteArgs = append(deleteArgs, []string{"-I", iface}...) } - glog.Infof("iscsi: log out target %s iqn %s iface %s", portal, iqn, iface) + klog.Infof("iscsi: log out target %s iqn %s iface %s", portal, iqn, iface) out, err := exec.Run("iscsiadm", logoutArgs...) if err != nil { - glog.Errorf("iscsi: failed to detach disk Error: %s", string(out)) + klog.Errorf("iscsi: failed to detach disk Error: %s", string(out)) } // Delete the node record - glog.Infof("iscsi: delete node record target %s iqn %s", portal, iqn) + klog.Infof("iscsi: delete node record target %s iqn %s", portal, iqn) out, err = exec.Run("iscsiadm", deleteArgs...) if err != nil { - glog.Errorf("iscsi: failed to delete node record Error: %s", string(out)) + klog.Errorf("iscsi: failed to delete node record Error: %s", string(out)) } } // Delete the iface after all sessions have logged out @@ -753,7 +753,7 @@ func (util *ISCSIUtil) detachISCSIDisk(exec mount.Exec, portals []string, iqn, i deleteArgs := []string{"-m", "iface", "-I", iface, "-o", "delete"} out, err := exec.Run("iscsiadm", deleteArgs...) if err != nil { - glog.Errorf("iscsi: failed to delete iface Error: %s", string(out)) + klog.Errorf("iscsi: failed to delete iface Error: %s", string(out)) } } diff --git a/pkg/volume/local/BUILD b/pkg/volume/local/BUILD index 8fe054c70bb..a17a11a58bc 100644 --- a/pkg/volume/local/BUILD +++ b/pkg/volume/local/BUILD @@ -20,7 +20,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/local/local.go b/pkg/volume/local/local.go index 3b0b59f563e..45249313424 100644 --- a/pkg/volume/local/local.go +++ b/pkg/volume/local/local.go @@ -23,7 +23,7 @@ import ( "runtime" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -257,7 +257,7 @@ func (plugin *localVolumePlugin) NewDeviceMounter() (volume.DeviceMounter, error } func (dm *deviceMounter) mountLocalBlockDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error { - glog.V(4).Infof("local: mounting device %s to %s", devicePath, deviceMountPath) + klog.V(4).Infof("local: mounting device %s to %s", devicePath, deviceMountPath) notMnt, err := dm.mounter.IsLikelyNotMountPoint(deviceMountPath) if err != nil { if os.IsNotExist(err) { @@ -291,7 +291,7 @@ func (dm *deviceMounter) mountLocalBlockDevice(spec *volume.Spec, devicePath str os.Remove(deviceMountPath) return fmt.Errorf("local: failed to mount device %s at %s (fstype: %s), error %v", devicePath, deviceMountPath, fstype, err) } - glog.V(3).Infof("local: successfully mount device %s at %s (fstype: %s)", devicePath, deviceMountPath, fstype) + klog.V(3).Infof("local: successfully mount device %s at %s (fstype: %s)", devicePath, deviceMountPath, fstype) return nil } @@ -322,10 +322,9 @@ func getVolumeSourceFSType(spec *volume.Spec) (string, error) { spec.PersistentVolume.Spec.Local != nil { if spec.PersistentVolume.Spec.Local.FSType != nil { return *spec.PersistentVolume.Spec.Local.FSType, nil - } else { - // if the FSType is not set in local PV spec, setting it to default ("ext4") - return defaultFSType, nil } + // if the FSType is not set in local PV spec, setting it to default ("ext4") + return defaultFSType, nil } return "", fmt.Errorf("spec does not reference a Local volume type") @@ -435,9 +434,9 @@ func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } notMnt, err := m.mounter.IsNotMountPoint(dir) - glog.V(4).Infof("LocalVolume mount setup: PodDir(%s) VolDir(%s) Mounted(%t) Error(%v), ReadOnly(%t)", dir, m.globalPath, !notMnt, err, m.readOnly) + klog.V(4).Infof("LocalVolume mount setup: PodDir(%s) VolDir(%s) Mounted(%t) Error(%v), ReadOnly(%t)", dir, m.globalPath, !notMnt, err, m.readOnly) if err != nil && !os.IsNotExist(err) { - glog.Errorf("cannot validate mount point: %s %v", dir, err) + klog.Errorf("cannot validate mount point: %s %v", dir, err) return err } @@ -447,7 +446,7 @@ func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { refs, err := m.mounter.GetMountRefs(m.globalPath) if fsGroup != nil { if err != nil { - glog.Errorf("cannot collect mounting information: %s %v", m.globalPath, err) + klog.Errorf("cannot collect mounting information: %s %v", m.globalPath, err) return err } @@ -469,7 +468,7 @@ func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if runtime.GOOS != "windows" { // skip below MkdirAll for windows since the "bind mount" logic is implemented differently in mount_wiondows.go if err := os.MkdirAll(dir, 0750); err != nil { - glog.Errorf("mkdir failed on disk %s (%v)", dir, err) + klog.Errorf("mkdir failed on disk %s (%v)", dir, err) return err } } @@ -480,29 +479,29 @@ func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } mountOptions := util.JoinMountOptions(options, m.mountOptions) - glog.V(4).Infof("attempting to mount %s", dir) + klog.V(4).Infof("attempting to mount %s", dir) globalPath := util.MakeAbsolutePath(runtime.GOOS, m.globalPath) err = m.mounter.Mount(globalPath, dir, "", mountOptions) if err != nil { - glog.Errorf("Mount of volume %s failed: %v", dir, err) + klog.Errorf("Mount of volume %s failed: %v", dir, err) notMnt, mntErr := m.mounter.IsNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsNotMountPoint check failed: %v", mntErr) return err } if !notMnt { if mntErr = m.mounter.Unmount(dir); mntErr != nil { - glog.Errorf("Failed to unmount: %v", mntErr) + klog.Errorf("Failed to unmount: %v", mntErr) return err } notMnt, mntErr = m.mounter.IsNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsNotMountPoint check failed: %v", mntErr) return err } if !notMnt { // This is very odd, we don't expect it. We'll try again next sync loop. - glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) + klog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) return err } } @@ -542,7 +541,7 @@ func (u *localVolumeUnmounter) TearDown() error { // TearDownAt unmounts the bind mount func (u *localVolumeUnmounter) TearDownAt(dir string) error { - glog.V(4).Infof("Unmounting volume %q at path %q\n", u.volName, dir) + klog.V(4).Infof("Unmounting volume %q at path %q\n", u.volName, dir) return util.UnmountMountPoint(dir, u.mounter, true) /* extensiveMountPointCheck = true */ } @@ -557,7 +556,7 @@ var _ volume.BlockVolumeMapper = &localVolumeMapper{} // SetUpDevice provides physical device path for the local PV. func (m *localVolumeMapper) SetUpDevice() (string, error) { globalPath := util.MakeAbsolutePath(runtime.GOOS, m.globalPath) - glog.V(4).Infof("SetupDevice returning path %s", globalPath) + klog.V(4).Infof("SetupDevice returning path %s", globalPath) return globalPath, nil } @@ -573,8 +572,8 @@ type localVolumeUnmapper struct { var _ volume.BlockVolumeUnmapper = &localVolumeUnmapper{} // TearDownDevice will undo SetUpDevice procedure. In local PV, all of this already handled by operation_generator. -func (u *localVolumeUnmapper) TearDownDevice(mapPath, devicePath string) error { - glog.V(4).Infof("local: TearDownDevice completed for: %s", mapPath) +func (u *localVolumeUnmapper) TearDownDevice(mapPath, _ string) error { + klog.V(4).Infof("local: TearDownDevice completed for: %s", mapPath) return nil } diff --git a/pkg/volume/local/local_test.go b/pkg/volume/local/local_test.go index c28c92dde02..b59152ca951 100644 --- a/pkg/volume/local/local_test.go +++ b/pkg/volume/local/local_test.go @@ -41,7 +41,6 @@ const ( testMountPath = "pods/poduid/volumes/kubernetes.io~local-volume/pvA" testGlobalPath = "plugins/kubernetes.io~local-volume/volumeDevices/pvA" testPodPath = "pods/poduid/volumeDevices/kubernetes.io~local-volume" - testNodeName = "fakeNodeName" testBlockFormattingToFSGlobalPath = "plugins/kubernetes.io/local-volume/mounts/pvA" ) diff --git a/pkg/volume/nfs/BUILD b/pkg/volume/nfs/BUILD index 81fae48e2a2..28885d7fb4f 100644 --- a/pkg/volume/nfs/BUILD +++ b/pkg/volume/nfs/BUILD @@ -22,7 +22,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/nfs/nfs.go b/pkg/volume/nfs/nfs.go index 588687cdebb..92384b3db01 100644 --- a/pkg/volume/nfs/nfs.go +++ b/pkg/volume/nfs/nfs.go @@ -21,10 +21,10 @@ import ( "os" "runtime" - "github.com/golang/glog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" @@ -235,7 +235,7 @@ func (b *nfsMounter) SetUp(fsGroup *int64) error { func (b *nfsMounter) SetUpAt(dir string, fsGroup *int64) error { notMnt, err := b.mounter.IsNotMountPoint(dir) - glog.V(4).Infof("NFS mount set up: %s %v %v", dir, !notMnt, err) + klog.V(4).Infof("NFS mount set up: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { return err } @@ -255,22 +255,22 @@ func (b *nfsMounter) SetUpAt(dir string, fsGroup *int64) error { if err != nil { notMnt, mntErr := b.mounter.IsNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsNotMountPoint check failed: %v", mntErr) return err } if !notMnt { if mntErr = b.mounter.Unmount(dir); mntErr != nil { - glog.Errorf("Failed to unmount: %v", mntErr) + klog.Errorf("Failed to unmount: %v", mntErr) return err } notMnt, mntErr := b.mounter.IsNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsNotMountPoint check failed: %v", mntErr) return err } if !notMnt { // This is very odd, we don't expect it. We'll try again next sync loop. - glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) + klog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) return err } } diff --git a/pkg/volume/photon_pd/BUILD b/pkg/volume/photon_pd/BUILD index 0ae0a316aed..61d5467e4df 100644 --- a/pkg/volume/photon_pd/BUILD +++ b/pkg/volume/photon_pd/BUILD @@ -25,7 +25,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -44,7 +44,7 @@ go_test( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/photon_pd/attacher.go b/pkg/volume/photon_pd/attacher.go index 016fd65b839..d2570a4a743 100644 --- a/pkg/volume/photon_pd/attacher.go +++ b/pkg/volume/photon_pd/attacher.go @@ -25,9 +25,9 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/photon" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" @@ -50,7 +50,7 @@ var _ volume.DeviceMountableVolumePlugin = &photonPersistentDiskPlugin{} func (plugin *photonPersistentDiskPlugin) NewAttacher() (volume.Attacher, error) { photonCloud, err := getCloudProvider(plugin.host.GetCloudProvider()) if err != nil { - glog.Errorf("Photon Controller attacher: NewAttacher failed to get cloud provider") + klog.Errorf("Photon Controller attacher: NewAttacher failed to get cloud provider") return nil, err } @@ -74,22 +74,22 @@ func (attacher *photonPersistentDiskAttacher) Attach(spec *volume.Spec, nodeName hostName := string(nodeName) volumeSource, _, err := getVolumeSource(spec) if err != nil { - glog.Errorf("Photon Controller attacher: Attach failed to get volume source") + klog.Errorf("Photon Controller attacher: Attach failed to get volume source") return "", err } attached, err := attacher.photonDisks.DiskIsAttached(context.TODO(), volumeSource.PdID, nodeName) if err != nil { - glog.Warningf("Photon Controller: couldn't check if disk is Attached for host %s, will try attach disk: %+v", hostName, err) + klog.Warningf("Photon Controller: couldn't check if disk is Attached for host %s, will try attach disk: %+v", hostName, err) attached = false } if !attached { - glog.V(4).Infof("Photon Controller: Attach disk called for host %s", hostName) + klog.V(4).Infof("Photon Controller: Attach disk called for host %s", hostName) err = attacher.photonDisks.AttachDisk(context.TODO(), volumeSource.PdID, nodeName) if err != nil { - glog.Errorf("Error attaching volume %q to node %q: %+v", volumeSource.PdID, nodeName, err) + klog.Errorf("Error attaching volume %q to node %q: %+v", volumeSource.PdID, nodeName, err) return "", err } } @@ -105,7 +105,7 @@ func (attacher *photonPersistentDiskAttacher) VolumesAreAttached(specs []*volume for _, spec := range specs { volumeSource, _, err := getVolumeSource(spec) if err != nil { - glog.Errorf("Error getting volume (%q) source : %v", spec.Name(), err) + klog.Errorf("Error getting volume (%q) source : %v", spec.Name(), err) continue } @@ -115,7 +115,7 @@ func (attacher *photonPersistentDiskAttacher) VolumesAreAttached(specs []*volume } attachedResult, err := attacher.photonDisks.DisksAreAttached(context.TODO(), pdIDList, nodeName) if err != nil { - glog.Errorf( + klog.Errorf( "Error checking if volumes (%v) are attached to current node (%q). err=%v", pdIDList, nodeName, err) return volumesAttachedCheck, err @@ -125,7 +125,7 @@ func (attacher *photonPersistentDiskAttacher) VolumesAreAttached(specs []*volume if !attached { spec := volumeSpecMap[pdID] volumesAttachedCheck[spec] = false - glog.V(2).Infof("VolumesAreAttached: check volume %q (specName: %q) is no longer attached", pdID, spec.Name()) + klog.V(2).Infof("VolumesAreAttached: check volume %q (specName: %q) is no longer attached", pdID, spec.Name()) } } return volumesAttachedCheck, nil @@ -134,7 +134,7 @@ func (attacher *photonPersistentDiskAttacher) VolumesAreAttached(specs []*volume func (attacher *photonPersistentDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) { volumeSource, _, err := getVolumeSource(spec) if err != nil { - glog.Errorf("Photon Controller attacher: WaitForAttach failed to get volume source") + klog.Errorf("Photon Controller attacher: WaitForAttach failed to get volume source") return "", err } @@ -154,14 +154,14 @@ func (attacher *photonPersistentDiskAttacher) WaitForAttach(spec *volume.Spec, d for { select { case <-ticker.C: - glog.V(4).Infof("Checking PD %s is attached", volumeSource.PdID) + klog.V(4).Infof("Checking PD %s is attached", volumeSource.PdID) checkPath, err := verifyDevicePath(devicePath) if err != nil { // Log error, if any, and continue checking periodically. See issue #11321 - glog.Warningf("Photon Controller attacher: WaitForAttach with devicePath %s Checking PD %s Error verify path", devicePath, volumeSource.PdID) + klog.Warningf("Photon Controller attacher: WaitForAttach with devicePath %s Checking PD %s Error verify path", devicePath, volumeSource.PdID) } else if checkPath != "" { // A device path has successfully been created for the VMDK - glog.V(4).Infof("Successfully found attached PD %s.", volumeSource.PdID) + klog.V(4).Infof("Successfully found attached PD %s.", volumeSource.PdID) // map path with spec.Name() volName := spec.Name() realPath, _ := filepath.EvalSymlinks(devicePath) @@ -180,7 +180,7 @@ func (attacher *photonPersistentDiskAttacher) WaitForAttach(spec *volume.Spec, d func (attacher *photonPersistentDiskAttacher) GetDeviceMountPath(spec *volume.Spec) (string, error) { volumeSource, _, err := getVolumeSource(spec) if err != nil { - glog.Errorf("Photon Controller attacher: GetDeviceMountPath failed to get volume source") + klog.Errorf("Photon Controller attacher: GetDeviceMountPath failed to get volume source") return "", err } @@ -201,7 +201,7 @@ func (attacher *photonPersistentDiskAttacher) MountDevice(spec *volume.Spec, dev if err != nil { if os.IsNotExist(err) { if err := os.MkdirAll(deviceMountPath, 0750); err != nil { - glog.Errorf("Failed to create directory at %#v. err: %s", deviceMountPath, err) + klog.Errorf("Failed to create directory at %#v. err: %s", deviceMountPath, err) return err } notMnt = true @@ -212,7 +212,7 @@ func (attacher *photonPersistentDiskAttacher) MountDevice(spec *volume.Spec, dev volumeSource, _, err := getVolumeSource(spec) if err != nil { - glog.Errorf("Photon Controller attacher: MountDevice failed to get volume source. err: %s", err) + klog.Errorf("Photon Controller attacher: MountDevice failed to get volume source. err: %s", err) return err } @@ -226,7 +226,7 @@ func (attacher *photonPersistentDiskAttacher) MountDevice(spec *volume.Spec, dev os.Remove(deviceMountPath) return err } - glog.V(4).Infof("formatting spec %v devicePath %v deviceMountPath %v fs %v with options %+v", spec.Name(), devicePath, deviceMountPath, volumeSource.FSType, options) + klog.V(4).Infof("formatting spec %v devicePath %v deviceMountPath %v fs %v with options %+v", spec.Name(), devicePath, deviceMountPath, volumeSource.FSType, options) } return nil } @@ -243,7 +243,7 @@ var _ volume.DeviceUnmounter = &photonPersistentDiskDetacher{} func (plugin *photonPersistentDiskPlugin) NewDetacher() (volume.Detacher, error) { photonCloud, err := getCloudProvider(plugin.host.GetCloudProvider()) if err != nil { - glog.Errorf("Photon Controller attacher: NewDetacher failed to get cloud provider. err: %s", err) + klog.Errorf("Photon Controller attacher: NewDetacher failed to get cloud provider. err: %s", err) return nil, err } @@ -265,19 +265,19 @@ func (detacher *photonPersistentDiskDetacher) Detach(volumeName string, nodeName attached, err := detacher.photonDisks.DiskIsAttached(context.TODO(), pdID, nodeName) if err != nil { // Log error and continue with detach - glog.Errorf( + klog.Errorf( "Error checking if persistent disk (%q) is already attached to current node (%q). Will continue and try detach anyway. err=%v", pdID, hostName, err) } if err == nil && !attached { // Volume is already detached from node. - glog.V(4).Infof("detach operation was successful. persistent disk %q is already detached from node %q.", pdID, hostName) + klog.V(4).Infof("detach operation was successful. persistent disk %q is already detached from node %q.", pdID, hostName) return nil } if err := detacher.photonDisks.DetachDisk(context.TODO(), pdID, nodeName); err != nil { - glog.Errorf("Error detaching volume %q: %v", pdID, err) + klog.Errorf("Error detaching volume %q: %v", pdID, err) return err } return nil @@ -292,7 +292,7 @@ func (detacher *photonPersistentDiskDetacher) WaitForDetach(devicePath string, t for { select { case <-ticker.C: - glog.V(4).Infof("Checking device %q is detached.", devicePath) + klog.V(4).Infof("Checking device %q is detached.", devicePath) if pathExists, err := volumeutil.PathExists(devicePath); err != nil { return fmt.Errorf("Error checking if device path exists: %v", err) } else if !pathExists { diff --git a/pkg/volume/photon_pd/attacher_test.go b/pkg/volume/photon_pd/attacher_test.go index cf055e9182f..077131610b5 100644 --- a/pkg/volume/photon_pd/attacher_test.go +++ b/pkg/volume/photon_pd/attacher_test.go @@ -26,8 +26,8 @@ import ( "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" ) func TestGetDeviceName_Volume(t *testing.T) { @@ -254,7 +254,7 @@ func (testcase *testcase) AttachDisk(ctx context.Context, diskName string, nodeN return errors.New("Unexpected AttachDisk call: wrong nodeName") } - glog.V(4).Infof("AttachDisk call: %s, %s, returning %v", diskName, nodeName, expected.ret) + klog.V(4).Infof("AttachDisk call: %s, %s, returning %v", diskName, nodeName, expected.ret) return expected.ret } @@ -279,7 +279,7 @@ func (testcase *testcase) DetachDisk(ctx context.Context, diskName string, nodeN return errors.New("Unexpected DetachDisk call: wrong nodeName") } - glog.V(4).Infof("DetachDisk call: %s, %s, returning %v", diskName, nodeName, expected.ret) + klog.V(4).Infof("DetachDisk call: %s, %s, returning %v", diskName, nodeName, expected.ret) return expected.ret } @@ -304,7 +304,7 @@ func (testcase *testcase) DiskIsAttached(ctx context.Context, diskName string, n return false, errors.New("Unexpected DiskIsAttached call: wrong nodeName") } - glog.V(4).Infof("DiskIsAttached call: %s, %s, returning %v, %v", diskName, nodeName, expected.isAttached, expected.ret) + klog.V(4).Infof("DiskIsAttached call: %s, %s, returning %v, %v", diskName, nodeName, expected.isAttached, expected.ret) return expected.isAttached, expected.ret } diff --git a/pkg/volume/photon_pd/photon_pd.go b/pkg/volume/photon_pd/photon_pd.go index b8d8c936e00..03cbd9b0ecb 100644 --- a/pkg/volume/photon_pd/photon_pd.go +++ b/pkg/volume/photon_pd/photon_pd.go @@ -21,11 +21,11 @@ import ( "os" "path" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" utilstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" @@ -62,7 +62,7 @@ func (plugin *photonPersistentDiskPlugin) GetPluginName() string { func (plugin *photonPersistentDiskPlugin) GetVolumeName(spec *volume.Spec) (string, error) { volumeSource, _, err := getVolumeSource(spec) if err != nil { - glog.Errorf("Photon volume plugin: GetVolumeName failed to get volume source") + klog.Errorf("Photon volume plugin: GetVolumeName failed to get volume source") return "", err } @@ -97,7 +97,7 @@ func (plugin *photonPersistentDiskPlugin) NewUnmounter(volName string, podUID ty func (plugin *photonPersistentDiskPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Mounter, error) { vvol, _, err := getVolumeSource(spec) if err != nil { - glog.Errorf("Photon volume plugin: newMounterInternal failed to get volume source") + klog.Errorf("Photon volume plugin: newMounterInternal failed to get volume source") return nil, err } @@ -202,12 +202,12 @@ func (b *photonPersistentDiskMounter) SetUp(fsGroup *int64) error { // SetUp attaches the disk and bind mounts to the volume path. func (b *photonPersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error { - glog.V(4).Infof("Photon Persistent Disk setup %s to %s", b.pdID, dir) + klog.V(4).Infof("Photon Persistent Disk setup %s to %s", b.pdID, dir) // TODO: handle failed mounts here. notmnt, err := b.mounter.IsLikelyNotMountPoint(dir) if err != nil && !os.IsNotExist(err) { - glog.Errorf("cannot validate mount point: %s %v", dir, err) + klog.Errorf("cannot validate mount point: %s %v", dir, err) return err } if !notmnt { @@ -215,7 +215,7 @@ func (b *photonPersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error } if err := os.MkdirAll(dir, 0750); err != nil { - glog.Errorf("mkdir failed on disk %s (%v)", dir, err) + klog.Errorf("mkdir failed on disk %s (%v)", dir, err) return err } @@ -223,33 +223,33 @@ func (b *photonPersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error // Perform a bind mount to the full path to allow duplicate mounts of the same PD. globalPDPath := makeGlobalPDPath(b.plugin.host, b.pdID) - glog.V(4).Infof("attempting to mount %s", dir) + klog.V(4).Infof("attempting to mount %s", dir) mountOptions := util.JoinMountOptions(options, b.mountOption) err = b.mounter.Mount(globalPDPath, dir, "", mountOptions) if err != nil { notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !notmnt { if mntErr = b.mounter.Unmount(dir); mntErr != nil { - glog.Errorf("Failed to unmount: %v", mntErr) + klog.Errorf("Failed to unmount: %v", mntErr) return err } notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !notmnt { - glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", b.GetPath()) + klog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", b.GetPath()) return err } } os.Remove(dir) - glog.Errorf("Mount of disk %s failed: %v", dir, err) + klog.Errorf("Mount of disk %s failed: %v", dir, err) return err } diff --git a/pkg/volume/photon_pd/photon_util.go b/pkg/volume/photon_pd/photon_util.go index d3e8c29e57a..fe3a6c24c6d 100644 --- a/pkg/volume/photon_pd/photon_util.go +++ b/pkg/volume/photon_pd/photon_util.go @@ -23,9 +23,9 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/photon" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" @@ -63,7 +63,7 @@ func scsiHostScan() { name := scsi_path + f.Name() + "/scan" data := []byte("- - -") ioutil.WriteFile(name, data, 0666) - glog.Errorf("scsiHostScan scan for %s", name) + klog.Errorf("scsiHostScan scan for %s", name) } } } @@ -75,7 +75,7 @@ func verifyDevicePath(path string) (string, error) { return path, nil } - glog.V(4).Infof("verifyDevicePath: path not exists yet") + klog.V(4).Infof("verifyDevicePath: path not exists yet") return "", nil } @@ -83,7 +83,7 @@ func verifyDevicePath(path string) (string, error) { func (util *PhotonDiskUtil) CreateVolume(p *photonPersistentDiskProvisioner) (pdID string, capacityGB int, fstype string, err error) { cloud, err := getCloudProvider(p.plugin.host.GetCloudProvider()) if err != nil { - glog.Errorf("Photon Controller Util: CreateVolume failed to get cloud provider. Error [%v]", err) + klog.Errorf("Photon Controller Util: CreateVolume failed to get cloud provider. Error [%v]", err) return "", 0, "", err } @@ -106,20 +106,20 @@ func (util *PhotonDiskUtil) CreateVolume(p *photonPersistentDiskProvisioner) (pd volumeOptions.Flavor = value case volume.VolumeParameterFSType: fstype = value - glog.V(4).Infof("Photon Controller Util: Setting fstype to %s", fstype) + klog.V(4).Infof("Photon Controller Util: Setting fstype to %s", fstype) default: - glog.Errorf("Photon Controller Util: invalid option %s for volume plugin %s.", parameter, p.plugin.GetPluginName()) + klog.Errorf("Photon Controller Util: invalid option %s for volume plugin %s.", parameter, p.plugin.GetPluginName()) return "", 0, "", fmt.Errorf("Photon Controller Util: invalid option %s for volume plugin %s.", parameter, p.plugin.GetPluginName()) } } pdID, err = cloud.CreateDisk(volumeOptions) if err != nil { - glog.Errorf("Photon Controller Util: failed to CreateDisk. Error [%v]", err) + klog.Errorf("Photon Controller Util: failed to CreateDisk. Error [%v]", err) return "", 0, "", err } - glog.V(4).Infof("Successfully created Photon Controller persistent disk %s", name) + klog.V(4).Infof("Successfully created Photon Controller persistent disk %s", name) return pdID, volSizeGB, "", nil } @@ -127,28 +127,28 @@ func (util *PhotonDiskUtil) CreateVolume(p *photonPersistentDiskProvisioner) (pd func (util *PhotonDiskUtil) DeleteVolume(pd *photonPersistentDiskDeleter) error { cloud, err := getCloudProvider(pd.plugin.host.GetCloudProvider()) if err != nil { - glog.Errorf("Photon Controller Util: DeleteVolume failed to get cloud provider. Error [%v]", err) + klog.Errorf("Photon Controller Util: DeleteVolume failed to get cloud provider. Error [%v]", err) return err } if err = cloud.DeleteDisk(pd.pdID); err != nil { - glog.Errorf("Photon Controller Util: failed to DeleteDisk for pdID %s. Error [%v]", pd.pdID, err) + klog.Errorf("Photon Controller Util: failed to DeleteDisk for pdID %s. Error [%v]", pd.pdID, err) return err } - glog.V(4).Infof("Successfully deleted PhotonController persistent disk %s", pd.pdID) + klog.V(4).Infof("Successfully deleted PhotonController persistent disk %s", pd.pdID) return nil } func getCloudProvider(cloud cloudprovider.Interface) (*photon.PCCloud, error) { if cloud == nil { - glog.Errorf("Photon Controller Util: Cloud provider not initialized properly") + klog.Errorf("Photon Controller Util: Cloud provider not initialized properly") return nil, fmt.Errorf("Photon Controller Util: Cloud provider not initialized properly") } pcc := cloud.(*photon.PCCloud) if pcc == nil { - glog.Errorf("Invalid cloud provider: expected Photon Controller") + klog.Errorf("Invalid cloud provider: expected Photon Controller") return nil, fmt.Errorf("Invalid cloud provider: expected Photon Controller") } return pcc, nil diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index abfb7bf8fd5..a0cb5716464 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -22,7 +22,6 @@ import ( "strings" "sync" - "github.com/golang/glog" authenticationv1 "k8s.io/api/authentication/v1" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -34,6 +33,7 @@ import ( "k8s.io/client-go/tools/record" cloudprovider "k8s.io/cloud-provider" csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume/util/recyclerclient" ) @@ -514,7 +514,7 @@ func (pm *VolumePluginMgr) InitPlugins(plugins []VolumePlugin, prober DynamicPlu } if err := pm.prober.Init(); err != nil { // Prober init failure should not affect the initialization of other plugins. - glog.Errorf("Error initializing dynamic plugin prober: %s", err) + klog.Errorf("Error initializing dynamic plugin prober: %s", err) pm.prober = &dummyPluginProber{} } @@ -539,12 +539,12 @@ func (pm *VolumePluginMgr) InitPlugins(plugins []VolumePlugin, prober DynamicPlu } err := plugin.Init(host) if err != nil { - glog.Errorf("Failed to load volume plugin %s, error: %s", name, err.Error()) + klog.Errorf("Failed to load volume plugin %s, error: %s", name, err.Error()) allErrs = append(allErrs, err) continue } pm.plugins[name] = plugin - glog.V(1).Infof("Loaded volume plugin %q", name) + klog.V(1).Infof("Loaded volume plugin %q", name) } return utilerrors.NewAggregate(allErrs) } @@ -560,7 +560,7 @@ func (pm *VolumePluginMgr) initProbedPlugin(probedPlugin VolumePlugin) error { return fmt.Errorf("Failed to load volume plugin %s, error: %s", name, err.Error()) } - glog.V(1).Infof("Loaded volume plugin %q", name) + klog.V(1).Infof("Loaded volume plugin %q", name) return nil } @@ -639,14 +639,14 @@ func (pm *VolumePluginMgr) FindPluginByName(name string) (VolumePlugin, error) { func (pm *VolumePluginMgr) refreshProbedPlugins() { events, err := pm.prober.Probe() if err != nil { - glog.Errorf("Error dynamically probing plugins: %s", err) + klog.Errorf("Error dynamically probing plugins: %s", err) return // Use cached plugins upon failure. } for _, event := range events { if event.Op == ProbeAddOrUpdate { if err := pm.initProbedPlugin(event.Plugin); err != nil { - glog.Errorf("Error initializing dynamically probed plugin %s; error: %s", + klog.Errorf("Error initializing dynamically probed plugin %s; error: %s", event.Plugin.GetPluginName(), err) continue } @@ -655,7 +655,7 @@ func (pm *VolumePluginMgr) refreshProbedPlugins() { // Plugin is not available on ProbeRemove event, only PluginName delete(pm.probedPlugins, event.PluginName) } else { - glog.Errorf("Unknown Operation on PluginName: %s.", + klog.Errorf("Unknown Operation on PluginName: %s.", event.Plugin.GetPluginName()) } } @@ -839,10 +839,10 @@ func (pm *VolumePluginMgr) FindExpandablePluginBySpec(spec *Spec) (ExpandableVol if spec.IsKubeletExpandable() { // for kubelet expandable volumes, return a noop plugin that // returns success for expand on the controller - glog.Warningf("FindExpandablePluginBySpec(%s) -> returning noopExpandableVolumePluginInstance", spec.Name()) + klog.Warningf("FindExpandablePluginBySpec(%s) -> returning noopExpandableVolumePluginInstance", spec.Name()) return &noopExpandableVolumePluginInstance{spec}, nil } - glog.Warningf("FindExpandablePluginBySpec(%s) -> err:%v", spec.Name(), err) + klog.Warningf("FindExpandablePluginBySpec(%s) -> err:%v", spec.Name(), err) return nil, err } diff --git a/pkg/volume/portworx/BUILD b/pkg/volume/portworx/BUILD index 19190114902..e43ba8d037f 100644 --- a/pkg/volume/portworx/BUILD +++ b/pkg/volume/portworx/BUILD @@ -39,12 +39,12 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/libopenstorage/openstorage/api:go_default_library", "//vendor/github.com/libopenstorage/openstorage/api/client:go_default_library", "//vendor/github.com/libopenstorage/openstorage/api/client/volume:go_default_library", "//vendor/github.com/libopenstorage/openstorage/api/spec:go_default_library", "//vendor/github.com/libopenstorage/openstorage/volume:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/portworx/portworx.go b/pkg/volume/portworx/portworx.go index 8c875e7dd77..212a4c23d79 100644 --- a/pkg/volume/portworx/portworx.go +++ b/pkg/volume/portworx/portworx.go @@ -20,11 +20,11 @@ import ( "fmt" "os" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" kstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" @@ -180,13 +180,13 @@ func (plugin *portworxVolumePlugin) ExpandVolumeDevice( spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) { - glog.V(4).Infof("Expanding: %s from %v to %v", spec.Name(), oldSize, newSize) + klog.V(4).Infof("Expanding: %s from %v to %v", spec.Name(), oldSize, newSize) err := plugin.util.ResizeVolume(spec, newSize, plugin.host) if err != nil { return oldSize, err } - glog.V(4).Infof("Successfully resized %s to %v", spec.Name(), newSize) + klog.V(4).Infof("Successfully resized %s to %v", spec.Name(), newSize) return newSize, nil } @@ -290,9 +290,9 @@ func (b *portworxVolumeMounter) SetUp(fsGroup *int64) error { // SetUpAt attaches the disk and bind mounts to the volume path. func (b *portworxVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) - glog.Infof("Portworx Volume set up. Dir: %s %v %v", dir, !notMnt, err) + klog.Infof("Portworx Volume set up. Dir: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { - glog.Errorf("Cannot validate mountpoint: %s", dir) + klog.Errorf("Cannot validate mountpoint: %s", dir) return err } if !notMnt { @@ -306,7 +306,7 @@ func (b *portworxVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { return err } - glog.V(4).Infof("Portworx Volume %s attached", b.volumeID) + klog.V(4).Infof("Portworx Volume %s attached", b.volumeID) if err := os.MkdirAll(dir, 0750); err != nil { return err @@ -318,7 +318,7 @@ func (b *portworxVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if !b.readOnly { volume.SetVolumeOwnership(b, fsGroup) } - glog.Infof("Portworx Volume %s setup at %s", b.volumeID, dir) + klog.Infof("Portworx Volume %s setup at %s", b.volumeID, dir) return nil } @@ -341,7 +341,7 @@ func (c *portworxVolumeUnmounter) TearDown() error { // Unmounts the bind mount, and detaches the disk only if the PD // resource was the last reference to that disk on the kubelet. func (c *portworxVolumeUnmounter) TearDownAt(dir string) error { - glog.Infof("Portworx Volume TearDown of %s", dir) + klog.Infof("Portworx Volume TearDown of %s", dir) if err := c.manager.UnmountVolume(c, dir); err != nil { return err diff --git a/pkg/volume/portworx/portworx_util.go b/pkg/volume/portworx/portworx_util.go index ab68e407268..62b3e3f4d31 100644 --- a/pkg/volume/portworx/portworx_util.go +++ b/pkg/volume/portworx/portworx_util.go @@ -19,7 +19,6 @@ package portworx import ( "fmt" - "github.com/golang/glog" osdapi "github.com/libopenstorage/openstorage/api" osdclient "github.com/libopenstorage/openstorage/api/client" volumeclient "github.com/libopenstorage/openstorage/api/client/volume" @@ -28,6 +27,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/volume" volutil "k8s.io/kubernetes/pkg/volume/util" @@ -51,11 +51,11 @@ type PortworxVolumeUtil struct { func (util *PortworxVolumeUtil) CreateVolume(p *portworxVolumeProvisioner) (string, int64, map[string]string, error) { driver, err := util.getPortworxDriver(p.plugin.host, false /*localOnly*/) if err != nil || driver == nil { - glog.Errorf("Failed to get portworx driver. Err: %v", err) + klog.Errorf("Failed to get portworx driver. Err: %v", err) return "", 0, nil, err } - glog.Infof("Creating Portworx volume for PVC: %v", p.options.PVC.Name) + klog.Infof("Creating Portworx volume for PVC: %v", p.options.PVC.Name) capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] // Portworx Volumes are specified in GiB @@ -95,7 +95,7 @@ func (util *PortworxVolumeUtil) CreateVolume(p *portworxVolumeProvisioner) (stri for k, v := range p.options.PVC.Annotations { if _, present := spec.VolumeLabels[k]; present { - glog.Warningf("not saving annotation: %s=%s in spec labels due to an existing key", k, v) + klog.Warningf("not saving annotation: %s=%s in spec labels due to an existing key", k, v) continue } spec.VolumeLabels[k] = v @@ -103,11 +103,11 @@ func (util *PortworxVolumeUtil) CreateVolume(p *portworxVolumeProvisioner) (stri volumeID, err := driver.Create(locator, source, spec) if err != nil { - glog.Errorf("Error creating Portworx Volume : %v", err) + klog.Errorf("Error creating Portworx Volume : %v", err) return "", 0, nil, err } - glog.Infof("Successfully created Portworx volume for PVC: %v", p.options.PVC.Name) + klog.Infof("Successfully created Portworx volume for PVC: %v", p.options.PVC.Name) return volumeID, requestGiB, nil, err } @@ -115,13 +115,13 @@ func (util *PortworxVolumeUtil) CreateVolume(p *portworxVolumeProvisioner) (stri func (util *PortworxVolumeUtil) DeleteVolume(d *portworxVolumeDeleter) error { driver, err := util.getPortworxDriver(d.plugin.host, false /*localOnly*/) if err != nil || driver == nil { - glog.Errorf("Failed to get portworx driver. Err: %v", err) + klog.Errorf("Failed to get portworx driver. Err: %v", err) return err } err = driver.Delete(d.volumeID) if err != nil { - glog.Errorf("Error deleting Portworx Volume (%v): %v", d.volName, err) + klog.Errorf("Error deleting Portworx Volume (%v): %v", d.volName, err) return err } return nil @@ -131,13 +131,13 @@ func (util *PortworxVolumeUtil) DeleteVolume(d *portworxVolumeDeleter) error { func (util *PortworxVolumeUtil) AttachVolume(m *portworxVolumeMounter, attachOptions map[string]string) (string, error) { driver, err := util.getPortworxDriver(m.plugin.host, true /*localOnly*/) if err != nil || driver == nil { - glog.Errorf("Failed to get portworx driver. Err: %v", err) + klog.Errorf("Failed to get portworx driver. Err: %v", err) return "", err } devicePath, err := driver.Attach(m.volName, attachOptions) if err != nil { - glog.Errorf("Error attaching Portworx Volume (%v): %v", m.volName, err) + klog.Errorf("Error attaching Portworx Volume (%v): %v", m.volName, err) return "", err } return devicePath, nil @@ -147,13 +147,13 @@ func (util *PortworxVolumeUtil) AttachVolume(m *portworxVolumeMounter, attachOpt func (util *PortworxVolumeUtil) DetachVolume(u *portworxVolumeUnmounter) error { driver, err := util.getPortworxDriver(u.plugin.host, true /*localOnly*/) if err != nil || driver == nil { - glog.Errorf("Failed to get portworx driver. Err: %v", err) + klog.Errorf("Failed to get portworx driver. Err: %v", err) return err } err = driver.Detach(u.volName, false /*doNotForceDetach*/) if err != nil { - glog.Errorf("Error detaching Portworx Volume (%v): %v", u.volName, err) + klog.Errorf("Error detaching Portworx Volume (%v): %v", u.volName, err) return err } return nil @@ -163,13 +163,13 @@ func (util *PortworxVolumeUtil) DetachVolume(u *portworxVolumeUnmounter) error { func (util *PortworxVolumeUtil) MountVolume(m *portworxVolumeMounter, mountPath string) error { driver, err := util.getPortworxDriver(m.plugin.host, true /*localOnly*/) if err != nil || driver == nil { - glog.Errorf("Failed to get portworx driver. Err: %v", err) + klog.Errorf("Failed to get portworx driver. Err: %v", err) return err } err = driver.Mount(m.volName, mountPath) if err != nil { - glog.Errorf("Error mounting Portworx Volume (%v) on Path (%v): %v", m.volName, mountPath, err) + klog.Errorf("Error mounting Portworx Volume (%v) on Path (%v): %v", m.volName, mountPath, err) return err } return nil @@ -179,13 +179,13 @@ func (util *PortworxVolumeUtil) MountVolume(m *portworxVolumeMounter, mountPath func (util *PortworxVolumeUtil) UnmountVolume(u *portworxVolumeUnmounter, mountPath string) error { driver, err := util.getPortworxDriver(u.plugin.host, true /*localOnly*/) if err != nil || driver == nil { - glog.Errorf("Failed to get portworx driver. Err: %v", err) + klog.Errorf("Failed to get portworx driver. Err: %v", err) return err } err = driver.Unmount(u.volName, mountPath) if err != nil { - glog.Errorf("Error unmounting Portworx Volume (%v) on Path (%v): %v", u.volName, mountPath, err) + klog.Errorf("Error unmounting Portworx Volume (%v) on Path (%v): %v", u.volName, mountPath, err) return err } return nil @@ -194,7 +194,7 @@ func (util *PortworxVolumeUtil) UnmountVolume(u *portworxVolumeUnmounter, mountP func (util *PortworxVolumeUtil) ResizeVolume(spec *volume.Spec, newSize resource.Quantity, volumeHost volume.VolumeHost) error { driver, err := util.getPortworxDriver(volumeHost, false /*localOnly*/) if err != nil || driver == nil { - glog.Errorf("Failed to get portworx driver. Err: %v", err) + klog.Errorf("Failed to get portworx driver. Err: %v", err) return err } @@ -210,7 +210,7 @@ func (util *PortworxVolumeUtil) ResizeVolume(spec *volume.Spec, newSize resource vol := vols[0] newSizeInBytes := uint64(volutil.RoundUpToGiB(newSize) * volutil.GIB) if vol.Spec.Size >= newSizeInBytes { - glog.Infof("Portworx volume: %s already at size: %d greater than or equal to new "+ + klog.Infof("Portworx volume: %s already at size: %d greater than or equal to new "+ "requested size: %d. Skipping resize.", spec.Name(), vol.Spec.Size, newSizeInBytes) return nil } @@ -247,7 +247,7 @@ func isClientValid(client *osdclient.Client) (bool, error) { _, err := client.Versions(osdapi.OsdVolumePath) if err != nil { - glog.Errorf("portworx client failed driver versions check. Err: %v", err) + klog.Errorf("portworx client failed driver versions check. Err: %v", err) return false, err } @@ -285,7 +285,7 @@ func (util *PortworxVolumeUtil) getPortworxDriver(volumeHost volume.VolumeHost, if err != nil { return nil, err } else { - glog.V(4).Infof("Using portworx local service at: %v as api endpoint", volumeHost.GetHostName()) + klog.V(4).Infof("Using portworx local service at: %v as api endpoint", volumeHost.GetHostName()) return volumeclient.VolumeDriver(util.portworxClient), nil } } @@ -301,31 +301,31 @@ func (util *PortworxVolumeUtil) getPortworxDriver(volumeHost volume.VolumeHost, // Create client from portworx service kubeClient := volumeHost.GetKubeClient() if kubeClient == nil { - glog.Error("Failed to get kubeclient when creating portworx client") + klog.Error("Failed to get kubeclient when creating portworx client") return nil, nil } opts := metav1.GetOptions{} svc, err := kubeClient.CoreV1().Services(api.NamespaceSystem).Get(pxServiceName, opts) if err != nil { - glog.Errorf("Failed to get service. Err: %v", err) + klog.Errorf("Failed to get service. Err: %v", err) return nil, err } if svc == nil { - glog.Errorf("Service: %v not found. Consult Portworx docs to deploy it.", pxServiceName) + klog.Errorf("Service: %v not found. Consult Portworx docs to deploy it.", pxServiceName) return nil, err } util.portworxClient, err = createDriverClient(svc.Spec.ClusterIP) if err != nil || util.portworxClient == nil { - glog.Errorf("Failed to connect to portworx service. Err: %v", err) + klog.Errorf("Failed to connect to portworx service. Err: %v", err) return nil, err } - glog.Infof("Using portworx cluster service at: %v as api endpoint", svc.Spec.ClusterIP) + klog.Infof("Using portworx cluster service at: %v as api endpoint", svc.Spec.ClusterIP) } else { - glog.Infof("Using portworx service at: %v as api endpoint", volumeHost.GetHostName()) + klog.Infof("Using portworx service at: %v as api endpoint", volumeHost.GetHostName()) } return volumeclient.VolumeDriver(util.portworxClient), nil diff --git a/pkg/volume/projected/BUILD b/pkg/volume/projected/BUILD index a1f2a07faf7..65883f906b3 100644 --- a/pkg/volume/projected/BUILD +++ b/pkg/volume/projected/BUILD @@ -11,15 +11,22 @@ go_test( srcs = ["projected_test.go"], embed = [":go_default_library"], deps = [ + "//pkg/apis/authentication/v1:go_default_library", + "//pkg/apis/core/v1:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/emptydir:go_default_library", "//pkg/volume/testing:go_default_library", "//pkg/volume/util:go_default_library", + "//staging/src/k8s.io/api/authentication/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + "//staging/src/k8s.io/client-go/testing:go_default_library", ], ) @@ -42,7 +49,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/projected/projected.go b/pkg/volume/projected/projected.go index 376199e120e..e93588ce676 100644 --- a/pkg/volume/projected/projected.go +++ b/pkg/volume/projected/projected.go @@ -34,7 +34,7 @@ import ( "k8s.io/kubernetes/pkg/volume/secret" volumeutil "k8s.io/kubernetes/pkg/volume/util" - "github.com/golang/glog" + "k8s.io/klog" ) // ProbeVolumePlugins is the entry point for plugin detection in a package. @@ -188,7 +188,7 @@ func (s *projectedVolumeMounter) SetUp(fsGroup *int64) error { } func (s *projectedVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { - glog.V(3).Infof("Setting up volume %v for pod %v at %v", s.volName, s.pod.UID, dir) + klog.V(3).Infof("Setting up volume %v for pod %v at %v", s.volName, s.pod.UID, dir) wrapped, err := s.plugin.host.NewWrapperMounter(s.volName, wrappedVolumeSpec(), s.pod, *s.opts) if err != nil { @@ -197,7 +197,7 @@ func (s *projectedVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { data, err := s.collectData() if err != nil { - glog.Errorf("Error preparing data for projected volume %v for pod %v/%v: %s", s.volName, s.pod.Namespace, s.pod.Name, err.Error()) + klog.Errorf("Error preparing data for projected volume %v for pod %v/%v: %s", s.volName, s.pod.Namespace, s.pod.Name, err.Error()) return err } @@ -215,12 +215,12 @@ func (s *projectedVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if !setupSuccess { unmounter, unmountCreateErr := s.plugin.NewUnmounter(s.volName, s.podUID) if unmountCreateErr != nil { - glog.Errorf("error cleaning up mount %s after failure. Create unmounter failed with %v", s.volName, unmountCreateErr) + klog.Errorf("error cleaning up mount %s after failure. Create unmounter failed with %v", s.volName, unmountCreateErr) return } tearDownErr := unmounter.TearDown() if tearDownErr != nil { - glog.Errorf("error tearing down volume %s with : %v", s.volName, tearDownErr) + klog.Errorf("error tearing down volume %s with : %v", s.volName, tearDownErr) } } }() @@ -228,19 +228,19 @@ func (s *projectedVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { writerContext := fmt.Sprintf("pod %v/%v volume %v", s.pod.Namespace, s.pod.Name, s.volName) writer, err := volumeutil.NewAtomicWriter(dir, writerContext) if err != nil { - glog.Errorf("Error creating atomic writer: %v", err) + klog.Errorf("Error creating atomic writer: %v", err) return err } err = writer.Write(data) if err != nil { - glog.Errorf("Error writing payload to dir: %v", err) + klog.Errorf("Error writing payload to dir: %v", err) return err } err = volume.SetVolumeOwnership(s, fsGroup) if err != nil { - glog.Errorf("Error applying volume ownership settings for group: %v", fsGroup) + klog.Errorf("Error applying volume ownership settings for group: %v", fsGroup) return err } setupSuccess = true @@ -266,7 +266,7 @@ func (s *projectedVolumeMounter) collectData() (map[string]volumeutil.FileProjec secretapi, err := s.plugin.getSecret(s.pod.Namespace, source.Secret.Name) if err != nil { if !(errors.IsNotFound(err) && optional) { - glog.Errorf("Couldn't get secret %v/%v: %v", s.pod.Namespace, source.Secret.Name, err) + klog.Errorf("Couldn't get secret %v/%v: %v", s.pod.Namespace, source.Secret.Name, err) errlist = append(errlist, err) continue } @@ -279,7 +279,7 @@ func (s *projectedVolumeMounter) collectData() (map[string]volumeutil.FileProjec } secretPayload, err := secret.MakePayload(source.Secret.Items, secretapi, s.source.DefaultMode, optional) if err != nil { - glog.Errorf("Couldn't get secret payload %v/%v: %v", s.pod.Namespace, source.Secret.Name, err) + klog.Errorf("Couldn't get secret payload %v/%v: %v", s.pod.Namespace, source.Secret.Name, err) errlist = append(errlist, err) continue } @@ -291,7 +291,7 @@ func (s *projectedVolumeMounter) collectData() (map[string]volumeutil.FileProjec configMap, err := s.plugin.getConfigMap(s.pod.Namespace, source.ConfigMap.Name) if err != nil { if !(errors.IsNotFound(err) && optional) { - glog.Errorf("Couldn't get configMap %v/%v: %v", s.pod.Namespace, source.ConfigMap.Name, err) + klog.Errorf("Couldn't get configMap %v/%v: %v", s.pod.Namespace, source.ConfigMap.Name, err) errlist = append(errlist, err) continue } @@ -304,7 +304,7 @@ func (s *projectedVolumeMounter) collectData() (map[string]volumeutil.FileProjec } configMapPayload, err := configmap.MakePayload(source.ConfigMap.Items, configMap, s.source.DefaultMode, optional) if err != nil { - glog.Errorf("Couldn't get configMap payload %v/%v: %v", s.pod.Namespace, source.ConfigMap.Name, err) + klog.Errorf("Couldn't get configMap payload %v/%v: %v", s.pod.Namespace, source.ConfigMap.Name, err) errlist = append(errlist, err) continue } @@ -326,11 +326,14 @@ func (s *projectedVolumeMounter) collectData() (map[string]volumeutil.FileProjec continue } tp := source.ServiceAccountToken + + var auds []string + if len(tp.Audience) != 0 { + auds = []string{tp.Audience} + } tr, err := s.plugin.getServiceAccountToken(s.pod.Namespace, s.pod.Spec.ServiceAccountName, &authenticationv1.TokenRequest{ Spec: authenticationv1.TokenRequestSpec{ - Audiences: []string{ - tp.Audience, - }, + Audiences: auds, ExpirationSeconds: tp.ExpirationSeconds, BoundObjectRef: &authenticationv1.BoundObjectReference{ APIVersion: "v1", @@ -364,7 +367,7 @@ func (c *projectedVolumeUnmounter) TearDown() error { } func (c *projectedVolumeUnmounter) TearDownAt(dir string) error { - glog.V(3).Infof("Tearing down volume %v for pod %v at %v", c.volName, c.podUID, dir) + klog.V(3).Infof("Tearing down volume %v for pod %v at %v", c.volName, c.podUID, dir) wrapped, err := c.plugin.host.NewWrapperUnmounter(c.volName, wrappedVolumeSpec(), c.podUID) if err != nil { diff --git a/pkg/volume/projected/projected_test.go b/pkg/volume/projected/projected_test.go index 10c7b14fa5d..932e2fe992c 100644 --- a/pkg/volume/projected/projected_test.go +++ b/pkg/volume/projected/projected_test.go @@ -25,11 +25,18 @@ import ( "strings" "testing" + authenticationv1 "k8s.io/api/authentication/v1" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/diff" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" + clitesting "k8s.io/client-go/testing" + pkgauthenticationv1 "k8s.io/kubernetes/pkg/apis/authentication/v1" + pkgcorev1 "k8s.io/kubernetes/pkg/apis/core/v1" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/emptydir" volumetest "k8s.io/kubernetes/pkg/volume/testing" @@ -699,6 +706,113 @@ func TestCollectDataWithDownwardAPI(t *testing.T) { } } +func TestCollectDataWithServiceAccountToken(t *testing.T) { + scheme := runtime.NewScheme() + utilruntime.Must(pkgauthenticationv1.RegisterDefaults(scheme)) + utilruntime.Must(pkgcorev1.RegisterDefaults(scheme)) + + minute := int64(60) + cases := []struct { + name string + svcacct string + audience string + expiration *int64 + path string + + payload map[string]util.FileProjection + }{ + { + name: "test good service account", + audience: "https://example.com", + path: "token", + expiration: &minute, + + payload: map[string]util.FileProjection{ + "token": {Data: []byte("test_projected_namespace:foo:60:[https://example.com]"), Mode: 0600}, + }, + }, + { + name: "test good service account other path", + audience: "https://example.com", + path: "other-token", + expiration: &minute, + + payload: map[string]util.FileProjection{ + "other-token": {Data: []byte("test_projected_namespace:foo:60:[https://example.com]"), Mode: 0600}, + }, + }, + { + name: "test good service account defaults audience", + path: "token", + expiration: &minute, + + payload: map[string]util.FileProjection{ + "token": {Data: []byte("test_projected_namespace:foo:60:[https://api]"), Mode: 0600}, + }, + }, + { + name: "test good service account defaults expiration", + audience: "https://example.com", + path: "token", + + payload: map[string]util.FileProjection{ + "token": {Data: []byte("test_projected_namespace:foo:3600:[https://example.com]"), Mode: 0600}, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + testNamespace := "test_projected_namespace" + source := makeProjection(tc.name, 0600, "serviceAccountToken") + source.Sources[0].ServiceAccountToken.Audience = tc.audience + source.Sources[0].ServiceAccountToken.ExpirationSeconds = tc.expiration + source.Sources[0].ServiceAccountToken.Path = tc.path + + testPodUID := types.UID("test_pod_uid") + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, UID: testPodUID}, + Spec: v1.PodSpec{ServiceAccountName: "foo"}, + } + scheme.Default(pod) + + client := &fake.Clientset{} + client.AddReactor("create", "serviceaccounts", clitesting.ReactionFunc(func(action clitesting.Action) (bool, runtime.Object, error) { + tr := action.(clitesting.CreateAction).GetObject().(*authenticationv1.TokenRequest) + scheme.Default(tr) + if len(tr.Spec.Audiences) == 0 { + tr.Spec.Audiences = []string{"https://api"} + } + tr.Status.Token = fmt.Sprintf("%v:%v:%d:%v", action.GetNamespace(), "foo", *tr.Spec.ExpirationSeconds, tr.Spec.Audiences) + return true, tr, nil + })) + + _, host := newTestHost(t, client) + + var myVolumeMounter = projectedVolumeMounter{ + projectedVolume: &projectedVolume{ + sources: source.Sources, + podUID: pod.UID, + plugin: &projectedPlugin{ + host: host, + getServiceAccountToken: host.GetServiceAccountTokenFunc(), + }, + }, + source: *source, + pod: pod, + } + + actualPayload, err := myVolumeMounter.collectData() + if err != nil { + t.Fatalf("unexpected failure making payload: %v", err) + } + if e, a := tc.payload, actualPayload; !reflect.DeepEqual(e, a) { + t.Errorf("expected and actual payload do not match:\n%s", diff.ObjectReflectDiff(e, a)) + } + }) + } +} + func newTestHost(t *testing.T, clientset clientset.Interface) (string, volume.VolumeHost) { tempDir, err := ioutil.TempDir("/tmp", "projected_volume_test.") if err != nil { @@ -1113,6 +1227,10 @@ func makeProjection(name string, defaultMode int32, kind string) *v1.ProjectedVo item = v1.VolumeProjection{ DownwardAPI: &v1.DownwardAPIProjection{}, } + case "serviceAccountToken": + item = v1.VolumeProjection{ + ServiceAccountToken: &v1.ServiceAccountTokenProjection{}, + } } return &v1.ProjectedVolumeSource{ diff --git a/pkg/volume/quobyte/BUILD b/pkg/volume/quobyte/BUILD index b120d9c2f7c..1142f5f633a 100644 --- a/pkg/volume/quobyte/BUILD +++ b/pkg/volume/quobyte/BUILD @@ -23,9 +23,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pborman/uuid:go_default_library", "//vendor/github.com/quobyte/api:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/quobyte/quobyte.go b/pkg/volume/quobyte/quobyte.go index 22be50e8f0c..c9d9e773fa7 100644 --- a/pkg/volume/quobyte/quobyte.go +++ b/pkg/volume/quobyte/quobyte.go @@ -22,12 +22,12 @@ import ( "path" gostrings "strings" - "github.com/golang/glog" "github.com/pborman/uuid" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" @@ -94,16 +94,16 @@ func (plugin *quobytePlugin) CanSupport(spec *volume.Spec) bool { qm, _ := mounter.(*quobyteMounter) pluginDir := plugin.host.GetPluginDir(strings.EscapeQualifiedNameForDisk(quobytePluginName)) if mounted, err := qm.pluginDirIsMounted(pluginDir); mounted && err == nil { - glog.V(4).Infof("quobyte: can support") + klog.V(4).Infof("quobyte: can support") return true } } else { - glog.V(4).Infof("quobyte: Error: %v", err) + klog.V(4).Infof("quobyte: Error: %v", err) } exec := plugin.host.GetExec(plugin.GetPluginName()) if out, err := exec.Run("ls", "/sbin/mount.quobyte"); err == nil { - glog.V(4).Infof("quobyte: can support: %s", string(out)) + klog.V(4).Infof("quobyte: can support: %s", string(out)) return true } @@ -260,7 +260,7 @@ func (mounter *quobyteMounter) SetUpAt(dir string, fsGroup *int64) error { return fmt.Errorf("quobyte: mount failed: %v", err) } - glog.V(4).Infof("quobyte: mount set up: %s", dir) + klog.V(4).Infof("quobyte: mount set up: %s", dir) return nil } diff --git a/pkg/volume/quobyte/quobyte_util.go b/pkg/volume/quobyte/quobyte_util.go index d61879b7bb6..107c1454bed 100644 --- a/pkg/volume/quobyte/quobyte_util.go +++ b/pkg/volume/quobyte/quobyte_util.go @@ -25,8 +25,8 @@ import ( "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/volume/util" - "github.com/golang/glog" quobyteapi "github.com/quobyte/api" + "k8s.io/klog" ) type quobyteVolumeManager struct { @@ -63,7 +63,7 @@ func (manager *quobyteVolumeManager) createVolume(provisioner *quobyteVolumeProv } } - glog.V(4).Infof("Created Quobyte volume %s", provisioner.volume) + klog.V(4).Infof("Created Quobyte volume %s", provisioner.volume) return &v1.QuobyteVolumeSource{ Registry: provisioner.registry, Volume: provisioner.volume, @@ -96,7 +96,7 @@ func (mounter *quobyteMounter) pluginDirIsMounted(pluginDir string) (bool, error } if mountPoint.Path == pluginDir { - glog.V(4).Infof("quobyte: found mountpoint %s in /proc/mounts", mountPoint.Path) + klog.V(4).Infof("quobyte: found mountpoint %s in /proc/mounts", mountPoint.Path) return true, nil } } diff --git a/pkg/volume/rbd/BUILD b/pkg/volume/rbd/BUILD index 41bf823ac4b..ccd413eefec 100644 --- a/pkg/volume/rbd/BUILD +++ b/pkg/volume/rbd/BUILD @@ -35,7 +35,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/rbd/attacher.go b/pkg/volume/rbd/attacher.go index 70fc15ea618..97d944f2b5c 100644 --- a/pkg/volume/rbd/attacher.go +++ b/pkg/volume/rbd/attacher.go @@ -21,9 +21,9 @@ import ( "os" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volutil "k8s.io/kubernetes/pkg/volume/util" @@ -107,17 +107,17 @@ func (attacher *rbdAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName t // attach volume onto the node. // This method is idempotent, callers are responsible for retrying on failure. func (attacher *rbdAttacher) WaitForAttach(spec *volume.Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) { - glog.V(4).Infof("rbd: waiting for attach volume (name: %s) for pod (name: %s, uid: %s)", spec.Name(), pod.Name, pod.UID) + klog.V(4).Infof("rbd: waiting for attach volume (name: %s) for pod (name: %s, uid: %s)", spec.Name(), pod.Name, pod.UID) mounter, err := attacher.plugin.createMounterFromVolumeSpecAndPod(spec, pod) if err != nil { - glog.Warningf("failed to create mounter: %v", spec) + klog.Warningf("failed to create mounter: %v", spec) return "", err } realDevicePath, err := attacher.manager.AttachDisk(*mounter) if err != nil { return "", err } - glog.V(3).Infof("rbd: successfully wait for attach volume (spec: %s, pool: %s, image: %s) at %s", spec.Name(), mounter.Pool, mounter.Image, realDevicePath) + klog.V(3).Infof("rbd: successfully wait for attach volume (spec: %s, pool: %s, image: %s) at %s", spec.Name(), mounter.Pool, mounter.Image, realDevicePath) return realDevicePath, nil } @@ -138,7 +138,7 @@ func (attacher *rbdAttacher) GetDeviceMountPath(spec *volume.Spec) (string, erro // mount device at the given mount path. // This method is idempotent, callers are responsible for retrying on failure. func (attacher *rbdAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error { - glog.V(4).Infof("rbd: mouting device %s to %s", devicePath, deviceMountPath) + klog.V(4).Infof("rbd: mouting device %s to %s", devicePath, deviceMountPath) notMnt, err := attacher.mounter.IsLikelyNotMountPoint(deviceMountPath) if err != nil { if os.IsNotExist(err) { @@ -171,7 +171,7 @@ func (attacher *rbdAttacher) MountDevice(spec *volume.Spec, devicePath string, d os.Remove(deviceMountPath) return fmt.Errorf("rbd: failed to mount device %s at %s (fstype: %s), error %v", devicePath, deviceMountPath, fstype, err) } - glog.V(3).Infof("rbd: successfully mount device %s at %s (fstype: %s)", devicePath, deviceMountPath, fstype) + klog.V(3).Infof("rbd: successfully mount device %s at %s (fstype: %s)", devicePath, deviceMountPath, fstype) return nil } @@ -200,7 +200,7 @@ func (detacher *rbdDetacher) UnmountDevice(deviceMountPath string) error { if pathExists, pathErr := volutil.PathExists(deviceMountPath); pathErr != nil { return fmt.Errorf("Error checking if path exists: %v", pathErr) } else if !pathExists { - glog.Warningf("Warning: Unmount skipped because path does not exist: %v", deviceMountPath) + klog.Warningf("Warning: Unmount skipped because path does not exist: %v", deviceMountPath) return nil } devicePath, _, err := mount.GetDeviceNameFromMount(detacher.mounter, deviceMountPath) @@ -208,23 +208,23 @@ func (detacher *rbdDetacher) UnmountDevice(deviceMountPath string) error { return err } // Unmount the device from the device mount point. - glog.V(4).Infof("rbd: unmouting device mountpoint %s", deviceMountPath) + klog.V(4).Infof("rbd: unmouting device mountpoint %s", deviceMountPath) if err = detacher.mounter.Unmount(deviceMountPath); err != nil { return err } - glog.V(3).Infof("rbd: successfully umount device mountpath %s", deviceMountPath) + klog.V(3).Infof("rbd: successfully umount device mountpath %s", deviceMountPath) - glog.V(4).Infof("rbd: detaching device %s", devicePath) + klog.V(4).Infof("rbd: detaching device %s", devicePath) err = detacher.manager.DetachDisk(detacher.plugin, deviceMountPath, devicePath) if err != nil { return err } - glog.V(3).Infof("rbd: successfully detach device %s", devicePath) + klog.V(3).Infof("rbd: successfully detach device %s", devicePath) err = os.Remove(deviceMountPath) if err != nil { return err } - glog.V(3).Infof("rbd: successfully remove device mount point %s", deviceMountPath) + klog.V(3).Infof("rbd: successfully remove device mount point %s", deviceMountPath) return nil } diff --git a/pkg/volume/rbd/disk_manager.go b/pkg/volume/rbd/disk_manager.go index 6f62f485444..6067916da75 100644 --- a/pkg/volume/rbd/disk_manager.go +++ b/pkg/volume/rbd/disk_manager.go @@ -26,9 +26,9 @@ import ( "fmt" "os" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" @@ -61,7 +61,7 @@ func diskSetUp(manager diskManager, b rbdMounter, volPath string, mounter mount. globalPDPath := manager.MakeGlobalPDName(*b.rbd) notMnt, err := mounter.IsLikelyNotMountPoint(globalPDPath) if err != nil && !os.IsNotExist(err) { - glog.Errorf("cannot validate mountpoint: %s", globalPDPath) + klog.Errorf("cannot validate mountpoint: %s", globalPDPath) return err } if notMnt { @@ -70,7 +70,7 @@ func diskSetUp(manager diskManager, b rbdMounter, volPath string, mounter mount. notMnt, err = mounter.IsLikelyNotMountPoint(volPath) if err != nil && !os.IsNotExist(err) { - glog.Errorf("cannot validate mountpoint: %s", volPath) + klog.Errorf("cannot validate mountpoint: %s", volPath) return err } if !notMnt { @@ -78,7 +78,7 @@ func diskSetUp(manager diskManager, b rbdMounter, volPath string, mounter mount. } if err := os.MkdirAll(volPath, 0750); err != nil { - glog.Errorf("failed to mkdir:%s", volPath) + klog.Errorf("failed to mkdir:%s", volPath) return err } // Perform a bind mount to the full path to allow duplicate mounts of the same disk. @@ -89,10 +89,10 @@ func diskSetUp(manager diskManager, b rbdMounter, volPath string, mounter mount. mountOptions := util.JoinMountOptions(b.mountOptions, options) err = mounter.Mount(globalPDPath, volPath, "", mountOptions) if err != nil { - glog.Errorf("failed to bind mount:%s", globalPDPath) + klog.Errorf("failed to bind mount:%s", globalPDPath) return err } - glog.V(3).Infof("rbd: successfully bind mount %s to %s with options %v", globalPDPath, volPath, mountOptions) + klog.V(3).Infof("rbd: successfully bind mount %s to %s with options %v", globalPDPath, volPath, mountOptions) if !b.ReadOnly { volume.SetVolumeOwnership(&b, fsGroup) @@ -105,28 +105,28 @@ func diskSetUp(manager diskManager, b rbdMounter, volPath string, mounter mount. func diskTearDown(manager diskManager, c rbdUnmounter, volPath string, mounter mount.Interface) error { notMnt, err := mounter.IsLikelyNotMountPoint(volPath) if err != nil && !os.IsNotExist(err) { - glog.Errorf("cannot validate mountpoint: %s", volPath) + klog.Errorf("cannot validate mountpoint: %s", volPath) return err } if notMnt { - glog.V(3).Infof("volume path %s is not a mountpoint, deleting", volPath) + klog.V(3).Infof("volume path %s is not a mountpoint, deleting", volPath) return os.Remove(volPath) } // Unmount the bind-mount inside this pod. if err := mounter.Unmount(volPath); err != nil { - glog.Errorf("failed to umount %s", volPath) + klog.Errorf("failed to umount %s", volPath) return err } notMnt, mntErr := mounter.IsLikelyNotMountPoint(volPath) if mntErr != nil && !os.IsNotExist(mntErr) { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return mntErr } if notMnt { if err := os.Remove(volPath); err != nil { - glog.V(2).Info("Error removing mountpoint ", volPath, ": ", err) + klog.V(2).Info("Error removing mountpoint ", volPath, ": ", err) return err } } diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index 4d4f61fbb26..d9197428f8b 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -23,7 +23,6 @@ import ( "regexp" dstrings "strings" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/strings" @@ -385,7 +385,7 @@ func (plugin *rbdPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*vol // the deprecated format: /var/lib/kubelet/plugins/kubernetes.io/rbd/rbd/{pool}-image-{image}. // So we will try to check whether this old style global device mount path exist or not. // If existed, extract the sourceName from this old style path, otherwise return an error. - glog.V(3).Infof("SourceName %s wrong, fallback to old format", sourceName) + klog.V(3).Infof("SourceName %s wrong, fallback to old format", sourceName) sourceName, err = plugin.getDeviceNameFromOldMountPath(mounter, mountPath) if err != nil { return nil, err @@ -415,7 +415,7 @@ func (plugin *rbdPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, if err != nil { return nil, err } - glog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err) + klog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err) globalMapPath := filepath.Dir(globalMapPathUUID) if len(globalMapPath) == 1 { return nil, fmt.Errorf("failed to retrieve volume plugin information from globalMapPathUUID: %v", globalMapPathUUID) @@ -680,10 +680,10 @@ func (r *rbdVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopologie r.rbdMounter.Image = image rbd, sizeMB, err := r.manager.CreateImage(r) if err != nil { - glog.Errorf("rbd: create volume failed, err: %v", err) + klog.Errorf("rbd: create volume failed, err: %v", err) return nil, err } - glog.Infof("successfully created rbd image %q", image) + klog.Infof("successfully created rbd image %q", image) pv := new(v1.PersistentVolume) metav1.SetMetaDataAnnotation(&pv.ObjectMeta, volutil.VolumeDynamicallyCreatedByKey, "rbd-dynamic-provisioner") @@ -824,12 +824,12 @@ func (b *rbdMounter) SetUp(fsGroup *int64) error { func (b *rbdMounter) SetUpAt(dir string, fsGroup *int64) error { // diskSetUp checks mountpoints and prevent repeated calls - glog.V(4).Infof("rbd: attempting to setup at %s", dir) + klog.V(4).Infof("rbd: attempting to setup at %s", dir) err := diskSetUp(b.manager, *b, dir, b.mounter, fsGroup) if err != nil { - glog.Errorf("rbd: failed to setup at %s %v", dir, err) + klog.Errorf("rbd: failed to setup at %s %v", dir, err) } - glog.V(3).Infof("rbd: successfully setup at %s", dir) + klog.V(3).Infof("rbd: successfully setup at %s", dir) return err } @@ -847,18 +847,18 @@ func (c *rbdUnmounter) TearDown() error { } func (c *rbdUnmounter) TearDownAt(dir string) error { - glog.V(4).Infof("rbd: attempting to teardown at %s", dir) + klog.V(4).Infof("rbd: attempting to teardown at %s", dir) if pathExists, pathErr := volutil.PathExists(dir); pathErr != nil { return fmt.Errorf("Error checking if path exists: %v", pathErr) } else if !pathExists { - glog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir) + klog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir) return nil } err := diskTearDown(c.manager, *c, dir, c.mounter) if err != nil { return err } - glog.V(3).Infof("rbd: successfully teardown at %s", dir) + klog.V(3).Infof("rbd: successfully teardown at %s", dir) return nil } @@ -971,13 +971,13 @@ func (rbd *rbdDiskUnmapper) TearDownDevice(mapPath, _ string) error { if err != nil { return fmt.Errorf("rbd: failed to detach disk: %s\nError: %v", mapPath, err) } - glog.V(4).Infof("rbd: %q is unmapped, deleting the directory", mapPath) + klog.V(4).Infof("rbd: %q is unmapped, deleting the directory", mapPath) err = os.RemoveAll(mapPath) if err != nil { return fmt.Errorf("rbd: failed to delete the directory: %s\nError: %v", mapPath, err) } - glog.V(4).Infof("rbd: successfully detached disk: %s", mapPath) + klog.V(4).Infof("rbd: successfully detached disk: %s", mapPath) return nil } @@ -1077,7 +1077,7 @@ func getVolumeAccessModes(spec *volume.Spec) ([]v1.PersistentVolumeAccessMode, e func parsePodSecret(pod *v1.Pod, secretName string, kubeClient clientset.Interface) (string, error) { secret, err := volutil.GetSecretForPod(pod, secretName, kubeClient) if err != nil { - glog.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName) + klog.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName) return "", fmt.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName) } return parseSecretMap(secret) @@ -1086,7 +1086,7 @@ func parsePodSecret(pod *v1.Pod, secretName string, kubeClient clientset.Interfa func parsePVSecret(namespace, secretName string, kubeClient clientset.Interface) (string, error) { secret, err := volutil.GetSecretForPV(namespace, secretName, rbdPluginName, kubeClient) if err != nil { - glog.Errorf("failed to get secret from [%q/%q]", namespace, secretName) + klog.Errorf("failed to get secret from [%q/%q]", namespace, secretName) return "", fmt.Errorf("failed to get secret from [%q/%q]", namespace, secretName) } return parseSecretMap(secret) diff --git a/pkg/volume/rbd/rbd_util.go b/pkg/volume/rbd/rbd_util.go index 0edd02b35d4..2e3de226442 100644 --- a/pkg/volume/rbd/rbd_util.go +++ b/pkg/volume/rbd/rbd_util.go @@ -32,11 +32,11 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" fileutil "k8s.io/kubernetes/pkg/util/file" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/node" @@ -82,21 +82,21 @@ func getRbdDevFromImageAndPool(pool string, image string) (string, bool) { poolFile := path.Join(sys_path, name, "pool") poolBytes, err := ioutil.ReadFile(poolFile) if err != nil { - glog.V(4).Infof("error reading %s: %v", poolFile, err) + klog.V(4).Infof("error reading %s: %v", poolFile, err) continue } if strings.TrimSpace(string(poolBytes)) != pool { - glog.V(4).Infof("device %s is not %q: %q", name, pool, string(poolBytes)) + klog.V(4).Infof("device %s is not %q: %q", name, pool, string(poolBytes)) continue } imgFile := path.Join(sys_path, name, "name") imgBytes, err := ioutil.ReadFile(imgFile) if err != nil { - glog.V(4).Infof("error reading %s: %v", imgFile, err) + klog.V(4).Infof("error reading %s: %v", imgFile, err) continue } if strings.TrimSpace(string(imgBytes)) != image { - glog.V(4).Infof("device %s is not %q: %q", name, image, string(imgBytes)) + klog.V(4).Infof("device %s is not %q: %q", name, image, string(imgBytes)) continue } // Found a match, check if device exists. @@ -119,7 +119,7 @@ func getMaxNbds() (int, error) { return 0, fmt.Errorf("rbd-nbd: failed to retrieve max_nbds from %s err: %q", maxNbdsPath, err) } - glog.V(4).Infof("found nbds max parameters file at %s", maxNbdsPath) + klog.V(4).Infof("found nbds max parameters file at %s", maxNbdsPath) maxNbdBytes, err := ioutil.ReadFile(maxNbdsPath) if err != nil { @@ -131,7 +131,7 @@ func getMaxNbds() (int, error) { return 0, fmt.Errorf("rbd-nbd: failed to read max_nbds err: %q", err) } - glog.V(4).Infof("rbd-nbd: max_nbds: %d", maxNbds) + klog.V(4).Infof("rbd-nbd: max_nbds: %d", maxNbds) return maxNbds, nil } @@ -148,7 +148,7 @@ func getNbdDevFromImageAndPool(pool string, image string) (string, bool) { maxNbds, maxNbdsErr := getMaxNbds() if maxNbdsErr != nil { - glog.V(4).Infof("error reading nbds_max %v", maxNbdsErr) + klog.V(4).Infof("error reading nbds_max %v", maxNbdsErr) return "", false } @@ -156,18 +156,18 @@ func getNbdDevFromImageAndPool(pool string, image string) (string, bool) { nbdPath := basePath + strconv.Itoa(i) _, err := os.Lstat(nbdPath) if err != nil { - glog.V(4).Infof("error reading nbd info directory %s: %v", nbdPath, err) + klog.V(4).Infof("error reading nbd info directory %s: %v", nbdPath, err) continue } pidBytes, err := ioutil.ReadFile(path.Join(nbdPath, "pid")) if err != nil { - glog.V(5).Infof("did not find valid pid file in dir %s: %v", nbdPath, err) + klog.V(5).Infof("did not find valid pid file in dir %s: %v", nbdPath, err) continue } cmdlineFileName := path.Join("/proc", strings.TrimSpace(string(pidBytes)), "cmdline") rawCmdline, err := ioutil.ReadFile(cmdlineFileName) if err != nil { - glog.V(4).Infof("failed to read cmdline file %s: %v", cmdlineFileName, err) + klog.V(4).Infof("failed to read cmdline file %s: %v", cmdlineFileName, err) continue } cmdlineArgs := strings.FieldsFunc(string(rawCmdline), func(r rune) bool { @@ -177,17 +177,17 @@ func getNbdDevFromImageAndPool(pool string, image string) (string, bool) { // Only accepted pattern of cmdline is from execRbdMap: // rbd-nbd map pool/image ... if len(cmdlineArgs) < 3 || cmdlineArgs[0] != "rbd-nbd" || cmdlineArgs[1] != "map" { - glog.V(4).Infof("nbd device %s is not used by rbd", nbdPath) + klog.V(4).Infof("nbd device %s is not used by rbd", nbdPath) continue } if cmdlineArgs[2] != imgPath { - glog.V(4).Infof("rbd-nbd device %s did not match expected image path: %s with path found: %s", + klog.V(4).Infof("rbd-nbd device %s did not match expected image path: %s with path found: %s", nbdPath, imgPath, cmdlineArgs[2]) continue } devicePath := path.Join("/dev", "nbd"+strconv.Itoa(i)) if _, err := os.Lstat(devicePath); err != nil { - glog.Warningf("Stat device %s for imgpath %s failed %v", devicePath, imgPath, err) + klog.Warningf("Stat device %s for imgpath %s failed %v", devicePath, imgPath, err) continue } return devicePath, true @@ -233,14 +233,14 @@ func execRbdMap(b rbdMounter, rbdCmd string, mon string) ([]byte, error) { func checkRbdNbdTools(e mount.Exec) bool { _, err := e.Run("modprobe", "nbd") if err != nil { - glog.V(5).Infof("rbd-nbd: nbd modprobe failed with error %v", err) + klog.V(5).Infof("rbd-nbd: nbd modprobe failed with error %v", err) return false } if _, err := e.Run("rbd-nbd", "--version"); err != nil { - glog.V(5).Infof("rbd-nbd: getting rbd-nbd version failed with error %v", err) + klog.V(5).Infof("rbd-nbd: getting rbd-nbd version failed with error %v", err) return false } - glog.V(3).Infof("rbd-nbd tools were found.") + klog.V(3).Infof("rbd-nbd tools were found.") return true } @@ -251,7 +251,7 @@ func makePDNameInternal(host volume.VolumeHost, pool string, image string) strin info, err := os.Stat(deprecatedDir) if err == nil && info.IsDir() { // The device mount path has already been created with the deprecated format, return it. - glog.V(5).Infof("Deprecated format path %s found", deprecatedDir) + klog.V(5).Infof("Deprecated format path %s found", deprecatedDir) return deprecatedDir } // Return the canonical format path. @@ -331,7 +331,7 @@ func (util *RBDUtil) rbdUnlock(b rbdMounter) error { args = append(args, secret_opt...) cmd, err = b.exec.Run("rbd", args...) output = string(cmd) - glog.V(4).Infof("lock list output %q", output) + klog.V(4).Infof("lock list output %q", output) if err != nil { return err } @@ -349,9 +349,9 @@ func (util *RBDUtil) rbdUnlock(b rbdMounter) error { args = append(args, secret_opt...) cmd, err = b.exec.Run("rbd", args...) if err == nil { - glog.V(4).Infof("rbd: successfully remove lock (locker_id: %s) on image: %s/%s with id %s mon %s", lock_id, b.Pool, b.Image, b.Id, mon) + klog.V(4).Infof("rbd: successfully remove lock (locker_id: %s) on image: %s/%s with id %s mon %s", lock_id, b.Pool, b.Image, b.Id, mon) } else { - glog.Warningf("rbd: failed to remove lock (lock_id: %s) on image: %s/%s with id %s mon %s: %v", lock_id, b.Pool, b.Image, b.Id, mon, err) + klog.Warningf("rbd: failed to remove lock (lock_id: %s) on image: %s/%s with id %s mon %s: %v", lock_id, b.Pool, b.Image, b.Id, mon, err) } } @@ -424,19 +424,19 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) (string, error) { } mon := util.kernelRBDMonitorsOpt(b.Mon) - glog.V(1).Infof("rbd: map mon %s", mon) + klog.V(1).Infof("rbd: map mon %s", mon) _, err := b.exec.Run("modprobe", "rbd") if err != nil { - glog.Warningf("rbd: failed to load rbd kernel module:%v", err) + klog.Warningf("rbd: failed to load rbd kernel module:%v", err) } output, err = execRbdMap(b, "rbd", mon) if err != nil { if !nbdToolsFound { - glog.V(1).Infof("rbd: map error %v, rbd output: %s", err, string(output)) + klog.V(1).Infof("rbd: map error %v, rbd output: %s", err, string(output)) return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output)) } - glog.V(3).Infof("rbd: map failed with %v, %s. Retrying with rbd-nbd", err, string(output)) + klog.V(3).Infof("rbd: map failed with %v, %s. Retrying with rbd-nbd", err, string(output)) errList := []error{err} outputList := output output, err = execRbdMap(b, "rbd-nbd", mon) @@ -481,7 +481,7 @@ func (util *RBDUtil) DetachDisk(plugin *rbdPlugin, deviceMountPath string, devic if err != nil { return rbdErrors(err, fmt.Errorf("rbd: failed to unmap device %s, error %v, rbd output: %v", device, err, output)) } - glog.V(3).Infof("rbd: successfully unmap device %s", device) + klog.V(3).Infof("rbd: successfully unmap device %s", device) // Currently, we don't persist rbd info on the disk, but for backward // compatbility, we need to clean it if found. @@ -491,13 +491,13 @@ func (util *RBDUtil) DetachDisk(plugin *rbdPlugin, deviceMountPath string, devic return err } if exists { - glog.V(3).Infof("rbd: old rbd.json is found under %s, cleaning it", deviceMountPath) + klog.V(3).Infof("rbd: old rbd.json is found under %s, cleaning it", deviceMountPath) err = util.cleanOldRBDFile(plugin, rbdFile) if err != nil { - glog.Errorf("rbd: failed to clean %s", rbdFile) + klog.Errorf("rbd: failed to clean %s", rbdFile) return err } - glog.V(3).Infof("rbd: successfully remove %s", rbdFile) + klog.V(3).Infof("rbd: successfully remove %s", rbdFile) } return nil } @@ -508,7 +508,7 @@ func (util *RBDUtil) DetachBlockDisk(disk rbdDiskUnmapper, mapPath string) error if pathExists, pathErr := volutil.PathExists(mapPath); pathErr != nil { return fmt.Errorf("Error checking if path exists: %v", pathErr) } else if !pathExists { - glog.Warningf("Warning: Unmap skipped because path does not exist: %v", mapPath) + klog.Warningf("Warning: Unmap skipped because path does not exist: %v", mapPath) return nil } // If we arrive here, device is no longer used, see if we need to logout of the target @@ -529,10 +529,10 @@ func (util *RBDUtil) DetachBlockDisk(disk rbdDiskUnmapper, mapPath string) error // Any nbd device must be unmapped by rbd-nbd if strings.HasPrefix(device, "/dev/nbd") { rbdCmd = "rbd-nbd" - glog.V(4).Infof("rbd: using rbd-nbd for unmap function") + klog.V(4).Infof("rbd: using rbd-nbd for unmap function") } else { rbdCmd = "rbd" - glog.V(4).Infof("rbd: using rbd for unmap function") + klog.V(4).Infof("rbd: using rbd for unmap function") } // rbd unmap @@ -540,7 +540,7 @@ func (util *RBDUtil) DetachBlockDisk(disk rbdDiskUnmapper, mapPath string) error if err != nil { return rbdErrors(err, fmt.Errorf("rbd: failed to unmap device %s, error %v, rbd output: %s", device, err, string(output))) } - glog.V(3).Infof("rbd: successfully unmap device %s", device) + klog.V(3).Infof("rbd: successfully unmap device %s", device) return nil } @@ -564,7 +564,7 @@ func (util *RBDUtil) cleanOldRBDFile(plugin *rbdPlugin, rbdFile string) error { } if err != nil { - glog.Errorf("failed to load rbd info from %s: %v", rbdFile, err) + klog.Errorf("failed to load rbd info from %s: %v", rbdFile, err) return err } // Remove rbd lock if found. @@ -589,9 +589,9 @@ func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDPersistentVo volSz := fmt.Sprintf("%d", sz) mon := util.kernelRBDMonitorsOpt(p.Mon) if p.rbdMounter.imageFormat == rbdImageFormat2 { - glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, p.rbdMounter.imageFormat, p.rbdMounter.imageFeatures, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret) + klog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, p.rbdMounter.imageFormat, p.rbdMounter.imageFeatures, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret) } else { - glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, p.rbdMounter.imageFormat, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret) + klog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, p.rbdMounter.imageFormat, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret) } args := []string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", p.rbdMounter.imageFormat} if p.rbdMounter.imageFormat == rbdImageFormat2 { @@ -603,7 +603,7 @@ func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDPersistentVo output, err = p.exec.Run("rbd", args...) if err != nil { - glog.Warningf("failed to create rbd image, output %v", string(output)) + klog.Warningf("failed to create rbd image, output %v", string(output)) return nil, 0, fmt.Errorf("failed to create rbd image: %v, command output: %s", err, string(output)) } @@ -621,19 +621,19 @@ func (util *RBDUtil) DeleteImage(p *rbdVolumeDeleter) error { return fmt.Errorf("error %v, rbd output: %v", err, rbdOutput) } if found { - glog.Info("rbd is still being used ", p.rbdMounter.Image) + klog.Info("rbd is still being used ", p.rbdMounter.Image) return fmt.Errorf("rbd image %s/%s is still being used, rbd output: %v", p.rbdMounter.Pool, p.rbdMounter.Image, rbdOutput) } // rbd rm. mon := util.kernelRBDMonitorsOpt(p.rbdMounter.Mon) - glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret) + klog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret) output, err = p.exec.Run("rbd", "rm", p.rbdMounter.Image, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key="+p.rbdMounter.adminSecret) if err == nil { return nil } - glog.Errorf("failed to delete rbd image: %v, command output: %s", err, string(output)) + klog.Errorf("failed to delete rbd image: %v, command output: %s", err, string(output)) return fmt.Errorf("error %v, rbd output: %v", err, string(output)) } @@ -658,14 +658,14 @@ func (util *RBDUtil) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resourc // rbd resize. mon := util.kernelRBDMonitorsOpt(rbdExpander.rbdMounter.Mon) - glog.V(4).Infof("rbd: resize %s using mon %s, pool %s id %s key %s", rbdExpander.rbdMounter.Image, mon, rbdExpander.rbdMounter.Pool, rbdExpander.rbdMounter.adminId, rbdExpander.rbdMounter.adminSecret) + klog.V(4).Infof("rbd: resize %s using mon %s, pool %s id %s key %s", rbdExpander.rbdMounter.Image, mon, rbdExpander.rbdMounter.Pool, rbdExpander.rbdMounter.adminId, rbdExpander.rbdMounter.adminSecret) output, err = rbdExpander.exec.Run("rbd", "resize", rbdExpander.rbdMounter.Image, "--size", newVolSz, "--pool", rbdExpander.rbdMounter.Pool, "--id", rbdExpander.rbdMounter.adminId, "-m", mon, "--key="+rbdExpander.rbdMounter.adminSecret) if err == nil { return newSizeQuant, nil } - glog.Errorf("failed to resize rbd image: %v, command output: %s", err, string(output)) + klog.Errorf("failed to resize rbd image: %v, command output: %s", err, string(output)) return oldSize, err } @@ -701,14 +701,14 @@ func (util *RBDUtil) rbdInfo(b *rbdMounter) (int, error) { // # image does not exist (exit=2) // rbd: error opening image 1234: (2) No such file or directory // - glog.V(4).Infof("rbd: info %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, id, secret) + klog.V(4).Infof("rbd: info %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, id, secret) cmd, err = b.exec.Run("rbd", "info", b.Image, "--pool", b.Pool, "-m", mon, "--id", id, "--key="+secret) output = string(cmd) if err, ok := err.(*exec.Error); ok { if err.Err == exec.ErrNotFound { - glog.Errorf("rbd cmd not found") + klog.Errorf("rbd cmd not found") // fail fast if rbd command is not found. return 0, err } @@ -767,14 +767,14 @@ func (util *RBDUtil) rbdStatus(b *rbdMounter) (bool, string, error) { // # image does not exist (exit=2) // rbd: error opening image kubernetes-dynamic-pvc-: (2) No such file or directory // - glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, id, secret) + klog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, id, secret) cmd, err = b.exec.Run("rbd", "status", b.Image, "--pool", b.Pool, "-m", mon, "--id", id, "--key="+secret) output = string(cmd) if err, ok := err.(*exec.Error); ok { if err.Err == exec.ErrNotFound { - glog.Errorf("rbd cmd not found") + klog.Errorf("rbd cmd not found") // fail fast if command not found return false, output, err } @@ -786,10 +786,10 @@ func (util *RBDUtil) rbdStatus(b *rbdMounter) (bool, string, error) { } if strings.Contains(output, imageWatcherStr) { - glog.V(4).Infof("rbd: watchers on %s: %s", b.Image, output) + klog.V(4).Infof("rbd: watchers on %s: %s", b.Image, output) return true, output, nil } else { - glog.Warningf("rbd: no watchers on %s", b.Image) + klog.Warningf("rbd: no watchers on %s", b.Image) return false, output, nil } } diff --git a/pkg/volume/scaleio/BUILD b/pkg/volume/scaleio/BUILD index 767ad4a4033..526fcbe582b 100644 --- a/pkg/volume/scaleio/BUILD +++ b/pkg/volume/scaleio/BUILD @@ -25,7 +25,7 @@ go_test( "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", "//vendor/github.com/codedellemc/goscaleio/types/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -52,7 +52,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//vendor/github.com/codedellemc/goscaleio:go_default_library", "//vendor/github.com/codedellemc/goscaleio/types/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/scaleio/sio_client.go b/pkg/volume/scaleio/sio_client.go index de5b050019f..2c7041f478e 100644 --- a/pkg/volume/scaleio/sio_client.go +++ b/pkg/volume/scaleio/sio_client.go @@ -33,7 +33,7 @@ import ( sio "github.com/codedellemc/goscaleio" siotypes "github.com/codedellemc/goscaleio/types/v1" - "github.com/golang/glog" + "k8s.io/klog" ) var ( @@ -74,7 +74,7 @@ type sioClient struct { spClient *sio.StoragePool provisionMode string sdcPath string - sdcGuid string + sdcGUID string instanceID string inited bool diskRegex *regexp.Regexp @@ -97,7 +97,7 @@ func newSioClient(gateway, username, password string, sslEnabled bool, exec moun } r, err := regexp.Compile(`^emc-vol-\w*-\w*$`) if err != nil { - glog.Error(log("failed to compile regex: %v", err)) + klog.Error(log("failed to compile regex: %v", err)) return nil, err } client.diskRegex = r @@ -113,10 +113,10 @@ func (c *sioClient) init() error { if c.inited { return nil } - glog.V(4).Infoln(log("initializing scaleio client")) + klog.V(4).Infoln(log("initializing scaleio client")) client, err := sio.NewClientWithArgs(c.gateway, "", c.insecure, c.certsEnabled) if err != nil { - glog.Error(log("failed to create client: %v", err)) + klog.Error(log("failed to create client: %v", err)) return err } c.client = client @@ -127,24 +127,24 @@ func (c *sioClient) init() error { Username: c.username, Password: c.password}, ); err != nil { - glog.Error(log("client authentication failed: %v", err)) + klog.Error(log("client authentication failed: %v", err)) return err } // retrieve system if c.system, err = c.findSystem(c.sysName); err != nil { - glog.Error(log("unable to find system %s: %v", c.sysName, err)) + klog.Error(log("unable to find system %s: %v", c.sysName, err)) return err } // retrieve protection domain if c.protectionDomain, err = c.findProtectionDomain(c.pdName); err != nil { - glog.Error(log("unable to find protection domain %s: %v", c.protectionDomain, err)) + klog.Error(log("unable to find protection domain %s: %v", c.protectionDomain, err)) return err } // retrieve storage pool if c.storagePool, err = c.findStoragePool(c.spName); err != nil { - glog.Error(log("unable to find storage pool %s: %v", c.storagePool, err)) + klog.Error(log("unable to find storage pool %s: %v", c.storagePool, err)) return err } c.inited = true @@ -157,7 +157,7 @@ func (c *sioClient) Volumes() ([]*siotypes.Volume, error) { } vols, err := c.getVolumes() if err != nil { - glog.Error(log("failed to retrieve volumes: %v", err)) + klog.Error(log("failed to retrieve volumes: %v", err)) return nil, err } return vols, nil @@ -170,12 +170,12 @@ func (c *sioClient) Volume(id sioVolumeID) (*siotypes.Volume, error) { vols, err := c.getVolumesByID(id) if err != nil { - glog.Error(log("failed to retrieve volume by id: %v", err)) + klog.Error(log("failed to retrieve volume by id: %v", err)) return nil, err } vol := vols[0] if vol == nil { - glog.V(4).Info(log("volume not found, id %s", id)) + klog.V(4).Info(log("volume not found, id %s", id)) return nil, errors.New("volume not found") } return vol, nil @@ -186,20 +186,20 @@ func (c *sioClient) FindVolume(name string) (*siotypes.Volume, error) { return nil, err } - glog.V(4).Info(log("searching for volume %s", name)) + klog.V(4).Info(log("searching for volume %s", name)) volumes, err := c.getVolumesByName(name) if err != nil { - glog.Error(log("failed to find volume by name %v", err)) + klog.Error(log("failed to find volume by name %v", err)) return nil, err } for _, volume := range volumes { if volume.Name == name { - glog.V(4).Info(log("found volume %s", name)) + klog.V(4).Info(log("found volume %s", name)) return volume, nil } } - glog.V(4).Info(log("volume not found, name %s", name)) + klog.V(4).Info(log("volume not found, name %s", name)) return nil, errors.New("volume not found") } @@ -215,7 +215,7 @@ func (c *sioClient) CreateVolume(name string, sizeGB int64) (*siotypes.Volume, e } createResponse, err := c.client.CreateVolume(params, c.storagePool.Name) if err != nil { - glog.Error(log("failed to create volume %s: %v", name, err)) + klog.Error(log("failed to create volume %s: %v", name, err)) return nil, err } return c.Volume(sioVolumeID(createResponse.ID)) @@ -225,13 +225,13 @@ func (c *sioClient) CreateVolume(name string, sizeGB int64) (*siotypes.Volume, e // is true, ScaleIO will allow other SDC to map to that volume. func (c *sioClient) AttachVolume(id sioVolumeID, multipleMappings bool) error { if err := c.init(); err != nil { - glog.Error(log("failed to init'd client in attach volume: %v", err)) + klog.Error(log("failed to init'd client in attach volume: %v", err)) return err } iid, err := c.IID() if err != nil { - glog.Error(log("failed to get instanceIID for attach volume: %v", err)) + klog.Error(log("failed to get instanceIID for attach volume: %v", err)) return err } @@ -244,11 +244,11 @@ func (c *sioClient) AttachVolume(id sioVolumeID, multipleMappings bool) error { volClient.Volume = &siotypes.Volume{ID: string(id)} if err := volClient.MapVolumeSdc(params); err != nil { - glog.Error(log("failed to attach volume id %s: %v", id, err)) + klog.Error(log("failed to attach volume id %s: %v", id, err)) return err } - glog.V(4).Info(log("volume %s attached successfully", id)) + klog.V(4).Info(log("volume %s attached successfully", id)) return nil } @@ -301,35 +301,35 @@ func (c *sioClient) IID() (string, error) { // if instanceID not set, retrieve it if c.instanceID == "" { - guid, err := c.getGuid() + guid, err := c.getGUID() if err != nil { return "", err } - sdc, err := c.sysClient.FindSdc("SdcGuid", guid) + sdc, err := c.sysClient.FindSdc("SdcGUID", guid) if err != nil { - glog.Error(log("failed to retrieve sdc info %s", err)) + klog.Error(log("failed to retrieve sdc info %s", err)) return "", err } c.instanceID = sdc.Sdc.ID - glog.V(4).Info(log("retrieved instanceID %s", c.instanceID)) + klog.V(4).Info(log("retrieved instanceID %s", c.instanceID)) } return c.instanceID, nil } -// getGuid returns instance GUID, if not set using resource labels +// getGUID returns instance GUID, if not set using resource labels // it attempts to fallback to using drv_cfg binary -func (c *sioClient) getGuid() (string, error) { - if c.sdcGuid == "" { - glog.V(4).Info(log("sdc guid label not set, falling back to using drv_cfg")) +func (c *sioClient) getGUID() (string, error) { + if c.sdcGUID == "" { + klog.V(4).Info(log("sdc guid label not set, falling back to using drv_cfg")) cmd := c.getSdcCmd() output, err := c.exec.Run(cmd, "--query_guid") if err != nil { - glog.Error(log("drv_cfg --query_guid failed: %v", err)) + klog.Error(log("drv_cfg --query_guid failed: %v", err)) return "", err } - c.sdcGuid = strings.TrimSpace(string(output)) + c.sdcGUID = strings.TrimSpace(string(output)) } - return c.sdcGuid, nil + return c.sdcGUID, nil } // getSioDiskPaths traverse local disk devices to retrieve device path @@ -342,10 +342,10 @@ func (c *sioClient) getSioDiskPaths() ([]os.FileInfo, error) { if os.IsNotExist(err) { // sioDiskIDPath may not exist yet which is fine return []os.FileInfo{}, nil - } else { - glog.Error(log("failed to ReadDir %s: %v", sioDiskIDPath, err)) - return nil, err } + klog.Error(log("failed to ReadDir %s: %v", sioDiskIDPath, err)) + return nil, err + } result := []os.FileInfo{} for _, file := range files { @@ -360,13 +360,13 @@ func (c *sioClient) getSioDiskPaths() ([]os.FileInfo, error) { // GetVolumeRefs counts the number of references an SIO volume has a disk device. // This is useful in preventing premature detach. -func (c *sioClient) GetVolumeRefs(volId sioVolumeID) (refs int, err error) { +func (c *sioClient) GetVolumeRefs(volID sioVolumeID) (refs int, err error) { files, err := c.getSioDiskPaths() if err != nil { return 0, err } for _, file := range files { - if strings.Contains(file.Name(), string(volId)) { + if strings.Contains(file.Name(), string(volID)) { refs++ } } @@ -391,7 +391,7 @@ func (c *sioClient) Devs() (map[string]string, error) { volumeID := parts[3] devPath, err := filepath.EvalSymlinks(fmt.Sprintf("%s/%s", sioDiskIDPath, f.Name())) if err != nil { - glog.Error(log("devicepath-to-volID mapping error: %v", err)) + klog.Error(log("devicepath-to-volID mapping error: %v", err)) return nil, err } // map volumeID to devicePath @@ -417,18 +417,18 @@ func (c *sioClient) WaitForAttachedDevice(token string) (string, error) { case <-ticker.C: devMap, err := c.Devs() if err != nil { - glog.Error(log("failed while waiting for volume to attach: %v", err)) + klog.Error(log("failed while waiting for volume to attach: %v", err)) return "", err } go func() { - glog.V(4).Info(log("waiting for volume %s to be mapped/attached", token)) + klog.V(4).Info(log("waiting for volume %s to be mapped/attached", token)) }() if path, ok := devMap[token]; ok { - glog.V(4).Info(log("device %s mapped to vol %s", path, token)) + klog.V(4).Info(log("device %s mapped to vol %s", path, token)) return path, nil } case <-timer.C: - glog.Error(log("timed out while waiting for volume to be mapped to a device")) + klog.Error(log("timed out while waiting for volume to be mapped to a device")) return "", fmt.Errorf("volume attach timeout") } } @@ -451,18 +451,18 @@ func (c *sioClient) WaitForDetachedDevice(token string) error { case <-ticker.C: devMap, err := c.Devs() if err != nil { - glog.Error(log("failed while waiting for volume to unmap/detach: %v", err)) + klog.Error(log("failed while waiting for volume to unmap/detach: %v", err)) return err } go func() { - glog.V(4).Info(log("waiting for volume %s to be unmapped/detached", token)) + klog.V(4).Info(log("waiting for volume %s to be unmapped/detached", token)) }() // cant find vol id, then ok. if _, ok := devMap[token]; !ok { return nil } case <-timer.C: - glog.Error(log("timed out while waiting for volume %s to be unmapped/detached", token)) + klog.Error(log("timed out while waiting for volume %s to be unmapped/detached", token)) return fmt.Errorf("volume detach timeout") } } @@ -477,7 +477,7 @@ func (c *sioClient) findSystem(sysname string) (sys *siotypes.System, err error) } systems, err := c.client.GetInstance("") if err != nil { - glog.Error(log("failed to retrieve instances: %v", err)) + klog.Error(log("failed to retrieve instances: %v", err)) return nil, err } for _, sys = range systems { @@ -485,7 +485,7 @@ func (c *sioClient) findSystem(sysname string) (sys *siotypes.System, err error) return sys, nil } } - glog.Error(log("system %s not found", sysname)) + klog.Error(log("system %s not found", sysname)) return nil, errors.New("system not found") } @@ -494,13 +494,13 @@ func (c *sioClient) findProtectionDomain(pdname string) (*siotypes.ProtectionDom if c.sysClient != nil { protectionDomain, err := c.sysClient.FindProtectionDomain("", pdname, "") if err != nil { - glog.Error(log("failed to retrieve protection domains: %v", err)) + klog.Error(log("failed to retrieve protection domains: %v", err)) return nil, err } c.pdClient.ProtectionDomain = protectionDomain return protectionDomain, nil } - glog.Error(log("protection domain %s not set", pdname)) + klog.Error(log("protection domain %s not set", pdname)) return nil, errors.New("protection domain not set") } @@ -509,13 +509,13 @@ func (c *sioClient) findStoragePool(spname string) (*siotypes.StoragePool, error if c.pdClient != nil { sp, err := c.pdClient.FindStoragePool("", spname, "") if err != nil { - glog.Error(log("failed to retrieve storage pool: %v", err)) + klog.Error(log("failed to retrieve storage pool: %v", err)) return nil, err } c.spClient.StoragePool = sp return sp, nil } - glog.Error(log("storage pool %s not set", spname)) + klog.Error(log("storage pool %s not set", spname)) return nil, errors.New("storage pool not set") } diff --git a/pkg/volume/scaleio/sio_mgr.go b/pkg/volume/scaleio/sio_mgr.go index 711fa7fce5d..a322276b1dd 100644 --- a/pkg/volume/scaleio/sio_mgr.go +++ b/pkg/volume/scaleio/sio_mgr.go @@ -22,7 +22,7 @@ import ( "k8s.io/kubernetes/pkg/util/mount" - "github.com/golang/glog" + "k8s.io/klog" siotypes "github.com/codedellemc/goscaleio/types/v1" ) @@ -57,22 +57,22 @@ func newSioMgr(configs map[string]string, exec mount.Exec) (*sioMgr, error) { // getClient safely returns an sioInterface func (m *sioMgr) getClient() (sioInterface, error) { if m.client == nil { - glog.V(4).Info(log("creating scaleio client")) + klog.V(4).Info(log("creating scaleio client")) configs := m.configData username := configs[confKey.username] password := configs[confKey.password] gateway := configs[confKey.gateway] b, err := strconv.ParseBool(configs[confKey.sslEnabled]) if err != nil { - glog.Error(log("failed to parse sslEnabled, must be either \"true\" or \"false\"")) + klog.Error(log("failed to parse sslEnabled, must be either \"true\" or \"false\"")) return nil, err } certsEnabled := b - glog.V(4).Info(log("creating new client for gateway %s", gateway)) + klog.V(4).Info(log("creating new client for gateway %s", gateway)) client, err := newSioClient(gateway, username, password, certsEnabled, m.exec) if err != nil { - glog.Error(log("failed to create scaleio client: %v", err)) + klog.Error(log("failed to create scaleio client: %v", err)) return nil, err } @@ -81,11 +81,11 @@ func (m *sioMgr) getClient() (sioInterface, error) { client.spName = configs[confKey.storagePool] client.sdcPath = configs[confKey.sdcRootPath] client.provisionMode = configs[confKey.storageMode] - client.sdcGuid = configs[confKey.sdcGuid] + client.sdcGUID = configs[confKey.sdcGUID] m.client = client - glog.V(4).Info(log("client created successfully [gateway=%s]", gateway)) + klog.V(4).Info(log("client created successfully [gateway=%s]", gateway)) } return m.client, nil } @@ -97,13 +97,13 @@ func (m *sioMgr) CreateVolume(volName string, sizeGB int64) (*siotypes.Volume, e return nil, err } - glog.V(4).Infof("scaleio: creating volume %s", volName) + klog.V(4).Infof("scaleio: creating volume %s", volName) vol, err := client.CreateVolume(volName, sizeGB) if err != nil { - glog.V(4).Infof("scaleio: failed creating volume %s: %v", volName, err) + klog.V(4).Infof("scaleio: failed creating volume %s: %v", volName, err) return nil, err } - glog.V(4).Infof("scaleio: created volume %s successfully", volName) + klog.V(4).Infof("scaleio: created volume %s successfully", volName) return vol, nil } @@ -112,17 +112,17 @@ func (m *sioMgr) CreateVolume(volName string, sizeGB int64) (*siotypes.Volume, e func (m *sioMgr) AttachVolume(volName string, multipleMappings bool) (string, error) { client, err := m.getClient() if err != nil { - glog.Error(log("attach volume failed: %v", err)) + klog.Error(log("attach volume failed: %v", err)) return "", err } - glog.V(4).Infoln(log("attaching volume %s", volName)) + klog.V(4).Infoln(log("attaching volume %s", volName)) iid, err := client.IID() if err != nil { - glog.Error(log("failed to get instanceID")) + klog.Error(log("failed to get instanceID")) return "", err } - glog.V(4).Info(log("attaching volume %s to host instance %s", volName, iid)) + klog.V(4).Info(log("attaching volume %s to host instance %s", volName, iid)) devs, err := client.Devs() if err != nil { @@ -131,29 +131,29 @@ func (m *sioMgr) AttachVolume(volName string, multipleMappings bool) (string, er vol, err := client.FindVolume(volName) if err != nil { - glog.Error(log("failed to find volume %s: %v", volName, err)) + klog.Error(log("failed to find volume %s: %v", volName, err)) return "", err } // handle vol if already attached if len(vol.MappedSdcInfo) > 0 { if m.isSdcMappedToVol(iid, vol) { - glog.V(4).Info(log("skippping attachment, volume %s already attached to sdc %s", volName, iid)) + klog.V(4).Info(log("skippping attachment, volume %s already attached to sdc %s", volName, iid)) return devs[vol.ID], nil } } // attach volume, get deviceName if err := client.AttachVolume(sioVolumeID(vol.ID), multipleMappings); err != nil { - glog.Error(log("attachment for volume %s failed :%v", volName, err)) + klog.Error(log("attachment for volume %s failed :%v", volName, err)) return "", err } device, err := client.WaitForAttachedDevice(vol.ID) if err != nil { - glog.Error(log("failed while waiting for device to attach: %v", err)) + klog.Error(log("failed while waiting for device to attach: %v", err)) return "", err } - glog.V(4).Info(log("volume %s attached successfully as %s to instance %s", volName, device, iid)) + klog.V(4).Info(log("volume %s attached successfully as %s to instance %s", volName, device, iid)) return device, nil } @@ -165,7 +165,7 @@ func (m *sioMgr) IsAttached(volName string) (bool, error) { } iid, err := client.IID() if err != nil { - glog.Error("scaleio: failed to get instanceID") + klog.Error("scaleio: failed to get instanceID") return false, err } @@ -184,7 +184,7 @@ func (m *sioMgr) DetachVolume(volName string) error { } iid, err := client.IID() if err != nil { - glog.Error(log("failed to get instanceID: %v", err)) + klog.Error(log("failed to get instanceID: %v", err)) return err } @@ -193,7 +193,7 @@ func (m *sioMgr) DetachVolume(volName string) error { return err } if !m.isSdcMappedToVol(iid, vol) { - glog.Warning(log( + klog.Warning(log( "skipping detached, vol %s not attached to instance %s", volName, iid, )) @@ -201,11 +201,11 @@ func (m *sioMgr) DetachVolume(volName string) error { } if err := client.DetachVolume(sioVolumeID(vol.ID)); err != nil { - glog.Error(log("failed to detach vol %s: %v", volName, err)) + klog.Error(log("failed to detach vol %s: %v", volName, err)) return err } - glog.V(4).Info(log("volume %s detached successfully", volName)) + klog.V(4).Info(log("volume %s detached successfully", volName)) return nil } @@ -223,11 +223,11 @@ func (m *sioMgr) DeleteVolume(volName string) error { } if err := client.DeleteVolume(sioVolumeID(vol.ID)); err != nil { - glog.Error(log("failed to delete volume %s: %v", volName, err)) + klog.Error(log("failed to delete volume %s: %v", volName, err)) return err } - glog.V(4).Info(log("deleted volume %s successfully", volName)) + klog.V(4).Info(log("deleted volume %s successfully", volName)) return nil } @@ -235,7 +235,7 @@ func (m *sioMgr) DeleteVolume(volName string) error { // isSdcMappedToVol returns true if the sdc is mapped to the volume func (m *sioMgr) isSdcMappedToVol(sdcID string, vol *siotypes.Volume) bool { if len(vol.MappedSdcInfo) == 0 { - glog.V(4).Info(log("no attachment found")) + klog.V(4).Info(log("no attachment found")) return false } diff --git a/pkg/volume/scaleio/sio_mgr_test.go b/pkg/volume/scaleio/sio_mgr_test.go index 49ff5d05a81..b1dff3ca1a8 100644 --- a/pkg/volume/scaleio/sio_mgr_test.go +++ b/pkg/volume/scaleio/sio_mgr_test.go @@ -311,7 +311,7 @@ func (f *fakeSio) Devs() (map[string]string, error) { return f.devs, nil } -func (f *fakeSio) GetVolumeRefs(volId sioVolumeID) (int, error) { +func (f *fakeSio) GetVolumeRefs(volID sioVolumeID) (int, error) { if f.volume == nil { return 0, nil } diff --git a/pkg/volume/scaleio/sio_plugin.go b/pkg/volume/scaleio/sio_plugin.go index 367c9b9638d..eda8556f7a9 100644 --- a/pkg/volume/scaleio/sio_plugin.go +++ b/pkg/volume/scaleio/sio_plugin.go @@ -19,9 +19,9 @@ package scaleio import ( "errors" - "github.com/golang/glog" api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/keymutex" "k8s.io/kubernetes/pkg/volume" ) @@ -36,6 +36,7 @@ type sioPlugin struct { volumeMtx keymutex.KeyMutex } +// ProbeVolumePlugins is the primary entrypoint for volume plugins. func ProbeVolumePlugins() []volume.VolumePlugin { p := &sioPlugin{ host: nil, @@ -107,7 +108,7 @@ func (p *sioPlugin) NewMounter( // NewUnmounter creates a representation of the volume to unmount func (p *sioPlugin) NewUnmounter(specName string, podUID types.UID) (volume.Unmounter, error) { - glog.V(4).Info(log("Unmounter for %s", specName)) + klog.V(4).Info(log("Unmounter for %s", specName)) return &sioVolume{ podUID: podUID, @@ -160,7 +161,7 @@ var _ volume.DeletableVolumePlugin = &sioPlugin{} func (p *sioPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) { attribs, err := getVolumeSourceAttribs(spec) if err != nil { - glog.Error(log("deleter failed to extract volume attributes from spec: %v", err)) + klog.Error(log("deleter failed to extract volume attributes from spec: %v", err)) return nil, err } @@ -186,11 +187,11 @@ func (p *sioPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) { var _ volume.ProvisionableVolumePlugin = &sioPlugin{} func (p *sioPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) { - glog.V(4).Info(log("creating Provisioner")) + klog.V(4).Info(log("creating Provisioner")) configData := options.Parameters if configData == nil { - glog.Error(log("provisioner missing parameters, unable to continue")) + klog.Error(log("provisioner missing parameters, unable to continue")) return nil, errors.New("option parameters missing") } diff --git a/pkg/volume/scaleio/sio_util.go b/pkg/volume/scaleio/sio_util.go index de0c5811665..e1b5116318f 100644 --- a/pkg/volume/scaleio/sio_util.go +++ b/pkg/volume/scaleio/sio_util.go @@ -24,7 +24,7 @@ import ( "path" "strconv" - "github.com/golang/glog" + "k8s.io/klog" api "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/volume" @@ -54,7 +54,7 @@ var ( username, password, secretNamespace, - sdcGuid string + sdcGUID string }{ gateway: "gateway", sslEnabled: "sslEnabled", @@ -71,9 +71,9 @@ var ( readOnly: "readOnly", username: "username", password: "password", - sdcGuid: "sdcGuid", + sdcGUID: "sdcGUID", } - sdcGuidLabelName = "scaleio.sdcGuid" + sdcGUIDLabelName = "scaleio.sdcGUID" sdcRootPath = "/opt/emc/scaleio/sdc/bin" secretNotFoundErr = errors.New("secret not found") @@ -140,7 +140,7 @@ func validateConfigs(config map[string]string) error { func applyConfigDefaults(config map[string]string) { b, err := strconv.ParseBool(config[confKey.sslEnabled]) if err != nil { - glog.Warning(log("failed to parse param sslEnabled, setting it to false")) + klog.Warning(log("failed to parse param sslEnabled, setting it to false")) b = false } config[confKey.sslEnabled] = strconv.FormatBool(b) @@ -148,7 +148,7 @@ func applyConfigDefaults(config map[string]string) { config[confKey.fsType] = defaultString(config[confKey.fsType], "xfs") b, err = strconv.ParseBool(config[confKey.readOnly]) if err != nil { - glog.Warning(log("failed to parse param readOnly, setting it to false")) + klog.Warning(log("failed to parse param readOnly, setting it to false")) b = false } config[confKey.readOnly] = strconv.FormatBool(b) @@ -163,21 +163,21 @@ func defaultString(val, defVal string) string { // loadConfig loads configuration data from a file on disk func loadConfig(configName string) (map[string]string, error) { - glog.V(4).Info(log("loading config file %s", configName)) + klog.V(4).Info(log("loading config file %s", configName)) file, err := os.Open(configName) if err != nil { - glog.Error(log("failed to open config file %s: %v", configName, err)) + klog.Error(log("failed to open config file %s: %v", configName, err)) return nil, err } defer file.Close() data := map[string]string{} if err := gob.NewDecoder(file).Decode(&data); err != nil { - glog.Error(log("failed to parse config data %s: %v", configName, err)) + klog.Error(log("failed to parse config data %s: %v", configName, err)) return nil, err } applyConfigDefaults(data) if err := validateConfigs(data); err != nil { - glog.Error(log("failed to load ConfigMap %s: %v", err)) + klog.Error(log("failed to load ConfigMap %s: %v", err)) return nil, err } @@ -186,31 +186,31 @@ func loadConfig(configName string) (map[string]string, error) { // saveConfig saves the configuration data to local disk func saveConfig(configName string, data map[string]string) error { - glog.V(4).Info(log("saving config file %s", configName)) + klog.V(4).Info(log("saving config file %s", configName)) dir := path.Dir(configName) if _, err := os.Stat(dir); err != nil { if !os.IsNotExist(err) { return err } - glog.V(4).Info(log("creating config dir for config data: %s", dir)) + klog.V(4).Info(log("creating config dir for config data: %s", dir)) if err := os.MkdirAll(dir, 0750); err != nil { - glog.Error(log("failed to create config data dir %v", err)) + klog.Error(log("failed to create config data dir %v", err)) return err } } file, err := os.Create(configName) if err != nil { - glog.V(4).Info(log("failed to save config data file %s: %v", configName, err)) + klog.V(4).Info(log("failed to save config data file %s: %v", configName, err)) return err } defer file.Close() if err := gob.NewEncoder(file).Encode(data); err != nil { - glog.Error(log("failed to save config %s: %v", configName, err)) + klog.Error(log("failed to save config %s: %v", configName, err)) return err } - glog.V(4).Info(log("config data file saved successfully as %s", configName)) + klog.V(4).Info(log("config data file saved successfully as %s", configName)) return nil } @@ -221,7 +221,7 @@ func attachSecret(plug *sioPlugin, namespace string, configData map[string]strin kubeClient := plug.host.GetKubeClient() secretMap, err := volutil.GetSecretForPV(namespace, secretRefName, sioPluginName, kubeClient) if err != nil { - glog.Error(log("failed to get secret: %v", err)) + klog.Error(log("failed to get secret: %v", err)) return secretNotFoundErr } // merge secret data @@ -232,30 +232,30 @@ func attachSecret(plug *sioPlugin, namespace string, configData map[string]strin return nil } -// attachSdcGuid injects the sdc guid node label value into config -func attachSdcGuid(plug *sioPlugin, conf map[string]string) error { - guid, err := getSdcGuidLabel(plug) +// attachSdcGUID injects the sdc guid node label value into config +func attachSdcGUID(plug *sioPlugin, conf map[string]string) error { + guid, err := getSdcGUIDLabel(plug) if err != nil { return err } - conf[confKey.sdcGuid] = guid + conf[confKey.sdcGUID] = guid return nil } -// getSdcGuidLabel fetches the scaleio.sdcGuid node label +// getSdcGUIDLabel fetches the scaleio.sdcGuid node label // associated with the node executing this code. -func getSdcGuidLabel(plug *sioPlugin) (string, error) { +func getSdcGUIDLabel(plug *sioPlugin) (string, error) { nodeLabels, err := plug.host.GetNodeLabels() if err != nil { return "", err } - label, ok := nodeLabels[sdcGuidLabelName] + label, ok := nodeLabels[sdcGUIDLabelName] if !ok { - glog.V(4).Info(log("node label %s not found", sdcGuidLabelName)) + klog.V(4).Info(log("node label %s not found", sdcGUIDLabelName)) return "", nil } - glog.V(4).Info(log("found node label %s=%s", sdcGuidLabelName, label)) + klog.V(4).Info(log("found node label %s=%s", sdcGUIDLabelName, label)) return label, nil } @@ -284,7 +284,7 @@ func getVolumeSourceAttribs(spec *volume.Spec) (*volSourceAttribs, error) { attribs.readOnly = pSource.ReadOnly } else { msg := log("failed to get ScaleIOVolumeSource or ScaleIOPersistentVolumeSource from spec") - glog.Error(msg) + klog.Error(msg) return nil, errors.New(msg) } return attribs, nil diff --git a/pkg/volume/scaleio/sio_volume.go b/pkg/volume/scaleio/sio_volume.go index 0f10dfa1e83..4a5cb35221e 100644 --- a/pkg/volume/scaleio/sio_volume.go +++ b/pkg/volume/scaleio/sio_volume.go @@ -23,12 +23,12 @@ import ( "strconv" "strings" - "github.com/golang/glog" api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" kstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" @@ -85,20 +85,20 @@ func (v *sioVolume) SetUpAt(dir string, fsGroup *int64) error { v.plugin.volumeMtx.LockKey(v.volSpecName) defer v.plugin.volumeMtx.UnlockKey(v.volSpecName) - glog.V(4).Info(log("setting up volume for PV.spec %s", v.volSpecName)) + klog.V(4).Info(log("setting up volume for PV.spec %s", v.volSpecName)) if err := v.setSioMgr(); err != nil { - glog.Error(log("setup failed to create scalio manager: %v", err)) + klog.Error(log("setup failed to create scalio manager: %v", err)) return err } mounter := v.plugin.host.GetMounter(v.plugin.GetPluginName()) notDevMnt, err := mounter.IsLikelyNotMountPoint(dir) if err != nil && !os.IsNotExist(err) { - glog.Error(log("IsLikelyNotMountPoint test failed for dir %v", dir)) + klog.Error(log("IsLikelyNotMountPoint test failed for dir %v", dir)) return err } if !notDevMnt { - glog.V(4).Info(log("skipping setup, dir %s already a mount point", v.volName)) + klog.V(4).Info(log("skipping setup, dir %s already a mount point", v.volName)) return nil } @@ -114,12 +114,12 @@ func (v *sioVolume) SetUpAt(dir string, fsGroup *int64) error { } } } - glog.V(4).Info(log("multiple mapping enabled = %v", enableMultiMaps)) + klog.V(4).Info(log("multiple mapping enabled = %v", enableMultiMaps)) volName := v.volName devicePath, err := v.sioMgr.AttachVolume(volName, enableMultiMaps) if err != nil { - glog.Error(log("setup of volume %v: %v", v.volSpecName, err)) + klog.Error(log("setup of volume %v: %v", v.volSpecName, err)) return err } options := []string{} @@ -134,31 +134,31 @@ func (v *sioVolume) SetUpAt(dir string, fsGroup *int64) error { options = append(options, "ro") } - glog.V(4).Info(log("mounting device %s -> %s", devicePath, dir)) + klog.V(4).Info(log("mounting device %s -> %s", devicePath, dir)) if err := os.MkdirAll(dir, 0750); err != nil { - glog.Error(log("failed to create dir %#v: %v", dir, err)) + klog.Error(log("failed to create dir %#v: %v", dir, err)) return err } - glog.V(4).Info(log("setup created mount point directory %s", dir)) + klog.V(4).Info(log("setup created mount point directory %s", dir)) diskMounter := util.NewSafeFormatAndMountFromHost(v.plugin.GetPluginName(), v.plugin.host) err = diskMounter.FormatAndMount(devicePath, dir, v.fsType, options) if err != nil { - glog.Error(log("mount operation failed during setup: %v", err)) + klog.Error(log("mount operation failed during setup: %v", err)) if err := os.Remove(dir); err != nil && !os.IsNotExist(err) { - glog.Error(log("failed to remove dir %s during a failed mount at setup: %v", dir, err)) + klog.Error(log("failed to remove dir %s during a failed mount at setup: %v", dir, err)) return err } return err } if !v.readOnly && fsGroup != nil { - glog.V(4).Info(log("applying value FSGroup ownership")) + klog.V(4).Info(log("applying value FSGroup ownership")) volume.SetVolumeOwnership(v, fsGroup) } - glog.V(4).Info(log("successfully setup PV %s: volume %s mapped as %s mounted at %s", v.volSpecName, v.volName, devicePath, dir)) + klog.V(4).Info(log("successfully setup PV %s: volume %s mapped as %s mounted at %s", v.volSpecName, v.volName, devicePath, dir)) return nil } @@ -188,21 +188,21 @@ func (v *sioVolume) TearDownAt(dir string) error { mounter := v.plugin.host.GetMounter(v.plugin.GetPluginName()) dev, _, err := mount.GetDeviceNameFromMount(mounter, dir) if err != nil { - glog.Errorf(log("failed to get reference count for volume: %s", dir)) + klog.Errorf(log("failed to get reference count for volume: %s", dir)) return err } - glog.V(4).Info(log("attempting to unmount %s", dir)) + klog.V(4).Info(log("attempting to unmount %s", dir)) if err := util.UnmountPath(dir, mounter); err != nil { - glog.Error(log("teardown failed while unmounting dir %s: %v ", dir, err)) + klog.Error(log("teardown failed while unmounting dir %s: %v ", dir, err)) return err } - glog.V(4).Info(log("dir %s unmounted successfully", dir)) + klog.V(4).Info(log("dir %s unmounted successfully", dir)) // detach/unmap deviceBusy, err := mounter.DeviceOpened(dev) if err != nil { - glog.Error(log("teardown unable to get status for device %s: %v", dev, err)) + klog.Error(log("teardown unable to get status for device %s: %v", dev, err)) return err } @@ -210,16 +210,16 @@ func (v *sioVolume) TearDownAt(dir string) error { // use "last attempt wins" strategy to detach volume from node // only allow volume to detach when it is not busy (not being used by other pods) if !deviceBusy { - glog.V(4).Info(log("teardown is attempting to detach/unmap volume for PV %s", v.volSpecName)) + klog.V(4).Info(log("teardown is attempting to detach/unmap volume for PV %s", v.volSpecName)) if err := v.resetSioMgr(); err != nil { - glog.Error(log("teardown failed, unable to reset scalio mgr: %v", err)) + klog.Error(log("teardown failed, unable to reset scalio mgr: %v", err)) } volName := v.volName if err := v.sioMgr.DetachVolume(volName); err != nil { - glog.Warning(log("warning: detaching failed for volume %s: %v", volName, err)) + klog.Warning(log("warning: detaching failed for volume %s: %v", volName, err)) return nil } - glog.V(4).Infof(log("teardown of volume %v detached successfully", volName)) + klog.V(4).Infof(log("teardown of volume %v detached successfully", volName)) } return nil } @@ -230,20 +230,20 @@ func (v *sioVolume) TearDownAt(dir string) error { var _ volume.Deleter = &sioVolume{} func (v *sioVolume) Delete() error { - glog.V(4).Info(log("deleting pvc %s", v.volSpecName)) + klog.V(4).Info(log("deleting pvc %s", v.volSpecName)) if err := v.setSioMgrFromSpec(); err != nil { - glog.Error(log("delete failed while setting sio manager: %v", err)) + klog.Error(log("delete failed while setting sio manager: %v", err)) return err } err := v.sioMgr.DeleteVolume(v.volName) if err != nil { - glog.Error(log("failed to delete volume %s: %v", v.volName, err)) + klog.Error(log("failed to delete volume %s: %v", v.volName, err)) return err } - glog.V(4).Info(log("successfully deleted PV %s with volume %s", v.volSpecName, v.volName)) + klog.V(4).Info(log("successfully deleted PV %s with volume %s", v.volSpecName, v.volName)) return nil } @@ -253,7 +253,7 @@ func (v *sioVolume) Delete() error { var _ volume.Provisioner = &sioVolume{} func (v *sioVolume) Provision(selectedNode *api.Node, allowedTopologies []api.TopologySelectorTerm) (*api.PersistentVolume, error) { - glog.V(4).Info(log("attempting to dynamically provision pvc %v", v.options.PVC.Name)) + klog.V(4).Info(log("attempting to dynamically provision pvc %v", v.options.PVC.Name)) if !util.AccessModesContainedInAll(v.plugin.GetAccessModes(), v.options.PVC.Spec.AccessModes) { return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", v.options.PVC.Spec.AccessModes, v.plugin.GetAccessModes()) @@ -266,7 +266,7 @@ func (v *sioVolume) Provision(selectedNode *api.Node, allowedTopologies []api.To // setup volume attrributes genName := v.generateName("k8svol", 11) var oneGig int64 = 1024 * 1024 * 1024 - var eightGig int64 = 8 * oneGig + eightGig := 8 * oneGig capacity := v.options.PVC.Spec.Resources.Requests[api.ResourceName(api.ResourceStorage)] volSizeBytes := capacity.Value() @@ -278,13 +278,13 @@ func (v *sioVolume) Provision(selectedNode *api.Node, allowedTopologies []api.To if volSizeBytes < eightGig { volSizeGB = int64(util.RoundUpSize(eightGig, oneGig)) - glog.V(4).Info(log("capacity less than 8Gi found, adjusted to %dGi", volSizeGB)) + klog.V(4).Info(log("capacity less than 8Gi found, adjusted to %dGi", volSizeGB)) } // create sio manager if err := v.setSioMgrFromConfig(); err != nil { - glog.Error(log("provision failed while setting up sio mgr: %v", err)) + klog.Error(log("provision failed while setting up sio mgr: %v", err)) return nil, err } @@ -292,7 +292,7 @@ func (v *sioVolume) Provision(selectedNode *api.Node, allowedTopologies []api.To volName := genName vol, err := v.sioMgr.CreateVolume(volName, volSizeGB) if err != nil { - glog.Error(log("provision failed while creating volume: %v", err)) + klog.Error(log("provision failed while creating volume: %v", err)) return nil, err } @@ -300,12 +300,12 @@ func (v *sioVolume) Provision(selectedNode *api.Node, allowedTopologies []api.To v.configData[confKey.volumeName] = volName sslEnabled, err := strconv.ParseBool(v.configData[confKey.sslEnabled]) if err != nil { - glog.Warning(log("failed to parse parameter sslEnabled, setting to false")) + klog.Warning(log("failed to parse parameter sslEnabled, setting to false")) sslEnabled = false } readOnly, err := strconv.ParseBool(v.configData[confKey.readOnly]) if err != nil { - glog.Warning(log("failed to parse parameter readOnly, setting it to false")) + klog.Warning(log("failed to parse parameter readOnly, setting it to false")) readOnly = false } @@ -348,24 +348,24 @@ func (v *sioVolume) Provision(selectedNode *api.Node, allowedTopologies []api.To pv.Spec.AccessModes = v.plugin.GetAccessModes() } - glog.V(4).Info(log("provisioner created pv %v and volume %s successfully", pvName, vol.Name)) + klog.V(4).Info(log("provisioner created pv %v and volume %s successfully", pvName, vol.Name)) return pv, nil } // setSioMgr creates scaleio mgr from cached config data if found // otherwise, setups new config data and create mgr func (v *sioVolume) setSioMgr() error { - glog.V(4).Info(log("setting up sio mgr for spec %s", v.volSpecName)) + klog.V(4).Info(log("setting up sio mgr for spec %s", v.volSpecName)) podDir := v.plugin.host.GetPodPluginDir(v.podUID, sioPluginName) configName := path.Join(podDir, sioConfigFileName) if v.sioMgr == nil { configData, err := loadConfig(configName) // try to load config if exist if err != nil { if !os.IsNotExist(err) { - glog.Error(log("failed to load config %s : %v", configName, err)) + klog.Error(log("failed to load config %s : %v", configName, err)) return err } - glog.V(4).Info(log("previous config file not found, creating new one")) + klog.V(4).Info(log("previous config file not found, creating new one")) // prepare config data configData = make(map[string]string) mapVolumeSpec(configData, v.spec) @@ -376,31 +376,31 @@ func (v *sioVolume) setSioMgr() error { configData[confKey.volSpecName] = v.volSpecName if err := validateConfigs(configData); err != nil { - glog.Error(log("config setup failed: %s", err)) + klog.Error(log("config setup failed: %s", err)) return err } // persist config if err := saveConfig(configName, configData); err != nil { - glog.Error(log("failed to save config data: %v", err)) + klog.Error(log("failed to save config data: %v", err)) return err } } // merge in secret if err := attachSecret(v.plugin, v.secretNamespace, configData); err != nil { - glog.Error(log("failed to load secret: %v", err)) + klog.Error(log("failed to load secret: %v", err)) return err } // merge in Sdc Guid label value - if err := attachSdcGuid(v.plugin, configData); err != nil { - glog.Error(log("failed to retrieve sdc guid: %v", err)) + if err := attachSdcGUID(v.plugin, configData); err != nil { + klog.Error(log("failed to retrieve sdc guid: %v", err)) return err } mgr, err := newSioMgr(configData, v.plugin.host.GetExec(v.plugin.GetPluginName())) if err != nil { - glog.Error(log("failed to reset sio manager: %v", err)) + klog.Error(log("failed to reset sio manager: %v", err)) return err } @@ -417,7 +417,7 @@ func (v *sioVolume) resetSioMgr() error { // load config data from disk configData, err := loadConfig(configName) if err != nil { - glog.Error(log("failed to load config data: %v", err)) + klog.Error(log("failed to load config data: %v", err)) return err } v.secretName = configData[confKey.secretName] @@ -427,20 +427,20 @@ func (v *sioVolume) resetSioMgr() error { // attach secret if err := attachSecret(v.plugin, v.secretNamespace, configData); err != nil { - glog.Error(log("failed to load secret: %v", err)) + klog.Error(log("failed to load secret: %v", err)) return err } // merge in Sdc Guid label value - if err := attachSdcGuid(v.plugin, configData); err != nil { - glog.Error(log("failed to retrieve sdc guid: %v", err)) + if err := attachSdcGUID(v.plugin, configData); err != nil { + klog.Error(log("failed to retrieve sdc guid: %v", err)) return err } mgr, err := newSioMgr(configData, v.plugin.host.GetExec(v.plugin.GetPluginName())) if err != nil { - glog.Error(log("failed to reset scaleio mgr: %v", err)) + klog.Error(log("failed to reset scaleio mgr: %v", err)) return err } v.sioMgr = mgr @@ -451,14 +451,14 @@ func (v *sioVolume) resetSioMgr() error { // setSioFromConfig sets up scaleio mgr from an available config data map // designed to be called from dynamic provisioner func (v *sioVolume) setSioMgrFromConfig() error { - glog.V(4).Info(log("setting scaleio mgr from available config")) + klog.V(4).Info(log("setting scaleio mgr from available config")) if v.sioMgr == nil { applyConfigDefaults(v.configData) v.configData[confKey.volSpecName] = v.volSpecName if err := validateConfigs(v.configData); err != nil { - glog.Error(log("config data setup failed: %s", err)) + klog.Error(log("config data setup failed: %s", err)) return err } @@ -469,14 +469,14 @@ func (v *sioVolume) setSioMgrFromConfig() error { } if err := attachSecret(v.plugin, v.secretNamespace, data); err != nil { - glog.Error(log("failed to load secret: %v", err)) + klog.Error(log("failed to load secret: %v", err)) return err } mgr, err := newSioMgr(data, v.plugin.host.GetExec(v.plugin.GetPluginName())) if err != nil { - glog.Error(log("failed while setting scaleio mgr from config: %v", err)) + klog.Error(log("failed while setting scaleio mgr from config: %v", err)) return err } v.sioMgr = mgr @@ -487,7 +487,7 @@ func (v *sioVolume) setSioMgrFromConfig() error { // setSioMgrFromSpec sets the scaleio manager from a spec object. // The spec may be complete or incomplete depending on lifecycle phase. func (v *sioVolume) setSioMgrFromSpec() error { - glog.V(4).Info(log("setting sio manager from spec")) + klog.V(4).Info(log("setting sio manager from spec")) if v.sioMgr == nil { // get config data form spec volume source configData := map[string]string{} @@ -499,20 +499,20 @@ func (v *sioVolume) setSioMgrFromSpec() error { configData[confKey.volSpecName] = v.volSpecName if err := validateConfigs(configData); err != nil { - glog.Error(log("config setup failed: %s", err)) + klog.Error(log("config setup failed: %s", err)) return err } // attach secret object to config data if err := attachSecret(v.plugin, v.secretNamespace, configData); err != nil { - glog.Error(log("failed to load secret: %v", err)) + klog.Error(log("failed to load secret: %v", err)) return err } mgr, err := newSioMgr(configData, v.plugin.host.GetExec(v.plugin.GetPluginName())) if err != nil { - glog.Error(log("failed to reset sio manager: %v", err)) + klog.Error(log("failed to reset sio manager: %v", err)) return err } v.sioMgr = mgr diff --git a/pkg/volume/scaleio/sio_volume_test.go b/pkg/volume/scaleio/sio_volume_test.go index 7f122143f51..199be8c918d 100644 --- a/pkg/volume/scaleio/sio_volume_test.go +++ b/pkg/volume/scaleio/sio_volume_test.go @@ -23,7 +23,7 @@ import ( "strings" "testing" - "github.com/golang/glog" + "k8s.io/klog" api "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -56,7 +56,7 @@ func newPluginMgr(t *testing.T, apiObject runtime.Object) (*volume.VolumePluginM tmpDir, fakeClient, nil, - map[string]string{sdcGuidLabelName: "abc-123"}, + map[string]string{sdcGUIDLabelName: "abc-123"}, ) plugMgr := &volume.VolumePluginMgr{} plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) @@ -206,9 +206,9 @@ func TestVolumeMounterUnmounter(t *testing.T) { t.Errorf("SetUp() - expecting multiple volume disabled by default") } - // did we read sdcGuid label - if _, ok := sioVol.sioMgr.configData[confKey.sdcGuid]; !ok { - t.Errorf("Expected to find node label scaleio.sdcGuid, but did not find it") + // did we read sdcGUID label + if _, ok := sioVol.sioMgr.configData[confKey.sdcGUID]; !ok { + t.Errorf("Expected to find node label scaleio.sdcGUID, but did not find it") } // rebuild spec @@ -328,7 +328,7 @@ func TestVolumeProvisioner(t *testing.T) { t.Errorf("expected volume name to be %s, got %s", actualVolName, vol.Name) } if vol.SizeInKb != 8*1024*1024 { - glog.V(4).Info(log("unexpected volume size")) + klog.V(4).Info(log("unexpected volume size")) } // mount dynamic vol @@ -349,9 +349,9 @@ func TestVolumeProvisioner(t *testing.T) { t.Fatalf("Expected success, got: %v", err) } - // did we read sdcGuid label - if _, ok := sioVol.sioMgr.configData[confKey.sdcGuid]; !ok { - t.Errorf("Expected to find node label scaleio.sdcGuid, but did not find it") + // did we read sdcGUID label + if _, ok := sioVol.sioMgr.configData[confKey.sdcGUID]; !ok { + t.Errorf("Expected to find node label scaleio.sdcGUID, but did not find it") } // isMultiMap applied diff --git a/pkg/volume/secret/BUILD b/pkg/volume/secret/BUILD index e07df1456f3..d44d59dd614 100644 --- a/pkg/volume/secret/BUILD +++ b/pkg/volume/secret/BUILD @@ -22,7 +22,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/secret/secret.go b/pkg/volume/secret/secret.go index 065b9bfb5af..14485181f92 100644 --- a/pkg/volume/secret/secret.go +++ b/pkg/volume/secret/secret.go @@ -19,18 +19,18 @@ package secret import ( "fmt" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" ) -// ProbeVolumePlugin is the entry point for plugin detection in a package. +// ProbeVolumePlugins is the entry point for plugin detection in a package. func ProbeVolumePlugins() []volume.VolumePlugin { return []volume.VolumePlugin{&secretPlugin{}} } @@ -179,7 +179,7 @@ func (b *secretVolumeMounter) SetUp(fsGroup *int64) error { } func (b *secretVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { - glog.V(3).Infof("Setting up volume %v for pod %v at %v", b.volName, b.pod.UID, dir) + klog.V(3).Infof("Setting up volume %v for pod %v at %v", b.volName, b.pod.UID, dir) // Wrap EmptyDir, let it do the setup. wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec(), &b.pod, *b.opts) @@ -191,7 +191,7 @@ func (b *secretVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { secret, err := b.getSecret(b.pod.Namespace, b.source.SecretName) if err != nil { if !(errors.IsNotFound(err) && optional) { - glog.Errorf("Couldn't get secret %v/%v: %v", b.pod.Namespace, b.source.SecretName, err) + klog.Errorf("Couldn't get secret %v/%v: %v", b.pod.Namespace, b.source.SecretName, err) return err } secret = &v1.Secret{ @@ -203,7 +203,7 @@ func (b *secretVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } totalBytes := totalSecretBytes(secret) - glog.V(3).Infof("Received secret %v/%v containing (%v) pieces of data, %v total bytes", + klog.V(3).Infof("Received secret %v/%v containing (%v) pieces of data, %v total bytes", b.pod.Namespace, b.source.SecretName, len(secret.Data), @@ -227,12 +227,12 @@ func (b *secretVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if !setupSuccess { unmounter, unmountCreateErr := b.plugin.NewUnmounter(b.volName, b.podUID) if unmountCreateErr != nil { - glog.Errorf("error cleaning up mount %s after failure. Create unmounter failed with %v", b.volName, unmountCreateErr) + klog.Errorf("error cleaning up mount %s after failure. Create unmounter failed with %v", b.volName, unmountCreateErr) return } tearDownErr := unmounter.TearDown() if tearDownErr != nil { - glog.Errorf("error tearing down volume %s with : %v", b.volName, tearDownErr) + klog.Errorf("error tearing down volume %s with : %v", b.volName, tearDownErr) } } }() @@ -240,26 +240,26 @@ func (b *secretVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { writerContext := fmt.Sprintf("pod %v/%v volume %v", b.pod.Namespace, b.pod.Name, b.volName) writer, err := volumeutil.NewAtomicWriter(dir, writerContext) if err != nil { - glog.Errorf("Error creating atomic writer: %v", err) + klog.Errorf("Error creating atomic writer: %v", err) return err } err = writer.Write(payload) if err != nil { - glog.Errorf("Error writing payload to dir: %v", err) + klog.Errorf("Error writing payload to dir: %v", err) return err } err = volume.SetVolumeOwnership(b, fsGroup) if err != nil { - glog.Errorf("Error applying volume ownership settings for group: %v", fsGroup) + klog.Errorf("Error applying volume ownership settings for group: %v", fsGroup) return err } setupSuccess = true return nil } -// Note: this function is exported so that it can be called from the projection volume driver +// MakePayload function is exported so that it can be called from the projection volume driver func MakePayload(mappings []v1.KeyToPath, secret *v1.Secret, defaultMode *int32, optional bool) (map[string]volumeutil.FileProjection, error) { if defaultMode == nil { return nil, fmt.Errorf("No defaultMode used, not even the default value for it") @@ -281,9 +281,9 @@ func MakePayload(mappings []v1.KeyToPath, secret *v1.Secret, defaultMode *int32, if optional { continue } - err_msg := "references non-existent secret key" - glog.Errorf(err_msg) - return nil, fmt.Errorf(err_msg) + errMsg := "references non-existent secret key" + klog.Errorf(errMsg) + return nil, fmt.Errorf(errMsg) } fileProjection.Data = []byte(content) diff --git a/pkg/volume/storageos/BUILD b/pkg/volume/storageos/BUILD index ee84092563f..c89c5fee55f 100644 --- a/pkg/volume/storageos/BUILD +++ b/pkg/volume/storageos/BUILD @@ -24,9 +24,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/storageos/go-api:go_default_library", "//vendor/github.com/storageos/go-api/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/storageos/storageos.go b/pkg/volume/storageos/storageos.go index 1130ae5b5ed..9cb56b353aa 100644 --- a/pkg/volume/storageos/storageos.go +++ b/pkg/volume/storageos/storageos.go @@ -24,7 +24,7 @@ import ( "path/filepath" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -343,14 +343,14 @@ func (b *storageosMounter) CanMount() error { func (b *storageosMounter) SetUp(fsGroup *int64) error { // Need a namespace to find the volume, try pod's namespace if not set. if b.volNamespace == "" { - glog.V(2).Infof("Setting StorageOS volume namespace to pod namespace: %s", b.podNamespace) + klog.V(2).Infof("Setting StorageOS volume namespace to pod namespace: %s", b.podNamespace) b.volNamespace = b.podNamespace } // Attach the StorageOS volume as a block device devicePath, err := b.manager.AttachVolume(b) if err != nil { - glog.Errorf("Failed to attach StorageOS volume %s: %s", b.volName, err.Error()) + klog.Errorf("Failed to attach StorageOS volume %s: %s", b.volName, err.Error()) return err } @@ -360,7 +360,7 @@ func (b *storageosMounter) SetUp(fsGroup *int64) error { if err != nil { return err } - glog.V(4).Infof("Successfully mounted StorageOS volume %s into global mount directory", b.volName) + klog.V(4).Infof("Successfully mounted StorageOS volume %s into global mount directory", b.volName) // Bind mount the volume into the pod return b.SetUpAt(b.GetPath(), fsGroup) @@ -369,9 +369,9 @@ func (b *storageosMounter) SetUp(fsGroup *int64) error { // SetUp bind mounts the disk global mount to the give volume path. func (b *storageosMounter) SetUpAt(dir string, fsGroup *int64) error { notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) - glog.V(4).Infof("StorageOS volume set up: %s %v %v", dir, !notMnt, err) + klog.V(4).Infof("StorageOS volume set up: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { - glog.Errorf("Cannot validate mount point: %s %v", dir, err) + klog.Errorf("Cannot validate mount point: %s %v", dir, err) return err } if !notMnt { @@ -379,7 +379,7 @@ func (b *storageosMounter) SetUpAt(dir string, fsGroup *int64) error { } if err = os.MkdirAll(dir, 0750); err != nil { - glog.Errorf("mkdir failed on disk %s (%v)", dir, err) + klog.Errorf("mkdir failed on disk %s (%v)", dir, err) return err } @@ -391,39 +391,39 @@ func (b *storageosMounter) SetUpAt(dir string, fsGroup *int64) error { mountOptions := util.JoinMountOptions(b.mountOptions, options) globalPDPath := makeGlobalPDName(b.plugin.host, b.pvName, b.volNamespace, b.volName) - glog.V(4).Infof("Attempting to bind mount to pod volume at %s", dir) + klog.V(4).Infof("Attempting to bind mount to pod volume at %s", dir) err = b.mounter.Mount(globalPDPath, dir, "", mountOptions) if err != nil { notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !notMnt { if mntErr = b.mounter.Unmount(dir); mntErr != nil { - glog.Errorf("Failed to unmount: %v", mntErr) + klog.Errorf("Failed to unmount: %v", mntErr) return err } notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !notMnt { - glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) + klog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", dir) return err } } os.Remove(dir) - glog.Errorf("Mount of disk %s failed: %v", dir, err) + klog.Errorf("Mount of disk %s failed: %v", dir, err) return err } if !b.readOnly { volume.SetVolumeOwnership(b, fsGroup) } - glog.V(4).Infof("StorageOS volume setup complete on %s", dir) + klog.V(4).Infof("StorageOS volume setup complete on %s", dir) return nil } @@ -487,7 +487,7 @@ func (b *storageosUnmounter) GetPath() string { // resource was the last reference to that disk on the kubelet. func (b *storageosUnmounter) TearDown() error { if len(b.volNamespace) == 0 || len(b.volName) == 0 { - glog.Warningf("volNamespace: %q, volName: %q not set, skipping TearDown", b.volNamespace, b.volName) + klog.Warningf("volNamespace: %q, volName: %q not set, skipping TearDown", b.volNamespace, b.volName) return fmt.Errorf("pvName not specified for TearDown, waiting for next sync loop") } // Unmount from pod @@ -495,7 +495,7 @@ func (b *storageosUnmounter) TearDown() error { err := b.TearDownAt(mountPath) if err != nil { - glog.Errorf("Unmount from pod failed: %v", err) + klog.Errorf("Unmount from pod failed: %v", err) return err } @@ -503,25 +503,25 @@ func (b *storageosUnmounter) TearDown() error { globalPDPath := makeGlobalPDName(b.plugin.host, b.pvName, b.volNamespace, b.volName) devicePath, _, err := mount.GetDeviceNameFromMount(b.mounter, globalPDPath) if err != nil { - glog.Errorf("Detach failed when getting device from global mount: %v", err) + klog.Errorf("Detach failed when getting device from global mount: %v", err) return err } // Unmount from plugin's disk global mount dir. err = b.TearDownAt(globalPDPath) if err != nil { - glog.Errorf("Detach failed during unmount: %v", err) + klog.Errorf("Detach failed during unmount: %v", err) return err } // Detach loop device err = b.manager.DetachVolume(b, devicePath) if err != nil { - glog.Errorf("Detach device %s failed for volume %s: %v", devicePath, b.pvName, err) + klog.Errorf("Detach device %s failed for volume %s: %v", devicePath, b.pvName, err) return err } - glog.V(4).Infof("Successfully unmounted StorageOS volume %s and detached devices", b.pvName) + klog.V(4).Infof("Successfully unmounted StorageOS volume %s and detached devices", b.pvName) return nil } @@ -530,10 +530,10 @@ func (b *storageosUnmounter) TearDown() error { // resource was the last reference to that disk on the kubelet. func (b *storageosUnmounter) TearDownAt(dir string) error { if err := util.UnmountPath(dir, b.mounter); err != nil { - glog.V(4).Infof("Unmounted StorageOS volume %s failed with: %v", b.pvName, err) + klog.V(4).Infof("Unmounted StorageOS volume %s failed with: %v", b.pvName, err) } if err := b.manager.UnmountVolume(b); err != nil { - glog.V(4).Infof("Mount reference for volume %s could not be removed from StorageOS: %v", b.pvName, err) + klog.V(4).Infof("Mount reference for volume %s could not be removed from StorageOS: %v", b.pvName, err) } return nil } @@ -616,7 +616,7 @@ func (c *storageosProvisioner) Provision(selectedNode *v1.Node, allowedTopologie vol, err := c.manager.CreateVolume(c) if err != nil { - glog.Errorf("failed to create volume: %v", err) + klog.Errorf("failed to create volume: %v", err) return nil, err } if vol.FSType == "" { @@ -717,7 +717,7 @@ func getAPICfg(spec *volume.Spec, pod *v1.Pod, kubeClient clientset.Interface) ( func parsePodSecret(pod *v1.Pod, secretName string, kubeClient clientset.Interface) (*storageosAPIConfig, error) { secret, err := util.GetSecretForPod(pod, secretName, kubeClient) if err != nil { - glog.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName) + klog.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName) return nil, fmt.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName) } return parseAPIConfig(secret) @@ -728,7 +728,7 @@ func parsePodSecret(pod *v1.Pod, secretName string, kubeClient clientset.Interfa func parsePVSecret(namespace, secretName string, kubeClient clientset.Interface) (*storageosAPIConfig, error) { secret, err := util.GetSecretForPV(namespace, secretName, storageosPluginName, kubeClient) if err != nil { - glog.Errorf("failed to get secret from [%q/%q]", namespace, secretName) + klog.Errorf("failed to get secret from [%q/%q]", namespace, secretName) return nil, fmt.Errorf("failed to get secret from [%q/%q]", namespace, secretName) } return parseAPIConfig(secret) diff --git a/pkg/volume/storageos/storageos_util.go b/pkg/volume/storageos/storageos_util.go index beac8924087..c102e4226d5 100644 --- a/pkg/volume/storageos/storageos_util.go +++ b/pkg/volume/storageos/storageos_util.go @@ -25,9 +25,9 @@ import ( "k8s.io/kubernetes/pkg/util/mount" - "github.com/golang/glog" storageosapi "github.com/storageos/go-api" storageostypes "github.com/storageos/go-api/types" + "k8s.io/klog" ) const ( @@ -88,7 +88,7 @@ func (u *storageosUtil) NewAPI(apiCfg *storageosAPIConfig) error { apiPass: defaultAPIPassword, apiVersion: defaultAPIVersion, } - glog.V(4).Infof("Using default StorageOS API settings: addr %s, version: %s", apiCfg.apiAddr, defaultAPIVersion) + klog.V(4).Infof("Using default StorageOS API settings: addr %s, version: %s", apiCfg.apiAddr, defaultAPIVersion) } api, err := storageosapi.NewVersionedClient(apiCfg.apiAddr, defaultAPIVersion) @@ -122,7 +122,7 @@ func (u *storageosUtil) CreateVolume(p *storageosProvisioner) (*storageosVolume, vol, err := u.api.VolumeCreate(opts) if err != nil { - glog.Errorf("volume create failed for volume %q (%v)", opts.Name, err) + klog.Errorf("volume create failed for volume %q (%v)", opts.Name, err) return nil, err } return &storageosVolume{ @@ -157,7 +157,7 @@ func (u *storageosUtil) AttachVolume(b *storageosMounter) (string, error) { vol, err := u.api.Volume(b.volNamespace, b.volName) if err != nil { - glog.Warningf("volume retrieve failed for volume %q with namespace %q (%v)", b.volName, b.volNamespace, err) + klog.Warningf("volume retrieve failed for volume %q with namespace %q (%v)", b.volName, b.volNamespace, err) return "", err } @@ -170,14 +170,14 @@ func (u *storageosUtil) AttachVolume(b *storageosMounter) (string, error) { Namespace: vol.Namespace, } if err := u.api.VolumeUnmount(opts); err != nil { - glog.Warningf("Couldn't clear existing StorageOS mount reference: %v", err) + klog.Warningf("Couldn't clear existing StorageOS mount reference: %v", err) } } srcPath := path.Join(b.deviceDir, vol.ID) dt, err := pathDeviceType(srcPath) if err != nil { - glog.Warningf("volume source path %q for volume %q not ready (%v)", srcPath, b.volName, err) + klog.Warningf("volume source path %q for volume %q not ready (%v)", srcPath, b.volName, err) return "", err } @@ -217,7 +217,7 @@ func (u *storageosUtil) MountVolume(b *storageosMounter, mntDevice, deviceMountP } } if err = os.MkdirAll(deviceMountPath, 0750); err != nil { - glog.Errorf("mkdir failed on disk %s (%v)", deviceMountPath, err) + klog.Errorf("mkdir failed on disk %s (%v)", deviceMountPath, err) return err } options := []string{} @@ -255,7 +255,7 @@ func (u *storageosUtil) UnmountVolume(b *storageosUnmounter) error { if err := u.NewAPI(b.apiCfg); err != nil { // We can't always get the config we need, so allow the unmount to // succeed even if we can't remove the mount reference from the API. - glog.V(4).Infof("Could not remove mount reference in the StorageOS API as no credentials available to the unmount operation") + klog.V(4).Infof("Could not remove mount reference in the StorageOS API as no credentials available to the unmount operation") return nil } @@ -291,11 +291,11 @@ func (u *storageosUtil) DeviceDir(b *storageosMounter) string { ctrl, err := u.api.Controller(b.plugin.host.GetHostName()) if err != nil { - glog.Warningf("node device path lookup failed: %v", err) + klog.Warningf("node device path lookup failed: %v", err) return defaultDeviceDir } if ctrl == nil || ctrl.DeviceDir == "" { - glog.Warningf("node device path not set, using default: %s", defaultDeviceDir) + klog.Warningf("node device path not set, using default: %s", defaultDeviceDir) return defaultDeviceDir } return ctrl.DeviceDir @@ -327,7 +327,7 @@ func attachFileDevice(path string, exec mount.Exec) (string, error) { // If no existing loop device for the path, create one if blockDevicePath == "" { - glog.V(4).Infof("Creating device for path: %s", path) + klog.V(4).Infof("Creating device for path: %s", path) blockDevicePath, err = makeLoopDevice(path, exec) if err != nil { return "", err @@ -349,7 +349,7 @@ func getLoopDevice(path string, exec mount.Exec) (string, error) { args := []string{"-j", path} out, err := exec.Run(losetupPath, args...) if err != nil { - glog.V(2).Infof("Failed device discover command for path %s: %v", path, err) + klog.V(2).Infof("Failed device discover command for path %s: %v", path, err) return "", err } return parseLosetupOutputForDevice(out) @@ -359,7 +359,7 @@ func makeLoopDevice(path string, exec mount.Exec) (string, error) { args := []string{"-f", "-P", "--show", path} out, err := exec.Run(losetupPath, args...) if err != nil { - glog.V(2).Infof("Failed device create command for path %s: %v", path, err) + klog.V(2).Infof("Failed device create command for path %s: %v", path, err) return "", err } return parseLosetupOutputForDevice(out) diff --git a/pkg/volume/storageos/storageos_util_test.go b/pkg/volume/storageos/storageos_util_test.go index 598bfd5a212..e4800cbe51e 100644 --- a/pkg/volume/storageos/storageos_util_test.go +++ b/pkg/volume/storageos/storageos_util_test.go @@ -30,7 +30,7 @@ import ( "testing" ) -var testApiSecretName = "storageos-api" +var testAPISecretName = "storageos-api" var testVolName = "storageos-test-vol" var testPVName = "storageos-test-pv" var testNamespace = "storageos-test-namespace" diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index 224db3e3b86..4c116aa5934 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -250,6 +250,10 @@ type FakeVolumePlugin struct { LimitKey string ProvisionDelaySeconds int + // Add callbacks as needed + WaitForAttachHook func(spec *Spec, devicePath string, pod *v1.Pod, spectimeout time.Duration) (string, error) + UnmountDeviceHook func(globalMountPath string) error + Mounters []*FakeVolume Unmounters []*FakeVolume Attachers []*FakeVolume @@ -269,7 +273,10 @@ var _ DeviceMountableVolumePlugin = &FakeVolumePlugin{} var _ FSResizableVolumePlugin = &FakeVolumePlugin{} func (plugin *FakeVolumePlugin) getFakeVolume(list *[]*FakeVolume) *FakeVolume { - volume := &FakeVolume{} + volume := &FakeVolume{ + WaitForAttachHook: plugin.WaitForAttachHook, + UnmountDeviceHook: plugin.UnmountDeviceHook, + } *list = append(*list, volume) return volume } @@ -551,6 +558,10 @@ type FakeVolume struct { Plugin *FakeVolumePlugin MetricsNil + // Add callbacks as needed + WaitForAttachHook func(spec *Spec, devicePath string, pod *v1.Pod, spectimeout time.Duration) (string, error) + UnmountDeviceHook func(globalMountPath string) error + SetUpCallCount int TearDownCallCount int AttachCallCount int @@ -724,6 +735,9 @@ func (fv *FakeVolume) WaitForAttach(spec *Spec, devicePath string, pod *v1.Pod, fv.Lock() defer fv.Unlock() fv.WaitForAttachCallCount++ + if fv.WaitForAttachHook != nil { + return fv.WaitForAttachHook(spec, devicePath, pod, spectimeout) + } return "/dev/sdb", nil } @@ -776,6 +790,9 @@ func (fv *FakeVolume) UnmountDevice(globalMountPath string) error { fv.Lock() defer fv.Unlock() fv.UnmountDeviceCallCount++ + if fv.UnmountDeviceHook != nil { + return fv.UnmountDeviceHook(globalMountPath) + } return nil } diff --git a/pkg/volume/util/BUILD b/pkg/volume/util/BUILD index cafe94a2235..64eac778de2 100644 --- a/pkg/volume/util/BUILD +++ b/pkg/volume/util/BUILD @@ -41,8 +41,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -52,6 +52,7 @@ go_test( "atomic_writer_test.go", "attach_limit_test.go", "device_util_linux_test.go", + "main_test.go", "nested_volumes_test.go", "resize_util_test.go", "util_test.go", @@ -60,6 +61,7 @@ go_test( deps = [ "//pkg/apis/core/install:go_default_library", "//pkg/apis/core/v1/helper:go_default_library", + "//pkg/features:go_default_library", "//pkg/kubelet/apis:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/util/slice:go_default_library", @@ -70,6 +72,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", ], ) diff --git a/pkg/volume/util/atomic_writer.go b/pkg/volume/util/atomic_writer.go index 99cc0c15d85..7b3d034012f 100644 --- a/pkg/volume/util/atomic_writer.go +++ b/pkg/volume/util/atomic_writer.go @@ -27,7 +27,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/sets" ) @@ -121,7 +121,7 @@ func (w *AtomicWriter) Write(payload map[string]FileProjection) error { // (1) cleanPayload, err := validatePayload(payload) if err != nil { - glog.Errorf("%s: invalid payload: %v", w.logContext, err) + klog.Errorf("%s: invalid payload: %v", w.logContext, err) return err } @@ -130,7 +130,7 @@ func (w *AtomicWriter) Write(payload map[string]FileProjection) error { oldTsDir, err := os.Readlink(dataDirPath) if err != nil { if !os.IsNotExist(err) { - glog.Errorf("%s: error reading link for data directory: %v", w.logContext, err) + klog.Errorf("%s: error reading link for data directory: %v", w.logContext, err) return err } // although Readlink() returns "" on err, don't be fragile by relying on it (since it's not specified in docs) @@ -145,40 +145,40 @@ func (w *AtomicWriter) Write(payload map[string]FileProjection) error { // (3) pathsToRemove, err = w.pathsToRemove(cleanPayload, oldTsPath) if err != nil { - glog.Errorf("%s: error determining user-visible files to remove: %v", w.logContext, err) + klog.Errorf("%s: error determining user-visible files to remove: %v", w.logContext, err) return err } // (4) if should, err := shouldWritePayload(cleanPayload, oldTsPath); err != nil { - glog.Errorf("%s: error determining whether payload should be written to disk: %v", w.logContext, err) + klog.Errorf("%s: error determining whether payload should be written to disk: %v", w.logContext, err) return err } else if !should && len(pathsToRemove) == 0 { - glog.V(4).Infof("%s: no update required for target directory %v", w.logContext, w.targetDir) + klog.V(4).Infof("%s: no update required for target directory %v", w.logContext, w.targetDir) return nil } else { - glog.V(4).Infof("%s: write required for target directory %v", w.logContext, w.targetDir) + klog.V(4).Infof("%s: write required for target directory %v", w.logContext, w.targetDir) } } // (5) tsDir, err := w.newTimestampDir() if err != nil { - glog.V(4).Infof("%s: error creating new ts data directory: %v", w.logContext, err) + klog.V(4).Infof("%s: error creating new ts data directory: %v", w.logContext, err) return err } tsDirName := filepath.Base(tsDir) // (6) if err = w.writePayloadToDir(cleanPayload, tsDir); err != nil { - glog.Errorf("%s: error writing payload to ts data directory %s: %v", w.logContext, tsDir, err) + klog.Errorf("%s: error writing payload to ts data directory %s: %v", w.logContext, tsDir, err) return err } - glog.V(4).Infof("%s: performed write of new data to ts data directory: %s", w.logContext, tsDir) + klog.V(4).Infof("%s: performed write of new data to ts data directory: %s", w.logContext, tsDir) // (7) if err = w.createUserVisibleFiles(cleanPayload); err != nil { - glog.Errorf("%s: error creating visible symlinks in %s: %v", w.logContext, w.targetDir, err) + klog.Errorf("%s: error creating visible symlinks in %s: %v", w.logContext, w.targetDir, err) return err } @@ -186,7 +186,7 @@ func (w *AtomicWriter) Write(payload map[string]FileProjection) error { newDataDirPath := path.Join(w.targetDir, newDataDirName) if err = os.Symlink(tsDirName, newDataDirPath); err != nil { os.RemoveAll(tsDir) - glog.Errorf("%s: error creating symbolic link for atomic update: %v", w.logContext, err) + klog.Errorf("%s: error creating symbolic link for atomic update: %v", w.logContext, err) return err } @@ -201,20 +201,20 @@ func (w *AtomicWriter) Write(payload map[string]FileProjection) error { if err != nil { os.Remove(newDataDirPath) os.RemoveAll(tsDir) - glog.Errorf("%s: error renaming symbolic link for data directory %s: %v", w.logContext, newDataDirPath, err) + klog.Errorf("%s: error renaming symbolic link for data directory %s: %v", w.logContext, newDataDirPath, err) return err } // (10) if err = w.removeUserVisiblePaths(pathsToRemove); err != nil { - glog.Errorf("%s: error removing old visible symlinks: %v", w.logContext, err) + klog.Errorf("%s: error removing old visible symlinks: %v", w.logContext, err) return err } // (11) if len(oldTsDir) > 0 { if err = os.RemoveAll(oldTsPath); err != nil { - glog.Errorf("%s: error removing old data directory %s: %v", w.logContext, oldTsDir, err) + klog.Errorf("%s: error removing old data directory %s: %v", w.logContext, oldTsDir, err) return err } } @@ -329,7 +329,7 @@ func (w *AtomicWriter) pathsToRemove(payload map[string]FileProjection, oldTsDir } else if err != nil { return nil, err } - glog.V(5).Infof("%s: current paths: %+v", w.targetDir, paths.List()) + klog.V(5).Infof("%s: current paths: %+v", w.targetDir, paths.List()) newPaths := sets.NewString() for file := range payload { @@ -341,10 +341,10 @@ func (w *AtomicWriter) pathsToRemove(payload map[string]FileProjection, oldTsDir subPath = strings.TrimSuffix(subPath, string(os.PathSeparator)) } } - glog.V(5).Infof("%s: new paths: %+v", w.targetDir, newPaths.List()) + klog.V(5).Infof("%s: new paths: %+v", w.targetDir, newPaths.List()) result := paths.Difference(newPaths) - glog.V(5).Infof("%s: paths to remove: %+v", w.targetDir, result) + klog.V(5).Infof("%s: paths to remove: %+v", w.targetDir, result) return result, nil } @@ -353,7 +353,7 @@ func (w *AtomicWriter) pathsToRemove(payload map[string]FileProjection, oldTsDir func (w *AtomicWriter) newTimestampDir() (string, error) { tsDir, err := ioutil.TempDir(w.targetDir, time.Now().UTC().Format("..2006_01_02_15_04_05.")) if err != nil { - glog.Errorf("%s: unable to create new temp directory: %v", w.logContext, err) + klog.Errorf("%s: unable to create new temp directory: %v", w.logContext, err) return "", err } @@ -362,7 +362,7 @@ func (w *AtomicWriter) newTimestampDir() (string, error) { // regardless of the process' umask. err = os.Chmod(tsDir, 0755) if err != nil { - glog.Errorf("%s: unable to set mode on new temp directory: %v", w.logContext, err) + klog.Errorf("%s: unable to set mode on new temp directory: %v", w.logContext, err) return "", err } @@ -380,13 +380,13 @@ func (w *AtomicWriter) writePayloadToDir(payload map[string]FileProjection, dir err := os.MkdirAll(baseDir, os.ModePerm) if err != nil { - glog.Errorf("%s: unable to create directory %s: %v", w.logContext, baseDir, err) + klog.Errorf("%s: unable to create directory %s: %v", w.logContext, baseDir, err) return err } err = ioutil.WriteFile(fullPath, content, mode) if err != nil { - glog.Errorf("%s: unable to write file %s with mode %v: %v", w.logContext, fullPath, mode, err) + klog.Errorf("%s: unable to write file %s with mode %v: %v", w.logContext, fullPath, mode, err) return err } // Chmod is needed because ioutil.WriteFile() ends up calling @@ -395,7 +395,7 @@ func (w *AtomicWriter) writePayloadToDir(payload map[string]FileProjection, dir // in the file no matter what the umask is. err = os.Chmod(fullPath, mode) if err != nil { - glog.Errorf("%s: unable to write file %s with mode %v: %v", w.logContext, fullPath, mode, err) + klog.Errorf("%s: unable to write file %s with mode %v: %v", w.logContext, fullPath, mode, err) } } @@ -445,7 +445,7 @@ func (w *AtomicWriter) removeUserVisiblePaths(paths sets.String) error { continue } if err := os.Remove(path.Join(w.targetDir, p)); err != nil { - glog.Errorf("%s: error pruning old user-visible path %s: %v", w.logContext, p, err) + klog.Errorf("%s: error pruning old user-visible path %s: %v", w.logContext, p, err) lasterr = err } } diff --git a/pkg/volume/util/device_util_linux.go b/pkg/volume/util/device_util_linux.go index a3d292d0f96..66e8564915b 100644 --- a/pkg/volume/util/device_util_linux.go +++ b/pkg/volume/util/device_util_linux.go @@ -21,7 +21,7 @@ package util import ( "errors" "fmt" - "github.com/golang/glog" + "k8s.io/klog" "path" "strconv" "strings" @@ -109,7 +109,7 @@ func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) ( } hostNumber, err := strconv.Atoi(strings.TrimPrefix(hostName, "host")) if err != nil { - glog.Errorf("Could not get number from iSCSI host: %s", hostName) + klog.Errorf("Could not get number from iSCSI host: %s", hostName) continue } @@ -222,7 +222,7 @@ func (handler *deviceHandler) FindDevicesForISCSILun(targetIqn string, lun int) } hostNumber, err := strconv.Atoi(strings.TrimPrefix(hostName, "host")) if err != nil { - glog.Errorf("Could not get number from iSCSI host: %s", hostName) + klog.Errorf("Could not get number from iSCSI host: %s", hostName) continue } diff --git a/pkg/volume/util/main_test.go b/pkg/volume/util/main_test.go new file mode 100644 index 00000000000..6af02d0a11d --- /dev/null +++ b/pkg/volume/util/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/pkg/volume/util/metrics.go b/pkg/volume/util/metrics.go index e3af12df890..338e812663b 100644 --- a/pkg/volume/util/metrics.go +++ b/pkg/volume/util/metrics.go @@ -17,9 +17,11 @@ limitations under the License. package util import ( + "fmt" "time" "github.com/prometheus/client_golang/prometheus" + "k8s.io/kubernetes/pkg/volume" ) var storageOperationMetric = prometheus.NewHistogramVec( @@ -62,3 +64,15 @@ func OperationCompleteHook(plugin, operationName string) func(*error) { } return opComplete } + +// GetFullQualifiedPluginNameForVolume returns full qualified plugin name for +// given volume. For CSI plugin, it appends plugin driver name at the end of +// plugin name, e.g. kubernetes.io/csi:csi-hostpath. It helps to distinguish +// between metrics emitted for CSI volumes which may be handled by different +// CSI plugin drivers. +func GetFullQualifiedPluginNameForVolume(pluginName string, spec *volume.Spec) string { + if spec != nil && spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CSI != nil { + return fmt.Sprintf("%s:%s", pluginName, spec.PersistentVolume.Spec.CSI.Driver) + } + return pluginName +} diff --git a/pkg/volume/util/nestedpendingoperations/BUILD b/pkg/volume/util/nestedpendingoperations/BUILD index 4622d91fbb1..945d33dfdf2 100644 --- a/pkg/volume/util/nestedpendingoperations/BUILD +++ b/pkg/volume/util/nestedpendingoperations/BUILD @@ -15,7 +15,7 @@ go_library( "//pkg/volume/util/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/util/nestedpendingoperations/nestedpendingoperations.go b/pkg/volume/util/nestedpendingoperations/nestedpendingoperations.go index 526ea403cee..d9d277e0b21 100644 --- a/pkg/volume/util/nestedpendingoperations/nestedpendingoperations.go +++ b/pkg/volume/util/nestedpendingoperations/nestedpendingoperations.go @@ -28,9 +28,9 @@ import ( "fmt" "sync" - "github.com/golang/glog" "k8s.io/api/core/v1" k8sRuntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff" "k8s.io/kubernetes/pkg/volume/util/types" ) @@ -240,7 +240,7 @@ func (grm *nestedPendingOperations) operationComplete( if *err != nil { // Log error logOperationName := getOperationName(volumeName, podName) - glog.Errorf("operation %s failed with: %v", + klog.Errorf("operation %s failed with: %v", logOperationName, *err) } @@ -252,7 +252,7 @@ func (grm *nestedPendingOperations) operationComplete( if getOpErr != nil { // Failed to find existing operation logOperationName := getOperationName(volumeName, podName) - glog.Errorf("Operation %s completed. error: %v. exponentialBackOffOnError is enabled, but failed to get operation to update.", + klog.Errorf("Operation %s completed. error: %v. exponentialBackOffOnError is enabled, but failed to get operation to update.", logOperationName, *err) return @@ -264,7 +264,7 @@ func (grm *nestedPendingOperations) operationComplete( // Log error operationName := getOperationName(volumeName, podName) - glog.Errorf("%v", grm.operations[existingOpIndex].expBackoff. + klog.Errorf("%v", grm.operations[existingOpIndex].expBackoff. GenerateNoRetriesPermittedMsg(operationName)) } diff --git a/pkg/volume/util/operationexecutor/BUILD b/pkg/volume/util/operationexecutor/BUILD index 2f42e04ea9f..3b2cd65d5d7 100644 --- a/pkg/volume/util/operationexecutor/BUILD +++ b/pkg/volume/util/operationexecutor/BUILD @@ -30,7 +30,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/util/operationexecutor/operation_executor.go b/pkg/volume/util/operationexecutor/operation_executor.go index 8983ae48d39..745910dc220 100644 --- a/pkg/volume/util/operationexecutor/operation_executor.go +++ b/pkg/volume/util/operationexecutor/operation_executor.go @@ -24,7 +24,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -627,14 +627,14 @@ func (oe *operationExecutor) VerifyVolumesAreAttached( for node, nodeAttachedVolumes := range attachedVolumes { for _, volumeAttached := range nodeAttachedVolumes { if volumeAttached.VolumeSpec == nil { - glog.Errorf("VerifyVolumesAreAttached: nil spec for volume %s", volumeAttached.VolumeName) + klog.Errorf("VerifyVolumesAreAttached: nil spec for volume %s", volumeAttached.VolumeName) continue } volumePlugin, err := oe.operationGenerator.GetVolumePluginMgr().FindPluginBySpec(volumeAttached.VolumeSpec) if err != nil || volumePlugin == nil { - glog.Errorf( + klog.Errorf( "VolumesAreAttached.FindPluginBySpec failed for volume %q (spec.Name: %q) on node %q with error: %v", volumeAttached.VolumeName, volumeAttached.VolumeSpec.Name(), @@ -673,7 +673,7 @@ func (oe *operationExecutor) VerifyVolumesAreAttached( // If node doesn't support Bulk volume polling it is best to poll individually nodeError := oe.VerifyVolumesAreAttachedPerNode(nodeAttachedVolumes, node, actualStateOfWorld) if nodeError != nil { - glog.Errorf("BulkVerifyVolumes.VerifyVolumesAreAttached verifying volumes on node %q with %v", node, nodeError) + klog.Errorf("BulkVerifyVolumes.VerifyVolumesAreAttached verifying volumes on node %q with %v", node, nodeError) } break } @@ -686,14 +686,14 @@ func (oe *operationExecutor) VerifyVolumesAreAttached( volumeSpecMapByPlugin[pluginName], actualStateOfWorld) if err != nil { - glog.Errorf("BulkVerifyVolumes.GenerateBulkVolumeVerifyFunc error bulk verifying volumes for plugin %q with %v", pluginName, err) + klog.Errorf("BulkVerifyVolumes.GenerateBulkVolumeVerifyFunc error bulk verifying volumes for plugin %q with %v", pluginName, err) } // Ugly hack to ensure - we don't do parallel bulk polling of same volume plugin uniquePluginName := v1.UniqueVolumeName(pluginName) err = oe.pendingOperations.Run(uniquePluginName, "" /* Pod Name */, generatedOperations) if err != nil { - glog.Errorf("BulkVerifyVolumes.Run Error bulk volume verification for plugin %q with %v", pluginName, err) + klog.Errorf("BulkVerifyVolumes.Run Error bulk volume verification for plugin %q with %v", pluginName, err) } } } @@ -862,7 +862,7 @@ func (oe *operationExecutor) ReconstructVolumeOperation( // Filesystem Volume case if volumeMode == v1.PersistentVolumeFilesystem { // Create volumeSpec from mount path - glog.V(5).Infof("Starting operationExecutor.ReconstructVolumepodName") + klog.V(5).Infof("Starting operationExecutor.ReconstructVolumepodName") volumeSpec, err := plugin.ConstructVolumeSpec(volumeSpecName, mountPath) if err != nil { return nil, err @@ -872,7 +872,7 @@ func (oe *operationExecutor) ReconstructVolumeOperation( // Block Volume case // Create volumeSpec from mount path - glog.V(5).Infof("Starting operationExecutor.ReconstructVolume") + klog.V(5).Infof("Starting operationExecutor.ReconstructVolume") if mapperPlugin == nil { return nil, fmt.Errorf("Could not find block volume plugin %q (spec.Name: %q) pod %q (UID: %q)", pluginName, diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 67ba7f6d04c..36c10b67752 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -22,7 +22,6 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,6 +29,7 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/record" + "k8s.io/klog" expandcache "k8s.io/kubernetes/pkg/controller/volume/expand/cache" "k8s.io/kubernetes/pkg/features" kevents "k8s.io/kubernetes/pkg/kubelet/events" @@ -139,13 +139,13 @@ func (og *operationGenerator) GenerateVolumesAreAttachedFunc( // Iterate each volume spec and put them into a map index by the pluginName for _, volumeAttached := range attachedVolumes { if volumeAttached.VolumeSpec == nil { - glog.Errorf("VerifyVolumesAreAttached.GenerateVolumesAreAttachedFunc: nil spec for volume %s", volumeAttached.VolumeName) + klog.Errorf("VerifyVolumesAreAttached.GenerateVolumesAreAttachedFunc: nil spec for volume %s", volumeAttached.VolumeName) continue } volumePlugin, err := og.volumePluginMgr.FindPluginBySpec(volumeAttached.VolumeSpec) if err != nil || volumePlugin == nil { - glog.Errorf(volumeAttached.GenerateErrorDetailed("VolumesAreAttached.FindPluginBySpec failed", err).Error()) + klog.Errorf(volumeAttached.GenerateErrorDetailed("VolumesAreAttached.FindPluginBySpec failed", err).Error()) continue } volumeSpecList, pluginExists := volumesPerPlugin[volumePlugin.GetPluginName()] @@ -165,7 +165,7 @@ func (og *operationGenerator) GenerateVolumesAreAttachedFunc( attachableVolumePlugin, err := og.volumePluginMgr.FindAttachablePluginByName(pluginName) if err != nil || attachableVolumePlugin == nil { - glog.Errorf( + klog.Errorf( "VolumeAreAttached.FindAttachablePluginBySpec failed for plugin %q with: %v", pluginName, err) @@ -174,7 +174,7 @@ func (og *operationGenerator) GenerateVolumesAreAttachedFunc( volumeAttacher, newAttacherErr := attachableVolumePlugin.NewAttacher() if newAttacherErr != nil { - glog.Errorf( + klog.Errorf( "VolumesAreAttached.NewAttacher failed for getting plugin %q with: %v", pluginName, newAttacherErr) @@ -183,7 +183,7 @@ func (og *operationGenerator) GenerateVolumesAreAttachedFunc( attached, areAttachedErr := volumeAttacher.VolumesAreAttached(volumesSpecs, nodeName) if areAttachedErr != nil { - glog.Errorf( + klog.Errorf( "VolumesAreAttached failed for checking on node %q with: %v", nodeName, areAttachedErr) @@ -193,7 +193,7 @@ func (og *operationGenerator) GenerateVolumesAreAttachedFunc( for spec, check := range attached { if !check { actualStateOfWorld.MarkVolumeAsDetached(volumeSpecMap[spec], nodeName) - glog.V(1).Infof("VerifyVolumesAreAttached determined volume %q (spec.Name: %q) is no longer attached to node %q, therefore it was marked as detached.", + klog.V(1).Infof("VerifyVolumesAreAttached determined volume %q (spec.Name: %q) is no longer attached to node %q, therefore it was marked as detached.", volumeSpecMap[spec], spec.Name(), nodeName) } } @@ -203,7 +203,7 @@ func (og *operationGenerator) GenerateVolumesAreAttachedFunc( return volumetypes.GeneratedOperations{ OperationFunc: volumesAreAttachedFunc, - CompleteFunc: util.OperationCompleteHook("", "verify_volumes_are_attached_per_node"), + CompleteFunc: util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume("", nil), "verify_volumes_are_attached_per_node"), EventRecorderFunc: nil, // nil because we do not want to generate event on error }, nil } @@ -218,7 +218,7 @@ func (og *operationGenerator) GenerateBulkVolumeVerifyFunc( attachableVolumePlugin, err := og.volumePluginMgr.FindAttachablePluginByName(pluginName) if err != nil || attachableVolumePlugin == nil { - glog.Errorf( + klog.Errorf( "BulkVerifyVolume.FindAttachablePluginBySpec failed for plugin %q with: %v", pluginName, err) @@ -228,7 +228,7 @@ func (og *operationGenerator) GenerateBulkVolumeVerifyFunc( volumeAttacher, newAttacherErr := attachableVolumePlugin.NewAttacher() if newAttacherErr != nil { - glog.Errorf( + klog.Errorf( "BulkVerifyVolume.NewAttacher failed for getting plugin %q with: %v", attachableVolumePlugin, newAttacherErr) @@ -237,13 +237,13 @@ func (og *operationGenerator) GenerateBulkVolumeVerifyFunc( bulkVolumeVerifier, ok := volumeAttacher.(volume.BulkVolumeVerifier) if !ok { - glog.Errorf("BulkVerifyVolume failed to type assert attacher %q", bulkVolumeVerifier) + klog.Errorf("BulkVerifyVolume failed to type assert attacher %q", bulkVolumeVerifier) return nil, nil } attached, bulkAttachErr := bulkVolumeVerifier.BulkVerifyVolumes(pluginNodeVolumes) if bulkAttachErr != nil { - glog.Errorf("BulkVerifyVolume.BulkVerifyVolumes Error checking volumes are attached with %v", bulkAttachErr) + klog.Errorf("BulkVerifyVolume.BulkVerifyVolumes Error checking volumes are attached with %v", bulkAttachErr) return nil, nil } @@ -252,7 +252,7 @@ func (og *operationGenerator) GenerateBulkVolumeVerifyFunc( nodeVolumeSpecs, nodeChecked := attached[nodeName] if !nodeChecked { - glog.V(2).Infof("VerifyVolumesAreAttached.BulkVerifyVolumes failed for node %q and leaving volume %q as attached", + klog.V(2).Infof("VerifyVolumesAreAttached.BulkVerifyVolumes failed for node %q and leaving volume %q as attached", nodeName, volumeSpec.Name()) continue @@ -261,7 +261,7 @@ func (og *operationGenerator) GenerateBulkVolumeVerifyFunc( check := nodeVolumeSpecs[volumeSpec] if !check { - glog.V(2).Infof("VerifyVolumesAreAttached.BulkVerifyVolumes failed for node %q and volume %q", + klog.V(2).Infof("VerifyVolumesAreAttached.BulkVerifyVolumes failed for node %q and volume %q", nodeName, volumeSpec.Name()) actualStateOfWorld.MarkVolumeAsDetached(volumeSpecMap[volumeSpec], nodeName) @@ -274,7 +274,7 @@ func (og *operationGenerator) GenerateBulkVolumeVerifyFunc( return volumetypes.GeneratedOperations{ OperationFunc: bulkVolumeVerifyFunc, - CompleteFunc: util.OperationCompleteHook(pluginName, "verify_volumes_are_attached"), + CompleteFunc: util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(pluginName, nil), "verify_volumes_are_attached"), EventRecorderFunc: nil, // nil because we do not want to generate event on error }, nil @@ -319,7 +319,7 @@ func (og *operationGenerator) GenerateAttachVolumeFunc( derr.DevicePath) if addErr != nil { - glog.Errorf("AttachVolume.MarkVolumeAsAttached failed to fix dangling volume error for volume %q with %s", volumeToAttach.VolumeName, addErr) + klog.Errorf("AttachVolume.MarkVolumeAsAttached failed to fix dangling volume error for volume %q with %s", volumeToAttach.VolumeName, addErr) } } @@ -332,7 +332,7 @@ func (og *operationGenerator) GenerateAttachVolumeFunc( for _, pod := range volumeToAttach.ScheduledPods { og.recorder.Eventf(pod, v1.EventTypeNormal, kevents.SuccessfulAttachVolume, simpleMsg) } - glog.Infof(volumeToAttach.GenerateMsgDetailed("AttachVolume.Attach succeeded", "")) + klog.Infof(volumeToAttach.GenerateMsgDetailed("AttachVolume.Attach succeeded", "")) // Update actual state of world addVolumeNodeErr := actualStateOfWorld.MarkVolumeAsAttached( @@ -348,7 +348,7 @@ func (og *operationGenerator) GenerateAttachVolumeFunc( return volumetypes.GeneratedOperations{ OperationFunc: attachVolumeFunc, EventRecorderFunc: eventRecorderFunc, - CompleteFunc: util.OperationCompleteHook(attachableVolumePlugin.GetPluginName(), "volume_attach"), + CompleteFunc: util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(attachableVolumePlugin.GetPluginName(), volumeToAttach.VolumeSpec), "volume_attach"), }, nil } @@ -416,7 +416,7 @@ func (og *operationGenerator) GenerateDetachVolumeFunc( return volumeToDetach.GenerateError("DetachVolume.Detach failed", err) } - glog.Infof(volumeToDetach.GenerateMsgDetailed("DetachVolume.Detach succeeded", "")) + klog.Infof(volumeToDetach.GenerateMsgDetailed("DetachVolume.Detach succeeded", "")) // Update actual state of world actualStateOfWorld.MarkVolumeAsDetached( @@ -427,7 +427,7 @@ func (og *operationGenerator) GenerateDetachVolumeFunc( return volumetypes.GeneratedOperations{ OperationFunc: getVolumePluginMgrFunc, - CompleteFunc: util.OperationCompleteHook(pluginName, "volume_detach"), + CompleteFunc: util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(pluginName, volumeToDetach.VolumeSpec), "volume_detach"), EventRecorderFunc: nil, // nil because we do not want to generate event on error }, nil } @@ -494,7 +494,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( devicePath := volumeToMount.DevicePath if volumeAttacher != nil { // Wait for attachable volumes to finish attaching - glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath))) + klog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath))) devicePath, err = volumeAttacher.WaitForAttach( volumeToMount.VolumeSpec, devicePath, volumeToMount.Pod, waitForAttachTimeout) @@ -503,7 +503,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( return volumeToMount.GenerateError("MountVolume.WaitForAttach failed", err) } - glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) + klog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) } if volumeDeviceMounter != nil { @@ -524,7 +524,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( return volumeToMount.GenerateError("MountVolume.MountDevice failed", err) } - glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.MountDevice succeeded", fmt.Sprintf("device mount path %q", deviceMountPath))) + klog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.MountDevice succeeded", fmt.Sprintf("device mount path %q", deviceMountPath))) // Update actual state of world to reflect volume is globally mounted markDeviceMountedErr := actualStateOfWorld.MarkDeviceAsMounted( @@ -560,11 +560,11 @@ func (og *operationGenerator) GenerateMountVolumeFunc( } _, detailedMsg := volumeToMount.GenerateMsg("MountVolume.SetUp succeeded", "") - verbosity := glog.Level(1) + verbosity := klog.Level(1) if isRemount { - verbosity = glog.Level(4) + verbosity = klog.Level(4) } - glog.V(verbosity).Infof(detailedMsg) + klog.V(verbosity).Infof(detailedMsg) // Update actual state of world markVolMountedErr := actualStateOfWorld.MarkVolumeAsMounted( @@ -593,13 +593,13 @@ func (og *operationGenerator) GenerateMountVolumeFunc( return volumetypes.GeneratedOperations{ OperationFunc: mountVolumeFunc, EventRecorderFunc: eventRecorderFunc, - CompleteFunc: util.OperationCompleteHook(volumePlugin.GetPluginName(), "volume_mount"), + CompleteFunc: util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(volumePlugin.GetPluginName(), volumeToMount.VolumeSpec), "volume_mount"), }, nil } func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath, deviceMountPath, pluginName string) (simpleErr, detailedErr error) { if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) { - glog.V(4).Infof("Resizing is not enabled for this volume %s", volumeToMount.VolumeName) + klog.V(4).Infof("Resizing is not enabled for this volume %s", volumeToMount.VolumeName) return nil, nil } @@ -621,11 +621,11 @@ func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devi pvSpecCap := pv.Spec.Capacity[v1.ResourceStorage] if pvcStatusCap.Cmp(pvSpecCap) < 0 { // File system resize was requested, proceed - glog.V(4).Infof(volumeToMount.GenerateMsgDetailed("MountVolume.resizeFileSystem entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath))) + klog.V(4).Infof(volumeToMount.GenerateMsgDetailed("MountVolume.resizeFileSystem entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath))) if volumeToMount.VolumeSpec.ReadOnly { simpleMsg, detailedMsg := volumeToMount.GenerateMsg("MountVolume.resizeFileSystem failed", "requested read-only file system") - glog.Warningf(detailedMsg) + klog.Warningf(detailedMsg) og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FileSystemResizeFailed, simpleMsg) return nil, nil } @@ -634,7 +634,7 @@ func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devi } simpleMsg, detailedMsg := volumeToMount.GenerateMsg("MountVolume.resizeFileSystem succeeded", "") og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeNormal, kevents.FileSystemResizeSuccess, simpleMsg) - glog.Infof(detailedMsg) + klog.Infof(detailedMsg) // File system resize succeeded, now update the PVC's Capacity to match the PV's err = util.MarkFSResizeFinished(pvc, pv.Spec.Capacity, og.kubeClient) if err != nil { @@ -680,7 +680,7 @@ func (og *operationGenerator) GenerateUnmountVolumeFunc( return volumeToUnmount.GenerateError("UnmountVolume.TearDown failed", unmountErr) } - glog.Infof( + klog.Infof( "UnmountVolume.TearDown succeeded for volume %q (OuterVolumeSpecName: %q) pod %q (UID: %q). InnerVolumeSpecName %q. PluginName %q, VolumeGidValue %q", volumeToUnmount.VolumeName, volumeToUnmount.OuterVolumeSpecName, @@ -695,7 +695,7 @@ func (og *operationGenerator) GenerateUnmountVolumeFunc( volumeToUnmount.PodName, volumeToUnmount.VolumeName) if markVolMountedErr != nil { // On failure, just log and exit - glog.Errorf(volumeToUnmount.GenerateErrorDetailed("UnmountVolume.MarkVolumeAsUnmounted failed", markVolMountedErr).Error()) + klog.Errorf(volumeToUnmount.GenerateErrorDetailed("UnmountVolume.MarkVolumeAsUnmounted failed", markVolMountedErr).Error()) } return nil, nil @@ -703,7 +703,7 @@ func (og *operationGenerator) GenerateUnmountVolumeFunc( return volumetypes.GeneratedOperations{ OperationFunc: unmountVolumeFunc, - CompleteFunc: util.OperationCompleteHook(volumePlugin.GetPluginName(), "volume_unmount"), + CompleteFunc: util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(volumePlugin.GetPluginName(), volumeToUnmount.VolumeSpec), "volume_unmount"), EventRecorderFunc: nil, // nil because we do not want to generate event on error }, nil } @@ -765,7 +765,7 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( fmt.Errorf("the device is in use when it was no longer expected to be in use")) } - glog.Infof(deviceToDetach.GenerateMsg("UnmountDevice succeeded", "")) + klog.Infof(deviceToDetach.GenerateMsg("UnmountDevice succeeded", "")) // Update actual state of world markDeviceUnmountedErr := actualStateOfWorld.MarkDeviceAsUnmounted( @@ -780,7 +780,7 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( return volumetypes.GeneratedOperations{ OperationFunc: unmountDeviceFunc, - CompleteFunc: util.OperationCompleteHook(deviceMountableVolumePlugin.GetPluginName(), "unmount_device"), + CompleteFunc: util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(deviceMountableVolumePlugin.GetPluginName(), deviceToDetach.VolumeSpec), "unmount_device"), EventRecorderFunc: nil, // nil because we do not want to generate event on error }, nil } @@ -844,7 +844,7 @@ func (og *operationGenerator) GenerateMapVolumeFunc( } if volumeAttacher != nil { // Wait for attachable volumes to finish attaching - glog.Infof(volumeToMount.GenerateMsgDetailed("MapVolume.WaitForAttach entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath))) + klog.Infof(volumeToMount.GenerateMsgDetailed("MapVolume.WaitForAttach entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath))) devicePath, err = volumeAttacher.WaitForAttach( volumeToMount.VolumeSpec, volumeToMount.DevicePath, volumeToMount.Pod, waitForAttachTimeout) @@ -853,7 +853,7 @@ func (og *operationGenerator) GenerateMapVolumeFunc( return volumeToMount.GenerateError("MapVolume.WaitForAttach failed", err) } - glog.Infof(volumeToMount.GenerateMsgDetailed("MapVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) + klog.Infof(volumeToMount.GenerateMsgDetailed("MapVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) } // A plugin doesn't have attacher also needs to map device to global map path with SetUpDevice() @@ -909,15 +909,15 @@ func (og *operationGenerator) GenerateMapVolumeFunc( // Device mapping for global map path succeeded simpleMsg, detailedMsg := volumeToMount.GenerateMsg("MapVolume.MapDevice succeeded", fmt.Sprintf("globalMapPath %q", globalMapPath)) - verbosity := glog.Level(4) + verbosity := klog.Level(4) og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeNormal, kevents.SuccessfulMountVolume, simpleMsg) - glog.V(verbosity).Infof(detailedMsg) + klog.V(verbosity).Infof(detailedMsg) // Device mapping for pod device map path succeeded simpleMsg, detailedMsg = volumeToMount.GenerateMsg("MapVolume.MapDevice succeeded", fmt.Sprintf("volumeMapPath %q", volumeMapPath)) - verbosity = glog.Level(1) + verbosity = klog.Level(1) og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeNormal, kevents.SuccessfulMountVolume, simpleMsg) - glog.V(verbosity).Infof(detailedMsg) + klog.V(verbosity).Infof(detailedMsg) // Update actual state of world markVolMountedErr := actualStateOfWorld.MarkVolumeAsMounted( @@ -946,7 +946,7 @@ func (og *operationGenerator) GenerateMapVolumeFunc( return volumetypes.GeneratedOperations{ OperationFunc: mapVolumeFunc, EventRecorderFunc: eventRecorderFunc, - CompleteFunc: util.OperationCompleteHook(blockVolumePlugin.GetPluginName(), "map_volume"), + CompleteFunc: util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(blockVolumePlugin.GetPluginName(), volumeToMount.VolumeSpec), "map_volume"), }, nil } @@ -992,7 +992,7 @@ func (og *operationGenerator) GenerateUnmapVolumeFunc( return volumeToUnmount.GenerateError("UnmapVolume.UnmapDevice on global map path failed", unmapDeviceErr) } - glog.Infof( + klog.Infof( "UnmapVolume succeeded for volume %q (OuterVolumeSpecName: %q) pod %q (UID: %q). InnerVolumeSpecName %q. PluginName %q, VolumeGidValue %q", volumeToUnmount.VolumeName, volumeToUnmount.OuterVolumeSpecName, @@ -1007,7 +1007,7 @@ func (og *operationGenerator) GenerateUnmapVolumeFunc( volumeToUnmount.PodName, volumeToUnmount.VolumeName) if markVolUnmountedErr != nil { // On failure, just log and exit - glog.Errorf(volumeToUnmount.GenerateErrorDetailed("UnmapVolume.MarkVolumeAsUnmounted failed", markVolUnmountedErr).Error()) + klog.Errorf(volumeToUnmount.GenerateErrorDetailed("UnmapVolume.MarkVolumeAsUnmounted failed", markVolUnmountedErr).Error()) } return nil, nil @@ -1015,7 +1015,7 @@ func (og *operationGenerator) GenerateUnmapVolumeFunc( return volumetypes.GeneratedOperations{ OperationFunc: unmapVolumeFunc, - CompleteFunc: util.OperationCompleteHook(blockVolumePlugin.GetPluginName(), "unmap_volume"), + CompleteFunc: util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(blockVolumePlugin.GetPluginName(), volumeToUnmount.VolumeSpec), "unmap_volume"), EventRecorderFunc: nil, // nil because we do not want to generate event on error }, nil } @@ -1045,7 +1045,7 @@ func (og *operationGenerator) GenerateUnmapDeviceFunc( } blockVolumeUnmapper, newUnmapperErr := blockVolumePlugin.NewBlockVolumeUnmapper( - string(deviceToDetach.VolumeName), + deviceToDetach.VolumeSpec.Name(), "" /* podUID */) if newUnmapperErr != nil { return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmapDevice.NewUnmapper failed", newUnmapperErr) @@ -1067,11 +1067,11 @@ func (og *operationGenerator) GenerateUnmapDeviceFunc( // The block volume is not referenced from Pods. Release file descriptor lock. // This should be done before calling TearDownDevice, because some plugins that do local detach // in TearDownDevice will fail in detaching device due to the refcnt on the loopback device. - glog.V(4).Infof("UnmapDevice: deviceToDetach.DevicePath: %v", deviceToDetach.DevicePath) + klog.V(4).Infof("UnmapDevice: deviceToDetach.DevicePath: %v", deviceToDetach.DevicePath) loopPath, err := og.blkUtil.GetLoopDevice(deviceToDetach.DevicePath) if err != nil { if err.Error() == volumepathhandler.ErrDeviceNotFound { - glog.Warningf(deviceToDetach.GenerateMsgDetailed("UnmapDevice: Couldn't find loopback device which takes file descriptor lock", fmt.Sprintf("device path: %q", deviceToDetach.DevicePath))) + klog.Warningf(deviceToDetach.GenerateMsgDetailed("UnmapDevice: Couldn't find loopback device which takes file descriptor lock", fmt.Sprintf("device path: %q", deviceToDetach.DevicePath))) } else { errInfo := "UnmapDevice.GetLoopDevice failed to get loopback device, " + fmt.Sprintf("device path: %q", deviceToDetach.DevicePath) return deviceToDetach.GenerateError(errInfo, err) @@ -1115,7 +1115,7 @@ func (og *operationGenerator) GenerateUnmapDeviceFunc( fmt.Errorf("the device is in use when it was no longer expected to be in use")) } - glog.Infof(deviceToDetach.GenerateMsgDetailed("UnmapDevice succeeded", "")) + klog.Infof(deviceToDetach.GenerateMsgDetailed("UnmapDevice succeeded", "")) // Update actual state of world markDeviceUnmountedErr := actualStateOfWorld.MarkDeviceAsUnmounted( @@ -1130,7 +1130,7 @@ func (og *operationGenerator) GenerateUnmapDeviceFunc( return volumetypes.GeneratedOperations{ OperationFunc: unmapDeviceFunc, - CompleteFunc: util.OperationCompleteHook(blockVolumePlugin.GetPluginName(), "unmap_device"), + CompleteFunc: util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(blockVolumePlugin.GetPluginName(), deviceToDetach.VolumeSpec), "unmap_device"), EventRecorderFunc: nil, // nil because we do not want to generate event on error }, nil } @@ -1189,7 +1189,7 @@ func (og *operationGenerator) GenerateVerifyControllerAttachedVolumeFunc( if attachedVolume.Name == volumeToMount.VolumeName { addVolumeNodeErr := actualStateOfWorld.MarkVolumeAsAttached( v1.UniqueVolumeName(""), volumeToMount.VolumeSpec, nodeName, attachedVolume.DevicePath) - glog.Infof(volumeToMount.GenerateMsgDetailed("Controller attach succeeded", fmt.Sprintf("device path: %q", attachedVolume.DevicePath))) + klog.Infof(volumeToMount.GenerateMsgDetailed("Controller attach succeeded", fmt.Sprintf("device path: %q", attachedVolume.DevicePath))) if addVolumeNodeErr != nil { // On failure, return error. Caller will log and retry. return volumeToMount.GenerateError("VerifyControllerAttachedVolume.MarkVolumeAsAttached failed", addVolumeNodeErr) @@ -1204,7 +1204,7 @@ func (og *operationGenerator) GenerateVerifyControllerAttachedVolumeFunc( return volumetypes.GeneratedOperations{ OperationFunc: verifyControllerAttachedVolumeFunc, - CompleteFunc: util.OperationCompleteHook(volumePlugin.GetPluginName(), "verify_controller_attached_volume"), + CompleteFunc: util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(volumePlugin.GetPluginName(), volumeToMount.VolumeSpec), "verify_controller_attached_volume"), EventRecorderFunc: nil, // nil because we do not want to generate event on error }, nil @@ -1216,7 +1216,7 @@ func (og *operationGenerator) verifyVolumeIsSafeToDetach( node, fetchErr := og.kubeClient.CoreV1().Nodes().Get(string(volumeToDetach.NodeName), metav1.GetOptions{}) if fetchErr != nil { if errors.IsNotFound(fetchErr) { - glog.Warningf(volumeToDetach.GenerateMsgDetailed("Node not found on API server. DetachVolume will skip safe to detach check", "")) + klog.Warningf(volumeToDetach.GenerateMsgDetailed("Node not found on API server. DetachVolume will skip safe to detach check", "")) return nil } @@ -1240,7 +1240,7 @@ func (og *operationGenerator) verifyVolumeIsSafeToDetach( } // Volume is not marked as in use by node - glog.Infof(volumeToDetach.GenerateMsgDetailed("Verified volume is safe to detach", "")) + klog.Infof(volumeToDetach.GenerateMsgDetailed("Verified volume is safe to detach", "")) return nil } @@ -1274,7 +1274,7 @@ func (og *operationGenerator) GenerateExpandVolumeFunc( return detailedErr, detailedErr } - glog.Infof("ExpandVolume succeeded for volume %s", pvcWithResizeRequest.QualifiedName()) + klog.Infof("ExpandVolume succeeded for volume %s", pvcWithResizeRequest.QualifiedName()) newSize = updatedSize // k8s doesn't have transactions, we can't guarantee that after updating PV - updating PVC will be // successful, that is why all PVCs for which pvc.Spec.Size > pvc.Status.Size must be reprocessed @@ -1285,14 +1285,14 @@ func (og *operationGenerator) GenerateExpandVolumeFunc( detailedErr := fmt.Errorf("Error updating PV spec capacity for volume %q with : %v", pvcWithResizeRequest.QualifiedName(), updateErr) return detailedErr, detailedErr } - glog.Infof("ExpandVolume.UpdatePV succeeded for volume %s", pvcWithResizeRequest.QualifiedName()) + klog.Infof("ExpandVolume.UpdatePV succeeded for volume %s", pvcWithResizeRequest.QualifiedName()) } // No Cloudprovider resize needed, lets mark resizing as done // Rest of the volume expand controller code will assume PVC as *not* resized until pvc.Status.Size // reflects user requested size. if !volumePlugin.RequiresFSResize() { - glog.V(4).Infof("Controller resizing done for PVC %s", pvcWithResizeRequest.QualifiedName()) + klog.V(4).Infof("Controller resizing done for PVC %s", pvcWithResizeRequest.QualifiedName()) err := resizeMap.MarkAsResized(pvcWithResizeRequest, newSize) if err != nil { @@ -1305,7 +1305,7 @@ func (og *operationGenerator) GenerateExpandVolumeFunc( err := resizeMap.MarkForFSResize(pvcWithResizeRequest) if err != nil { detailedErr := fmt.Errorf("Error updating pvc %s condition for fs resize : %v", pvcWithResizeRequest.QualifiedName(), err) - glog.Warning(detailedErr) + klog.Warning(detailedErr) return nil, nil } } @@ -1321,7 +1321,7 @@ func (og *operationGenerator) GenerateExpandVolumeFunc( return volumetypes.GeneratedOperations{ OperationFunc: expandVolumeFunc, EventRecorderFunc: eventRecorderFunc, - CompleteFunc: util.OperationCompleteHook(volumePlugin.GetPluginName(), "expand_volume"), + CompleteFunc: util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(volumePlugin.GetPluginName(), volumeSpec), "expand_volume"), }, nil } @@ -1378,7 +1378,7 @@ func (og *operationGenerator) GenerateExpandVolumeFSWithoutUnmountingFunc( return volumetypes.GeneratedOperations{ OperationFunc: fsResizeFunc, EventRecorderFunc: eventRecorderFunc, - CompleteFunc: util.OperationCompleteHook(volumePlugin.GetPluginName(), "volume_fs_resize"), + CompleteFunc: util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(volumePlugin.GetPluginName(), volumeToMount.VolumeSpec), "volume_fs_resize"), }, nil } @@ -1421,7 +1421,7 @@ func isDeviceOpened(deviceToDetach AttachedVolume, mounter mount.Interface) (boo (devicePathErr != nil && strings.Contains(devicePathErr.Error(), "does not exist")) { // not a device path or path doesn't exist //TODO: refer to #36092 - glog.V(3).Infof("The path isn't device path or doesn't exist. Skip checking device path: %s", deviceToDetach.DevicePath) + klog.V(3).Infof("The path isn't device path or doesn't exist. Skip checking device path: %s", deviceToDetach.DevicePath) deviceOpened = false } else if devicePathErr != nil { return false, deviceToDetach.GenerateErrorDetailed("PathIsDevice failed", devicePathErr) diff --git a/pkg/volume/util/recyclerclient/BUILD b/pkg/volume/util/recyclerclient/BUILD index e43b35af736..3ea30eb4491 100644 --- a/pkg/volume/util/recyclerclient/BUILD +++ b/pkg/volume/util/recyclerclient/BUILD @@ -12,7 +12,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/util/recyclerclient/recycler_client.go b/pkg/volume/util/recyclerclient/recycler_client.go index 275f55bbe33..c7a8f147f87 100644 --- a/pkg/volume/util/recyclerclient/recycler_client.go +++ b/pkg/volume/util/recyclerclient/recycler_client.go @@ -20,13 +20,13 @@ import ( "fmt" "sync" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" ) type RecycleEventRecorder func(eventtype, message string) @@ -51,7 +51,7 @@ func RecycleVolumeByWatchingPodUntilCompletion(pvName string, pod *v1.Pod, kubeC // same as above func comments, except 'recyclerClient' is a narrower pod API // interface to ease testing func internalRecycleVolumeByWatchingPodUntilCompletion(pvName string, pod *v1.Pod, recyclerClient recyclerClient) error { - glog.V(5).Infof("creating recycler pod for volume %s\n", pod.Name) + klog.V(5).Infof("creating recycler pod for volume %s\n", pod.Name) // Generate unique name for the recycler pod - we need to get "already // exists" error when a previous controller has already started recycling @@ -63,7 +63,7 @@ func internalRecycleVolumeByWatchingPodUntilCompletion(pvName string, pod *v1.Po defer close(stopChannel) podCh, err := recyclerClient.WatchPod(pod.Name, pod.Namespace, stopChannel) if err != nil { - glog.V(4).Infof("cannot start watcher for pod %s/%s: %v", pod.Namespace, pod.Name, err) + klog.V(4).Infof("cannot start watcher for pod %s/%s: %v", pod.Namespace, pod.Name, err) return err } @@ -84,10 +84,10 @@ func internalRecycleVolumeByWatchingPodUntilCompletion(pvName string, pod *v1.Po err = waitForPod(pod, recyclerClient, podCh) // In all cases delete the recycler pod and log its result. - glog.V(2).Infof("deleting recycler pod %s/%s", pod.Namespace, pod.Name) + klog.V(2).Infof("deleting recycler pod %s/%s", pod.Namespace, pod.Name) deleteErr := recyclerClient.DeletePod(pod.Name, pod.Namespace) if deleteErr != nil { - glog.Errorf("failed to delete recycler pod %s/%s: %v", pod.Namespace, pod.Name, err) + klog.Errorf("failed to delete recycler pod %s/%s: %v", pod.Namespace, pod.Name, err) } // Returning recycler error is preferred, the pod will be deleted again on @@ -117,7 +117,7 @@ func waitForPod(pod *v1.Pod, recyclerClient recyclerClient, podCh <-chan watch.E case *v1.Pod: // POD changed pod := event.Object.(*v1.Pod) - glog.V(4).Infof("recycler pod update received: %s %s/%s %s", event.Type, pod.Namespace, pod.Name, pod.Status.Phase) + klog.V(4).Infof("recycler pod update received: %s %s/%s %s", event.Type, pod.Namespace, pod.Name, pod.Status.Phase) switch event.Type { case watch.Added, watch.Modified: if pod.Status.Phase == v1.PodSucceeded { @@ -142,7 +142,7 @@ func waitForPod(pod *v1.Pod, recyclerClient recyclerClient, podCh <-chan watch.E case *v1.Event: // Event received podEvent := event.Object.(*v1.Event) - glog.V(4).Infof("recycler event received: %s %s/%s %s/%s %s", event.Type, podEvent.Namespace, podEvent.Name, podEvent.InvolvedObject.Namespace, podEvent.InvolvedObject.Name, podEvent.Message) + klog.V(4).Infof("recycler event received: %s %s/%s %s/%s %s", event.Type, podEvent.Namespace, podEvent.Name, podEvent.InvolvedObject.Namespace, podEvent.InvolvedObject.Name, podEvent.Message) if event.Type == watch.Added { recyclerClient.Event(podEvent.Type, podEvent.Message) } diff --git a/pkg/volume/util/util.go b/pkg/volume/util/util.go index faf0d34b61d..070961c2822 100644 --- a/pkg/volume/util/util.go +++ b/pkg/volume/util/util.go @@ -25,7 +25,6 @@ import ( "strings" "syscall" - "github.com/golang/glog" "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" "k8s.io/kubernetes/pkg/api/legacyscheme" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/features" @@ -101,7 +101,7 @@ func IsReady(dir string) bool { } if !s.Mode().IsRegular() { - glog.Errorf("ready-file is not a file: %s", readyFile) + klog.Errorf("ready-file is not a file: %s", readyFile) return false } @@ -113,14 +113,14 @@ func IsReady(dir string) bool { // created. func SetReady(dir string) { if err := os.MkdirAll(dir, 0750); err != nil && !os.IsExist(err) { - glog.Errorf("Can't mkdir %s: %v", dir, err) + klog.Errorf("Can't mkdir %s: %v", dir, err) return } readyFile := path.Join(dir, readyFileName) file, err := os.Create(readyFile) if err != nil { - glog.Errorf("Can't touch %s: %v", readyFile, err) + klog.Errorf("Can't touch %s: %v", readyFile, err) return } file.Close() @@ -140,7 +140,7 @@ func UnmountPath(mountPath string, mounter mount.Interface) error { func UnmountMountPoint(mountPath string, mounter mount.Interface, extensiveMountPointCheck bool) error { pathExists, pathErr := PathExists(mountPath) if !pathExists { - glog.Warningf("Warning: Unmount skipped because path does not exist: %v", mountPath) + klog.Warningf("Warning: Unmount skipped because path does not exist: %v", mountPath) return nil } corruptedMnt := IsCorruptedMnt(pathErr) @@ -171,13 +171,13 @@ func doUnmountMountPoint(mountPath string, mounter mount.Interface, extensiveMou } if notMnt { - glog.Warningf("Warning: %q is not a mountpoint, deleting", mountPath) + klog.Warningf("Warning: %q is not a mountpoint, deleting", mountPath) return os.Remove(mountPath) } } // Unmount the mount path - glog.V(4).Infof("%q is a mountpoint, unmounting", mountPath) + klog.V(4).Infof("%q is a mountpoint, unmounting", mountPath) if err := mounter.Unmount(mountPath); err != nil { return err } @@ -186,7 +186,7 @@ func doUnmountMountPoint(mountPath string, mounter mount.Interface, extensiveMou return mntErr } if notMnt { - glog.V(4).Infof("%q is unmounted, deleting the directory", mountPath) + klog.V(4).Infof("%q is unmounted, deleting the directory", mountPath) return os.Remove(mountPath) } return fmt.Errorf("Failed to unmount path %v", mountPath) @@ -291,7 +291,7 @@ func checkVolumeNodeAffinity(pv *v1.PersistentVolume, nodeLabels map[string]stri if pv.Spec.NodeAffinity.Required != nil { terms := pv.Spec.NodeAffinity.Required.NodeSelectorTerms - glog.V(10).Infof("Match for Required node selector terms %+v", terms) + klog.V(10).Infof("Match for Required node selector terms %+v", terms) if !v1helper.MatchNodeSelectorTerms(terms, labels.Set(nodeLabels), nil) { return fmt.Errorf("No matching NodeSelectorTerms") } @@ -607,7 +607,7 @@ func ChooseZoneForVolume(zones sets.String, pvcName string) string { zoneSlice := zones.List() zone := zoneSlice[(hash+index)%uint32(len(zoneSlice))] - glog.V(2).Infof("Creating volume for PVC %q; chose zone=%q from zones=%q", pvcName, zone, zoneSlice) + klog.V(2).Infof("Creating volume for PVC %q; chose zone=%q from zones=%q", pvcName, zone, zoneSlice) return zone } @@ -666,7 +666,7 @@ func ChooseZonesForVolume(zones sets.String, pvcName string, numZones uint32) se replicaZones.Insert(zone) } - glog.V(2).Infof("Creating volume for replicated PVC %q; chosen zones=%q from zones=%q", + klog.V(2).Infof("Creating volume for replicated PVC %q; chosen zones=%q from zones=%q", pvcName, replicaZones.UnsortedList(), zoneSlice) return replicaZones } @@ -674,7 +674,7 @@ func ChooseZonesForVolume(zones sets.String, pvcName string, numZones uint32) se func getPVCNameHashAndIndexOffset(pvcName string) (hash uint32, index uint32) { if pvcName == "" { // We should always be called with a name; this shouldn't happen - glog.Warningf("No name defined during volume create; choosing random zone") + klog.Warningf("No name defined during volume create; choosing random zone") hash = rand.Uint32() } else { @@ -710,7 +710,7 @@ func getPVCNameHashAndIndexOffset(pvcName string) (hash uint32, index uint32) { hashString = hashString[lastDash+1:] } - glog.V(2).Infof("Detected StatefulSet-style volume name %q; index=%d", pvcName, index) + klog.V(2).Infof("Detected StatefulSet-style volume name %q; index=%d", pvcName, index) } } @@ -726,7 +726,7 @@ func getPVCNameHashAndIndexOffset(pvcName string) (hash uint32, index uint32) { // UnmountViaEmptyDir delegates the tear down operation for secret, configmap, git_repo and downwardapi // to empty_dir func UnmountViaEmptyDir(dir string, host volume.VolumeHost, volName string, volSpec volume.Spec, podUID utypes.UID) error { - glog.V(3).Infof("Tearing down volume %v for pod %v at %v", volName, podUID, dir) + klog.V(3).Infof("Tearing down volume %v for pod %v at %v", volName, podUID, dir) // Wrap EmptyDir, let it do the teardown. wrapped, err := host.NewWrapperUnmounter(volName, volSpec, podUID) diff --git a/pkg/volume/util/util_test.go b/pkg/volume/util/util_test.go index c5e4e8574f3..8f376a3adb3 100644 --- a/pkg/volume/util/util_test.go +++ b/pkg/volume/util/util_test.go @@ -24,12 +24,15 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" utiltesting "k8s.io/client-go/util/testing" + // util.go uses api.Codecs.LegacyCodec so import this package to do some // resource initialization. "hash/fnv" _ "k8s.io/kubernetes/pkg/apis/core/install" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "reflect" @@ -1426,59 +1429,54 @@ func TestSelectZoneForVolume(t *testing.T) { } for _, test := range tests { - utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") - if test.VolumeScheduling { - utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - } + t.Run(test.Name, func(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, test.VolumeScheduling)() - var zonesParameter, zonesWithNodes sets.String - var err error + var zonesParameter, zonesWithNodes sets.String + var err error - if test.Zones != "" { - zonesParameter, err = ZonesToSet(test.Zones) - if err != nil { - t.Errorf("Could not convert Zones to a set: %s. This is a test error %s", test.Zones, test.Name) - continue - } - } - - if test.ZonesWithNodes != "" { - zonesWithNodes, err = ZonesToSet(test.ZonesWithNodes) - if err != nil { - t.Errorf("Could not convert specified ZonesWithNodes to a set: %s. This is a test error %s", test.ZonesWithNodes, test.Name) - continue - } - } - - zone, err := SelectZoneForVolume(test.ZonePresent, test.ZonesPresent, test.Zone, zonesParameter, zonesWithNodes, test.Node, test.AllowedTopologies, test.Name) - - if test.Reject && err == nil { - t.Errorf("Unexpected zone from SelectZoneForVolume for %s", zone) - continue - } - - if !test.Reject { - if err != nil { - t.Errorf("Unexpected error from SelectZoneForVolume for %s; Error: %v", test.Name, err) - continue - } - - if test.ExpectSpecificZone == true { - if zone != test.ExpectedZone { - t.Errorf("Expected zone %v does not match obtained zone %v for %s", test.ExpectedZone, zone, test.Name) + if test.Zones != "" { + zonesParameter, err = ZonesToSet(test.Zones) + if err != nil { + t.Fatalf("Could not convert Zones to a set: %s. This is a test error %s", test.Zones, test.Name) } - continue } - expectedZones, err := ZonesToSet(test.ExpectedZones) - if err != nil { - t.Errorf("Could not convert ExpectedZones to a set: %s. This is a test error", test.ExpectedZones) - continue + if test.ZonesWithNodes != "" { + zonesWithNodes, err = ZonesToSet(test.ZonesWithNodes) + if err != nil { + t.Fatalf("Could not convert specified ZonesWithNodes to a set: %s. This is a test error %s", test.ZonesWithNodes, test.Name) + } } - if !expectedZones.Has(zone) { - t.Errorf("Obtained zone %s not member of expectedZones %s", zone, expectedZones) + + zone, err := SelectZoneForVolume(test.ZonePresent, test.ZonesPresent, test.Zone, zonesParameter, zonesWithNodes, test.Node, test.AllowedTopologies, test.Name) + + if test.Reject && err == nil { + t.Errorf("Unexpected zone from SelectZoneForVolume for %s", zone) } - } + + if !test.Reject { + if err != nil { + t.Errorf("Unexpected error from SelectZoneForVolume for %s; Error: %v", test.Name, err) + } + + if test.ExpectSpecificZone == true { + if zone != test.ExpectedZone { + t.Errorf("Expected zone %v does not match obtained zone %v for %s", test.ExpectedZone, zone, test.Name) + } + } + + if test.ExpectedZones != "" { + expectedZones, err := ZonesToSet(test.ExpectedZones) + if err != nil { + t.Fatalf("Could not convert ExpectedZones to a set: %s. This is a test error", test.ExpectedZones) + } + if !expectedZones.Has(zone) { + t.Errorf("Obtained zone %s not member of expectedZones %s", zone, expectedZones) + } + } + } + }) } } diff --git a/pkg/volume/util/volumepathhandler/BUILD b/pkg/volume/util/volumepathhandler/BUILD index e61e3973ea5..e344849a517 100644 --- a/pkg/volume/util/volumepathhandler/BUILD +++ b/pkg/volume/util/volumepathhandler/BUILD @@ -11,7 +11,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/util/volumepathhandler/volume_path_handler.go b/pkg/volume/util/volumepathhandler/volume_path_handler.go index 61680c11577..a7822efc3e5 100644 --- a/pkg/volume/util/volumepathhandler/volume_path_handler.go +++ b/pkg/volume/util/volumepathhandler/volume_path_handler.go @@ -23,7 +23,7 @@ import ( "path" "path/filepath" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/types" ) @@ -86,14 +86,14 @@ func (v VolumePathHandler) MapDevice(devicePath string, mapPath string, linkName if !filepath.IsAbs(mapPath) { return fmt.Errorf("The map path should be absolute: map path: %s", mapPath) } - glog.V(5).Infof("MapDevice: devicePath %s", devicePath) - glog.V(5).Infof("MapDevice: mapPath %s", mapPath) - glog.V(5).Infof("MapDevice: linkName %s", linkName) + klog.V(5).Infof("MapDevice: devicePath %s", devicePath) + klog.V(5).Infof("MapDevice: mapPath %s", mapPath) + klog.V(5).Infof("MapDevice: linkName %s", linkName) // Check and create mapPath _, err := os.Stat(mapPath) if err != nil && !os.IsNotExist(err) { - glog.Errorf("cannot validate map path: %s", mapPath) + klog.Errorf("cannot validate map path: %s", mapPath) return err } if err = os.MkdirAll(mapPath, 0750); err != nil { @@ -115,15 +115,15 @@ func (v VolumePathHandler) UnmapDevice(mapPath string, linkName string) error { if len(mapPath) == 0 { return fmt.Errorf("Failed to unmap device from map path. mapPath is empty") } - glog.V(5).Infof("UnmapDevice: mapPath %s", mapPath) - glog.V(5).Infof("UnmapDevice: linkName %s", linkName) + klog.V(5).Infof("UnmapDevice: mapPath %s", mapPath) + klog.V(5).Infof("UnmapDevice: linkName %s", linkName) // Check symbolic link exists linkPath := path.Join(mapPath, string(linkName)) if islinkExist, checkErr := v.IsSymlinkExist(linkPath); checkErr != nil { return checkErr } else if !islinkExist { - glog.Warningf("Warning: Unmap skipped because symlink does not exist on the path: %v", linkPath) + klog.Warningf("Warning: Unmap skipped because symlink does not exist on the path: %v", linkPath) return nil } err := os.Remove(linkPath) @@ -135,7 +135,7 @@ func (v VolumePathHandler) RemoveMapPath(mapPath string) error { if len(mapPath) == 0 { return fmt.Errorf("Failed to remove map path. mapPath is empty") } - glog.V(5).Infof("RemoveMapPath: mapPath %s", mapPath) + klog.V(5).Infof("RemoveMapPath: mapPath %s", mapPath) err := os.RemoveAll(mapPath) if err != nil && !os.IsNotExist(err) { return err @@ -180,12 +180,12 @@ func (v VolumePathHandler) GetDeviceSymlinkRefs(devPath string, mapPath string) if err != nil { return nil, fmt.Errorf("Symbolic link cannot be retrieved %v", err) } - glog.V(5).Infof("GetDeviceSymlinkRefs: filepath: %v, devPath: %v", filepath, devPath) + klog.V(5).Infof("GetDeviceSymlinkRefs: filepath: %v, devPath: %v", filepath, devPath) if filepath == devPath { refs = append(refs, path.Join(mapPath, filename)) } } - glog.V(5).Infof("GetDeviceSymlinkRefs: refs %v", refs) + klog.V(5).Infof("GetDeviceSymlinkRefs: refs %v", refs) return refs, nil } @@ -201,7 +201,7 @@ func (v VolumePathHandler) FindGlobalMapPathUUIDFromPod(pluginDir, mapPath strin return err } if (fi.Mode()&os.ModeSymlink == os.ModeSymlink) && (fi.Name() == string(podUID)) { - glog.V(5).Infof("FindGlobalMapPathFromPod: path %s, mapPath %s", path, mapPath) + klog.V(5).Infof("FindGlobalMapPathFromPod: path %s, mapPath %s", path, mapPath) if res, err := compareSymlinks(path, mapPath); err == nil && res { globalMapPathUUID = path } @@ -211,7 +211,7 @@ func (v VolumePathHandler) FindGlobalMapPathUUIDFromPod(pluginDir, mapPath strin if err != nil { return "", err } - glog.V(5).Infof("FindGlobalMapPathFromPod: globalMapPathUUID %s", globalMapPathUUID) + klog.V(5).Infof("FindGlobalMapPathFromPod: globalMapPathUUID %s", globalMapPathUUID) // Return path contains global map path + {pod uuid} return globalMapPathUUID, nil } @@ -225,7 +225,7 @@ func compareSymlinks(global, pod string) (bool, error) { if err != nil { return false, err } - glog.V(5).Infof("CompareSymlinks: devGloBal %s, devPod %s", devGlobal, devPod) + klog.V(5).Infof("CompareSymlinks: devGloBal %s, devPod %s", devGlobal, devPod) if devGlobal == devPod { return true, nil } diff --git a/pkg/volume/util/volumepathhandler/volume_path_handler_linux.go b/pkg/volume/util/volumepathhandler/volume_path_handler_linux.go index f9a886d7dc6..7170edc7de0 100644 --- a/pkg/volume/util/volumepathhandler/volume_path_handler_linux.go +++ b/pkg/volume/util/volumepathhandler/volume_path_handler_linux.go @@ -25,7 +25,7 @@ import ( "os/exec" "strings" - "github.com/golang/glog" + "k8s.io/klog" ) // AttachFileDevice takes a path to a regular file and makes it available as an @@ -38,7 +38,7 @@ func (v VolumePathHandler) AttachFileDevice(path string) (string, error) { // If no existing loop device for the path, create one if blockDevicePath == "" { - glog.V(4).Infof("Creating device for path: %s", path) + klog.V(4).Infof("Creating device for path: %s", path) blockDevicePath, err = makeLoopDevice(path) if err != nil { return "", err @@ -61,7 +61,7 @@ func (v VolumePathHandler) GetLoopDevice(path string) (string, error) { cmd := exec.Command(losetupPath, args...) out, err := cmd.CombinedOutput() if err != nil { - glog.V(2).Infof("Failed device discover command for path %s: %v %s", path, err, out) + klog.V(2).Infof("Failed device discover command for path %s: %v %s", path, err, out) return "", err } return parseLosetupOutputForDevice(out) @@ -72,7 +72,7 @@ func makeLoopDevice(path string) (string, error) { cmd := exec.Command(losetupPath, args...) out, err := cmd.CombinedOutput() if err != nil { - glog.V(2).Infof("Failed device create command for path: %s %v %s ", path, err, out) + klog.V(2).Infof("Failed device create command for path: %s %v %s ", path, err, out) return "", err } return parseLosetupOutputForDevice(out) @@ -87,7 +87,7 @@ func (v VolumePathHandler) RemoveLoopDevice(device string) error { if _, err := os.Stat(device); os.IsNotExist(err) { return nil } - glog.V(2).Infof("Failed to remove loopback device: %s: %v %s", device, err, out) + klog.V(2).Infof("Failed to remove loopback device: %s: %v %s", device, err, out) return err } return nil diff --git a/pkg/volume/volume_linux.go b/pkg/volume/volume_linux.go index ef1f45208c9..eb44d5f162f 100644 --- a/pkg/volume/volume_linux.go +++ b/pkg/volume/volume_linux.go @@ -24,7 +24,7 @@ import ( "os" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -63,13 +63,13 @@ func SetVolumeOwnership(mounter Mounter, fsGroup *int64) error { } if stat == nil { - glog.Errorf("Got nil stat_t for path %v while setting ownership of volume", path) + klog.Errorf("Got nil stat_t for path %v while setting ownership of volume", path) return nil } err = os.Chown(path, int(stat.Uid), int(*fsGroup)) if err != nil { - glog.Errorf("Chown failed on %v: %v", path, err) + klog.Errorf("Chown failed on %v: %v", path, err) } mask := rwMask @@ -83,7 +83,7 @@ func SetVolumeOwnership(mounter Mounter, fsGroup *int64) error { err = os.Chmod(path, info.Mode()|mask) if err != nil { - glog.Errorf("Chmod failed on %v: %v", path, err) + klog.Errorf("Chmod failed on %v: %v", path, err) } return nil diff --git a/pkg/volume/vsphere_volume/BUILD b/pkg/volume/vsphere_volume/BUILD index 087276bcad5..89892c7621d 100644 --- a/pkg/volume/vsphere_volume/BUILD +++ b/pkg/volume/vsphere_volume/BUILD @@ -11,23 +11,27 @@ go_library( srcs = [ "attacher.go", "vsphere_volume.go", + "vsphere_volume_block.go", "vsphere_volume_util.go", ], importpath = "k8s.io/kubernetes/pkg/volume/vsphere_volume", deps = [ "//pkg/cloudprovider/providers/vsphere:go_default_library", "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library", + "//pkg/features:go_default_library", "//pkg/util/keymutex:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/util/strings:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", + "//pkg/volume/util/volumepathhandler:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -35,6 +39,7 @@ go_test( name = "go_default_test", srcs = [ "attacher_test.go", + "vsphere_volume_block_test.go", "vsphere_volume_test.go", ], embed = [":go_default_library"], @@ -46,10 +51,11 @@ go_test( "//pkg/volume:go_default_library", "//pkg/volume/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/vsphere_volume/attacher.go b/pkg/volume/vsphere_volume/attacher.go index 9f292e348fa..ca934b45abd 100644 --- a/pkg/volume/vsphere_volume/attacher.go +++ b/pkg/volume/vsphere_volume/attacher.go @@ -22,9 +22,9 @@ import ( "path" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" "k8s.io/kubernetes/pkg/util/keymutex" "k8s.io/kubernetes/pkg/util/mount" @@ -76,7 +76,7 @@ func (attacher *vsphereVMDKAttacher) Attach(spec *volume.Spec, nodeName types.No return "", err } - glog.V(4).Infof("vSphere: Attach disk called for node %s", nodeName) + klog.V(4).Infof("vSphere: Attach disk called for node %s", nodeName) // Keeps concurrent attach operations to same host atomic attachdetachMutex.LockKey(string(nodeName)) @@ -86,7 +86,7 @@ func (attacher *vsphereVMDKAttacher) Attach(spec *volume.Spec, nodeName types.No // succeeds in that case, so no need to do that separately. diskUUID, err := attacher.vsphereVolumes.AttachDisk(volumeSource.VolumePath, volumeSource.StoragePolicyName, nodeName) if err != nil { - glog.Errorf("Error attaching volume %q to node %q: %+v", volumeSource.VolumePath, nodeName, err) + klog.Errorf("Error attaching volume %q to node %q: %+v", volumeSource.VolumePath, nodeName, err) return "", err } @@ -94,14 +94,14 @@ func (attacher *vsphereVMDKAttacher) Attach(spec *volume.Spec, nodeName types.No } func (attacher *vsphereVMDKAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) { - glog.Warningf("Attacher.VolumesAreAttached called for node %q - Please use BulkVerifyVolumes for vSphere", nodeName) + klog.Warningf("Attacher.VolumesAreAttached called for node %q - Please use BulkVerifyVolumes for vSphere", nodeName) volumeNodeMap := map[types.NodeName][]*volume.Spec{ nodeName: specs, } nodeVolumesResult := make(map[*volume.Spec]bool) nodesVerificationMap, err := attacher.BulkVerifyVolumes(volumeNodeMap) if err != nil { - glog.Errorf("Attacher.VolumesAreAttached - error checking volumes for node %q with %v", nodeName, err) + klog.Errorf("Attacher.VolumesAreAttached - error checking volumes for node %q with %v", nodeName, err) return nodeVolumesResult, err } if result, ok := nodesVerificationMap[nodeName]; ok { @@ -119,7 +119,7 @@ func (attacher *vsphereVMDKAttacher) BulkVerifyVolumes(volumesByNode map[types.N for _, volumeSpec := range volumeSpecs { volumeSource, _, err := getVolumeSource(volumeSpec) if err != nil { - glog.Errorf("Error getting volume (%q) source : %v", volumeSpec.Name(), err) + klog.Errorf("Error getting volume (%q) source : %v", volumeSpec.Name(), err) continue } volPath := volumeSource.VolumePath @@ -135,7 +135,7 @@ func (attacher *vsphereVMDKAttacher) BulkVerifyVolumes(volumesByNode map[types.N } attachedResult, err := attacher.vsphereVolumes.DisksAreAttached(volumePathsByNode) if err != nil { - glog.Errorf("Error checking if volumes are attached to nodes: %+v. err: %v", volumePathsByNode, err) + klog.Errorf("Error checking if volumes are attached to nodes: %+v. err: %v", volumePathsByNode, err) return volumesAttachedCheck, err } @@ -169,14 +169,14 @@ func (attacher *vsphereVMDKAttacher) WaitForAttach(spec *volume.Spec, devicePath for { select { case <-ticker.C: - glog.V(5).Infof("Checking VMDK %q is attached", volumeSource.VolumePath) + klog.V(5).Infof("Checking VMDK %q is attached", volumeSource.VolumePath) path, err := verifyDevicePath(devicePath) if err != nil { // Log error, if any, and continue checking periodically. See issue #11321 - glog.Warningf("Error verifying VMDK (%q) is attached: %v", volumeSource.VolumePath, err) + klog.Warningf("Error verifying VMDK (%q) is attached: %v", volumeSource.VolumePath, err) } else if path != "" { // A device path has successfully been created for the VMDK - glog.Infof("Successfully found attached VMDK %q.", volumeSource.VolumePath) + klog.Infof("Successfully found attached VMDK %q.", volumeSource.VolumePath) return path, nil } case <-timer.C: @@ -210,7 +210,7 @@ func (attacher *vsphereVMDKAttacher) MountDevice(spec *volume.Spec, devicePath s if err != nil { if os.IsNotExist(err) { if err := os.MkdirAll(deviceMountPath, 0750); err != nil { - glog.Errorf("Failed to create directory at %#v. err: %s", deviceMountPath, err) + klog.Errorf("Failed to create directory at %#v. err: %s", deviceMountPath, err) return err } notMnt = true @@ -234,7 +234,7 @@ func (attacher *vsphereVMDKAttacher) MountDevice(spec *volume.Spec, devicePath s os.Remove(deviceMountPath) return err } - glog.V(4).Infof("formatting spec %v devicePath %v deviceMountPath %v fs %v with options %+v", spec.Name(), devicePath, deviceMountPath, volumeSource.FSType, options) + klog.V(4).Infof("formatting spec %v devicePath %v deviceMountPath %v fs %v with options %+v", spec.Name(), devicePath, deviceMountPath, volumeSource.FSType, options) } return nil } @@ -271,21 +271,21 @@ func (detacher *vsphereVMDKDetacher) Detach(volumeName string, nodeName types.No attached, err := detacher.vsphereVolumes.DiskIsAttached(volPath, nodeName) if err != nil { // Log error and continue with detach - glog.Errorf( + klog.Errorf( "Error checking if volume (%q) is already attached to current node (%q). Will continue and try detach anyway. err=%v", volPath, nodeName, err) } if err == nil && !attached { // Volume is already detached from node. - glog.Infof("detach operation was successful. volume %q is already detached from node %q.", volPath, nodeName) + klog.Infof("detach operation was successful. volume %q is already detached from node %q.", volPath, nodeName) return nil } attachdetachMutex.LockKey(string(nodeName)) defer attachdetachMutex.UnlockKey(string(nodeName)) if err := detacher.vsphereVolumes.DetachDisk(volPath, nodeName); err != nil { - glog.Errorf("Error detaching volume %q: %v", volPath, err) + klog.Errorf("Error detaching volume %q: %v", volPath, err) return err } return nil diff --git a/pkg/volume/vsphere_volume/attacher_test.go b/pkg/volume/vsphere_volume/attacher_test.go index 5922c00eef7..08a59f593c4 100644 --- a/pkg/volume/vsphere_volume/attacher_test.go +++ b/pkg/volume/vsphere_volume/attacher_test.go @@ -26,7 +26,7 @@ import ( "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" - "github.com/golang/glog" + "k8s.io/klog" ) func TestGetDeviceName_Volume(t *testing.T) { @@ -253,7 +253,7 @@ func (testcase *testcase) AttachDisk(diskName string, storagePolicyName string, return "", errors.New("Unexpected AttachDisk call: wrong nodeName") } - glog.V(4).Infof("AttachDisk call: %s, %s, returning %q, %v", diskName, nodeName, expected.retDeviceUUID, expected.ret) + klog.V(4).Infof("AttachDisk call: %s, %s, returning %q, %v", diskName, nodeName, expected.retDeviceUUID, expected.ret) return expected.retDeviceUUID, expected.ret } @@ -278,7 +278,7 @@ func (testcase *testcase) DetachDisk(diskName string, nodeName types.NodeName) e return errors.New("Unexpected DetachDisk call: wrong nodeName") } - glog.V(4).Infof("DetachDisk call: %s, %s, returning %v", diskName, nodeName, expected.ret) + klog.V(4).Infof("DetachDisk call: %s, %s, returning %v", diskName, nodeName, expected.ret) return expected.ret } @@ -303,7 +303,7 @@ func (testcase *testcase) DiskIsAttached(diskName string, nodeName types.NodeNam return false, errors.New("Unexpected DiskIsAttached call: wrong nodeName") } - glog.V(4).Infof("DiskIsAttached call: %s, %s, returning %v, %v", diskName, nodeName, expected.isAttached, expected.ret) + klog.V(4).Infof("DiskIsAttached call: %s, %s, returning %v, %v", diskName, nodeName, expected.isAttached, expected.ret) return expected.isAttached, expected.ret } diff --git a/pkg/volume/vsphere_volume/vsphere_volume.go b/pkg/volume/vsphere_volume/vsphere_volume.go index be896d22d8c..7097005ef9e 100644 --- a/pkg/volume/vsphere_volume/vsphere_volume.go +++ b/pkg/volume/vsphere_volume/vsphere_volume.go @@ -22,11 +22,13 @@ import ( "path" "strings" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" utilstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" @@ -144,7 +146,7 @@ func (plugin *vsphereVolumePlugin) ConstructVolumeSpec(volumeName, mountPath str return nil, err } volumePath = strings.Replace(volumePath, "\\040", " ", -1) - glog.V(5).Infof("vSphere volume path is %q", volumePath) + klog.V(5).Infof("vSphere volume path is %q", volumePath) vsphereVolume := &v1.Volume{ Name: volumeName, VolumeSource: v1.VolumeSource{ @@ -214,21 +216,21 @@ func (b *vsphereVolumeMounter) CanMount() error { // SetUp attaches the disk and bind mounts to the volume path. func (b *vsphereVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { - glog.V(5).Infof("vSphere volume setup %s to %s", b.volPath, dir) + klog.V(5).Infof("vSphere volume setup %s to %s", b.volPath, dir) // TODO: handle failed mounts here. notmnt, err := b.mounter.IsLikelyNotMountPoint(dir) if err != nil && !os.IsNotExist(err) { - glog.V(4).Infof("IsLikelyNotMountPoint failed: %v", err) + klog.V(4).Infof("IsLikelyNotMountPoint failed: %v", err) return err } if !notmnt { - glog.V(4).Infof("Something is already mounted to target %s", dir) + klog.V(4).Infof("Something is already mounted to target %s", dir) return nil } if err := os.MkdirAll(dir, 0750); err != nil { - glog.V(4).Infof("Could not create directory %s: %v", dir, err) + klog.V(4).Infof("Could not create directory %s: %v", dir, err) return err } @@ -241,21 +243,21 @@ func (b *vsphereVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if err != nil { notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !notmnt { if mntErr = b.mounter.Unmount(dir); mntErr != nil { - glog.Errorf("Failed to unmount: %v", mntErr) + klog.Errorf("Failed to unmount: %v", mntErr) return err } notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) if mntErr != nil { - glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err } if !notmnt { - glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", b.GetPath()) + klog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", b.GetPath()) return err } } @@ -263,7 +265,7 @@ func (b *vsphereVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { return err } volume.SetVolumeOwnership(b, fsGroup) - glog.V(3).Infof("vSphere volume %s mounted to %s", b.volPath, dir) + klog.V(3).Infof("vSphere volume %s mounted to %s", b.volPath, dir) return nil } @@ -356,9 +358,6 @@ func (v *vsphereVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopol if !util.AccessModesContainedInAll(v.plugin.GetAccessModes(), v.options.PVC.Spec.AccessModes) { return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", v.options.PVC.Spec.AccessModes, v.plugin.GetAccessModes()) } - if util.CheckPersistentVolumeClaimModeBlock(v.options.PVC) { - return nil, fmt.Errorf("%s does not support block volume provisioning", v.plugin.GetPluginName()) - } volSpec, err := v.manager.CreateVolume(v) if err != nil { @@ -369,6 +368,15 @@ func (v *vsphereVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopol volSpec.Fstype = "ext4" } + var volumeMode *v1.PersistentVolumeMode + if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) { + volumeMode = v.options.PVC.Spec.VolumeMode + if volumeMode != nil && *volumeMode == v1.PersistentVolumeBlock { + klog.V(5).Infof("vSphere block volume should not have any FSType") + volSpec.Fstype = "" + } + } + pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: v.options.PVName, @@ -383,6 +391,7 @@ func (v *vsphereVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopol Capacity: v1.ResourceList{ v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dKi", volSpec.Size)), }, + VolumeMode: volumeMode, PersistentVolumeSource: v1.PersistentVolumeSource{ VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{ VolumePath: volSpec.Path, diff --git a/pkg/volume/vsphere_volume/vsphere_volume_block.go b/pkg/volume/vsphere_volume/vsphere_volume_block.go new file mode 100644 index 00000000000..40342b8aacd --- /dev/null +++ b/pkg/volume/vsphere_volume/vsphere_volume_block.go @@ -0,0 +1,161 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere_volume + +import ( + "fmt" + "path" + "path/filepath" + "strings" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/util/mount" + kstrings "k8s.io/kubernetes/pkg/util/strings" + "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/util" + "k8s.io/kubernetes/pkg/volume/util/volumepathhandler" +) + +var _ volume.BlockVolumePlugin = &vsphereVolumePlugin{} + +func (plugin *vsphereVolumePlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) { + + pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName()) + blkUtil := volumepathhandler.NewBlockVolumePathHandler() + globalMapPathUUID, err := blkUtil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID) + if err != nil { + klog.Errorf("Failed to find GlobalMapPathUUID from Pod: %s with error: %+v", podUID, err) + return nil, err + } + klog.V(5).Infof("globalMapPathUUID: %v", globalMapPathUUID) + globalMapPath := filepath.Dir(globalMapPathUUID) + if len(globalMapPath) <= 1 { + return nil, fmt.Errorf("failed to get volume plugin information from globalMapPathUUID: %v", globalMapPathUUID) + } + return getVolumeSpecFromGlobalMapPath(globalMapPath) +} + +func getVolumeSpecFromGlobalMapPath(globalMapPath string) (*volume.Spec, error) { + // Construct volume spec from globalMapPath + // globalMapPath example: + // plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumeID} + // plugins/kubernetes.io/vsphere-volume/volumeDevices/[datastore1]\\040volumes/myDisk + volPath := filepath.Base(globalMapPath) + volPath = strings.Replace(volPath, "\\040", "", -1) + if len(volPath) <= 1 { + return nil, fmt.Errorf("failed to get volume path from global path=%s", globalMapPath) + } + block := v1.PersistentVolumeBlock + vsphereVolume := &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{ + VolumePath: volPath, + }, + }, + VolumeMode: &block, + }, + } + return volume.NewSpecFromPersistentVolume(vsphereVolume, true), nil +} + +func (plugin *vsphereVolumePlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) { + // If this called via GenerateUnmapDeviceFunc(), pod is nil. + // Pass empty string as dummy uid since uid isn't used in the case. + var uid types.UID + if pod != nil { + uid = pod.UID + } + return plugin.newBlockVolumeMapperInternal(spec, uid, &VsphereDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) +} + +func (plugin *vsphereVolumePlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager vdManager, mounter mount.Interface) (volume.BlockVolumeMapper, error) { + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + klog.Errorf("Failed to get Volume source from volume Spec: %+v with error: %+v", *spec, err) + return nil, err + } + volPath := volumeSource.VolumePath + return &vsphereBlockVolumeMapper{ + vsphereVolume: &vsphereVolume{ + volName: spec.Name(), + podUID: podUID, + volPath: volPath, + manager: manager, + mounter: mounter, + plugin: plugin, + MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)), + }, + }, nil + +} + +func (plugin *vsphereVolumePlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) { + return plugin.newUnmapperInternal(volName, podUID, &VsphereDiskUtil{}) +} + +func (plugin *vsphereVolumePlugin) newUnmapperInternal(volName string, podUID types.UID, manager vdManager) (volume.BlockVolumeUnmapper, error) { + return &vsphereBlockVolumeUnmapper{ + vsphereVolume: &vsphereVolume{ + volName: volName, + podUID: podUID, + volPath: volName, + manager: manager, + plugin: plugin, + }, + }, nil +} + +var _ volume.BlockVolumeMapper = &vsphereBlockVolumeMapper{} + +type vsphereBlockVolumeMapper struct { + *vsphereVolume +} + +func (v vsphereBlockVolumeMapper) SetUpDevice() (string, error) { + return "", nil +} + +func (v vsphereBlockVolumeMapper) MapDevice(devicePath, globalMapPath, volumeMapPath, volumeMapName string, podUID types.UID) error { + return util.MapBlockVolume(devicePath, globalMapPath, volumeMapPath, volumeMapName, podUID) +} + +var _ volume.BlockVolumeUnmapper = &vsphereBlockVolumeUnmapper{} + +type vsphereBlockVolumeUnmapper struct { + *vsphereVolume +} + +func (v *vsphereBlockVolumeUnmapper) TearDownDevice(mapPath, devicePath string) error { + return nil +} + +// GetGlobalMapPath returns global map path and error +// path: plugins/kubernetes.io/{PluginName}/volumeDevices/volumePath +func (v *vsphereVolume) GetGlobalMapPath(spec *volume.Spec) (string, error) { + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + return "", err + } + return path.Join(v.plugin.host.GetVolumeDevicePluginDir(vsphereVolumePluginName), string(volumeSource.VolumePath)), nil +} + +func (v *vsphereVolume) GetPodDeviceMapPath() (string, string) { + return v.plugin.host.GetPodVolumeDeviceDir(v.podUID, kstrings.EscapeQualifiedNameForDisk(vsphereVolumePluginName)), v.volName +} diff --git a/pkg/volume/vsphere_volume/vsphere_volume_block_test.go b/pkg/volume/vsphere_volume/vsphere_volume_block_test.go new file mode 100644 index 00000000000..4443e7bef72 --- /dev/null +++ b/pkg/volume/vsphere_volume/vsphere_volume_block_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere_volume + +import ( + "os" + "path" + "testing" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + utiltesting "k8s.io/client-go/util/testing" + "k8s.io/kubernetes/pkg/volume" + volumetest "k8s.io/kubernetes/pkg/volume/testing" +) + +var ( + testVolumePath = "volPath1" + testGlobalPath = "plugins/kubernetes.io/vsphere-volume/volumeDevices/volPath1" + testPodPath = "pods/poduid/volumeDevices/kubernetes.io~vsphere-volume" +) + +func TestGetVolumeSpecFromGlobalMapPath(t *testing.T) { + // make our test path for fake GlobalMapPath + // /tmp symbolized our pluginDir + // /tmp/testGlobalPathXXXXX/plugins/kubernetes.io/vsphere-volume/volumeDevices/ + tmpVDir, err := utiltesting.MkTmpdir("vsphereBlockVolume") + if err != nil { + t.Fatalf("cant' make a temp dir: %s", err) + } + // deferred clean up + defer os.RemoveAll(tmpVDir) + + expectedGlobalPath := path.Join(tmpVDir, testGlobalPath) + + // Bad Path + badspec, err := getVolumeSpecFromGlobalMapPath("") + if badspec != nil || err == nil { + t.Errorf("Expected not to get spec from GlobalMapPath but did") + } + + // Good Path + spec, err := getVolumeSpecFromGlobalMapPath(expectedGlobalPath) + if spec == nil || err != nil { + t.Fatalf("Failed to get spec from GlobalMapPath: %s", err) + } + if spec.PersistentVolume.Spec.VsphereVolume.VolumePath != testVolumePath { + t.Fatalf("Invalid volumePath from GlobalMapPath spec: %s", spec.PersistentVolume.Spec.VsphereVolume.VolumePath) + } + block := v1.PersistentVolumeBlock + specMode := spec.PersistentVolume.Spec.VolumeMode + if &specMode == nil { + t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", &specMode, block) + } + if *specMode != block { + t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", *specMode, block) + } +} + +func TestGetPodAndPluginMapPaths(t *testing.T) { + tmpVDir, err := utiltesting.MkTmpdir("vsphereBlockVolume") + if err != nil { + t.Fatalf("cant' make a temp dir: %s", err) + } + // deferred clean up + defer os.RemoveAll(tmpVDir) + + expectedGlobalPath := path.Join(tmpVDir, testGlobalPath) + expectedPodPath := path.Join(tmpVDir, testPodPath) + + spec := getTestVolume(true) // block volume + pluginMgr := volume.VolumePluginMgr{} + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil, volumetest.NewFakeVolumeHost(tmpVDir, nil, nil)) + plugin, err := pluginMgr.FindMapperPluginByName(vsphereVolumePluginName) + if err != nil { + os.RemoveAll(tmpVDir) + t.Fatalf("Can't find the plugin by name: %q", vsphereVolumePluginName) + } + if plugin.GetPluginName() != vsphereVolumePluginName { + t.Fatalf("Wrong name: %s", plugin.GetPluginName()) + } + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: types.UID("poduid"), + }, + } + mapper, err := plugin.NewBlockVolumeMapper(spec, pod, volume.VolumeOptions{}) + if err != nil { + t.Fatalf("Failed to make a new Mounter: %v", err) + } + if mapper == nil { + t.Fatalf("Got a nil Mounter") + } + + // GetGlobalMapPath + globalMapPath, err := mapper.GetGlobalMapPath(spec) + if err != nil || len(globalMapPath) == 0 { + t.Fatalf("Invalid GlobalMapPath from spec: %s", spec.PersistentVolume.Spec.VsphereVolume.VolumePath) + } + if globalMapPath != expectedGlobalPath { + t.Errorf("Failed to get GlobalMapPath: %s %s", globalMapPath, expectedGlobalPath) + } + + // GetPodDeviceMapPath + devicePath, volumeName := mapper.GetPodDeviceMapPath() + if devicePath != expectedPodPath { + t.Errorf("Got unexpected pod path: %s, expected %s", devicePath, expectedPodPath) + } + if volumeName != testVolumePath { + t.Errorf("Got unexpected volNamne: %s, expected %s", volumeName, testVolumePath) + } +} + +func getTestVolume(isBlock bool) *volume.Spec { + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: testVolumePath, + }, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{ + VolumePath: testVolumePath, + }, + }, + }, + } + if isBlock { + blockMode := v1.PersistentVolumeBlock + pv.Spec.VolumeMode = &blockMode + } + return volume.NewSpecFromPersistentVolume(pv, true) +} diff --git a/pkg/volume/vsphere_volume/vsphere_volume_util.go b/pkg/volume/vsphere_volume/vsphere_volume_util.go index 0603b36e552..a5880ea7fbf 100644 --- a/pkg/volume/vsphere_volume/vsphere_volume_util.go +++ b/pkg/volume/vsphere_volume/vsphere_volume_util.go @@ -23,9 +23,9 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib" "k8s.io/kubernetes/pkg/volume" @@ -114,10 +114,10 @@ func (util *VsphereDiskUtil) CreateVolume(v *vsphereVolumeProvisioner) (volSpec volumeOptions.Datastore = value case volume.VolumeParameterFSType: fstype = value - glog.V(4).Infof("Setting fstype as %q", fstype) + klog.V(4).Infof("Setting fstype as %q", fstype) case StoragePolicyName: volumeOptions.StoragePolicyName = value - glog.V(4).Infof("Setting StoragePolicyName as %q", volumeOptions.StoragePolicyName) + klog.V(4).Infof("Setting StoragePolicyName as %q", volumeOptions.StoragePolicyName) case HostFailuresToTolerateCapability, ForceProvisioningCapability, CacheReservationCapability, DiskStripesCapability, ObjectSpaceReservationCapability, IopsLimitCapability: @@ -137,7 +137,7 @@ func (util *VsphereDiskUtil) CreateVolume(v *vsphereVolumeProvisioner) (volSpec } volumeOptions.VSANStorageProfileData = "(" + volumeOptions.VSANStorageProfileData + ")" } - glog.V(4).Infof("VSANStorageProfileData in vsphere volume %q", volumeOptions.VSANStorageProfileData) + klog.V(4).Infof("VSANStorageProfileData in vsphere volume %q", volumeOptions.VSANStorageProfileData) // TODO: implement PVC.Selector parsing if v.options.PVC.Spec.Selector != nil { return nil, fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on vSphere") @@ -154,7 +154,7 @@ func (util *VsphereDiskUtil) CreateVolume(v *vsphereVolumeProvisioner) (volSpec StoragePolicyName: volumeOptions.StoragePolicyName, StoragePolicyID: volumeOptions.StoragePolicyID, } - glog.V(2).Infof("Successfully created vsphere volume %s", name) + klog.V(2).Infof("Successfully created vsphere volume %s", name) return volSpec, nil } @@ -166,10 +166,10 @@ func (util *VsphereDiskUtil) DeleteVolume(vd *vsphereVolumeDeleter) error { } if err = cloud.DeleteVolume(vd.volPath); err != nil { - glog.V(2).Infof("Error deleting vsphere volume %s: %v", vd.volPath, err) + klog.V(2).Infof("Error deleting vsphere volume %s: %v", vd.volPath, err) return err } - glog.V(2).Infof("Successfully deleted vsphere volume %s", vd.volPath) + klog.V(2).Infof("Successfully deleted vsphere volume %s", vd.volPath) return nil } @@ -184,7 +184,7 @@ func getVolPathfromVolumeName(deviceMountPath string) string { func getCloudProvider(cloud cloudprovider.Interface) (*vsphere.VSphere, error) { if cloud == nil { - glog.Errorf("Cloud provider not initialized properly") + klog.Errorf("Cloud provider not initialized properly") return nil, errors.New("Cloud provider not initialized properly") } diff --git a/pkg/windows/service/BUILD b/pkg/windows/service/BUILD index 3e9cf1d3bdc..e64c40488b4 100644 --- a/pkg/windows/service/BUILD +++ b/pkg/windows/service/BUILD @@ -11,9 +11,9 @@ go_library( importpath = "k8s.io/kubernetes/pkg/windows/service", deps = select({ "@io_bazel_rules_go//go/platform:windows": [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/sys/windows:go_default_library", "//vendor/golang.org/x/sys/windows/svc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "//conditions:default": [], }), diff --git a/pkg/windows/service/service.go b/pkg/windows/service/service.go index acc48246f1e..a5bffa1822e 100644 --- a/pkg/windows/service/service.go +++ b/pkg/windows/service/service.go @@ -21,7 +21,7 @@ package service import ( "os" - "github.com/golang/glog" + "k8s.io/klog" "golang.org/x/sys/windows" "golang.org/x/sys/windows/svc" @@ -57,7 +57,7 @@ func InitService(serviceName string) error { if err != nil { return err } - glog.Infof("Running %s as a Windows service!", serviceName) + klog.Infof("Running %s as a Windows service!", serviceName) return nil } @@ -67,7 +67,7 @@ func (h *handler) Execute(_ []string, r <-chan svc.ChangeRequest, s chan<- svc.S h.fromsvc <- nil s <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown | svc.Accepted(windows.SERVICE_ACCEPT_PARAMCHANGE)} - glog.Infof("Service running") + klog.Infof("Service running") Loop: for { select { diff --git a/plugin/pkg/admission/admit/BUILD b/plugin/pkg/admission/admit/BUILD index 7fe5dcf695c..bd93027e33c 100644 --- a/plugin/pkg/admission/admit/BUILD +++ b/plugin/pkg/admission/admit/BUILD @@ -12,7 +12,7 @@ go_library( importpath = "k8s.io/kubernetes/plugin/pkg/admission/admit", deps = [ "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/plugin/pkg/admission/admit/admission.go b/plugin/pkg/admission/admit/admission.go index 863184d32e9..867bbcdae4d 100644 --- a/plugin/pkg/admission/admit/admission.go +++ b/plugin/pkg/admission/admit/admission.go @@ -19,8 +19,8 @@ package admit import ( "io" - "github.com/golang/glog" "k8s.io/apiserver/pkg/admission" + "k8s.io/klog" ) // PluginName indicates name of admission plugin. @@ -58,7 +58,7 @@ func (alwaysAdmit) Handles(operation admission.Operation) bool { // NewAlwaysAdmit creates a new always admit admission handler func NewAlwaysAdmit() admission.Interface { // DEPRECATED: AlwaysAdmit admit all admission request, it is no use. - glog.Warningf("%s admission controller is deprecated. "+ + klog.Warningf("%s admission controller is deprecated. "+ "Please remove this controller from your configuration files and scripts.", PluginName) return new(alwaysAdmit) } diff --git a/plugin/pkg/admission/deny/BUILD b/plugin/pkg/admission/deny/BUILD index 989df96ae8b..cdbe0e929ad 100644 --- a/plugin/pkg/admission/deny/BUILD +++ b/plugin/pkg/admission/deny/BUILD @@ -12,7 +12,7 @@ go_library( importpath = "k8s.io/kubernetes/plugin/pkg/admission/deny", deps = [ "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/plugin/pkg/admission/deny/admission.go b/plugin/pkg/admission/deny/admission.go index 386cb78b1e2..bf484590cbe 100644 --- a/plugin/pkg/admission/deny/admission.go +++ b/plugin/pkg/admission/deny/admission.go @@ -20,7 +20,7 @@ import ( "errors" "io" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apiserver/pkg/admission" ) @@ -60,7 +60,7 @@ func (alwaysDeny) Handles(operation admission.Operation) bool { // NewAlwaysDeny creates an always deny admission handler func NewAlwaysDeny() admission.Interface { // DEPRECATED: AlwaysDeny denys all admission request, it is no use. - glog.Warningf("%s admission controller is deprecated. "+ + klog.Warningf("%s admission controller is deprecated. "+ "Please remove this controller from your configuration files and scripts.", PluginName) return new(alwaysDeny) } diff --git a/plugin/pkg/admission/gc/BUILD b/plugin/pkg/admission/gc/BUILD index 2d71285ed05..354e386505b 100644 --- a/plugin/pkg/admission/gc/BUILD +++ b/plugin/pkg/admission/gc/BUILD @@ -27,11 +27,10 @@ go_test( srcs = ["gc_admission_test.go"], embed = [":go_default_library"], deps = [ - "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/core:go_default_library", - "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/kubeapiserver/admission:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/meta/testrestmapper:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", @@ -39,6 +38,9 @@ go_test( "//staging/src/k8s.io/apiserver/pkg/admission/initializer:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", + "//staging/src/k8s.io/client-go/discovery/fake:go_default_library", + "//staging/src/k8s.io/client-go/restmapper:go_default_library", + "//staging/src/k8s.io/client-go/testing:go_default_library", ], ) diff --git a/plugin/pkg/admission/gc/gc_admission_test.go b/plugin/pkg/admission/gc/gc_admission_test.go index 08deb51dea2..034e5179e45 100644 --- a/plugin/pkg/admission/gc/gc_admission_test.go +++ b/plugin/pkg/admission/gc/gc_admission_test.go @@ -17,10 +17,12 @@ limitations under the License. package gc import ( + "fmt" "strings" "testing" - "k8s.io/apimachinery/pkg/api/meta/testrestmapper" + corev1 "k8s.io/api/core/v1" + extensionv1beta1 "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -28,13 +30,11 @@ import ( "k8s.io/apiserver/pkg/admission/initializer" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/kubernetes/pkg/api/legacyscheme" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/restmapper" + coretesting "k8s.io/client-go/testing" api "k8s.io/kubernetes/pkg/apis/core" kubeadmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" - - // Manually initialize legacy scheme to make test rest mapper work with built-in resources. - // See issue #70192 for more details - _ "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" ) type fakeAuthorizer struct{} @@ -101,7 +101,30 @@ func newGCPermissionsEnforcement() (*gcPermissionsEnforcement, error) { } genericPluginInitializer := initializer.New(nil, nil, fakeAuthorizer{}, nil) - pluginInitializer := kubeadmission.NewPluginInitializer(nil, testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme), nil) + fakeDiscoveryClient := &fakediscovery.FakeDiscovery{Fake: &coretesting.Fake{}} + fakeDiscoveryClient.Resources = []*metav1.APIResourceList{ + { + GroupVersion: corev1.SchemeGroupVersion.String(), + APIResources: []metav1.APIResource{ + {Name: "nodes", Namespaced: false, Kind: "Node"}, + {Name: "pods", Namespaced: true, Kind: "Pod"}, + {Name: "replicationcontrollers", Namespaced: true, Kind: "ReplicationController"}, + }, + }, + { + GroupVersion: extensionv1beta1.SchemeGroupVersion.String(), + APIResources: []metav1.APIResource{ + {Name: "daemonsets", Namespaced: true, Kind: "DaemonSet"}, + }, + }, + } + + restMapperRes, err := restmapper.GetAPIGroupResources(fakeDiscoveryClient) + if err != nil { + return nil, fmt.Errorf("unexpected error while constructing resource list from fake discovery client: %v", err) + } + restMapper := restmapper.NewDiscoveryRESTMapper(restMapperRes) + pluginInitializer := kubeadmission.NewPluginInitializer(nil, restMapper, nil) initializersChain := admission.PluginInitializers{} initializersChain = append(initializersChain, genericPluginInitializer) initializersChain = append(initializersChain, pluginInitializer) diff --git a/plugin/pkg/admission/imagepolicy/BUILD b/plugin/pkg/admission/imagepolicy/BUILD index ea26d351e7f..fb8c1c55e54 100644 --- a/plugin/pkg/admission/imagepolicy/BUILD +++ b/plugin/pkg/admission/imagepolicy/BUILD @@ -26,7 +26,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/plugin/pkg/admission/imagepolicy/admission.go b/plugin/pkg/admission/imagepolicy/admission.go index e37d272bfa1..cfcb70a0472 100644 --- a/plugin/pkg/admission/imagepolicy/admission.go +++ b/plugin/pkg/admission/imagepolicy/admission.go @@ -26,7 +26,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/imagepolicy/v1alpha1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -110,7 +110,7 @@ func (a *Plugin) filterAnnotations(allAnnotations map[string]string) map[string] // Function to call on webhook failure; behavior determined by defaultAllow flag func (a *Plugin) webhookError(pod *api.Pod, attributes admission.Attributes, err error) error { if err != nil { - glog.V(2).Infof("error contacting webhook backend: %s", err) + klog.V(2).Infof("error contacting webhook backend: %s", err) if a.defaultAllow { attributes.AddAnnotation(AuditKeyPrefix+ImagePolicyFailedOpenKeySuffix, "true") // TODO(wteiken): Remove the annotation code for the 1.13 release @@ -121,10 +121,10 @@ func (a *Plugin) webhookError(pod *api.Pod, attributes admission.Attributes, err annotations[api.ImagePolicyFailedOpenKey] = "true" pod.ObjectMeta.SetAnnotations(annotations) - glog.V(2).Infof("resource allowed in spite of webhook backend failure") + klog.V(2).Infof("resource allowed in spite of webhook backend failure") return nil } - glog.V(2).Infof("resource not allowed due to webhook backend failure ") + klog.V(2).Infof("resource not allowed due to webhook backend failure ") return admission.NewForbidden(attributes, err) } return nil @@ -194,7 +194,7 @@ func (a *Plugin) admitPod(pod *api.Pod, attributes admission.Attributes, review for k, v := range review.Status.AuditAnnotations { if err := attributes.AddAnnotation(AuditKeyPrefix+k, v); err != nil { - glog.Warningf("failed to set admission audit annotation %s to %s: %v", AuditKeyPrefix+k, v, err) + klog.Warningf("failed to set admission audit annotation %s to %s: %v", AuditKeyPrefix+k, v, err) } } if !review.Status.Allowed { diff --git a/plugin/pkg/admission/imagepolicy/config.go b/plugin/pkg/admission/imagepolicy/config.go index df34a4906b4..cbe2ece4a31 100644 --- a/plugin/pkg/admission/imagepolicy/config.go +++ b/plugin/pkg/admission/imagepolicy/config.go @@ -22,7 +22,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -72,13 +72,13 @@ func normalizeWebhookConfig(config *imagePolicyWebhookConfig) (err error) { func normalizeConfigDuration(name string, scale, value, min, max, defaultValue time.Duration) (time.Duration, error) { // disable with -1 sentinel if value == disableTTL { - glog.V(2).Infof("image policy webhook %s disabled", name) + klog.V(2).Infof("image policy webhook %s disabled", name) return time.Duration(0), nil } // use default with 0 sentinel if value == useDefault { - glog.V(2).Infof("image policy webhook %s using default value", name) + klog.V(2).Infof("image policy webhook %s using default value", name) return defaultValue, nil } diff --git a/plugin/pkg/admission/noderestriction/BUILD b/plugin/pkg/admission/noderestriction/BUILD index 9d6850b5b2c..5c3bd1e93dc 100644 --- a/plugin/pkg/admission/noderestriction/BUILD +++ b/plugin/pkg/admission/noderestriction/BUILD @@ -18,16 +18,19 @@ go_library( "//pkg/apis/policy:go_default_library", "//pkg/auth/nodeidentifier:go_default_library", "//pkg/features:go_default_library", + "//pkg/kubelet/apis:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission/initializer:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -42,10 +45,12 @@ go_test( "//pkg/apis/policy:go_default_library", "//pkg/auth/nodeidentifier:go_default_library", "//pkg/features:go_default_library", + "//pkg/kubelet/apis:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", diff --git a/plugin/pkg/admission/noderestriction/OWNERS b/plugin/pkg/admission/noderestriction/OWNERS index 77eb1f5a7f6..5a7794867cf 100644 --- a/plugin/pkg/admission/noderestriction/OWNERS +++ b/plugin/pkg/admission/noderestriction/OWNERS @@ -1,10 +1,7 @@ approvers: -- deads2k -- liggitt -- tallclair -- mikedanese +- sig-auth-node-isolation-approvers reviewers: -- deads2k -- liggitt -- tallclair -- mikedanese +- sig-auth-node-isolation-reviewers +labels: +- sig/auth + diff --git a/plugin/pkg/admission/noderestriction/admission.go b/plugin/pkg/admission/noderestriction/admission.go index 7434ffb589f..222156a100a 100644 --- a/plugin/pkg/admission/noderestriction/admission.go +++ b/plugin/pkg/admission/noderestriction/admission.go @@ -19,17 +19,20 @@ package noderestriction import ( "fmt" "io" + "strings" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/admission" apiserveradmission "k8s.io/apiserver/pkg/admission/initializer" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" corev1lister "k8s.io/client-go/listers/core/v1" csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" + "k8s.io/klog" podutil "k8s.io/kubernetes/pkg/api/pod" authenticationapi "k8s.io/kubernetes/pkg/apis/authentication" coordapi "k8s.io/kubernetes/pkg/apis/coordination" @@ -37,6 +40,7 @@ import ( "k8s.io/kubernetes/pkg/apis/policy" "k8s.io/kubernetes/pkg/auth/nodeidentifier" "k8s.io/kubernetes/pkg/features" + kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" ) const ( @@ -330,6 +334,18 @@ func (c *nodePlugin) admitNode(nodeName string, a admission.Attributes) error { return admission.NewForbidden(a, fmt.Errorf("cannot create with non-nil configSource")) } + // Don't allow a node to register with labels outside the allowed set. + // This would allow a node to add or modify its labels in a way that would let it steer privileged workloads to itself. + modifiedLabels := getModifiedLabels(node.Labels, nil) + if forbiddenLabels := c.getForbiddenCreateLabels(modifiedLabels); len(forbiddenLabels) > 0 { + return admission.NewForbidden(a, fmt.Errorf("cannot set labels: %s", strings.Join(forbiddenLabels.List(), ", "))) + } + // check and warn if nodes set labels on create that would have been forbidden on update + // TODO(liggitt): in 1.17, expand getForbiddenCreateLabels to match getForbiddenUpdateLabels and drop this + if forbiddenUpdateLabels := c.getForbiddenUpdateLabels(modifiedLabels); len(forbiddenUpdateLabels) > 0 { + klog.Warningf("node %q added disallowed labels on node creation: %s", nodeName, strings.Join(forbiddenUpdateLabels.List(), ", ")) + } + // On create, get name from new object if unset in admission if len(requestedName) == 0 { requestedName = node.Name @@ -353,19 +369,100 @@ func (c *nodePlugin) admitNode(nodeName string, a admission.Attributes) error { // We scope node access to things listed in the Node.Spec, so allowing this would allow a view escalation. // We only do the check if the new node's configSource is non-nil; old kubelets might drop the field during a status update. if node.Spec.ConfigSource != nil && !apiequality.Semantic.DeepEqual(node.Spec.ConfigSource, oldNode.Spec.ConfigSource) { - return admission.NewForbidden(a, fmt.Errorf("cannot update configSource to a new non-nil configSource")) + return admission.NewForbidden(a, fmt.Errorf("node %q cannot update configSource to a new non-nil configSource", nodeName)) } // Don't allow a node to update its own taints. This would allow a node to remove or modify its // taints in a way that would let it steer disallowed workloads to itself. if !apiequality.Semantic.DeepEqual(node.Spec.Taints, oldNode.Spec.Taints) { - return admission.NewForbidden(a, fmt.Errorf("cannot modify taints")) + return admission.NewForbidden(a, fmt.Errorf("node %q cannot modify taints", nodeName)) + } + + // Don't allow a node to update labels outside the allowed set. + // This would allow a node to add or modify its labels in a way that would let it steer privileged workloads to itself. + modifiedLabels := getModifiedLabels(node.Labels, oldNode.Labels) + if forbiddenUpdateLabels := c.getForbiddenUpdateLabels(modifiedLabels); len(forbiddenUpdateLabels) > 0 { + return admission.NewForbidden(a, fmt.Errorf("cannot modify labels: %s", strings.Join(forbiddenUpdateLabels.List(), ", "))) } } return nil } +// getModifiedLabels returns the set of label keys that are different between the two maps +func getModifiedLabels(a, b map[string]string) sets.String { + modified := sets.NewString() + for k, v1 := range a { + if v2, ok := b[k]; !ok || v1 != v2 { + modified.Insert(k) + } + } + for k, v1 := range b { + if v2, ok := a[k]; !ok || v1 != v2 { + modified.Insert(k) + } + } + return modified +} + +func isKubernetesLabel(key string) bool { + namespace := getLabelNamespace(key) + if namespace == "kubernetes.io" || strings.HasSuffix(namespace, ".kubernetes.io") { + return true + } + if namespace == "k8s.io" || strings.HasSuffix(namespace, ".k8s.io") { + return true + } + return false +} + +func getLabelNamespace(key string) string { + if parts := strings.SplitN(key, "/", 2); len(parts) == 2 { + return parts[0] + } + return "" +} + +// getForbiddenCreateLabels returns the set of labels that may not be set by the node. +// TODO(liggitt): in 1.17, expand to match getForbiddenUpdateLabels() +func (c *nodePlugin) getForbiddenCreateLabels(modifiedLabels sets.String) sets.String { + if len(modifiedLabels) == 0 { + return nil + } + + forbiddenLabels := sets.NewString() + for label := range modifiedLabels { + namespace := getLabelNamespace(label) + // forbid kubelets from setting node-restriction labels + if namespace == kubeletapis.LabelNamespaceNodeRestriction || strings.HasSuffix(namespace, "."+kubeletapis.LabelNamespaceNodeRestriction) { + forbiddenLabels.Insert(label) + } + } + return forbiddenLabels +} + +// getForbiddenLabels returns the set of labels that may not be set by the node on update. +func (c *nodePlugin) getForbiddenUpdateLabels(modifiedLabels sets.String) sets.String { + if len(modifiedLabels) == 0 { + return nil + } + + forbiddenLabels := sets.NewString() + for label := range modifiedLabels { + namespace := getLabelNamespace(label) + // forbid kubelets from setting node-restriction labels + if namespace == kubeletapis.LabelNamespaceNodeRestriction || strings.HasSuffix(namespace, "."+kubeletapis.LabelNamespaceNodeRestriction) { + forbiddenLabels.Insert(label) + } + // forbid kubelets from setting unknown kubernetes.io and k8s.io labels on update + if isKubernetesLabel(label) && !kubeletapis.IsKubeletLabel(label) { + // TODO: defer to label policy once available + forbiddenLabels.Insert(label) + } + } + return forbiddenLabels +} + func (c *nodePlugin) admitServiceAccount(nodeName string, a admission.Attributes) error { if a.GetOperation() != admission.Create { return nil diff --git a/plugin/pkg/admission/noderestriction/admission_test.go b/plugin/pkg/admission/noderestriction/admission_test.go index 75ef000d399..b271109e2a1 100644 --- a/plugin/pkg/admission/noderestriction/admission_test.go +++ b/plugin/pkg/admission/noderestriction/admission_test.go @@ -17,15 +17,17 @@ limitations under the License. package noderestriction import ( + "fmt" + "reflect" "strings" "testing" "time" - "fmt" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/authentication/user" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -38,6 +40,7 @@ import ( "k8s.io/kubernetes/pkg/apis/policy" "k8s.io/kubernetes/pkg/auth/nodeidentifier" "k8s.io/kubernetes/pkg/features" + kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/utils/pointer" ) @@ -119,6 +122,99 @@ func makeTokenRequest(podname string, poduid types.UID) *authenticationapi.Token return tr } +func setAllLabels(node *api.Node, value string) *api.Node { + node = setAllowedCreateLabels(node, value) + node = setAllowedUpdateLabels(node, value) + node = setForbiddenCreateLabels(node, value) + node = setForbiddenUpdateLabels(node, value) + return node +} + +func setAllowedCreateLabels(node *api.Node, value string) *api.Node { + node = setAllowedUpdateLabels(node, value) + // also allow other kubernetes labels on create until 1.17 (TODO: remove this in 1.17) + node.Labels["other.kubernetes.io/foo"] = value + node.Labels["other.k8s.io/foo"] = value + return node +} + +func setAllowedUpdateLabels(node *api.Node, value string) *api.Node { + node = node.DeepCopy() + if node.Labels == nil { + node.Labels = map[string]string{} + } + if value == "" { + value = "value" + } + // non-kube labels + node.Labels["foo"] = value + node.Labels["example.com/foo"] = value + + // kubelet labels + node.Labels["kubernetes.io/hostname"] = value + node.Labels["failure-domain.beta.kubernetes.io/zone"] = value + node.Labels["failure-domain.beta.kubernetes.io/region"] = value + node.Labels["beta.kubernetes.io/instance-type"] = value + node.Labels["beta.kubernetes.io/os"] = value + node.Labels["beta.kubernetes.io/arch"] = value + node.Labels["failure-domain.kubernetes.io/zone"] = value + node.Labels["failure-domain.kubernetes.io/region"] = value + node.Labels["kubernetes.io/instance-type"] = value + node.Labels["kubernetes.io/os"] = value + node.Labels["kubernetes.io/arch"] = value + + // kubelet label prefixes + node.Labels["kubelet.kubernetes.io/foo"] = value + node.Labels["foo.kubelet.kubernetes.io/foo"] = value + node.Labels["node.kubernetes.io/foo"] = value + node.Labels["foo.node.kubernetes.io/foo"] = value + + // test all explicitly allowed labels and prefixes + for _, key := range kubeletapis.KubeletLabels() { + node.Labels[key] = value + } + for _, namespace := range kubeletapis.KubeletLabelNamespaces() { + node.Labels[namespace+"/foo"] = value + node.Labels["foo."+namespace+"/foo"] = value + } + + return node +} + +func setForbiddenCreateLabels(node *api.Node, value string) *api.Node { + node = node.DeepCopy() + if node.Labels == nil { + node.Labels = map[string]string{} + } + if value == "" { + value = "value" + } + // node restriction labels are forbidden + node.Labels["node-restriction.kubernetes.io/foo"] = value + node.Labels["foo.node-restriction.kubernetes.io/foo"] = value + // TODO: in 1.17, forbid arbitrary kubernetes labels on create + // node.Labels["other.kubernetes.io/foo"] = value + // node.Labels["other.k8s.io/foo"] = value + return node +} + +func setForbiddenUpdateLabels(node *api.Node, value string) *api.Node { + node = node.DeepCopy() + if node.Labels == nil { + node.Labels = map[string]string{} + } + if value == "" { + value = "value" + } + // node restriction labels are forbidden + node.Labels["node-restriction.kubernetes.io/foo"] = value + node.Labels["foo.node-restriction.kubernetes.io/foo"] = value + // arbitrary kubernetes labels are forbidden on update + node.Labels["other.kubernetes.io/foo"] = value + node.Labels["other.k8s.io/foo"] = value + return node +} + func Test_nodePlugin_Admit(t *testing.T) { var ( mynode = &user.DefaultInfo{Name: "system:node:mynode", Groups: []string{"system:nodes"}} @@ -216,11 +312,22 @@ func Test_nodePlugin_Admit(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "mynode", }, - CSIDrivers: []csiv1alpha1.CSIDriverInfo{ - { - Driver: "com.example.csi/mydriver", - NodeID: "com.example.csi/mynode", - TopologyKeys: []string{"com.example.csi/zone"}, + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: []csiv1alpha1.CSIDriverInfoSpec{ + { + Name: "com.example.csi/mydriver", + NodeID: "com.example.csi/mynode", + TopologyKeys: []string{"com.example.csi/zone"}, + }, + }, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: []csiv1alpha1.CSIDriverInfoStatus{ + { + Name: "com.example.csi/mydriver", + Available: true, + VolumePluginMechanism: csiv1alpha1.VolumePluginMechanismInTree, + }, }, }, } @@ -228,11 +335,22 @@ func Test_nodePlugin_Admit(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, - CSIDrivers: []csiv1alpha1.CSIDriverInfo{ - { - Driver: "com.example.csi/mydriver", - NodeID: "com.example.csi/foo", - TopologyKeys: []string{"com.example.csi/zone"}, + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: []csiv1alpha1.CSIDriverInfoSpec{ + { + Name: "com.example.csi/mydriver", + NodeID: "com.example.csi/foo", + TopologyKeys: []string{"com.example.csi/zone"}, + }, + }, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: []csiv1alpha1.CSIDriverInfoStatus{ + { + Name: "com.example.csi/mydriver", + Available: true, + VolumePluginMechanism: csiv1alpha1.VolumePluginMechanismInTree, + }, }, }, } @@ -740,6 +858,18 @@ func Test_nodePlugin_Admit(t *testing.T) { attributes: admission.NewAttributesRecord(mynodeObjTaintA, nil, nodeKind, mynodeObj.Namespace, "", nodeResource, "", admission.Create, false, mynode), err: "", }, + { + name: "allow create of my node with labels", + podsGetter: noExistingPods, + attributes: admission.NewAttributesRecord(setAllowedCreateLabels(mynodeObj, ""), nil, nodeKind, mynodeObj.Namespace, "", nodeResource, "", admission.Create, false, mynode), + err: "", + }, + { + name: "forbid create of my node with forbidden labels", + podsGetter: noExistingPods, + attributes: admission.NewAttributesRecord(setForbiddenCreateLabels(mynodeObj, ""), nil, nodeKind, mynodeObj.Namespace, "", nodeResource, "", admission.Create, false, mynode), + err: `cannot set labels: foo.node-restriction.kubernetes.io/foo, node-restriction.kubernetes.io/foo`, + }, { name: "allow update of my node", podsGetter: existingPods, @@ -794,6 +924,42 @@ func Test_nodePlugin_Admit(t *testing.T) { attributes: admission.NewAttributesRecord(mynodeObjTaintA, mynodeObjTaintA, nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Update, false, mynode), err: "", }, + { + name: "allow update of my node: add allowed labels", + podsGetter: existingPods, + attributes: admission.NewAttributesRecord(setAllowedUpdateLabels(mynodeObj, ""), mynodeObj, nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Update, false, mynode), + err: "", + }, + { + name: "allow update of my node: remove allowed labels", + podsGetter: existingPods, + attributes: admission.NewAttributesRecord(mynodeObj, setAllowedUpdateLabels(mynodeObj, ""), nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Update, false, mynode), + err: "", + }, + { + name: "allow update of my node: modify allowed labels", + podsGetter: existingPods, + attributes: admission.NewAttributesRecord(setAllowedUpdateLabels(mynodeObj, "b"), setAllowedUpdateLabels(mynodeObj, "a"), nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Update, false, mynode), + err: "", + }, + { + name: "allow update of my node: no change to labels", + podsGetter: existingPods, + attributes: admission.NewAttributesRecord(setAllLabels(mynodeObj, ""), setAllLabels(mynodeObj, ""), nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Update, false, mynode), + err: "", + }, + { + name: "allow update of my node: add allowed labels while forbidden labels exist unmodified", + podsGetter: existingPods, + attributes: admission.NewAttributesRecord(setAllLabels(mynodeObj, ""), setForbiddenUpdateLabels(mynodeObj, ""), nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Update, false, mynode), + err: "", + }, + { + name: "allow update of my node: remove allowed labels while forbidden labels exist unmodified", + podsGetter: existingPods, + attributes: admission.NewAttributesRecord(setForbiddenUpdateLabels(mynodeObj, ""), setAllLabels(mynodeObj, ""), nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Update, false, mynode), + err: "", + }, { name: "forbid update of my node: add taints", podsGetter: existingPods, @@ -812,6 +978,24 @@ func Test_nodePlugin_Admit(t *testing.T) { attributes: admission.NewAttributesRecord(mynodeObjTaintA, mynodeObjTaintB, nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Update, false, mynode), err: "cannot modify taints", }, + { + name: "forbid update of my node: add labels", + podsGetter: existingPods, + attributes: admission.NewAttributesRecord(setForbiddenUpdateLabels(mynodeObj, ""), mynodeObj, nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Update, false, mynode), + err: `cannot modify labels: foo.node-restriction.kubernetes.io/foo, node-restriction.kubernetes.io/foo, other.k8s.io/foo, other.kubernetes.io/foo`, + }, + { + name: "forbid update of my node: remove labels", + podsGetter: existingPods, + attributes: admission.NewAttributesRecord(mynodeObj, setForbiddenUpdateLabels(mynodeObj, ""), nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Update, false, mynode), + err: `cannot modify labels: foo.node-restriction.kubernetes.io/foo, node-restriction.kubernetes.io/foo, other.k8s.io/foo, other.kubernetes.io/foo`, + }, + { + name: "forbid update of my node: change labels", + podsGetter: existingPods, + attributes: admission.NewAttributesRecord(setForbiddenUpdateLabels(mynodeObj, "new"), setForbiddenUpdateLabels(mynodeObj, "old"), nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Update, false, mynode), + err: `cannot modify labels: foo.node-restriction.kubernetes.io/foo, node-restriction.kubernetes.io/foo, other.k8s.io/foo, other.kubernetes.io/foo`, + }, // Other node object { @@ -1061,3 +1245,80 @@ func Test_nodePlugin_Admit(t *testing.T) { }) } } + +func Test_getModifiedLabels(t *testing.T) { + tests := []struct { + name string + a map[string]string + b map[string]string + want sets.String + }{ + { + name: "empty", + a: nil, + b: nil, + want: sets.NewString(), + }, + { + name: "no change", + a: map[string]string{"x": "1", "y": "2", "z": "3"}, + b: map[string]string{"x": "1", "y": "2", "z": "3"}, + want: sets.NewString(), + }, + { + name: "added", + a: map[string]string{}, + b: map[string]string{"a": "0"}, + want: sets.NewString("a"), + }, + { + name: "removed", + a: map[string]string{"z": "3"}, + b: map[string]string{}, + want: sets.NewString("z"), + }, + { + name: "changed", + a: map[string]string{"z": "3"}, + b: map[string]string{"z": "4"}, + want: sets.NewString("z"), + }, + { + name: "added empty", + a: map[string]string{}, + b: map[string]string{"a": ""}, + want: sets.NewString("a"), + }, + { + name: "removed empty", + a: map[string]string{"z": ""}, + b: map[string]string{}, + want: sets.NewString("z"), + }, + { + name: "changed to empty", + a: map[string]string{"z": "3"}, + b: map[string]string{"z": ""}, + want: sets.NewString("z"), + }, + { + name: "changed from empty", + a: map[string]string{"z": ""}, + b: map[string]string{"z": "3"}, + want: sets.NewString("z"), + }, + { + name: "added, removed, and changed", + a: map[string]string{"a": "1", "b": "2"}, + b: map[string]string{"a": "2", "c": "3"}, + want: sets.NewString("a", "b", "c"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getModifiedLabels(tt.a, tt.b); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getModifiedLabels() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/plugin/pkg/admission/podnodeselector/BUILD b/plugin/pkg/admission/podnodeselector/BUILD index c7ef7642951..0144ccd207d 100644 --- a/plugin/pkg/admission/podnodeselector/BUILD +++ b/plugin/pkg/admission/podnodeselector/BUILD @@ -23,7 +23,7 @@ go_library( "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/plugin/pkg/admission/podnodeselector/admission.go b/plugin/pkg/admission/podnodeselector/admission.go index 8dc1daa66c6..1d502a0de12 100644 --- a/plugin/pkg/admission/podnodeselector/admission.go +++ b/plugin/pkg/admission/podnodeselector/admission.go @@ -21,7 +21,7 @@ import ( "io" "reflect" - "github.com/golang/glog" + "k8s.io/klog" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -190,7 +190,7 @@ func shouldIgnore(a admission.Attributes) bool { _, ok := a.GetObject().(*api.Pod) if !ok { - glog.Errorf("expected pod but got %s", a.GetKind().Kind) + klog.Errorf("expected pod but got %s", a.GetKind().Kind) return true } diff --git a/plugin/pkg/admission/podpreset/BUILD b/plugin/pkg/admission/podpreset/BUILD index c27fcc83eb0..b3a3b6a4bff 100644 --- a/plugin/pkg/admission/podpreset/BUILD +++ b/plugin/pkg/admission/podpreset/BUILD @@ -42,7 +42,7 @@ go_library( "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/listers/settings/v1alpha1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/plugin/pkg/admission/podpreset/admission.go b/plugin/pkg/admission/podpreset/admission.go index d91bdd78896..e86f4511c4d 100644 --- a/plugin/pkg/admission/podpreset/admission.go +++ b/plugin/pkg/admission/podpreset/admission.go @@ -22,7 +22,7 @@ import ( "reflect" "strings" - "github.com/golang/glog" + "k8s.io/klog" settingsv1alpha1 "k8s.io/api/settings/v1alpha1" "k8s.io/apimachinery/pkg/api/errors" @@ -108,7 +108,7 @@ func (c *podPresetPlugin) Admit(a admission.Attributes) error { // Ignore if exclusion annotation is present if podAnnotations := pod.GetAnnotations(); podAnnotations != nil { - glog.V(5).Infof("Looking at pod annotations, found: %v", podAnnotations) + klog.V(5).Infof("Looking at pod annotations, found: %v", podAnnotations) if podAnnotations[api.PodPresetOptOutAnnotationKey] == "true" { return nil } @@ -137,14 +137,14 @@ func (c *podPresetPlugin) Admit(a admission.Attributes) error { err = safeToApplyPodPresetsOnPod(pod, matchingPPs) if err != nil { // conflict, ignore the error, but raise an event - glog.Warningf("conflict occurred while applying podpresets: %s on pod: %v err: %v", + klog.Warningf("conflict occurred while applying podpresets: %s on pod: %v err: %v", strings.Join(presetNames, ","), pod.GetGenerateName(), err) return nil } applyPodPresetsOnPod(pod, matchingPPs) - glog.Infof("applied podpresets: %s successfully on Pod: %+v ", strings.Join(presetNames, ","), pod.GetGenerateName()) + klog.Infof("applied podpresets: %s successfully on Pod: %+v ", strings.Join(presetNames, ","), pod.GetGenerateName()) return nil } @@ -163,7 +163,7 @@ func filterPodPresets(list []*settingsv1alpha1.PodPreset, pod *api.Pod) ([]*sett if !selector.Matches(labels.Set(pod.Labels)) { continue } - glog.V(4).Infof("PodPreset %s matches pod %s labels", pp.GetName(), pod.GetName()) + klog.V(4).Infof("PodPreset %s matches pod %s labels", pp.GetName(), pod.GetName()) matchingPPs = append(matchingPPs, pp) } return matchingPPs, nil diff --git a/plugin/pkg/admission/podtolerationrestriction/BUILD b/plugin/pkg/admission/podtolerationrestriction/BUILD index d5652163389..b519bf07ee7 100644 --- a/plugin/pkg/admission/podtolerationrestriction/BUILD +++ b/plugin/pkg/admission/podtolerationrestriction/BUILD @@ -8,10 +8,14 @@ load( go_test( name = "go_default_test", - srcs = ["admission_test.go"], + srcs = [ + "admission_test.go", + "main_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", + "//pkg/features:go_default_library", "//pkg/scheduler/api:go_default_library", "//pkg/util/tolerations:go_default_library", "//plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction:go_default_library", @@ -21,6 +25,7 @@ go_test( "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission/initializer:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", @@ -55,7 +60,7 @@ go_library( "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/plugin/pkg/admission/podtolerationrestriction/admission.go b/plugin/pkg/admission/podtolerationrestriction/admission.go index 6e75d8dc2bc..35823f5a058 100644 --- a/plugin/pkg/admission/podtolerationrestriction/admission.go +++ b/plugin/pkg/admission/podtolerationrestriction/admission.go @@ -21,7 +21,7 @@ import ( "fmt" "io" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" @@ -188,7 +188,7 @@ func shouldIgnore(a admission.Attributes) bool { obj := a.GetObject() _, ok := obj.(*api.Pod) if !ok { - glog.Errorf("expected pod but got %s", a.GetKind().Kind) + klog.Errorf("expected pod but got %s", a.GetKind().Kind) return true } diff --git a/plugin/pkg/admission/podtolerationrestriction/admission_test.go b/plugin/pkg/admission/podtolerationrestriction/admission_test.go index 688c0671c86..10bfa2e6a63 100644 --- a/plugin/pkg/admission/podtolerationrestriction/admission_test.go +++ b/plugin/pkg/admission/podtolerationrestriction/admission_test.go @@ -27,10 +27,12 @@ import ( "k8s.io/apiserver/pkg/admission" genericadmissioninitializer "k8s.io/apiserver/pkg/admission/initializer" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" "k8s.io/kubernetes/pkg/util/tolerations" pluginapi "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction" @@ -83,9 +85,7 @@ func TestPodAdmission(t *testing.T) { }, } - if err := utilfeature.DefaultFeatureGate.Set("TaintNodesByCondition=true"); err != nil { - t.Errorf("Failed to enable TaintByCondition feature: %v.", err) - } + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.TaintNodesByCondition, true)() tests := []struct { pod *api.Pod diff --git a/plugin/pkg/admission/podtolerationrestriction/main_test.go b/plugin/pkg/admission/podtolerationrestriction/main_test.go new file mode 100644 index 00000000000..d500cb63af7 --- /dev/null +++ b/plugin/pkg/admission/podtolerationrestriction/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podtolerationrestriction + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/plugin/pkg/admission/priority/BUILD b/plugin/pkg/admission/priority/BUILD index 513cc84def3..3737f0c2361 100644 --- a/plugin/pkg/admission/priority/BUILD +++ b/plugin/pkg/admission/priority/BUILD @@ -8,7 +8,10 @@ load( go_test( name = "go_default_test", - srcs = ["admission_test.go"], + srcs = [ + "admission_test.go", + "main_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", @@ -21,8 +24,9 @@ go_test( "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/plugin/pkg/admission/priority/admission_test.go b/plugin/pkg/admission/priority/admission_test.go index ef1c43c2a11..8404467c82c 100644 --- a/plugin/pkg/admission/priority/admission_test.go +++ b/plugin/pkg/admission/priority/admission_test.go @@ -17,16 +17,16 @@ limitations under the License. package priority import ( - "fmt" "testing" - "github.com/golang/glog" + "k8s.io/klog" schedulingv1beta1 "k8s.io/api/scheduling/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/authentication/user" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/informers" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/scheduling" @@ -138,7 +138,7 @@ func TestPriorityClassAdmission(t *testing.T) { } for _, test := range tests { - glog.V(4).Infof("starting test %q", test.name) + klog.V(4).Infof("starting test %q", test.name) ctrl := newPlugin() // Add existing priority classes. @@ -159,7 +159,7 @@ func TestPriorityClassAdmission(t *testing.T) { test.userInfo, ) err := ctrl.Validate(attrs) - glog.Infof("Got %v", err) + klog.Infof("Got %v", err) if err != nil && !test.expectError { t.Errorf("Test %q: unexpected error received: %v", test.name, err) } @@ -239,7 +239,7 @@ func TestDefaultPriority(t *testing.T) { } for _, test := range tests { - glog.V(4).Infof("starting test %q", test.name) + klog.V(4).Infof("starting test %q", test.name) ctrl := newPlugin() if err := addPriorityClasses(ctrl, test.classesBefore); err != nil { t.Errorf("Test %q: unable to add object to informer: %v", test.name, err) @@ -468,9 +468,9 @@ func TestPodAdmission(t *testing.T) { }, } // Enable PodPriority feature gate. - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority)) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)() // Enable ExperimentalCriticalPodAnnotation feature gate. - utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.ExperimentalCriticalPodAnnotation)) + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() tests := []struct { name string existingClasses []*scheduling.PriorityClass @@ -582,7 +582,7 @@ func TestPodAdmission(t *testing.T) { } for _, test := range tests { - glog.V(4).Infof("starting test %q", test.name) + klog.V(4).Infof("starting test %q", test.name) ctrl := newPlugin() // Add existing priority classes. @@ -604,7 +604,7 @@ func TestPodAdmission(t *testing.T) { nil, ) err := ctrl.Admit(attrs) - glog.Infof("Got %v", err) + klog.Infof("Got %v", err) if !test.expectError { if err != nil { t.Errorf("Test %q: unexpected error received: %v", test.name, err) diff --git a/plugin/pkg/admission/priority/main_test.go b/plugin/pkg/admission/priority/main_test.go new file mode 100644 index 00000000000..c17952fd3eb --- /dev/null +++ b/plugin/pkg/admission/priority/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package priority + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/plugin/pkg/admission/resourcequota/BUILD b/plugin/pkg/admission/resourcequota/BUILD index e638b0691a0..96d889890e1 100644 --- a/plugin/pkg/admission/resourcequota/BUILD +++ b/plugin/pkg/admission/resourcequota/BUILD @@ -44,18 +44,22 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/hashicorp/golang-lru:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) go_test( name = "go_default_test", - srcs = ["admission_test.go"], + srcs = [ + "admission_test.go", + "main_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", "//pkg/controller:go_default_library", + "//pkg/features:go_default_library", "//pkg/quota/v1/generic:go_default_library", "//pkg/quota/v1/install:go_default_library", "//plugin/pkg/admission/resourcequota/apis/resourcequota:go_default_library", @@ -65,6 +69,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", diff --git a/plugin/pkg/admission/resourcequota/admission_test.go b/plugin/pkg/admission/resourcequota/admission_test.go index 09e29b0b8fc..be7a8af6bb1 100644 --- a/plugin/pkg/admission/resourcequota/admission_test.go +++ b/plugin/pkg/admission/resourcequota/admission_test.go @@ -31,12 +31,14 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/admission" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" testcore "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/controller" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/quota/v1/generic" "k8s.io/kubernetes/pkg/quota/v1/install" resourcequotaapi "k8s.io/kubernetes/plugin/pkg/admission/resourcequota/apis/resourcequota" @@ -452,15 +454,7 @@ func TestAdmitHandlesNegativePVCUpdates(t *testing.T) { stopCh := make(chan struct{}) defer close(stopCh) - err := utilfeature.DefaultFeatureGate.Set("ExpandPersistentVolumes=true") - if err != nil { - t.Errorf("Failed to enable feature gate for LocalPersistentVolumes: %v", err) - return - } - - defer func() { - utilfeature.DefaultFeatureGate.Set("ExpandPersistentVolumes=false") - }() + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExpandPersistentVolumes, true)() kubeClient := fake.NewSimpleClientset(resourceQuota) informerFactory := informers.NewSharedInformerFactory(kubeClient, controller.NoResyncPeriodFunc()) @@ -491,7 +485,7 @@ func TestAdmitHandlesNegativePVCUpdates(t *testing.T) { }, } - err = handler.Validate(admission.NewAttributesRecord(newPVC, oldPVC, api.Kind("PersistentVolumeClaim").WithVersion("version"), newPVC.Namespace, newPVC.Name, corev1.Resource("persistentvolumeclaims").WithVersion("version"), "", admission.Update, false, nil)) + err := handler.Validate(admission.NewAttributesRecord(newPVC, oldPVC, api.Kind("PersistentVolumeClaim").WithVersion("version"), newPVC.Namespace, newPVC.Name, corev1.Resource("persistentvolumeclaims").WithVersion("version"), "", admission.Update, false, nil)) if err != nil { t.Errorf("Unexpected error: %v", err) } @@ -515,15 +509,7 @@ func TestAdmitHandlesPVCUpdates(t *testing.T) { }, } - err := utilfeature.DefaultFeatureGate.Set("ExpandPersistentVolumes=true") - if err != nil { - t.Errorf("Failed to enable feature gate for LocalPersistentVolumes: %v", err) - return - } - - defer func() { - utilfeature.DefaultFeatureGate.Set("ExpandPersistentVolumes=false") - }() + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExpandPersistentVolumes, true)() // start up quota system stopCh := make(chan struct{}) @@ -558,7 +544,7 @@ func TestAdmitHandlesPVCUpdates(t *testing.T) { }, } - err = handler.Validate(admission.NewAttributesRecord(newPVC, oldPVC, api.Kind("PersistentVolumeClaim").WithVersion("version"), newPVC.Namespace, newPVC.Name, corev1.Resource("persistentvolumeclaims").WithVersion("version"), "", admission.Update, false, nil)) + err := handler.Validate(admission.NewAttributesRecord(newPVC, oldPVC, api.Kind("PersistentVolumeClaim").WithVersion("version"), newPVC.Namespace, newPVC.Name, corev1.Resource("persistentvolumeclaims").WithVersion("version"), "", admission.Update, false, nil)) if err != nil { t.Errorf("Unexpected error: %v", err) } diff --git a/plugin/pkg/admission/resourcequota/controller.go b/plugin/pkg/admission/resourcequota/controller.go index 34c621903b9..13f46d1d5c9 100644 --- a/plugin/pkg/admission/resourcequota/controller.go +++ b/plugin/pkg/admission/resourcequota/controller.go @@ -23,7 +23,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -143,7 +143,7 @@ func (e *quotaEvaluator) run() { go wait.Until(e.doWork, time.Second, e.stopCh) } <-e.stopCh - glog.Infof("Shutting down quota evaluator") + klog.Infof("Shutting down quota evaluator") e.queue.ShutDown() } @@ -162,7 +162,7 @@ func (e *quotaEvaluator) doWork() { } for { if quit := workFunc(); quit { - glog.Infof("quota evaluator worker shutdown") + klog.Infof("quota evaluator worker shutdown") return } } @@ -379,7 +379,7 @@ func getMatchedLimitedScopes(evaluator quota.Evaluator, inputObject runtime.Obje for _, limitedResource := range limitedResources { matched, err := evaluator.MatchingScopes(inputObject, limitedResource.MatchScopes) if err != nil { - glog.Errorf("Error while matching limited Scopes: %v", err) + klog.Errorf("Error while matching limited Scopes: %v", err) return []corev1.ScopedResourceSelectorRequirement{}, err } for _, scope := range matched { @@ -450,7 +450,7 @@ func CheckRequest(quotas []corev1.ResourceQuota, a admission.Attributes, evaluat match, err := evaluator.Matches(&resourceQuota, inputObject) if err != nil { - glog.Errorf("Error occurred while matching resource quota, %v, against input object. Err: %v", resourceQuota, err) + klog.Errorf("Error occurred while matching resource quota, %v, against input object. Err: %v", resourceQuota, err) return quotas, err } if !match { @@ -605,7 +605,7 @@ func (e *quotaEvaluator) Evaluate(a admission.Attributes) error { // note, we do not need aggregate usage here, so we pass a nil informer func evaluator = generic.NewObjectCountEvaluator(gr, nil, "") e.registry.Add(evaluator) - glog.Infof("quota admission added evaluator for: %s", gr) + klog.Infof("quota admission added evaluator for: %s", gr) } // for this kind, check if the operation could mutate any quota resources // if no resources tracked by quota are impacted, then just return diff --git a/plugin/pkg/admission/resourcequota/main_test.go b/plugin/pkg/admission/resourcequota/main_test.go new file mode 100644 index 00000000000..a9d7da30438 --- /dev/null +++ b/plugin/pkg/admission/resourcequota/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcequota + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/plugin/pkg/admission/security/podsecuritypolicy/BUILD b/plugin/pkg/admission/security/podsecuritypolicy/BUILD index 673c7fd666c..6480ddde4dd 100644 --- a/plugin/pkg/admission/security/podsecuritypolicy/BUILD +++ b/plugin/pkg/admission/security/podsecuritypolicy/BUILD @@ -28,7 +28,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/listers/policy/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/plugin/pkg/admission/security/podsecuritypolicy/OWNERS b/plugin/pkg/admission/security/podsecuritypolicy/OWNERS index f3ca3b7dd14..787630abd2f 100644 --- a/plugin/pkg/admission/security/podsecuritypolicy/OWNERS +++ b/plugin/pkg/admission/security/podsecuritypolicy/OWNERS @@ -1,5 +1,7 @@ approvers: -- tallclair -- liggitt +- sig-auth-policy-approvers reviewers: -- pweil- +- sig-auth-policy-reviewers +labels: +- sig/auth + diff --git a/plugin/pkg/admission/security/podsecuritypolicy/admission.go b/plugin/pkg/admission/security/podsecuritypolicy/admission.go index 6c4ac7e25f0..f980b932212 100644 --- a/plugin/pkg/admission/security/podsecuritypolicy/admission.go +++ b/plugin/pkg/admission/security/podsecuritypolicy/admission.go @@ -22,7 +22,7 @@ import ( "sort" "strings" - "github.com/golang/glog" + "k8s.io/klog" policyv1beta1 "k8s.io/api/policy/v1beta1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -132,20 +132,20 @@ func (c *PodSecurityPolicyPlugin) Admit(a admission.Attributes) error { if allowedPod != nil { *pod = *allowedPod // annotate and accept the pod - glog.V(4).Infof("pod %s (generate: %s) in namespace %s validated against provider %s", pod.Name, pod.GenerateName, a.GetNamespace(), pspName) + klog.V(4).Infof("pod %s (generate: %s) in namespace %s validated against provider %s", pod.Name, pod.GenerateName, a.GetNamespace(), pspName) if pod.ObjectMeta.Annotations == nil { pod.ObjectMeta.Annotations = map[string]string{} } pod.ObjectMeta.Annotations[psputil.ValidatedPSPAnnotation] = pspName key := auditKeyPrefix + "/" + "admit-policy" if err := a.AddAnnotation(key, pspName); err != nil { - glog.Warningf("failed to set admission audit annotation %s to %s: %v", key, pspName, err) + klog.Warningf("failed to set admission audit annotation %s to %s: %v", key, pspName, err) } return nil } // we didn't validate against any provider, reject the pod and give the errors for each attempt - glog.V(4).Infof("unable to validate pod %s (generate: %s) in namespace %s against any pod security policy: %v", pod.Name, pod.GenerateName, a.GetNamespace(), validationErrs) + klog.V(4).Infof("unable to validate pod %s (generate: %s) in namespace %s against any pod security policy: %v", pod.Name, pod.GenerateName, a.GetNamespace(), validationErrs) return admission.NewForbidden(a, fmt.Errorf("unable to validate against any pod security policy: %v", validationErrs)) } @@ -166,13 +166,13 @@ func (c *PodSecurityPolicyPlugin) Validate(a admission.Attributes) error { if apiequality.Semantic.DeepEqual(pod, allowedPod) { key := auditKeyPrefix + "/" + "validate-policy" if err := a.AddAnnotation(key, pspName); err != nil { - glog.Warningf("failed to set admission audit annotation %s to %s: %v", key, pspName, err) + klog.Warningf("failed to set admission audit annotation %s to %s: %v", key, pspName, err) } return nil } // we didn't validate against any provider, reject the pod and give the errors for each attempt - glog.V(4).Infof("unable to validate pod %s (generate: %s) in namespace %s against any pod security policy: %v", pod.Name, pod.GenerateName, a.GetNamespace(), validationErrs) + klog.V(4).Infof("unable to validate pod %s (generate: %s) in namespace %s against any pod security policy: %v", pod.Name, pod.GenerateName, a.GetNamespace(), validationErrs) return admission.NewForbidden(a, fmt.Errorf("unable to validate against any pod security policy: %v", validationErrs)) } @@ -207,7 +207,7 @@ func shouldIgnore(a admission.Attributes) (bool, error) { // saved in kubernetes.io/psp annotation. This psp is usually the one we are looking for. func (c *PodSecurityPolicyPlugin) computeSecurityContext(a admission.Attributes, pod *api.Pod, specMutationAllowed bool, validatedPSPHint string) (*api.Pod, string, field.ErrorList, error) { // get all constraints that are usable by the user - glog.V(4).Infof("getting pod security policies for pod %s (generate: %s)", pod.Name, pod.GenerateName) + klog.V(4).Infof("getting pod security policies for pod %s (generate: %s)", pod.Name, pod.GenerateName) var saInfo user.Info if len(pod.Spec.ServiceAccountName) > 0 { saInfo = serviceaccount.UserInfo(a.GetNamespace(), pod.Spec.ServiceAccountName, "") @@ -241,7 +241,7 @@ func (c *PodSecurityPolicyPlugin) computeSecurityContext(a admission.Attributes, providers, errs := c.createProvidersFromPolicies(policies, pod.Namespace) for _, err := range errs { - glog.V(4).Infof("provider creation error: %v", err) + klog.V(4).Infof("provider creation error: %v", err) } if len(providers) == 0 { @@ -379,7 +379,7 @@ func authorizedForPolicyInAPIGroup(info user.Info, namespace, policyName, apiGro attr := buildAttributes(info, namespace, policyName, apiGroupName) decision, reason, err := authz.Authorize(attr) if err != nil { - glog.V(5).Infof("cannot authorize for policy: %v,%v", reason, err) + klog.V(5).Infof("cannot authorize for policy: %v,%v", reason, err) } return (decision == authorizer.DecisionAllow) } diff --git a/plugin/pkg/admission/serviceaccount/OWNERS b/plugin/pkg/admission/serviceaccount/OWNERS index af89d3e6d65..d914c0d7195 100644 --- a/plugin/pkg/admission/serviceaccount/OWNERS +++ b/plugin/pkg/admission/serviceaccount/OWNERS @@ -1,9 +1,7 @@ approvers: -- liggitt -- deads2k -- mikedanese +- sig-auth-serviceaccounts-approvers reviewers: -- liggitt -- deads2k -- mikedanese -- enj +- sig-auth-serviceaccounts-reviewers +labels: +- sig/auth + diff --git a/plugin/pkg/admission/storage/persistentvolume/label/BUILD b/plugin/pkg/admission/storage/persistentvolume/label/BUILD index 5f504ab267c..35b7366258d 100644 --- a/plugin/pkg/admission/storage/persistentvolume/label/BUILD +++ b/plugin/pkg/admission/storage/persistentvolume/label/BUILD @@ -26,17 +26,21 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) go_test( name = "go_default_test", - srcs = ["admission_test.go"], + srcs = [ + "admission_test.go", + "main_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", "//pkg/cloudprovider/providers/aws:go_default_library", + "//pkg/features:go_default_library", "//pkg/kubelet/apis:go_default_library", "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", @@ -44,6 +48,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/plugin/pkg/admission/storage/persistentvolume/label/admission.go b/plugin/pkg/admission/storage/persistentvolume/label/admission.go index a1683e3d297..db291c86f44 100644 --- a/plugin/pkg/admission/storage/persistentvolume/label/admission.go +++ b/plugin/pkg/admission/storage/persistentvolume/label/admission.go @@ -22,10 +22,10 @@ import ( "io" "sync" - "github.com/golang/glog" "k8s.io/apiserver/pkg/admission" utilfeature "k8s.io/apiserver/pkg/util/feature" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" "k8s.io/kubernetes/pkg/cloudprovider/providers/azure" @@ -73,7 +73,7 @@ func newPersistentVolumeLabel() *persistentVolumeLabel { // DEPRECATED: cloud-controller-manager will now start NewPersistentVolumeLabelController // which does exactly what this admission controller used to do. So once GCE, AWS and AZURE can // run externally, we can remove this admission controller. - glog.Warning("PersistentVolumeLabel admission controller is deprecated. " + + klog.Warning("PersistentVolumeLabel admission controller is deprecated. " + "Please remove this controller from your configuration files and scripts.") return &persistentVolumeLabel{ Handler: admission.NewHandler(admission.Create), @@ -170,7 +170,7 @@ func (l *persistentVolumeLabel) Admit(a admission.Attributes) (err error) { volume.Spec.NodeAffinity.Required.NodeSelectorTerms = make([]api.NodeSelectorTerm, 1) } if nodeSelectorRequirementKeysExistInNodeSelectorTerms(requirements, volume.Spec.NodeAffinity.Required.NodeSelectorTerms) { - glog.V(4).Infof("NodeSelectorRequirements for cloud labels %v conflict with existing NodeAffinity %v. Skipping addition of NodeSelectorRequirements for cloud labels.", + klog.V(4).Infof("NodeSelectorRequirements for cloud labels %v conflict with existing NodeAffinity %v. Skipping addition of NodeSelectorRequirements for cloud labels.", requirements, volume.Spec.NodeAffinity) } else { for _, req := range requirements { diff --git a/plugin/pkg/admission/storage/persistentvolume/label/admission_test.go b/plugin/pkg/admission/storage/persistentvolume/label/admission_test.go index bc462e74106..5b6dc02041d 100644 --- a/plugin/pkg/admission/storage/persistentvolume/label/admission_test.go +++ b/plugin/pkg/admission/storage/persistentvolume/label/admission_test.go @@ -28,8 +28,10 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/admission" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" + "k8s.io/kubernetes/pkg/features" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" volumeutil "k8s.io/kubernetes/pkg/volume/util" ) @@ -122,8 +124,8 @@ func TestAdmission(t *testing.T) { }, }, } - utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true") - defer utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") + + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() // Non-cloud PVs are ignored err := handler.Admit(admission.NewAttributesRecord(&ignoredPV, nil, api.Kind("PersistentVolume").WithVersion("version"), ignoredPV.Namespace, ignoredPV.Name, api.Resource("persistentvolumes").WithVersion("version"), "", admission.Create, false, nil)) diff --git a/plugin/pkg/admission/storage/persistentvolume/label/main_test.go b/plugin/pkg/admission/storage/persistentvolume/label/main_test.go new file mode 100644 index 00000000000..60a5d5020e5 --- /dev/null +++ b/plugin/pkg/admission/storage/persistentvolume/label/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package label + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/plugin/pkg/admission/storage/storageclass/setdefault/BUILD b/plugin/pkg/admission/storage/storageclass/setdefault/BUILD index ca19332f2fa..78f92102b00 100644 --- a/plugin/pkg/admission/storage/storageclass/setdefault/BUILD +++ b/plugin/pkg/admission/storage/storageclass/setdefault/BUILD @@ -21,7 +21,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/admission/initializer:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/listers/storage/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -37,7 +37,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/plugin/pkg/admission/storage/storageclass/setdefault/admission.go b/plugin/pkg/admission/storage/storageclass/setdefault/admission.go index f90ac56d080..98610ae060e 100644 --- a/plugin/pkg/admission/storage/storageclass/setdefault/admission.go +++ b/plugin/pkg/admission/storage/storageclass/setdefault/admission.go @@ -20,7 +20,7 @@ import ( "fmt" "io" - "github.com/golang/glog" + "k8s.io/klog" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -105,7 +105,7 @@ func (a *claimDefaulterPlugin) Admit(attr admission.Attributes) error { return nil } - glog.V(4).Infof("no storage class for claim %s (generate: %s)", pvc.Name, pvc.GenerateName) + klog.V(4).Infof("no storage class for claim %s (generate: %s)", pvc.Name, pvc.GenerateName) def, err := getDefaultClass(a.lister) if err != nil { @@ -116,7 +116,7 @@ func (a *claimDefaulterPlugin) Admit(attr admission.Attributes) error { return nil } - glog.V(4).Infof("defaulting storage class for claim %s (generate: %s) to %s", pvc.Name, pvc.GenerateName, def.Name) + klog.V(4).Infof("defaulting storage class for claim %s (generate: %s) to %s", pvc.Name, pvc.GenerateName, def.Name) pvc.Spec.StorageClassName = &def.Name return nil } @@ -132,7 +132,7 @@ func getDefaultClass(lister storagev1listers.StorageClassLister) (*storagev1.Sto for _, class := range list { if storageutil.IsDefaultAnnotation(class.ObjectMeta) { defaultClasses = append(defaultClasses, class) - glog.V(4).Infof("getDefaultClass added: %s", class.Name) + klog.V(4).Infof("getDefaultClass added: %s", class.Name) } } @@ -140,7 +140,7 @@ func getDefaultClass(lister storagev1listers.StorageClassLister) (*storagev1.Sto return nil, nil } if len(defaultClasses) > 1 { - glog.V(4).Infof("getDefaultClass %d defaults found", len(defaultClasses)) + klog.V(4).Infof("getDefaultClass %d defaults found", len(defaultClasses)) return nil, errors.NewInternalError(fmt.Errorf("%d default StorageClasses were found", len(defaultClasses))) } return defaultClasses[0], nil diff --git a/plugin/pkg/admission/storage/storageclass/setdefault/admission_test.go b/plugin/pkg/admission/storage/storageclass/setdefault/admission_test.go index cdd8d73eb90..e9d5fcbaf43 100644 --- a/plugin/pkg/admission/storage/storageclass/setdefault/admission_test.go +++ b/plugin/pkg/admission/storage/storageclass/setdefault/admission_test.go @@ -19,7 +19,7 @@ package setdefault import ( "testing" - "github.com/golang/glog" + "k8s.io/klog" storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -188,7 +188,7 @@ func TestAdmission(t *testing.T) { } for _, test := range tests { - glog.V(4).Infof("starting test %q", test.name) + klog.V(4).Infof("starting test %q", test.name) // clone the claim, it's going to be modified claim := test.claim.DeepCopy() @@ -212,7 +212,7 @@ func TestAdmission(t *testing.T) { nil, // userInfo ) err := ctrl.Admit(attrs) - glog.Infof("Got %v", err) + klog.Infof("Got %v", err) if err != nil && !test.expectError { t.Errorf("Test %q: unexpected error received: %v", test.name, err) } diff --git a/plugin/pkg/admission/storage/storageobjectinuseprotection/BUILD b/plugin/pkg/admission/storage/storageobjectinuseprotection/BUILD index 1cccceca387..6073526d6bc 100644 --- a/plugin/pkg/admission/storage/storageobjectinuseprotection/BUILD +++ b/plugin/pkg/admission/storage/storageobjectinuseprotection/BUILD @@ -11,22 +11,27 @@ go_library( "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) go_test( name = "go_default_test", - srcs = ["admission_test.go"], + srcs = [ + "admission_test.go", + "main_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", + "//pkg/features:go_default_library", "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", ], ) diff --git a/plugin/pkg/admission/storage/storageobjectinuseprotection/admission.go b/plugin/pkg/admission/storage/storageobjectinuseprotection/admission.go index 2a8e24c6dfa..f9d769c3266 100644 --- a/plugin/pkg/admission/storage/storageobjectinuseprotection/admission.go +++ b/plugin/pkg/admission/storage/storageobjectinuseprotection/admission.go @@ -19,7 +19,7 @@ package storageobjectinuseprotection import ( "io" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/util/feature" @@ -97,7 +97,7 @@ func (c *storageProtectionPlugin) admitPV(a admission.Attributes) error { return nil } } - glog.V(4).Infof("adding PV protection finalizer to %s", pv.Name) + klog.V(4).Infof("adding PV protection finalizer to %s", pv.Name) pv.Finalizers = append(pv.Finalizers, volumeutil.PVProtectionFinalizer) return nil @@ -121,7 +121,7 @@ func (c *storageProtectionPlugin) admitPVC(a admission.Attributes) error { } } - glog.V(4).Infof("adding PVC protection finalizer to %s/%s", pvc.Namespace, pvc.Name) + klog.V(4).Infof("adding PVC protection finalizer to %s/%s", pvc.Namespace, pvc.Name) pvc.Finalizers = append(pvc.Finalizers, volumeutil.PVCProtectionFinalizer) return nil } diff --git a/plugin/pkg/admission/storage/storageobjectinuseprotection/admission_test.go b/plugin/pkg/admission/storage/storageobjectinuseprotection/admission_test.go index 757a93749e6..54ee9da6ae7 100644 --- a/plugin/pkg/admission/storage/storageobjectinuseprotection/admission_test.go +++ b/plugin/pkg/admission/storage/storageobjectinuseprotection/admission_test.go @@ -17,7 +17,6 @@ limitations under the License. package storageobjectinuseprotection import ( - "fmt" "reflect" "testing" @@ -27,8 +26,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/util/feature" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" volumeutil "k8s.io/kubernetes/pkg/volume/util" ) @@ -118,31 +119,29 @@ func TestAdmit(t *testing.T) { ctrl := newPlugin() for _, test := range tests { - feature.DefaultFeatureGate.Set(fmt.Sprintf("StorageObjectInUseProtection=%v", test.featureEnabled)) - obj := test.object.DeepCopyObject() - attrs := admission.NewAttributesRecord( - obj, // new object - obj.DeepCopyObject(), // old object, copy to be sure it's not modified - schema.GroupVersionKind{}, - test.namespace, - "foo", - test.resource, - "", // subresource - admission.Create, - false, // dryRun - nil, // userInfo - ) + t.Run(test.name, func(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageObjectInUseProtection, test.featureEnabled)() + obj := test.object.DeepCopyObject() + attrs := admission.NewAttributesRecord( + obj, // new object + obj.DeepCopyObject(), // old object, copy to be sure it's not modified + schema.GroupVersionKind{}, + test.namespace, + "foo", + test.resource, + "", // subresource + admission.Create, + false, // dryRun + nil, // userInfo + ) - err := ctrl.Admit(attrs) - if err != nil { - t.Errorf("Test %q: got unexpected error: %v", test.name, err) - } - if !reflect.DeepEqual(test.expectedObject, obj) { - t.Errorf("Test %q: Expected object:\n%s\ngot:\n%s", test.name, spew.Sdump(test.expectedObject), spew.Sdump(obj)) - } + err := ctrl.Admit(attrs) + if err != nil { + t.Errorf("Test %q: got unexpected error: %v", test.name, err) + } + if !reflect.DeepEqual(test.expectedObject, obj) { + t.Errorf("Test %q: Expected object:\n%s\ngot:\n%s", test.name, spew.Sdump(test.expectedObject), spew.Sdump(obj)) + } + }) } - - // Disable the feature for rest of the tests. - // TODO: remove after alpha - feature.DefaultFeatureGate.Set("StorageObjectInUseProtection=false") } diff --git a/plugin/pkg/admission/storage/storageobjectinuseprotection/main_test.go b/plugin/pkg/admission/storage/storageobjectinuseprotection/main_test.go new file mode 100644 index 00000000000..1c7d89390a1 --- /dev/null +++ b/plugin/pkg/admission/storage/storageobjectinuseprotection/main_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storageobjectinuseprotection + +import ( + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + _ "k8s.io/kubernetes/pkg/features" +) + +func TestMain(m *testing.M) { + utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +} diff --git a/plugin/pkg/auth/authenticator/OWNERS b/plugin/pkg/auth/authenticator/OWNERS new file mode 100644 index 00000000000..c607d2aa8c5 --- /dev/null +++ b/plugin/pkg/auth/authenticator/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authenticators-approvers +reviewers: +- sig-auth-authenticators-reviewers +labels: +- sig/auth + diff --git a/plugin/pkg/auth/authenticator/token/bootstrap/BUILD b/plugin/pkg/auth/authenticator/token/bootstrap/BUILD index 15f6da57969..4432fe3faa3 100644 --- a/plugin/pkg/auth/authenticator/token/bootstrap/BUILD +++ b/plugin/pkg/auth/authenticator/token/bootstrap/BUILD @@ -34,7 +34,7 @@ go_library( "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/plugin/pkg/auth/authenticator/token/bootstrap/bootstrap.go b/plugin/pkg/auth/authenticator/token/bootstrap/bootstrap.go index 93b4e1cddd8..44596156653 100644 --- a/plugin/pkg/auth/authenticator/token/bootstrap/bootstrap.go +++ b/plugin/pkg/auth/authenticator/token/bootstrap/bootstrap.go @@ -27,7 +27,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -62,7 +62,7 @@ type TokenAuthenticator struct { // func tokenErrorf(s *corev1.Secret, format string, i ...interface{}) { format = fmt.Sprintf("Bootstrap secret %s/%s matching bearer token ", s.Namespace, s.Name) + format - glog.V(3).Infof(format, i...) + klog.V(3).Infof(format, i...) } // AuthenticateToken tries to match the provided token to a bootstrap token secret @@ -102,7 +102,7 @@ func (t *TokenAuthenticator) AuthenticateToken(ctx context.Context, token string secret, err := t.lister.Get(secretName) if err != nil { if errors.IsNotFound(err) { - glog.V(3).Infof("No secret of name %s to match bootstrap bearer token", secretName) + klog.V(3).Infof("No secret of name %s to match bootstrap bearer token", secretName) return nil, false, nil } return nil, false, err @@ -170,12 +170,12 @@ func isSecretExpired(secret *corev1.Secret) bool { if len(expiration) > 0 { expTime, err2 := time.Parse(time.RFC3339, expiration) if err2 != nil { - glog.V(3).Infof("Unparseable expiration time (%s) in %s/%s Secret: %v. Treating as expired.", + klog.V(3).Infof("Unparseable expiration time (%s) in %s/%s Secret: %v. Treating as expired.", expiration, secret.Namespace, secret.Name, err2) return true } if time.Now().After(expTime) { - glog.V(3).Infof("Expired bootstrap token in %s/%s Secret: %v", + klog.V(3).Infof("Expired bootstrap token in %s/%s Secret: %v", secret.Namespace, secret.Name, expiration) return true } diff --git a/plugin/pkg/auth/authorizer/OWNERS b/plugin/pkg/auth/authorizer/OWNERS new file mode 100644 index 00000000000..cd0d70a0f8f --- /dev/null +++ b/plugin/pkg/auth/authorizer/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authorizers-approvers +reviewers: +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/plugin/pkg/auth/authorizer/node/BUILD b/plugin/pkg/auth/authorizer/node/BUILD index 5faaf3d6396..5ae8b4a5369 100644 --- a/plugin/pkg/auth/authorizer/node/BUILD +++ b/plugin/pkg/auth/authorizer/node/BUILD @@ -60,7 +60,7 @@ go_library( "//third_party/forked/gonum/graph:go_default_library", "//third_party/forked/gonum/graph/simple:go_default_library", "//third_party/forked/gonum/graph/traverse:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/plugin/pkg/auth/authorizer/node/OWNERS b/plugin/pkg/auth/authorizer/node/OWNERS index 00846326417..5a7794867cf 100644 --- a/plugin/pkg/auth/authorizer/node/OWNERS +++ b/plugin/pkg/auth/authorizer/node/OWNERS @@ -1,8 +1,7 @@ approvers: -- tallclair -- liggitt -- deads2k +- sig-auth-node-isolation-approvers reviewers: -- tallclair -- liggitt -- deads2k +- sig-auth-node-isolation-reviewers +labels: +- sig/auth + diff --git a/plugin/pkg/auth/authorizer/node/graph_populator.go b/plugin/pkg/auth/authorizer/node/graph_populator.go index 7bdf9b76857..f2cc1a5135c 100644 --- a/plugin/pkg/auth/authorizer/node/graph_populator.go +++ b/plugin/pkg/auth/authorizer/node/graph_populator.go @@ -18,7 +18,7 @@ package node import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" corev1 "k8s.io/api/core/v1" storagev1beta1 "k8s.io/api/storage/v1beta1" @@ -109,7 +109,7 @@ func (g *graphPopulator) updateNode(oldObj, obj interface{}) { if node.Spec.ConfigSource != nil { path = fmt.Sprintf("%s/%s", namespace, name) } - glog.V(4).Infof("updateNode configSource reference to %s for node %s", path, node.Name) + klog.V(4).Infof("updateNode configSource reference to %s for node %s", path, node.Name) g.graph.SetNodeConfigMap(node.Name, name, namespace) } @@ -119,7 +119,7 @@ func (g *graphPopulator) deleteNode(obj interface{}) { } node, ok := obj.(*corev1.Node) if !ok { - glog.Infof("unexpected type %T", obj) + klog.Infof("unexpected type %T", obj) return } @@ -137,17 +137,17 @@ func (g *graphPopulator) updatePod(oldObj, obj interface{}) { pod := obj.(*corev1.Pod) if len(pod.Spec.NodeName) == 0 { // No node assigned - glog.V(5).Infof("updatePod %s/%s, no node", pod.Namespace, pod.Name) + klog.V(5).Infof("updatePod %s/%s, no node", pod.Namespace, pod.Name) return } if oldPod, ok := oldObj.(*corev1.Pod); ok && oldPod != nil { if (pod.Spec.NodeName == oldPod.Spec.NodeName) && (pod.UID == oldPod.UID) { // Node and uid are unchanged, all object references in the pod spec are immutable - glog.V(5).Infof("updatePod %s/%s, node unchanged", pod.Namespace, pod.Name) + klog.V(5).Infof("updatePod %s/%s, node unchanged", pod.Namespace, pod.Name) return } } - glog.V(4).Infof("updatePod %s/%s for node %s", pod.Namespace, pod.Name, pod.Spec.NodeName) + klog.V(4).Infof("updatePod %s/%s for node %s", pod.Namespace, pod.Name, pod.Spec.NodeName) g.graph.AddPod(pod) } @@ -157,14 +157,14 @@ func (g *graphPopulator) deletePod(obj interface{}) { } pod, ok := obj.(*corev1.Pod) if !ok { - glog.Infof("unexpected type %T", obj) + klog.Infof("unexpected type %T", obj) return } if len(pod.Spec.NodeName) == 0 { - glog.V(5).Infof("deletePod %s/%s, no node", pod.Namespace, pod.Name) + klog.V(5).Infof("deletePod %s/%s, no node", pod.Namespace, pod.Name) return } - glog.V(4).Infof("deletePod %s/%s for node %s", pod.Namespace, pod.Name, pod.Spec.NodeName) + klog.V(4).Infof("deletePod %s/%s for node %s", pod.Namespace, pod.Name, pod.Spec.NodeName) g.graph.DeletePod(pod.Name, pod.Namespace) } @@ -184,7 +184,7 @@ func (g *graphPopulator) deletePV(obj interface{}) { } pv, ok := obj.(*corev1.PersistentVolume) if !ok { - glog.Infof("unexpected type %T", obj) + klog.Infof("unexpected type %T", obj) return } g.graph.DeletePV(pv.Name) @@ -212,7 +212,7 @@ func (g *graphPopulator) deleteVolumeAttachment(obj interface{}) { } attachment, ok := obj.(*storagev1beta1.VolumeAttachment) if !ok { - glog.Infof("unexpected type %T", obj) + klog.Infof("unexpected type %T", obj) return } g.graph.DeleteVolumeAttachment(attachment.Name) diff --git a/plugin/pkg/auth/authorizer/node/node_authorizer.go b/plugin/pkg/auth/authorizer/node/node_authorizer.go index caeb7d4c387..757b3868611 100644 --- a/plugin/pkg/auth/authorizer/node/node_authorizer.go +++ b/plugin/pkg/auth/authorizer/node/node_authorizer.go @@ -19,7 +19,7 @@ package node import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -86,7 +86,7 @@ func (r *NodeAuthorizer) Authorize(attrs authorizer.Attributes) (authorizer.Deci } if len(nodeName) == 0 { // reject requests from unidentifiable nodes - glog.V(2).Infof("NODE DENY: unknown node for user %q", attrs.GetUser().GetName()) + klog.V(2).Infof("NODE DENY: unknown node for user %q", attrs.GetUser().GetName()) return authorizer.DecisionNoOpinion, fmt.Sprintf("unknown node for user %q", attrs.GetUser().GetName()), nil } @@ -144,12 +144,12 @@ func (r *NodeAuthorizer) authorizeStatusUpdate(nodeName string, startingType ver case "update", "patch": // ok default: - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "can only get/update/patch this type", nil } if attrs.GetSubresource() != "status" { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "can only update status subresource", nil } @@ -159,11 +159,11 @@ func (r *NodeAuthorizer) authorizeStatusUpdate(nodeName string, startingType ver // authorizeGet authorizes "get" requests to objects of the specified type if they are related to the specified node func (r *NodeAuthorizer) authorizeGet(nodeName string, startingType vertexType, attrs authorizer.Attributes) (authorizer.Decision, string, error) { if attrs.GetVerb() != "get" { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "can only get individual resources of this type", nil } if len(attrs.GetSubresource()) > 0 { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "cannot get subresource", nil } return r.authorize(nodeName, startingType, attrs) @@ -173,15 +173,15 @@ func (r *NodeAuthorizer) authorizeGet(nodeName string, startingType vertexType, // specified types if they are related to the specified node. func (r *NodeAuthorizer) authorizeReadNamespacedObject(nodeName string, startingType vertexType, attrs authorizer.Attributes) (authorizer.Decision, string, error) { if attrs.GetVerb() != "get" && attrs.GetVerb() != "list" && attrs.GetVerb() != "watch" { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "can only read resources of this type", nil } if len(attrs.GetSubresource()) > 0 { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "cannot read subresource", nil } if len(attrs.GetNamespace()) == 0 { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "can only read namespaced object of this type", nil } return r.authorize(nodeName, startingType, attrs) @@ -189,17 +189,17 @@ func (r *NodeAuthorizer) authorizeReadNamespacedObject(nodeName string, starting func (r *NodeAuthorizer) authorize(nodeName string, startingType vertexType, attrs authorizer.Attributes) (authorizer.Decision, string, error) { if len(attrs.GetName()) == 0 { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "No Object name found", nil } ok, err := r.hasPathFrom(nodeName, startingType, attrs.GetNamespace(), attrs.GetName()) if err != nil { - glog.V(2).Infof("NODE DENY: %v", err) + klog.V(2).Infof("NODE DENY: %v", err) return authorizer.DecisionNoOpinion, "no path found to object", nil } if !ok { - glog.V(2).Infof("NODE DENY: %q %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %q %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "no path found to object", nil } return authorizer.DecisionAllow, "", nil @@ -209,22 +209,22 @@ func (r *NodeAuthorizer) authorize(nodeName string, startingType vertexType, att // subresource of pods running on a node func (r *NodeAuthorizer) authorizeCreateToken(nodeName string, startingType vertexType, attrs authorizer.Attributes) (authorizer.Decision, string, error) { if attrs.GetVerb() != "create" || len(attrs.GetName()) == 0 { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "can only create tokens for individual service accounts", nil } if attrs.GetSubresource() != "token" { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "can only create token subresource of serviceaccount", nil } ok, err := r.hasPathFrom(nodeName, startingType, attrs.GetNamespace(), attrs.GetName()) if err != nil { - glog.V(2).Infof("NODE DENY: %v", err) + klog.V(2).Infof("NODE DENY: %v", err) return authorizer.DecisionNoOpinion, "no path found to object", nil } if !ok { - glog.V(2).Infof("NODE DENY: %q %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %q %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "no path found to object", nil } return authorizer.DecisionAllow, "", nil @@ -239,13 +239,13 @@ func (r *NodeAuthorizer) authorizeLease(nodeName string, attrs authorizer.Attrib verb != "update" && verb != "patch" && verb != "delete" { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "can only get, create, update, patch, or delete a node lease", nil } // the request must be against the system namespace reserved for node leases if attrs.GetNamespace() != api.NamespaceNodeLease { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, fmt.Sprintf("can only access leases in the %q system namespace", api.NamespaceNodeLease), nil } @@ -253,7 +253,7 @@ func (r *NodeAuthorizer) authorizeLease(nodeName string, attrs authorizer.Attrib // note we skip this check for create, since the authorizer doesn't know the name on create // the noderestriction admission plugin is capable of performing this check at create time if verb != "create" && attrs.GetName() != nodeName { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "can only access node lease with the same name as the requesting node", nil } @@ -269,12 +269,12 @@ func (r *NodeAuthorizer) authorizeCSINodeInfo(nodeName string, attrs authorizer. verb != "update" && verb != "patch" && verb != "delete" { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "can only get, create, update, patch, or delete a CSINodeInfo", nil } if len(attrs.GetSubresource()) > 0 { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "cannot authorize CSINodeInfo subresources", nil } @@ -282,7 +282,7 @@ func (r *NodeAuthorizer) authorizeCSINodeInfo(nodeName string, attrs authorizer. // note we skip this check for create, since the authorizer doesn't know the name on create // the noderestriction admission plugin is capable of performing this check at create time if verb != "create" && attrs.GetName() != nodeName { - glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + klog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "can only access CSINodeInfo with the same name as the requesting node", nil } diff --git a/plugin/pkg/auth/authorizer/rbac/BUILD b/plugin/pkg/auth/authorizer/rbac/BUILD index 6425852b9d8..772cc13ae34 100644 --- a/plugin/pkg/auth/authorizer/rbac/BUILD +++ b/plugin/pkg/auth/authorizer/rbac/BUILD @@ -22,7 +22,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//staging/src/k8s.io/client-go/listers/rbac/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/BUILD b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/BUILD index 2a83c5c9338..51c72997834 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/BUILD +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/BUILD @@ -23,7 +23,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -48,7 +48,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go index ef96322cad0..b20c927d0a4 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go @@ -19,7 +19,7 @@ package bootstrappolicy import ( "strings" - "github.com/golang/glog" + "k8s.io/klog" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,12 +32,12 @@ const saRolePrefix = "system:controller:" func addControllerRole(controllerRoles *[]rbacv1.ClusterRole, controllerRoleBindings *[]rbacv1.ClusterRoleBinding, role rbacv1.ClusterRole) { if !strings.HasPrefix(role.Name, saRolePrefix) { - glog.Fatalf(`role %q must start with %q`, role.Name, saRolePrefix) + klog.Fatalf(`role %q must start with %q`, role.Name, saRolePrefix) } for _, existingRole := range *controllerRoles { if role.Name == existingRole.Name { - glog.Fatalf("role %q was already registered", role.Name) + klog.Fatalf("role %q was already registered", role.Name) } } @@ -353,6 +353,16 @@ func buildControllerRoles() ([]rbacv1.ClusterRole, []rbacv1.ClusterRoleBinding) }) } + if utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) { + addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "root-ca-cert-publisher"}, + Rules: []rbacv1.PolicyRule{ + rbacv1helpers.NewRule("create", "update").Groups(legacyGroup).Resources("configmaps").RuleOrDie(), + eventsRule(), + }, + }) + } + return controllerRoles, controllerRoleBindings } diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/namespace_policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/namespace_policy.go index fefe26d2ceb..31ca13db521 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/namespace_policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/namespace_policy.go @@ -19,7 +19,7 @@ package bootstrappolicy import ( "strings" - "github.com/golang/glog" + "k8s.io/klog" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,13 +36,13 @@ var ( func addNamespaceRole(namespace string, role rbacv1.Role) { if !strings.HasPrefix(namespace, "kube-") { - glog.Fatalf(`roles can only be bootstrapped into reserved namespaces starting with "kube-", not %q`, namespace) + klog.Fatalf(`roles can only be bootstrapped into reserved namespaces starting with "kube-", not %q`, namespace) } existingRoles := namespaceRoles[namespace] for _, existingRole := range existingRoles { if role.Name == existingRole.Name { - glog.Fatalf("role %q was already registered in %q", role.Name, namespace) + klog.Fatalf("role %q was already registered in %q", role.Name, namespace) } } @@ -54,13 +54,13 @@ func addNamespaceRole(namespace string, role rbacv1.Role) { func addNamespaceRoleBinding(namespace string, roleBinding rbacv1.RoleBinding) { if !strings.HasPrefix(namespace, "kube-") { - glog.Fatalf(`rolebindings can only be bootstrapped into reserved namespaces starting with "kube-", not %q`, namespace) + klog.Fatalf(`rolebindings can only be bootstrapped into reserved namespaces starting with "kube-", not %q`, namespace) } existingRoleBindings := namespaceRoleBindings[namespace] for _, existingRoleBinding := range existingRoleBindings { if roleBinding.Name == existingRoleBinding.Name { - glog.Fatalf("rolebinding %q was already registered in %q", roleBinding.Name, namespace) + klog.Fatalf("rolebinding %q was already registered in %q", roleBinding.Name, namespace) } } diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go index 33b3e610685..1972220c25e 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go @@ -107,7 +107,7 @@ func NodeRules() []rbacv1.PolicyRule { // Use the NodeRestriction admission plugin to limit a node to creating/updating its own API object. rbacv1helpers.NewRule("create", "get", "list", "watch").Groups(legacyGroup).Resources("nodes").RuleOrDie(), rbacv1helpers.NewRule("update", "patch").Groups(legacyGroup).Resources("nodes/status").RuleOrDie(), - rbacv1helpers.NewRule("update", "patch", "delete").Groups(legacyGroup).Resources("nodes").RuleOrDie(), + rbacv1helpers.NewRule("update", "patch").Groups(legacyGroup).Resources("nodes").RuleOrDie(), // TODO: restrict to the bound node as creator in the NodeRestrictions admission plugin rbacv1helpers.NewRule("create", "update", "patch").Groups(legacyGroup).Resources("events").RuleOrDie(), @@ -306,6 +306,7 @@ func ClusterRoles() []rbacv1.ClusterRole { rbacv1helpers.NewRule(Read...).Groups(legacyGroup).Resources("namespaces").RuleOrDie(), rbacv1helpers.NewRule(Read...).Groups(appsGroup).Resources( + "controllerrevisions", "statefulsets", "statefulsets/scale", "daemonsets", "deployments", "deployments/scale", diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy_test.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy_test.go index 20ba3b3c0b1..4f1a48b945d 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy_test.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy_test.go @@ -23,7 +23,7 @@ import ( "reflect" "testing" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml index 1df60152a91..5f97d6d8308 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml @@ -271,6 +271,7 @@ items: - apiGroups: - apps resources: + - controllerrevisions - daemonsets - deployments - deployments/scale @@ -881,7 +882,6 @@ items: resources: - nodes verbs: - - delete - patch - update - apiGroups: diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml index 6d5cb73e50d..4f259b16819 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml @@ -357,6 +357,23 @@ items: - kind: ServiceAccount name: resourcequota-controller namespace: kube-system +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + creationTimestamp: null + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:controller:root-ca-cert-publisher + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:controller:root-ca-cert-publisher + subjects: + - kind: ServiceAccount + name: root-ca-cert-publisher + namespace: kube-system - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml index 3344def7c28..a853a9d9e9b 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml @@ -1031,6 +1031,31 @@ items: - create - patch - update +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + creationTimestamp: null + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:controller:root-ca-cert-publisher + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - update + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/plugin/pkg/auth/authorizer/rbac/rbac.go b/plugin/pkg/auth/authorizer/rbac/rbac.go index a0f173c393b..5cd339675b9 100644 --- a/plugin/pkg/auth/authorizer/rbac/rbac.go +++ b/plugin/pkg/auth/authorizer/rbac/rbac.go @@ -21,7 +21,7 @@ import ( "bytes" "fmt" - "github.com/golang/glog" + "k8s.io/klog" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/labels" @@ -81,7 +81,7 @@ func (r *RBACAuthorizer) Authorize(requestAttributes authorizer.Attributes) (aut // Build a detailed log of the denial. // Make the whole block conditional so we don't do a lot of string-building we won't use. - if glog.V(5) { + if klog.V(5) { var operation string if requestAttributes.IsResourceRequest() { b := &bytes.Buffer{} @@ -115,7 +115,7 @@ func (r *RBACAuthorizer) Authorize(requestAttributes authorizer.Attributes) (aut scope = "cluster-wide" } - glog.Infof("RBAC DENY: user %q groups %q cannot %s %s", requestAttributes.GetUser().GetName(), requestAttributes.GetUser().GetGroups(), operation, scope) + klog.Infof("RBAC DENY: user %q groups %q cannot %s %s", requestAttributes.GetUser().GetName(), requestAttributes.GetUser().GetGroups(), operation, scope) } reason := "" diff --git a/staging/src/BUILD b/staging/src/BUILD index b771f0dcb1d..07cb474cafe 100644 --- a/staging/src/BUILD +++ b/staging/src/BUILD @@ -102,6 +102,7 @@ filegroup( "//staging/src/k8s.io/apiserver/pkg/util/feature:all-srcs", "//staging/src/k8s.io/apiserver/pkg/util/flag:all-srcs", "//staging/src/k8s.io/apiserver/pkg/util/flushwriter:all-srcs", + "//staging/src/k8s.io/apiserver/pkg/util/globalflag:all-srcs", "//staging/src/k8s.io/apiserver/pkg/util/logs:all-srcs", "//staging/src/k8s.io/apiserver/pkg/util/openapi:all-srcs", "//staging/src/k8s.io/apiserver/pkg/util/proxy:all-srcs", @@ -123,6 +124,7 @@ filegroup( "//staging/src/k8s.io/client-go/examples/workqueue:all-srcs", "//staging/src/k8s.io/client-go/informers:all-srcs", "//staging/src/k8s.io/client-go/kubernetes:all-srcs", + "//staging/src/k8s.io/client-go/kubernetes_test:all-srcs", "//staging/src/k8s.io/client-go/listers/admissionregistration/v1alpha1:all-srcs", "//staging/src/k8s.io/client-go/listers/admissionregistration/v1beta1:all-srcs", "//staging/src/k8s.io/client-go/listers/apps/v1:all-srcs", @@ -199,7 +201,6 @@ filegroup( "//staging/src/k8s.io/code-generator/cmd/import-boss:all-srcs", "//staging/src/k8s.io/code-generator/cmd/informer-gen:all-srcs", "//staging/src/k8s.io/code-generator/cmd/lister-gen:all-srcs", - "//staging/src/k8s.io/code-generator/cmd/openapi-gen:all-srcs", "//staging/src/k8s.io/code-generator/cmd/register-gen:all-srcs", "//staging/src/k8s.io/code-generator/cmd/set-gen:all-srcs", "//staging/src/k8s.io/code-generator/hack:all-srcs", diff --git a/staging/src/k8s.io/api/Godeps/Godeps.json b/staging/src/k8s.io/api/Godeps/Godeps.json index a0dad5a5eea..953c5a7e02d 100644 --- a/staging/src/k8s.io/api/Godeps/Godeps.json +++ b/staging/src/k8s.io/api/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/api", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -10,10 +10,6 @@ "ImportPath": "github.com/davecgh/go-spew/spew", "Rev": "782f4967f2dc4564575ca782fe2d04090b5faca8" }, - { - "ImportPath": "github.com/ghodss/yaml", - "Rev": "c7ce16629ff4cd059ed96ed06419dd3856fd3577" - }, { "ImportPath": "github.com/gogo/protobuf/proto", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" @@ -22,10 +18,6 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/golang/protobuf/proto", "Rev": "b4deda0973fb4c70b50d226b1af49f3da59f5265" @@ -64,19 +56,19 @@ }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/text/secure/bidirule", @@ -246,6 +238,10 @@ "ImportPath": "k8s.io/apimachinery/third_party/forked/golang/reflect", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/apimachinery/pkg/api/apitesting/fuzzer", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -285,6 +281,10 @@ { "ImportPath": "k8s.io/apimachinery/pkg/util/intstr", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" } ] } diff --git a/staging/src/k8s.io/api/auditregistration/OWNERS b/staging/src/k8s.io/api/auditregistration/OWNERS index df96f16bf05..c342e5e0cb7 100644 --- a/staging/src/k8s.io/api/auditregistration/OWNERS +++ b/staging/src/k8s.io/api/auditregistration/OWNERS @@ -1,5 +1,6 @@ reviewers: -- lavalamp -- sttts -- tallclair -- pbarker \ No newline at end of file +- sig-auth-audit-approvers +- sig-auth-audit-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/api/authentication/OWNERS b/staging/src/k8s.io/api/authentication/OWNERS index 2bdfd0ce5bc..1b888f0b803 100755 --- a/staging/src/k8s.io/api/authentication/OWNERS +++ b/staging/src/k8s.io/api/authentication/OWNERS @@ -1,9 +1,6 @@ reviewers: -- liggitt -- lavalamp -- wojtek-t -- deads2k -- sttts -- mbohlool -- jianhuiz -- enj +- sig-auth-authenticators-approvers +- sig-auth-authenticators-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/api/authentication/v1/generated.pb.go b/staging/src/k8s.io/api/authentication/v1/generated.pb.go index dcea42283cb..4e7f28d8c97 100644 --- a/staging/src/k8s.io/api/authentication/v1/generated.pb.go +++ b/staging/src/k8s.io/api/authentication/v1/generated.pb.go @@ -355,6 +355,21 @@ func (m *TokenReviewSpec) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Token))) i += copy(dAtA[i:], m.Token) + if len(m.Audiences) > 0 { + for _, s := range m.Audiences { + dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } return i, nil } @@ -393,6 +408,21 @@ func (m *TokenReviewStatus) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Error))) i += copy(dAtA[i:], m.Error) + if len(m.Audiences) > 0 { + for _, s := range m.Audiences { + dAtA[i] = 0x22 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } return i, nil } @@ -561,6 +591,12 @@ func (m *TokenReviewSpec) Size() (n int) { _ = l l = len(m.Token) n += 1 + l + sovGenerated(uint64(l)) + if len(m.Audiences) > 0 { + for _, s := range m.Audiences { + l = len(s) + n += 1 + l + sovGenerated(uint64(l)) + } + } return n } @@ -572,6 +608,12 @@ func (m *TokenReviewStatus) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) l = len(m.Error) n += 1 + l + sovGenerated(uint64(l)) + if len(m.Audiences) > 0 { + for _, s := range m.Audiences { + l = len(s) + n += 1 + l + sovGenerated(uint64(l)) + } + } return n } @@ -679,6 +721,7 @@ func (this *TokenReviewSpec) String() string { } s := strings.Join([]string{`&TokenReviewSpec{`, `Token:` + fmt.Sprintf("%v", this.Token) + `,`, + `Audiences:` + fmt.Sprintf("%v", this.Audiences) + `,`, `}`, }, "") return s @@ -691,6 +734,7 @@ func (this *TokenReviewStatus) String() string { `Authenticated:` + fmt.Sprintf("%v", this.Authenticated) + `,`, `User:` + strings.Replace(strings.Replace(this.User.String(), "UserInfo", "UserInfo", 1), `&`, ``, 1) + `,`, `Error:` + fmt.Sprintf("%v", this.Error) + `,`, + `Audiences:` + fmt.Sprintf("%v", this.Audiences) + `,`, `}`, }, "") return s @@ -1550,6 +1594,35 @@ func (m *TokenReviewSpec) Unmarshal(dAtA []byte) error { } m.Token = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Audiences", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Audiences = append(m.Audiences, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -1679,6 +1752,35 @@ func (m *TokenReviewStatus) Unmarshal(dAtA []byte) error { } m.Error = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Audiences", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Audiences = append(m.Audiences, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -2070,61 +2172,62 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 892 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x55, 0xcf, 0x8f, 0xdb, 0x44, - 0x14, 0x8e, 0xf3, 0x63, 0xb5, 0x99, 0x74, 0x97, 0xdd, 0x29, 0x95, 0xa2, 0x05, 0xec, 0x60, 0x24, - 0x14, 0x01, 0xb5, 0x9b, 0x08, 0x95, 0xaa, 0x48, 0x48, 0x6b, 0x36, 0x82, 0x08, 0x41, 0xab, 0x69, - 0x77, 0x41, 0x9c, 0x98, 0xd8, 0x6f, 0xb3, 0x26, 0x78, 0x6c, 0xec, 0x71, 0x68, 0x6e, 0xfd, 0x13, - 0x38, 0x82, 0xc4, 0x81, 0x3f, 0x02, 0x89, 0x23, 0xd7, 0x3d, 0x56, 0x9c, 0x7a, 0x40, 0x11, 0x6b, - 0xfe, 0x05, 0x4e, 0x9c, 0xd0, 0x8c, 0x67, 0xe3, 0xfc, 0xd8, 0x4d, 0x73, 0xea, 0x2d, 0xf3, 0xde, - 0xf7, 0xbe, 0x79, 0xef, 0x9b, 0x2f, 0xcf, 0xa8, 0x37, 0xba, 0x97, 0x58, 0x7e, 0x68, 0x8f, 0xd2, - 0x01, 0xc4, 0x0c, 0x38, 0x24, 0xf6, 0x18, 0x98, 0x17, 0xc6, 0xb6, 0x4a, 0xd0, 0xc8, 0xb7, 0x69, - 0xca, 0xcf, 0x80, 0x71, 0xdf, 0xa5, 0xdc, 0x0f, 0x99, 0x3d, 0xee, 0xd8, 0x43, 0x60, 0x10, 0x53, - 0x0e, 0x9e, 0x15, 0xc5, 0x21, 0x0f, 0xf1, 0xeb, 0x39, 0xda, 0xa2, 0x91, 0x6f, 0x2d, 0xa2, 0xad, - 0x71, 0xe7, 0xe0, 0xf6, 0xd0, 0xe7, 0x67, 0xe9, 0xc0, 0x72, 0xc3, 0xc0, 0x1e, 0x86, 0xc3, 0xd0, - 0x96, 0x45, 0x83, 0xf4, 0x54, 0x9e, 0xe4, 0x41, 0xfe, 0xca, 0xc9, 0x0e, 0xde, 0x2f, 0xae, 0x0e, - 0xa8, 0x7b, 0xe6, 0x33, 0x88, 0x27, 0x76, 0x34, 0x1a, 0x8a, 0x40, 0x62, 0x07, 0xc0, 0xe9, 0x15, - 0x2d, 0x1c, 0xd8, 0xd7, 0x55, 0xc5, 0x29, 0xe3, 0x7e, 0x00, 0x2b, 0x05, 0x77, 0x5f, 0x54, 0x90, - 0xb8, 0x67, 0x10, 0xd0, 0xe5, 0x3a, 0xf3, 0x4f, 0x0d, 0xbd, 0xea, 0x84, 0x29, 0xf3, 0x1e, 0x0c, - 0xbe, 0x05, 0x97, 0x13, 0x38, 0x85, 0x18, 0x98, 0x0b, 0xb8, 0x85, 0xaa, 0x23, 0x9f, 0x79, 0x4d, - 0xad, 0xa5, 0xb5, 0xeb, 0xce, 0x8d, 0xf3, 0xa9, 0x51, 0xca, 0xa6, 0x46, 0xf5, 0x33, 0x9f, 0x79, - 0x44, 0x66, 0x70, 0x17, 0x21, 0xfa, 0xb0, 0x7f, 0x02, 0x71, 0xe2, 0x87, 0xac, 0x59, 0x96, 0x38, - 0xac, 0x70, 0xe8, 0x70, 0x96, 0x21, 0x73, 0x28, 0xc1, 0xca, 0x68, 0x00, 0xcd, 0xca, 0x22, 0xeb, - 0x17, 0x34, 0x00, 0x22, 0x33, 0xd8, 0x41, 0x95, 0xb4, 0x7f, 0xd4, 0xac, 0x4a, 0xc0, 0x1d, 0x05, - 0xa8, 0x1c, 0xf7, 0x8f, 0xfe, 0x9b, 0x1a, 0x6f, 0x5e, 0x37, 0x24, 0x9f, 0x44, 0x90, 0x58, 0xc7, - 0xfd, 0x23, 0x22, 0x8a, 0xcd, 0x0f, 0x10, 0xea, 0x3d, 0xe1, 0x31, 0x3d, 0xa1, 0xdf, 0xa5, 0x80, - 0x0d, 0x54, 0xf3, 0x39, 0x04, 0x49, 0x53, 0x6b, 0x55, 0xda, 0x75, 0xa7, 0x9e, 0x4d, 0x8d, 0x5a, - 0x5f, 0x04, 0x48, 0x1e, 0xbf, 0xbf, 0xfd, 0xd3, 0xaf, 0x46, 0xe9, 0xe9, 0x5f, 0xad, 0x92, 0xf9, - 0x4b, 0x19, 0xdd, 0x78, 0x1c, 0x8e, 0x80, 0x11, 0xf8, 0x3e, 0x85, 0x84, 0xe3, 0x6f, 0xd0, 0xb6, - 0x78, 0x22, 0x8f, 0x72, 0x2a, 0x95, 0x68, 0x74, 0xef, 0x58, 0x85, 0x3b, 0x66, 0x4d, 0x58, 0xd1, - 0x68, 0x28, 0x02, 0x89, 0x25, 0xd0, 0xd6, 0xb8, 0x63, 0xe5, 0x72, 0x7e, 0x0e, 0x9c, 0x16, 0x9a, - 0x14, 0x31, 0x32, 0x63, 0xc5, 0x0f, 0x51, 0x35, 0x89, 0xc0, 0x95, 0xfa, 0x35, 0xba, 0x96, 0xb5, - 0xce, 0x7b, 0xd6, 0x7c, 0x6f, 0x8f, 0x22, 0x70, 0x0b, 0x05, 0xc5, 0x89, 0x48, 0x26, 0xfc, 0x15, - 0xda, 0x4a, 0x38, 0xe5, 0x69, 0x22, 0x55, 0x5e, 0xec, 0xf8, 0x45, 0x9c, 0xb2, 0xce, 0xd9, 0x55, - 0xac, 0x5b, 0xf9, 0x99, 0x28, 0x3e, 0xf3, 0x5f, 0x0d, 0xed, 0x2d, 0xb7, 0x80, 0xdf, 0x45, 0x75, - 0x9a, 0x7a, 0xbe, 0x30, 0xcd, 0xa5, 0xc4, 0x3b, 0xd9, 0xd4, 0xa8, 0x1f, 0x5e, 0x06, 0x49, 0x91, - 0xc7, 0x0c, 0xed, 0x0e, 0x16, 0xdc, 0xa6, 0x7a, 0xec, 0xae, 0xef, 0xf1, 0x2a, 0x87, 0x3a, 0x38, - 0x9b, 0x1a, 0xbb, 0x8b, 0x19, 0xb2, 0xc4, 0x8e, 0x3f, 0x46, 0xfb, 0xf0, 0x24, 0xf2, 0x63, 0xc9, - 0xf4, 0x08, 0xdc, 0x90, 0x79, 0x89, 0xf4, 0x56, 0xc5, 0xb9, 0x95, 0x4d, 0x8d, 0xfd, 0xde, 0x72, - 0x92, 0xac, 0xe2, 0xcd, 0xdf, 0x34, 0x84, 0x57, 0x55, 0xc2, 0x6f, 0xa1, 0x1a, 0x17, 0x51, 0xf5, - 0x17, 0xd9, 0x51, 0xa2, 0xd5, 0x72, 0x68, 0x9e, 0xc3, 0x13, 0x74, 0xb3, 0x20, 0x7c, 0xec, 0x07, - 0x90, 0x70, 0x1a, 0x44, 0xea, 0xb5, 0xdf, 0xd9, 0xcc, 0x4b, 0xa2, 0xcc, 0x79, 0x4d, 0xd1, 0xdf, - 0xec, 0xad, 0xd2, 0x91, 0xab, 0xee, 0x30, 0x7f, 0x2e, 0xa3, 0x86, 0x6a, 0x7b, 0xec, 0xc3, 0x0f, - 0x2f, 0xc1, 0xcb, 0x0f, 0x16, 0xbc, 0x7c, 0x7b, 0x23, 0xdf, 0x89, 0xd6, 0xae, 0xb5, 0xf2, 0x97, - 0x4b, 0x56, 0xb6, 0x37, 0xa7, 0x5c, 0xef, 0xe4, 0xbb, 0xe8, 0x95, 0xa5, 0xfb, 0x37, 0x7a, 0x4e, - 0xf3, 0x0f, 0x0d, 0xed, 0xaf, 0xdc, 0x82, 0x3f, 0x44, 0x3b, 0x73, 0xcd, 0x40, 0xbe, 0x34, 0xb7, - 0x9d, 0x5b, 0x8a, 0x62, 0xe7, 0x70, 0x3e, 0x49, 0x16, 0xb1, 0xf8, 0x53, 0x54, 0x4d, 0x13, 0x88, - 0x95, 0x68, 0x6f, 0xaf, 0x9f, 0xf0, 0x38, 0x81, 0xb8, 0xcf, 0x4e, 0xc3, 0x42, 0x2d, 0x11, 0x21, - 0x92, 0x41, 0x4c, 0x00, 0x71, 0x1c, 0xc6, 0x6a, 0xbb, 0xce, 0x26, 0xe8, 0x89, 0x20, 0xc9, 0x73, - 0xe6, 0xef, 0x65, 0xb4, 0x7d, 0xc9, 0x82, 0xdf, 0x43, 0xdb, 0xa2, 0x52, 0xae, 0xe4, 0x7c, 0xec, - 0x3d, 0x55, 0x24, 0x31, 0x22, 0x4e, 0x66, 0x08, 0xfc, 0x06, 0xaa, 0xa4, 0xbe, 0xa7, 0x36, 0x7d, - 0x63, 0x6e, 0x35, 0x13, 0x11, 0xc7, 0x26, 0xda, 0x1a, 0xc6, 0x61, 0x1a, 0x89, 0xc7, 0x12, 0x5b, - 0x00, 0x09, 0xdd, 0x3f, 0x91, 0x11, 0xa2, 0x32, 0xf8, 0x04, 0xd5, 0x40, 0x6c, 0xe6, 0x66, 0xb5, - 0x55, 0x69, 0x37, 0xba, 0x9d, 0xcd, 0xa6, 0xb5, 0xe4, 0x36, 0xef, 0x31, 0x1e, 0x4f, 0xe6, 0xa6, - 0x12, 0x31, 0x92, 0xd3, 0x1d, 0x0c, 0xd4, 0xc6, 0x97, 0x18, 0xbc, 0x87, 0x2a, 0x23, 0x98, 0xe4, - 0x13, 0x11, 0xf1, 0x13, 0x7f, 0x84, 0x6a, 0x63, 0xf1, 0x31, 0x50, 0x2a, 0xb7, 0xd7, 0xdf, 0x5b, - 0x7c, 0x3c, 0x48, 0x5e, 0x76, 0xbf, 0x7c, 0x4f, 0x73, 0xda, 0xe7, 0x17, 0x7a, 0xe9, 0xd9, 0x85, - 0x5e, 0x7a, 0x7e, 0xa1, 0x97, 0x9e, 0x66, 0xba, 0x76, 0x9e, 0xe9, 0xda, 0xb3, 0x4c, 0xd7, 0x9e, - 0x67, 0xba, 0xf6, 0x77, 0xa6, 0x6b, 0x3f, 0xfe, 0xa3, 0x97, 0xbe, 0x2e, 0x8f, 0x3b, 0xff, 0x07, - 0x00, 0x00, 0xff, 0xff, 0x5e, 0x8d, 0x94, 0x78, 0x88, 0x08, 0x00, 0x00, + // 900 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x55, 0xcf, 0x6f, 0xe3, 0x44, + 0x14, 0x8e, 0xf3, 0xa3, 0x4a, 0x26, 0xdb, 0xd2, 0xce, 0xb2, 0x52, 0x54, 0xc0, 0x2e, 0x41, 0x42, + 0x15, 0xb0, 0xf6, 0x26, 0x42, 0xb0, 0x5a, 0x24, 0xa4, 0x9a, 0x46, 0x10, 0x21, 0xd8, 0xd5, 0xec, + 0xb6, 0x20, 0x4e, 0x4c, 0xec, 0xd7, 0xc4, 0x04, 0x8f, 0x8d, 0x3d, 0x0e, 0x9b, 0xdb, 0xfe, 0x09, + 0x1c, 0x41, 0xe2, 0xc0, 0x1f, 0x81, 0xc4, 0xbf, 0xd0, 0xe3, 0x8a, 0xd3, 0x1e, 0x50, 0x44, 0xcd, + 0x95, 0x23, 0x27, 0x4e, 0x68, 0xc6, 0xd3, 0x38, 0x4e, 0xda, 0x34, 0x27, 0x6e, 0x9e, 0xf7, 0xbe, + 0xf7, 0xbd, 0x37, 0xdf, 0x7c, 0x9e, 0x41, 0xbd, 0xf1, 0xfd, 0xd8, 0xf4, 0x02, 0x6b, 0x9c, 0x0c, + 0x20, 0x62, 0xc0, 0x21, 0xb6, 0x26, 0xc0, 0xdc, 0x20, 0xb2, 0x54, 0x82, 0x86, 0x9e, 0x45, 0x13, + 0x3e, 0x02, 0xc6, 0x3d, 0x87, 0x72, 0x2f, 0x60, 0xd6, 0xa4, 0x63, 0x0d, 0x81, 0x41, 0x44, 0x39, + 0xb8, 0x66, 0x18, 0x05, 0x3c, 0xc0, 0xaf, 0x66, 0x68, 0x93, 0x86, 0x9e, 0x59, 0x44, 0x9b, 0x93, + 0xce, 0xfe, 0xdd, 0xa1, 0xc7, 0x47, 0xc9, 0xc0, 0x74, 0x02, 0xdf, 0x1a, 0x06, 0xc3, 0xc0, 0x92, + 0x45, 0x83, 0xe4, 0x4c, 0xae, 0xe4, 0x42, 0x7e, 0x65, 0x64, 0xfb, 0xef, 0xe6, 0xad, 0x7d, 0xea, + 0x8c, 0x3c, 0x06, 0xd1, 0xd4, 0x0a, 0xc7, 0x43, 0x11, 0x88, 0x2d, 0x1f, 0x38, 0xbd, 0x62, 0x84, + 0x7d, 0xeb, 0xba, 0xaa, 0x28, 0x61, 0xdc, 0xf3, 0x61, 0xa5, 0xe0, 0xbd, 0x9b, 0x0a, 0x62, 0x67, + 0x04, 0x3e, 0x5d, 0xae, 0x6b, 0xff, 0xae, 0xa1, 0x97, 0xed, 0x20, 0x61, 0xee, 0xc3, 0xc1, 0x37, + 0xe0, 0x70, 0x02, 0x67, 0x10, 0x01, 0x73, 0x00, 0x1f, 0xa0, 0xea, 0xd8, 0x63, 0x6e, 0x4b, 0x3b, + 0xd0, 0x0e, 0x1b, 0xf6, 0xad, 0xf3, 0x99, 0x51, 0x4a, 0x67, 0x46, 0xf5, 0x53, 0x8f, 0xb9, 0x44, + 0x66, 0x70, 0x17, 0x21, 0xfa, 0xa8, 0x7f, 0x0a, 0x51, 0xec, 0x05, 0xac, 0x55, 0x96, 0x38, 0xac, + 0x70, 0xe8, 0x68, 0x9e, 0x21, 0x0b, 0x28, 0xc1, 0xca, 0xa8, 0x0f, 0xad, 0x4a, 0x91, 0xf5, 0x73, + 0xea, 0x03, 0x91, 0x19, 0x6c, 0xa3, 0x4a, 0xd2, 0x3f, 0x6e, 0x55, 0x25, 0xe0, 0x9e, 0x02, 0x54, + 0x4e, 0xfa, 0xc7, 0xff, 0xce, 0x8c, 0xd7, 0xaf, 0xdb, 0x24, 0x9f, 0x86, 0x10, 0x9b, 0x27, 0xfd, + 0x63, 0x22, 0x8a, 0xdb, 0xef, 0x23, 0xd4, 0x7b, 0xca, 0x23, 0x7a, 0x4a, 0xbf, 0x4d, 0x00, 0x1b, + 0xa8, 0xe6, 0x71, 0xf0, 0xe3, 0x96, 0x76, 0x50, 0x39, 0x6c, 0xd8, 0x8d, 0x74, 0x66, 0xd4, 0xfa, + 0x22, 0x40, 0xb2, 0xf8, 0x83, 0xfa, 0x8f, 0xbf, 0x18, 0xa5, 0x67, 0x7f, 0x1c, 0x94, 0xda, 0x3f, + 0x97, 0xd1, 0xad, 0x27, 0xc1, 0x18, 0x18, 0x81, 0xef, 0x12, 0x88, 0x39, 0xfe, 0x1a, 0xd5, 0xc5, + 0x11, 0xb9, 0x94, 0x53, 0xa9, 0x44, 0xb3, 0x7b, 0xcf, 0xcc, 0xdd, 0x31, 0x1f, 0xc2, 0x0c, 0xc7, + 0x43, 0x11, 0x88, 0x4d, 0x81, 0x36, 0x27, 0x1d, 0x33, 0x93, 0xf3, 0x33, 0xe0, 0x34, 0xd7, 0x24, + 0x8f, 0x91, 0x39, 0x2b, 0x7e, 0x84, 0xaa, 0x71, 0x08, 0x8e, 0xd4, 0xaf, 0xd9, 0x35, 0xcd, 0x75, + 0xde, 0x33, 0x17, 0x67, 0x7b, 0x1c, 0x82, 0x93, 0x2b, 0x28, 0x56, 0x44, 0x32, 0xe1, 0x2f, 0xd1, + 0x56, 0xcc, 0x29, 0x4f, 0x62, 0xa9, 0x72, 0x71, 0xe2, 0x9b, 0x38, 0x65, 0x9d, 0xbd, 0xa3, 0x58, + 0xb7, 0xb2, 0x35, 0x51, 0x7c, 0xed, 0x7f, 0x34, 0xb4, 0xbb, 0x3c, 0x02, 0x7e, 0x1b, 0x35, 0x68, + 0xe2, 0x7a, 0xc2, 0x34, 0x97, 0x12, 0x6f, 0xa7, 0x33, 0xa3, 0x71, 0x74, 0x19, 0x24, 0x79, 0x1e, + 0x33, 0xb4, 0x33, 0x28, 0xb8, 0x4d, 0xcd, 0xd8, 0x5d, 0x3f, 0xe3, 0x55, 0x0e, 0xb5, 0x71, 0x3a, + 0x33, 0x76, 0x8a, 0x19, 0xb2, 0xc4, 0x8e, 0x3f, 0x42, 0x7b, 0xf0, 0x34, 0xf4, 0x22, 0xc9, 0xf4, + 0x18, 0x9c, 0x80, 0xb9, 0xb1, 0xf4, 0x56, 0xc5, 0xbe, 0x93, 0xce, 0x8c, 0xbd, 0xde, 0x72, 0x92, + 0xac, 0xe2, 0xdb, 0xbf, 0x6a, 0x08, 0xaf, 0xaa, 0x84, 0xdf, 0x40, 0x35, 0x2e, 0xa2, 0xea, 0x17, + 0xd9, 0x56, 0xa2, 0xd5, 0x32, 0x68, 0x96, 0xc3, 0x53, 0x74, 0x3b, 0x27, 0x7c, 0xe2, 0xf9, 0x10, + 0x73, 0xea, 0x87, 0xea, 0xb4, 0xdf, 0xda, 0xcc, 0x4b, 0xa2, 0xcc, 0x7e, 0x45, 0xd1, 0xdf, 0xee, + 0xad, 0xd2, 0x91, 0xab, 0x7a, 0xb4, 0x7f, 0x2a, 0xa3, 0xa6, 0x1a, 0x7b, 0xe2, 0xc1, 0xf7, 0xff, + 0x83, 0x97, 0x1f, 0x16, 0xbc, 0x7c, 0x77, 0x23, 0xdf, 0x89, 0xd1, 0xae, 0xb5, 0xf2, 0x17, 0x4b, + 0x56, 0xb6, 0x36, 0xa7, 0x5c, 0xef, 0x64, 0x07, 0xbd, 0xb4, 0xd4, 0x7f, 0xb3, 0xe3, 0x2c, 0x98, + 0xbd, 0xbc, 0xde, 0xec, 0xed, 0xbf, 0x35, 0xb4, 0xb7, 0x32, 0x12, 0xfe, 0x00, 0x6d, 0x2f, 0x4c, + 0x0e, 0xd9, 0x0d, 0x5b, 0xb7, 0xef, 0xa8, 0x7e, 0xdb, 0x47, 0x8b, 0x49, 0x52, 0xc4, 0xe2, 0x4f, + 0x50, 0x35, 0x89, 0x21, 0x52, 0x0a, 0xbf, 0xb9, 0x5e, 0x8e, 0x93, 0x18, 0xa2, 0x3e, 0x3b, 0x0b, + 0x72, 0x69, 0x45, 0x84, 0x48, 0x06, 0xb1, 0x5d, 0x88, 0xa2, 0x20, 0x52, 0x57, 0xf1, 0x7c, 0xbb, + 0x3d, 0x11, 0x24, 0x59, 0xae, 0xb8, 0xdd, 0xea, 0x0d, 0xdb, 0xfd, 0xad, 0x8c, 0xea, 0x97, 0x2d, + 0xf1, 0x3b, 0xa8, 0x2e, 0xda, 0xc8, 0xcb, 0x3e, 0x13, 0x74, 0x57, 0x75, 0x90, 0x18, 0x11, 0x27, + 0x73, 0x04, 0x7e, 0x0d, 0x55, 0x12, 0xcf, 0x55, 0x6f, 0x48, 0x73, 0xe1, 0xd2, 0x27, 0x22, 0x8e, + 0xdb, 0x68, 0x6b, 0x18, 0x05, 0x49, 0x28, 0x6c, 0x20, 0x66, 0x40, 0xe2, 0x44, 0x3f, 0x96, 0x11, + 0xa2, 0x32, 0xf8, 0x14, 0xd5, 0x40, 0xdc, 0xf9, 0x72, 0xcc, 0x66, 0xb7, 0xb3, 0x99, 0x34, 0xa6, + 0x7c, 0x27, 0x7a, 0x8c, 0x47, 0xd3, 0x05, 0x09, 0x44, 0x8c, 0x64, 0x74, 0xfb, 0x03, 0xf5, 0x96, + 0x48, 0x0c, 0xde, 0x45, 0x95, 0x31, 0x4c, 0xb3, 0x1d, 0x11, 0xf1, 0x89, 0x3f, 0x44, 0xb5, 0x89, + 0x78, 0x66, 0xd4, 0x91, 0x1c, 0xae, 0xef, 0x9b, 0x3f, 0x4b, 0x24, 0x2b, 0x7b, 0x50, 0xbe, 0xaf, + 0xd9, 0x87, 0xe7, 0x17, 0x7a, 0xe9, 0xf9, 0x85, 0x5e, 0x7a, 0x71, 0xa1, 0x97, 0x9e, 0xa5, 0xba, + 0x76, 0x9e, 0xea, 0xda, 0xf3, 0x54, 0xd7, 0x5e, 0xa4, 0xba, 0xf6, 0x67, 0xaa, 0x6b, 0x3f, 0xfc, + 0xa5, 0x97, 0xbe, 0x2a, 0x4f, 0x3a, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x5f, 0x04, 0x81, 0x6f, + 0xe2, 0x08, 0x00, 0x00, } diff --git a/staging/src/k8s.io/api/authentication/v1/generated.proto b/staging/src/k8s.io/api/authentication/v1/generated.proto index 10c792171d2..b69636a814d 100644 --- a/staging/src/k8s.io/api/authentication/v1/generated.proto +++ b/staging/src/k8s.io/api/authentication/v1/generated.proto @@ -118,6 +118,14 @@ message TokenReviewSpec { // Token is the opaque bearer token. // +optional optional string token = 1; + + // Audiences is a list of the identifiers that the resource server presented + // with the token identifies as. Audience-aware token authenticators will + // verify that the token was intended for at least one of the audiences in + // this list. If no audiences are provided, the audience will default to the + // audience of the Kubernetes apiserver. + // +optional + repeated string audiences = 2; } // TokenReviewStatus is the result of the token authentication request. @@ -130,6 +138,18 @@ message TokenReviewStatus { // +optional optional UserInfo user = 2; + // Audiences are audience identifiers chosen by the authenticator that are + // compatible with both the TokenReview and token. An identifier is any + // identifier in the intersection of the TokenReviewSpec audiences and the + // token's audiences. A client of the TokenReview API that sets the + // spec.audiences field should validate that a compatible audience identifier + // is returned in the status.audiences field to ensure that the TokenReview + // server is audience aware. If a TokenReview returns an empty + // status.audience field where status.authenticated is "true", the token is + // valid against the audience of the Kubernetes API server. + // +optional + repeated string audiences = 4; + // Error indicates that the token couldn't be checked // +optional optional string error = 3; diff --git a/staging/src/k8s.io/api/authentication/v1/types.go b/staging/src/k8s.io/api/authentication/v1/types.go index 723457a3dd2..d348c6fd405 100644 --- a/staging/src/k8s.io/api/authentication/v1/types.go +++ b/staging/src/k8s.io/api/authentication/v1/types.go @@ -64,6 +64,13 @@ type TokenReviewSpec struct { // Token is the opaque bearer token. // +optional Token string `json:"token,omitempty" protobuf:"bytes,1,opt,name=token"` + // Audiences is a list of the identifiers that the resource server presented + // with the token identifies as. Audience-aware token authenticators will + // verify that the token was intended for at least one of the audiences in + // this list. If no audiences are provided, the audience will default to the + // audience of the Kubernetes apiserver. + // +optional + Audiences []string `json:"audiences,omitempty" protobuf:"bytes,2,rep,name=audiences"` } // TokenReviewStatus is the result of the token authentication request. @@ -74,6 +81,17 @@ type TokenReviewStatus struct { // User is the UserInfo associated with the provided token. // +optional User UserInfo `json:"user,omitempty" protobuf:"bytes,2,opt,name=user"` + // Audiences are audience identifiers chosen by the authenticator that are + // compatible with both the TokenReview and token. An identifier is any + // identifier in the intersection of the TokenReviewSpec audiences and the + // token's audiences. A client of the TokenReview API that sets the + // spec.audiences field should validate that a compatible audience identifier + // is returned in the status.audiences field to ensure that the TokenReview + // server is audience aware. If a TokenReview returns an empty + // status.audience field where status.authenticated is "true", the token is + // valid against the audience of the Kubernetes API server. + // +optional + Audiences []string `json:"audiences,omitempty" protobuf:"bytes,4,rep,name=audiences"` // Error indicates that the token couldn't be checked // +optional Error string `json:"error,omitempty" protobuf:"bytes,3,opt,name=error"` diff --git a/staging/src/k8s.io/api/authentication/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/authentication/v1/types_swagger_doc_generated.go index 6632a5dd5b2..f2c9b95c71f 100644 --- a/staging/src/k8s.io/api/authentication/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/authentication/v1/types_swagger_doc_generated.go @@ -79,8 +79,9 @@ func (TokenReview) SwaggerDoc() map[string]string { } var map_TokenReviewSpec = map[string]string{ - "": "TokenReviewSpec is a description of the token authentication request.", - "token": "Token is the opaque bearer token.", + "": "TokenReviewSpec is a description of the token authentication request.", + "token": "Token is the opaque bearer token.", + "audiences": "Audiences is a list of the identifiers that the resource server presented with the token identifies as. Audience-aware token authenticators will verify that the token was intended for at least one of the audiences in this list. If no audiences are provided, the audience will default to the audience of the Kubernetes apiserver.", } func (TokenReviewSpec) SwaggerDoc() map[string]string { @@ -91,6 +92,7 @@ var map_TokenReviewStatus = map[string]string{ "": "TokenReviewStatus is the result of the token authentication request.", "authenticated": "Authenticated indicates that the token was associated with a known user.", "user": "User is the UserInfo associated with the provided token.", + "audiences": "Audiences are audience identifiers chosen by the authenticator that are compatible with both the TokenReview and token. An identifier is any identifier in the intersection of the TokenReviewSpec audiences and the token's audiences. A client of the TokenReview API that sets the spec.audiences field should validate that a compatible audience identifier is returned in the status.audiences field to ensure that the TokenReview server is audience aware. If a TokenReview returns an empty status.audience field where status.authenticated is \"true\", the token is valid against the audience of the Kubernetes API server.", "error": "Error indicates that the token couldn't be checked", } diff --git a/staging/src/k8s.io/api/authentication/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/authentication/v1/zz_generated.deepcopy.go index f36c253b2e7..aca99c42b76 100644 --- a/staging/src/k8s.io/api/authentication/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/authentication/v1/zz_generated.deepcopy.go @@ -141,7 +141,7 @@ func (in *TokenReview) DeepCopyInto(out *TokenReview) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) return } @@ -167,6 +167,11 @@ func (in *TokenReview) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TokenReviewSpec) DeepCopyInto(out *TokenReviewSpec) { *out = *in + if in.Audiences != nil { + in, out := &in.Audiences, &out.Audiences + *out = make([]string, len(*in)) + copy(*out, *in) + } return } @@ -184,6 +189,11 @@ func (in *TokenReviewSpec) DeepCopy() *TokenReviewSpec { func (in *TokenReviewStatus) DeepCopyInto(out *TokenReviewStatus) { *out = *in in.User.DeepCopyInto(&out.User) + if in.Audiences != nil { + in, out := &in.Audiences, &out.Audiences + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/staging/src/k8s.io/api/authentication/v1beta1/generated.pb.go b/staging/src/k8s.io/api/authentication/v1beta1/generated.pb.go index 55886b47818..5f34e76a9c3 100644 --- a/staging/src/k8s.io/api/authentication/v1beta1/generated.pb.go +++ b/staging/src/k8s.io/api/authentication/v1beta1/generated.pb.go @@ -175,6 +175,21 @@ func (m *TokenReviewSpec) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Token))) i += copy(dAtA[i:], m.Token) + if len(m.Audiences) > 0 { + for _, s := range m.Audiences { + dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } return i, nil } @@ -213,6 +228,21 @@ func (m *TokenReviewStatus) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Error))) i += copy(dAtA[i:], m.Error) + if len(m.Audiences) > 0 { + for _, s := range m.Audiences { + dAtA[i] = 0x22 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } return i, nil } @@ -326,6 +356,12 @@ func (m *TokenReviewSpec) Size() (n int) { _ = l l = len(m.Token) n += 1 + l + sovGenerated(uint64(l)) + if len(m.Audiences) > 0 { + for _, s := range m.Audiences { + l = len(s) + n += 1 + l + sovGenerated(uint64(l)) + } + } return n } @@ -337,6 +373,12 @@ func (m *TokenReviewStatus) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) l = len(m.Error) n += 1 + l + sovGenerated(uint64(l)) + if len(m.Audiences) > 0 { + for _, s := range m.Audiences { + l = len(s) + n += 1 + l + sovGenerated(uint64(l)) + } + } return n } @@ -396,6 +438,7 @@ func (this *TokenReviewSpec) String() string { } s := strings.Join([]string{`&TokenReviewSpec{`, `Token:` + fmt.Sprintf("%v", this.Token) + `,`, + `Audiences:` + fmt.Sprintf("%v", this.Audiences) + `,`, `}`, }, "") return s @@ -408,6 +451,7 @@ func (this *TokenReviewStatus) String() string { `Authenticated:` + fmt.Sprintf("%v", this.Authenticated) + `,`, `User:` + strings.Replace(strings.Replace(this.User.String(), "UserInfo", "UserInfo", 1), `&`, ``, 1) + `,`, `Error:` + fmt.Sprintf("%v", this.Error) + `,`, + `Audiences:` + fmt.Sprintf("%v", this.Audiences) + `,`, `}`, }, "") return s @@ -720,6 +764,35 @@ func (m *TokenReviewSpec) Unmarshal(dAtA []byte) error { } m.Token = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Audiences", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Audiences = append(m.Audiences, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -849,6 +922,35 @@ func (m *TokenReviewStatus) Unmarshal(dAtA []byte) error { } m.Error = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Audiences", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Audiences = append(m.Audiences, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -1240,45 +1342,47 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 635 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xcf, 0x4f, 0xd4, 0x40, - 0x14, 0x6e, 0xf7, 0x07, 0xee, 0xce, 0x8a, 0xe2, 0x24, 0x26, 0x9b, 0x4d, 0xec, 0xae, 0xeb, 0x85, - 0x44, 0x99, 0x0a, 0x21, 0x48, 0xf0, 0x64, 0x95, 0x18, 0x4c, 0x88, 0xc9, 0x08, 0x1e, 0xd4, 0x83, - 0xb3, 0xdd, 0x47, 0xb7, 0xae, 0xed, 0x34, 0xd3, 0x69, 0x95, 0x1b, 0x7f, 0x82, 0x47, 0x8f, 0x26, - 0xfe, 0x25, 0x26, 0x1e, 0x38, 0x72, 0xe4, 0x60, 0x88, 0xd4, 0x7f, 0xc4, 0xcc, 0x74, 0x64, 0x17, - 0x88, 0x01, 0x6e, 0xf3, 0xbe, 0xf7, 0xbe, 0x6f, 0xde, 0xf7, 0x66, 0x1e, 0x7a, 0x31, 0x5e, 0x4d, - 0x49, 0xc8, 0xdd, 0x71, 0x36, 0x00, 0x11, 0x83, 0x84, 0xd4, 0xcd, 0x21, 0x1e, 0x72, 0xe1, 0x9a, - 0x04, 0x4b, 0x42, 0x97, 0x65, 0x72, 0x04, 0xb1, 0x0c, 0x7d, 0x26, 0x43, 0x1e, 0xbb, 0xf9, 0xe2, - 0x00, 0x24, 0x5b, 0x74, 0x03, 0x88, 0x41, 0x30, 0x09, 0x43, 0x92, 0x08, 0x2e, 0x39, 0xbe, 0x5b, - 0x52, 0x08, 0x4b, 0x42, 0x72, 0x9a, 0x42, 0x0c, 0xa5, 0xb3, 0x10, 0x84, 0x72, 0x94, 0x0d, 0x88, - 0xcf, 0x23, 0x37, 0xe0, 0x01, 0x77, 0x35, 0x73, 0x90, 0xed, 0xe8, 0x48, 0x07, 0xfa, 0x54, 0x2a, - 0x76, 0x96, 0x27, 0x4d, 0x44, 0xcc, 0x1f, 0x85, 0x31, 0x88, 0x5d, 0x37, 0x19, 0x07, 0x0a, 0x48, - 0xdd, 0x08, 0x24, 0x73, 0xf3, 0x73, 0x7d, 0x74, 0xdc, 0xff, 0xb1, 0x44, 0x16, 0xcb, 0x30, 0x82, - 0x73, 0x84, 0x95, 0x8b, 0x08, 0xa9, 0x3f, 0x82, 0x88, 0x9d, 0xe5, 0xf5, 0x1f, 0x21, 0xb4, 0xfe, - 0x59, 0x0a, 0xf6, 0x9a, 0x7d, 0xcc, 0x00, 0x77, 0x51, 0x3d, 0x94, 0x10, 0xa5, 0x6d, 0xbb, 0x57, - 0x9d, 0x6f, 0x7a, 0xcd, 0xe2, 0xa8, 0x5b, 0xdf, 0x50, 0x00, 0x2d, 0xf1, 0xb5, 0xc6, 0xd7, 0x6f, - 0x5d, 0x6b, 0xef, 0x57, 0xcf, 0xea, 0x7f, 0xaf, 0xa0, 0xd6, 0x16, 0x1f, 0x43, 0x4c, 0x21, 0x0f, - 0xe1, 0x13, 0x7e, 0x8f, 0x1a, 0xca, 0xcc, 0x90, 0x49, 0xd6, 0xb6, 0x7b, 0xf6, 0x7c, 0x6b, 0xe9, - 0x21, 0x99, 0x0c, 0xf3, 0xa4, 0x27, 0x92, 0x8c, 0x03, 0x05, 0xa4, 0x44, 0x55, 0x93, 0x7c, 0x91, - 0xbc, 0x1c, 0x7c, 0x00, 0x5f, 0x6e, 0x82, 0x64, 0x1e, 0xde, 0x3f, 0xea, 0x5a, 0xc5, 0x51, 0x17, - 0x4d, 0x30, 0x7a, 0xa2, 0x8a, 0xb7, 0x50, 0x2d, 0x4d, 0xc0, 0x6f, 0x57, 0xb4, 0xfa, 0x12, 0xb9, - 0xf0, 0xa9, 0xc8, 0x54, 0x7f, 0xaf, 0x12, 0xf0, 0xbd, 0xeb, 0x46, 0xbf, 0xa6, 0x22, 0xaa, 0xd5, - 0xf0, 0x3b, 0x34, 0x93, 0x4a, 0x26, 0xb3, 0xb4, 0x5d, 0xd5, 0xba, 0xcb, 0x57, 0xd4, 0xd5, 0x5c, - 0xef, 0x86, 0x51, 0x9e, 0x29, 0x63, 0x6a, 0x34, 0xfb, 0x2b, 0xe8, 0xe6, 0x99, 0x26, 0xf0, 0x3d, - 0x54, 0x97, 0x0a, 0xd2, 0x53, 0x6a, 0x7a, 0xb3, 0x86, 0x59, 0x2f, 0xeb, 0xca, 0x5c, 0xff, 0xa7, - 0x8d, 0x6e, 0x9d, 0xbb, 0x05, 0x3f, 0x46, 0xb3, 0x53, 0x1d, 0xc1, 0x50, 0x4b, 0x34, 0xbc, 0xdb, - 0x46, 0x62, 0xf6, 0xc9, 0x74, 0x92, 0x9e, 0xae, 0xc5, 0x9b, 0xa8, 0x96, 0xa5, 0x20, 0xcc, 0xf8, - 0xee, 0x5f, 0xc2, 0xe6, 0x76, 0x0a, 0x62, 0x23, 0xde, 0xe1, 0x93, 0xb9, 0x29, 0x84, 0x6a, 0x19, - 0x65, 0x03, 0x84, 0xe0, 0x42, 0x8f, 0x6d, 0xca, 0xc6, 0xba, 0x02, 0x69, 0x99, 0xeb, 0xff, 0xa8, - 0xa0, 0xc6, 0x3f, 0x15, 0xfc, 0x00, 0x35, 0x14, 0x33, 0x66, 0x11, 0x18, 0xef, 0x73, 0x86, 0xa4, - 0x6b, 0x14, 0x4e, 0x4f, 0x2a, 0xf0, 0x1d, 0x54, 0xcd, 0xc2, 0xa1, 0xee, 0xb6, 0xe9, 0xb5, 0x4c, - 0x61, 0x75, 0x7b, 0xe3, 0x19, 0x55, 0x38, 0xee, 0xa3, 0x99, 0x40, 0xf0, 0x2c, 0x51, 0xcf, 0xa6, - 0xbe, 0x2a, 0x52, 0xc3, 0x7f, 0xae, 0x11, 0x6a, 0x32, 0xf8, 0x2d, 0xaa, 0x83, 0xfa, 0xdb, 0xed, - 0x5a, 0xaf, 0x3a, 0xdf, 0x5a, 0x5a, 0xb9, 0x82, 0x65, 0xa2, 0x97, 0x62, 0x3d, 0x96, 0x62, 0x77, - 0xca, 0x9a, 0xc2, 0x68, 0xa9, 0xd9, 0x09, 0xcc, 0xe2, 0xe8, 0x1a, 0x3c, 0x87, 0xaa, 0x63, 0xd8, - 0x2d, 0x6d, 0x51, 0x75, 0xc4, 0x4f, 0x51, 0x3d, 0x57, 0x3b, 0x65, 0xe6, 0xbd, 0x70, 0x89, 0xcb, - 0x27, 0x8b, 0x48, 0x4b, 0xee, 0x5a, 0x65, 0xd5, 0xf6, 0x16, 0xf6, 0x8f, 0x1d, 0xeb, 0xe0, 0xd8, - 0xb1, 0x0e, 0x8f, 0x1d, 0x6b, 0xaf, 0x70, 0xec, 0xfd, 0xc2, 0xb1, 0x0f, 0x0a, 0xc7, 0x3e, 0x2c, - 0x1c, 0xfb, 0x77, 0xe1, 0xd8, 0x5f, 0xfe, 0x38, 0xd6, 0x9b, 0x6b, 0x46, 0xe4, 0x6f, 0x00, 0x00, - 0x00, 0xff, 0xff, 0x39, 0x00, 0xe7, 0xfa, 0x0e, 0x05, 0x00, 0x00, + // 663 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xcd, 0x4e, 0x14, 0x4d, + 0x14, 0xed, 0x9e, 0x1f, 0xbe, 0x99, 0x9a, 0x6f, 0x14, 0x2b, 0x31, 0x99, 0x4c, 0x62, 0x0f, 0x8e, + 0x1b, 0x12, 0xa4, 0x5a, 0x08, 0x41, 0x82, 0x2b, 0x5a, 0x89, 0xc1, 0x84, 0x98, 0x94, 0xe0, 0x42, + 0x5d, 0x58, 0xd3, 0x73, 0xe9, 0x69, 0xc7, 0xfe, 0x49, 0x55, 0xf5, 0x28, 0x3b, 0x1e, 0xc1, 0xa5, + 0x4b, 0x13, 0x9f, 0xc4, 0x1d, 0x4b, 0x96, 0x2c, 0xcc, 0x44, 0xda, 0x27, 0xf0, 0x0d, 0x4c, 0x55, + 0x17, 0xcc, 0x00, 0x31, 0xc0, 0xae, 0xeb, 0xdc, 0x7b, 0xce, 0x3d, 0xf7, 0x54, 0x17, 0x7a, 0x31, + 0x5c, 0x13, 0x24, 0x4c, 0xdc, 0x61, 0xd6, 0x03, 0x1e, 0x83, 0x04, 0xe1, 0x8e, 0x20, 0xee, 0x27, + 0xdc, 0x35, 0x05, 0x96, 0x86, 0x2e, 0xcb, 0xe4, 0x00, 0x62, 0x19, 0xfa, 0x4c, 0x86, 0x49, 0xec, + 0x8e, 0x96, 0x7a, 0x20, 0xd9, 0x92, 0x1b, 0x40, 0x0c, 0x9c, 0x49, 0xe8, 0x93, 0x94, 0x27, 0x32, + 0xc1, 0xf7, 0x0b, 0x0a, 0x61, 0x69, 0x48, 0xce, 0x53, 0x88, 0xa1, 0xb4, 0x17, 0x83, 0x50, 0x0e, + 0xb2, 0x1e, 0xf1, 0x93, 0xc8, 0x0d, 0x92, 0x20, 0x71, 0x35, 0xb3, 0x97, 0xed, 0xe9, 0x93, 0x3e, + 0xe8, 0xaf, 0x42, 0xb1, 0xbd, 0x32, 0x31, 0x11, 0x31, 0x7f, 0x10, 0xc6, 0xc0, 0xf7, 0xdd, 0x74, + 0x18, 0x28, 0x40, 0xb8, 0x11, 0x48, 0xe6, 0x8e, 0x2e, 0xf9, 0x68, 0xbb, 0xff, 0x62, 0xf1, 0x2c, + 0x96, 0x61, 0x04, 0x97, 0x08, 0xab, 0x57, 0x11, 0x84, 0x3f, 0x80, 0x88, 0x5d, 0xe4, 0x75, 0x1f, + 0x23, 0xb4, 0xf9, 0x59, 0x72, 0xf6, 0x9a, 0x7d, 0xcc, 0x00, 0x77, 0x50, 0x35, 0x94, 0x10, 0x89, + 0x96, 0x3d, 0x57, 0x9e, 0xaf, 0x7b, 0xf5, 0x7c, 0xdc, 0xa9, 0x6e, 0x29, 0x80, 0x16, 0xf8, 0x7a, + 0xed, 0xeb, 0xb7, 0x8e, 0x75, 0xf0, 0x73, 0xce, 0xea, 0x7e, 0x2f, 0xa1, 0xc6, 0x4e, 0x32, 0x84, + 0x98, 0xc2, 0x28, 0x84, 0x4f, 0xf8, 0x3d, 0xaa, 0xa9, 0x65, 0xfa, 0x4c, 0xb2, 0x96, 0x3d, 0x67, + 0xcf, 0x37, 0x96, 0x1f, 0x91, 0x49, 0x98, 0x67, 0x9e, 0x48, 0x3a, 0x0c, 0x14, 0x20, 0x88, 0xea, + 0x26, 0xa3, 0x25, 0xf2, 0xb2, 0xf7, 0x01, 0x7c, 0xb9, 0x0d, 0x92, 0x79, 0xf8, 0x70, 0xdc, 0xb1, + 0xf2, 0x71, 0x07, 0x4d, 0x30, 0x7a, 0xa6, 0x8a, 0x77, 0x50, 0x45, 0xa4, 0xe0, 0xb7, 0x4a, 0x5a, + 0x7d, 0x99, 0x5c, 0x79, 0x55, 0x64, 0xca, 0xdf, 0xab, 0x14, 0x7c, 0xef, 0x7f, 0xa3, 0x5f, 0x51, + 0x27, 0xaa, 0xd5, 0xf0, 0x3b, 0x34, 0x23, 0x24, 0x93, 0x99, 0x68, 0x95, 0xb5, 0xee, 0xca, 0x0d, + 0x75, 0x35, 0xd7, 0xbb, 0x65, 0x94, 0x67, 0x8a, 0x33, 0x35, 0x9a, 0x5d, 0x1f, 0xdd, 0xbe, 0x60, + 0x02, 0x3f, 0x40, 0x55, 0xa9, 0x20, 0x9d, 0x52, 0xdd, 0x6b, 0x1a, 0x66, 0xb5, 0xe8, 0x2b, 0x6a, + 0x78, 0x01, 0xd5, 0x59, 0xd6, 0x0f, 0x21, 0xf6, 0x41, 0xb4, 0x4a, 0xfa, 0x32, 0x9a, 0xf9, 0xb8, + 0x53, 0xdf, 0x38, 0x05, 0xe9, 0xa4, 0xde, 0xfd, 0x63, 0xa3, 0x3b, 0x97, 0x2c, 0xe1, 0x27, 0xa8, + 0x39, 0x65, 0x1f, 0xfa, 0x7a, 0x5e, 0xcd, 0xbb, 0x6b, 0xe6, 0x35, 0x37, 0xa6, 0x8b, 0xf4, 0x7c, + 0x2f, 0xde, 0x46, 0x95, 0x4c, 0x00, 0x37, 0x59, 0x2f, 0x5c, 0x23, 0x93, 0x5d, 0x01, 0x7c, 0x2b, + 0xde, 0x4b, 0x26, 0x21, 0x2b, 0x84, 0x6a, 0x19, 0xb5, 0x33, 0x70, 0x9e, 0x70, 0x9d, 0xf1, 0xd4, + 0xce, 0x9b, 0x0a, 0xa4, 0x45, 0xed, 0xfc, 0xce, 0x95, 0x2b, 0x76, 0xfe, 0x51, 0x42, 0xb5, 0xd3, + 0x91, 0xf8, 0x21, 0xaa, 0xa9, 0x31, 0x31, 0x8b, 0xc0, 0xa4, 0x3a, 0x6b, 0x26, 0xe8, 0x1e, 0x85, + 0xd3, 0xb3, 0x0e, 0x7c, 0x0f, 0x95, 0xb3, 0xb0, 0xaf, 0x57, 0xab, 0x7b, 0x0d, 0xd3, 0x58, 0xde, + 0xdd, 0x7a, 0x46, 0x15, 0x8e, 0xbb, 0x68, 0x26, 0xe0, 0x49, 0x96, 0xaa, 0x1f, 0x42, 0x79, 0x40, + 0xea, 0x5a, 0x9f, 0x6b, 0x84, 0x9a, 0x0a, 0x7e, 0x8b, 0xaa, 0xa0, 0x5e, 0x8d, 0xb6, 0xd9, 0x58, + 0x5e, 0xbd, 0x41, 0x3e, 0x44, 0x3f, 0xb7, 0xcd, 0x58, 0xf2, 0xfd, 0xa9, 0x1c, 0x14, 0x46, 0x0b, + 0xcd, 0x76, 0x60, 0x9e, 0xa4, 0xee, 0xc1, 0xb3, 0xa8, 0x3c, 0x84, 0xfd, 0x62, 0x2d, 0xaa, 0x3e, + 0xf1, 0x53, 0x54, 0x1d, 0xa9, 0xd7, 0x6a, 0x2e, 0x67, 0xf1, 0x1a, 0xc3, 0x27, 0x4f, 0x9c, 0x16, + 0xdc, 0xf5, 0xd2, 0x9a, 0xed, 0x2d, 0x1e, 0x9e, 0x38, 0xd6, 0xd1, 0x89, 0x63, 0x1d, 0x9f, 0x38, + 0xd6, 0x41, 0xee, 0xd8, 0x87, 0xb9, 0x63, 0x1f, 0xe5, 0x8e, 0x7d, 0x9c, 0x3b, 0xf6, 0xaf, 0xdc, + 0xb1, 0xbf, 0xfc, 0x76, 0xac, 0x37, 0xff, 0x19, 0x91, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xf7, + 0xd6, 0x32, 0x28, 0x68, 0x05, 0x00, 0x00, } diff --git a/staging/src/k8s.io/api/authentication/v1beta1/generated.proto b/staging/src/k8s.io/api/authentication/v1beta1/generated.proto index a057bc591cf..caf2a6a53af 100644 --- a/staging/src/k8s.io/api/authentication/v1beta1/generated.proto +++ b/staging/src/k8s.io/api/authentication/v1beta1/generated.proto @@ -57,6 +57,14 @@ message TokenReviewSpec { // Token is the opaque bearer token. // +optional optional string token = 1; + + // Audiences is a list of the identifiers that the resource server presented + // with the token identifies as. Audience-aware token authenticators will + // verify that the token was intended for at least one of the audiences in + // this list. If no audiences are provided, the audience will default to the + // audience of the Kubernetes apiserver. + // +optional + repeated string audiences = 2; } // TokenReviewStatus is the result of the token authentication request. @@ -69,6 +77,18 @@ message TokenReviewStatus { // +optional optional UserInfo user = 2; + // Audiences are audience identifiers chosen by the authenticator that are + // compatible with both the TokenReview and token. An identifier is any + // identifier in the intersection of the TokenReviewSpec audiences and the + // token's audiences. A client of the TokenReview API that sets the + // spec.audiences field should validate that a compatible audience identifier + // is returned in the status.audiences field to ensure that the TokenReview + // server is audience aware. If a TokenReview returns an empty + // status.audience field where status.authenticated is "true", the token is + // valid against the audience of the Kubernetes API server. + // +optional + repeated string audiences = 4; + // Error indicates that the token couldn't be checked // +optional optional string error = 3; diff --git a/staging/src/k8s.io/api/authentication/v1beta1/types.go b/staging/src/k8s.io/api/authentication/v1beta1/types.go index a90949dc37d..0b6cba822a2 100644 --- a/staging/src/k8s.io/api/authentication/v1beta1/types.go +++ b/staging/src/k8s.io/api/authentication/v1beta1/types.go @@ -48,6 +48,13 @@ type TokenReviewSpec struct { // Token is the opaque bearer token. // +optional Token string `json:"token,omitempty" protobuf:"bytes,1,opt,name=token"` + // Audiences is a list of the identifiers that the resource server presented + // with the token identifies as. Audience-aware token authenticators will + // verify that the token was intended for at least one of the audiences in + // this list. If no audiences are provided, the audience will default to the + // audience of the Kubernetes apiserver. + // +optional + Audiences []string `json:"audiences,omitempty" protobuf:"bytes,2,rep,name=audiences"` } // TokenReviewStatus is the result of the token authentication request. @@ -58,6 +65,17 @@ type TokenReviewStatus struct { // User is the UserInfo associated with the provided token. // +optional User UserInfo `json:"user,omitempty" protobuf:"bytes,2,opt,name=user"` + // Audiences are audience identifiers chosen by the authenticator that are + // compatible with both the TokenReview and token. An identifier is any + // identifier in the intersection of the TokenReviewSpec audiences and the + // token's audiences. A client of the TokenReview API that sets the + // spec.audiences field should validate that a compatible audience identifier + // is returned in the status.audiences field to ensure that the TokenReview + // server is audience aware. If a TokenReview returns an empty + // status.audience field where status.authenticated is "true", the token is + // valid against the audience of the Kubernetes API server. + // +optional + Audiences []string `json:"audiences,omitempty" protobuf:"bytes,4,rep,name=audiences"` // Error indicates that the token couldn't be checked // +optional Error string `json:"error,omitempty" protobuf:"bytes,3,opt,name=error"` diff --git a/staging/src/k8s.io/api/authentication/v1beta1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/authentication/v1beta1/types_swagger_doc_generated.go index 968999d1ebe..8c9acfb5b24 100644 --- a/staging/src/k8s.io/api/authentication/v1beta1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/authentication/v1beta1/types_swagger_doc_generated.go @@ -38,8 +38,9 @@ func (TokenReview) SwaggerDoc() map[string]string { } var map_TokenReviewSpec = map[string]string{ - "": "TokenReviewSpec is a description of the token authentication request.", - "token": "Token is the opaque bearer token.", + "": "TokenReviewSpec is a description of the token authentication request.", + "token": "Token is the opaque bearer token.", + "audiences": "Audiences is a list of the identifiers that the resource server presented with the token identifies as. Audience-aware token authenticators will verify that the token was intended for at least one of the audiences in this list. If no audiences are provided, the audience will default to the audience of the Kubernetes apiserver.", } func (TokenReviewSpec) SwaggerDoc() map[string]string { @@ -50,6 +51,7 @@ var map_TokenReviewStatus = map[string]string{ "": "TokenReviewStatus is the result of the token authentication request.", "authenticated": "Authenticated indicates that the token was associated with a known user.", "user": "User is the UserInfo associated with the provided token.", + "audiences": "Audiences are audience identifiers chosen by the authenticator that are compatible with both the TokenReview and token. An identifier is any identifier in the intersection of the TokenReviewSpec audiences and the token's audiences. A client of the TokenReview API that sets the spec.audiences field should validate that a compatible audience identifier is returned in the status.audiences field to ensure that the TokenReview server is audience aware. If a TokenReview returns an empty status.audience field where status.authenticated is \"true\", the token is valid against the audience of the Kubernetes API server.", "error": "Error indicates that the token couldn't be checked", } diff --git a/staging/src/k8s.io/api/authentication/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/authentication/v1beta1/zz_generated.deepcopy.go index 3a5f6d5a936..a5d82a8100a 100644 --- a/staging/src/k8s.io/api/authentication/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/authentication/v1beta1/zz_generated.deepcopy.go @@ -49,7 +49,7 @@ func (in *TokenReview) DeepCopyInto(out *TokenReview) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) return } @@ -75,6 +75,11 @@ func (in *TokenReview) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TokenReviewSpec) DeepCopyInto(out *TokenReviewSpec) { *out = *in + if in.Audiences != nil { + in, out := &in.Audiences, &out.Audiences + *out = make([]string, len(*in)) + copy(*out, *in) + } return } @@ -92,6 +97,11 @@ func (in *TokenReviewSpec) DeepCopy() *TokenReviewSpec { func (in *TokenReviewStatus) DeepCopyInto(out *TokenReviewStatus) { *out = *in in.User.DeepCopyInto(&out.User) + if in.Audiences != nil { + in, out := &in.Audiences, &out.Audiences + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/staging/src/k8s.io/api/authorization/OWNERS b/staging/src/k8s.io/api/authorization/OWNERS index c1613fc2e02..ff4a7f4bf9a 100755 --- a/staging/src/k8s.io/api/authorization/OWNERS +++ b/staging/src/k8s.io/api/authorization/OWNERS @@ -1,17 +1,7 @@ +# approval on api packages bubbles to api-approvers reviewers: -- thockin -- lavalamp -- smarterclayton -- wojtek-t -- deads2k -- liggitt -- nikhiljindal -- erictune -- sttts -- ncdc -- dims -- mml -- mbohlool -- david-mcmahon -- jianhuiz -- enj +- sig-auth-authorizers-approvers +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/api/core/v1/generated.pb.go b/staging/src/k8s.io/api/core/v1/generated.pb.go index 80e8aad5f36..05cc6d62844 100644 --- a/staging/src/k8s.io/api/core/v1/generated.pb.go +++ b/staging/src/k8s.io/api/core/v1/generated.pb.go @@ -81,6 +81,7 @@ limitations under the License. FlockerVolumeSource GCEPersistentDiskVolumeSource GitRepoVolumeSource + GlusterfsPersistentVolumeSource GlusterfsVolumeSource HTTPGetAction HTTPHeader @@ -498,604 +499,610 @@ func (m *GitRepoVolumeSource) Reset() { *m = GitRepoVolumeSou func (*GitRepoVolumeSource) ProtoMessage() {} func (*GitRepoVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{56} } +func (m *GlusterfsPersistentVolumeSource) Reset() { *m = GlusterfsPersistentVolumeSource{} } +func (*GlusterfsPersistentVolumeSource) ProtoMessage() {} +func (*GlusterfsPersistentVolumeSource) Descriptor() ([]byte, []int) { + return fileDescriptorGenerated, []int{57} +} + func (m *GlusterfsVolumeSource) Reset() { *m = GlusterfsVolumeSource{} } func (*GlusterfsVolumeSource) ProtoMessage() {} -func (*GlusterfsVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{57} } +func (*GlusterfsVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{58} } func (m *HTTPGetAction) Reset() { *m = HTTPGetAction{} } func (*HTTPGetAction) ProtoMessage() {} -func (*HTTPGetAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{58} } +func (*HTTPGetAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{59} } func (m *HTTPHeader) Reset() { *m = HTTPHeader{} } func (*HTTPHeader) ProtoMessage() {} -func (*HTTPHeader) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{59} } +func (*HTTPHeader) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{60} } func (m *Handler) Reset() { *m = Handler{} } func (*Handler) ProtoMessage() {} -func (*Handler) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{60} } +func (*Handler) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{61} } func (m *HostAlias) Reset() { *m = HostAlias{} } func (*HostAlias) ProtoMessage() {} -func (*HostAlias) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{61} } +func (*HostAlias) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{62} } func (m *HostPathVolumeSource) Reset() { *m = HostPathVolumeSource{} } func (*HostPathVolumeSource) ProtoMessage() {} -func (*HostPathVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{62} } +func (*HostPathVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{63} } func (m *ISCSIPersistentVolumeSource) Reset() { *m = ISCSIPersistentVolumeSource{} } func (*ISCSIPersistentVolumeSource) ProtoMessage() {} func (*ISCSIPersistentVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{63} + return fileDescriptorGenerated, []int{64} } func (m *ISCSIVolumeSource) Reset() { *m = ISCSIVolumeSource{} } func (*ISCSIVolumeSource) ProtoMessage() {} -func (*ISCSIVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{64} } +func (*ISCSIVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{65} } func (m *KeyToPath) Reset() { *m = KeyToPath{} } func (*KeyToPath) ProtoMessage() {} -func (*KeyToPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{65} } +func (*KeyToPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{66} } func (m *Lifecycle) Reset() { *m = Lifecycle{} } func (*Lifecycle) ProtoMessage() {} -func (*Lifecycle) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{66} } +func (*Lifecycle) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{67} } func (m *LimitRange) Reset() { *m = LimitRange{} } func (*LimitRange) ProtoMessage() {} -func (*LimitRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{67} } +func (*LimitRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{68} } func (m *LimitRangeItem) Reset() { *m = LimitRangeItem{} } func (*LimitRangeItem) ProtoMessage() {} -func (*LimitRangeItem) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{68} } +func (*LimitRangeItem) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{69} } func (m *LimitRangeList) Reset() { *m = LimitRangeList{} } func (*LimitRangeList) ProtoMessage() {} -func (*LimitRangeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{69} } +func (*LimitRangeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{70} } func (m *LimitRangeSpec) Reset() { *m = LimitRangeSpec{} } func (*LimitRangeSpec) ProtoMessage() {} -func (*LimitRangeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{70} } +func (*LimitRangeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{71} } func (m *List) Reset() { *m = List{} } func (*List) ProtoMessage() {} -func (*List) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{71} } +func (*List) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{72} } func (m *LoadBalancerIngress) Reset() { *m = LoadBalancerIngress{} } func (*LoadBalancerIngress) ProtoMessage() {} -func (*LoadBalancerIngress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{72} } +func (*LoadBalancerIngress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{73} } func (m *LoadBalancerStatus) Reset() { *m = LoadBalancerStatus{} } func (*LoadBalancerStatus) ProtoMessage() {} -func (*LoadBalancerStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{73} } +func (*LoadBalancerStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{74} } func (m *LocalObjectReference) Reset() { *m = LocalObjectReference{} } func (*LocalObjectReference) ProtoMessage() {} -func (*LocalObjectReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{74} } +func (*LocalObjectReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{75} } func (m *LocalVolumeSource) Reset() { *m = LocalVolumeSource{} } func (*LocalVolumeSource) ProtoMessage() {} -func (*LocalVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{75} } +func (*LocalVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{76} } func (m *NFSVolumeSource) Reset() { *m = NFSVolumeSource{} } func (*NFSVolumeSource) ProtoMessage() {} -func (*NFSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{76} } +func (*NFSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{77} } func (m *Namespace) Reset() { *m = Namespace{} } func (*Namespace) ProtoMessage() {} -func (*Namespace) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{77} } +func (*Namespace) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{78} } func (m *NamespaceList) Reset() { *m = NamespaceList{} } func (*NamespaceList) ProtoMessage() {} -func (*NamespaceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{78} } +func (*NamespaceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{79} } func (m *NamespaceSpec) Reset() { *m = NamespaceSpec{} } func (*NamespaceSpec) ProtoMessage() {} -func (*NamespaceSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{79} } +func (*NamespaceSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{80} } func (m *NamespaceStatus) Reset() { *m = NamespaceStatus{} } func (*NamespaceStatus) ProtoMessage() {} -func (*NamespaceStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{80} } +func (*NamespaceStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{81} } func (m *Node) Reset() { *m = Node{} } func (*Node) ProtoMessage() {} -func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{81} } +func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{82} } func (m *NodeAddress) Reset() { *m = NodeAddress{} } func (*NodeAddress) ProtoMessage() {} -func (*NodeAddress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{82} } +func (*NodeAddress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{83} } func (m *NodeAffinity) Reset() { *m = NodeAffinity{} } func (*NodeAffinity) ProtoMessage() {} -func (*NodeAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{83} } +func (*NodeAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{84} } func (m *NodeCondition) Reset() { *m = NodeCondition{} } func (*NodeCondition) ProtoMessage() {} -func (*NodeCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{84} } +func (*NodeCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{85} } func (m *NodeConfigSource) Reset() { *m = NodeConfigSource{} } func (*NodeConfigSource) ProtoMessage() {} -func (*NodeConfigSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{85} } +func (*NodeConfigSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{86} } func (m *NodeConfigStatus) Reset() { *m = NodeConfigStatus{} } func (*NodeConfigStatus) ProtoMessage() {} -func (*NodeConfigStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{86} } +func (*NodeConfigStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{87} } func (m *NodeDaemonEndpoints) Reset() { *m = NodeDaemonEndpoints{} } func (*NodeDaemonEndpoints) ProtoMessage() {} -func (*NodeDaemonEndpoints) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{87} } +func (*NodeDaemonEndpoints) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{88} } func (m *NodeList) Reset() { *m = NodeList{} } func (*NodeList) ProtoMessage() {} -func (*NodeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{88} } +func (*NodeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{89} } func (m *NodeProxyOptions) Reset() { *m = NodeProxyOptions{} } func (*NodeProxyOptions) ProtoMessage() {} -func (*NodeProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{89} } +func (*NodeProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{90} } func (m *NodeResources) Reset() { *m = NodeResources{} } func (*NodeResources) ProtoMessage() {} -func (*NodeResources) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{90} } +func (*NodeResources) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{91} } func (m *NodeSelector) Reset() { *m = NodeSelector{} } func (*NodeSelector) ProtoMessage() {} -func (*NodeSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{91} } +func (*NodeSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{92} } func (m *NodeSelectorRequirement) Reset() { *m = NodeSelectorRequirement{} } func (*NodeSelectorRequirement) ProtoMessage() {} func (*NodeSelectorRequirement) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{92} + return fileDescriptorGenerated, []int{93} } func (m *NodeSelectorTerm) Reset() { *m = NodeSelectorTerm{} } func (*NodeSelectorTerm) ProtoMessage() {} -func (*NodeSelectorTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{93} } +func (*NodeSelectorTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{94} } func (m *NodeSpec) Reset() { *m = NodeSpec{} } func (*NodeSpec) ProtoMessage() {} -func (*NodeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{94} } +func (*NodeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{95} } func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{95} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{96} } func (m *NodeSystemInfo) Reset() { *m = NodeSystemInfo{} } func (*NodeSystemInfo) ProtoMessage() {} -func (*NodeSystemInfo) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{96} } +func (*NodeSystemInfo) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{97} } func (m *ObjectFieldSelector) Reset() { *m = ObjectFieldSelector{} } func (*ObjectFieldSelector) ProtoMessage() {} -func (*ObjectFieldSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{97} } +func (*ObjectFieldSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{98} } func (m *ObjectReference) Reset() { *m = ObjectReference{} } func (*ObjectReference) ProtoMessage() {} -func (*ObjectReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{98} } +func (*ObjectReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{99} } func (m *PersistentVolume) Reset() { *m = PersistentVolume{} } func (*PersistentVolume) ProtoMessage() {} -func (*PersistentVolume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{99} } +func (*PersistentVolume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{100} } func (m *PersistentVolumeClaim) Reset() { *m = PersistentVolumeClaim{} } func (*PersistentVolumeClaim) ProtoMessage() {} -func (*PersistentVolumeClaim) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{100} } +func (*PersistentVolumeClaim) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{101} } func (m *PersistentVolumeClaimCondition) Reset() { *m = PersistentVolumeClaimCondition{} } func (*PersistentVolumeClaimCondition) ProtoMessage() {} func (*PersistentVolumeClaimCondition) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{101} + return fileDescriptorGenerated, []int{102} } func (m *PersistentVolumeClaimList) Reset() { *m = PersistentVolumeClaimList{} } func (*PersistentVolumeClaimList) ProtoMessage() {} func (*PersistentVolumeClaimList) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{102} + return fileDescriptorGenerated, []int{103} } func (m *PersistentVolumeClaimSpec) Reset() { *m = PersistentVolumeClaimSpec{} } func (*PersistentVolumeClaimSpec) ProtoMessage() {} func (*PersistentVolumeClaimSpec) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{103} + return fileDescriptorGenerated, []int{104} } func (m *PersistentVolumeClaimStatus) Reset() { *m = PersistentVolumeClaimStatus{} } func (*PersistentVolumeClaimStatus) ProtoMessage() {} func (*PersistentVolumeClaimStatus) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{104} + return fileDescriptorGenerated, []int{105} } func (m *PersistentVolumeClaimVolumeSource) Reset() { *m = PersistentVolumeClaimVolumeSource{} } func (*PersistentVolumeClaimVolumeSource) ProtoMessage() {} func (*PersistentVolumeClaimVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{105} + return fileDescriptorGenerated, []int{106} } func (m *PersistentVolumeList) Reset() { *m = PersistentVolumeList{} } func (*PersistentVolumeList) ProtoMessage() {} -func (*PersistentVolumeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{106} } +func (*PersistentVolumeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{107} } func (m *PersistentVolumeSource) Reset() { *m = PersistentVolumeSource{} } func (*PersistentVolumeSource) ProtoMessage() {} func (*PersistentVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{107} + return fileDescriptorGenerated, []int{108} } func (m *PersistentVolumeSpec) Reset() { *m = PersistentVolumeSpec{} } func (*PersistentVolumeSpec) ProtoMessage() {} -func (*PersistentVolumeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{108} } +func (*PersistentVolumeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{109} } func (m *PersistentVolumeStatus) Reset() { *m = PersistentVolumeStatus{} } func (*PersistentVolumeStatus) ProtoMessage() {} func (*PersistentVolumeStatus) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{109} + return fileDescriptorGenerated, []int{110} } func (m *PhotonPersistentDiskVolumeSource) Reset() { *m = PhotonPersistentDiskVolumeSource{} } func (*PhotonPersistentDiskVolumeSource) ProtoMessage() {} func (*PhotonPersistentDiskVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{110} + return fileDescriptorGenerated, []int{111} } func (m *Pod) Reset() { *m = Pod{} } func (*Pod) ProtoMessage() {} -func (*Pod) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{111} } +func (*Pod) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{112} } func (m *PodAffinity) Reset() { *m = PodAffinity{} } func (*PodAffinity) ProtoMessage() {} -func (*PodAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{112} } +func (*PodAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{113} } func (m *PodAffinityTerm) Reset() { *m = PodAffinityTerm{} } func (*PodAffinityTerm) ProtoMessage() {} -func (*PodAffinityTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{113} } +func (*PodAffinityTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{114} } func (m *PodAntiAffinity) Reset() { *m = PodAntiAffinity{} } func (*PodAntiAffinity) ProtoMessage() {} -func (*PodAntiAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{114} } +func (*PodAntiAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{115} } func (m *PodAttachOptions) Reset() { *m = PodAttachOptions{} } func (*PodAttachOptions) ProtoMessage() {} -func (*PodAttachOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{115} } +func (*PodAttachOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{116} } func (m *PodCondition) Reset() { *m = PodCondition{} } func (*PodCondition) ProtoMessage() {} -func (*PodCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{116} } +func (*PodCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{117} } func (m *PodDNSConfig) Reset() { *m = PodDNSConfig{} } func (*PodDNSConfig) ProtoMessage() {} -func (*PodDNSConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{117} } +func (*PodDNSConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{118} } func (m *PodDNSConfigOption) Reset() { *m = PodDNSConfigOption{} } func (*PodDNSConfigOption) ProtoMessage() {} -func (*PodDNSConfigOption) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{118} } +func (*PodDNSConfigOption) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{119} } func (m *PodExecOptions) Reset() { *m = PodExecOptions{} } func (*PodExecOptions) ProtoMessage() {} -func (*PodExecOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{119} } +func (*PodExecOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{120} } func (m *PodList) Reset() { *m = PodList{} } func (*PodList) ProtoMessage() {} -func (*PodList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{120} } +func (*PodList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{121} } func (m *PodLogOptions) Reset() { *m = PodLogOptions{} } func (*PodLogOptions) ProtoMessage() {} -func (*PodLogOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{121} } +func (*PodLogOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{122} } func (m *PodPortForwardOptions) Reset() { *m = PodPortForwardOptions{} } func (*PodPortForwardOptions) ProtoMessage() {} -func (*PodPortForwardOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{122} } +func (*PodPortForwardOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{123} } func (m *PodProxyOptions) Reset() { *m = PodProxyOptions{} } func (*PodProxyOptions) ProtoMessage() {} -func (*PodProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{123} } +func (*PodProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{124} } func (m *PodReadinessGate) Reset() { *m = PodReadinessGate{} } func (*PodReadinessGate) ProtoMessage() {} -func (*PodReadinessGate) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{124} } +func (*PodReadinessGate) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{125} } func (m *PodSecurityContext) Reset() { *m = PodSecurityContext{} } func (*PodSecurityContext) ProtoMessage() {} -func (*PodSecurityContext) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{125} } +func (*PodSecurityContext) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{126} } func (m *PodSignature) Reset() { *m = PodSignature{} } func (*PodSignature) ProtoMessage() {} -func (*PodSignature) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{126} } +func (*PodSignature) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{127} } func (m *PodSpec) Reset() { *m = PodSpec{} } func (*PodSpec) ProtoMessage() {} -func (*PodSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{127} } +func (*PodSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{128} } func (m *PodStatus) Reset() { *m = PodStatus{} } func (*PodStatus) ProtoMessage() {} -func (*PodStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{128} } +func (*PodStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{129} } func (m *PodStatusResult) Reset() { *m = PodStatusResult{} } func (*PodStatusResult) ProtoMessage() {} -func (*PodStatusResult) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{129} } +func (*PodStatusResult) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{130} } func (m *PodTemplate) Reset() { *m = PodTemplate{} } func (*PodTemplate) ProtoMessage() {} -func (*PodTemplate) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{130} } +func (*PodTemplate) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{131} } func (m *PodTemplateList) Reset() { *m = PodTemplateList{} } func (*PodTemplateList) ProtoMessage() {} -func (*PodTemplateList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{131} } +func (*PodTemplateList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{132} } func (m *PodTemplateSpec) Reset() { *m = PodTemplateSpec{} } func (*PodTemplateSpec) ProtoMessage() {} -func (*PodTemplateSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{132} } +func (*PodTemplateSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{133} } func (m *PortworxVolumeSource) Reset() { *m = PortworxVolumeSource{} } func (*PortworxVolumeSource) ProtoMessage() {} -func (*PortworxVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{133} } +func (*PortworxVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{134} } func (m *Preconditions) Reset() { *m = Preconditions{} } func (*Preconditions) ProtoMessage() {} -func (*Preconditions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{134} } +func (*Preconditions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{135} } func (m *PreferAvoidPodsEntry) Reset() { *m = PreferAvoidPodsEntry{} } func (*PreferAvoidPodsEntry) ProtoMessage() {} -func (*PreferAvoidPodsEntry) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{135} } +func (*PreferAvoidPodsEntry) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{136} } func (m *PreferredSchedulingTerm) Reset() { *m = PreferredSchedulingTerm{} } func (*PreferredSchedulingTerm) ProtoMessage() {} func (*PreferredSchedulingTerm) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{136} + return fileDescriptorGenerated, []int{137} } func (m *Probe) Reset() { *m = Probe{} } func (*Probe) ProtoMessage() {} -func (*Probe) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{137} } +func (*Probe) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{138} } func (m *ProjectedVolumeSource) Reset() { *m = ProjectedVolumeSource{} } func (*ProjectedVolumeSource) ProtoMessage() {} -func (*ProjectedVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{138} } +func (*ProjectedVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{139} } func (m *QuobyteVolumeSource) Reset() { *m = QuobyteVolumeSource{} } func (*QuobyteVolumeSource) ProtoMessage() {} -func (*QuobyteVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{139} } +func (*QuobyteVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{140} } func (m *RBDPersistentVolumeSource) Reset() { *m = RBDPersistentVolumeSource{} } func (*RBDPersistentVolumeSource) ProtoMessage() {} func (*RBDPersistentVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{140} + return fileDescriptorGenerated, []int{141} } func (m *RBDVolumeSource) Reset() { *m = RBDVolumeSource{} } func (*RBDVolumeSource) ProtoMessage() {} -func (*RBDVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{141} } +func (*RBDVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{142} } func (m *RangeAllocation) Reset() { *m = RangeAllocation{} } func (*RangeAllocation) ProtoMessage() {} -func (*RangeAllocation) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{142} } +func (*RangeAllocation) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{143} } func (m *ReplicationController) Reset() { *m = ReplicationController{} } func (*ReplicationController) ProtoMessage() {} -func (*ReplicationController) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{143} } +func (*ReplicationController) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{144} } func (m *ReplicationControllerCondition) Reset() { *m = ReplicationControllerCondition{} } func (*ReplicationControllerCondition) ProtoMessage() {} func (*ReplicationControllerCondition) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{144} + return fileDescriptorGenerated, []int{145} } func (m *ReplicationControllerList) Reset() { *m = ReplicationControllerList{} } func (*ReplicationControllerList) ProtoMessage() {} func (*ReplicationControllerList) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{145} + return fileDescriptorGenerated, []int{146} } func (m *ReplicationControllerSpec) Reset() { *m = ReplicationControllerSpec{} } func (*ReplicationControllerSpec) ProtoMessage() {} func (*ReplicationControllerSpec) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{146} + return fileDescriptorGenerated, []int{147} } func (m *ReplicationControllerStatus) Reset() { *m = ReplicationControllerStatus{} } func (*ReplicationControllerStatus) ProtoMessage() {} func (*ReplicationControllerStatus) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{147} + return fileDescriptorGenerated, []int{148} } func (m *ResourceFieldSelector) Reset() { *m = ResourceFieldSelector{} } func (*ResourceFieldSelector) ProtoMessage() {} -func (*ResourceFieldSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{148} } +func (*ResourceFieldSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{149} } func (m *ResourceQuota) Reset() { *m = ResourceQuota{} } func (*ResourceQuota) ProtoMessage() {} -func (*ResourceQuota) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{149} } +func (*ResourceQuota) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{150} } func (m *ResourceQuotaList) Reset() { *m = ResourceQuotaList{} } func (*ResourceQuotaList) ProtoMessage() {} -func (*ResourceQuotaList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{150} } +func (*ResourceQuotaList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{151} } func (m *ResourceQuotaSpec) Reset() { *m = ResourceQuotaSpec{} } func (*ResourceQuotaSpec) ProtoMessage() {} -func (*ResourceQuotaSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{151} } +func (*ResourceQuotaSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{152} } func (m *ResourceQuotaStatus) Reset() { *m = ResourceQuotaStatus{} } func (*ResourceQuotaStatus) ProtoMessage() {} -func (*ResourceQuotaStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{152} } +func (*ResourceQuotaStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{153} } func (m *ResourceRequirements) Reset() { *m = ResourceRequirements{} } func (*ResourceRequirements) ProtoMessage() {} -func (*ResourceRequirements) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{153} } +func (*ResourceRequirements) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{154} } func (m *SELinuxOptions) Reset() { *m = SELinuxOptions{} } func (*SELinuxOptions) ProtoMessage() {} -func (*SELinuxOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{154} } +func (*SELinuxOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{155} } func (m *ScaleIOPersistentVolumeSource) Reset() { *m = ScaleIOPersistentVolumeSource{} } func (*ScaleIOPersistentVolumeSource) ProtoMessage() {} func (*ScaleIOPersistentVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{155} + return fileDescriptorGenerated, []int{156} } func (m *ScaleIOVolumeSource) Reset() { *m = ScaleIOVolumeSource{} } func (*ScaleIOVolumeSource) ProtoMessage() {} -func (*ScaleIOVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{156} } +func (*ScaleIOVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{157} } func (m *ScopeSelector) Reset() { *m = ScopeSelector{} } func (*ScopeSelector) ProtoMessage() {} -func (*ScopeSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{157} } +func (*ScopeSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{158} } func (m *ScopedResourceSelectorRequirement) Reset() { *m = ScopedResourceSelectorRequirement{} } func (*ScopedResourceSelectorRequirement) ProtoMessage() {} func (*ScopedResourceSelectorRequirement) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{158} + return fileDescriptorGenerated, []int{159} } func (m *Secret) Reset() { *m = Secret{} } func (*Secret) ProtoMessage() {} -func (*Secret) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{159} } +func (*Secret) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{160} } func (m *SecretEnvSource) Reset() { *m = SecretEnvSource{} } func (*SecretEnvSource) ProtoMessage() {} -func (*SecretEnvSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{160} } +func (*SecretEnvSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{161} } func (m *SecretKeySelector) Reset() { *m = SecretKeySelector{} } func (*SecretKeySelector) ProtoMessage() {} -func (*SecretKeySelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{161} } +func (*SecretKeySelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{162} } func (m *SecretList) Reset() { *m = SecretList{} } func (*SecretList) ProtoMessage() {} -func (*SecretList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{162} } +func (*SecretList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{163} } func (m *SecretProjection) Reset() { *m = SecretProjection{} } func (*SecretProjection) ProtoMessage() {} -func (*SecretProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{163} } +func (*SecretProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{164} } func (m *SecretReference) Reset() { *m = SecretReference{} } func (*SecretReference) ProtoMessage() {} -func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{164} } +func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{165} } func (m *SecretVolumeSource) Reset() { *m = SecretVolumeSource{} } func (*SecretVolumeSource) ProtoMessage() {} -func (*SecretVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{165} } +func (*SecretVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{166} } func (m *SecurityContext) Reset() { *m = SecurityContext{} } func (*SecurityContext) ProtoMessage() {} -func (*SecurityContext) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{166} } +func (*SecurityContext) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{167} } func (m *SerializedReference) Reset() { *m = SerializedReference{} } func (*SerializedReference) ProtoMessage() {} -func (*SerializedReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{167} } +func (*SerializedReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{168} } func (m *Service) Reset() { *m = Service{} } func (*Service) ProtoMessage() {} -func (*Service) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{168} } +func (*Service) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{169} } func (m *ServiceAccount) Reset() { *m = ServiceAccount{} } func (*ServiceAccount) ProtoMessage() {} -func (*ServiceAccount) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{169} } +func (*ServiceAccount) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{170} } func (m *ServiceAccountList) Reset() { *m = ServiceAccountList{} } func (*ServiceAccountList) ProtoMessage() {} -func (*ServiceAccountList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{170} } +func (*ServiceAccountList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{171} } func (m *ServiceAccountTokenProjection) Reset() { *m = ServiceAccountTokenProjection{} } func (*ServiceAccountTokenProjection) ProtoMessage() {} func (*ServiceAccountTokenProjection) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{171} + return fileDescriptorGenerated, []int{172} } func (m *ServiceList) Reset() { *m = ServiceList{} } func (*ServiceList) ProtoMessage() {} -func (*ServiceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{172} } +func (*ServiceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{173} } func (m *ServicePort) Reset() { *m = ServicePort{} } func (*ServicePort) ProtoMessage() {} -func (*ServicePort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{173} } +func (*ServicePort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{174} } func (m *ServiceProxyOptions) Reset() { *m = ServiceProxyOptions{} } func (*ServiceProxyOptions) ProtoMessage() {} -func (*ServiceProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{174} } +func (*ServiceProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{175} } func (m *ServiceSpec) Reset() { *m = ServiceSpec{} } func (*ServiceSpec) ProtoMessage() {} -func (*ServiceSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{175} } +func (*ServiceSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{176} } func (m *ServiceStatus) Reset() { *m = ServiceStatus{} } func (*ServiceStatus) ProtoMessage() {} -func (*ServiceStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{176} } +func (*ServiceStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{177} } func (m *SessionAffinityConfig) Reset() { *m = SessionAffinityConfig{} } func (*SessionAffinityConfig) ProtoMessage() {} -func (*SessionAffinityConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{177} } +func (*SessionAffinityConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{178} } func (m *StorageOSPersistentVolumeSource) Reset() { *m = StorageOSPersistentVolumeSource{} } func (*StorageOSPersistentVolumeSource) ProtoMessage() {} func (*StorageOSPersistentVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{178} + return fileDescriptorGenerated, []int{179} } func (m *StorageOSVolumeSource) Reset() { *m = StorageOSVolumeSource{} } func (*StorageOSVolumeSource) ProtoMessage() {} -func (*StorageOSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{179} } +func (*StorageOSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{180} } func (m *Sysctl) Reset() { *m = Sysctl{} } func (*Sysctl) ProtoMessage() {} -func (*Sysctl) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{180} } +func (*Sysctl) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{181} } func (m *TCPSocketAction) Reset() { *m = TCPSocketAction{} } func (*TCPSocketAction) ProtoMessage() {} -func (*TCPSocketAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{181} } +func (*TCPSocketAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{182} } func (m *Taint) Reset() { *m = Taint{} } func (*Taint) ProtoMessage() {} -func (*Taint) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{182} } +func (*Taint) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{183} } func (m *Toleration) Reset() { *m = Toleration{} } func (*Toleration) ProtoMessage() {} -func (*Toleration) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{183} } +func (*Toleration) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{184} } func (m *TopologySelectorLabelRequirement) Reset() { *m = TopologySelectorLabelRequirement{} } func (*TopologySelectorLabelRequirement) ProtoMessage() {} func (*TopologySelectorLabelRequirement) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{184} + return fileDescriptorGenerated, []int{185} } func (m *TopologySelectorTerm) Reset() { *m = TopologySelectorTerm{} } func (*TopologySelectorTerm) ProtoMessage() {} -func (*TopologySelectorTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{185} } +func (*TopologySelectorTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{186} } func (m *TypedLocalObjectReference) Reset() { *m = TypedLocalObjectReference{} } func (*TypedLocalObjectReference) ProtoMessage() {} func (*TypedLocalObjectReference) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{186} + return fileDescriptorGenerated, []int{187} } func (m *Volume) Reset() { *m = Volume{} } func (*Volume) ProtoMessage() {} -func (*Volume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{187} } +func (*Volume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{188} } func (m *VolumeDevice) Reset() { *m = VolumeDevice{} } func (*VolumeDevice) ProtoMessage() {} -func (*VolumeDevice) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{188} } +func (*VolumeDevice) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{189} } func (m *VolumeMount) Reset() { *m = VolumeMount{} } func (*VolumeMount) ProtoMessage() {} -func (*VolumeMount) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{189} } +func (*VolumeMount) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{190} } func (m *VolumeNodeAffinity) Reset() { *m = VolumeNodeAffinity{} } func (*VolumeNodeAffinity) ProtoMessage() {} -func (*VolumeNodeAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{190} } +func (*VolumeNodeAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{191} } func (m *VolumeProjection) Reset() { *m = VolumeProjection{} } func (*VolumeProjection) ProtoMessage() {} -func (*VolumeProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{191} } +func (*VolumeProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{192} } func (m *VolumeSource) Reset() { *m = VolumeSource{} } func (*VolumeSource) ProtoMessage() {} -func (*VolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{192} } +func (*VolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{193} } func (m *VsphereVirtualDiskVolumeSource) Reset() { *m = VsphereVirtualDiskVolumeSource{} } func (*VsphereVirtualDiskVolumeSource) ProtoMessage() {} func (*VsphereVirtualDiskVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{193} + return fileDescriptorGenerated, []int{194} } func (m *WeightedPodAffinityTerm) Reset() { *m = WeightedPodAffinityTerm{} } func (*WeightedPodAffinityTerm) ProtoMessage() {} func (*WeightedPodAffinityTerm) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{194} + return fileDescriptorGenerated, []int{195} } func init() { @@ -1156,6 +1163,7 @@ func init() { proto.RegisterType((*FlockerVolumeSource)(nil), "k8s.io.api.core.v1.FlockerVolumeSource") proto.RegisterType((*GCEPersistentDiskVolumeSource)(nil), "k8s.io.api.core.v1.GCEPersistentDiskVolumeSource") proto.RegisterType((*GitRepoVolumeSource)(nil), "k8s.io.api.core.v1.GitRepoVolumeSource") + proto.RegisterType((*GlusterfsPersistentVolumeSource)(nil), "k8s.io.api.core.v1.GlusterfsPersistentVolumeSource") proto.RegisterType((*GlusterfsVolumeSource)(nil), "k8s.io.api.core.v1.GlusterfsVolumeSource") proto.RegisterType((*HTTPGetAction)(nil), "k8s.io.api.core.v1.HTTPGetAction") proto.RegisterType((*HTTPHeader)(nil), "k8s.io.api.core.v1.HTTPHeader") @@ -3948,6 +3956,46 @@ func (m *GitRepoVolumeSource) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *GlusterfsPersistentVolumeSource) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GlusterfsPersistentVolumeSource) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.EndpointsName))) + i += copy(dAtA[i:], m.EndpointsName) + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Path))) + i += copy(dAtA[i:], m.Path) + dAtA[i] = 0x18 + i++ + if m.ReadOnly { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + if m.EndpointsNamespace != nil { + dAtA[i] = 0x22 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(*m.EndpointsNamespace))) + i += copy(dAtA[i:], *m.EndpointsNamespace) + } + return i, nil +} + func (m *GlusterfsVolumeSource) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -11946,6 +11994,21 @@ func (m *GitRepoVolumeSource) Size() (n int) { return n } +func (m *GlusterfsPersistentVolumeSource) Size() (n int) { + var l int + _ = l + l = len(m.EndpointsName) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.Path) + n += 1 + l + sovGenerated(uint64(l)) + n += 2 + if m.EndpointsNamespace != nil { + l = len(*m.EndpointsNamespace) + n += 1 + l + sovGenerated(uint64(l)) + } + return n +} + func (m *GlusterfsVolumeSource) Size() (n int) { var l int _ = l @@ -15291,6 +15354,19 @@ func (this *GitRepoVolumeSource) String() string { }, "") return s } +func (this *GlusterfsPersistentVolumeSource) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&GlusterfsPersistentVolumeSource{`, + `EndpointsName:` + fmt.Sprintf("%v", this.EndpointsName) + `,`, + `Path:` + fmt.Sprintf("%v", this.Path) + `,`, + `ReadOnly:` + fmt.Sprintf("%v", this.ReadOnly) + `,`, + `EndpointsNamespace:` + valueToStringGenerated(this.EndpointsNamespace) + `,`, + `}`, + }, "") + return s +} func (this *GlusterfsVolumeSource) String() string { if this == nil { return "nil" @@ -16006,7 +16082,7 @@ func (this *PersistentVolumeSource) String() string { `GCEPersistentDisk:` + strings.Replace(fmt.Sprintf("%v", this.GCEPersistentDisk), "GCEPersistentDiskVolumeSource", "GCEPersistentDiskVolumeSource", 1) + `,`, `AWSElasticBlockStore:` + strings.Replace(fmt.Sprintf("%v", this.AWSElasticBlockStore), "AWSElasticBlockStoreVolumeSource", "AWSElasticBlockStoreVolumeSource", 1) + `,`, `HostPath:` + strings.Replace(fmt.Sprintf("%v", this.HostPath), "HostPathVolumeSource", "HostPathVolumeSource", 1) + `,`, - `Glusterfs:` + strings.Replace(fmt.Sprintf("%v", this.Glusterfs), "GlusterfsVolumeSource", "GlusterfsVolumeSource", 1) + `,`, + `Glusterfs:` + strings.Replace(fmt.Sprintf("%v", this.Glusterfs), "GlusterfsPersistentVolumeSource", "GlusterfsPersistentVolumeSource", 1) + `,`, `NFS:` + strings.Replace(fmt.Sprintf("%v", this.NFS), "NFSVolumeSource", "NFSVolumeSource", 1) + `,`, `RBD:` + strings.Replace(fmt.Sprintf("%v", this.RBD), "RBDPersistentVolumeSource", "RBDPersistentVolumeSource", 1) + `,`, `ISCSI:` + strings.Replace(fmt.Sprintf("%v", this.ISCSI), "ISCSIPersistentVolumeSource", "ISCSIPersistentVolumeSource", 1) + `,`, @@ -26700,6 +26776,164 @@ func (m *GitRepoVolumeSource) Unmarshal(dAtA []byte) error { } return nil } +func (m *GlusterfsPersistentVolumeSource) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GlusterfsPersistentVolumeSource: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GlusterfsPersistentVolumeSource: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EndpointsName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.EndpointsName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Path", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Path = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ReadOnly", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ReadOnly = bool(v != 0) + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EndpointsNamespace", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(dAtA[iNdEx:postIndex]) + m.EndpointsNamespace = &s + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *GlusterfsVolumeSource) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -35062,7 +35296,7 @@ func (m *PersistentVolumeSource) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Glusterfs == nil { - m.Glusterfs = &GlusterfsVolumeSource{} + m.Glusterfs = &GlusterfsPersistentVolumeSource{} } if err := m.Glusterfs.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -51414,806 +51648,808 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 12807 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0x6b, 0x70, 0x24, 0xd7, - 0x75, 0x18, 0xac, 0x9e, 0x19, 0x3c, 0xe6, 0xe0, 0x7d, 0x77, 0x97, 0xc4, 0x82, 0xdc, 0x9d, 0x65, - 0x53, 0x5a, 0x2e, 0x45, 0x12, 0x2b, 0x2e, 0x49, 0x91, 0x16, 0x29, 0xda, 0x00, 0x06, 0xd8, 0x1d, - 0xee, 0x02, 0x3b, 0xbc, 0x83, 0x5d, 0x4a, 0x34, 0x25, 0xab, 0x31, 0x73, 0x01, 0x34, 0xd1, 0xe8, - 0x1e, 0x76, 0xf7, 0x60, 0x17, 0xfc, 0xec, 0xaa, 0x2f, 0x72, 0xec, 0x44, 0xb1, 0x2b, 0xa5, 0x8a, - 0x55, 0x79, 0xd8, 0x2e, 0xa7, 0xca, 0x71, 0xca, 0x76, 0x9c, 0xa4, 0xe2, 0xd8, 0xb1, 0x1d, 0xcb, - 0x4e, 0x1c, 0x3b, 0x3f, 0x9c, 0x3f, 0x8a, 0x93, 0xaa, 0x94, 0x5c, 0xe5, 0x0a, 0x62, 0xc3, 0x79, - 0x94, 0x7f, 0xe4, 0x51, 0x71, 0x7e, 0xc4, 0x88, 0x2b, 0x4e, 0xdd, 0x67, 0xdf, 0xdb, 0xd3, 0x3d, - 0x33, 0x58, 0x62, 0x41, 0x5a, 0xa5, 0x7f, 0x33, 0xf7, 0x9c, 0x7b, 0xee, 0xed, 0xfb, 0x3c, 0xe7, - 0xdc, 0xf3, 0x80, 0x57, 0x77, 0x5e, 0x89, 0xe6, 0xdd, 0xe0, 0xea, 0x4e, 0x67, 0x83, 0x84, 0x3e, - 0x89, 0x49, 0x74, 0x75, 0x8f, 0xf8, 0xad, 0x20, 0xbc, 0x2a, 0x00, 0x4e, 0xdb, 0xbd, 0xda, 0x0c, - 0x42, 0x72, 0x75, 0xef, 0xf9, 0xab, 0x5b, 0xc4, 0x27, 0xa1, 0x13, 0x93, 0xd6, 0x7c, 0x3b, 0x0c, - 0xe2, 0x00, 0x21, 0x8e, 0x33, 0xef, 0xb4, 0xdd, 0x79, 0x8a, 0x33, 0xbf, 0xf7, 0xfc, 0xdc, 0x73, - 0x5b, 0x6e, 0xbc, 0xdd, 0xd9, 0x98, 0x6f, 0x06, 0xbb, 0x57, 0xb7, 0x82, 0xad, 0xe0, 0x2a, 0x43, - 0xdd, 0xe8, 0x6c, 0xb2, 0x7f, 0xec, 0x0f, 0xfb, 0xc5, 0x49, 0xcc, 0xbd, 0x98, 0x34, 0xb3, 0xeb, - 0x34, 0xb7, 0x5d, 0x9f, 0x84, 0xfb, 0x57, 0xdb, 0x3b, 0x5b, 0xac, 0xdd, 0x90, 0x44, 0x41, 0x27, - 0x6c, 0x92, 0x74, 0xc3, 0x3d, 0x6b, 0x45, 0x57, 0x77, 0x49, 0xec, 0x64, 0x74, 0x77, 0xee, 0x6a, - 0x5e, 0xad, 0xb0, 0xe3, 0xc7, 0xee, 0x6e, 0x77, 0x33, 0x9f, 0xee, 0x57, 0x21, 0x6a, 0x6e, 0x93, - 0x5d, 0xa7, 0xab, 0xde, 0x0b, 0x79, 0xf5, 0x3a, 0xb1, 0xeb, 0x5d, 0x75, 0xfd, 0x38, 0x8a, 0xc3, - 0x74, 0x25, 0xfb, 0x9b, 0x16, 0x5c, 0x5a, 0x78, 0xab, 0xb1, 0xec, 0x39, 0x51, 0xec, 0x36, 0x17, - 0xbd, 0xa0, 0xb9, 0xd3, 0x88, 0x83, 0x90, 0xdc, 0x0d, 0xbc, 0xce, 0x2e, 0x69, 0xb0, 0x81, 0x40, - 0xcf, 0xc2, 0xe8, 0x1e, 0xfb, 0x5f, 0xab, 0xce, 0x5a, 0x97, 0xac, 0x2b, 0xe5, 0xc5, 0xe9, 0xdf, - 0x3e, 0xa8, 0x7c, 0xec, 0xf0, 0xa0, 0x32, 0x7a, 0x57, 0x94, 0x63, 0x85, 0x81, 0x2e, 0xc3, 0xf0, - 0x66, 0xb4, 0xbe, 0xdf, 0x26, 0xb3, 0x05, 0x86, 0x3b, 0x29, 0x70, 0x87, 0x57, 0x1a, 0xb4, 0x14, - 0x0b, 0x28, 0xba, 0x0a, 0xe5, 0xb6, 0x13, 0xc6, 0x6e, 0xec, 0x06, 0xfe, 0x6c, 0xf1, 0x92, 0x75, - 0x65, 0x68, 0x71, 0x46, 0xa0, 0x96, 0xeb, 0x12, 0x80, 0x13, 0x1c, 0xda, 0x8d, 0x90, 0x38, 0xad, - 0xdb, 0xbe, 0xb7, 0x3f, 0x5b, 0xba, 0x64, 0x5d, 0x19, 0x4d, 0xba, 0x81, 0x45, 0x39, 0x56, 0x18, - 0xf6, 0x8f, 0x16, 0x60, 0x74, 0x61, 0x73, 0xd3, 0xf5, 0xdd, 0x78, 0x1f, 0xdd, 0x85, 0x71, 0x3f, - 0x68, 0x11, 0xf9, 0x9f, 0x7d, 0xc5, 0xd8, 0xb5, 0x4b, 0xf3, 0xdd, 0x4b, 0x69, 0x7e, 0x4d, 0xc3, - 0x5b, 0x9c, 0x3e, 0x3c, 0xa8, 0x8c, 0xeb, 0x25, 0xd8, 0xa0, 0x83, 0x30, 0x8c, 0xb5, 0x83, 0x96, - 0x22, 0x5b, 0x60, 0x64, 0x2b, 0x59, 0x64, 0xeb, 0x09, 0xda, 0xe2, 0xd4, 0xe1, 0x41, 0x65, 0x4c, - 0x2b, 0xc0, 0x3a, 0x11, 0xb4, 0x01, 0x53, 0xf4, 0xaf, 0x1f, 0xbb, 0x8a, 0x6e, 0x91, 0xd1, 0x7d, - 0x32, 0x8f, 0xae, 0x86, 0xba, 0x78, 0xe6, 0xf0, 0xa0, 0x32, 0x95, 0x2a, 0xc4, 0x69, 0x82, 0xf6, - 0xfb, 0x30, 0xb9, 0x10, 0xc7, 0x4e, 0x73, 0x9b, 0xb4, 0xf8, 0x0c, 0xa2, 0x17, 0xa1, 0xe4, 0x3b, - 0xbb, 0x44, 0xcc, 0xef, 0x25, 0x31, 0xb0, 0xa5, 0x35, 0x67, 0x97, 0x1c, 0x1d, 0x54, 0xa6, 0xef, - 0xf8, 0xee, 0x7b, 0x1d, 0xb1, 0x2a, 0x68, 0x19, 0x66, 0xd8, 0xe8, 0x1a, 0x40, 0x8b, 0xec, 0xb9, - 0x4d, 0x52, 0x77, 0xe2, 0x6d, 0x31, 0xdf, 0x48, 0xd4, 0x85, 0xaa, 0x82, 0x60, 0x0d, 0xcb, 0xbe, - 0x0f, 0xe5, 0x85, 0xbd, 0xc0, 0x6d, 0xd5, 0x83, 0x56, 0x84, 0x76, 0x60, 0xaa, 0x1d, 0x92, 0x4d, - 0x12, 0xaa, 0xa2, 0x59, 0xeb, 0x52, 0xf1, 0xca, 0xd8, 0xb5, 0x2b, 0x99, 0x1f, 0x6b, 0xa2, 0x2e, - 0xfb, 0x71, 0xb8, 0xbf, 0xf8, 0xa8, 0x68, 0x6f, 0x2a, 0x05, 0xc5, 0x69, 0xca, 0xf6, 0xbf, 0x2c, - 0xc0, 0xb9, 0x85, 0xf7, 0x3b, 0x21, 0xa9, 0xba, 0xd1, 0x4e, 0x7a, 0x85, 0xb7, 0xdc, 0x68, 0x67, - 0x2d, 0x19, 0x01, 0xb5, 0xb4, 0xaa, 0xa2, 0x1c, 0x2b, 0x0c, 0xf4, 0x1c, 0x8c, 0xd0, 0xdf, 0x77, - 0x70, 0x4d, 0x7c, 0xf2, 0x19, 0x81, 0x3c, 0x56, 0x75, 0x62, 0xa7, 0xca, 0x41, 0x58, 0xe2, 0xa0, - 0x55, 0x18, 0x6b, 0xb2, 0x0d, 0xb9, 0xb5, 0x1a, 0xb4, 0x08, 0x9b, 0xcc, 0xf2, 0xe2, 0x33, 0x14, - 0x7d, 0x29, 0x29, 0x3e, 0x3a, 0xa8, 0xcc, 0xf2, 0xbe, 0x09, 0x12, 0x1a, 0x0c, 0xeb, 0xf5, 0x91, - 0xad, 0xf6, 0x57, 0x89, 0x51, 0x82, 0x8c, 0xbd, 0x75, 0x45, 0xdb, 0x2a, 0x43, 0x6c, 0xab, 0x8c, - 0x67, 0x6f, 0x13, 0xf4, 0x3c, 0x94, 0x76, 0x5c, 0xbf, 0x35, 0x3b, 0xcc, 0x68, 0x5d, 0xa0, 0x73, - 0x7e, 0xd3, 0xf5, 0x5b, 0x47, 0x07, 0x95, 0x19, 0xa3, 0x3b, 0xb4, 0x10, 0x33, 0x54, 0xfb, 0x8f, - 0x2d, 0xa8, 0x30, 0xd8, 0x8a, 0xeb, 0x91, 0x3a, 0x09, 0x23, 0x37, 0x8a, 0x89, 0x1f, 0x1b, 0x03, - 0x7a, 0x0d, 0x20, 0x22, 0xcd, 0x90, 0xc4, 0xda, 0x90, 0xaa, 0x85, 0xd1, 0x50, 0x10, 0xac, 0x61, - 0xd1, 0x03, 0x21, 0xda, 0x76, 0x42, 0xb6, 0xbe, 0xc4, 0xc0, 0xaa, 0x03, 0xa1, 0x21, 0x01, 0x38, - 0xc1, 0x31, 0x0e, 0x84, 0x62, 0xbf, 0x03, 0x01, 0x7d, 0x16, 0xa6, 0x92, 0xc6, 0xa2, 0xb6, 0xd3, - 0x94, 0x03, 0xc8, 0xb6, 0x4c, 0xc3, 0x04, 0xe1, 0x34, 0xae, 0xfd, 0xf7, 0x2c, 0xb1, 0x78, 0xe8, - 0x57, 0x7f, 0xc4, 0xbf, 0xd5, 0xfe, 0x15, 0x0b, 0x46, 0x16, 0x5d, 0xbf, 0xe5, 0xfa, 0x5b, 0xe8, - 0x4b, 0x30, 0x4a, 0xef, 0xa6, 0x96, 0x13, 0x3b, 0xe2, 0xdc, 0xfb, 0x94, 0xb6, 0xb7, 0xd4, 0x55, - 0x31, 0xdf, 0xde, 0xd9, 0xa2, 0x05, 0xd1, 0x3c, 0xc5, 0xa6, 0xbb, 0xed, 0xf6, 0xc6, 0xbb, 0xa4, - 0x19, 0xaf, 0x92, 0xd8, 0x49, 0x3e, 0x27, 0x29, 0xc3, 0x8a, 0x2a, 0xba, 0x09, 0xc3, 0xb1, 0x13, - 0x6e, 0x91, 0x58, 0x1c, 0x80, 0x99, 0x07, 0x15, 0xaf, 0x89, 0xe9, 0x8e, 0x24, 0x7e, 0x93, 0x24, - 0xd7, 0xc2, 0x3a, 0xab, 0x8a, 0x05, 0x09, 0xfb, 0xaf, 0x0c, 0xc3, 0xf9, 0xa5, 0x46, 0x2d, 0x67, - 0x5d, 0x5d, 0x86, 0xe1, 0x56, 0xe8, 0xee, 0x91, 0x50, 0x8c, 0xb3, 0xa2, 0x52, 0x65, 0xa5, 0x58, - 0x40, 0xd1, 0x2b, 0x30, 0xce, 0x2f, 0xa4, 0x1b, 0x8e, 0xdf, 0xf2, 0xe4, 0x10, 0x9f, 0x15, 0xd8, - 0xe3, 0x77, 0x35, 0x18, 0x36, 0x30, 0x8f, 0xb9, 0xa8, 0x2e, 0xa7, 0x36, 0x63, 0xde, 0x65, 0xf7, - 0x15, 0x0b, 0xa6, 0x79, 0x33, 0x0b, 0x71, 0x1c, 0xba, 0x1b, 0x9d, 0x98, 0x44, 0xb3, 0x43, 0xec, - 0xa4, 0x5b, 0xca, 0x1a, 0xad, 0xdc, 0x11, 0x98, 0xbf, 0x9b, 0xa2, 0xc2, 0x0f, 0xc1, 0x59, 0xd1, - 0xee, 0x74, 0x1a, 0x8c, 0xbb, 0x9a, 0x45, 0xdf, 0x6f, 0xc1, 0x5c, 0x33, 0xf0, 0xe3, 0x30, 0xf0, - 0x3c, 0x12, 0xd6, 0x3b, 0x1b, 0x9e, 0x1b, 0x6d, 0xf3, 0x75, 0x8a, 0xc9, 0x26, 0x3b, 0x09, 0x72, - 0xe6, 0x50, 0x21, 0x89, 0x39, 0xbc, 0x78, 0x78, 0x50, 0x99, 0x5b, 0xca, 0x25, 0x85, 0x7b, 0x34, - 0x83, 0x76, 0x00, 0xd1, 0xab, 0xb4, 0x11, 0x3b, 0x5b, 0x24, 0x69, 0x7c, 0x64, 0xf0, 0xc6, 0x1f, - 0x39, 0x3c, 0xa8, 0xa0, 0xb5, 0x2e, 0x12, 0x38, 0x83, 0x2c, 0x7a, 0x0f, 0xce, 0xd2, 0xd2, 0xae, - 0x6f, 0x1d, 0x1d, 0xbc, 0xb9, 0xd9, 0xc3, 0x83, 0xca, 0xd9, 0xb5, 0x0c, 0x22, 0x38, 0x93, 0xf4, - 0xdc, 0x12, 0x9c, 0xcb, 0x9c, 0x2a, 0x34, 0x0d, 0xc5, 0x1d, 0xc2, 0x59, 0x90, 0x32, 0xa6, 0x3f, - 0xd1, 0x59, 0x18, 0xda, 0x73, 0xbc, 0x8e, 0x58, 0xa5, 0x98, 0xff, 0xf9, 0x4c, 0xe1, 0x15, 0xcb, - 0x6e, 0xc2, 0xf8, 0x92, 0xd3, 0x76, 0x36, 0x5c, 0xcf, 0x8d, 0x5d, 0x12, 0xa1, 0xa7, 0xa0, 0xe8, - 0xb4, 0x5a, 0xec, 0x8a, 0x2c, 0x2f, 0x9e, 0x3b, 0x3c, 0xa8, 0x14, 0x17, 0x5a, 0xf4, 0xac, 0x06, - 0x85, 0xb5, 0x8f, 0x29, 0x06, 0xfa, 0x24, 0x94, 0x5a, 0x61, 0xd0, 0x9e, 0x2d, 0x30, 0x4c, 0x3a, - 0x54, 0xa5, 0x6a, 0x18, 0xb4, 0x53, 0xa8, 0x0c, 0xc7, 0xfe, 0x8d, 0x02, 0x3c, 0xbe, 0x44, 0xda, - 0xdb, 0x2b, 0x8d, 0x9c, 0x4d, 0x77, 0x05, 0x46, 0x77, 0x03, 0xdf, 0x8d, 0x83, 0x30, 0x12, 0x4d, - 0xb3, 0xdb, 0x64, 0x55, 0x94, 0x61, 0x05, 0x45, 0x97, 0xa0, 0xd4, 0x4e, 0x38, 0x81, 0x71, 0xc9, - 0x45, 0x30, 0x1e, 0x80, 0x41, 0x28, 0x46, 0x27, 0x22, 0xa1, 0xb8, 0x05, 0x15, 0xc6, 0x9d, 0x88, - 0x84, 0x98, 0x41, 0x92, 0xe3, 0x94, 0x1e, 0xb4, 0x62, 0x5b, 0xa5, 0x8e, 0x53, 0x0a, 0xc1, 0x1a, - 0x16, 0xaa, 0x43, 0x39, 0x52, 0x93, 0x3a, 0x34, 0xf8, 0xa4, 0x4e, 0xb0, 0xf3, 0x56, 0xcd, 0x64, - 0x42, 0xc4, 0x38, 0x06, 0x86, 0xfb, 0x9e, 0xb7, 0x5f, 0x2f, 0x00, 0xe2, 0x43, 0xf8, 0xe7, 0x6c, - 0xe0, 0xee, 0x74, 0x0f, 0x5c, 0x26, 0xe7, 0x75, 0x2b, 0x68, 0x3a, 0x5e, 0xfa, 0x08, 0x3f, 0xa9, - 0xd1, 0xfb, 0x5f, 0x16, 0x3c, 0xbe, 0xe4, 0xfa, 0x2d, 0x12, 0xe6, 0x2c, 0xc0, 0x87, 0x23, 0x80, - 0x1c, 0xef, 0xa4, 0x37, 0x96, 0x58, 0xe9, 0x04, 0x96, 0x98, 0xfd, 0xdf, 0x2d, 0x40, 0xfc, 0xb3, - 0x3f, 0x72, 0x1f, 0x7b, 0xa7, 0xfb, 0x63, 0x4f, 0x60, 0x59, 0xd8, 0xb7, 0x60, 0x72, 0xc9, 0x73, - 0x89, 0x1f, 0xd7, 0xea, 0x4b, 0x81, 0xbf, 0xe9, 0x6e, 0xa1, 0xcf, 0xc0, 0x24, 0x95, 0x69, 0x83, - 0x4e, 0xdc, 0x20, 0xcd, 0xc0, 0x67, 0xec, 0x3f, 0x95, 0x04, 0xd1, 0xe1, 0x41, 0x65, 0x72, 0xdd, - 0x80, 0xe0, 0x14, 0xa6, 0xfd, 0x7b, 0x74, 0xfc, 0x82, 0xdd, 0x76, 0xe0, 0x13, 0x3f, 0x5e, 0x0a, - 0xfc, 0x16, 0x17, 0x13, 0x3f, 0x03, 0xa5, 0x98, 0x8e, 0x07, 0x1f, 0xbb, 0xcb, 0x72, 0xa3, 0xd0, - 0x51, 0x38, 0x3a, 0xa8, 0x3c, 0xd2, 0x5d, 0x83, 0x8d, 0x13, 0xab, 0x83, 0xbe, 0x03, 0x86, 0xa3, - 0xd8, 0x89, 0x3b, 0x91, 0x18, 0xcd, 0x27, 0xe4, 0x68, 0x36, 0x58, 0xe9, 0xd1, 0x41, 0x65, 0x4a, - 0x55, 0xe3, 0x45, 0x58, 0x54, 0x40, 0x4f, 0xc3, 0xc8, 0x2e, 0x89, 0x22, 0x67, 0x4b, 0x72, 0xf8, - 0x53, 0xa2, 0xee, 0xc8, 0x2a, 0x2f, 0xc6, 0x12, 0x8e, 0x9e, 0x84, 0x21, 0x12, 0x86, 0x41, 0x28, - 0xf6, 0xe8, 0x84, 0x40, 0x1c, 0x5a, 0xa6, 0x85, 0x98, 0xc3, 0xec, 0x7f, 0x6d, 0xc1, 0x94, 0xea, - 0x2b, 0x6f, 0xeb, 0x14, 0x58, 0xb9, 0xb7, 0x01, 0x9a, 0xf2, 0x03, 0x23, 0x76, 0x7b, 0x8c, 0x5d, - 0xbb, 0x9c, 0xc9, 0xa0, 0x74, 0x0d, 0x63, 0x42, 0x59, 0x15, 0x45, 0x58, 0xa3, 0x66, 0xff, 0xba, - 0x05, 0x67, 0x52, 0x5f, 0x74, 0xcb, 0x8d, 0x62, 0xf4, 0x4e, 0xd7, 0x57, 0xcd, 0x0f, 0xf6, 0x55, - 0xb4, 0x36, 0xfb, 0x26, 0xb5, 0x94, 0x65, 0x89, 0xf6, 0x45, 0x37, 0x60, 0xc8, 0x8d, 0xc9, 0xae, - 0xfc, 0x98, 0x27, 0x7b, 0x7e, 0x0c, 0xef, 0x55, 0x32, 0x23, 0x35, 0x5a, 0x13, 0x73, 0x02, 0xf6, - 0x8f, 0x14, 0xa1, 0xcc, 0x97, 0xed, 0xaa, 0xd3, 0x3e, 0x85, 0xb9, 0xa8, 0x41, 0x89, 0x51, 0xe7, - 0x1d, 0x7f, 0x2a, 0xbb, 0xe3, 0xa2, 0x3b, 0xf3, 0x54, 0x4e, 0xe3, 0xac, 0xa0, 0xba, 0x1a, 0x68, - 0x11, 0x66, 0x24, 0x90, 0x03, 0xb0, 0xe1, 0xfa, 0x4e, 0xb8, 0x4f, 0xcb, 0x66, 0x8b, 0x8c, 0xe0, - 0x73, 0xbd, 0x09, 0x2e, 0x2a, 0x7c, 0x4e, 0x56, 0xf5, 0x35, 0x01, 0x60, 0x8d, 0xe8, 0xdc, 0xcb, - 0x50, 0x56, 0xc8, 0xc7, 0xe1, 0x71, 0xe6, 0x3e, 0x0b, 0x53, 0xa9, 0xb6, 0xfa, 0x55, 0x1f, 0xd7, - 0x59, 0xa4, 0x5f, 0x65, 0xa7, 0x80, 0xe8, 0xf5, 0xb2, 0xbf, 0x27, 0x4e, 0xd1, 0xf7, 0xe1, 0xac, - 0x97, 0x71, 0x38, 0x89, 0xa9, 0x1a, 0xfc, 0x30, 0x7b, 0x5c, 0x7c, 0xf6, 0xd9, 0x2c, 0x28, 0xce, - 0x6c, 0x83, 0x5e, 0xfb, 0x41, 0x9b, 0xae, 0x79, 0xc7, 0x63, 0xfd, 0x15, 0xd2, 0xf7, 0x6d, 0x51, - 0x86, 0x15, 0x94, 0x1e, 0x61, 0x67, 0x55, 0xe7, 0x6f, 0x92, 0xfd, 0x06, 0xf1, 0x48, 0x33, 0x0e, - 0xc2, 0x0f, 0xb5, 0xfb, 0x17, 0xf8, 0xe8, 0xf3, 0x13, 0x70, 0x4c, 0x10, 0x28, 0xde, 0x24, 0xfb, - 0x7c, 0x2a, 0xf4, 0xaf, 0x2b, 0xf6, 0xfc, 0xba, 0x9f, 0xb7, 0x60, 0x42, 0x7d, 0xdd, 0x29, 0x6c, - 0xf5, 0x45, 0x73, 0xab, 0x5f, 0xe8, 0xb9, 0xc0, 0x73, 0x36, 0xf9, 0xd7, 0x0b, 0x70, 0x5e, 0xe1, - 0x50, 0x76, 0x9f, 0xff, 0x11, 0xab, 0xea, 0x2a, 0x94, 0x7d, 0xa5, 0x3d, 0xb0, 0x4c, 0xb1, 0x3d, - 0xd1, 0x1d, 0x24, 0x38, 0x94, 0x6b, 0xf3, 0x13, 0x11, 0x7f, 0x5c, 0x57, 0xab, 0x09, 0x15, 0xda, - 0x22, 0x14, 0x3b, 0x6e, 0x4b, 0xdc, 0x19, 0x9f, 0x92, 0xa3, 0x7d, 0xa7, 0x56, 0x3d, 0x3a, 0xa8, - 0x3c, 0x91, 0xa7, 0xd2, 0xa5, 0x97, 0x55, 0x34, 0x7f, 0xa7, 0x56, 0xc5, 0xb4, 0x32, 0x5a, 0x80, - 0x29, 0xa9, 0xb5, 0xbe, 0x4b, 0x39, 0xa8, 0xc0, 0x17, 0x57, 0x8b, 0xd2, 0x8d, 0x61, 0x13, 0x8c, - 0xd3, 0xf8, 0xa8, 0x0a, 0xd3, 0x3b, 0x9d, 0x0d, 0xe2, 0x91, 0x98, 0x7f, 0xf0, 0x4d, 0xc2, 0x35, - 0x47, 0xe5, 0x44, 0xb4, 0xbc, 0x99, 0x82, 0xe3, 0xae, 0x1a, 0xf6, 0x9f, 0xb1, 0x23, 0x5e, 0x8c, - 0x5e, 0x3d, 0x0c, 0xe8, 0xc2, 0xa2, 0xd4, 0x3f, 0xcc, 0xe5, 0x3c, 0xc8, 0xaa, 0xb8, 0x49, 0xf6, - 0xd7, 0x03, 0xca, 0x6c, 0x67, 0xaf, 0x0a, 0x63, 0xcd, 0x97, 0x7a, 0xae, 0xf9, 0x5f, 0x2c, 0xc0, - 0x39, 0x35, 0x02, 0x06, 0x5f, 0xf7, 0xe7, 0x7d, 0x0c, 0x9e, 0x87, 0xb1, 0x16, 0xd9, 0x74, 0x3a, - 0x5e, 0xac, 0xd4, 0x98, 0x43, 0x5c, 0x95, 0x5d, 0x4d, 0x8a, 0xb1, 0x8e, 0x73, 0x8c, 0x61, 0xfb, - 0xa9, 0x31, 0x76, 0xb7, 0xc6, 0x0e, 0x5d, 0xe3, 0x6a, 0xd7, 0x58, 0xb9, 0xbb, 0xe6, 0x49, 0x18, - 0x72, 0x77, 0x29, 0xaf, 0x55, 0x30, 0x59, 0xa8, 0x1a, 0x2d, 0xc4, 0x1c, 0x86, 0x3e, 0x01, 0x23, - 0xcd, 0x60, 0x77, 0xd7, 0xf1, 0x5b, 0xec, 0xca, 0x2b, 0x2f, 0x8e, 0x51, 0x76, 0x6c, 0x89, 0x17, - 0x61, 0x09, 0x43, 0x8f, 0x43, 0xc9, 0x09, 0xb7, 0xa2, 0xd9, 0x12, 0xc3, 0x19, 0xa5, 0x2d, 0x2d, - 0x84, 0x5b, 0x11, 0x66, 0xa5, 0x54, 0xaa, 0xba, 0x17, 0x84, 0x3b, 0xae, 0xbf, 0x55, 0x75, 0x43, - 0xb1, 0x25, 0xd4, 0x5d, 0xf8, 0x96, 0x82, 0x60, 0x0d, 0x0b, 0xad, 0xc0, 0x50, 0x3b, 0x08, 0xe3, - 0x68, 0x76, 0x98, 0x0d, 0xf7, 0x13, 0x39, 0x07, 0x11, 0xff, 0xda, 0x7a, 0x10, 0xc6, 0xc9, 0x07, - 0xd0, 0x7f, 0x11, 0xe6, 0xd5, 0xd1, 0x77, 0x40, 0x91, 0xf8, 0x7b, 0xb3, 0x23, 0x8c, 0xca, 0x5c, - 0x16, 0x95, 0x65, 0x7f, 0xef, 0xae, 0x13, 0x26, 0xa7, 0xf4, 0xb2, 0xbf, 0x87, 0x69, 0x1d, 0xf4, - 0x79, 0x28, 0xcb, 0x2d, 0x1e, 0x09, 0x35, 0x47, 0xe6, 0x12, 0x93, 0x07, 0x03, 0x26, 0xef, 0x75, - 0xdc, 0x90, 0xec, 0x12, 0x3f, 0x8e, 0x92, 0x33, 0x4d, 0x42, 0x23, 0x9c, 0x50, 0x43, 0x9f, 0x97, - 0xba, 0xb5, 0xd5, 0xa0, 0xe3, 0xc7, 0xd1, 0x6c, 0x99, 0x75, 0x2f, 0xf3, 0xd5, 0xe3, 0x6e, 0x82, - 0x97, 0x56, 0xbe, 0xf1, 0xca, 0xd8, 0x20, 0x85, 0x30, 0x4c, 0x78, 0xee, 0x1e, 0xf1, 0x49, 0x14, - 0xd5, 0xc3, 0x60, 0x83, 0xcc, 0x02, 0xeb, 0xf9, 0xf9, 0xec, 0xc7, 0x80, 0x60, 0x83, 0x2c, 0xce, - 0x1c, 0x1e, 0x54, 0x26, 0x6e, 0xe9, 0x75, 0xb0, 0x49, 0x02, 0xdd, 0x81, 0x49, 0x2a, 0xd7, 0xb8, - 0x09, 0xd1, 0xb1, 0x7e, 0x44, 0x99, 0xf4, 0x81, 0x8d, 0x4a, 0x38, 0x45, 0x04, 0xbd, 0x01, 0x65, - 0xcf, 0xdd, 0x24, 0xcd, 0xfd, 0xa6, 0x47, 0x66, 0xc7, 0x19, 0xc5, 0xcc, 0x6d, 0x75, 0x4b, 0x22, - 0x71, 0xb9, 0x48, 0xfd, 0xc5, 0x49, 0x75, 0x74, 0x17, 0x1e, 0x89, 0x49, 0xb8, 0xeb, 0xfa, 0x0e, - 0xdd, 0x0e, 0x42, 0x5e, 0x60, 0x4f, 0x2a, 0x13, 0x6c, 0xbd, 0x5d, 0x14, 0x43, 0xf7, 0xc8, 0x7a, - 0x26, 0x16, 0xce, 0xa9, 0x8d, 0x6e, 0xc3, 0x14, 0xdb, 0x09, 0xf5, 0x8e, 0xe7, 0xd5, 0x03, 0xcf, - 0x6d, 0xee, 0xcf, 0x4e, 0x32, 0x82, 0x9f, 0x90, 0xf7, 0x42, 0xcd, 0x04, 0x1f, 0x1d, 0x54, 0x20, - 0xf9, 0x87, 0xd3, 0xb5, 0xd1, 0x06, 0xd3, 0xa1, 0x77, 0x42, 0x37, 0xde, 0xa7, 0xeb, 0x97, 0xdc, - 0x8f, 0x67, 0xa7, 0x7a, 0x8a, 0xc2, 0x3a, 0xaa, 0x52, 0xb4, 0xeb, 0x85, 0x38, 0x4d, 0x90, 0x6e, - 0xed, 0x28, 0x6e, 0xb9, 0xfe, 0xec, 0x34, 0x3b, 0x31, 0xd4, 0xce, 0x68, 0xd0, 0x42, 0xcc, 0x61, - 0x4c, 0x7f, 0x4e, 0x7f, 0xdc, 0xa6, 0x27, 0xe8, 0x0c, 0x43, 0x4c, 0xf4, 0xe7, 0x12, 0x80, 0x13, - 0x1c, 0xca, 0xd4, 0xc4, 0xf1, 0xfe, 0x2c, 0x62, 0xa8, 0x6a, 0xbb, 0xac, 0xaf, 0x7f, 0x1e, 0xd3, - 0x72, 0x74, 0x0b, 0x46, 0x88, 0xbf, 0xb7, 0x12, 0x06, 0xbb, 0xb3, 0x67, 0xf2, 0xf7, 0xec, 0x32, - 0x47, 0xe1, 0x07, 0x7a, 0x22, 0xe0, 0x89, 0x62, 0x2c, 0x49, 0xa0, 0xfb, 0x30, 0x9b, 0x31, 0x23, - 0x7c, 0x02, 0xce, 0xb2, 0x09, 0x78, 0x4d, 0xd4, 0x9d, 0x5d, 0xcf, 0xc1, 0x3b, 0xea, 0x01, 0xc3, - 0xb9, 0xd4, 0xd1, 0x17, 0x60, 0x82, 0x6f, 0x28, 0xfe, 0xf8, 0x16, 0xcd, 0x9e, 0x63, 0x5f, 0x73, - 0x29, 0x7f, 0x73, 0x72, 0xc4, 0xc5, 0x73, 0xa2, 0x43, 0x13, 0x7a, 0x69, 0x84, 0x4d, 0x6a, 0xf6, - 0x06, 0x4c, 0xaa, 0x73, 0x8b, 0x2d, 0x1d, 0x54, 0x81, 0x21, 0xc6, 0xed, 0x08, 0xfd, 0x56, 0x99, - 0xce, 0x14, 0xe3, 0x84, 0x30, 0x2f, 0x67, 0x33, 0xe5, 0xbe, 0x4f, 0x16, 0xf7, 0x63, 0xc2, 0xa5, - 0xea, 0xa2, 0x36, 0x53, 0x12, 0x80, 0x13, 0x1c, 0xfb, 0xff, 0x72, 0xae, 0x31, 0x39, 0x1c, 0x07, - 0xb8, 0x0e, 0x9e, 0x85, 0xd1, 0xed, 0x20, 0x8a, 0x29, 0x36, 0x6b, 0x63, 0x28, 0xe1, 0x13, 0x6f, - 0x88, 0x72, 0xac, 0x30, 0xd0, 0xab, 0x30, 0xd1, 0xd4, 0x1b, 0x10, 0x77, 0x99, 0x1a, 0x02, 0xa3, - 0x75, 0x6c, 0xe2, 0xa2, 0x57, 0x60, 0x94, 0x3d, 0x9d, 0x37, 0x03, 0x4f, 0x30, 0x59, 0xf2, 0x42, - 0x1e, 0xad, 0x8b, 0xf2, 0x23, 0xed, 0x37, 0x56, 0xd8, 0xe8, 0x32, 0x0c, 0xd3, 0x2e, 0xd4, 0xea, - 0xe2, 0x16, 0x51, 0xaa, 0x9a, 0x1b, 0xac, 0x14, 0x0b, 0xa8, 0xfd, 0xd7, 0x0a, 0xda, 0x28, 0x53, - 0x89, 0x94, 0xa0, 0x3a, 0x8c, 0xdc, 0x73, 0xdc, 0xd8, 0xf5, 0xb7, 0x04, 0xbb, 0xf0, 0x74, 0xcf, - 0x2b, 0x85, 0x55, 0x7a, 0x8b, 0x57, 0xe0, 0x97, 0x9e, 0xf8, 0x83, 0x25, 0x19, 0x4a, 0x31, 0xec, - 0xf8, 0x3e, 0xa5, 0x58, 0x18, 0x94, 0x22, 0xe6, 0x15, 0x38, 0x45, 0xf1, 0x07, 0x4b, 0x32, 0xe8, - 0x1d, 0x00, 0xb9, 0x2c, 0x49, 0x4b, 0x3c, 0x59, 0x3f, 0xdb, 0x9f, 0xe8, 0xba, 0xaa, 0xb3, 0x38, - 0x49, 0xaf, 0xd4, 0xe4, 0x3f, 0xd6, 0xe8, 0xd9, 0x31, 0x63, 0xab, 0xba, 0x3b, 0x83, 0xbe, 0x9b, - 0x9e, 0x04, 0x4e, 0x18, 0x93, 0xd6, 0x42, 0x2c, 0x06, 0xe7, 0x93, 0x83, 0xc9, 0x14, 0xeb, 0xee, - 0x2e, 0xd1, 0x4f, 0x0d, 0x41, 0x04, 0x27, 0xf4, 0xec, 0x5f, 0x2e, 0xc2, 0x6c, 0x5e, 0x77, 0xe9, - 0xa2, 0x23, 0xf7, 0xdd, 0x78, 0x89, 0x72, 0x43, 0x96, 0xb9, 0xe8, 0x96, 0x45, 0x39, 0x56, 0x18, - 0x74, 0xf6, 0x23, 0x77, 0x4b, 0x8a, 0x84, 0x43, 0xc9, 0xec, 0x37, 0x58, 0x29, 0x16, 0x50, 0x8a, - 0x17, 0x12, 0x27, 0x12, 0x36, 0x11, 0xda, 0x2a, 0xc1, 0xac, 0x14, 0x0b, 0xa8, 0xae, 0x6f, 0x2a, - 0xf5, 0xd1, 0x37, 0x19, 0x43, 0x34, 0x74, 0xb2, 0x43, 0x84, 0xbe, 0x08, 0xb0, 0xe9, 0xfa, 0x6e, - 0xb4, 0xcd, 0xa8, 0x0f, 0x1f, 0x9b, 0xba, 0xe2, 0xa5, 0x56, 0x14, 0x15, 0xac, 0x51, 0x44, 0x2f, - 0xc1, 0x98, 0xda, 0x80, 0xb5, 0x2a, 0x7b, 0x20, 0xd2, 0x1e, 0xdc, 0x93, 0xd3, 0xa8, 0x8a, 0x75, - 0x3c, 0xfb, 0xdd, 0xf4, 0x7a, 0x11, 0x3b, 0x40, 0x1b, 0x5f, 0x6b, 0xd0, 0xf1, 0x2d, 0xf4, 0x1e, - 0x5f, 0xfb, 0x37, 0x8b, 0x30, 0x65, 0x34, 0xd6, 0x89, 0x06, 0x38, 0xb3, 0xae, 0xd3, 0x7b, 0xce, - 0x89, 0x89, 0xd8, 0x7f, 0x76, 0xff, 0xad, 0xa2, 0xdf, 0x85, 0x74, 0x07, 0xf0, 0xfa, 0xe8, 0x8b, - 0x50, 0xf6, 0x9c, 0x88, 0xe9, 0xae, 0x88, 0xd8, 0x77, 0x83, 0x10, 0x4b, 0xe4, 0x08, 0x27, 0x8a, - 0xb5, 0xab, 0x86, 0xd3, 0x4e, 0x48, 0xd2, 0x0b, 0x99, 0xf2, 0x3e, 0xd2, 0xe8, 0x46, 0x75, 0x82, - 0x32, 0x48, 0xfb, 0x98, 0xc3, 0xd0, 0x2b, 0x30, 0x1e, 0x12, 0xb6, 0x2a, 0x96, 0x28, 0x2b, 0xc7, - 0x96, 0xd9, 0x50, 0xc2, 0xf3, 0x61, 0x0d, 0x86, 0x0d, 0xcc, 0x84, 0x95, 0x1f, 0xee, 0xc1, 0xca, - 0x3f, 0x0d, 0x23, 0xec, 0x87, 0x5a, 0x01, 0x6a, 0x36, 0x6a, 0xbc, 0x18, 0x4b, 0x78, 0x7a, 0xc1, - 0x8c, 0x0e, 0xb8, 0x60, 0x3e, 0x09, 0x93, 0x55, 0x87, 0xec, 0x06, 0xfe, 0xb2, 0xdf, 0x6a, 0x07, - 0xae, 0x1f, 0xa3, 0x59, 0x28, 0xb1, 0xdb, 0x81, 0xef, 0xed, 0x12, 0xa5, 0x80, 0x4b, 0x94, 0x31, - 0xb7, 0xb7, 0xe0, 0x5c, 0x35, 0xb8, 0xe7, 0xdf, 0x73, 0xc2, 0xd6, 0x42, 0xbd, 0xa6, 0xc9, 0xb9, - 0x6b, 0x52, 0xce, 0xe2, 0x46, 0x2c, 0x99, 0x67, 0xaa, 0x56, 0x93, 0xdf, 0xb5, 0x2b, 0xae, 0x47, - 0x72, 0xb4, 0x11, 0x7f, 0xa3, 0x60, 0xb4, 0x94, 0xe0, 0xab, 0x07, 0x23, 0x2b, 0xf7, 0xc1, 0xe8, - 0x4d, 0x18, 0xdd, 0x74, 0x89, 0xd7, 0xc2, 0x64, 0x53, 0x2c, 0xb1, 0xa7, 0xf2, 0xdf, 0xe5, 0x57, - 0x28, 0xa6, 0xd4, 0x3e, 0x71, 0x29, 0x6d, 0x45, 0x54, 0xc6, 0x8a, 0x0c, 0xda, 0x81, 0x69, 0x29, - 0x06, 0x48, 0xa8, 0x58, 0x70, 0x4f, 0xf7, 0x92, 0x2d, 0x4c, 0xe2, 0x67, 0x0f, 0x0f, 0x2a, 0xd3, - 0x38, 0x45, 0x06, 0x77, 0x11, 0xa6, 0x62, 0xd9, 0x2e, 0x3d, 0x5a, 0x4b, 0x6c, 0xf8, 0x99, 0x58, - 0xc6, 0x24, 0x4c, 0x56, 0x6a, 0xff, 0xb8, 0x05, 0x8f, 0x76, 0x8d, 0x8c, 0x90, 0xb4, 0x4f, 0x78, - 0x16, 0xd2, 0x92, 0x6f, 0xa1, 0xbf, 0xe4, 0x6b, 0xff, 0x7d, 0x0b, 0xce, 0x2e, 0xef, 0xb6, 0xe3, - 0xfd, 0xaa, 0x6b, 0xbe, 0xee, 0xbc, 0x0c, 0xc3, 0xbb, 0xa4, 0xe5, 0x76, 0x76, 0xc5, 0xcc, 0x55, - 0xe4, 0xf1, 0xb3, 0xca, 0x4a, 0x8f, 0x0e, 0x2a, 0x13, 0x8d, 0x38, 0x08, 0x9d, 0x2d, 0xc2, 0x0b, - 0xb0, 0x40, 0x67, 0x87, 0xb8, 0xfb, 0x3e, 0xb9, 0xe5, 0xee, 0xba, 0xd2, 0xce, 0xa2, 0xa7, 0xee, - 0x6c, 0x5e, 0x0e, 0xe8, 0xfc, 0x9b, 0x1d, 0xc7, 0x8f, 0xdd, 0x78, 0x5f, 0x3c, 0xcc, 0x48, 0x22, - 0x38, 0xa1, 0x67, 0x7f, 0xd3, 0x82, 0x29, 0xb9, 0xee, 0x17, 0x5a, 0xad, 0x90, 0x44, 0x11, 0x9a, - 0x83, 0x82, 0xdb, 0x16, 0xbd, 0x04, 0xd1, 0xcb, 0x42, 0xad, 0x8e, 0x0b, 0x6e, 0x1b, 0xd5, 0xa1, - 0xcc, 0xcd, 0x35, 0x92, 0xc5, 0x35, 0x90, 0xd1, 0x07, 0xeb, 0xc1, 0xba, 0xac, 0x89, 0x13, 0x22, - 0x92, 0x83, 0x63, 0x67, 0x66, 0xd1, 0x7c, 0xf5, 0xba, 0x21, 0xca, 0xb1, 0xc2, 0x40, 0x57, 0x60, - 0xd4, 0x0f, 0x5a, 0xdc, 0x7a, 0x86, 0xdf, 0x7e, 0x6c, 0xc9, 0xae, 0x89, 0x32, 0xac, 0xa0, 0xf6, - 0x0f, 0x5b, 0x30, 0x2e, 0xbf, 0x6c, 0x40, 0x66, 0x92, 0x6e, 0xad, 0x84, 0x91, 0x4c, 0xb6, 0x16, - 0x65, 0x06, 0x19, 0xc4, 0xe0, 0x01, 0x8b, 0xc7, 0xe1, 0x01, 0xed, 0x1f, 0x2b, 0xc0, 0xa4, 0xec, - 0x4e, 0xa3, 0xb3, 0x11, 0x91, 0x18, 0xad, 0x43, 0xd9, 0xe1, 0x43, 0x4e, 0xe4, 0x8a, 0x7d, 0x32, - 0x5b, 0xf8, 0x30, 0xe6, 0x27, 0xb9, 0x96, 0x17, 0x64, 0x6d, 0x9c, 0x10, 0x42, 0x1e, 0xcc, 0xf8, - 0x41, 0xcc, 0x8e, 0x68, 0x05, 0xef, 0xf5, 0x04, 0x92, 0xa6, 0x7e, 0x5e, 0x50, 0x9f, 0x59, 0x4b, - 0x53, 0xc1, 0xdd, 0x84, 0xd1, 0xb2, 0x54, 0x78, 0x14, 0xf3, 0xc5, 0x0d, 0x7d, 0x16, 0xb2, 0xf5, - 0x1d, 0xf6, 0xaf, 0x59, 0x50, 0x96, 0x68, 0xa7, 0xf1, 0xda, 0xb5, 0x0a, 0x23, 0x11, 0x9b, 0x04, - 0x39, 0x34, 0x76, 0xaf, 0x8e, 0xf3, 0xf9, 0x4a, 0x6e, 0x1e, 0xfe, 0x3f, 0xc2, 0x92, 0x06, 0xd3, - 0x77, 0xab, 0xee, 0x7f, 0x44, 0xf4, 0xdd, 0xaa, 0x3f, 0x39, 0x37, 0xcc, 0x7f, 0x61, 0x7d, 0xd6, - 0xc4, 0x5a, 0xca, 0x20, 0xb5, 0x43, 0xb2, 0xe9, 0xde, 0x4f, 0x33, 0x48, 0x75, 0x56, 0x8a, 0x05, - 0x14, 0xbd, 0x03, 0xe3, 0x4d, 0xa9, 0xe8, 0x4c, 0x8e, 0x81, 0xcb, 0x3d, 0x95, 0xee, 0xea, 0x7d, - 0x86, 0x5b, 0xd6, 0x2e, 0x69, 0xf5, 0xb1, 0x41, 0xcd, 0x7c, 0x6e, 0x2f, 0xf6, 0x7b, 0x6e, 0x4f, - 0xe8, 0xe6, 0x3f, 0x3e, 0xff, 0x84, 0x05, 0xc3, 0x5c, 0x5d, 0x36, 0x98, 0x7e, 0x51, 0x7b, 0xae, - 0x4a, 0xc6, 0xee, 0x2e, 0x2d, 0x14, 0xcf, 0x4f, 0x68, 0x15, 0xca, 0xec, 0x07, 0x53, 0x1b, 0x14, - 0xf3, 0x4d, 0x8a, 0x79, 0xab, 0x7a, 0x07, 0xef, 0xca, 0x6a, 0x38, 0xa1, 0x60, 0x7f, 0xad, 0x48, - 0x8f, 0xaa, 0x04, 0xd5, 0xb8, 0xc1, 0xad, 0x87, 0x77, 0x83, 0x17, 0x1e, 0xd6, 0x0d, 0xbe, 0x05, - 0x53, 0x4d, 0xed, 0x71, 0x2b, 0x99, 0xc9, 0x2b, 0x3d, 0x17, 0x89, 0xf6, 0x0e, 0xc6, 0x55, 0x46, - 0x4b, 0x26, 0x11, 0x9c, 0xa6, 0x8a, 0xbe, 0x1b, 0xc6, 0xf9, 0x3c, 0x8b, 0x56, 0xb8, 0xc5, 0xc2, - 0x27, 0xf2, 0xd7, 0x8b, 0xde, 0x04, 0x5b, 0x89, 0x0d, 0xad, 0x3a, 0x36, 0x88, 0xd9, 0xbf, 0x3c, - 0x0a, 0x43, 0xcb, 0x7b, 0xc4, 0x8f, 0x4f, 0xe1, 0x40, 0x6a, 0xc2, 0xa4, 0xeb, 0xef, 0x05, 0xde, - 0x1e, 0x69, 0x71, 0xf8, 0x71, 0x2e, 0xd7, 0x47, 0x04, 0xe9, 0xc9, 0x9a, 0x41, 0x02, 0xa7, 0x48, - 0x3e, 0x0c, 0x09, 0xf3, 0x3a, 0x0c, 0xf3, 0xb9, 0x17, 0xe2, 0x65, 0xa6, 0x32, 0x98, 0x0d, 0xa2, - 0xd8, 0x05, 0x89, 0xf4, 0xcb, 0xb5, 0xcf, 0xa2, 0x3a, 0x7a, 0x17, 0x26, 0x37, 0xdd, 0x30, 0x8a, - 0xa9, 0x68, 0x18, 0xc5, 0xce, 0x6e, 0xfb, 0x01, 0x24, 0x4a, 0x35, 0x0e, 0x2b, 0x06, 0x25, 0x9c, - 0xa2, 0x8c, 0xb6, 0x60, 0x82, 0x0a, 0x39, 0x49, 0x53, 0x23, 0xc7, 0x6e, 0x4a, 0xa9, 0x8c, 0x6e, - 0xe9, 0x84, 0xb0, 0x49, 0x97, 0x1e, 0x26, 0x4d, 0x26, 0x14, 0x8d, 0x32, 0x8e, 0x42, 0x1d, 0x26, - 0x5c, 0x1a, 0xe2, 0x30, 0x7a, 0x26, 0x31, 0xb3, 0x95, 0xb2, 0x79, 0x26, 0x69, 0xc6, 0x29, 0x5f, - 0x82, 0x32, 0xa1, 0x43, 0x48, 0x09, 0x0b, 0xc5, 0xf8, 0xd5, 0xc1, 0xfa, 0xba, 0xea, 0x36, 0xc3, - 0xc0, 0x94, 0xe5, 0x97, 0x25, 0x25, 0x9c, 0x10, 0x45, 0x4b, 0x30, 0x1c, 0x91, 0xd0, 0x25, 0x91, - 0x50, 0x91, 0xf7, 0x98, 0x46, 0x86, 0xc6, 0x6d, 0xcf, 0xf9, 0x6f, 0x2c, 0xaa, 0xd2, 0xe5, 0xe5, - 0x30, 0x69, 0x88, 0x69, 0xc5, 0xb5, 0xe5, 0xb5, 0xc0, 0x4a, 0xb1, 0x80, 0xa2, 0x37, 0x60, 0x24, - 0x24, 0x1e, 0x53, 0x16, 0x4d, 0x0c, 0xbe, 0xc8, 0xb9, 0xee, 0x89, 0xd7, 0xc3, 0x92, 0x00, 0xba, - 0x09, 0x28, 0x24, 0x94, 0x87, 0x70, 0xfd, 0x2d, 0x65, 0xcc, 0x21, 0x74, 0xdd, 0x8f, 0x89, 0xf6, - 0xcf, 0xe0, 0x04, 0x43, 0x5a, 0xa5, 0xe2, 0x8c, 0x6a, 0xe8, 0x3a, 0xcc, 0xa8, 0xd2, 0x9a, 0x1f, - 0xc5, 0x8e, 0xdf, 0x24, 0x4c, 0xcd, 0x5d, 0x4e, 0xb8, 0x22, 0x9c, 0x46, 0xc0, 0xdd, 0x75, 0xec, - 0x9f, 0xa5, 0xec, 0x0c, 0x1d, 0xad, 0x53, 0xe0, 0x05, 0x5e, 0x37, 0x79, 0x81, 0xf3, 0xb9, 0x33, - 0x97, 0xc3, 0x07, 0x1c, 0x5a, 0x30, 0xa6, 0xcd, 0x6c, 0xb2, 0x66, 0xad, 0x1e, 0x6b, 0xb6, 0x03, - 0xd3, 0x74, 0xa5, 0xdf, 0xde, 0x88, 0x48, 0xb8, 0x47, 0x5a, 0x6c, 0x61, 0x16, 0x1e, 0x6c, 0x61, - 0xaa, 0x57, 0xe6, 0x5b, 0x29, 0x82, 0xb8, 0xab, 0x09, 0xf4, 0xb2, 0xd4, 0x9c, 0x14, 0x0d, 0x23, - 0x2d, 0xae, 0x15, 0x39, 0x3a, 0xa8, 0x4c, 0x6b, 0x1f, 0xa2, 0x6b, 0x4a, 0xec, 0x2f, 0xc9, 0x6f, - 0x54, 0xaf, 0xf9, 0x4d, 0xb5, 0x58, 0x52, 0xaf, 0xf9, 0x6a, 0x39, 0xe0, 0x04, 0x87, 0xee, 0x51, - 0x2a, 0x82, 0xa4, 0x5f, 0xf3, 0xa9, 0x80, 0x82, 0x19, 0xc4, 0x7e, 0x01, 0x60, 0xf9, 0x3e, 0x69, - 0xf2, 0xa5, 0xae, 0x3f, 0x40, 0x5a, 0xf9, 0x0f, 0x90, 0xf6, 0xbf, 0xb5, 0x60, 0x72, 0x65, 0xc9, - 0x10, 0x13, 0xe7, 0x01, 0xb8, 0x6c, 0xf4, 0xd6, 0x5b, 0x6b, 0x52, 0xb7, 0xce, 0xd5, 0xa3, 0xaa, - 0x14, 0x6b, 0x18, 0xe8, 0x3c, 0x14, 0xbd, 0x8e, 0x2f, 0x44, 0x96, 0x91, 0xc3, 0x83, 0x4a, 0xf1, - 0x56, 0xc7, 0xc7, 0xb4, 0x4c, 0xb3, 0x10, 0x2c, 0x0e, 0x6c, 0x21, 0xd8, 0xd7, 0xbd, 0x0a, 0x55, - 0x60, 0xe8, 0xde, 0x3d, 0xb7, 0xc5, 0x8d, 0xd8, 0x85, 0xde, 0xff, 0xad, 0xb7, 0x6a, 0xd5, 0x08, - 0xf3, 0x72, 0xfb, 0xab, 0x45, 0x98, 0x5b, 0xf1, 0xc8, 0xfd, 0x0f, 0x68, 0xc8, 0x3f, 0xa8, 0x7d, - 0xe3, 0xf1, 0xf8, 0xc5, 0xe3, 0xda, 0xb0, 0xf6, 0x1f, 0x8f, 0x4d, 0x18, 0xe1, 0x8f, 0xd9, 0xd2, - 0xac, 0xff, 0xd5, 0xac, 0xd6, 0xf3, 0x07, 0x64, 0x9e, 0x3f, 0x8a, 0x0b, 0x73, 0x7e, 0x75, 0xd3, - 0x8a, 0x52, 0x2c, 0x89, 0xcf, 0x7d, 0x06, 0xc6, 0x75, 0xcc, 0x63, 0x59, 0x93, 0xff, 0x85, 0x22, - 0x4c, 0xd3, 0x1e, 0x3c, 0xd4, 0x89, 0xb8, 0xd3, 0x3d, 0x11, 0x27, 0x6d, 0x51, 0xdc, 0x7f, 0x36, - 0xde, 0x49, 0xcf, 0xc6, 0xf3, 0x79, 0xb3, 0x71, 0xda, 0x73, 0xf0, 0xfd, 0x16, 0x9c, 0x59, 0xf1, - 0x82, 0xe6, 0x4e, 0xca, 0xea, 0xf7, 0x25, 0x18, 0xa3, 0xe7, 0x78, 0x64, 0x78, 0x11, 0x19, 0x7e, - 0x65, 0x02, 0x84, 0x75, 0x3c, 0xad, 0xda, 0x9d, 0x3b, 0xb5, 0x6a, 0x96, 0x3b, 0x9a, 0x00, 0x61, - 0x1d, 0xcf, 0xfe, 0x86, 0x05, 0x17, 0xae, 0x2f, 0x2d, 0x27, 0x4b, 0xb1, 0xcb, 0x23, 0x8e, 0x4a, - 0x81, 0x2d, 0xad, 0x2b, 0x89, 0x14, 0x58, 0x65, 0xbd, 0x10, 0xd0, 0x8f, 0x8a, 0xb7, 0xe7, 0xcf, - 0x58, 0x70, 0xe6, 0xba, 0x1b, 0xd3, 0x6b, 0x39, 0xed, 0x9b, 0x45, 0xef, 0xe5, 0xc8, 0x8d, 0x83, - 0x70, 0x3f, 0xed, 0x9b, 0x85, 0x15, 0x04, 0x6b, 0x58, 0xbc, 0xe5, 0x3d, 0x97, 0x99, 0x51, 0x15, - 0x4c, 0x55, 0x14, 0x16, 0xe5, 0x58, 0x61, 0xd0, 0x0f, 0x6b, 0xb9, 0x21, 0x13, 0x25, 0xf6, 0xc5, - 0x09, 0xab, 0x3e, 0xac, 0x2a, 0x01, 0x38, 0xc1, 0xb1, 0x7f, 0xdc, 0x82, 0x73, 0xd7, 0xbd, 0x4e, - 0x14, 0x93, 0x70, 0x33, 0x32, 0x3a, 0xfb, 0x02, 0x94, 0x89, 0x14, 0xd7, 0x45, 0x5f, 0x15, 0x83, - 0xa9, 0xe4, 0x78, 0xee, 0x18, 0xa6, 0xf0, 0x06, 0xf0, 0x1c, 0x38, 0x9e, 0xeb, 0xd8, 0x2f, 0x14, - 0x60, 0xe2, 0xc6, 0xfa, 0x7a, 0xfd, 0x3a, 0x89, 0xc5, 0x2d, 0xd6, 0x5f, 0xd5, 0x8c, 0x35, 0x8d, - 0x59, 0x2f, 0xa1, 0xa8, 0x13, 0xbb, 0xde, 0x3c, 0xf7, 0x44, 0x9e, 0xaf, 0xf9, 0xf1, 0xed, 0xb0, - 0x11, 0x87, 0xae, 0xbf, 0x95, 0xa9, 0x63, 0x93, 0x77, 0x6d, 0x31, 0xef, 0xae, 0x45, 0x2f, 0xc0, - 0x30, 0x73, 0x85, 0x96, 0xe2, 0xc9, 0x63, 0x4a, 0xa6, 0x60, 0xa5, 0x47, 0x07, 0x95, 0xf2, 0x1d, - 0x5c, 0xe3, 0x7f, 0xb0, 0x40, 0x45, 0x77, 0x60, 0x6c, 0x3b, 0x8e, 0xdb, 0x37, 0x88, 0xd3, 0x22, - 0xa1, 0x3c, 0x1d, 0x2e, 0x66, 0x9d, 0x0e, 0x74, 0x10, 0x38, 0x5a, 0xb2, 0xa1, 0x92, 0xb2, 0x08, - 0xeb, 0x74, 0xec, 0x06, 0x40, 0x02, 0x3b, 0x21, 0xfd, 0x82, 0xfd, 0x87, 0x16, 0x8c, 0x70, 0xaf, - 0xb4, 0x10, 0xbd, 0x06, 0x25, 0x72, 0x9f, 0x34, 0x05, 0xe7, 0x98, 0xd9, 0xe1, 0x84, 0xf1, 0xe0, - 0xda, 0x72, 0xfa, 0x1f, 0xb3, 0x5a, 0xe8, 0x06, 0x8c, 0xd0, 0xde, 0x5e, 0x57, 0x2e, 0x7a, 0x4f, - 0xe4, 0x7d, 0xb1, 0x9a, 0x76, 0xce, 0xab, 0x88, 0x22, 0x2c, 0xab, 0x33, 0xcd, 0x6f, 0xb3, 0xdd, - 0xa0, 0x07, 0x58, 0xdc, 0xeb, 0x9e, 0x5d, 0x5f, 0xaa, 0x73, 0x24, 0x41, 0x8d, 0x6b, 0x7e, 0x65, - 0x21, 0x4e, 0x88, 0xd8, 0xeb, 0x50, 0xa6, 0x93, 0xba, 0xe0, 0xb9, 0x4e, 0x6f, 0xa5, 0xf3, 0x33, - 0x50, 0x96, 0x0a, 0xe0, 0x48, 0x38, 0x36, 0x31, 0xaa, 0x52, 0x3f, 0x1c, 0xe1, 0x04, 0x6e, 0x6f, - 0xc2, 0x59, 0xf6, 0xf2, 0xef, 0xc4, 0xdb, 0xc6, 0x1e, 0xeb, 0xbf, 0x98, 0x9f, 0x15, 0x82, 0x18, - 0x9f, 0x99, 0x59, 0xcd, 0x77, 0x60, 0x5c, 0x52, 0x4c, 0x84, 0x32, 0xfb, 0x8f, 0x4a, 0xf0, 0x58, - 0xad, 0x91, 0xef, 0xb0, 0xf8, 0x0a, 0x8c, 0x73, 0x36, 0x8d, 0x2e, 0x6d, 0xc7, 0x13, 0xed, 0xaa, - 0x77, 0xb1, 0x75, 0x0d, 0x86, 0x0d, 0x4c, 0x74, 0x01, 0x8a, 0xee, 0x7b, 0x7e, 0xda, 0x0c, 0xb7, - 0xf6, 0xe6, 0x1a, 0xa6, 0xe5, 0x14, 0x4c, 0x39, 0x3e, 0x7e, 0x94, 0x2a, 0xb0, 0xe2, 0xfa, 0x5e, - 0x87, 0x49, 0x37, 0x6a, 0x46, 0x6e, 0xcd, 0xa7, 0xe7, 0x4c, 0xe2, 0xec, 0x9a, 0x28, 0x09, 0x68, - 0xa7, 0x15, 0x14, 0xa7, 0xb0, 0xb5, 0x73, 0x7d, 0x68, 0x60, 0xae, 0xb1, 0xaf, 0xa7, 0x0f, 0x65, - 0x88, 0xdb, 0xec, 0xeb, 0x22, 0x66, 0xd4, 0x26, 0x18, 0x62, 0xfe, 0xc1, 0x11, 0x96, 0x30, 0x2a, - 0x81, 0x35, 0xb7, 0x9d, 0xf6, 0x42, 0x27, 0xde, 0xae, 0xba, 0x51, 0x33, 0xd8, 0x23, 0xe1, 0x3e, - 0x13, 0x9e, 0x47, 0x13, 0x09, 0x4c, 0x01, 0x96, 0x6e, 0x2c, 0xd4, 0x29, 0x26, 0xee, 0xae, 0x63, - 0x72, 0x85, 0x70, 0x12, 0x5c, 0xe1, 0x02, 0x4c, 0xc9, 0x66, 0x1a, 0x24, 0x62, 0x77, 0xc4, 0x18, - 0xeb, 0x98, 0x32, 0xb5, 0x15, 0xc5, 0xaa, 0x5b, 0x69, 0x7c, 0xf4, 0x32, 0x4c, 0xb8, 0xbe, 0x1b, - 0xbb, 0x4e, 0x1c, 0x84, 0xec, 0x86, 0xe5, 0x72, 0x32, 0xb3, 0x64, 0xab, 0xe9, 0x00, 0x6c, 0xe2, - 0xd9, 0xff, 0xb1, 0x04, 0x33, 0x6c, 0xda, 0xbe, 0xbd, 0xc2, 0x3e, 0x32, 0x2b, 0xec, 0x4e, 0xf7, - 0x0a, 0x3b, 0x09, 0x76, 0xf7, 0xc3, 0x5c, 0x66, 0xef, 0x42, 0x59, 0xd9, 0x02, 0x4b, 0x67, 0x00, - 0x2b, 0xc7, 0x19, 0xa0, 0x3f, 0xf7, 0x21, 0x9f, 0x71, 0x8b, 0x99, 0xcf, 0xb8, 0x7f, 0xcb, 0x82, - 0xc4, 0x24, 0x12, 0xdd, 0x80, 0x72, 0x3b, 0x60, 0x66, 0x07, 0xa1, 0xb4, 0xe5, 0x79, 0x2c, 0xf3, - 0xa2, 0xe2, 0x97, 0x22, 0x1f, 0xbf, 0xba, 0xac, 0x81, 0x93, 0xca, 0x68, 0x11, 0x46, 0xda, 0x21, - 0x69, 0xc4, 0xcc, 0x05, 0xb6, 0x2f, 0x1d, 0xbe, 0x46, 0x38, 0x3e, 0x96, 0x15, 0xed, 0x5f, 0xb4, - 0x00, 0xf8, 0x4b, 0xa9, 0xe3, 0x6f, 0x91, 0x53, 0xd0, 0xfe, 0x56, 0xa1, 0x14, 0xb5, 0x49, 0xb3, - 0x97, 0x41, 0x48, 0xd2, 0x9f, 0x46, 0x9b, 0x34, 0x93, 0x01, 0xa7, 0xff, 0x30, 0xab, 0x6d, 0xff, - 0x00, 0xc0, 0x64, 0x82, 0x56, 0x8b, 0xc9, 0x2e, 0x7a, 0xce, 0x70, 0x89, 0x3b, 0x9f, 0x72, 0x89, - 0x2b, 0x33, 0x6c, 0x4d, 0xd1, 0xf8, 0x2e, 0x14, 0x77, 0x9d, 0xfb, 0x42, 0x93, 0xf4, 0x4c, 0xef, - 0x6e, 0x50, 0xfa, 0xf3, 0xab, 0xce, 0x7d, 0x2e, 0x33, 0x3d, 0x23, 0x17, 0xc8, 0xaa, 0x73, 0xff, - 0x88, 0x9b, 0x7d, 0xb0, 0x43, 0xea, 0x96, 0x1b, 0xc5, 0x5f, 0xfe, 0x0f, 0xc9, 0x7f, 0xb6, 0xec, - 0x68, 0x23, 0xac, 0x2d, 0xd7, 0x17, 0xef, 0x86, 0x03, 0xb5, 0xe5, 0xfa, 0xe9, 0xb6, 0x5c, 0x7f, - 0x80, 0xb6, 0x5c, 0x1f, 0xbd, 0x0f, 0x23, 0xe2, 0x8d, 0x9e, 0xd9, 0x7a, 0x9b, 0x5a, 0xaa, 0xbc, - 0xf6, 0xc4, 0x13, 0x3f, 0x6f, 0xf3, 0xaa, 0x94, 0x09, 0x45, 0x69, 0xdf, 0x76, 0x65, 0x83, 0xe8, - 0xaf, 0x5b, 0x30, 0x29, 0x7e, 0x63, 0xf2, 0x5e, 0x87, 0x44, 0xb1, 0xe0, 0x3d, 0x3f, 0x3d, 0x78, - 0x1f, 0x44, 0x45, 0xde, 0x95, 0x4f, 0xcb, 0x63, 0xd6, 0x04, 0xf6, 0xed, 0x51, 0xaa, 0x17, 0xe8, - 0x1f, 0x5a, 0x70, 0x76, 0xd7, 0xb9, 0xcf, 0x5b, 0xe4, 0x65, 0xd8, 0x89, 0xdd, 0x40, 0xd8, 0xae, - 0xbf, 0x36, 0xd8, 0xf4, 0x77, 0x55, 0xe7, 0x9d, 0x94, 0x66, 0xae, 0x67, 0xb3, 0x50, 0xfa, 0x76, - 0x35, 0xb3, 0x5f, 0x73, 0x9b, 0x30, 0x2a, 0xd7, 0x5b, 0x86, 0xe4, 0x5d, 0xd5, 0x19, 0xeb, 0x63, - 0x9b, 0x48, 0xe8, 0x7e, 0x69, 0xb4, 0x1d, 0xb1, 0xd6, 0x1e, 0x6a, 0x3b, 0xef, 0xc2, 0xb8, 0xbe, - 0xc6, 0x1e, 0x6a, 0x5b, 0xef, 0xc1, 0x99, 0x8c, 0xb5, 0xf4, 0x50, 0x9b, 0xbc, 0x07, 0xe7, 0x73, - 0xd7, 0xc7, 0xc3, 0x6c, 0xd8, 0xfe, 0x05, 0x4b, 0x3f, 0x07, 0x4f, 0x41, 0x05, 0xbf, 0x64, 0xaa, - 0xe0, 0x2f, 0xf6, 0xde, 0x39, 0x39, 0x7a, 0xf8, 0x77, 0xf4, 0x4e, 0xd3, 0x53, 0x1d, 0xbd, 0x01, - 0xc3, 0x1e, 0x2d, 0x91, 0xc6, 0x21, 0x76, 0xff, 0x1d, 0x99, 0xf0, 0x52, 0xac, 0x3c, 0xc2, 0x82, - 0x82, 0xfd, 0x2b, 0x16, 0x94, 0x4e, 0x61, 0x24, 0xb0, 0x39, 0x12, 0xcf, 0xe5, 0x92, 0x16, 0x21, - 0xcd, 0xe6, 0xb1, 0x73, 0x6f, 0xf9, 0x7e, 0x4c, 0xfc, 0x88, 0x89, 0x8a, 0x99, 0x03, 0xf3, 0x3d, - 0x70, 0xe6, 0x56, 0xe0, 0xb4, 0x16, 0x1d, 0xcf, 0xf1, 0x9b, 0x24, 0xac, 0xf9, 0x5b, 0x7d, 0xad, - 0x94, 0x74, 0x9b, 0xa2, 0x42, 0x3f, 0x9b, 0x22, 0x7b, 0x1b, 0x90, 0xde, 0x80, 0xb0, 0xe3, 0xc4, - 0x30, 0xe2, 0xf2, 0xa6, 0xc4, 0xf0, 0x3f, 0x95, 0xcd, 0xdd, 0x75, 0xf5, 0x4c, 0xb3, 0x50, 0xe4, - 0x05, 0x58, 0x12, 0xb2, 0x5f, 0x81, 0x4c, 0xdf, 0xad, 0xfe, 0x6a, 0x03, 0xfb, 0xf3, 0x30, 0xc3, - 0x6a, 0x1e, 0x53, 0xa4, 0xb5, 0x53, 0x4a, 0xba, 0x8c, 0x90, 0x51, 0xf6, 0x57, 0x2c, 0x98, 0x5a, - 0x4b, 0xc5, 0xaf, 0xb8, 0xcc, 0xde, 0x03, 0x33, 0x74, 0xc3, 0x0d, 0x56, 0x8a, 0x05, 0xf4, 0xc4, - 0x75, 0x50, 0x7f, 0x66, 0x41, 0xe2, 0x4e, 0x79, 0x0a, 0x8c, 0xd7, 0x92, 0xc1, 0x78, 0x65, 0xea, - 0x46, 0x54, 0x77, 0xf2, 0xf8, 0x2e, 0x74, 0x53, 0xc5, 0x0e, 0xe8, 0xa1, 0x16, 0x49, 0xc8, 0x70, - 0x4f, 0xf3, 0x49, 0x33, 0xc0, 0x80, 0x8c, 0x26, 0xc0, 0x4c, 0x89, 0x14, 0xee, 0x47, 0xc4, 0x94, - 0x48, 0xf5, 0x27, 0x67, 0x87, 0xd6, 0xb5, 0x2e, 0xb3, 0x93, 0xeb, 0x3b, 0x99, 0x69, 0xb8, 0xe3, - 0xb9, 0xef, 0x13, 0x15, 0x00, 0xa5, 0x22, 0x4c, 0xbd, 0x45, 0xe9, 0xd1, 0x41, 0x65, 0x42, 0xfd, - 0xe3, 0x51, 0xb2, 0x92, 0x2a, 0xf6, 0x0d, 0x98, 0x4a, 0x0d, 0x18, 0x7a, 0x09, 0x86, 0xda, 0xdb, - 0x4e, 0x44, 0x52, 0xe6, 0x93, 0x43, 0x75, 0x5a, 0x78, 0x74, 0x50, 0x99, 0x54, 0x15, 0x58, 0x09, - 0xe6, 0xd8, 0xf6, 0xff, 0xb0, 0xa0, 0xb4, 0x16, 0xb4, 0x4e, 0x63, 0x31, 0xbd, 0x6e, 0x2c, 0xa6, - 0xc7, 0xf3, 0x62, 0x0c, 0xe6, 0xae, 0xa3, 0x95, 0xd4, 0x3a, 0xba, 0x98, 0x4b, 0xa1, 0xf7, 0x12, - 0xda, 0x85, 0x31, 0x16, 0xb9, 0x50, 0x98, 0x73, 0xbe, 0x60, 0xc8, 0x00, 0x95, 0x94, 0x0c, 0x30, - 0xa5, 0xa1, 0x6a, 0x92, 0xc0, 0xd3, 0x30, 0x22, 0x4c, 0x0a, 0xd3, 0x46, 0xf0, 0x02, 0x17, 0x4b, - 0xb8, 0xfd, 0x13, 0x45, 0x30, 0x22, 0x25, 0xa2, 0x5f, 0xb3, 0x60, 0x3e, 0xe4, 0x5e, 0x85, 0xad, - 0x6a, 0x27, 0x74, 0xfd, 0xad, 0x46, 0x73, 0x9b, 0xb4, 0x3a, 0x9e, 0xeb, 0x6f, 0xd5, 0xb6, 0xfc, - 0x40, 0x15, 0x2f, 0xdf, 0x27, 0xcd, 0x0e, 0x7b, 0x17, 0xe8, 0x13, 0x96, 0x51, 0x99, 0xec, 0x5c, - 0x3b, 0x3c, 0xa8, 0xcc, 0xe3, 0x63, 0xd1, 0xc6, 0xc7, 0xec, 0x0b, 0xfa, 0x86, 0x05, 0x57, 0x79, - 0x00, 0xc1, 0xc1, 0xfb, 0xdf, 0x43, 0x62, 0xaa, 0x4b, 0x52, 0x09, 0x91, 0x75, 0x12, 0xee, 0x2e, - 0xbe, 0x2c, 0x06, 0xf4, 0x6a, 0xfd, 0x78, 0x6d, 0xe1, 0xe3, 0x76, 0xce, 0xfe, 0x17, 0x45, 0x98, - 0x10, 0x0e, 0xed, 0x22, 0x52, 0xca, 0x4b, 0xc6, 0x92, 0x78, 0x22, 0xb5, 0x24, 0x66, 0x0c, 0xe4, - 0x93, 0x09, 0x92, 0x12, 0xc1, 0x8c, 0xe7, 0x44, 0xf1, 0x0d, 0xe2, 0x84, 0xf1, 0x06, 0x71, 0xb8, - 0x29, 0x4b, 0xf1, 0xd8, 0x66, 0x37, 0x4a, 0x45, 0x73, 0x2b, 0x4d, 0x0c, 0x77, 0xd3, 0x47, 0x7b, - 0x80, 0x98, 0x3d, 0x4e, 0xe8, 0xf8, 0x11, 0xff, 0x16, 0x57, 0xbc, 0x19, 0x1c, 0xaf, 0xd5, 0x39, - 0xd1, 0x2a, 0xba, 0xd5, 0x45, 0x0d, 0x67, 0xb4, 0xa0, 0xd9, 0x59, 0x0d, 0x0d, 0x6a, 0x67, 0x35, - 0xdc, 0xc7, 0xd3, 0xc4, 0x87, 0xe9, 0xae, 0x98, 0x04, 0x6f, 0x43, 0x59, 0xd9, 0xc3, 0x89, 0x43, - 0xa7, 0x77, 0x68, 0x8f, 0x34, 0x05, 0xae, 0x46, 0x49, 0x6c, 0x31, 0x13, 0x72, 0xf6, 0x3f, 0x2a, - 0x18, 0x0d, 0xf2, 0x49, 0x5c, 0x83, 0x51, 0x27, 0x8a, 0xdc, 0x2d, 0x9f, 0xb4, 0xc4, 0x8e, 0xfd, - 0x78, 0xde, 0x8e, 0x35, 0x9a, 0x61, 0x36, 0x89, 0x0b, 0xa2, 0x26, 0x56, 0x34, 0xd0, 0x0d, 0x6e, - 0x30, 0xb4, 0x27, 0x79, 0xfe, 0xc1, 0xa8, 0x81, 0x34, 0x29, 0xda, 0x23, 0x58, 0xd4, 0x47, 0x5f, - 0xe0, 0x16, 0x5d, 0x37, 0xfd, 0xe0, 0x9e, 0x7f, 0x3d, 0x08, 0xa4, 0x17, 0xda, 0x60, 0x04, 0x67, - 0xa4, 0x1d, 0x97, 0xaa, 0x8e, 0x4d, 0x6a, 0x83, 0xc5, 0xed, 0xf9, 0x5e, 0x38, 0x43, 0x49, 0x9b, - 0xbe, 0x24, 0x11, 0x22, 0x30, 0x25, 0xa2, 0x25, 0xc8, 0x32, 0x31, 0x76, 0x99, 0xec, 0xbc, 0x59, - 0x3b, 0x51, 0xfa, 0xdd, 0x34, 0x49, 0xe0, 0x34, 0x4d, 0xfb, 0xa7, 0x2d, 0x60, 0x56, 0xf0, 0xa7, - 0xc0, 0x32, 0x7c, 0xd6, 0x64, 0x19, 0x66, 0xf3, 0x06, 0x39, 0x87, 0x5b, 0x78, 0x91, 0xaf, 0xac, - 0x7a, 0x18, 0xdc, 0xdf, 0x17, 0xaf, 0xe9, 0xfd, 0x39, 0x59, 0xfb, 0xff, 0x58, 0xfc, 0x10, 0x53, - 0x8e, 0xe9, 0xe8, 0xfb, 0x60, 0xb4, 0xe9, 0xb4, 0x9d, 0x26, 0x0f, 0xeb, 0x9b, 0xab, 0xd5, 0x31, - 0x2a, 0xcd, 0x2f, 0x89, 0x1a, 0x5c, 0x4b, 0x21, 0xa3, 0x6e, 0x8c, 0xca, 0xe2, 0xbe, 0x9a, 0x09, - 0xd5, 0xe4, 0xdc, 0x0e, 0x4c, 0x18, 0xc4, 0x1e, 0xaa, 0x48, 0xfb, 0x7d, 0xfc, 0x8a, 0x55, 0x51, - 0x62, 0x76, 0x61, 0xc6, 0xd7, 0xfe, 0xd3, 0x0b, 0x45, 0x8a, 0x29, 0x1f, 0xef, 0x77, 0x89, 0xb2, - 0xdb, 0x47, 0xb3, 0xf2, 0x4f, 0x91, 0xc1, 0xdd, 0x94, 0xed, 0x9f, 0xb4, 0xe0, 0x51, 0x1d, 0x51, - 0x8b, 0x19, 0xd0, 0x4f, 0x4f, 0x5c, 0x85, 0xd1, 0xa0, 0x4d, 0x42, 0x27, 0x0e, 0x42, 0x71, 0x6b, - 0x5c, 0x91, 0x83, 0x7e, 0x5b, 0x94, 0x1f, 0x89, 0xf8, 0x8a, 0x92, 0xba, 0x2c, 0xc7, 0xaa, 0x26, - 0x95, 0x63, 0xd8, 0x60, 0x44, 0x22, 0x9e, 0x03, 0x3b, 0x03, 0xd8, 0x93, 0x69, 0x84, 0x05, 0xc4, - 0xfe, 0x23, 0x8b, 0x2f, 0x2c, 0xbd, 0xeb, 0xe8, 0x3d, 0x98, 0xde, 0x75, 0xe2, 0xe6, 0xf6, 0xf2, - 0xfd, 0x76, 0xc8, 0xd5, 0xe3, 0x72, 0x9c, 0x9e, 0xe9, 0x37, 0x4e, 0xda, 0x47, 0x26, 0x46, 0x6a, - 0xab, 0x29, 0x62, 0xb8, 0x8b, 0x3c, 0xda, 0x80, 0x31, 0x56, 0xc6, 0xac, 0xa1, 0xa3, 0x5e, 0xac, - 0x41, 0x5e, 0x6b, 0xea, 0xd5, 0x79, 0x35, 0xa1, 0x83, 0x75, 0xa2, 0xf6, 0x97, 0x8b, 0x7c, 0xb7, - 0x33, 0x6e, 0xfb, 0x69, 0x18, 0x69, 0x07, 0xad, 0xa5, 0x5a, 0x15, 0x8b, 0x59, 0x50, 0xd7, 0x48, - 0x9d, 0x17, 0x63, 0x09, 0x47, 0xaf, 0x02, 0x90, 0xfb, 0x31, 0x09, 0x7d, 0xc7, 0x53, 0x46, 0x23, - 0xca, 0x4c, 0xb2, 0x1a, 0xac, 0x05, 0xf1, 0x9d, 0x88, 0x7c, 0xcf, 0xb2, 0x42, 0xc1, 0x1a, 0x3a, - 0xba, 0x06, 0xd0, 0x0e, 0x83, 0x3d, 0xb7, 0xc5, 0xdc, 0xeb, 0x8a, 0xa6, 0x49, 0x45, 0x5d, 0x41, - 0xb0, 0x86, 0x85, 0x5e, 0x85, 0x89, 0x8e, 0x1f, 0x71, 0x0e, 0xc5, 0xd9, 0x10, 0xd1, 0x09, 0x47, - 0x13, 0xeb, 0x86, 0x3b, 0x3a, 0x10, 0x9b, 0xb8, 0x68, 0x01, 0x86, 0x63, 0x87, 0xd9, 0x44, 0x0c, - 0xe5, 0xdb, 0x36, 0xae, 0x53, 0x0c, 0x3d, 0xa8, 0x2c, 0xad, 0x80, 0x45, 0x45, 0xf4, 0xb6, 0xf4, - 0x55, 0xe0, 0x67, 0xbd, 0x30, 0x2a, 0x1e, 0xec, 0x5e, 0xd0, 0x3c, 0x15, 0x84, 0xb1, 0xb2, 0x41, - 0xcb, 0xfe, 0x46, 0x19, 0x20, 0x61, 0xc7, 0xd1, 0xfb, 0x5d, 0xe7, 0xd1, 0xb3, 0xbd, 0x19, 0xf8, - 0x93, 0x3b, 0x8c, 0xd0, 0x0f, 0x5a, 0x30, 0xe6, 0x78, 0x5e, 0xd0, 0x74, 0x62, 0x36, 0xca, 0x85, - 0xde, 0xe7, 0xa1, 0x68, 0x7f, 0x21, 0xa9, 0xc1, 0xbb, 0xf0, 0x82, 0x5c, 0x78, 0x1a, 0xa4, 0x6f, - 0x2f, 0xf4, 0x86, 0xd1, 0xa7, 0xa4, 0x94, 0xc6, 0x97, 0xc7, 0x5c, 0x5a, 0x4a, 0x2b, 0xb3, 0xa3, - 0x5f, 0x13, 0xd0, 0xd0, 0x1d, 0x23, 0xf0, 0x5c, 0x29, 0x3f, 0x06, 0x83, 0xc1, 0x95, 0xf6, 0x8b, - 0x39, 0x87, 0xea, 0xba, 0x73, 0xd5, 0x50, 0x7e, 0xa0, 0x12, 0x4d, 0xfc, 0xe9, 0xe3, 0x58, 0xf5, - 0x2e, 0x4c, 0xb5, 0xcc, 0xbb, 0x5d, 0xac, 0xa6, 0xa7, 0xf2, 0xe8, 0xa6, 0x58, 0x81, 0xe4, 0x36, - 0x4f, 0x01, 0x70, 0x9a, 0x30, 0xaa, 0x73, 0x37, 0xb7, 0x9a, 0xbf, 0x19, 0x08, 0xe3, 0x74, 0x3b, - 0x77, 0x2e, 0xf7, 0xa3, 0x98, 0xec, 0x52, 0xcc, 0xe4, 0xd2, 0x5e, 0x13, 0x75, 0xb1, 0xa2, 0x82, - 0xde, 0x80, 0x61, 0xe6, 0x27, 0x1b, 0xcd, 0x8e, 0xe6, 0x2b, 0x13, 0xcd, 0x10, 0x0f, 0xc9, 0xa6, - 0x62, 0x7f, 0x23, 0x2c, 0x28, 0xa0, 0x1b, 0x32, 0x0e, 0x4c, 0x54, 0xf3, 0xef, 0x44, 0x84, 0xc5, - 0x81, 0x29, 0x2f, 0x7e, 0x3c, 0x09, 0xf1, 0xc2, 0xcb, 0x33, 0xc3, 0xc7, 0x1b, 0x35, 0x29, 0x73, - 0x24, 0xfe, 0xcb, 0xa8, 0xf4, 0xb3, 0x90, 0xdf, 0x3d, 0x33, 0x72, 0x7d, 0x32, 0x9c, 0x77, 0x4d, - 0x12, 0x38, 0x4d, 0x93, 0x32, 0x9a, 0x7c, 0xe7, 0x0a, 0xf3, 0xf6, 0x7e, 0xfb, 0x9f, 0xcb, 0xd7, - 0xec, 0x92, 0xe1, 0x25, 0x58, 0xd4, 0x3f, 0xd5, 0x5b, 0x7f, 0xce, 0x87, 0xe9, 0xf4, 0x16, 0x7d, - 0xa8, 0x5c, 0xc6, 0x1f, 0x96, 0x60, 0xd2, 0x5c, 0x52, 0xe8, 0x2a, 0x94, 0x05, 0x11, 0x15, 0x94, - 0x54, 0xed, 0x92, 0x55, 0x09, 0xc0, 0x09, 0x0e, 0x8b, 0x45, 0xcb, 0xaa, 0x6b, 0x66, 0x89, 0x49, - 0x2c, 0x5a, 0x05, 0xc1, 0x1a, 0x16, 0x95, 0x97, 0x36, 0x82, 0x20, 0x56, 0x97, 0x8a, 0x5a, 0x77, - 0x8b, 0xac, 0x14, 0x0b, 0x28, 0xbd, 0x4c, 0x76, 0x48, 0xe8, 0x13, 0xcf, 0x8c, 0x75, 0xa6, 0x2e, - 0x93, 0x9b, 0x3a, 0x10, 0x9b, 0xb8, 0xf4, 0x96, 0x0c, 0x22, 0xb6, 0x90, 0x85, 0x54, 0x96, 0x98, - 0x79, 0x36, 0xb8, 0xc7, 0xb9, 0x84, 0xa3, 0xcf, 0xc3, 0xa3, 0xca, 0x41, 0x1c, 0x73, 0x45, 0xb5, - 0x6c, 0x71, 0xd8, 0x50, 0xa2, 0x3c, 0xba, 0x94, 0x8d, 0x86, 0xf3, 0xea, 0xa3, 0xd7, 0x61, 0x52, - 0x70, 0xee, 0x92, 0xe2, 0x88, 0x69, 0x3b, 0x71, 0xd3, 0x80, 0xe2, 0x14, 0xb6, 0x8c, 0xd6, 0xc6, - 0x98, 0x67, 0x49, 0x61, 0xb4, 0x3b, 0x5a, 0x9b, 0x0e, 0xc7, 0x5d, 0x35, 0xd0, 0x02, 0x4c, 0x71, - 0xd6, 0xca, 0xf5, 0xb7, 0xf8, 0x9c, 0x08, 0xef, 0x13, 0xb5, 0xa5, 0x6e, 0x9b, 0x60, 0x9c, 0xc6, - 0x47, 0xaf, 0xc0, 0xb8, 0x13, 0x36, 0xb7, 0xdd, 0x98, 0x34, 0xe3, 0x4e, 0xc8, 0xdd, 0x52, 0x34, - 0xe3, 0x93, 0x05, 0x0d, 0x86, 0x0d, 0x4c, 0xfb, 0x7d, 0x38, 0x93, 0xe1, 0xb8, 0x46, 0x17, 0x8e, - 0xd3, 0x76, 0xe5, 0x37, 0xa5, 0x0c, 0x36, 0x17, 0xea, 0x35, 0xf9, 0x35, 0x1a, 0x16, 0x5d, 0x9d, - 0xcc, 0xc1, 0x4d, 0x4b, 0x42, 0xa1, 0x56, 0xe7, 0x8a, 0x04, 0xe0, 0x04, 0xc7, 0xfe, 0x9f, 0x05, - 0x98, 0xca, 0x50, 0xbe, 0xb3, 0x44, 0x08, 0x29, 0xd9, 0x23, 0xc9, 0x7b, 0x60, 0x06, 0xff, 0x2b, - 0x1c, 0x23, 0xf8, 0x5f, 0xb1, 0x5f, 0xf0, 0xbf, 0xd2, 0x07, 0x09, 0xfe, 0x67, 0x8e, 0xd8, 0xd0, - 0x40, 0x23, 0x96, 0x11, 0x30, 0x70, 0xf8, 0x98, 0x01, 0x03, 0x8d, 0x41, 0x1f, 0x19, 0x60, 0xd0, - 0xbf, 0x56, 0x80, 0xe9, 0xb4, 0x91, 0xdc, 0x29, 0xa8, 0x63, 0xdf, 0x30, 0xd4, 0xb1, 0xd9, 0x69, - 0x45, 0xd2, 0xa6, 0x7b, 0x79, 0xaa, 0x59, 0x9c, 0x52, 0xcd, 0x7e, 0x72, 0x20, 0x6a, 0xbd, 0xd5, - 0xb4, 0x7f, 0xa7, 0x00, 0xe7, 0xd2, 0x55, 0x96, 0x3c, 0xc7, 0xdd, 0x3d, 0x85, 0xb1, 0xb9, 0x6d, - 0x8c, 0xcd, 0x73, 0x83, 0x7c, 0x0d, 0xeb, 0x5a, 0xee, 0x00, 0xbd, 0x95, 0x1a, 0xa0, 0xab, 0x83, - 0x93, 0xec, 0x3d, 0x4a, 0xdf, 0x2c, 0xc2, 0xc5, 0xcc, 0x7a, 0x89, 0x36, 0x73, 0xc5, 0xd0, 0x66, - 0x5e, 0x4b, 0x69, 0x33, 0xed, 0xde, 0xb5, 0x4f, 0x46, 0xbd, 0x29, 0x3c, 0x0a, 0x59, 0x80, 0xb8, - 0x07, 0x54, 0x6d, 0x1a, 0x1e, 0x85, 0x8a, 0x10, 0x36, 0xe9, 0x7e, 0x2b, 0xa9, 0x34, 0xff, 0x95, - 0x05, 0xe7, 0x33, 0xe7, 0xe6, 0x14, 0x54, 0x58, 0x6b, 0xa6, 0x0a, 0xeb, 0xe9, 0x81, 0x57, 0x6b, - 0x8e, 0x4e, 0xeb, 0xb7, 0x4a, 0x39, 0xdf, 0xc2, 0x04, 0xf4, 0xdb, 0x30, 0xe6, 0x34, 0x9b, 0x24, - 0x8a, 0x56, 0x83, 0x96, 0x0a, 0x98, 0xf6, 0x1c, 0x93, 0xb3, 0x92, 0xe2, 0xa3, 0x83, 0xca, 0x5c, - 0x9a, 0x44, 0x02, 0xc6, 0x3a, 0x05, 0x33, 0xc6, 0x63, 0xe1, 0x44, 0x63, 0x3c, 0x5e, 0x03, 0xd8, - 0x53, 0xdc, 0x7a, 0x5a, 0xc8, 0xd7, 0xf8, 0x78, 0x0d, 0x0b, 0x7d, 0x01, 0x46, 0x23, 0x71, 0x8d, - 0x8b, 0xa5, 0xf8, 0xc2, 0x80, 0x73, 0xe5, 0x6c, 0x10, 0xcf, 0x74, 0x5d, 0x57, 0xfa, 0x10, 0x45, - 0x12, 0x7d, 0x17, 0x4c, 0x47, 0x3c, 0x32, 0xca, 0x92, 0xe7, 0x44, 0xcc, 0x0f, 0x42, 0xac, 0x42, - 0xe6, 0x8f, 0xde, 0x48, 0xc1, 0x70, 0x17, 0x36, 0x5a, 0x91, 0x1f, 0xc5, 0xc2, 0xb8, 0xf0, 0x85, - 0x79, 0x39, 0xf9, 0x20, 0x91, 0x86, 0xe9, 0x6c, 0x7a, 0xf8, 0xd9, 0xc0, 0x6b, 0x35, 0xd1, 0x17, - 0x00, 0xe8, 0xf2, 0x11, 0xba, 0x84, 0x91, 0xfc, 0xc3, 0x93, 0x9e, 0x2a, 0xad, 0x4c, 0xcb, 0x4f, - 0xe6, 0xcb, 0x57, 0x55, 0x44, 0xb0, 0x46, 0xd0, 0xfe, 0x5a, 0x09, 0x1e, 0xeb, 0x71, 0x46, 0xa2, - 0x05, 0xf3, 0x09, 0xf4, 0x99, 0xb4, 0x70, 0x3d, 0x97, 0x59, 0xd9, 0x90, 0xb6, 0x53, 0x4b, 0xb1, - 0xf0, 0x81, 0x97, 0xe2, 0x0f, 0x59, 0x9a, 0xda, 0x83, 0x1b, 0xf3, 0x7d, 0xf6, 0x98, 0x67, 0xff, - 0x09, 0xea, 0x41, 0x36, 0x33, 0x94, 0x09, 0xd7, 0x06, 0xee, 0xce, 0xc0, 0xda, 0x85, 0xd3, 0x55, - 0xfe, 0x7e, 0xd9, 0x82, 0x27, 0x32, 0xfb, 0x6b, 0x98, 0x6c, 0x5c, 0x85, 0x72, 0x93, 0x16, 0x6a, - 0xae, 0x5b, 0x89, 0x4f, 0xab, 0x04, 0xe0, 0x04, 0xc7, 0xb0, 0xcc, 0x28, 0xf4, 0xb5, 0xcc, 0xf8, - 0xe7, 0x16, 0x74, 0xed, 0x8f, 0x53, 0x38, 0xa8, 0x6b, 0xe6, 0x41, 0xfd, 0xf1, 0x41, 0xe6, 0x32, - 0xe7, 0x8c, 0xfe, 0x4f, 0x53, 0xf0, 0x48, 0x8e, 0xaf, 0xc6, 0x1e, 0xcc, 0x6c, 0x35, 0x89, 0xe9, - 0x14, 0x27, 0x3e, 0x26, 0xd3, 0x7f, 0xb0, 0xa7, 0x07, 0x1d, 0x4b, 0xcf, 0x33, 0xd3, 0x85, 0x82, - 0xbb, 0x9b, 0x40, 0x5f, 0xb6, 0xe0, 0xac, 0x73, 0x2f, 0xea, 0x4a, 0xc2, 0x28, 0xd6, 0xcc, 0x8b, - 0x99, 0x4a, 0x90, 0x3e, 0x49, 0x1b, 0x79, 0xbe, 0xa2, 0x2c, 0x2c, 0x9c, 0xd9, 0x16, 0xc2, 0x22, - 0x84, 0x26, 0x65, 0xe7, 0x7b, 0xb8, 0x6d, 0x66, 0x39, 0xd5, 0xf0, 0x23, 0x5b, 0x42, 0xb0, 0xa2, - 0x83, 0xee, 0x42, 0x79, 0x4b, 0x7a, 0xba, 0x89, 0x2b, 0x21, 0xf3, 0x8e, 0xcd, 0x74, 0x87, 0xe3, - 0xcf, 0x92, 0x0a, 0x84, 0x13, 0x52, 0xe8, 0x75, 0x28, 0xfa, 0x9b, 0x51, 0xaf, 0x44, 0x3f, 0x29, - 0x4b, 0x26, 0xee, 0x12, 0xbd, 0xb6, 0xd2, 0xc0, 0xb4, 0x22, 0xba, 0x01, 0xc5, 0x70, 0xa3, 0x25, - 0xf4, 0x76, 0x99, 0x27, 0x37, 0x5e, 0xac, 0x66, 0x2f, 0x12, 0x4e, 0x09, 0x2f, 0x56, 0x31, 0x25, - 0x81, 0xea, 0x30, 0xc4, 0xdc, 0x1a, 0xc4, 0x2d, 0x90, 0xc9, 0xef, 0xf6, 0x70, 0x0f, 0xe2, 0x7e, - 0xd3, 0x0c, 0x01, 0x73, 0x42, 0x68, 0x1d, 0x86, 0x9b, 0x2c, 0x29, 0x8c, 0x88, 0xda, 0xfc, 0xa9, - 0x4c, 0x0d, 0x5d, 0x8f, 0x6c, 0x39, 0x42, 0x61, 0xc5, 0x30, 0xb0, 0xa0, 0xc5, 0xa8, 0x92, 0xf6, - 0xf6, 0x66, 0xc4, 0x24, 0xfc, 0x3c, 0xaa, 0x3d, 0x92, 0x40, 0x09, 0xaa, 0x0c, 0x03, 0x0b, 0x5a, - 0xe8, 0x33, 0x50, 0xd8, 0x6c, 0x0a, 0xaf, 0x87, 0x4c, 0x55, 0x9d, 0xe9, 0xd5, 0xbe, 0x38, 0x7c, - 0x78, 0x50, 0x29, 0xac, 0x2c, 0xe1, 0xc2, 0x66, 0x13, 0xad, 0xc1, 0xc8, 0x26, 0xf7, 0x83, 0x15, - 0xda, 0xb8, 0xa7, 0xb2, 0x5d, 0x74, 0xbb, 0x5c, 0x65, 0xb9, 0xb5, 0xbe, 0x00, 0x60, 0x49, 0x84, - 0xc5, 0xa1, 0x54, 0xfe, 0xbc, 0x22, 0x20, 0xf3, 0xfc, 0xf1, 0x7c, 0xb0, 0xf9, 0xad, 0x9c, 0x78, - 0x05, 0x63, 0x8d, 0x22, 0xfa, 0x12, 0x94, 0x1d, 0x99, 0xfe, 0x4f, 0x04, 0xac, 0x78, 0x21, 0x73, - 0x63, 0xf6, 0xce, 0x8c, 0xc8, 0x57, 0xb5, 0x42, 0xc2, 0x09, 0x51, 0xb4, 0x03, 0x13, 0x7b, 0x51, - 0x7b, 0x9b, 0xc8, 0x8d, 0xcc, 0xe2, 0x57, 0xe4, 0x5c, 0x5c, 0x77, 0x05, 0xa2, 0x1b, 0xc6, 0x1d, - 0xc7, 0xeb, 0x3a, 0x7b, 0xd8, 0x5b, 0xf6, 0x5d, 0x9d, 0x18, 0x36, 0x69, 0xd3, 0xe1, 0x7f, 0xaf, - 0x13, 0x6c, 0xec, 0xc7, 0x44, 0x44, 0x70, 0xce, 0x1c, 0xfe, 0x37, 0x39, 0x4a, 0xf7, 0xf0, 0x0b, - 0x00, 0x96, 0x44, 0xe8, 0x56, 0x77, 0x64, 0x6a, 0x4d, 0x16, 0xb9, 0x39, 0x67, 0xab, 0x67, 0xe6, - 0xdf, 0xd4, 0x06, 0x85, 0x9d, 0x91, 0x09, 0x29, 0x76, 0x36, 0xb6, 0xb7, 0x83, 0x38, 0xf0, 0x53, - 0xe7, 0xf2, 0x4c, 0xfe, 0xd9, 0x58, 0xcf, 0xc0, 0xef, 0x3e, 0x1b, 0xb3, 0xb0, 0x70, 0x66, 0x5b, - 0xa8, 0x05, 0x93, 0xed, 0x20, 0x8c, 0xef, 0x05, 0xa1, 0x5c, 0x5f, 0xa8, 0x87, 0x36, 0xc1, 0xc0, - 0x14, 0x2d, 0xb2, 0x88, 0xe2, 0x26, 0x04, 0xa7, 0x68, 0xa2, 0xcf, 0xc1, 0x48, 0xd4, 0x74, 0x3c, - 0x52, 0xbb, 0x3d, 0x7b, 0x26, 0xff, 0xd2, 0x69, 0x70, 0x94, 0x9c, 0xd5, 0xc5, 0x26, 0x47, 0xa0, - 0x60, 0x49, 0x0e, 0xad, 0xc0, 0x10, 0x4b, 0x0b, 0xc0, 0x82, 0x4f, 0xe7, 0x04, 0x46, 0xea, 0xb2, - 0x2b, 0xe5, 0x67, 0x13, 0x2b, 0xc6, 0xbc, 0x3a, 0xdd, 0x03, 0x82, 0xa9, 0x0e, 0xa2, 0xd9, 0x73, - 0xf9, 0x7b, 0x40, 0xf0, 0xe2, 0xb7, 0x1b, 0xbd, 0xf6, 0x80, 0x42, 0xc2, 0x09, 0x51, 0x7a, 0x32, - 0xd3, 0xd3, 0xf4, 0x91, 0x1e, 0x66, 0x2c, 0xb9, 0x67, 0x29, 0x3b, 0x99, 0xe9, 0x49, 0x4a, 0x49, - 0xd8, 0xbf, 0x3f, 0xd2, 0xcd, 0xa9, 0x30, 0x31, 0xec, 0x2f, 0x5a, 0x5d, 0x2f, 0x74, 0x9f, 0x1e, - 0x54, 0x2b, 0x74, 0x82, 0x3c, 0xea, 0x97, 0x2d, 0x78, 0xa4, 0x9d, 0xf9, 0x21, 0xe2, 0xda, 0x1f, - 0x4c, 0xb9, 0xc4, 0x3f, 0x5d, 0x05, 0x88, 0xcf, 0x86, 0xe3, 0x9c, 0x96, 0xd2, 0x72, 0x40, 0xf1, - 0x03, 0xcb, 0x01, 0xab, 0x30, 0xca, 0x58, 0xcb, 0x3e, 0x49, 0xd2, 0xd2, 0xe2, 0x10, 0x63, 0x20, - 0x96, 0x44, 0x45, 0xac, 0x48, 0xa0, 0x1f, 0xb6, 0xe0, 0x42, 0xba, 0xeb, 0x98, 0x30, 0xb0, 0x08, - 0xa7, 0xce, 0x25, 0xc0, 0x15, 0xf1, 0xfd, 0x17, 0xea, 0xbd, 0x90, 0x8f, 0xfa, 0x21, 0xe0, 0xde, - 0x8d, 0xa1, 0x6a, 0x86, 0x08, 0x3a, 0x6c, 0xaa, 0xdd, 0x07, 0x10, 0x43, 0x5f, 0x84, 0xf1, 0xdd, - 0xa0, 0xe3, 0xc7, 0xc2, 0xea, 0x45, 0xf8, 0x29, 0xb2, 0x67, 0xe6, 0x55, 0xad, 0x1c, 0x1b, 0x58, - 0x29, 0xe1, 0x75, 0xf4, 0x81, 0x85, 0xd7, 0x77, 0x52, 0xa9, 0xb0, 0xcb, 0xf9, 0x61, 0xfb, 0x84, - 0x9c, 0x7f, 0x8c, 0x84, 0xd8, 0xa7, 0x2b, 0x11, 0xfd, 0xac, 0x95, 0xc1, 0xca, 0x73, 0x19, 0xf9, - 0x35, 0x53, 0x46, 0xbe, 0x9c, 0x96, 0x91, 0xbb, 0x54, 0xae, 0x86, 0x78, 0x3c, 0x78, 0xec, 0xe7, - 0x41, 0x83, 0xa9, 0xd9, 0x1e, 0x5c, 0xea, 0x77, 0x2d, 0x31, 0xf3, 0xa7, 0x96, 0x7a, 0x60, 0x4b, - 0xcc, 0x9f, 0x5a, 0xb5, 0x2a, 0x66, 0x90, 0x41, 0xa3, 0x6d, 0xd8, 0xff, 0xd5, 0x82, 0x62, 0x3d, - 0x68, 0x9d, 0x82, 0x0a, 0xf9, 0xb3, 0x86, 0x0a, 0xf9, 0xb1, 0x9c, 0x14, 0xe5, 0xb9, 0x0a, 0xe3, - 0xe5, 0x94, 0xc2, 0xf8, 0x42, 0x1e, 0x81, 0xde, 0xea, 0xe1, 0x9f, 0x2a, 0x82, 0x9e, 0x50, 0x1d, - 0xfd, 0xd6, 0x83, 0xd8, 0x1e, 0x17, 0x7b, 0xe5, 0x58, 0x17, 0x94, 0x99, 0xd5, 0x94, 0x74, 0xbd, - 0xfb, 0x73, 0x66, 0x82, 0xfc, 0x16, 0x71, 0xb7, 0xb6, 0x63, 0xd2, 0x4a, 0x7f, 0xce, 0xe9, 0x99, - 0x20, 0xff, 0x67, 0x0b, 0xa6, 0x52, 0xad, 0x23, 0x0f, 0x26, 0x3c, 0x5d, 0xff, 0x27, 0xd6, 0xe9, - 0x03, 0xa9, 0x0e, 0x85, 0x09, 0xa7, 0x56, 0x84, 0x4d, 0xe2, 0x68, 0x1e, 0x40, 0xbd, 0xcf, 0x49, - 0xbd, 0x17, 0xe3, 0xfa, 0xd5, 0x03, 0x5e, 0x84, 0x35, 0x0c, 0xf4, 0x12, 0x8c, 0xc5, 0x41, 0x3b, - 0xf0, 0x82, 0xad, 0xfd, 0x9b, 0x44, 0xc6, 0x77, 0x51, 0x86, 0x59, 0xeb, 0x09, 0x08, 0xeb, 0x78, - 0xf6, 0xcf, 0x14, 0x21, 0x9d, 0x84, 0xff, 0xdb, 0x6b, 0xf2, 0xa3, 0xb9, 0x26, 0xbf, 0x69, 0xc1, - 0x34, 0x6d, 0x9d, 0x19, 0x89, 0xc8, 0xcb, 0x56, 0xe5, 0xa0, 0xb1, 0x7a, 0xe4, 0xa0, 0xb9, 0x4c, - 0xcf, 0xae, 0x56, 0xd0, 0x89, 0x85, 0xde, 0x4c, 0x3b, 0x9c, 0x68, 0x29, 0x16, 0x50, 0x81, 0x47, - 0xc2, 0x50, 0x78, 0x3e, 0xe9, 0x78, 0x24, 0x0c, 0xb1, 0x80, 0xca, 0x14, 0x35, 0xa5, 0x9c, 0x14, - 0x35, 0x2c, 0x5a, 0x9d, 0x30, 0x27, 0x10, 0x6c, 0x8f, 0x16, 0xad, 0x4e, 0xda, 0x19, 0x24, 0x38, - 0xf6, 0x2f, 0x14, 0x61, 0xbc, 0x1e, 0xb4, 0x92, 0x17, 0xb2, 0x17, 0x8d, 0x17, 0xb2, 0x4b, 0xa9, - 0x17, 0xb2, 0x69, 0x1d, 0xf7, 0xdb, 0xef, 0x61, 0x1f, 0xd6, 0x7b, 0xd8, 0x3f, 0xb3, 0xd8, 0xac, - 0x55, 0xd7, 0x1a, 0x22, 0x45, 0xee, 0xf3, 0x30, 0xc6, 0x0e, 0x24, 0xe6, 0x6a, 0x27, 0x9f, 0x8d, - 0x58, 0xf4, 0xf9, 0xb5, 0xa4, 0x18, 0xeb, 0x38, 0xe8, 0x0a, 0x8c, 0x46, 0xc4, 0x09, 0x9b, 0xdb, - 0xea, 0x8c, 0x13, 0x8f, 0x2a, 0xbc, 0x0c, 0x2b, 0x28, 0x7a, 0x33, 0x09, 0x94, 0x56, 0xcc, 0x4f, - 0xf6, 0xaa, 0xf7, 0x87, 0x6f, 0x91, 0xfc, 0xe8, 0x68, 0xf6, 0x5b, 0x80, 0xba, 0xf1, 0x07, 0x08, - 0x89, 0x54, 0x31, 0x43, 0x22, 0x95, 0xbb, 0xc2, 0x21, 0xfd, 0xa9, 0x05, 0x93, 0xf5, 0xa0, 0x45, - 0xb7, 0xee, 0xb7, 0xd2, 0x3e, 0xd5, 0xa3, 0x44, 0x0e, 0xf7, 0x88, 0x12, 0xf9, 0x77, 0x2d, 0x18, - 0xa9, 0x07, 0xad, 0x53, 0xd0, 0xb6, 0xbf, 0x66, 0x6a, 0xdb, 0x1f, 0xcd, 0x59, 0x12, 0x39, 0x0a, - 0xf6, 0x5f, 0x2a, 0xc2, 0x04, 0xed, 0x67, 0xb0, 0x25, 0x67, 0xc9, 0x18, 0x11, 0x6b, 0x80, 0x11, - 0xa1, 0x6c, 0x6e, 0xe0, 0x79, 0xc1, 0xbd, 0xf4, 0x8c, 0xad, 0xb0, 0x52, 0x2c, 0xa0, 0xe8, 0x59, - 0x18, 0x6d, 0x87, 0x64, 0xcf, 0x0d, 0x04, 0xff, 0xa8, 0xbd, 0x5d, 0xd4, 0x45, 0x39, 0x56, 0x18, - 0x54, 0xee, 0x8a, 0x5c, 0xbf, 0x49, 0x64, 0xa6, 0xe9, 0x12, 0x4b, 0x46, 0xc5, 0xc3, 0x3f, 0x6b, - 0xe5, 0xd8, 0xc0, 0x42, 0x6f, 0x41, 0x99, 0xfd, 0x67, 0x27, 0xca, 0xf1, 0x93, 0xe7, 0x88, 0x9c, - 0x0b, 0x82, 0x00, 0x4e, 0x68, 0xa1, 0x6b, 0x00, 0xb1, 0x0c, 0x11, 0x1c, 0x89, 0xc8, 0x36, 0x8a, - 0xd7, 0x56, 0xc1, 0x83, 0x23, 0xac, 0x61, 0xa1, 0x67, 0xa0, 0x1c, 0x3b, 0xae, 0x77, 0xcb, 0xf5, - 0x49, 0xc4, 0x54, 0xce, 0x45, 0x99, 0x52, 0x41, 0x14, 0xe2, 0x04, 0x4e, 0x79, 0x1d, 0xe6, 0xf6, - 0xcd, 0x53, 0x6f, 0x8d, 0x32, 0x6c, 0xc6, 0xeb, 0xdc, 0x52, 0xa5, 0x58, 0xc3, 0xb0, 0x5f, 0x81, - 0x73, 0xf5, 0xa0, 0x55, 0x0f, 0xc2, 0x78, 0x25, 0x08, 0xef, 0x39, 0x61, 0x4b, 0xce, 0x5f, 0x45, - 0x46, 0xf7, 0xa7, 0x67, 0xcf, 0x10, 0xdf, 0x99, 0x46, 0xdc, 0xfe, 0x17, 0x18, 0xb7, 0x73, 0x4c, - 0x57, 0x8e, 0x26, 0xbb, 0x77, 0x55, 0x96, 0xbd, 0xeb, 0x4e, 0x4c, 0xd0, 0x6d, 0x96, 0x99, 0x2b, - 0xb9, 0x82, 0x44, 0xf5, 0xa7, 0xb5, 0xcc, 0x5c, 0x09, 0x30, 0xf3, 0xce, 0x32, 0xeb, 0xdb, 0xbf, - 0x5e, 0x64, 0xa7, 0x51, 0x2a, 0xe9, 0x1c, 0xfa, 0x22, 0x4c, 0x46, 0xe4, 0x96, 0xeb, 0x77, 0xee, - 0x4b, 0x21, 0xbc, 0x87, 0x33, 0x4e, 0x63, 0x59, 0xc7, 0xe4, 0xaa, 0x3c, 0xb3, 0x0c, 0xa7, 0xa8, - 0xd1, 0x79, 0x0a, 0x3b, 0xfe, 0x42, 0x74, 0x27, 0x22, 0xa1, 0x48, 0x7a, 0xc6, 0xe6, 0x09, 0xcb, - 0x42, 0x9c, 0xc0, 0xe9, 0xba, 0x64, 0x7f, 0xd6, 0x02, 0x1f, 0x07, 0x41, 0x2c, 0x57, 0x32, 0x4b, - 0x9b, 0xa3, 0x95, 0x63, 0x03, 0x0b, 0xad, 0x00, 0x8a, 0x3a, 0xed, 0xb6, 0xc7, 0x9e, 0xf3, 0x1d, - 0xef, 0x7a, 0x18, 0x74, 0xda, 0xfc, 0xad, 0xb3, 0xb8, 0xf8, 0x08, 0xbd, 0xc2, 0x1a, 0x5d, 0x50, - 0x9c, 0x51, 0x83, 0x9e, 0x3e, 0x9b, 0x11, 0xfb, 0xcd, 0x56, 0x77, 0x51, 0xa8, 0xd7, 0x1b, 0xac, - 0x08, 0x4b, 0x18, 0x5d, 0x4c, 0xac, 0x79, 0x8e, 0x39, 0x9c, 0x2c, 0x26, 0xac, 0x4a, 0xb1, 0x86, - 0x81, 0x96, 0x61, 0x24, 0xda, 0x8f, 0x9a, 0xb1, 0x88, 0xc3, 0x94, 0x93, 0xbe, 0xb2, 0xc1, 0x50, - 0xb4, 0x94, 0x0a, 0xbc, 0x0a, 0x96, 0x75, 0xed, 0xef, 0x63, 0x97, 0x21, 0x4b, 0x91, 0x15, 0x77, - 0x42, 0x82, 0x76, 0x61, 0xa2, 0xcd, 0xa6, 0x5c, 0x04, 0x70, 0x16, 0xf3, 0xf6, 0xe2, 0x80, 0x52, - 0xed, 0x3d, 0x7a, 0xd0, 0x28, 0xad, 0x13, 0x13, 0x17, 0xea, 0x3a, 0x39, 0x6c, 0x52, 0xb7, 0xbf, - 0x86, 0xd8, 0x99, 0xdb, 0xe0, 0xa2, 0xea, 0x88, 0x30, 0x28, 0x16, 0x7c, 0xf9, 0x5c, 0xbe, 0xce, - 0x24, 0xf9, 0x22, 0x61, 0x94, 0x8c, 0x65, 0x5d, 0xf4, 0x26, 0x7b, 0x9b, 0xe6, 0x07, 0x5d, 0xbf, - 0x4c, 0xc5, 0x1c, 0xcb, 0x78, 0x86, 0x16, 0x15, 0xb1, 0x46, 0x04, 0xdd, 0x82, 0x09, 0x91, 0x51, - 0x49, 0x28, 0xc5, 0x8a, 0x86, 0xd2, 0x63, 0x02, 0xeb, 0xc0, 0xa3, 0x74, 0x01, 0x36, 0x2b, 0xa3, - 0x2d, 0xb8, 0xa0, 0xa5, 0x17, 0xbc, 0x1e, 0x3a, 0xec, 0xbd, 0xd2, 0x65, 0x9b, 0x48, 0x3b, 0x37, - 0x9f, 0x38, 0x3c, 0xa8, 0x5c, 0x58, 0xef, 0x85, 0x88, 0x7b, 0xd3, 0x41, 0xb7, 0xe1, 0x1c, 0xf7, - 0xdb, 0xab, 0x12, 0xa7, 0xe5, 0xb9, 0xbe, 0x3a, 0x98, 0xf9, 0x3a, 0x3c, 0x7f, 0x78, 0x50, 0x39, - 0xb7, 0x90, 0x85, 0x80, 0xb3, 0xeb, 0xa1, 0xd7, 0xa0, 0xdc, 0xf2, 0x23, 0x31, 0x06, 0xc3, 0x46, - 0xe6, 0xcc, 0x72, 0x75, 0xad, 0xa1, 0xbe, 0x3f, 0xf9, 0x83, 0x93, 0x0a, 0x68, 0x8b, 0x2b, 0xc6, - 0x94, 0x1c, 0x3a, 0x92, 0x9f, 0x25, 0x5d, 0x2c, 0x09, 0xc3, 0x73, 0x87, 0x6b, 0x84, 0x95, 0xe5, - 0xab, 0xe1, 0xd4, 0x63, 0x10, 0x46, 0x6f, 0x00, 0xa2, 0x8c, 0x9a, 0xdb, 0x24, 0x0b, 0x4d, 0x16, - 0x47, 0x9b, 0xe9, 0x11, 0x47, 0x0d, 0x4f, 0x09, 0xd4, 0xe8, 0xc2, 0xc0, 0x19, 0xb5, 0xd0, 0x0d, - 0x7a, 0x90, 0xe9, 0xa5, 0xc2, 0x82, 0x57, 0x32, 0xf7, 0xb3, 0x55, 0xd2, 0x0e, 0x49, 0xd3, 0x89, - 0x49, 0xcb, 0xa4, 0x88, 0x53, 0xf5, 0xe8, 0x5d, 0xaa, 0x52, 0xea, 0x80, 0x19, 0x2c, 0xa3, 0x3b, - 0xad, 0x0e, 0x95, 0x8b, 0xb7, 0x83, 0x28, 0x5e, 0x23, 0xf1, 0xbd, 0x20, 0xdc, 0x11, 0xb1, 0xc9, - 0x92, 0x30, 0x99, 0x09, 0x08, 0xeb, 0x78, 0x94, 0x0f, 0x66, 0x8f, 0xc3, 0xb5, 0x2a, 0x7b, 0xa1, - 0x1b, 0x4d, 0xf6, 0xc9, 0x0d, 0x5e, 0x8c, 0x25, 0x5c, 0xa2, 0xd6, 0xea, 0x4b, 0xec, 0xb5, 0x2d, - 0x85, 0x5a, 0xab, 0x2f, 0x61, 0x09, 0x47, 0xa4, 0x3b, 0x2b, 0xe9, 0x64, 0xbe, 0x56, 0xb3, 0xfb, - 0x3a, 0x18, 0x30, 0x31, 0xa9, 0x0f, 0xd3, 0x2a, 0x1f, 0x2a, 0x0f, 0xda, 0x16, 0xcd, 0x4e, 0xb1, - 0x45, 0x32, 0x78, 0xc4, 0x37, 0xa5, 0x27, 0xae, 0xa5, 0x28, 0xe1, 0x2e, 0xda, 0x46, 0xf8, 0x92, - 0xe9, 0xbe, 0x29, 0x91, 0xae, 0x42, 0x39, 0xea, 0x6c, 0xb4, 0x82, 0x5d, 0xc7, 0xf5, 0xd9, 0xe3, - 0x98, 0xc6, 0x64, 0x35, 0x24, 0x00, 0x27, 0x38, 0x68, 0x05, 0x46, 0x1d, 0xa9, 0x04, 0x46, 0xf9, - 0xb1, 0x0a, 0x94, 0xea, 0x97, 0xbb, 0xef, 0x4a, 0xb5, 0xaf, 0xaa, 0x8b, 0x5e, 0x85, 0x09, 0xe1, - 0xad, 0xc5, 0x23, 0x38, 0xb0, 0xc7, 0x2b, 0xcd, 0x1c, 0xbf, 0xa1, 0x03, 0xb1, 0x89, 0x8b, 0xbe, - 0x00, 0x93, 0x94, 0x4a, 0x72, 0xb0, 0xcd, 0x9e, 0x1d, 0xe4, 0x44, 0xd4, 0x52, 0x5d, 0xe8, 0x95, - 0x71, 0x8a, 0x18, 0x6a, 0xc1, 0xe3, 0x4e, 0x27, 0x0e, 0x98, 0x22, 0xdd, 0x5c, 0xff, 0xeb, 0xc1, - 0x0e, 0xf1, 0xd9, 0x1b, 0xd6, 0xe8, 0xe2, 0xa5, 0xc3, 0x83, 0xca, 0xe3, 0x0b, 0x3d, 0xf0, 0x70, - 0x4f, 0x2a, 0xe8, 0x0e, 0x8c, 0xc5, 0x81, 0xc7, 0x0c, 0xe3, 0x29, 0x2b, 0xf1, 0x48, 0x7e, 0xf8, - 0x9f, 0x75, 0x85, 0xa6, 0x2b, 0x91, 0x54, 0x55, 0xac, 0xd3, 0x41, 0xeb, 0x7c, 0x8f, 0xb1, 0xc0, - 0xa8, 0x24, 0x9a, 0x7d, 0x34, 0x7f, 0x60, 0x54, 0xfc, 0x54, 0x73, 0x0b, 0x8a, 0x9a, 0x58, 0x27, - 0x83, 0xae, 0xc3, 0x4c, 0x3b, 0x74, 0x03, 0xb6, 0xb0, 0xd5, 0x23, 0xc6, 0xac, 0x99, 0xdd, 0xa0, - 0x9e, 0x46, 0xc0, 0xdd, 0x75, 0xa8, 0x90, 0x29, 0x0b, 0x67, 0xcf, 0xf3, 0x54, 0x59, 0x9c, 0xf1, - 0xe6, 0x65, 0x58, 0x41, 0xd1, 0x2a, 0x3b, 0x97, 0xb9, 0x38, 0x38, 0x3b, 0x97, 0x1f, 0xe3, 0x41, - 0x17, 0x1b, 0x39, 0xbf, 0xa4, 0xfe, 0xe2, 0x84, 0x02, 0xbd, 0x37, 0xa2, 0x6d, 0x27, 0x24, 0xf5, - 0x30, 0x68, 0x12, 0xde, 0x19, 0x6e, 0x93, 0xff, 0x18, 0x8f, 0xdf, 0x48, 0xef, 0x8d, 0x46, 0x16, - 0x02, 0xce, 0xae, 0x87, 0x5a, 0x5a, 0x86, 0x68, 0xca, 0x86, 0x46, 0xb3, 0x8f, 0xf7, 0x30, 0x33, - 0x4a, 0xf1, 0xac, 0xc9, 0x5a, 0x34, 0x8a, 0x23, 0x9c, 0xa2, 0x89, 0xbe, 0x0b, 0xa6, 0x45, 0xb8, - 0xa3, 0x64, 0xdc, 0x2f, 0x24, 0xf6, 0x8b, 0x38, 0x05, 0xc3, 0x5d, 0xd8, 0x94, 0xe5, 0x23, 0xbe, - 0xb3, 0xe1, 0x11, 0xb1, 0x08, 0x6f, 0xb9, 0xfe, 0x4e, 0x34, 0x7b, 0x91, 0x7d, 0x35, 0x63, 0xf9, - 0x96, 0xbb, 0xa0, 0x38, 0xa3, 0xc6, 0xdc, 0x77, 0xc2, 0x4c, 0xd7, 0xcd, 0x75, 0xac, 0x20, 0xe6, - 0x7f, 0x32, 0x04, 0x65, 0xa5, 0x94, 0x47, 0x57, 0xcd, 0xb7, 0x96, 0xf3, 0xe9, 0xb7, 0x96, 0x51, - 0x2a, 0x1b, 0xe8, 0xcf, 0x2b, 0xeb, 0x86, 0x79, 0x5e, 0x21, 0x3f, 0x65, 0x98, 0xce, 0xdd, 0xf7, - 0x75, 0xf5, 0xd3, 0x74, 0x2c, 0xc5, 0x81, 0x1f, 0x6d, 0x4a, 0x3d, 0xd5, 0x36, 0x03, 0x66, 0xec, - 0x45, 0x4f, 0x52, 0x01, 0xa9, 0x55, 0xab, 0xa7, 0x53, 0x58, 0xd6, 0x69, 0x21, 0xe6, 0x30, 0x26, - 0x48, 0x52, 0x36, 0x8b, 0x09, 0x92, 0x23, 0x0f, 0x28, 0x48, 0x4a, 0x02, 0x38, 0xa1, 0x85, 0x3c, - 0x98, 0x69, 0x9a, 0xd9, 0x47, 0x95, 0x7b, 0xdf, 0x93, 0x7d, 0xf3, 0x80, 0x76, 0xb4, 0x54, 0x6f, - 0x4b, 0x69, 0x2a, 0xb8, 0x9b, 0x30, 0x7a, 0x15, 0x46, 0xdf, 0x0b, 0x22, 0xb6, 0x28, 0x05, 0xaf, - 0x21, 0xdd, 0xa0, 0x46, 0xdf, 0xbc, 0xdd, 0x60, 0xe5, 0x47, 0x07, 0x95, 0xb1, 0x7a, 0xd0, 0x92, - 0x7f, 0xb1, 0xaa, 0x80, 0xee, 0xc3, 0x39, 0xe3, 0x84, 0x56, 0xdd, 0x85, 0xc1, 0xbb, 0x7b, 0x41, - 0x34, 0x77, 0xae, 0x96, 0x45, 0x09, 0x67, 0x37, 0x40, 0x8f, 0x3d, 0x3f, 0x10, 0x99, 0x7b, 0x25, - 0x3f, 0xc3, 0xd8, 0x96, 0xb2, 0xee, 0x04, 0x9f, 0x42, 0xc0, 0xdd, 0x75, 0xec, 0x5f, 0xe5, 0x6f, - 0x18, 0x42, 0xd3, 0x49, 0xa2, 0x8e, 0x77, 0x1a, 0x89, 0xa1, 0x96, 0x0d, 0x25, 0xec, 0x03, 0xbf, - 0x93, 0xfd, 0xa6, 0xc5, 0xde, 0xc9, 0xd6, 0xc9, 0x6e, 0xdb, 0xa3, 0xf2, 0xf6, 0xc3, 0xef, 0xf8, - 0x9b, 0x30, 0x1a, 0x8b, 0xd6, 0x7a, 0xe5, 0xb2, 0xd2, 0x3a, 0xc5, 0xde, 0x0a, 0x15, 0xa7, 0x23, - 0x4b, 0xb1, 0x22, 0x63, 0xff, 0x13, 0x3e, 0x03, 0x12, 0x72, 0x0a, 0x0a, 0xb1, 0xaa, 0xa9, 0x10, - 0xab, 0xf4, 0xf9, 0x82, 0x1c, 0xc5, 0xd8, 0x3f, 0x36, 0xfb, 0xcd, 0x84, 0xca, 0x8f, 0xfa, 0x03, - 0xad, 0xfd, 0xa3, 0x16, 0x9c, 0xcd, 0xb2, 0x68, 0xa2, 0xdc, 0x29, 0x17, 0x69, 0xd5, 0x83, 0xb5, - 0x1a, 0xc1, 0xbb, 0xa2, 0x1c, 0x2b, 0x8c, 0x81, 0xd3, 0x44, 0x1c, 0x2f, 0x4e, 0xdc, 0x6d, 0x98, - 0xa8, 0x87, 0x44, 0xbb, 0x03, 0x5e, 0xe7, 0xfe, 0x74, 0xbc, 0x3f, 0xcf, 0x1e, 0xdb, 0x97, 0xce, - 0xfe, 0xb9, 0x02, 0x9c, 0xe5, 0x2f, 0x4e, 0x0b, 0x7b, 0x81, 0xdb, 0xaa, 0x07, 0x2d, 0x91, 0xe2, - 0xe3, 0x6d, 0x18, 0x6f, 0x6b, 0x7a, 0x88, 0x5e, 0x91, 0xaa, 0x74, 0x7d, 0x45, 0x22, 0x0f, 0xea, - 0xa5, 0xd8, 0xa0, 0x85, 0x5a, 0x30, 0x4e, 0xf6, 0xdc, 0xa6, 0x7a, 0xb6, 0x28, 0x1c, 0xfb, 0x6e, - 0x50, 0xad, 0x2c, 0x6b, 0x74, 0xb0, 0x41, 0xf5, 0x21, 0x64, 0x7d, 0xb3, 0x7f, 0xcc, 0x82, 0x47, - 0x73, 0xe2, 0x5a, 0xd1, 0xe6, 0xee, 0xb1, 0xb7, 0x3d, 0x91, 0x40, 0x4a, 0x35, 0xc7, 0x5f, 0xfc, - 0xb0, 0x80, 0xa2, 0xcf, 0x01, 0xf0, 0x17, 0x3b, 0x2a, 0x1e, 0xf5, 0x0b, 0x00, 0x64, 0xc4, 0x2e, - 0xd1, 0x62, 0x4e, 0xc8, 0xfa, 0x58, 0xa3, 0x65, 0xff, 0x74, 0x11, 0x86, 0xd8, 0x0b, 0x11, 0x5a, - 0x81, 0x91, 0x6d, 0x1e, 0xe9, 0x79, 0x90, 0xa0, 0xd2, 0x89, 0x9c, 0xc9, 0x0b, 0xb0, 0xac, 0x8c, - 0x56, 0xe1, 0x0c, 0x8f, 0x94, 0xed, 0x55, 0x89, 0xe7, 0xec, 0x4b, 0x75, 0x05, 0x4f, 0xba, 0xa4, - 0xe2, 0x67, 0xd4, 0xba, 0x51, 0x70, 0x56, 0x3d, 0xf4, 0x3a, 0x4c, 0x52, 0xfe, 0x2e, 0xe8, 0xc4, - 0x92, 0x12, 0x8f, 0x91, 0xad, 0x18, 0xca, 0x75, 0x03, 0x8a, 0x53, 0xd8, 0x54, 0xf0, 0x6a, 0x77, - 0x29, 0x66, 0x86, 0x12, 0xc1, 0xcb, 0x54, 0xc6, 0x98, 0xb8, 0xcc, 0x94, 0xa9, 0xc3, 0x0c, 0xb7, - 0xd6, 0xb7, 0x43, 0x12, 0x6d, 0x07, 0x5e, 0x4b, 0xe4, 0xec, 0x4e, 0x4c, 0x99, 0x52, 0x70, 0xdc, - 0x55, 0x83, 0x52, 0xd9, 0x74, 0x5c, 0xaf, 0x13, 0x92, 0x84, 0xca, 0xb0, 0x49, 0x65, 0x25, 0x05, - 0xc7, 0x5d, 0x35, 0xe8, 0x3a, 0x3a, 0x27, 0x92, 0x68, 0x4b, 0xaf, 0x7e, 0x65, 0x9f, 0x36, 0x22, - 0xfd, 0x9b, 0x7a, 0x84, 0xb5, 0x11, 0x16, 0x3c, 0x2a, 0x0d, 0xb7, 0xa6, 0x4f, 0x14, 0x9e, 0x4d, - 0x92, 0xca, 0x83, 0xa4, 0x72, 0xfe, 0x7d, 0x0b, 0xce, 0x64, 0xd8, 0xc1, 0xf2, 0xa3, 0x6a, 0xcb, - 0x8d, 0x62, 0x95, 0x58, 0x46, 0x3b, 0xaa, 0x78, 0x39, 0x56, 0x18, 0x74, 0x3f, 0xf0, 0xc3, 0x30, - 0x7d, 0x00, 0x0a, 0x3b, 0x33, 0x01, 0x3d, 0xde, 0x01, 0x88, 0x2e, 0x41, 0xa9, 0x13, 0x11, 0x19, - 0x90, 0x4a, 0x9d, 0xdf, 0x4c, 0xc3, 0xcc, 0x20, 0x94, 0x35, 0xdd, 0x52, 0xca, 0x5d, 0x8d, 0x35, - 0xe5, 0x1a, 0x5b, 0x0e, 0xb3, 0xbf, 0x5a, 0x84, 0xf3, 0xb9, 0x16, 0xef, 0xb4, 0x4b, 0xbb, 0x81, - 0xef, 0xc6, 0x81, 0x7a, 0x7d, 0xe4, 0x21, 0x51, 0x48, 0x7b, 0x7b, 0x55, 0x94, 0x63, 0x85, 0x81, - 0x2e, 0xcb, 0x74, 0xee, 0xe9, 0xd4, 0x39, 0x8b, 0x55, 0x23, 0xa3, 0xfb, 0xa0, 0x69, 0xc9, 0x9e, - 0x84, 0x52, 0x3b, 0x08, 0xbc, 0xf4, 0x61, 0x44, 0xbb, 0x1b, 0x04, 0x1e, 0x66, 0x40, 0xf4, 0x09, - 0x31, 0x0e, 0xa9, 0xe7, 0x36, 0xec, 0xb4, 0x82, 0x48, 0x1b, 0x8c, 0xa7, 0x61, 0x64, 0x87, 0xec, - 0x87, 0xae, 0xbf, 0x95, 0x7e, 0x86, 0xbd, 0xc9, 0x8b, 0xb1, 0x84, 0x9b, 0x99, 0x23, 0x46, 0x4e, - 0x3a, 0x9f, 0xd8, 0x68, 0xdf, 0xab, 0xed, 0x87, 0x8a, 0x30, 0x85, 0x17, 0xab, 0xdf, 0x9e, 0x88, - 0x3b, 0xdd, 0x13, 0x71, 0xd2, 0xf9, 0xc4, 0xfa, 0xcf, 0xc6, 0x2f, 0x59, 0x30, 0xc5, 0xa2, 0x2b, - 0x8b, 0x40, 0x1c, 0x6e, 0xe0, 0x9f, 0x02, 0xeb, 0xf6, 0x24, 0x0c, 0x85, 0xb4, 0xd1, 0x74, 0x92, - 0x20, 0xd6, 0x13, 0xcc, 0x61, 0xe8, 0x71, 0x28, 0xb1, 0x2e, 0xd0, 0xc9, 0x1b, 0xe7, 0xf9, 0x15, - 0xaa, 0x4e, 0xec, 0x60, 0x56, 0xca, 0xbc, 0xcb, 0x31, 0x69, 0x7b, 0x2e, 0xef, 0x74, 0xf2, 0xb4, - 0xf1, 0xd1, 0xf0, 0x2e, 0xcf, 0xec, 0xda, 0x07, 0xf3, 0x2e, 0xcf, 0x26, 0xd9, 0x5b, 0x2c, 0xfa, - 0x6f, 0x05, 0xb8, 0x98, 0x59, 0x6f, 0x60, 0xef, 0xf2, 0xde, 0xb5, 0x4f, 0xc6, 0x9a, 0x26, 0xdb, - 0xc8, 0xa5, 0x78, 0x8a, 0x46, 0x2e, 0xa5, 0x41, 0x39, 0xc7, 0xa1, 0x01, 0x9c, 0xbe, 0x33, 0x87, - 0xec, 0x23, 0xe2, 0xf4, 0x9d, 0xd9, 0xb7, 0x1c, 0xb1, 0xee, 0xcf, 0x0a, 0x39, 0xdf, 0xc2, 0x04, - 0xbc, 0x2b, 0xf4, 0x9c, 0x61, 0xc0, 0x48, 0x70, 0xc2, 0xe3, 0xfc, 0x8c, 0xe1, 0x65, 0x58, 0x41, - 0x91, 0xab, 0xb9, 0x4f, 0x17, 0xf2, 0x53, 0x48, 0xe6, 0x36, 0x35, 0x6f, 0xbe, 0x44, 0xa9, 0x21, - 0xc8, 0x70, 0xa5, 0x5e, 0xd5, 0x84, 0xf2, 0xe2, 0xe0, 0x42, 0xf9, 0x78, 0xb6, 0x40, 0x8e, 0x16, - 0x60, 0x6a, 0xd7, 0xf5, 0xe9, 0xb1, 0xb9, 0x6f, 0xb2, 0xa2, 0x2a, 0x9a, 0xc8, 0xaa, 0x09, 0xc6, - 0x69, 0xfc, 0xb9, 0x57, 0x61, 0xe2, 0xc1, 0xd5, 0x91, 0xdf, 0x2c, 0xc2, 0x63, 0x3d, 0xb6, 0x3d, - 0x3f, 0xeb, 0x8d, 0x39, 0xd0, 0xce, 0xfa, 0xae, 0x79, 0xa8, 0xc3, 0xd9, 0xcd, 0x8e, 0xe7, 0xed, - 0x33, 0x3b, 0x52, 0xd2, 0x92, 0x18, 0x82, 0x57, 0x7c, 0x5c, 0x66, 0xb4, 0x58, 0xc9, 0xc0, 0xc1, - 0x99, 0x35, 0xd1, 0x1b, 0x80, 0x02, 0x91, 0xbf, 0xf6, 0x3a, 0xf1, 0x85, 0x7e, 0x9f, 0x0d, 0x7c, - 0x31, 0xd9, 0x8c, 0xb7, 0xbb, 0x30, 0x70, 0x46, 0x2d, 0xca, 0xf4, 0xd3, 0x5b, 0x69, 0x5f, 0x75, - 0x2b, 0xc5, 0xf4, 0x63, 0x1d, 0x88, 0x4d, 0x5c, 0x74, 0x1d, 0x66, 0x9c, 0x3d, 0xc7, 0xe5, 0x51, - 0xf6, 0x24, 0x01, 0xce, 0xf5, 0x2b, 0x25, 0xd8, 0x42, 0x1a, 0x01, 0x77, 0xd7, 0x49, 0x39, 0x58, - 0x0f, 0xe7, 0x3b, 0x58, 0xf7, 0x3e, 0x17, 0xfb, 0xe9, 0x74, 0xed, 0x7f, 0x6f, 0xd1, 0xeb, 0x2b, - 0x23, 0x07, 0x3d, 0x1d, 0x07, 0xa5, 0x9b, 0xd4, 0x7c, 0x9d, 0xcf, 0x69, 0x96, 0x22, 0x09, 0x10, - 0x9b, 0xb8, 0x7c, 0x41, 0x44, 0x89, 0xb3, 0x8d, 0xc1, 0xba, 0x8b, 0x58, 0x09, 0x0a, 0x03, 0x7d, - 0x1e, 0x46, 0x5a, 0xee, 0x9e, 0x1b, 0x05, 0xa1, 0xd8, 0x2c, 0xc7, 0x74, 0x59, 0x48, 0xce, 0xc1, - 0x2a, 0x27, 0x83, 0x25, 0x3d, 0xfb, 0x87, 0x0a, 0x30, 0x21, 0x5b, 0x7c, 0xb3, 0x13, 0xc4, 0xce, - 0x29, 0x5c, 0xcb, 0xd7, 0x8d, 0x6b, 0xf9, 0x13, 0xbd, 0x02, 0x46, 0xb0, 0x2e, 0xe5, 0x5e, 0xc7, - 0xb7, 0x53, 0xd7, 0xf1, 0x53, 0xfd, 0x49, 0xf5, 0xbe, 0x86, 0xff, 0xa9, 0x05, 0x33, 0x06, 0xfe, - 0x29, 0xdc, 0x06, 0x2b, 0xe6, 0x6d, 0xf0, 0x44, 0xdf, 0x6f, 0xc8, 0xb9, 0x05, 0x7e, 0xa0, 0x98, - 0xea, 0x3b, 0x3b, 0xfd, 0xdf, 0x83, 0xd2, 0xb6, 0x13, 0xb6, 0x7a, 0x05, 0xa6, 0xed, 0xaa, 0x34, - 0x7f, 0xc3, 0x09, 0x5b, 0xfc, 0x0c, 0x7f, 0x56, 0x65, 0xbd, 0x74, 0xc2, 0x56, 0x5f, 0xdf, 0x32, - 0xd6, 0x14, 0x7a, 0x05, 0x86, 0xa3, 0x66, 0xd0, 0x56, 0x96, 0x9f, 0x97, 0x78, 0x46, 0x4c, 0x5a, - 0x72, 0x74, 0x50, 0x41, 0x66, 0x73, 0xb4, 0x18, 0x0b, 0x7c, 0xf4, 0x36, 0x4c, 0xb0, 0x5f, 0xca, - 0x02, 0xa2, 0x98, 0x9f, 0x0e, 0xa1, 0xa1, 0x23, 0x72, 0x43, 0x1a, 0xa3, 0x08, 0x9b, 0xa4, 0xe6, - 0xb6, 0xa0, 0xac, 0x3e, 0xeb, 0xa1, 0xfa, 0x04, 0xfd, 0x9b, 0x22, 0x9c, 0xc9, 0x58, 0x73, 0x28, - 0x32, 0x66, 0xe2, 0xf9, 0x01, 0x97, 0xea, 0x07, 0x9c, 0x8b, 0x88, 0x49, 0x43, 0x2d, 0xb1, 0xb6, - 0x06, 0x6e, 0xf4, 0x4e, 0x44, 0xd2, 0x8d, 0xd2, 0xa2, 0xfe, 0x8d, 0xd2, 0xc6, 0x4e, 0x6d, 0xa8, - 0x69, 0x43, 0xaa, 0xa7, 0x0f, 0x75, 0x4e, 0xff, 0xb8, 0x08, 0x67, 0xb3, 0x62, 0xd8, 0xa0, 0xef, - 0x4d, 0xa5, 0xc6, 0x79, 0x71, 0xd0, 0xe8, 0x37, 0x3c, 0x5f, 0x8e, 0x48, 0xf4, 0x3c, 0x6f, 0x26, - 0xcb, 0xe9, 0x3b, 0xcc, 0xa2, 0x4d, 0xe6, 0x48, 0x1a, 0xf2, 0x94, 0x46, 0xf2, 0xf8, 0xf8, 0xf4, - 0xc0, 0x1d, 0x10, 0xb9, 0x90, 0xa2, 0x94, 0x23, 0xa9, 0x2c, 0xee, 0xef, 0x48, 0x2a, 0x5b, 0x9e, - 0x73, 0x61, 0x4c, 0xfb, 0x9a, 0x87, 0x3a, 0xe3, 0x3b, 0xf4, 0xb6, 0xd2, 0xfa, 0xfd, 0x50, 0x67, - 0xfd, 0xc7, 0x2c, 0x48, 0x99, 0x59, 0x2a, 0x75, 0x97, 0x95, 0xab, 0xee, 0xba, 0x04, 0xa5, 0x30, - 0xf0, 0x48, 0x3a, 0x13, 0x0d, 0x0e, 0x3c, 0x82, 0x19, 0x84, 0x62, 0xc4, 0x89, 0xb2, 0x63, 0x5c, - 0x17, 0xe4, 0x84, 0x88, 0xf6, 0x24, 0x0c, 0x79, 0x64, 0x8f, 0x78, 0xe9, 0x30, 0xef, 0xb7, 0x68, - 0x21, 0xe6, 0x30, 0xfb, 0x97, 0x4a, 0x70, 0xa1, 0xa7, 0x2b, 0x36, 0x15, 0x87, 0xb6, 0x9c, 0x98, - 0xdc, 0x73, 0xf6, 0xd3, 0xf1, 0x98, 0xaf, 0xf3, 0x62, 0x2c, 0xe1, 0xcc, 0xf2, 0x9c, 0xc7, 0x5f, - 0x4c, 0x29, 0x07, 0x45, 0xd8, 0x45, 0x01, 0x7d, 0x08, 0x49, 0xee, 0xaf, 0x01, 0x44, 0x91, 0xc7, - 0xed, 0x06, 0x5a, 0xc2, 0xa4, 0x3d, 0x89, 0xd3, 0xd9, 0xb8, 0x25, 0x20, 0x58, 0xc3, 0x42, 0x55, - 0x98, 0x6e, 0x87, 0x41, 0xcc, 0x75, 0xad, 0x55, 0x6e, 0x70, 0x34, 0x64, 0x7a, 0xc1, 0xd6, 0x53, - 0x70, 0xdc, 0x55, 0x03, 0xbd, 0x04, 0x63, 0xc2, 0x33, 0xb6, 0x1e, 0x04, 0x9e, 0x50, 0x03, 0x29, - 0xf3, 0x95, 0x46, 0x02, 0xc2, 0x3a, 0x9e, 0x56, 0x8d, 0x29, 0x70, 0x47, 0x32, 0xab, 0x71, 0x25, - 0xae, 0x86, 0x97, 0x8a, 0x67, 0x35, 0x3a, 0x50, 0x3c, 0xab, 0x44, 0x31, 0x56, 0x1e, 0xf8, 0xcd, - 0x0a, 0xfa, 0xaa, 0x92, 0x7e, 0xbe, 0x04, 0x67, 0xc4, 0xc2, 0x79, 0xd8, 0xcb, 0xe5, 0x21, 0xa5, - 0xe2, 0xff, 0xf6, 0x9a, 0x39, 0xed, 0x35, 0xf3, 0xc3, 0x16, 0x98, 0xec, 0x15, 0xfa, 0xff, 0x72, - 0x03, 0xda, 0xbf, 0x94, 0xcb, 0xae, 0xb5, 0xe4, 0x05, 0xf2, 0x01, 0x43, 0xdb, 0xdb, 0xff, 0xce, - 0x82, 0x27, 0xfa, 0x52, 0x44, 0xcb, 0x50, 0x66, 0x3c, 0xa0, 0x26, 0x9d, 0x3d, 0xa5, 0x0c, 0x12, - 0x25, 0x20, 0x87, 0x25, 0x4d, 0x6a, 0xa2, 0xe5, 0xae, 0xcc, 0x01, 0x4f, 0x67, 0x64, 0x0e, 0x38, - 0x67, 0x0c, 0xcf, 0x03, 0xa6, 0x0e, 0xf8, 0xd5, 0x22, 0x0c, 0xf3, 0x15, 0x7f, 0x0a, 0x62, 0xd8, - 0x8a, 0xd0, 0xdb, 0xf6, 0x88, 0x68, 0xc5, 0xfb, 0x32, 0x5f, 0x75, 0x62, 0x87, 0xb3, 0x09, 0xea, - 0xb6, 0x4a, 0x34, 0xbc, 0x68, 0xde, 0xb8, 0xcf, 0xe6, 0x52, 0x8a, 0x49, 0xe0, 0x34, 0xb4, 0xdb, - 0xed, 0x8b, 0x00, 0x11, 0xcb, 0xba, 0x4f, 0x69, 0x88, 0xd8, 0x68, 0x9f, 0xec, 0xd1, 0x7a, 0x43, - 0x21, 0xf3, 0x3e, 0x24, 0x3b, 0x5d, 0x01, 0xb0, 0x46, 0x71, 0xee, 0x65, 0x28, 0x2b, 0xe4, 0x7e, - 0x5a, 0x9c, 0x71, 0x9d, 0xb9, 0xf8, 0x2c, 0x4c, 0xa5, 0xda, 0x3a, 0x96, 0x12, 0xe8, 0x97, 0x2d, - 0x98, 0xe2, 0x5d, 0x5e, 0xf6, 0xf7, 0xc4, 0x99, 0xfa, 0x3e, 0x9c, 0xf5, 0x32, 0xce, 0x36, 0x31, - 0xa3, 0x83, 0x9f, 0x85, 0x4a, 0xe9, 0x93, 0x05, 0xc5, 0x99, 0x6d, 0xa0, 0x2b, 0x74, 0xdd, 0xd2, - 0xb3, 0xcb, 0xf1, 0x84, 0x17, 0xd3, 0x38, 0x5f, 0xb3, 0xbc, 0x0c, 0x2b, 0xa8, 0xfd, 0xbb, 0x16, - 0xcc, 0xf0, 0x9e, 0xdf, 0x24, 0xfb, 0x6a, 0x87, 0x7f, 0x98, 0x7d, 0x17, 0xc9, 0x3c, 0x0a, 0x39, - 0xc9, 0x3c, 0xf4, 0x4f, 0x2b, 0xf6, 0xfc, 0xb4, 0x9f, 0xb3, 0x40, 0xac, 0xc0, 0x53, 0x10, 0xe5, - 0xbf, 0xd3, 0x14, 0xe5, 0xe7, 0xf2, 0x17, 0x75, 0x8e, 0x0c, 0xff, 0xa7, 0x16, 0x4c, 0x73, 0x84, - 0xe4, 0x2d, 0xf9, 0x43, 0x9d, 0x87, 0x41, 0xb2, 0xf2, 0xa9, 0x54, 0xdd, 0xd9, 0x1f, 0x65, 0x4c, - 0x56, 0xa9, 0xe7, 0x64, 0xb5, 0xe4, 0x06, 0x3a, 0x46, 0x46, 0xca, 0x63, 0x07, 0xc5, 0xb6, 0xff, - 0xc8, 0x02, 0xc4, 0x9b, 0x31, 0xd8, 0x1f, 0xca, 0x54, 0xb0, 0x52, 0xed, 0xba, 0x48, 0x8e, 0x1a, - 0x05, 0xc1, 0x1a, 0xd6, 0x89, 0x0c, 0x4f, 0xca, 0x20, 0xa0, 0xd8, 0xdf, 0x20, 0xe0, 0x18, 0x23, - 0xfa, 0xbf, 0x4b, 0x90, 0x76, 0x2b, 0x40, 0x77, 0x61, 0xbc, 0xe9, 0xb4, 0x9d, 0x0d, 0xd7, 0x73, - 0x63, 0x97, 0x44, 0xbd, 0x2c, 0x89, 0x96, 0x34, 0x3c, 0xf1, 0xd4, 0xab, 0x95, 0x60, 0x83, 0x0e, - 0x9a, 0x07, 0x68, 0x87, 0xee, 0x9e, 0xeb, 0x91, 0x2d, 0xa6, 0x71, 0x60, 0x7e, 0x93, 0xdc, 0x3c, - 0x46, 0x96, 0x62, 0x0d, 0x23, 0xc3, 0x05, 0xae, 0xf8, 0xf0, 0x5c, 0xe0, 0x4a, 0xc7, 0x74, 0x81, - 0x1b, 0x1a, 0xc8, 0x05, 0x0e, 0xc3, 0x23, 0x92, 0x45, 0xa2, 0xff, 0x57, 0x5c, 0x8f, 0x08, 0xbe, - 0x98, 0x7b, 0x53, 0xce, 0x1d, 0x1e, 0x54, 0x1e, 0xc1, 0x99, 0x18, 0x38, 0xa7, 0x26, 0xfa, 0x1c, - 0xcc, 0x3a, 0x9e, 0x17, 0xdc, 0x53, 0xa3, 0xb6, 0x1c, 0x35, 0x1d, 0x8f, 0x6b, 0xec, 0x47, 0x18, - 0xd5, 0xc7, 0x0f, 0x0f, 0x2a, 0xb3, 0x0b, 0x39, 0x38, 0x38, 0xb7, 0x76, 0xca, 0x83, 0x6e, 0xb4, - 0xaf, 0x07, 0xdd, 0x6b, 0x50, 0x6e, 0x87, 0x41, 0x73, 0x55, 0xf3, 0xea, 0xb9, 0xc8, 0xf2, 0xdd, - 0xcb, 0xc2, 0xa3, 0x83, 0xca, 0x84, 0xfa, 0xc3, 0x6e, 0xf8, 0xa4, 0x82, 0xbd, 0x03, 0x67, 0x1a, - 0x24, 0x74, 0x59, 0x26, 0xcd, 0x56, 0xb2, 0xa1, 0xd7, 0xa1, 0x1c, 0xa6, 0x8e, 0xb0, 0x81, 0x02, - 0x34, 0x69, 0xd1, 0x82, 0xe5, 0x91, 0x95, 0x10, 0xb2, 0xff, 0xc4, 0x82, 0x11, 0x61, 0x61, 0x7e, - 0x0a, 0x9c, 0xd3, 0x82, 0xa1, 0xc0, 0xae, 0x64, 0x1f, 0xf3, 0xac, 0x33, 0xb9, 0xaa, 0xeb, 0x5a, - 0x4a, 0x75, 0xfd, 0x44, 0x2f, 0x22, 0xbd, 0x95, 0xd6, 0x7f, 0xb3, 0x08, 0x93, 0xa6, 0x53, 0xc8, - 0x29, 0x0c, 0xc1, 0x1a, 0x8c, 0x44, 0xc2, 0x03, 0xa9, 0x90, 0x6f, 0x39, 0x9d, 0x9e, 0xc4, 0xc4, - 0x2c, 0x4a, 0xf8, 0x1c, 0x49, 0x22, 0x99, 0xae, 0x4d, 0xc5, 0x87, 0xe8, 0xda, 0xd4, 0xcf, 0x2f, - 0xa7, 0x74, 0x12, 0x7e, 0x39, 0xf6, 0xd7, 0xd9, 0x55, 0xa3, 0x97, 0x9f, 0x02, 0x17, 0x72, 0xdd, - 0xbc, 0x94, 0xec, 0x1e, 0x2b, 0x4b, 0x74, 0x2a, 0x87, 0x1b, 0xf9, 0x45, 0x0b, 0x2e, 0x64, 0x7c, - 0x95, 0xc6, 0x9a, 0x3c, 0x0b, 0xa3, 0x4e, 0xa7, 0xe5, 0xaa, 0xbd, 0xac, 0x3d, 0x63, 0x2d, 0x88, - 0x72, 0xac, 0x30, 0xd0, 0x12, 0xcc, 0x90, 0xfb, 0x6d, 0x97, 0xbf, 0x23, 0xea, 0xb6, 0x8b, 0x45, - 0x1e, 0xaa, 0x76, 0x39, 0x0d, 0xc4, 0xdd, 0xf8, 0xca, 0xad, 0xbb, 0x98, 0xeb, 0xd6, 0xfd, 0x0f, - 0x2c, 0x18, 0x53, 0xde, 0x26, 0x0f, 0x7d, 0xb4, 0xbf, 0xcb, 0x1c, 0xed, 0xc7, 0x7a, 0x8c, 0x76, - 0xce, 0x30, 0xff, 0xed, 0x82, 0xea, 0x6f, 0x3d, 0x08, 0xe3, 0x01, 0x58, 0x9e, 0x57, 0x60, 0xb4, - 0x1d, 0x06, 0x71, 0xd0, 0x0c, 0x3c, 0xc1, 0xf1, 0x3c, 0x9e, 0x44, 0x1d, 0xe0, 0xe5, 0x47, 0xda, - 0x6f, 0xac, 0xb0, 0xd9, 0xe8, 0x05, 0x61, 0x2c, 0xb8, 0x8c, 0x64, 0xf4, 0x82, 0x30, 0xc6, 0x0c, - 0x82, 0x5a, 0x00, 0xb1, 0x13, 0x6e, 0x91, 0x98, 0x96, 0x89, 0x00, 0x26, 0xf9, 0x87, 0x47, 0x27, - 0x76, 0xbd, 0x79, 0xd7, 0x8f, 0xa3, 0x38, 0x9c, 0xaf, 0xf9, 0xf1, 0xed, 0x90, 0x0b, 0x50, 0x5a, - 0x18, 0x01, 0x45, 0x0b, 0x6b, 0x74, 0xa5, 0xaf, 0x27, 0x6b, 0x63, 0xc8, 0x7c, 0x10, 0x5f, 0x13, - 0xe5, 0x58, 0x61, 0xd8, 0x2f, 0xb3, 0xab, 0x84, 0x0d, 0xd0, 0xf1, 0x3c, 0xfc, 0xbf, 0x31, 0xaa, - 0x86, 0x96, 0xbd, 0x86, 0x55, 0xf5, 0x38, 0x02, 0xbd, 0x4f, 0x6e, 0xda, 0xb0, 0xee, 0x47, 0x93, - 0x04, 0x1b, 0x40, 0xdf, 0xdd, 0x65, 0x27, 0xf1, 0x5c, 0x9f, 0x2b, 0xe0, 0x18, 0x96, 0x11, 0x2c, - 0x7c, 0x36, 0x0b, 0x33, 0x5c, 0xab, 0x8b, 0x45, 0xae, 0x85, 0xcf, 0x16, 0x00, 0x9c, 0xe0, 0xa0, - 0xab, 0x42, 0xfc, 0x2e, 0x19, 0x49, 0xf4, 0xa4, 0xf8, 0x2d, 0x3f, 0x5f, 0x93, 0xbf, 0x9f, 0x87, - 0x31, 0x95, 0x4c, 0xaf, 0xce, 0x73, 0x92, 0x89, 0x70, 0x2e, 0xcb, 0x49, 0x31, 0xd6, 0x71, 0xd0, - 0x3a, 0x4c, 0x45, 0x5c, 0xf7, 0xa2, 0xa2, 0xf6, 0x71, 0x1d, 0xd6, 0x27, 0xa5, 0x7d, 0x45, 0xc3, - 0x04, 0x1f, 0xb1, 0x22, 0x7e, 0x74, 0x48, 0x87, 0xcd, 0x34, 0x09, 0xf4, 0x3a, 0x4c, 0x7a, 0x7a, - 0xda, 0xfa, 0xba, 0x50, 0x71, 0x29, 0xf3, 0x63, 0x23, 0xa9, 0x7d, 0x1d, 0xa7, 0xb0, 0x29, 0xa7, - 0xa4, 0x97, 0x88, 0x48, 0x93, 0x8e, 0xbf, 0x45, 0x22, 0x91, 0x0a, 0x8c, 0x71, 0x4a, 0xb7, 0x72, - 0x70, 0x70, 0x6e, 0x6d, 0xf4, 0x0a, 0x8c, 0xcb, 0xcf, 0xd7, 0xdc, 0x91, 0x13, 0x23, 0x77, 0x0d, - 0x86, 0x0d, 0x4c, 0x74, 0x0f, 0xce, 0xc9, 0xff, 0xeb, 0xa1, 0xb3, 0xb9, 0xe9, 0x36, 0x85, 0x37, - 0x38, 0xf7, 0xf4, 0x59, 0x90, 0xae, 0x43, 0xcb, 0x59, 0x48, 0x47, 0x07, 0x95, 0x4b, 0x62, 0xd4, - 0x32, 0xe1, 0x6c, 0x12, 0xb3, 0xe9, 0xa3, 0x55, 0x38, 0xb3, 0x4d, 0x1c, 0x2f, 0xde, 0x5e, 0xda, - 0x26, 0xcd, 0x1d, 0xb9, 0x89, 0x98, 0x93, 0xb3, 0x66, 0x1a, 0x7e, 0xa3, 0x1b, 0x05, 0x67, 0xd5, - 0x43, 0xef, 0xc0, 0x6c, 0xbb, 0xb3, 0xe1, 0xb9, 0xd1, 0xf6, 0x5a, 0x10, 0x33, 0x93, 0x0e, 0x95, - 0x8b, 0x4e, 0x78, 0x43, 0x2b, 0x07, 0xef, 0x7a, 0x0e, 0x1e, 0xce, 0xa5, 0x80, 0xde, 0x87, 0x73, - 0xa9, 0xc5, 0x20, 0x7c, 0x33, 0x27, 0xf3, 0xe3, 0xf6, 0x36, 0xb2, 0x2a, 0x08, 0x5f, 0xcb, 0x2c, - 0x10, 0xce, 0x6e, 0xe2, 0x83, 0x19, 0xfa, 0xbc, 0x47, 0x2b, 0x6b, 0x4c, 0x19, 0xfa, 0x12, 0x8c, - 0xeb, 0xab, 0x48, 0x5c, 0x30, 0x97, 0xb3, 0x79, 0x16, 0x6d, 0xb5, 0x71, 0x96, 0x4e, 0xad, 0x28, - 0x1d, 0x86, 0x0d, 0x8a, 0x36, 0x81, 0xec, 0xef, 0x43, 0xb7, 0x60, 0xb4, 0xe9, 0xb9, 0xc4, 0x8f, - 0x6b, 0xf5, 0x5e, 0xc1, 0x43, 0x96, 0x04, 0x8e, 0x18, 0x30, 0x11, 0xe8, 0x94, 0x97, 0x61, 0x45, - 0xc1, 0xfe, 0x8d, 0x02, 0x54, 0xfa, 0x44, 0xcd, 0x4d, 0xe9, 0xa3, 0xad, 0x81, 0xf4, 0xd1, 0x0b, - 0x32, 0xb3, 0xde, 0x5a, 0x4a, 0x48, 0x4f, 0x65, 0xcd, 0x4b, 0x44, 0xf5, 0x34, 0xfe, 0xc0, 0xf6, - 0xc1, 0xba, 0x4a, 0xbb, 0xd4, 0xd7, 0x72, 0xdd, 0x78, 0xca, 0x1a, 0x1a, 0x5c, 0x10, 0xc9, 0x7d, - 0x96, 0xb0, 0xbf, 0x5e, 0x80, 0x73, 0x6a, 0x08, 0xbf, 0x75, 0x07, 0xee, 0x4e, 0xf7, 0xc0, 0x9d, - 0xc0, 0xa3, 0x8e, 0x7d, 0x1b, 0x86, 0x79, 0xf0, 0x95, 0x01, 0x18, 0xa0, 0x27, 0xcd, 0x48, 0x5d, - 0xea, 0x9a, 0x36, 0xa2, 0x75, 0xfd, 0x25, 0x0b, 0xa6, 0xd6, 0x97, 0xea, 0x8d, 0xa0, 0xb9, 0x43, - 0xe2, 0x05, 0xce, 0xb0, 0x62, 0xc1, 0xff, 0x58, 0x0f, 0xc8, 0xd7, 0x64, 0x71, 0x4c, 0x97, 0xa0, - 0xb4, 0x1d, 0x44, 0x71, 0xfa, 0xc5, 0xf7, 0x46, 0x10, 0xc5, 0x98, 0x41, 0xec, 0xdf, 0xb3, 0x60, - 0x88, 0xe5, 0x83, 0xed, 0x97, 0xa4, 0x78, 0x90, 0xef, 0x42, 0x2f, 0xc1, 0x30, 0xd9, 0xdc, 0x24, - 0xcd, 0x58, 0xcc, 0xaa, 0x74, 0x47, 0x1d, 0x5e, 0x66, 0xa5, 0xf4, 0xd2, 0x67, 0x8d, 0xf1, 0xbf, - 0x58, 0x20, 0xa3, 0xb7, 0xa0, 0x1c, 0xbb, 0xbb, 0x64, 0xa1, 0xd5, 0x12, 0x6f, 0x66, 0x0f, 0xe0, - 0xfd, 0xbb, 0x2e, 0x09, 0xe0, 0x84, 0x96, 0xfd, 0xd5, 0x02, 0x40, 0x12, 0x42, 0xa0, 0xdf, 0x27, - 0x2e, 0x76, 0xbd, 0xa6, 0x5c, 0xce, 0x78, 0x4d, 0x41, 0x09, 0xc1, 0x8c, 0xa7, 0x14, 0x35, 0x4c, - 0xc5, 0x81, 0x86, 0xa9, 0x74, 0x9c, 0x61, 0x5a, 0x82, 0x99, 0x24, 0x04, 0x82, 0x19, 0x0f, 0x86, - 0x09, 0x29, 0xeb, 0x69, 0x20, 0xee, 0xc6, 0xb7, 0x09, 0x5c, 0x92, 0x91, 0x39, 0xe5, 0x5d, 0xc3, - 0x4c, 0x32, 0x8f, 0x91, 0xaf, 0x3a, 0x79, 0x2e, 0x2a, 0xe4, 0x3e, 0x17, 0xfd, 0xa4, 0x05, 0x67, - 0xd3, 0xed, 0x30, 0xdf, 0xb7, 0xaf, 0x58, 0x70, 0x8e, 0x3d, 0x9a, 0xb1, 0x56, 0xbb, 0x9f, 0xe8, - 0x5e, 0xcc, 0x0e, 0x0d, 0xd1, 0xbb, 0xc7, 0x89, 0xdf, 0xf3, 0x6a, 0x16, 0x69, 0x9c, 0xdd, 0xa2, - 0xfd, 0x15, 0x0b, 0xce, 0xe7, 0xa6, 0x21, 0x42, 0x57, 0x60, 0xd4, 0x69, 0xbb, 0x5c, 0x23, 0x25, - 0xf6, 0x3b, 0x93, 0x1e, 0xeb, 0x35, 0xae, 0x8f, 0x52, 0x50, 0x95, 0x1e, 0xb1, 0x90, 0x9b, 0x1e, - 0xb1, 0x6f, 0xb6, 0x43, 0xfb, 0x07, 0x2d, 0x10, 0xee, 0x4e, 0x03, 0x1c, 0x32, 0x6f, 0xcb, 0xec, - 0xb2, 0x46, 0x50, 0xf4, 0x4b, 0xf9, 0xfe, 0x5f, 0x22, 0x14, 0xba, 0xba, 0xd4, 0x8d, 0x00, 0xe8, - 0x06, 0x2d, 0xbb, 0x05, 0x02, 0x5a, 0x25, 0x4c, 0x67, 0xd5, 0xbf, 0x37, 0xd7, 0x00, 0x5a, 0x0c, - 0x57, 0xcb, 0x31, 0xa9, 0xae, 0x90, 0xaa, 0x82, 0x60, 0x0d, 0xcb, 0xfe, 0x91, 0x02, 0x8c, 0xc9, - 0x20, 0xdc, 0x1d, 0x7f, 0x10, 0xc9, 0xf2, 0x58, 0xb9, 0x78, 0x58, 0x52, 0x56, 0x4a, 0xb8, 0x9e, - 0x08, 0xe4, 0x49, 0x52, 0x56, 0x09, 0xc0, 0x09, 0x0e, 0x7a, 0x1a, 0x46, 0xa2, 0xce, 0x06, 0x43, - 0x4f, 0x39, 0xf1, 0x34, 0x78, 0x31, 0x96, 0x70, 0xf4, 0x39, 0x98, 0xe6, 0xf5, 0xc2, 0xa0, 0xed, - 0x6c, 0x71, 0xf5, 0xe7, 0x90, 0xf2, 0xaa, 0x9d, 0x5e, 0x4d, 0xc1, 0x8e, 0x0e, 0x2a, 0x67, 0xd3, - 0x65, 0x4c, 0x71, 0xde, 0x45, 0xc5, 0xfe, 0x12, 0xa0, 0xee, 0xb8, 0xe2, 0xe8, 0x0d, 0x6e, 0x4a, - 0xe5, 0x86, 0xa4, 0xd5, 0x4b, 0x23, 0xae, 0x3b, 0x81, 0x4a, 0x43, 0x7a, 0x5e, 0x0b, 0xab, 0xfa, - 0xf6, 0x5f, 0x2d, 0xc2, 0x74, 0xda, 0x25, 0x10, 0xdd, 0x80, 0x61, 0x7e, 0xd9, 0x09, 0xf2, 0x3d, - 0x1e, 0x5c, 0x35, 0x47, 0x42, 0xb6, 0xed, 0xc5, 0x7d, 0x29, 0xea, 0xa3, 0x77, 0x60, 0xac, 0x15, - 0xdc, 0xf3, 0xef, 0x39, 0x61, 0x6b, 0xa1, 0x5e, 0x13, 0xeb, 0x32, 0x93, 0x67, 0xae, 0x26, 0x68, - 0xba, 0x73, 0x22, 0x7b, 0x5c, 0x48, 0x40, 0x58, 0x27, 0x87, 0xd6, 0x59, 0xac, 0xc4, 0x4d, 0x77, - 0x6b, 0xd5, 0x69, 0xf7, 0xb2, 0xab, 0x5d, 0x92, 0x48, 0x1a, 0xe5, 0x09, 0x11, 0x50, 0x91, 0x03, - 0x70, 0x42, 0x08, 0x7d, 0x2f, 0x9c, 0x89, 0x72, 0xd4, 0x6c, 0x79, 0x69, 0x26, 0x7a, 0x69, 0x9e, - 0x16, 0x1f, 0xa5, 0xd2, 0x4c, 0x96, 0x42, 0x2e, 0xab, 0x19, 0xfb, 0xcb, 0x67, 0xc0, 0xd8, 0x8d, - 0x46, 0xae, 0x21, 0xeb, 0x84, 0x72, 0x0d, 0x61, 0x18, 0x25, 0xbb, 0xed, 0x78, 0xbf, 0xea, 0x86, - 0xbd, 0x72, 0xe1, 0x2d, 0x0b, 0x9c, 0x6e, 0x9a, 0x12, 0x82, 0x15, 0x9d, 0xec, 0x84, 0x50, 0xc5, - 0x0f, 0x31, 0x21, 0x54, 0xe9, 0x14, 0x13, 0x42, 0xad, 0xc1, 0xc8, 0x96, 0x1b, 0x63, 0xd2, 0x0e, - 0x04, 0x9b, 0x99, 0xb9, 0x0e, 0xaf, 0x73, 0x94, 0xee, 0x24, 0x24, 0x02, 0x80, 0x25, 0x11, 0xf4, - 0x86, 0xda, 0x81, 0xc3, 0xf9, 0x52, 0x5a, 0xf7, 0xcb, 0x60, 0xe6, 0x1e, 0x14, 0x09, 0xa0, 0x46, - 0x1e, 0x34, 0x01, 0xd4, 0x8a, 0x4c, 0xdb, 0x34, 0x9a, 0x6f, 0x04, 0xcf, 0xb2, 0x32, 0xf5, 0x49, - 0xd6, 0x64, 0x24, 0xb8, 0x2a, 0x9f, 0x5c, 0x82, 0xab, 0x1f, 0xb4, 0xe0, 0x5c, 0x3b, 0x2b, 0xd7, - 0x9b, 0x48, 0xb6, 0xf4, 0xd2, 0xc0, 0xc9, 0xec, 0x8c, 0x06, 0x99, 0xb8, 0x9e, 0x89, 0x86, 0xb3, - 0x9b, 0xa3, 0x03, 0x1d, 0x6e, 0xb4, 0x44, 0x86, 0xa6, 0x27, 0x73, 0x32, 0x65, 0xf5, 0xc8, 0x8f, - 0xb5, 0x9e, 0x91, 0x95, 0xe9, 0xe3, 0x79, 0x59, 0x99, 0x06, 0xce, 0xc5, 0xf4, 0x86, 0xca, 0x91, - 0x35, 0x91, 0xbf, 0x94, 0x78, 0x06, 0xac, 0xbe, 0x99, 0xb1, 0xde, 0x50, 0x99, 0xb1, 0x7a, 0xc4, - 0x8c, 0xe3, 0x79, 0xaf, 0xfa, 0xe6, 0xc3, 0xd2, 0x72, 0x5a, 0x4d, 0x9d, 0x4c, 0x4e, 0x2b, 0xe3, - 0xaa, 0xe1, 0x69, 0x95, 0x9e, 0xe9, 0x73, 0xd5, 0x18, 0x74, 0x7b, 0x5f, 0x36, 0x3c, 0x7f, 0xd7, - 0xcc, 0x03, 0xe5, 0xef, 0xba, 0xab, 0xe7, 0xc3, 0x42, 0x7d, 0x12, 0x3e, 0x51, 0xa4, 0x01, 0xb3, - 0x60, 0xdd, 0xd5, 0x2f, 0xc0, 0x33, 0xf9, 0x74, 0xd5, 0x3d, 0xd7, 0x4d, 0x37, 0xf3, 0x0a, 0xec, - 0xca, 0xae, 0x75, 0xf6, 0x74, 0xb2, 0x6b, 0x9d, 0x3b, 0xf1, 0xec, 0x5a, 0x8f, 0x9c, 0x42, 0x76, - 0xad, 0x47, 0x3f, 0xd4, 0xec, 0x5a, 0xb3, 0x0f, 0x21, 0xbb, 0xd6, 0x5a, 0x92, 0x5d, 0xeb, 0x7c, - 0xfe, 0x94, 0x64, 0x58, 0xe6, 0xe6, 0xe4, 0xd4, 0xba, 0xcb, 0x9e, 0xe7, 0x79, 0xcc, 0x0a, 0x11, - 0xd4, 0x2e, 0x3b, 0x7f, 0x70, 0x56, 0x60, 0x0b, 0x3e, 0x25, 0x0a, 0x84, 0x13, 0x52, 0x94, 0x6e, - 0x92, 0x63, 0xeb, 0xb1, 0x1e, 0x0a, 0xd9, 0x2c, 0x55, 0x57, 0x7e, 0x66, 0x2d, 0xfb, 0x2f, 0x17, - 0xe0, 0x62, 0xef, 0x75, 0x9d, 0xe8, 0xc9, 0xea, 0xc9, 0xbb, 0x4e, 0x4a, 0x4f, 0xc6, 0x85, 0x9c, - 0x04, 0x6b, 0xe0, 0xc0, 0x3e, 0xd7, 0x61, 0x46, 0x99, 0xe4, 0x7a, 0x6e, 0x73, 0x5f, 0xcb, 0x2b, - 0xac, 0x5c, 0x0f, 0x1b, 0x69, 0x04, 0xdc, 0x5d, 0x07, 0x2d, 0xc0, 0x94, 0x51, 0x58, 0xab, 0x0a, - 0x61, 0x46, 0x29, 0xe6, 0x1a, 0x26, 0x18, 0xa7, 0xf1, 0xed, 0x9f, 0xb5, 0xe0, 0xd1, 0x9c, 0xc4, - 0x13, 0x03, 0xc7, 0xad, 0xd9, 0x84, 0xa9, 0xb6, 0x59, 0xb5, 0x4f, 0x78, 0x2b, 0x23, 0xbd, 0x85, - 0xea, 0x6b, 0x0a, 0x80, 0xd3, 0x44, 0x17, 0xaf, 0xfc, 0xf6, 0x1f, 0x5c, 0xfc, 0xd8, 0xef, 0xfc, - 0xc1, 0xc5, 0x8f, 0xfd, 0xee, 0x1f, 0x5c, 0xfc, 0xd8, 0xff, 0x7f, 0x78, 0xd1, 0xfa, 0xed, 0xc3, - 0x8b, 0xd6, 0xef, 0x1c, 0x5e, 0xb4, 0x7e, 0xf7, 0xf0, 0xa2, 0xf5, 0xfb, 0x87, 0x17, 0xad, 0xaf, - 0xfe, 0xe1, 0xc5, 0x8f, 0xbd, 0x5d, 0xd8, 0x7b, 0xfe, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0xd1, - 0x90, 0x44, 0x98, 0x55, 0xe7, 0x00, 0x00, + // 12835 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7d, 0x6d, 0x70, 0x64, 0x57, + 0x56, 0xd8, 0xbe, 0xee, 0xd6, 0x47, 0x1f, 0x7d, 0xdf, 0x99, 0xb1, 0x35, 0xb2, 0x67, 0x7a, 0xfc, + 0xbc, 0x3b, 0x1e, 0xaf, 0x6d, 0xcd, 0x7a, 0x6c, 0xaf, 0xcd, 0xda, 0x6b, 0x90, 0xd4, 0xd2, 0x4c, + 0x7b, 0x46, 0x9a, 0xf6, 0x6d, 0xcd, 0x78, 0xd7, 0x78, 0x97, 0x7d, 0xea, 0xbe, 0x92, 0x9e, 0xf5, + 0xf4, 0x5e, 0xfb, 0xbd, 0xd7, 0x9a, 0x91, 0x03, 0x55, 0xc9, 0x12, 0x48, 0x36, 0x50, 0xa9, 0xad, + 0xb0, 0x95, 0x0f, 0xa0, 0x48, 0x15, 0x21, 0x05, 0x84, 0x24, 0x15, 0x02, 0x01, 0xc2, 0x42, 0x42, + 0x20, 0x3f, 0xc8, 0x9f, 0x0d, 0x49, 0x55, 0x6a, 0xa9, 0xa2, 0xa2, 0x80, 0x48, 0x25, 0xc5, 0x8f, + 0x40, 0x2a, 0xe4, 0x47, 0x50, 0xa8, 0x90, 0xba, 0x9f, 0xef, 0xde, 0xd7, 0xef, 0x75, 0xb7, 0xc6, + 0x1a, 0xd9, 0x50, 0xfb, 0xaf, 0xfb, 0x9e, 0x73, 0xcf, 0xbd, 0xef, 0x7e, 0x9e, 0x73, 0xee, 0xf9, + 0x80, 0x57, 0x77, 0x5e, 0x89, 0xe6, 0xdd, 0xe0, 0xea, 0x4e, 0x67, 0x83, 0x84, 0x3e, 0x89, 0x49, + 0x74, 0x75, 0x8f, 0xf8, 0xad, 0x20, 0xbc, 0x2a, 0x00, 0x4e, 0xdb, 0xbd, 0xda, 0x0c, 0x42, 0x72, + 0x75, 0xef, 0xf9, 0xab, 0x5b, 0xc4, 0x27, 0xa1, 0x13, 0x93, 0xd6, 0x7c, 0x3b, 0x0c, 0xe2, 0x00, + 0x21, 0x8e, 0x33, 0xef, 0xb4, 0xdd, 0x79, 0x8a, 0x33, 0xbf, 0xf7, 0xfc, 0xdc, 0x73, 0x5b, 0x6e, + 0xbc, 0xdd, 0xd9, 0x98, 0x6f, 0x06, 0xbb, 0x57, 0xb7, 0x82, 0xad, 0xe0, 0x2a, 0x43, 0xdd, 0xe8, + 0x6c, 0xb2, 0x7f, 0xec, 0x0f, 0xfb, 0xc5, 0x49, 0xcc, 0xbd, 0x98, 0x34, 0xb3, 0xeb, 0x34, 0xb7, + 0x5d, 0x9f, 0x84, 0xfb, 0x57, 0xdb, 0x3b, 0x5b, 0xac, 0xdd, 0x90, 0x44, 0x41, 0x27, 0x6c, 0x92, + 0x74, 0xc3, 0x3d, 0x6b, 0x45, 0x57, 0x77, 0x49, 0xec, 0x64, 0x74, 0x77, 0xee, 0x6a, 0x5e, 0xad, + 0xb0, 0xe3, 0xc7, 0xee, 0x6e, 0x77, 0x33, 0x9f, 0xee, 0x57, 0x21, 0x6a, 0x6e, 0x93, 0x5d, 0xa7, + 0xab, 0xde, 0x0b, 0x79, 0xf5, 0x3a, 0xb1, 0xeb, 0x5d, 0x75, 0xfd, 0x38, 0x8a, 0xc3, 0x74, 0x25, + 0xfb, 0x9b, 0x16, 0x5c, 0x5a, 0x78, 0xab, 0xb1, 0xec, 0x39, 0x51, 0xec, 0x36, 0x17, 0xbd, 0xa0, + 0xb9, 0xd3, 0x88, 0x83, 0x90, 0xdc, 0x0d, 0xbc, 0xce, 0x2e, 0x69, 0xb0, 0x81, 0x40, 0xcf, 0xc2, + 0xe8, 0x1e, 0xfb, 0x5f, 0xab, 0xce, 0x5a, 0x97, 0xac, 0x2b, 0xe5, 0xc5, 0xe9, 0xdf, 0x3c, 0xa8, + 0x7c, 0xec, 0xf0, 0xa0, 0x32, 0x7a, 0x57, 0x94, 0x63, 0x85, 0x81, 0x2e, 0xc3, 0xf0, 0x66, 0xb4, + 0xbe, 0xdf, 0x26, 0xb3, 0x05, 0x86, 0x3b, 0x29, 0x70, 0x87, 0x57, 0x1a, 0xb4, 0x14, 0x0b, 0x28, + 0xba, 0x0a, 0xe5, 0xb6, 0x13, 0xc6, 0x6e, 0xec, 0x06, 0xfe, 0x6c, 0xf1, 0x92, 0x75, 0x65, 0x68, + 0x71, 0x46, 0xa0, 0x96, 0xeb, 0x12, 0x80, 0x13, 0x1c, 0xda, 0x8d, 0x90, 0x38, 0xad, 0xdb, 0xbe, + 0xb7, 0x3f, 0x5b, 0xba, 0x64, 0x5d, 0x19, 0x4d, 0xba, 0x81, 0x45, 0x39, 0x56, 0x18, 0xf6, 0x0f, + 0x17, 0x60, 0x74, 0x61, 0x73, 0xd3, 0xf5, 0xdd, 0x78, 0x1f, 0xdd, 0x85, 0x71, 0x3f, 0x68, 0x11, + 0xf9, 0x9f, 0x7d, 0xc5, 0xd8, 0xb5, 0x4b, 0xf3, 0xdd, 0x4b, 0x69, 0x7e, 0x4d, 0xc3, 0x5b, 0x9c, + 0x3e, 0x3c, 0xa8, 0x8c, 0xeb, 0x25, 0xd8, 0xa0, 0x83, 0x30, 0x8c, 0xb5, 0x83, 0x96, 0x22, 0x5b, + 0x60, 0x64, 0x2b, 0x59, 0x64, 0xeb, 0x09, 0xda, 0xe2, 0xd4, 0xe1, 0x41, 0x65, 0x4c, 0x2b, 0xc0, + 0x3a, 0x11, 0xb4, 0x01, 0x53, 0xf4, 0xaf, 0x1f, 0xbb, 0x8a, 0x6e, 0x91, 0xd1, 0x7d, 0x32, 0x8f, + 0xae, 0x86, 0xba, 0x78, 0xe6, 0xf0, 0xa0, 0x32, 0x95, 0x2a, 0xc4, 0x69, 0x82, 0xf6, 0xfb, 0x30, + 0xb9, 0x10, 0xc7, 0x4e, 0x73, 0x9b, 0xb4, 0xf8, 0x0c, 0xa2, 0x17, 0xa1, 0xe4, 0x3b, 0xbb, 0x44, + 0xcc, 0xef, 0x25, 0x31, 0xb0, 0xa5, 0x35, 0x67, 0x97, 0x1c, 0x1d, 0x54, 0xa6, 0xef, 0xf8, 0xee, + 0x7b, 0x1d, 0xb1, 0x2a, 0x68, 0x19, 0x66, 0xd8, 0xe8, 0x1a, 0x40, 0x8b, 0xec, 0xb9, 0x4d, 0x52, + 0x77, 0xe2, 0x6d, 0x31, 0xdf, 0x48, 0xd4, 0x85, 0xaa, 0x82, 0x60, 0x0d, 0xcb, 0xbe, 0x0f, 0xe5, + 0x85, 0xbd, 0xc0, 0x6d, 0xd5, 0x83, 0x56, 0x84, 0x76, 0x60, 0xaa, 0x1d, 0x92, 0x4d, 0x12, 0xaa, + 0xa2, 0x59, 0xeb, 0x52, 0xf1, 0xca, 0xd8, 0xb5, 0x2b, 0x99, 0x1f, 0x6b, 0xa2, 0x2e, 0xfb, 0x71, + 0xb8, 0xbf, 0xf8, 0xa8, 0x68, 0x6f, 0x2a, 0x05, 0xc5, 0x69, 0xca, 0xf6, 0xbf, 0x2d, 0xc0, 0xb9, + 0x85, 0xf7, 0x3b, 0x21, 0xa9, 0xba, 0xd1, 0x4e, 0x7a, 0x85, 0xb7, 0xdc, 0x68, 0x67, 0x2d, 0x19, + 0x01, 0xb5, 0xb4, 0xaa, 0xa2, 0x1c, 0x2b, 0x0c, 0xf4, 0x1c, 0x8c, 0xd0, 0xdf, 0x77, 0x70, 0x4d, + 0x7c, 0xf2, 0x19, 0x81, 0x3c, 0x56, 0x75, 0x62, 0xa7, 0xca, 0x41, 0x58, 0xe2, 0xa0, 0x55, 0x18, + 0x6b, 0xb2, 0x0d, 0xb9, 0xb5, 0x1a, 0xb4, 0x08, 0x9b, 0xcc, 0xf2, 0xe2, 0x33, 0x14, 0x7d, 0x29, + 0x29, 0x3e, 0x3a, 0xa8, 0xcc, 0xf2, 0xbe, 0x09, 0x12, 0x1a, 0x0c, 0xeb, 0xf5, 0x91, 0xad, 0xf6, + 0x57, 0x89, 0x51, 0x82, 0x8c, 0xbd, 0x75, 0x45, 0xdb, 0x2a, 0x43, 0x6c, 0xab, 0x8c, 0x67, 0x6f, + 0x13, 0xf4, 0x3c, 0x94, 0x76, 0x5c, 0xbf, 0x35, 0x3b, 0xcc, 0x68, 0x5d, 0xa0, 0x73, 0x7e, 0xd3, + 0xf5, 0x5b, 0x47, 0x07, 0x95, 0x19, 0xa3, 0x3b, 0xb4, 0x10, 0x33, 0x54, 0xfb, 0x8f, 0x2d, 0xa8, + 0x30, 0xd8, 0x8a, 0xeb, 0x91, 0x3a, 0x09, 0x23, 0x37, 0x8a, 0x89, 0x1f, 0x1b, 0x03, 0x7a, 0x0d, + 0x20, 0x22, 0xcd, 0x90, 0xc4, 0xda, 0x90, 0xaa, 0x85, 0xd1, 0x50, 0x10, 0xac, 0x61, 0xd1, 0x03, + 0x21, 0xda, 0x76, 0x42, 0xb6, 0xbe, 0xc4, 0xc0, 0xaa, 0x03, 0xa1, 0x21, 0x01, 0x38, 0xc1, 0x31, + 0x0e, 0x84, 0x62, 0xbf, 0x03, 0x01, 0x7d, 0x16, 0xa6, 0x92, 0xc6, 0xa2, 0xb6, 0xd3, 0x94, 0x03, + 0xc8, 0xb6, 0x4c, 0xc3, 0x04, 0xe1, 0x34, 0xae, 0xfd, 0x8f, 0x2c, 0xb1, 0x78, 0xe8, 0x57, 0x7f, + 0xc4, 0xbf, 0xd5, 0xfe, 0x25, 0x0b, 0x46, 0x16, 0x5d, 0xbf, 0xe5, 0xfa, 0x5b, 0xe8, 0x4b, 0x30, + 0x4a, 0xef, 0xa6, 0x96, 0x13, 0x3b, 0xe2, 0xdc, 0xfb, 0x94, 0xb6, 0xb7, 0xd4, 0x55, 0x31, 0xdf, + 0xde, 0xd9, 0xa2, 0x05, 0xd1, 0x3c, 0xc5, 0xa6, 0xbb, 0xed, 0xf6, 0xc6, 0xbb, 0xa4, 0x19, 0xaf, + 0x92, 0xd8, 0x49, 0x3e, 0x27, 0x29, 0xc3, 0x8a, 0x2a, 0xba, 0x09, 0xc3, 0xb1, 0x13, 0x6e, 0x91, + 0x58, 0x1c, 0x80, 0x99, 0x07, 0x15, 0xaf, 0x89, 0xe9, 0x8e, 0x24, 0x7e, 0x93, 0x24, 0xd7, 0xc2, + 0x3a, 0xab, 0x8a, 0x05, 0x09, 0xfb, 0x6f, 0x0c, 0xc3, 0xf9, 0xa5, 0x46, 0x2d, 0x67, 0x5d, 0x5d, + 0x86, 0xe1, 0x56, 0xe8, 0xee, 0x91, 0x50, 0x8c, 0xb3, 0xa2, 0x52, 0x65, 0xa5, 0x58, 0x40, 0xd1, + 0x2b, 0x30, 0xce, 0x2f, 0xa4, 0x1b, 0x8e, 0xdf, 0xf2, 0xe4, 0x10, 0x9f, 0x15, 0xd8, 0xe3, 0x77, + 0x35, 0x18, 0x36, 0x30, 0x8f, 0xb9, 0xa8, 0x2e, 0xa7, 0x36, 0x63, 0xde, 0x65, 0xf7, 0x15, 0x0b, + 0xa6, 0x79, 0x33, 0x0b, 0x71, 0x1c, 0xba, 0x1b, 0x9d, 0x98, 0x44, 0xb3, 0x43, 0xec, 0xa4, 0x5b, + 0xca, 0x1a, 0xad, 0xdc, 0x11, 0x98, 0xbf, 0x9b, 0xa2, 0xc2, 0x0f, 0xc1, 0x59, 0xd1, 0xee, 0x74, + 0x1a, 0x8c, 0xbb, 0x9a, 0x45, 0xdf, 0x6b, 0xc1, 0x5c, 0x33, 0xf0, 0xe3, 0x30, 0xf0, 0x3c, 0x12, + 0xd6, 0x3b, 0x1b, 0x9e, 0x1b, 0x6d, 0xf3, 0x75, 0x8a, 0xc9, 0x26, 0x3b, 0x09, 0x72, 0xe6, 0x50, + 0x21, 0x89, 0x39, 0xbc, 0x78, 0x78, 0x50, 0x99, 0x5b, 0xca, 0x25, 0x85, 0x7b, 0x34, 0x83, 0x76, + 0x00, 0xd1, 0xab, 0xb4, 0x11, 0x3b, 0x5b, 0x24, 0x69, 0x7c, 0x64, 0xf0, 0xc6, 0x1f, 0x39, 0x3c, + 0xa8, 0xa0, 0xb5, 0x2e, 0x12, 0x38, 0x83, 0x2c, 0x7a, 0x0f, 0xce, 0xd2, 0xd2, 0xae, 0x6f, 0x1d, + 0x1d, 0xbc, 0xb9, 0xd9, 0xc3, 0x83, 0xca, 0xd9, 0xb5, 0x0c, 0x22, 0x38, 0x93, 0xf4, 0xdc, 0x12, + 0x9c, 0xcb, 0x9c, 0x2a, 0x34, 0x0d, 0xc5, 0x1d, 0xc2, 0x59, 0x90, 0x32, 0xa6, 0x3f, 0xd1, 0x59, + 0x18, 0xda, 0x73, 0xbc, 0x8e, 0x58, 0xa5, 0x98, 0xff, 0xf9, 0x4c, 0xe1, 0x15, 0xcb, 0x6e, 0xc2, + 0xf8, 0x92, 0xd3, 0x76, 0x36, 0x5c, 0xcf, 0x8d, 0x5d, 0x12, 0xa1, 0xa7, 0xa0, 0xe8, 0xb4, 0x5a, + 0xec, 0x8a, 0x2c, 0x2f, 0x9e, 0x3b, 0x3c, 0xa8, 0x14, 0x17, 0x5a, 0xf4, 0xac, 0x06, 0x85, 0xb5, + 0x8f, 0x29, 0x06, 0xfa, 0x24, 0x94, 0x5a, 0x61, 0xd0, 0x9e, 0x2d, 0x30, 0x4c, 0x3a, 0x54, 0xa5, + 0x6a, 0x18, 0xb4, 0x53, 0xa8, 0x0c, 0xc7, 0xfe, 0xb5, 0x02, 0x3c, 0xbe, 0x44, 0xda, 0xdb, 0x2b, + 0x8d, 0x9c, 0x4d, 0x77, 0x05, 0x46, 0x77, 0x03, 0xdf, 0x8d, 0x83, 0x30, 0x12, 0x4d, 0xb3, 0xdb, + 0x64, 0x55, 0x94, 0x61, 0x05, 0x45, 0x97, 0xa0, 0xd4, 0x4e, 0x38, 0x81, 0x71, 0xc9, 0x45, 0x30, + 0x1e, 0x80, 0x41, 0x28, 0x46, 0x27, 0x22, 0xa1, 0xb8, 0x05, 0x15, 0xc6, 0x9d, 0x88, 0x84, 0x98, + 0x41, 0x92, 0xe3, 0x94, 0x1e, 0xb4, 0x62, 0x5b, 0xa5, 0x8e, 0x53, 0x0a, 0xc1, 0x1a, 0x16, 0xaa, + 0x43, 0x39, 0x52, 0x93, 0x3a, 0x34, 0xf8, 0xa4, 0x4e, 0xb0, 0xf3, 0x56, 0xcd, 0x64, 0x42, 0xc4, + 0x38, 0x06, 0x86, 0xfb, 0x9e, 0xb7, 0x5f, 0x2f, 0x00, 0xe2, 0x43, 0xf8, 0xe7, 0x6c, 0xe0, 0xee, + 0x74, 0x0f, 0x5c, 0x26, 0xe7, 0x75, 0x2b, 0x68, 0x3a, 0x5e, 0xfa, 0x08, 0x3f, 0xa9, 0xd1, 0xfb, + 0xdf, 0x16, 0x3c, 0xbe, 0xe4, 0xfa, 0x2d, 0x12, 0xe6, 0x2c, 0xc0, 0x87, 0x23, 0x80, 0x1c, 0xef, + 0xa4, 0x37, 0x96, 0x58, 0xe9, 0x04, 0x96, 0x98, 0xfd, 0x47, 0x16, 0x20, 0xfe, 0xd9, 0x1f, 0xb9, + 0x8f, 0xbd, 0xd3, 0xfd, 0xb1, 0x27, 0xb0, 0x2c, 0xec, 0x5b, 0x30, 0xb9, 0xe4, 0xb9, 0xc4, 0x8f, + 0x6b, 0xf5, 0xa5, 0xc0, 0xdf, 0x74, 0xb7, 0xd0, 0x67, 0x60, 0x92, 0xca, 0xb4, 0x41, 0x27, 0x6e, + 0x90, 0x66, 0xe0, 0x33, 0xf6, 0x9f, 0x4a, 0x82, 0xe8, 0xf0, 0xa0, 0x32, 0xb9, 0x6e, 0x40, 0x70, + 0x0a, 0xd3, 0xfe, 0x1d, 0x3a, 0x7e, 0xc1, 0x6e, 0x3b, 0xf0, 0x89, 0x1f, 0x2f, 0x05, 0x7e, 0x8b, + 0x8b, 0x89, 0x9f, 0x81, 0x52, 0x4c, 0xc7, 0x83, 0x8f, 0xdd, 0x65, 0xb9, 0x51, 0xe8, 0x28, 0x1c, + 0x1d, 0x54, 0x1e, 0xe9, 0xae, 0xc1, 0xc6, 0x89, 0xd5, 0x41, 0xdf, 0x06, 0xc3, 0x51, 0xec, 0xc4, + 0x9d, 0x48, 0x8c, 0xe6, 0x13, 0x72, 0x34, 0x1b, 0xac, 0xf4, 0xe8, 0xa0, 0x32, 0xa5, 0xaa, 0xf1, + 0x22, 0x2c, 0x2a, 0xa0, 0xa7, 0x61, 0x64, 0x97, 0x44, 0x91, 0xb3, 0x25, 0x39, 0xfc, 0x29, 0x51, + 0x77, 0x64, 0x95, 0x17, 0x63, 0x09, 0x47, 0x4f, 0xc2, 0x10, 0x09, 0xc3, 0x20, 0x14, 0x7b, 0x74, + 0x42, 0x20, 0x0e, 0x2d, 0xd3, 0x42, 0xcc, 0x61, 0xf6, 0xbf, 0xb7, 0x60, 0x4a, 0xf5, 0x95, 0xb7, + 0x75, 0x0a, 0xac, 0xdc, 0xdb, 0x00, 0x4d, 0xf9, 0x81, 0x11, 0xbb, 0x3d, 0xc6, 0xae, 0x5d, 0xce, + 0x64, 0x50, 0xba, 0x86, 0x31, 0xa1, 0xac, 0x8a, 0x22, 0xac, 0x51, 0xb3, 0x7f, 0xd5, 0x82, 0x33, + 0xa9, 0x2f, 0xba, 0xe5, 0x46, 0x31, 0x7a, 0xa7, 0xeb, 0xab, 0xe6, 0x07, 0xfb, 0x2a, 0x5a, 0x9b, + 0x7d, 0x93, 0x5a, 0xca, 0xb2, 0x44, 0xfb, 0xa2, 0x1b, 0x30, 0xe4, 0xc6, 0x64, 0x57, 0x7e, 0xcc, + 0x93, 0x3d, 0x3f, 0x86, 0xf7, 0x2a, 0x99, 0x91, 0x1a, 0xad, 0x89, 0x39, 0x01, 0xfb, 0x87, 0x8a, + 0x50, 0xe6, 0xcb, 0x76, 0xd5, 0x69, 0x9f, 0xc2, 0x5c, 0xd4, 0xa0, 0xc4, 0xa8, 0xf3, 0x8e, 0x3f, + 0x95, 0xdd, 0x71, 0xd1, 0x9d, 0x79, 0x2a, 0xa7, 0x71, 0x56, 0x50, 0x5d, 0x0d, 0xb4, 0x08, 0x33, + 0x12, 0xc8, 0x01, 0xd8, 0x70, 0x7d, 0x27, 0xdc, 0xa7, 0x65, 0xb3, 0x45, 0x46, 0xf0, 0xb9, 0xde, + 0x04, 0x17, 0x15, 0x3e, 0x27, 0xab, 0xfa, 0x9a, 0x00, 0xb0, 0x46, 0x74, 0xee, 0x65, 0x28, 0x2b, + 0xe4, 0xe3, 0xf0, 0x38, 0x73, 0x9f, 0x85, 0xa9, 0x54, 0x5b, 0xfd, 0xaa, 0x8f, 0xeb, 0x2c, 0xd2, + 0x2f, 0xb3, 0x53, 0x40, 0xf4, 0x7a, 0xd9, 0xdf, 0x13, 0xa7, 0xe8, 0xfb, 0x70, 0xd6, 0xcb, 0x38, + 0x9c, 0xc4, 0x54, 0x0d, 0x7e, 0x98, 0x3d, 0x2e, 0x3e, 0xfb, 0x6c, 0x16, 0x14, 0x67, 0xb6, 0x41, + 0xaf, 0xfd, 0xa0, 0x4d, 0xd7, 0xbc, 0xe3, 0xb1, 0xfe, 0x0a, 0xe9, 0xfb, 0xb6, 0x28, 0xc3, 0x0a, + 0x4a, 0x8f, 0xb0, 0xb3, 0xaa, 0xf3, 0x37, 0xc9, 0x7e, 0x83, 0x78, 0xa4, 0x19, 0x07, 0xe1, 0x87, + 0xda, 0xfd, 0x0b, 0x7c, 0xf4, 0xf9, 0x09, 0x38, 0x26, 0x08, 0x14, 0x6f, 0x92, 0x7d, 0x3e, 0x15, + 0xfa, 0xd7, 0x15, 0x7b, 0x7e, 0xdd, 0xcf, 0x5a, 0x30, 0xa1, 0xbe, 0xee, 0x14, 0xb6, 0xfa, 0xa2, + 0xb9, 0xd5, 0x2f, 0xf4, 0x5c, 0xe0, 0x39, 0x9b, 0xfc, 0xeb, 0x05, 0x38, 0xaf, 0x70, 0x28, 0xbb, + 0xcf, 0xff, 0x88, 0x55, 0x75, 0x15, 0xca, 0xbe, 0xd2, 0x1e, 0x58, 0xa6, 0xd8, 0x9e, 0xe8, 0x0e, + 0x12, 0x1c, 0xca, 0xb5, 0xf9, 0x89, 0x88, 0x3f, 0xae, 0xab, 0xd5, 0x84, 0x0a, 0x6d, 0x11, 0x8a, + 0x1d, 0xb7, 0x25, 0xee, 0x8c, 0x4f, 0xc9, 0xd1, 0xbe, 0x53, 0xab, 0x1e, 0x1d, 0x54, 0x9e, 0xc8, + 0x53, 0xe9, 0xd2, 0xcb, 0x2a, 0x9a, 0xbf, 0x53, 0xab, 0x62, 0x5a, 0x19, 0x2d, 0xc0, 0x94, 0xd4, + 0x5a, 0xdf, 0xa5, 0x1c, 0x54, 0xe0, 0x8b, 0xab, 0x45, 0xe9, 0xc6, 0xb0, 0x09, 0xc6, 0x69, 0x7c, + 0x54, 0x85, 0xe9, 0x9d, 0xce, 0x06, 0xf1, 0x48, 0xcc, 0x3f, 0xf8, 0x26, 0xe1, 0x9a, 0xa3, 0x72, + 0x22, 0x5a, 0xde, 0x4c, 0xc1, 0x71, 0x57, 0x0d, 0xfb, 0xcf, 0xd8, 0x11, 0x2f, 0x46, 0xaf, 0x1e, + 0x06, 0x74, 0x61, 0x51, 0xea, 0x1f, 0xe6, 0x72, 0x1e, 0x64, 0x55, 0xdc, 0x24, 0xfb, 0xeb, 0x01, + 0x65, 0xb6, 0xb3, 0x57, 0x85, 0xb1, 0xe6, 0x4b, 0x3d, 0xd7, 0xfc, 0xcf, 0x17, 0xe0, 0x9c, 0x1a, + 0x01, 0x83, 0xaf, 0xfb, 0xf3, 0x3e, 0x06, 0xcf, 0xc3, 0x58, 0x8b, 0x6c, 0x3a, 0x1d, 0x2f, 0x56, + 0x6a, 0xcc, 0x21, 0xae, 0xca, 0xae, 0x26, 0xc5, 0x58, 0xc7, 0x39, 0xc6, 0xb0, 0xfd, 0xc4, 0x18, + 0xbb, 0x5b, 0x63, 0x87, 0xae, 0x71, 0xb5, 0x6b, 0xac, 0xdc, 0x5d, 0xf3, 0x24, 0x0c, 0xb9, 0xbb, + 0x94, 0xd7, 0x2a, 0x98, 0x2c, 0x54, 0x8d, 0x16, 0x62, 0x0e, 0x43, 0x9f, 0x80, 0x91, 0x66, 0xb0, + 0xbb, 0xeb, 0xf8, 0x2d, 0x76, 0xe5, 0x95, 0x17, 0xc7, 0x28, 0x3b, 0xb6, 0xc4, 0x8b, 0xb0, 0x84, + 0xa1, 0xc7, 0xa1, 0xe4, 0x84, 0x5b, 0xd1, 0x6c, 0x89, 0xe1, 0x8c, 0xd2, 0x96, 0x16, 0xc2, 0xad, + 0x08, 0xb3, 0x52, 0x2a, 0x55, 0xdd, 0x0b, 0xc2, 0x1d, 0xd7, 0xdf, 0xaa, 0xba, 0xa1, 0xd8, 0x12, + 0xea, 0x2e, 0x7c, 0x4b, 0x41, 0xb0, 0x86, 0x85, 0x56, 0x60, 0xa8, 0x1d, 0x84, 0x71, 0x34, 0x3b, + 0xcc, 0x86, 0xfb, 0x89, 0x9c, 0x83, 0x88, 0x7f, 0x6d, 0x3d, 0x08, 0xe3, 0xe4, 0x03, 0xe8, 0xbf, + 0x08, 0xf3, 0xea, 0xe8, 0xdb, 0xa0, 0x48, 0xfc, 0xbd, 0xd9, 0x11, 0x46, 0x65, 0x2e, 0x8b, 0xca, + 0xb2, 0xbf, 0x77, 0xd7, 0x09, 0x93, 0x53, 0x7a, 0xd9, 0xdf, 0xc3, 0xb4, 0x0e, 0xfa, 0x3c, 0x94, + 0xe5, 0x16, 0x8f, 0x84, 0x9a, 0x23, 0x73, 0x89, 0xc9, 0x83, 0x01, 0x93, 0xf7, 0x3a, 0x6e, 0x48, + 0x76, 0x89, 0x1f, 0x47, 0xc9, 0x99, 0x26, 0xa1, 0x11, 0x4e, 0xa8, 0xa1, 0xcf, 0x4b, 0xdd, 0xda, + 0x6a, 0xd0, 0xf1, 0xe3, 0x68, 0xb6, 0xcc, 0xba, 0x97, 0xf9, 0xea, 0x71, 0x37, 0xc1, 0x4b, 0x2b, + 0xdf, 0x78, 0x65, 0x6c, 0x90, 0x42, 0x18, 0x26, 0x3c, 0x77, 0x8f, 0xf8, 0x24, 0x8a, 0xea, 0x61, + 0xb0, 0x41, 0x66, 0x81, 0xf5, 0xfc, 0x7c, 0xf6, 0x63, 0x40, 0xb0, 0x41, 0x16, 0x67, 0x0e, 0x0f, + 0x2a, 0x13, 0xb7, 0xf4, 0x3a, 0xd8, 0x24, 0x81, 0xee, 0xc0, 0x24, 0x95, 0x6b, 0xdc, 0x84, 0xe8, + 0x58, 0x3f, 0xa2, 0x4c, 0xfa, 0xc0, 0x46, 0x25, 0x9c, 0x22, 0x82, 0xde, 0x80, 0xb2, 0xe7, 0x6e, + 0x92, 0xe6, 0x7e, 0xd3, 0x23, 0xb3, 0xe3, 0x8c, 0x62, 0xe6, 0xb6, 0xba, 0x25, 0x91, 0xb8, 0x5c, + 0xa4, 0xfe, 0xe2, 0xa4, 0x3a, 0xba, 0x0b, 0x8f, 0xc4, 0x24, 0xdc, 0x75, 0x7d, 0x87, 0x6e, 0x07, + 0x21, 0x2f, 0xb0, 0x27, 0x95, 0x09, 0xb6, 0xde, 0x2e, 0x8a, 0xa1, 0x7b, 0x64, 0x3d, 0x13, 0x0b, + 0xe7, 0xd4, 0x46, 0xb7, 0x61, 0x8a, 0xed, 0x84, 0x7a, 0xc7, 0xf3, 0xea, 0x81, 0xe7, 0x36, 0xf7, + 0x67, 0x27, 0x19, 0xc1, 0x4f, 0xc8, 0x7b, 0xa1, 0x66, 0x82, 0x8f, 0x0e, 0x2a, 0x90, 0xfc, 0xc3, + 0xe9, 0xda, 0x68, 0x83, 0xe9, 0xd0, 0x3b, 0xa1, 0x1b, 0xef, 0xd3, 0xf5, 0x4b, 0xee, 0xc7, 0xb3, + 0x53, 0x3d, 0x45, 0x61, 0x1d, 0x55, 0x29, 0xda, 0xf5, 0x42, 0x9c, 0x26, 0x48, 0xb7, 0x76, 0x14, + 0xb7, 0x5c, 0x7f, 0x76, 0x9a, 0x9d, 0x18, 0x6a, 0x67, 0x34, 0x68, 0x21, 0xe6, 0x30, 0xa6, 0x3f, + 0xa7, 0x3f, 0x6e, 0xd3, 0x13, 0x74, 0x86, 0x21, 0x26, 0xfa, 0x73, 0x09, 0xc0, 0x09, 0x0e, 0x65, + 0x6a, 0xe2, 0x78, 0x7f, 0x16, 0x31, 0x54, 0xb5, 0x5d, 0xd6, 0xd7, 0x3f, 0x8f, 0x69, 0x39, 0xba, + 0x05, 0x23, 0xc4, 0xdf, 0x5b, 0x09, 0x83, 0xdd, 0xd9, 0x33, 0xf9, 0x7b, 0x76, 0x99, 0xa3, 0xf0, + 0x03, 0x3d, 0x11, 0xf0, 0x44, 0x31, 0x96, 0x24, 0xd0, 0x7d, 0x98, 0xcd, 0x98, 0x11, 0x3e, 0x01, + 0x67, 0xd9, 0x04, 0xbc, 0x26, 0xea, 0xce, 0xae, 0xe7, 0xe0, 0x1d, 0xf5, 0x80, 0xe1, 0x5c, 0xea, + 0xe8, 0x0b, 0x30, 0xc1, 0x37, 0x14, 0x7f, 0x7c, 0x8b, 0x66, 0xcf, 0xb1, 0xaf, 0xb9, 0x94, 0xbf, + 0x39, 0x39, 0xe2, 0xe2, 0x39, 0xd1, 0xa1, 0x09, 0xbd, 0x34, 0xc2, 0x26, 0x35, 0x7b, 0x03, 0x26, + 0xd5, 0xb9, 0xc5, 0x96, 0x0e, 0xaa, 0xc0, 0x10, 0xe3, 0x76, 0x84, 0x7e, 0xab, 0x4c, 0x67, 0x8a, + 0x71, 0x42, 0x98, 0x97, 0xb3, 0x99, 0x72, 0xdf, 0x27, 0x8b, 0xfb, 0x31, 0xe1, 0x52, 0x75, 0x51, + 0x9b, 0x29, 0x09, 0xc0, 0x09, 0x8e, 0xfd, 0xff, 0x38, 0xd7, 0x98, 0x1c, 0x8e, 0x03, 0x5c, 0x07, + 0xcf, 0xc2, 0xe8, 0x76, 0x10, 0xc5, 0x14, 0x9b, 0xb5, 0x31, 0x94, 0xf0, 0x89, 0x37, 0x44, 0x39, + 0x56, 0x18, 0xe8, 0x55, 0x98, 0x68, 0xea, 0x0d, 0x88, 0xbb, 0x4c, 0x0d, 0x81, 0xd1, 0x3a, 0x36, + 0x71, 0xd1, 0x2b, 0x30, 0xca, 0x9e, 0xce, 0x9b, 0x81, 0x27, 0x98, 0x2c, 0x79, 0x21, 0x8f, 0xd6, + 0x45, 0xf9, 0x91, 0xf6, 0x1b, 0x2b, 0x6c, 0x74, 0x19, 0x86, 0x69, 0x17, 0x6a, 0x75, 0x71, 0x8b, + 0x28, 0x55, 0xcd, 0x0d, 0x56, 0x8a, 0x05, 0xd4, 0xfe, 0x5b, 0x05, 0x6d, 0x94, 0xa9, 0x44, 0x4a, + 0x50, 0x1d, 0x46, 0xee, 0x39, 0x6e, 0xec, 0xfa, 0x5b, 0x82, 0x5d, 0x78, 0xba, 0xe7, 0x95, 0xc2, + 0x2a, 0xbd, 0xc5, 0x2b, 0xf0, 0x4b, 0x4f, 0xfc, 0xc1, 0x92, 0x0c, 0xa5, 0x18, 0x76, 0x7c, 0x9f, + 0x52, 0x2c, 0x0c, 0x4a, 0x11, 0xf3, 0x0a, 0x9c, 0xa2, 0xf8, 0x83, 0x25, 0x19, 0xf4, 0x0e, 0x80, + 0x5c, 0x96, 0xa4, 0x25, 0x9e, 0xac, 0x9f, 0xed, 0x4f, 0x74, 0x5d, 0xd5, 0x59, 0x9c, 0xa4, 0x57, + 0x6a, 0xf2, 0x1f, 0x6b, 0xf4, 0xec, 0x98, 0xb1, 0x55, 0xdd, 0x9d, 0x41, 0xdf, 0x49, 0x4f, 0x02, + 0x27, 0x8c, 0x49, 0x6b, 0x21, 0x16, 0x83, 0xf3, 0xc9, 0xc1, 0x64, 0x8a, 0x75, 0x77, 0x97, 0xe8, + 0xa7, 0x86, 0x20, 0x82, 0x13, 0x7a, 0xf6, 0x2f, 0x16, 0x61, 0x36, 0xaf, 0xbb, 0x74, 0xd1, 0x91, + 0xfb, 0x6e, 0xbc, 0x44, 0xb9, 0x21, 0xcb, 0x5c, 0x74, 0xcb, 0xa2, 0x1c, 0x2b, 0x0c, 0x3a, 0xfb, + 0x91, 0xbb, 0x25, 0x45, 0xc2, 0xa1, 0x64, 0xf6, 0x1b, 0xac, 0x14, 0x0b, 0x28, 0xc5, 0x0b, 0x89, + 0x13, 0x09, 0x9b, 0x08, 0x6d, 0x95, 0x60, 0x56, 0x8a, 0x05, 0x54, 0xd7, 0x37, 0x95, 0xfa, 0xe8, + 0x9b, 0x8c, 0x21, 0x1a, 0x3a, 0xd9, 0x21, 0x42, 0x5f, 0x04, 0xd8, 0x74, 0x7d, 0x37, 0xda, 0x66, + 0xd4, 0x87, 0x8f, 0x4d, 0x5d, 0xf1, 0x52, 0x2b, 0x8a, 0x0a, 0xd6, 0x28, 0xa2, 0x97, 0x60, 0x4c, + 0x6d, 0xc0, 0x5a, 0x95, 0x3d, 0x10, 0x69, 0x0f, 0xee, 0xc9, 0x69, 0x54, 0xc5, 0x3a, 0x9e, 0xfd, + 0x6e, 0x7a, 0xbd, 0x88, 0x1d, 0xa0, 0x8d, 0xaf, 0x35, 0xe8, 0xf8, 0x16, 0x7a, 0x8f, 0xaf, 0xfd, + 0xeb, 0x45, 0x98, 0x32, 0x1a, 0xeb, 0x44, 0x03, 0x9c, 0x59, 0xd7, 0xe9, 0x3d, 0xe7, 0xc4, 0x44, + 0xec, 0x3f, 0xbb, 0xff, 0x56, 0xd1, 0xef, 0x42, 0xba, 0x03, 0x78, 0x7d, 0xf4, 0x45, 0x28, 0x7b, + 0x4e, 0xc4, 0x74, 0x57, 0x44, 0xec, 0xbb, 0x41, 0x88, 0x25, 0x72, 0x84, 0x13, 0xc5, 0xda, 0x55, + 0xc3, 0x69, 0x27, 0x24, 0xe9, 0x85, 0x4c, 0x79, 0x1f, 0x69, 0x74, 0xa3, 0x3a, 0x41, 0x19, 0xa4, + 0x7d, 0xcc, 0x61, 0xe8, 0x15, 0x18, 0x0f, 0x09, 0x5b, 0x15, 0x4b, 0x94, 0x95, 0x63, 0xcb, 0x6c, + 0x28, 0xe1, 0xf9, 0xb0, 0x06, 0xc3, 0x06, 0x66, 0xc2, 0xca, 0x0f, 0xf7, 0x60, 0xe5, 0x9f, 0x86, + 0x11, 0xf6, 0x43, 0xad, 0x00, 0x35, 0x1b, 0x35, 0x5e, 0x8c, 0x25, 0x3c, 0xbd, 0x60, 0x46, 0x07, + 0x5c, 0x30, 0x9f, 0x84, 0xc9, 0xaa, 0x43, 0x76, 0x03, 0x7f, 0xd9, 0x6f, 0xb5, 0x03, 0xd7, 0x8f, + 0xd1, 0x2c, 0x94, 0xd8, 0xed, 0xc0, 0xf7, 0x76, 0x89, 0x52, 0xc0, 0x25, 0xca, 0x98, 0xdb, 0x5b, + 0x70, 0xae, 0x1a, 0xdc, 0xf3, 0xef, 0x39, 0x61, 0x6b, 0xa1, 0x5e, 0xd3, 0xe4, 0xdc, 0x35, 0x29, + 0x67, 0x71, 0x23, 0x96, 0xcc, 0x33, 0x55, 0xab, 0xc9, 0xef, 0xda, 0x15, 0xd7, 0x23, 0x39, 0xda, + 0x88, 0xbf, 0x53, 0x30, 0x5a, 0x4a, 0xf0, 0xd5, 0x83, 0x91, 0x95, 0xfb, 0x60, 0xf4, 0x26, 0x8c, + 0x6e, 0xba, 0xc4, 0x6b, 0x61, 0xb2, 0x29, 0x96, 0xd8, 0x53, 0xf9, 0xef, 0xf2, 0x2b, 0x14, 0x53, + 0x6a, 0x9f, 0xb8, 0x94, 0xb6, 0x22, 0x2a, 0x63, 0x45, 0x06, 0xed, 0xc0, 0xb4, 0x14, 0x03, 0x24, + 0x54, 0x2c, 0xb8, 0xa7, 0x7b, 0xc9, 0x16, 0x26, 0xf1, 0xb3, 0x87, 0x07, 0x95, 0x69, 0x9c, 0x22, + 0x83, 0xbb, 0x08, 0x53, 0xb1, 0x6c, 0x97, 0x1e, 0xad, 0x25, 0x36, 0xfc, 0x4c, 0x2c, 0x63, 0x12, + 0x26, 0x2b, 0xb5, 0x7f, 0xd4, 0x82, 0x47, 0xbb, 0x46, 0x46, 0x48, 0xda, 0x27, 0x3c, 0x0b, 0x69, + 0xc9, 0xb7, 0xd0, 0x5f, 0xf2, 0xb5, 0xff, 0xb1, 0x05, 0x67, 0x97, 0x77, 0xdb, 0xf1, 0x7e, 0xd5, + 0x35, 0x5f, 0x77, 0x5e, 0x86, 0xe1, 0x5d, 0xd2, 0x72, 0x3b, 0xbb, 0x62, 0xe6, 0x2a, 0xf2, 0xf8, + 0x59, 0x65, 0xa5, 0x47, 0x07, 0x95, 0x89, 0x46, 0x1c, 0x84, 0xce, 0x16, 0xe1, 0x05, 0x58, 0xa0, + 0xb3, 0x43, 0xdc, 0x7d, 0x9f, 0xdc, 0x72, 0x77, 0x5d, 0x69, 0x67, 0xd1, 0x53, 0x77, 0x36, 0x2f, + 0x07, 0x74, 0xfe, 0xcd, 0x8e, 0xe3, 0xc7, 0x6e, 0xbc, 0x2f, 0x1e, 0x66, 0x24, 0x11, 0x9c, 0xd0, + 0xb3, 0xbf, 0x69, 0xc1, 0x94, 0x5c, 0xf7, 0x0b, 0xad, 0x56, 0x48, 0xa2, 0x08, 0xcd, 0x41, 0xc1, + 0x6d, 0x8b, 0x5e, 0x82, 0xe8, 0x65, 0xa1, 0x56, 0xc7, 0x05, 0xb7, 0x8d, 0xea, 0x50, 0xe6, 0xe6, + 0x1a, 0xc9, 0xe2, 0x1a, 0xc8, 0xe8, 0x83, 0xf5, 0x60, 0x5d, 0xd6, 0xc4, 0x09, 0x11, 0xc9, 0xc1, + 0xb1, 0x33, 0xb3, 0x68, 0xbe, 0x7a, 0xdd, 0x10, 0xe5, 0x58, 0x61, 0xa0, 0x2b, 0x30, 0xea, 0x07, + 0x2d, 0x6e, 0x3d, 0xc3, 0x6f, 0x3f, 0xb6, 0x64, 0xd7, 0x44, 0x19, 0x56, 0x50, 0xfb, 0x07, 0x2d, + 0x18, 0x97, 0x5f, 0x36, 0x20, 0x33, 0x49, 0xb7, 0x56, 0xc2, 0x48, 0x26, 0x5b, 0x8b, 0x32, 0x83, + 0x0c, 0x62, 0xf0, 0x80, 0xc5, 0xe3, 0xf0, 0x80, 0xf6, 0x8f, 0x14, 0x60, 0x52, 0x76, 0xa7, 0xd1, + 0xd9, 0x88, 0x48, 0x8c, 0xd6, 0xa1, 0xec, 0xf0, 0x21, 0x27, 0x72, 0xc5, 0x3e, 0x99, 0x2d, 0x7c, + 0x18, 0xf3, 0x93, 0x5c, 0xcb, 0x0b, 0xb2, 0x36, 0x4e, 0x08, 0x21, 0x0f, 0x66, 0xfc, 0x20, 0x66, + 0x47, 0xb4, 0x82, 0xf7, 0x7a, 0x02, 0x49, 0x53, 0x3f, 0x2f, 0xa8, 0xcf, 0xac, 0xa5, 0xa9, 0xe0, + 0x6e, 0xc2, 0x68, 0x59, 0x2a, 0x3c, 0x8a, 0xf9, 0xe2, 0x86, 0x3e, 0x0b, 0xd9, 0xfa, 0x0e, 0xfb, + 0x57, 0x2c, 0x28, 0x4b, 0xb4, 0xd3, 0x78, 0xed, 0x5a, 0x85, 0x91, 0x88, 0x4d, 0x82, 0x1c, 0x1a, + 0xbb, 0x57, 0xc7, 0xf9, 0x7c, 0x25, 0x37, 0x0f, 0xff, 0x1f, 0x61, 0x49, 0x83, 0xe9, 0xbb, 0x55, + 0xf7, 0x3f, 0x22, 0xfa, 0x6e, 0xd5, 0x9f, 0x9c, 0x1b, 0xe6, 0xbf, 0xb3, 0x3e, 0x6b, 0x62, 0x2d, + 0x65, 0x90, 0xda, 0x21, 0xd9, 0x74, 0xef, 0xa7, 0x19, 0xa4, 0x3a, 0x2b, 0xc5, 0x02, 0x8a, 0xde, + 0x81, 0xf1, 0xa6, 0x54, 0x74, 0x26, 0xc7, 0xc0, 0xe5, 0x9e, 0x4a, 0x77, 0xf5, 0x3e, 0xc3, 0x2d, + 0x6b, 0x97, 0xb4, 0xfa, 0xd8, 0xa0, 0x66, 0x3e, 0xb7, 0x17, 0xfb, 0x3d, 0xb7, 0x27, 0x74, 0xf3, + 0x1f, 0x9f, 0x7f, 0xcc, 0x82, 0x61, 0xae, 0x2e, 0x1b, 0x4c, 0xbf, 0xa8, 0x3d, 0x57, 0x25, 0x63, + 0x77, 0x97, 0x16, 0x8a, 0xe7, 0x27, 0xb4, 0x0a, 0x65, 0xf6, 0x83, 0xa9, 0x0d, 0x8a, 0xf9, 0x26, + 0xc5, 0xbc, 0x55, 0xbd, 0x83, 0x77, 0x65, 0x35, 0x9c, 0x50, 0xb0, 0xbf, 0x56, 0xa4, 0x47, 0x55, + 0x82, 0x6a, 0xdc, 0xe0, 0xd6, 0xc3, 0xbb, 0xc1, 0x0b, 0x0f, 0xeb, 0x06, 0xdf, 0x82, 0xa9, 0xa6, + 0xf6, 0xb8, 0x95, 0xcc, 0xe4, 0x95, 0x9e, 0x8b, 0x44, 0x7b, 0x07, 0xe3, 0x2a, 0xa3, 0x25, 0x93, + 0x08, 0x4e, 0x53, 0x45, 0xdf, 0x09, 0xe3, 0x7c, 0x9e, 0x45, 0x2b, 0xdc, 0x62, 0xe1, 0x13, 0xf9, + 0xeb, 0x45, 0x6f, 0x82, 0xad, 0xc4, 0x86, 0x56, 0x1d, 0x1b, 0xc4, 0xec, 0x5f, 0x1c, 0x85, 0xa1, + 0xe5, 0x3d, 0xe2, 0xc7, 0xa7, 0x70, 0x20, 0x35, 0x61, 0xd2, 0xf5, 0xf7, 0x02, 0x6f, 0x8f, 0xb4, + 0x38, 0xfc, 0x38, 0x97, 0xeb, 0x23, 0x82, 0xf4, 0x64, 0xcd, 0x20, 0x81, 0x53, 0x24, 0x1f, 0x86, + 0x84, 0x79, 0x1d, 0x86, 0xf9, 0xdc, 0x0b, 0xf1, 0x32, 0x53, 0x19, 0xcc, 0x06, 0x51, 0xec, 0x82, + 0x44, 0xfa, 0xe5, 0xda, 0x67, 0x51, 0x1d, 0xbd, 0x0b, 0x93, 0x9b, 0x6e, 0x18, 0xc5, 0x54, 0x34, + 0x8c, 0x62, 0x67, 0xb7, 0xfd, 0x00, 0x12, 0xa5, 0x1a, 0x87, 0x15, 0x83, 0x12, 0x4e, 0x51, 0x46, + 0x5b, 0x30, 0x41, 0x85, 0x9c, 0xa4, 0xa9, 0x91, 0x63, 0x37, 0xa5, 0x54, 0x46, 0xb7, 0x74, 0x42, + 0xd8, 0xa4, 0x4b, 0x0f, 0x93, 0x26, 0x13, 0x8a, 0x46, 0x19, 0x47, 0xa1, 0x0e, 0x13, 0x2e, 0x0d, + 0x71, 0x18, 0x3d, 0x93, 0x98, 0xd9, 0x4a, 0xd9, 0x3c, 0x93, 0x34, 0xe3, 0x94, 0x2f, 0x41, 0x99, + 0xd0, 0x21, 0xa4, 0x84, 0x85, 0x62, 0xfc, 0xea, 0x60, 0x7d, 0x5d, 0x75, 0x9b, 0x61, 0x60, 0xca, + 0xf2, 0xcb, 0x92, 0x12, 0x4e, 0x88, 0xa2, 0x25, 0x18, 0x8e, 0x48, 0xe8, 0x92, 0x48, 0xa8, 0xc8, + 0x7b, 0x4c, 0x23, 0x43, 0xe3, 0xb6, 0xe7, 0xfc, 0x37, 0x16, 0x55, 0xe9, 0xf2, 0x72, 0x98, 0x34, + 0xc4, 0xb4, 0xe2, 0xda, 0xf2, 0x5a, 0x60, 0xa5, 0x58, 0x40, 0xd1, 0x1b, 0x30, 0x12, 0x12, 0x8f, + 0x29, 0x8b, 0x26, 0x06, 0x5f, 0xe4, 0x5c, 0xf7, 0xc4, 0xeb, 0x61, 0x49, 0x00, 0xdd, 0x04, 0x14, + 0x12, 0xca, 0x43, 0xb8, 0xfe, 0x96, 0x32, 0xe6, 0x10, 0xba, 0xee, 0xc7, 0x44, 0xfb, 0x67, 0x70, + 0x82, 0x21, 0xad, 0x52, 0x71, 0x46, 0x35, 0x74, 0x1d, 0x66, 0x54, 0x69, 0xcd, 0x8f, 0x62, 0xc7, + 0x6f, 0x12, 0xa6, 0xe6, 0x2e, 0x27, 0x5c, 0x11, 0x4e, 0x23, 0xe0, 0xee, 0x3a, 0xf6, 0x4f, 0x53, + 0x76, 0x86, 0x8e, 0xd6, 0x29, 0xf0, 0x02, 0xaf, 0x9b, 0xbc, 0xc0, 0xf9, 0xdc, 0x99, 0xcb, 0xe1, + 0x03, 0x0e, 0x2d, 0x18, 0xd3, 0x66, 0x36, 0x59, 0xb3, 0x56, 0x8f, 0x35, 0xdb, 0x81, 0x69, 0xba, + 0xd2, 0x6f, 0x6f, 0x44, 0x24, 0xdc, 0x23, 0x2d, 0xb6, 0x30, 0x0b, 0x0f, 0xb6, 0x30, 0xd5, 0x2b, + 0xf3, 0xad, 0x14, 0x41, 0xdc, 0xd5, 0x04, 0x7a, 0x59, 0x6a, 0x4e, 0x8a, 0x86, 0x91, 0x16, 0xd7, + 0x8a, 0x1c, 0x1d, 0x54, 0xa6, 0xb5, 0x0f, 0xd1, 0x35, 0x25, 0xf6, 0x97, 0xe4, 0x37, 0xaa, 0xd7, + 0xfc, 0xa6, 0x5a, 0x2c, 0xa9, 0xd7, 0x7c, 0xb5, 0x1c, 0x70, 0x82, 0x43, 0xf7, 0x28, 0x15, 0x41, + 0xd2, 0xaf, 0xf9, 0x54, 0x40, 0xc1, 0x0c, 0x62, 0xbf, 0x00, 0xb0, 0x7c, 0x9f, 0x34, 0xf9, 0x52, + 0xd7, 0x1f, 0x20, 0xad, 0xfc, 0x07, 0x48, 0xfb, 0x3f, 0x5a, 0x30, 0xb9, 0xb2, 0x64, 0x88, 0x89, + 0xf3, 0x00, 0x5c, 0x36, 0x7a, 0xeb, 0xad, 0x35, 0xa9, 0x5b, 0xe7, 0xea, 0x51, 0x55, 0x8a, 0x35, + 0x0c, 0x74, 0x1e, 0x8a, 0x5e, 0xc7, 0x17, 0x22, 0xcb, 0xc8, 0xe1, 0x41, 0xa5, 0x78, 0xab, 0xe3, + 0x63, 0x5a, 0xa6, 0x59, 0x08, 0x16, 0x07, 0xb6, 0x10, 0xec, 0xeb, 0x5e, 0x85, 0x2a, 0x30, 0x74, + 0xef, 0x9e, 0xdb, 0xe2, 0x46, 0xec, 0x42, 0xef, 0xff, 0xd6, 0x5b, 0xb5, 0x6a, 0x84, 0x79, 0xb9, + 0xfd, 0xd5, 0x22, 0xcc, 0xad, 0x78, 0xe4, 0xfe, 0x07, 0x34, 0xe4, 0x1f, 0xd4, 0xbe, 0xf1, 0x78, + 0xfc, 0xe2, 0x71, 0x6d, 0x58, 0xfb, 0x8f, 0xc7, 0x26, 0x8c, 0xf0, 0xc7, 0x6c, 0x69, 0xd6, 0xff, + 0x6a, 0x56, 0xeb, 0xf9, 0x03, 0x32, 0xcf, 0x1f, 0xc5, 0x85, 0x39, 0xbf, 0xba, 0x69, 0x45, 0x29, + 0x96, 0xc4, 0xe7, 0x3e, 0x03, 0xe3, 0x3a, 0xe6, 0xb1, 0xac, 0xc9, 0xff, 0x4a, 0x11, 0xa6, 0x69, + 0x0f, 0x1e, 0xea, 0x44, 0xdc, 0xe9, 0x9e, 0x88, 0x93, 0xb6, 0x28, 0xee, 0x3f, 0x1b, 0xef, 0xa4, + 0x67, 0xe3, 0xf9, 0xbc, 0xd9, 0x38, 0xed, 0x39, 0xf8, 0x5e, 0x0b, 0xce, 0xac, 0x78, 0x41, 0x73, + 0x27, 0x65, 0xf5, 0xfb, 0x12, 0x8c, 0xd1, 0x73, 0x3c, 0x32, 0xbc, 0x88, 0x0c, 0xbf, 0x32, 0x01, + 0xc2, 0x3a, 0x9e, 0x56, 0xed, 0xce, 0x9d, 0x5a, 0x35, 0xcb, 0x1d, 0x4d, 0x80, 0xb0, 0x8e, 0x67, + 0x7f, 0xc3, 0x82, 0x0b, 0xd7, 0x97, 0x96, 0x93, 0xa5, 0xd8, 0xe5, 0x11, 0x47, 0xa5, 0xc0, 0x96, + 0xd6, 0x95, 0x44, 0x0a, 0xac, 0xb2, 0x5e, 0x08, 0xe8, 0x47, 0xc5, 0xdb, 0xf3, 0xa7, 0x2c, 0x38, + 0x73, 0xdd, 0x8d, 0xe9, 0xb5, 0x9c, 0xf6, 0xcd, 0xa2, 0xf7, 0x72, 0xe4, 0xc6, 0x41, 0xb8, 0x9f, + 0xf6, 0xcd, 0xc2, 0x0a, 0x82, 0x35, 0x2c, 0xde, 0xf2, 0x9e, 0xcb, 0xcc, 0xa8, 0x0a, 0xa6, 0x2a, + 0x0a, 0x8b, 0x72, 0xac, 0x30, 0xe8, 0x87, 0xb5, 0xdc, 0x90, 0x89, 0x12, 0xfb, 0xe2, 0x84, 0x55, + 0x1f, 0x56, 0x95, 0x00, 0x9c, 0xe0, 0xd8, 0x7f, 0x68, 0x41, 0xe5, 0xba, 0xd7, 0x89, 0x62, 0x12, + 0x6e, 0x46, 0x39, 0xa7, 0xe3, 0x0b, 0x50, 0x26, 0x52, 0x70, 0x17, 0xbd, 0x56, 0xac, 0xa6, 0x92, + 0xe8, 0xb9, 0x8b, 0x98, 0xc2, 0x1b, 0xc0, 0x87, 0xe0, 0x78, 0x46, 0xe0, 0x2b, 0x80, 0x88, 0xde, + 0x96, 0xee, 0x33, 0xc7, 0x9c, 0x6f, 0x96, 0xbb, 0xa0, 0x38, 0xa3, 0x86, 0xfd, 0xa3, 0x16, 0x9c, + 0x53, 0x1f, 0xfc, 0x91, 0xfb, 0x4c, 0xfb, 0xe7, 0x0a, 0x30, 0x71, 0x63, 0x7d, 0xbd, 0x7e, 0x9d, + 0xc4, 0xe2, 0xda, 0xee, 0xaf, 0x5b, 0xc7, 0x9a, 0x8a, 0xb0, 0x97, 0x14, 0xd8, 0x89, 0x5d, 0x6f, + 0x9e, 0xbb, 0x5e, 0xcf, 0xd7, 0xfc, 0xf8, 0x76, 0xd8, 0x88, 0x43, 0xd7, 0xdf, 0xca, 0x54, 0x2a, + 0x4a, 0xe6, 0xa2, 0x98, 0xc7, 0x5c, 0xa0, 0x17, 0x60, 0x98, 0xf9, 0x7e, 0xcb, 0x49, 0x78, 0x4c, + 0x09, 0x51, 0xac, 0xf4, 0xe8, 0xa0, 0x52, 0xbe, 0x83, 0x6b, 0xfc, 0x0f, 0x16, 0xa8, 0xe8, 0x0e, + 0x8c, 0x6d, 0xc7, 0x71, 0xfb, 0x06, 0x71, 0x5a, 0x24, 0x94, 0xc7, 0xe1, 0xc5, 0xac, 0xe3, 0x90, + 0x0e, 0x02, 0x47, 0x4b, 0x4e, 0x90, 0xa4, 0x2c, 0xc2, 0x3a, 0x1d, 0xbb, 0x01, 0x90, 0xc0, 0x4e, + 0x48, 0xa1, 0x62, 0xff, 0xbe, 0x05, 0x23, 0xdc, 0x0d, 0x2f, 0x44, 0xaf, 0x41, 0x89, 0xdc, 0x27, + 0x4d, 0xc1, 0x2a, 0x67, 0x76, 0x38, 0xe1, 0xb4, 0xf8, 0xf3, 0x00, 0xfd, 0x8f, 0x59, 0x2d, 0x74, + 0x03, 0x46, 0x68, 0x6f, 0xaf, 0x2b, 0x9f, 0xc4, 0x27, 0xf2, 0xbe, 0x58, 0x4d, 0x3b, 0x67, 0xce, + 0x44, 0x11, 0x96, 0xd5, 0x99, 0xaa, 0xbb, 0xd9, 0x6e, 0xd0, 0x13, 0x3b, 0xee, 0xc5, 0x58, 0xac, + 0x2f, 0xd5, 0x39, 0x92, 0xa0, 0xc6, 0x55, 0xdd, 0xb2, 0x10, 0x27, 0x44, 0xec, 0x75, 0x28, 0xd3, + 0x49, 0x5d, 0xf0, 0x5c, 0xa7, 0xb7, 0x96, 0xfd, 0x19, 0x28, 0x4b, 0x8d, 0x77, 0x24, 0x3c, 0xb9, + 0x18, 0x55, 0xa9, 0x10, 0x8f, 0x70, 0x02, 0xb7, 0x37, 0xe1, 0x2c, 0x33, 0x75, 0x70, 0xe2, 0x6d, + 0x63, 0x8f, 0xf5, 0x5f, 0xcc, 0xcf, 0x0a, 0xc9, 0x93, 0xcf, 0xcc, 0xac, 0xe6, 0x2c, 0x31, 0x2e, + 0x29, 0x26, 0x52, 0xa8, 0xfd, 0x07, 0x25, 0x78, 0xac, 0xd6, 0xc8, 0xf7, 0xd0, 0x7c, 0x05, 0xc6, + 0x39, 0x5f, 0x4a, 0x97, 0xb6, 0xe3, 0x89, 0x76, 0xd5, 0x43, 0xe0, 0xba, 0x06, 0xc3, 0x06, 0x26, + 0xba, 0x00, 0x45, 0xf7, 0x3d, 0x3f, 0x6d, 0x77, 0x5c, 0x7b, 0x73, 0x0d, 0xd3, 0x72, 0x0a, 0xa6, + 0x2c, 0x2e, 0xbf, 0x3b, 0x14, 0x58, 0xb1, 0xb9, 0xaf, 0xc3, 0xa4, 0x1b, 0x35, 0x23, 0xb7, 0xe6, + 0xd3, 0x73, 0x46, 0x3b, 0xa9, 0x94, 0x56, 0x84, 0x76, 0x5a, 0x41, 0x71, 0x0a, 0x5b, 0xbb, 0xc8, + 0x86, 0x06, 0x66, 0x93, 0xfb, 0xba, 0x36, 0x51, 0x09, 0xa0, 0xcd, 0xbe, 0x2e, 0x62, 0x56, 0x7c, + 0x42, 0x02, 0xe0, 0x1f, 0x1c, 0x61, 0x09, 0xa3, 0x22, 0x67, 0x73, 0xdb, 0x69, 0x2f, 0x74, 0xe2, + 0xed, 0xaa, 0x1b, 0x35, 0x83, 0x3d, 0x12, 0xee, 0x33, 0x6d, 0xc1, 0x68, 0x22, 0x72, 0x2a, 0xc0, + 0xd2, 0x8d, 0x85, 0x3a, 0xc5, 0xc4, 0xdd, 0x75, 0x4c, 0x36, 0x18, 0x4e, 0x82, 0x0d, 0x5e, 0x80, + 0x29, 0xd9, 0x4c, 0x83, 0x44, 0xec, 0x52, 0x1c, 0x63, 0x1d, 0x53, 0xb6, 0xc5, 0xa2, 0x58, 0x75, + 0x2b, 0x8d, 0x8f, 0x5e, 0x86, 0x09, 0xd7, 0x77, 0x63, 0xd7, 0x89, 0x83, 0x90, 0xb1, 0x14, 0x5c, + 0x31, 0xc0, 0x4c, 0xf7, 0x6a, 0x3a, 0x00, 0x9b, 0x78, 0xf6, 0x7f, 0x2d, 0xc1, 0x0c, 0x9b, 0xb6, + 0x6f, 0xad, 0xb0, 0x8f, 0xcc, 0x0a, 0xbb, 0xd3, 0xbd, 0xc2, 0x4e, 0x82, 0xbf, 0xff, 0x30, 0x97, + 0xd9, 0xbb, 0x50, 0x56, 0xc6, 0xcf, 0xd2, 0xfb, 0xc1, 0xca, 0xf1, 0x7e, 0xe8, 0xcf, 0x7d, 0xc8, + 0x77, 0xeb, 0x62, 0xe6, 0xbb, 0xf5, 0xdf, 0xb3, 0x20, 0xb1, 0x01, 0x45, 0x37, 0xa0, 0xdc, 0x0e, + 0x98, 0x9d, 0x45, 0x28, 0x8d, 0x97, 0x1e, 0xcb, 0xbc, 0xa8, 0xf8, 0xa5, 0xc8, 0xc7, 0xaf, 0x2e, + 0x6b, 0xe0, 0xa4, 0x32, 0x5a, 0x84, 0x91, 0x76, 0x48, 0x1a, 0x31, 0xf3, 0xf9, 0xed, 0x4b, 0x87, + 0xaf, 0x11, 0x8e, 0x8f, 0x65, 0x45, 0xfb, 0xe7, 0x2d, 0x00, 0xfe, 0x34, 0xec, 0xf8, 0x5b, 0xe4, + 0x14, 0xd4, 0xdd, 0x55, 0x28, 0x45, 0x6d, 0xd2, 0xec, 0x65, 0x01, 0x93, 0xf4, 0xa7, 0xd1, 0x26, + 0xcd, 0x64, 0xc0, 0xe9, 0x3f, 0xcc, 0x6a, 0xdb, 0xdf, 0x07, 0x30, 0x99, 0xa0, 0xd5, 0x62, 0xb2, + 0x8b, 0x9e, 0x33, 0x7c, 0x00, 0xcf, 0xa7, 0x7c, 0x00, 0xcb, 0x0c, 0x5b, 0xd3, 0xac, 0xbe, 0x0b, + 0xc5, 0x5d, 0xe7, 0xbe, 0x50, 0x9d, 0x3d, 0xd3, 0xbb, 0x1b, 0x94, 0xfe, 0xfc, 0xaa, 0x73, 0x9f, + 0x0b, 0x89, 0xcf, 0xc8, 0x05, 0xb2, 0xea, 0xdc, 0x3f, 0xe2, 0x76, 0x2e, 0xec, 0x90, 0xba, 0xe5, + 0x46, 0xf1, 0x97, 0xff, 0x4b, 0xf2, 0x9f, 0x2d, 0x3b, 0xda, 0x08, 0x6b, 0xcb, 0xf5, 0xc5, 0x43, + 0xe9, 0x40, 0x6d, 0xb9, 0x7e, 0xba, 0x2d, 0xd7, 0x1f, 0xa0, 0x2d, 0xd7, 0x47, 0xef, 0xc3, 0x88, + 0x30, 0x4a, 0x60, 0xc6, 0xed, 0xa6, 0x5a, 0x2e, 0xaf, 0x3d, 0x61, 0xd3, 0xc0, 0xdb, 0xbc, 0x2a, + 0x85, 0x60, 0x51, 0xda, 0xb7, 0x5d, 0xd9, 0x20, 0xfa, 0xdb, 0x16, 0x4c, 0x8a, 0xdf, 0x98, 0xbc, + 0xd7, 0x21, 0x51, 0x2c, 0x78, 0xcf, 0x4f, 0x0f, 0xde, 0x07, 0x51, 0x91, 0x77, 0xe5, 0xd3, 0xf2, + 0x98, 0x35, 0x81, 0x7d, 0x7b, 0x94, 0xea, 0x05, 0xfa, 0xa7, 0x16, 0x9c, 0xdd, 0x75, 0xee, 0xf3, + 0x16, 0x79, 0x19, 0x76, 0x62, 0x37, 0x10, 0xc6, 0xfa, 0xaf, 0x0d, 0x36, 0xfd, 0x5d, 0xd5, 0x79, + 0x27, 0xa5, 0x5d, 0xef, 0xd9, 0x2c, 0x94, 0xbe, 0x5d, 0xcd, 0xec, 0xd7, 0xdc, 0x26, 0x8c, 0xca, + 0xf5, 0x96, 0xa1, 0x6a, 0xa8, 0xea, 0x8c, 0xf5, 0xb1, 0x6d, 0x42, 0x74, 0x47, 0x3c, 0xda, 0x8e, + 0x58, 0x6b, 0x0f, 0xb5, 0x9d, 0x77, 0x61, 0x5c, 0x5f, 0x63, 0x0f, 0xb5, 0xad, 0xf7, 0xe0, 0x4c, + 0xc6, 0x5a, 0x7a, 0xa8, 0x4d, 0xde, 0x83, 0xf3, 0xb9, 0xeb, 0xe3, 0x61, 0x36, 0x6c, 0xff, 0x9c, + 0xa5, 0x9f, 0x83, 0xa7, 0xf0, 0xe6, 0xb0, 0x64, 0xbe, 0x39, 0x5c, 0xec, 0xbd, 0x73, 0x72, 0x1e, + 0x1e, 0xde, 0xd1, 0x3b, 0x4d, 0x4f, 0x75, 0xf4, 0x06, 0x0c, 0x7b, 0xb4, 0x44, 0x5a, 0xc3, 0xd8, + 0xfd, 0x77, 0x64, 0xc2, 0x4b, 0xb1, 0xf2, 0x08, 0x0b, 0x0a, 0xf6, 0x2f, 0x59, 0x50, 0x3a, 0x85, + 0x91, 0xc0, 0xe6, 0x48, 0x3c, 0x97, 0x4b, 0x5a, 0xc4, 0x70, 0x9b, 0xc7, 0xce, 0xbd, 0xe5, 0xfb, + 0x31, 0xf1, 0x23, 0x26, 0x2a, 0x66, 0x0e, 0xcc, 0x77, 0xc1, 0x99, 0x5b, 0x81, 0xd3, 0x5a, 0x74, + 0x3c, 0xc7, 0x6f, 0x92, 0xb0, 0xe6, 0x6f, 0xf5, 0x35, 0xcb, 0xd2, 0x8d, 0xa8, 0x0a, 0xfd, 0x8c, + 0xa8, 0xec, 0x6d, 0x40, 0x7a, 0x03, 0xc2, 0x70, 0x15, 0xc3, 0x88, 0xcb, 0x9b, 0x12, 0xc3, 0xff, + 0x54, 0x36, 0x77, 0xd7, 0xd5, 0x33, 0xcd, 0x24, 0x93, 0x17, 0x60, 0x49, 0xc8, 0x7e, 0x05, 0x32, + 0x9d, 0xd5, 0xfa, 0xab, 0x0d, 0xec, 0xcf, 0xc3, 0x0c, 0xab, 0x79, 0x4c, 0x91, 0xd6, 0x4e, 0x69, + 0x25, 0x33, 0x62, 0x64, 0xd9, 0x5f, 0xb1, 0x60, 0x6a, 0x2d, 0x15, 0xb0, 0xe3, 0x32, 0x7b, 0x00, + 0xcd, 0x50, 0x86, 0x37, 0x58, 0x29, 0x16, 0xd0, 0x13, 0xd7, 0x41, 0xfd, 0x99, 0x05, 0x89, 0xff, + 0xe8, 0x29, 0x30, 0x5e, 0x4b, 0x06, 0xe3, 0x95, 0xa9, 0x1b, 0x51, 0xdd, 0xc9, 0xe3, 0xbb, 0xd0, + 0x4d, 0x15, 0x2c, 0xa1, 0x87, 0x5a, 0x24, 0x21, 0xc3, 0x5d, 0xeb, 0x27, 0xcd, 0x88, 0x0a, 0x32, + 0x7c, 0x02, 0xb3, 0x9d, 0x52, 0xb8, 0x1f, 0x11, 0xdb, 0x29, 0xd5, 0x9f, 0x9c, 0x1d, 0x5a, 0xd7, + 0xba, 0xcc, 0x4e, 0xae, 0x6f, 0x67, 0xb6, 0xf0, 0x8e, 0xe7, 0xbe, 0x4f, 0x54, 0xc4, 0x97, 0x8a, + 0xb0, 0x6d, 0x17, 0xa5, 0x47, 0x07, 0x95, 0x09, 0xf5, 0x8f, 0x87, 0x05, 0x4b, 0xaa, 0xd8, 0x37, + 0x60, 0x2a, 0x35, 0x60, 0xe8, 0x25, 0x18, 0x6a, 0x6f, 0x3b, 0x11, 0x49, 0xd9, 0x8b, 0x0e, 0xd5, + 0x69, 0xe1, 0xd1, 0x41, 0x65, 0x52, 0x55, 0x60, 0x25, 0x98, 0x63, 0xdb, 0xff, 0xd3, 0x82, 0xd2, + 0x5a, 0xd0, 0x3a, 0x8d, 0xc5, 0xf4, 0xba, 0xb1, 0x98, 0x1e, 0xcf, 0x0b, 0xaa, 0x98, 0xbb, 0x8e, + 0x56, 0x52, 0xeb, 0xe8, 0x62, 0x2e, 0x85, 0xde, 0x4b, 0x68, 0x17, 0xc6, 0x58, 0xa8, 0x46, 0x61, + 0xbf, 0xfa, 0x82, 0x21, 0x03, 0x54, 0x52, 0x32, 0xc0, 0x94, 0x86, 0xaa, 0x49, 0x02, 0x4f, 0xc3, + 0x88, 0xb0, 0xa1, 0x4c, 0x5b, 0xfd, 0x0b, 0x5c, 0x2c, 0xe1, 0xf6, 0x8f, 0x15, 0xc1, 0x08, 0x0d, + 0x89, 0x7e, 0xc5, 0x82, 0xf9, 0x90, 0xbb, 0x51, 0xb6, 0xaa, 0x9d, 0xd0, 0xf5, 0xb7, 0x1a, 0xcd, + 0x6d, 0xd2, 0xea, 0x78, 0xae, 0xbf, 0x55, 0xdb, 0xf2, 0x03, 0x55, 0xbc, 0x7c, 0x9f, 0x34, 0x3b, + 0xec, 0x21, 0xa4, 0x4f, 0x1c, 0x4a, 0x65, 0xa3, 0x74, 0xed, 0xf0, 0xa0, 0x32, 0x8f, 0x8f, 0x45, + 0x1b, 0x1f, 0xb3, 0x2f, 0xe8, 0x1b, 0x16, 0x5c, 0xe5, 0x11, 0x13, 0x07, 0xef, 0x7f, 0x0f, 0x89, + 0xa9, 0x2e, 0x49, 0x25, 0x44, 0xd6, 0x49, 0xb8, 0xbb, 0xf8, 0xb2, 0x18, 0xd0, 0xab, 0xf5, 0xe3, + 0xb5, 0x85, 0x8f, 0xdb, 0x39, 0xfb, 0xdf, 0x14, 0x61, 0x42, 0x78, 0xf0, 0x8b, 0xd0, 0x30, 0x2f, + 0x19, 0x4b, 0xe2, 0x89, 0xd4, 0x92, 0x98, 0x31, 0x90, 0x4f, 0x26, 0x2a, 0x4c, 0x04, 0x33, 0x9e, + 0x13, 0xc5, 0x37, 0x88, 0x13, 0xc6, 0x1b, 0xc4, 0xe1, 0xb6, 0x3b, 0xc5, 0x63, 0xdb, 0x19, 0x29, + 0x15, 0xcd, 0xad, 0x34, 0x31, 0xdc, 0x4d, 0x1f, 0xed, 0x01, 0x62, 0x06, 0x48, 0xa1, 0xe3, 0x47, + 0xfc, 0x5b, 0x5c, 0xf1, 0x66, 0x70, 0xbc, 0x56, 0xe7, 0x44, 0xab, 0xe8, 0x56, 0x17, 0x35, 0x9c, + 0xd1, 0x82, 0x66, 0x58, 0x36, 0x34, 0xa8, 0x61, 0xd9, 0x70, 0x1f, 0xd7, 0x1a, 0x1f, 0xa6, 0xbb, + 0x82, 0x30, 0xbc, 0x0d, 0x65, 0x65, 0x00, 0x28, 0x0e, 0x9d, 0xde, 0xb1, 0x4c, 0xd2, 0x14, 0xb8, + 0x1a, 0x25, 0x31, 0x3e, 0x4d, 0xc8, 0xd9, 0xff, 0xac, 0x60, 0x34, 0xc8, 0x27, 0x71, 0x0d, 0x46, + 0x9d, 0x28, 0x72, 0xb7, 0x7c, 0xd2, 0x12, 0x3b, 0xf6, 0xe3, 0x79, 0x3b, 0xd6, 0x68, 0x86, 0x19, + 0x61, 0x2e, 0x88, 0x9a, 0x58, 0xd1, 0x40, 0x37, 0xb8, 0x85, 0xd4, 0x9e, 0xe4, 0xf9, 0x07, 0xa3, + 0x06, 0xd2, 0x86, 0x6a, 0x8f, 0x60, 0x51, 0x1f, 0x7d, 0x81, 0x9b, 0xb0, 0xdd, 0xf4, 0x83, 0x7b, + 0xfe, 0xf5, 0x20, 0x90, 0x6e, 0x77, 0x83, 0x11, 0x9c, 0x91, 0x86, 0x6b, 0xaa, 0x3a, 0x36, 0xa9, + 0x0d, 0x16, 0xa8, 0xe8, 0xbb, 0xe1, 0x0c, 0x25, 0x6d, 0x3a, 0xcf, 0x44, 0x88, 0xc0, 0x94, 0x08, + 0x0f, 0x21, 0xcb, 0xc4, 0xd8, 0x65, 0xb2, 0xf3, 0x66, 0xed, 0x44, 0xe9, 0x77, 0xd3, 0x24, 0x81, + 0xd3, 0x34, 0xed, 0x9f, 0xb4, 0x80, 0x99, 0xfd, 0x9f, 0x02, 0xcb, 0xf0, 0x59, 0x93, 0x65, 0x98, + 0xcd, 0x1b, 0xe4, 0x1c, 0x6e, 0xe1, 0x45, 0xbe, 0xb2, 0xea, 0x61, 0x70, 0x7f, 0x5f, 0x98, 0x0f, + 0xf4, 0xe7, 0x64, 0xed, 0xff, 0x6b, 0xf1, 0x43, 0x4c, 0x79, 0xe2, 0xa3, 0xef, 0x81, 0xd1, 0xa6, + 0xd3, 0x76, 0x9a, 0x3c, 0x8e, 0x71, 0xae, 0x56, 0xc7, 0xa8, 0x34, 0xbf, 0x24, 0x6a, 0x70, 0x2d, + 0x85, 0x0c, 0x33, 0x32, 0x2a, 0x8b, 0xfb, 0x6a, 0x26, 0x54, 0x93, 0x73, 0x3b, 0x30, 0x61, 0x10, + 0x7b, 0xa8, 0x22, 0xed, 0xf7, 0xf0, 0x2b, 0x56, 0x85, 0xc5, 0xd9, 0x85, 0x19, 0x5f, 0xfb, 0x4f, + 0x2f, 0x14, 0x29, 0xa6, 0x7c, 0xbc, 0xdf, 0x25, 0xca, 0x6e, 0x1f, 0xcd, 0xad, 0x21, 0x45, 0x06, + 0x77, 0x53, 0xb6, 0x7f, 0xdc, 0x82, 0x47, 0x75, 0x44, 0x2d, 0x48, 0x42, 0x3f, 0x3d, 0x71, 0x15, + 0x46, 0x83, 0x36, 0x09, 0x9d, 0x38, 0x08, 0xc5, 0xad, 0x71, 0x45, 0x0e, 0xfa, 0x6d, 0x51, 0x7e, + 0x24, 0x02, 0x4a, 0x4a, 0xea, 0xb2, 0x1c, 0xab, 0x9a, 0x54, 0x8e, 0x61, 0x83, 0x11, 0x89, 0x00, + 0x16, 0xec, 0x0c, 0x60, 0x4f, 0xa6, 0x11, 0x16, 0x10, 0xfb, 0x0f, 0x2c, 0xbe, 0xb0, 0xf4, 0xae, + 0xa3, 0xf7, 0x60, 0x7a, 0xd7, 0x89, 0x9b, 0xdb, 0xcb, 0xf7, 0xdb, 0x21, 0x57, 0x8f, 0xcb, 0x71, + 0x7a, 0xa6, 0xdf, 0x38, 0x69, 0x1f, 0x99, 0x58, 0xe5, 0xad, 0xa6, 0x88, 0xe1, 0x2e, 0xf2, 0x68, + 0x03, 0xc6, 0x58, 0x19, 0x33, 0xff, 0x8e, 0x7a, 0xb1, 0x06, 0x79, 0xad, 0xa9, 0x57, 0xe7, 0xd5, + 0x84, 0x0e, 0xd6, 0x89, 0xda, 0x5f, 0x2e, 0xf2, 0xdd, 0xce, 0xb8, 0xed, 0xa7, 0x61, 0xa4, 0x1d, + 0xb4, 0x96, 0x6a, 0x55, 0x2c, 0x66, 0x41, 0x5d, 0x23, 0x75, 0x5e, 0x8c, 0x25, 0x1c, 0xbd, 0x0a, + 0x40, 0xee, 0xc7, 0x24, 0xf4, 0x1d, 0x4f, 0x59, 0xc9, 0x28, 0xbb, 0xd0, 0x6a, 0xb0, 0x16, 0xc4, + 0x77, 0x22, 0xf2, 0x5d, 0xcb, 0x0a, 0x05, 0x6b, 0xe8, 0xe8, 0x1a, 0x40, 0x3b, 0x0c, 0xf6, 0xdc, + 0x16, 0xf3, 0x27, 0x2c, 0x9a, 0x36, 0x24, 0x75, 0x05, 0xc1, 0x1a, 0x16, 0x7a, 0x15, 0x26, 0x3a, + 0x7e, 0xc4, 0x39, 0x14, 0x67, 0x43, 0x84, 0x63, 0x1c, 0x4d, 0xac, 0x1b, 0xee, 0xe8, 0x40, 0x6c, + 0xe2, 0xa2, 0x05, 0x18, 0x8e, 0x1d, 0x66, 0x13, 0x31, 0x94, 0x6f, 0xcc, 0xb9, 0x4e, 0x31, 0xf4, + 0x28, 0xba, 0xb4, 0x02, 0x16, 0x15, 0xd1, 0xdb, 0xd2, 0x39, 0x83, 0x9f, 0xf5, 0xc2, 0x8a, 0x7a, + 0xb0, 0x7b, 0x41, 0x73, 0xcd, 0x10, 0xd6, 0xd9, 0x06, 0x2d, 0xfb, 0x1b, 0x65, 0x80, 0x84, 0x1d, + 0x47, 0xef, 0x77, 0x9d, 0x47, 0xcf, 0xf6, 0x66, 0xe0, 0x4f, 0xee, 0x30, 0x42, 0xdf, 0x6f, 0xc1, + 0x98, 0xe3, 0x79, 0x41, 0xd3, 0x89, 0xd9, 0x28, 0x17, 0x7a, 0x9f, 0x87, 0xa2, 0xfd, 0x85, 0xa4, + 0x06, 0xef, 0xc2, 0x0b, 0x72, 0xe1, 0x69, 0x90, 0xbe, 0xbd, 0xd0, 0x1b, 0x46, 0x9f, 0x92, 0x52, + 0x1a, 0x5f, 0x1e, 0x73, 0x69, 0x29, 0xad, 0xcc, 0x8e, 0x7e, 0x4d, 0x40, 0x43, 0x77, 0x8c, 0x48, + 0x7b, 0xa5, 0xfc, 0xa0, 0x13, 0x06, 0x57, 0xda, 0x2f, 0xc8, 0x1e, 0xaa, 0xeb, 0xde, 0x64, 0x43, + 0xf9, 0x91, 0x59, 0x34, 0xf1, 0xa7, 0x8f, 0x27, 0xd9, 0xbb, 0x30, 0xd5, 0x32, 0xef, 0x76, 0xb1, + 0x9a, 0x9e, 0xca, 0xa3, 0x9b, 0x62, 0x05, 0x92, 0xdb, 0x3c, 0x05, 0xc0, 0x69, 0xc2, 0xa8, 0xce, + 0xfd, 0xfa, 0x6a, 0xfe, 0x66, 0x20, 0xac, 0xf1, 0xed, 0xdc, 0xb9, 0xdc, 0x8f, 0x62, 0xb2, 0x4b, + 0x31, 0x93, 0x4b, 0x7b, 0x4d, 0xd4, 0xc5, 0x8a, 0x0a, 0x7a, 0x03, 0x86, 0x99, 0x63, 0x70, 0x34, + 0x3b, 0x9a, 0xaf, 0x4c, 0x34, 0x63, 0x5a, 0x24, 0x9b, 0x8a, 0xfd, 0x8d, 0xb0, 0xa0, 0x80, 0x6e, + 0xc8, 0xc0, 0x37, 0x51, 0xcd, 0xbf, 0x13, 0x11, 0x16, 0xf8, 0xa6, 0xbc, 0xf8, 0xf1, 0x24, 0xa6, + 0x0d, 0x2f, 0xcf, 0x8c, 0x97, 0x6f, 0xd4, 0xa4, 0xcc, 0x91, 0xf8, 0x2f, 0xc3, 0xf0, 0xcf, 0x42, + 0x7e, 0xf7, 0xcc, 0x50, 0xfd, 0xc9, 0x70, 0xde, 0x35, 0x49, 0xe0, 0x34, 0x4d, 0xca, 0x68, 0xf2, + 0x9d, 0x2b, 0xec, 0xf9, 0xfb, 0xed, 0x7f, 0x2e, 0x5f, 0xb3, 0x4b, 0x86, 0x97, 0x60, 0x51, 0xff, + 0x54, 0x6f, 0xfd, 0x39, 0x1f, 0xa6, 0xd3, 0x5b, 0xf4, 0xa1, 0x72, 0x19, 0xbf, 0x5f, 0x82, 0x49, + 0x73, 0x49, 0xa1, 0xab, 0x50, 0x16, 0x44, 0x54, 0x14, 0x56, 0xb5, 0x4b, 0x56, 0x25, 0x00, 0x27, + 0x38, 0x2c, 0xf8, 0x2e, 0xab, 0xae, 0xd9, 0x61, 0x26, 0xc1, 0x77, 0x15, 0x04, 0x6b, 0x58, 0x54, + 0x5e, 0xda, 0x08, 0x82, 0x58, 0x5d, 0x2a, 0x6a, 0xdd, 0x2d, 0xb2, 0x52, 0x2c, 0xa0, 0xf4, 0x32, + 0xd9, 0x21, 0xa1, 0x4f, 0x3c, 0x33, 0xb8, 0x9b, 0xba, 0x4c, 0x6e, 0xea, 0x40, 0x6c, 0xe2, 0xd2, + 0x5b, 0x32, 0x88, 0xd8, 0x42, 0x16, 0x52, 0x59, 0x62, 0xd7, 0xda, 0xe0, 0x2e, 0xf6, 0x12, 0x8e, + 0x3e, 0x0f, 0x8f, 0x2a, 0x8f, 0x78, 0xcc, 0x15, 0xd5, 0xb2, 0xc5, 0x61, 0x43, 0x89, 0xf2, 0xe8, + 0x52, 0x36, 0x1a, 0xce, 0xab, 0x8f, 0x5e, 0x87, 0x49, 0xc1, 0xb9, 0x4b, 0x8a, 0x23, 0xa6, 0xed, + 0xc4, 0x4d, 0x03, 0x8a, 0x53, 0xd8, 0x32, 0x3c, 0x1d, 0x63, 0x9e, 0x25, 0x85, 0xd1, 0xee, 0xf0, + 0x74, 0x3a, 0x1c, 0x77, 0xd5, 0x40, 0x0b, 0x30, 0xc5, 0x59, 0x2b, 0xd7, 0xdf, 0xe2, 0x73, 0x22, + 0xdc, 0x6d, 0xd4, 0x96, 0xba, 0x6d, 0x82, 0x71, 0x1a, 0x1f, 0xbd, 0x02, 0xe3, 0x4e, 0xd8, 0xdc, + 0x76, 0x63, 0xd2, 0x8c, 0x3b, 0x21, 0xf7, 0xc3, 0xd1, 0x8c, 0x4f, 0x16, 0x34, 0x18, 0x36, 0x30, + 0xed, 0xf7, 0xe1, 0x4c, 0x86, 0xa7, 0x1e, 0x5d, 0x38, 0x4e, 0xdb, 0x95, 0xdf, 0x94, 0xb2, 0x50, + 0x5d, 0xa8, 0xd7, 0xe4, 0xd7, 0x68, 0x58, 0x74, 0x75, 0x32, 0x8f, 0x3e, 0x2d, 0xeb, 0x86, 0x5a, + 0x9d, 0x2b, 0x12, 0x80, 0x13, 0x1c, 0xfb, 0x7f, 0x15, 0x60, 0x2a, 0x43, 0xf9, 0xce, 0x32, 0x3f, + 0xa4, 0x64, 0x8f, 0x24, 0xd1, 0x83, 0x19, 0xed, 0xb0, 0x70, 0x8c, 0x68, 0x87, 0xc5, 0x7e, 0xd1, + 0x0e, 0x4b, 0x1f, 0x24, 0xda, 0xa1, 0x39, 0x62, 0x43, 0x03, 0x8d, 0x58, 0x46, 0x84, 0xc4, 0xe1, + 0x63, 0x46, 0x48, 0x34, 0x06, 0x7d, 0x64, 0x80, 0x41, 0xff, 0x5a, 0x01, 0xa6, 0xd3, 0x46, 0x72, + 0xa7, 0xa0, 0x8e, 0x7d, 0xc3, 0x50, 0xc7, 0x66, 0xe7, 0x51, 0x49, 0x9b, 0xee, 0xe5, 0xa9, 0x66, + 0x71, 0x4a, 0x35, 0xfb, 0xc9, 0x81, 0xa8, 0xf5, 0x56, 0xd3, 0xfe, 0x83, 0x02, 0x9c, 0x4b, 0x57, + 0x59, 0xf2, 0x1c, 0x77, 0xf7, 0x14, 0xc6, 0xe6, 0xb6, 0x31, 0x36, 0xcf, 0x0d, 0xf2, 0x35, 0xac, + 0x6b, 0xb9, 0x03, 0xf4, 0x56, 0x6a, 0x80, 0xae, 0x0e, 0x4e, 0xb2, 0xf7, 0x28, 0x7d, 0xb3, 0x08, + 0x17, 0x33, 0xeb, 0x25, 0xda, 0xcc, 0x15, 0x43, 0x9b, 0x79, 0x2d, 0xa5, 0xcd, 0xb4, 0x7b, 0xd7, + 0x3e, 0x19, 0xf5, 0xa6, 0x70, 0xa1, 0x64, 0x11, 0xf1, 0x1e, 0x50, 0xb5, 0x69, 0xb8, 0x50, 0x2a, + 0x42, 0xd8, 0xa4, 0xfb, 0x17, 0x49, 0xa5, 0xf9, 0xef, 0x2c, 0x38, 0x9f, 0x39, 0x37, 0xa7, 0xa0, + 0xc2, 0x5a, 0x33, 0x55, 0x58, 0x4f, 0x0f, 0xbc, 0x5a, 0x73, 0x74, 0x5a, 0xbf, 0x51, 0xca, 0xf9, + 0x16, 0x26, 0xa0, 0xdf, 0x86, 0x31, 0xa7, 0xd9, 0x24, 0x51, 0xb4, 0x1a, 0xb4, 0x54, 0x84, 0xb8, + 0xe7, 0x98, 0x9c, 0x95, 0x14, 0x1f, 0x1d, 0x54, 0xe6, 0xd2, 0x24, 0x12, 0x30, 0xd6, 0x29, 0x98, + 0x41, 0x2d, 0x0b, 0x27, 0x1a, 0xd4, 0xf2, 0x1a, 0xc0, 0x9e, 0xe2, 0xd6, 0xd3, 0x42, 0xbe, 0xc6, + 0xc7, 0x6b, 0x58, 0xe8, 0x0b, 0x30, 0x1a, 0x89, 0x6b, 0x5c, 0x2c, 0xc5, 0x17, 0x06, 0x9c, 0x2b, + 0x67, 0x83, 0x78, 0xa6, 0xaf, 0xbe, 0xd2, 0x87, 0x28, 0x92, 0xe8, 0x3b, 0x60, 0x3a, 0xe2, 0xa1, + 0x60, 0x96, 0x3c, 0x27, 0x62, 0x7e, 0x10, 0x62, 0x15, 0x32, 0x07, 0xfc, 0x46, 0x0a, 0x86, 0xbb, + 0xb0, 0xd1, 0x8a, 0xfc, 0x28, 0x16, 0xb7, 0x86, 0x2f, 0xcc, 0xcb, 0xc9, 0x07, 0x89, 0xbc, 0x53, + 0x67, 0xd3, 0xc3, 0xcf, 0x06, 0x5e, 0xab, 0x89, 0xbe, 0x00, 0x40, 0x97, 0x8f, 0xd0, 0x25, 0x8c, + 0xe4, 0x1f, 0x9e, 0xf4, 0x54, 0x69, 0x65, 0x5a, 0x7e, 0x32, 0xe7, 0xc5, 0xaa, 0x22, 0x82, 0x35, + 0x82, 0xf6, 0xd7, 0x4a, 0xf0, 0x58, 0x8f, 0x33, 0x12, 0x2d, 0x98, 0x4f, 0xa0, 0xcf, 0xa4, 0x85, + 0xeb, 0xb9, 0xcc, 0xca, 0x86, 0xb4, 0x9d, 0x5a, 0x8a, 0x85, 0x0f, 0xbc, 0x14, 0x7f, 0xc0, 0xd2, + 0xd4, 0x1e, 0xdc, 0x98, 0xef, 0xb3, 0xc7, 0x3c, 0xfb, 0x4f, 0x50, 0x0f, 0xb2, 0x99, 0xa1, 0x4c, + 0xb8, 0x36, 0x70, 0x77, 0x06, 0xd6, 0x2e, 0x9c, 0xae, 0xf2, 0xf7, 0xcb, 0x16, 0x3c, 0x91, 0xd9, + 0x5f, 0xc3, 0x64, 0xe3, 0x2a, 0x94, 0x9b, 0xb4, 0x50, 0xf3, 0x55, 0x4b, 0x9c, 0x78, 0x25, 0x00, + 0x27, 0x38, 0x86, 0x65, 0x46, 0xa1, 0xaf, 0x65, 0xc6, 0xbf, 0xb6, 0xa0, 0x6b, 0x7f, 0x9c, 0xc2, + 0x41, 0x5d, 0x33, 0x0f, 0xea, 0x8f, 0x0f, 0x32, 0x97, 0x39, 0x67, 0xf4, 0x1f, 0x4d, 0xc1, 0x23, + 0x39, 0xbe, 0x1a, 0x7b, 0x30, 0xb3, 0xd5, 0x24, 0xa6, 0x17, 0xa0, 0xf8, 0x98, 0x4c, 0x87, 0xc9, + 0x9e, 0x2e, 0x83, 0x2c, 0x1f, 0xd1, 0x4c, 0x17, 0x0a, 0xee, 0x6e, 0x02, 0x7d, 0xd9, 0x82, 0xb3, + 0xce, 0xbd, 0xa8, 0x2b, 0xeb, 0xa4, 0x58, 0x33, 0x2f, 0x66, 0x2a, 0x41, 0xfa, 0x64, 0xa9, 0xe4, + 0x09, 0x9a, 0xb2, 0xb0, 0x70, 0x66, 0x5b, 0x08, 0x8b, 0x98, 0xa1, 0x94, 0x9d, 0xef, 0xe1, 0xa7, + 0x9a, 0xe5, 0x54, 0xc3, 0x8f, 0x6c, 0x09, 0xc1, 0x8a, 0x0e, 0xfa, 0x12, 0x94, 0xb7, 0xa4, 0xa7, + 0x5b, 0xc6, 0x95, 0x90, 0x0c, 0x64, 0x6f, 0xff, 0x3f, 0xfe, 0x40, 0xa9, 0x90, 0x70, 0x42, 0x14, + 0xbd, 0x0e, 0x45, 0x7f, 0x33, 0xea, 0x95, 0xe3, 0x28, 0x65, 0xd3, 0xc4, 0xbd, 0xc1, 0xd7, 0x56, + 0x1a, 0x98, 0x56, 0x44, 0x37, 0xa0, 0x18, 0x6e, 0xb4, 0x84, 0x06, 0x2f, 0xf3, 0x0c, 0xc7, 0x8b, + 0xd5, 0x9c, 0x5e, 0x31, 0x4a, 0x78, 0xb1, 0x8a, 0x29, 0x09, 0x54, 0x87, 0x21, 0xe6, 0xe0, 0x20, + 0xee, 0x83, 0x4c, 0xce, 0xb7, 0x87, 0xa3, 0x10, 0x77, 0x19, 0x67, 0x08, 0x98, 0x13, 0x42, 0xeb, + 0x30, 0xdc, 0x64, 0xf9, 0x70, 0x44, 0xc0, 0xea, 0x4f, 0x65, 0xea, 0xea, 0x7a, 0x24, 0x0a, 0x12, + 0xaa, 0x2b, 0x86, 0x81, 0x05, 0x2d, 0x46, 0x95, 0xb4, 0xb7, 0x37, 0x23, 0x26, 0xeb, 0xe7, 0x51, + 0xed, 0x91, 0xff, 0x4a, 0x50, 0x65, 0x18, 0x58, 0xd0, 0x42, 0x9f, 0x81, 0xc2, 0x66, 0x53, 0xf8, + 0x3f, 0x64, 0x2a, 0xed, 0x4c, 0x87, 0xfe, 0xc5, 0xe1, 0xc3, 0x83, 0x4a, 0x61, 0x65, 0x09, 0x17, + 0x36, 0x9b, 0x68, 0x0d, 0x46, 0x36, 0xb9, 0x0b, 0xb0, 0xd0, 0xcb, 0x3d, 0x95, 0xed, 0x9d, 0xdc, + 0xe5, 0x25, 0xcc, 0xed, 0xf6, 0x05, 0x00, 0x4b, 0x22, 0x2c, 0x04, 0xa7, 0x72, 0x65, 0x16, 0xb1, + 0xa8, 0xe7, 0x8f, 0xe7, 0x7e, 0xce, 0xef, 0xe7, 0xc4, 0x21, 0x1a, 0x6b, 0x14, 0xe9, 0xaa, 0x76, + 0x64, 0xe6, 0x43, 0x11, 0xab, 0x23, 0x73, 0x55, 0xf7, 0x49, 0x0a, 0xc9, 0x57, 0xb5, 0x42, 0xc2, + 0x09, 0x51, 0xb4, 0x03, 0x13, 0x7b, 0x51, 0x7b, 0x9b, 0xc8, 0x2d, 0xcd, 0x42, 0x77, 0xe4, 0x5c, + 0x61, 0x77, 0x05, 0xa2, 0x1b, 0xc6, 0x1d, 0xc7, 0xeb, 0x3a, 0x85, 0xd8, 0xab, 0xf6, 0x5d, 0x9d, + 0x18, 0x36, 0x69, 0xd3, 0xe1, 0x7f, 0xaf, 0x13, 0x6c, 0xec, 0xc7, 0x44, 0x04, 0xaf, 0xce, 0x1c, + 0xfe, 0x37, 0x39, 0x4a, 0xf7, 0xf0, 0x0b, 0x00, 0x96, 0x44, 0xd0, 0x5d, 0x31, 0x3c, 0xec, 0xf4, + 0x9c, 0xce, 0x0f, 0xa6, 0x94, 0x99, 0x7a, 0x54, 0x1b, 0x14, 0x76, 0x5a, 0x26, 0xa4, 0xd8, 0x29, + 0xd9, 0xde, 0x0e, 0xe2, 0xc0, 0x4f, 0x9d, 0xd0, 0x33, 0xf9, 0xa7, 0x64, 0x3d, 0x03, 0xbf, 0xfb, + 0x94, 0xcc, 0xc2, 0xc2, 0x99, 0x6d, 0xa1, 0x16, 0x4c, 0xb6, 0x83, 0x30, 0xbe, 0x17, 0x84, 0x72, + 0x7d, 0xa1, 0x1e, 0x7a, 0x05, 0x03, 0x53, 0xb4, 0xc8, 0x82, 0xa9, 0x9b, 0x10, 0x9c, 0xa2, 0x89, + 0x3e, 0x07, 0x23, 0x51, 0xd3, 0xf1, 0x48, 0xed, 0xf6, 0xec, 0x99, 0xfc, 0xeb, 0xa7, 0xc1, 0x51, + 0x72, 0x56, 0x17, 0x9b, 0x1c, 0x81, 0x82, 0x25, 0x39, 0xb4, 0x02, 0x43, 0x2c, 0x23, 0x02, 0x8b, + 0xbb, 0x9d, 0x13, 0x13, 0xaa, 0xcb, 0xc2, 0x94, 0x9f, 0x4d, 0xac, 0x18, 0xf3, 0xea, 0x74, 0x0f, + 0x08, 0xf6, 0x3a, 0x88, 0x66, 0xcf, 0xe5, 0xef, 0x01, 0xc1, 0x95, 0xdf, 0x6e, 0xf4, 0xda, 0x03, + 0x0a, 0x09, 0x27, 0x44, 0xe9, 0xc9, 0x4c, 0x4f, 0xd3, 0x47, 0x7a, 0x18, 0xb4, 0xe4, 0x9e, 0xa5, + 0xec, 0x64, 0xa6, 0x27, 0x29, 0x25, 0x61, 0xff, 0xee, 0x48, 0x37, 0xcf, 0xc2, 0x04, 0xb2, 0xbf, + 0x6a, 0x75, 0xbd, 0xd5, 0x7d, 0x7a, 0x50, 0xfd, 0xd0, 0x09, 0x72, 0xab, 0x5f, 0xb6, 0xe0, 0x91, + 0x76, 0xe6, 0x87, 0x08, 0x06, 0x60, 0x30, 0x35, 0x13, 0xff, 0x74, 0x15, 0x1b, 0x3f, 0x1b, 0x8e, + 0x73, 0x5a, 0x4a, 0x4b, 0x04, 0xc5, 0x0f, 0x2c, 0x11, 0xac, 0xc2, 0x28, 0x63, 0x32, 0xfb, 0xe4, + 0x87, 0x4b, 0x0b, 0x46, 0x8c, 0x95, 0x58, 0x12, 0x15, 0xb1, 0x22, 0x81, 0x7e, 0xd0, 0x82, 0x0b, + 0xe9, 0xae, 0x63, 0xc2, 0xc0, 0x22, 0x92, 0x3c, 0x97, 0x05, 0x57, 0xc4, 0xf7, 0x5f, 0xa8, 0xf7, + 0x42, 0x3e, 0xea, 0x87, 0x80, 0x7b, 0x37, 0x86, 0xaa, 0x19, 0xc2, 0xe8, 0xb0, 0xa9, 0x80, 0x1f, + 0x40, 0x20, 0x7d, 0x11, 0xc6, 0x77, 0x83, 0x8e, 0x1f, 0x0b, 0xfb, 0x17, 0xe1, 0xb1, 0xc8, 0x1e, + 0x9c, 0x57, 0xb5, 0x72, 0x6c, 0x60, 0xa5, 0xc4, 0xd8, 0xd1, 0x07, 0x16, 0x63, 0xdf, 0x49, 0x65, + 0x01, 0x2f, 0xe7, 0x47, 0x2c, 0x14, 0x12, 0xff, 0x31, 0x72, 0x81, 0x9f, 0xae, 0x6c, 0xf4, 0xd3, + 0x56, 0x06, 0x53, 0xcf, 0xa5, 0xe5, 0xd7, 0x4c, 0x69, 0xf9, 0x72, 0x5a, 0x5a, 0xee, 0x52, 0xbe, + 0x1a, 0x82, 0xf2, 0xe0, 0x61, 0xaf, 0x07, 0x8d, 0x23, 0x67, 0x7b, 0x70, 0xa9, 0xdf, 0xb5, 0xc4, + 0x0c, 0xa1, 0x5a, 0xea, 0xa9, 0x2d, 0x31, 0x84, 0x6a, 0xd5, 0xaa, 0x98, 0x41, 0x06, 0x0d, 0x34, + 0x62, 0xff, 0x0f, 0x0b, 0x8a, 0xf5, 0xa0, 0x75, 0x0a, 0xca, 0xe4, 0xcf, 0x1a, 0xca, 0xe4, 0xc7, + 0x72, 0xb2, 0xb3, 0xe7, 0xaa, 0x8e, 0x97, 0x53, 0xaa, 0xe3, 0x0b, 0x79, 0x04, 0x7a, 0x2b, 0x8a, + 0x7f, 0xa2, 0x08, 0x7a, 0x2e, 0x79, 0xf4, 0x1b, 0x0f, 0x62, 0x85, 0x5c, 0xec, 0x95, 0x5e, 0x5e, + 0x50, 0x66, 0xf6, 0x53, 0xd2, 0x09, 0xef, 0xcf, 0x99, 0x31, 0xf2, 0x5b, 0xc4, 0xdd, 0xda, 0x8e, + 0x49, 0x2b, 0xfd, 0x39, 0xa7, 0x67, 0x8c, 0xfc, 0xdf, 0x2c, 0x98, 0x4a, 0xb5, 0x8e, 0x3c, 0x98, + 0xf0, 0x74, 0x4d, 0xa0, 0x58, 0xa7, 0x0f, 0xa4, 0x44, 0x14, 0xc6, 0x9c, 0x5a, 0x11, 0x36, 0x89, + 0xa3, 0x79, 0x00, 0xf5, 0x52, 0x27, 0x35, 0x60, 0x8c, 0xeb, 0x57, 0x4f, 0x79, 0x11, 0xd6, 0x30, + 0xd0, 0x4b, 0x30, 0x16, 0x07, 0xed, 0xc0, 0x0b, 0xb6, 0xf6, 0x6f, 0x12, 0x19, 0xda, 0x46, 0x99, + 0x68, 0xad, 0x27, 0x20, 0xac, 0xe3, 0xd9, 0x3f, 0x55, 0xe4, 0x1f, 0xea, 0xc7, 0xee, 0xb7, 0xd6, + 0xe4, 0x47, 0x7b, 0x4d, 0x7e, 0xd3, 0x82, 0x69, 0xda, 0x3a, 0x33, 0x17, 0x91, 0x97, 0xad, 0x4a, + 0xbf, 0x63, 0xf5, 0x48, 0xbf, 0x73, 0x99, 0x9e, 0x5d, 0xad, 0xa0, 0x13, 0x0b, 0x0d, 0x9a, 0x76, + 0x38, 0xd1, 0x52, 0x2c, 0xa0, 0x02, 0x8f, 0x84, 0xa1, 0xf0, 0x81, 0xd2, 0xf1, 0x48, 0x18, 0x62, + 0x01, 0x95, 0xd9, 0x79, 0x4a, 0x39, 0xd9, 0x79, 0x58, 0xa0, 0x3e, 0x61, 0x58, 0x20, 0xd8, 0x1e, + 0x2d, 0x50, 0x9f, 0xb4, 0x38, 0x48, 0x70, 0xec, 0x9f, 0x2b, 0xc2, 0x78, 0x3d, 0x68, 0x25, 0x6f, + 0x65, 0x2f, 0x1a, 0x6f, 0x65, 0x97, 0x52, 0x6f, 0x65, 0xd3, 0x3a, 0xee, 0xb7, 0x5e, 0xc6, 0x3e, + 0xac, 0x97, 0xb1, 0x7f, 0x65, 0xb1, 0x59, 0xab, 0xae, 0x35, 0x44, 0x76, 0xe0, 0xe7, 0x61, 0x8c, + 0x1d, 0x48, 0xcc, 0xe9, 0x4e, 0x3e, 0x20, 0xb1, 0xc0, 0xfb, 0x6b, 0x49, 0x31, 0xd6, 0x71, 0xd0, + 0x15, 0x18, 0x8d, 0x88, 0x13, 0x36, 0xb7, 0xd5, 0x19, 0x27, 0x9e, 0x57, 0x78, 0x19, 0x56, 0x50, + 0xf4, 0x66, 0x12, 0x23, 0xae, 0x98, 0x9f, 0xe7, 0x56, 0xef, 0x0f, 0xdf, 0x22, 0xf9, 0x81, 0xe1, + 0xec, 0xb7, 0x00, 0x75, 0xe3, 0x0f, 0x10, 0x1c, 0xa9, 0x62, 0x06, 0x47, 0x2a, 0x77, 0x05, 0x46, + 0xfa, 0x53, 0x0b, 0x26, 0xeb, 0x41, 0x8b, 0x6e, 0xdd, 0xbf, 0x48, 0xfb, 0x54, 0x0f, 0x90, 0x39, + 0xdc, 0x23, 0x40, 0xe6, 0x3f, 0xb4, 0x60, 0xa4, 0x1e, 0xb4, 0x4e, 0x41, 0xef, 0xfe, 0x9a, 0xa9, + 0x77, 0x7f, 0x34, 0x67, 0x49, 0xe4, 0xa8, 0xda, 0x7f, 0xa1, 0x08, 0x13, 0xb4, 0x9f, 0xc1, 0x96, + 0x9c, 0x25, 0x63, 0x44, 0xac, 0x01, 0x46, 0x84, 0xb2, 0xb9, 0x81, 0xe7, 0x05, 0xf7, 0xd2, 0x33, + 0xb6, 0xc2, 0x4a, 0xb1, 0x80, 0xa2, 0x67, 0x61, 0xb4, 0x1d, 0x92, 0x3d, 0x37, 0x10, 0xfc, 0xa3, + 0xf6, 0x8a, 0x51, 0x17, 0xe5, 0x58, 0x61, 0x50, 0xb9, 0x2b, 0x72, 0xfd, 0x26, 0x91, 0x49, 0xb6, + 0x4b, 0x2c, 0x0f, 0x17, 0x8f, 0x7c, 0xad, 0x95, 0x63, 0x03, 0x0b, 0xbd, 0x05, 0x65, 0xf6, 0x9f, + 0x9d, 0x28, 0xc7, 0xcf, 0x1b, 0x24, 0xd2, 0x4d, 0x08, 0x02, 0x38, 0xa1, 0x85, 0xae, 0x01, 0xc4, + 0x32, 0x3a, 0x72, 0x24, 0x62, 0xdc, 0x28, 0x5e, 0x5b, 0xc5, 0x4d, 0x8e, 0xb0, 0x86, 0x85, 0x9e, + 0x81, 0x72, 0xec, 0xb8, 0xde, 0x2d, 0xd7, 0x27, 0x11, 0x53, 0x39, 0x17, 0x65, 0x36, 0x09, 0x51, + 0x88, 0x13, 0x38, 0xe5, 0x75, 0x98, 0x03, 0x38, 0xcf, 0x3a, 0x36, 0xca, 0xb0, 0x19, 0xaf, 0x73, + 0x4b, 0x95, 0x62, 0x0d, 0xc3, 0x7e, 0x05, 0xce, 0xd5, 0x83, 0x56, 0x3d, 0x08, 0xe3, 0x95, 0x20, + 0xbc, 0xe7, 0x84, 0x2d, 0x39, 0x7f, 0x15, 0x99, 0xd8, 0x80, 0x9e, 0x3d, 0x43, 0x7c, 0x67, 0x1a, + 0x29, 0x0b, 0x5e, 0x60, 0xdc, 0xce, 0x31, 0x9d, 0x3a, 0x9a, 0xec, 0xde, 0x55, 0x09, 0x06, 0xaf, + 0x3b, 0x31, 0x41, 0xb7, 0x59, 0x52, 0xb2, 0xe4, 0x0a, 0x12, 0xd5, 0x9f, 0xd6, 0x92, 0x92, 0x25, + 0xc0, 0xcc, 0x3b, 0xcb, 0xac, 0x6f, 0xff, 0x6a, 0x91, 0x9d, 0x46, 0xa9, 0x7c, 0x7b, 0xe8, 0x8b, + 0x30, 0x19, 0x91, 0x5b, 0xae, 0xdf, 0xb9, 0x2f, 0x85, 0xf0, 0x1e, 0x6e, 0x39, 0x8d, 0x65, 0x1d, + 0x93, 0xab, 0xf2, 0xcc, 0x32, 0x9c, 0xa2, 0x46, 0xe7, 0x29, 0xec, 0xf8, 0x0b, 0xd1, 0x9d, 0x88, + 0x84, 0x22, 0xdf, 0x1b, 0x9b, 0x27, 0x2c, 0x0b, 0x71, 0x02, 0xa7, 0xeb, 0x92, 0xfd, 0x59, 0x0b, + 0x7c, 0x1c, 0x04, 0xb1, 0x5c, 0xc9, 0x2c, 0x63, 0x90, 0x56, 0x8e, 0x0d, 0x2c, 0xb4, 0x02, 0x28, + 0xea, 0xb4, 0xdb, 0x1e, 0x7b, 0xd8, 0x77, 0xbc, 0xeb, 0x61, 0xd0, 0x69, 0xf3, 0x57, 0xcf, 0x22, + 0x0f, 0x4c, 0xd8, 0xe8, 0x82, 0xe2, 0x8c, 0x1a, 0xf4, 0xf4, 0xd9, 0x8c, 0xd8, 0x6f, 0xb6, 0xba, + 0x8b, 0x42, 0xbd, 0xde, 0x60, 0x45, 0x58, 0xc2, 0xe8, 0x62, 0x62, 0xcd, 0x73, 0xcc, 0xe1, 0x64, + 0x31, 0x61, 0x55, 0x8a, 0x35, 0x0c, 0xb4, 0x0c, 0x23, 0xd1, 0x7e, 0xd4, 0x8c, 0x45, 0x44, 0xa6, + 0x9c, 0xcc, 0x9d, 0x0d, 0x86, 0xa2, 0x65, 0x93, 0xe0, 0x55, 0xb0, 0xac, 0x6b, 0x7f, 0x0f, 0xbb, + 0x0c, 0x59, 0x76, 0xb0, 0xb8, 0x13, 0x12, 0xb4, 0x0b, 0x13, 0x6d, 0x36, 0xe5, 0x22, 0x76, 0xb5, + 0x98, 0xb7, 0x17, 0x07, 0x94, 0x6a, 0xef, 0xd1, 0x83, 0x46, 0x69, 0x9d, 0x98, 0xb8, 0x50, 0xd7, + 0xc9, 0x61, 0x93, 0xba, 0xfd, 0x35, 0xc4, 0xce, 0xdc, 0x06, 0x17, 0x55, 0x47, 0x84, 0x69, 0xb1, + 0xe0, 0xcb, 0xe7, 0xf2, 0x75, 0x26, 0xc9, 0x17, 0x09, 0xf3, 0x64, 0x2c, 0xeb, 0xa2, 0x37, 0xd9, + 0x2b, 0x35, 0x3f, 0xe8, 0xfa, 0x25, 0x69, 0xe6, 0x58, 0xc6, 0x83, 0xb4, 0xa8, 0x88, 0x35, 0x22, + 0xe8, 0x16, 0x4c, 0x88, 0x64, 0x52, 0x42, 0x29, 0x56, 0x34, 0x94, 0x1e, 0x13, 0x58, 0x07, 0x1e, + 0xa5, 0x0b, 0xb0, 0x59, 0x19, 0x6d, 0xc1, 0x05, 0x2d, 0xb3, 0xe2, 0xf5, 0xd0, 0x61, 0x2f, 0x97, + 0x2e, 0xdb, 0x44, 0xda, 0xb9, 0xf9, 0xc4, 0xe1, 0x41, 0xe5, 0xc2, 0x7a, 0x2f, 0x44, 0xdc, 0x9b, + 0x0e, 0xba, 0x0d, 0xe7, 0xb8, 0x07, 0x5f, 0x95, 0x38, 0x2d, 0xcf, 0xf5, 0xd5, 0xc1, 0xcc, 0xd7, + 0xe1, 0xf9, 0xc3, 0x83, 0xca, 0xb9, 0x85, 0x2c, 0x04, 0x9c, 0x5d, 0x0f, 0xbd, 0x06, 0xe5, 0x96, + 0x1f, 0x89, 0x31, 0x18, 0x36, 0x92, 0x86, 0x96, 0xab, 0x6b, 0x0d, 0xf5, 0xfd, 0xc9, 0x1f, 0x9c, + 0x54, 0x40, 0x5b, 0x5c, 0x31, 0xa6, 0xe4, 0xd0, 0x91, 0xfc, 0x04, 0xf1, 0x62, 0x49, 0x18, 0x3e, + 0x3c, 0x5c, 0x23, 0xac, 0x6c, 0x60, 0x0d, 0xf7, 0x1e, 0x83, 0x30, 0x7a, 0x03, 0x10, 0x65, 0xd4, + 0xdc, 0x26, 0x59, 0x68, 0xb2, 0x10, 0xe2, 0x4c, 0x8f, 0x38, 0x6a, 0xf8, 0x4c, 0xa0, 0x46, 0x17, + 0x06, 0xce, 0xa8, 0x85, 0x6e, 0xd0, 0x83, 0x4c, 0x2f, 0x15, 0xb6, 0xbc, 0x92, 0xb9, 0x9f, 0xad, + 0x92, 0x76, 0x48, 0x9a, 0x4e, 0x4c, 0x5a, 0x26, 0x45, 0x9c, 0xaa, 0x47, 0xef, 0x52, 0x95, 0x4d, + 0x08, 0xcc, 0xb0, 0x19, 0xdd, 0x19, 0x85, 0xa8, 0x5c, 0xbc, 0x1d, 0x44, 0xf1, 0x1a, 0x89, 0xef, + 0x05, 0xe1, 0x8e, 0x88, 0x52, 0x96, 0x04, 0xcc, 0x4c, 0x40, 0x58, 0xc7, 0xa3, 0x7c, 0x30, 0x7b, + 0x26, 0xae, 0x55, 0xd9, 0x0b, 0xdd, 0x68, 0xb2, 0x4f, 0x6e, 0xf0, 0x62, 0x2c, 0xe1, 0x12, 0xb5, + 0x56, 0x5f, 0x62, 0xaf, 0x6d, 0x29, 0xd4, 0x5a, 0x7d, 0x09, 0x4b, 0x38, 0x22, 0xdd, 0x09, 0x59, + 0x27, 0xf3, 0xb5, 0x9a, 0xdd, 0xd7, 0xc1, 0x80, 0x39, 0x59, 0x7d, 0x98, 0x56, 0xa9, 0x60, 0x79, + 0xf8, 0xb6, 0x68, 0x76, 0x8a, 0x2d, 0x92, 0xc1, 0x63, 0xbf, 0x29, 0x3d, 0x71, 0x2d, 0x45, 0x09, + 0x77, 0xd1, 0x36, 0x02, 0x99, 0x4c, 0xf7, 0xcd, 0x06, 0x75, 0x15, 0xca, 0x51, 0x67, 0xa3, 0x15, + 0xec, 0x3a, 0xae, 0xcf, 0x1e, 0xc7, 0x34, 0x26, 0xab, 0x21, 0x01, 0x38, 0xc1, 0x41, 0x2b, 0x30, + 0xea, 0x48, 0x25, 0x30, 0xca, 0x8f, 0x5a, 0xa0, 0x54, 0xbf, 0xdc, 0x91, 0x57, 0xaa, 0x7d, 0x55, + 0x5d, 0xf4, 0x2a, 0x4c, 0x08, 0xbf, 0x2d, 0x1e, 0xcb, 0x81, 0x3d, 0x5e, 0x69, 0x86, 0xf9, 0x0d, + 0x1d, 0x88, 0x4d, 0x5c, 0xf4, 0x05, 0x98, 0xa4, 0x54, 0x92, 0x83, 0x6d, 0xf6, 0xec, 0x20, 0x27, + 0xa2, 0x96, 0xe5, 0x43, 0xaf, 0x8c, 0x53, 0xc4, 0x50, 0x0b, 0x1e, 0x77, 0x3a, 0x71, 0xc0, 0x14, + 0xe9, 0xe6, 0xfa, 0x5f, 0x0f, 0x76, 0x88, 0xcf, 0xde, 0xb0, 0x46, 0x17, 0x2f, 0x1d, 0x1e, 0x54, + 0x1e, 0x5f, 0xe8, 0x81, 0x87, 0x7b, 0x52, 0x41, 0x77, 0x60, 0x2c, 0x0e, 0x3c, 0x66, 0x22, 0x4f, + 0x59, 0x89, 0x47, 0xf2, 0x03, 0x01, 0xad, 0x2b, 0x34, 0x5d, 0x89, 0xa4, 0xaa, 0x62, 0x9d, 0x0e, + 0x5a, 0xe7, 0x7b, 0x8c, 0x85, 0x48, 0x25, 0xd1, 0xec, 0xa3, 0xf9, 0x03, 0xa3, 0x22, 0xa9, 0x9a, + 0x5b, 0x50, 0xd4, 0xc4, 0x3a, 0x19, 0x74, 0x1d, 0x66, 0xda, 0xa1, 0x1b, 0xb0, 0x85, 0xad, 0x1e, + 0x31, 0x66, 0xcd, 0xc4, 0x0e, 0xf5, 0x34, 0x02, 0xee, 0xae, 0x43, 0x85, 0x4c, 0x59, 0x38, 0x7b, + 0x9e, 0x67, 0x09, 0xe3, 0x8c, 0x37, 0x2f, 0xc3, 0x0a, 0x8a, 0x56, 0xd9, 0xb9, 0xcc, 0xc5, 0xc1, + 0xd9, 0xb9, 0xfc, 0x68, 0x0f, 0xba, 0xd8, 0xc8, 0xf9, 0x25, 0xf5, 0x17, 0x27, 0x14, 0xe8, 0xbd, + 0x11, 0x6d, 0x3b, 0x21, 0xa9, 0x87, 0x41, 0x93, 0x44, 0x5a, 0x54, 0xe6, 0xc7, 0x78, 0x24, 0x47, + 0x7a, 0x6f, 0x34, 0xb2, 0x10, 0x70, 0x76, 0x3d, 0xd4, 0xd2, 0x92, 0x63, 0x53, 0x36, 0x34, 0x9a, + 0x7d, 0xbc, 0x87, 0xc1, 0x51, 0x8a, 0x67, 0x4d, 0xd6, 0xa2, 0x51, 0x1c, 0xe1, 0x14, 0x4d, 0xf4, + 0x1d, 0x30, 0x2d, 0x02, 0x1f, 0x25, 0xe3, 0x7e, 0x21, 0xb1, 0x64, 0xc4, 0x29, 0x18, 0xee, 0xc2, + 0xe6, 0xb1, 0xa8, 0x9d, 0x0d, 0x8f, 0x88, 0x45, 0x78, 0xcb, 0xf5, 0x77, 0xa2, 0xd9, 0x8b, 0xec, + 0xab, 0x45, 0x2c, 0xea, 0x34, 0x14, 0x67, 0xd4, 0x98, 0xfb, 0x76, 0x98, 0xe9, 0xba, 0xb9, 0x8e, + 0x15, 0xbf, 0xfd, 0x4f, 0x86, 0xa0, 0xac, 0x94, 0xf2, 0xe8, 0xaa, 0xf9, 0xd6, 0x72, 0x3e, 0xfd, + 0xd6, 0x32, 0x4a, 0x65, 0x03, 0xfd, 0x79, 0x65, 0xdd, 0x30, 0xd4, 0x2b, 0xe4, 0x67, 0x4b, 0xd3, + 0xb9, 0xfb, 0xbe, 0x4e, 0x7f, 0x9a, 0x8e, 0xa5, 0x38, 0xf0, 0xa3, 0x4d, 0xa9, 0xa7, 0xda, 0x66, + 0xc0, 0x64, 0xc5, 0xe8, 0x49, 0x2a, 0x20, 0xb5, 0x6a, 0xf5, 0x74, 0xf6, 0xce, 0x3a, 0x2d, 0xc4, + 0x1c, 0xc6, 0x04, 0x49, 0xca, 0x66, 0x31, 0x41, 0x72, 0xe4, 0x01, 0x05, 0x49, 0x49, 0x00, 0x27, + 0xb4, 0x90, 0x07, 0x33, 0x4d, 0x33, 0xf1, 0xaa, 0x72, 0xf4, 0x7b, 0xb2, 0x6f, 0x0a, 0xd4, 0x8e, + 0x96, 0xe5, 0x6e, 0x29, 0x4d, 0x05, 0x77, 0x13, 0x46, 0xaf, 0xc2, 0xe8, 0x7b, 0x41, 0xc4, 0x16, + 0xa5, 0xe0, 0x35, 0xa4, 0x43, 0xd4, 0xe8, 0x9b, 0xb7, 0x1b, 0xac, 0xfc, 0xe8, 0xa0, 0x32, 0x56, + 0x0f, 0x5a, 0xf2, 0x2f, 0x56, 0x15, 0xd0, 0x7d, 0x38, 0x67, 0x9c, 0xd0, 0xaa, 0xbb, 0x30, 0x78, + 0x77, 0x2f, 0x88, 0xe6, 0xce, 0xd5, 0xb2, 0x28, 0xe1, 0xec, 0x06, 0xe8, 0xb1, 0xe7, 0x07, 0x22, + 0x69, 0xb1, 0xe4, 0x67, 0x18, 0xdb, 0x52, 0xd6, 0xdd, 0xe1, 0x53, 0x08, 0xb8, 0xbb, 0x8e, 0xfd, + 0xcb, 0xfc, 0x0d, 0x43, 0x68, 0x3a, 0x49, 0xd4, 0xf1, 0x4e, 0x23, 0x27, 0xd6, 0xb2, 0xa1, 0x84, + 0x7d, 0xe0, 0x77, 0xb2, 0x5f, 0xb7, 0xd8, 0x3b, 0xd9, 0x3a, 0xd9, 0x6d, 0x7b, 0x54, 0xde, 0x7e, + 0xf8, 0x1d, 0x7f, 0x13, 0x46, 0x63, 0xd1, 0x5a, 0xaf, 0x34, 0x5e, 0x5a, 0xa7, 0xd8, 0x5b, 0xa1, + 0xe2, 0x74, 0x64, 0x29, 0x56, 0x64, 0xec, 0x7f, 0xc1, 0x67, 0x40, 0x42, 0x4e, 0x41, 0x21, 0x56, + 0x35, 0x15, 0x62, 0x95, 0x3e, 0x5f, 0x90, 0xa3, 0x18, 0xfb, 0xe7, 0x66, 0xbf, 0x99, 0x50, 0xf9, + 0x51, 0x7f, 0xa0, 0xb5, 0x7f, 0xd8, 0x82, 0xb3, 0x59, 0x16, 0x4d, 0x94, 0x3b, 0xe5, 0x22, 0xad, + 0x7a, 0xb0, 0x56, 0x23, 0x78, 0x57, 0x94, 0x63, 0x85, 0x31, 0x70, 0x86, 0x8c, 0xe3, 0x45, 0x8c, + 0xbb, 0x0d, 0x13, 0xf5, 0x90, 0x68, 0x77, 0xc0, 0xeb, 0xdc, 0xb3, 0x8e, 0xf7, 0xe7, 0xd9, 0x63, + 0x7b, 0xd5, 0xd9, 0x3f, 0x53, 0x80, 0xb3, 0xfc, 0xc5, 0x69, 0x61, 0x2f, 0x70, 0x5b, 0xf5, 0xa0, + 0x25, 0xb2, 0x9b, 0xbc, 0x0d, 0xe3, 0x6d, 0x4d, 0x0f, 0xd1, 0x2b, 0x66, 0x95, 0xae, 0xaf, 0x48, + 0xe4, 0x41, 0xbd, 0x14, 0x1b, 0xb4, 0x50, 0x0b, 0xc6, 0xc9, 0x9e, 0xdb, 0x54, 0xcf, 0x16, 0x85, + 0x63, 0xdf, 0x0d, 0xaa, 0x95, 0x65, 0x8d, 0x0e, 0x36, 0xa8, 0x3e, 0x84, 0x84, 0x77, 0xf6, 0x8f, + 0x58, 0xf0, 0x68, 0x4e, 0x84, 0x2b, 0xda, 0xdc, 0x3d, 0xf6, 0xb6, 0x27, 0x72, 0x67, 0xa9, 0xe6, + 0xf8, 0x8b, 0x1f, 0x16, 0x50, 0xf4, 0x39, 0x00, 0xfe, 0x62, 0x47, 0xc5, 0xa3, 0x7e, 0xa1, 0x80, + 0x8c, 0x28, 0x26, 0x5a, 0xf4, 0x09, 0x59, 0x1f, 0x6b, 0xb4, 0xec, 0x9f, 0x2c, 0xc2, 0x10, 0x7b, + 0x21, 0x42, 0x2b, 0x30, 0xb2, 0xcd, 0x63, 0x3e, 0x0f, 0x12, 0x5e, 0x3a, 0x91, 0x33, 0x79, 0x01, + 0x96, 0x95, 0xd1, 0x2a, 0x9c, 0xe1, 0x31, 0xb3, 0xbd, 0x2a, 0xf1, 0x9c, 0x7d, 0xa9, 0xae, 0xe0, + 0xf9, 0xa6, 0x54, 0x24, 0x8d, 0x5a, 0x37, 0x0a, 0xce, 0xaa, 0x87, 0x5e, 0x87, 0x49, 0xca, 0xdf, + 0x05, 0x9d, 0x58, 0x52, 0xe2, 0xd1, 0xb2, 0x15, 0x43, 0xb9, 0x6e, 0x40, 0x71, 0x0a, 0x9b, 0x0a, + 0x5e, 0xed, 0x2e, 0xc5, 0xcc, 0x50, 0x22, 0x78, 0x99, 0xca, 0x18, 0x13, 0x97, 0x99, 0x32, 0x75, + 0x98, 0xe1, 0xd6, 0xfa, 0x76, 0x48, 0xa2, 0xed, 0xc0, 0x6b, 0x89, 0x74, 0xe5, 0x89, 0x29, 0x53, + 0x0a, 0x8e, 0xbb, 0x6a, 0x50, 0x2a, 0x9b, 0x8e, 0xeb, 0x75, 0x42, 0x92, 0x50, 0x19, 0x36, 0xa9, + 0xac, 0xa4, 0xe0, 0xb8, 0xab, 0x06, 0x5d, 0x47, 0xe7, 0x44, 0xfe, 0x70, 0xe9, 0xdf, 0xaf, 0xec, + 0xd3, 0x46, 0xa4, 0xa7, 0x53, 0x8f, 0x00, 0x37, 0xc2, 0x82, 0x47, 0x65, 0x20, 0xd7, 0xf4, 0x89, + 0xc2, 0xc7, 0x49, 0x52, 0x79, 0x90, 0x2c, 0xd6, 0xbf, 0x6b, 0xc1, 0x99, 0x0c, 0x3b, 0x58, 0x7e, + 0x54, 0x6d, 0xb9, 0x51, 0xac, 0x72, 0xea, 0x68, 0x47, 0x15, 0x2f, 0xc7, 0x0a, 0x83, 0xee, 0x07, + 0x7e, 0x18, 0xa6, 0x0f, 0x40, 0x61, 0x67, 0x26, 0xa0, 0xc7, 0xcc, 0x4e, 0x73, 0x09, 0x4a, 0x9d, + 0x88, 0xc8, 0xd0, 0x54, 0xea, 0xfc, 0x66, 0x1a, 0x66, 0x06, 0xa1, 0xac, 0xe9, 0x96, 0x52, 0xee, + 0x6a, 0xac, 0x29, 0xd7, 0xd8, 0x72, 0x98, 0xfd, 0xd5, 0x22, 0x9c, 0xcf, 0xb5, 0x78, 0xa7, 0x5d, + 0xda, 0x0d, 0x7c, 0x37, 0x0e, 0xd4, 0xeb, 0x23, 0x0f, 0x8e, 0x42, 0xda, 0xdb, 0xab, 0xa2, 0x1c, + 0x2b, 0x0c, 0x74, 0x59, 0x66, 0xb2, 0x4f, 0x67, 0x0d, 0x5a, 0xac, 0x1a, 0xc9, 0xec, 0x07, 0xcd, + 0xc8, 0xf6, 0x24, 0x94, 0xda, 0x41, 0xe0, 0xa5, 0x0f, 0x23, 0xda, 0xdd, 0x20, 0xf0, 0x30, 0x03, + 0xa2, 0x4f, 0x88, 0x71, 0x48, 0x3d, 0xb7, 0x61, 0xa7, 0x15, 0x44, 0xda, 0x60, 0x3c, 0x0d, 0x23, + 0x3b, 0x64, 0x3f, 0x74, 0xfd, 0xad, 0xf4, 0x33, 0xec, 0x4d, 0x5e, 0x8c, 0x25, 0xdc, 0xcc, 0x21, + 0x31, 0x72, 0xd2, 0xa9, 0xd4, 0x46, 0xfb, 0x5e, 0x6d, 0x3f, 0x50, 0x84, 0x29, 0xbc, 0x58, 0xfd, + 0xd6, 0x44, 0xdc, 0xe9, 0x9e, 0x88, 0x93, 0x4e, 0xa5, 0xd6, 0x7f, 0x36, 0x7e, 0xc1, 0x82, 0x29, + 0x16, 0x67, 0x59, 0x84, 0xe4, 0x70, 0x03, 0xff, 0x14, 0x58, 0xb7, 0x27, 0x61, 0x28, 0xa4, 0x8d, + 0xa6, 0xd3, 0x05, 0xb1, 0x9e, 0x60, 0x0e, 0x43, 0x8f, 0x43, 0x89, 0x75, 0x81, 0x4e, 0xde, 0x38, + 0xcf, 0xb4, 0x50, 0x75, 0x62, 0x07, 0xb3, 0x52, 0xe6, 0x67, 0x8e, 0x49, 0xdb, 0x73, 0x79, 0xa7, + 0x93, 0xa7, 0x8d, 0x8f, 0x86, 0x9f, 0x79, 0x66, 0xd7, 0x3e, 0x98, 0x9f, 0x79, 0x36, 0xc9, 0xde, + 0x62, 0xd1, 0x1f, 0x16, 0xe0, 0x62, 0x66, 0xbd, 0x81, 0xfd, 0xcc, 0x7b, 0xd7, 0x3e, 0x19, 0x6b, + 0x9a, 0x6c, 0x23, 0x97, 0xe2, 0x29, 0x1a, 0xb9, 0x94, 0x06, 0xe5, 0x1c, 0x87, 0x06, 0x70, 0xff, + 0xce, 0x1c, 0xb2, 0x8f, 0x88, 0xfb, 0x77, 0x66, 0xdf, 0x72, 0xc4, 0xba, 0x3f, 0x2b, 0xe4, 0x7c, + 0x0b, 0x13, 0xf0, 0xae, 0xd0, 0x73, 0x86, 0x01, 0x23, 0xc1, 0x09, 0x8f, 0xf3, 0x33, 0x86, 0x97, + 0x61, 0x05, 0x45, 0xae, 0xe6, 0x48, 0x5d, 0xc8, 0xcf, 0x9e, 0x99, 0xdb, 0xd4, 0xbc, 0xf9, 0x12, + 0xa5, 0x86, 0x20, 0xc3, 0xa9, 0x7a, 0x55, 0x13, 0xca, 0x8b, 0x83, 0x0b, 0xe5, 0xe3, 0xd9, 0x02, + 0x39, 0x5a, 0x80, 0xa9, 0x5d, 0xd7, 0xa7, 0xc7, 0xe6, 0xbe, 0xc9, 0x8a, 0xaa, 0xb8, 0x22, 0xab, + 0x26, 0x18, 0xa7, 0xf1, 0xe7, 0x5e, 0x85, 0x89, 0x07, 0x57, 0x47, 0x7e, 0xb3, 0x08, 0x8f, 0xf5, + 0xd8, 0xf6, 0xfc, 0xac, 0x37, 0xe6, 0x40, 0x3b, 0xeb, 0xbb, 0xe6, 0xa1, 0x0e, 0x67, 0x37, 0x3b, + 0x9e, 0xb7, 0xcf, 0xec, 0x48, 0x49, 0x4b, 0x62, 0x08, 0x5e, 0xf1, 0x71, 0x99, 0xdb, 0x62, 0x25, + 0x03, 0x07, 0x67, 0xd6, 0x44, 0x6f, 0x00, 0x0a, 0x44, 0xea, 0xde, 0xeb, 0xc4, 0x17, 0xfa, 0x7d, + 0x36, 0xf0, 0xc5, 0x64, 0x33, 0xde, 0xee, 0xc2, 0xc0, 0x19, 0xb5, 0x28, 0xd3, 0x4f, 0x6f, 0xa5, + 0x7d, 0xd5, 0xad, 0x14, 0xd3, 0x8f, 0x75, 0x20, 0x36, 0x71, 0xd1, 0x75, 0x98, 0x71, 0xf6, 0x1c, + 0x97, 0xc7, 0xdb, 0x93, 0x04, 0x38, 0xd7, 0xaf, 0x94, 0x60, 0x0b, 0x69, 0x04, 0xdc, 0x5d, 0x27, + 0xe5, 0x6a, 0x3d, 0x9c, 0xef, 0x6a, 0xdd, 0xfb, 0x5c, 0xec, 0xa7, 0xd3, 0xb5, 0xff, 0xb3, 0x45, + 0xaf, 0xaf, 0x8c, 0xf4, 0xfb, 0x74, 0x1c, 0x94, 0x6e, 0x52, 0xf3, 0x7a, 0x3e, 0xa7, 0x59, 0x8a, + 0x24, 0x40, 0x6c, 0xe2, 0xf2, 0x05, 0x11, 0x25, 0xce, 0x36, 0x06, 0xeb, 0x2e, 0xa2, 0x26, 0x28, + 0x0c, 0xf4, 0x79, 0x18, 0x69, 0xb9, 0x7b, 0x6e, 0x14, 0x84, 0x62, 0xb3, 0x1c, 0xd3, 0x65, 0x21, + 0x39, 0x07, 0xab, 0x9c, 0x0c, 0x96, 0xf4, 0xec, 0x1f, 0x28, 0xc0, 0x84, 0x6c, 0xf1, 0xcd, 0x4e, + 0x10, 0x3b, 0xa7, 0x70, 0x2d, 0x5f, 0x37, 0xae, 0xe5, 0x4f, 0xf4, 0x0a, 0x1d, 0xc1, 0xba, 0x94, + 0x7b, 0x1d, 0xdf, 0x4e, 0x5d, 0xc7, 0x4f, 0xf5, 0x27, 0xd5, 0xfb, 0x1a, 0xfe, 0x97, 0x16, 0xcc, + 0x18, 0xf8, 0xa7, 0x70, 0x1b, 0xac, 0x98, 0xb7, 0xc1, 0x13, 0x7d, 0xbf, 0x21, 0xe7, 0x16, 0xf8, + 0xbe, 0x62, 0xaa, 0xef, 0xec, 0xf4, 0x7f, 0x0f, 0x4a, 0xdb, 0x4e, 0xd8, 0xea, 0x15, 0xa2, 0xb6, + 0xab, 0xd2, 0xfc, 0x0d, 0x27, 0x6c, 0xf1, 0x33, 0xfc, 0x59, 0x95, 0xff, 0xd2, 0x09, 0x5b, 0x7d, + 0x7d, 0xcb, 0x58, 0x53, 0xe8, 0x15, 0x18, 0x8e, 0x9a, 0x41, 0x5b, 0x59, 0x7e, 0x5e, 0xe2, 0xb9, + 0x31, 0x69, 0xc9, 0xd1, 0x41, 0x05, 0x99, 0xcd, 0xd1, 0x62, 0x2c, 0xf0, 0xd1, 0xdb, 0x30, 0xc1, + 0x7e, 0x29, 0x0b, 0x88, 0x62, 0x7e, 0x62, 0x84, 0x86, 0x8e, 0xc8, 0x0d, 0x69, 0x8c, 0x22, 0x6c, + 0x92, 0x9a, 0xdb, 0x82, 0xb2, 0xfa, 0xac, 0x87, 0xea, 0x13, 0xf4, 0x1f, 0x8a, 0x70, 0x26, 0x63, + 0xcd, 0xa1, 0xc8, 0x98, 0x89, 0xe7, 0x07, 0x5c, 0xaa, 0x1f, 0x70, 0x2e, 0x22, 0x26, 0x0d, 0xb5, + 0xc4, 0xda, 0x1a, 0xb8, 0xd1, 0x3b, 0x11, 0x49, 0x37, 0x4a, 0x8b, 0xfa, 0x37, 0x4a, 0x1b, 0x3b, + 0xb5, 0xa1, 0xa6, 0x0d, 0xa9, 0x9e, 0x3e, 0xd4, 0x39, 0xfd, 0xe3, 0x22, 0x9c, 0xcd, 0x8a, 0x66, + 0x83, 0xbe, 0x3b, 0x95, 0x24, 0xe7, 0xc5, 0x41, 0xe3, 0xe0, 0xf0, 0xcc, 0x39, 0x22, 0xc7, 0xf5, + 0xbc, 0x99, 0x36, 0xa7, 0xef, 0x30, 0x8b, 0x36, 0x99, 0x23, 0x69, 0xc8, 0x93, 0x1b, 0xc9, 0xe3, + 0xe3, 0xd3, 0x03, 0x77, 0x40, 0x64, 0x45, 0x8a, 0x52, 0x8e, 0xa4, 0xb2, 0xb8, 0xbf, 0x23, 0xa9, + 0x6c, 0x79, 0xce, 0x85, 0x31, 0xed, 0x6b, 0x1e, 0xea, 0x8c, 0xef, 0xd0, 0xdb, 0x4a, 0xeb, 0xf7, + 0x43, 0x9d, 0xf5, 0x1f, 0xb1, 0x20, 0x65, 0x66, 0xa9, 0xd4, 0x5d, 0x56, 0xae, 0xba, 0xeb, 0x12, + 0x94, 0xc2, 0xc0, 0x23, 0xe9, 0x9c, 0x34, 0x38, 0xf0, 0x08, 0x66, 0x10, 0x8a, 0x11, 0x27, 0xca, + 0x8e, 0x71, 0x5d, 0x90, 0x13, 0x22, 0xda, 0x93, 0x30, 0xe4, 0x91, 0x3d, 0xe2, 0xa5, 0x03, 0xbe, + 0xdf, 0xa2, 0x85, 0x98, 0xc3, 0xec, 0x5f, 0x28, 0xc1, 0x85, 0x9e, 0xae, 0xd8, 0x54, 0x1c, 0xda, + 0x72, 0x62, 0x72, 0xcf, 0xd9, 0x4f, 0x47, 0x66, 0xbe, 0xce, 0x8b, 0xb1, 0x84, 0x33, 0xcb, 0x73, + 0x1e, 0x89, 0x31, 0xa5, 0x1c, 0x14, 0x01, 0x18, 0x05, 0xf4, 0x21, 0xe4, 0xf7, 0xbf, 0x06, 0x10, + 0x45, 0x1e, 0xb7, 0x1b, 0x68, 0x09, 0x93, 0xf6, 0x24, 0x62, 0x67, 0xe3, 0x96, 0x80, 0x60, 0x0d, + 0x0b, 0x55, 0x61, 0xba, 0x1d, 0x06, 0x31, 0xd7, 0xb5, 0x56, 0xb9, 0xc1, 0xd1, 0x90, 0xe9, 0x05, + 0x5b, 0x4f, 0xc1, 0x71, 0x57, 0x0d, 0xf4, 0x12, 0x8c, 0x09, 0xcf, 0xd8, 0x7a, 0x10, 0x78, 0x42, + 0x0d, 0xa4, 0xcc, 0x57, 0x1a, 0x09, 0x08, 0xeb, 0x78, 0x5a, 0x35, 0xa6, 0xc0, 0x1d, 0xc9, 0xac, + 0xc6, 0x95, 0xb8, 0x1a, 0x5e, 0x2a, 0xb2, 0xd5, 0xe8, 0x40, 0x91, 0xad, 0x12, 0xc5, 0x58, 0x79, + 0xe0, 0x37, 0x2b, 0xe8, 0xab, 0x4a, 0xfa, 0xd9, 0x12, 0x9c, 0x11, 0x0b, 0xe7, 0x61, 0x2f, 0x97, + 0x3b, 0xdd, 0xcb, 0xe5, 0x24, 0x54, 0x67, 0xdf, 0x5a, 0x33, 0xa7, 0xbd, 0x66, 0x7e, 0xd0, 0x02, + 0x93, 0xbd, 0x42, 0x7f, 0x29, 0x37, 0xb4, 0xfd, 0x4b, 0xb9, 0xec, 0x5a, 0x4b, 0x5e, 0x20, 0x1f, + 0x30, 0xc8, 0xbd, 0xfd, 0x9f, 0x2c, 0x78, 0xa2, 0x2f, 0x45, 0xb4, 0x0c, 0x65, 0xc6, 0x03, 0x6a, + 0xd2, 0xd9, 0x53, 0xca, 0x20, 0x51, 0x02, 0x72, 0x58, 0xd2, 0xa4, 0x26, 0x5a, 0xee, 0xca, 0x21, + 0xf0, 0x74, 0x46, 0x0e, 0x81, 0x73, 0xc6, 0xf0, 0x3c, 0x60, 0x12, 0x81, 0x5f, 0x2e, 0xc2, 0x30, + 0x5f, 0xf1, 0xa7, 0x20, 0x86, 0xad, 0x08, 0xbd, 0x6d, 0x8f, 0xd8, 0x56, 0xbc, 0x2f, 0xf3, 0x55, + 0x27, 0x76, 0x38, 0x9b, 0xa0, 0x6e, 0xab, 0x44, 0xc3, 0x8b, 0xe6, 0x8d, 0xfb, 0x6c, 0x2e, 0xa5, + 0x98, 0x04, 0x4e, 0x43, 0xbb, 0xdd, 0xbe, 0x08, 0x10, 0xb1, 0xfc, 0xfb, 0x94, 0x86, 0x88, 0x92, + 0xf6, 0xc9, 0x1e, 0xad, 0x37, 0x14, 0x32, 0xef, 0x43, 0xb2, 0xd3, 0x15, 0x00, 0x6b, 0x14, 0xe7, + 0x5e, 0x86, 0xb2, 0x42, 0xee, 0xa7, 0xc5, 0x19, 0xd7, 0x99, 0x8b, 0xcf, 0xc2, 0x54, 0xaa, 0xad, + 0x63, 0x29, 0x81, 0x7e, 0xd1, 0x82, 0x29, 0xde, 0xe5, 0x65, 0x7f, 0x4f, 0x9c, 0xa9, 0xef, 0xc3, + 0x59, 0x2f, 0xe3, 0x6c, 0x13, 0x33, 0x3a, 0xf8, 0x59, 0xa8, 0x94, 0x3e, 0x59, 0x50, 0x9c, 0xd9, + 0x06, 0xba, 0x42, 0xd7, 0x2d, 0x3d, 0xbb, 0x1c, 0x4f, 0x78, 0x31, 0x8d, 0xf3, 0x35, 0xcb, 0xcb, + 0xb0, 0x82, 0xda, 0xbf, 0x6d, 0xc1, 0x0c, 0xef, 0xf9, 0x4d, 0xb2, 0xaf, 0x76, 0xf8, 0x87, 0xd9, + 0x77, 0x91, 0xd6, 0xa3, 0x90, 0x93, 0xd6, 0x43, 0xff, 0xb4, 0x62, 0xcf, 0x4f, 0xfb, 0x19, 0x0b, + 0xc4, 0x0a, 0x3c, 0x05, 0x51, 0xfe, 0xdb, 0x4d, 0x51, 0x7e, 0x2e, 0x7f, 0x51, 0xe7, 0xc8, 0xf0, + 0x7f, 0x6a, 0xc1, 0x34, 0x47, 0x48, 0xde, 0x92, 0x3f, 0xd4, 0x79, 0x18, 0x24, 0x3f, 0x9f, 0x4a, + 0xda, 0x9d, 0xfd, 0x51, 0xc6, 0x64, 0x95, 0x7a, 0x4e, 0x56, 0x4b, 0x6e, 0xa0, 0x63, 0xe4, 0xa6, + 0x3c, 0x76, 0x78, 0x6c, 0xfb, 0x0f, 0x2c, 0x40, 0xbc, 0x19, 0x83, 0xfd, 0xa1, 0x4c, 0x05, 0x2b, + 0xd5, 0xae, 0x8b, 0xe4, 0xa8, 0x51, 0x10, 0xac, 0x61, 0x9d, 0xc8, 0xf0, 0xa4, 0x0c, 0x02, 0x8a, + 0xfd, 0x0d, 0x02, 0x8e, 0x31, 0xa2, 0xff, 0xa7, 0x04, 0x69, 0xb7, 0x02, 0x74, 0x17, 0xc6, 0x9b, + 0x4e, 0xdb, 0xd9, 0x70, 0x3d, 0x37, 0x76, 0x49, 0xd4, 0xcb, 0x92, 0x68, 0x49, 0xc3, 0x13, 0x4f, + 0xbd, 0x5a, 0x09, 0x36, 0xe8, 0xa0, 0x79, 0x80, 0x76, 0xe8, 0xee, 0xb9, 0x1e, 0xd9, 0x62, 0x1a, + 0x07, 0xe6, 0x37, 0xc9, 0xcd, 0x63, 0x64, 0x29, 0xd6, 0x30, 0x32, 0x5c, 0xe0, 0x8a, 0x0f, 0xcf, + 0x05, 0xae, 0x74, 0x4c, 0x17, 0xb8, 0xa1, 0x81, 0x5c, 0xe0, 0x30, 0x3c, 0x22, 0x59, 0x24, 0xfa, + 0x7f, 0xc5, 0xf5, 0x88, 0xe0, 0x8b, 0xb9, 0x37, 0xe5, 0xdc, 0xe1, 0x41, 0xe5, 0x11, 0x9c, 0x89, + 0x81, 0x73, 0x6a, 0xa2, 0xcf, 0xc1, 0xac, 0xe3, 0x79, 0xc1, 0x3d, 0x35, 0x6a, 0xcb, 0x51, 0xd3, + 0xf1, 0xb8, 0xc6, 0x7e, 0x84, 0x51, 0x7d, 0xfc, 0xf0, 0xa0, 0x32, 0xbb, 0x90, 0x83, 0x83, 0x73, + 0x6b, 0xa7, 0x3c, 0xe8, 0x46, 0xfb, 0x7a, 0xd0, 0xbd, 0x06, 0xe5, 0x76, 0x18, 0x34, 0x57, 0x35, + 0xaf, 0x9e, 0x8b, 0x2c, 0xf3, 0xbd, 0x2c, 0x3c, 0x3a, 0xa8, 0x4c, 0xa8, 0x3f, 0xec, 0x86, 0x4f, + 0x2a, 0xd8, 0x3b, 0x70, 0xa6, 0x41, 0x42, 0x97, 0xe5, 0xd4, 0x6c, 0x25, 0x1b, 0x7a, 0x1d, 0xca, + 0x61, 0xea, 0x08, 0x1b, 0x28, 0x40, 0x93, 0x16, 0x37, 0x58, 0x1e, 0x59, 0x09, 0x21, 0xfb, 0x4f, + 0x2c, 0x18, 0x11, 0x16, 0xe6, 0xa7, 0xc0, 0x39, 0x2d, 0x18, 0x0a, 0xec, 0x4a, 0xf6, 0x31, 0xcf, + 0x3a, 0x93, 0xab, 0xba, 0xae, 0xa5, 0x54, 0xd7, 0x4f, 0xf4, 0x22, 0xd2, 0x5b, 0x69, 0xfd, 0x77, + 0x8b, 0x30, 0x69, 0x3a, 0x85, 0x9c, 0xc2, 0x10, 0xac, 0xc1, 0x48, 0x24, 0x3c, 0x90, 0x0a, 0xf9, + 0x96, 0xd3, 0xe9, 0x49, 0x4c, 0xcc, 0xa2, 0x84, 0xcf, 0x91, 0x24, 0x92, 0xe9, 0xda, 0x54, 0x7c, + 0x88, 0xae, 0x4d, 0xfd, 0xfc, 0x72, 0x4a, 0x27, 0xe1, 0x97, 0x63, 0x7f, 0x9d, 0x5d, 0x35, 0x7a, + 0xf9, 0x29, 0x70, 0x21, 0xd7, 0xcd, 0x4b, 0xc9, 0xee, 0xb1, 0xb2, 0x44, 0xa7, 0x72, 0xb8, 0x91, + 0x9f, 0xb7, 0xe0, 0x42, 0xc6, 0x57, 0x69, 0xac, 0xc9, 0xb3, 0x30, 0xea, 0x74, 0x5a, 0xae, 0xda, + 0xcb, 0xda, 0x33, 0xd6, 0x82, 0x28, 0xc7, 0x0a, 0x03, 0x2d, 0xc1, 0x0c, 0xb9, 0xdf, 0x76, 0xf9, + 0x3b, 0xa2, 0x6e, 0xbb, 0x58, 0xe4, 0x41, 0x6b, 0x97, 0xd3, 0x40, 0xdc, 0x8d, 0xaf, 0xdc, 0xba, + 0x8b, 0xb9, 0x6e, 0xdd, 0xff, 0xc4, 0x82, 0x31, 0xe5, 0x6d, 0xf2, 0xd0, 0x47, 0xfb, 0x3b, 0xcc, + 0xd1, 0x7e, 0xac, 0xc7, 0x68, 0xe7, 0x0c, 0xf3, 0xdf, 0x2f, 0xa8, 0xfe, 0xd6, 0x83, 0x30, 0x1e, + 0x80, 0xe5, 0x79, 0x05, 0x46, 0xdb, 0x61, 0x10, 0x07, 0xcd, 0xc0, 0x13, 0x1c, 0xcf, 0xe3, 0x49, + 0xd4, 0x01, 0x5e, 0x7e, 0xa4, 0xfd, 0xc6, 0x0a, 0x9b, 0x8d, 0x5e, 0x10, 0xc6, 0x82, 0xcb, 0x48, + 0x46, 0x2f, 0x08, 0x63, 0xcc, 0x20, 0xa8, 0x05, 0x10, 0x3b, 0xe1, 0x16, 0x89, 0x69, 0x99, 0x08, + 0x60, 0x92, 0x7f, 0x78, 0x74, 0x62, 0xd7, 0x9b, 0x77, 0xfd, 0x38, 0x8a, 0xc3, 0xf9, 0x9a, 0x1f, + 0xdf, 0x0e, 0xb9, 0x00, 0xa5, 0x85, 0x11, 0x50, 0xb4, 0xb0, 0x46, 0x57, 0xfa, 0x7a, 0xb2, 0x36, + 0x86, 0xcc, 0x07, 0xf1, 0x35, 0x51, 0x8e, 0x15, 0x86, 0xfd, 0x32, 0xbb, 0x4a, 0xd8, 0x00, 0x1d, + 0xcf, 0xc3, 0xff, 0x1b, 0xa3, 0x6a, 0x68, 0xd9, 0x6b, 0x58, 0x55, 0x8f, 0x23, 0xd0, 0xfb, 0xe4, + 0xa6, 0x0d, 0xeb, 0x7e, 0x34, 0x49, 0xb0, 0x01, 0xf4, 0x9d, 0x5d, 0x76, 0x12, 0xcf, 0xf5, 0xb9, + 0x02, 0x8e, 0x61, 0x19, 0xc1, 0x02, 0x69, 0xb3, 0x30, 0xc3, 0xb5, 0xba, 0x58, 0xe4, 0x5a, 0x20, + 0x6d, 0x01, 0xc0, 0x09, 0x0e, 0xba, 0x2a, 0xc4, 0xef, 0x92, 0x91, 0x4e, 0x4f, 0x8a, 0xdf, 0xf2, + 0xf3, 0x35, 0xf9, 0xfb, 0x79, 0x18, 0x53, 0x69, 0xf5, 0xea, 0x3c, 0x3b, 0x99, 0x08, 0xe7, 0xb2, + 0x9c, 0x14, 0x63, 0x1d, 0x07, 0xad, 0xc3, 0x54, 0xc4, 0x75, 0x2f, 0x2a, 0x6a, 0x1f, 0xd7, 0x61, + 0x7d, 0x52, 0xda, 0x57, 0x34, 0x4c, 0xf0, 0x11, 0x2b, 0xe2, 0x47, 0x87, 0x74, 0xd8, 0x4c, 0x93, + 0x40, 0xaf, 0xc3, 0xa4, 0xa7, 0x27, 0xb0, 0xaf, 0x0b, 0x15, 0x97, 0x32, 0x3f, 0x36, 0xd2, 0xdb, + 0xd7, 0x71, 0x0a, 0x9b, 0x72, 0x4a, 0x7a, 0x89, 0x88, 0x34, 0xe9, 0xf8, 0x5b, 0x24, 0x12, 0x49, + 0xc1, 0x18, 0xa7, 0x74, 0x2b, 0x07, 0x07, 0xe7, 0xd6, 0x46, 0xaf, 0xc0, 0xb8, 0xfc, 0x7c, 0xcd, + 0x1d, 0x39, 0x31, 0x72, 0xd7, 0x60, 0xd8, 0xc0, 0x44, 0xf7, 0xe0, 0x9c, 0xfc, 0xbf, 0x1e, 0x3a, + 0x9b, 0x9b, 0x6e, 0x53, 0x78, 0x83, 0x73, 0x4f, 0x9f, 0x05, 0xe9, 0x3a, 0xb4, 0x9c, 0x85, 0x74, + 0x74, 0x50, 0xb9, 0x24, 0x46, 0x2d, 0x13, 0xce, 0x26, 0x31, 0x9b, 0x3e, 0x5a, 0x85, 0x33, 0xdb, + 0xc4, 0xf1, 0xe2, 0xed, 0xa5, 0x6d, 0xd2, 0xdc, 0x91, 0x9b, 0x88, 0x39, 0x39, 0x6b, 0xa6, 0xe1, + 0x37, 0xba, 0x51, 0x70, 0x56, 0x3d, 0xf4, 0x0e, 0xcc, 0xb6, 0x3b, 0x1b, 0x9e, 0x1b, 0x6d, 0xaf, + 0x05, 0x31, 0x33, 0xe9, 0x50, 0x59, 0xe9, 0x84, 0x37, 0xb4, 0x72, 0xf0, 0xae, 0xe7, 0xe0, 0xe1, + 0x5c, 0x0a, 0xe8, 0x7d, 0x38, 0x97, 0x5a, 0x0c, 0xc2, 0x37, 0x73, 0x32, 0x3f, 0x6e, 0x6f, 0x23, + 0xab, 0x82, 0xf0, 0xb5, 0xcc, 0x02, 0xe1, 0xec, 0x26, 0x3e, 0x98, 0xa1, 0xcf, 0x7b, 0xb4, 0xb2, + 0xc6, 0x94, 0xa1, 0x2f, 0xc1, 0xb8, 0xbe, 0x8a, 0xc4, 0x05, 0x73, 0x39, 0x9b, 0x67, 0xd1, 0x56, + 0x1b, 0x67, 0xe9, 0xd4, 0x8a, 0xd2, 0x61, 0xd8, 0xa0, 0x68, 0x13, 0xc8, 0xfe, 0x3e, 0x74, 0x0b, + 0x46, 0x9b, 0x9e, 0x4b, 0xfc, 0xb8, 0x56, 0xef, 0x15, 0x3c, 0x64, 0x49, 0xe0, 0x88, 0x01, 0x13, + 0x81, 0x4e, 0x79, 0x19, 0x56, 0x14, 0xec, 0x5f, 0x2b, 0x40, 0xa5, 0x4f, 0xd4, 0xdc, 0x94, 0x3e, + 0xda, 0x1a, 0x48, 0x1f, 0xbd, 0x20, 0x73, 0xec, 0xad, 0xa5, 0x84, 0xf4, 0x54, 0xfe, 0xbc, 0x44, + 0x54, 0x4f, 0xe3, 0x0f, 0x6c, 0x1f, 0xac, 0xab, 0xb4, 0x4b, 0x7d, 0x2d, 0xd7, 0x8d, 0xa7, 0xac, + 0xa1, 0xc1, 0x05, 0x91, 0xdc, 0x67, 0x09, 0xfb, 0xeb, 0x05, 0x38, 0xa7, 0x86, 0xf0, 0x2f, 0xee, + 0xc0, 0xdd, 0xe9, 0x1e, 0xb8, 0x13, 0x78, 0xd4, 0xb1, 0x6f, 0xc3, 0x30, 0x0f, 0xbe, 0x32, 0x00, + 0x03, 0xf4, 0xa4, 0x19, 0xa9, 0x4b, 0x5d, 0xd3, 0x46, 0xb4, 0xae, 0xbf, 0x66, 0xc1, 0xd4, 0xfa, + 0x52, 0xbd, 0x11, 0x34, 0x77, 0x48, 0xbc, 0xc0, 0x19, 0x56, 0x2c, 0xf8, 0x1f, 0xeb, 0x01, 0xf9, + 0x9a, 0x2c, 0x8e, 0xe9, 0x12, 0x94, 0xb6, 0x83, 0x28, 0x4e, 0xbf, 0xf8, 0xde, 0x08, 0xa2, 0x18, + 0x33, 0x88, 0xfd, 0x3b, 0x16, 0x0c, 0xb1, 0xcc, 0xb0, 0xfd, 0xd2, 0x15, 0x0f, 0xf2, 0x5d, 0xe8, + 0x25, 0x18, 0x26, 0x9b, 0x9b, 0xa4, 0x19, 0x8b, 0x59, 0x95, 0xee, 0xa8, 0xc3, 0xcb, 0xac, 0x94, + 0x5e, 0xfa, 0xac, 0x31, 0xfe, 0x17, 0x0b, 0x64, 0xf4, 0x16, 0x94, 0x63, 0x77, 0x97, 0x2c, 0xb4, + 0x5a, 0xe2, 0xcd, 0xec, 0x01, 0xbc, 0x7f, 0xd7, 0x25, 0x01, 0x9c, 0xd0, 0xb2, 0xbf, 0x5a, 0x00, + 0x48, 0x42, 0x08, 0xf4, 0xfb, 0xc4, 0xc5, 0xae, 0xd7, 0x94, 0xcb, 0x19, 0xaf, 0x29, 0x28, 0x21, + 0x98, 0xf1, 0x94, 0xa2, 0x86, 0xa9, 0x38, 0xd0, 0x30, 0x95, 0x8e, 0x33, 0x4c, 0x4b, 0x30, 0x93, + 0x84, 0x40, 0x30, 0xe3, 0xc1, 0x30, 0x21, 0x65, 0x3d, 0x0d, 0xc4, 0xdd, 0xf8, 0x36, 0x81, 0x4b, + 0x32, 0x32, 0xa7, 0xbc, 0x6b, 0x98, 0x49, 0xe6, 0x31, 0x32, 0x57, 0x27, 0xcf, 0x45, 0x85, 0xdc, + 0xe7, 0xa2, 0x1f, 0xb7, 0xe0, 0x6c, 0xba, 0x1d, 0xe6, 0xfb, 0xf6, 0x15, 0x0b, 0xce, 0xb1, 0x47, + 0x33, 0xd6, 0x6a, 0xf7, 0x13, 0xdd, 0x8b, 0xd9, 0xa1, 0x21, 0x7a, 0xf7, 0x38, 0xf1, 0x7b, 0x5e, + 0xcd, 0x22, 0x8d, 0xb3, 0x5b, 0xb4, 0xbf, 0x62, 0xc1, 0xf9, 0xdc, 0x84, 0x44, 0xe8, 0x0a, 0x8c, + 0x3a, 0x6d, 0x97, 0x6b, 0xa4, 0xc4, 0x7e, 0x67, 0xd2, 0x63, 0xbd, 0xc6, 0xf5, 0x51, 0x0a, 0xaa, + 0x12, 0x25, 0x16, 0x72, 0x13, 0x25, 0xf6, 0xcd, 0x7b, 0x68, 0x7f, 0xbf, 0x05, 0xc2, 0xdd, 0x69, + 0x80, 0x43, 0xe6, 0x6d, 0x99, 0x67, 0xd6, 0x08, 0x8a, 0x7e, 0x29, 0xdf, 0xff, 0x4b, 0x84, 0x42, + 0x57, 0x97, 0xba, 0x11, 0x00, 0xdd, 0xa0, 0x65, 0xb7, 0x40, 0x40, 0xab, 0x84, 0xe9, 0xac, 0xfa, + 0xf7, 0xe6, 0x1a, 0x40, 0x8b, 0xe1, 0x6a, 0xd9, 0x26, 0xd5, 0x15, 0x52, 0x55, 0x10, 0xac, 0x61, + 0xd9, 0x3f, 0x54, 0x80, 0x31, 0x19, 0x84, 0xbb, 0xe3, 0x0f, 0x22, 0x59, 0x1e, 0x2b, 0x2b, 0x0f, + 0x4b, 0xcf, 0x4a, 0x09, 0xd7, 0x13, 0x81, 0x3c, 0x49, 0xcf, 0x2a, 0x01, 0x38, 0xc1, 0x41, 0x4f, + 0xc3, 0x48, 0xd4, 0xd9, 0x60, 0xe8, 0x29, 0x27, 0x9e, 0x06, 0x2f, 0xc6, 0x12, 0x8e, 0x3e, 0x07, + 0xd3, 0xbc, 0x5e, 0x18, 0xb4, 0x9d, 0x2d, 0xae, 0xfe, 0x1c, 0x52, 0x5e, 0xb5, 0xd3, 0xab, 0x29, + 0xd8, 0xd1, 0x41, 0xe5, 0x6c, 0xba, 0x8c, 0x29, 0xce, 0xbb, 0xa8, 0xd8, 0x5f, 0x02, 0xd4, 0x1d, + 0x57, 0x1c, 0xbd, 0xc1, 0x4d, 0xa9, 0xdc, 0x90, 0xb4, 0x7a, 0x69, 0xc4, 0x75, 0x27, 0x50, 0x69, + 0x48, 0xcf, 0x6b, 0x61, 0x55, 0xdf, 0xfe, 0x9b, 0x45, 0x98, 0x4e, 0xbb, 0x04, 0xa2, 0x1b, 0x30, + 0xcc, 0x2f, 0x3b, 0x41, 0xbe, 0xc7, 0x83, 0xab, 0xe6, 0x48, 0xc8, 0xb6, 0xbd, 0xb8, 0x2f, 0x45, + 0x7d, 0xf4, 0x0e, 0x8c, 0xb5, 0x82, 0x7b, 0xfe, 0x3d, 0x27, 0x6c, 0x2d, 0xd4, 0x6b, 0x62, 0x5d, + 0x66, 0xf2, 0xcc, 0xd5, 0x04, 0x4d, 0x77, 0x4e, 0x64, 0x8f, 0x0b, 0x09, 0x08, 0xeb, 0xe4, 0xd0, + 0x3a, 0x8b, 0x95, 0xb8, 0xe9, 0x6e, 0xad, 0x3a, 0xed, 0x5e, 0x76, 0xb5, 0x4b, 0x12, 0x49, 0xa3, + 0x3c, 0x21, 0x02, 0x2a, 0x72, 0x00, 0x4e, 0x08, 0xa1, 0xef, 0x86, 0x33, 0x51, 0x8e, 0x9a, 0x2d, + 0x2f, 0xcd, 0x44, 0x2f, 0xcd, 0xd3, 0xe2, 0xa3, 0x54, 0x9a, 0xc9, 0x52, 0xc8, 0x65, 0x35, 0x63, + 0x7f, 0xf9, 0x0c, 0x18, 0xbb, 0xd1, 0xc8, 0x3a, 0x64, 0x9d, 0x50, 0xd6, 0x21, 0x0c, 0xa3, 0x64, + 0xb7, 0x1d, 0xef, 0x57, 0xdd, 0xb0, 0x57, 0x56, 0xbc, 0x65, 0x81, 0xd3, 0x4d, 0x53, 0x42, 0xb0, + 0xa2, 0x93, 0x9d, 0x1a, 0xaa, 0xf8, 0x21, 0xa6, 0x86, 0x2a, 0x9d, 0x62, 0x6a, 0xa8, 0x35, 0x18, + 0xd9, 0x72, 0x63, 0x4c, 0xda, 0x81, 0x60, 0x33, 0x33, 0xd7, 0xe1, 0x75, 0x8e, 0xd2, 0x9d, 0x84, + 0x44, 0x00, 0xb0, 0x24, 0x82, 0xde, 0x50, 0x3b, 0x70, 0x38, 0x5f, 0x4a, 0xeb, 0x7e, 0x19, 0xcc, + 0xdc, 0x83, 0x22, 0x01, 0xd4, 0xc8, 0x83, 0x26, 0x80, 0x5a, 0x91, 0x69, 0x9b, 0x46, 0xf3, 0x8d, + 0xe0, 0x59, 0x56, 0xa6, 0x3e, 0xc9, 0x9a, 0xee, 0xea, 0xa9, 0xae, 0xca, 0xf9, 0x27, 0x81, 0xca, + 0x62, 0x35, 0x60, 0x82, 0xab, 0xef, 0xb7, 0xe0, 0x5c, 0x3b, 0x2b, 0xeb, 0x9b, 0x48, 0xb6, 0xf4, + 0xd2, 0xc0, 0x69, 0xed, 0x8c, 0x06, 0x99, 0xb8, 0x9e, 0x89, 0x86, 0xb3, 0x9b, 0xa3, 0x03, 0x1d, + 0x6e, 0xb4, 0x44, 0x86, 0xa6, 0x27, 0x73, 0x32, 0x65, 0xf5, 0xc8, 0x8f, 0xb5, 0x9e, 0x91, 0x95, + 0xe9, 0xe3, 0x79, 0x59, 0x99, 0x06, 0xce, 0xc5, 0xf4, 0x86, 0xca, 0x91, 0x35, 0x91, 0xbf, 0x94, + 0x78, 0x06, 0xac, 0xbe, 0x99, 0xb1, 0xde, 0x50, 0x99, 0xb1, 0x7a, 0xc4, 0x8c, 0xe3, 0x79, 0xaf, + 0xfa, 0xe6, 0xc3, 0xd2, 0x72, 0x5a, 0x4d, 0x9d, 0x4c, 0x4e, 0x2b, 0xe3, 0xaa, 0xe1, 0x69, 0x95, + 0x9e, 0xe9, 0x73, 0xd5, 0x18, 0x74, 0x7b, 0x5f, 0x36, 0x3c, 0x7f, 0xd7, 0xcc, 0x03, 0xe5, 0xef, + 0xba, 0xab, 0xe7, 0xc3, 0x42, 0x7d, 0x12, 0x3e, 0x51, 0xa4, 0x01, 0xb3, 0x60, 0xdd, 0xd5, 0x2f, + 0xc0, 0x33, 0xf9, 0x74, 0xd5, 0x3d, 0xd7, 0x4d, 0x37, 0xf3, 0x0a, 0xec, 0xca, 0xae, 0x75, 0xf6, + 0x74, 0xb2, 0x6b, 0x9d, 0x3b, 0xf1, 0xec, 0x5a, 0x8f, 0x9c, 0x42, 0x76, 0xad, 0x47, 0x3f, 0xd4, + 0xec, 0x5a, 0xb3, 0x0f, 0x21, 0xbb, 0xd6, 0x5a, 0x92, 0x5d, 0xeb, 0x7c, 0xfe, 0x94, 0x64, 0x58, + 0xe6, 0xe6, 0xe4, 0xd4, 0xba, 0xcb, 0x9e, 0xe7, 0x79, 0xcc, 0x0a, 0x11, 0xd4, 0x2e, 0x3b, 0x93, + 0x70, 0x56, 0x60, 0x0b, 0x3e, 0x25, 0x0a, 0x84, 0x13, 0x52, 0x94, 0x6e, 0x92, 0x63, 0xeb, 0xb1, + 0x1e, 0x0a, 0xd9, 0x2c, 0x55, 0x57, 0x7e, 0x66, 0x2d, 0xfb, 0xaf, 0x17, 0xe0, 0x62, 0xef, 0x75, + 0x9d, 0xe8, 0xc9, 0xea, 0xc9, 0xbb, 0x4e, 0x4a, 0x4f, 0xc6, 0x85, 0x9c, 0x04, 0x6b, 0xe0, 0xc0, + 0x3e, 0xd7, 0x61, 0x46, 0x99, 0xe4, 0x7a, 0x6e, 0x73, 0x5f, 0xcb, 0x30, 0xac, 0x5c, 0x0f, 0x1b, + 0x69, 0x04, 0xdc, 0x5d, 0x07, 0x2d, 0xc0, 0x94, 0x51, 0x58, 0xab, 0x0a, 0x61, 0x46, 0x29, 0xe6, + 0x1a, 0x26, 0x18, 0xa7, 0xf1, 0xed, 0x9f, 0xb6, 0xe0, 0xd1, 0x9c, 0xc4, 0x13, 0x03, 0xc7, 0xad, + 0xd9, 0x84, 0xa9, 0xb6, 0x59, 0xb5, 0x4f, 0x78, 0x2b, 0x23, 0xbd, 0x85, 0xea, 0x6b, 0x0a, 0x80, + 0xd3, 0x44, 0x17, 0xaf, 0xfc, 0xe6, 0xef, 0x5d, 0xfc, 0xd8, 0x6f, 0xfd, 0xde, 0xc5, 0x8f, 0xfd, + 0xf6, 0xef, 0x5d, 0xfc, 0xd8, 0x5f, 0x3e, 0xbc, 0x68, 0xfd, 0xe6, 0xe1, 0x45, 0xeb, 0xb7, 0x0e, + 0x2f, 0x5a, 0xbf, 0x7d, 0x78, 0xd1, 0xfa, 0xdd, 0xc3, 0x8b, 0xd6, 0x57, 0x7f, 0xff, 0xe2, 0xc7, + 0xde, 0x2e, 0xec, 0x3d, 0xff, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0x84, 0x97, 0x9c, 0xb4, 0x50, + 0xe8, 0x00, 0x00, } diff --git a/staging/src/k8s.io/api/core/v1/generated.proto b/staging/src/k8s.io/api/core/v1/generated.proto index c76482ffbdb..165aa2a2450 100644 --- a/staging/src/k8s.io/api/core/v1/generated.proto +++ b/staging/src/k8s.io/api/core/v1/generated.proto @@ -641,7 +641,7 @@ message Container { repeated VolumeMount volumeMounts = 9; // volumeDevices is the list of block devices to be used by the container. - // This is an alpha feature and may change in the future. + // This is a beta feature. // +patchMergeKey=devicePath // +patchStrategy=merge // +optional @@ -1373,6 +1373,30 @@ message GitRepoVolumeSource { optional string directory = 3; } +// Represents a Glusterfs mount that lasts the lifetime of a pod. +// Glusterfs volumes do not support ownership management or SELinux relabeling. +message GlusterfsPersistentVolumeSource { + // EndpointsName is the endpoint name that details Glusterfs topology. + // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod + optional string endpoints = 1; + + // Path is the Glusterfs volume path. + // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod + optional string path = 2; + + // ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. + // Defaults to false. + // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod + // +optional + optional bool readOnly = 3; + + // EndpointsNamespace is the namespace that contains Glusterfs endpoint. + // If this field is empty, the EndpointNamespace defaults to the same namespace as the bound PVC. + // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod + // +optional + optional string endpointsNamespace = 4; +} + // Represents a Glusterfs mount that lasts the lifetime of a pod. // Glusterfs volumes do not support ownership management or SELinux relabeling. message GlusterfsVolumeSource { @@ -2296,7 +2320,7 @@ message PersistentVolumeClaimSpec { // volumeMode defines what type of volume is required by the claim. // Value of Filesystem is implied when not included in claim spec. - // This is an alpha feature and may change in the future. + // This is a beta feature. // +optional optional string volumeMode = 6; @@ -2389,7 +2413,7 @@ message PersistentVolumeSource { // exposed to the pod. Provisioned by an admin. // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md // +optional - optional GlusterfsVolumeSource glusterfs = 4; + optional GlusterfsPersistentVolumeSource glusterfs = 4; // NFS represents an NFS mount on the host. Provisioned by an admin. // More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs @@ -2512,7 +2536,7 @@ message PersistentVolumeSpec { // volumeMode defines if a volume is intended to be used with a formatted filesystem // or to remain in raw block state. Value of Filesystem is implied when not included in spec. - // This is an alpha feature and may change in the future. + // This is a beta feature. // +optional optional string volumeMode = 8; diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index 67c67f849c8..43e70b9bad5 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -191,7 +191,7 @@ type PersistentVolumeSource struct { // exposed to the pod. Provisioned by an admin. // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md // +optional - Glusterfs *GlusterfsVolumeSource `json:"glusterfs,omitempty" protobuf:"bytes,4,opt,name=glusterfs"` + Glusterfs *GlusterfsPersistentVolumeSource `json:"glusterfs,omitempty" protobuf:"bytes,4,opt,name=glusterfs"` // NFS represents an NFS mount on the host. Provisioned by an admin. // More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs // +optional @@ -326,7 +326,7 @@ type PersistentVolumeSpec struct { MountOptions []string `json:"mountOptions,omitempty" protobuf:"bytes,7,opt,name=mountOptions"` // volumeMode defines if a volume is intended to be used with a formatted filesystem // or to remain in raw block state. Value of Filesystem is implied when not included in spec. - // This is an alpha feature and may change in the future. + // This is a beta feature. // +optional VolumeMode *PersistentVolumeMode `json:"volumeMode,omitempty" protobuf:"bytes,8,opt,name=volumeMode,casttype=PersistentVolumeMode"` // NodeAffinity defines constraints that limit what nodes this volume can be accessed from. @@ -455,7 +455,7 @@ type PersistentVolumeClaimSpec struct { StorageClassName *string `json:"storageClassName,omitempty" protobuf:"bytes,5,opt,name=storageClassName"` // volumeMode defines what type of volume is required by the claim. // Value of Filesystem is implied when not included in claim spec. - // This is an alpha feature and may change in the future. + // This is a beta feature. // +optional VolumeMode *PersistentVolumeMode `json:"volumeMode,omitempty" protobuf:"bytes,6,opt,name=volumeMode,casttype=PersistentVolumeMode"` // This field requires the VolumeSnapshotDataSource alpha feature gate to be @@ -636,6 +636,30 @@ type GlusterfsVolumeSource struct { ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,3,opt,name=readOnly"` } +// Represents a Glusterfs mount that lasts the lifetime of a pod. +// Glusterfs volumes do not support ownership management or SELinux relabeling. +type GlusterfsPersistentVolumeSource struct { + // EndpointsName is the endpoint name that details Glusterfs topology. + // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod + EndpointsName string `json:"endpoints" protobuf:"bytes,1,opt,name=endpoints"` + + // Path is the Glusterfs volume path. + // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod + Path string `json:"path" protobuf:"bytes,2,opt,name=path"` + + // ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. + // Defaults to false. + // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod + // +optional + ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,3,opt,name=readOnly"` + + // EndpointsNamespace is the namespace that contains Glusterfs endpoint. + // If this field is empty, the EndpointNamespace defaults to the same namespace as the bound PVC. + // More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod + // +optional + EndpointsNamespace *string `json:"endpointsNamespace,omitempty" protobuf:"bytes,4,opt,name=endpointsNamespace"` +} + // Represents a Rados Block Device mount that lasts the lifetime of a pod. // RBD volumes support ownership management and SELinux relabeling. type RBDVolumeSource struct { @@ -2090,7 +2114,7 @@ type Container struct { // +patchStrategy=merge VolumeMounts []VolumeMount `json:"volumeMounts,omitempty" patchStrategy:"merge" patchMergeKey:"mountPath" protobuf:"bytes,9,rep,name=volumeMounts"` // volumeDevices is the list of block devices to be used by the container. - // This is an alpha feature and may change in the future. + // This is a beta feature. // +patchMergeKey=devicePath // +patchStrategy=merge // +optional diff --git a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go index ba95e388ba5..71f90685f35 100644 --- a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go @@ -321,7 +321,7 @@ var map_Container = map[string]string{ "env": "List of environment variables to set in the container. Cannot be updated.", "resources": "Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", "volumeMounts": "Pod volumes to mount into the container's filesystem. Cannot be updated.", - "volumeDevices": "volumeDevices is the list of block devices to be used by the container. This is an alpha feature and may change in the future.", + "volumeDevices": "volumeDevices is the list of block devices to be used by the container. This is a beta feature.", "livenessProbe": "Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "readinessProbe": "Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", "lifecycle": "Actions that the management system should take in response to container lifecycle events. Cannot be updated.", @@ -695,6 +695,18 @@ func (GitRepoVolumeSource) SwaggerDoc() map[string]string { return map_GitRepoVolumeSource } +var map_GlusterfsPersistentVolumeSource = map[string]string{ + "": "Represents a Glusterfs mount that lasts the lifetime of a pod. Glusterfs volumes do not support ownership management or SELinux relabeling.", + "endpoints": "EndpointsName is the endpoint name that details Glusterfs topology. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod", + "path": "Path is the Glusterfs volume path. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod", + "readOnly": "ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod", + "endpointsNamespace": "EndpointsNamespace is the namespace that contains Glusterfs endpoint. If this field is empty, the EndpointNamespace defaults to the same namespace as the bound PVC. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod", +} + +func (GlusterfsPersistentVolumeSource) SwaggerDoc() map[string]string { + return map_GlusterfsPersistentVolumeSource +} + var map_GlusterfsVolumeSource = map[string]string{ "": "Represents a Glusterfs mount that lasts the lifetime of a pod. Glusterfs volumes do not support ownership management or SELinux relabeling.", "endpoints": "EndpointsName is the endpoint name that details Glusterfs topology. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod", @@ -1210,7 +1222,7 @@ var map_PersistentVolumeClaimSpec = map[string]string{ "resources": "Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources", "volumeName": "VolumeName is the binding reference to the PersistentVolume backing this claim.", "storageClassName": "Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1", - "volumeMode": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is an alpha feature and may change in the future.", + "volumeMode": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.", "dataSource": "This field requires the VolumeSnapshotDataSource alpha feature gate to be enabled and currently VolumeSnapshot is the only supported data source. If the provisioner can support VolumeSnapshot data source, it will create a new volume and data will be restored to the volume at the same time. If the provisioner does not support VolumeSnapshot data source, volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.", } @@ -1288,7 +1300,7 @@ var map_PersistentVolumeSpec = map[string]string{ "persistentVolumeReclaimPolicy": "What happens to a persistent volume when released from its claim. Valid options are Retain (default for manually created PersistentVolumes), Delete (default for dynamically provisioned PersistentVolumes), and Recycle (deprecated). Recycle must be supported by the volume plugin underlying this PersistentVolume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#reclaiming", "storageClassName": "Name of StorageClass to which this persistent volume belongs. Empty value means that this volume does not belong to any StorageClass.", "mountOptions": "A list of mount options, e.g. [\"ro\", \"soft\"]. Not validated - mount will simply fail if one is invalid. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#mount-options", - "volumeMode": "volumeMode defines if a volume is intended to be used with a formatted filesystem or to remain in raw block state. Value of Filesystem is implied when not included in spec. This is an alpha feature and may change in the future.", + "volumeMode": "volumeMode defines if a volume is intended to be used with a formatted filesystem or to remain in raw block state. Value of Filesystem is implied when not included in spec. This is a beta feature.", "nodeAffinity": "NodeAffinity defines constraints that limit what nodes this volume can be accessed from. This field influences the scheduling of pods that use this volume.", } diff --git a/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go index 42a5ca07518..4219c95eb09 100644 --- a/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go @@ -1498,6 +1498,27 @@ func (in *GitRepoVolumeSource) DeepCopy() *GitRepoVolumeSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlusterfsPersistentVolumeSource) DeepCopyInto(out *GlusterfsPersistentVolumeSource) { + *out = *in + if in.EndpointsNamespace != nil { + in, out := &in.EndpointsNamespace, &out.EndpointsNamespace + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlusterfsPersistentVolumeSource. +func (in *GlusterfsPersistentVolumeSource) DeepCopy() *GlusterfsPersistentVolumeSource { + if in == nil { + return nil + } + out := new(GlusterfsPersistentVolumeSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GlusterfsVolumeSource) DeepCopyInto(out *GlusterfsVolumeSource) { *out = *in @@ -2806,8 +2827,8 @@ func (in *PersistentVolumeSource) DeepCopyInto(out *PersistentVolumeSource) { } if in.Glusterfs != nil { in, out := &in.Glusterfs, &out.Glusterfs - *out = new(GlusterfsVolumeSource) - **out = **in + *out = new(GlusterfsPersistentVolumeSource) + (*in).DeepCopyInto(*out) } if in.NFS != nil { in, out := &in.NFS, &out.NFS diff --git a/staging/src/k8s.io/api/imagepolicy/OWNERS b/staging/src/k8s.io/api/imagepolicy/OWNERS index 96ae42e027a..02b8511e465 100755 --- a/staging/src/k8s.io/api/imagepolicy/OWNERS +++ b/staging/src/k8s.io/api/imagepolicy/OWNERS @@ -1,4 +1,7 @@ +# approval on api packages bubbles to api-approvers reviewers: -- deads2k -- mbohlool -- jianhuiz +- sig-auth-policy-approvers +- sig-auth-policy-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/api/policy/OWNERS b/staging/src/k8s.io/api/policy/OWNERS index 872e16329b0..61a5da4cf3e 100755 --- a/staging/src/k8s.io/api/policy/OWNERS +++ b/staging/src/k8s.io/api/policy/OWNERS @@ -1,7 +1,8 @@ -approvers: -- sig-apps-api-approvers +# approval on api packages bubbles to api-approvers reviewers: - sig-apps-reviewers -- pweil- -- liggitt -- tallclair +- sig-auth-policy-approvers +- sig-auth-policy-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/api/rbac/OWNERS b/staging/src/k8s.io/api/rbac/OWNERS index 45b66b0a8ca..ff4a7f4bf9a 100755 --- a/staging/src/k8s.io/api/rbac/OWNERS +++ b/staging/src/k8s.io/api/rbac/OWNERS @@ -1,16 +1,7 @@ +# approval on api packages bubbles to api-approvers reviewers: -- thockin -- lavalamp -- smarterclayton -- deads2k -- sttts -- ncdc -- dims -- krousey -- mml -- mbohlool -- david-mcmahon -- lixiaobing10051267 -- jianhuiz -- liggitt -- enj +- sig-auth-authorizers-approvers +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/api/storage/v1/generated.pb.go b/staging/src/k8s.io/api/storage/v1/generated.pb.go index 9ad0a015d85..e4b29311b68 100644 --- a/staging/src/k8s.io/api/storage/v1/generated.pb.go +++ b/staging/src/k8s.io/api/storage/v1/generated.pb.go @@ -26,6 +26,12 @@ limitations under the License. It has these top-level messages: StorageClass StorageClassList + VolumeAttachment + VolumeAttachmentList + VolumeAttachmentSource + VolumeAttachmentSpec + VolumeAttachmentStatus + VolumeError */ package v1 @@ -61,9 +67,39 @@ func (m *StorageClassList) Reset() { *m = StorageClassList{} func (*StorageClassList) ProtoMessage() {} func (*StorageClassList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{1} } +func (m *VolumeAttachment) Reset() { *m = VolumeAttachment{} } +func (*VolumeAttachment) ProtoMessage() {} +func (*VolumeAttachment) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{2} } + +func (m *VolumeAttachmentList) Reset() { *m = VolumeAttachmentList{} } +func (*VolumeAttachmentList) ProtoMessage() {} +func (*VolumeAttachmentList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{3} } + +func (m *VolumeAttachmentSource) Reset() { *m = VolumeAttachmentSource{} } +func (*VolumeAttachmentSource) ProtoMessage() {} +func (*VolumeAttachmentSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{4} } + +func (m *VolumeAttachmentSpec) Reset() { *m = VolumeAttachmentSpec{} } +func (*VolumeAttachmentSpec) ProtoMessage() {} +func (*VolumeAttachmentSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{5} } + +func (m *VolumeAttachmentStatus) Reset() { *m = VolumeAttachmentStatus{} } +func (*VolumeAttachmentStatus) ProtoMessage() {} +func (*VolumeAttachmentStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{6} } + +func (m *VolumeError) Reset() { *m = VolumeError{} } +func (*VolumeError) ProtoMessage() {} +func (*VolumeError) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{7} } + func init() { proto.RegisterType((*StorageClass)(nil), "k8s.io.api.storage.v1.StorageClass") proto.RegisterType((*StorageClassList)(nil), "k8s.io.api.storage.v1.StorageClassList") + proto.RegisterType((*VolumeAttachment)(nil), "k8s.io.api.storage.v1.VolumeAttachment") + proto.RegisterType((*VolumeAttachmentList)(nil), "k8s.io.api.storage.v1.VolumeAttachmentList") + proto.RegisterType((*VolumeAttachmentSource)(nil), "k8s.io.api.storage.v1.VolumeAttachmentSource") + proto.RegisterType((*VolumeAttachmentSpec)(nil), "k8s.io.api.storage.v1.VolumeAttachmentSpec") + proto.RegisterType((*VolumeAttachmentStatus)(nil), "k8s.io.api.storage.v1.VolumeAttachmentStatus") + proto.RegisterType((*VolumeError)(nil), "k8s.io.api.storage.v1.VolumeError") } func (m *StorageClass) Marshal() (dAtA []byte, err error) { size := m.Size() @@ -204,6 +240,242 @@ func (m *StorageClassList) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *VolumeAttachment) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeAttachment) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) + n3, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n3 + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) + n4, err := m.Spec.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n4 + dAtA[i] = 0x1a + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.Status.Size())) + n5, err := m.Status.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n5 + return i, nil +} + +func (m *VolumeAttachmentList) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeAttachmentList) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) + n6, err := m.ListMeta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n6 + if len(m.Items) > 0 { + for _, msg := range m.Items { + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *VolumeAttachmentSource) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeAttachmentSource) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.PersistentVolumeName != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(*m.PersistentVolumeName))) + i += copy(dAtA[i:], *m.PersistentVolumeName) + } + return i, nil +} + +func (m *VolumeAttachmentSpec) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeAttachmentSpec) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Attacher))) + i += copy(dAtA[i:], m.Attacher) + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.Source.Size())) + n7, err := m.Source.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n7 + dAtA[i] = 0x1a + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.NodeName))) + i += copy(dAtA[i:], m.NodeName) + return i, nil +} + +func (m *VolumeAttachmentStatus) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeAttachmentStatus) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0x8 + i++ + if m.Attached { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + if len(m.AttachmentMetadata) > 0 { + keysForAttachmentMetadata := make([]string, 0, len(m.AttachmentMetadata)) + for k := range m.AttachmentMetadata { + keysForAttachmentMetadata = append(keysForAttachmentMetadata, string(k)) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForAttachmentMetadata) + for _, k := range keysForAttachmentMetadata { + dAtA[i] = 0x12 + i++ + v := m.AttachmentMetadata[string(k)] + mapSize := 1 + len(k) + sovGenerated(uint64(len(k))) + 1 + len(v) + sovGenerated(uint64(len(v))) + i = encodeVarintGenerated(dAtA, i, uint64(mapSize)) + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(k))) + i += copy(dAtA[i:], k) + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(v))) + i += copy(dAtA[i:], v) + } + } + if m.AttachError != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.AttachError.Size())) + n8, err := m.AttachError.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n8 + } + if m.DetachError != nil { + dAtA[i] = 0x22 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.DetachError.Size())) + n9, err := m.DetachError.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n9 + } + return i, nil +} + +func (m *VolumeError) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeError) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.Time.Size())) + n10, err := m.Time.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n10 + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Message))) + i += copy(dAtA[i:], m.Message) + return i, nil +} + func encodeVarintGenerated(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -268,6 +540,87 @@ func (m *StorageClassList) Size() (n int) { return n } +func (m *VolumeAttachment) Size() (n int) { + var l int + _ = l + l = m.ObjectMeta.Size() + n += 1 + l + sovGenerated(uint64(l)) + l = m.Spec.Size() + n += 1 + l + sovGenerated(uint64(l)) + l = m.Status.Size() + n += 1 + l + sovGenerated(uint64(l)) + return n +} + +func (m *VolumeAttachmentList) Size() (n int) { + var l int + _ = l + l = m.ListMeta.Size() + n += 1 + l + sovGenerated(uint64(l)) + if len(m.Items) > 0 { + for _, e := range m.Items { + l = e.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + } + return n +} + +func (m *VolumeAttachmentSource) Size() (n int) { + var l int + _ = l + if m.PersistentVolumeName != nil { + l = len(*m.PersistentVolumeName) + n += 1 + l + sovGenerated(uint64(l)) + } + return n +} + +func (m *VolumeAttachmentSpec) Size() (n int) { + var l int + _ = l + l = len(m.Attacher) + n += 1 + l + sovGenerated(uint64(l)) + l = m.Source.Size() + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.NodeName) + n += 1 + l + sovGenerated(uint64(l)) + return n +} + +func (m *VolumeAttachmentStatus) Size() (n int) { + var l int + _ = l + n += 2 + if len(m.AttachmentMetadata) > 0 { + for k, v := range m.AttachmentMetadata { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovGenerated(uint64(len(k))) + 1 + len(v) + sovGenerated(uint64(len(v))) + n += mapEntrySize + 1 + sovGenerated(uint64(mapEntrySize)) + } + } + if m.AttachError != nil { + l = m.AttachError.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + if m.DetachError != nil { + l = m.DetachError.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + return n +} + +func (m *VolumeError) Size() (n int) { + var l int + _ = l + l = m.Time.Size() + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.Message) + n += 1 + l + sovGenerated(uint64(l)) + return n +} + func sovGenerated(x uint64) (n int) { for { n++ @@ -319,6 +672,85 @@ func (this *StorageClassList) String() string { }, "") return s } +func (this *VolumeAttachment) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&VolumeAttachment{`, + `ObjectMeta:` + strings.Replace(strings.Replace(this.ObjectMeta.String(), "ObjectMeta", "k8s_io_apimachinery_pkg_apis_meta_v1.ObjectMeta", 1), `&`, ``, 1) + `,`, + `Spec:` + strings.Replace(strings.Replace(this.Spec.String(), "VolumeAttachmentSpec", "VolumeAttachmentSpec", 1), `&`, ``, 1) + `,`, + `Status:` + strings.Replace(strings.Replace(this.Status.String(), "VolumeAttachmentStatus", "VolumeAttachmentStatus", 1), `&`, ``, 1) + `,`, + `}`, + }, "") + return s +} +func (this *VolumeAttachmentList) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&VolumeAttachmentList{`, + `ListMeta:` + strings.Replace(strings.Replace(this.ListMeta.String(), "ListMeta", "k8s_io_apimachinery_pkg_apis_meta_v1.ListMeta", 1), `&`, ``, 1) + `,`, + `Items:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.Items), "VolumeAttachment", "VolumeAttachment", 1), `&`, ``, 1) + `,`, + `}`, + }, "") + return s +} +func (this *VolumeAttachmentSource) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&VolumeAttachmentSource{`, + `PersistentVolumeName:` + valueToStringGenerated(this.PersistentVolumeName) + `,`, + `}`, + }, "") + return s +} +func (this *VolumeAttachmentSpec) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&VolumeAttachmentSpec{`, + `Attacher:` + fmt.Sprintf("%v", this.Attacher) + `,`, + `Source:` + strings.Replace(strings.Replace(this.Source.String(), "VolumeAttachmentSource", "VolumeAttachmentSource", 1), `&`, ``, 1) + `,`, + `NodeName:` + fmt.Sprintf("%v", this.NodeName) + `,`, + `}`, + }, "") + return s +} +func (this *VolumeAttachmentStatus) String() string { + if this == nil { + return "nil" + } + keysForAttachmentMetadata := make([]string, 0, len(this.AttachmentMetadata)) + for k := range this.AttachmentMetadata { + keysForAttachmentMetadata = append(keysForAttachmentMetadata, k) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForAttachmentMetadata) + mapStringForAttachmentMetadata := "map[string]string{" + for _, k := range keysForAttachmentMetadata { + mapStringForAttachmentMetadata += fmt.Sprintf("%v: %v,", k, this.AttachmentMetadata[k]) + } + mapStringForAttachmentMetadata += "}" + s := strings.Join([]string{`&VolumeAttachmentStatus{`, + `Attached:` + fmt.Sprintf("%v", this.Attached) + `,`, + `AttachmentMetadata:` + mapStringForAttachmentMetadata + `,`, + `AttachError:` + strings.Replace(fmt.Sprintf("%v", this.AttachError), "VolumeError", "VolumeError", 1) + `,`, + `DetachError:` + strings.Replace(fmt.Sprintf("%v", this.DetachError), "VolumeError", "VolumeError", 1) + `,`, + `}`, + }, "") + return s +} +func (this *VolumeError) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&VolumeError{`, + `Time:` + strings.Replace(strings.Replace(this.Time.String(), "Time", "k8s_io_apimachinery_pkg_apis_meta_v1.Time", 1), `&`, ``, 1) + `,`, + `Message:` + fmt.Sprintf("%v", this.Message) + `,`, + `}`, + }, "") + return s +} func valueToStringGenerated(v interface{}) string { rv := reflect.ValueOf(v) if rv.IsNil() { @@ -806,6 +1238,838 @@ func (m *StorageClassList) Unmarshal(dAtA []byte) error { } return nil } +func (m *VolumeAttachment) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeAttachment: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeAttachment: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ObjectMeta", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.ObjectMeta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Spec", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Spec.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Status.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VolumeAttachmentList) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeAttachmentList: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeAttachmentList: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ListMeta", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.ListMeta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Items", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Items = append(m.Items, VolumeAttachment{}) + if err := m.Items[len(m.Items)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VolumeAttachmentSource) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeAttachmentSource: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeAttachmentSource: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PersistentVolumeName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(dAtA[iNdEx:postIndex]) + m.PersistentVolumeName = &s + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VolumeAttachmentSpec) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeAttachmentSpec: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeAttachmentSpec: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attacher", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Attacher = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Source", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Source.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NodeName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VolumeAttachmentStatus) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeAttachmentStatus: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeAttachmentStatus: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Attached", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Attached = bool(v != 0) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AttachmentMetadata", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.AttachmentMetadata == nil { + m.AttachmentMetadata = make(map[string]string) + } + var mapkey string + var mapvalue string + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthGenerated + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapvalue := int(stringLenmapvalue) + if intStringLenmapvalue < 0 { + return ErrInvalidLengthGenerated + } + postStringIndexmapvalue := iNdEx + intStringLenmapvalue + if postStringIndexmapvalue > l { + return io.ErrUnexpectedEOF + } + mapvalue = string(dAtA[iNdEx:postStringIndexmapvalue]) + iNdEx = postStringIndexmapvalue + } else { + iNdEx = entryPreIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.AttachmentMetadata[mapkey] = mapvalue + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AttachError", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.AttachError == nil { + m.AttachError = &VolumeError{} + } + if err := m.AttachError.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DetachError", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.DetachError == nil { + m.DetachError = &VolumeError{} + } + if err := m.DetachError.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VolumeError) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeError: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeError: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Time", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Time.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Message = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipGenerated(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -916,46 +2180,67 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 656 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xcd, 0x6e, 0xd3, 0x4a, - 0x14, 0x8e, 0x93, 0x9b, 0xde, 0x74, 0xd2, 0xea, 0x26, 0xbe, 0xbd, 0x92, 0x6f, 0x16, 0x4e, 0x54, - 0x36, 0x11, 0x12, 0xe3, 0xa6, 0x14, 0x54, 0x21, 0x81, 0x54, 0xa3, 0x4a, 0x20, 0xb5, 0x6a, 0xe4, - 0x56, 0x15, 0x42, 0x2c, 0x98, 0x38, 0x07, 0x77, 0x88, 0xed, 0x31, 0x33, 0x63, 0x43, 0x76, 0xbc, - 0x00, 0x12, 0xcf, 0xc3, 0x13, 0x74, 0xd9, 0x65, 0x57, 0x16, 0x35, 0x6f, 0xd1, 0x15, 0xf2, 0x0f, - 0x8d, 0x9b, 0x04, 0xd1, 0xdd, 0xcc, 0x77, 0xbe, 0xef, 0x3b, 0x33, 0xe7, 0x07, 0x3d, 0x9b, 0xec, - 0x0a, 0x4c, 0x99, 0x31, 0x09, 0x47, 0xc0, 0x7d, 0x90, 0x20, 0x8c, 0x08, 0xfc, 0x31, 0xe3, 0x46, - 0x11, 0x20, 0x01, 0x35, 0x84, 0x64, 0x9c, 0x38, 0x60, 0x44, 0x03, 0xc3, 0x01, 0x1f, 0x38, 0x91, - 0x30, 0xc6, 0x01, 0x67, 0x92, 0xa9, 0xff, 0xe5, 0x34, 0x4c, 0x02, 0x8a, 0x0b, 0x1a, 0x8e, 0x06, - 0x9d, 0x07, 0x0e, 0x95, 0x67, 0xe1, 0x08, 0xdb, 0xcc, 0x33, 0x1c, 0xe6, 0x30, 0x23, 0x63, 0x8f, - 0xc2, 0x77, 0xd9, 0x2d, 0xbb, 0x64, 0xa7, 0xdc, 0xa5, 0xb3, 0x59, 0x4a, 0x66, 0x33, 0xbe, 0x2c, - 0x53, 0x67, 0x67, 0xc6, 0xf1, 0x88, 0x7d, 0x46, 0x7d, 0xe0, 0x53, 0x23, 0x98, 0x38, 0x29, 0x20, - 0x0c, 0x0f, 0x24, 0x59, 0xa6, 0x32, 0x7e, 0xa7, 0xe2, 0xa1, 0x2f, 0xa9, 0x07, 0x0b, 0x82, 0xc7, - 0x7f, 0x12, 0x08, 0xfb, 0x0c, 0x3c, 0x32, 0xaf, 0xdb, 0xfc, 0xb2, 0x82, 0xd6, 0x8e, 0xf3, 0x02, - 0x3c, 0x77, 0x89, 0x10, 0xea, 0x5b, 0xd4, 0x48, 0x1f, 0x35, 0x26, 0x92, 0x68, 0x4a, 0x4f, 0xe9, - 0x37, 0xb7, 0xb7, 0xf0, 0xac, 0x58, 0x37, 0xde, 0x38, 0x98, 0x38, 0x29, 0x20, 0x70, 0xca, 0xc6, - 0xd1, 0x00, 0x1f, 0x8d, 0xde, 0x83, 0x2d, 0x0f, 0x41, 0x12, 0x53, 0x3d, 0x8f, 0xbb, 0x95, 0x24, - 0xee, 0xa2, 0x19, 0x66, 0xdd, 0xb8, 0xaa, 0x8f, 0x50, 0x33, 0xe0, 0x2c, 0xa2, 0x82, 0x32, 0x1f, - 0xb8, 0x56, 0xed, 0x29, 0xfd, 0x55, 0xf3, 0xdf, 0x42, 0xd2, 0x1c, 0xce, 0x42, 0x56, 0x99, 0xa7, - 0x3a, 0x08, 0x05, 0x84, 0x13, 0x0f, 0x24, 0x70, 0xa1, 0xd5, 0x7a, 0xb5, 0x7e, 0x73, 0xfb, 0x21, - 0x5e, 0xda, 0x47, 0x5c, 0xfe, 0x11, 0x1e, 0xde, 0xa8, 0xf6, 0x7d, 0xc9, 0xa7, 0xb3, 0xd7, 0xcd, - 0x02, 0x56, 0xc9, 0x5a, 0x9d, 0xa0, 0x75, 0x0e, 0xb6, 0x4b, 0xa8, 0x37, 0x64, 0x2e, 0xb5, 0xa7, - 0xda, 0x5f, 0xd9, 0x0b, 0xf7, 0x93, 0xb8, 0xbb, 0x6e, 0x95, 0x03, 0xd7, 0x71, 0x77, 0x6b, 0x71, - 0x02, 0xf0, 0x10, 0xb8, 0xa0, 0x42, 0x82, 0x2f, 0x4f, 0x99, 0x1b, 0x7a, 0x70, 0x4b, 0x63, 0xdd, - 0xf6, 0x56, 0x77, 0xd0, 0x9a, 0xc7, 0x42, 0x5f, 0x1e, 0x05, 0x92, 0x32, 0x5f, 0x68, 0xf5, 0x5e, - 0xad, 0xbf, 0x6a, 0xb6, 0x92, 0xb8, 0xbb, 0x76, 0x58, 0xc2, 0xad, 0x5b, 0x2c, 0xf5, 0x00, 0x6d, - 0x10, 0xd7, 0x65, 0x1f, 0xf3, 0x04, 0xfb, 0x9f, 0x02, 0xe2, 0xa7, 0x55, 0xd2, 0x56, 0x7a, 0x4a, - 0xbf, 0x61, 0x6a, 0x49, 0xdc, 0xdd, 0xd8, 0x5b, 0x12, 0xb7, 0x96, 0xaa, 0xd4, 0x57, 0xa8, 0x1d, - 0x65, 0x90, 0x49, 0xfd, 0x31, 0xf5, 0x9d, 0x43, 0x36, 0x06, 0xed, 0xef, 0xec, 0xd3, 0xf7, 0x93, - 0xb8, 0xdb, 0x3e, 0x9d, 0x0f, 0x5e, 0x2f, 0x03, 0xad, 0x45, 0x13, 0xf5, 0x03, 0x6a, 0x67, 0x19, - 0x61, 0x7c, 0xc2, 0x02, 0xe6, 0x32, 0x87, 0x82, 0xd0, 0x1a, 0x59, 0xeb, 0xfa, 0xe5, 0xd6, 0xa5, - 0xa5, 0x4b, 0xfb, 0x56, 0xb0, 0xa6, 0xc7, 0xe0, 0x82, 0x2d, 0x19, 0x3f, 0x01, 0xee, 0x99, 0xff, - 0x17, 0xfd, 0x6a, 0xef, 0xcd, 0x5b, 0x59, 0x8b, 0xee, 0x9d, 0xa7, 0xe8, 0x9f, 0xb9, 0x86, 0xab, - 0x2d, 0x54, 0x9b, 0xc0, 0x34, 0x9b, 0xe6, 0x55, 0x2b, 0x3d, 0xaa, 0x1b, 0xa8, 0x1e, 0x11, 0x37, - 0x84, 0x7c, 0xf8, 0xac, 0xfc, 0xf2, 0xa4, 0xba, 0xab, 0x6c, 0x7e, 0x53, 0x50, 0xab, 0x3c, 0x3d, - 0x07, 0x54, 0x48, 0xf5, 0xcd, 0xc2, 0x4e, 0xe0, 0xbb, 0xed, 0x44, 0xaa, 0xce, 0x36, 0xa2, 0x55, - 0xfc, 0xa1, 0xf1, 0x0b, 0x29, 0xed, 0xc3, 0x0b, 0x54, 0xa7, 0x12, 0x3c, 0xa1, 0x55, 0xb3, 0xc2, - 0xdc, 0xbb, 0xc3, 0x4c, 0x9b, 0xeb, 0x85, 0x5f, 0xfd, 0x65, 0xaa, 0xb4, 0x72, 0x03, 0xb3, 0x7f, - 0x7e, 0xa5, 0x57, 0x2e, 0xae, 0xf4, 0xca, 0xe5, 0x95, 0x5e, 0xf9, 0x9c, 0xe8, 0xca, 0x79, 0xa2, - 0x2b, 0x17, 0x89, 0xae, 0x5c, 0x26, 0xba, 0xf2, 0x3d, 0xd1, 0x95, 0xaf, 0x3f, 0xf4, 0xca, 0xeb, - 0x6a, 0x34, 0xf8, 0x19, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x64, 0x41, 0x83, 0x40, 0x05, 0x00, 0x00, + // 984 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x56, 0x3d, 0x6f, 0x23, 0x45, + 0x18, 0xce, 0xc6, 0xf9, 0x70, 0xc6, 0x09, 0x97, 0x0c, 0x01, 0x8c, 0x0b, 0x3b, 0x32, 0x05, 0xe6, + 0xe0, 0x76, 0x2f, 0xe1, 0x40, 0x27, 0x24, 0x90, 0xbc, 0x60, 0x09, 0xa4, 0xf8, 0x2e, 0x9a, 0x84, + 0x13, 0x42, 0x14, 0x4c, 0x76, 0xdf, 0xdb, 0x2c, 0xf6, 0xee, 0x2c, 0x33, 0x63, 0x43, 0x3a, 0x2a, + 0x3a, 0x24, 0x68, 0xf9, 0x29, 0x94, 0x54, 0xa1, 0xbb, 0xf2, 0x2a, 0x8b, 0x2c, 0x35, 0x7f, 0x20, + 0x15, 0x9a, 0xd9, 0x89, 0xbd, 0xb1, 0xd7, 0x9c, 0xd3, 0x5c, 0xe7, 0xf7, 0xe3, 0x79, 0xde, 0xef, + 0x59, 0xa3, 0x4f, 0x7a, 0x0f, 0x85, 0x1d, 0x32, 0xa7, 0x37, 0x38, 0x05, 0x1e, 0x83, 0x04, 0xe1, + 0x0c, 0x21, 0xf6, 0x19, 0x77, 0x8c, 0x81, 0x26, 0xa1, 0x23, 0x24, 0xe3, 0x34, 0x00, 0x67, 0xb8, + 0xef, 0x04, 0x10, 0x03, 0xa7, 0x12, 0x7c, 0x3b, 0xe1, 0x4c, 0x32, 0xfc, 0x5a, 0xe6, 0x66, 0xd3, + 0x24, 0xb4, 0x8d, 0x9b, 0x3d, 0xdc, 0xaf, 0xdd, 0x0b, 0x42, 0x79, 0x36, 0x38, 0xb5, 0x3d, 0x16, + 0x39, 0x01, 0x0b, 0x98, 0xa3, 0xbd, 0x4f, 0x07, 0x4f, 0xb5, 0xa4, 0x05, 0xfd, 0x2b, 0x63, 0xa9, + 0x35, 0x73, 0xc1, 0x3c, 0xc6, 0x8b, 0x22, 0xd5, 0x1e, 0x4c, 0x7c, 0x22, 0xea, 0x9d, 0x85, 0x31, + 0xf0, 0x73, 0x27, 0xe9, 0x05, 0x4a, 0x21, 0x9c, 0x08, 0x24, 0x2d, 0x42, 0x39, 0xf3, 0x50, 0x7c, + 0x10, 0xcb, 0x30, 0x82, 0x19, 0xc0, 0x87, 0x2f, 0x02, 0x08, 0xef, 0x0c, 0x22, 0x3a, 0x8d, 0x6b, + 0xfe, 0xb2, 0x86, 0x36, 0x8f, 0xb3, 0x06, 0x7c, 0xda, 0xa7, 0x42, 0xe0, 0x6f, 0x51, 0x59, 0x25, + 0xe5, 0x53, 0x49, 0xab, 0xd6, 0x9e, 0xd5, 0xaa, 0x1c, 0xdc, 0xb7, 0x27, 0xcd, 0x1a, 0x73, 0xdb, + 0x49, 0x2f, 0x50, 0x0a, 0x61, 0x2b, 0x6f, 0x7b, 0xb8, 0x6f, 0x3f, 0x3e, 0xfd, 0x0e, 0x3c, 0xd9, + 0x05, 0x49, 0x5d, 0x7c, 0x31, 0x6a, 0x2c, 0xa5, 0xa3, 0x06, 0x9a, 0xe8, 0xc8, 0x98, 0x15, 0x7f, + 0x80, 0x2a, 0x09, 0x67, 0xc3, 0x50, 0x84, 0x2c, 0x06, 0x5e, 0x5d, 0xde, 0xb3, 0x5a, 0x1b, 0xee, + 0xab, 0x06, 0x52, 0x39, 0x9a, 0x98, 0x48, 0xde, 0x0f, 0x07, 0x08, 0x25, 0x94, 0xd3, 0x08, 0x24, + 0x70, 0x51, 0x2d, 0xed, 0x95, 0x5a, 0x95, 0x83, 0xf7, 0xed, 0xc2, 0x39, 0xda, 0xf9, 0x8a, 0xec, + 0xa3, 0x31, 0xaa, 0x13, 0x4b, 0x7e, 0x3e, 0xc9, 0x6e, 0x62, 0x20, 0x39, 0x6a, 0xdc, 0x43, 0x5b, + 0x1c, 0xbc, 0x3e, 0x0d, 0xa3, 0x23, 0xd6, 0x0f, 0xbd, 0xf3, 0xea, 0x8a, 0xce, 0xb0, 0x93, 0x8e, + 0x1a, 0x5b, 0x24, 0x6f, 0xb8, 0x1a, 0x35, 0xee, 0xcf, 0x6e, 0x80, 0x7d, 0x04, 0x5c, 0x84, 0x42, + 0x42, 0x2c, 0x9f, 0xb0, 0xfe, 0x20, 0x82, 0x1b, 0x18, 0x72, 0x93, 0x1b, 0x3f, 0x40, 0x9b, 0x11, + 0x1b, 0xc4, 0xf2, 0x71, 0x22, 0x43, 0x16, 0x8b, 0xea, 0xea, 0x5e, 0xa9, 0xb5, 0xe1, 0x6e, 0xa7, + 0xa3, 0xc6, 0x66, 0x37, 0xa7, 0x27, 0x37, 0xbc, 0xf0, 0x21, 0xda, 0xa5, 0xfd, 0x3e, 0xfb, 0x21, + 0x0b, 0xd0, 0xf9, 0x31, 0xa1, 0xb1, 0xea, 0x52, 0x75, 0x6d, 0xcf, 0x6a, 0x95, 0xdd, 0x6a, 0x3a, + 0x6a, 0xec, 0xb6, 0x0b, 0xec, 0xa4, 0x10, 0x85, 0xbf, 0x42, 0x3b, 0x43, 0xad, 0x72, 0xc3, 0xd8, + 0x0f, 0xe3, 0xa0, 0xcb, 0x7c, 0xa8, 0xae, 0xeb, 0xa2, 0xef, 0xa6, 0xa3, 0xc6, 0xce, 0x93, 0x69, + 0xe3, 0x55, 0x91, 0x92, 0xcc, 0x92, 0xe0, 0xef, 0xd1, 0x8e, 0x8e, 0x08, 0xfe, 0x09, 0x4b, 0x58, + 0x9f, 0x05, 0x21, 0x88, 0x6a, 0x59, 0x8f, 0xae, 0x95, 0x1f, 0x9d, 0x6a, 0x9d, 0x9a, 0x9b, 0xf1, + 0x3a, 0x3f, 0x86, 0x3e, 0x78, 0x92, 0xf1, 0x13, 0xe0, 0x91, 0xfb, 0xa6, 0x99, 0xd7, 0x4e, 0x7b, + 0x9a, 0x8a, 0xcc, 0xb2, 0xd7, 0x3e, 0x46, 0x77, 0xa6, 0x06, 0x8e, 0xb7, 0x51, 0xa9, 0x07, 0xe7, + 0x7a, 0x9b, 0x37, 0x88, 0xfa, 0x89, 0x77, 0xd1, 0xea, 0x90, 0xf6, 0x07, 0x90, 0x2d, 0x1f, 0xc9, + 0x84, 0x8f, 0x96, 0x1f, 0x5a, 0xcd, 0x3f, 0x2c, 0xb4, 0x9d, 0xdf, 0x9e, 0xc3, 0x50, 0x48, 0xfc, + 0xcd, 0xcc, 0x4d, 0xd8, 0x8b, 0xdd, 0x84, 0x42, 0xeb, 0x8b, 0xd8, 0x36, 0x35, 0x94, 0xaf, 0x35, + 0xb9, 0x7b, 0xf8, 0x1c, 0xad, 0x86, 0x12, 0x22, 0x51, 0x5d, 0xd6, 0x8d, 0x79, 0x6b, 0x81, 0x9d, + 0x76, 0xb7, 0x0c, 0xdf, 0xea, 0x17, 0x0a, 0x49, 0x32, 0x82, 0xe6, 0xef, 0xcb, 0x68, 0x3b, 0x9b, + 0x4b, 0x5b, 0x4a, 0xea, 0x9d, 0x45, 0x10, 0xcb, 0x97, 0x70, 0xd0, 0x5d, 0xb4, 0x22, 0x12, 0xf0, + 0x74, 0x33, 0x2b, 0x07, 0xef, 0xce, 0xc9, 0x7f, 0x3a, 0xb1, 0xe3, 0x04, 0x3c, 0x77, 0xd3, 0x10, + 0xaf, 0x28, 0x89, 0x68, 0x1a, 0xfc, 0x25, 0x5a, 0x13, 0x92, 0xca, 0x81, 0x3a, 0x72, 0x45, 0x78, + 0x6f, 0x51, 0x42, 0x0d, 0x72, 0x5f, 0x31, 0x94, 0x6b, 0x99, 0x4c, 0x0c, 0x59, 0xf3, 0x4f, 0x0b, + 0xed, 0x4e, 0x43, 0x5e, 0xc2, 0x74, 0x0f, 0x6f, 0x4e, 0xf7, 0xed, 0x05, 0x8b, 0x99, 0x33, 0xe1, + 0xa7, 0xe8, 0xf5, 0x99, 0xb2, 0xd9, 0x80, 0x7b, 0xa0, 0x9e, 0x84, 0x64, 0xea, 0xe1, 0x79, 0x44, + 0x23, 0xc8, 0xb6, 0x3e, 0x7b, 0x12, 0x8e, 0x0a, 0xec, 0xa4, 0x10, 0xd5, 0xfc, 0xab, 0xa0, 0x59, + 0x6a, 0x44, 0xf8, 0x3d, 0x54, 0xa6, 0x5a, 0x03, 0xdc, 0x50, 0x8f, 0x8b, 0x6f, 0x1b, 0x3d, 0x19, + 0x7b, 0xe8, 0x51, 0xea, 0xf4, 0xcc, 0x6e, 0x2c, 0x3c, 0x4a, 0x0d, 0xca, 0x8d, 0x52, 0xcb, 0xc4, + 0x90, 0xa9, 0x24, 0x62, 0xe6, 0x67, 0xf5, 0x95, 0x6e, 0x26, 0xf1, 0xc8, 0xe8, 0xc9, 0xd8, 0xa3, + 0xf9, 0x6f, 0xa9, 0xa0, 0x69, 0x7a, 0x27, 0x72, 0xd5, 0xf8, 0xba, 0x9a, 0xf2, 0x4c, 0x35, 0xfe, + 0xb8, 0x1a, 0x1f, 0xff, 0x66, 0x21, 0x4c, 0xc7, 0x14, 0xdd, 0xeb, 0x9d, 0xc9, 0x06, 0xdb, 0xb9, + 0xd5, 0x96, 0xda, 0xed, 0x19, 0x9e, 0xec, 0xe3, 0x54, 0x33, 0xf1, 0xf1, 0xac, 0x03, 0x29, 0x08, + 0x8e, 0x7d, 0x54, 0xc9, 0xb4, 0x1d, 0xce, 0x19, 0x37, 0x17, 0xd3, 0xfc, 0xdf, 0x5c, 0xb4, 0xa7, + 0x5b, 0x57, 0x1f, 0xdb, 0xf6, 0x04, 0x7a, 0x35, 0x6a, 0x54, 0x72, 0x76, 0x92, 0xa7, 0x55, 0x51, + 0x7c, 0x98, 0x44, 0x59, 0xb9, 0x5d, 0x94, 0xcf, 0x60, 0x7e, 0x94, 0x1c, 0x6d, 0xad, 0x83, 0xde, + 0x98, 0xd3, 0x96, 0x5b, 0x3d, 0xe1, 0x3f, 0x5b, 0x28, 0x1f, 0x03, 0x1f, 0xa2, 0x15, 0xf5, 0x0f, + 0xc8, 0xdc, 0xf6, 0xdd, 0xc5, 0x6e, 0xfb, 0x24, 0x8c, 0x60, 0xf2, 0x3a, 0x29, 0x89, 0x68, 0x16, + 0xfc, 0x0e, 0x5a, 0x8f, 0x40, 0x08, 0x1a, 0x98, 0xc8, 0xee, 0x1d, 0xe3, 0xb4, 0xde, 0xcd, 0xd4, + 0xe4, 0xda, 0xee, 0xb6, 0x2e, 0x2e, 0xeb, 0x4b, 0xcf, 0x2e, 0xeb, 0x4b, 0xcf, 0x2f, 0xeb, 0x4b, + 0x3f, 0xa5, 0x75, 0xeb, 0x22, 0xad, 0x5b, 0xcf, 0xd2, 0xba, 0xf5, 0x3c, 0xad, 0x5b, 0x7f, 0xa7, + 0x75, 0xeb, 0xd7, 0x7f, 0xea, 0x4b, 0x5f, 0x2f, 0x0f, 0xf7, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, + 0x85, 0x2a, 0x88, 0xc0, 0xcf, 0x0a, 0x00, 0x00, } diff --git a/staging/src/k8s.io/api/storage/v1/generated.proto b/staging/src/k8s.io/api/storage/v1/generated.proto index 1b8102c0539..668c8544743 100644 --- a/staging/src/k8s.io/api/storage/v1/generated.proto +++ b/staging/src/k8s.io/api/storage/v1/generated.proto @@ -88,3 +88,99 @@ message StorageClassList { repeated StorageClass items = 2; } +// VolumeAttachment captures the intent to attach or detach the specified volume +// to/from the specified node. +// +// VolumeAttachment objects are non-namespaced. +message VolumeAttachment { + // Standard object metadata. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + optional k8s.io.apimachinery.pkg.apis.meta.v1.ObjectMeta metadata = 1; + + // Specification of the desired attach/detach volume behavior. + // Populated by the Kubernetes system. + optional VolumeAttachmentSpec spec = 2; + + // Status of the VolumeAttachment request. + // Populated by the entity completing the attach or detach + // operation, i.e. the external-attacher. + // +optional + optional VolumeAttachmentStatus status = 3; +} + +// VolumeAttachmentList is a collection of VolumeAttachment objects. +message VolumeAttachmentList { + // Standard list metadata + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + optional k8s.io.apimachinery.pkg.apis.meta.v1.ListMeta metadata = 1; + + // Items is the list of VolumeAttachments + repeated VolumeAttachment items = 2; +} + +// VolumeAttachmentSource represents a volume that should be attached. +// Right now only PersistenVolumes can be attached via external attacher, +// in future we may allow also inline volumes in pods. +// Exactly one member can be set. +message VolumeAttachmentSource { + // Name of the persistent volume to attach. + // +optional + optional string persistentVolumeName = 1; +} + +// VolumeAttachmentSpec is the specification of a VolumeAttachment request. +message VolumeAttachmentSpec { + // Attacher indicates the name of the volume driver that MUST handle this + // request. This is the name returned by GetPluginName(). + optional string attacher = 1; + + // Source represents the volume that should be attached. + optional VolumeAttachmentSource source = 2; + + // The node that the volume should be attached to. + optional string nodeName = 3; +} + +// VolumeAttachmentStatus is the status of a VolumeAttachment request. +message VolumeAttachmentStatus { + // Indicates the volume is successfully attached. + // This field must only be set by the entity completing the attach + // operation, i.e. the external-attacher. + optional bool attached = 1; + + // Upon successful attach, this field is populated with any + // information returned by the attach operation that must be passed + // into subsequent WaitForAttach or Mount calls. + // This field must only be set by the entity completing the attach + // operation, i.e. the external-attacher. + // +optional + map attachmentMetadata = 2; + + // The last error encountered during attach operation, if any. + // This field must only be set by the entity completing the attach + // operation, i.e. the external-attacher. + // +optional + optional VolumeError attachError = 3; + + // The last error encountered during detach operation, if any. + // This field must only be set by the entity completing the detach + // operation, i.e. the external-attacher. + // +optional + optional VolumeError detachError = 4; +} + +// VolumeError captures an error encountered during a volume operation. +message VolumeError { + // Time the error was encountered. + // +optional + optional k8s.io.apimachinery.pkg.apis.meta.v1.Time time = 1; + + // String detailing the error encountered during Attach or Detach operation. + // This string maybe logged, so it should not contain sensitive + // information. + // +optional + optional string message = 2; +} + diff --git a/staging/src/k8s.io/api/storage/v1/register.go b/staging/src/k8s.io/api/storage/v1/register.go index c058add8400..473c687278b 100644 --- a/staging/src/k8s.io/api/storage/v1/register.go +++ b/staging/src/k8s.io/api/storage/v1/register.go @@ -46,6 +46,9 @@ func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &StorageClass{}, &StorageClassList{}, + + &VolumeAttachment{}, + &VolumeAttachmentList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) diff --git a/staging/src/k8s.io/api/storage/v1/types.go b/staging/src/k8s.io/api/storage/v1/types.go index 30e6d6d29bc..9f2f67b6b76 100644 --- a/staging/src/k8s.io/api/storage/v1/types.go +++ b/staging/src/k8s.io/api/storage/v1/types.go @@ -102,3 +102,110 @@ const ( // binding will occur during Pod scheduing. VolumeBindingWaitForFirstConsumer VolumeBindingMode = "WaitForFirstConsumer" ) + +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// VolumeAttachment captures the intent to attach or detach the specified volume +// to/from the specified node. +// +// VolumeAttachment objects are non-namespaced. +type VolumeAttachment struct { + metav1.TypeMeta `json:",inline"` + + // Standard object metadata. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // Specification of the desired attach/detach volume behavior. + // Populated by the Kubernetes system. + Spec VolumeAttachmentSpec `json:"spec" protobuf:"bytes,2,opt,name=spec"` + + // Status of the VolumeAttachment request. + // Populated by the entity completing the attach or detach + // operation, i.e. the external-attacher. + // +optional + Status VolumeAttachmentStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// VolumeAttachmentList is a collection of VolumeAttachment objects. +type VolumeAttachmentList struct { + metav1.TypeMeta `json:",inline"` + // Standard list metadata + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // Items is the list of VolumeAttachments + Items []VolumeAttachment `json:"items" protobuf:"bytes,2,rep,name=items"` +} + +// VolumeAttachmentSpec is the specification of a VolumeAttachment request. +type VolumeAttachmentSpec struct { + // Attacher indicates the name of the volume driver that MUST handle this + // request. This is the name returned by GetPluginName(). + Attacher string `json:"attacher" protobuf:"bytes,1,opt,name=attacher"` + + // Source represents the volume that should be attached. + Source VolumeAttachmentSource `json:"source" protobuf:"bytes,2,opt,name=source"` + + // The node that the volume should be attached to. + NodeName string `json:"nodeName" protobuf:"bytes,3,opt,name=nodeName"` +} + +// VolumeAttachmentSource represents a volume that should be attached. +// Right now only PersistenVolumes can be attached via external attacher, +// in future we may allow also inline volumes in pods. +// Exactly one member can be set. +type VolumeAttachmentSource struct { + // Name of the persistent volume to attach. + // +optional + PersistentVolumeName *string `json:"persistentVolumeName,omitempty" protobuf:"bytes,1,opt,name=persistentVolumeName"` + + // Placeholder for *VolumeSource to accommodate inline volumes in pods. +} + +// VolumeAttachmentStatus is the status of a VolumeAttachment request. +type VolumeAttachmentStatus struct { + // Indicates the volume is successfully attached. + // This field must only be set by the entity completing the attach + // operation, i.e. the external-attacher. + Attached bool `json:"attached" protobuf:"varint,1,opt,name=attached"` + + // Upon successful attach, this field is populated with any + // information returned by the attach operation that must be passed + // into subsequent WaitForAttach or Mount calls. + // This field must only be set by the entity completing the attach + // operation, i.e. the external-attacher. + // +optional + AttachmentMetadata map[string]string `json:"attachmentMetadata,omitempty" protobuf:"bytes,2,rep,name=attachmentMetadata"` + + // The last error encountered during attach operation, if any. + // This field must only be set by the entity completing the attach + // operation, i.e. the external-attacher. + // +optional + AttachError *VolumeError `json:"attachError,omitempty" protobuf:"bytes,3,opt,name=attachError,casttype=VolumeError"` + + // The last error encountered during detach operation, if any. + // This field must only be set by the entity completing the detach + // operation, i.e. the external-attacher. + // +optional + DetachError *VolumeError `json:"detachError,omitempty" protobuf:"bytes,4,opt,name=detachError,casttype=VolumeError"` +} + +// VolumeError captures an error encountered during a volume operation. +type VolumeError struct { + // Time the error was encountered. + // +optional + Time metav1.Time `json:"time,omitempty" protobuf:"bytes,1,opt,name=time"` + + // String detailing the error encountered during Attach or Detach operation. + // This string maybe logged, so it should not contain sensitive + // information. + // +optional + Message string `json:"message,omitempty" protobuf:"bytes,2,opt,name=message"` +} diff --git a/staging/src/k8s.io/api/storage/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/storage/v1/types_swagger_doc_generated.go index 23b76e28de9..d4a022d52ec 100644 --- a/staging/src/k8s.io/api/storage/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/storage/v1/types_swagger_doc_generated.go @@ -53,4 +53,67 @@ func (StorageClassList) SwaggerDoc() map[string]string { return map_StorageClassList } +var map_VolumeAttachment = map[string]string{ + "": "VolumeAttachment captures the intent to attach or detach the specified volume to/from the specified node.\n\nVolumeAttachment objects are non-namespaced.", + "metadata": "Standard object metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata", + "spec": "Specification of the desired attach/detach volume behavior. Populated by the Kubernetes system.", + "status": "Status of the VolumeAttachment request. Populated by the entity completing the attach or detach operation, i.e. the external-attacher.", +} + +func (VolumeAttachment) SwaggerDoc() map[string]string { + return map_VolumeAttachment +} + +var map_VolumeAttachmentList = map[string]string{ + "": "VolumeAttachmentList is a collection of VolumeAttachment objects.", + "metadata": "Standard list metadata More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata", + "items": "Items is the list of VolumeAttachments", +} + +func (VolumeAttachmentList) SwaggerDoc() map[string]string { + return map_VolumeAttachmentList +} + +var map_VolumeAttachmentSource = map[string]string{ + "": "VolumeAttachmentSource represents a volume that should be attached. Right now only PersistenVolumes can be attached via external attacher, in future we may allow also inline volumes in pods. Exactly one member can be set.", + "persistentVolumeName": "Name of the persistent volume to attach.", +} + +func (VolumeAttachmentSource) SwaggerDoc() map[string]string { + return map_VolumeAttachmentSource +} + +var map_VolumeAttachmentSpec = map[string]string{ + "": "VolumeAttachmentSpec is the specification of a VolumeAttachment request.", + "attacher": "Attacher indicates the name of the volume driver that MUST handle this request. This is the name returned by GetPluginName().", + "source": "Source represents the volume that should be attached.", + "nodeName": "The node that the volume should be attached to.", +} + +func (VolumeAttachmentSpec) SwaggerDoc() map[string]string { + return map_VolumeAttachmentSpec +} + +var map_VolumeAttachmentStatus = map[string]string{ + "": "VolumeAttachmentStatus is the status of a VolumeAttachment request.", + "attached": "Indicates the volume is successfully attached. This field must only be set by the entity completing the attach operation, i.e. the external-attacher.", + "attachmentMetadata": "Upon successful attach, this field is populated with any information returned by the attach operation that must be passed into subsequent WaitForAttach or Mount calls. This field must only be set by the entity completing the attach operation, i.e. the external-attacher.", + "attachError": "The last error encountered during attach operation, if any. This field must only be set by the entity completing the attach operation, i.e. the external-attacher.", + "detachError": "The last error encountered during detach operation, if any. This field must only be set by the entity completing the detach operation, i.e. the external-attacher.", +} + +func (VolumeAttachmentStatus) SwaggerDoc() map[string]string { + return map_VolumeAttachmentStatus +} + +var map_VolumeError = map[string]string{ + "": "VolumeError captures an error encountered during a volume operation.", + "time": "Time the error was encountered.", + "message": "String detailing the error encountered during Attach or Detach operation. This string maybe logged, so it should not contain sensitive information.", +} + +func (VolumeError) SwaggerDoc() map[string]string { + return map_VolumeError +} + // AUTO-GENERATED FUNCTIONS END HERE diff --git a/staging/src/k8s.io/api/storage/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/storage/v1/zz_generated.deepcopy.go index 0e850dc34f0..3157ec67812 100644 --- a/staging/src/k8s.io/api/storage/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/storage/v1/zz_generated.deepcopy.go @@ -117,3 +117,152 @@ func (in *StorageClassList) DeepCopyObject() runtime.Object { } return nil } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeAttachment) DeepCopyInto(out *VolumeAttachment) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeAttachment. +func (in *VolumeAttachment) DeepCopy() *VolumeAttachment { + if in == nil { + return nil + } + out := new(VolumeAttachment) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VolumeAttachment) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeAttachmentList) DeepCopyInto(out *VolumeAttachmentList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VolumeAttachment, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeAttachmentList. +func (in *VolumeAttachmentList) DeepCopy() *VolumeAttachmentList { + if in == nil { + return nil + } + out := new(VolumeAttachmentList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VolumeAttachmentList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeAttachmentSource) DeepCopyInto(out *VolumeAttachmentSource) { + *out = *in + if in.PersistentVolumeName != nil { + in, out := &in.PersistentVolumeName, &out.PersistentVolumeName + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeAttachmentSource. +func (in *VolumeAttachmentSource) DeepCopy() *VolumeAttachmentSource { + if in == nil { + return nil + } + out := new(VolumeAttachmentSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeAttachmentSpec) DeepCopyInto(out *VolumeAttachmentSpec) { + *out = *in + in.Source.DeepCopyInto(&out.Source) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeAttachmentSpec. +func (in *VolumeAttachmentSpec) DeepCopy() *VolumeAttachmentSpec { + if in == nil { + return nil + } + out := new(VolumeAttachmentSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeAttachmentStatus) DeepCopyInto(out *VolumeAttachmentStatus) { + *out = *in + if in.AttachmentMetadata != nil { + in, out := &in.AttachmentMetadata, &out.AttachmentMetadata + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.AttachError != nil { + in, out := &in.AttachError, &out.AttachError + *out = new(VolumeError) + (*in).DeepCopyInto(*out) + } + if in.DetachError != nil { + in, out := &in.DetachError, &out.DetachError + *out = new(VolumeError) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeAttachmentStatus. +func (in *VolumeAttachmentStatus) DeepCopy() *VolumeAttachmentStatus { + if in == nil { + return nil + } + out := new(VolumeAttachmentStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeError) DeepCopyInto(out *VolumeError) { + *out = *in + in.Time.DeepCopyInto(&out.Time) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeError. +func (in *VolumeError) DeepCopy() *VolumeError { + if in == nil { + return nil + } + out := new(VolumeError) + in.DeepCopyInto(out) + return out +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/BUILD b/staging/src/k8s.io/apiextensions-apiserver/BUILD index dee7cb342be..e555fb761c0 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/BUILD @@ -20,7 +20,7 @@ go_library( "//staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json b/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json index d16135f56de..df164ef7363 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json +++ b/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/apiextensions-apiserver", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -458,10 +458,6 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/golang/groupcache/lru", "Rev": "02826c3e79038b59d737d3b1c0a1d937f71a4433" @@ -724,35 +720,35 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/internal/timeseries", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/trace", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/websocket", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/oauth2", @@ -824,79 +820,111 @@ }, { "ImportPath": "google.golang.org/grpc", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/balancer", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/balancer/base", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/balancer/roundrobin", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/codes", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/connectivity", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/credentials", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { - "ImportPath": "google.golang.org/grpc/grpclb/grpc_lb_v1/messages", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "ImportPath": "google.golang.org/grpc/encoding", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/encoding/proto", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/grpclog", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/health", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/health/grpc_health_v1", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/internal", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/backoff", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/channelz", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/grpcrand", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/keepalive", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/metadata", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/naming", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/peer", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/resolver", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/resolver/dns", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/resolver/passthrough", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/stats", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/status", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/tap", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/transport", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "gopkg.in/inf.v0", @@ -1362,10 +1390,18 @@ "ImportPath": "k8s.io/apiserver/pkg/audit", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/audit/event", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/audit/policy", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/audit/util", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/authentication/authenticator", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -1406,6 +1442,10 @@ "ImportPath": "k8s.io/apiserver/pkg/authentication/serviceaccount", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/authentication/token/cache", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/authentication/token/tokenfile", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -1614,6 +1654,14 @@ "ImportPath": "k8s.io/apiserver/plugin/pkg/audit/buffered", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/plugin/pkg/audit/dynamic", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "ImportPath": "k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/plugin/pkg/audit/log", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -2170,6 +2218,10 @@ "ImportPath": "k8s.io/client-go/tools/pager", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/client-go/tools/record", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/client-go/tools/reference", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -2206,25 +2258,29 @@ "ImportPath": "k8s.io/client-go/util/retry", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/kube-openapi/pkg/builder", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/common", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/handler", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/apimachinery/pkg/api/apitesting/fuzzer", @@ -2338,6 +2394,10 @@ "ImportPath": "k8s.io/apimachinery/pkg/util/sets", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apimachinery/pkg/util/uuid", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apimachinery/pkg/util/validation", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -2438,10 +2498,18 @@ "ImportPath": "k8s.io/apiserver/pkg/util/feature", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/util/feature/testing", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/util/logs", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/util/proxy", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/util/webhook", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -2466,6 +2534,10 @@ "ImportPath": "k8s.io/client-go/kubernetes/scheme", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/client-go/listers/core/v1", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/client-go/rest", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -2501,6 +2573,10 @@ { "ImportPath": "k8s.io/client-go/util/workqueue", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" } ] } diff --git a/staging/src/k8s.io/apiextensions-apiserver/examples/client-go/pkg/client/clientset/versioned/typed/cr/v1/example.go b/staging/src/k8s.io/apiextensions-apiserver/examples/client-go/pkg/client/clientset/versioned/typed/cr/v1/example.go index b9408f1c5a8..5b5df88f660 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/examples/client-go/pkg/client/clientset/versioned/typed/cr/v1/example.go +++ b/staging/src/k8s.io/apiextensions-apiserver/examples/client-go/pkg/client/clientset/versioned/typed/cr/v1/example.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/apiextensions-apiserver/examples/client-go/pkg/apis/cr/v1" scheme "k8s.io/apiextensions-apiserver/examples/client-go/pkg/client/clientset/versioned/scheme" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -75,11 +77,16 @@ func (c *examples) Get(name string, options metav1.GetOptions) (result *v1.Examp // List takes label and field selectors, and returns the list of Examples that match those selectors. func (c *examples) List(opts metav1.ListOptions) (result *v1.ExampleList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.ExampleList{} err = c.client.Get(). Namespace(c.ns). Resource("examples"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *examples) List(opts metav1.ListOptions) (result *v1.ExampleList, err er // Watch returns a watch.Interface that watches the requested examples. func (c *examples) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("examples"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *examples) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *examples) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("examples"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/apiextensions-apiserver/main.go b/staging/src/k8s.io/apiextensions-apiserver/main.go index 888a04b1a46..eeac2e685a7 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/main.go +++ b/staging/src/k8s.io/apiextensions-apiserver/main.go @@ -20,7 +20,7 @@ import ( "flag" "os" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apiextensions-apiserver/pkg/cmd/server" genericapiserver "k8s.io/apiserver/pkg/server" @@ -35,6 +35,6 @@ func main() { cmd := server.NewServerCommand(os.Stdout, os.Stderr, stopCh) cmd.Flags().AddGoFlagSet(flag.CommandLine) if err := cmd.Execute(); err != nil { - glog.Fatal(err) + klog.Fatal(err) } } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go index cda76ff75b0..fcbd8bd197a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go @@ -44,8 +44,14 @@ type CustomResourceDefinitionSpec struct { // Scope indicates whether this resource is cluster or namespace scoped. Default is namespaced Scope ResourceScope // Validation describes the validation methods for CustomResources + // Optional, the global validation schema for all versions. + // Top-level and per-version schemas are mutually exclusive. + // +optional Validation *CustomResourceValidation - // Subresources describes the subresources for CustomResources + // Subresources describes the subresources for CustomResource + // Optional, the global subresources for all versions. + // Top-level and per-version subresources are mutually exclusive. + // +optional Subresources *CustomResourceSubresources // Versions is the list of all supported versions for this resource. // If Version field is provided, this field is optional. @@ -60,6 +66,9 @@ type CustomResourceDefinitionSpec struct { // v10, v2, v1, v11beta2, v10beta3, v3beta1, v12alpha1, v11alpha2, foo1, foo10. Versions []CustomResourceDefinitionVersion // AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column. + // Optional, the global columns for all versions. + // Top-level and per-version columns are mutually exclusive. + // +optional AdditionalPrinterColumns []CustomResourceColumnDefinition // `conversion` defines conversion settings for the CRD. @@ -149,6 +158,27 @@ type CustomResourceDefinitionVersion struct { // Storage flags the version as storage version. There must be exactly one flagged // as storage version. Storage bool + // Schema describes the schema for CustomResource used in validation, pruning, and defaulting. + // Top-level and per-version schemas are mutually exclusive. + // Per-version schemas must not all be set to identical values (top-level validation schema should be used instead) + // This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature. + // +optional + Schema *CustomResourceValidation + // Subresources describes the subresources for CustomResource + // Top-level and per-version subresources are mutually exclusive. + // Per-version subresources must not all be set to identical values (top-level subresources should be used instead) + // This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature. + // +optional + Subresources *CustomResourceSubresources + // AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column. + // Top-level and per-version columns are mutually exclusive. + // Per-version columns must not all be set to identical values (top-level columns should be used instead) + // This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature. + // NOTE: CRDs created prior to 1.13 populated the top-level additionalPrinterColumns field by default. To apply an + // update that changes to per-version additionalPrinterColumns, the top-level additionalPrinterColumns field must + // be explicitly set to null + // +optional + AdditionalPrinterColumns []CustomResourceColumnDefinition } // CustomResourceColumnDefinition specifies a column for server side printing. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/defaults.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/defaults.go index 2e3c2146a0a..5aae97cf1a2 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/defaults.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/defaults.go @@ -19,12 +19,9 @@ package v1beta1 import ( "strings" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) -var swaggerMetadataDescriptions = metav1.ObjectMeta{}.SwaggerDoc() - func addDefaultingFuncs(scheme *runtime.Scheme) error { scheme.AddTypeDefaultingFunc(&CustomResourceDefinition{}, func(obj interface{}) { SetDefaults_CustomResourceDefinition(obj.(*CustomResourceDefinition)) }) // TODO figure out why I can't seem to get my defaulter generated @@ -66,14 +63,19 @@ func SetDefaults_CustomResourceDefinitionSpec(obj *CustomResourceDefinitionSpec) if len(obj.Version) == 0 && len(obj.Versions) != 0 { obj.Version = obj.Versions[0].Name } - if len(obj.AdditionalPrinterColumns) == 0 { - obj.AdditionalPrinterColumns = []CustomResourceColumnDefinition{ - {Name: "Age", Type: "date", Description: swaggerMetadataDescriptions["creationTimestamp"], JSONPath: ".metadata.creationTimestamp"}, - } - } if obj.Conversion == nil { obj.Conversion = &CustomResourceConversion{ Strategy: NoneConverter, } } } + +// hasPerVersionColumns returns true if a CRD uses per-version columns. +func hasPerVersionColumns(versions []CustomResourceDefinitionVersion) bool { + for _, v := range versions { + if len(v.AdditionalPrinterColumns) > 0 { + return true + } + } + return false +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.pb.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.pb.go index 12aa2ddd9fa..ef2dfe16aa6 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.pb.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.pb.go @@ -785,6 +785,38 @@ func (m *CustomResourceDefinitionVersion) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0 } i++ + if m.Schema != nil { + dAtA[i] = 0x22 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.Schema.Size())) + n15, err := m.Schema.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n15 + } + if m.Subresources != nil { + dAtA[i] = 0x2a + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.Subresources.Size())) + n16, err := m.Subresources.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n16 + } + if len(m.AdditionalPrinterColumns) > 0 { + for _, msg := range m.AdditionalPrinterColumns { + dAtA[i] = 0x32 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } return i, nil } @@ -857,21 +889,21 @@ func (m *CustomResourceSubresources) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Status.Size())) - n15, err := m.Status.MarshalTo(dAtA[i:]) + n17, err := m.Status.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n15 + i += n17 } if m.Scale != nil { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Scale.Size())) - n16, err := m.Scale.MarshalTo(dAtA[i:]) + n18, err := m.Scale.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n16 + i += n18 } return i, nil } @@ -895,11 +927,11 @@ func (m *CustomResourceValidation) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.OpenAPIV3Schema.Size())) - n17, err := m.OpenAPIV3Schema.MarshalTo(dAtA[i:]) + n19, err := m.OpenAPIV3Schema.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n17 + i += n19 } return i, nil } @@ -1003,11 +1035,11 @@ func (m *JSONSchemaProps) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x42 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Default.Size())) - n18, err := m.Default.MarshalTo(dAtA[i:]) + n20, err := m.Default.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n18 + i += n20 } if m.Maximum != nil { dAtA[i] = 0x49 @@ -1134,11 +1166,11 @@ func (m *JSONSchemaProps) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Items.Size())) - n19, err := m.Items.MarshalTo(dAtA[i:]) + n21, err := m.Items.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n19 + i += n21 } if len(m.AllOf) > 0 { for _, msg := range m.AllOf { @@ -1188,11 +1220,11 @@ func (m *JSONSchemaProps) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Not.Size())) - n20, err := m.Not.MarshalTo(dAtA[i:]) + n22, err := m.Not.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n20 + i += n22 } if len(m.Properties) > 0 { keysForProperties := make([]string, 0, len(m.Properties)) @@ -1220,11 +1252,11 @@ func (m *JSONSchemaProps) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n21, err := (&v).MarshalTo(dAtA[i:]) + n23, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n21 + i += n23 } } if m.AdditionalProperties != nil { @@ -1233,11 +1265,11 @@ func (m *JSONSchemaProps) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.AdditionalProperties.Size())) - n22, err := m.AdditionalProperties.MarshalTo(dAtA[i:]) + n24, err := m.AdditionalProperties.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n22 + i += n24 } if len(m.PatternProperties) > 0 { keysForPatternProperties := make([]string, 0, len(m.PatternProperties)) @@ -1265,11 +1297,11 @@ func (m *JSONSchemaProps) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n23, err := (&v).MarshalTo(dAtA[i:]) + n25, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n23 + i += n25 } } if len(m.Dependencies) > 0 { @@ -1298,11 +1330,11 @@ func (m *JSONSchemaProps) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n24, err := (&v).MarshalTo(dAtA[i:]) + n26, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n24 + i += n26 } } if m.AdditionalItems != nil { @@ -1311,11 +1343,11 @@ func (m *JSONSchemaProps) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.AdditionalItems.Size())) - n25, err := m.AdditionalItems.MarshalTo(dAtA[i:]) + n27, err := m.AdditionalItems.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n25 + i += n27 } if len(m.Definitions) > 0 { keysForDefinitions := make([]string, 0, len(m.Definitions)) @@ -1343,11 +1375,11 @@ func (m *JSONSchemaProps) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n26, err := (&v).MarshalTo(dAtA[i:]) + n28, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n26 + i += n28 } } if m.ExternalDocs != nil { @@ -1356,11 +1388,11 @@ func (m *JSONSchemaProps) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ExternalDocs.Size())) - n27, err := m.ExternalDocs.MarshalTo(dAtA[i:]) + n29, err := m.ExternalDocs.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n27 + i += n29 } if m.Example != nil { dAtA[i] = 0xa2 @@ -1368,11 +1400,11 @@ func (m *JSONSchemaProps) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Example.Size())) - n28, err := m.Example.MarshalTo(dAtA[i:]) + n30, err := m.Example.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n28 + i += n30 } return i, nil } @@ -1396,11 +1428,11 @@ func (m *JSONSchemaPropsOrArray) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Schema.Size())) - n29, err := m.Schema.MarshalTo(dAtA[i:]) + n31, err := m.Schema.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n29 + i += n31 } if len(m.JSONSchemas) > 0 { for _, msg := range m.JSONSchemas { @@ -1444,11 +1476,11 @@ func (m *JSONSchemaPropsOrBool) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Schema.Size())) - n30, err := m.Schema.MarshalTo(dAtA[i:]) + n32, err := m.Schema.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n30 + i += n32 } return i, nil } @@ -1472,11 +1504,11 @@ func (m *JSONSchemaPropsOrStringArray) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Schema.Size())) - n31, err := m.Schema.MarshalTo(dAtA[i:]) + n33, err := m.Schema.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n31 + i += n33 } if len(m.Property) > 0 { for _, s := range m.Property { @@ -1547,11 +1579,11 @@ func (m *WebhookClientConfig) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Service.Size())) - n32, err := m.Service.MarshalTo(dAtA[i:]) + n34, err := m.Service.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n32 + i += n34 } if m.CABundle != nil { dAtA[i] = 0x12 @@ -1785,6 +1817,20 @@ func (m *CustomResourceDefinitionVersion) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) n += 2 n += 2 + if m.Schema != nil { + l = m.Schema.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + if m.Subresources != nil { + l = m.Subresources.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + if len(m.AdditionalPrinterColumns) > 0 { + for _, e := range m.AdditionalPrinterColumns { + l = e.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + } return n } @@ -2239,6 +2285,9 @@ func (this *CustomResourceDefinitionVersion) String() string { `Name:` + fmt.Sprintf("%v", this.Name) + `,`, `Served:` + fmt.Sprintf("%v", this.Served) + `,`, `Storage:` + fmt.Sprintf("%v", this.Storage) + `,`, + `Schema:` + strings.Replace(fmt.Sprintf("%v", this.Schema), "CustomResourceValidation", "CustomResourceValidation", 1) + `,`, + `Subresources:` + strings.Replace(fmt.Sprintf("%v", this.Subresources), "CustomResourceSubresources", "CustomResourceSubresources", 1) + `,`, + `AdditionalPrinterColumns:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.AdditionalPrinterColumns), "CustomResourceColumnDefinition", "CustomResourceColumnDefinition", 1), `&`, ``, 1) + `,`, `}`, }, "") return s @@ -4414,6 +4463,103 @@ func (m *CustomResourceDefinitionVersion) Unmarshal(dAtA []byte) error { } } m.Storage = bool(v != 0) + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Schema == nil { + m.Schema = &CustomResourceValidation{} + } + if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Subresources", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Subresources == nil { + m.Subresources = &CustomResourceSubresources{} + } + if err := m.Subresources.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AdditionalPrinterColumns", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AdditionalPrinterColumns = append(m.AdditionalPrinterColumns, CustomResourceColumnDefinition{}) + if err := m.AdditionalPrinterColumns[len(m.AdditionalPrinterColumns)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -7107,175 +7253,177 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 2711 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x5a, 0xdd, 0x6f, 0x24, 0x47, - 0x11, 0xbf, 0xd9, 0xf5, 0xda, 0xeb, 0xb6, 0x7d, 0xb6, 0xfb, 0xe2, 0xcb, 0x9c, 0xb9, 0xec, 0xda, - 0x1b, 0x12, 0x99, 0x70, 0xb7, 0xce, 0x1d, 0x09, 0x09, 0x91, 0x78, 0xf0, 0xda, 0x4e, 0xe4, 0x70, - 0x3e, 0x9b, 0xde, 0xbb, 0x0b, 0x90, 0xcf, 0xf6, 0x4c, 0xef, 0x7a, 0xce, 0xf3, 0x75, 0xd3, 0x33, - 0x6b, 0x5b, 0x7c, 0x08, 0x12, 0x45, 0x20, 0x04, 0x04, 0x91, 0x13, 0x12, 0x02, 0x1e, 0x00, 0xf1, - 0x82, 0x10, 0x3c, 0xc0, 0x1b, 0xfc, 0x01, 0xf7, 0x18, 0xf1, 0x94, 0xa7, 0x15, 0xb7, 0xfc, 0x0b, - 0x48, 0x48, 0x7e, 0x42, 0xfd, 0x31, 0x3d, 0xb3, 0xb3, 0xbb, 0x39, 0x2b, 0xb7, 0x9b, 0x7b, 0xf3, - 0x54, 0x55, 0xd7, 0xaf, 0xba, 0xba, 0xaa, 0xba, 0xaa, 0xd7, 0xa0, 0x71, 0xf0, 0x22, 0xad, 0x5a, - 0xde, 0xea, 0x41, 0xb4, 0x47, 0x02, 0x97, 0x84, 0x84, 0xae, 0xb6, 0x88, 0x6b, 0x7a, 0xc1, 0xaa, - 0x64, 0x60, 0xdf, 0x22, 0x47, 0x21, 0x71, 0xa9, 0xe5, 0xb9, 0xf4, 0x32, 0xf6, 0x2d, 0x4a, 0x82, - 0x16, 0x09, 0x56, 0xfd, 0x83, 0x26, 0xe3, 0xd1, 0x6e, 0x81, 0xd5, 0xd6, 0x95, 0x3d, 0x12, 0xe2, - 0x2b, 0xab, 0x4d, 0xe2, 0x92, 0x00, 0x87, 0xc4, 0xac, 0xfa, 0x81, 0x17, 0x7a, 0xf0, 0xab, 0x42, - 0x5d, 0xb5, 0x4b, 0xfa, 0x6d, 0xa5, 0xae, 0xea, 0x1f, 0x34, 0x19, 0x8f, 0x76, 0x0b, 0x54, 0xa5, - 0xba, 0xc5, 0xcb, 0x4d, 0x2b, 0xdc, 0x8f, 0xf6, 0xaa, 0x86, 0xe7, 0xac, 0x36, 0xbd, 0xa6, 0xb7, - 0xca, 0xb5, 0xee, 0x45, 0x0d, 0xfe, 0xc5, 0x3f, 0xf8, 0x5f, 0x02, 0x6d, 0xf1, 0xb9, 0xc4, 0x78, - 0x07, 0x1b, 0xfb, 0x96, 0x4b, 0x82, 0xe3, 0xc4, 0x62, 0x87, 0x84, 0x78, 0xb5, 0xd5, 0x63, 0xe3, - 0xe2, 0xea, 0xa0, 0x55, 0x41, 0xe4, 0x86, 0x96, 0x43, 0x7a, 0x16, 0x7c, 0xf9, 0x41, 0x0b, 0xa8, - 0xb1, 0x4f, 0x1c, 0x9c, 0x5d, 0x57, 0x39, 0xd1, 0xc0, 0xfc, 0xba, 0xe7, 0xb6, 0x48, 0xc0, 0x76, - 0x89, 0xc8, 0x9d, 0x88, 0xd0, 0x10, 0xd6, 0x40, 0x3e, 0xb2, 0x4c, 0x5d, 0x5b, 0xd2, 0x56, 0x26, - 0x6b, 0xcf, 0xde, 0x6b, 0x97, 0xcf, 0x74, 0xda, 0xe5, 0xfc, 0xcd, 0xad, 0x8d, 0x93, 0x76, 0x79, - 0x79, 0x10, 0x52, 0x78, 0xec, 0x13, 0x5a, 0xbd, 0xb9, 0xb5, 0x81, 0xd8, 0x62, 0xf8, 0x0a, 0x98, - 0x37, 0x09, 0xb5, 0x02, 0x62, 0xae, 0xed, 0x6e, 0xdd, 0x12, 0xfa, 0xf5, 0x1c, 0xd7, 0x78, 0x41, - 0x6a, 0x9c, 0xdf, 0xc8, 0x0a, 0xa0, 0xde, 0x35, 0xf0, 0x1b, 0x60, 0xc2, 0xdb, 0xbb, 0x4d, 0x8c, - 0x90, 0xea, 0xf9, 0xa5, 0xfc, 0xca, 0xd4, 0xd5, 0xcb, 0xd5, 0xe4, 0x04, 0x95, 0x09, 0xfc, 0xd8, - 0xe4, 0x66, 0xab, 0x08, 0x1f, 0x6e, 0xc6, 0x27, 0x57, 0x9b, 0x95, 0x68, 0x13, 0x3b, 0x42, 0x0b, - 0x8a, 0xd5, 0x55, 0xfe, 0x90, 0x03, 0x30, 0xbd, 0x79, 0xea, 0x7b, 0x2e, 0x25, 0x43, 0xd9, 0x3d, - 0x05, 0x73, 0x06, 0xd7, 0x1c, 0x12, 0x53, 0xe2, 0xea, 0xb9, 0x4f, 0x63, 0xbd, 0x2e, 0xf1, 0xe7, - 0xd6, 0x33, 0xea, 0x50, 0x0f, 0x00, 0xbc, 0x01, 0xc6, 0x03, 0x42, 0x23, 0x3b, 0xd4, 0xf3, 0x4b, - 0xda, 0xca, 0xd4, 0xd5, 0x4b, 0x03, 0xa1, 0x78, 0x7c, 0xb3, 0xe0, 0xab, 0xb6, 0xae, 0x54, 0xeb, - 0x21, 0x0e, 0x23, 0x5a, 0x3b, 0x2b, 0x91, 0xc6, 0x11, 0xd7, 0x81, 0xa4, 0xae, 0xca, 0x8f, 0x72, - 0x60, 0x2e, 0xed, 0xa5, 0x96, 0x45, 0x0e, 0xe1, 0x21, 0x98, 0x08, 0x44, 0xb0, 0x70, 0x3f, 0x4d, - 0x5d, 0xdd, 0xad, 0x3e, 0x54, 0x5a, 0x55, 0x7b, 0x82, 0xb0, 0x36, 0xc5, 0xce, 0x4c, 0x7e, 0xa0, - 0x18, 0x0d, 0x7e, 0x1b, 0x14, 0x03, 0x79, 0x50, 0x3c, 0x9a, 0xa6, 0xae, 0x7e, 0x7d, 0x88, 0xc8, - 0x42, 0x71, 0x6d, 0xba, 0xd3, 0x2e, 0x17, 0xe3, 0x2f, 0xa4, 0x00, 0x2b, 0x1f, 0xe6, 0x40, 0x69, - 0x3d, 0xa2, 0xa1, 0xe7, 0x20, 0x42, 0xbd, 0x28, 0x30, 0xc8, 0xba, 0x67, 0x47, 0x8e, 0xbb, 0x41, - 0x1a, 0x96, 0x6b, 0x85, 0x2c, 0x5a, 0x97, 0xc0, 0x98, 0x8b, 0x1d, 0x22, 0xa3, 0x67, 0x5a, 0xfa, - 0x74, 0xec, 0x3a, 0x76, 0x08, 0xe2, 0x1c, 0x26, 0xc1, 0x82, 0x45, 0xe6, 0x82, 0x92, 0xb8, 0x71, - 0xec, 0x13, 0xc4, 0x39, 0xf0, 0x69, 0x30, 0xde, 0xf0, 0x02, 0x07, 0x8b, 0x73, 0x9c, 0x4c, 0x4e, - 0xe6, 0x65, 0x4e, 0x45, 0x92, 0x0b, 0x9f, 0x07, 0x53, 0x26, 0xa1, 0x46, 0x60, 0xf9, 0x0c, 0x5a, - 0x1f, 0xe3, 0xc2, 0xe7, 0xa4, 0xf0, 0xd4, 0x46, 0xc2, 0x42, 0x69, 0x39, 0x78, 0x09, 0x14, 0xfd, - 0xc0, 0xf2, 0x02, 0x2b, 0x3c, 0xd6, 0x0b, 0x4b, 0xda, 0x4a, 0xa1, 0x36, 0x27, 0xd7, 0x14, 0x77, - 0x25, 0x1d, 0x29, 0x09, 0xb8, 0x04, 0x8a, 0xaf, 0xd6, 0x77, 0xae, 0xef, 0xe2, 0x70, 0x5f, 0x1f, - 0xe7, 0x08, 0x63, 0x4c, 0x1a, 0x15, 0x6f, 0x4b, 0x6a, 0xe5, 0xdd, 0x1c, 0xd0, 0xb3, 0x5e, 0x89, - 0x5d, 0x0a, 0x5f, 0x06, 0x45, 0x1a, 0xb2, 0x8a, 0xd3, 0x3c, 0x96, 0x3e, 0x79, 0x26, 0x06, 0xab, - 0x4b, 0xfa, 0x49, 0xbb, 0x7c, 0x3e, 0x59, 0x11, 0x53, 0xb9, 0x3f, 0xd4, 0x5a, 0xf8, 0x5b, 0x0d, - 0x9c, 0x3b, 0x24, 0x7b, 0xfb, 0x9e, 0x77, 0xb0, 0x6e, 0x5b, 0xc4, 0x0d, 0xd7, 0x3d, 0xb7, 0x61, - 0x35, 0x65, 0x0c, 0xa0, 0x87, 0x8c, 0x81, 0xd7, 0x7a, 0x35, 0xd7, 0x1e, 0xef, 0xb4, 0xcb, 0xe7, - 0xfa, 0x30, 0x50, 0x3f, 0x3b, 0x2a, 0xef, 0xe5, 0xb3, 0x4e, 0x48, 0x05, 0xc5, 0x3b, 0xa0, 0xc8, - 0x92, 0xcd, 0xc4, 0x21, 0x96, 0xe9, 0xf2, 0xec, 0xe9, 0x52, 0x53, 0x64, 0xf6, 0x36, 0x09, 0x71, - 0x0d, 0x4a, 0xb7, 0x81, 0x84, 0x86, 0x94, 0x56, 0xf8, 0x5d, 0x30, 0x46, 0x7d, 0x62, 0x48, 0x77, - 0xbc, 0xfe, 0xb0, 0x29, 0x31, 0x60, 0x23, 0x75, 0x9f, 0x18, 0x49, 0xc4, 0xb2, 0x2f, 0xc4, 0x61, - 0xe1, 0xfb, 0x1a, 0x18, 0xa7, 0xbc, 0x8c, 0xc8, 0xd2, 0xf3, 0xe6, 0xa8, 0x2c, 0xc8, 0xd4, 0x2a, - 0xf1, 0x8d, 0x24, 0x78, 0xe5, 0xbf, 0x39, 0xb0, 0x3c, 0x68, 0xe9, 0xba, 0xe7, 0x9a, 0xe2, 0x38, - 0xb6, 0x64, 0x06, 0x8a, 0x78, 0x7c, 0x3e, 0x9d, 0x81, 0x27, 0xed, 0xf2, 0x53, 0x0f, 0x54, 0x90, - 0x4a, 0xd5, 0xaf, 0xa8, 0x7d, 0x8b, 0x74, 0x5e, 0xee, 0x36, 0xec, 0xa4, 0x5d, 0x9e, 0x55, 0xcb, - 0xba, 0x6d, 0x85, 0x2d, 0x00, 0x6d, 0x4c, 0xc3, 0x1b, 0x01, 0x76, 0xa9, 0x50, 0x6b, 0x39, 0x44, - 0xba, 0xef, 0x99, 0xd3, 0x85, 0x07, 0x5b, 0x51, 0x5b, 0x94, 0x90, 0xf0, 0x5a, 0x8f, 0x36, 0xd4, - 0x07, 0x81, 0x55, 0x97, 0x80, 0x60, 0xaa, 0x0a, 0x46, 0xaa, 0xee, 0x33, 0x2a, 0x92, 0x5c, 0xf8, - 0x05, 0x30, 0xe1, 0x10, 0x4a, 0x71, 0x93, 0xf0, 0x2a, 0x31, 0x99, 0x5c, 0xa4, 0xdb, 0x82, 0x8c, - 0x62, 0x3e, 0xeb, 0x22, 0x2e, 0x0e, 0xf2, 0xda, 0x35, 0x8b, 0x86, 0xf0, 0x8d, 0x9e, 0x04, 0xa8, - 0x9e, 0x6e, 0x87, 0x6c, 0x35, 0x0f, 0x7f, 0x55, 0xa2, 0x62, 0x4a, 0x2a, 0xf8, 0xbf, 0x03, 0x0a, - 0x56, 0x48, 0x9c, 0xf8, 0x86, 0x7d, 0x6d, 0x44, 0xb1, 0x57, 0x9b, 0x91, 0x36, 0x14, 0xb6, 0x18, - 0x1a, 0x12, 0xa0, 0x95, 0x3f, 0xe6, 0xc0, 0x13, 0x83, 0x96, 0xb0, 0xb2, 0x4f, 0x99, 0xc7, 0x7d, - 0x3b, 0x0a, 0xb0, 0x2d, 0x23, 0x4e, 0x79, 0x7c, 0x97, 0x53, 0x91, 0xe4, 0xb2, 0xc2, 0x4c, 0x2d, - 0xb7, 0x19, 0xd9, 0x38, 0x90, 0xe1, 0xa4, 0x76, 0x5d, 0x97, 0x74, 0xa4, 0x24, 0x60, 0x15, 0x00, - 0xba, 0xef, 0x05, 0x21, 0xc7, 0xe0, 0xad, 0xd1, 0x64, 0xed, 0x2c, 0x2b, 0x10, 0x75, 0x45, 0x45, - 0x29, 0x09, 0x76, 0xef, 0x1c, 0x58, 0xae, 0x29, 0x4f, 0x5d, 0x65, 0xf1, 0xd7, 0x2c, 0xd7, 0x44, - 0x9c, 0xc3, 0xf0, 0x6d, 0x8b, 0x86, 0x8c, 0x22, 0x8f, 0xbc, 0xcb, 0xeb, 0x5c, 0x52, 0x49, 0x30, - 0x7c, 0x83, 0xd5, 0x66, 0x2f, 0xb0, 0x08, 0xd5, 0xc7, 0x13, 0xfc, 0x75, 0x45, 0x45, 0x29, 0x89, - 0xca, 0xaf, 0x8b, 0x83, 0x83, 0x84, 0x95, 0x12, 0xf8, 0x24, 0x28, 0x34, 0x03, 0x2f, 0xf2, 0xa5, - 0x97, 0x94, 0xb7, 0x5f, 0x61, 0x44, 0x24, 0x78, 0x2c, 0x2a, 0x5b, 0x5d, 0xcd, 0xa4, 0x8a, 0xca, - 0xb8, 0x85, 0x8c, 0xf9, 0xf0, 0x07, 0x1a, 0x28, 0xb8, 0xd2, 0x39, 0x2c, 0xe4, 0xde, 0x18, 0x51, - 0x5c, 0x70, 0xf7, 0x26, 0xe6, 0x0a, 0xcf, 0x0b, 0x64, 0xf8, 0x1c, 0x28, 0x50, 0xc3, 0xf3, 0x89, - 0xf4, 0x7a, 0x29, 0x16, 0xaa, 0x33, 0xe2, 0x49, 0xbb, 0x3c, 0x13, 0xab, 0xe3, 0x04, 0x24, 0x84, - 0xe1, 0x0f, 0x35, 0x00, 0x5a, 0xd8, 0xb6, 0x4c, 0xcc, 0x2f, 0xf6, 0x02, 0x37, 0x7f, 0xb8, 0x61, - 0x7d, 0x4b, 0xa9, 0x17, 0x87, 0x96, 0x7c, 0xa3, 0x14, 0x34, 0xfc, 0x40, 0x03, 0xd3, 0x34, 0xda, - 0x0b, 0xe4, 0x2a, 0xca, 0x5b, 0x80, 0xa9, 0xab, 0xdf, 0x1c, 0xaa, 0x2d, 0xf5, 0x14, 0x40, 0x6d, - 0xae, 0xd3, 0x2e, 0x4f, 0xa7, 0x29, 0xa8, 0xcb, 0x00, 0xf8, 0x13, 0x0d, 0x14, 0xe5, 0x09, 0x53, - 0x7d, 0x82, 0x27, 0xfc, 0x5b, 0x23, 0x3a, 0x58, 0x19, 0x51, 0x49, 0x16, 0x48, 0x02, 0x45, 0xca, - 0x02, 0xf8, 0x0f, 0x0d, 0xe8, 0xd8, 0x14, 0x05, 0x1e, 0xdb, 0xbb, 0x81, 0xe5, 0x86, 0x24, 0x10, - 0x5d, 0x21, 0xd5, 0x8b, 0xdc, 0xbc, 0xe1, 0xde, 0x85, 0xd9, 0x8e, 0xb3, 0xb6, 0x24, 0xad, 0xd3, - 0xd7, 0x06, 0x98, 0x81, 0x06, 0x1a, 0xc8, 0x03, 0xcd, 0x50, 0xad, 0x97, 0x3e, 0x39, 0x82, 0x40, - 0x4b, 0x3a, 0x3b, 0x59, 0x1d, 0x92, 0x76, 0x3b, 0x05, 0x5d, 0xf9, 0x20, 0x9f, 0x6d, 0xad, 0xb3, - 0x97, 0x3e, 0xbc, 0x2b, 0x8c, 0x15, 0x5b, 0xa1, 0xba, 0xc6, 0x9d, 0xfb, 0xce, 0x88, 0xce, 0x5e, - 0xdd, 0xda, 0x49, 0xe3, 0xa5, 0x48, 0x14, 0xa5, 0xec, 0x80, 0xbf, 0xd2, 0xc0, 0x0c, 0x36, 0x0c, - 0xe2, 0x87, 0xc4, 0x14, 0xb5, 0x38, 0xf7, 0x19, 0x94, 0x9b, 0x05, 0x69, 0xd5, 0xcc, 0x5a, 0x1a, - 0x1a, 0x75, 0x5b, 0x02, 0x5f, 0x02, 0x67, 0x69, 0xe8, 0x05, 0xc4, 0x8c, 0x23, 0x57, 0xde, 0x13, - 0xb0, 0xd3, 0x2e, 0x9f, 0xad, 0x77, 0x71, 0x50, 0x46, 0xb2, 0xf2, 0x4b, 0x0d, 0x94, 0x1f, 0x90, - 0x19, 0xa7, 0x98, 0x76, 0x9e, 0x06, 0xe3, 0x7c, 0xbb, 0x26, 0xf7, 0x4a, 0x31, 0xd5, 0xb9, 0x71, - 0x2a, 0x92, 0x5c, 0x56, 0xd7, 0x19, 0x3e, 0xeb, 0x36, 0xf2, 0x5c, 0x50, 0xd5, 0xf5, 0xba, 0x20, - 0xa3, 0x98, 0x5f, 0xf9, 0x9f, 0x96, 0x0d, 0x95, 0x54, 0xb9, 0xa8, 0x1b, 0xd8, 0x26, 0x70, 0x03, - 0xcc, 0xb1, 0xbe, 0x14, 0x11, 0xdf, 0xb6, 0x0c, 0x4c, 0xf9, 0xf0, 0x22, 0x6c, 0x54, 0xf3, 0x74, - 0x3d, 0xc3, 0x47, 0x3d, 0x2b, 0xe0, 0xab, 0x00, 0x8a, 0x5e, 0xad, 0x4b, 0x8f, 0xb8, 0x76, 0x54, - 0xd7, 0x55, 0xef, 0x91, 0x40, 0x7d, 0x56, 0xc1, 0x75, 0x30, 0x6f, 0xe3, 0x3d, 0x62, 0xd7, 0x89, - 0x4d, 0x8c, 0xd0, 0x0b, 0xb8, 0x2a, 0x31, 0xde, 0x2d, 0x74, 0xda, 0xe5, 0xf9, 0x6b, 0x59, 0x26, - 0xea, 0x95, 0xaf, 0x2c, 0x67, 0x4f, 0x24, 0xbd, 0x71, 0xd1, 0x01, 0xff, 0x2e, 0x07, 0x16, 0x07, - 0x57, 0x57, 0xf8, 0x6e, 0xd2, 0xa8, 0x8b, 0x3e, 0xec, 0xad, 0x51, 0x55, 0x72, 0xd9, 0xa9, 0x83, - 0xde, 0x2e, 0x1d, 0x7e, 0x8f, 0x5d, 0x8a, 0xd8, 0x8e, 0x07, 0xf8, 0x37, 0x47, 0x66, 0x02, 0x03, - 0xa9, 0x4d, 0x8a, 0xfb, 0x16, 0xdb, 0xfc, 0x7a, 0xc5, 0x36, 0xa9, 0xfc, 0x49, 0xcb, 0xce, 0x6a, - 0xc9, 0xed, 0x07, 0x7f, 0xaa, 0x81, 0x59, 0xcf, 0x27, 0xee, 0xda, 0xee, 0xd6, 0xad, 0x2f, 0xd5, - 0xf9, 0xab, 0x99, 0x74, 0xd5, 0xf5, 0x87, 0xb4, 0x93, 0x8d, 0xd1, 0x42, 0xe1, 0x6e, 0xe0, 0xf9, - 0xb4, 0x76, 0xae, 0xd3, 0x2e, 0xcf, 0xee, 0x74, 0x43, 0xa1, 0x2c, 0x76, 0xc5, 0x01, 0x0b, 0x9b, - 0x47, 0x21, 0x09, 0x5c, 0x6c, 0x6f, 0x78, 0x46, 0xe4, 0x10, 0x37, 0x14, 0x86, 0x66, 0xa6, 0x7f, - 0xed, 0x94, 0xd3, 0xff, 0x13, 0x20, 0x1f, 0x05, 0xb6, 0x8c, 0xe2, 0x29, 0xf5, 0xba, 0x85, 0xae, - 0x21, 0x46, 0xaf, 0x2c, 0x83, 0x31, 0x66, 0x27, 0xbc, 0x00, 0xf2, 0x01, 0x3e, 0xe4, 0x5a, 0xa7, - 0x6b, 0x13, 0x4c, 0x04, 0xe1, 0x43, 0xc4, 0x68, 0x95, 0x3f, 0x5f, 0x04, 0xb3, 0x99, 0xbd, 0xc0, - 0x45, 0x90, 0x53, 0x4f, 0x66, 0x40, 0x2a, 0xcd, 0x6d, 0x6d, 0xa0, 0x9c, 0x65, 0xc2, 0x17, 0xc0, - 0xb8, 0x78, 0x7d, 0x94, 0xa0, 0x65, 0x55, 0x02, 0x38, 0x95, 0x75, 0x41, 0x89, 0x3a, 0x66, 0x88, - 0x14, 0xe7, 0x36, 0x90, 0x86, 0xcc, 0x12, 0x61, 0x03, 0x69, 0x20, 0x46, 0xfb, 0xb4, 0x4f, 0x1f, - 0xf1, 0xdb, 0x4b, 0xe1, 0x14, 0x6f, 0x2f, 0xe3, 0x9f, 0xf8, 0xf6, 0xf2, 0x24, 0x28, 0x84, 0x56, - 0x68, 0x13, 0x7d, 0xa2, 0xbb, 0x59, 0xbd, 0xc1, 0x88, 0x48, 0xf0, 0xe0, 0x6d, 0x30, 0x61, 0x92, - 0x06, 0x8e, 0xec, 0x50, 0x2f, 0xf2, 0x10, 0x5a, 0x1f, 0x42, 0x08, 0x89, 0x87, 0xb1, 0x0d, 0xa1, - 0x17, 0xc5, 0x00, 0xf0, 0x29, 0x30, 0xe1, 0xe0, 0x23, 0xcb, 0x89, 0x1c, 0x7e, 0x8d, 0x6b, 0x42, - 0x6c, 0x5b, 0x90, 0x50, 0xcc, 0x63, 0x95, 0x91, 0x1c, 0x19, 0x76, 0x44, 0xad, 0x16, 0x91, 0x4c, - 0x1d, 0xf0, 0x82, 0xab, 0x2a, 0xe3, 0x66, 0x86, 0x8f, 0x7a, 0x56, 0x70, 0x30, 0xcb, 0xe5, 0x8b, - 0xa7, 0x52, 0x60, 0x82, 0x84, 0x62, 0x5e, 0x37, 0x98, 0x94, 0x9f, 0x1e, 0x04, 0x26, 0x17, 0xf7, - 0xac, 0x80, 0x5f, 0x04, 0x93, 0x0e, 0x3e, 0xba, 0x46, 0xdc, 0x66, 0xb8, 0xaf, 0xcf, 0x2c, 0x69, - 0x2b, 0xf9, 0xda, 0x4c, 0xa7, 0x5d, 0x9e, 0xdc, 0x8e, 0x89, 0x28, 0xe1, 0x73, 0x61, 0xcb, 0x95, - 0xc2, 0x67, 0x53, 0xc2, 0x31, 0x11, 0x25, 0x7c, 0x76, 0xe9, 0xf8, 0x38, 0x64, 0xc9, 0xa5, 0xcf, - 0x76, 0x0f, 0x13, 0xbb, 0x82, 0x8c, 0x62, 0x3e, 0x5c, 0x01, 0x45, 0x07, 0x1f, 0xf1, 0xc1, 0x4f, - 0x9f, 0xe3, 0x6a, 0xf9, 0x23, 0xe1, 0xb6, 0xa4, 0x21, 0xc5, 0xe5, 0x92, 0x96, 0x2b, 0x24, 0xe7, - 0x53, 0x92, 0x92, 0x86, 0x14, 0x97, 0x05, 0x71, 0xe4, 0x5a, 0x77, 0x22, 0x22, 0x84, 0x21, 0xf7, - 0x8c, 0x0a, 0xe2, 0x9b, 0x09, 0x0b, 0xa5, 0xe5, 0xd8, 0xe0, 0xe5, 0x44, 0x76, 0x68, 0xf9, 0x36, - 0xd9, 0x69, 0xe8, 0xe7, 0xb8, 0xff, 0x79, 0x6b, 0xb5, 0xad, 0xa8, 0x28, 0x25, 0x01, 0x09, 0x18, - 0x23, 0x6e, 0xe4, 0xe8, 0x8f, 0xf1, 0x86, 0x69, 0x28, 0x21, 0xa8, 0x32, 0x67, 0xd3, 0x8d, 0x1c, - 0xc4, 0xd5, 0xc3, 0x17, 0xc0, 0x8c, 0x83, 0x8f, 0x58, 0x39, 0x20, 0x41, 0xc8, 0x46, 0xc2, 0x05, - 0xbe, 0xf9, 0x79, 0xd6, 0xa4, 0x6c, 0xa7, 0x19, 0xa8, 0x5b, 0x8e, 0x2f, 0xb4, 0xdc, 0xd4, 0xc2, - 0xf3, 0xa9, 0x85, 0x69, 0x06, 0xea, 0x96, 0x63, 0x9e, 0x0e, 0xc8, 0x9d, 0xc8, 0x0a, 0x88, 0xa9, - 0x3f, 0xce, 0xfb, 0x1a, 0xf9, 0x70, 0x2b, 0x68, 0x48, 0x71, 0x61, 0x2b, 0x7e, 0x21, 0xd0, 0x79, - 0x1a, 0xde, 0x1c, 0x6e, 0x25, 0xdf, 0x09, 0xd6, 0x82, 0x00, 0x1f, 0x8b, 0x9b, 0x26, 0xfd, 0x36, - 0x00, 0x29, 0x28, 0x60, 0xdb, 0xde, 0x69, 0xe8, 0x17, 0xb8, 0xef, 0x87, 0x7d, 0x83, 0xa8, 0xaa, - 0xb3, 0xc6, 0x40, 0x90, 0xc0, 0x62, 0xa0, 0x9e, 0xcb, 0x42, 0x63, 0x71, 0xb4, 0xa0, 0x3b, 0x0c, - 0x04, 0x09, 0x2c, 0xbe, 0x53, 0xf7, 0x78, 0xa7, 0xa1, 0x7f, 0x6e, 0xc4, 0x3b, 0x65, 0x20, 0x48, - 0x60, 0x41, 0x0b, 0xe4, 0x5d, 0x2f, 0xd4, 0x2f, 0x8e, 0xe4, 0x7a, 0xe6, 0x17, 0xce, 0x75, 0x2f, - 0x44, 0x0c, 0x03, 0xfe, 0x42, 0x03, 0xc0, 0x4f, 0x42, 0xf4, 0x89, 0xa1, 0x0c, 0x9e, 0x19, 0xc8, - 0x6a, 0x12, 0xdb, 0x9b, 0x6e, 0x18, 0x1c, 0x27, 0xa3, 0x47, 0x2a, 0x07, 0x52, 0x56, 0xc0, 0xdf, - 0x6b, 0xe0, 0xb1, 0xf4, 0x6c, 0xa7, 0xcc, 0x2b, 0x71, 0x8f, 0xdc, 0x18, 0x76, 0x98, 0xd7, 0x3c, - 0xcf, 0xae, 0xe9, 0x9d, 0x76, 0xf9, 0xb1, 0xb5, 0x3e, 0xa8, 0xa8, 0xaf, 0x2d, 0xf0, 0x2f, 0x1a, - 0x98, 0x97, 0x55, 0x34, 0x65, 0x61, 0x99, 0x3b, 0x90, 0x0c, 0xdb, 0x81, 0x59, 0x1c, 0xe1, 0x47, - 0xf5, 0x83, 0x63, 0x0f, 0x1f, 0xf5, 0x9a, 0x06, 0xff, 0xae, 0x81, 0x69, 0x93, 0xf8, 0xc4, 0x35, - 0x89, 0x6b, 0x30, 0x5b, 0x97, 0x86, 0x32, 0x69, 0x66, 0x6d, 0xdd, 0x48, 0x41, 0x08, 0x33, 0xab, - 0xd2, 0xcc, 0xe9, 0x34, 0xeb, 0xa4, 0x5d, 0x3e, 0x9f, 0x2c, 0x4d, 0x73, 0x50, 0x97, 0x95, 0xf0, - 0x43, 0x0d, 0xcc, 0x26, 0x07, 0x20, 0xae, 0x94, 0xe5, 0x11, 0xc6, 0x01, 0x6f, 0x5f, 0xd7, 0xba, - 0x01, 0x51, 0xd6, 0x02, 0xf8, 0x57, 0x8d, 0x75, 0x6a, 0xf1, 0xdc, 0x48, 0xf5, 0x0a, 0xf7, 0xe5, - 0xdb, 0x43, 0xf7, 0xa5, 0x42, 0x10, 0xae, 0xbc, 0x94, 0xb4, 0x82, 0x8a, 0x73, 0xd2, 0x2e, 0x2f, - 0xa4, 0x3d, 0xa9, 0x18, 0x28, 0x6d, 0x21, 0xfc, 0xb1, 0x06, 0xa6, 0x49, 0xd2, 0x71, 0x53, 0xfd, - 0xc9, 0xa1, 0x38, 0xb1, 0x6f, 0x13, 0x2f, 0x5e, 0xbb, 0x52, 0x2c, 0x8a, 0xba, 0xb0, 0x59, 0x07, - 0x49, 0x8e, 0xb0, 0xe3, 0xdb, 0x44, 0xff, 0xfc, 0x90, 0x3b, 0xc8, 0x4d, 0xa1, 0x17, 0xc5, 0x00, - 0x8b, 0x6c, 0xf2, 0xc9, 0x64, 0x0e, 0x9c, 0x03, 0xf9, 0x03, 0x22, 0x7f, 0xb9, 0x43, 0xec, 0x4f, - 0x68, 0x82, 0x42, 0x0b, 0xdb, 0x51, 0x3c, 0xbc, 0x0d, 0xb9, 0xea, 0x22, 0xa1, 0xfc, 0xa5, 0xdc, - 0x8b, 0xda, 0xe2, 0x5d, 0x0d, 0x9c, 0xef, 0x9f, 0xd0, 0x8f, 0xd4, 0xac, 0xdf, 0x68, 0x60, 0xbe, - 0x27, 0x77, 0xfb, 0x58, 0x74, 0xa7, 0xdb, 0xa2, 0xd7, 0x87, 0x9d, 0x84, 0xf5, 0x30, 0xb0, 0xdc, - 0x26, 0xef, 0x3c, 0xd2, 0xe6, 0xfd, 0x4c, 0x03, 0x73, 0xd9, 0x74, 0x78, 0x94, 0xfe, 0xaa, 0xdc, - 0xcd, 0x81, 0xf3, 0xfd, 0x1b, 0x26, 0x18, 0xa8, 0xc9, 0x70, 0x34, 0x13, 0x36, 0x48, 0xa6, 0x4c, - 0x35, 0x54, 0xbe, 0xaf, 0x81, 0xa9, 0xdb, 0x4a, 0x2e, 0xfe, 0xcd, 0x68, 0xe8, 0xb3, 0x7d, 0x5c, - 0x7f, 0x12, 0x06, 0x45, 0x69, 0xdc, 0xca, 0xdf, 0x34, 0xb0, 0xd0, 0xb7, 0xb0, 0xb2, 0x11, 0x14, - 0xdb, 0xb6, 0x77, 0x28, 0x9e, 0x68, 0x52, 0x4f, 0x66, 0x6b, 0x9c, 0x8a, 0x24, 0x37, 0xe5, 0xbd, - 0xdc, 0x67, 0xe5, 0xbd, 0xca, 0x3f, 0x35, 0x70, 0xf1, 0x93, 0x22, 0xf1, 0x91, 0x1c, 0xe9, 0x0a, - 0x28, 0xca, 0xa6, 0xe8, 0x98, 0x1f, 0xa7, 0x9c, 0x03, 0x64, 0xd1, 0xe0, 0xff, 0xcc, 0x20, 0xfe, - 0xaa, 0xbc, 0xa7, 0x81, 0xb9, 0x3a, 0x09, 0x5a, 0x96, 0x41, 0x10, 0x69, 0x90, 0x80, 0xb8, 0x06, - 0x81, 0xab, 0x60, 0x92, 0xff, 0x58, 0xe3, 0x63, 0x23, 0x7e, 0xc9, 0x9c, 0x97, 0x2e, 0x9f, 0xbc, - 0x1e, 0x33, 0x50, 0x22, 0xa3, 0x5e, 0x3d, 0x73, 0x03, 0x5f, 0x3d, 0x2f, 0x82, 0x31, 0x3f, 0x79, - 0xe0, 0x2b, 0x32, 0x2e, 0x7f, 0xd3, 0xe3, 0xd4, 0xca, 0xbf, 0x34, 0xd0, 0xef, 0x1f, 0x0b, 0x60, - 0x0b, 0x4c, 0x50, 0x61, 0x9c, 0x74, 0xde, 0xce, 0x43, 0x3a, 0x2f, 0xbb, 0x55, 0x51, 0xf8, 0x63, - 0x6a, 0x0c, 0xc6, 0xfc, 0x67, 0xe0, 0x5a, 0xe4, 0x9a, 0xf2, 0x49, 0x6e, 0x5a, 0xf8, 0x6f, 0x7d, - 0x4d, 0xd0, 0x90, 0xe2, 0xc2, 0x0b, 0xe2, 0xf1, 0x28, 0xf5, 0x22, 0x13, 0x3f, 0x1c, 0xd5, 0x2e, - 0xdf, 0xbb, 0x5f, 0x3a, 0xf3, 0xd1, 0xfd, 0xd2, 0x99, 0x8f, 0xef, 0x97, 0xce, 0x7c, 0xbf, 0x53, - 0xd2, 0xee, 0x75, 0x4a, 0xda, 0x47, 0x9d, 0x92, 0xf6, 0x71, 0xa7, 0xa4, 0xfd, 0xbb, 0x53, 0xd2, - 0x7e, 0xfe, 0x9f, 0xd2, 0x99, 0x6f, 0x4d, 0x48, 0xd3, 0xfe, 0x1f, 0x00, 0x00, 0xff, 0xff, 0xc9, - 0x4a, 0x8d, 0x8a, 0xee, 0x27, 0x00, 0x00, + // 2740 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x5a, 0xcd, 0x73, 0x1c, 0x47, + 0x15, 0xf7, 0xec, 0x6a, 0xa5, 0x55, 0x4b, 0xb2, 0xa4, 0xb6, 0xe5, 0x8c, 0x85, 0xbd, 0x2b, 0xad, + 0x49, 0x4a, 0x04, 0x7b, 0x15, 0x9b, 0x84, 0x84, 0x54, 0x71, 0xd0, 0x4a, 0x4a, 0x4a, 0xc6, 0xfa, + 0xa0, 0xd7, 0x76, 0x80, 0x7c, 0xb6, 0x66, 0x7b, 0x57, 0x63, 0xcd, 0x97, 0xa7, 0x67, 0x56, 0x52, + 0x05, 0x28, 0x48, 0x2a, 0x05, 0x45, 0x01, 0xa1, 0x88, 0x2f, 0x14, 0x70, 0x00, 0x8a, 0x0b, 0x45, + 0xc1, 0x01, 0x6e, 0xf0, 0x07, 0xf8, 0x98, 0xe2, 0x94, 0xd3, 0x16, 0xde, 0xfc, 0x0b, 0x54, 0x51, + 0xa5, 0x13, 0xd5, 0x1f, 0xd3, 0x33, 0x3b, 0xbb, 0x6b, 0xab, 0xe2, 0xdd, 0x98, 0x9b, 0xe6, 0xbd, + 0xd7, 0xef, 0xf7, 0xfa, 0xf5, 0x7b, 0xaf, 0x5f, 0xbf, 0x15, 0xa8, 0xef, 0xbf, 0x44, 0xcb, 0xa6, + 0xbb, 0xbc, 0x1f, 0xee, 0x12, 0xdf, 0x21, 0x01, 0xa1, 0xcb, 0x4d, 0xe2, 0xd4, 0x5c, 0x7f, 0x59, + 0x32, 0xb0, 0x67, 0x92, 0xc3, 0x80, 0x38, 0xd4, 0x74, 0x1d, 0x7a, 0x05, 0x7b, 0x26, 0x25, 0x7e, + 0x93, 0xf8, 0xcb, 0xde, 0x7e, 0x83, 0xf1, 0x68, 0xa7, 0xc0, 0x72, 0xf3, 0xea, 0x2e, 0x09, 0xf0, + 0xd5, 0xe5, 0x06, 0x71, 0x88, 0x8f, 0x03, 0x52, 0x2b, 0x7b, 0xbe, 0x1b, 0xb8, 0xf0, 0xeb, 0x42, + 0x5d, 0xb9, 0x43, 0xfa, 0x6d, 0xa5, 0xae, 0xec, 0xed, 0x37, 0x18, 0x8f, 0x76, 0x0a, 0x94, 0xa5, + 0xba, 0xf9, 0x2b, 0x0d, 0x33, 0xd8, 0x0b, 0x77, 0xcb, 0x86, 0x6b, 0x2f, 0x37, 0xdc, 0x86, 0xbb, + 0xcc, 0xb5, 0xee, 0x86, 0x75, 0xfe, 0xc5, 0x3f, 0xf8, 0x5f, 0x02, 0x6d, 0xfe, 0xf9, 0xd8, 0x78, + 0x1b, 0x1b, 0x7b, 0xa6, 0x43, 0xfc, 0xa3, 0xd8, 0x62, 0x9b, 0x04, 0x78, 0xb9, 0xd9, 0x65, 0xe3, + 0xfc, 0x72, 0xbf, 0x55, 0x7e, 0xe8, 0x04, 0xa6, 0x4d, 0xba, 0x16, 0x7c, 0xf5, 0x51, 0x0b, 0xa8, + 0xb1, 0x47, 0x6c, 0x9c, 0x5e, 0x57, 0x3a, 0xd6, 0xc0, 0xec, 0xaa, 0xeb, 0x34, 0x89, 0xcf, 0x76, + 0x89, 0xc8, 0xdd, 0x90, 0xd0, 0x00, 0x56, 0x40, 0x36, 0x34, 0x6b, 0xba, 0xb6, 0xa0, 0x2d, 0x8d, + 0x57, 0x9e, 0xbb, 0xdf, 0x2a, 0x9e, 0x6a, 0xb7, 0x8a, 0xd9, 0x5b, 0x1b, 0x6b, 0xc7, 0xad, 0xe2, + 0x62, 0x3f, 0xa4, 0xe0, 0xc8, 0x23, 0xb4, 0x7c, 0x6b, 0x63, 0x0d, 0xb1, 0xc5, 0xf0, 0x55, 0x30, + 0x5b, 0x23, 0xd4, 0xf4, 0x49, 0x6d, 0x65, 0x67, 0xe3, 0xb6, 0xd0, 0xaf, 0x67, 0xb8, 0xc6, 0xf3, + 0x52, 0xe3, 0xec, 0x5a, 0x5a, 0x00, 0x75, 0xaf, 0x81, 0xdf, 0x02, 0x63, 0xee, 0xee, 0x1d, 0x62, + 0x04, 0x54, 0xcf, 0x2e, 0x64, 0x97, 0x26, 0xae, 0x5d, 0x29, 0xc7, 0x27, 0xa8, 0x4c, 0xe0, 0xc7, + 0x26, 0x37, 0x5b, 0x46, 0xf8, 0x60, 0x3d, 0x3a, 0xb9, 0xca, 0xb4, 0x44, 0x1b, 0xdb, 0x16, 0x5a, + 0x50, 0xa4, 0xae, 0xf4, 0x87, 0x0c, 0x80, 0xc9, 0xcd, 0x53, 0xcf, 0x75, 0x28, 0x19, 0xc8, 0xee, + 0x29, 0x98, 0x31, 0xb8, 0xe6, 0x80, 0xd4, 0x24, 0xae, 0x9e, 0xf9, 0x2c, 0xd6, 0xeb, 0x12, 0x7f, + 0x66, 0x35, 0xa5, 0x0e, 0x75, 0x01, 0xc0, 0x9b, 0x60, 0xd4, 0x27, 0x34, 0xb4, 0x02, 0x3d, 0xbb, + 0xa0, 0x2d, 0x4d, 0x5c, 0xbb, 0xdc, 0x17, 0x8a, 0xc7, 0x37, 0x0b, 0xbe, 0x72, 0xf3, 0x6a, 0xb9, + 0x1a, 0xe0, 0x20, 0xa4, 0x95, 0xd3, 0x12, 0x69, 0x14, 0x71, 0x1d, 0x48, 0xea, 0x2a, 0xfd, 0x38, + 0x03, 0x66, 0x92, 0x5e, 0x6a, 0x9a, 0xe4, 0x00, 0x1e, 0x80, 0x31, 0x5f, 0x04, 0x0b, 0xf7, 0xd3, + 0xc4, 0xb5, 0x9d, 0xf2, 0x63, 0xa5, 0x55, 0xb9, 0x2b, 0x08, 0x2b, 0x13, 0xec, 0xcc, 0xe4, 0x07, + 0x8a, 0xd0, 0xe0, 0xbb, 0x20, 0xef, 0xcb, 0x83, 0xe2, 0xd1, 0x34, 0x71, 0xed, 0x9b, 0x03, 0x44, + 0x16, 0x8a, 0x2b, 0x93, 0xed, 0x56, 0x31, 0x1f, 0x7d, 0x21, 0x05, 0x58, 0xfa, 0x28, 0x03, 0x0a, + 0xab, 0x21, 0x0d, 0x5c, 0x1b, 0x11, 0xea, 0x86, 0xbe, 0x41, 0x56, 0x5d, 0x2b, 0xb4, 0x9d, 0x35, + 0x52, 0x37, 0x1d, 0x33, 0x60, 0xd1, 0xba, 0x00, 0x46, 0x1c, 0x6c, 0x13, 0x19, 0x3d, 0x93, 0xd2, + 0xa7, 0x23, 0x5b, 0xd8, 0x26, 0x88, 0x73, 0x98, 0x04, 0x0b, 0x16, 0x99, 0x0b, 0x4a, 0xe2, 0xe6, + 0x91, 0x47, 0x10, 0xe7, 0xc0, 0x67, 0xc0, 0x68, 0xdd, 0xf5, 0x6d, 0x2c, 0xce, 0x71, 0x3c, 0x3e, + 0x99, 0x57, 0x38, 0x15, 0x49, 0x2e, 0x7c, 0x01, 0x4c, 0xd4, 0x08, 0x35, 0x7c, 0xd3, 0x63, 0xd0, + 0xfa, 0x08, 0x17, 0x3e, 0x23, 0x85, 0x27, 0xd6, 0x62, 0x16, 0x4a, 0xca, 0xc1, 0xcb, 0x20, 0xef, + 0xf9, 0xa6, 0xeb, 0x9b, 0xc1, 0x91, 0x9e, 0x5b, 0xd0, 0x96, 0x72, 0x95, 0x19, 0xb9, 0x26, 0xbf, + 0x23, 0xe9, 0x48, 0x49, 0xc0, 0x05, 0x90, 0xbf, 0x5e, 0xdd, 0xde, 0xda, 0xc1, 0xc1, 0x9e, 0x3e, + 0xca, 0x11, 0x46, 0x98, 0x34, 0xca, 0xdf, 0x91, 0xd4, 0xd2, 0x7b, 0x19, 0xa0, 0xa7, 0xbd, 0x12, + 0xb9, 0x14, 0xbe, 0x02, 0xf2, 0x34, 0x60, 0x15, 0xa7, 0x71, 0x24, 0x7d, 0xf2, 0x6c, 0x04, 0x56, + 0x95, 0xf4, 0xe3, 0x56, 0xf1, 0x5c, 0xbc, 0x22, 0xa2, 0x72, 0x7f, 0xa8, 0xb5, 0xf0, 0xb7, 0x1a, + 0x38, 0x73, 0x40, 0x76, 0xf7, 0x5c, 0x77, 0x7f, 0xd5, 0x32, 0x89, 0x13, 0xac, 0xba, 0x4e, 0xdd, + 0x6c, 0xc8, 0x18, 0x40, 0x8f, 0x19, 0x03, 0xaf, 0x75, 0x6b, 0xae, 0x3c, 0xd5, 0x6e, 0x15, 0xcf, + 0xf4, 0x60, 0xa0, 0x5e, 0x76, 0x94, 0xde, 0xcf, 0xa6, 0x9d, 0x90, 0x08, 0x8a, 0x77, 0x40, 0x9e, + 0x25, 0x5b, 0x0d, 0x07, 0x58, 0xa6, 0xcb, 0x73, 0x27, 0x4b, 0x4d, 0x91, 0xd9, 0x9b, 0x24, 0xc0, + 0x15, 0x28, 0xdd, 0x06, 0x62, 0x1a, 0x52, 0x5a, 0xe1, 0xf7, 0xc0, 0x08, 0xf5, 0x88, 0x21, 0xdd, + 0xf1, 0xfa, 0xe3, 0xa6, 0x44, 0x9f, 0x8d, 0x54, 0x3d, 0x62, 0xc4, 0x11, 0xcb, 0xbe, 0x10, 0x87, + 0x85, 0x1f, 0x68, 0x60, 0x94, 0xf2, 0x32, 0x22, 0x4b, 0xcf, 0x9b, 0xc3, 0xb2, 0x20, 0x55, 0xab, + 0xc4, 0x37, 0x92, 0xe0, 0xa5, 0xff, 0x64, 0xc0, 0x62, 0xbf, 0xa5, 0xab, 0xae, 0x53, 0x13, 0xc7, + 0xb1, 0x21, 0x33, 0x50, 0xc4, 0xe3, 0x0b, 0xc9, 0x0c, 0x3c, 0x6e, 0x15, 0x9f, 0x7e, 0xa4, 0x82, + 0x44, 0xaa, 0x7e, 0x4d, 0xed, 0x5b, 0xa4, 0xf3, 0x62, 0xa7, 0x61, 0xc7, 0xad, 0xe2, 0xb4, 0x5a, + 0xd6, 0x69, 0x2b, 0x6c, 0x02, 0x68, 0x61, 0x1a, 0xdc, 0xf4, 0xb1, 0x43, 0x85, 0x5a, 0xd3, 0x26, + 0xd2, 0x7d, 0xcf, 0x9e, 0x2c, 0x3c, 0xd8, 0x8a, 0xca, 0xbc, 0x84, 0x84, 0x37, 0xba, 0xb4, 0xa1, + 0x1e, 0x08, 0xac, 0xba, 0xf8, 0x04, 0x53, 0x55, 0x30, 0x12, 0x75, 0x9f, 0x51, 0x91, 0xe4, 0xc2, + 0x2f, 0x81, 0x31, 0x9b, 0x50, 0x8a, 0x1b, 0x84, 0x57, 0x89, 0xf1, 0xf8, 0x22, 0xdd, 0x14, 0x64, + 0x14, 0xf1, 0x59, 0x17, 0x71, 0xa1, 0x9f, 0xd7, 0x6e, 0x98, 0x34, 0x80, 0x6f, 0x74, 0x25, 0x40, + 0xf9, 0x64, 0x3b, 0x64, 0xab, 0x79, 0xf8, 0xab, 0x12, 0x15, 0x51, 0x12, 0xc1, 0xff, 0x5d, 0x90, + 0x33, 0x03, 0x62, 0x47, 0x37, 0xec, 0x6b, 0x43, 0x8a, 0xbd, 0xca, 0x94, 0xb4, 0x21, 0xb7, 0xc1, + 0xd0, 0x90, 0x00, 0x2d, 0xfd, 0x31, 0x03, 0x2e, 0xf6, 0x5b, 0xc2, 0xca, 0x3e, 0x65, 0x1e, 0xf7, + 0xac, 0xd0, 0xc7, 0x96, 0x8c, 0x38, 0xe5, 0xf1, 0x1d, 0x4e, 0x45, 0x92, 0xcb, 0x0a, 0x33, 0x35, + 0x9d, 0x46, 0x68, 0x61, 0x5f, 0x86, 0x93, 0xda, 0x75, 0x55, 0xd2, 0x91, 0x92, 0x80, 0x65, 0x00, + 0xe8, 0x9e, 0xeb, 0x07, 0x1c, 0x83, 0xb7, 0x46, 0xe3, 0x95, 0xd3, 0xac, 0x40, 0x54, 0x15, 0x15, + 0x25, 0x24, 0xd8, 0xbd, 0xb3, 0x6f, 0x3a, 0x35, 0x79, 0xea, 0x2a, 0x8b, 0xbf, 0x61, 0x3a, 0x35, + 0xc4, 0x39, 0x0c, 0xdf, 0x32, 0x69, 0xc0, 0x28, 0xf2, 0xc8, 0x3b, 0xbc, 0xce, 0x25, 0x95, 0x04, + 0xc3, 0x37, 0x58, 0x6d, 0x76, 0x7d, 0x93, 0x50, 0x7d, 0x34, 0xc6, 0x5f, 0x55, 0x54, 0x94, 0x90, + 0x28, 0xfd, 0x3a, 0xdf, 0x3f, 0x48, 0x58, 0x29, 0x81, 0x97, 0x40, 0xae, 0xe1, 0xbb, 0xa1, 0x27, + 0xbd, 0xa4, 0xbc, 0xfd, 0x2a, 0x23, 0x22, 0xc1, 0x63, 0x51, 0xd9, 0xec, 0x68, 0x26, 0x55, 0x54, + 0x46, 0x2d, 0x64, 0xc4, 0x87, 0x3f, 0xd4, 0x40, 0xce, 0x91, 0xce, 0x61, 0x21, 0xf7, 0xc6, 0x90, + 0xe2, 0x82, 0xbb, 0x37, 0x36, 0x57, 0x78, 0x5e, 0x20, 0xc3, 0xe7, 0x41, 0x8e, 0x1a, 0xae, 0x47, + 0xa4, 0xd7, 0x0b, 0x91, 0x50, 0x95, 0x11, 0x8f, 0x5b, 0xc5, 0xa9, 0x48, 0x1d, 0x27, 0x20, 0x21, + 0x0c, 0x7f, 0xa4, 0x01, 0xd0, 0xc4, 0x96, 0x59, 0xc3, 0xfc, 0x62, 0xcf, 0x71, 0xf3, 0x07, 0x1b, + 0xd6, 0xb7, 0x95, 0x7a, 0x71, 0x68, 0xf1, 0x37, 0x4a, 0x40, 0xc3, 0x0f, 0x35, 0x30, 0x49, 0xc3, + 0x5d, 0x5f, 0xae, 0xa2, 0xbc, 0x05, 0x98, 0xb8, 0xf6, 0xed, 0x81, 0xda, 0x52, 0x4d, 0x00, 0x54, + 0x66, 0xda, 0xad, 0xe2, 0x64, 0x92, 0x82, 0x3a, 0x0c, 0x80, 0x3f, 0xd5, 0x40, 0x5e, 0x9e, 0x30, + 0xd5, 0xc7, 0x78, 0xc2, 0xbf, 0x35, 0xa4, 0x83, 0x95, 0x11, 0x15, 0x67, 0x81, 0x24, 0x50, 0xa4, + 0x2c, 0x80, 0xff, 0xd0, 0x80, 0x8e, 0x6b, 0xa2, 0xc0, 0x63, 0x6b, 0xc7, 0x37, 0x9d, 0x80, 0xf8, + 0xa2, 0x2b, 0xa4, 0x7a, 0x9e, 0x9b, 0x37, 0xd8, 0xbb, 0x30, 0xdd, 0x71, 0x56, 0x16, 0xa4, 0x75, + 0xfa, 0x4a, 0x1f, 0x33, 0x50, 0x5f, 0x03, 0x79, 0xa0, 0x19, 0xaa, 0xf5, 0xd2, 0xc7, 0x87, 0x10, + 0x68, 0x71, 0x67, 0x27, 0xab, 0x43, 0xdc, 0x6e, 0x27, 0xa0, 0x4b, 0x1f, 0x66, 0xd3, 0xad, 0x75, + 0xfa, 0xd2, 0x87, 0xf7, 0x84, 0xb1, 0x62, 0x2b, 0x54, 0xd7, 0xb8, 0x73, 0xdf, 0x19, 0xd2, 0xd9, + 0xab, 0x5b, 0x3b, 0x6e, 0xbc, 0x14, 0x89, 0xa2, 0x84, 0x1d, 0xf0, 0x57, 0x1a, 0x98, 0xc2, 0x86, + 0x41, 0xbc, 0x80, 0xd4, 0x44, 0x2d, 0xce, 0x7c, 0x0e, 0xe5, 0x66, 0x4e, 0x5a, 0x35, 0xb5, 0x92, + 0x84, 0x46, 0x9d, 0x96, 0xc0, 0x97, 0xc1, 0x69, 0x1a, 0xb8, 0x3e, 0xa9, 0x45, 0x91, 0x2b, 0xef, + 0x09, 0xd8, 0x6e, 0x15, 0x4f, 0x57, 0x3b, 0x38, 0x28, 0x25, 0x59, 0xfa, 0x74, 0x04, 0x14, 0x1f, + 0x91, 0x19, 0x27, 0x78, 0xed, 0x3c, 0x03, 0x46, 0xf9, 0x76, 0x6b, 0xdc, 0x2b, 0xf9, 0x44, 0xe7, + 0xc6, 0xa9, 0x48, 0x72, 0x59, 0x5d, 0x67, 0xf8, 0xac, 0xdb, 0xc8, 0x72, 0x41, 0x55, 0xd7, 0xab, + 0x82, 0x8c, 0x22, 0x3e, 0x7c, 0x17, 0x8c, 0x8a, 0x69, 0x06, 0x2f, 0xaa, 0x43, 0x2c, 0x8c, 0x80, + 0xdb, 0xc9, 0xa1, 0x90, 0x84, 0xec, 0x2e, 0x88, 0xb9, 0x27, 0x5d, 0x10, 0x1f, 0x5a, 0x81, 0x46, + 0xff, 0xcf, 0x2b, 0x50, 0xe9, 0xbf, 0x5a, 0x3a, 0xef, 0x13, 0x5b, 0xad, 0x1a, 0xd8, 0x22, 0x70, + 0x0d, 0xcc, 0xb0, 0x47, 0x06, 0x22, 0x9e, 0x65, 0x1a, 0x98, 0xf2, 0x97, 0xa8, 0x08, 0x38, 0x35, + 0x1c, 0xa9, 0xa6, 0xf8, 0xa8, 0x6b, 0x05, 0xbc, 0x0e, 0xa0, 0x68, 0xbc, 0x3b, 0xf4, 0x88, 0x1e, + 0x42, 0xb5, 0xd0, 0xd5, 0x2e, 0x09, 0xd4, 0x63, 0x15, 0x5c, 0x05, 0xb3, 0x16, 0xde, 0x25, 0x56, + 0x95, 0x58, 0xc4, 0x08, 0x5c, 0x9f, 0xab, 0x12, 0x6f, 0xf5, 0xb9, 0x76, 0xab, 0x38, 0x7b, 0x23, + 0xcd, 0x44, 0xdd, 0xf2, 0xa5, 0xc5, 0x74, 0x7a, 0x25, 0x37, 0x2e, 0x9e, 0x33, 0xbf, 0xcb, 0x80, + 0xf9, 0xfe, 0x91, 0x01, 0xdf, 0x8b, 0x5f, 0x5d, 0xa2, 0xa9, 0x7e, 0x6b, 0x58, 0x51, 0x28, 0x9f, + 0x5d, 0xa0, 0xfb, 0xc9, 0x05, 0xbf, 0xcf, 0x3a, 0x1c, 0x6c, 0x45, 0xd3, 0x98, 0x37, 0x87, 0x66, + 0x02, 0x03, 0xa9, 0x8c, 0x8b, 0xe6, 0x09, 0x5b, 0xbc, 0x57, 0xc2, 0x16, 0x29, 0xfd, 0x49, 0x4b, + 0x3f, 0xbc, 0xe3, 0x0c, 0x86, 0x3f, 0xd3, 0xc0, 0xb4, 0xeb, 0x11, 0x67, 0x65, 0x67, 0xe3, 0xf6, + 0x57, 0x44, 0x26, 0x4b, 0x57, 0x6d, 0x3d, 0xa6, 0x9d, 0xd7, 0xab, 0xdb, 0x5b, 0x42, 0xe1, 0x8e, + 0xef, 0x7a, 0xb4, 0x72, 0xa6, 0xdd, 0x2a, 0x4e, 0x6f, 0x77, 0x42, 0xa1, 0x34, 0x76, 0xc9, 0x06, + 0x73, 0xeb, 0x87, 0x01, 0xf1, 0x1d, 0x6c, 0xad, 0xb9, 0x46, 0x68, 0x13, 0x27, 0x10, 0x86, 0xa6, + 0x46, 0x39, 0xda, 0x09, 0x47, 0x39, 0x17, 0x41, 0x36, 0xf4, 0x2d, 0x19, 0xc5, 0x13, 0x6a, 0x54, + 0x89, 0x6e, 0x20, 0x46, 0x2f, 0x2d, 0x82, 0x11, 0x66, 0x27, 0x3c, 0x0f, 0xb2, 0x3e, 0x3e, 0xe0, + 0x5a, 0x27, 0x2b, 0x63, 0x4c, 0x04, 0xe1, 0x03, 0xc4, 0x68, 0xa5, 0x3f, 0x5f, 0x00, 0xd3, 0xa9, + 0xbd, 0xc0, 0x79, 0x90, 0x51, 0xf3, 0x4f, 0x20, 0x95, 0x66, 0x36, 0xd6, 0x50, 0xc6, 0xac, 0xc1, + 0x17, 0x55, 0xf1, 0x15, 0xa0, 0x45, 0x55, 0xcf, 0x39, 0x95, 0xb5, 0xb4, 0xb1, 0x3a, 0x66, 0x48, + 0x54, 0x38, 0x99, 0x0d, 0xa4, 0x2e, 0xb3, 0x44, 0xd8, 0x40, 0xea, 0x88, 0xd1, 0x3e, 0xeb, 0x1c, + 0x2b, 0x1a, 0xa4, 0xe5, 0x4e, 0x30, 0x48, 0x1b, 0x7d, 0xe8, 0x20, 0xed, 0x12, 0xc8, 0x05, 0x66, + 0x60, 0x11, 0x7d, 0xac, 0xf3, 0xe5, 0x71, 0x93, 0x11, 0x91, 0xe0, 0xc1, 0x3b, 0x60, 0xac, 0x46, + 0xea, 0x38, 0xb4, 0x02, 0x3d, 0xcf, 0x43, 0x68, 0x75, 0x00, 0x21, 0x24, 0xa6, 0x9c, 0x6b, 0x42, + 0x2f, 0x8a, 0x00, 0xe0, 0xd3, 0x60, 0xcc, 0xc6, 0x87, 0xa6, 0x1d, 0xda, 0xbc, 0x27, 0xd3, 0x84, + 0xd8, 0xa6, 0x20, 0xa1, 0x88, 0xc7, 0x2a, 0x23, 0x39, 0x34, 0xac, 0x90, 0x9a, 0x4d, 0x22, 0x99, + 0x3a, 0xe0, 0xb7, 0xa7, 0xaa, 0x8c, 0xeb, 0x29, 0x3e, 0xea, 0x5a, 0xc1, 0xc1, 0x4c, 0x87, 0x2f, + 0x9e, 0x48, 0x80, 0x09, 0x12, 0x8a, 0x78, 0x9d, 0x60, 0x52, 0x7e, 0xb2, 0x1f, 0x98, 0x5c, 0xdc, + 0xb5, 0x02, 0x7e, 0x19, 0x8c, 0xdb, 0xf8, 0xf0, 0x06, 0x71, 0x1a, 0xc1, 0x9e, 0x3e, 0xb5, 0xa0, + 0x2d, 0x65, 0x2b, 0x53, 0xed, 0x56, 0x71, 0x7c, 0x33, 0x22, 0xa2, 0x98, 0xcf, 0x85, 0x4d, 0x47, + 0x0a, 0x9f, 0x4e, 0x08, 0x47, 0x44, 0x14, 0xf3, 0x59, 0x07, 0xe1, 0xe1, 0x80, 0x25, 0x97, 0x3e, + 0xdd, 0xf9, 0x32, 0xdc, 0x11, 0x64, 0x14, 0xf1, 0xe1, 0x12, 0xc8, 0xdb, 0xf8, 0x90, 0xbf, 0xe2, + 0xf5, 0x19, 0xae, 0x96, 0x4f, 0x7c, 0x37, 0x25, 0x0d, 0x29, 0x2e, 0x97, 0x34, 0x1d, 0x21, 0x39, + 0x9b, 0x90, 0x94, 0x34, 0xa4, 0xb8, 0x2c, 0x88, 0x43, 0xc7, 0xbc, 0x1b, 0x12, 0x21, 0x0c, 0xb9, + 0x67, 0x54, 0x10, 0xdf, 0x8a, 0x59, 0x28, 0x29, 0xc7, 0x5e, 0xd1, 0x76, 0x68, 0x05, 0xa6, 0x67, + 0x91, 0xed, 0xba, 0x7e, 0x86, 0xfb, 0x9f, 0xf7, 0xc9, 0x9b, 0x8a, 0x8a, 0x12, 0x12, 0x90, 0x80, + 0x11, 0xe2, 0x84, 0xb6, 0x7e, 0x96, 0x5f, 0xec, 0x03, 0x09, 0x41, 0x95, 0x39, 0xeb, 0x4e, 0x68, + 0x23, 0xae, 0x1e, 0xbe, 0x08, 0xa6, 0x6c, 0x7c, 0xc8, 0xca, 0x01, 0xf1, 0x03, 0xf6, 0xbe, 0x9f, + 0xe3, 0x9b, 0x9f, 0x65, 0x1d, 0xe7, 0x66, 0x92, 0x81, 0x3a, 0xe5, 0xf8, 0x42, 0xd3, 0x49, 0x2c, + 0x3c, 0x97, 0x58, 0x98, 0x64, 0xa0, 0x4e, 0x39, 0xe6, 0x69, 0x9f, 0xdc, 0x0d, 0x4d, 0x9f, 0xd4, + 0xf4, 0xa7, 0x78, 0x93, 0x2a, 0xa7, 0xf0, 0x82, 0x86, 0x14, 0x17, 0x36, 0xa3, 0x71, 0x8f, 0xce, + 0xd3, 0xf0, 0xd6, 0x60, 0x2b, 0xf9, 0xb6, 0xbf, 0xe2, 0xfb, 0xf8, 0x48, 0xdc, 0x34, 0xc9, 0x41, + 0x0f, 0xa4, 0x20, 0x87, 0x2d, 0x6b, 0xbb, 0xae, 0x9f, 0xe7, 0xbe, 0x1f, 0xf4, 0x0d, 0xa2, 0xaa, + 0xce, 0x0a, 0x03, 0x41, 0x02, 0x8b, 0x81, 0xba, 0x0e, 0x0b, 0x8d, 0xf9, 0xe1, 0x82, 0x6e, 0x33, + 0x10, 0x24, 0xb0, 0xf8, 0x4e, 0x9d, 0xa3, 0xed, 0xba, 0xfe, 0x85, 0x21, 0xef, 0x94, 0x81, 0x20, + 0x81, 0x05, 0x4d, 0x90, 0x75, 0xdc, 0x40, 0xbf, 0x30, 0x94, 0xeb, 0x99, 0x5f, 0x38, 0x5b, 0x6e, + 0x80, 0x18, 0x06, 0xfc, 0xa5, 0x06, 0x80, 0x17, 0x87, 0xe8, 0xc5, 0x81, 0x4c, 0x11, 0x52, 0x90, + 0xe5, 0x38, 0xb6, 0xd7, 0x9d, 0xc0, 0x3f, 0x8a, 0xdf, 0x91, 0x89, 0x1c, 0x48, 0x58, 0x01, 0x7f, + 0xaf, 0x81, 0xb3, 0xc9, 0x36, 0x59, 0x99, 0x57, 0xe0, 0x1e, 0xb9, 0x39, 0xe8, 0x30, 0xaf, 0xb8, + 0xae, 0x55, 0xd1, 0xdb, 0xad, 0xe2, 0xd9, 0x95, 0x1e, 0xa8, 0xa8, 0xa7, 0x2d, 0xf0, 0x2f, 0x1a, + 0x98, 0x95, 0x55, 0x34, 0x61, 0x61, 0x91, 0x3b, 0x90, 0x0c, 0xda, 0x81, 0x69, 0x1c, 0xe1, 0x47, + 0xf5, 0xeb, 0x71, 0x17, 0x1f, 0x75, 0x9b, 0x06, 0xff, 0xae, 0x81, 0xc9, 0x1a, 0xf1, 0x88, 0x53, + 0x23, 0x8e, 0xc1, 0x6c, 0x5d, 0x18, 0xc8, 0xd8, 0x20, 0x6d, 0xeb, 0x5a, 0x02, 0x42, 0x98, 0x59, + 0x96, 0x66, 0x4e, 0x26, 0x59, 0xc7, 0xad, 0xe2, 0xb9, 0x78, 0x69, 0x92, 0x83, 0x3a, 0xac, 0x84, + 0x1f, 0x69, 0x60, 0x3a, 0x3e, 0x00, 0x71, 0xa5, 0x2c, 0x0e, 0x31, 0x0e, 0x78, 0xfb, 0xba, 0xd2, + 0x09, 0x88, 0xd2, 0x16, 0xc0, 0xbf, 0x6a, 0xac, 0x53, 0x8b, 0xde, 0x7d, 0x54, 0x2f, 0x71, 0x5f, + 0xbe, 0x3d, 0x70, 0x5f, 0x2a, 0x04, 0xe1, 0xca, 0xcb, 0x71, 0x2b, 0xa8, 0x38, 0xc7, 0xad, 0xe2, + 0x5c, 0xd2, 0x93, 0x8a, 0x81, 0x92, 0x16, 0xc2, 0x9f, 0x68, 0x60, 0x92, 0xc4, 0x1d, 0x37, 0xd5, + 0x2f, 0x0d, 0xc4, 0x89, 0x3d, 0x9b, 0x78, 0xf1, 0x52, 0x4f, 0xb0, 0x28, 0xea, 0xc0, 0x66, 0x1d, + 0x24, 0x39, 0xc4, 0xb6, 0x67, 0x11, 0xfd, 0x8b, 0x03, 0xee, 0x20, 0xd7, 0x85, 0x5e, 0x14, 0x01, + 0xcc, 0xb3, 0x97, 0x4f, 0x2a, 0x73, 0xe0, 0x0c, 0xc8, 0xee, 0x13, 0xf9, 0x33, 0x2c, 0x62, 0x7f, + 0xc2, 0x1a, 0xc8, 0x35, 0xb1, 0x15, 0x46, 0x8f, 0xb7, 0x01, 0x57, 0x5d, 0x24, 0x94, 0xbf, 0x9c, + 0x79, 0x49, 0x9b, 0xbf, 0xa7, 0x81, 0x73, 0xbd, 0x13, 0xfa, 0x89, 0x9a, 0xf5, 0x1b, 0x0d, 0xcc, + 0x76, 0xe5, 0x6e, 0x0f, 0x8b, 0xee, 0x76, 0x5a, 0xf4, 0xfa, 0xa0, 0x93, 0xb0, 0x1a, 0xf8, 0xa6, + 0xd3, 0xe0, 0x9d, 0x47, 0xd2, 0xbc, 0x9f, 0x6b, 0x60, 0x26, 0x9d, 0x0e, 0x4f, 0xd2, 0x5f, 0xa5, + 0x7b, 0x19, 0x70, 0xae, 0x77, 0xc3, 0x04, 0x7d, 0xf5, 0x32, 0x1c, 0xce, 0x0b, 0xbb, 0xd7, 0x34, + 0xee, 0x03, 0x0d, 0x4c, 0xdc, 0x51, 0x72, 0xd1, 0x0f, 0x80, 0x03, 0x7f, 0xdb, 0x47, 0xf5, 0x27, + 0x66, 0x50, 0x94, 0xc4, 0x2d, 0xfd, 0x4d, 0x03, 0x73, 0x3d, 0x0b, 0x2b, 0x7b, 0x82, 0x62, 0xcb, + 0x72, 0x0f, 0xc4, 0x88, 0x26, 0x31, 0xff, 0x5c, 0xe1, 0x54, 0x24, 0xb9, 0x09, 0xef, 0x65, 0x3e, + 0x2f, 0xef, 0x95, 0xfe, 0xa9, 0x81, 0x0b, 0x0f, 0x8b, 0xc4, 0x27, 0x72, 0xa4, 0x4b, 0x20, 0x2f, + 0x9b, 0xa2, 0x23, 0x7e, 0x9c, 0xf2, 0x1d, 0x20, 0x8b, 0x06, 0xff, 0xcf, 0x14, 0xf1, 0x57, 0xe9, + 0x7d, 0x0d, 0xcc, 0x54, 0x89, 0xdf, 0x34, 0x0d, 0x82, 0x48, 0x9d, 0xf8, 0xc4, 0x31, 0x08, 0x5c, + 0x06, 0xe3, 0xfc, 0x97, 0x37, 0x0f, 0x1b, 0xd1, 0x58, 0x7a, 0x56, 0xba, 0x7c, 0x7c, 0x2b, 0x62, + 0xa0, 0x58, 0x46, 0x8d, 0xb0, 0x33, 0x7d, 0x47, 0xd8, 0x17, 0xc0, 0x88, 0x17, 0x0f, 0xf8, 0xf2, + 0x8c, 0xcb, 0x67, 0x7a, 0x9c, 0x5a, 0xfa, 0x97, 0x06, 0x7a, 0xfd, 0x97, 0x08, 0x6c, 0x82, 0x31, + 0x2a, 0x8c, 0x93, 0xce, 0xdb, 0x7e, 0x4c, 0xe7, 0xa5, 0xb7, 0x2a, 0x0a, 0x7f, 0x44, 0x8d, 0xc0, + 0x98, 0xff, 0x0c, 0x5c, 0x09, 0x9d, 0x9a, 0x1c, 0xc9, 0x4d, 0x0a, 0xff, 0xad, 0xae, 0x08, 0x1a, + 0x52, 0x5c, 0x78, 0x5e, 0x0c, 0x8f, 0x12, 0x13, 0x99, 0x68, 0x70, 0x54, 0xb9, 0x72, 0xff, 0x41, + 0xe1, 0xd4, 0xc7, 0x0f, 0x0a, 0xa7, 0x3e, 0x79, 0x50, 0x38, 0xf5, 0x83, 0x76, 0x41, 0xbb, 0xdf, + 0x2e, 0x68, 0x1f, 0xb7, 0x0b, 0xda, 0x27, 0xed, 0x82, 0xf6, 0xef, 0x76, 0x41, 0xfb, 0xc5, 0xa7, + 0x85, 0x53, 0xdf, 0x19, 0x93, 0xa6, 0xfd, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xe7, 0xc5, 0xe4, 0x3a, + 0xbb, 0x29, 0x00, 0x00, } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto index 615c03060b1..8ee83cf5670 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto @@ -203,10 +203,14 @@ message CustomResourceDefinitionSpec { optional string scope = 4; // Validation describes the validation methods for CustomResources + // Optional, the global validation schema for all versions. + // Top-level and per-version schemas are mutually exclusive. // +optional optional CustomResourceValidation validation = 5; - // Subresources describes the subresources for CustomResources + // Subresources describes the subresources for CustomResource + // Optional, the global subresources for all versions. + // Top-level and per-version subresources are mutually exclusive. // +optional optional CustomResourceSubresources subresources = 6; @@ -225,6 +229,8 @@ message CustomResourceDefinitionSpec { repeated CustomResourceDefinitionVersion versions = 7; // AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column. + // Optional, the global columns for all versions. + // Top-level and per-version columns are mutually exclusive. // +optional repeated CustomResourceColumnDefinition additionalPrinterColumns = 8; @@ -262,6 +268,30 @@ message CustomResourceDefinitionVersion { // Storage flags the version as storage version. There must be exactly one // flagged as storage version. optional bool storage = 3; + + // Schema describes the schema for CustomResource used in validation, pruning, and defaulting. + // Top-level and per-version schemas are mutually exclusive. + // Per-version schemas must not all be set to identical values (top-level validation schema should be used instead) + // This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature. + // +optional + optional CustomResourceValidation schema = 4; + + // Subresources describes the subresources for CustomResource + // Top-level and per-version subresources are mutually exclusive. + // Per-version subresources must not all be set to identical values (top-level subresources should be used instead) + // This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature. + // +optional + optional CustomResourceSubresources subresources = 5; + + // AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column. + // Top-level and per-version columns are mutually exclusive. + // Per-version columns must not all be set to identical values (top-level columns should be used instead) + // This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature. + // NOTE: CRDs created prior to 1.13 populated the top-level additionalPrinterColumns field by default. To apply an + // update that changes to per-version additionalPrinterColumns, the top-level additionalPrinterColumns field must + // be explicitly set to null + // +optional + repeated CustomResourceColumnDefinition additionalPrinterColumns = 6; } // CustomResourceSubresourceScale defines how to serve the scale subresource for CustomResources. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go index f810ef768ab..e99d9e49b52 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go @@ -47,9 +47,13 @@ type CustomResourceDefinitionSpec struct { // Scope indicates whether this resource is cluster or namespace scoped. Default is namespaced Scope ResourceScope `json:"scope" protobuf:"bytes,4,opt,name=scope,casttype=ResourceScope"` // Validation describes the validation methods for CustomResources + // Optional, the global validation schema for all versions. + // Top-level and per-version schemas are mutually exclusive. // +optional Validation *CustomResourceValidation `json:"validation,omitempty" protobuf:"bytes,5,opt,name=validation"` - // Subresources describes the subresources for CustomResources + // Subresources describes the subresources for CustomResource + // Optional, the global subresources for all versions. + // Top-level and per-version subresources are mutually exclusive. // +optional Subresources *CustomResourceSubresources `json:"subresources,omitempty" protobuf:"bytes,6,opt,name=subresources"` // Versions is the list of all supported versions for this resource. @@ -66,6 +70,8 @@ type CustomResourceDefinitionSpec struct { // +optional Versions []CustomResourceDefinitionVersion `json:"versions,omitempty" protobuf:"bytes,7,rep,name=versions"` // AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column. + // Optional, the global columns for all versions. + // Top-level and per-version columns are mutually exclusive. // +optional AdditionalPrinterColumns []CustomResourceColumnDefinition `json:"additionalPrinterColumns,omitempty" protobuf:"bytes,8,rep,name=additionalPrinterColumns"` @@ -159,6 +165,27 @@ type CustomResourceDefinitionVersion struct { // Storage flags the version as storage version. There must be exactly one // flagged as storage version. Storage bool `json:"storage" protobuf:"varint,3,opt,name=storage"` + // Schema describes the schema for CustomResource used in validation, pruning, and defaulting. + // Top-level and per-version schemas are mutually exclusive. + // Per-version schemas must not all be set to identical values (top-level validation schema should be used instead) + // This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature. + // +optional + Schema *CustomResourceValidation `json:"schema,omitempty" protobuf:"bytes,4,opt,name=schema"` + // Subresources describes the subresources for CustomResource + // Top-level and per-version subresources are mutually exclusive. + // Per-version subresources must not all be set to identical values (top-level subresources should be used instead) + // This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature. + // +optional + Subresources *CustomResourceSubresources `json:"subresources,omitempty" protobuf:"bytes,5,opt,name=subresources"` + // AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column. + // Top-level and per-version columns are mutually exclusive. + // Per-version columns must not all be set to identical values (top-level columns should be used instead) + // This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature. + // NOTE: CRDs created prior to 1.13 populated the top-level additionalPrinterColumns field by default. To apply an + // update that changes to per-version additionalPrinterColumns, the top-level additionalPrinterColumns field must + // be explicitly set to null + // +optional + AdditionalPrinterColumns []CustomResourceColumnDefinition `json:"additionalPrinterColumns,omitempty" protobuf:"bytes,6,rep,name=additionalPrinterColumns"` } // CustomResourceColumnDefinition specifies a column for server side printing. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go index 0980caef8bc..cab8019b458 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go @@ -464,7 +464,17 @@ func autoConvert_v1beta1_CustomResourceDefinitionSpec_To_apiextensions_CustomRes out.Validation = nil } out.Subresources = (*apiextensions.CustomResourceSubresources)(unsafe.Pointer(in.Subresources)) - out.Versions = *(*[]apiextensions.CustomResourceDefinitionVersion)(unsafe.Pointer(&in.Versions)) + if in.Versions != nil { + in, out := &in.Versions, &out.Versions + *out = make([]apiextensions.CustomResourceDefinitionVersion, len(*in)) + for i := range *in { + if err := Convert_v1beta1_CustomResourceDefinitionVersion_To_apiextensions_CustomResourceDefinitionVersion(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Versions = nil + } out.AdditionalPrinterColumns = *(*[]apiextensions.CustomResourceColumnDefinition)(unsafe.Pointer(&in.AdditionalPrinterColumns)) out.Conversion = (*apiextensions.CustomResourceConversion)(unsafe.Pointer(in.Conversion)) return nil @@ -492,7 +502,17 @@ func autoConvert_apiextensions_CustomResourceDefinitionSpec_To_v1beta1_CustomRes out.Validation = nil } out.Subresources = (*CustomResourceSubresources)(unsafe.Pointer(in.Subresources)) - out.Versions = *(*[]CustomResourceDefinitionVersion)(unsafe.Pointer(&in.Versions)) + if in.Versions != nil { + in, out := &in.Versions, &out.Versions + *out = make([]CustomResourceDefinitionVersion, len(*in)) + for i := range *in { + if err := Convert_apiextensions_CustomResourceDefinitionVersion_To_v1beta1_CustomResourceDefinitionVersion(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Versions = nil + } out.AdditionalPrinterColumns = *(*[]CustomResourceColumnDefinition)(unsafe.Pointer(&in.AdditionalPrinterColumns)) out.Conversion = (*CustomResourceConversion)(unsafe.Pointer(in.Conversion)) return nil @@ -535,6 +555,17 @@ func autoConvert_v1beta1_CustomResourceDefinitionVersion_To_apiextensions_Custom out.Name = in.Name out.Served = in.Served out.Storage = in.Storage + if in.Schema != nil { + in, out := &in.Schema, &out.Schema + *out = new(apiextensions.CustomResourceValidation) + if err := Convert_v1beta1_CustomResourceValidation_To_apiextensions_CustomResourceValidation(*in, *out, s); err != nil { + return err + } + } else { + out.Schema = nil + } + out.Subresources = (*apiextensions.CustomResourceSubresources)(unsafe.Pointer(in.Subresources)) + out.AdditionalPrinterColumns = *(*[]apiextensions.CustomResourceColumnDefinition)(unsafe.Pointer(&in.AdditionalPrinterColumns)) return nil } @@ -547,6 +578,17 @@ func autoConvert_apiextensions_CustomResourceDefinitionVersion_To_v1beta1_Custom out.Name = in.Name out.Served = in.Served out.Storage = in.Storage + if in.Schema != nil { + in, out := &in.Schema, &out.Schema + *out = new(CustomResourceValidation) + if err := Convert_apiextensions_CustomResourceValidation_To_v1beta1_CustomResourceValidation(*in, *out, s); err != nil { + return err + } + } else { + out.Schema = nil + } + out.Subresources = (*CustomResourceSubresources)(unsafe.Pointer(in.Subresources)) + out.AdditionalPrinterColumns = *(*[]CustomResourceColumnDefinition)(unsafe.Pointer(&in.AdditionalPrinterColumns)) return nil } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.deepcopy.go index e98a7acf923..8dd7a87bfcf 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.deepcopy.go @@ -264,7 +264,9 @@ func (in *CustomResourceDefinitionSpec) DeepCopyInto(out *CustomResourceDefiniti if in.Versions != nil { in, out := &in.Versions, &out.Versions *out = make([]CustomResourceDefinitionVersion, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.AdditionalPrinterColumns != nil { in, out := &in.AdditionalPrinterColumns, &out.AdditionalPrinterColumns @@ -321,6 +323,21 @@ func (in *CustomResourceDefinitionStatus) DeepCopy() *CustomResourceDefinitionSt // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CustomResourceDefinitionVersion) DeepCopyInto(out *CustomResourceDefinitionVersion) { *out = *in + if in.Schema != nil { + in, out := &in.Schema, &out.Schema + *out = new(CustomResourceValidation) + (*in).DeepCopyInto(*out) + } + if in.Subresources != nil { + in, out := &in.Subresources, &out.Subresources + *out = new(CustomResourceSubresources) + (*in).DeepCopyInto(*out) + } + if in.AdditionalPrinterColumns != nil { + in, out := &in.AdditionalPrinterColumns, &out.AdditionalPrinterColumns + *out = make([]CustomResourceColumnDefinition, len(*in)) + copy(*out, *in) + } return } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/BUILD index ed76581c30e..1c80d0c5a0d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/BUILD @@ -15,6 +15,7 @@ go_library( "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/validation:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go index 91b01ce70ed..4f16ccce878 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go @@ -18,15 +18,16 @@ package validation import ( "fmt" - "k8s.io/apiserver/pkg/util/webhook" "reflect" "strings" + apiequality "k8s.io/apimachinery/pkg/api/equality" genericvalidation "k8s.io/apimachinery/pkg/api/validation" "k8s.io/apimachinery/pkg/util/sets" validationutil "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/apiserver/pkg/util/webhook" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" @@ -99,6 +100,17 @@ func ValidateUpdateCustomResourceDefinitionStatus(obj, oldObj *apiextensions.Cus return allErrs } +// ValidateCustomResourceDefinitionVersion statically validates. +func ValidateCustomResourceDefinitionVersion(version *apiextensions.CustomResourceDefinitionVersion, fldPath *field.Path, statusEnabled bool) field.ErrorList { + allErrs := field.ErrorList{} + allErrs = append(allErrs, ValidateCustomResourceDefinitionValidation(version.Schema, statusEnabled, fldPath.Child("schema"))...) + allErrs = append(allErrs, ValidateCustomResourceDefinitionSubresources(version.Subresources, fldPath.Child("subresources"))...) + for i := range version.AdditionalPrinterColumns { + allErrs = append(allErrs, ValidateCustomResourceColumnDefinition(&version.AdditionalPrinterColumns[i], fldPath.Child("additionalPrinterColumns").Index(i))...) + } + return allErrs +} + // ValidateCustomResourceDefinitionSpec statically validates func ValidateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefinitionSpec, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} @@ -128,7 +140,32 @@ func ValidateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefi if errs := validationutil.IsDNS1035Label(version.Name); len(errs) > 0 { allErrs = append(allErrs, field.Invalid(fldPath.Child("versions").Index(i).Child("name"), spec.Versions[i].Name, strings.Join(errs, ","))) } + subresources := getSubresourcesForVersion(spec, version.Name) + allErrs = append(allErrs, ValidateCustomResourceDefinitionVersion(&version, fldPath.Child("versions").Index(i), hasStatusEnabled(subresources))...) } + + // The top-level and per-version fields are mutual exclusive + if spec.Validation != nil && hasPerVersionSchema(spec.Versions) { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("validation"), "top-level and per-version schemas are mutually exclusive")) + } + if spec.Subresources != nil && hasPerVersionSubresources(spec.Versions) { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("subresources"), "top-level and per-version subresources are mutually exclusive")) + } + if len(spec.AdditionalPrinterColumns) > 0 && hasPerVersionColumns(spec.Versions) { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("additionalPrinterColumns"), "top-level and per-version additionalPrinterColumns are mutually exclusive")) + } + + // Per-version fields may not all be set to identical values (top-level field should be used instead) + if hasIdenticalPerVersionSchema(spec.Versions) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("versions"), spec.Versions, "per-version schemas may not all be set to identical values (top-level validation should be used instead)")) + } + if hasIdenticalPerVersionSubresources(spec.Versions) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("versions"), spec.Versions, "per-version subresources may not all be set to identical values (top-level subresources should be used instead)")) + } + if hasIdenticalPerVersionColumns(spec.Versions) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("versions"), spec.Versions, "per-version additionalPrinterColumns may not all be set to identical values (top-level additionalPrinterColumns should be used instead)")) + } + if !uniqueNames { allErrs = append(allErrs, field.Invalid(fldPath.Child("versions"), spec.Versions, "must contain unique version names")) } @@ -161,11 +198,7 @@ func ValidateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefi allErrs = append(allErrs, ValidateCustomResourceDefinitionNames(&spec.Names, fldPath.Child("names"))...) if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceValidation) { - statusEnabled := false - if spec.Subresources != nil && spec.Subresources.Status != nil { - statusEnabled = true - } - allErrs = append(allErrs, ValidateCustomResourceDefinitionValidation(spec.Validation, statusEnabled, fldPath.Child("validation"))...) + allErrs = append(allErrs, ValidateCustomResourceDefinitionValidation(spec.Validation, hasAnyStatusEnabled(spec), fldPath.Child("validation"))...) } else if spec.Validation != nil { allErrs = append(allErrs, field.Forbidden(fldPath.Child("validation"), "disabled by feature-gate CustomResourceValidation")) } @@ -177,7 +210,7 @@ func ValidateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefi } for i := range spec.AdditionalPrinterColumns { - if errs := ValidateCustomResourceColumnDefinition(&spec.AdditionalPrinterColumns[i], fldPath.Child("columns").Index(i)); len(errs) > 0 { + if errs := ValidateCustomResourceColumnDefinition(&spec.AdditionalPrinterColumns[i], fldPath.Child("additionalPrinterColumns").Index(i)); len(errs) > 0 { allErrs = append(allErrs, errs...) } } @@ -250,6 +283,118 @@ func ValidateCustomResourceDefinitionSpecUpdate(spec, oldSpec *apiextensions.Cus return allErrs } +// getSubresourcesForVersion returns the subresources for given version in given CRD spec. +// NOTE That this function assumes version always exist since it's used by the validation process +// that iterates through the existing versions. +func getSubresourcesForVersion(crd *apiextensions.CustomResourceDefinitionSpec, version string) *apiextensions.CustomResourceSubresources { + if !hasPerVersionSubresources(crd.Versions) { + return crd.Subresources + } + for _, v := range crd.Versions { + if version == v.Name { + return v.Subresources + } + } + return nil +} + +// hasAnyStatusEnabled returns true if given CRD spec has at least one Status Subresource set +// among the top-level and per-version Subresources. +func hasAnyStatusEnabled(crd *apiextensions.CustomResourceDefinitionSpec) bool { + if hasStatusEnabled(crd.Subresources) { + return true + } + for _, v := range crd.Versions { + if hasStatusEnabled(v.Subresources) { + return true + } + } + return false +} + +// hasStatusEnabled returns true if given CRD Subresources has non-nil Status set. +func hasStatusEnabled(subresources *apiextensions.CustomResourceSubresources) bool { + if subresources != nil && subresources.Status != nil { + return true + } + return false +} + +// hasPerVersionSchema returns true if a CRD uses per-version schema. +func hasPerVersionSchema(versions []apiextensions.CustomResourceDefinitionVersion) bool { + for _, v := range versions { + if v.Schema != nil { + return true + } + } + return false +} + +// hasPerVersionSubresources returns true if a CRD uses per-version subresources. +func hasPerVersionSubresources(versions []apiextensions.CustomResourceDefinitionVersion) bool { + for _, v := range versions { + if v.Subresources != nil { + return true + } + } + return false +} + +// hasPerVersionColumns returns true if a CRD uses per-version columns. +func hasPerVersionColumns(versions []apiextensions.CustomResourceDefinitionVersion) bool { + for _, v := range versions { + if len(v.AdditionalPrinterColumns) > 0 { + return true + } + } + return false +} + +// hasIdenticalPerVersionSchema returns true if a CRD sets identical non-nil values +// to all per-version schemas +func hasIdenticalPerVersionSchema(versions []apiextensions.CustomResourceDefinitionVersion) bool { + if len(versions) == 0 { + return false + } + value := versions[0].Schema + for _, v := range versions { + if v.Schema == nil || !apiequality.Semantic.DeepEqual(v.Schema, value) { + return false + } + } + return true +} + +// hasIdenticalPerVersionSubresources returns true if a CRD sets identical non-nil values +// to all per-version subresources +func hasIdenticalPerVersionSubresources(versions []apiextensions.CustomResourceDefinitionVersion) bool { + if len(versions) == 0 { + return false + } + value := versions[0].Subresources + for _, v := range versions { + if v.Subresources == nil || !apiequality.Semantic.DeepEqual(v.Subresources, value) { + return false + } + } + return true +} + +// hasIdenticalPerVersionColumns returns true if a CRD sets identical non-nil values +// to all per-version columns +func hasIdenticalPerVersionColumns(versions []apiextensions.CustomResourceDefinitionVersion) bool { + if len(versions) == 0 { + return false + } + value := versions[0].AdditionalPrinterColumns + for _, v := range versions { + if len(v.AdditionalPrinterColumns) == 0 || !apiequality.Semantic.DeepEqual(v.AdditionalPrinterColumns, value) { + return false + } + } + return true +} + // ValidateCustomResourceDefinitionStatus statically validates func ValidateCustomResourceDefinitionStatus(status *apiextensions.CustomResourceDefinitionStatus, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go index 24cd454bb81..692aa8b0c0a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go @@ -594,6 +594,55 @@ func TestValidateCustomResourceDefinition(t *testing.T) { }, errors: []validationMatch{}, }, + { + name: "per-version fields may not all be set to identical values (top-level field should be used instead)", + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validValidationSchema, + }, + Subresources: &apiextensions.CustomResourceSubresources{}, + AdditionalPrinterColumns: []apiextensions.CustomResourceColumnDefinition{{Name: "Alpha", Type: "string", JSONPath: ".spec.alpha"}}, + }, + { + Name: "version2", + Served: true, + Storage: false, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validValidationSchema, + }, + Subresources: &apiextensions.CustomResourceSubresources{}, + AdditionalPrinterColumns: []apiextensions.CustomResourceColumnDefinition{{Name: "Alpha", Type: "string", JSONPath: ".spec.alpha"}}, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{ + // Per-version schema/subresources/columns may not all be set to identical values. + // Note that the test will fail if we de-duplicate the expected errors below. + invalid("spec", "versions"), + invalid("spec", "versions"), + invalid("spec", "versions"), + }, + }, } for _, tc := range tests { @@ -1003,6 +1052,87 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { immutable("spec", "names", "plural"), }, }, + { + name: "top-level and per-version fields are mutually exclusive", + old: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "plural.group.com", + ResourceVersion: "42", + }, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + Subresources: &apiextensions.CustomResourceSubresources{}, + }, + { + Name: "version2", + Served: true, + Storage: false, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "plural.group.com", + ResourceVersion: "42", + }, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + }, + { + Name: "version2", + Served: true, + Storage: false, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validValidationSchema, + }, + Subresources: &apiextensions.CustomResourceSubresources{}, + AdditionalPrinterColumns: []apiextensions.CustomResourceColumnDefinition{{Name: "Alpha", Type: "string", JSONPath: ".spec.alpha"}}, + }, + }, + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validValidationSchema, + }, + Subresources: &apiextensions.CustomResourceSubresources{}, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{ + forbidden("spec", "validation"), + forbidden("spec", "subresources"), + }, + }, } for _, tc := range tests { @@ -1090,36 +1220,7 @@ func TestValidateCustomResourceDefinitionValidation(t *testing.T) { { name: "all allowed fields at the root of the schema with status", input: apiextensions.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ - Description: "This is a description", - Type: "object", - Format: "date-time", - Title: "This is a title", - Maximum: float64Ptr(10), - ExclusiveMaximum: true, - Minimum: float64Ptr(5), - ExclusiveMinimum: true, - MaxLength: int64Ptr(10), - MinLength: int64Ptr(5), - Pattern: "^[a-z]$", - MaxItems: int64Ptr(10), - MinItems: int64Ptr(5), - MultipleOf: float64Ptr(3), - Required: []string{"spec", "status"}, - Items: &apiextensions.JSONSchemaPropsOrArray{ - Schema: &apiextensions.JSONSchemaProps{ - Description: "This is a schema nested under Items", - }, - }, - Properties: map[string]apiextensions.JSONSchemaProps{ - "spec": {}, - "status": {}, - }, - ExternalDocs: &apiextensions.ExternalDocumentation{ - Description: "This is an external documentation description", - }, - Example: &example, - }, + OpenAPIV3Schema: validValidationSchema, }, statusEnabled: true, wantError: false, @@ -1139,6 +1240,37 @@ func TestValidateCustomResourceDefinitionValidation(t *testing.T) { var example = apiextensions.JSON(`"This is an example"`) +var validValidationSchema = &apiextensions.JSONSchemaProps{ + Description: "This is a description", + Type: "object", + Format: "date-time", + Title: "This is a title", + Maximum: float64Ptr(10), + ExclusiveMaximum: true, + Minimum: float64Ptr(5), + ExclusiveMinimum: true, + MaxLength: int64Ptr(10), + MinLength: int64Ptr(5), + Pattern: "^[a-z]$", + MaxItems: int64Ptr(10), + MinItems: int64Ptr(5), + MultipleOf: float64Ptr(3), + Required: []string{"spec", "status"}, + Items: &apiextensions.JSONSchemaPropsOrArray{ + Schema: &apiextensions.JSONSchemaProps{ + Description: "This is a schema nested under Items", + }, + }, + Properties: map[string]apiextensions.JSONSchemaProps{ + "spec": {}, + "status": {}, + }, + ExternalDocs: &apiextensions.ExternalDocumentation{ + Description: "This is an external documentation description", + }, + Example: &example, +} + func float64Ptr(f float64) *float64 { return &f } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/zz_generated.deepcopy.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/zz_generated.deepcopy.go index 1b4fb335382..5a01307539d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/zz_generated.deepcopy.go @@ -182,7 +182,9 @@ func (in *CustomResourceDefinitionSpec) DeepCopyInto(out *CustomResourceDefiniti if in.Versions != nil { in, out := &in.Versions, &out.Versions *out = make([]CustomResourceDefinitionVersion, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.AdditionalPrinterColumns != nil { in, out := &in.AdditionalPrinterColumns, &out.AdditionalPrinterColumns @@ -239,6 +241,21 @@ func (in *CustomResourceDefinitionStatus) DeepCopy() *CustomResourceDefinitionSt // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CustomResourceDefinitionVersion) DeepCopyInto(out *CustomResourceDefinitionVersion) { *out = *in + if in.Schema != nil { + in, out := &in.Schema, &out.Schema + *out = new(CustomResourceValidation) + (*in).DeepCopyInto(*out) + } + if in.Subresources != nil { + in, out := &in.Subresources, &out.Subresources + *out = new(CustomResourceSubresources) + (*in).DeepCopyInto(*out) + } + if in.AdditionalPrinterColumns != nil { + in, out := &in.AdditionalPrinterColumns, &out.AdditionalPrinterColumns + *out = make([]CustomResourceColumnDefinition, len(*in)) + copy(*out, *in) + } return } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD index 7bdf1a397fd..5da6c14cda3 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD @@ -13,6 +13,7 @@ go_library( "customresource_discovery.go", "customresource_discovery_controller.go", "customresource_handler.go", + "helpers.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver", importpath = "k8s.io/apiextensions-apiserver/pkg/apiserver", @@ -66,6 +67,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", "//staging/src/k8s.io/client-go/scale:go_default_library", "//staging/src/k8s.io/client-go/scale/scheme/autoscalingv1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", @@ -73,7 +75,7 @@ go_library( "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/github.com/go-openapi/strfmt:go_default_library", "//vendor/github.com/go-openapi/validate:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go index f1fc89ba96c..b6830d762c8 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go @@ -31,6 +31,7 @@ import ( "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" serverstorage "k8s.io/apiserver/pkg/server/storage" + "k8s.io/apiserver/pkg/util/webhook" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" @@ -78,6 +79,11 @@ type ExtraConfig struct { // MasterCount is used to detect whether cluster is HA, and if it is // the CRD Establishing will be hold by 5 seconds. MasterCount int + + // ServiceResolver is used in CR webhook converters to resolve webhook's service names + ServiceResolver webhook.ServiceResolver + // AuthResolverWrapper is used in CR webhook converters + AuthResolverWrapper webhook.AuthenticationInfoResolverWrapper } type Config struct { @@ -167,7 +173,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) delegate: delegateHandler, } establishingController := establish.NewEstablishingController(s.Informers.Apiextensions().InternalVersion().CustomResourceDefinitions(), crdClient.Apiextensions()) - crdHandler := NewCustomResourceDefinitionHandler( + crdHandler, err := NewCustomResourceDefinitionHandler( versionDiscoveryHandler, groupDiscoveryHandler, s.Informers.Apiextensions().InternalVersion().CustomResourceDefinitions(), @@ -175,8 +181,13 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) c.ExtraConfig.CRDRESTOptionsGetter, c.GenericConfig.AdmissionControl, establishingController, + c.ExtraConfig.ServiceResolver, + c.ExtraConfig.AuthResolverWrapper, c.ExtraConfig.MasterCount, ) + if err != nil { + return nil, err + } s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", crdHandler) s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/apis/", crdHandler) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD index 93a4b1364d1..560e9388341 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD @@ -5,15 +5,23 @@ go_library( srcs = [ "converter.go", "nop_converter.go", + "webhook_converter.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion", importpath = "k8s.io/apiextensions-apiserver/pkg/apiserver/conversion", visibility = ["//visibility:public"], deps = [ "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go index a66f82969dc..9345e9bd27b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go @@ -20,39 +20,74 @@ import ( "fmt" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/apiserver/pkg/util/webhook" ) -// NewCRDConverter returns a new CRD converter based on the conversion settings in crd object. -func NewCRDConverter(crd *apiextensions.CustomResourceDefinition) (safe, unsafe runtime.ObjectConvertor) { +// CRConverterFactory is the factory for all CR converters. +type CRConverterFactory struct { + // webhookConverterFactory is the factory for webhook converters. + // This field should not be used if CustomResourceWebhookConversion feature is disabled. + webhookConverterFactory *webhookConverterFactory +} + +// NewCRConverterFactory creates a new CRConverterFactory +func NewCRConverterFactory(serviceResolver webhook.ServiceResolver, authResolverWrapper webhook.AuthenticationInfoResolverWrapper) (*CRConverterFactory, error) { + converterFactory := &CRConverterFactory{} + if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) { + webhookConverterFactory, err := newWebhookConverterFactory(serviceResolver, authResolverWrapper) + if err != nil { + return nil, err + } + converterFactory.webhookConverterFactory = webhookConverterFactory + } + return converterFactory, nil +} + +// NewConverter returns a new CR converter based on the conversion settings in crd object. +func (m *CRConverterFactory) NewConverter(crd *apiextensions.CustomResourceDefinition) (safe, unsafe runtime.ObjectConvertor, err error) { validVersions := map[schema.GroupVersion]bool{} for _, version := range crd.Spec.Versions { validVersions[schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name}] = true } - // The only converter right now is nopConverter. More converters will be returned based on the - // CRD object when they introduced. - unsafe = &crdConverter{ - clusterScoped: crd.Spec.Scope == apiextensions.ClusterScoped, - delegate: &nopConverter{ - validVersions: validVersions, - }, + switch crd.Spec.Conversion.Strategy { + case apiextensions.NoneConverter: + unsafe = &crConverter{ + clusterScoped: crd.Spec.Scope == apiextensions.ClusterScoped, + delegate: &nopConverter{ + validVersions: validVersions, + }, + } + return &safeConverterWrapper{unsafe}, unsafe, nil + case apiextensions.WebhookConverter: + if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) { + return nil, nil, fmt.Errorf("webhook conversion is disabled on this cluster") + } + unsafe, err := m.webhookConverterFactory.NewWebhookConverter(validVersions, crd) + if err != nil { + return nil, nil, err + } + return &safeConverterWrapper{unsafe}, unsafe, nil } - return &safeConverterWrapper{unsafe}, unsafe + + return nil, nil, fmt.Errorf("unknown conversion strategy %q for CRD %s", crd.Spec.Conversion.Strategy, crd.Name) } -var _ runtime.ObjectConvertor = &crdConverter{} +var _ runtime.ObjectConvertor = &crConverter{} -// crdConverter extends the delegate with generic CRD conversion behaviour. The delegate will implement the +// crConverter extends the delegate with generic CR conversion behaviour. The delegate will implement the // user defined conversion strategy given in the CustomResourceDefinition. -type crdConverter struct { +type crConverter struct { delegate runtime.ObjectConvertor clusterScoped bool } -func (c *crdConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, value string) (string, string, error) { +func (c *crConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, value string) (string, string, error) { // We currently only support metadata.namespace and metadata.name. switch { case label == "metadata.name": @@ -64,12 +99,12 @@ func (c *crdConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, val } } -func (c *crdConverter) Convert(in, out, context interface{}) error { +func (c *crConverter) Convert(in, out, context interface{}) error { return c.delegate.Convert(in, out, context) } // ConvertToVersion converts in object to the given gvk in place and returns the same `in` object. -func (c *crdConverter) ConvertToVersion(in runtime.Object, target runtime.GroupVersioner) (runtime.Object, error) { +func (c *crConverter) ConvertToVersion(in runtime.Object, target runtime.GroupVersioner) (runtime.Object, error) { // Run the converter on the list items instead of list itself if list, ok := in.(*unstructured.UnstructuredList); ok { for i := range list.Items { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/nop_converter.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/nop_converter.go index 7fae8137596..8791238c5d0 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/nop_converter.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/nop_converter.go @@ -49,11 +49,11 @@ func (c *nopConverter) Convert(in, out, context interface{}) error { outGVK := unstructOut.GroupVersionKind() if !c.validVersions[outGVK.GroupVersion()] { - return fmt.Errorf("request to convert CRD from an invalid group/version: %s", outGVK.String()) + return fmt.Errorf("request to convert CR from an invalid group/version: %s", outGVK.String()) } inGVK := unstructIn.GroupVersionKind() if !c.validVersions[inGVK.GroupVersion()] { - return fmt.Errorf("request to convert CRD to an invalid group/version: %s", inGVK.String()) + return fmt.Errorf("request to convert CR to an invalid group/version: %s", inGVK.String()) } unstructOut.SetUnstructuredContent(unstructIn.UnstructuredContent()) @@ -72,7 +72,7 @@ func (c *nopConverter) ConvertToVersion(in runtime.Object, target runtime.GroupV return nil, fmt.Errorf("%v is unstructured and is not suitable for converting to %q", kind, target) } if !c.validVersions[gvk.GroupVersion()] { - return nil, fmt.Errorf("request to convert CRD to an invalid group/version: %s", gvk.String()) + return nil, fmt.Errorf("request to convert CR to an invalid group/version: %s", gvk.String()) } in.GetObjectKind().SetGroupVersionKind(gvk) return in, nil diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/webhook_converter.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/webhook_converter.go new file mode 100644 index 00000000000..3cc295ea187 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/webhook_converter.go @@ -0,0 +1,350 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conversion + +import ( + "context" + "errors" + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apiserver/pkg/util/webhook" + "k8s.io/client-go/rest" + + internal "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" +) + +type webhookConverterFactory struct { + clientManager webhook.ClientManager +} + +func newWebhookConverterFactory(serviceResolver webhook.ServiceResolver, authResolverWrapper webhook.AuthenticationInfoResolverWrapper) (*webhookConverterFactory, error) { + clientManager, err := webhook.NewClientManager(v1beta1.SchemeGroupVersion, v1beta1.AddToScheme) + if err != nil { + return nil, err + } + authInfoResolver, err := webhook.NewDefaultAuthenticationInfoResolver("") + if err != nil { + return nil, err + } + // Set defaults which may be overridden later. + clientManager.SetAuthenticationInfoResolver(authInfoResolver) + clientManager.SetAuthenticationInfoResolverWrapper(authResolverWrapper) + clientManager.SetServiceResolver(serviceResolver) + return &webhookConverterFactory{clientManager}, nil +} + +// webhookConverter is a converter that calls an external webhook to do the CR conversion. +type webhookConverter struct { + validVersions map[schema.GroupVersion]bool + clientManager webhook.ClientManager + restClient *rest.RESTClient + name string + nopConverter nopConverter +} + +func webhookClientConfigForCRD(crd *internal.CustomResourceDefinition) *webhook.ClientConfig { + apiConfig := crd.Spec.Conversion.WebhookClientConfig + ret := webhook.ClientConfig{ + Name: fmt.Sprintf("conversion_webhook_for_%s", crd.Name), + CABundle: apiConfig.CABundle, + } + if apiConfig.URL != nil { + ret.URL = *apiConfig.URL + } + if apiConfig.Service != nil { + ret.Service = &webhook.ClientConfigService{ + Name: apiConfig.Service.Name, + Namespace: apiConfig.Service.Namespace, + } + if apiConfig.Service.Path != nil { + ret.Service.Path = *apiConfig.Service.Path + } + } + return &ret +} + +var _ runtime.ObjectConvertor = &webhookConverter{} + +func (f *webhookConverterFactory) NewWebhookConverter(validVersions map[schema.GroupVersion]bool, crd *internal.CustomResourceDefinition) (*webhookConverter, error) { + restClient, err := f.clientManager.HookClient(*webhookClientConfigForCRD(crd)) + if err != nil { + return nil, err + } + return &webhookConverter{ + clientManager: f.clientManager, + validVersions: validVersions, + restClient: restClient, + name: crd.Name, + nopConverter: nopConverter{validVersions: validVersions}, + }, nil +} + +func (webhookConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, value string) (string, string, error) { + return "", "", errors.New("unstructured cannot convert field labels") +} + +func (c *webhookConverter) Convert(in, out, context interface{}) error { + unstructIn, ok := in.(*unstructured.Unstructured) + if !ok { + return fmt.Errorf("input type %T in not valid for unstructured conversion", in) + } + + unstructOut, ok := out.(*unstructured.Unstructured) + if !ok { + return fmt.Errorf("output type %T in not valid for unstructured conversion", out) + } + + outGVK := unstructOut.GroupVersionKind() + if !c.validVersions[outGVK.GroupVersion()] { + return fmt.Errorf("request to convert CR from an invalid group/version: %s", outGVK.String()) + } + inGVK := unstructIn.GroupVersionKind() + if !c.validVersions[inGVK.GroupVersion()] { + return fmt.Errorf("request to convert CR to an invalid group/version: %s", inGVK.String()) + } + + converted, err := c.ConvertToVersion(unstructIn, outGVK.GroupVersion()) + if err != nil { + return err + } + unstructuredConverted, ok := converted.(runtime.Unstructured) + if !ok { + // this should not happened + return fmt.Errorf("CR conversion failed") + } + unstructOut.SetUnstructuredContent(unstructuredConverted.UnstructuredContent()) + return nil +} + +func createConversionReview(obj runtime.Object, apiVersion string) *v1beta1.ConversionReview { + listObj, isList := obj.(*unstructured.UnstructuredList) + var objects []runtime.RawExtension + if isList { + for i := 0; i < len(listObj.Items); i++ { + // Only sent item for conversion, if the apiVersion is different + if listObj.Items[i].GetAPIVersion() != apiVersion { + objects = append(objects, runtime.RawExtension{Object: &listObj.Items[i]}) + } + } + } else { + if obj.GetObjectKind().GroupVersionKind().GroupVersion().String() != apiVersion { + objects = []runtime.RawExtension{{Object: obj}} + } + } + return &v1beta1.ConversionReview{ + Request: &v1beta1.ConversionRequest{ + Objects: objects, + DesiredAPIVersion: apiVersion, + UID: uuid.NewUUID(), + }, + Response: &v1beta1.ConversionResponse{}, + } +} + +func getRawExtensionObject(rx runtime.RawExtension) (runtime.Object, error) { + if rx.Object != nil { + return rx.Object, nil + } + u := unstructured.Unstructured{} + err := u.UnmarshalJSON(rx.Raw) + if err != nil { + return nil, err + } + return &u, nil +} + +// getTargetGroupVersion returns group/version which should be used to convert in objects to. +// String version of the return item is APIVersion. +func getTargetGroupVersion(in runtime.Object, target runtime.GroupVersioner) (schema.GroupVersion, error) { + fromGVK := in.GetObjectKind().GroupVersionKind() + toGVK, ok := target.KindForGroupVersionKinds([]schema.GroupVersionKind{fromGVK}) + if !ok { + // TODO: should this be a typed error? + return schema.GroupVersion{}, fmt.Errorf("%v is unstructured and is not suitable for converting to %q", fromGVK.String(), target) + } + return toGVK.GroupVersion(), nil +} + +func (c *webhookConverter) ConvertToVersion(in runtime.Object, target runtime.GroupVersioner) (runtime.Object, error) { + // In general, the webhook should not do any defaulting or validation. A special case of that is an empty object + // conversion that must result an empty object and practically is the same as nopConverter. + // A smoke test in API machinery calls the converter on empty objects. As this case happens consistently + // it special cased here not to call webhook converter. The test initiated here: + // https://github.com/kubernetes/kubernetes/blob/dbb448bbdcb9e440eee57024ffa5f1698956a054/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go#L201 + if isEmptyUnstructuredObject(in) { + return c.nopConverter.ConvertToVersion(in, target) + } + + toGV, err := getTargetGroupVersion(in, target) + if err != nil { + return nil, err + } + if !c.validVersions[toGV] { + return nil, fmt.Errorf("request to convert CR to an invalid group/version: %s", toGV.String()) + } + fromGV := in.GetObjectKind().GroupVersionKind().GroupVersion() + if !c.validVersions[fromGV] { + return nil, fmt.Errorf("request to convert CR from an invalid group/version: %s", fromGV.String()) + } + listObj, isList := in.(*unstructured.UnstructuredList) + if isList { + for i, item := range listObj.Items { + fromGV := item.GroupVersionKind().GroupVersion() + if !c.validVersions[fromGV] { + return nil, fmt.Errorf("input list has invalid group/version `%v` at `%v` index", fromGV, i) + } + } + } + + request := createConversionReview(in, toGV.String()) + if len(request.Request.Objects) == 0 { + if !isList { + return in, nil + } + out := listObj.DeepCopy() + out.SetAPIVersion(toGV.String()) + return out, nil + } + response := &v1beta1.ConversionReview{} + // TODO: Figure out if adding one second timeout make sense here. + ctx := context.TODO() + r := c.restClient.Post().Context(ctx).Body(request).Do() + if err := r.Into(response); err != nil { + // TODO: Return a webhook specific error to be able to convert it to meta.Status + return nil, fmt.Errorf("calling to conversion webhook failed for %s: %v", c.name, err) + } + + if response.Response == nil { + // TODO: Return a webhook specific error to be able to convert it to meta.Status + return nil, fmt.Errorf("conversion webhook response was absent for %s", c.name) + } + + if response.Response.Result.Status != v1.StatusSuccess { + // TODO return status message as error + return nil, fmt.Errorf("conversion request failed for %v, Response: %v", in.GetObjectKind(), response) + } + + if len(response.Response.ConvertedObjects) != len(request.Request.Objects) { + return nil, fmt.Errorf("expected %v converted objects, got %v", len(request.Request.Objects), len(response.Response.ConvertedObjects)) + } + + if isList { + convertedList := listObj.DeepCopy() + // Collection of items sent for conversion is different than list items + // because only items that needed conversion has been sent. + convertedIndex := 0 + for i := 0; i < len(listObj.Items); i++ { + if listObj.Items[i].GetAPIVersion() == toGV.String() { + // This item has not been sent for conversion, skip it. + continue + } + converted, err := getRawExtensionObject(response.Response.ConvertedObjects[convertedIndex]) + convertedIndex++ + original := listObj.Items[i] + if err != nil { + return nil, fmt.Errorf("invalid converted object at index %v: %v", convertedIndex, err) + } + if e, a := toGV, converted.GetObjectKind().GroupVersionKind().GroupVersion(); e != a { + return nil, fmt.Errorf("invalid converted object at index %v: invalid groupVersion, e=%v, a=%v", convertedIndex, e, a) + } + if e, a := original.GetObjectKind().GroupVersionKind().Kind, converted.GetObjectKind().GroupVersionKind().Kind; e != a { + return nil, fmt.Errorf("invalid converted object at index %v: invalid kind, e=%v, a=%v", convertedIndex, e, a) + } + unstructConverted, ok := converted.(*unstructured.Unstructured) + if !ok { + // this should not happened + return nil, fmt.Errorf("CR conversion failed") + } + if err := validateConvertedObject(&listObj.Items[i], unstructConverted); err != nil { + return nil, fmt.Errorf("invalid converted object at index %v: %v", convertedIndex, err) + } + convertedList.Items[i] = *unstructConverted + } + convertedList.SetAPIVersion(toGV.String()) + return convertedList, nil + } + + if len(response.Response.ConvertedObjects) != 1 { + // This should not happened + return nil, fmt.Errorf("CR conversion failed") + } + converted, err := getRawExtensionObject(response.Response.ConvertedObjects[0]) + if err != nil { + return nil, err + } + if e, a := toGV, converted.GetObjectKind().GroupVersionKind().GroupVersion(); e != a { + return nil, fmt.Errorf("invalid converted object: invalid groupVersion, e=%v, a=%v", e, a) + } + if e, a := in.GetObjectKind().GroupVersionKind().Kind, converted.GetObjectKind().GroupVersionKind().Kind; e != a { + return nil, fmt.Errorf("invalid converted object: invalid kind, e=%v, a=%v", e, a) + } + unstructConverted, ok := converted.(*unstructured.Unstructured) + if !ok { + // this should not happened + return nil, fmt.Errorf("CR conversion failed") + } + unstructIn, ok := in.(*unstructured.Unstructured) + if !ok { + // this should not happened + return nil, fmt.Errorf("CR conversion failed") + } + if err := validateConvertedObject(unstructIn, unstructConverted); err != nil { + return nil, fmt.Errorf("invalid converted object: %v", err) + } + return converted, nil +} + +func validateConvertedObject(unstructIn, unstructOut *unstructured.Unstructured) error { + if e, a := unstructIn.GetKind(), unstructOut.GetKind(); e != a { + return fmt.Errorf("must have the same kind: %v != %v", e, a) + } + if e, a := unstructIn.GetName(), unstructOut.GetName(); e != a { + return fmt.Errorf("must have the same name: %v != %v", e, a) + } + if e, a := unstructIn.GetNamespace(), unstructOut.GetNamespace(); e != a { + return fmt.Errorf("must have the same namespace: %v != %v", e, a) + } + if e, a := unstructIn.GetUID(), unstructOut.GetUID(); e != a { + return fmt.Errorf("must have the same UID: %v != %v", e, a) + } + return nil +} + +// isEmptyUnstructuredObject returns true if in is an empty unstructured object, i.e. an unstructured object that does +// not have any field except apiVersion and kind. +func isEmptyUnstructuredObject(in runtime.Object) bool { + u, ok := in.(*unstructured.Unstructured) + if !ok { + return false + } + if len(u.Object) != 2 { + return false + } + if _, ok := u.Object["kind"]; !ok { + return false + } + if _, ok := u.Object["apiVersion"]; !ok { + return false + } + return true +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go index e3b3d0a4413..3b13e8357db 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go @@ -21,7 +21,7 @@ import ( "sort" "time" - "github.com/golang/glog" + "k8s.io/klog" autoscaling "k8s.io/api/autoscaling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -136,7 +136,11 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { Categories: crd.Status.AcceptedNames.Categories, }) - if crd.Spec.Subresources != nil && crd.Spec.Subresources.Status != nil { + subresources, err := getSubresourcesForVersion(crd, version.Version) + if err != nil { + return err + } + if subresources != nil && subresources.Status != nil { apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{ Name: crd.Status.AcceptedNames.Plural + "/status", Namespaced: crd.Spec.Scope == apiextensions.NamespaceScoped, @@ -145,7 +149,7 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { }) } - if crd.Spec.Subresources != nil && crd.Spec.Subresources.Scale != nil { + if subresources != nil && subresources.Scale != nil { apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{ Group: autoscaling.GroupName, Version: "v1", @@ -194,9 +198,9 @@ func sortGroupDiscoveryByKubeAwareVersion(gd []metav1.GroupVersionForDiscovery) func (c *DiscoveryController) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() - defer glog.Infof("Shutting down DiscoveryController") + defer klog.Infof("Shutting down DiscoveryController") - glog.Infof("Starting DiscoveryController") + klog.Infof("Starting DiscoveryController") if !cache.WaitForCacheSync(stopCh, c.crdsSynced) { utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) @@ -242,14 +246,14 @@ func (c *DiscoveryController) enqueue(obj *apiextensions.CustomResourceDefinitio func (c *DiscoveryController) addCustomResourceDefinition(obj interface{}) { castObj := obj.(*apiextensions.CustomResourceDefinition) - glog.V(4).Infof("Adding customresourcedefinition %s", castObj.Name) + klog.V(4).Infof("Adding customresourcedefinition %s", castObj.Name) c.enqueue(castObj) } func (c *DiscoveryController) updateCustomResourceDefinition(oldObj, newObj interface{}) { castNewObj := newObj.(*apiextensions.CustomResourceDefinition) castOldObj := oldObj.(*apiextensions.CustomResourceDefinition) - glog.V(4).Infof("Updating customresourcedefinition %s", castOldObj.Name) + klog.V(4).Infof("Updating customresourcedefinition %s", castOldObj.Name) // Enqueue both old and new object to make sure we remove and add appropriate Versions. // The working queue will resolve any duplicates and only changes will stay in the queue. c.enqueue(castNewObj) @@ -261,15 +265,15 @@ func (c *DiscoveryController) deleteCustomResourceDefinition(obj interface{}) { if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) + klog.Errorf("Couldn't get object from tombstone %#v", obj) return } castObj, ok = tombstone.Obj.(*apiextensions.CustomResourceDefinition) if !ok { - glog.Errorf("Tombstone contained object that is not expected %#v", obj) + klog.Errorf("Tombstone contained object that is not expected %#v", obj) return } } - glog.V(4).Infof("Deleting customresourcedefinition %q", castObj.Name) + klog.V(4).Infof("Deleting customresourcedefinition %q", castObj.Name) c.enqueue(castObj) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index e8d175ad99d..2ad59dfe937 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -28,7 +28,7 @@ import ( "github.com/go-openapi/spec" "github.com/go-openapi/strfmt" "github.com/go-openapi/validate" - "github.com/golang/glog" + "k8s.io/klog" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -67,6 +67,7 @@ import ( apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apiextensions-apiserver/pkg/registry/customresource" "k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor" + "k8s.io/apiserver/pkg/util/webhook" ) // crdHandler serves the `/apis` endpoint. @@ -93,6 +94,8 @@ type crdHandler struct { // MasterCount is used to implement sleep to improve // CRD establishing process for HA clusters. masterCount int + + converterFactory *conversion.CRConverterFactory } // crdInfo stores enough information to serve the storage for the custom resource @@ -129,7 +132,9 @@ func NewCustomResourceDefinitionHandler( restOptionsGetter generic.RESTOptionsGetter, admission admission.Interface, establishingController *establish.EstablishingController, - masterCount int) *crdHandler { + serviceResolver webhook.ServiceResolver, + authResolverWrapper webhook.AuthenticationInfoResolverWrapper, + masterCount int) (*crdHandler, error) { ret := &crdHandler{ versionDiscoveryHandler: versionDiscoveryHandler, groupDiscoveryHandler: groupDiscoveryHandler, @@ -147,10 +152,15 @@ func NewCustomResourceDefinitionHandler( ret.removeDeadStorage() }, }) + crConverterFactory, err := conversion.NewCRConverterFactory(serviceResolver, authResolverWrapper) + if err != nil { + return nil, err + } + ret.converterFactory = crConverterFactory ret.customStorage.Store(crdStorageMap{}) - return ret + return ret, nil } func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { @@ -220,10 +230,16 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } var handler http.HandlerFunc + subresources, err := getSubresourcesForVersion(crd, requestInfo.APIVersion) + if err != nil { + utilruntime.HandleError(err) + http.Error(w, "the server could not properly serve the CR subresources", http.StatusInternalServerError) + return + } switch { - case subresource == "status" && crd.Spec.Subresources != nil && crd.Spec.Subresources.Status != nil: + case subresource == "status" && subresources != nil && subresources.Status != nil: handler = r.serveStatus(w, req, requestInfo, crdInfo, terminating, supportedTypes) - case subresource == "scale" && crd.Spec.Subresources != nil && crd.Spec.Subresources.Scale != nil: + case subresource == "scale" && subresources != nil && subresources.Scale != nil: handler = r.serveScale(w, req, requestInfo, crdInfo, terminating, supportedTypes) case len(subresource) == 0: handler = r.serveResource(w, req, requestInfo, crdInfo, terminating, supportedTypes) @@ -334,11 +350,11 @@ func (r *crdHandler) updateCustomResourceDefinition(oldObj, newObj interface{}) return } if apiequality.Semantic.DeepEqual(&newCRD.Spec, oldInfo.spec) && apiequality.Semantic.DeepEqual(&newCRD.Status.AcceptedNames, oldInfo.acceptedNames) { - glog.V(6).Infof("Ignoring customresourcedefinition %s update because neither spec, nor accepted names changed", oldCRD.Name) + klog.V(6).Infof("Ignoring customresourcedefinition %s update because neither spec, nor accepted names changed", oldCRD.Name) return } - glog.V(4).Infof("Updating customresourcedefinition %s", oldCRD.Name) + klog.V(4).Infof("Updating customresourcedefinition %s", oldCRD.Name) // Copy because we cannot write to storageMap without a race // as it is used without locking elsewhere. @@ -378,7 +394,7 @@ func (r *crdHandler) removeDeadStorage() { } } if !found { - glog.V(4).Infof("Removing dead CRD storage for %s/%s", s.spec.Group, s.spec.Names.Kind) + klog.V(4).Infof("Removing dead CRD storage for %s/%s", s.spec.Group, s.spec.Names.Kind) for _, storage := range s.storages { // destroy only the main storage. Those for the subresources share cacher and etcd clients. storage.CustomResource.DestroyFunc() @@ -427,7 +443,10 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource scaleScopes := map[string]handlers.RequestScope{} for _, v := range crd.Spec.Versions { - safeConverter, unsafeConverter := conversion.NewCRDConverter(crd) + safeConverter, unsafeConverter, err := r.converterFactory.NewConverter(crd) + if err != nil { + return nil, err + } // In addition to Unstructured objects (Custom Resources), we also may sometimes need to // decode unversioned Options objects, so we delegate to parameterScheme for such types. parameterScheme := runtime.NewScheme() @@ -443,19 +462,28 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource typer := newUnstructuredObjectTyper(parameterScheme) creator := unstructuredCreator{} - validator, _, err := apiservervalidation.NewSchemaValidator(crd.Spec.Validation) + validationSchema, err := getSchemaForVersion(crd, v.Name) + if err != nil { + utilruntime.HandleError(err) + return nil, fmt.Errorf("the server could not properly serve the CR schema") + } + validator, _, err := apiservervalidation.NewSchemaValidator(validationSchema) if err != nil { return nil, err } var statusSpec *apiextensions.CustomResourceSubresourceStatus var statusValidator *validate.SchemaValidator - if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) && crd.Spec.Subresources != nil && crd.Spec.Subresources.Status != nil { - statusSpec = crd.Spec.Subresources.Status - + subresources, err := getSubresourcesForVersion(crd, v.Name) + if err != nil { + utilruntime.HandleError(err) + return nil, fmt.Errorf("the server could not properly serve the CR subresources") + } + if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) && subresources != nil && subresources.Status != nil { + statusSpec = subresources.Status // for the status subresource, validate only against the status schema - if crd.Spec.Validation != nil && crd.Spec.Validation.OpenAPIV3Schema != nil && crd.Spec.Validation.OpenAPIV3Schema.Properties != nil { - if statusSchema, ok := crd.Spec.Validation.OpenAPIV3Schema.Properties["status"]; ok { + if validationSchema != nil && validationSchema.OpenAPIV3Schema != nil && validationSchema.OpenAPIV3Schema.Properties != nil { + if statusSchema, ok := validationSchema.OpenAPIV3Schema.Properties["status"]; ok { openapiSchema := &spec.Schema{} if err := apiservervalidation.ConvertJSONSchemaProps(&statusSchema, openapiSchema); err != nil { return nil, err @@ -466,13 +494,18 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource } var scaleSpec *apiextensions.CustomResourceSubresourceScale - if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) && crd.Spec.Subresources != nil && crd.Spec.Subresources.Scale != nil { - scaleSpec = crd.Spec.Subresources.Scale + if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) && subresources != nil && subresources.Scale != nil { + scaleSpec = subresources.Scale } - table, err := tableconvertor.New(crd.Spec.AdditionalPrinterColumns) + columns, err := getColumnsForVersion(crd, v.Name) if err != nil { - glog.V(2).Infof("The CRD for %v has an invalid printer specification, falling back to default printing: %v", kind, err) + utilruntime.HandleError(err) + return nil, fmt.Errorf("the server could not properly serve the CR columns") + } + table, err := tableconvertor.New(columns) + if err != nil { + klog.V(2).Infof("The CRD for %v has an invalid printer specification, falling back to default printing: %v", kind, err) } storages[v.Name] = customresource.NewStorage( diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler_test.go index fa7c84e6631..62ed7cca25d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler_test.go @@ -79,14 +79,24 @@ func TestConvertFieldLabel(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - crd := apiextensions.CustomResourceDefinition{} + crd := apiextensions.CustomResourceDefinition{ + Spec: apiextensions.CustomResourceDefinitionSpec{ + Conversion: &apiextensions.CustomResourceConversion{ + Strategy: "None", + }, + }, + } if test.clusterScoped { crd.Spec.Scope = apiextensions.ClusterScoped } else { crd.Spec.Scope = apiextensions.NamespaceScoped } - _, c := conversion.NewCRDConverter(&crd) + f, err := conversion.NewCRConverterFactory(nil, nil) + if err != nil { + t.Fatal(err) + } + _, c, err := f.NewConverter(&crd) label, value, err := c.ConvertFieldLabel(schema.GroupVersionKind{}, test.label, "value") if e, a := test.expectError, err != nil; e != a { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/helpers.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/helpers.go new file mode 100644 index 00000000000..ae77bfcd5a8 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/helpers.go @@ -0,0 +1,117 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiserver + +import ( + "fmt" + + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" +) + +// getSchemaForVersion returns the validation schema for given version in given CRD. +func getSchemaForVersion(crd *apiextensions.CustomResourceDefinition, version string) (*apiextensions.CustomResourceValidation, error) { + if !hasPerVersionSchema(crd.Spec.Versions) { + return crd.Spec.Validation, nil + } + if crd.Spec.Validation != nil { + return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version schemas must be mutual exclusive", crd.Name, version) + } + for _, v := range crd.Spec.Versions { + if version == v.Name { + return v.Schema, nil + } + } + return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name) +} + +// getSubresourcesForVersion returns the subresources for given version in given CRD. +func getSubresourcesForVersion(crd *apiextensions.CustomResourceDefinition, version string) (*apiextensions.CustomResourceSubresources, error) { + if !hasPerVersionSubresources(crd.Spec.Versions) { + return crd.Spec.Subresources, nil + } + if crd.Spec.Subresources != nil { + return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version subresources must be mutual exclusive", crd.Name, version) + } + for _, v := range crd.Spec.Versions { + if version == v.Name { + return v.Subresources, nil + } + } + return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name) +} + +// getColumnsForVersion returns the columns for given version in given CRD. +// NOTE: the newly logically-defaulted columns is not pointing to the original CRD object. +// One cannot mutate the original CRD columns using the logically-defaulted columns. Please iterate through +// the original CRD object instead. +func getColumnsForVersion(crd *apiextensions.CustomResourceDefinition, version string) ([]apiextensions.CustomResourceColumnDefinition, error) { + if !hasPerVersionColumns(crd.Spec.Versions) { + return serveDefaultColumnsIfEmpty(crd.Spec.AdditionalPrinterColumns), nil + } + if len(crd.Spec.AdditionalPrinterColumns) > 0 { + return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version additionalPrinterColumns must be mutual exclusive", crd.Name, version) + } + for _, v := range crd.Spec.Versions { + if version == v.Name { + return serveDefaultColumnsIfEmpty(v.AdditionalPrinterColumns), nil + } + } + return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name) +} + +// serveDefaultColumnsIfEmpty applies logically defaulting to columns, if the input columns is empty. +// NOTE: in this way, the newly logically-defaulted columns is not pointing to the original CRD object. +// One cannot mutate the original CRD columns using the logically-defaulted columns. Please iterate through +// the original CRD object instead. +func serveDefaultColumnsIfEmpty(columns []apiextensions.CustomResourceColumnDefinition) []apiextensions.CustomResourceColumnDefinition { + if len(columns) > 0 { + return columns + } + return []apiextensions.CustomResourceColumnDefinition{ + {Name: "Age", Type: "date", Description: swaggerMetadataDescriptions["creationTimestamp"], JSONPath: ".metadata.creationTimestamp"}, + } +} + +// hasPerVersionSchema returns true if a CRD uses per-version schema. +func hasPerVersionSchema(versions []apiextensions.CustomResourceDefinitionVersion) bool { + for _, v := range versions { + if v.Schema != nil { + return true + } + } + return false +} + +// hasPerVersionSubresources returns true if a CRD uses per-version subresources. +func hasPerVersionSubresources(versions []apiextensions.CustomResourceDefinitionVersion) bool { + for _, v := range versions { + if v.Subresources != nil { + return true + } + } + return false +} + +// hasPerVersionColumns returns true if a CRD uses per-version columns. +func hasPerVersionColumns(versions []apiextensions.CustomResourceDefinitionVersion) bool { + for _, v := range versions { + if len(v.AdditionalPrinterColumns) > 0 { + return true + } + } + return false +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1/customresourcedefinition.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1/customresourcedefinition.go index f25a6ce345a..c925313a7c9 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1/customresourcedefinition.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1/customresourcedefinition.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" scheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,10 +75,15 @@ func (c *customResourceDefinitions) Get(name string, options v1.GetOptions) (res // List takes label and field selectors, and returns the list of CustomResourceDefinitions that match those selectors. func (c *customResourceDefinitions) List(opts v1.ListOptions) (result *v1beta1.CustomResourceDefinitionList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.CustomResourceDefinitionList{} err = c.client.Get(). Resource("customresourcedefinitions"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *customResourceDefinitions) List(opts v1.ListOptions) (result *v1beta1.C // Watch returns a watch.Interface that watches the requested customResourceDefinitions. func (c *customResourceDefinitions) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("customresourcedefinitions"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *customResourceDefinitions) Delete(name string, options *v1.DeleteOption // DeleteCollection deletes a collection of objects. func (c *customResourceDefinitions) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("customresourcedefinitions"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/typed/apiextensions/internalversion/customresourcedefinition.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/typed/apiextensions/internalversion/customresourcedefinition.go index f3ddc8044b7..5724386879f 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/typed/apiextensions/internalversion/customresourcedefinition.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/typed/apiextensions/internalversion/customresourcedefinition.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" scheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,10 +75,15 @@ func (c *customResourceDefinitions) Get(name string, options v1.GetOptions) (res // List takes label and field selectors, and returns the list of CustomResourceDefinitions that match those selectors. func (c *customResourceDefinitions) List(opts v1.ListOptions) (result *apiextensions.CustomResourceDefinitionList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &apiextensions.CustomResourceDefinitionList{} err = c.client.Get(). Resource("customresourcedefinitions"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *customResourceDefinitions) List(opts v1.ListOptions) (result *apiextens // Watch returns a watch.Interface that watches the requested customResourceDefinitions. func (c *customResourceDefinitions) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("customresourcedefinitions"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *customResourceDefinitions) Delete(name string, options *v1.DeleteOption // DeleteCollection deletes a collection of objects. func (c *customResourceDefinitions) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("customresourcedefinitions"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/BUILD index 48e77981168..08a69b3436c 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/BUILD @@ -14,6 +14,9 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/proxy:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", + "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go index 995a3155555..c787aa6419f 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "net" + "net/url" "github.com/spf13/pflag" @@ -30,6 +31,9 @@ import ( genericregistry "k8s.io/apiserver/pkg/registry/generic" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/util/proxy" + "k8s.io/apiserver/pkg/util/webhook" + "k8s.io/client-go/listers/core/v1" ) const defaultEtcdPathPrefix = "/registry/apiextensions.kubernetes.io" @@ -46,8 +50,12 @@ type CustomResourceDefinitionsServerOptions struct { // NewCustomResourceDefinitionsServerOptions creates default options of an apiextensions-apiserver. func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer) *CustomResourceDefinitionsServerOptions { o := &CustomResourceDefinitionsServerOptions{ - RecommendedOptions: genericoptions.NewRecommendedOptions(defaultEtcdPathPrefix, apiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion)), - APIEnablement: genericoptions.NewAPIEnablementOptions(), + RecommendedOptions: genericoptions.NewRecommendedOptions( + defaultEtcdPathPrefix, + apiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion), + genericoptions.NewProcessInfo("apiextensions-apiserver", "kube-system"), + ), + APIEnablement: genericoptions.NewAPIEnablementOptions(), StdOut: out, StdErr: errOut, @@ -94,6 +102,8 @@ func (o CustomResourceDefinitionsServerOptions) Config() (*apiserver.Config, err GenericConfig: serverConfig, ExtraConfig: apiserver.ExtraConfig{ CRDRESTOptionsGetter: NewCRDRESTOptionsGetter(*o.RecommendedOptions.Etcd), + ServiceResolver: &serviceResolver{serverConfig.SharedInformerFactory.Core().V1().Services().Lister()}, + AuthResolverWrapper: webhook.NewDefaultAuthenticationInfoResolverWrapper(nil, serverConfig.LoopbackClientConfig), }, } return config, nil @@ -114,3 +124,11 @@ func NewCRDRESTOptionsGetter(etcdOptions genericoptions.EtcdOptions) genericregi return ret } + +type serviceResolver struct { + services v1.ServiceLister +} + +func (r *serviceResolver) ResolveEndpoint(namespace, name string) (*url.URL, error) { + return proxy.ResolveCluster(r.services, namespace, name) +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/establish/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/establish/BUILD index 40c07208d8a..03d72c68487 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/establish/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/establish/BUILD @@ -16,7 +16,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/establish/establishing_controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/establish/establishing_controller.go index 6420c3a6f1e..d041485536e 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/establish/establishing_controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/establish/establishing_controller.go @@ -20,12 +20,12 @@ import ( "fmt" "time" - "github.com/golang/glog" apierrors "k8s.io/apimachinery/pkg/api/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" client "k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/typed/apiextensions/internalversion" @@ -70,8 +70,8 @@ func (ec *EstablishingController) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer ec.queue.ShutDown() - glog.Infof("Starting EstablishingController") - defer glog.Infof("Shutting down EstablishingController") + klog.Infof("Starting EstablishingController") + defer klog.Infof("Shutting down EstablishingController") if !cache.WaitForCacheSync(stopCh, ec.crdSynced) { return diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/BUILD index ecafd6a92ee..5acb5f3ce99 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/BUILD @@ -26,7 +26,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go index f881427efdf..c2ebdcf1709 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go @@ -21,7 +21,7 @@ import ( "reflect" "time" - "github.com/golang/glog" + "k8s.io/klog" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -216,7 +216,7 @@ func (c *CRDFinalizer) deleteInstances(crd *apiextensions.CustomResourceDefiniti if len(listObj.(*unstructured.UnstructuredList).Items) == 0 { return true, nil } - glog.V(2).Infof("%s.%s waiting for %d items to be removed", crd.Status.AcceptedNames.Plural, crd.Spec.Group, len(listObj.(*unstructured.UnstructuredList).Items)) + klog.V(2).Infof("%s.%s waiting for %d items to be removed", crd.Status.AcceptedNames.Plural, crd.Spec.Group, len(listObj.(*unstructured.UnstructuredList).Items)) return false, nil }) if err != nil { @@ -239,8 +239,8 @@ func (c *CRDFinalizer) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() - glog.Infof("Starting CRDFinalizer") - defer glog.Infof("Shutting down CRDFinalizer") + klog.Infof("Starting CRDFinalizer") + defer klog.Infof("Shutting down CRDFinalizer") if !cache.WaitForCacheSync(stopCh, c.crdSynced) { return diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/BUILD index d75d3dbe501..1d6f1cbbccc 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/BUILD @@ -37,7 +37,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go index f00def4b124..fe5c9479847 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -281,8 +281,8 @@ func (c *NamingConditionController) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() - glog.Infof("Starting NamingConditionController") - defer glog.Infof("Shutting down NamingConditionController") + klog.Infof("Starting NamingConditionController") + defer klog.Infof("Shutting down NamingConditionController") if !cache.WaitForCacheSync(stopCh, c.crdSynced) { return @@ -331,13 +331,13 @@ func (c *NamingConditionController) enqueue(obj *apiextensions.CustomResourceDef func (c *NamingConditionController) addCustomResourceDefinition(obj interface{}) { castObj := obj.(*apiextensions.CustomResourceDefinition) - glog.V(4).Infof("Adding %s", castObj.Name) + klog.V(4).Infof("Adding %s", castObj.Name) c.enqueue(castObj) } func (c *NamingConditionController) updateCustomResourceDefinition(obj, _ interface{}) { castObj := obj.(*apiextensions.CustomResourceDefinition) - glog.V(4).Infof("Updating %s", castObj.Name) + klog.V(4).Infof("Updating %s", castObj.Name) c.enqueue(castObj) } @@ -346,16 +346,16 @@ func (c *NamingConditionController) deleteCustomResourceDefinition(obj interface if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) + klog.Errorf("Couldn't get object from tombstone %#v", obj) return } castObj, ok = tombstone.Obj.(*apiextensions.CustomResourceDefinition) if !ok { - glog.Errorf("Tombstone contained object that is not expected %#v", obj) + klog.Errorf("Tombstone contained object that is not expected %#v", obj) return } } - glog.V(4).Infof("Deleting %q", castObj.Name) + klog.V(4).Infof("Deleting %q", castObj.Name) c.enqueue(castObj) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go index 0690e10d613..8af73d2e196 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go @@ -41,7 +41,7 @@ const ( // CustomResourceSubresources defines the subresources for CustomResources CustomResourceSubresources utilfeature.Feature = "CustomResourceSubresources" - // owner: @mbohlool + // owner: @mbohlool, @roycaihw // alpha: v1.13 // // CustomResourceWebhookConversion defines the webhook conversion for Custom Resources. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go index 8beeaa9d8f8..ee3d8eaf042 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go @@ -57,9 +57,25 @@ func (strategy) PrepareForCreate(ctx context.Context, obj runtime.Object) { // if the feature gate is disabled, drop the feature. if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceValidation) { crd.Spec.Validation = nil + for i := range crd.Spec.Versions { + crd.Spec.Versions[i].Schema = nil + } } if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) { crd.Spec.Subresources = nil + for i := range crd.Spec.Versions { + crd.Spec.Versions[i].Subresources = nil + } + } + // On CREATE, if the CustomResourceWebhookConversion feature gate is off, we auto-clear + // the per-version fields. This is to be consistent with the other built-in types, as the + // apiserver drops unknown fields. + if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) { + for i := range crd.Spec.Versions { + crd.Spec.Versions[i].Schema = nil + crd.Spec.Versions[i].Subresources = nil + crd.Spec.Versions[i].AdditionalPrinterColumns = nil + } } if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) && crd.Spec.Conversion != nil { crd.Spec.Conversion.WebhookClientConfig = nil @@ -96,10 +112,36 @@ func (strategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceValidation) { newCRD.Spec.Validation = nil oldCRD.Spec.Validation = nil + for i := range newCRD.Spec.Versions { + newCRD.Spec.Versions[i].Schema = nil + } + for i := range oldCRD.Spec.Versions { + oldCRD.Spec.Versions[i].Schema = nil + } } if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) { newCRD.Spec.Subresources = nil oldCRD.Spec.Subresources = nil + for i := range newCRD.Spec.Versions { + newCRD.Spec.Versions[i].Subresources = nil + } + for i := range oldCRD.Spec.Versions { + oldCRD.Spec.Versions[i].Subresources = nil + } + } + + // On UPDATE, if the CustomResourceWebhookConversion feature gate is off, we auto-clear + // the per-version fields if the old CRD doesn't use per-version fields already. + // This is to be consistent with the other built-in types, as the apiserver drops unknown + // fields. If the old CRD already uses per-version fields, the CRD is allowed to continue + // use per-version fields. + if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) && + !hasPerVersionField(oldCRD.Spec.Versions) { + for i := range newCRD.Spec.Versions { + newCRD.Spec.Versions[i].Schema = nil + newCRD.Spec.Versions[i].Subresources = nil + newCRD.Spec.Versions[i].AdditionalPrinterColumns = nil + } } if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) && newCRD.Spec.Conversion != nil { if oldCRD.Spec.Conversion == nil || newCRD.Spec.Conversion.WebhookClientConfig == nil { @@ -117,6 +159,16 @@ func (strategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { } } +// hasPerVersionField returns true if a CRD uses per-version schema/subresources/columns fields. +func hasPerVersionField(versions []apiextensions.CustomResourceDefinitionVersion) bool { + for _, v := range versions { + if v.Schema != nil || v.Subresources != nil || len(v.AdditionalPrinterColumns) > 0 { + return true + } + } + return false +} + // Validate validates a new CustomResourceDefinition. func (strategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { return validation.ValidateCustomResourceDefinition(obj.(*apiextensions.CustomResourceDefinition)) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD b/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD index 4ac71b28dba..eb430540562 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD @@ -26,6 +26,7 @@ go_test( "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", @@ -40,13 +41,15 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//vendor/github.com/coreos/etcd/clientv3:go_default_library", "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/resources.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/resources.go index 9a1f2d84845..7230725d4db 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/resources.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/resources.go @@ -370,13 +370,13 @@ func DeleteCustomResourceDefinition(crd *apiextensionsv1beta1.CustomResourceDefi return nil } -// CreateNewScaleClient returns a scale client. -func CreateNewScaleClient(crd *apiextensionsv1beta1.CustomResourceDefinition, config *rest.Config) (scale.ScalesGetter, error) { +// CreateNewVersionedScaleClient returns a scale client. +func CreateNewVersionedScaleClient(crd *apiextensionsv1beta1.CustomResourceDefinition, config *rest.Config, version string) (scale.ScalesGetter, error) { discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) if err != nil { return nil, err } - groupResource, err := discoveryClient.ServerResourcesForGroupVersion(crd.Spec.Group + "/" + crd.Spec.Version) + groupResource, err := discoveryClient.ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version) if err != nil { return nil, err } @@ -386,12 +386,12 @@ func CreateNewScaleClient(crd *apiextensionsv1beta1.CustomResourceDefinition, co Group: metav1.APIGroup{ Name: crd.Spec.Group, Versions: []metav1.GroupVersionForDiscovery{ - {Version: crd.Spec.Version}, + {Version: version}, }, - PreferredVersion: metav1.GroupVersionForDiscovery{Version: crd.Spec.Version}, + PreferredVersion: metav1.GroupVersionForDiscovery{Version: version}, }, VersionedResources: map[string][]metav1.APIResource{ - crd.Spec.Version: groupResource.APIResources, + version: groupResource.APIResources, }, }, } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go index 76344034564..7a7d6611e86 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go @@ -30,6 +30,8 @@ import ( "k8s.io/client-go/dynamic" ) +var swaggerMetadataDescriptions = metav1.ObjectMeta{}.SwaggerDoc() + func instantiateCustomResource(t *testing.T, instanceToCreate *unstructured.Unstructured, client dynamic.ResourceInterface, definition *apiextensionsv1beta1.CustomResourceDefinition) (*unstructured.Unstructured, error) { return instantiateVersionedCustomResource(t, instanceToCreate, client, definition, definition.Spec.Versions[0].Name) } @@ -74,8 +76,8 @@ func newNamespacedCustomResourceClient(ns string, client dynamic.Interface, crd return newNamespacedCustomResourceVersionedClient(ns, client, crd, crd.Spec.Versions[0].Name) } -// updateCustomResourceDefinitionWithRetry updates a CRD, retrying up to 5 times on version conflict errors. -func updateCustomResourceDefinitionWithRetry(client clientset.Interface, name string, update func(*apiextensionsv1beta1.CustomResourceDefinition)) (*apiextensionsv1beta1.CustomResourceDefinition, error) { +// UpdateCustomResourceDefinitionWithRetry updates a CRD, retrying up to 5 times on version conflict errors. +func UpdateCustomResourceDefinitionWithRetry(client clientset.Interface, name string, update func(*apiextensionsv1beta1.CustomResourceDefinition)) (*apiextensionsv1beta1.CustomResourceDefinition, error) { for i := 0; i < 5; i++ { crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) if err != nil { @@ -92,3 +94,97 @@ func updateCustomResourceDefinitionWithRetry(client clientset.Interface, name st } return nil, fmt.Errorf("too many retries after conflicts updating CustomResourceDefinition %q", name) } + +// getSchemaForVersion returns the validation schema for given version in given CRD. +func getSchemaForVersion(crd *apiextensionsv1beta1.CustomResourceDefinition, version string) (*apiextensionsv1beta1.CustomResourceValidation, error) { + if !hasPerVersionSchema(crd.Spec.Versions) { + return crd.Spec.Validation, nil + } + if crd.Spec.Validation != nil { + return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version schemas must be mutual exclusive", crd.Name, version) + } + for _, v := range crd.Spec.Versions { + if version == v.Name { + return v.Schema, nil + } + } + return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name) +} + +// getSubresourcesForVersion returns the subresources for given version in given CRD. +func getSubresourcesForVersion(crd *apiextensionsv1beta1.CustomResourceDefinition, version string) (*apiextensionsv1beta1.CustomResourceSubresources, error) { + if !hasPerVersionSubresources(crd.Spec.Versions) { + return crd.Spec.Subresources, nil + } + if crd.Spec.Subresources != nil { + return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version subresources must be mutual exclusive", crd.Name, version) + } + for _, v := range crd.Spec.Versions { + if version == v.Name { + return v.Subresources, nil + } + } + return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name) +} + +// getColumnsForVersion returns the columns for given version in given CRD. +// NOTE: the newly logically-defaulted columns is not pointing to the original CRD object. +// One cannot mutate the original CRD columns using the logically-defaulted columns. Please iterate through +// the original CRD object instead. +func getColumnsForVersion(crd *apiextensionsv1beta1.CustomResourceDefinition, version string) ([]apiextensionsv1beta1.CustomResourceColumnDefinition, error) { + if !hasPerVersionColumns(crd.Spec.Versions) { + return serveDefaultColumnsIfEmpty(crd.Spec.AdditionalPrinterColumns), nil + } + if len(crd.Spec.AdditionalPrinterColumns) > 0 { + return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version additionalPrinterColumns must be mutual exclusive", crd.Name, version) + } + for _, v := range crd.Spec.Versions { + if version == v.Name { + return serveDefaultColumnsIfEmpty(v.AdditionalPrinterColumns), nil + } + } + return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name) +} + +// serveDefaultColumnsIfEmpty applies logically defaulting to columns, if the input columns is empty. +// NOTE: in this way, the newly logically-defaulted columns is not pointing to the original CRD object. +// One cannot mutate the original CRD columns using the logically-defaulted columns. Please iterate through +// the original CRD object instead. +func serveDefaultColumnsIfEmpty(columns []apiextensionsv1beta1.CustomResourceColumnDefinition) []apiextensionsv1beta1.CustomResourceColumnDefinition { + if len(columns) > 0 { + return columns + } + return []apiextensionsv1beta1.CustomResourceColumnDefinition{ + {Name: "Age", Type: "date", Description: swaggerMetadataDescriptions["creationTimestamp"], JSONPath: ".metadata.creationTimestamp"}, + } +} + +// hasPerVersionSchema returns true if a CRD uses per-version schema. +func hasPerVersionSchema(versions []apiextensionsv1beta1.CustomResourceDefinitionVersion) bool { + for _, v := range versions { + if v.Schema != nil { + return true + } + } + return false +} + +// hasPerVersionSubresources returns true if a CRD uses per-version subresources. +func hasPerVersionSubresources(versions []apiextensionsv1beta1.CustomResourceDefinitionVersion) bool { + for _, v := range versions { + if v.Subresources != nil { + return true + } + } + return false +} + +// hasPerVersionColumns returns true if a CRD uses per-version columns. +func hasPerVersionColumns(versions []apiextensionsv1beta1.CustomResourceDefinitionVersion) bool { + for _, v := range versions { + if len(v.AdditionalPrinterColumns) > 0 { + return true + } + } + return false +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/subresources_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/subresources_test.go index 3c0f8976456..c3a7c1cb214 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/subresources_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/subresources_test.go @@ -17,6 +17,7 @@ limitations under the License. package integration import ( + "fmt" "math" "reflect" "sort" @@ -29,49 +30,108 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/dynamic" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apiextensions-apiserver/test/integration/fixtures" ) var labelSelectorPath = ".status.labelSelector" +var anotherLabelSelectorPath = ".status.anotherLabelSelector" -func NewNoxuSubresourcesCRD(scope apiextensionsv1beta1.ResourceScope) *apiextensionsv1beta1.CustomResourceDefinition { - return &apiextensionsv1beta1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"}, - Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: "mygroup.example.com", - Version: "v1beta1", - Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ - Plural: "noxus", - Singular: "nonenglishnoxu", - Kind: "WishIHadChosenNoxu", - ShortNames: []string{"foo", "bar", "abc", "def"}, - ListKind: "NoxuItemList", +func NewNoxuSubresourcesCRDs(scope apiextensionsv1beta1.ResourceScope) []*apiextensionsv1beta1.CustomResourceDefinition { + return []*apiextensionsv1beta1.CustomResourceDefinition{ + // CRD that uses top-level subresources + { + ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"}, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "mygroup.example.com", + Version: "v1beta1", + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "noxus", + Singular: "nonenglishnoxu", + Kind: "WishIHadChosenNoxu", + ShortNames: []string{"foo", "bar", "abc", "def"}, + ListKind: "NoxuItemList", + }, + Scope: scope, + Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Served: true, + Storage: true, + }, + { + Name: "v1", + Served: true, + Storage: false, + }, + }, + Subresources: &apiextensionsv1beta1.CustomResourceSubresources{ + Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{}, + Scale: &apiextensionsv1beta1.CustomResourceSubresourceScale{ + SpecReplicasPath: ".spec.replicas", + StatusReplicasPath: ".status.replicas", + LabelSelectorPath: &labelSelectorPath, + }, + }, }, - Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{ - {Name: "v1beta1", Served: true, Storage: false}, - {Name: "v1", Served: true, Storage: true}, - }, - Scope: scope, - Subresources: &apiextensionsv1beta1.CustomResourceSubresources{ - Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{}, - Scale: &apiextensionsv1beta1.CustomResourceSubresourceScale{ - SpecReplicasPath: ".spec.replicas", - StatusReplicasPath: ".status.replicas", - LabelSelectorPath: &labelSelectorPath, + }, + // CRD that uses per-version subresources + { + ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"}, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "mygroup.example.com", + Version: "v1beta1", + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "noxus", + Singular: "nonenglishnoxu", + Kind: "WishIHadChosenNoxu", + ShortNames: []string{"foo", "bar", "abc", "def"}, + ListKind: "NoxuItemList", + }, + Scope: scope, + Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Served: true, + Storage: true, + Subresources: &apiextensionsv1beta1.CustomResourceSubresources{ + Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{}, + Scale: &apiextensionsv1beta1.CustomResourceSubresourceScale{ + SpecReplicasPath: ".spec.replicas", + StatusReplicasPath: ".status.replicas", + LabelSelectorPath: &labelSelectorPath, + }, + }, + }, + { + Name: "v1", + Served: true, + Storage: false, + Subresources: &apiextensionsv1beta1.CustomResourceSubresources{ + Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{}, + Scale: &apiextensionsv1beta1.CustomResourceSubresourceScale{ + SpecReplicasPath: ".spec.replicas", + StatusReplicasPath: ".status.replicas", + LabelSelectorPath: &anotherLabelSelectorPath, + }, + }, + }, }, }, }, } } -func NewNoxuSubresourceInstance(namespace, name string) *unstructured.Unstructured { +func NewNoxuSubresourceInstance(namespace, name, version string) *unstructured.Unstructured { return &unstructured.Unstructured{ Object: map[string]interface{}{ - "apiVersion": "mygroup.example.com/v1beta1", + "apiVersion": fmt.Sprintf("mygroup.example.com/%s", version), "kind": "WishIHadChosenNoxu", "metadata": map[string]interface{}{ "namespace": namespace, @@ -89,112 +149,120 @@ func NewNoxuSubresourceInstance(namespace, name string) *unstructured.Unstructur } func TestStatusSubresource(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t) if err != nil { t.Fatal(err) } defer tearDown() - noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.NamespaceScoped) - noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err != nil { - t.Fatal(err) - } + noxuDefinitions := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped) + for _, noxuDefinition := range noxuDefinitions { + noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } - ns := "not-the-default" - noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition) - _, err = instantiateCustomResource(t, NewNoxuSubresourceInstance(ns, "foo"), noxuResourceClient, noxuDefinition) - if err != nil { - t.Fatalf("unable to create noxu instance: %v", err) - } + ns := "not-the-default" + for _, v := range noxuDefinition.Spec.Versions { + noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name) + _, err = instantiateVersionedCustomResource(t, NewNoxuSubresourceInstance(ns, "foo", v.Name), noxuResourceClient, noxuDefinition, v.Name) + if err != nil { + t.Fatalf("unable to create noxu instance: %v", err) + } + gottenNoxuInstance, err := noxuResourceClient.Get("foo", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + // status should not be set after creation + if val, ok := gottenNoxuInstance.Object["status"]; ok { + t.Fatalf("status should not be set after creation, got %v", val) + } - gottenNoxuInstance, err := noxuResourceClient.Get("foo", metav1.GetOptions{}) - if err != nil { - t.Fatal(err) - } + // .status.num = 20 + err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(20), "status", "num") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } - // status should not be set after creation - if val, ok := gottenNoxuInstance.Object["status"]; ok { - t.Fatalf("status should not be set after creation, got %v", val) - } + // .spec.num = 20 + err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(20), "spec", "num") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } - // .status.num = 20 - err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(20), "status", "num") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + // UpdateStatus should not update spec. + // Check that .spec.num = 10 and .status.num = 20 + updatedStatusInstance, err := noxuResourceClient.UpdateStatus(gottenNoxuInstance, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("unable to update status: %v", err) + } - // .spec.num = 20 - err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(20), "spec", "num") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + specNum, found, err := unstructured.NestedInt64(updatedStatusInstance.Object, "spec", "num") + if !found || err != nil { + t.Fatalf("unable to get .spec.num") + } + if specNum != int64(10) { + t.Fatalf(".spec.num: expected: %v, got: %v", int64(10), specNum) + } - // UpdateStatus should not update spec. - // Check that .spec.num = 10 and .status.num = 20 - updatedStatusInstance, err := noxuResourceClient.UpdateStatus(gottenNoxuInstance, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("unable to update status: %v", err) - } + statusNum, found, err := unstructured.NestedInt64(updatedStatusInstance.Object, "status", "num") + if !found || err != nil { + t.Fatalf("unable to get .status.num") + } + if statusNum != int64(20) { + t.Fatalf(".status.num: expected: %v, got: %v", int64(20), statusNum) + } - specNum, found, err := unstructured.NestedInt64(updatedStatusInstance.Object, "spec", "num") - if !found || err != nil { - t.Fatalf("unable to get .spec.num") - } - if specNum != int64(10) { - t.Fatalf(".spec.num: expected: %v, got: %v", int64(10), specNum) - } + gottenNoxuInstance, err = noxuResourceClient.Get("foo", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } - statusNum, found, err := unstructured.NestedInt64(updatedStatusInstance.Object, "status", "num") - if !found || err != nil { - t.Fatalf("unable to get .status.num") - } - if statusNum != int64(20) { - t.Fatalf(".status.num: expected: %v, got: %v", int64(20), statusNum) - } + // .status.num = 40 + err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(40), "status", "num") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } - gottenNoxuInstance, err = noxuResourceClient.Get("foo", metav1.GetOptions{}) - if err != nil { - t.Fatal(err) - } + // .spec.num = 40 + err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(40), "spec", "num") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } - // .status.num = 40 - err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(40), "status", "num") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + // Update should not update status. + // Check that .spec.num = 40 and .status.num = 20 + updatedInstance, err := noxuResourceClient.Update(gottenNoxuInstance, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("unable to update instance: %v", err) + } - // .spec.num = 40 - err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(40), "spec", "num") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + specNum, found, err = unstructured.NestedInt64(updatedInstance.Object, "spec", "num") + if !found || err != nil { + t.Fatalf("unable to get .spec.num") + } + if specNum != int64(40) { + t.Fatalf(".spec.num: expected: %v, got: %v", int64(40), specNum) + } - // Update should not update status. - // Check that .spec.num = 40 and .status.num = 20 - updatedInstance, err := noxuResourceClient.Update(gottenNoxuInstance, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("unable to update instance: %v", err) - } - - specNum, found, err = unstructured.NestedInt64(updatedInstance.Object, "spec", "num") - if !found || err != nil { - t.Fatalf("unable to get .spec.num") - } - if specNum != int64(40) { - t.Fatalf(".spec.num: expected: %v, got: %v", int64(40), specNum) - } - - statusNum, found, err = unstructured.NestedInt64(updatedInstance.Object, "status", "num") - if !found || err != nil { - t.Fatalf("unable to get .status.num") - } - if statusNum != int64(20) { - t.Fatalf(".status.num: expected: %v, got: %v", int64(20), statusNum) + statusNum, found, err = unstructured.NestedInt64(updatedInstance.Object, "status", "num") + if !found || err != nil { + t.Fatalf("unable to get .status.num") + } + if statusNum != int64(20) { + t.Fatalf(".status.num: expected: %v, got: %v", int64(20), statusNum) + } + noxuResourceClient.Delete("foo", &metav1.DeleteOptions{}) + } + if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil { + t.Fatal(err) + } } } func TestScaleSubresource(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() groupResource := schema.GroupResource{ Group: "mygroup.example.com", Resource: "noxus", @@ -215,117 +283,132 @@ func TestScaleSubresource(t *testing.T) { t.Fatal(err) } - noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.NamespaceScoped) + noxuDefinitions := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped) + for _, noxuDefinition := range noxuDefinitions { + for _, v := range noxuDefinition.Spec.Versions { + // Start with a new CRD, so that the object doesn't have resourceVersion + noxuDefinition := noxuDefinition.DeepCopy() - // set invalid json path for specReplicasPath - noxuDefinition.Spec.Subresources.Scale.SpecReplicasPath = "foo,bar" - _, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err == nil { - t.Fatalf("unexpected non-error: specReplicasPath should be a valid json path under .spec") - } + subresources, err := getSubresourcesForVersion(noxuDefinition, v.Name) + if err != nil { + t.Fatal(err) + } + // set invalid json path for specReplicasPath + subresources.Scale.SpecReplicasPath = "foo,bar" + _, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err == nil { + t.Fatalf("unexpected non-error: specReplicasPath should be a valid json path under .spec") + } - noxuDefinition.Spec.Subresources.Scale.SpecReplicasPath = ".spec.replicas" - noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err != nil { - t.Fatal(err) - } + subresources.Scale.SpecReplicasPath = ".spec.replicas" + noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } - ns := "not-the-default" - noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition) - _, err = instantiateCustomResource(t, NewNoxuSubresourceInstance(ns, "foo"), noxuResourceClient, noxuDefinition) - if err != nil { - t.Fatalf("unable to create noxu instance: %v", err) - } + ns := "not-the-default" + noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name) + _, err = instantiateVersionedCustomResource(t, NewNoxuSubresourceInstance(ns, "foo", v.Name), noxuResourceClient, noxuDefinition, v.Name) + if err != nil { + t.Fatalf("unable to create noxu instance: %v", err) + } - scaleClient, err := fixtures.CreateNewScaleClient(noxuDefinition, config) - if err != nil { - t.Fatal(err) - } + scaleClient, err := fixtures.CreateNewVersionedScaleClient(noxuDefinition, config, v.Name) + if err != nil { + t.Fatal(err) + } - // set .status.labelSelector = bar - gottenNoxuInstance, err := noxuResourceClient.Get("foo", metav1.GetOptions{}) - if err != nil { - t.Fatal(err) - } - err = unstructured.SetNestedField(gottenNoxuInstance.Object, "bar", "status", "labelSelector") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - _, err = noxuResourceClient.UpdateStatus(gottenNoxuInstance, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("unable to update status: %v", err) - } + // set .status.labelSelector = bar + gottenNoxuInstance, err := noxuResourceClient.Get("foo", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + err = unstructured.SetNestedField(gottenNoxuInstance.Object, "bar", strings.Split((*subresources.Scale.LabelSelectorPath)[1:], ".")...) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + _, err = noxuResourceClient.UpdateStatus(gottenNoxuInstance, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("unable to update status: %v", err) + } - // get the scale object - gottenScale, err := scaleClient.Scales("not-the-default").Get(groupResource, "foo") - if err != nil { - t.Fatal(err) - } - if gottenScale.Spec.Replicas != 3 { - t.Fatalf("Scale.Spec.Replicas: expected: %v, got: %v", 3, gottenScale.Spec.Replicas) - } - if gottenScale.Status.Selector != "bar" { - t.Fatalf("Scale.Status.Selector: expected: %v, got: %v", "bar", gottenScale.Status.Selector) - } + // get the scale object + gottenScale, err := scaleClient.Scales("not-the-default").Get(groupResource, "foo") + if err != nil { + t.Fatal(err) + } + if gottenScale.Spec.Replicas != 3 { + t.Fatalf("Scale.Spec.Replicas: expected: %v, got: %v", 3, gottenScale.Spec.Replicas) + } + if gottenScale.Status.Selector != "bar" { + t.Fatalf("Scale.Status.Selector: expected: %v, got: %v", "bar", gottenScale.Status.Selector) + } - // check self link - expectedSelfLink := "/apis/mygroup.example.com/v1beta1/namespaces/not-the-default/noxus/foo/scale" - if gottenScale.GetSelfLink() != expectedSelfLink { - t.Fatalf("Scale.Metadata.SelfLink: expected: %v, got: %v", expectedSelfLink, gottenScale.GetSelfLink()) - } + // check self link + expectedSelfLink := fmt.Sprintf("/apis/mygroup.example.com/%s/namespaces/not-the-default/noxus/foo/scale", v.Name) + if gottenScale.GetSelfLink() != expectedSelfLink { + t.Fatalf("Scale.Metadata.SelfLink: expected: %v, got: %v", expectedSelfLink, gottenScale.GetSelfLink()) + } - // update the scale object - // check that spec is updated, but status is not - gottenScale.Spec.Replicas = 5 - gottenScale.Status.Selector = "baz" - updatedScale, err := scaleClient.Scales("not-the-default").Update(groupResource, gottenScale) - if err != nil { - t.Fatal(err) - } - if updatedScale.Spec.Replicas != 5 { - t.Fatalf("replicas: expected: %v, got: %v", 5, updatedScale.Spec.Replicas) - } - if updatedScale.Status.Selector != "bar" { - t.Fatalf("scale should not update status: expected %v, got: %v", "bar", updatedScale.Status.Selector) - } + // update the scale object + // check that spec is updated, but status is not + gottenScale.Spec.Replicas = 5 + gottenScale.Status.Selector = "baz" + updatedScale, err := scaleClient.Scales("not-the-default").Update(groupResource, gottenScale) + if err != nil { + t.Fatal(err) + } + if updatedScale.Spec.Replicas != 5 { + t.Fatalf("replicas: expected: %v, got: %v", 5, updatedScale.Spec.Replicas) + } + if updatedScale.Status.Selector != "bar" { + t.Fatalf("scale should not update status: expected %v, got: %v", "bar", updatedScale.Status.Selector) + } - // check that .spec.replicas = 5, but status is not updated - updatedNoxuInstance, err := noxuResourceClient.Get("foo", metav1.GetOptions{}) - if err != nil { - t.Fatal(err) - } - specReplicas, found, err := unstructured.NestedInt64(updatedNoxuInstance.Object, "spec", "replicas") - if !found || err != nil { - t.Fatalf("unable to get .spec.replicas") - } - if specReplicas != 5 { - t.Fatalf("replicas: expected: %v, got: %v", 5, specReplicas) - } - statusLabelSelector, found, err := unstructured.NestedString(updatedNoxuInstance.Object, "status", "labelSelector") - if !found || err != nil { - t.Fatalf("unable to get .status.labelSelector") - } - if statusLabelSelector != "bar" { - t.Fatalf("scale should not update status: expected %v, got: %v", "bar", statusLabelSelector) - } + // check that .spec.replicas = 5, but status is not updated + updatedNoxuInstance, err := noxuResourceClient.Get("foo", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + specReplicas, found, err := unstructured.NestedInt64(updatedNoxuInstance.Object, "spec", "replicas") + if !found || err != nil { + t.Fatalf("unable to get .spec.replicas") + } + if specReplicas != 5 { + t.Fatalf("replicas: expected: %v, got: %v", 5, specReplicas) + } + statusLabelSelector, found, err := unstructured.NestedString(updatedNoxuInstance.Object, strings.Split((*subresources.Scale.LabelSelectorPath)[1:], ".")...) + if !found || err != nil { + t.Fatalf("unable to get %s", *subresources.Scale.LabelSelectorPath) + } + if statusLabelSelector != "bar" { + t.Fatalf("scale should not update status: expected %v, got: %v", "bar", statusLabelSelector) + } - // validate maximum value - // set .spec.replicas = math.MaxInt64 - gottenNoxuInstance, err = noxuResourceClient.Get("foo", metav1.GetOptions{}) - if err != nil { - t.Fatal(err) - } - err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(math.MaxInt64), "spec", "replicas") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - _, err = noxuResourceClient.Update(gottenNoxuInstance, metav1.UpdateOptions{}) - if err == nil { - t.Fatalf("unexpected non-error: .spec.replicas should be less than 2147483647") + // validate maximum value + // set .spec.replicas = math.MaxInt64 + gottenNoxuInstance, err = noxuResourceClient.Get("foo", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(math.MaxInt64), "spec", "replicas") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + _, err = noxuResourceClient.Update(gottenNoxuInstance, metav1.UpdateOptions{}) + if err == nil { + t.Fatalf("unexpected non-error: .spec.replicas should be less than 2147483647") + } + noxuResourceClient.Delete("foo", &metav1.DeleteOptions{}) + if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil { + t.Fatal(err) + } + } } } func TestValidationSchemaWithStatus(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() tearDown, config, _, err := fixtures.StartDefaultServer(t) if err != nil { t.Fatal(err) @@ -342,7 +425,7 @@ func TestValidationSchemaWithStatus(t *testing.T) { } // fields other than properties in root schema are not allowed - noxuDefinition := newNoxuValidationCRD(apiextensionsv1beta1.NamespaceScoped) + noxuDefinition := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped)[0] noxuDefinition.Spec.Subresources = &apiextensionsv1beta1.CustomResourceSubresources{ Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{}, } @@ -373,6 +456,7 @@ func TestValidationSchemaWithStatus(t *testing.T) { } func TestValidateOnlyStatus(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t) if err != nil { t.Fatal(err) @@ -407,59 +491,79 @@ func TestValidateOnlyStatus(t *testing.T) { }, } - noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.NamespaceScoped) - noxuDefinition.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{ - OpenAPIV3Schema: schema, - } + noxuDefinitions := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped) + for i, noxuDefinition := range noxuDefinitions { + if i == 0 { + noxuDefinition.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: schema, + } + } else { + noxuDefinition.Spec.Versions[0].Schema = &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: schema, + } + schemaWithDescription := schema.DeepCopy() + schemaWithDescription.Description = "test" + noxuDefinition.Spec.Versions[1].Schema = &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: schemaWithDescription, + } + } - noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err != nil { - t.Fatal(err) - } - ns := "not-the-default" - noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition) + noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } + ns := "not-the-default" + for _, v := range noxuDefinition.Spec.Versions { + noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name) - // set .spec.num = 10 and .status.num = 10 - noxuInstance := NewNoxuSubresourceInstance(ns, "foo") - err = unstructured.SetNestedField(noxuInstance.Object, int64(10), "status", "num") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + // set .spec.num = 10 and .status.num = 10 + noxuInstance := NewNoxuSubresourceInstance(ns, "foo", v.Name) + err = unstructured.SetNestedField(noxuInstance.Object, int64(10), "status", "num") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } - createdNoxuInstance, err := instantiateCustomResource(t, noxuInstance, noxuResourceClient, noxuDefinition) - if err != nil { - t.Fatalf("unable to create noxu instance: %v", err) - } + createdNoxuInstance, err := instantiateVersionedCustomResource(t, noxuInstance, noxuResourceClient, noxuDefinition, v.Name) + if err != nil { + t.Fatalf("unable to create noxu instance: %v", err) + } - // update the spec with .spec.num = 15, expecting no error - err = unstructured.SetNestedField(createdNoxuInstance.Object, int64(15), "spec", "num") - if err != nil { - t.Fatalf("unexpected error setting .spec.num: %v", err) - } - createdNoxuInstance, err = noxuResourceClient.UpdateStatus(createdNoxuInstance, metav1.UpdateOptions{}) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + // update the spec with .spec.num = 15, expecting no error + err = unstructured.SetNestedField(createdNoxuInstance.Object, int64(15), "spec", "num") + if err != nil { + t.Fatalf("unexpected error setting .spec.num: %v", err) + } + createdNoxuInstance, err = noxuResourceClient.UpdateStatus(createdNoxuInstance, metav1.UpdateOptions{}) + if err != nil { + t.Errorf("unexpected error: %v", err) + } - // update with .status.num = 15, expecting an error - err = unstructured.SetNestedField(createdNoxuInstance.Object, int64(15), "status", "num") - if err != nil { - t.Fatalf("unexpected error setting .status.num: %v", err) - } - createdNoxuInstance, err = noxuResourceClient.UpdateStatus(createdNoxuInstance, metav1.UpdateOptions{}) - if err == nil { - t.Fatal("expected error, but got none") - } - statusError, isStatus := err.(*apierrors.StatusError) - if !isStatus || statusError == nil { - t.Fatalf("expected status error, got %T: %v", err, err) - } - if !strings.Contains(statusError.Error(), "Invalid value") { - t.Fatalf("expected 'Invalid value' in error, got: %v", err) + // update with .status.num = 15, expecting an error + err = unstructured.SetNestedField(createdNoxuInstance.Object, int64(15), "status", "num") + if err != nil { + t.Fatalf("unexpected error setting .status.num: %v", err) + } + createdNoxuInstance, err = noxuResourceClient.UpdateStatus(createdNoxuInstance, metav1.UpdateOptions{}) + if err == nil { + t.Fatal("expected error, but got none") + } + statusError, isStatus := err.(*apierrors.StatusError) + if !isStatus || statusError == nil { + t.Fatalf("expected status error, got %T: %v", err, err) + } + if !strings.Contains(statusError.Error(), "Invalid value") { + t.Fatalf("expected 'Invalid value' in error, got: %v", err) + } + noxuResourceClient.Delete("foo", &metav1.DeleteOptions{}) + } + if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil { + t.Fatal(err) + } } } func TestSubresourcesDiscovery(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() tearDown, config, _, err := fixtures.StartDefaultServer(t) if err != nil { t.Fatal(err) @@ -475,140 +579,157 @@ func TestSubresourcesDiscovery(t *testing.T) { t.Fatal(err) } - noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.NamespaceScoped) - noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err != nil { - t.Fatal(err) - } + noxuDefinitions := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped) + for _, noxuDefinition := range noxuDefinitions { + noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } - group := "mygroup.example.com" - version := "v1beta1" + for _, v := range noxuDefinition.Spec.Versions { + group := "mygroup.example.com" + version := v.Name - resources, err := apiExtensionClient.Discovery().ServerResourcesForGroupVersion(group + "/" + version) - if err != nil { - t.Fatal(err) - } + resources, err := apiExtensionClient.Discovery().ServerResourcesForGroupVersion(group + "/" + version) + if err != nil { + t.Fatal(err) + } - if len(resources.APIResources) != 3 { - t.Fatalf("Expected exactly the resources \"noxus\", \"noxus/status\" and \"noxus/scale\" in group version %v/%v via discovery, got: %v", group, version, resources.APIResources) - } + if len(resources.APIResources) != 3 { + t.Fatalf("Expected exactly the resources \"noxus\", \"noxus/status\" and \"noxus/scale\" in group version %v/%v via discovery, got: %v", group, version, resources.APIResources) + } - // check discovery info for status - status := resources.APIResources[1] + // check discovery info for status + status := resources.APIResources[1] - if status.Name != "noxus/status" { - t.Fatalf("incorrect status via discovery: expected name: %v, got: %v", "noxus/status", status.Name) - } + if status.Name != "noxus/status" { + t.Fatalf("incorrect status via discovery: expected name: %v, got: %v", "noxus/status", status.Name) + } - if status.Namespaced != true { - t.Fatalf("incorrect status via discovery: expected namespace: %v, got: %v", true, status.Namespaced) - } + if status.Namespaced != true { + t.Fatalf("incorrect status via discovery: expected namespace: %v, got: %v", true, status.Namespaced) + } - if status.Kind != "WishIHadChosenNoxu" { - t.Fatalf("incorrect status via discovery: expected kind: %v, got: %v", "WishIHadChosenNoxu", status.Kind) - } + if status.Kind != "WishIHadChosenNoxu" { + t.Fatalf("incorrect status via discovery: expected kind: %v, got: %v", "WishIHadChosenNoxu", status.Kind) + } - expectedVerbs := []string{"get", "patch", "update"} - sort.Strings(status.Verbs) - if !reflect.DeepEqual([]string(status.Verbs), expectedVerbs) { - t.Fatalf("incorrect status via discovery: expected: %v, got: %v", expectedVerbs, status.Verbs) - } + expectedVerbs := []string{"get", "patch", "update"} + sort.Strings(status.Verbs) + if !reflect.DeepEqual([]string(status.Verbs), expectedVerbs) { + t.Fatalf("incorrect status via discovery: expected: %v, got: %v", expectedVerbs, status.Verbs) + } - // check discovery info for scale - scale := resources.APIResources[2] + // check discovery info for scale + scale := resources.APIResources[2] - if scale.Group != autoscaling.GroupName { - t.Fatalf("incorrect scale via discovery: expected group: %v, got: %v", autoscaling.GroupName, scale.Group) - } + if scale.Group != autoscaling.GroupName { + t.Fatalf("incorrect scale via discovery: expected group: %v, got: %v", autoscaling.GroupName, scale.Group) + } - if scale.Version != "v1" { - t.Fatalf("incorrect scale via discovery: expected version: %v, got %v", "v1", scale.Version) - } + if scale.Version != "v1" { + t.Fatalf("incorrect scale via discovery: expected version: %v, got %v", "v1", scale.Version) + } - if scale.Name != "noxus/scale" { - t.Fatalf("incorrect scale via discovery: expected name: %v, got: %v", "noxus/scale", scale.Name) - } + if scale.Name != "noxus/scale" { + t.Fatalf("incorrect scale via discovery: expected name: %v, got: %v", "noxus/scale", scale.Name) + } - if scale.Namespaced != true { - t.Fatalf("incorrect scale via discovery: expected namespace: %v, got: %v", true, scale.Namespaced) - } + if scale.Namespaced != true { + t.Fatalf("incorrect scale via discovery: expected namespace: %v, got: %v", true, scale.Namespaced) + } - if scale.Kind != "Scale" { - t.Fatalf("incorrect scale via discovery: expected kind: %v, got: %v", "Scale", scale.Kind) - } + if scale.Kind != "Scale" { + t.Fatalf("incorrect scale via discovery: expected kind: %v, got: %v", "Scale", scale.Kind) + } - sort.Strings(scale.Verbs) - if !reflect.DeepEqual([]string(scale.Verbs), expectedVerbs) { - t.Fatalf("incorrect scale via discovery: expected: %v, got: %v", expectedVerbs, scale.Verbs) + sort.Strings(scale.Verbs) + if !reflect.DeepEqual([]string(scale.Verbs), expectedVerbs) { + t.Fatalf("incorrect scale via discovery: expected: %v, got: %v", expectedVerbs, scale.Verbs) + } + } + if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil { + t.Fatal(err) + } } } func TestGeneration(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t) if err != nil { t.Fatal(err) } defer tearDown() - noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.NamespaceScoped) - noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err != nil { - t.Fatal(err) - } + noxuDefinitions := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped) + for _, noxuDefinition := range noxuDefinitions { + noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } - ns := "not-the-default" - noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition) - _, err = instantiateCustomResource(t, NewNoxuSubresourceInstance(ns, "foo"), noxuResourceClient, noxuDefinition) - if err != nil { - t.Fatalf("unable to create noxu instance: %v", err) - } + ns := "not-the-default" + for _, v := range noxuDefinition.Spec.Versions { + noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name) + _, err = instantiateVersionedCustomResource(t, NewNoxuSubresourceInstance(ns, "foo", v.Name), noxuResourceClient, noxuDefinition, v.Name) + if err != nil { + t.Fatalf("unable to create noxu instance: %v", err) + } - // .metadata.generation = 1 - gottenNoxuInstance, err := noxuResourceClient.Get("foo", metav1.GetOptions{}) - if err != nil { - t.Fatal(err) - } - if gottenNoxuInstance.GetGeneration() != 1 { - t.Fatalf(".metadata.generation should be 1 after creation") - } + // .metadata.generation = 1 + gottenNoxuInstance, err := noxuResourceClient.Get("foo", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if gottenNoxuInstance.GetGeneration() != 1 { + t.Fatalf(".metadata.generation should be 1 after creation") + } - // .status.num = 20 - err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(20), "status", "num") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + // .status.num = 20 + err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(20), "status", "num") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } - // UpdateStatus does not increment generation - updatedStatusInstance, err := noxuResourceClient.UpdateStatus(gottenNoxuInstance, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("unable to update status: %v", err) - } - if updatedStatusInstance.GetGeneration() != 1 { - t.Fatalf("updating status should not increment .metadata.generation: expected: %v, got: %v", 1, updatedStatusInstance.GetGeneration()) - } + // UpdateStatus does not increment generation + updatedStatusInstance, err := noxuResourceClient.UpdateStatus(gottenNoxuInstance, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("unable to update status: %v", err) + } + if updatedStatusInstance.GetGeneration() != 1 { + t.Fatalf("updating status should not increment .metadata.generation: expected: %v, got: %v", 1, updatedStatusInstance.GetGeneration()) + } - gottenNoxuInstance, err = noxuResourceClient.Get("foo", metav1.GetOptions{}) - if err != nil { - t.Fatal(err) - } + gottenNoxuInstance, err = noxuResourceClient.Get("foo", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } - // .spec.num = 20 - err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(20), "spec", "num") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + // .spec.num = 20 + err = unstructured.SetNestedField(gottenNoxuInstance.Object, int64(20), "spec", "num") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } - // Update increments generation - updatedInstance, err := noxuResourceClient.Update(gottenNoxuInstance, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("unable to update instance: %v", err) - } - if updatedInstance.GetGeneration() != 2 { - t.Fatalf("updating spec should increment .metadata.generation: expected: %v, got: %v", 2, updatedStatusInstance.GetGeneration()) + // Update increments generation + updatedInstance, err := noxuResourceClient.Update(gottenNoxuInstance, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("unable to update instance: %v", err) + } + if updatedInstance.GetGeneration() != 2 { + t.Fatalf("updating spec should increment .metadata.generation: expected: %v, got: %v", 2, updatedStatusInstance.GetGeneration()) + } + noxuResourceClient.Delete("foo", &metav1.DeleteOptions{}) + } + if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil { + t.Fatal(err) + } } } func TestSubresourcePatch(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() groupResource := schema.GroupResource{ Group: "mygroup.example.com", Resource: "noxus", @@ -629,146 +750,155 @@ func TestSubresourcePatch(t *testing.T) { t.Fatal(err) } - noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.NamespaceScoped) - noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err != nil { - t.Fatal(err) - } + noxuDefinitions := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped) + for _, noxuDefinition := range noxuDefinitions { + noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } - ns := "not-the-default" - noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition) + ns := "not-the-default" + for _, v := range noxuDefinition.Spec.Versions { + noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name) - t.Logf("Creating foo") - _, err = instantiateCustomResource(t, NewNoxuSubresourceInstance(ns, "foo"), noxuResourceClient, noxuDefinition) - if err != nil { - t.Fatalf("unable to create noxu instance: %v", err) - } + t.Logf("Creating foo") + _, err = instantiateVersionedCustomResource(t, NewNoxuSubresourceInstance(ns, "foo", v.Name), noxuResourceClient, noxuDefinition, v.Name) + if err != nil { + t.Fatalf("unable to create noxu instance: %v", err) + } - scaleClient, err := fixtures.CreateNewScaleClient(noxuDefinition, config) - if err != nil { - t.Fatal(err) - } + scaleClient, err := fixtures.CreateNewVersionedScaleClient(noxuDefinition, config, v.Name) + if err != nil { + t.Fatal(err) + } - t.Logf("Patching .status.num to 999") - patch := []byte(`{"spec": {"num":999}, "status": {"num":999}}`) - patchedNoxuInstance, err := noxuResourceClient.Patch("foo", types.MergePatchType, patch, metav1.UpdateOptions{}, "status") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + t.Logf("Patching .status.num to 999") + patch := []byte(`{"spec": {"num":999}, "status": {"num":999}}`) + patchedNoxuInstance, err := noxuResourceClient.Patch("foo", types.MergePatchType, patch, metav1.UpdateOptions{}, "status") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } - expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 999, "status", "num") // .status.num should be 999 - expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 10, "spec", "num") // .spec.num should remain 10 - rv, found, err := unstructured.NestedString(patchedNoxuInstance.UnstructuredContent(), "metadata", "resourceVersion") - if err != nil { - t.Fatal(err) - } - if !found { - t.Fatalf("metadata.resourceVersion not found") - } + expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 999, "status", "num") // .status.num should be 999 + expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 10, "spec", "num") // .spec.num should remain 10 + rv, found, err := unstructured.NestedString(patchedNoxuInstance.UnstructuredContent(), "metadata", "resourceVersion") + if err != nil { + t.Fatal(err) + } + if !found { + t.Fatalf("metadata.resourceVersion not found") + } - // this call waits for the resourceVersion to be reached in the cache before returning. - // We need to do this because the patch gets its initial object from the storage, and the cache serves that. - // If it is out of date, then our initial patch is applied to an old resource version, which conflicts - // and then the updated object shows a conflicting diff, which permanently fails the patch. - // This gives expected stability in the patch without retrying on an known number of conflicts below in the test. - // See https://issue.k8s.io/42644 - _, err = noxuResourceClient.Get("foo", metav1.GetOptions{ResourceVersion: patchedNoxuInstance.GetResourceVersion()}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + // this call waits for the resourceVersion to be reached in the cache before returning. + // We need to do this because the patch gets its initial object from the storage, and the cache serves that. + // If it is out of date, then our initial patch is applied to an old resource version, which conflicts + // and then the updated object shows a conflicting diff, which permanently fails the patch. + // This gives expected stability in the patch without retrying on an known number of conflicts below in the test. + // See https://issue.k8s.io/42644 + _, err = noxuResourceClient.Get("foo", metav1.GetOptions{ResourceVersion: patchedNoxuInstance.GetResourceVersion()}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } - // no-op patch - t.Logf("Patching .status.num again to 999") - patchedNoxuInstance, err = noxuResourceClient.Patch("foo", types.MergePatchType, patch, metav1.UpdateOptions{}, "status") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - // make sure no-op patch does not increment resourceVersion - expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 999, "status", "num") - expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 10, "spec", "num") - expectString(t, patchedNoxuInstance.UnstructuredContent(), rv, "metadata", "resourceVersion") + // no-op patch + t.Logf("Patching .status.num again to 999") + patchedNoxuInstance, err = noxuResourceClient.Patch("foo", types.MergePatchType, patch, metav1.UpdateOptions{}, "status") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // make sure no-op patch does not increment resourceVersion + expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 999, "status", "num") + expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 10, "spec", "num") + expectString(t, patchedNoxuInstance.UnstructuredContent(), rv, "metadata", "resourceVersion") - // empty patch - t.Logf("Applying empty patch") - patchedNoxuInstance, err = noxuResourceClient.Patch("foo", types.MergePatchType, []byte(`{}`), metav1.UpdateOptions{}, "status") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - // an empty patch is a no-op patch. make sure it does not increment resourceVersion - expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 999, "status", "num") - expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 10, "spec", "num") - expectString(t, patchedNoxuInstance.UnstructuredContent(), rv, "metadata", "resourceVersion") + // empty patch + t.Logf("Applying empty patch") + patchedNoxuInstance, err = noxuResourceClient.Patch("foo", types.MergePatchType, []byte(`{}`), metav1.UpdateOptions{}, "status") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } - t.Logf("Patching .spec.replicas to 7") - patch = []byte(`{"spec": {"replicas":7}, "status": {"replicas":7}}`) - patchedNoxuInstance, err = noxuResourceClient.Patch("foo", types.MergePatchType, patch, metav1.UpdateOptions{}, "scale") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + // an empty patch is a no-op patch. make sure it does not increment resourceVersion + expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 999, "status", "num") + expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 10, "spec", "num") + expectString(t, patchedNoxuInstance.UnstructuredContent(), rv, "metadata", "resourceVersion") - expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 7, "spec", "replicas") - expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 0, "status", "replicas") // .status.replicas should remain 0 - rv, found, err = unstructured.NestedString(patchedNoxuInstance.UnstructuredContent(), "metadata", "resourceVersion") - if err != nil { - t.Fatal(err) - } - if !found { - t.Fatalf("metadata.resourceVersion not found") - } + t.Logf("Patching .spec.replicas to 7") + patch = []byte(`{"spec": {"replicas":7}, "status": {"replicas":7}}`) + patchedNoxuInstance, err = noxuResourceClient.Patch("foo", types.MergePatchType, patch, metav1.UpdateOptions{}, "scale") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } - // this call waits for the resourceVersion to be reached in the cache before returning. - // We need to do this because the patch gets its initial object from the storage, and the cache serves that. - // If it is out of date, then our initial patch is applied to an old resource version, which conflicts - // and then the updated object shows a conflicting diff, which permanently fails the patch. - // This gives expected stability in the patch without retrying on an known number of conflicts below in the test. - // See https://issue.k8s.io/42644 - _, err = noxuResourceClient.Get("foo", metav1.GetOptions{ResourceVersion: patchedNoxuInstance.GetResourceVersion()}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 7, "spec", "replicas") + expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 0, "status", "replicas") // .status.replicas should remain 0 + rv, found, err = unstructured.NestedString(patchedNoxuInstance.UnstructuredContent(), "metadata", "resourceVersion") + if err != nil { + t.Fatal(err) + } + if !found { + t.Fatalf("metadata.resourceVersion not found") + } - // Scale.Spec.Replicas = 7 but Scale.Status.Replicas should remain 0 - gottenScale, err := scaleClient.Scales("not-the-default").Get(groupResource, "foo") - if err != nil { - t.Fatal(err) - } - if gottenScale.Spec.Replicas != 7 { - t.Fatalf("Scale.Spec.Replicas: expected: %v, got: %v", 7, gottenScale.Spec.Replicas) - } - if gottenScale.Status.Replicas != 0 { - t.Fatalf("Scale.Status.Replicas: expected: %v, got: %v", 0, gottenScale.Spec.Replicas) - } + // this call waits for the resourceVersion to be reached in the cache before returning. + // We need to do this because the patch gets its initial object from the storage, and the cache serves that. + // If it is out of date, then our initial patch is applied to an old resource version, which conflicts + // and then the updated object shows a conflicting diff, which permanently fails the patch. + // This gives expected stability in the patch without retrying on an known number of conflicts below in the test. + // See https://issue.k8s.io/42644 + _, err = noxuResourceClient.Get("foo", metav1.GetOptions{ResourceVersion: patchedNoxuInstance.GetResourceVersion()}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } - // no-op patch - t.Logf("Patching .spec.replicas again to 7") - patchedNoxuInstance, err = noxuResourceClient.Patch("foo", types.MergePatchType, patch, metav1.UpdateOptions{}, "scale") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - // make sure no-op patch does not increment resourceVersion - expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 7, "spec", "replicas") - expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 0, "status", "replicas") - expectString(t, patchedNoxuInstance.UnstructuredContent(), rv, "metadata", "resourceVersion") + // Scale.Spec.Replicas = 7 but Scale.Status.Replicas should remain 0 + gottenScale, err := scaleClient.Scales("not-the-default").Get(groupResource, "foo") + if err != nil { + t.Fatal(err) + } + if gottenScale.Spec.Replicas != 7 { + t.Fatalf("Scale.Spec.Replicas: expected: %v, got: %v", 7, gottenScale.Spec.Replicas) + } + if gottenScale.Status.Replicas != 0 { + t.Fatalf("Scale.Status.Replicas: expected: %v, got: %v", 0, gottenScale.Spec.Replicas) + } - // empty patch - t.Logf("Applying empty patch") - patchedNoxuInstance, err = noxuResourceClient.Patch("foo", types.MergePatchType, []byte(`{}`), metav1.UpdateOptions{}, "scale") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - // an empty patch is a no-op patch. make sure it does not increment resourceVersion - expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 7, "spec", "replicas") - expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 0, "status", "replicas") - expectString(t, patchedNoxuInstance.UnstructuredContent(), rv, "metadata", "resourceVersion") + // no-op patch + t.Logf("Patching .spec.replicas again to 7") + patchedNoxuInstance, err = noxuResourceClient.Patch("foo", types.MergePatchType, patch, metav1.UpdateOptions{}, "scale") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // make sure no-op patch does not increment resourceVersion + expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 7, "spec", "replicas") + expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 0, "status", "replicas") + expectString(t, patchedNoxuInstance.UnstructuredContent(), rv, "metadata", "resourceVersion") - // make sure strategic merge patch is not supported for both status and scale - _, err = noxuResourceClient.Patch("foo", types.StrategicMergePatchType, patch, metav1.UpdateOptions{}, "status") - if err == nil { - t.Fatalf("unexpected non-error: strategic merge patch is not supported for custom resources") - } + // empty patch + t.Logf("Applying empty patch") + patchedNoxuInstance, err = noxuResourceClient.Patch("foo", types.MergePatchType, []byte(`{}`), metav1.UpdateOptions{}, "scale") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // an empty patch is a no-op patch. make sure it does not increment resourceVersion + expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 7, "spec", "replicas") + expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 0, "status", "replicas") + expectString(t, patchedNoxuInstance.UnstructuredContent(), rv, "metadata", "resourceVersion") - _, err = noxuResourceClient.Patch("foo", types.StrategicMergePatchType, patch, metav1.UpdateOptions{}, "scale") - if err == nil { - t.Fatalf("unexpected non-error: strategic merge patch is not supported for custom resources") + // make sure strategic merge patch is not supported for both status and scale + _, err = noxuResourceClient.Patch("foo", types.StrategicMergePatchType, patch, metav1.UpdateOptions{}, "status") + if err == nil { + t.Fatalf("unexpected non-error: strategic merge patch is not supported for custom resources") + } + + _, err = noxuResourceClient.Patch("foo", types.StrategicMergePatchType, patch, metav1.UpdateOptions{}, "scale") + if err == nil { + t.Fatalf("unexpected non-error: strategic merge patch is not supported for custom resources") + } + noxuResourceClient.Delete("foo", &metav1.DeleteOptions{}) + } + if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil { + t.Fatal(err) + } } } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/table_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/table_test.go index 7aab4f4086d..a10a72ecca4 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/table_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/table_test.go @@ -28,10 +28,14 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apiextensions-apiserver/test/integration/fixtures" ) @@ -48,12 +52,32 @@ func newTableCRD() *apiextensionsv1beta1.CustomResourceDefinition { ListKind: "TablemList", }, Scope: apiextensionsv1beta1.ClusterScoped, - AdditionalPrinterColumns: []apiextensionsv1beta1.CustomResourceColumnDefinition{ - {Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp"}, - {Name: "Alpha", Type: "string", JSONPath: ".spec.alpha"}, - {Name: "Beta", Type: "integer", Description: "the beta field", Format: "int64", Priority: 42, JSONPath: ".spec.beta"}, - {Name: "Gamma", Type: "integer", Description: "a column with wrongly typed values", JSONPath: ".spec.gamma"}, - {Name: "Epsilon", Type: "string", Description: "an array of integers as string", JSONPath: ".spec.epsilon"}, + Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Served: true, + Storage: false, + AdditionalPrinterColumns: []apiextensionsv1beta1.CustomResourceColumnDefinition{ + {Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp"}, + {Name: "Alpha", Type: "string", JSONPath: ".spec.alpha"}, + {Name: "Beta", Type: "integer", Description: "the beta field", Format: "int64", Priority: 42, JSONPath: ".spec.beta"}, + {Name: "Gamma", Type: "integer", Description: "a column with wrongly typed values", JSONPath: ".spec.gamma"}, + {Name: "Epsilon", Type: "string", Description: "an array of integers as string", JSONPath: ".spec.epsilon"}, + }, + }, + { + Name: "v1", + Served: true, + Storage: true, + AdditionalPrinterColumns: []apiextensionsv1beta1.CustomResourceColumnDefinition{ + {Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp"}, + {Name: "Alpha", Type: "string", JSONPath: ".spec.alpha"}, + {Name: "Beta", Type: "integer", Description: "the beta field", Format: "int64", Priority: 42, JSONPath: ".spec.beta"}, + {Name: "Gamma", Type: "integer", Description: "a column with wrongly typed values", JSONPath: ".spec.gamma"}, + {Name: "Epsilon", Type: "string", Description: "an array of integers as string", JSONPath: ".spec.epsilon"}, + {Name: "Zeta", Type: "integer", Description: "the zeta field", Format: "int64", Priority: 42, JSONPath: ".spec.zeta"}, + }, + }, }, }, } @@ -62,7 +86,7 @@ func newTableCRD() *apiextensionsv1beta1.CustomResourceDefinition { func newTableInstance(name string) *unstructured.Unstructured { return &unstructured.Unstructured{ Object: map[string]interface{}{ - "apiVersion": "mygroup.example.com/v1beta1", + "apiVersion": "mygroup.example.com/v1", "kind": "Table", "metadata": map[string]interface{}{ "name": name, @@ -73,12 +97,14 @@ func newTableInstance(name string) *unstructured.Unstructured { "gamma": "bar", "delta": "hello", "epsilon": []int64{1, 2, 3}, + "zeta": 5, }, }, } } func TestTableGet(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() tearDown, config, _, err := fixtures.StartDefaultServer(t) if err != nil { t.Fatal(err) @@ -107,107 +133,219 @@ func TestTableGet(t *testing.T) { } t.Logf("table crd created: %#v", crd) - crClient := newNamespacedCustomResourceClient("", dynamicClient, crd) + crClient := newNamespacedCustomResourceVersionedClient("", dynamicClient, crd, "v1") foo, err := crClient.Create(newTableInstance("foo"), metav1.CreateOptions{}) if err != nil { t.Fatalf("unable to create noxu instance: %v", err) } t.Logf("foo created: %#v", foo.UnstructuredContent()) - gv := schema.GroupVersion{Group: crd.Spec.Group, Version: crd.Spec.Version} - gvk := gv.WithKind(crd.Spec.Names.Kind) + for i, v := range crd.Spec.Versions { + gv := schema.GroupVersion{Group: crd.Spec.Group, Version: v.Name} + gvk := gv.WithKind(crd.Spec.Names.Kind) - scheme := runtime.NewScheme() - codecs := serializer.NewCodecFactory(scheme) - parameterCodec := runtime.NewParameterCodec(scheme) - metav1.AddToGroupVersion(scheme, gv) - scheme.AddKnownTypes(gv, &metav1beta1.Table{}, &metav1beta1.TableOptions{}) - scheme.AddKnownTypes(metav1beta1.SchemeGroupVersion, &metav1beta1.Table{}, &metav1beta1.TableOptions{}) + scheme := runtime.NewScheme() + codecs := serializer.NewCodecFactory(scheme) + parameterCodec := runtime.NewParameterCodec(scheme) + metav1.AddToGroupVersion(scheme, gv) + scheme.AddKnownTypes(gv, &metav1beta1.Table{}, &metav1beta1.TableOptions{}) + scheme.AddKnownTypes(metav1beta1.SchemeGroupVersion, &metav1beta1.Table{}, &metav1beta1.TableOptions{}) - crConfig := *config - crConfig.GroupVersion = &gv - crConfig.APIPath = "/apis" - crConfig.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: codecs} - crRestClient, err := rest.RESTClientFor(&crConfig) + crConfig := *config + crConfig.GroupVersion = &gv + crConfig.APIPath = "/apis" + crConfig.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: codecs} + crRestClient, err := rest.RESTClientFor(&crConfig) + if err != nil { + t.Fatal(err) + } + + ret, err := crRestClient.Get(). + Resource(crd.Spec.Names.Plural). + SetHeader("Accept", fmt.Sprintf("application/json;as=Table;v=%s;g=%s, application/json", metav1beta1.SchemeGroupVersion.Version, metav1beta1.GroupName)). + VersionedParams(&metav1beta1.TableOptions{}, parameterCodec). + Do(). + Get() + if err != nil { + t.Fatalf("failed to list %v resources: %v", gvk, err) + } + + tbl, ok := ret.(*metav1beta1.Table) + if !ok { + t.Fatalf("expected metav1beta1.Table, got %T", ret) + } + t.Logf("%v table list: %#v", gvk, tbl) + + columns, err := getColumnsForVersion(crd, v.Name) + if err != nil { + t.Fatal(err) + } + expectColumnNum := len(columns) + 1 + if got, expected := len(tbl.ColumnDefinitions), expectColumnNum; got != expected { + t.Errorf("expected %d headers, got %d", expected, got) + } else { + age := metav1beta1.TableColumnDefinition{Name: "Age", Type: "date", Format: "", Description: "Custom resource definition column (in JSONPath format): .metadata.creationTimestamp", Priority: 0} + if got, expected := tbl.ColumnDefinitions[1], age; got != expected { + t.Errorf("expected column definition %#v, got %#v", expected, got) + } + + alpha := metav1beta1.TableColumnDefinition{Name: "Alpha", Type: "string", Format: "", Description: "Custom resource definition column (in JSONPath format): .spec.alpha", Priority: 0} + if got, expected := tbl.ColumnDefinitions[2], alpha; got != expected { + t.Errorf("expected column definition %#v, got %#v", expected, got) + } + + beta := metav1beta1.TableColumnDefinition{Name: "Beta", Type: "integer", Format: "int64", Description: "the beta field", Priority: 42} + if got, expected := tbl.ColumnDefinitions[3], beta; got != expected { + t.Errorf("expected column definition %#v, got %#v", expected, got) + } + + gamma := metav1beta1.TableColumnDefinition{Name: "Gamma", Type: "integer", Description: "a column with wrongly typed values"} + if got, expected := tbl.ColumnDefinitions[4], gamma; got != expected { + t.Errorf("expected column definition %#v, got %#v", expected, got) + } + + epsilon := metav1beta1.TableColumnDefinition{Name: "Epsilon", Type: "string", Description: "an array of integers as string"} + if got, expected := tbl.ColumnDefinitions[5], epsilon; got != expected { + t.Errorf("expected column definition %#v, got %#v", expected, got) + } + + // Validate extra column for v1 + if i == 1 { + zeta := metav1beta1.TableColumnDefinition{Name: "Zeta", Type: "integer", Format: "int64", Description: "the zeta field", Priority: 42} + if got, expected := tbl.ColumnDefinitions[6], zeta; got != expected { + t.Errorf("expected column definition %#v, got %#v", expected, got) + } + } + } + if got, expected := len(tbl.Rows), 1; got != expected { + t.Errorf("expected %d rows, got %d", expected, got) + } else if got, expected := len(tbl.Rows[0].Cells), expectColumnNum; got != expected { + t.Errorf("expected %d cells, got %d", expected, got) + } else { + if got, expected := tbl.Rows[0].Cells[0], "foo"; got != expected { + t.Errorf("expected cell[0] to equal %q, got %q", expected, got) + } + if s, ok := tbl.Rows[0].Cells[1].(string); !ok { + t.Errorf("expected cell[1] to be a string, got: %#v", tbl.Rows[0].Cells[1]) + } else { + dur, err := time.ParseDuration(s) + if err != nil { + t.Errorf("expected cell[1] to be a duration: %v", err) + } else if abs(dur.Seconds()) > 30.0 { + t.Errorf("expected cell[1] to be a small age, but got: %v", dur) + } + } + if got, expected := tbl.Rows[0].Cells[2], "foo_123"; got != expected { + t.Errorf("expected cell[2] to equal %q, got %q", expected, got) + } + if got, expected := tbl.Rows[0].Cells[3], int64(10); got != expected { + t.Errorf("expected cell[3] to equal %#v, got %#v", expected, got) + } + if got, expected := tbl.Rows[0].Cells[4], interface{}(nil); got != expected { + t.Errorf("expected cell[4] to equal %#v although the type does not match the column, got %#v", expected, got) + } + if got, expected := tbl.Rows[0].Cells[5], "[1 2 3]"; got != expected { + t.Errorf("expected cell[5] to equal %q, got %q", expected, got) + } + // Validate extra column for v1 + if i == 1 { + if got, expected := tbl.Rows[0].Cells[6], int64(5); got != expected { + t.Errorf("expected cell[6] to equal %q, got %q", expected, got) + } + } + } + } +} + +// TestColumnsPatch tests the case that a CRD was created with no top-level or +// per-version columns. One should be able to PATCH the CRD setting per-version columns. +func TestColumnsPatch(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() + tearDown, config, _, err := fixtures.StartDefaultServer(t) + if err != nil { + t.Fatal(err) + } + defer tearDown() + + apiExtensionClient, err := clientset.NewForConfig(config) if err != nil { t.Fatal(err) } - ret, err := crRestClient.Get(). - Resource(crd.Spec.Names.Plural). - SetHeader("Accept", fmt.Sprintf("application/json;as=Table;v=%s;g=%s, application/json", metav1beta1.SchemeGroupVersion.Version, metav1beta1.GroupName)). - VersionedParams(&metav1beta1.TableOptions{}, parameterCodec). - Do(). - Get() + dynamicClient, err := dynamic.NewForConfig(config) if err != nil { - t.Fatalf("failed to list %v resources: %v", gvk, err) + t.Fatal(err) } - tbl, ok := ret.(*metav1beta1.Table) - if !ok { - t.Fatalf("expected metav1beta1.Table, got %T", ret) + // CRD with no top-level and per-version columns should be created successfully + crd := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped)[0] + crd, err = fixtures.CreateNewCustomResourceDefinition(crd, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) } - t.Logf("%v table list: %#v", gvk, tbl) - if got, expected := len(tbl.ColumnDefinitions), 6; got != expected { - t.Errorf("expected %d headers, got %d", expected, got) - } else { - age := metav1beta1.TableColumnDefinition{Name: "Age", Type: "date", Format: "", Description: "Custom resource definition column (in JSONPath format): .metadata.creationTimestamp", Priority: 0} - if got, expected := tbl.ColumnDefinitions[1], age; got != expected { - t.Errorf("expected column definition %#v, got %#v", expected, got) - } + // One should be able to patch the CRD to use per-version columns. The top-level columns + // should not be defaulted during creation, and apiserver should not return validation + // error about top-level and per-version columns being mutual exclusive. + patch := []byte(`{"spec":{"versions":[{"name":"v1beta1","served":true,"storage":true,"additionalPrinterColumns":[{"name":"Age","type":"date","JSONPath":".metadata.creationTimestamp"}]},{"name":"v1","served":true,"storage":false,"additionalPrinterColumns":[{"name":"Age2","type":"date","JSONPath":".metadata.creationTimestamp"}]}]}}`) - alpha := metav1beta1.TableColumnDefinition{Name: "Alpha", Type: "string", Format: "", Description: "Custom resource definition column (in JSONPath format): .spec.alpha", Priority: 0} - if got, expected := tbl.ColumnDefinitions[2], alpha; got != expected { - t.Errorf("expected column definition %#v, got %#v", expected, got) - } - - beta := metav1beta1.TableColumnDefinition{Name: "Beta", Type: "integer", Format: "int64", Description: "the beta field", Priority: 42} - if got, expected := tbl.ColumnDefinitions[3], beta; got != expected { - t.Errorf("expected column definition %#v, got %#v", expected, got) - } - - gamma := metav1beta1.TableColumnDefinition{Name: "Gamma", Type: "integer", Description: "a column with wrongly typed values"} - if got, expected := tbl.ColumnDefinitions[4], gamma; got != expected { - t.Errorf("expected column definition %#v, got %#v", expected, got) - } - - epsilon := metav1beta1.TableColumnDefinition{Name: "Epsilon", Type: "string", Description: "an array of integers as string"} - if got, expected := tbl.ColumnDefinitions[5], epsilon; got != expected { - t.Errorf("expected column definition %#v, got %#v", expected, got) - } + _, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crd.Name, types.MergePatchType, patch) + if err != nil { + t.Fatal(err) } - if got, expected := len(tbl.Rows), 1; got != expected { - t.Errorf("expected %d rows, got %d", expected, got) - } else if got, expected := len(tbl.Rows[0].Cells), 6; got != expected { - t.Errorf("expected %d cells, got %d", expected, got) - } else { - if got, expected := tbl.Rows[0].Cells[0], "foo"; got != expected { - t.Errorf("expected cell[0] to equal %q, got %q", expected, got) - } - if s, ok := tbl.Rows[0].Cells[1].(string); !ok { - t.Errorf("expected cell[1] to be a string, got: %#v", tbl.Rows[0].Cells[1]) - } else { - dur, err := time.ParseDuration(s) - if err != nil { - t.Errorf("expected cell[1] to be a duration: %v", err) - } else if abs(dur.Seconds()) > 30.0 { - t.Errorf("expected cell[1] to be a small age, but got: %v", dur) - } - } - if got, expected := tbl.Rows[0].Cells[2], "foo_123"; got != expected { - t.Errorf("expected cell[2] to equal %q, got %q", expected, got) - } - if got, expected := tbl.Rows[0].Cells[3], int64(10); got != expected { - t.Errorf("expected cell[3] to equal %#v, got %#v", expected, got) - } - if got, expected := tbl.Rows[0].Cells[4], interface{}(nil); got != expected { - t.Errorf("expected cell[4] to equal %#v although the type does not match the column, got %#v", expected, got) - } - if got, expected := tbl.Rows[0].Cells[5], "[1 2 3]"; got != expected { - t.Errorf("expected cell[5] to equal %q, got %q", expected, got) - } + + crd, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) } + t.Logf("columns crd patched: %#v", crd) +} + +// TestPatchCleanTopLevelColumns tests the case that a CRD was created with top-level columns. +// One should be able to PATCH the CRD cleaning the top-level columns and setting per-version +// columns. +func TestPatchCleanTopLevelColumns(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() + tearDown, config, _, err := fixtures.StartDefaultServer(t) + if err != nil { + t.Fatal(err) + } + defer tearDown() + + apiExtensionClient, err := clientset.NewForConfig(config) + if err != nil { + t.Fatal(err) + } + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + t.Fatal(err) + } + + crd := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped)[0] + crd.Spec.AdditionalPrinterColumns = []apiextensionsv1beta1.CustomResourceColumnDefinition{ + {Name: "Age", Type: "date", Description: swaggerMetadataDescriptions["creationTimestamp"], JSONPath: ".metadata.creationTimestamp"}, + } + crd, err = fixtures.CreateNewCustomResourceDefinition(crd, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } + t.Logf("columns crd created: %#v", crd) + + // One should be able to patch the CRD to use per-version columns by cleaning + // the top-level columns. + patch := []byte(`{"spec":{"additionalPrinterColumns":null,"versions":[{"name":"v1beta1","served":true,"storage":true,"additionalPrinterColumns":[{"name":"Age","type":"date","JSONPath":".metadata.creationTimestamp"}]},{"name":"v1","served":true,"storage":false,"additionalPrinterColumns":[{"name":"Age2","type":"date","JSONPath":".metadata.creationTimestamp"}]}]}}`) + + _, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crd.Name, types.MergePatchType, patch) + if err != nil { + t.Fatal(err) + } + + crd, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + t.Logf("columns crd patched: %#v", crd) } func abs(x float64) float64 { diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go index 37f9d094381..997183c519f 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go @@ -17,6 +17,7 @@ limitations under the License. package integration import ( + "fmt" "strings" "testing" "time" @@ -25,8 +26,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apiextensions-apiserver/test/integration/fixtures" ) @@ -84,64 +88,114 @@ func TestForProperValidationErrors(t *testing.T) { } } -func newNoxuValidationCRD(scope apiextensionsv1beta1.ResourceScope) *apiextensionsv1beta1.CustomResourceDefinition { - return &apiextensionsv1beta1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"}, - Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: "mygroup.example.com", - Version: "v1beta1", - Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ - Plural: "noxus", - Singular: "nonenglishnoxu", - Kind: "WishIHadChosenNoxu", - ShortNames: []string{"foo", "bar", "abc", "def"}, - ListKind: "NoxuItemList", +func newNoxuValidationCRDs(scope apiextensionsv1beta1.ResourceScope) []*apiextensionsv1beta1.CustomResourceDefinition { + validationSchema := &apiextensionsv1beta1.JSONSchemaProps{ + Required: []string{"alpha", "beta"}, + AdditionalProperties: &apiextensionsv1beta1.JSONSchemaPropsOrBool{ + Allows: true, + }, + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "alpha": { + Description: "Alpha is an alphanumeric string with underscores", + Type: "string", + Pattern: "^[a-zA-Z0-9_]*$", }, - Scope: apiextensionsv1beta1.NamespaceScoped, - Validation: &apiextensionsv1beta1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ - Required: []string{"alpha", "beta"}, - AdditionalProperties: &apiextensionsv1beta1.JSONSchemaPropsOrBool{ - Allows: true, + "beta": { + Description: "Minimum value of beta is 10", + Type: "number", + Minimum: float64Ptr(10), + }, + "gamma": { + Description: "Gamma is restricted to foo, bar and baz", + Type: "string", + Enum: []apiextensionsv1beta1.JSON{ + { + Raw: []byte(`"foo"`), }, - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "alpha": { - Description: "Alpha is an alphanumeric string with underscores", - Type: "string", - Pattern: "^[a-zA-Z0-9_]*$", + { + Raw: []byte(`"bar"`), + }, + { + Raw: []byte(`"baz"`), + }, + }, + }, + "delta": { + Description: "Delta is a string with a maximum length of 5 or a number with a minimum value of 0", + AnyOf: []apiextensionsv1beta1.JSONSchemaProps{ + { + Type: "string", + MaxLength: int64Ptr(5), + }, + { + Type: "number", + Minimum: float64Ptr(0), + }, + }, + }, + }, + } + validationSchemaWithDescription := validationSchema.DeepCopy() + validationSchemaWithDescription.Description = "test" + return []*apiextensionsv1beta1.CustomResourceDefinition{ + { + ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"}, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "mygroup.example.com", + Version: "v1beta1", + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "noxus", + Singular: "nonenglishnoxu", + Kind: "WishIHadChosenNoxu", + ShortNames: []string{"foo", "bar", "abc", "def"}, + ListKind: "NoxuItemList", + }, + Scope: apiextensionsv1beta1.NamespaceScoped, + Validation: &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: validationSchema, + }, + Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Served: true, + Storage: true, + }, + { + Name: "v1", + Served: true, + Storage: false, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"}, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "mygroup.example.com", + Version: "v1beta1", + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "noxus", + Singular: "nonenglishnoxu", + Kind: "WishIHadChosenNoxu", + ShortNames: []string{"foo", "bar", "abc", "def"}, + ListKind: "NoxuItemList", + }, + Scope: apiextensionsv1beta1.NamespaceScoped, + Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Served: true, + Storage: true, + Schema: &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: validationSchema, }, - "beta": { - Description: "Minimum value of beta is 10", - Type: "number", - Minimum: float64Ptr(10), - }, - "gamma": { - Description: "Gamma is restricted to foo, bar and baz", - Type: "string", - Enum: []apiextensionsv1beta1.JSON{ - { - Raw: []byte(`"foo"`), - }, - { - Raw: []byte(`"bar"`), - }, - { - Raw: []byte(`"baz"`), - }, - }, - }, - "delta": { - Description: "Delta is a string with a maximum length of 5 or a number with a minimum value of 0", - AnyOf: []apiextensionsv1beta1.JSONSchemaProps{ - { - Type: "string", - MaxLength: int64Ptr(5), - }, - { - Type: "number", - Minimum: float64Ptr(0), - }, - }, + }, + { + Name: "v1", + Served: true, + Storage: false, + Schema: &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: validationSchemaWithDescription, }, }, }, @@ -168,253 +222,320 @@ func newNoxuValidationInstance(namespace, name string) *unstructured.Unstructure } func TestCustomResourceValidation(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t) if err != nil { t.Fatal(err) } defer tearDown() - noxuDefinition := newNoxuValidationCRD(apiextensionsv1beta1.NamespaceScoped) - noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err != nil { - t.Fatal(err) - } + noxuDefinitions := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped) + for _, noxuDefinition := range noxuDefinitions { + noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } - ns := "not-the-default" - noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition) - _, err = instantiateCustomResource(t, newNoxuValidationInstance(ns, "foo"), noxuResourceClient, noxuDefinition) - if err != nil { - t.Fatalf("unable to create noxu instance: %v", err) + ns := "not-the-default" + for _, v := range noxuDefinition.Spec.Versions { + noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name) + instanceToCreate := newNoxuValidationInstance(ns, "foo") + instanceToCreate.Object["apiVersion"] = fmt.Sprintf("%s/%s", noxuDefinition.Spec.Group, v.Name) + _, err = instantiateVersionedCustomResource(t, instanceToCreate, noxuResourceClient, noxuDefinition, v.Name) + if err != nil { + t.Fatalf("unable to create noxu instance: %v", err) + } + noxuResourceClient.Delete("foo", &metav1.DeleteOptions{}) + } + if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil { + t.Fatal(err) + } } } func TestCustomResourceUpdateValidation(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t) if err != nil { t.Fatal(err) } defer tearDown() - noxuDefinition := newNoxuValidationCRD(apiextensionsv1beta1.NamespaceScoped) - noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err != nil { - t.Fatal(err) - } + noxuDefinitions := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped) + for _, noxuDefinition := range noxuDefinitions { + noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } - ns := "not-the-default" - noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition) - _, err = instantiateCustomResource(t, newNoxuValidationInstance(ns, "foo"), noxuResourceClient, noxuDefinition) - if err != nil { - t.Fatalf("unable to create noxu instance: %v", err) - } + ns := "not-the-default" + for _, v := range noxuDefinition.Spec.Versions { + noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name) + instanceToCreate := newNoxuValidationInstance(ns, "foo") + instanceToCreate.Object["apiVersion"] = fmt.Sprintf("%s/%s", noxuDefinition.Spec.Group, v.Name) + _, err = instantiateVersionedCustomResource(t, instanceToCreate, noxuResourceClient, noxuDefinition, v.Name) + if err != nil { + t.Fatalf("unable to create noxu instance: %v", err) + } - gottenNoxuInstance, err := noxuResourceClient.Get("foo", metav1.GetOptions{}) - if err != nil { - t.Fatal(err) - } + gottenNoxuInstance, err := noxuResourceClient.Get("foo", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } - // invalidate the instance - gottenNoxuInstance.Object = map[string]interface{}{ - "apiVersion": "mygroup.example.com/v1beta1", - "kind": "WishIHadChosenNoxu", - "metadata": map[string]interface{}{ - "namespace": "not-the-default", - "name": "foo", - }, - "gamma": "bar", - "delta": "hello", - } + // invalidate the instance + gottenNoxuInstance.Object = map[string]interface{}{ + "apiVersion": "mygroup.example.com/v1beta1", + "kind": "WishIHadChosenNoxu", + "metadata": map[string]interface{}{ + "namespace": "not-the-default", + "name": "foo", + }, + "gamma": "bar", + "delta": "hello", + } - _, err = noxuResourceClient.Update(gottenNoxuInstance, metav1.UpdateOptions{}) - if err == nil { - t.Fatalf("unexpected non-error: alpha and beta should be present while updating %v", gottenNoxuInstance) + _, err = noxuResourceClient.Update(gottenNoxuInstance, metav1.UpdateOptions{}) + if err == nil { + t.Fatalf("unexpected non-error: alpha and beta should be present while updating %v", gottenNoxuInstance) + } + noxuResourceClient.Delete("foo", &metav1.DeleteOptions{}) + } + if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil { + t.Fatal(err) + } } } func TestCustomResourceValidationErrors(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t) if err != nil { t.Fatal(err) } defer tearDown() - noxuDefinition := newNoxuValidationCRD(apiextensionsv1beta1.NamespaceScoped) - noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err != nil { - t.Fatal(err) - } - - ns := "not-the-default" - noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition) - - tests := []struct { - name string - instanceFn func() *unstructured.Unstructured - expectedError string - }{ - { - name: "bad alpha", - instanceFn: func() *unstructured.Unstructured { - instance := newNoxuValidationInstance(ns, "foo") - instance.Object["alpha"] = "foo_123!" - return instance - }, - expectedError: "alpha in body should match '^[a-zA-Z0-9_]*$'", - }, - { - name: "bad beta", - instanceFn: func() *unstructured.Unstructured { - instance := newNoxuValidationInstance(ns, "foo") - instance.Object["beta"] = 5 - return instance - }, - expectedError: "beta in body should be greater than or equal to 10", - }, - { - name: "bad gamma", - instanceFn: func() *unstructured.Unstructured { - instance := newNoxuValidationInstance(ns, "foo") - instance.Object["gamma"] = "qux" - return instance - }, - expectedError: "gamma in body should be one of [foo bar baz]", - }, - { - name: "bad delta", - instanceFn: func() *unstructured.Unstructured { - instance := newNoxuValidationInstance(ns, "foo") - instance.Object["delta"] = "foobarbaz" - return instance - }, - expectedError: "must validate at least one schema (anyOf)\ndelta in body should be at most 5 chars long", - }, - { - name: "absent alpha and beta", - instanceFn: func() *unstructured.Unstructured { - instance := newNoxuValidationInstance(ns, "foo") - instance.Object = map[string]interface{}{ - "apiVersion": "mygroup.example.com/v1beta1", - "kind": "WishIHadChosenNoxu", - "metadata": map[string]interface{}{ - "namespace": "not-the-default", - "name": "foo", - }, - "gamma": "bar", - "delta": "hello", - } - return instance - }, - expectedError: ".alpha in body is required\n.beta in body is required", - }, - } - - for _, tc := range tests { - _, err := noxuResourceClient.Create(tc.instanceFn(), metav1.CreateOptions{}) - if err == nil { - t.Errorf("%v: expected %v", tc.name, tc.expectedError) - continue + noxuDefinitions := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped) + for _, noxuDefinition := range noxuDefinitions { + noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) } - // this only works when status errors contain the expect kind and version, so this effectively tests serializations too - if !strings.Contains(err.Error(), tc.expectedError) { - t.Errorf("%v: expected %v, got %v", tc.name, tc.expectedError, err) - continue + + ns := "not-the-default" + + tests := []struct { + name string + instanceFn func() *unstructured.Unstructured + expectedError string + }{ + { + name: "bad alpha", + instanceFn: func() *unstructured.Unstructured { + instance := newNoxuValidationInstance(ns, "foo") + instance.Object["alpha"] = "foo_123!" + return instance + }, + expectedError: "alpha in body should match '^[a-zA-Z0-9_]*$'", + }, + { + name: "bad beta", + instanceFn: func() *unstructured.Unstructured { + instance := newNoxuValidationInstance(ns, "foo") + instance.Object["beta"] = 5 + return instance + }, + expectedError: "beta in body should be greater than or equal to 10", + }, + { + name: "bad gamma", + instanceFn: func() *unstructured.Unstructured { + instance := newNoxuValidationInstance(ns, "foo") + instance.Object["gamma"] = "qux" + return instance + }, + expectedError: "gamma in body should be one of [foo bar baz]", + }, + { + name: "bad delta", + instanceFn: func() *unstructured.Unstructured { + instance := newNoxuValidationInstance(ns, "foo") + instance.Object["delta"] = "foobarbaz" + return instance + }, + expectedError: "must validate at least one schema (anyOf)\ndelta in body should be at most 5 chars long", + }, + { + name: "absent alpha and beta", + instanceFn: func() *unstructured.Unstructured { + instance := newNoxuValidationInstance(ns, "foo") + instance.Object = map[string]interface{}{ + "apiVersion": "mygroup.example.com/v1beta1", + "kind": "WishIHadChosenNoxu", + "metadata": map[string]interface{}{ + "namespace": "not-the-default", + "name": "foo", + }, + "gamma": "bar", + "delta": "hello", + } + return instance + }, + expectedError: ".alpha in body is required\n.beta in body is required", + }, + } + + for _, tc := range tests { + for _, v := range noxuDefinition.Spec.Versions { + noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name) + instanceToCreate := tc.instanceFn() + instanceToCreate.Object["apiVersion"] = fmt.Sprintf("%s/%s", noxuDefinition.Spec.Group, v.Name) + _, err := noxuResourceClient.Create(instanceToCreate, metav1.CreateOptions{}) + if err == nil { + t.Errorf("%v: expected %v", tc.name, tc.expectedError) + continue + } + // this only works when status errors contain the expect kind and version, so this effectively tests serializations too + if !strings.Contains(err.Error(), tc.expectedError) { + t.Errorf("%v: expected %v, got %v", tc.name, tc.expectedError, err) + continue + } + } + } + if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil { + t.Fatal(err) } } } func TestCRValidationOnCRDUpdate(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t) if err != nil { t.Fatal(err) } defer tearDown() - noxuDefinition := newNoxuValidationCRD(apiextensionsv1beta1.NamespaceScoped) + noxuDefinitions := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped) + for i, noxuDefinition := range noxuDefinitions { + for _, v := range noxuDefinition.Spec.Versions { + // Re-define the CRD to make sure we start with a clean CRD + noxuDefinition := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped)[i] + validationSchema, err := getSchemaForVersion(noxuDefinition, v.Name) + if err != nil { + t.Fatal(err) + } - // set stricter schema - noxuDefinition.Spec.Validation.OpenAPIV3Schema.Required = []string{"alpha", "beta", "epsilon"} + // set stricter schema + validationSchema.OpenAPIV3Schema.Required = []string{"alpha", "beta", "epsilon"} - noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err != nil { - t.Fatal(err) - } - ns := "not-the-default" - noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition) + noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } + ns := "not-the-default" + noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name) + instanceToCreate := newNoxuValidationInstance(ns, "foo") + instanceToCreate.Object["apiVersion"] = fmt.Sprintf("%s/%s", noxuDefinition.Spec.Group, v.Name) - // CR is rejected - _, err = instantiateCustomResource(t, newNoxuValidationInstance(ns, "foo"), noxuResourceClient, noxuDefinition) - if err == nil { - t.Fatalf("unexpected non-error: CR should be rejected") - } + // CR is rejected + _, err = instantiateVersionedCustomResource(t, instanceToCreate, noxuResourceClient, noxuDefinition, v.Name) + if err == nil { + t.Fatalf("unexpected non-error: CR should be rejected") + } - // update the CRD to a less stricter schema - _, err = updateCustomResourceDefinitionWithRetry(apiExtensionClient, "noxus.mygroup.example.com", func(crd *apiextensionsv1beta1.CustomResourceDefinition) { - crd.Spec.Validation.OpenAPIV3Schema.Required = []string{"alpha", "beta"} - }) - if err != nil { - t.Fatal(err) - } + // update the CRD to a less stricter schema + _, err = UpdateCustomResourceDefinitionWithRetry(apiExtensionClient, "noxus.mygroup.example.com", func(crd *apiextensionsv1beta1.CustomResourceDefinition) { + validationSchema, err := getSchemaForVersion(crd, v.Name) + if err != nil { + t.Fatal(err) + } + validationSchema.OpenAPIV3Schema.Required = []string{"alpha", "beta"} + }) + if err != nil { + t.Fatal(err) + } - // CR is now accepted - err = wait.Poll(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { - _, err := noxuResourceClient.Create(newNoxuValidationInstance(ns, "foo"), metav1.CreateOptions{}) - if statusError, isStatus := err.(*apierrors.StatusError); isStatus { - if strings.Contains(statusError.Error(), "is invalid") { - return false, nil + // CR is now accepted + err = wait.Poll(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { + _, err := noxuResourceClient.Create(instanceToCreate, metav1.CreateOptions{}) + if _, isStatus := err.(*apierrors.StatusError); isStatus { + if apierrors.IsInvalid(err) { + return false, nil + } + } + if err != nil { + return false, err + } + return true, nil + }) + if err != nil { + t.Fatal(err) + } + noxuResourceClient.Delete("foo", &metav1.DeleteOptions{}) + if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil { + t.Fatal(err) } } - if err != nil { - return false, err - } - return true, nil - }) - if err != nil { - t.Fatal(err) } } func TestForbiddenFieldsInSchema(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t) if err != nil { t.Fatal(err) } defer tearDown() - noxuDefinition := newNoxuValidationCRD(apiextensionsv1beta1.NamespaceScoped) - noxuDefinition.Spec.Validation.OpenAPIV3Schema.AdditionalProperties.Allows = false + noxuDefinitions := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped) + for i, noxuDefinition := range noxuDefinitions { + for _, v := range noxuDefinition.Spec.Versions { + // Re-define the CRD to make sure we start with a clean CRD + noxuDefinition := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped)[i] + validationSchema, err := getSchemaForVersion(noxuDefinition, v.Name) + if err != nil { + t.Fatal(err) + } + validationSchema.OpenAPIV3Schema.AdditionalProperties.Allows = false - _, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err == nil { - t.Fatalf("unexpected non-error: additionalProperties cannot be set to false") - } + _, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err == nil { + t.Fatalf("unexpected non-error: additionalProperties cannot be set to false") + } - noxuDefinition.Spec.Validation.OpenAPIV3Schema.Properties["zeta"] = apiextensionsv1beta1.JSONSchemaProps{ - Type: "array", - UniqueItems: true, - } - noxuDefinition.Spec.Validation.OpenAPIV3Schema.AdditionalProperties.Allows = true + validationSchema.OpenAPIV3Schema.Properties["zeta"] = apiextensionsv1beta1.JSONSchemaProps{ + Type: "array", + UniqueItems: true, + } + validationSchema.OpenAPIV3Schema.AdditionalProperties.Allows = true - _, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err == nil { - t.Fatalf("unexpected non-error: uniqueItems cannot be set to true") - } + _, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err == nil { + t.Fatalf("unexpected non-error: uniqueItems cannot be set to true") + } - noxuDefinition.Spec.Validation.OpenAPIV3Schema.Ref = strPtr("#/definition/zeta") - noxuDefinition.Spec.Validation.OpenAPIV3Schema.Properties["zeta"] = apiextensionsv1beta1.JSONSchemaProps{ - Type: "array", - UniqueItems: false, - } + validationSchema.OpenAPIV3Schema.Ref = strPtr("#/definition/zeta") + validationSchema.OpenAPIV3Schema.Properties["zeta"] = apiextensionsv1beta1.JSONSchemaProps{ + Type: "array", + UniqueItems: false, + } - _, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err == nil { - t.Fatal("unexpected non-error: $ref cannot be non-empty string") - } + _, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err == nil { + t.Fatal("unexpected non-error: $ref cannot be non-empty string") + } - noxuDefinition.Spec.Validation.OpenAPIV3Schema.Ref = nil + validationSchema.OpenAPIV3Schema.Ref = nil - noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) - if err != nil { - t.Fatal(err) + noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } + if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil { + t.Fatal(err) + } + } } } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/versioning_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/versioning_test.go index 7a813ab49cf..b56d3950e7d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/versioning_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/versioning_test.go @@ -64,7 +64,7 @@ func TestInternalVersionIsHandlerVersion(t *testing.T) { // update validation via update because the cache priming in CreateNewCustomResourceDefinition will fail otherwise t.Logf("Updating CRD to validate apiVersion") - noxuDefinition, err = updateCustomResourceDefinitionWithRetry(apiExtensionClient, noxuDefinition.Name, func(crd *apiextensionsv1beta1.CustomResourceDefinition) { + noxuDefinition, err = UpdateCustomResourceDefinitionWithRetry(apiExtensionClient, noxuDefinition.Name, func(crd *apiextensionsv1beta1.CustomResourceDefinition) { crd.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{ OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/yaml_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/yaml_test.go index 45e5176b000..e4989d2535a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/yaml_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/yaml_test.go @@ -22,15 +22,18 @@ import ( "net/http" "testing" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/dynamic" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apiextensions-apiserver/test/integration/fixtures" ) @@ -354,6 +357,7 @@ values: } func TestYAMLSubresource(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() tearDown, config, _, err := fixtures.StartDefaultServer(t) if err != nil { t.Fatal(err) @@ -369,7 +373,7 @@ func TestYAMLSubresource(t *testing.T) { t.Fatal(err) } - noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.ClusterScoped) + noxuDefinition := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.ClusterScoped)[0] noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) if err != nil { t.Fatal(err) diff --git a/staging/src/k8s.io/apimachinery/Godeps/Godeps.json b/staging/src/k8s.io/apimachinery/Godeps/Godeps.json index bbb5a8a3388..c9bd57586b8 100644 --- a/staging/src/k8s.io/apimachinery/Godeps/Godeps.json +++ b/staging/src/k8s.io/apimachinery/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/apimachinery", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -26,10 +26,6 @@ "ImportPath": "github.com/evanphx/json-patch", "Rev": "36442dbdb585210f8d5a1b45e67aa323c197d5c4" }, - { - "ImportPath": "github.com/ghodss/yaml", - "Rev": "c7ce16629ff4cd059ed96ed06419dd3856fd3577" - }, { "ImportPath": "github.com/gogo/protobuf/proto", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" @@ -38,10 +34,6 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/golang/groupcache/lru", "Rev": "02826c3e79038b59d737d3b1c0a1d937f71a4433" @@ -128,31 +120,31 @@ }, { "ImportPath": "golang.org/x/net/html", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/html/atom", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/websocket", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/text/secure/bidirule", @@ -178,9 +170,17 @@ "ImportPath": "gopkg.in/yaml.v2", "Rev": "5420a8b6744d3b0345ab293f6fcba19c978f1183" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" + }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" } ] } diff --git a/staging/src/k8s.io/apimachinery/pkg/api/meta/BUILD b/staging/src/k8s.io/apimachinery/pkg/api/meta/BUILD index 938ba9b5b73..4ced3ce5663 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/meta/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/api/meta/BUILD @@ -49,7 +49,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/api/meta/meta.go b/staging/src/k8s.io/apimachinery/pkg/api/meta/meta.go index 854bd30fa3d..6fe7458f6c4 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/meta/meta.go +++ b/staging/src/k8s.io/apimachinery/pkg/api/meta/meta.go @@ -20,7 +20,7 @@ import ( "fmt" "reflect" - "github.com/golang/glog" + "k8s.io/klog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1beta1 "k8s.io/apimachinery/pkg/apis/meta/v1beta1" @@ -607,7 +607,7 @@ func (a genericAccessor) GetOwnerReferences() []metav1.OwnerReference { var ret []metav1.OwnerReference s := a.ownerReferences if s.Kind() != reflect.Ptr || s.Elem().Kind() != reflect.Slice { - glog.Errorf("expect %v to be a pointer to slice", s) + klog.Errorf("expect %v to be a pointer to slice", s) return ret } s = s.Elem() @@ -615,7 +615,7 @@ func (a genericAccessor) GetOwnerReferences() []metav1.OwnerReference { ret = make([]metav1.OwnerReference, s.Len(), s.Len()+1) for i := 0; i < s.Len(); i++ { if err := extractFromOwnerReference(s.Index(i), &ret[i]); err != nil { - glog.Errorf("extractFromOwnerReference failed: %v", err) + klog.Errorf("extractFromOwnerReference failed: %v", err) return ret } } @@ -625,13 +625,13 @@ func (a genericAccessor) GetOwnerReferences() []metav1.OwnerReference { func (a genericAccessor) SetOwnerReferences(references []metav1.OwnerReference) { s := a.ownerReferences if s.Kind() != reflect.Ptr || s.Elem().Kind() != reflect.Slice { - glog.Errorf("expect %v to be a pointer to slice", s) + klog.Errorf("expect %v to be a pointer to slice", s) } s = s.Elem() newReferences := reflect.MakeSlice(s.Type(), len(references), len(references)) for i := 0; i < len(references); i++ { if err := setOwnerReference(newReferences.Index(i), &references[i]); err != nil { - glog.Errorf("setOwnerReference failed: %v", err) + klog.Errorf("setOwnerReference failed: %v", err) return } } diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/BUILD b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/BUILD index 8b02313678e..267bccc4aa9 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/BUILD @@ -25,7 +25,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/duration_test.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/duration_test.go index 7230cb28ab4..34ca6b5ded2 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/duration_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/duration_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) type DurationHolder struct { diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/micro_time_test.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/micro_time_test.go index 985aa70b0dc..b08e3f236c5 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/micro_time_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/micro_time_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) type MicroTimeHolder struct { diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/time_test.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/time_test.go index 0bf6c360513..7ade16f6197 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/time_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/time_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) type TimeHolder struct { diff --git a/staging/src/k8s.io/apimachinery/pkg/labels/BUILD b/staging/src/k8s.io/apimachinery/pkg/labels/BUILD index 14666aec8ec..eeffddf89e5 100644 --- a/staging/src/k8s.io/apimachinery/pkg/labels/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/labels/BUILD @@ -33,7 +33,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/selection:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/labels/selector.go b/staging/src/k8s.io/apimachinery/pkg/labels/selector.go index 374d2ef1377..f5a0888932f 100644 --- a/staging/src/k8s.io/apimachinery/pkg/labels/selector.go +++ b/staging/src/k8s.io/apimachinery/pkg/labels/selector.go @@ -23,10 +23,10 @@ import ( "strconv" "strings" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/klog" ) // Requirements is AND of all requirements. @@ -211,13 +211,13 @@ func (r *Requirement) Matches(ls Labels) bool { } lsValue, err := strconv.ParseInt(ls.Get(r.key), 10, 64) if err != nil { - glog.V(10).Infof("ParseInt failed for value %+v in label %+v, %+v", ls.Get(r.key), ls, err) + klog.V(10).Infof("ParseInt failed for value %+v in label %+v, %+v", ls.Get(r.key), ls, err) return false } // There should be only one strValue in r.strValues, and can be converted to a integer. if len(r.strValues) != 1 { - glog.V(10).Infof("Invalid values count %+v of requirement %#v, for 'Gt', 'Lt' operators, exactly one value is required", len(r.strValues), r) + klog.V(10).Infof("Invalid values count %+v of requirement %#v, for 'Gt', 'Lt' operators, exactly one value is required", len(r.strValues), r) return false } @@ -225,7 +225,7 @@ func (r *Requirement) Matches(ls Labels) bool { for i := range r.strValues { rValue, err = strconv.ParseInt(r.strValues[i], 10, 64) if err != nil { - glog.V(10).Infof("ParseInt failed for value %+v in requirement %#v, for 'Gt', 'Lt' operators, the value must be an integer", r.strValues[i], r) + klog.V(10).Infof("ParseInt failed for value %+v in requirement %#v, for 'Gt', 'Lt' operators, the value must be an integer", r.strValues[i], r) return false } } diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/BUILD b/staging/src/k8s.io/apimachinery/pkg/runtime/BUILD index 3c07e159539..6b52ceff0fe 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/BUILD @@ -67,7 +67,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/github.com/gogo/protobuf/proto:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/converter.go b/staging/src/k8s.io/apimachinery/pkg/runtime/converter.go index 291d7a4e888..dff56e03401 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/converter.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/converter.go @@ -33,7 +33,7 @@ import ( "k8s.io/apimachinery/pkg/util/json" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "github.com/golang/glog" + "k8s.io/klog" ) // UnstructuredConverter is an interface for converting between interface{} @@ -133,10 +133,10 @@ func (c *unstructuredConverter) FromUnstructured(u map[string]interface{}, obj i newObj := reflect.New(t.Elem()).Interface() newErr := fromUnstructuredViaJSON(u, newObj) if (err != nil) != (newErr != nil) { - glog.Fatalf("FromUnstructured unexpected error for %v: error: %v", u, err) + klog.Fatalf("FromUnstructured unexpected error for %v: error: %v", u, err) } if err == nil && !c.comparison.DeepEqual(obj, newObj) { - glog.Fatalf("FromUnstructured mismatch\nobj1: %#v\nobj2: %#v", obj, newObj) + klog.Fatalf("FromUnstructured mismatch\nobj1: %#v\nobj2: %#v", obj, newObj) } } return err @@ -424,10 +424,10 @@ func (c *unstructuredConverter) ToUnstructured(obj interface{}) (map[string]inte newUnstr := map[string]interface{}{} newErr := toUnstructuredViaJSON(obj, &newUnstr) if (err != nil) != (newErr != nil) { - glog.Fatalf("ToUnstructured unexpected error for %v: error: %v; newErr: %v", obj, err, newErr) + klog.Fatalf("ToUnstructured unexpected error for %v: error: %v; newErr: %v", obj, err, newErr) } if err == nil && !c.comparison.DeepEqual(u, newUnstr) { - glog.Fatalf("ToUnstructured mismatch\nobj1: %#v\nobj2: %#v", u, newUnstr) + klog.Fatalf("ToUnstructured mismatch\nobj1: %#v\nobj2: %#v", u, newUnstr) } } if err != nil { diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/BUILD b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/BUILD index 05764c34b7d..e323052f4d4 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/BUILD @@ -21,9 +21,9 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer/testing:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/google/gofuzz:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/codec_test.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/codec_test.go index d27da113a96..90a2962bf6f 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/codec_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/codec_test.go @@ -32,9 +32,9 @@ import ( serializertesting "k8s.io/apimachinery/pkg/runtime/serializer/testing" "k8s.io/apimachinery/pkg/util/diff" - "github.com/ghodss/yaml" "github.com/google/gofuzz" flag "github.com/spf13/pflag" + "sigs.k8s.io/yaml" ) var fuzzIters = flag.Int("fuzz-iters", 50, "How many fuzzing iterations to do.") diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/BUILD b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/BUILD index 4857211b347..8ff68498d0a 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/BUILD @@ -34,9 +34,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer/recognizer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/framer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/json-iterator/go:go_default_library", "//vendor/github.com/modern-go/reflect2:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go index 382c4858e7f..8987e74c680 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go @@ -22,9 +22,9 @@ import ( "strconv" "unsafe" - "github.com/ghodss/yaml" jsoniter "github.com/json-iterator/go" "github.com/modern-go/reflect2" + "sigs.k8s.io/yaml" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/staging/src/k8s.io/apimachinery/pkg/util/httpstream/spdy/BUILD b/staging/src/k8s.io/apimachinery/pkg/util/httpstream/spdy/BUILD index 9721de40006..2c7d03c9560 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/httpstream/spdy/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/util/httpstream/spdy/BUILD @@ -39,7 +39,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/third_party/forked/golang/netutil:go_default_library", "//vendor/github.com/docker/spdystream:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/httpstream/spdy/connection.go b/staging/src/k8s.io/apimachinery/pkg/util/httpstream/spdy/connection.go index 3dc8e23ae14..9d222faa898 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/httpstream/spdy/connection.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/httpstream/spdy/connection.go @@ -23,8 +23,8 @@ import ( "time" "github.com/docker/spdystream" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/httpstream" + "k8s.io/klog" ) // connection maintains state about a spdystream.Connection and its associated @@ -128,7 +128,7 @@ func (c *connection) newSpdyStream(stream *spdystream.Stream) { err := c.newStreamHandler(stream, replySent) rejectStream := (err != nil) if rejectStream { - glog.Warningf("Stream rejected: %v", err) + klog.Warningf("Stream rejected: %v", err) stream.Reset() return } diff --git a/staging/src/k8s.io/apimachinery/pkg/util/intstr/BUILD b/staging/src/k8s.io/apimachinery/pkg/util/intstr/BUILD index 3e4b9eb71e1..50bb5fa4c70 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/intstr/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/util/intstr/BUILD @@ -10,7 +10,7 @@ go_test( name = "go_default_test", srcs = ["intstr_test.go"], embed = [":go_default_library"], - deps = ["//vendor/github.com/ghodss/yaml:go_default_library"], + deps = ["//vendor/sigs.k8s.io/yaml:go_default_library"], ) go_library( @@ -23,8 +23,8 @@ go_library( importpath = "k8s.io/apimachinery/pkg/util/intstr", deps = [ "//vendor/github.com/gogo/protobuf/proto:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/gofuzz:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/intstr/intstr.go b/staging/src/k8s.io/apimachinery/pkg/util/intstr/intstr.go index 642b83cec21..5b26ed26263 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/intstr/intstr.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/intstr/intstr.go @@ -25,8 +25,8 @@ import ( "strconv" "strings" - "github.com/golang/glog" "github.com/google/gofuzz" + "k8s.io/klog" ) // IntOrString is a type that can hold an int32 or a string. When used in @@ -58,7 +58,7 @@ const ( // TODO: convert to (val int32) func FromInt(val int) IntOrString { if val > math.MaxInt32 || val < math.MinInt32 { - glog.Errorf("value: %d overflows int32\n%s\n", val, debug.Stack()) + klog.Errorf("value: %d overflows int32\n%s\n", val, debug.Stack()) } return IntOrString{Type: Int, IntVal: int32(val)} } diff --git a/staging/src/k8s.io/apimachinery/pkg/util/intstr/intstr_test.go b/staging/src/k8s.io/apimachinery/pkg/util/intstr/intstr_test.go index 690fe2d5331..f6e082987d8 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/intstr/intstr_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/intstr/intstr_test.go @@ -21,7 +21,7 @@ import ( "reflect" "testing" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) func TestFromInt(t *testing.T) { diff --git a/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/BUILD b/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/BUILD index 107492bb1df..88b93d25815 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/BUILD @@ -14,7 +14,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/json:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/patch_test.go b/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/patch_test.go index 9672deaad61..25435206570 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/patch_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/patch_test.go @@ -23,8 +23,8 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/evanphx/json-patch" - "github.com/ghodss/yaml" "k8s.io/apimachinery/pkg/util/json" + "sigs.k8s.io/yaml" ) type FilterNullTestCases struct { diff --git a/staging/src/k8s.io/apimachinery/pkg/util/mergepatch/BUILD b/staging/src/k8s.io/apimachinery/pkg/util/mergepatch/BUILD index 1ed8649a9ec..aa444d4f6c2 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/mergepatch/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/util/mergepatch/BUILD @@ -22,7 +22,7 @@ go_library( importpath = "k8s.io/apimachinery/pkg/util/mergepatch", deps = [ "//vendor/github.com/davecgh/go-spew/spew:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/mergepatch/util.go b/staging/src/k8s.io/apimachinery/pkg/util/mergepatch/util.go index d09a939be30..990fa0d43a6 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/mergepatch/util.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/mergepatch/util.go @@ -21,7 +21,7 @@ import ( "reflect" "github.com/davecgh/go-spew/spew" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) // PreconditionFunc asserts that an incompatible change is not present within a patch. diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/BUILD b/staging/src/k8s.io/apimachinery/pkg/util/net/BUILD index d38670a8c87..6f4542e72c1 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/BUILD @@ -37,8 +37,8 @@ go_library( importpath = "k8s.io/apimachinery/pkg/util/net", deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/net/http2:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/http.go b/staging/src/k8s.io/apimachinery/pkg/util/net/http.go index 7c2a5e6286d..155667cdfc7 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/http.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/http.go @@ -31,8 +31,8 @@ import ( "strconv" "strings" - "github.com/golang/glog" "golang.org/x/net/http2" + "k8s.io/klog" ) // JoinPreservingTrailingSlash does a path.Join of the specified elements, @@ -107,10 +107,10 @@ func SetTransportDefaults(t *http.Transport) *http.Transport { t = SetOldTransportDefaults(t) // Allow clients to disable http2 if needed. if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 { - glog.Infof("HTTP2 has been explicitly disabled") + klog.Infof("HTTP2 has been explicitly disabled") } else { if err := http2.ConfigureTransport(t); err != nil { - glog.Warningf("Transport failed http2 configuration: %v", err) + klog.Warningf("Transport failed http2 configuration: %v", err) } } return t @@ -368,7 +368,7 @@ redirectLoop: resp, err := http.ReadResponse(respReader, nil) if err != nil { // Unable to read the backend response; let the client handle it. - glog.Warningf("Error reading backend response: %v", err) + klog.Warningf("Error reading backend response: %v", err) break redirectLoop } diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go b/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go index 0ab9b36080b..daf5d249645 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go @@ -26,7 +26,7 @@ import ( "strings" - "github.com/golang/glog" + "k8s.io/klog" ) type AddressFamily uint @@ -193,7 +193,7 @@ func isInterfaceUp(intf *net.Interface) bool { return false } if intf.Flags&net.FlagUp != 0 { - glog.V(4).Infof("Interface %v is up", intf.Name) + klog.V(4).Infof("Interface %v is up", intf.Name) return true } return false @@ -208,20 +208,20 @@ func isLoopbackOrPointToPoint(intf *net.Interface) bool { func getMatchingGlobalIP(addrs []net.Addr, family AddressFamily) (net.IP, error) { if len(addrs) > 0 { for i := range addrs { - glog.V(4).Infof("Checking addr %s.", addrs[i].String()) + klog.V(4).Infof("Checking addr %s.", addrs[i].String()) ip, _, err := net.ParseCIDR(addrs[i].String()) if err != nil { return nil, err } if memberOf(ip, family) { if ip.IsGlobalUnicast() { - glog.V(4).Infof("IP found %v", ip) + klog.V(4).Infof("IP found %v", ip) return ip, nil } else { - glog.V(4).Infof("Non-global unicast address found %v", ip) + klog.V(4).Infof("Non-global unicast address found %v", ip) } } else { - glog.V(4).Infof("%v is not an IPv%d address", ip, int(family)) + klog.V(4).Infof("%v is not an IPv%d address", ip, int(family)) } } @@ -241,13 +241,13 @@ func getIPFromInterface(intfName string, forFamily AddressFamily, nw networkInte if err != nil { return nil, err } - glog.V(4).Infof("Interface %q has %d addresses :%v.", intfName, len(addrs), addrs) + klog.V(4).Infof("Interface %q has %d addresses :%v.", intfName, len(addrs), addrs) matchingIP, err := getMatchingGlobalIP(addrs, forFamily) if err != nil { return nil, err } if matchingIP != nil { - glog.V(4).Infof("Found valid IPv%d address %v for interface %q.", int(forFamily), matchingIP, intfName) + klog.V(4).Infof("Found valid IPv%d address %v for interface %q.", int(forFamily), matchingIP, intfName) return matchingIP, nil } } @@ -275,14 +275,14 @@ func chooseIPFromHostInterfaces(nw networkInterfacer) (net.IP, error) { return nil, fmt.Errorf("no interfaces found on host.") } for _, family := range []AddressFamily{familyIPv4, familyIPv6} { - glog.V(4).Infof("Looking for system interface with a global IPv%d address", uint(family)) + klog.V(4).Infof("Looking for system interface with a global IPv%d address", uint(family)) for _, intf := range intfs { if !isInterfaceUp(&intf) { - glog.V(4).Infof("Skipping: down interface %q", intf.Name) + klog.V(4).Infof("Skipping: down interface %q", intf.Name) continue } if isLoopbackOrPointToPoint(&intf) { - glog.V(4).Infof("Skipping: LB or P2P interface %q", intf.Name) + klog.V(4).Infof("Skipping: LB or P2P interface %q", intf.Name) continue } addrs, err := nw.Addrs(&intf) @@ -290,7 +290,7 @@ func chooseIPFromHostInterfaces(nw networkInterfacer) (net.IP, error) { return nil, err } if len(addrs) == 0 { - glog.V(4).Infof("Skipping: no addresses on interface %q", intf.Name) + klog.V(4).Infof("Skipping: no addresses on interface %q", intf.Name) continue } for _, addr := range addrs { @@ -299,15 +299,15 @@ func chooseIPFromHostInterfaces(nw networkInterfacer) (net.IP, error) { return nil, fmt.Errorf("Unable to parse CIDR for interface %q: %s", intf.Name, err) } if !memberOf(ip, family) { - glog.V(4).Infof("Skipping: no address family match for %q on interface %q.", ip, intf.Name) + klog.V(4).Infof("Skipping: no address family match for %q on interface %q.", ip, intf.Name) continue } // TODO: Decide if should open up to allow IPv6 LLAs in future. if !ip.IsGlobalUnicast() { - glog.V(4).Infof("Skipping: non-global address %q on interface %q.", ip, intf.Name) + klog.V(4).Infof("Skipping: non-global address %q on interface %q.", ip, intf.Name) continue } - glog.V(4).Infof("Found global unicast address %q on interface %q.", ip, intf.Name) + klog.V(4).Infof("Found global unicast address %q on interface %q.", ip, intf.Name) return ip, nil } } @@ -381,23 +381,23 @@ func getAllDefaultRoutes() ([]Route, error) { // an IPv4 IP, and then will look at each IPv6 route for an IPv6 IP. func chooseHostInterfaceFromRoute(routes []Route, nw networkInterfacer) (net.IP, error) { for _, family := range []AddressFamily{familyIPv4, familyIPv6} { - glog.V(4).Infof("Looking for default routes with IPv%d addresses", uint(family)) + klog.V(4).Infof("Looking for default routes with IPv%d addresses", uint(family)) for _, route := range routes { if route.Family != family { continue } - glog.V(4).Infof("Default route transits interface %q", route.Interface) + klog.V(4).Infof("Default route transits interface %q", route.Interface) finalIP, err := getIPFromInterface(route.Interface, family, nw) if err != nil { return nil, err } if finalIP != nil { - glog.V(4).Infof("Found active IP %v ", finalIP) + klog.V(4).Infof("Found active IP %v ", finalIP) return finalIP, nil } } } - glog.V(4).Infof("No active IP found by looking at default routes") + klog.V(4).Infof("No active IP found by looking at default routes") return nil, fmt.Errorf("unable to select an IP from default routes.") } diff --git a/staging/src/k8s.io/apimachinery/pkg/util/proxy/BUILD b/staging/src/k8s.io/apimachinery/pkg/util/proxy/BUILD index bd1b0cc23d0..6e9d6bababf 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/proxy/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/util/proxy/BUILD @@ -41,10 +41,10 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/third_party/forked/golang/netutil:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/mxk/go-flowrate/flowrate:go_default_library", "//vendor/golang.org/x/net/html:go_default_library", "//vendor/golang.org/x/net/html/atom:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/proxy/dial.go b/staging/src/k8s.io/apimachinery/pkg/util/proxy/dial.go index 37a5be487c0..a59b24c8dc3 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/proxy/dial.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/proxy/dial.go @@ -24,7 +24,7 @@ import ( "net/http" "net/url" - "github.com/golang/glog" + "k8s.io/klog" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/third_party/forked/golang/netutil" @@ -35,7 +35,7 @@ func DialURL(ctx context.Context, url *url.URL, transport http.RoundTripper) (ne dialer, err := utilnet.DialerFor(transport) if err != nil { - glog.V(5).Infof("Unable to unwrap transport %T to get dialer: %v", transport, err) + klog.V(5).Infof("Unable to unwrap transport %T to get dialer: %v", transport, err) } switch url.Scheme { @@ -52,7 +52,7 @@ func DialURL(ctx context.Context, url *url.URL, transport http.RoundTripper) (ne var err error tlsConfig, err = utilnet.TLSClientConfig(transport) if err != nil { - glog.V(5).Infof("Unable to unwrap transport %T to get at TLS config: %v", transport, err) + klog.V(5).Infof("Unable to unwrap transport %T to get at TLS config: %v", transport, err) } if dialer != nil { @@ -64,7 +64,7 @@ func DialURL(ctx context.Context, url *url.URL, transport http.RoundTripper) (ne } if tlsConfig == nil { // tls.Client requires non-nil config - glog.Warningf("using custom dialer with no TLSClientConfig. Defaulting to InsecureSkipVerify") + klog.Warningf("using custom dialer with no TLSClientConfig. Defaulting to InsecureSkipVerify") // tls.Handshake() requires ServerName or InsecureSkipVerify tlsConfig = &tls.Config{ InsecureSkipVerify: true, diff --git a/staging/src/k8s.io/apimachinery/pkg/util/proxy/transport.go b/staging/src/k8s.io/apimachinery/pkg/util/proxy/transport.go index 6c34ab5241d..3c8cf6da737 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/proxy/transport.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/proxy/transport.go @@ -27,9 +27,9 @@ import ( "path" "strings" - "github.com/golang/glog" "golang.org/x/net/html" "golang.org/x/net/html/atom" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" @@ -236,7 +236,7 @@ func (t *Transport) rewriteResponse(req *http.Request, resp *http.Response) (*ht // This is fine default: // Some encoding we don't understand-- don't try to parse this - glog.Errorf("Proxy encountered encoding %v for text/html; can't understand this so not fixing links.", encoding) + klog.Errorf("Proxy encountered encoding %v for text/html; can't understand this so not fixing links.", encoding) return resp, nil } @@ -245,7 +245,7 @@ func (t *Transport) rewriteResponse(req *http.Request, resp *http.Response) (*ht } err := rewriteHTML(reader, writer, urlRewriter) if err != nil { - glog.Errorf("Failed to rewrite URLs: %v", err) + klog.Errorf("Failed to rewrite URLs: %v", err) return resp, err } diff --git a/staging/src/k8s.io/apimachinery/pkg/util/proxy/upgradeaware.go b/staging/src/k8s.io/apimachinery/pkg/util/proxy/upgradeaware.go index 269c5331046..596b1888975 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/proxy/upgradeaware.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/proxy/upgradeaware.go @@ -34,8 +34,8 @@ import ( utilnet "k8s.io/apimachinery/pkg/util/net" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "github.com/golang/glog" "github.com/mxk/go-flowrate/flowrate" + "k8s.io/klog" ) // UpgradeRequestRoundTripper provides an additional method to decorate a request @@ -235,7 +235,7 @@ func (h *UpgradeAwareHandler) ServeHTTP(w http.ResponseWriter, req *http.Request // tryUpgrade returns true if the request was handled. func (h *UpgradeAwareHandler) tryUpgrade(w http.ResponseWriter, req *http.Request) bool { if !httpstream.IsUpgradeRequest(req) { - glog.V(6).Infof("Request was not an upgrade") + klog.V(6).Infof("Request was not an upgrade") return false } @@ -257,15 +257,15 @@ func (h *UpgradeAwareHandler) tryUpgrade(w http.ResponseWriter, req *http.Reques // handles this in the non-upgrade path. utilnet.AppendForwardedForHeader(clone) if h.InterceptRedirects { - glog.V(6).Infof("Connecting to backend proxy (intercepting redirects) %s\n Headers: %v", &location, clone.Header) + klog.V(6).Infof("Connecting to backend proxy (intercepting redirects) %s\n Headers: %v", &location, clone.Header) backendConn, rawResponse, err = utilnet.ConnectWithRedirects(req.Method, &location, clone.Header, req.Body, utilnet.DialerFunc(h.DialForUpgrade), h.RequireSameHostRedirects) } else { - glog.V(6).Infof("Connecting to backend proxy (direct dial) %s\n Headers: %v", &location, clone.Header) + klog.V(6).Infof("Connecting to backend proxy (direct dial) %s\n Headers: %v", &location, clone.Header) clone.URL = &location backendConn, err = h.DialForUpgrade(clone) } if err != nil { - glog.V(6).Infof("Proxy connection error: %v", err) + klog.V(6).Infof("Proxy connection error: %v", err) h.Responder.Error(w, req, err) return true } @@ -275,13 +275,13 @@ func (h *UpgradeAwareHandler) tryUpgrade(w http.ResponseWriter, req *http.Reques // hijacking should be the last step in the upgrade. requestHijacker, ok := w.(http.Hijacker) if !ok { - glog.V(6).Infof("Unable to hijack response writer: %T", w) + klog.V(6).Infof("Unable to hijack response writer: %T", w) h.Responder.Error(w, req, fmt.Errorf("request connection cannot be hijacked: %T", w)) return true } requestHijackedConn, _, err := requestHijacker.Hijack() if err != nil { - glog.V(6).Infof("Unable to hijack response: %v", err) + klog.V(6).Infof("Unable to hijack response: %v", err) h.Responder.Error(w, req, fmt.Errorf("error hijacking connection: %v", err)) return true } @@ -289,7 +289,7 @@ func (h *UpgradeAwareHandler) tryUpgrade(w http.ResponseWriter, req *http.Reques // Forward raw response bytes back to client. if len(rawResponse) > 0 { - glog.V(6).Infof("Writing %d bytes to hijacked connection", len(rawResponse)) + klog.V(6).Infof("Writing %d bytes to hijacked connection", len(rawResponse)) if _, err = requestHijackedConn.Write(rawResponse); err != nil { utilruntime.HandleError(fmt.Errorf("Error proxying response from backend to client: %v", err)) } @@ -311,7 +311,7 @@ func (h *UpgradeAwareHandler) tryUpgrade(w http.ResponseWriter, req *http.Reques } _, err := io.Copy(writer, requestHijackedConn) if err != nil && !strings.Contains(err.Error(), "use of closed network connection") { - glog.Errorf("Error proxying data from client to backend: %v", err) + klog.Errorf("Error proxying data from client to backend: %v", err) } close(writerComplete) }() @@ -325,7 +325,7 @@ func (h *UpgradeAwareHandler) tryUpgrade(w http.ResponseWriter, req *http.Reques } _, err := io.Copy(requestHijackedConn, reader) if err != nil && !strings.Contains(err.Error(), "use of closed network connection") { - glog.Errorf("Error proxying data from backend to client: %v", err) + klog.Errorf("Error proxying data from backend to client: %v", err) } close(readerComplete) }() @@ -336,7 +336,7 @@ func (h *UpgradeAwareHandler) tryUpgrade(w http.ResponseWriter, req *http.Reques case <-writerComplete: case <-readerComplete: } - glog.V(6).Infof("Disconnecting from backend proxy %s\n Headers: %v", &location, clone.Header) + klog.V(6).Infof("Disconnecting from backend proxy %s\n Headers: %v", &location, clone.Header) return true } diff --git a/staging/src/k8s.io/apimachinery/pkg/util/runtime/BUILD b/staging/src/k8s.io/apimachinery/pkg/util/runtime/BUILD index 6bdeeb3d246..cf8f2141673 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/runtime/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/util/runtime/BUILD @@ -17,7 +17,7 @@ go_library( srcs = ["runtime.go"], importmap = "k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/runtime", importpath = "k8s.io/apimachinery/pkg/util/runtime", - deps = ["//vendor/github.com/golang/glog:go_default_library"], + deps = ["//vendor/k8s.io/klog:go_default_library"], ) filegroup( diff --git a/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go b/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go index da32fe12f33..8e34f926139 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" ) var ( @@ -63,7 +63,11 @@ func HandleCrash(additionalHandlers ...func(interface{})) { // logPanic logs the caller tree when a panic occurs. func logPanic(r interface{}) { callers := getCallers(r) - glog.Errorf("Observed a panic: %#v (%v)\n%v", r, r, callers) + if _, ok := r.(string); ok { + klog.Errorf("Observed a panic: %s\n%v", r, callers) + } else { + klog.Errorf("Observed a panic: %#v (%v)\n%v", r, r, callers) + } } func getCallers(r interface{}) string { @@ -111,7 +115,7 @@ func HandleError(err error) { // logError prints an error with the call stack of the location it was reported func logError(err error) { - glog.ErrorDepth(2, err) + klog.ErrorDepth(2, err) } type rudimentaryErrorBackoff struct { diff --git a/staging/src/k8s.io/apimachinery/pkg/util/strategicpatch/BUILD b/staging/src/k8s.io/apimachinery/pkg/util/strategicpatch/BUILD index e0a947ece3f..5e3c9b419ed 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/strategicpatch/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/util/strategicpatch/BUILD @@ -21,7 +21,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch/testing:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/strategicpatch/patch_test.go b/staging/src/k8s.io/apimachinery/pkg/util/strategicpatch/patch_test.go index 4721803ccd1..6a0166e57d2 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/strategicpatch/patch_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/strategicpatch/patch_test.go @@ -24,7 +24,7 @@ import ( "testing" "github.com/davecgh/go-spew/spew" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/json" diff --git a/staging/src/k8s.io/apimachinery/pkg/util/yaml/BUILD b/staging/src/k8s.io/apimachinery/pkg/util/yaml/BUILD index 64fa65e7566..ad28913e082 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/yaml/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/util/yaml/BUILD @@ -18,8 +18,8 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/yaml", importpath = "k8s.io/apimachinery/pkg/util/yaml", deps = [ - "//vendor/github.com/ghodss/yaml:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go b/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go index 3cd85515d43..63d735a804c 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go @@ -26,8 +26,8 @@ import ( "strings" "unicode" - "github.com/ghodss/yaml" - "github.com/golang/glog" + "k8s.io/klog" + "sigs.k8s.io/yaml" ) // ToJSON converts a single YAML document into a JSON document @@ -217,11 +217,11 @@ func (d *YAMLOrJSONDecoder) Decode(into interface{}) error { if d.decoder == nil { buffer, origData, isJSON := GuessJSONStream(d.r, d.bufferSize) if isJSON { - glog.V(4).Infof("decoding stream as JSON") + klog.V(4).Infof("decoding stream as JSON") d.decoder = json.NewDecoder(buffer) d.rawData = origData } else { - glog.V(4).Infof("decoding stream as YAML") + klog.V(4).Infof("decoding stream as YAML") d.decoder = NewYAMLToJSONDecoder(buffer) } } @@ -230,7 +230,7 @@ func (d *YAMLOrJSONDecoder) Decode(into interface{}) error { if syntax, ok := err.(*json.SyntaxError); ok { data, readErr := ioutil.ReadAll(jsonDecoder.Buffered()) if readErr != nil { - glog.V(4).Infof("reading stream failed: %v", readErr) + klog.V(4).Infof("reading stream failed: %v", readErr) } js := string(data) diff --git a/staging/src/k8s.io/apimachinery/pkg/watch/BUILD b/staging/src/k8s.io/apimachinery/pkg/watch/BUILD index 49ebc15246e..5b9e1880993 100644 --- a/staging/src/k8s.io/apimachinery/pkg/watch/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/watch/BUILD @@ -23,7 +23,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/watch/streamwatcher.go b/staging/src/k8s.io/apimachinery/pkg/watch/streamwatcher.go index 93bb1cdf7f6..d61cf5a2e58 100644 --- a/staging/src/k8s.io/apimachinery/pkg/watch/streamwatcher.go +++ b/staging/src/k8s.io/apimachinery/pkg/watch/streamwatcher.go @@ -20,10 +20,10 @@ import ( "io" "sync" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/net" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog" ) // Decoder allows StreamWatcher to watch any stream for which a Decoder can be written. @@ -100,13 +100,13 @@ func (sw *StreamWatcher) receive() { case io.EOF: // watch closed normally case io.ErrUnexpectedEOF: - glog.V(1).Infof("Unexpected EOF during watch stream event decoding: %v", err) + klog.V(1).Infof("Unexpected EOF during watch stream event decoding: %v", err) default: msg := "Unable to decode an event from the watch stream: %v" if net.IsProbableEOF(err) { - glog.V(5).Infof(msg, err) + klog.V(5).Infof(msg, err) } else { - glog.Errorf(msg, err) + klog.Errorf(msg, err) } } return diff --git a/staging/src/k8s.io/apimachinery/pkg/watch/watch.go b/staging/src/k8s.io/apimachinery/pkg/watch/watch.go index a627d1d572c..be9c90c03d1 100644 --- a/staging/src/k8s.io/apimachinery/pkg/watch/watch.go +++ b/staging/src/k8s.io/apimachinery/pkg/watch/watch.go @@ -20,7 +20,7 @@ import ( "fmt" "sync" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/runtime" ) @@ -106,7 +106,7 @@ func (f *FakeWatcher) Stop() { f.Lock() defer f.Unlock() if !f.Stopped { - glog.V(4).Infof("Stopping fake watcher.") + klog.V(4).Infof("Stopping fake watcher.") close(f.result) f.Stopped = true } @@ -173,7 +173,7 @@ func (f *RaceFreeFakeWatcher) Stop() { f.Lock() defer f.Unlock() if !f.Stopped { - glog.V(4).Infof("Stopping fake watcher.") + klog.V(4).Infof("Stopping fake watcher.") close(f.result) f.Stopped = true } diff --git a/staging/src/k8s.io/apiserver/Godeps/Godeps.json b/staging/src/k8s.io/apiserver/Godeps/Godeps.json index 74c66c6dbab..9ef4b940b32 100644 --- a/staging/src/k8s.io/apiserver/Godeps/Godeps.json +++ b/staging/src/k8s.io/apiserver/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/apiserver", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -422,10 +422,6 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/golang/groupcache/lru", "Rev": "02826c3e79038b59d737d3b1c0a1d937f71a4433" @@ -716,35 +712,35 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/internal/timeseries", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/trace", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/websocket", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/oauth2", @@ -816,79 +812,111 @@ }, { "ImportPath": "google.golang.org/grpc", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/balancer", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/balancer/base", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/balancer/roundrobin", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/codes", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/connectivity", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/credentials", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { - "ImportPath": "google.golang.org/grpc/grpclb/grpc_lb_v1/messages", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "ImportPath": "google.golang.org/grpc/encoding", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/encoding/proto", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/grpclog", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/health", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/health/grpc_health_v1", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/internal", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/backoff", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/channelz", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/grpcrand", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/keepalive", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/metadata", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/naming", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/peer", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/resolver", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/resolver/dns", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/resolver/passthrough", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/stats", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/status", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/tap", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/transport", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "gopkg.in/inf.v0", @@ -1918,25 +1946,29 @@ "ImportPath": "k8s.io/client-go/util/retry", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/kube-openapi/pkg/builder", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/common", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/handler", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/client-go/discovery", @@ -1950,6 +1982,10 @@ "ImportPath": "k8s.io/client-go/informers", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/client-go/informers/auditregistration/v1alpha1", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/client-go/kubernetes", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -1970,6 +2006,10 @@ "ImportPath": "k8s.io/client-go/kubernetes/typed/authorization/v1beta1", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/client-go/kubernetes/typed/core/v1", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/client-go/listers/admissionregistration/v1beta1", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -2002,6 +2042,10 @@ "ImportPath": "k8s.io/client-go/tools/clientcmd/api/v1", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/client-go/tools/record", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/client-go/util/cert", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -2013,6 +2057,10 @@ { "ImportPath": "k8s.io/utils/pointer", "Rev": "66066c83e385e385ccc3c964b44fd7dcd413d0ed" + }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" } ] } diff --git a/staging/src/k8s.io/apiserver/pkg/admission/BUILD b/staging/src/k8s.io/apiserver/pkg/admission/BUILD index f519566c49a..e45f33e1ecf 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/admission/BUILD @@ -59,8 +59,8 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", "//staging/src/k8s.io/apiserver/pkg/audit:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/admission/config.go b/staging/src/k8s.io/apiserver/pkg/admission/config.go index f59d0608bce..ffda2f3262c 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/config.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/config.go @@ -25,8 +25,8 @@ import ( "path" "path/filepath" - "github.com/ghodss/yaml" - "github.com/golang/glog" + "k8s.io/klog" + "sigs.k8s.io/yaml" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -146,7 +146,7 @@ func GetAdmissionPluginConfigurationFor(pluginCfg apiserver.AdmissionPluginConfi if pluginCfg.Path != "" { content, err := ioutil.ReadFile(pluginCfg.Path) if err != nil { - glog.Fatalf("Couldn't open admission plugin configuration %s: %#v", pluginCfg.Path, err) + klog.Fatalf("Couldn't open admission plugin configuration %s: %#v", pluginCfg.Path, err) return nil, err } return bytes.NewBuffer(content), nil diff --git a/staging/src/k8s.io/apiserver/pkg/admission/configuration/BUILD b/staging/src/k8s.io/apiserver/pkg/admission/configuration/BUILD index 598d6d404c0..10136848ecc 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/configuration/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/admission/configuration/BUILD @@ -51,7 +51,7 @@ go_library( "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/listers/admissionregistration/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/admission/configuration/initializer_manager.go b/staging/src/k8s.io/apiserver/pkg/admission/configuration/initializer_manager.go index 986524b5ba2..f2b7e909942 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/configuration/initializer_manager.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/configuration/initializer_manager.go @@ -21,7 +21,7 @@ import ( "reflect" "sort" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/admissionregistration/v1alpha1" "k8s.io/apimachinery/pkg/api/errors" @@ -42,7 +42,7 @@ func NewInitializerConfigurationManager(c InitializerConfigurationLister) *Initi list, err := c.List(metav1.ListOptions{}) if err != nil { if errors.IsNotFound(err) || errors.IsForbidden(err) { - glog.V(5).Infof("Initializers are disabled due to an error: %v", err) + klog.V(5).Infof("Initializers are disabled due to an error: %v", err) return nil, ErrDisabled } return nil, err diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/initialization/BUILD b/staging/src/k8s.io/apiserver/pkg/admission/plugin/initialization/BUILD index 05ac8fe0fc4..ad3b9b5f58b 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/initialization/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/initialization/BUILD @@ -27,7 +27,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/initialization/initialization.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/initialization/initialization.go index 9ebd2db3f7b..d4d184a5747 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/initialization/initialization.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/initialization/initialization.go @@ -21,7 +21,7 @@ import ( "io" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/admissionregistration/v1alpha1" "k8s.io/api/core/v1" @@ -86,9 +86,9 @@ func (i *initializer) ValidateInitialization() error { if !utilfeature.DefaultFeatureGate.Enabled(features.Initializers) { if err := utilfeature.DefaultFeatureGate.Set(string(features.Initializers) + "=true"); err != nil { - glog.Errorf("error enabling Initializers feature as part of admission plugin setup: %v", err) + klog.Errorf("error enabling Initializers feature as part of admission plugin setup: %v", err) } else { - glog.Infof("enabled Initializers feature as part of admission plugin setup") + klog.Infof("enabled Initializers feature as part of admission plugin setup") } } @@ -170,7 +170,7 @@ func (i *initializer) Admit(a admission.Attributes) (err error) { } existing := accessor.GetInitializers() if existing != nil { - glog.V(5).Infof("Admin bypassing initialization for %s", a.GetResource()) + klog.V(5).Infof("Admin bypassing initialization for %s", a.GetResource()) // it must be possible for some users to bypass initialization - for now, check the initialize operation if err := i.canInitialize(a, "create with initializers denied"); err != nil { @@ -182,7 +182,7 @@ func (i *initializer) Admit(a admission.Attributes) (err error) { return nil } } else { - glog.V(5).Infof("Checking initialization for %s", a.GetResource()) + klog.V(5).Infof("Checking initialization for %s", a.GetResource()) config, err := i.readConfig(a) if err != nil { @@ -205,11 +205,11 @@ func (i *initializer) Admit(a admission.Attributes) (err error) { names := findInitializers(config, a.GetResource()) if len(names) == 0 { - glog.V(5).Infof("No initializers needed") + klog.V(5).Infof("No initializers needed") return nil } - glog.V(5).Infof("Found initializers for %s: %v", a.GetResource(), names) + klog.V(5).Infof("Found initializers for %s: %v", a.GetResource(), names) accessor.SetInitializers(newInitializers(names)) } @@ -241,7 +241,7 @@ func (i *initializer) Admit(a admission.Attributes) (err error) { return nil } - glog.V(5).Infof("Modifying uninitialized resource %s", a.GetResource()) + klog.V(5).Infof("Modifying uninitialized resource %s", a.GetResource()) // because we are called before validation, we need to ensure the update transition is valid. if errs := validation.ValidateInitializersUpdate(updated, existing, initializerFieldPath); len(errs) > 0 { diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle/BUILD b/staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle/BUILD index 731684adfcc..5e605f4af59 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle/BUILD @@ -24,7 +24,7 @@ go_library( "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle/admission.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle/admission.go index ab01c0aaa77..d7bb0215b98 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle/admission.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle/admission.go @@ -21,7 +21,7 @@ import ( "io" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -139,7 +139,7 @@ func (l *Lifecycle) Admit(a admission.Attributes) error { exists = true } if exists { - glog.V(4).Infof("found %s in cache after waiting", a.GetNamespace()) + klog.V(4).Infof("found %s in cache after waiting", a.GetNamespace()) } } @@ -160,7 +160,7 @@ func (l *Lifecycle) Admit(a admission.Attributes) error { case err != nil: return errors.NewInternalError(err) } - glog.V(4).Infof("found %s via storage lookup", a.GetNamespace()) + klog.V(4).Infof("found %s via storage lookup", a.GetNamespace()) } // ensure that we're not trying to create objects in terminating namespaces diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go index 408187fd1f7..13b898bca9b 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go @@ -164,6 +164,7 @@ func (a *Webhook) Dispatch(attr admission.Attributes) error { return admission.NewForbidden(attr, fmt.Errorf("not yet ready to handle request")) } hooks := a.hookSource.Webhooks() + // TODO: Figure out if adding one second timeout make sense here. ctx := context.TODO() var relevantHooks []*v1beta1.Webhook diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/BUILD b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/BUILD index a1693818ad6..da393cf2283 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/BUILD @@ -27,7 +27,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/util:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go index 4f95a6adf9a..d646bacb535 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go @@ -24,7 +24,7 @@ import ( "time" jsonpatch "github.com/evanphx/json-patch" - "github.com/golang/glog" + "k8s.io/klog" admissionv1beta1 "k8s.io/api/admission/v1beta1" "k8s.io/api/admissionregistration/v1beta1" @@ -65,11 +65,11 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr *generic.Version ignoreClientCallFailures := hook.FailurePolicy != nil && *hook.FailurePolicy == v1beta1.Ignore if callErr, ok := err.(*webhook.ErrCallingWebhook); ok { if ignoreClientCallFailures { - glog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr) + klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr) utilruntime.HandleError(callErr) continue } - glog.Warningf("Failed calling webhook, failing closed %v: %v", hook.Name, err) + klog.Warningf("Failed calling webhook, failing closed %v: %v", hook.Name, err) } return apierrors.NewInternalError(err) } @@ -110,7 +110,7 @@ func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *v1beta for k, v := range response.Response.AuditAnnotations { key := h.Name + "/" + k if err := attr.AddAnnotation(key, v); err != nil { - glog.Warningf("Failed to set admission audit annotation %s to %s for mutating webhook %s: %v", key, v, h.Name, err) + klog.Warningf("Failed to set admission audit annotation %s to %s for mutating webhook %s: %v", key, v, h.Name, err) } } diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/BUILD b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/BUILD index e180eeb76d4..ea9f489f8ec 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/BUILD @@ -23,7 +23,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/util:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go index 42e4262d090..166e21adcdf 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" admissionv1beta1 "k8s.io/api/admission/v1beta1" "k8s.io/api/admissionregistration/v1beta1" @@ -64,17 +64,17 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr *generic.Versi ignoreClientCallFailures := hook.FailurePolicy != nil && *hook.FailurePolicy == v1beta1.Ignore if callErr, ok := err.(*webhook.ErrCallingWebhook); ok { if ignoreClientCallFailures { - glog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr) + klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr) utilruntime.HandleError(callErr) return } - glog.Warningf("Failed calling webhook, failing closed %v: %v", hook.Name, err) + klog.Warningf("Failed calling webhook, failing closed %v: %v", hook.Name, err) errCh <- apierrors.NewInternalError(err) return } - glog.Warningf("rejected by webhook %q: %#v", hook.Name, err) + klog.Warningf("rejected by webhook %q: %#v", hook.Name, err) errCh <- err }(relevantHooks[i]) } @@ -124,7 +124,7 @@ func (d *validatingDispatcher) callHook(ctx context.Context, h *v1beta1.Webhook, for k, v := range response.Response.AuditAnnotations { key := h.Name + "/" + k if err := attr.AddAnnotation(key, v); err != nil { - glog.Warningf("Failed to set admission audit annotation %s to %s for validating webhook %s: %v", key, v, h.Name, err) + klog.Warningf("Failed to set admission audit annotation %s to %s for validating webhook %s: %v", key, v, h.Name, err) } } if response.Response.Allowed { diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugins.go b/staging/src/k8s.io/apiserver/pkg/admission/plugins.go index c17d62cd4e6..bdf087e564f 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugins.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugins.go @@ -26,7 +26,7 @@ import ( "strings" "sync" - "github.com/golang/glog" + "k8s.io/klog" ) // Factory is a function that returns an Interface for admission decisions. @@ -75,13 +75,13 @@ func (ps *Plugins) Register(name string, plugin Factory) { if ps.registry != nil { _, found := ps.registry[name] if found { - glog.Fatalf("Admission plugin %q was registered twice", name) + klog.Fatalf("Admission plugin %q was registered twice", name) } } else { ps.registry = map[string]Factory{} } - glog.V(1).Infof("Registered admission plugin %q", name) + klog.V(1).Infof("Registered admission plugin %q", name) ps.registry[name] = plugin } @@ -155,10 +155,10 @@ func (ps *Plugins) NewFromPlugins(pluginNames []string, configProvider ConfigPro } } if len(mutationPlugins) != 0 { - glog.Infof("Loaded %d mutating admission controller(s) successfully in the following order: %s.", len(mutationPlugins), strings.Join(mutationPlugins, ",")) + klog.Infof("Loaded %d mutating admission controller(s) successfully in the following order: %s.", len(mutationPlugins), strings.Join(mutationPlugins, ",")) } if len(validationPlugins) != 0 { - glog.Infof("Loaded %d validating admission controller(s) successfully in the following order: %s.", len(validationPlugins), strings.Join(validationPlugins, ",")) + klog.Infof("Loaded %d validating admission controller(s) successfully in the following order: %s.", len(validationPlugins), strings.Join(validationPlugins, ",")) } return chainAdmissionHandler(handlers), nil } @@ -166,7 +166,7 @@ func (ps *Plugins) NewFromPlugins(pluginNames []string, configProvider ConfigPro // InitPlugin creates an instance of the named interface. func (ps *Plugins) InitPlugin(name string, config io.Reader, pluginInitializer PluginInitializer) (Interface, error) { if name == "" { - glog.Info("No admission plugin specified.") + klog.Info("No admission plugin specified.") return nil, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/audit/OWNERS b/staging/src/k8s.io/apiserver/pkg/apis/audit/OWNERS new file mode 100644 index 00000000000..8389778900f --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/audit/OWNERS @@ -0,0 +1,7 @@ +# approval on api packages bubbles to api-approvers +reviewers: +- sig-auth-audit-approvers +- sig-auth-audit-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/BUILD b/staging/src/k8s.io/apiserver/pkg/apis/config/BUILD index eadb944d515..feaa8a3c370 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/config/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/BUILD @@ -4,13 +4,18 @@ go_library( name = "go_default_library", srcs = [ "doc.go", + "register.go", "types.go", "zz_generated.deepcopy.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/apis/config", importpath = "k8s.io/apiserver/pkg/apis/config", visibility = ["//visibility:public"], - deps = ["//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library"], + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + ], ) filegroup( @@ -24,6 +29,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//staging/src/k8s.io/apiserver/pkg/apis/config/v1:all-srcs", "//staging/src/k8s.io/apiserver/pkg/apis/config/v1alpha1:all-srcs", "//staging/src/k8s.io/apiserver/pkg/apis/config/validation:all-srcs", ], diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/register.go b/staging/src/k8s.io/apiserver/pkg/apis/config/register.go new file mode 100644 index 00000000000..6a0aae8e560 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/register.go @@ -0,0 +1,53 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + // SchemeBuilder points to a list of functions added to Scheme. + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + // AddToScheme adds this group to a scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +// GroupName is the group name use in this package. +const GroupName = "apiserver.config.k8s.io" + +// SchemeGroupVersion is group version used to register these objects. +var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal} + +// Kind takes an unqualified kind and returns a Group qualified GroupKind. +func Kind(kind string) schema.GroupKind { + return SchemeGroupVersion.WithKind(kind).GroupKind() +} + +// Resource takes an unqualified resource and returns a Group qualified GroupResource. +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +func addKnownTypes(scheme *runtime.Scheme) error { + // TODO this will get cleaned up with the scheme types are fixed + scheme.AddKnownTypes(SchemeGroupVersion, + &EncryptionConfiguration{}, + ) + return nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/types.go b/staging/src/k8s.io/apiserver/pkg/apis/config/types.go index f6424ec4e8d..822806d7e5d 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/config/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/types.go @@ -56,3 +56,71 @@ type DebuggingConfiguration struct { // enableProfiling is true. EnableContentionProfiling bool } + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// EncryptionConfiguration stores the complete configuration for encryption providers. +type EncryptionConfiguration struct { + metav1.TypeMeta + // resources is a list containing resources, and their corresponding encryption providers. + Resources []ResourceConfiguration +} + +// ResourceConfiguration stores per resource configuration. +type ResourceConfiguration struct { + // resources is a list of kubernetes resources which have to be encrypted. + Resources []string + // providers is a list of transformers to be used for reading and writing the resources to disk. + // eg: aesgcm, aescbc, secretbox, identity. + Providers []ProviderConfiguration +} + +// ProviderConfiguration stores the provided configuration for an encryption provider. +type ProviderConfiguration struct { + // aesgcm is the configuration for the AES-GCM transformer. + AESGCM *AESConfiguration + // aescbc is the configuration for the AES-CBC transformer. + AESCBC *AESConfiguration + // secretbox is the configuration for the Secretbox based transformer. + Secretbox *SecretboxConfiguration + // identity is the (empty) configuration for the identity transformer. + Identity *IdentityConfiguration + // kms contains the name, cache size and path to configuration file for a KMS based envelope transformer. + KMS *KMSConfiguration +} + +// AESConfiguration contains the API configuration for an AES transformer. +type AESConfiguration struct { + // keys is a list of keys to be used for creating the AES transformer. + // Each key has to be 32 bytes long for AES-CBC and 16, 24 or 32 bytes for AES-GCM. + Keys []Key +} + +// SecretboxConfiguration contains the API configuration for an Secretbox transformer. +type SecretboxConfiguration struct { + // keys is a list of keys to be used for creating the Secretbox transformer. + // Each key has to be 32 bytes long. + Keys []Key +} + +// Key contains name and secret of the provided key for a transformer. +type Key struct { + // name is the name of the key to be used while storing data to disk. + Name string + // secret is the actual key, encoded in base64. + Secret string +} + +// IdentityConfiguration is an empty struct to allow identity transformer in provider configuration. +type IdentityConfiguration struct{} + +// KMSConfiguration contains the name, cache size and path to configuration file for a KMS based envelope transformer. +type KMSConfiguration struct { + // name is the name of the KMS plugin to be used. + Name string + // cacheSize is the maximum number of secrets which are cached in memory. The default value is 1000. + // +optional + CacheSize int32 + // endpoint is the gRPC server listening address, for example "unix:///var/run/kms-provider.sock". + Endpoint string +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/v1/BUILD b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/BUILD new file mode 100644 index 00000000000..ca9eea027c4 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/BUILD @@ -0,0 +1,37 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "doc.go", + "register.go", + "types.go", + "zz_generated.conversion.go", + "zz_generated.deepcopy.go", + "zz_generated.defaults.go", + ], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/apis/config/v1", + importpath = "k8s.io/apiserver/pkg/apis/config/v1", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/config:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/v1/doc.go b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/doc.go new file mode 100644 index 00000000000..b1a18ccab57 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/doc.go @@ -0,0 +1,23 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:conversion-gen=k8s.io/apiserver/pkg/apis/config +// +k8s:deepcopy-gen=package +// +k8s:defaulter-gen=TypeMeta +// +groupName=apiserver.config.k8s.io + +// Package v1 is the v1 version of the API. +package v1 diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/v1/register.go b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/register.go new file mode 100644 index 00000000000..2e3ecfff2ea --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/register.go @@ -0,0 +1,52 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName is the group name use in this package. +const GroupName = "apiserver.config.k8s.io" + +// SchemeGroupVersion is group version used to register these objects. +var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1"} + +var ( + // SchemeBuilder points to a list of functions added to Scheme. + SchemeBuilder runtime.SchemeBuilder + localSchemeBuilder = &SchemeBuilder + // AddToScheme adds this group to a scheme. + AddToScheme = localSchemeBuilder.AddToScheme +) + +func init() { + // We only register manually written functions here. The registration of the + // generated functions takes place in the generated files. The separation + // makes the code compile even when the generated files are missing. + localSchemeBuilder.Register(addKnownTypes) +} + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &EncryptionConfiguration{}, + ) + // also register into the v1 group as EncryptionConfig (due to a docs bug) + scheme.AddKnownTypeWithName(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "EncryptionConfig"}, &EncryptionConfiguration{}) + return nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/types.go b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/types.go similarity index 58% rename from staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/types.go rename to staging/src/k8s.io/apiserver/pkg/apis/config/v1/types.go index 67dfa6c9fce..e2c123d1dc2 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/types.go @@ -14,50 +14,51 @@ See the License for the specific language governing permissions and limitations under the License. */ -package encryptionconfig +package v1 -// EncryptionConfig stores the complete configuration for encryption providers. -type EncryptionConfig struct { - // kind is the type of configuration file. - Kind string `json:"kind"` - // apiVersion is the API version this file has to be parsed as. - APIVersion string `json:"apiVersion"` +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// EncryptionConfiguration stores the complete configuration for encryption providers. +type EncryptionConfiguration struct { + metav1.TypeMeta // resources is a list containing resources, and their corresponding encryption providers. - Resources []ResourceConfig `json:"resources"` + Resources []ResourceConfiguration `json:"resources"` } -// ResourceConfig stores per resource configuration. -type ResourceConfig struct { +// ResourceConfiguration stores per resource configuration. +type ResourceConfiguration struct { // resources is a list of kubernetes resources which have to be encrypted. Resources []string `json:"resources"` // providers is a list of transformers to be used for reading and writing the resources to disk. // eg: aesgcm, aescbc, secretbox, identity. - Providers []ProviderConfig `json:"providers"` + Providers []ProviderConfiguration `json:"providers"` } -// ProviderConfig stores the provided configuration for an encryption provider. -type ProviderConfig struct { +// ProviderConfiguration stores the provided configuration for an encryption provider. +type ProviderConfiguration struct { // aesgcm is the configuration for the AES-GCM transformer. - AESGCM *AESConfig `json:"aesgcm,omitempty"` + AESGCM *AESConfiguration `json:"aesgcm,omitempty"` // aescbc is the configuration for the AES-CBC transformer. - AESCBC *AESConfig `json:"aescbc,omitempty"` + AESCBC *AESConfiguration `json:"aescbc,omitempty"` // secretbox is the configuration for the Secretbox based transformer. - Secretbox *SecretboxConfig `json:"secretbox,omitempty"` + Secretbox *SecretboxConfiguration `json:"secretbox,omitempty"` // identity is the (empty) configuration for the identity transformer. - Identity *IdentityConfig `json:"identity,omitempty"` + Identity *IdentityConfiguration `json:"identity,omitempty"` // kms contains the name, cache size and path to configuration file for a KMS based envelope transformer. - KMS *KMSConfig `json:"kms,omitempty"` + KMS *KMSConfiguration `json:"kms,omitempty"` } -// AESConfig contains the API configuration for an AES transformer. -type AESConfig struct { +// AESConfiguration contains the API configuration for an AES transformer. +type AESConfiguration struct { // keys is a list of keys to be used for creating the AES transformer. // Each key has to be 32 bytes long for AES-CBC and 16, 24 or 32 bytes for AES-GCM. Keys []Key `json:"keys"` } -// SecretboxConfig contains the API configuration for an Secretbox transformer. -type SecretboxConfig struct { +// SecretboxConfiguration contains the API configuration for an Secretbox transformer. +type SecretboxConfiguration struct { // keys is a list of keys to be used for creating the Secretbox transformer. // Each key has to be 32 bytes long. Keys []Key `json:"keys"` @@ -71,16 +72,16 @@ type Key struct { Secret string `json:"secret"` } -// IdentityConfig is an empty struct to allow identity transformer in provider configuration. -type IdentityConfig struct{} +// IdentityConfiguration is an empty struct to allow identity transformer in provider configuration. +type IdentityConfiguration struct{} -// KMSConfig contains the name, cache size and path to configuration file for a KMS based envelope transformer. -type KMSConfig struct { +// KMSConfiguration contains the name, cache size and path to configuration file for a KMS based envelope transformer. +type KMSConfiguration struct { // name is the name of the KMS plugin to be used. Name string `json:"name"` // cacheSize is the maximum number of secrets which are cached in memory. The default value is 1000. // +optional - CacheSize int `json:"cachesize,omitempty"` - // the gRPC server listening address, for example "unix:///var/run/kms-provider.sock". + CacheSize int32 `json:"cachesize,omitempty"` + // endpoint is the gRPC server listening address, for example "unix:///var/run/kms-provider.sock". Endpoint string `json:"endpoint"` } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/v1/zz_generated.conversion.go b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/zz_generated.conversion.go new file mode 100644 index 00000000000..27fb16d3183 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/zz_generated.conversion.go @@ -0,0 +1,293 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by conversion-gen. DO NOT EDIT. + +package v1 + +import ( + unsafe "unsafe" + + conversion "k8s.io/apimachinery/pkg/conversion" + runtime "k8s.io/apimachinery/pkg/runtime" + config "k8s.io/apiserver/pkg/apis/config" +) + +func init() { + localSchemeBuilder.Register(RegisterConversions) +} + +// RegisterConversions adds conversion functions to the given scheme. +// Public to allow building arbitrary schemes. +func RegisterConversions(s *runtime.Scheme) error { + if err := s.AddGeneratedConversionFunc((*AESConfiguration)(nil), (*config.AESConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_AESConfiguration_To_config_AESConfiguration(a.(*AESConfiguration), b.(*config.AESConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.AESConfiguration)(nil), (*AESConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_AESConfiguration_To_v1_AESConfiguration(a.(*config.AESConfiguration), b.(*AESConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*EncryptionConfiguration)(nil), (*config.EncryptionConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_EncryptionConfiguration_To_config_EncryptionConfiguration(a.(*EncryptionConfiguration), b.(*config.EncryptionConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.EncryptionConfiguration)(nil), (*EncryptionConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_EncryptionConfiguration_To_v1_EncryptionConfiguration(a.(*config.EncryptionConfiguration), b.(*EncryptionConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*IdentityConfiguration)(nil), (*config.IdentityConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_IdentityConfiguration_To_config_IdentityConfiguration(a.(*IdentityConfiguration), b.(*config.IdentityConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.IdentityConfiguration)(nil), (*IdentityConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_IdentityConfiguration_To_v1_IdentityConfiguration(a.(*config.IdentityConfiguration), b.(*IdentityConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*KMSConfiguration)(nil), (*config.KMSConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_KMSConfiguration_To_config_KMSConfiguration(a.(*KMSConfiguration), b.(*config.KMSConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.KMSConfiguration)(nil), (*KMSConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_KMSConfiguration_To_v1_KMSConfiguration(a.(*config.KMSConfiguration), b.(*KMSConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*Key)(nil), (*config.Key)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_Key_To_config_Key(a.(*Key), b.(*config.Key), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.Key)(nil), (*Key)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_Key_To_v1_Key(a.(*config.Key), b.(*Key), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ProviderConfiguration)(nil), (*config.ProviderConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_ProviderConfiguration_To_config_ProviderConfiguration(a.(*ProviderConfiguration), b.(*config.ProviderConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ProviderConfiguration)(nil), (*ProviderConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ProviderConfiguration_To_v1_ProviderConfiguration(a.(*config.ProviderConfiguration), b.(*ProviderConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ResourceConfiguration)(nil), (*config.ResourceConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_ResourceConfiguration_To_config_ResourceConfiguration(a.(*ResourceConfiguration), b.(*config.ResourceConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ResourceConfiguration)(nil), (*ResourceConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ResourceConfiguration_To_v1_ResourceConfiguration(a.(*config.ResourceConfiguration), b.(*ResourceConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*SecretboxConfiguration)(nil), (*config.SecretboxConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_SecretboxConfiguration_To_config_SecretboxConfiguration(a.(*SecretboxConfiguration), b.(*config.SecretboxConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.SecretboxConfiguration)(nil), (*SecretboxConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_SecretboxConfiguration_To_v1_SecretboxConfiguration(a.(*config.SecretboxConfiguration), b.(*SecretboxConfiguration), scope) + }); err != nil { + return err + } + return nil +} + +func autoConvert_v1_AESConfiguration_To_config_AESConfiguration(in *AESConfiguration, out *config.AESConfiguration, s conversion.Scope) error { + out.Keys = *(*[]config.Key)(unsafe.Pointer(&in.Keys)) + return nil +} + +// Convert_v1_AESConfiguration_To_config_AESConfiguration is an autogenerated conversion function. +func Convert_v1_AESConfiguration_To_config_AESConfiguration(in *AESConfiguration, out *config.AESConfiguration, s conversion.Scope) error { + return autoConvert_v1_AESConfiguration_To_config_AESConfiguration(in, out, s) +} + +func autoConvert_config_AESConfiguration_To_v1_AESConfiguration(in *config.AESConfiguration, out *AESConfiguration, s conversion.Scope) error { + out.Keys = *(*[]Key)(unsafe.Pointer(&in.Keys)) + return nil +} + +// Convert_config_AESConfiguration_To_v1_AESConfiguration is an autogenerated conversion function. +func Convert_config_AESConfiguration_To_v1_AESConfiguration(in *config.AESConfiguration, out *AESConfiguration, s conversion.Scope) error { + return autoConvert_config_AESConfiguration_To_v1_AESConfiguration(in, out, s) +} + +func autoConvert_v1_EncryptionConfiguration_To_config_EncryptionConfiguration(in *EncryptionConfiguration, out *config.EncryptionConfiguration, s conversion.Scope) error { + out.Resources = *(*[]config.ResourceConfiguration)(unsafe.Pointer(&in.Resources)) + return nil +} + +// Convert_v1_EncryptionConfiguration_To_config_EncryptionConfiguration is an autogenerated conversion function. +func Convert_v1_EncryptionConfiguration_To_config_EncryptionConfiguration(in *EncryptionConfiguration, out *config.EncryptionConfiguration, s conversion.Scope) error { + return autoConvert_v1_EncryptionConfiguration_To_config_EncryptionConfiguration(in, out, s) +} + +func autoConvert_config_EncryptionConfiguration_To_v1_EncryptionConfiguration(in *config.EncryptionConfiguration, out *EncryptionConfiguration, s conversion.Scope) error { + out.Resources = *(*[]ResourceConfiguration)(unsafe.Pointer(&in.Resources)) + return nil +} + +// Convert_config_EncryptionConfiguration_To_v1_EncryptionConfiguration is an autogenerated conversion function. +func Convert_config_EncryptionConfiguration_To_v1_EncryptionConfiguration(in *config.EncryptionConfiguration, out *EncryptionConfiguration, s conversion.Scope) error { + return autoConvert_config_EncryptionConfiguration_To_v1_EncryptionConfiguration(in, out, s) +} + +func autoConvert_v1_IdentityConfiguration_To_config_IdentityConfiguration(in *IdentityConfiguration, out *config.IdentityConfiguration, s conversion.Scope) error { + return nil +} + +// Convert_v1_IdentityConfiguration_To_config_IdentityConfiguration is an autogenerated conversion function. +func Convert_v1_IdentityConfiguration_To_config_IdentityConfiguration(in *IdentityConfiguration, out *config.IdentityConfiguration, s conversion.Scope) error { + return autoConvert_v1_IdentityConfiguration_To_config_IdentityConfiguration(in, out, s) +} + +func autoConvert_config_IdentityConfiguration_To_v1_IdentityConfiguration(in *config.IdentityConfiguration, out *IdentityConfiguration, s conversion.Scope) error { + return nil +} + +// Convert_config_IdentityConfiguration_To_v1_IdentityConfiguration is an autogenerated conversion function. +func Convert_config_IdentityConfiguration_To_v1_IdentityConfiguration(in *config.IdentityConfiguration, out *IdentityConfiguration, s conversion.Scope) error { + return autoConvert_config_IdentityConfiguration_To_v1_IdentityConfiguration(in, out, s) +} + +func autoConvert_v1_KMSConfiguration_To_config_KMSConfiguration(in *KMSConfiguration, out *config.KMSConfiguration, s conversion.Scope) error { + out.Name = in.Name + out.CacheSize = in.CacheSize + out.Endpoint = in.Endpoint + return nil +} + +// Convert_v1_KMSConfiguration_To_config_KMSConfiguration is an autogenerated conversion function. +func Convert_v1_KMSConfiguration_To_config_KMSConfiguration(in *KMSConfiguration, out *config.KMSConfiguration, s conversion.Scope) error { + return autoConvert_v1_KMSConfiguration_To_config_KMSConfiguration(in, out, s) +} + +func autoConvert_config_KMSConfiguration_To_v1_KMSConfiguration(in *config.KMSConfiguration, out *KMSConfiguration, s conversion.Scope) error { + out.Name = in.Name + out.CacheSize = in.CacheSize + out.Endpoint = in.Endpoint + return nil +} + +// Convert_config_KMSConfiguration_To_v1_KMSConfiguration is an autogenerated conversion function. +func Convert_config_KMSConfiguration_To_v1_KMSConfiguration(in *config.KMSConfiguration, out *KMSConfiguration, s conversion.Scope) error { + return autoConvert_config_KMSConfiguration_To_v1_KMSConfiguration(in, out, s) +} + +func autoConvert_v1_Key_To_config_Key(in *Key, out *config.Key, s conversion.Scope) error { + out.Name = in.Name + out.Secret = in.Secret + return nil +} + +// Convert_v1_Key_To_config_Key is an autogenerated conversion function. +func Convert_v1_Key_To_config_Key(in *Key, out *config.Key, s conversion.Scope) error { + return autoConvert_v1_Key_To_config_Key(in, out, s) +} + +func autoConvert_config_Key_To_v1_Key(in *config.Key, out *Key, s conversion.Scope) error { + out.Name = in.Name + out.Secret = in.Secret + return nil +} + +// Convert_config_Key_To_v1_Key is an autogenerated conversion function. +func Convert_config_Key_To_v1_Key(in *config.Key, out *Key, s conversion.Scope) error { + return autoConvert_config_Key_To_v1_Key(in, out, s) +} + +func autoConvert_v1_ProviderConfiguration_To_config_ProviderConfiguration(in *ProviderConfiguration, out *config.ProviderConfiguration, s conversion.Scope) error { + out.AESGCM = (*config.AESConfiguration)(unsafe.Pointer(in.AESGCM)) + out.AESCBC = (*config.AESConfiguration)(unsafe.Pointer(in.AESCBC)) + out.Secretbox = (*config.SecretboxConfiguration)(unsafe.Pointer(in.Secretbox)) + out.Identity = (*config.IdentityConfiguration)(unsafe.Pointer(in.Identity)) + out.KMS = (*config.KMSConfiguration)(unsafe.Pointer(in.KMS)) + return nil +} + +// Convert_v1_ProviderConfiguration_To_config_ProviderConfiguration is an autogenerated conversion function. +func Convert_v1_ProviderConfiguration_To_config_ProviderConfiguration(in *ProviderConfiguration, out *config.ProviderConfiguration, s conversion.Scope) error { + return autoConvert_v1_ProviderConfiguration_To_config_ProviderConfiguration(in, out, s) +} + +func autoConvert_config_ProviderConfiguration_To_v1_ProviderConfiguration(in *config.ProviderConfiguration, out *ProviderConfiguration, s conversion.Scope) error { + out.AESGCM = (*AESConfiguration)(unsafe.Pointer(in.AESGCM)) + out.AESCBC = (*AESConfiguration)(unsafe.Pointer(in.AESCBC)) + out.Secretbox = (*SecretboxConfiguration)(unsafe.Pointer(in.Secretbox)) + out.Identity = (*IdentityConfiguration)(unsafe.Pointer(in.Identity)) + out.KMS = (*KMSConfiguration)(unsafe.Pointer(in.KMS)) + return nil +} + +// Convert_config_ProviderConfiguration_To_v1_ProviderConfiguration is an autogenerated conversion function. +func Convert_config_ProviderConfiguration_To_v1_ProviderConfiguration(in *config.ProviderConfiguration, out *ProviderConfiguration, s conversion.Scope) error { + return autoConvert_config_ProviderConfiguration_To_v1_ProviderConfiguration(in, out, s) +} + +func autoConvert_v1_ResourceConfiguration_To_config_ResourceConfiguration(in *ResourceConfiguration, out *config.ResourceConfiguration, s conversion.Scope) error { + out.Resources = *(*[]string)(unsafe.Pointer(&in.Resources)) + out.Providers = *(*[]config.ProviderConfiguration)(unsafe.Pointer(&in.Providers)) + return nil +} + +// Convert_v1_ResourceConfiguration_To_config_ResourceConfiguration is an autogenerated conversion function. +func Convert_v1_ResourceConfiguration_To_config_ResourceConfiguration(in *ResourceConfiguration, out *config.ResourceConfiguration, s conversion.Scope) error { + return autoConvert_v1_ResourceConfiguration_To_config_ResourceConfiguration(in, out, s) +} + +func autoConvert_config_ResourceConfiguration_To_v1_ResourceConfiguration(in *config.ResourceConfiguration, out *ResourceConfiguration, s conversion.Scope) error { + out.Resources = *(*[]string)(unsafe.Pointer(&in.Resources)) + out.Providers = *(*[]ProviderConfiguration)(unsafe.Pointer(&in.Providers)) + return nil +} + +// Convert_config_ResourceConfiguration_To_v1_ResourceConfiguration is an autogenerated conversion function. +func Convert_config_ResourceConfiguration_To_v1_ResourceConfiguration(in *config.ResourceConfiguration, out *ResourceConfiguration, s conversion.Scope) error { + return autoConvert_config_ResourceConfiguration_To_v1_ResourceConfiguration(in, out, s) +} + +func autoConvert_v1_SecretboxConfiguration_To_config_SecretboxConfiguration(in *SecretboxConfiguration, out *config.SecretboxConfiguration, s conversion.Scope) error { + out.Keys = *(*[]config.Key)(unsafe.Pointer(&in.Keys)) + return nil +} + +// Convert_v1_SecretboxConfiguration_To_config_SecretboxConfiguration is an autogenerated conversion function. +func Convert_v1_SecretboxConfiguration_To_config_SecretboxConfiguration(in *SecretboxConfiguration, out *config.SecretboxConfiguration, s conversion.Scope) error { + return autoConvert_v1_SecretboxConfiguration_To_config_SecretboxConfiguration(in, out, s) +} + +func autoConvert_config_SecretboxConfiguration_To_v1_SecretboxConfiguration(in *config.SecretboxConfiguration, out *SecretboxConfiguration, s conversion.Scope) error { + out.Keys = *(*[]Key)(unsafe.Pointer(&in.Keys)) + return nil +} + +// Convert_config_SecretboxConfiguration_To_v1_SecretboxConfiguration is an autogenerated conversion function. +func Convert_config_SecretboxConfiguration_To_v1_SecretboxConfiguration(in *config.SecretboxConfiguration, out *SecretboxConfiguration, s conversion.Scope) error { + return autoConvert_config_SecretboxConfiguration_To_v1_SecretboxConfiguration(in, out, s) +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/zz_generated.deepcopy.go new file mode 100644 index 00000000000..e5f28d46a3d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/zz_generated.deepcopy.go @@ -0,0 +1,216 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AESConfiguration) DeepCopyInto(out *AESConfiguration) { + *out = *in + if in.Keys != nil { + in, out := &in.Keys, &out.Keys + *out = make([]Key, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AESConfiguration. +func (in *AESConfiguration) DeepCopy() *AESConfiguration { + if in == nil { + return nil + } + out := new(AESConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EncryptionConfiguration) DeepCopyInto(out *EncryptionConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = make([]ResourceConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionConfiguration. +func (in *EncryptionConfiguration) DeepCopy() *EncryptionConfiguration { + if in == nil { + return nil + } + out := new(EncryptionConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EncryptionConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IdentityConfiguration) DeepCopyInto(out *IdentityConfiguration) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IdentityConfiguration. +func (in *IdentityConfiguration) DeepCopy() *IdentityConfiguration { + if in == nil { + return nil + } + out := new(IdentityConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KMSConfiguration) DeepCopyInto(out *KMSConfiguration) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KMSConfiguration. +func (in *KMSConfiguration) DeepCopy() *KMSConfiguration { + if in == nil { + return nil + } + out := new(KMSConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Key) DeepCopyInto(out *Key) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Key. +func (in *Key) DeepCopy() *Key { + if in == nil { + return nil + } + out := new(Key) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProviderConfiguration) DeepCopyInto(out *ProviderConfiguration) { + *out = *in + if in.AESGCM != nil { + in, out := &in.AESGCM, &out.AESGCM + *out = new(AESConfiguration) + (*in).DeepCopyInto(*out) + } + if in.AESCBC != nil { + in, out := &in.AESCBC, &out.AESCBC + *out = new(AESConfiguration) + (*in).DeepCopyInto(*out) + } + if in.Secretbox != nil { + in, out := &in.Secretbox, &out.Secretbox + *out = new(SecretboxConfiguration) + (*in).DeepCopyInto(*out) + } + if in.Identity != nil { + in, out := &in.Identity, &out.Identity + *out = new(IdentityConfiguration) + **out = **in + } + if in.KMS != nil { + in, out := &in.KMS, &out.KMS + *out = new(KMSConfiguration) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderConfiguration. +func (in *ProviderConfiguration) DeepCopy() *ProviderConfiguration { + if in == nil { + return nil + } + out := new(ProviderConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceConfiguration) DeepCopyInto(out *ResourceConfiguration) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Providers != nil { + in, out := &in.Providers, &out.Providers + *out = make([]ProviderConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceConfiguration. +func (in *ResourceConfiguration) DeepCopy() *ResourceConfiguration { + if in == nil { + return nil + } + out := new(ResourceConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretboxConfiguration) DeepCopyInto(out *SecretboxConfiguration) { + *out = *in + if in.Keys != nil { + in, out := &in.Keys, &out.Keys + *out = make([]Key, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretboxConfiguration. +func (in *SecretboxConfiguration) DeepCopy() *SecretboxConfiguration { + if in == nil { + return nil + } + out := new(SecretboxConfiguration) + in.DeepCopyInto(out) + return out +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/v1/zz_generated.defaults.go b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/zz_generated.defaults.go new file mode 100644 index 00000000000..cce2e603a69 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/zz_generated.defaults.go @@ -0,0 +1,32 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by defaulter-gen. DO NOT EDIT. + +package v1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + return nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/zz_generated.deepcopy.go b/staging/src/k8s.io/apiserver/pkg/apis/config/zz_generated.deepcopy.go index 0b81eb6919e..438dff997d5 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/config/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/zz_generated.deepcopy.go @@ -20,6 +20,31 @@ limitations under the License. package config +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AESConfiguration) DeepCopyInto(out *AESConfiguration) { + *out = *in + if in.Keys != nil { + in, out := &in.Keys, &out.Keys + *out = make([]Key, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AESConfiguration. +func (in *AESConfiguration) DeepCopy() *AESConfiguration { + if in == nil { + return nil + } + out := new(AESConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DebuggingConfiguration) DeepCopyInto(out *DebuggingConfiguration) { *out = *in @@ -36,6 +61,86 @@ func (in *DebuggingConfiguration) DeepCopy() *DebuggingConfiguration { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EncryptionConfiguration) DeepCopyInto(out *EncryptionConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = make([]ResourceConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionConfiguration. +func (in *EncryptionConfiguration) DeepCopy() *EncryptionConfiguration { + if in == nil { + return nil + } + out := new(EncryptionConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EncryptionConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IdentityConfiguration) DeepCopyInto(out *IdentityConfiguration) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IdentityConfiguration. +func (in *IdentityConfiguration) DeepCopy() *IdentityConfiguration { + if in == nil { + return nil + } + out := new(IdentityConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KMSConfiguration) DeepCopyInto(out *KMSConfiguration) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KMSConfiguration. +func (in *KMSConfiguration) DeepCopy() *KMSConfiguration { + if in == nil { + return nil + } + out := new(KMSConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Key) DeepCopyInto(out *Key) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Key. +func (in *Key) DeepCopy() *Key { + if in == nil { + return nil + } + out := new(Key) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LeaderElectionConfiguration) DeepCopyInto(out *LeaderElectionConfiguration) { *out = *in @@ -54,3 +159,93 @@ func (in *LeaderElectionConfiguration) DeepCopy() *LeaderElectionConfiguration { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProviderConfiguration) DeepCopyInto(out *ProviderConfiguration) { + *out = *in + if in.AESGCM != nil { + in, out := &in.AESGCM, &out.AESGCM + *out = new(AESConfiguration) + (*in).DeepCopyInto(*out) + } + if in.AESCBC != nil { + in, out := &in.AESCBC, &out.AESCBC + *out = new(AESConfiguration) + (*in).DeepCopyInto(*out) + } + if in.Secretbox != nil { + in, out := &in.Secretbox, &out.Secretbox + *out = new(SecretboxConfiguration) + (*in).DeepCopyInto(*out) + } + if in.Identity != nil { + in, out := &in.Identity, &out.Identity + *out = new(IdentityConfiguration) + **out = **in + } + if in.KMS != nil { + in, out := &in.KMS, &out.KMS + *out = new(KMSConfiguration) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderConfiguration. +func (in *ProviderConfiguration) DeepCopy() *ProviderConfiguration { + if in == nil { + return nil + } + out := new(ProviderConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceConfiguration) DeepCopyInto(out *ResourceConfiguration) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Providers != nil { + in, out := &in.Providers, &out.Providers + *out = make([]ProviderConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceConfiguration. +func (in *ResourceConfiguration) DeepCopy() *ResourceConfiguration { + if in == nil { + return nil + } + out := new(ResourceConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretboxConfiguration) DeepCopyInto(out *SecretboxConfiguration) { + *out = *in + if in.Keys != nil { + in, out := &in.Keys, &out.Keys + *out = make([]Key, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretboxConfiguration. +func (in *SecretboxConfiguration) DeepCopy() *SecretboxConfiguration { + if in == nil { + return nil + } + out := new(SecretboxConfiguration) + in.DeepCopyInto(out) + return out +} diff --git a/staging/src/k8s.io/apiserver/pkg/audit/BUILD b/staging/src/k8s.io/apiserver/pkg/audit/BUILD index bb74cb151c9..f71da4d4243 100644 --- a/staging/src/k8s.io/apiserver/pkg/audit/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/audit/BUILD @@ -33,9 +33,9 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pborman/uuid:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -64,7 +64,9 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//staging/src/k8s.io/apiserver/pkg/audit/event:all-srcs", "//staging/src/k8s.io/apiserver/pkg/audit/policy:all-srcs", + "//staging/src/k8s.io/apiserver/pkg/audit/util:all-srcs", ], tags = ["automanaged"], ) diff --git a/staging/src/k8s.io/apiserver/pkg/audit/OWNERS b/staging/src/k8s.io/apiserver/pkg/audit/OWNERS new file mode 100644 index 00000000000..178ce84a5ce --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/audit/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-audit-approvers +reviewers: +- sig-auth-audit-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/apiserver/pkg/audit/event/BUILD b/staging/src/k8s.io/apiserver/pkg/audit/event/BUILD new file mode 100644 index 00000000000..9aa798bc729 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/audit/event/BUILD @@ -0,0 +1,38 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["attributes.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/audit/event", + importpath = "k8s.io/apiserver/pkg/audit/event", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["attributes_test.go"], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", + "//vendor/github.com/stretchr/testify/require:go_default_library", + ], +) diff --git a/staging/src/k8s.io/apiserver/pkg/audit/event/attributes.go b/staging/src/k8s.io/apiserver/pkg/audit/event/attributes.go new file mode 100644 index 00000000000..576b8db8482 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/audit/event/attributes.go @@ -0,0 +1,147 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package event + +import ( + "fmt" + "net/url" + + "k8s.io/apiserver/pkg/apis/audit" + authuser "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/authorization/authorizer" +) + +var _ authorizer.Attributes = &attributes{} + +// attributes implements the authorizer attributes interface +// with event data. This is used for enforced audit backends +type attributes struct { + event *audit.Event + path string +} + +// NewAttributes returns a new attributes struct and parsed request uri +// if needed +func NewAttributes(event *audit.Event) (authorizer.Attributes, error) { + a := attributes{ + event: event, + } + if event.ObjectRef == nil { + u, err := url.ParseRequestURI(a.event.RequestURI) + if err != nil { + return nil, fmt.Errorf("could not parse url: %v", err) + } + a.path = u.Path + } + return &a, nil +} + +// GetUser returns the user. This is only used for checking audit policy, +// and the audit policy user check is based off the original user, +// not the impersonated user. +func (a *attributes) GetUser() authuser.Info { + return user(a.event.User) +} + +// GetVerb returns the verb +func (a *attributes) GetVerb() string { + return a.event.Verb +} + +// IsReadOnly determines if the verb is a read only action +func (a *attributes) IsReadOnly() bool { + return a.event.Verb == "get" || a.event.Verb == "list" || a.event.Verb == "watch" +} + +// GetNamespace returns the object namespace if present +func (a *attributes) GetNamespace() string { + if a.event.ObjectRef == nil { + return "" + } + return a.event.ObjectRef.Namespace +} + +// GetResource returns the object resource if present +func (a *attributes) GetResource() string { + if a.event.ObjectRef == nil { + return "" + } + return a.event.ObjectRef.Resource +} + +// GetSubresource returns the object subresource if present +func (a *attributes) GetSubresource() string { + if a.event.ObjectRef == nil { + return "" + } + return a.event.ObjectRef.Subresource +} + +// GetName returns the object name if present +func (a *attributes) GetName() string { + if a.event.ObjectRef == nil { + return "" + } + return a.event.ObjectRef.Name +} + +// GetAPIGroup returns the object api group if present +func (a *attributes) GetAPIGroup() string { + if a.event.ObjectRef == nil { + return "" + } + return a.event.ObjectRef.APIGroup +} + +// GetAPIVersion returns the object api version if present +func (a *attributes) GetAPIVersion() string { + if a.event.ObjectRef == nil { + return "" + } + return a.event.ObjectRef.APIVersion +} + +// IsResourceRequest determines if the request was acted on a resource +func (a *attributes) IsResourceRequest() bool { + return a.event.ObjectRef != nil +} + +// GetPath returns the path uri accessed +func (a *attributes) GetPath() string { + return a.path +} + +// user represents the event user +type user audit.UserInfo + +// GetName returns the user name +func (u user) GetName() string { return u.Username } + +// GetUID returns the user uid +func (u user) GetUID() string { return u.UID } + +// GetGroups returns the user groups +func (u user) GetGroups() []string { return u.Groups } + +// GetExtra returns the user extra data +func (u user) GetExtra() map[string][]string { + m := map[string][]string{} + for k, v := range u.Extra { + m[k] = []string(v) + } + return m +} diff --git a/staging/src/k8s.io/apiserver/pkg/audit/event/attributes_test.go b/staging/src/k8s.io/apiserver/pkg/audit/event/attributes_test.go new file mode 100644 index 00000000000..e97dbf45bc4 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/audit/event/attributes_test.go @@ -0,0 +1,96 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package event + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "k8s.io/apiserver/pkg/apis/audit" +) + +func TestAttributes(t *testing.T) { + for _, tc := range []struct { + desc string + ev *audit.Event + path string + isReadOnly bool + resourceName string + shouldErr bool + }{ + { + desc: "has resources", + ev: &audit.Event{ + Verb: "get", + ObjectRef: &audit.ObjectReference{ + Resource: "pod", + Name: "mypod", + Namespace: "test", + }, + RequestURI: "/api/v1/namespaces/test/pods", + }, + path: "", + isReadOnly: true, + resourceName: "mypod", + shouldErr: false, + }, + { + desc: "no resources", + ev: &audit.Event{ + Verb: "create", + RequestURI: "/api/v1/namespaces/test/pods", + }, + path: "/api/v1/namespaces/test/pods", + isReadOnly: false, + resourceName: "", + shouldErr: false, + }, + { + desc: "no path or resource", + ev: &audit.Event{ + Verb: "create", + }, + path: "", + isReadOnly: false, + resourceName: "", + shouldErr: true, + }, + { + desc: "invalid path", + ev: &audit.Event{ + Verb: "create", + }, + path: "a/bad/path", + isReadOnly: false, + resourceName: "", + shouldErr: true, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + attr, err := NewAttributes(tc.ev) + if tc.shouldErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.path, attr.GetPath()) + require.Equal(t, tc.isReadOnly, attr.IsReadOnly()) + require.Equal(t, tc.resourceName, attr.GetName()) + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/audit/metrics.go b/staging/src/k8s.io/apiserver/pkg/audit/metrics.go index 10280e0d88e..46b480eeaf5 100644 --- a/staging/src/k8s.io/apiserver/pkg/audit/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/audit/metrics.go @@ -19,9 +19,9 @@ package audit import ( "fmt" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" auditinternal "k8s.io/apiserver/pkg/apis/audit" + "k8s.io/klog" ) const ( @@ -83,5 +83,5 @@ func HandlePluginError(plugin string, err error, impacted ...*auditinternal.Even for _, ev := range impacted { msg = msg + EventString(ev) + "\n" } - glog.Error(msg) + klog.Error(msg) } diff --git a/staging/src/k8s.io/apiserver/pkg/audit/policy/BUILD b/staging/src/k8s.io/apiserver/pkg/audit/policy/BUILD index a844d732132..eb5905be653 100644 --- a/staging/src/k8s.io/apiserver/pkg/audit/policy/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/audit/policy/BUILD @@ -10,12 +10,21 @@ go_test( name = "go_default_test", srcs = [ "checker_test.go", + "dynamic_test.go", + "enforce_test.go", "reader_test.go", + "util_test.go", ], embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/api/auditregistration/v1alpha1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/apitesting/fuzzer:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/audit/fuzzer:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook:go_default_library", @@ -28,12 +37,17 @@ go_library( name = "go_default_library", srcs = [ "checker.go", + "dynamic.go", + "enforce.go", "reader.go", + "util.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/audit/policy", importpath = "k8s.io/apiserver/pkg/audit/policy", deps = [ + "//staging/src/k8s.io/api/auditregistration/v1alpha1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1alpha1:go_default_library", @@ -41,7 +55,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/apis/audit/validation:go_default_library", "//staging/src/k8s.io/apiserver/pkg/audit:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/audit/policy/dynamic.go b/staging/src/k8s.io/apiserver/pkg/audit/policy/dynamic.go new file mode 100644 index 00000000000..4b5f29a1132 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/audit/policy/dynamic.go @@ -0,0 +1,54 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "k8s.io/api/auditregistration/v1alpha1" + "k8s.io/apiserver/pkg/apis/audit" + "k8s.io/apiserver/pkg/authorization/authorizer" +) + +// ConvertDynamicPolicyToInternal constructs an internal policy type from a +// v1alpha1 dynamic type +func ConvertDynamicPolicyToInternal(p *v1alpha1.Policy) *audit.Policy { + stages := make([]audit.Stage, len(p.Stages)) + for i, stage := range p.Stages { + stages[i] = audit.Stage(stage) + } + return &audit.Policy{ + Rules: []audit.PolicyRule{ + { + Level: audit.Level(p.Level), + }, + }, + OmitStages: InvertStages(stages), + } +} + +// NewDynamicChecker returns a new dynamic policy checker +func NewDynamicChecker() Checker { + return &dynamicPolicyChecker{} +} + +type dynamicPolicyChecker struct{} + +// LevelAndStages returns returns a fixed level of the full event, this is so that the downstream policy +// can be applied per sink. +// TODO: this needs benchmarking before the API moves to beta to determine the effect this has on the apiserver +func (d *dynamicPolicyChecker) LevelAndStages(authorizer.Attributes) (audit.Level, []audit.Stage) { + return audit.LevelRequestResponse, []audit.Stage{} +} diff --git a/staging/src/k8s.io/apiserver/pkg/audit/policy/dynamic_test.go b/staging/src/k8s.io/apiserver/pkg/audit/policy/dynamic_test.go new file mode 100644 index 00000000000..3697160014d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/audit/policy/dynamic_test.go @@ -0,0 +1,81 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "k8s.io/api/auditregistration/v1alpha1" + "k8s.io/apiserver/pkg/apis/audit" +) + +func TestConvertDynamicPolicyToInternal(t *testing.T) { + for _, test := range []struct { + desc string + dynamic *v1alpha1.Policy + internal *audit.Policy + }{ + { + desc: "should convert full", + dynamic: &v1alpha1.Policy{ + Level: v1alpha1.LevelMetadata, + Stages: []v1alpha1.Stage{ + v1alpha1.StageResponseComplete, + }, + }, + internal: &audit.Policy{ + Rules: []audit.PolicyRule{ + { + Level: audit.LevelMetadata, + }, + }, + OmitStages: []audit.Stage{ + audit.StageRequestReceived, + audit.StageResponseStarted, + audit.StagePanic, + }, + }, + }, + { + desc: "should convert missing stages", + dynamic: &v1alpha1.Policy{ + Level: v1alpha1.LevelMetadata, + }, + internal: &audit.Policy{ + Rules: []audit.PolicyRule{ + { + Level: audit.LevelMetadata, + }, + }, + OmitStages: []audit.Stage{ + audit.StageRequestReceived, + audit.StageResponseStarted, + audit.StageResponseComplete, + audit.StagePanic, + }, + }, + }, + } { + t.Run(test.desc, func(t *testing.T) { + d := ConvertDynamicPolicyToInternal(test.dynamic) + require.ElementsMatch(t, test.internal.OmitStages, d.OmitStages) + require.Equal(t, test.internal.Rules, d.Rules) + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/audit/policy/enforce.go b/staging/src/k8s.io/apiserver/pkg/audit/policy/enforce.go new file mode 100644 index 00000000000..e2b107b9f1d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/audit/policy/enforce.go @@ -0,0 +1,53 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "fmt" + + "k8s.io/apiserver/pkg/apis/audit" +) + +// EnforcePolicy drops any part of the event that doesn't conform to a policy level +// or omitStages and sets the event level accordingly +func EnforcePolicy(event *audit.Event, level audit.Level, omitStages []audit.Stage) (*audit.Event, error) { + for _, stage := range omitStages { + if event.Stage == stage { + return nil, nil + } + } + return enforceLevel(event, level) +} + +func enforceLevel(event *audit.Event, level audit.Level) (*audit.Event, error) { + switch level { + case audit.LevelMetadata: + event.Level = audit.LevelMetadata + event.ResponseObject = nil + event.RequestObject = nil + case audit.LevelRequest: + event.Level = audit.LevelRequest + event.ResponseObject = nil + case audit.LevelRequestResponse: + event.Level = audit.LevelRequestResponse + case audit.LevelNone: + return nil, nil + default: + return nil, fmt.Errorf("level unknown: %s", level) + } + return event, nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/audit/policy/enforce_test.go b/staging/src/k8s.io/apiserver/pkg/audit/policy/enforce_test.go new file mode 100644 index 00000000000..b7fd775a19d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/audit/policy/enforce_test.go @@ -0,0 +1,146 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "k8s.io/apimachinery/pkg/api/apitesting/fuzzer" + "k8s.io/apimachinery/pkg/runtime" + runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/apis/audit" + auditfuzz "k8s.io/apiserver/pkg/apis/audit/fuzzer" +) + +func TestEnforcePolicy(t *testing.T) { + scheme := runtime.NewScheme() + audit.SchemeBuilder.AddToScheme(scheme) + codecs := runtimeserializer.NewCodecFactory(scheme) + rs := rand.NewSource(time.Now().UnixNano()) + objectFuzzer := fuzzer.FuzzerFor(auditfuzz.Funcs, rs, codecs) + + for _, tc := range []struct { + name string + level audit.Level + omitStages []audit.Stage + }{ + { + name: "level metadata", + level: audit.LevelMetadata, + }, + { + name: "level request", + level: audit.LevelRequest, + }, + { + name: "level requestresponse", + level: audit.LevelRequestResponse, + }, + { + name: "level none", + level: audit.LevelNone, + }, + { + name: "level unknown", + level: audit.Level("unknown"), + }, + { + name: "stage valid", + level: audit.LevelRequest, + omitStages: []audit.Stage{audit.StageRequestReceived}, + }, + { + name: "stage unknown", + level: audit.LevelRequest, + omitStages: []audit.Stage{"unknown"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + events := make([]audit.Event, 20) + omitSet := sets.NewString(ConvertStagesToStrings(tc.omitStages)...) + for i := range events { + e := &events[i] + objectFuzzer.Fuzz(e) + ev, err := EnforcePolicy(e, tc.level, tc.omitStages) + if omitSet.Has(string(e.Stage)) { + require.Nil(t, err) + require.Nil(t, ev) + return + } + switch tc.level { + case audit.LevelNone: + require.Nil(t, ev) + case audit.LevelMetadata: + expected := &audit.Event{ + TypeMeta: e.TypeMeta, + Level: tc.level, + AuditID: e.AuditID, + Stage: e.Stage, + RequestURI: e.RequestURI, + Verb: e.Verb, + User: e.User, + ImpersonatedUser: e.ImpersonatedUser, + SourceIPs: e.SourceIPs, + UserAgent: e.UserAgent, + ObjectRef: e.ObjectRef, + ResponseStatus: e.ResponseStatus, + RequestReceivedTimestamp: e.RequestReceivedTimestamp, + StageTimestamp: e.StageTimestamp, + Annotations: e.Annotations, + RequestObject: nil, + ResponseObject: nil, + } + require.Equal(t, expected, ev) + case audit.LevelRequest: + expected := &audit.Event{ + TypeMeta: e.TypeMeta, + Level: tc.level, + AuditID: e.AuditID, + Stage: e.Stage, + RequestURI: e.RequestURI, + Verb: e.Verb, + User: e.User, + ImpersonatedUser: e.ImpersonatedUser, + SourceIPs: e.SourceIPs, + UserAgent: e.UserAgent, + ObjectRef: e.ObjectRef, + ResponseStatus: e.ResponseStatus, + RequestReceivedTimestamp: e.RequestReceivedTimestamp, + StageTimestamp: e.StageTimestamp, + Annotations: e.Annotations, + RequestObject: e.RequestObject, + ResponseObject: nil, + } + require.Equal(t, expected, ev) + case audit.LevelRequestResponse: + expected := e.DeepCopy() + expected.Level = tc.level + require.Equal(t, expected, ev) + default: + require.NotNil(t, err) + return + } + require.Nil(t, err) + } + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/audit/policy/reader.go b/staging/src/k8s.io/apiserver/pkg/audit/policy/reader.go index e78f86e8c44..3d669fe699d 100644 --- a/staging/src/k8s.io/apiserver/pkg/audit/policy/reader.go +++ b/staging/src/k8s.io/apiserver/pkg/audit/policy/reader.go @@ -28,7 +28,7 @@ import ( "k8s.io/apiserver/pkg/apis/audit/validation" "k8s.io/apiserver/pkg/audit" - "github.com/golang/glog" + "k8s.io/klog" ) var ( @@ -85,6 +85,6 @@ func LoadPolicyFromBytes(policyDef []byte) (*auditinternal.Policy, error) { if policyCnt == 0 { return nil, fmt.Errorf("loaded illegal policy with 0 rules") } - glog.V(4).Infof("Loaded %d audit policy rules", policyCnt) + klog.V(4).Infof("Loaded %d audit policy rules", policyCnt) return policy, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/audit/policy/util.go b/staging/src/k8s.io/apiserver/pkg/audit/policy/util.go new file mode 100644 index 00000000000..29be912303c --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/audit/policy/util.go @@ -0,0 +1,68 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/apis/audit" +) + +// AllStages returns all possible stages +func AllStages() sets.String { + return sets.NewString( + audit.StageRequestReceived, + audit.StageResponseStarted, + audit.StageResponseComplete, + audit.StagePanic, + ) +} + +// AllLevels returns all possible levels +func AllLevels() sets.String { + return sets.NewString( + string(audit.LevelNone), + string(audit.LevelMetadata), + string(audit.LevelRequest), + string(audit.LevelRequestResponse), + ) +} + +// InvertStages subtracts the given array of stages from all stages +func InvertStages(stages []audit.Stage) []audit.Stage { + s := ConvertStagesToStrings(stages) + a := AllStages() + a.Delete(s...) + return ConvertStringSetToStages(a) +} + +// ConvertStagesToStrings converts an array of stages to a string array +func ConvertStagesToStrings(stages []audit.Stage) []string { + s := make([]string, len(stages)) + for i, stage := range stages { + s[i] = string(stage) + } + return s +} + +// ConvertStringSetToStages converts a string set to an array of stages +func ConvertStringSetToStages(set sets.String) []audit.Stage { + stages := make([]audit.Stage, len(set)) + for i, stage := range set.List() { + stages[i] = audit.Stage(stage) + } + return stages +} diff --git a/staging/src/k8s.io/apiserver/pkg/audit/policy/util_test.go b/staging/src/k8s.io/apiserver/pkg/audit/policy/util_test.go new file mode 100644 index 00000000000..3181b03a622 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/audit/policy/util_test.go @@ -0,0 +1,81 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "k8s.io/apiserver/pkg/apis/audit" +) + +func TestInvertStages(t *testing.T) { + for _, test := range []struct { + desc string + stages []audit.Stage + expectedStages []audit.Stage + }{ + { + desc: "should remove one", + stages: []audit.Stage{ + audit.StageResponseStarted, + }, + expectedStages: []audit.Stage{ + audit.StageRequestReceived, + audit.StageResponseComplete, + audit.StagePanic, + }, + }, + { + desc: "should remove both", + stages: []audit.Stage{ + audit.StageResponseStarted, + audit.StageRequestReceived, + }, + expectedStages: []audit.Stage{ + audit.StageResponseComplete, + audit.StagePanic, + }, + }, + { + desc: "should remove none", + stages: []audit.Stage{}, + expectedStages: []audit.Stage{ + audit.StageResponseComplete, + audit.StageResponseStarted, + audit.StageRequestReceived, + audit.StagePanic, + }, + }, + { + desc: "should remove all", + stages: []audit.Stage{ + audit.StageResponseComplete, + audit.StageResponseStarted, + audit.StageRequestReceived, + audit.StagePanic, + }, + expectedStages: []audit.Stage{}, + }, + } { + t.Run(test.desc, func(t *testing.T) { + e := InvertStages(test.stages) + require.ElementsMatch(t, e, test.expectedStages) + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/audit/request.go b/staging/src/k8s.io/apiserver/pkg/audit/request.go index 3aafde910f9..d4b12770eab 100644 --- a/staging/src/k8s.io/apiserver/pkg/audit/request.go +++ b/staging/src/k8s.io/apiserver/pkg/audit/request.go @@ -22,8 +22,8 @@ import ( "net/http" "time" - "github.com/golang/glog" "github.com/pborman/uuid" + "k8s.io/klog" "reflect" @@ -152,7 +152,7 @@ func LogRequestObject(ae *auditinternal.Event, obj runtime.Object, gvr schema.Gr ae.RequestObject, err = encodeObject(obj, gvr.GroupVersion(), s) if err != nil { // TODO(audit): add error slice to audit event struct - glog.Warningf("Auditing failed of %v request: %v", reflect.TypeOf(obj).Name(), err) + klog.Warningf("Auditing failed of %v request: %v", reflect.TypeOf(obj).Name(), err) return } } @@ -191,7 +191,7 @@ func LogResponseObject(ae *auditinternal.Event, obj runtime.Object, gv schema.Gr var err error ae.ResponseObject, err = encodeObject(obj, gv, s) if err != nil { - glog.Warningf("Audit failed for %q response: %v", reflect.TypeOf(obj).Name(), err) + klog.Warningf("Audit failed for %q response: %v", reflect.TypeOf(obj).Name(), err) } } @@ -223,7 +223,7 @@ func LogAnnotation(ae *auditinternal.Event, key, value string) { ae.Annotations = make(map[string]string) } if v, ok := ae.Annotations[key]; ok && v != value { - glog.Warningf("Failed to set annotations[%q] to %q for audit:%q, it has already been set to %q", key, value, ae.AuditID, ae.Annotations[key]) + klog.Warningf("Failed to set annotations[%q] to %q for audit:%q, it has already been set to %q", key, value, ae.AuditID, ae.Annotations[key]) return } ae.Annotations[key] = value diff --git a/staging/src/k8s.io/apiserver/pkg/audit/util/BUILD b/staging/src/k8s.io/apiserver/pkg/audit/util/BUILD new file mode 100644 index 00000000000..037503622eb --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/audit/util/BUILD @@ -0,0 +1,39 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["conversion.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/audit/util", + importpath = "k8s.io/apiserver/pkg/audit/util", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/api/auditregistration/v1alpha1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["conversion_test.go"], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/api/auditregistration/v1alpha1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", + "//vendor/github.com/stretchr/testify/require:go_default_library", + ], +) diff --git a/staging/src/k8s.io/apiserver/pkg/audit/util/conversion.go b/staging/src/k8s.io/apiserver/pkg/audit/util/conversion.go new file mode 100644 index 00000000000..6b1f35c4395 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/audit/util/conversion.go @@ -0,0 +1,43 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "k8s.io/api/auditregistration/v1alpha1" + "k8s.io/apiserver/pkg/util/webhook" +) + +// HookClientConfigForSink constructs a webhook.ClientConfig using a v1alpha1.AuditSink API object. +// webhook.ClientConfig is used to create a HookClient and the purpose of the config struct is to +// share that with other packages that need to create a HookClient. +func HookClientConfigForSink(a *v1alpha1.AuditSink) webhook.ClientConfig { + c := a.Spec.Webhook.ClientConfig + ret := webhook.ClientConfig{Name: a.Name, CABundle: c.CABundle} + if c.URL != nil { + ret.URL = *c.URL + } + if c.Service != nil { + ret.Service = &webhook.ClientConfigService{ + Name: c.Service.Name, + Namespace: c.Service.Namespace, + } + if c.Service.Path != nil { + ret.Service.Path = *c.Service.Path + } + } + return ret +} diff --git a/staging/src/k8s.io/apiserver/pkg/audit/util/conversion_test.go b/staging/src/k8s.io/apiserver/pkg/audit/util/conversion_test.go new file mode 100644 index 00000000000..4cd23c13d19 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/audit/util/conversion_test.go @@ -0,0 +1,88 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + "github.com/stretchr/testify/require" + + auditregv1alpha1 "k8s.io/api/auditregistration/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/util/webhook" +) + +func TestHookClientConfigForSink(t *testing.T) { + testURL := "http://localhost" + path := "/path" + for _, tc := range []struct { + desc string + sink *auditregv1alpha1.AuditSink + clientConfig webhook.ClientConfig + }{ + { + desc: "build full", + sink: &auditregv1alpha1.AuditSink{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: auditregv1alpha1.AuditSinkSpec{ + Webhook: auditregv1alpha1.Webhook{ + ClientConfig: auditregv1alpha1.WebhookClientConfig{ + URL: &testURL, + Service: &auditregv1alpha1.ServiceReference{ + Name: "test", + Path: &path, + Namespace: "test", + }, + }, + }, + }, + }, + clientConfig: webhook.ClientConfig{ + Name: "test", + URL: testURL, + Service: &webhook.ClientConfigService{ + Name: "test", + Namespace: "test", + Path: path, + }, + }, + }, + { + desc: "build empty client config", + sink: &auditregv1alpha1.AuditSink{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: auditregv1alpha1.AuditSinkSpec{ + Webhook: auditregv1alpha1.Webhook{ + ClientConfig: auditregv1alpha1.WebhookClientConfig{}, + }, + }, + }, + clientConfig: webhook.ClientConfig{ + Name: "test", + }, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + ret := HookClientConfigForSink(tc.sink) + require.Equal(t, tc.clientConfig, ret) + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/OWNERS b/staging/src/k8s.io/apiserver/pkg/authentication/OWNERS new file mode 100644 index 00000000000..c607d2aa8c5 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/authentication/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authenticators-approvers +reviewers: +- sig-auth-authenticators-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/BUILD b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/BUILD index 750e8b4eae4..1da12201175 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/BUILD @@ -23,6 +23,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/authentication/request/union:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/request/websocket:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/request/x509:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authentication/token/cache:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/token/tokenfile:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go index d8e18345545..67958c3639b 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go +++ b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go @@ -31,6 +31,7 @@ import ( unionauth "k8s.io/apiserver/pkg/authentication/request/union" "k8s.io/apiserver/pkg/authentication/request/websocket" "k8s.io/apiserver/pkg/authentication/request/x509" + "k8s.io/apiserver/pkg/authentication/token/cache" webhooktoken "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook" authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1" "k8s.io/client-go/util/cert" @@ -50,6 +51,8 @@ type DelegatingAuthenticatorConfig struct { // ClientCAFile is the CA bundle file used to authenticate client certificates ClientCAFile string + APIAudiences authenticator.Audiences + RequestHeaderConfig *RequestHeaderConfig } @@ -85,11 +88,12 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur } if c.TokenAccessReviewClient != nil { - tokenAuth, err := webhooktoken.NewFromInterface(c.TokenAccessReviewClient, c.CacheTTL) + tokenAuth, err := webhooktoken.NewFromInterface(c.TokenAccessReviewClient, c.APIAudiences) if err != nil { return nil, nil, err } - authenticators = append(authenticators, bearertoken.New(tokenAuth), websocket.NewProtocolAuthenticator(tokenAuth)) + cachingTokenAuth := cache.New(tokenAuth, false, c.CacheTTL, c.CacheTTL) + authenticators = append(authenticators, bearertoken.New(cachingTokenAuth), websocket.NewProtocolAuthenticator(cachingTokenAuth)) securityDefinitions["BearerToken"] = &spec.SecurityScheme{ SecuritySchemeProps: spec.SecuritySchemeProps{ diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/OWNERS b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/OWNERS new file mode 100644 index 00000000000..470b7a1c92d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-certificates-approvers +reviewers: +- sig-auth-certificates-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator.go b/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator.go index 457770aa73d..ea3853a38b4 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator.go +++ b/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator.go @@ -35,6 +35,7 @@ type cacheRecord struct { type cachedTokenAuthenticator struct { authenticator authenticator.Token + cacheErrs bool successTTL time.Duration failureTTL time.Duration @@ -51,13 +52,14 @@ type cache interface { } // New returns a token authenticator that caches the results of the specified authenticator. A ttl of 0 bypasses the cache. -func New(authenticator authenticator.Token, successTTL, failureTTL time.Duration) authenticator.Token { - return newWithClock(authenticator, successTTL, failureTTL, utilclock.RealClock{}) +func New(authenticator authenticator.Token, cacheErrs bool, successTTL, failureTTL time.Duration) authenticator.Token { + return newWithClock(authenticator, cacheErrs, successTTL, failureTTL, utilclock.RealClock{}) } -func newWithClock(authenticator authenticator.Token, successTTL, failureTTL time.Duration, clock utilclock.Clock) authenticator.Token { +func newWithClock(authenticator authenticator.Token, cacheErrs bool, successTTL, failureTTL time.Duration, clock utilclock.Clock) authenticator.Token { return &cachedTokenAuthenticator{ authenticator: authenticator, + cacheErrs: cacheErrs, successTTL: successTTL, failureTTL: failureTTL, cache: newStripedCache(32, fnvHashFunc, func() cache { return newSimpleCache(128, clock) }), @@ -74,6 +76,9 @@ func (a *cachedTokenAuthenticator) AuthenticateToken(ctx context.Context, token } resp, ok, err := a.authenticator.AuthenticateToken(ctx, token) + if !a.cacheErrs && err != nil { + return resp, ok, err + } switch { case ok && a.successTTL > 0: diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator_test.go b/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator_test.go index 9215fefc076..09901b40033 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator_test.go +++ b/staging/src/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator_test.go @@ -41,7 +41,7 @@ func TestCachedTokenAuthenticator(t *testing.T) { }) fakeClock := utilclock.NewFakeClock(time.Now()) - a := newWithClock(fakeAuth, time.Minute, 0, fakeClock) + a := newWithClock(fakeAuth, true, time.Minute, 0, fakeClock) calledWithToken, resultUsers, resultOk, resultErr = []string{}, nil, false, nil a.AuthenticateToken(context.Background(), "bad1") @@ -113,7 +113,7 @@ func TestCachedTokenAuthenticatorWithAudiences(t *testing.T) { }) fakeClock := utilclock.NewFakeClock(time.Now()) - a := newWithClock(fakeAuth, time.Minute, 0, fakeClock) + a := newWithClock(fakeAuth, true, time.Minute, 0, fakeClock) resultUsers["audAusertoken1"] = &user.DefaultInfo{Name: "user1"} resultUsers["audBusertoken1"] = &user.DefaultInfo{Name: "user1-different"} diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/token/tokenfile/BUILD b/staging/src/k8s.io/apiserver/pkg/authentication/token/tokenfile/BUILD index 172e0fedf6c..b43d1ff609c 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/token/tokenfile/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/authentication/token/tokenfile/BUILD @@ -21,7 +21,7 @@ go_library( deps = [ "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/token/tokenfile/tokenfile.go b/staging/src/k8s.io/apiserver/pkg/authentication/token/tokenfile/tokenfile.go index 4735357f9a8..69568f17dd2 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/token/tokenfile/tokenfile.go +++ b/staging/src/k8s.io/apiserver/pkg/authentication/token/tokenfile/tokenfile.go @@ -24,9 +24,9 @@ import ( "os" "strings" - "github.com/golang/glog" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/klog" ) type TokenAuthenticator struct { @@ -67,7 +67,7 @@ func NewCSV(path string) (*TokenAuthenticator, error) { recordNum++ if record[0] == "" { - glog.Warningf("empty token has been found in token file '%s', record number '%d'", path, recordNum) + klog.Warningf("empty token has been found in token file '%s', record number '%d'", path, recordNum) continue } @@ -76,7 +76,7 @@ func NewCSV(path string) (*TokenAuthenticator, error) { UID: record[2], } if _, exist := tokens[record[0]]; exist { - glog.Warningf("duplicate token has been found in token file '%s', record number '%d'", path, recordNum) + klog.Warningf("duplicate token has been found in token file '%s', record number '%d'", path, recordNum) } tokens[record[0]] = obj diff --git a/staging/src/k8s.io/apiserver/pkg/authorization/OWNERS b/staging/src/k8s.io/apiserver/pkg/authorization/OWNERS new file mode 100644 index 00000000000..cd0d70a0f8f --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/authorization/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authorizers-approvers +reviewers: +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/BUILD index 45f63ebeb55..ad36f85ea72 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/BUILD @@ -86,11 +86,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/endpoints/metrics:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/filters:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/openapi:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", - "//vendor/k8s.io/kube-openapi/pkg/builder:go_default_library", - "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", - "//vendor/k8s.io/kube-openapi/pkg/util:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/util/proto:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/audit_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/audit_test.go index 5301f95ea18..e0c2de2d439 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/audit_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/audit_test.go @@ -42,7 +42,10 @@ type fakeAuditSink struct { func (s *fakeAuditSink) ProcessEvents(evs ...*auditinternal.Event) { s.lock.Lock() defer s.lock.Unlock() - s.events = append(s.events, evs...) + for _, ev := range evs { + e := ev.DeepCopy() + s.events = append(s.events, e) + } } func (s *fakeAuditSink) Events() []*auditinternal.Event { @@ -90,6 +93,9 @@ func TestAudit(t *testing.T) { } requestBodyMatches := func(i int, pattern string) eventCheck { return func(events []*auditinternal.Event) error { + if events[i].RequestObject == nil { + return fmt.Errorf("expected non nil request object") + } if matched, _ := regexp.Match(pattern, events[i].RequestObject.Raw); !matched { return fmt.Errorf("expected RequestBody to match %q, but didn't: %q", pattern, string(events[i].RequestObject.Raw)) } @@ -106,6 +112,9 @@ func TestAudit(t *testing.T) { } responseBodyMatches := func(i int, pattern string) eventCheck { return func(events []*auditinternal.Event) error { + if events[i].ResponseObject == nil { + return fmt.Errorf("expected non nil response object") + } if matched, _ := regexp.Match(pattern, events[i].ResponseObject.Raw); !matched { return fmt.Errorf("expected ResponseBody to match %q, but didn't: %q", pattern, string(events[i].ResponseObject.Raw)) } @@ -122,6 +131,19 @@ func TestAudit(t *testing.T) { return nil } } + expectedStages := func(stages ...auditinternal.Stage) eventCheck { + return func(events []*auditinternal.Event) error { + if len(stages) != len(events) { + return fmt.Errorf("expected %d stages, but got %d events", len(stages), len(events)) + } + for i, stage := range stages { + if events[i].Stage != stage { + return fmt.Errorf("expected stage %q, got %q", stage, events[i].Stage) + } + } + return nil + } + } for _, test := range []struct { desc string @@ -140,8 +162,9 @@ func TestAudit(t *testing.T) { 200, 2, []eventCheck{ - noRequestBody(0), - responseBodyMatches(0, `{.*"name":"c".*}`), + noRequestBody(1), + responseBodyMatches(1, `{.*"name":"c".*}`), + expectedStages(auditinternal.StageRequestReceived, auditinternal.StageResponseComplete), }, }, { @@ -157,8 +180,9 @@ func TestAudit(t *testing.T) { 200, 2, []eventCheck{ - noRequestBody(0), - responseBodyMatches(0, `{.*"name":"a".*"name":"b".*}`), + noRequestBody(1), + responseBodyMatches(1, `{.*"name":"a".*"name":"b".*}`), + expectedStages(auditinternal.StageRequestReceived, auditinternal.StageResponseComplete), }, }, { @@ -170,8 +194,9 @@ func TestAudit(t *testing.T) { 201, 2, []eventCheck{ - requestBodyIs(0, string(simpleFooJSON)), - responseBodyMatches(0, `{.*"foo".*}`), + requestBodyIs(1, string(simpleFooJSON)), + responseBodyMatches(1, `{.*"foo".*}`), + expectedStages(auditinternal.StageRequestReceived, auditinternal.StageResponseComplete), }, }, { @@ -183,8 +208,9 @@ func TestAudit(t *testing.T) { 405, 2, []eventCheck{ - noRequestBody(0), // the 405 is thrown long before the create handler would be executed - noResponseBody(0), // the 405 is thrown long before the create handler would be executed + noRequestBody(1), // the 405 is thrown long before the create handler would be executed + noResponseBody(1), // the 405 is thrown long before the create handler would be executed + expectedStages(auditinternal.StageRequestReceived, auditinternal.StageResponseComplete), }, }, { @@ -196,8 +222,9 @@ func TestAudit(t *testing.T) { 200, 2, []eventCheck{ - noRequestBody(0), - responseBodyMatches(0, `{.*"kind":"Status".*"status":"Success".*}`), + noRequestBody(1), + responseBodyMatches(1, `{.*"kind":"Status".*"status":"Success".*}`), + expectedStages(auditinternal.StageRequestReceived, auditinternal.StageResponseComplete), }, }, { @@ -209,8 +236,9 @@ func TestAudit(t *testing.T) { 200, 2, []eventCheck{ - requestBodyMatches(0, "DeleteOptions"), - responseBodyMatches(0, `{.*"kind":"Status".*"status":"Success".*}`), + requestBodyMatches(1, "DeleteOptions"), + responseBodyMatches(1, `{.*"kind":"Status".*"status":"Success".*}`), + expectedStages(auditinternal.StageRequestReceived, auditinternal.StageResponseComplete), }, }, { @@ -222,8 +250,9 @@ func TestAudit(t *testing.T) { 200, 2, []eventCheck{ - requestBodyIs(0, string(simpleCPrimeJSON)), - responseBodyMatches(0, `{.*"bla".*}`), + requestBodyIs(1, string(simpleCPrimeJSON)), + responseBodyMatches(1, `{.*"bla".*}`), + expectedStages(auditinternal.StageRequestReceived, auditinternal.StageResponseComplete), }, }, { @@ -235,8 +264,9 @@ func TestAudit(t *testing.T) { 400, 2, []eventCheck{ - requestBodyIs(0, string(simpleCPrimeJSON)), - responseBodyMatches(0, `"Status".*"status":"Failure".*"code":400}`), + requestBodyIs(1, string(simpleCPrimeJSON)), + responseBodyMatches(1, `"Status".*"status":"Failure".*"code":400}`), + expectedStages(auditinternal.StageRequestReceived, auditinternal.StageResponseComplete), }, }, { @@ -255,8 +285,9 @@ func TestAudit(t *testing.T) { 200, 2, []eventCheck{ - requestBodyIs(0, `{"labels":{"foo":"bar"}}`), - responseBodyMatches(0, `"name":"c".*"labels":{"foo":"bar"}`), + requestBodyIs(1, `{"labels":{"foo":"bar"}}`), + responseBodyMatches(1, `"name":"c".*"labels":{"foo":"bar"}`), + expectedStages(auditinternal.StageRequestReceived, auditinternal.StageResponseComplete), }, }, { @@ -272,8 +303,9 @@ func TestAudit(t *testing.T) { 200, 3, []eventCheck{ - noRequestBody(0), - noResponseBody(0), + noRequestBody(2), + noResponseBody(2), + expectedStages(auditinternal.StageRequestReceived, auditinternal.StageResponseStarted, auditinternal.StageResponseComplete), }, }, } { diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/BUILD index 5ceffa3df4d..0fc8e02381b 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/BUILD @@ -68,8 +68,8 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/httplog:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication.go b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication.go index 70c14e088a2..d9f70efac26 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication.go @@ -21,8 +21,8 @@ import ( "net/http" "strings" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" + "k8s.io/klog" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -52,7 +52,7 @@ func init() { // is invoked to serve the request. func WithAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler, apiAuds authenticator.Audiences) http.Handler { if auth == nil { - glog.Warningf("Authentication is disabled") + klog.Warningf("Authentication is disabled") return handler } return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -62,7 +62,7 @@ func WithAuthentication(handler http.Handler, auth authenticator.Request, failed resp, ok, err := auth.AuthenticateRequest(req) if err != nil || !ok { if err != nil { - glog.Errorf("Unable to authenticate the request due to an error: %v", err) + klog.Errorf("Unable to authenticate the request due to an error: %v", err) } failed.ServeHTTP(w, req) return diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authorization.go b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authorization.go index 4c9f140ca30..c6ab15b3d67 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authorization.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/authorization.go @@ -21,7 +21,7 @@ import ( "errors" "net/http" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/audit" @@ -44,7 +44,7 @@ const ( // WithAuthorizationCheck passes all authorized requests on to handler, and returns a forbidden error otherwise. func WithAuthorization(handler http.Handler, a authorizer.Authorizer, s runtime.NegotiatedSerializer) http.Handler { if a == nil { - glog.Warningf("Authorization is disabled") + klog.Warningf("Authorization is disabled") return handler } return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -70,7 +70,7 @@ func WithAuthorization(handler http.Handler, a authorizer.Authorizer, s runtime. return } - glog.V(4).Infof("Forbidden: %#v, Reason: %q", req.RequestURI, reason) + klog.V(4).Infof("Forbidden: %#v, Reason: %q", req.RequestURI, reason) audit.LogAnnotation(ae, decisionAnnotationKey, decisionForbid) audit.LogAnnotation(ae, reasonAnnotationKey, reason) responsewriters.Forbidden(ctx, attributes, w, req, reason, s) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/impersonation.go b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/impersonation.go index 726cbe4d565..d017f2bf687 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/impersonation.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/impersonation.go @@ -23,7 +23,7 @@ import ( "net/url" "strings" - "github.com/golang/glog" + "k8s.io/klog" authenticationv1 "k8s.io/api/authentication/v1" "k8s.io/api/core/v1" @@ -42,7 +42,7 @@ func WithImpersonation(handler http.Handler, a authorizer.Authorizer, s runtime. return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { impersonationRequests, err := buildImpersonationRequests(req.Header) if err != nil { - glog.V(4).Infof("%v", err) + klog.V(4).Infof("%v", err) responsewriters.InternalError(w, req, err) return } @@ -102,14 +102,14 @@ func WithImpersonation(handler http.Handler, a authorizer.Authorizer, s runtime. userExtra[extraKey] = append(userExtra[extraKey], extraValue) default: - glog.V(4).Infof("unknown impersonation request type: %v", impersonationRequest) + klog.V(4).Infof("unknown impersonation request type: %v", impersonationRequest) responsewriters.Forbidden(ctx, actingAsAttributes, w, req, fmt.Sprintf("unknown impersonation request type: %v", impersonationRequest), s) return } decision, reason, err := a.Authorize(actingAsAttributes) if err != nil || decision != authorizer.DecisionAllow { - glog.V(4).Infof("Forbidden: %#v, Reason: %s, Error: %v", req.RequestURI, reason, err) + klog.V(4).Infof("Forbidden: %#v, Reason: %s, Error: %v", req.RequestURI, reason, err) responsewriters.Forbidden(ctx, actingAsAttributes, w, req, reason, s) return } diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/groupversion.go b/staging/src/k8s.io/apiserver/pkg/endpoints/groupversion.go index 695c62b5984..a1ba7bdbbeb 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/groupversion.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/groupversion.go @@ -31,7 +31,7 @@ import ( "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/endpoints/discovery" "k8s.io/apiserver/pkg/registry/rest" - openapicommon "k8s.io/kube-openapi/pkg/common" + openapiproto "k8s.io/kube-openapi/pkg/util/proto" ) // APIGroupVersion is a helper for exposing rest.Storage objects as http.Handlers via go-restful @@ -85,8 +85,8 @@ type APIGroupVersion struct { // if the client requests it via Accept-Encoding EnableAPIResponseCompression bool - // OpenAPIConfig lets the individual handlers build a subset of the OpenAPI schema before they are installed. - OpenAPIConfig *openapicommon.Config + // OpenAPIModels exposes the OpenAPI models to each individual handler. + OpenAPIModels openapiproto.Models } // InstallREST registers the REST handlers (storage, watch, proxy and redirect) into a restful Container. diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD index fede49cf337..734fc9004b2 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD @@ -85,8 +85,8 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/trace:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/wsstream:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/net/websocket:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/util/proto:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/get.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/get.go index 8526f8066ec..0f1c59946a3 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/get.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/get.go @@ -25,7 +25,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -244,7 +244,7 @@ func ListResource(r rest.Lister, rw rest.Watcher, scope RequestScope, forceWatch if timeout == 0 && minRequestTimeout > 0 { timeout = time.Duration(float64(minRequestTimeout) * (rand.Float64() + 1.0)) } - glog.V(3).Infof("Starting watch for %s, rv=%s labels=%s fields=%s timeout=%s", req.URL.Path, opts.ResourceVersion, opts.LabelSelector, opts.FieldSelector, timeout) + klog.V(3).Infof("Starting watch for %s, rv=%s labels=%s fields=%s timeout=%s", req.URL.Path, opts.ResourceVersion, opts.LabelSelector, opts.FieldSelector, timeout) watcher, err := rw.Watch(ctx, &opts) if err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go index b3504df3f23..56da2271838 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go @@ -23,9 +23,11 @@ import ( "io/ioutil" "net/http" "net/url" + goruntime "runtime" + "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -33,7 +35,6 @@ import ( metav1beta1 "k8s.io/apimachinery/pkg/apis/meta/v1beta1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" @@ -60,7 +61,7 @@ type RequestScope struct { Trace *utiltrace.Trace TableConvertor rest.TableConvertor - OpenAPISchema openapiproto.Schema + OpenAPIModels openapiproto.Models Resource schema.GroupVersionResource Kind schema.GroupVersionKind @@ -182,10 +183,17 @@ func finishRequest(timeout time.Duration, fn resultFunc) (result runtime.Object, panicCh := make(chan interface{}, 1) go func() { // panics don't cross goroutine boundaries, so we have to handle ourselves - defer utilruntime.HandleCrash(func(panicReason interface{}) { + defer func() { + panicReason := recover() + if panicReason != nil { + const size = 64 << 10 + buf := make([]byte, size) + buf = buf[:goruntime.Stack(buf, false)] + panicReason = strings.TrimSuffix(fmt.Sprintf("%v\n%s", panicReason, string(buf)), "\n") + } // Propagate to parent goroutine panicCh <- panicReason - }) + }() if result, err := fn(); err != nil { errCh <- err @@ -284,7 +292,7 @@ func setListSelfLink(obj runtime.Object, ctx context.Context, req *http.Request, return 0, err } if err := namer.SetSelfLink(obj, uri); err != nil { - glog.V(4).Infof("Unable to set self link on object: %v", err) + klog.V(4).Infof("Unable to set self link on object: %v", err) } requestInfo, ok := request.RequestInfoFrom(ctx) if !ok { @@ -327,7 +335,7 @@ func parseTimeout(str string) time.Duration { if err == nil { return timeout } - glog.Errorf("Failed to parse %q: %v", str, err) + klog.Errorf("Failed to parse %q: %v", str, err) } return 30 * time.Second } diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest_test.go index 66d9e9c2cf8..47b198704a0 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest_test.go @@ -22,6 +22,7 @@ import ( "fmt" "net/http" "reflect" + "strings" "testing" "time" @@ -835,13 +836,15 @@ func TestFinishRequest(t *testing.T) { successStatusObj := &metav1.Status{Status: metav1.StatusSuccess, Message: "success message"} errorStatusObj := &metav1.Status{Status: metav1.StatusFailure, Message: "error message"} testcases := []struct { - timeout time.Duration - fn resultFunc - expectedObj runtime.Object - expectedErr error + name string + timeout time.Duration + fn resultFunc + expectedObj runtime.Object + expectedErr error + expectedPanic string }{ { - // Expected obj is returned. + name: "Expected obj is returned", timeout: time.Second, fn: func() (runtime.Object, error) { return exampleObj, nil @@ -850,7 +853,7 @@ func TestFinishRequest(t *testing.T) { expectedErr: nil, }, { - // Expected error is returned. + name: "Expected error is returned", timeout: time.Second, fn: func() (runtime.Object, error) { return nil, exampleErr @@ -859,7 +862,7 @@ func TestFinishRequest(t *testing.T) { expectedErr: exampleErr, }, { - // Successful status object is returned as expected. + name: "Successful status object is returned as expected", timeout: time.Second, fn: func() (runtime.Object, error) { return successStatusObj, nil @@ -868,7 +871,7 @@ func TestFinishRequest(t *testing.T) { expectedErr: nil, }, { - // Error status object is converted to StatusError. + name: "Error status object is converted to StatusError", timeout: time.Second, fn: func() (runtime.Object, error) { return errorStatusObj, nil @@ -876,15 +879,50 @@ func TestFinishRequest(t *testing.T) { expectedObj: nil, expectedErr: apierrors.FromObject(errorStatusObj), }, + { + name: "Panic is propagated up", + timeout: time.Second, + fn: func() (runtime.Object, error) { + panic("my panic") + return nil, nil + }, + expectedObj: nil, + expectedErr: nil, + expectedPanic: "my panic", + }, + { + name: "Panic is propagated with stack", + timeout: time.Second, + fn: func() (runtime.Object, error) { + panic("my panic") + return nil, nil + }, + expectedObj: nil, + expectedErr: nil, + expectedPanic: "rest_test.go", + }, } for i, tc := range testcases { - obj, err := finishRequest(tc.timeout, tc.fn) - if (err == nil && tc.expectedErr != nil) || (err != nil && tc.expectedErr == nil) || (err != nil && tc.expectedErr != nil && err.Error() != tc.expectedErr.Error()) { - t.Errorf("%d: unexpected err. expected: %v, got: %v", i, tc.expectedErr, err) - } - if !apiequality.Semantic.DeepEqual(obj, tc.expectedObj) { - t.Errorf("%d: unexpected obj. expected %#v, got %#v", i, tc.expectedObj, obj) - } + t.Run(tc.name, func(t *testing.T) { + defer func() { + r := recover() + switch { + case r == nil && len(tc.expectedPanic) > 0: + t.Errorf("expected panic containing '%s', got none", tc.expectedPanic) + case r != nil && len(tc.expectedPanic) == 0: + t.Errorf("unexpected panic: %v", r) + case r != nil && len(tc.expectedPanic) > 0 && !strings.Contains(fmt.Sprintf("%v", r), tc.expectedPanic): + t.Errorf("expected panic containing '%s', got '%v'", tc.expectedPanic, r) + } + }() + obj, err := finishRequest(tc.timeout, tc.fn) + if (err == nil && tc.expectedErr != nil) || (err != nil && tc.expectedErr == nil) || (err != nil && tc.expectedErr != nil && err.Error() != tc.expectedErr.Error()) { + t.Errorf("%d: unexpected err. expected: %v, got: %v", i, tc.expectedErr, err) + } + if !apiequality.Semantic.DeepEqual(obj, tc.expectedObj) { + t.Errorf("%d: unexpected obj. expected %#v, got %#v", i, tc.expectedObj, obj) + } + }) } } diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go index 0b18d992696..f387403dce0 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go @@ -39,10 +39,6 @@ import ( "k8s.io/apiserver/pkg/endpoints/metrics" "k8s.io/apiserver/pkg/registry/rest" genericfilters "k8s.io/apiserver/pkg/server/filters" - utilopenapi "k8s.io/apiserver/pkg/util/openapi" - openapibuilder "k8s.io/kube-openapi/pkg/builder" - openapiutil "k8s.io/kube-openapi/pkg/util" - openapiproto "k8s.io/kube-openapi/pkg/util/proto" ) const ( @@ -135,17 +131,17 @@ func (a *APIInstaller) newWebService() *restful.WebService { return ws } -// getResourceKind returns the external group version kind registered for the given storage +// GetResourceKind returns the external group version kind registered for the given storage // object. If the storage object is a subresource and has an override supplied for it, it returns // the group version kind supplied in the override. -func (a *APIInstaller) getResourceKind(path string, storage rest.Storage) (schema.GroupVersionKind, error) { +func GetResourceKind(groupVersion schema.GroupVersion, storage rest.Storage, typer runtime.ObjectTyper) (schema.GroupVersionKind, error) { // Let the storage tell us exactly what GVK it has if gvkProvider, ok := storage.(rest.GroupVersionKindProvider); ok { - return gvkProvider.GroupVersionKind(a.group.GroupVersion), nil + return gvkProvider.GroupVersionKind(groupVersion), nil } object := storage.New() - fqKinds, _, err := a.group.Typer.ObjectKinds(object) + fqKinds, _, err := typer.ObjectKinds(object) if err != nil { return schema.GroupVersionKind{}, err } @@ -154,13 +150,13 @@ func (a *APIInstaller) getResourceKind(path string, storage rest.Storage) (schem // we're trying to register here fqKindToRegister := schema.GroupVersionKind{} for _, fqKind := range fqKinds { - if fqKind.Group == a.group.GroupVersion.Group { - fqKindToRegister = a.group.GroupVersion.WithKind(fqKind.Kind) + if fqKind.Group == groupVersion.Group { + fqKindToRegister = groupVersion.WithKind(fqKind.Kind) break } } if fqKindToRegister.Empty() { - return schema.GroupVersionKind{}, fmt.Errorf("unable to locate fully qualified kind for %v: found %v when registering for %v", reflect.TypeOf(object), fqKinds, a.group.GroupVersion) + return schema.GroupVersionKind{}, fmt.Errorf("unable to locate fully qualified kind for %v: found %v when registering for %v", reflect.TypeOf(object), fqKinds, groupVersion) } // group is guaranteed to match based on the check above @@ -180,7 +176,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag return nil, err } - fqKindToRegister, err := a.getResourceKind(path, storage) + fqKindToRegister, err := GetResourceKind(a.group.GroupVersion, storage, a.group.Typer) if err != nil { return nil, err } @@ -513,10 +509,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if a.group.MetaGroupVersion != nil { reqScope.MetaGroupVersion = *a.group.MetaGroupVersion } - reqScope.OpenAPISchema, err = a.getOpenAPISchema(ws.RootPath(), resource, fqKindToRegister, defaultVersionedObject) - if err != nil { - return nil, fmt.Errorf("unable to get openapi schema for %v: %v", fqKindToRegister, err) - } + reqScope.OpenAPIModels = a.group.OpenAPIModels for _, action := range actions { producedObject := storageMeta.ProducesObject(action.Verb) if producedObject == nil { @@ -558,7 +551,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag return nil, fmt.Errorf("missing parent storage: %q", resource) } - fqParentKind, err := a.getResourceKind(resource, parentStorage) + fqParentKind, err := GetResourceKind(a.group.GroupVersion, parentStorage, a.group.Typer) if err != nil { return nil, err } @@ -873,24 +866,6 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag return &apiResource, nil } -// getOpenAPISchema builds the openapi schema for a single resource model to be given to each handler. It will -// return nil if the apiserver doesn't have openapi enabled, or if the specific path should be ignored by openapi. -func (a *APIInstaller) getOpenAPISchema(rootPath, resource string, kind schema.GroupVersionKind, sampleObject interface{}) (openapiproto.Schema, error) { - path := gpath.Join(rootPath, resource) - if a.group.OpenAPIConfig == nil { - return nil, nil - } - pathsToIgnore := openapiutil.NewTrie(a.group.OpenAPIConfig.IgnorePrefixes) - if pathsToIgnore.HasPrefix(path) { - return nil, nil - } - openAPIDefinitions, err := openapibuilder.BuildOpenAPIDefinitionsForResource(sampleObject, a.group.OpenAPIConfig) - if err != nil { - return nil, err - } - return utilopenapi.ToProtoSchema(openAPIDefinitions, kind) -} - // indirectArbitraryPointer returns *ptrToObject for an arbitrary pointer func indirectArbitraryPointer(ptrToObject interface{}) interface{} { return reflect.Indirect(reflect.ValueOf(ptrToObject)).Interface() diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/request/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/request/BUILD index 5e84006a1cf..388c1846e3a 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/request/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/request/BUILD @@ -36,7 +36,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/request/requestinfo.go b/staging/src/k8s.io/apiserver/pkg/endpoints/request/requestinfo.go index 1520bb3c9e5..cc8ae39fa2c 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/request/requestinfo.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/request/requestinfo.go @@ -27,7 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" - "github.com/golang/glog" + "k8s.io/klog" ) // LongRunningRequestCheck is a predicate which is true for long-running http requests. @@ -210,7 +210,7 @@ func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, er opts := metainternalversion.ListOptions{} if err := metainternalversion.ParameterCodec.DecodeParameters(req.URL.Query(), metav1.SchemeGroupVersion, &opts); err != nil { // An error in parsing request will result in default to "list" and not setting "name" field. - glog.Errorf("Couldn't parse request %#v: %v", req.URL.Query(), err) + klog.Errorf("Couldn't parse request %#v: %v", req.URL.Query(), err) // Reset opts to not rely on partial results from parsing. // However, if watch is set, let's report it. opts = metainternalversion.ListOptions{} diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index d5323ca7622..92418256814 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -52,6 +52,13 @@ const ( // audited. AdvancedAuditing utilfeature.Feature = "AdvancedAuditing" + // owner: @pbarker + // alpha: v1.13 + // + // DynamicAuditing enables configuration of audit policy and webhook backends through an + // AuditSink API object. + DynamicAuditing utilfeature.Feature = "DynamicAuditing" + // owner: @ilackams // alpha: v1.7 // @@ -94,6 +101,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS StreamingProxyRedirects: {Default: true, PreRelease: utilfeature.Beta}, ValidateProxyRedirects: {Default: false, PreRelease: utilfeature.Alpha}, AdvancedAuditing: {Default: true, PreRelease: utilfeature.GA}, + DynamicAuditing: {Default: false, PreRelease: utilfeature.Alpha}, APIResponseCompression: {Default: false, PreRelease: utilfeature.Alpha}, Initializers: {Default: false, PreRelease: utilfeature.Alpha}, APIListChunking: {Default: true, PreRelease: utilfeature.Beta}, diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/BUILD b/staging/src/k8s.io/apiserver/pkg/registry/generic/BUILD index 87b78db56c5..9f23d1223a9 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/BUILD @@ -23,7 +23,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/storage:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/BUILD b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/BUILD index 8281a8e5a1b..69db1062085 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/BUILD @@ -89,7 +89,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/dryrun:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/storage_factory.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/storage_factory.go index fc93cc4d25b..45524750705 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/storage_factory.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/storage_factory.go @@ -19,7 +19,7 @@ package registry import ( "sync" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic" @@ -43,10 +43,10 @@ func StorageWithCacher(capacity int) generic.StorageDecorator { s, d := generic.NewRawStorage(storageConfig) if capacity == 0 { - glog.V(5).Infof("Storage caching is disabled for %T", objectType) + klog.V(5).Infof("Storage caching is disabled for %T", objectType) return s, d } - glog.V(5).Infof("Storage caching is enabled for %T with capacity %v", objectType, capacity) + klog.V(5).Infof("Storage caching is enabled for %T with capacity %v", objectType, capacity) // TODO: we would change this later to make storage always have cacher and hide low level KV layer inside. // Currently it has two layers of same storage interface -- cacher and low level kv. diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go index 615637d8dee..2dcf99eae52 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go @@ -47,7 +47,7 @@ import ( "k8s.io/apiserver/pkg/storage/etcd/metrics" "k8s.io/apiserver/pkg/util/dryrun" - "github.com/golang/glog" + "k8s.io/klog" ) // ObjectFunc is a function to act on a given object. An error may be returned @@ -501,7 +501,7 @@ func (e *Store) shouldDeleteForFailedInitialization(ctx context.Context, obj run // Used for objects that are either been finalized or have never initialized. func (e *Store) deleteWithoutFinalizers(ctx context.Context, name, key string, obj runtime.Object, preconditions *storage.Preconditions, dryRun bool) (runtime.Object, bool, error) { out := e.NewFunc() - glog.V(6).Infof("going to delete %s from registry, triggered by update", name) + klog.V(6).Infof("going to delete %s from registry, triggered by update", name) if err := e.Storage.Delete(ctx, key, out, preconditions, dryRun); err != nil { // Deletion is racy, i.e., there could be multiple update // requests to remove all finalizers from the object, so we @@ -909,7 +909,7 @@ func (e *Store) updateForGracefulDeletionAndFinalizers(ctx context.Context, name if !graceful { // set the DeleteGracePeriods to 0 if the object has pendingFinalizers but not supporting graceful deletion if pendingFinalizers { - glog.V(6).Infof("update the DeletionTimestamp to \"now\" and GracePeriodSeconds to 0 for object %s, because it has pending finalizers", name) + klog.V(6).Infof("update the DeletionTimestamp to \"now\" and GracePeriodSeconds to 0 for object %s, because it has pending finalizers", name) err = markAsDeleting(existing) if err != nil { return nil, err @@ -1017,7 +1017,7 @@ func (e *Store) Delete(ctx context.Context, name string, options *metav1.DeleteO } // delete immediately, or no graceful deletion supported - glog.V(6).Infof("going to delete %s from registry: ", name) + klog.V(6).Infof("going to delete %s from registry: ", name) out = e.NewFunc() if err := e.Storage.Delete(ctx, key, out, &preconditions, dryrun.IsDryRun(options.DryRun)); err != nil { // Please refer to the place where we set ignoreNotFound for the reason @@ -1103,7 +1103,7 @@ func (e *Store) DeleteCollection(ctx context.Context, options *metav1.DeleteOpti return } if _, _, err := e.Delete(ctx, accessor.GetName(), options); err != nil && !kubeerr.IsNotFound(err) { - glog.V(4).Infof("Delete %s in DeleteCollection failed: %v", accessor.GetName(), err) + klog.V(4).Infof("Delete %s in DeleteCollection failed: %v", accessor.GetName(), err) errs <- err return } @@ -1246,7 +1246,7 @@ func (e *Store) Export(ctx context.Context, name string, opts metav1.ExportOptio if accessor, err := meta.Accessor(obj); err == nil { exportObjectMeta(accessor, opts.Exact) } else { - glog.V(4).Infof("Object of type %v does not have ObjectMeta: %v", reflect.TypeOf(obj), err) + klog.V(4).Infof("Object of type %v does not have ObjectMeta: %v", reflect.TypeOf(obj), err) } if e.ExportStrategy != nil { @@ -1411,12 +1411,12 @@ func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error { func (e *Store) startObservingCount(period time.Duration) func() { prefix := e.KeyRootFunc(genericapirequest.NewContext()) resourceName := e.DefaultQualifiedResource.String() - glog.V(2).Infof("Monitoring %v count at /%v", resourceName, prefix) + klog.V(2).Infof("Monitoring %v count at /%v", resourceName, prefix) stopCh := make(chan struct{}) go wait.JitterUntil(func() { count, err := e.Storage.Count(prefix) if err != nil { - glog.V(5).Infof("Failed to update storage count metric: %v", err) + klog.V(5).Infof("Failed to update storage count metric: %v", err) metrics.UpdateObjectCount(resourceName, -1) } else { metrics.UpdateObjectCount(resourceName, count) diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/storage_decorator.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/storage_decorator.go index 94a4794422c..858ad922af4 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/storage_decorator.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/storage_decorator.go @@ -17,11 +17,11 @@ limitations under the License. package generic import ( - "github.com/golang/glog" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/apiserver/pkg/storage/storagebackend/factory" + "k8s.io/klog" ) // StorageDecorator is a function signature for producing a storage.Interface @@ -54,7 +54,7 @@ func UndecoratedStorage( func NewRawStorage(config *storagebackend.Config) (storage.Interface, factory.DestroyFunc) { s, d, err := factory.Create(*config) if err != nil { - glog.Fatalf("Unable to create storage backend: config (%v), err (%v)", config, err) + klog.Fatalf("Unable to create storage backend: config (%v), err (%v)", config, err) } return s, d } diff --git a/staging/src/k8s.io/apiserver/pkg/server/BUILD b/staging/src/k8s.io/apiserver/pkg/server/BUILD index c71377a41ed..68588c5a83c 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/BUILD @@ -103,6 +103,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/openapi:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", @@ -110,10 +111,13 @@ go_library( "//vendor/github.com/emicklei/go-restful:go_default_library", "//vendor/github.com/emicklei/go-restful-swagger12:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pborman/uuid:go_default_library", "//vendor/golang.org/x/net/http2:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/k8s.io/kube-openapi/pkg/builder:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", + "//vendor/k8s.io/kube-openapi/pkg/util:go_default_library", + "//vendor/k8s.io/kube-openapi/pkg/util/proto:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index 10621f84f60..4d6f90891fc 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -30,8 +30,8 @@ import ( "github.com/emicklei/go-restful-swagger12" "github.com/go-openapi/spec" - "github.com/golang/glog" "github.com/pborman/uuid" + "k8s.io/klog" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -358,11 +358,11 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo // if there is no port, and we listen on one securely, use that one if _, _, err := net.SplitHostPort(c.ExternalAddress); err != nil { if c.SecureServing == nil { - glog.Fatalf("cannot derive external address port without listening on a secure port.") + klog.Fatalf("cannot derive external address port without listening on a secure port.") } _, port, err := c.SecureServing.HostPort() if err != nil { - glog.Fatalf("cannot derive external address from the secure port: %v", err) + klog.Fatalf("cannot derive external address from the secure port: %v", err) } c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port)) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/deprecated_insecure_serving.go b/staging/src/k8s.io/apiserver/pkg/server/deprecated_insecure_serving.go index 6cf6c1a64fc..a78250edae9 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/deprecated_insecure_serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/deprecated_insecure_serving.go @@ -21,7 +21,7 @@ import ( "net/http" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" @@ -46,9 +46,9 @@ func (s *DeprecatedInsecureServingInfo) Serve(handler http.Handler, shutdownTime } if len(s.Name) > 0 { - glog.Infof("Serving %s insecurely on %s", s.Name, s.Listener.Addr()) + klog.Infof("Serving %s insecurely on %s", s.Name, s.Listener.Addr()) } else { - glog.Infof("Serving insecurely on %s", s.Listener.Addr()) + klog.Infof("Serving insecurely on %s", s.Listener.Addr()) } return RunServer(insecureServer, s.Listener, shutdownTimeout, stopCh) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD b/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD index 900c24b656b..b75b3de9824 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD @@ -54,7 +54,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/httplog:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/cors.go b/staging/src/k8s.io/apiserver/pkg/server/filters/cors.go index 2c6e66ed6ba..96ff58dc7c8 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/cors.go +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/cors.go @@ -21,7 +21,7 @@ import ( "regexp" "strings" - "github.com/golang/glog" + "k8s.io/klog" ) // TODO: use restful.CrossOriginResourceSharing @@ -79,7 +79,7 @@ func WithCORS(handler http.Handler, allowedOriginPatterns []string, allowedMetho func allowedOriginRegexps(allowedOrigins []string) []*regexp.Regexp { res, err := compileRegexps(allowedOrigins) if err != nil { - glog.Fatalf("Invalid CORS allowed origin, --cors-allowed-origins flag was set to %v - %v", strings.Join(allowedOrigins, ","), err) + klog.Fatalf("Invalid CORS allowed origin, --cors-allowed-origins flag was set to %v - %v", strings.Join(allowedOrigins, ","), err) } return res } diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/maxinflight.go b/staging/src/k8s.io/apiserver/pkg/server/filters/maxinflight.go index 4f56e48a03a..8818cb5633f 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/maxinflight.go +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/maxinflight.go @@ -28,7 +28,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/metrics" apirequest "k8s.io/apiserver/pkg/endpoints/request" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -47,7 +47,7 @@ var nonMutatingRequestVerbs = sets.NewString("get", "list", "watch") func handleError(w http.ResponseWriter, r *http.Request, err error) { errorMsg := fmt.Sprintf("Internal Server Error: %#v", r.RequestURI) http.Error(w, errorMsg, http.StatusInternalServerError) - glog.Errorf(err.Error()) + klog.Errorf(err.Error()) } // requestWatermark is used to trak maximal usage of inflight requests. diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/timeout.go b/staging/src/k8s.io/apiserver/pkg/server/filters/timeout.go index 7d37ed05a3d..adb179f8235 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/timeout.go +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/timeout.go @@ -23,6 +23,7 @@ import ( "fmt" "net" "net/http" + "runtime" "sync" "time" @@ -91,16 +92,23 @@ func (t *timeoutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - result := make(chan interface{}) + errCh := make(chan interface{}) tw := newTimeoutWriter(w) go func() { defer func() { - result <- recover() + err := recover() + if err != nil { + const size = 64 << 10 + buf := make([]byte, size) + buf = buf[:runtime.Stack(buf, false)] + err = fmt.Sprintf("%v\n%s", err, buf) + } + errCh <- err }() t.handler.ServeHTTP(tw, r) }() select { - case err := <-result: + case err := <-errCh: if err != nil { panic(err) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/timeout_test.go b/staging/src/k8s.io/apiserver/pkg/server/filters/timeout_test.go index 28403f93509..26f571d08af 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/timeout_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/timeout_test.go @@ -18,10 +18,12 @@ package filters import ( "encoding/json" + "fmt" "io/ioutil" "net/http" "net/http/httptest" "reflect" + "strings" "sync" "testing" "time" @@ -50,6 +52,18 @@ func (r *recorder) Count() int { return r.count } +func newHandler(responseCh <-chan string, panicCh <-chan struct{}, writeErrCh chan<- error) http.HandlerFunc { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case resp := <-responseCh: + _, err := w.Write([]byte(resp)) + writeErrCh <- err + case <-panicCh: + panic("inner handler panics") + } + }) +} + func TestTimeout(t *testing.T) { origReallyCrash := runtime.ReallyCrash runtime.ReallyCrash = false @@ -57,31 +71,28 @@ func TestTimeout(t *testing.T) { runtime.ReallyCrash = origReallyCrash }() - sendResponse := make(chan struct{}, 1) + sendResponse := make(chan string, 1) doPanic := make(chan struct{}, 1) writeErrors := make(chan error, 1) + gotPanic := make(chan interface{}, 1) timeout := make(chan time.Time, 1) resp := "test response" timeoutErr := apierrors.NewServerTimeout(schema.GroupResource{Group: "foo", Resource: "bar"}, "get", 0) record := &recorder{} - ts := httptest.NewServer(WithPanicRecovery(WithTimeout(http.HandlerFunc( - func(w http.ResponseWriter, r *http.Request) { - select { - case <-sendResponse: - _, err := w.Write([]byte(resp)) - writeErrors <- err - case <-doPanic: - panic("inner handler panics") - } - }), - func(req *http.Request) (*http.Request, <-chan time.Time, func(), *apierrors.StatusError) { + handler := newHandler(sendResponse, doPanic, writeErrors) + ts := httptest.NewServer(withPanicRecovery( + WithTimeout(handler, func(req *http.Request) (*http.Request, <-chan time.Time, func(), *apierrors.StatusError) { return req, timeout, record.Record, timeoutErr - }))) + }), func(w http.ResponseWriter, req *http.Request, err interface{}) { + gotPanic <- err + http.Error(w, "This request caused apiserver to panic. Look in the logs for details.", http.StatusInternalServerError) + }), + ) defer ts.Close() // No timeouts - sendResponse <- struct{}{} + sendResponse <- resp res, err := http.Get(ts.URL) if err != nil { t.Fatal(err) @@ -122,7 +133,7 @@ func TestTimeout(t *testing.T) { } // Now try to send a response - sendResponse <- struct{}{} + sendResponse <- resp if err := <-writeErrors; err != http.ErrHandlerTimeout { t.Errorf("got Write error of %v; expected %v", err, http.ErrHandlerTimeout) } @@ -136,4 +147,13 @@ func TestTimeout(t *testing.T) { if res.StatusCode != http.StatusInternalServerError { t.Errorf("got res.StatusCode %d; expected %d due to panic", res.StatusCode, http.StatusInternalServerError) } + select { + case err := <-gotPanic: + msg := fmt.Sprintf("%v", err) + if !strings.Contains(msg, "newHandler") { + t.Errorf("expected line with root cause panic in the stack trace, but didn't: %v", err) + } + case <-time.After(30 * time.Second): + t.Fatalf("expected to see a handler panic, but didn't") + } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/wrap.go b/staging/src/k8s.io/apiserver/pkg/server/filters/wrap.go index 38742ffd9a2..0a75845611d 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/wrap.go +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/wrap.go @@ -18,9 +18,8 @@ package filters import ( "net/http" - "runtime/debug" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/server/httplog" @@ -28,10 +27,16 @@ import ( // WithPanicRecovery wraps an http Handler to recover and log panics. func WithPanicRecovery(handler http.Handler) http.Handler { + return withPanicRecovery(handler, func(w http.ResponseWriter, req *http.Request, err interface{}) { + http.Error(w, "This request caused apiserver to panic. Look in the logs for details.", http.StatusInternalServerError) + klog.Errorf("apiserver panic'd on %v %v", req.Method, req.RequestURI) + }) +} + +func withPanicRecovery(handler http.Handler, crashHandler func(http.ResponseWriter, *http.Request, interface{})) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { defer runtime.HandleCrash(func(err interface{}) { - http.Error(w, "This request caused apiserver to panic. Look in the logs for details.", http.StatusInternalServerError) - glog.Errorf("apiserver panic'd on %v %v: %v\n%s\n", req.Method, req.RequestURI, err, debug.Stack()) + crashHandler(w, req, err) }) logger := httplog.NewLogged(req, &w) diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index b6e500c6123..4efc82427e2 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -19,13 +19,14 @@ package server import ( "fmt" "net/http" + gpath "path" "strings" "sync" "time" systemd "github.com/coreos/go-systemd/daemon" "github.com/emicklei/go-restful-swagger12" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -42,8 +43,12 @@ import ( "k8s.io/apiserver/pkg/registry/rest" "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/routes" + utilopenapi "k8s.io/apiserver/pkg/util/openapi" restclient "k8s.io/client-go/rest" + openapibuilder "k8s.io/kube-openapi/pkg/builder" openapicommon "k8s.io/kube-openapi/pkg/common" + openapiutil "k8s.io/kube-openapi/pkg/util" + openapiproto "k8s.io/kube-openapi/pkg/util/proto" ) // Info about an API group. @@ -312,7 +317,7 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error { s.RunPostStartHooks(stopCh) if _, err := systemd.SdNotify(true, "READY=1\n"); err != nil { - glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) + klog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) } return nil @@ -320,9 +325,13 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error { // installAPIResources is a private method for installing the REST storage backing each api groupversionresource func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo) error { + openAPIGroupModels, err := s.getOpenAPIModelsForGroup(apiPrefix, apiGroupInfo) + if err != nil { + return fmt.Errorf("unable to get openapi models for group %v: %v", apiPrefix, err) + } for _, groupVersion := range apiGroupInfo.PrioritizedVersions { if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 { - glog.Warningf("Skipping API %v because it has no resources.", groupVersion) + klog.Warningf("Skipping API %v because it has no resources.", groupVersion) continue } @@ -330,6 +339,7 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A if apiGroupInfo.OptionsExternalVersion != nil { apiGroupVersion.OptionsExternalVersion = apiGroupInfo.OptionsExternalVersion } + apiGroupVersion.OpenAPIModels = openAPIGroupModels if err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer); err != nil { return fmt.Errorf("unable to setup API %v: %v", apiGroupInfo, err) @@ -427,7 +437,6 @@ func (s *GenericAPIServer) newAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupV Admit: s.admissionControl, MinRequestTimeout: s.minRequestTimeout, EnableAPIResponseCompression: s.enableAPIResponseCompression, - OpenAPIConfig: s.openAPIConfig, Authorizer: s.Authorizer, } } @@ -445,3 +454,37 @@ func NewDefaultAPIGroupInfo(group string, scheme *runtime.Scheme, parameterCodec NegotiatedSerializer: codecs, } } + +// getOpenAPIModelsForGroup is a private method for getting the OpenAPI Schemas for each api group +func (s *GenericAPIServer) getOpenAPIModelsForGroup(apiPrefix string, apiGroupInfo *APIGroupInfo) (openapiproto.Models, error) { + if s.openAPIConfig == nil { + return nil, nil + } + pathsToIgnore := openapiutil.NewTrie(s.openAPIConfig.IgnorePrefixes) + // Get the canonical names of every resource we need to build in this api group + resourceNames := make([]string, 0) + for _, groupVersion := range apiGroupInfo.PrioritizedVersions { + for resource, storage := range apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version] { + path := gpath.Join(apiPrefix, groupVersion.Group, groupVersion.Version, resource) + if !pathsToIgnore.HasPrefix(path) { + kind, err := genericapi.GetResourceKind(groupVersion, storage, apiGroupInfo.Scheme) + if err != nil { + return nil, err + } + sampleObject, err := apiGroupInfo.Scheme.New(kind) + if err != nil { + return nil, err + } + name := openapiutil.GetCanonicalTypeName(sampleObject) + resourceNames = append(resourceNames, name) + } + } + } + + // Build the openapi definitions for those resources and convert it to proto models + openAPISpec, err := openapibuilder.BuildOpenAPIDefinitionsForResources(s.openAPIConfig, resourceNames...) + if err != nil { + return nil, err + } + return utilopenapi.ToProtoModels(openAPISpec) +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/handler.go b/staging/src/k8s.io/apiserver/pkg/server/handler.go index e4e7d9aee06..0277bac7788 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/handler.go +++ b/staging/src/k8s.io/apiserver/pkg/server/handler.go @@ -25,7 +25,7 @@ import ( "strings" "github.com/emicklei/go-restful" - "github.com/golang/glog" + "k8s.io/klog" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -130,7 +130,7 @@ func (d director) ServeHTTP(w http.ResponseWriter, req *http.Request) { // normally these are passed to the nonGoRestfulMux, but if discovery is enabled, it will go directly. // We can't rely on a prefix match since /apis matches everything (see the big comment on Director above) if path == "/apis" || path == "/apis/" { - glog.V(5).Infof("%v: %v %q satisfied by gorestful with webservice %v", d.name, req.Method, path, ws.RootPath()) + klog.V(5).Infof("%v: %v %q satisfied by gorestful with webservice %v", d.name, req.Method, path, ws.RootPath()) // don't use servemux here because gorestful servemuxes get messed up when removing webservices // TODO fix gorestful, remove TPRs, or stop using gorestful d.goRestfulContainer.Dispatch(w, req) @@ -140,7 +140,7 @@ func (d director) ServeHTTP(w http.ResponseWriter, req *http.Request) { case strings.HasPrefix(path, ws.RootPath()): // ensure an exact match or a path boundary match if len(path) == len(ws.RootPath()) || path[len(ws.RootPath())] == '/' { - glog.V(5).Infof("%v: %v %q satisfied by gorestful with webservice %v", d.name, req.Method, path, ws.RootPath()) + klog.V(5).Infof("%v: %v %q satisfied by gorestful with webservice %v", d.name, req.Method, path, ws.RootPath()) // don't use servemux here because gorestful servemuxes get messed up when removing webservices // TODO fix gorestful, remove TPRs, or stop using gorestful d.goRestfulContainer.Dispatch(w, req) @@ -150,7 +150,7 @@ func (d director) ServeHTTP(w http.ResponseWriter, req *http.Request) { } // if we didn't find a match, then we just skip gorestful altogether - glog.V(5).Infof("%v: %v %q satisfied by nonGoRestful", d.name, req.Method, path) + klog.V(5).Infof("%v: %v %q satisfied by nonGoRestful", d.name, req.Method, path) d.nonGoRestfulMux.ServeHTTP(w, req) } @@ -165,7 +165,7 @@ func logStackOnRecover(s runtime.NegotiatedSerializer, panicReason interface{}, } buffer.WriteString(fmt.Sprintf(" %s:%d\r\n", file, line)) } - glog.Errorln(buffer.String()) + klog.Errorln(buffer.String()) headers := http.Header{} if ct := w.Header().Get("Content-Type"); len(ct) > 0 { diff --git a/staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD b/staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD index aab24c34ef1..7907d0e6111 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD @@ -10,6 +10,9 @@ go_test( name = "go_default_test", srcs = ["healthz_test.go"], embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + ], ) go_library( @@ -21,8 +24,9 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/healthz", importpath = "k8s.io/apiserver/pkg/server/healthz", deps = [ + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go index 24fafcefa0f..17d85fbe637 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go +++ b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go @@ -25,8 +25,9 @@ import ( "sync/atomic" "time" - "github.com/golang/glog" + "k8s.io/klog" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" ) @@ -76,7 +77,7 @@ func (l *log) Check(_ *http.Request) error { l.startOnce.Do(func() { l.lastVerified.Store(time.Now()) go wait.Forever(func() { - glog.Flush() + klog.Flush() l.lastVerified.Store(time.Now()) }, time.Minute) }) @@ -108,11 +109,11 @@ func InstallHandler(mux mux, checks ...HealthzChecker) { // result in a panic. func InstallPathHandler(mux mux, path string, checks ...HealthzChecker) { if len(checks) == 0 { - glog.V(5).Info("No default health checks specified. Installing the ping handler.") + klog.V(5).Info("No default health checks specified. Installing the ping handler.") checks = []HealthzChecker{PingHealthz} } - glog.V(5).Info("Installing healthz checkers:", strings.Join(checkerNames(checks...), ", ")) + klog.V(5).Info("Installing healthz checkers:", formatQuoted(checkerNames(checks...)...)) mux.Handle(path, handleRootHealthz(checks...)) for _, check := range checks { @@ -141,22 +142,43 @@ func (c *healthzCheck) Check(r *http.Request) error { return c.check(r) } +// getExcludedChecks extracts the health check names to be excluded from the query param +func getExcludedChecks(r *http.Request) sets.String { + checks, found := r.URL.Query()["exclude"] + if found { + return sets.NewString(checks...) + } + return sets.NewString() +} + // handleRootHealthz returns an http.HandlerFunc that serves the provided checks. func handleRootHealthz(checks ...HealthzChecker) http.HandlerFunc { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { failed := false + excluded := getExcludedChecks(r) var verboseOut bytes.Buffer for _, check := range checks { + // no-op the check if we've specified we want to exclude the check + if excluded.Has(check.Name()) { + excluded.Delete(check.Name()) + fmt.Fprintf(&verboseOut, "[+]%v excluded: ok\n", check.Name()) + continue + } if err := check.Check(r); err != nil { // don't include the error since this endpoint is public. If someone wants more detail // they should have explicit permission to the detailed checks. - glog.V(6).Infof("healthz check %v failed: %v", check.Name(), err) + klog.V(6).Infof("healthz check %v failed: %v", check.Name(), err) fmt.Fprintf(&verboseOut, "[-]%v failed: reason withheld\n", check.Name()) failed = true } else { fmt.Fprintf(&verboseOut, "[+]%v ok\n", check.Name()) } } + if excluded.Len() > 0 { + fmt.Fprintf(&verboseOut, "warn: some health checks cannot be excluded: no matches for %v\n", formatQuoted(excluded.List()...)) + klog.Warningf("cannot exclude some health checks, no health checks are installed matching %v", + formatQuoted(excluded.List()...)) + } // always be verbose on failure if failed { http.Error(w, fmt.Sprintf("%vhealthz check failed", verboseOut.String()), http.StatusInternalServerError) @@ -187,14 +209,20 @@ func adaptCheckToHandler(c func(r *http.Request) error) http.HandlerFunc { // checkerNames returns the names of the checks in the same order as passed in. func checkerNames(checks ...HealthzChecker) []string { - if len(checks) > 0 { - // accumulate the names of checks for printing them out. - checkerNames := make([]string, 0, len(checks)) - for _, check := range checks { - // quote the Name so we can disambiguate - checkerNames = append(checkerNames, fmt.Sprintf("%q", check.Name())) - } - return checkerNames + // accumulate the names of checks for printing them out. + checkerNames := make([]string, 0, len(checks)) + for _, check := range checks { + checkerNames = append(checkerNames, check.Name()) } - return nil + return checkerNames +} + +// formatQuoted returns a formatted string of the health check names, +// preserving the order passed in. +func formatQuoted(names ...string) string { + quoted := make([]string, 0, len(names)) + for _, name := range names { + quoted = append(quoted, fmt.Sprintf("%q", name)) + } + return strings.Join(quoted, ",") } diff --git a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go index 7ae26c4e9ce..704532237b7 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go @@ -21,8 +21,11 @@ import ( "fmt" "net/http" "net/http/httptest" + "net/url" "reflect" "testing" + + "k8s.io/apimachinery/pkg/util/sets" ) func TestInstallHandler(t *testing.T) { @@ -82,6 +85,10 @@ func testMultipleChecks(path string, t *testing.T) { addBadCheck bool }{ {"?verbose", "[+]ping ok\nhealthz check passed\n", http.StatusOK, false}, + {"?exclude=dontexist", "ok", http.StatusOK, false}, + {"?exclude=bad", "ok", http.StatusOK, true}, + {"?verbose=true&exclude=bad", "[+]ping ok\n[+]bad excluded: ok\nhealthz check passed\n", http.StatusOK, true}, + {"?verbose=true&exclude=dontexist", "[+]ping ok\nwarn: some health checks cannot be excluded: no matches for \"dontexist\"\nhealthz check passed\n", http.StatusOK, false}, {"/ping", "ok", http.StatusOK, false}, {"", "ok", http.StatusOK, false}, {"?verbose", "[+]ping ok\n[-]bad failed: reason withheld\nhealthz check failed\n", http.StatusInternalServerError, true}, @@ -148,10 +155,72 @@ func TestCheckerNames(t *testing.T) { for _, tc := range testCases { result := checkerNames(tc.have...) t.Run(tc.desc, func(t *testing.T) { - if reflect.DeepEqual(tc.want, result) { + if !reflect.DeepEqual(tc.want, result) { t.Errorf("want %#v, got %#v", tc.want, result) } }) } - +} + +func TestFormatQuoted(t *testing.T) { + n1 := "n1" + n2 := "n2" + testCases := []struct { + desc string + names []string + expected string + }{ + {"empty", []string{}, ""}, + {"single name", []string{n1}, "\"n1\""}, + {"two names", []string{n1, n2}, "\"n1\",\"n2\""}, + {"two names, reverse order", []string{n2, n1}, "\"n2\",\"n1\""}, + } + for _, tc := range testCases { + result := formatQuoted(tc.names...) + t.Run(tc.desc, func(t *testing.T) { + if result != tc.expected { + t.Errorf("expected %#v, got %#v", tc.expected, result) + } + }) + } +} + +func TestGetExcludedChecks(t *testing.T) { + type args struct { + r *http.Request + } + tests := []struct { + name string + r *http.Request + want sets.String + }{ + {"Should have no excluded health checks", + createGetRequestWithUrl("/healthz?verbose=true"), + sets.NewString(), + }, + {"Should extract out the ping health check", + createGetRequestWithUrl("/healthz?exclude=ping"), + sets.NewString("ping"), + }, + {"Should extract out ping and log health check", + createGetRequestWithUrl("/healthz?exclude=ping&exclude=log"), + sets.NewString("ping", "log"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getExcludedChecks(tt.r); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getExcludedChecks() = %v, want %v", got, tt.want) + } + }) + } +} + +func createGetRequestWithUrl(rawUrlString string) *http.Request { + url, _ := url.Parse(rawUrlString) + return &http.Request{ + Method: http.MethodGet, + Proto: "HTTP/1.1", + URL: url, + } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/hooks.go b/staging/src/k8s.io/apiserver/pkg/server/hooks.go index ccf8ee17ad0..921255218bc 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/hooks.go +++ b/staging/src/k8s.io/apiserver/pkg/server/hooks.go @@ -21,7 +21,7 @@ import ( "fmt" "net/http" - "github.com/golang/glog" + "k8s.io/klog" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -101,7 +101,7 @@ func (s *GenericAPIServer) AddPostStartHook(name string, hook PostStartHookFunc) // AddPostStartHookOrDie allows you to add a PostStartHook, but dies on failure func (s *GenericAPIServer) AddPostStartHookOrDie(name string, hook PostStartHookFunc) { if err := s.AddPostStartHook(name, hook); err != nil { - glog.Fatalf("Error registering PostStartHook %q: %v", name, err) + klog.Fatalf("Error registering PostStartHook %q: %v", name, err) } } @@ -132,7 +132,7 @@ func (s *GenericAPIServer) AddPreShutdownHook(name string, hook PreShutdownHookF // AddPreShutdownHookOrDie allows you to add a PostStartHook, but dies on failure func (s *GenericAPIServer) AddPreShutdownHookOrDie(name string, hook PreShutdownHookFunc) { if err := s.AddPreShutdownHook(name, hook); err != nil { - glog.Fatalf("Error registering PreShutdownHook %q: %v", name, err) + klog.Fatalf("Error registering PreShutdownHook %q: %v", name, err) } } @@ -185,7 +185,7 @@ func runPostStartHook(name string, entry postStartHookEntry, context PostStartHo }() // if the hook intentionally wants to kill server, let it. if err != nil { - glog.Fatalf("PostStartHook %q failed: %v", name, err) + klog.Fatalf("PostStartHook %q failed: %v", name, err) } close(entry.done) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/httplog/BUILD b/staging/src/k8s.io/apiserver/pkg/server/httplog/BUILD index 22d299ba555..9626af306f2 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/httplog/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/httplog/BUILD @@ -20,7 +20,7 @@ go_library( ], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/httplog", importpath = "k8s.io/apiserver/pkg/server/httplog", - deps = ["//vendor/github.com/golang/glog:go_default_library"], + deps = ["//vendor/k8s.io/klog:go_default_library"], ) filegroup( diff --git a/staging/src/k8s.io/apiserver/pkg/server/httplog/httplog.go b/staging/src/k8s.io/apiserver/pkg/server/httplog/httplog.go index f8a8a5307aa..dcdba69225d 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/httplog/httplog.go +++ b/staging/src/k8s.io/apiserver/pkg/server/httplog/httplog.go @@ -24,7 +24,7 @@ import ( "runtime" "time" - "github.com/golang/glog" + "k8s.io/klog" ) // StacktracePred returns true if a stacktrace should be logged for this status. @@ -61,7 +61,7 @@ type passthroughLogger struct{} // Addf logs info immediately. func (passthroughLogger) Addf(format string, data ...interface{}) { - glog.V(2).Info(fmt.Sprintf(format, data...)) + klog.V(2).Info(fmt.Sprintf(format, data...)) } // DefaultStacktracePred is the default implementation of StacktracePred. @@ -143,11 +143,11 @@ func (rl *respLogger) Addf(format string, data ...interface{}) { // Log is intended to be called once at the end of your request handler, via defer func (rl *respLogger) Log() { latency := time.Since(rl.startTime) - if glog.V(3) { + if klog.V(3) { if !rl.hijacked { - glog.InfoDepth(1, fmt.Sprintf("%s %s: (%v) %v%v%v [%s %s]", rl.req.Method, rl.req.RequestURI, latency, rl.status, rl.statusStack, rl.addedInfo, rl.req.UserAgent(), rl.req.RemoteAddr)) + klog.InfoDepth(1, fmt.Sprintf("%s %s: (%v) %v%v%v [%s %s]", rl.req.Method, rl.req.RequestURI, latency, rl.status, rl.statusStack, rl.addedInfo, rl.req.UserAgent(), rl.req.RemoteAddr)) } else { - glog.InfoDepth(1, fmt.Sprintf("%s %s: (%v) hijacked [%s %s]", rl.req.Method, rl.req.RequestURI, latency, rl.req.UserAgent(), rl.req.RemoteAddr)) + klog.InfoDepth(1, fmt.Sprintf("%s %s: (%v) hijacked [%s %s]", rl.req.Method, rl.req.RequestURI, latency, rl.req.UserAgent(), rl.req.RemoteAddr)) } } } @@ -173,8 +173,8 @@ func (rl *respLogger) Write(b []byte) (int, error) { func (rl *respLogger) Flush() { if flusher, ok := rl.w.(http.Flusher); ok { flusher.Flush() - } else if glog.V(2) { - glog.InfoDepth(1, fmt.Sprintf("Unable to convert %+v into http.Flusher", rl.w)) + } else if klog.V(2) { + klog.InfoDepth(1, fmt.Sprintf("Unable to convert %+v into http.Flusher", rl.w)) } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/mux/BUILD b/staging/src/k8s.io/apiserver/pkg/server/mux/BUILD index e60551c82db..dd150e0fc69 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/mux/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/mux/BUILD @@ -24,7 +24,7 @@ go_library( deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/server/mux/pathrecorder.go b/staging/src/k8s.io/apiserver/pkg/server/mux/pathrecorder.go index 2f0eb7aa5b2..16857cc8a6b 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/mux/pathrecorder.go +++ b/staging/src/k8s.io/apiserver/pkg/server/mux/pathrecorder.go @@ -25,7 +25,7 @@ import ( "sync" "sync/atomic" - "github.com/golang/glog" + "k8s.io/klog" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -237,20 +237,20 @@ func (m *PathRecorderMux) ServeHTTP(w http.ResponseWriter, r *http.Request) { // ServeHTTP makes it an http.Handler func (h *pathHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if exactHandler, ok := h.pathToHandler[r.URL.Path]; ok { - glog.V(5).Infof("%v: %q satisfied by exact match", h.muxName, r.URL.Path) + klog.V(5).Infof("%v: %q satisfied by exact match", h.muxName, r.URL.Path) exactHandler.ServeHTTP(w, r) return } for _, prefixHandler := range h.prefixHandlers { if strings.HasPrefix(r.URL.Path, prefixHandler.prefix) { - glog.V(5).Infof("%v: %q satisfied by prefix %v", h.muxName, r.URL.Path, prefixHandler.prefix) + klog.V(5).Infof("%v: %q satisfied by prefix %v", h.muxName, r.URL.Path, prefixHandler.prefix) prefixHandler.handler.ServeHTTP(w, r) return } } - glog.V(5).Infof("%v: %q satisfied by NotFoundHandler", h.muxName, r.URL.Path) + klog.V(5).Infof("%v: %q satisfied by NotFoundHandler", h.muxName, r.URL.Path) h.notFoundHandler.ServeHTTP(w, r) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD index 7516c6e75ba..4125ba1343b 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD @@ -12,11 +12,13 @@ go_library( "deprecated_insecure_serving.go", "doc.go", "etcd.go", + "events.go", "feature.go", "recommended.go", "server_run_options.go", "serving.go", "serving_with_loopback.go", + "webhook.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/options", importpath = "k8s.io/apiserver/pkg/server/options", @@ -61,19 +63,23 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/buffered:go_default_library", + "//staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic:go_default_library", + "//staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/log:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/truncate:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pborman/uuid:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/gopkg.in/natefinch/lumberjack.v2:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", ], ) @@ -92,6 +98,7 @@ go_test( data = glob(["testdata/**"]), embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", @@ -99,11 +106,17 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", "//staging/src/k8s.io/client-go/discovery:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api/v1:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/audit.go b/staging/src/k8s.io/apiserver/pkg/server/options/audit.go index 71476c229a0..f997c53081f 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/audit.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/audit.go @@ -23,21 +23,30 @@ import ( "strings" "time" - "github.com/golang/glog" "github.com/spf13/pflag" "gopkg.in/natefinch/lumberjack.v2" + "k8s.io/klog" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" auditv1alpha1 "k8s.io/apiserver/pkg/apis/audit/v1alpha1" auditv1beta1 "k8s.io/apiserver/pkg/apis/audit/v1beta1" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/audit/policy" + "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/server" + utilfeature "k8s.io/apiserver/pkg/util/feature" pluginbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered" + plugindynamic "k8s.io/apiserver/plugin/pkg/audit/dynamic" + pluginenforced "k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced" pluginlog "k8s.io/apiserver/plugin/pkg/audit/log" plugintruncate "k8s.io/apiserver/plugin/pkg/audit/truncate" pluginwebhook "k8s.io/apiserver/plugin/pkg/audit/webhook" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" + restclient "k8s.io/client-go/rest" ) const ( @@ -54,6 +63,9 @@ func appendBackend(existing, newBackend audit.Backend) audit.Backend { if existing == nil { return newBackend } + if newBackend == nil { + return existing + } return audit.Union(existing, newBackend) } @@ -65,6 +77,7 @@ type AuditOptions struct { // Plugin options LogOptions AuditLogOptions WebhookOptions AuditWebhookOptions + DynamicOptions AuditDynamicOptions } const ( @@ -129,6 +142,11 @@ type AuditWebhookOptions struct { GroupVersionString string } +type AuditDynamicOptions struct { + // Enabled tells whether the dynamic audit capability is enabled. + Enabled bool +} + func NewAuditOptions() *AuditOptions { return &AuditOptions{ WebhookOptions: AuditWebhookOptions{ @@ -137,9 +155,8 @@ func NewAuditOptions() *AuditOptions { Mode: ModeBatch, BatchConfig: defaultWebhookBatchConfig(), }, - TruncateOptions: NewAuditTruncateOptions(), - // TODO(audit): use v1 API in release 1.13 - GroupVersionString: "audit.k8s.io/v1beta1", + TruncateOptions: NewAuditTruncateOptions(), + GroupVersionString: "audit.k8s.io/v1", }, LogOptions: AuditLogOptions{ Format: pluginlog.FormatJson, @@ -147,9 +164,11 @@ func NewAuditOptions() *AuditOptions { Mode: ModeBlocking, BatchConfig: defaultLogBatchConfig(), }, - TruncateOptions: NewAuditTruncateOptions(), - // TODO(audit): use v1 API in release 1.13 - GroupVersionString: "audit.k8s.io/v1beta1", + TruncateOptions: NewAuditTruncateOptions(), + GroupVersionString: "audit.k8s.io/v1", + }, + DynamicOptions: AuditDynamicOptions{ + Enabled: false, }, } } @@ -173,6 +192,7 @@ func (o *AuditOptions) Validate() []error { var allErrors []error allErrors = append(allErrors, o.LogOptions.Validate()...) allErrors = append(allErrors, o.WebhookOptions.Validate()...) + allErrors = append(allErrors, o.DynamicOptions.Validate()...) return allErrors } @@ -252,44 +272,102 @@ func (o *AuditOptions) AddFlags(fs *pflag.FlagSet) { o.WebhookOptions.AddFlags(fs) o.WebhookOptions.BatchOptions.AddFlags(pluginwebhook.PluginName, fs) o.WebhookOptions.TruncateOptions.AddFlags(pluginwebhook.PluginName, fs) + o.DynamicOptions.AddFlags(fs) } -func (o *AuditOptions) ApplyTo(c *server.Config) error { +func (o *AuditOptions) ApplyTo( + c *server.Config, + kubeClientConfig *restclient.Config, + informers informers.SharedInformerFactory, + processInfo *ProcessInfo, + webhookOptions *WebhookOptions, +) error { if o == nil { return nil } + if c == nil { + return fmt.Errorf("server config must be non-nil") + } - // Apply advanced options. - // 1. Apply generic options. - if err := o.applyTo(c); err != nil { + // 1. Build policy checker + checker, err := o.newPolicyChecker() + if err != nil { return err } - // 2. Apply plugin options. - if err := o.LogOptions.applyTo(c); err != nil { - return err + // 2. Build log backend + var logBackend audit.Backend + if w := o.LogOptions.getWriter(); w != nil { + if checker == nil { + klog.V(2).Info("No audit policy file provided, no events will be recorded for log backend") + } else { + logBackend = o.LogOptions.newBackend(w) + } } - if err := o.WebhookOptions.applyTo(c); err != nil { + + // 3. Build webhook backend + var webhookBackend audit.Backend + if o.WebhookOptions.enabled() { + if checker == nil { + klog.V(2).Info("No audit policy file provided, no events will be recorded for webhook backend") + } else { + webhookBackend, err = o.WebhookOptions.newUntruncatedBackend() + if err != nil { + return err + } + } + } + + groupVersion, err := schema.ParseGroupVersion(o.WebhookOptions.GroupVersionString) + if err != nil { return err } - if c.AuditBackend != nil && c.AuditPolicyChecker == nil { - glog.V(2).Info("No audit policy file provided for AdvancedAuditing, no events will be recorded.") + // 4. Apply dynamic options. + var dynamicBackend audit.Backend + if o.DynamicOptions.enabled() { + // if dynamic is enabled the webhook and log backends need to be wrapped in an enforced backend with the static policy + if webhookBackend != nil { + webhookBackend = pluginenforced.NewBackend(webhookBackend, checker) + } + if logBackend != nil { + logBackend = pluginenforced.NewBackend(logBackend, checker) + } + // build dynamic backend + dynamicBackend, checker, err = o.DynamicOptions.newBackend(c.ExternalAddress, kubeClientConfig, informers, processInfo, webhookOptions) + if err != nil { + return err + } + // union dynamic and webhook backends so that truncate options can be applied to both + dynamicBackend = appendBackend(webhookBackend, dynamicBackend) + dynamicBackend = o.WebhookOptions.TruncateOptions.wrapBackend(dynamicBackend, groupVersion) + } else if webhookBackend != nil { + // if only webhook is enabled wrap it in the truncate options + dynamicBackend = o.WebhookOptions.TruncateOptions.wrapBackend(webhookBackend, groupVersion) + } + + // 5. Set the policy checker + c.AuditPolicyChecker = checker + + // 6. Join the log backend with the webhooks + c.AuditBackend = appendBackend(logBackend, dynamicBackend) + + if c.AuditBackend != nil { + klog.V(2).Infof("Using audit backend: %s", c.AuditBackend) } return nil } -func (o *AuditOptions) applyTo(c *server.Config) error { +func (o *AuditOptions) newPolicyChecker() (policy.Checker, error) { if o.PolicyFile == "" { - return nil + return nil, nil } p, err := policy.LoadPolicyFromFile(o.PolicyFile) if err != nil { - return fmt.Errorf("loading audit policy file: %v", err) + return nil, fmt.Errorf("loading audit policy file: %v", err) } - c.AuditPolicyChecker = policy.NewChecker(p) - return nil + return policy.NewChecker(p), nil } func (o *AuditBatchOptions) AddFlags(pluginName string, fs *pflag.FlagSet) { @@ -438,15 +516,12 @@ func (o *AuditLogOptions) getWriter() io.Writer { return w } -func (o *AuditLogOptions) applyTo(c *server.Config) error { - if w := o.getWriter(); w != nil { - groupVersion, _ := schema.ParseGroupVersion(o.GroupVersionString) - log := pluginlog.NewBackend(w, o.Format, groupVersion) - log = o.BatchOptions.wrapBackend(log) - log = o.TruncateOptions.wrapBackend(log, groupVersion) - c.AuditBackend = appendBackend(c.AuditBackend, log) - } - return nil +func (o *AuditLogOptions) newBackend(w io.Writer) audit.Backend { + groupVersion, _ := schema.ParseGroupVersion(o.GroupVersionString) + log := pluginlog.NewBackend(w, o.Format, groupVersion) + log = o.BatchOptions.wrapBackend(log) + log = o.TruncateOptions.wrapBackend(log, groupVersion) + return log } func (o *AuditWebhookOptions) AddFlags(fs *pflag.FlagSet) { @@ -485,20 +560,76 @@ func (o *AuditWebhookOptions) enabled() bool { return o != nil && o.ConfigFile != "" } -func (o *AuditWebhookOptions) applyTo(c *server.Config) error { - if !o.enabled() { - return nil - } - +// newUntruncatedBackend returns a webhook backend without the truncate options applied +// this is done so that the same trucate backend can wrap both the webhook and dynamic backends +func (o *AuditWebhookOptions) newUntruncatedBackend() (audit.Backend, error) { groupVersion, _ := schema.ParseGroupVersion(o.GroupVersionString) webhook, err := pluginwebhook.NewBackend(o.ConfigFile, groupVersion, o.InitialBackoff) if err != nil { - return fmt.Errorf("initializing audit webhook: %v", err) + return nil, fmt.Errorf("initializing audit webhook: %v", err) } webhook = o.BatchOptions.wrapBackend(webhook) - webhook = o.TruncateOptions.wrapBackend(webhook, groupVersion) - c.AuditBackend = appendBackend(c.AuditBackend, webhook) - return nil + return webhook, nil +} + +func (o *AuditDynamicOptions) AddFlags(fs *pflag.FlagSet) { + fs.BoolVar(&o.Enabled, "audit-dynamic-configuration", o.Enabled, + "Enables dynamic audit configuration. This feature also requires the DynamicAuditing feature flag") +} + +func (o *AuditDynamicOptions) enabled() bool { + return o.Enabled && utilfeature.DefaultFeatureGate.Enabled(features.DynamicAuditing) +} + +func (o *AuditDynamicOptions) Validate() []error { + var allErrors []error + if o.Enabled && !utilfeature.DefaultFeatureGate.Enabled(features.DynamicAuditing) { + allErrors = append(allErrors, fmt.Errorf("--audit-dynamic-configuration set, but DynamicAuditing feature gate is not enabled")) + } + return allErrors +} + +func (o *AuditDynamicOptions) newBackend( + hostname string, + kubeClientConfig *restclient.Config, + informers informers.SharedInformerFactory, + processInfo *ProcessInfo, + webhookOptions *WebhookOptions, +) (audit.Backend, policy.Checker, error) { + if err := validateProcessInfo(processInfo); err != nil { + return nil, nil, err + } + clientset, err := kubernetes.NewForConfig(kubeClientConfig) + if err != nil { + return nil, nil, err + } + if webhookOptions == nil { + webhookOptions = NewWebhookOptions() + } + checker := policy.NewDynamicChecker() + informer := informers.Auditregistration().V1alpha1().AuditSinks() + eventSink := &v1core.EventSinkImpl{Interface: clientset.CoreV1().Events(processInfo.Namespace)} + + dc := &plugindynamic.Config{ + Informer: informer, + BufferedConfig: plugindynamic.NewDefaultWebhookBatchConfig(), + EventConfig: plugindynamic.EventConfig{ + Sink: eventSink, + Source: corev1.EventSource{ + Component: processInfo.Name, + Host: hostname, + }, + }, + WebhookConfig: plugindynamic.WebhookConfig{ + AuthInfoResolverWrapper: webhookOptions.AuthInfoResolverWrapper, + ServiceResolver: webhookOptions.ServiceResolver, + }, + } + backend, err := plugindynamic.NewBackend(dc) + if err != nil { + return nil, nil, fmt.Errorf("could not create dynamic audit backend: %v", err) + } + return backend, checker, nil } // defaultWebhookBatchConfig returns the default BatchConfig used by the Webhook backend. diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/audit_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/audit_test.go index 78ed4f6a207..b1014ef54b0 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/audit_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/audit_test.go @@ -23,7 +23,15 @@ import ( "os" "testing" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" + "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/server" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd/api/v1" "github.com/spf13/pflag" @@ -35,6 +43,15 @@ func TestAuditValidOptions(t *testing.T) { webhookConfig := makeTmpWebhookConfig(t) defer os.Remove(webhookConfig) + policy := makeTmpPolicy(t) + defer os.Remove(policy) + + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DynamicAuditing, true)() + + clientConfig := &restclient.Config{} + informerFactory := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 0) + processInfo := &ProcessInfo{"test", "test"} + testCases := []struct { name string options func() *AuditOptions @@ -47,23 +64,42 @@ func TestAuditValidOptions(t *testing.T) { options: func() *AuditOptions { o := NewAuditOptions() o.LogOptions.Path = "/audit" + o.PolicyFile = policy return o }, expected: "log", + }, { + name: "default log no policy", + options: func() *AuditOptions { + o := NewAuditOptions() + o.LogOptions.Path = "/audit" + return o + }, + expected: "", }, { name: "default webhook", options: func() *AuditOptions { o := NewAuditOptions() o.WebhookOptions.ConfigFile = webhookConfig + o.PolicyFile = policy return o }, expected: "buffered", + }, { + name: "default webhook no policy", + options: func() *AuditOptions { + o := NewAuditOptions() + o.WebhookOptions.ConfigFile = webhookConfig + return o + }, + expected: "", }, { name: "default union", options: func() *AuditOptions { o := NewAuditOptions() o.LogOptions.Path = "/audit" o.WebhookOptions.ConfigFile = webhookConfig + o.PolicyFile = policy return o }, expected: "union[log,buffered]", @@ -75,6 +111,7 @@ func TestAuditValidOptions(t *testing.T) { o.LogOptions.Path = "/audit" o.WebhookOptions.BatchOptions.Mode = ModeBlocking o.WebhookOptions.ConfigFile = webhookConfig + o.PolicyFile = policy return o }, expected: "union[buffered,webhook]", @@ -84,10 +121,62 @@ func TestAuditValidOptions(t *testing.T) { o := NewAuditOptions() o.WebhookOptions.ConfigFile = webhookConfig o.WebhookOptions.TruncateOptions.Enabled = true + o.PolicyFile = policy return o }, expected: "truncate>", - }} + }, { + name: "dynamic", + options: func() *AuditOptions { + o := NewAuditOptions() + o.DynamicOptions.Enabled = true + return o + }, + expected: "dynamic[]", + }, { + name: "dynamic with truncating", + options: func() *AuditOptions { + o := NewAuditOptions() + o.DynamicOptions.Enabled = true + o.WebhookOptions.TruncateOptions.Enabled = true + return o + }, + expected: "truncate", + }, { + name: "dynamic with log", + options: func() *AuditOptions { + o := NewAuditOptions() + o.DynamicOptions.Enabled = true + o.LogOptions.Path = "/audit" + o.PolicyFile = policy + return o + }, + expected: "union[enforced,dynamic[]]", + }, { + name: "dynamic with truncating and webhook", + options: func() *AuditOptions { + o := NewAuditOptions() + o.DynamicOptions.Enabled = true + o.WebhookOptions.TruncateOptions.Enabled = true + o.WebhookOptions.ConfigFile = webhookConfig + o.PolicyFile = policy + return o + }, + expected: "truncate>,dynamic[]]>", + }, { + name: "dynamic with truncating and webhook and log", + options: func() *AuditOptions { + o := NewAuditOptions() + o.DynamicOptions.Enabled = true + o.WebhookOptions.TruncateOptions.Enabled = true + o.WebhookOptions.ConfigFile = webhookConfig + o.PolicyFile = policy + o.LogOptions.Path = "/audit" + return o + }, + expected: "union[enforced,truncate>,dynamic[]]>]", + }, + } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { options := tc.options() @@ -101,7 +190,7 @@ func TestAuditValidOptions(t *testing.T) { assert.Empty(t, options.Validate(), "Options should be valid.") config := &server.Config{} - require.NoError(t, options.ApplyTo(config)) + require.NoError(t, options.ApplyTo(config, clientConfig, informerFactory, processInfo, nil)) if tc.expected == "" { assert.Nil(t, config.AuditBackend) } else { @@ -176,7 +265,15 @@ func TestAuditInvalidOptions(t *testing.T) { o.WebhookOptions.TruncateOptions.TruncateConfig.MaxBatchSize = 1 return o }, - }} + }, { + name: "invalid dynamic flag group", + options: func() *AuditOptions { + o := NewAuditOptions() + o.DynamicOptions.Enabled = true + return o + }, + }, + } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { options := tc.options() @@ -198,3 +295,21 @@ func makeTmpWebhookConfig(t *testing.T) string { require.NoError(t, f.Close()) return f.Name() } + +func makeTmpPolicy(t *testing.T) string { + pol := auditv1.Policy{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "audit.k8s.io/v1", + }, + Rules: []auditv1.PolicyRule{ + { + Level: auditv1.LevelRequestResponse, + }, + }, + } + f, err := ioutil.TempFile("", "k8s_audit_policy_test_") + require.NoError(t, err, "creating temp file") + require.NoError(t, stdjson.NewEncoder(f).Encode(pol), "writing policy file") + require.NoError(t, f.Close()) + return f.Name() +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go b/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go index 043a9341538..04331794e54 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go @@ -22,8 +22,8 @@ import ( "io/ioutil" "time" - "github.com/golang/glog" "github.com/spf13/pflag" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -232,10 +232,10 @@ func (s *DelegatingAuthenticationOptions) lookupMissingConfigInCluster(client ku } if client == nil { if len(s.ClientCert.ClientCA) == 0 { - glog.Warningf("No authentication-kubeconfig provided in order to lookup client-ca-file in configmap/%s in %s, so client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace) + klog.Warningf("No authentication-kubeconfig provided in order to lookup client-ca-file in configmap/%s in %s, so client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace) } if len(s.RequestHeader.ClientCAFile) == 0 { - glog.Warningf("No authentication-kubeconfig provided in order to lookup requestheader-client-ca-file in configmap/%s in %s, so request-header client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace) + klog.Warningf("No authentication-kubeconfig provided in order to lookup requestheader-client-ca-file in configmap/%s in %s, so request-header client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace) } return nil } @@ -245,7 +245,7 @@ func (s *DelegatingAuthenticationOptions) lookupMissingConfigInCluster(client ku case errors.IsNotFound(err): // ignore, authConfigMap is nil now case errors.IsForbidden(err): - glog.Warningf("Unable to get configmap/%s in %s. Usually fixed by "+ + klog.Warningf("Unable to get configmap/%s in %s. Usually fixed by "+ "'kubectl create rolebinding -n %s ROLE_NAME --role=%s --serviceaccount=YOUR_NS:YOUR_SA'", authenticationConfigMapName, authenticationConfigMapNamespace, authenticationConfigMapNamespace, authenticationRoleName) return err @@ -264,7 +264,7 @@ func (s *DelegatingAuthenticationOptions) lookupMissingConfigInCluster(client ku } } if len(s.ClientCert.ClientCA) == 0 { - glog.Warningf("Cluster doesn't provide client-ca-file in configmap/%s in %s, so client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace) + klog.Warningf("Cluster doesn't provide client-ca-file in configmap/%s in %s, so client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace) } } @@ -279,7 +279,7 @@ func (s *DelegatingAuthenticationOptions) lookupMissingConfigInCluster(client ku } } if len(s.RequestHeader.ClientCAFile) == 0 { - glog.Warningf("Cluster doesn't provide requestheader-client-ca-file in configmap/%s in %s, so request-header client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace) + klog.Warningf("Cluster doesn't provide requestheader-client-ca-file in configmap/%s in %s, so request-header client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace) } } @@ -370,7 +370,7 @@ func (s *DelegatingAuthenticationOptions) getClient() (kubernetes.Interface, err clientConfig, err = rest.InClusterConfig() if err != nil && s.RemoteKubeConfigFileOptional { if err != rest.ErrNotInCluster { - glog.Warningf("failed to read in-cluster kubeconfig for delegated authentication: %v", err) + klog.Warningf("failed to read in-cluster kubeconfig for delegated authentication: %v", err) } return nil, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go b/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go index 7c65dd39184..5d81d9e8660 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authorization.go @@ -20,8 +20,8 @@ import ( "fmt" "time" - "github.com/golang/glog" "github.com/spf13/pflag" + "k8s.io/klog" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/authorization/authorizerfactory" @@ -56,6 +56,9 @@ type DelegatingAuthorizationOptions struct { // AlwaysAllowPaths are HTTP paths which are excluded from authorization. They can be plain // paths or end in * in which case prefix-match is applied. A leading / is optional. AlwaysAllowPaths []string + + // AlwaysAllowGroups are groups which are allowed to take any actions. In kube, this is system:masters. + AlwaysAllowGroups []string } func NewDelegatingAuthorizationOptions() *DelegatingAuthorizationOptions { @@ -66,6 +69,18 @@ func NewDelegatingAuthorizationOptions() *DelegatingAuthorizationOptions { } } +// WithAlwaysAllowGroups appends the list of paths to AlwaysAllowGroups +func (s *DelegatingAuthorizationOptions) WithAlwaysAllowGroups(groups ...string) *DelegatingAuthorizationOptions { + s.AlwaysAllowGroups = append(s.AlwaysAllowGroups, groups...) + return s +} + +// WithAlwaysAllowPaths appends the list of paths to AlwaysAllowPaths +func (s *DelegatingAuthorizationOptions) WithAlwaysAllowPaths(paths ...string) *DelegatingAuthorizationOptions { + s.AlwaysAllowPaths = append(s.AlwaysAllowPaths, paths...) + return s +} + func (s *DelegatingAuthorizationOptions) Validate() []error { allErrors := []error{} return allErrors @@ -115,6 +130,10 @@ func (s *DelegatingAuthorizationOptions) ApplyTo(c *server.AuthorizationInfo) er func (s *DelegatingAuthorizationOptions) toAuthorizer(client kubernetes.Interface) (authorizer.Authorizer, error) { var authorizers []authorizer.Authorizer + if len(s.AlwaysAllowGroups) > 0 { + authorizers = append(authorizers, authorizerfactory.NewPrivilegedGroups(s.AlwaysAllowGroups...)) + } + if len(s.AlwaysAllowPaths) > 0 { a, err := path.NewAuthorizer(s.AlwaysAllowPaths) if err != nil { @@ -124,7 +143,7 @@ func (s *DelegatingAuthorizationOptions) toAuthorizer(client kubernetes.Interfac } if client == nil { - glog.Warningf("No authorization-kubeconfig provided, so SubjectAccessReview of authorization tokens won't work.") + klog.Warningf("No authorization-kubeconfig provided, so SubjectAccessReview of authorization tokens won't work.") } else { cfg := authorizerfactory.DelegatingAuthorizerConfig{ SubjectAccessReviewClient: client.AuthorizationV1beta1().SubjectAccessReviews(), @@ -155,7 +174,7 @@ func (s *DelegatingAuthorizationOptions) getClient() (kubernetes.Interface, erro clientConfig, err = rest.InClusterConfig() if err != nil && s.RemoteKubeConfigFileOptional { if err != rest.ErrNotInCluster { - glog.Warningf("failed to read in-cluster kubeconfig for delegated authorization: %v", err) + klog.Warningf("failed to read in-cluster kubeconfig for delegated authorization: %v", err) } return nil, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go index e8e3d7feb3f..f1cc4430b8b 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go @@ -150,11 +150,11 @@ func (s *DeprecatedInsecureServingOptionsWithLoopback) ApplyTo(insecureServingIn secureLoopbackClientConfig, err := (*insecureServingInfo).NewLoopbackClientConfig() switch { // if we failed and there's no fallback loopback client config, we need to fail - case err != nil && secureLoopbackClientConfig == nil: + case err != nil && *loopbackClientConfig == nil: return err // if we failed, but we already have a fallback loopback client config (usually insecure), allow it - case err != nil && secureLoopbackClientConfig != nil: + case err != nil && *loopbackClientConfig != nil: default: *loopbackClientConfig = secureLoopbackClientConfig diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/BUILD b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/BUILD index d409f8cfca3..5e4da1762f8 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/BUILD @@ -8,20 +8,20 @@ load( go_library( name = "go_default_library", - srcs = [ - "config.go", - "types.go", - ], + srcs = ["config.go"], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig", importpath = "k8s.io/apiserver/pkg/server/options/encryptionconfig", deps = [ + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/config:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/config/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/identity:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/secretbox:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", ], ) @@ -31,6 +31,8 @@ go_test( embed = [":go_default_library"], deps = [ "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/config:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope:go_default_library", ], diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/OWNERS b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/OWNERS new file mode 100644 index 00000000000..498b470c9bf --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-encryption-at-rest-approvers +reviewers: +- sig-auth-encryption-at-rest-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go index feece9f029a..d86a3e10181 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go @@ -26,9 +26,11 @@ import ( "os" "time" - yaml "github.com/ghodss/yaml" - + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + apiserverconfig "k8s.io/apiserver/pkg/apis/config" + apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1" "k8s.io/apiserver/pkg/storage/value" aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes" "k8s.io/apiserver/pkg/storage/value/encrypt/envelope" @@ -66,20 +68,11 @@ func ParseEncryptionConfiguration(f io.Reader) (map[schema.GroupResource]value.T return nil, fmt.Errorf("could not read contents: %v", err) } - var config EncryptionConfig - err = yaml.Unmarshal(configFileContents, &config) + config, err := loadConfig(configFileContents) if err != nil { return nil, fmt.Errorf("error while parsing file: %v", err) } - if config.Kind == "" { - return nil, fmt.Errorf("invalid configuration file, missing Kind") - } - if config.Kind != "EncryptionConfig" { - return nil, fmt.Errorf("invalid configuration kind %q provided", config.Kind) - } - // TODO config.APIVersion is unchecked - resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{} // For each entry in the configuration @@ -102,13 +95,32 @@ func ParseEncryptionConfiguration(f io.Reader) (map[schema.GroupResource]value.T result[gr] = value.NewMutableTransformer(value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)) } return result, nil + +} + +// loadConfig decodes data as a EncryptionConfiguration object. +func loadConfig(data []byte) (*apiserverconfig.EncryptionConfiguration, error) { + scheme := runtime.NewScheme() + codecs := serializer.NewCodecFactory(scheme) + apiserverconfig.AddToScheme(scheme) + apiserverconfigv1.AddToScheme(scheme) + + configObj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil) + if err != nil { + return nil, err + } + config, ok := configObj.(*apiserverconfig.EncryptionConfiguration) + if !ok { + return nil, fmt.Errorf("got unexpected config type: %v", gvk) + } + return config, nil } // The factory to create kms service. This is to make writing test easier. var envelopeServiceFactory = envelope.NewGRPCService -// GetPrefixTransformers constructs and returns the appropriate prefix transformers for the passed resource using its configuration -func GetPrefixTransformers(config *ResourceConfig) ([]value.PrefixTransformer, error) { +// GetPrefixTransformers constructs and returns the appropriate prefix transformers for the passed resource using its configuration. +func GetPrefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]value.PrefixTransformer, error) { var result []value.PrefixTransformer for _, provider := range config.Providers { found := false @@ -188,7 +200,7 @@ type BlockTransformerFunc func(cipher.Block) value.Transformer // GetAESPrefixTransformer returns a prefix transformer from the provided configuration. // Returns an AES transformer based on the provided prefix and block transformer. -func GetAESPrefixTransformer(config *AESConfig, fn BlockTransformerFunc, prefix string) (value.PrefixTransformer, error) { +func GetAESPrefixTransformer(config *apiserverconfig.AESConfiguration, fn BlockTransformerFunc, prefix string) (value.PrefixTransformer, error) { var result value.PrefixTransformer if len(config.Keys) == 0 { @@ -236,7 +248,7 @@ func GetAESPrefixTransformer(config *AESConfig, fn BlockTransformerFunc, prefix } // GetSecretboxPrefixTransformer returns a prefix transformer from the provided configuration -func GetSecretboxPrefixTransformer(config *SecretboxConfig) (value.PrefixTransformer, error) { +func GetSecretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (value.PrefixTransformer, error) { var result value.PrefixTransformer if len(config.Keys) == 0 { @@ -288,8 +300,8 @@ func GetSecretboxPrefixTransformer(config *SecretboxConfig) (value.PrefixTransfo // getEnvelopePrefixTransformer returns a prefix transformer from the provided config. // envelopeService is used as the root of trust. -func getEnvelopePrefixTransformer(config *KMSConfig, envelopeService envelope.Service, prefix string) (value.PrefixTransformer, error) { - envelopeTransformer, err := envelope.NewEnvelopeTransformer(envelopeService, config.CacheSize, aestransformer.NewCBCTransformer) +func getEnvelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) (value.PrefixTransformer, error) { + envelopeTransformer, err := envelope.NewEnvelopeTransformer(envelopeService, int(config.CacheSize), aestransformer.NewCBCTransformer) if err != nil { return value.PrefixTransformer{}, err } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go index 8cd6027e904..6ba28763749 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go @@ -19,11 +19,14 @@ package encryptionconfig import ( "bytes" "encoding/base64" + "reflect" "strings" "testing" "time" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/diff" + apiserverconfig "k8s.io/apiserver/pkg/apis/config" "k8s.io/apiserver/pkg/storage/value" "k8s.io/apiserver/pkg/storage/value/encrypt/envelope" ) @@ -33,9 +36,40 @@ const ( sampleContextText = "0123456789" + legacyV1Config = ` + kind: EncryptionConfig + apiVersion: v1 + resources: + - resources: + - secrets + - namespaces + providers: + - identity: {} + - aesgcm: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ== + - name: key2 + secret: dGhpcyBpcyBwYXNzd29yZA== + - kms: + name: testprovider + endpoint: unix:///tmp/testprovider.sock + cachesize: 10 + - aescbc: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ== + - name: key2 + secret: dGhpcyBpcyBwYXNzd29yZA== + - secretbox: + keys: + - name: key1 + secret: YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoxMjM0NTY= + ` + correctConfigWithIdentityFirst = ` -kind: EncryptionConfig -apiVersion: v1 +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 resources: - resources: - secrets @@ -65,8 +99,8 @@ resources: ` correctConfigWithAesGcmFirst = ` -kind: EncryptionConfig -apiVersion: v1 +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 resources: - resources: - secrets @@ -95,8 +129,8 @@ resources: ` correctConfigWithAesCbcFirst = ` -kind: EncryptionConfig -apiVersion: v1 +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 resources: - resources: - secrets @@ -125,8 +159,8 @@ resources: ` correctConfigWithSecretboxFirst = ` -kind: EncryptionConfig -apiVersion: v1 +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 resources: - resources: - secrets @@ -155,8 +189,8 @@ resources: ` correctConfigWithKMSFirst = ` -kind: EncryptionConfig -apiVersion: v1 +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 resources: - resources: - secrets @@ -185,8 +219,8 @@ resources: ` incorrectConfigNoSecretForKey = ` -kind: EncryptionConfig -apiVersion: v1 +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 resources: - resources: - namespaces @@ -198,8 +232,8 @@ resources: ` incorrectConfigInvalidKey = ` -kind: EncryptionConfig -apiVersion: v1 +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 resources: - resources: - namespaces @@ -214,8 +248,8 @@ resources: ` incorrectConfigNoEndpointForKMS = ` -kind: EncryptionConfig -apiVersion: v1 +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 resources: - resources: - secrets @@ -244,6 +278,48 @@ func newMockEnvelopeService(endpoint string, timeout time.Duration) (envelope.Se return &testEnvelopeService{}, nil } +func TestLegacyConfig(t *testing.T) { + legacyConfigObject, err := loadConfig([]byte(legacyV1Config)) + if err != nil { + t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, legacyV1Config) + } + + expected := &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{"secrets", "namespaces"}, + Providers: []apiserverconfig.ProviderConfiguration{ + {Identity: &apiserverconfig.IdentityConfiguration{}}, + {AESGCM: &apiserverconfig.AESConfiguration{ + Keys: []apiserverconfig.Key{ + {Name: "key1", Secret: "c2VjcmV0IGlzIHNlY3VyZQ=="}, + {Name: "key2", Secret: "dGhpcyBpcyBwYXNzd29yZA=="}, + }, + }}, + {KMS: &apiserverconfig.KMSConfiguration{ + Name: "testprovider", + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: 10, + }}, + {AESCBC: &apiserverconfig.AESConfiguration{ + Keys: []apiserverconfig.Key{ + {Name: "key1", Secret: "c2VjcmV0IGlzIHNlY3VyZQ=="}, + {Name: "key2", Secret: "dGhpcyBpcyBwYXNzd29yZA=="}, + }, + }}, + {Secretbox: &apiserverconfig.SecretboxConfiguration{ + Keys: []apiserverconfig.Key{ + {Name: "key1", Secret: "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoxMjM0NTY="}, + }, + }}, + }, + }, + }, + } + if !reflect.DeepEqual(legacyConfigObject, expected) { + t.Fatal(diff.ObjectReflectDiff(expected, legacyConfigObject)) + } +} func TestEncryptionProviderConfigCorrect(t *testing.T) { // Set factory for mock envelope service factory := envelopeServiceFactory diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/events.go b/staging/src/k8s.io/apiserver/pkg/server/options/events.go new file mode 100644 index 00000000000..2dfc0111fcc --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/options/events.go @@ -0,0 +1,56 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "os" +) + +// ProcessInfo holds the apiserver process information used to send events +type ProcessInfo struct { + // Name of the api process to identify events + Name string + + // Namespace of the api process to send events + Namespace string +} + +// NewProcessInfo returns a new process info with the hostname concatenated to the name given +func NewProcessInfo(name, namespace string) *ProcessInfo { + // try to concat the hostname if available + host, _ := os.Hostname() + if host != "" { + name = fmt.Sprintf("%s-%s", name, host) + } + return &ProcessInfo{ + Name: name, + Namespace: namespace, + } +} + +// validateProcessInfo checks for a complete process info +func validateProcessInfo(p *ProcessInfo) error { + if p == nil { + return fmt.Errorf("ProcessInfo must be set") + } else if p.Name == "" { + return fmt.Errorf("ProcessInfo name must be set") + } else if p.Namespace == "" { + return fmt.Errorf("ProcessInfo namespace must be set") + } + return nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go index 5016145bd14..500d578d6bd 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go @@ -41,9 +41,12 @@ type RecommendedOptions struct { // admission plugin initializers to Admission.ApplyTo. ExtraAdmissionInitializers func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error) Admission *AdmissionOptions + // ProcessInfo is used to identify events created by the server. + ProcessInfo *ProcessInfo + Webhook *WebhookOptions } -func NewRecommendedOptions(prefix string, codec runtime.Codec) *RecommendedOptions { +func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *ProcessInfo) *RecommendedOptions { sso := NewSecureServingOptions() // We are composing recommended options for an aggregated api-server, @@ -62,6 +65,8 @@ func NewRecommendedOptions(prefix string, codec runtime.Codec) *RecommendedOptio CoreAPI: NewCoreAPIOptions(), ExtraAdmissionInitializers: func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error) { return nil, nil }, Admission: NewAdmissionOptions(), + ProcessInfo: processInfo, + Webhook: NewWebhookOptions(), } } @@ -92,7 +97,7 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig, scheme *r if err := o.Authorization.ApplyTo(&config.Config.Authorization); err != nil { return err } - if err := o.Audit.ApplyTo(&config.Config); err != nil { + if err := o.Audit.ApplyTo(&config.Config, config.ClientConfig, config.SharedInformerFactory, o.ProcessInfo, o.Webhook); err != nil { return err } if err := o.Features.ApplyTo(&config.Config); err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go index 998c7ce358a..939e05741da 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go @@ -24,8 +24,8 @@ import ( "strconv" "strings" - "github.com/golang/glog" "github.com/spf13/pflag" + "k8s.io/klog" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apiserver/pkg/server" @@ -308,14 +308,14 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str if err := certutil.WriteKey(keyCert.KeyFile, key); err != nil { return err } - glog.Infof("Generated self-signed cert (%s, %s)", keyCert.CertFile, keyCert.KeyFile) + klog.Infof("Generated self-signed cert (%s, %s)", keyCert.CertFile, keyCert.KeyFile) } else { tlsCert, err := tls.X509KeyPair(cert, key) if err != nil { return fmt.Errorf("unable to generate self signed cert: %v", err) } s.ServerCert.GeneratedCert = &tlsCert - glog.Infof("Generated self-signed cert in-memory") + klog.Infof("Generated self-signed cert in-memory") } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go index ac9635d2dd3..7f19206425e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go @@ -63,11 +63,11 @@ func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.Se secureLoopbackClientConfig, err := (*secureServingInfo).NewLoopbackClientConfig(uuid.NewRandom().String(), certPem) switch { // if we failed and there's no fallback loopback client config, we need to fail - case err != nil && secureLoopbackClientConfig == nil: + case err != nil && *loopbackClientConfig == nil: return err // if we failed, but we already have a fallback loopback client config (usually insecure), allow it - case err != nil && secureLoopbackClientConfig != nil: + case err != nil && *loopbackClientConfig != nil: default: *loopbackClientConfig = secureLoopbackClientConfig diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/webhook.go b/staging/src/k8s.io/apiserver/pkg/server/options/webhook.go new file mode 100644 index 00000000000..bd3ec124d6d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/options/webhook.go @@ -0,0 +1,34 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + utilwebhook "k8s.io/apiserver/pkg/util/webhook" +) + +// WebhookOptions holds the outgoing webhook options +type WebhookOptions struct { + ServiceResolver utilwebhook.ServiceResolver + AuthInfoResolverWrapper utilwebhook.AuthenticationInfoResolverWrapper +} + +// NewWebhookOptions returns the default options for outgoing webhooks +func NewWebhookOptions() *WebhookOptions { + return &WebhookOptions{ + ServiceResolver: utilwebhook.NewDefaultServiceResolver(), + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/routes/BUILD b/staging/src/k8s.io/apiserver/pkg/server/routes/BUILD index f4d56f9e304..0b81332a3a4 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/routes/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/routes/BUILD @@ -32,8 +32,8 @@ go_library( "//vendor/github.com/elazarl/go-bindata-assetfs:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", "//vendor/github.com/emicklei/go-restful-swagger12:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/handler:go_default_library", ], diff --git a/staging/src/k8s.io/apiserver/pkg/server/routes/flags.go b/staging/src/k8s.io/apiserver/pkg/server/routes/flags.go index d40f11499b3..a03b80d3ce7 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/routes/flags.go +++ b/staging/src/k8s.io/apiserver/pkg/server/routes/flags.go @@ -24,7 +24,7 @@ import ( "path" "sync" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apiserver/pkg/server/mux" ) @@ -57,7 +57,7 @@ func (f DebugFlags) Index(w http.ResponseWriter, r *http.Request) { lock.RLock() defer lock.RUnlock() if err := indexTmpl.Execute(w, registeredFlags); err != nil { - glog.Error(err) + klog.Error(err) } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/routes/openapi.go b/staging/src/k8s.io/apiserver/pkg/server/routes/openapi.go index 06c723d3753..934bbf84a04 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/routes/openapi.go +++ b/staging/src/k8s.io/apiserver/pkg/server/routes/openapi.go @@ -18,7 +18,7 @@ package routes import ( restful "github.com/emicklei/go-restful" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apiserver/pkg/server/mux" "k8s.io/kube-openapi/pkg/common" @@ -37,10 +37,10 @@ func (oa OpenAPI) Install(c *restful.Container, mux *mux.PathRecorderMux) { // are tracked at: https://docs.google.com/document/d/19lEqE9lc4yHJ3WJAJxS_G7TcORIJXGHyq3wpwcH28nU. _, err := handler.BuildAndRegisterOpenAPIService("/swagger.json", c.RegisteredWebServices(), oa.Config, mux) if err != nil { - glog.Fatalf("Failed to register open api spec for root: %v", err) + klog.Fatalf("Failed to register open api spec for root: %v", err) } _, err = handler.BuildAndRegisterOpenAPIVersionedService("/openapi/v2", c.RegisteredWebServices(), oa.Config, mux) if err != nil { - glog.Fatalf("Failed to register versioned open api spec for root: %v", err) + klog.Fatalf("Failed to register versioned open api spec for root: %v", err) } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go b/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go index 67a45d5779f..08006c96550 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go @@ -26,8 +26,8 @@ import ( "strings" "time" - "github.com/golang/glog" "golang.org/x/net/http2" + "k8s.io/klog" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/validation" @@ -113,7 +113,7 @@ func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Dur return fmt.Errorf("error configuring http2: %v", err) } - glog.Infof("Serving securely on %s", secureServer.Addr) + klog.Infof("Serving securely on %s", secureServer.Addr) return RunServer(secureServer, s.Listener, shutdownTimeout, stopCh) } @@ -153,7 +153,7 @@ func RunServer( msg := fmt.Sprintf("Stopped listening on %s", ln.Addr().String()) select { case <-stopCh: - glog.Info(msg) + klog.Info(msg) default: panic(fmt.Sprintf("%s due to error: %v", msg, err)) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/BUILD b/staging/src/k8s.io/apiserver/pkg/server/storage/BUILD index 8995549c5f8..612eefe7b08 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/BUILD @@ -45,7 +45,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go index 50c06825463..a87ce4a5efb 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go @@ -22,7 +22,7 @@ import ( "io/ioutil" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -282,7 +282,7 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (* if err != nil { return nil, err } - glog.V(3).Infof("storing %v in %v, reading as %v from %#v", groupResource, codecConfig.StorageVersion, codecConfig.MemoryVersion, codecConfig.Config) + klog.V(3).Infof("storing %v in %v, reading as %v from %#v", groupResource, codecConfig.StorageVersion, codecConfig.MemoryVersion, codecConfig.Config) return &storageConfig, nil } @@ -302,14 +302,14 @@ func (s *DefaultStorageFactory) Backends() []Backend { if len(s.StorageConfig.CertFile) > 0 && len(s.StorageConfig.KeyFile) > 0 { cert, err := tls.LoadX509KeyPair(s.StorageConfig.CertFile, s.StorageConfig.KeyFile) if err != nil { - glog.Errorf("failed to load key pair while getting backends: %s", err) + klog.Errorf("failed to load key pair while getting backends: %s", err) } else { tlsConfig.Certificates = []tls.Certificate{cert} } } if len(s.StorageConfig.CAFile) > 0 { if caCert, err := ioutil.ReadFile(s.StorageConfig.CAFile); err != nil { - glog.Errorf("failed to read ca file while getting backends: %s", err) + klog.Errorf("failed to read ca file while getting backends: %s", err) } else { caPool := x509.NewCertPool() caPool.AppendCertsFromPEM(caCert) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/cacher/BUILD index 9d4b3fe24c7..af3a14a8fba 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher/BUILD @@ -28,7 +28,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/trace:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -50,10 +50,14 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/example:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/example/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/etcd:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go b/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go index f73634cd951..e3515148c35 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go @@ -24,7 +24,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -122,7 +122,7 @@ func (i *indexedWatchers) deleteWatcher(number int, value string, supported bool func (i *indexedWatchers) terminateAll(objectType reflect.Type) { if len(i.allWatchers) > 0 || len(i.valueWatchers) > 0 { - glog.Warningf("Terminating all watchers from cacher %v", objectType) + klog.Warningf("Terminating all watchers from cacher %v", objectType) } i.allWatchers.terminateAll() for index, watchers := range i.valueWatchers { @@ -269,7 +269,7 @@ func (c *Cacher) startCaching(stopChannel <-chan struct{}) { // Also note that startCaching is called in a loop, so there's no need // to have another loop here. if err := c.reflector.ListAndWatch(stopChannel); err != nil { - glog.Errorf("unexpected ListAndWatch error: %v", err) + klog.Errorf("unexpected ListAndWatch error: %v", err) } } @@ -404,10 +404,15 @@ func (c *Cacher) Get(ctx context.Context, key string, resourceVersion string, ob // GetToList implements storage.Interface. func (c *Cacher) GetToList(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate, listObj runtime.Object) error { pagingEnabled := utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking) - if resourceVersion == "" || (pagingEnabled && (len(pred.Continue) > 0 || pred.Limit > 0)) { + hasContinuation := pagingEnabled && len(pred.Continue) > 0 + hasLimit := pagingEnabled && pred.Limit > 0 && resourceVersion != "0" + if resourceVersion == "" || hasContinuation || hasLimit { // If resourceVersion is not specified, serve it from underlying - // storage (for backward compatibility). If a continuation or limit is + // storage (for backward compatibility). If a continuation is // requested, serve it from the underlying storage as well. + // Limits are only sent to storage when resourceVersion is non-zero + // since the watch cache isn't able to perform continuations, and + // limits are ignored when resource version is zero return c.storage.GetToList(ctx, key, resourceVersion, pred, listObj) } @@ -547,7 +552,7 @@ func (c *Cacher) GuaranteedUpdate( // Ignore the suggestion and try to pass down the current version of the object // read from cache. if elem, exists, err := c.watchCache.GetByKey(key); err != nil { - glog.Errorf("GetByKey returned error: %v", err) + klog.Errorf("GetByKey returned error: %v", err) } else if exists { currObj := elem.(*storeElement).Object.DeepCopyObject() return c.storage.GuaranteedUpdate(ctx, key, ptrToType, ignoreNotFound, preconditions, tryUpdate, currObj) @@ -590,7 +595,7 @@ func (c *Cacher) triggerValues(event *watchCacheEvent) ([]string, bool) { func (c *Cacher) processEvent(event *watchCacheEvent) { if curLen := int64(len(c.incoming)); c.incomingHWM.Update(curLen) { // Monitor if this gets backed up, and how much. - glog.V(1).Infof("cacher (%v): %v objects queued in incoming channel.", c.objectType.String(), curLen) + klog.V(1).Infof("cacher (%v): %v objects queued in incoming channel.", c.objectType.String(), curLen) } c.incoming <- *event } @@ -679,7 +684,7 @@ func forgetWatcher(c *Cacher, index int, triggerValue string, triggerSupported b // false is currently passed only if we are forcing watcher to close due // to its unresponsiveness and blocking other watchers. // TODO: Get this information in cleaner way. - glog.V(1).Infof("Forcing watcher close due to unresponsiveness: %v", c.objectType.String()) + klog.V(1).Infof("Forcing watcher close due to unresponsiveness: %v", c.objectType.String()) } // It's possible that the watcher is already not in the structure (e.g. in case of // simultaneous Stop() and terminateAllWatchers(), but it doesn't break anything. @@ -942,7 +947,7 @@ func (c *cacheWatcher) process(initEvents []*watchCacheEvent, resourceVersion ui if len(initEvents) > 0 { objType = reflect.TypeOf(initEvents[0].Object).String() } - glog.V(2).Infof("processing %d initEvents of %s took %v", len(initEvents), objType, processingTime) + klog.V(2).Infof("processing %d initEvents of %s took %v", len(initEvents), objType, processingTime) } defer close(c.result) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher_whitebox_test.go b/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher_whitebox_test.go index 11e43f4d746..780af4b2d10 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher_whitebox_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher_whitebox_test.go @@ -17,6 +17,7 @@ limitations under the License. package cacher import ( + "context" "fmt" "reflect" "strconv" @@ -30,9 +31,14 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/diff" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" + "k8s.io/apiserver/pkg/apis/example" + examplev1 "k8s.io/apiserver/pkg/apis/example/v1" + "k8s.io/apiserver/pkg/storage" ) // verifies the cacheWatcher.process goroutine is properly cleaned up even if @@ -195,7 +201,13 @@ func (testVersioner) UpdateObject(obj runtime.Object, resourceVersion uint64) er return meta.NewAccessor().SetResourceVersion(obj, strconv.FormatUint(resourceVersion, 10)) } func (testVersioner) UpdateList(obj runtime.Object, resourceVersion uint64, continueValue string) error { - return fmt.Errorf("unimplemented") + listAccessor, err := meta.ListAccessor(obj) + if err != nil || listAccessor == nil { + return err + } + listAccessor.SetResourceVersion(strconv.FormatUint(resourceVersion, 10)) + listAccessor.SetContinue(continueValue) + return nil } func (testVersioner) PrepareObjectForStorage(obj runtime.Object) error { return fmt.Errorf("unimplemented") @@ -206,3 +218,136 @@ func (testVersioner) ObjectResourceVersion(obj runtime.Object) (uint64, error) { func (testVersioner) ParseResourceVersion(resourceVersion string) (uint64, error) { return strconv.ParseUint(resourceVersion, 10, 64) } + +var ( + scheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(scheme) + errDummy = fmt.Errorf("dummy error") +) + +func init() { + metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion) + utilruntime.Must(example.AddToScheme(scheme)) + utilruntime.Must(examplev1.AddToScheme(scheme)) +} + +func newTestCacher(s storage.Interface, cap int) (*Cacher, storage.Versioner) { + prefix := "pods" + config := Config{ + CacheCapacity: cap, + Storage: s, + Versioner: testVersioner{}, + Type: &example.Pod{}, + ResourcePrefix: prefix, + KeyFunc: func(obj runtime.Object) (string, error) { return storage.NamespaceKeyFunc(prefix, obj) }, + GetAttrsFunc: func(obj runtime.Object) (labels.Set, fields.Set, bool, error) { return nil, nil, true, nil }, + NewListFunc: func() runtime.Object { return &example.PodList{} }, + Codec: codecs.LegacyCodec(examplev1.SchemeGroupVersion), + } + return NewCacherFromConfig(config), testVersioner{} +} + +type dummyStorage struct { + err error +} + +type dummyWatch struct { + ch chan watch.Event +} + +func (w *dummyWatch) ResultChan() <-chan watch.Event { + return w.ch +} + +func (w *dummyWatch) Stop() { + close(w.ch) +} + +func newDummyWatch() watch.Interface { + return &dummyWatch{ + ch: make(chan watch.Event), + } +} + +func (d *dummyStorage) Versioner() storage.Versioner { return nil } +func (d *dummyStorage) Create(_ context.Context, _ string, _, _ runtime.Object, _ uint64) error { + return fmt.Errorf("unimplemented") +} +func (d *dummyStorage) Delete(_ context.Context, _ string, _ runtime.Object, _ *storage.Preconditions) error { + return fmt.Errorf("unimplemented") +} +func (d *dummyStorage) Watch(_ context.Context, _ string, _ string, _ storage.SelectionPredicate) (watch.Interface, error) { + return newDummyWatch(), nil +} +func (d *dummyStorage) WatchList(_ context.Context, _ string, _ string, _ storage.SelectionPredicate) (watch.Interface, error) { + return newDummyWatch(), nil +} +func (d *dummyStorage) Get(_ context.Context, _ string, _ string, _ runtime.Object, _ bool) error { + return fmt.Errorf("unimplemented") +} +func (d *dummyStorage) GetToList(_ context.Context, _ string, _ string, _ storage.SelectionPredicate, _ runtime.Object) error { + return d.err +} +func (d *dummyStorage) List(_ context.Context, _ string, _ string, _ storage.SelectionPredicate, listObj runtime.Object) error { + podList := listObj.(*example.PodList) + podList.ListMeta = metav1.ListMeta{ResourceVersion: "100"} + return d.err +} +func (d *dummyStorage) GuaranteedUpdate(_ context.Context, _ string, _ runtime.Object, _ bool, _ *storage.Preconditions, _ storage.UpdateFunc, _ ...runtime.Object) error { + return fmt.Errorf("unimplemented") +} +func (d *dummyStorage) Count(_ string) (int64, error) { + return 0, fmt.Errorf("unimplemented") +} + +func TestListWithLimitAndRV0(t *testing.T) { + backingStorage := &dummyStorage{} + cacher, _ := newTestCacher(backingStorage, 0) + defer cacher.Stop() + + pred := storage.SelectionPredicate{ + Limit: 500, + } + result := &example.PodList{} + + // Wait until cacher is initialized. + cacher.ready.wait() + + // Inject error to underlying layer and check if cacher is not bypassed. + backingStorage.err = errDummy + err := cacher.List(context.TODO(), "pods/ns", "0", pred, result) + if err != nil { + t.Errorf("List with Limit and RV=0 should be served from cache: %v", err) + } + + err = cacher.List(context.TODO(), "pods/ns", "", pred, result) + if err != errDummy { + t.Errorf("List with Limit without RV=0 should bypass cacher: %v", err) + } +} + +func TestGetToListWithLimitAndRV0(t *testing.T) { + backingStorage := &dummyStorage{} + cacher, _ := newTestCacher(backingStorage, 0) + defer cacher.Stop() + + pred := storage.SelectionPredicate{ + Limit: 500, + } + result := &example.PodList{} + + // Wait until cacher is initialized. + cacher.ready.wait() + + // Inject error to underlying layer and check if cacher is not bypassed. + backingStorage.err = errDummy + err := cacher.GetToList(context.TODO(), "pods/ns", "0", pred, result) + if err != nil { + t.Errorf("GetToList with Limit and RV=0 should be served from cache: %v", err) + } + + err = cacher.GetToList(context.TODO(), "pods/ns", "", pred, result) + if err != errDummy { + t.Errorf("List with Limit without RV=0 should bypass cacher: %v", err) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd/testing/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/etcd/testing/BUILD index f5b0c4c2603..0bbe6add4ec 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd/testing/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd/testing/BUILD @@ -24,7 +24,7 @@ go_library( "//vendor/github.com/coreos/etcd/pkg/testutil:go_default_library", "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library", "//vendor/github.com/coreos/etcd/pkg/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd/testing/utils.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd/testing/utils.go index d8e1f20f84c..493abaa2feb 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd/testing/utils.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd/testing/utils.go @@ -43,7 +43,7 @@ import ( "github.com/coreos/etcd/pkg/testutil" "github.com/coreos/etcd/pkg/transport" "github.com/coreos/etcd/pkg/types" - "github.com/golang/glog" + "k8s.io/klog" ) // EtcdTestServer encapsulates the datastructures needed to start local instance for testing @@ -220,7 +220,7 @@ func (m *EtcdTestServer) waitUntilUp() error { for start := time.Now(); time.Since(start) < wait.ForeverTestTimeout; time.Sleep(10 * time.Millisecond) { members, err := membersAPI.List(context.TODO()) if err != nil { - glog.Errorf("Error when getting etcd cluster members") + klog.Errorf("Error when getting etcd cluster members") continue } if len(members) == 1 && len(members[0].ClientURLs) > 0 { diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/BUILD index 1e683a48d7f..1b177714b27 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/BUILD @@ -66,7 +66,7 @@ go_library( "//vendor/github.com/coreos/etcd/clientv3:go_default_library", "//vendor/github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes:go_default_library", "//vendor/github.com/coreos/etcd/mvcc/mvccpb:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/compact.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/compact.go index bdcd5bcb60d..d4524f49221 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/compact.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/compact.go @@ -23,7 +23,7 @@ import ( "time" "github.com/coreos/etcd/clientv3" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -51,7 +51,7 @@ func StartCompactor(ctx context.Context, client *clientv3.Client, compactInterva // Currently we rely on endpoints to differentiate clusters. for _, ep := range client.Endpoints() { if _, ok := endpointsMap[ep]; ok { - glog.V(4).Infof("compactor already exists for endpoints %v", client.Endpoints()) + klog.V(4).Infof("compactor already exists for endpoints %v", client.Endpoints()) return } } @@ -121,7 +121,7 @@ func compactor(ctx context.Context, client *clientv3.Client, interval time.Durat compactTime, rev, err = compact(ctx, client, compactTime, rev) if err != nil { - glog.Errorf("etcd: endpoint (%v) compact failed: %v", client.Endpoints(), err) + klog.Errorf("etcd: endpoint (%v) compact failed: %v", client.Endpoints(), err) continue } } @@ -157,6 +157,6 @@ func compact(ctx context.Context, client *clientv3.Client, t, rev int64) (int64, if _, err = client.Compact(ctx, rev); err != nil { return curTime, curRev, err } - glog.V(4).Infof("etcd: compacted rev (%d), endpoints (%v)", rev, client.Endpoints()) + klog.V(4).Infof("etcd: compacted rev (%d), endpoints (%v)", rev, client.Endpoints()) return curTime, curRev, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go index 43fa584296e..129b593f53b 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go @@ -29,7 +29,7 @@ import ( "time" "github.com/coreos/etcd/clientv3" - "github.com/golang/glog" + "k8s.io/klog" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -236,7 +236,7 @@ func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.O } if !txnResp.Succeeded { getResp = (*clientv3.GetResponse)(txnResp.Responses[0].GetResponseRange()) - glog.V(4).Infof("deletion of %s failed because of a conflict, going to retry", key) + klog.V(4).Infof("deletion of %s failed because of a conflict, going to retry", key) continue } return decode(s.codec, s.versioner, origState.data, out, origState.rev) @@ -352,7 +352,7 @@ func (s *store) GuaranteedUpdate( trace.Step("Transaction committed") if !txnResp.Succeeded { getResp := (*clientv3.GetResponse)(txnResp.Responses[0].GetResponseRange()) - glog.V(4).Infof("GuaranteedUpdate of %s failed because of a conflict, going to retry", key) + klog.V(4).Infof("GuaranteedUpdate of %s failed because of a conflict, going to retry", key) origState, err = s.getState(getResp, key, v, ignoreNotFound) if err != nil { return err diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go index c1216d5884c..d450038eff7 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go @@ -32,7 +32,7 @@ import ( "k8s.io/apiserver/pkg/storage/value" "github.com/coreos/etcd/clientv3" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -191,7 +191,7 @@ func (wc *watchChan) sync() error { func (wc *watchChan) startWatching(watchClosedCh chan struct{}) { if wc.initialRev == 0 { if err := wc.sync(); err != nil { - glog.Errorf("failed to sync with latest state: %v", err) + klog.Errorf("failed to sync with latest state: %v", err) wc.sendError(err) return } @@ -205,7 +205,7 @@ func (wc *watchChan) startWatching(watchClosedCh chan struct{}) { if wres.Err() != nil { err := wres.Err() // If there is an error on server (e.g. compaction), the channel will return it before closed. - glog.Errorf("watch chan error: %v", err) + klog.Errorf("watch chan error: %v", err) wc.sendError(err) return } @@ -232,7 +232,7 @@ func (wc *watchChan) processEvent(wg *sync.WaitGroup) { continue } if len(wc.resultChan) == outgoingBufSize { - glog.V(3).Infof("Fast watcher, slow processing. Number of buffered events: %d."+ + klog.V(3).Infof("Fast watcher, slow processing. Number of buffered events: %d."+ "Probably caused by slow dispatching events to watchers", outgoingBufSize) } // If user couldn't receive results fast enough, we also block incoming events from watcher. @@ -265,7 +265,7 @@ func (wc *watchChan) acceptAll() bool { func (wc *watchChan) transform(e *event) (res *watch.Event) { curObj, oldObj, err := wc.prepareObjs(e) if err != nil { - glog.Errorf("failed to prepare current and previous objects: %v", err) + klog.Errorf("failed to prepare current and previous objects: %v", err) wc.sendError(err) return nil } @@ -339,7 +339,7 @@ func (wc *watchChan) sendError(err error) { func (wc *watchChan) sendEvent(e *event) { if len(wc.incomingEventChan) == incomingBufSize { - glog.V(3).Infof("Fast watcher, slow processing. Number of buffered events: %d."+ + klog.V(3).Infof("Fast watcher, slow processing. Number of buffered events: %d."+ "Probably caused by slow decoding, user not receiving fast, or other processing logic", incomingBufSize) } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/OWNERS b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/OWNERS new file mode 100644 index 00000000000..498b470c9bf --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-encryption-at-rest-approvers +reviewers: +- sig-auth-encryption-at-rest-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/BUILD index 8beca7163d1..bd42719d88d 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/BUILD @@ -17,10 +17,10 @@ go_library( deps = [ "//staging/src/k8s.io/apiserver/pkg/storage/value:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/hashicorp/golang-lru:go_default_library", "//vendor/golang.org/x/crypto/cryptobyte:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service.go index a39ceeca0da..818da7dec58 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service.go @@ -26,7 +26,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "google.golang.org/grpc" @@ -54,7 +54,7 @@ type gRPCService struct { // NewGRPCService returns an envelope.Service which use gRPC to communicate the remote KMS provider. func NewGRPCService(endpoint string, callTimeout time.Duration) (Service, error) { - glog.V(4).Infof("Configure KMS provider with endpoint: %s", endpoint) + klog.V(4).Infof("Configure KMS provider with endpoint: %s", endpoint) addr, err := parseEndpoint(endpoint) if err != nil { @@ -68,7 +68,7 @@ func NewGRPCService(endpoint string, callTimeout time.Duration) (Service, error) // timeout - is ignored since we are connecting in a non-blocking configuration c, err := net.DialTimeout(unixProtocol, addr, 0) if err != nil { - glog.Errorf("failed to create connection to unix socket: %s, error: %v", addr, err) + klog.Errorf("failed to create connection to unix socket: %s, error: %v", addr, err) } return c, err })) @@ -129,7 +129,7 @@ func (g *gRPCService) checkAPIVersion(ctx context.Context) error { } g.versionChecked = true - glog.V(4).Infof("Version of KMS provider is %s", response.Version) + klog.V(4).Infof("Version of KMS provider is %s", response.Version) return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/util/feature/BUILD b/staging/src/k8s.io/apiserver/pkg/util/feature/BUILD index 31274c71d86..2b0386ab76b 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/feature/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/util/feature/BUILD @@ -22,8 +22,8 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/util/feature", importpath = "k8s.io/apiserver/pkg/util/feature", deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go b/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go index 8847c1fb625..a83dafd56ab 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go +++ b/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go @@ -24,8 +24,8 @@ import ( "sync" "sync/atomic" - "github.com/golang/glog" "github.com/spf13/pflag" + "k8s.io/klog" ) type Feature string @@ -193,9 +193,9 @@ func (f *featureGate) SetFromMap(m map[string]bool) error { } if featureSpec.PreRelease == Deprecated { - glog.Warningf("Setting deprecated feature gate %s=%t. It will be removed in a future release.", k, v) + klog.Warningf("Setting deprecated feature gate %s=%t. It will be removed in a future release.", k, v) } else if featureSpec.PreRelease == GA { - glog.Warningf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v) + klog.Warningf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v) } } @@ -203,7 +203,7 @@ func (f *featureGate) SetFromMap(m map[string]bool) error { f.known.Store(known) f.enabled.Store(enabled) - glog.V(1).Infof("feature gates: %v", f.enabled) + klog.V(1).Infof("feature gates: %v", f.enabled) return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/util/feature/testing/feature_gate_testing.go b/staging/src/k8s.io/apiserver/pkg/util/feature/testing/feature_gate_testing.go index fdbd7735a01..bcae2566bc3 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/feature/testing/feature_gate_testing.go +++ b/staging/src/k8s.io/apiserver/pkg/util/feature/testing/feature_gate_testing.go @@ -18,13 +18,57 @@ package testing import ( "fmt" + "os" + "strings" "testing" "k8s.io/apiserver/pkg/util/feature" ) +// VerifyFeatureGatesUnchanged ensures the provided gate does not change any values when tests() are completed. +// Intended to be placed into unit test packages that mess with feature gates. +// +// Example use: +// +// import ( +// "testing" +// +// utilfeature "k8s.io/apiserver/pkg/util/feature" +// utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" +// _ "k8s.io/kubernetes/pkg/features" +// ) +// +// func TestMain(m *testing.M) { +// utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run) +// } +func VerifyFeatureGatesUnchanged(gate feature.FeatureGate, tests func() int) { + originalGates := gate.DeepCopy() + originalSet := fmt.Sprint(gate) + + rc := tests() + + finalSet := fmt.Sprint(gate) + if finalSet != originalSet { + for _, kv := range strings.Split(finalSet, ",") { + k := strings.Split(kv, "=")[0] + if originalGates.Enabled(feature.Feature(k)) != gate.Enabled(feature.Feature(k)) { + fmt.Println(fmt.Sprintf("VerifyFeatureGatesUnchanged: mutated %s feature gate from %v to %v", k, originalGates.Enabled(feature.Feature(k)), gate.Enabled(feature.Feature(k)))) + rc = 1 + } + } + } + + if rc != 0 { + os.Exit(rc) + } +} + // SetFeatureGateDuringTest sets the specified gate to the specified value, and returns a function that restores the original value. // Failures to set or restore cause the test to fail. +// +// Example use: +// +// defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features., true)() func SetFeatureGateDuringTest(t *testing.T, gate feature.FeatureGate, feature feature.Feature, value bool) func() { originalValue := gate.Enabled(feature) diff --git a/staging/src/k8s.io/apiserver/pkg/util/flag/BUILD b/staging/src/k8s.io/apiserver/pkg/util/flag/BUILD index 6370f398c22..7841681a86e 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flag/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/util/flag/BUILD @@ -42,8 +42,8 @@ go_library( deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/github.com/docker/docker/pkg/term:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/util/flag/flags.go b/staging/src/k8s.io/apiserver/pkg/util/flag/flags.go index 55a3ed34a8e..d0fff8db2e4 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flag/flags.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flag/flags.go @@ -20,8 +20,8 @@ import ( goflag "flag" "strings" - "github.com/golang/glog" "github.com/spf13/pflag" + "k8s.io/klog" ) // WordSepNormalizeFunc changes all flags that contain "_" separators @@ -36,7 +36,7 @@ func WordSepNormalizeFunc(f *pflag.FlagSet, name string) pflag.NormalizedName { func WarnWordSepNormalizeFunc(f *pflag.FlagSet, name string) pflag.NormalizedName { if strings.Contains(name, "_") { nname := strings.Replace(name, "_", "-", -1) - glog.Warningf("%s is DEPRECATED and will be removed in a future version. Use %s instead.", name, nname) + klog.Warningf("%s is DEPRECATED and will be removed in a future version. Use %s instead.", name, nname) return pflag.NormalizedName(nname) } @@ -49,6 +49,6 @@ func InitFlags() { pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) pflag.Parse() pflag.VisitAll(func(flag *pflag.Flag) { - glog.V(2).Infof("FLAG: --%s=%q", flag.Name, flag.Value) + klog.V(2).Infof("FLAG: --%s=%q", flag.Name, flag.Value) }) } diff --git a/staging/src/k8s.io/apiserver/pkg/util/globalflag/BUILD b/staging/src/k8s.io/apiserver/pkg/util/globalflag/BUILD new file mode 100644 index 00000000000..9600afb0b49 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/globalflag/BUILD @@ -0,0 +1,37 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["globalflags.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/util/globalflag", + importpath = "k8s.io/apiserver/pkg/util/globalflag", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["globalflags_test.go"], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/apiserver/pkg/util/globalflag/globalflags.go b/staging/src/k8s.io/apiserver/pkg/util/globalflag/globalflags.go new file mode 100644 index 00000000000..c00ef7b9f1e --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/globalflag/globalflags.go @@ -0,0 +1,74 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package globalflag + +import ( + "flag" + "fmt" + "os" + "strings" + + "github.com/spf13/pflag" + + "k8s.io/apiserver/pkg/util/logs" +) + +// AddGlobalFlags explicitly registers flags that libraries (klog, verflag, etc.) register +// against the global flagsets from "flag" and "github.com/spf13/pflag". +// We do this in order to prevent unwanted flags from leaking into the component's flagset. +func AddGlobalFlags(fs *pflag.FlagSet, name string) { + addGlogFlags(fs) + logs.AddFlags(fs) + + fs.BoolP("help", "h", false, fmt.Sprintf("help for %s", name)) +} + +// addGlogFlags explicitly registers flags that klog libraries(k8s.io/klog) register. +func addGlogFlags(fs *pflag.FlagSet) { + // lookup flags in global flag set and re-register the values with our flagset + global := flag.CommandLine + local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError) + + register(global, local, "logtostderr") + register(global, local, "alsologtostderr") + register(global, local, "v") + register(global, local, "skip_headers") + register(global, local, "stderrthreshold") + register(global, local, "vmodule") + register(global, local, "log_backtrace_at") + register(global, local, "log_dir") + register(global, local, "log_file") + + fs.AddFlagSet(local) +} + +// normalize replaces underscores with hyphens +// we should always use hyphens instead of underscores when registering component flags +func normalize(s string) string { + return strings.Replace(s, "_", "-", -1) +} + +// register adds a flag to local that targets the Value associated with the Flag named globalName in global +func register(global *flag.FlagSet, local *pflag.FlagSet, globalName string) { + if f := global.Lookup(globalName); f != nil { + pflagFlag := pflag.PFlagFromGoFlag(f) + pflagFlag.Name = normalize(pflagFlag.Name) + local.AddFlag(pflagFlag) + } else { + panic(fmt.Sprintf("failed to find flag in global flagset (flag): %s", globalName)) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/globalflag/globalflags_test.go b/staging/src/k8s.io/apiserver/pkg/util/globalflag/globalflags_test.go new file mode 100644 index 00000000000..16475154ab6 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/globalflag/globalflags_test.go @@ -0,0 +1,86 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package globalflag + +import ( + "flag" + "reflect" + "sort" + "strings" + "testing" + + "github.com/spf13/pflag" + + apiserverflag "k8s.io/apiserver/pkg/util/flag" +) + +func TestAddGlobalFlags(t *testing.T) { + namedFlagSets := &apiserverflag.NamedFlagSets{} + nfs := namedFlagSets.FlagSet("global") + AddGlobalFlags(nfs, "test-cmd") + + actualFlag := []string{} + nfs.VisitAll(func(flag *pflag.Flag) { + actualFlag = append(actualFlag, flag.Name) + }) + + // Get all flags from flags.CommandLine, except flag `test.*`. + wantedFlag := []string{"help"} + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.VisitAll(func(flag *pflag.Flag) { + if !strings.Contains(flag.Name, "test.") { + wantedFlag = append(wantedFlag, normalize(flag.Name)) + } + }) + sort.Strings(wantedFlag) + + if !reflect.DeepEqual(wantedFlag, actualFlag) { + t.Errorf("[Default]: expected %+v, got %+v", wantedFlag, actualFlag) + } + + tests := []struct { + expectedFlag []string + matchExpected bool + }{ + { + // Happy case + expectedFlag: []string{"alsologtostderr", "help", "log-backtrace-at", "log-dir", "log-file", "log-flush-frequency", "logtostderr", "skip-headers", "stderrthreshold", "v", "vmodule"}, + matchExpected: false, + }, + { + // Missing flag + expectedFlag: []string{"logtostderr", "log-dir"}, + matchExpected: true, + }, + { + // Empty flag + expectedFlag: []string{}, + matchExpected: true, + }, + { + // Invalid flag + expectedFlag: []string{"foo"}, + matchExpected: true, + }, + } + + for i, test := range tests { + if reflect.DeepEqual(test.expectedFlag, actualFlag) == test.matchExpected { + t.Errorf("[%d]: expected %+v, got %+v", i, test.expectedFlag, actualFlag) + } + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/logs/BUILD b/staging/src/k8s.io/apiserver/pkg/util/logs/BUILD index 1e5cf49750a..d51cc643926 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/logs/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/util/logs/BUILD @@ -12,8 +12,8 @@ go_library( importpath = "k8s.io/apiserver/pkg/util/logs", deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/util/logs/logs.go b/staging/src/k8s.io/apiserver/pkg/util/logs/logs.go index a62c06094dd..3ffe9eeb29b 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/logs/logs.go +++ b/staging/src/k8s.io/apiserver/pkg/util/logs/logs.go @@ -22,9 +22,9 @@ import ( "log" "time" - "github.com/golang/glog" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" ) const logFlushFreqFlagName = "log-flush-frequency" @@ -33,6 +33,7 @@ var logFlushFreq = pflag.Duration(logFlushFreqFlagName, 5*time.Second, "Maximum // TODO(thockin): This is temporary until we agree on log dirs and put those into each cmd. func init() { + klog.InitFlags(flag.CommandLine) flag.Set("logtostderr", "true") } @@ -42,38 +43,38 @@ func AddFlags(fs *pflag.FlagSet) { fs.AddFlag(pflag.Lookup(logFlushFreqFlagName)) } -// GlogWriter serves as a bridge between the standard log package and the glog package. -type GlogWriter struct{} +// KlogWriter serves as a bridge between the standard log package and the glog package. +type KlogWriter struct{} // Write implements the io.Writer interface. -func (writer GlogWriter) Write(data []byte) (n int, err error) { - glog.InfoDepth(1, string(data)) +func (writer KlogWriter) Write(data []byte) (n int, err error) { + klog.InfoDepth(1, string(data)) return len(data), nil } // InitLogs initializes logs the way we want for kubernetes. func InitLogs() { - log.SetOutput(GlogWriter{}) + log.SetOutput(KlogWriter{}) log.SetFlags(0) // The default glog flush interval is 5 seconds. - go wait.Forever(glog.Flush, *logFlushFreq) + go wait.Forever(klog.Flush, *logFlushFreq) } // FlushLogs flushes logs immediately. func FlushLogs() { - glog.Flush() + klog.Flush() } -// NewLogger creates a new log.Logger which sends logs to glog.Info. +// NewLogger creates a new log.Logger which sends logs to klog.Info. func NewLogger(prefix string) *log.Logger { - return log.New(GlogWriter{}, prefix, 0) + return log.New(KlogWriter{}, prefix, 0) } // GlogSetter is a setter to set glog level. func GlogSetter(val string) (string, error) { - var level glog.Level + var level klog.Level if err := level.Set(val); err != nil { - return "", fmt.Errorf("failed set glog.logging.verbosity %s: %v", val, err) + return "", fmt.Errorf("failed set klog.logging.verbosity %s: %v", val, err) } - return fmt.Sprintf("successfully set glog.logging.verbosity to %s", val), nil + return fmt.Sprintf("successfully set klog.logging.verbosity to %s", val), nil } diff --git a/staging/src/k8s.io/apiserver/pkg/util/openapi/BUILD b/staging/src/k8s.io/apiserver/pkg/util/openapi/BUILD index 1d90479fd17..872a693c4be 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/openapi/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/util/openapi/BUILD @@ -7,7 +7,6 @@ go_library( importpath = "k8s.io/apiserver/pkg/util/openapi", visibility = ["//visibility:public"], deps = [ - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library", "//vendor/github.com/googleapis/gnostic/compiler:go_default_library", @@ -35,7 +34,6 @@ go_test( srcs = ["proto_test.go"], embed = [":go_default_library"], deps = [ - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/util/proto:go_default_library", ], diff --git a/staging/src/k8s.io/apiserver/pkg/util/openapi/proto.go b/staging/src/k8s.io/apiserver/pkg/util/openapi/proto.go index 5641d1a141f..ba51ba5329b 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/openapi/proto.go +++ b/staging/src/k8s.io/apiserver/pkg/util/openapi/proto.go @@ -18,29 +18,17 @@ package openapi import ( "encoding/json" - "fmt" "github.com/go-openapi/spec" openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2" "github.com/googleapis/gnostic/compiler" yaml "gopkg.in/yaml.v2" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kube-openapi/pkg/util/proto" ) -const ( - // groupVersionKindExtensionKey is the key used to lookup the - // GroupVersionKind value for an object definition from the - // definition's "extensions" map. - groupVersionKindExtensionKey = "x-kubernetes-group-version-kind" -) - -// ToProtoSchema builds the proto formatted schema from an OpenAPI spec -func ToProtoSchema(openAPIDefinitions *spec.Definitions, gvk schema.GroupVersionKind) (proto.Schema, error) { - openAPISpec := newMinimalValidOpenAPISpec() - openAPISpec.Definitions = *openAPIDefinitions - +// ToProtoModels builds the proto formatted models from OpenAPI spec +func ToProtoModels(openAPISpec *spec.Swagger) (proto.Models, error) { specBytes, err := json.MarshalIndent(openAPISpec, " ", " ") if err != nil { return nil, err @@ -62,81 +50,5 @@ func ToProtoSchema(openAPIDefinitions *spec.Definitions, gvk schema.GroupVersion return nil, err } - for _, modelName := range models.ListModels() { - model := models.LookupModel(modelName) - if model == nil { - return nil, fmt.Errorf("the ListModels function returned a model that can't be looked-up") - } - gvkList := parseGroupVersionKind(model) - for _, modelGVK := range gvkList { - if modelGVK == gvk { - return model, nil - } - } - } - - return nil, fmt.Errorf("no model found with a %v tag matching %v", groupVersionKindExtensionKey, gvk) -} - -// newMinimalValidOpenAPISpec creates a minimal openapi spec with only the required fields filled in -func newMinimalValidOpenAPISpec() *spec.Swagger { - return &spec.Swagger{ - SwaggerProps: spec.SwaggerProps{ - Swagger: "2.0", - Info: &spec.Info{ - InfoProps: spec.InfoProps{ - Title: "Kubernetes", - Version: "0.0.0", - }, - }, - }, - } -} - -// parseGroupVersionKind gets and parses GroupVersionKind from the extension. Returns empty if it doesn't have one. -func parseGroupVersionKind(s proto.Schema) []schema.GroupVersionKind { - extensions := s.GetExtensions() - - gvkListResult := []schema.GroupVersionKind{} - - // Get the extensions - gvkExtension, ok := extensions[groupVersionKindExtensionKey] - if !ok { - return []schema.GroupVersionKind{} - } - - // gvk extension must be a list of at least 1 element. - gvkList, ok := gvkExtension.([]interface{}) - if !ok { - return []schema.GroupVersionKind{} - } - - for _, gvk := range gvkList { - // gvk extension list must be a map with group, version, and - // kind fields - gvkMap, ok := gvk.(map[interface{}]interface{}) - if !ok { - continue - } - group, ok := gvkMap["group"].(string) - if !ok { - continue - } - version, ok := gvkMap["version"].(string) - if !ok { - continue - } - kind, ok := gvkMap["kind"].(string) - if !ok { - continue - } - - gvkListResult = append(gvkListResult, schema.GroupVersionKind{ - Group: group, - Version: version, - Kind: kind, - }) - } - - return gvkListResult + return models, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/util/openapi/proto_test.go b/staging/src/k8s.io/apiserver/pkg/util/openapi/proto_test.go index 64421a7ff8f..d4ee737caa2 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/openapi/proto_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/openapi/proto_test.go @@ -22,36 +22,41 @@ import ( "github.com/go-openapi/spec" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kube-openapi/pkg/util/proto" ) // TestOpenAPIDefinitionsToProtoSchema tests the openapi parser -func TestOpenAPIDefinitionsToProtoSchema(t *testing.T) { - openAPIDefinitions := &spec.Definitions{ - "io.k8s.api.testgroup.v1.Foo": spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "Description of Foos", - Properties: map[string]spec.Schema{}, +func TestOpenAPIDefinitionsToProtoModels(t *testing.T) { + openAPISpec := &spec.Swagger{ + SwaggerProps: spec.SwaggerProps{ + Swagger: "2.0", + Info: &spec.Info{ + InfoProps: spec.InfoProps{ + Title: "Kubernetes", + Version: "0.0.0", + }, }, - VendorExtensible: spec.VendorExtensible{ - Extensions: spec.Extensions{ - "x-kubernetes-group-version-kind": []map[string]string{ - { - "group": "testgroup.k8s.io", - "version": "v1", - "kind": "Foo", + Definitions: spec.Definitions{ + "io.k8s.api.testgroup.v1.Foo": spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Description of Foos", + Properties: map[string]spec.Schema{}, + }, + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-group-version-kind": []map[string]string{ + { + "group": "testgroup.k8s.io", + "version": "v1", + "kind": "Foo", + }, + }, }, }, }, }, }, } - gvk := schema.GroupVersionKind{ - Group: "testgroup.k8s.io", - Version: "v1", - Kind: "Foo", - } expectedSchema := &proto.Arbitrary{ BaseSchema: proto.BaseSchema{ Description: "Description of Foos", @@ -67,10 +72,11 @@ func TestOpenAPIDefinitionsToProtoSchema(t *testing.T) { Path: proto.NewPath("io.k8s.api.testgroup.v1.Foo"), }, } - actualSchema, err := ToProtoSchema(openAPIDefinitions, gvk) + protoModels, err := ToProtoModels(openAPISpec) if err != nil { - t.Fatalf("expected ToProtoSchema not to return an error") + t.Fatalf("expected ToProtoModels not to return an error") } + actualSchema := protoModels.LookupModel("io.k8s.api.testgroup.v1.Foo") if !reflect.DeepEqual(expectedSchema, actualSchema) { t.Fatalf("expected schema:\n%v\nbut got:\n%v", expectedSchema, actualSchema) } diff --git a/staging/src/k8s.io/apiserver/pkg/util/trace/BUILD b/staging/src/k8s.io/apiserver/pkg/util/trace/BUILD index 2e94b1d0d42..4e93ff5b498 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/trace/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/util/trace/BUILD @@ -10,7 +10,7 @@ go_library( srcs = ["trace.go"], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/util/trace", importpath = "k8s.io/apiserver/pkg/util/trace", - deps = ["//vendor/github.com/golang/glog:go_default_library"], + deps = ["//vendor/k8s.io/klog:go_default_library"], ) filegroup( diff --git a/staging/src/k8s.io/apiserver/pkg/util/trace/trace.go b/staging/src/k8s.io/apiserver/pkg/util/trace/trace.go index b2f31c5275d..9049a17d0de 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/trace/trace.go +++ b/staging/src/k8s.io/apiserver/pkg/util/trace/trace.go @@ -22,7 +22,7 @@ import ( "math/rand" "time" - "github.com/golang/glog" + "k8s.io/klog" ) type traceStep struct { @@ -63,17 +63,17 @@ func (t *Trace) logWithStepThreshold(stepThreshold time.Duration) { lastStepTime := t.startTime for _, step := range t.steps { stepDuration := step.stepTime.Sub(lastStepTime) - if stepThreshold == 0 || stepDuration > stepThreshold || glog.V(4) { + if stepThreshold == 0 || stepDuration > stepThreshold || klog.V(4) { buffer.WriteString(fmt.Sprintf("Trace[%d]: [%v] [%v] %v\n", tracenum, step.stepTime.Sub(t.startTime), stepDuration, step.msg)) } lastStepTime = step.stepTime } stepDuration := endTime.Sub(lastStepTime) - if stepThreshold == 0 || stepDuration > stepThreshold || glog.V(4) { + if stepThreshold == 0 || stepDuration > stepThreshold || klog.V(4) { buffer.WriteString(fmt.Sprintf("Trace[%d]: [%v] [%v] END\n", tracenum, endTime.Sub(t.startTime), stepDuration)) } - glog.Info(buffer.String()) + klog.Info(buffer.String()) } func (t *Trace) LogIfLong(threshold time.Duration) { diff --git a/staging/src/k8s.io/apiserver/pkg/util/wsstream/BUILD b/staging/src/k8s.io/apiserver/pkg/util/wsstream/BUILD index 269af15e5ce..5367ab2f5c6 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/wsstream/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/util/wsstream/BUILD @@ -27,8 +27,8 @@ go_library( importpath = "k8s.io/apiserver/pkg/util/wsstream", deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/net/websocket:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/util/wsstream/conn.go b/staging/src/k8s.io/apiserver/pkg/util/wsstream/conn.go index 05faf48eba1..2d1a7902136 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/wsstream/conn.go +++ b/staging/src/k8s.io/apiserver/pkg/util/wsstream/conn.go @@ -25,8 +25,8 @@ import ( "strings" "time" - "github.com/golang/glog" "golang.org/x/net/websocket" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/runtime" ) @@ -251,7 +251,7 @@ func (conn *Conn) handle(ws *websocket.Conn) { var data []byte if err := websocket.Message.Receive(ws, &data); err != nil { if err != io.EOF { - glog.Errorf("Error on socket receive: %v", err) + klog.Errorf("Error on socket receive: %v", err) } break } @@ -264,11 +264,11 @@ func (conn *Conn) handle(ws *websocket.Conn) { } data = data[1:] if int(channel) >= len(conn.channels) { - glog.V(6).Infof("Frame is targeted for a reader %d that is not valid, possible protocol error", channel) + klog.V(6).Infof("Frame is targeted for a reader %d that is not valid, possible protocol error", channel) continue } if _, err := conn.channels[channel].DataFromSocket(data); err != nil { - glog.Errorf("Unable to write frame to %d: %v\n%s", channel, err, string(data)) + klog.Errorf("Unable to write frame to %d: %v\n%s", channel, err, string(data)) continue } } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/audit/BUILD index 12df108356d..c2c4d07fcdf 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/audit/BUILD +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/BUILD @@ -24,6 +24,7 @@ filegroup( srcs = [ ":package-srcs", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/buffered:all-srcs", + "//staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic:all-srcs", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/fake:all-srcs", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/log:all-srcs", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/truncate:all-srcs", diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/OWNERS b/staging/src/k8s.io/apiserver/plugin/pkg/audit/OWNERS new file mode 100644 index 00000000000..178ce84a5ce --- /dev/null +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-audit-approvers +reviewers: +- sig-auth-audit-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/BUILD new file mode 100644 index 00000000000..a838ddc87c7 --- /dev/null +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/BUILD @@ -0,0 +1,75 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "defaults.go", + "dynamic.go", + "factory.go", + ], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/plugin/pkg/audit/dynamic", + importpath = "k8s.io/apiserver/plugin/pkg/audit/dynamic", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/api/auditregistration/v1alpha1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/audit/install:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/audit:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/audit/policy:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/audit/util:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", + "//staging/src/k8s.io/apiserver/plugin/pkg/audit/buffered:go_default_library", + "//staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced:go_default_library", + "//staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook:go_default_library", + "//staging/src/k8s.io/client-go/informers/auditregistration/v1alpha1:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "dynamic_test.go", + "factory_test.go", + ], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/api/auditregistration/v1alpha1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/audit:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", + "//vendor/github.com/stretchr/testify/require:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/defaults.go b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/defaults.go new file mode 100644 index 00000000000..f442954b500 --- /dev/null +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/defaults.go @@ -0,0 +1,46 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamic + +import ( + "time" + + bufferedplugin "k8s.io/apiserver/plugin/pkg/audit/buffered" +) + +const ( + // Default configuration values for ModeBatch when applied to a dynamic plugin + defaultBatchBufferSize = 5000 // Buffer up to 5000 events before starting discarding. + defaultBatchMaxSize = 400 // Only send up to 400 events at a time. + defaultBatchMaxWait = 30 * time.Second // Send events at least twice a minute. + defaultBatchThrottleQPS = 10 // Limit the send rate by 10 QPS. + defaultBatchThrottleBurst = 15 // Allow up to 15 QPS burst. +) + +// NewDefaultWebhookBatchConfig returns new Batch Config objects populated by default values +// for dynamic webhooks +func NewDefaultWebhookBatchConfig() *bufferedplugin.BatchConfig { + return &bufferedplugin.BatchConfig{ + BufferSize: defaultBatchBufferSize, + MaxBatchSize: defaultBatchMaxSize, + MaxBatchWait: defaultBatchMaxWait, + ThrottleEnable: true, + ThrottleQPS: defaultBatchThrottleQPS, + ThrottleBurst: defaultBatchThrottleBurst, + AsyncDelegate: true, + } +} diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/dynamic.go b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/dynamic.go new file mode 100644 index 00000000000..a8347aa3e72 --- /dev/null +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/dynamic.go @@ -0,0 +1,335 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamic + +import ( + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" + + "k8s.io/klog" + + auditregv1alpha1 "k8s.io/api/auditregistration/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + auditinternal "k8s.io/apiserver/pkg/apis/audit" + auditinstall "k8s.io/apiserver/pkg/apis/audit/install" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" + "k8s.io/apiserver/pkg/audit" + webhook "k8s.io/apiserver/pkg/util/webhook" + bufferedplugin "k8s.io/apiserver/plugin/pkg/audit/buffered" + auditinformer "k8s.io/client-go/informers/auditregistration/v1alpha1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" +) + +// PluginName is the name reported in error metrics. +const PluginName = "dynamic" + +// Config holds the configuration for the dynamic backend +type Config struct { + // Informer for the audit sinks + Informer auditinformer.AuditSinkInformer + // EventConfig holds the configuration for event notifications about the AuditSink API objects + EventConfig EventConfig + // BufferedConfig is the runtime buffered configuration + BufferedConfig *bufferedplugin.BatchConfig + // WebhookConfig holds the configuration for outgoing webhooks + WebhookConfig WebhookConfig +} + +// WebhookConfig holds the configurations for outgoing webhooks +type WebhookConfig struct { + // AuthInfoResolverWrapper provides the webhook authentication for in-cluster endpoints + AuthInfoResolverWrapper webhook.AuthenticationInfoResolverWrapper + // ServiceResolver knows how to convert a webhook service reference into an actual location. + ServiceResolver webhook.ServiceResolver +} + +// EventConfig holds the configurations for sending event notifiations about AuditSink API objects +type EventConfig struct { + // Sink for emitting events + Sink record.EventSink + // Source holds the source information about the event emitter + Source corev1.EventSource +} + +// delegate represents a delegate backend that was created from an audit sink configuration +type delegate struct { + audit.Backend + configuration *auditregv1alpha1.AuditSink + stopChan chan struct{} +} + +// gracefulShutdown will gracefully shutdown the delegate +func (d *delegate) gracefulShutdown() { + close(d.stopChan) + d.Shutdown() +} + +// NewBackend returns a backend that dynamically updates its configuration +// based on a shared informer. +func NewBackend(c *Config) (audit.Backend, error) { + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(klog.Infof) + eventBroadcaster.StartRecordingToSink(c.EventConfig.Sink) + + scheme := runtime.NewScheme() + err := auditregv1alpha1.AddToScheme(scheme) + if err != nil { + return nil, err + } + recorder := eventBroadcaster.NewRecorder(scheme, c.EventConfig.Source) + + if c.BufferedConfig == nil { + c.BufferedConfig = NewDefaultWebhookBatchConfig() + } + cm, err := webhook.NewClientManager(auditv1.SchemeGroupVersion, func(s *runtime.Scheme) error { + auditinstall.Install(s) + return nil + }) + if err != nil { + return nil, err + } + + // TODO: need a way of injecting authentication before beta + authInfoResolver, err := webhook.NewDefaultAuthenticationInfoResolver("") + if err != nil { + return nil, err + } + cm.SetAuthenticationInfoResolver(authInfoResolver) + cm.SetServiceResolver(c.WebhookConfig.ServiceResolver) + cm.SetAuthenticationInfoResolverWrapper(c.WebhookConfig.AuthInfoResolverWrapper) + + manager := &backend{ + config: c, + delegates: atomic.Value{}, + delegateUpdateMutex: sync.Mutex{}, + webhookClientManager: cm, + recorder: recorder, + } + manager.delegates.Store(syncedDelegates{}) + + c.Informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + manager.addSink(obj.(*auditregv1alpha1.AuditSink)) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + manager.updateSink(oldObj.(*auditregv1alpha1.AuditSink), newObj.(*auditregv1alpha1.AuditSink)) + }, + DeleteFunc: func(obj interface{}) { + sink, ok := obj.(*auditregv1alpha1.AuditSink) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + klog.V(2).Infof("Couldn't get object from tombstone %#v", obj) + return + } + sink, ok = tombstone.Obj.(*auditregv1alpha1.AuditSink) + if !ok { + klog.V(2).Infof("Tombstone contained object that is not an AuditSink: %#v", obj) + return + } + } + manager.deleteSink(sink) + }, + }) + + return manager, nil +} + +type backend struct { + // delegateUpdateMutex holds an update lock on the delegates + delegateUpdateMutex sync.Mutex + config *Config + delegates atomic.Value + webhookClientManager webhook.ClientManager + recorder record.EventRecorder +} + +type syncedDelegates map[types.UID]*delegate + +// Names returns the names of the delegate configurations +func (s syncedDelegates) Names() []string { + names := []string{} + for _, delegate := range s { + names = append(names, delegate.configuration.Name) + } + return names +} + +// ProcessEvents proccesses the given events per current delegate map +func (b *backend) ProcessEvents(events ...*auditinternal.Event) { + for _, d := range b.GetDelegates() { + d.ProcessEvents(events...) + } +} + +// Run starts a goroutine that propagates the shutdown signal, +// individual delegates are ran as they are created. +func (b *backend) Run(stopCh <-chan struct{}) error { + go func() { + <-stopCh + b.stopAllDelegates() + }() + return nil +} + +// stopAllDelegates closes the stopChan for every delegate to enable +// goroutines to terminate gracefully. This is a helper method to propagate +// the primary stopChan to the current delegate map. +func (b *backend) stopAllDelegates() { + b.delegateUpdateMutex.Lock() + for _, d := range b.GetDelegates() { + close(d.stopChan) + } +} + +// Shutdown calls the shutdown method on all delegates. The stopChan should +// be closed before this is called. +func (b *backend) Shutdown() { + for _, d := range b.GetDelegates() { + d.Shutdown() + } +} + +// GetDelegates retrieves current delegates in a safe manner +func (b *backend) GetDelegates() syncedDelegates { + return b.delegates.Load().(syncedDelegates) +} + +// copyDelegates returns a copied delegate map +func (b *backend) copyDelegates() syncedDelegates { + c := make(syncedDelegates) + for u, s := range b.GetDelegates() { + c[u] = s + } + return c +} + +// setDelegates sets the current delegates in a safe manner +func (b *backend) setDelegates(delegates syncedDelegates) { + b.delegates.Store(delegates) +} + +// addSink is called by the shared informer when a sink is added +func (b *backend) addSink(sink *auditregv1alpha1.AuditSink) { + b.delegateUpdateMutex.Lock() + defer b.delegateUpdateMutex.Unlock() + delegates := b.copyDelegates() + if _, ok := delegates[sink.UID]; ok { + klog.Errorf("Audit sink %q uid: %s already exists, could not readd", sink.Name, sink.UID) + return + } + d, err := b.createAndStartDelegate(sink) + if err != nil { + msg := fmt.Sprintf("Could not add audit sink %q: %v", sink.Name, err) + klog.Error(msg) + b.recorder.Event(sink, corev1.EventTypeWarning, "CreateFailed", msg) + return + } + delegates[sink.UID] = d + b.setDelegates(delegates) + klog.V(2).Infof("Added audit sink: %s", sink.Name) + klog.V(2).Infof("Current audit sinks: %v", delegates.Names()) +} + +// updateSink is called by the shared informer when a sink is updated. +// The new sink is only rebuilt on spec changes. The new sink must not have +// the same uid as the previous. The new sink will be started before the old +// one is shutdown so no events will be lost +func (b *backend) updateSink(oldSink, newSink *auditregv1alpha1.AuditSink) { + b.delegateUpdateMutex.Lock() + defer b.delegateUpdateMutex.Unlock() + delegates := b.copyDelegates() + oldDelegate, ok := delegates[oldSink.UID] + if !ok { + klog.Errorf("Could not update audit sink %q uid: %s, old sink does not exist", + oldSink.Name, oldSink.UID) + return + } + + // check if spec has changed + eq := reflect.DeepEqual(oldSink.Spec, newSink.Spec) + if eq { + delete(delegates, oldSink.UID) + delegates[newSink.UID] = oldDelegate + b.setDelegates(delegates) + } else { + d, err := b.createAndStartDelegate(newSink) + if err != nil { + msg := fmt.Sprintf("Could not update audit sink %q: %v", oldSink.Name, err) + klog.Error(msg) + b.recorder.Event(newSink, corev1.EventTypeWarning, "UpdateFailed", msg) + return + } + delete(delegates, oldSink.UID) + delegates[newSink.UID] = d + b.setDelegates(delegates) + oldDelegate.gracefulShutdown() + } + + klog.V(2).Infof("Updated audit sink: %s", newSink.Name) + klog.V(2).Infof("Current audit sinks: %v", delegates.Names()) +} + +// deleteSink is called by the shared informer when a sink is deleted +func (b *backend) deleteSink(sink *auditregv1alpha1.AuditSink) { + b.delegateUpdateMutex.Lock() + defer b.delegateUpdateMutex.Unlock() + delegates := b.copyDelegates() + delegate, ok := delegates[sink.UID] + if !ok { + klog.Errorf("Could not delete audit sink %q uid: %s, does not exist", sink.Name, sink.UID) + return + } + delete(delegates, sink.UID) + b.setDelegates(delegates) + delegate.gracefulShutdown() + klog.V(2).Infof("Deleted audit sink: %s", sink.Name) + klog.V(2).Infof("Current audit sinks: %v", delegates.Names()) +} + +// createAndStartDelegate will build a delegate from an audit sink configuration and run it +func (b *backend) createAndStartDelegate(sink *auditregv1alpha1.AuditSink) (*delegate, error) { + f := factory{ + config: b.config, + webhookClientManager: b.webhookClientManager, + sink: sink, + } + delegate, err := f.BuildDelegate() + if err != nil { + return nil, err + } + err = delegate.Run(delegate.stopChan) + if err != nil { + return nil, err + } + return delegate, nil +} + +// String returns a string representation of the backend +func (b *backend) String() string { + var delegateStrings []string + for _, delegate := range b.GetDelegates() { + delegateStrings = append(delegateStrings, fmt.Sprintf("%s", delegate)) + } + return fmt.Sprintf("%s[%s]", PluginName, strings.Join(delegateStrings, ",")) +} diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/dynamic_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/dynamic_test.go new file mode 100644 index 00000000000..8b9c91407a3 --- /dev/null +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/dynamic_test.go @@ -0,0 +1,275 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamic + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "reflect" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + auditregv1alpha1 "k8s.io/api/auditregistration/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + auditinternal "k8s.io/apiserver/pkg/apis/audit" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" + "k8s.io/apiserver/pkg/audit" + webhook "k8s.io/apiserver/pkg/util/webhook" + informers "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" +) + +func TestDynamic(t *testing.T) { + eventList1 := &atomic.Value{} + eventList1.Store(auditinternal.EventList{}) + eventList2 := &atomic.Value{} + eventList2.Store(auditinternal.EventList{}) + + // start test servers + server1 := httptest.NewServer(buildTestHandler(t, eventList1)) + defer server1.Close() + server2 := httptest.NewServer(buildTestHandler(t, eventList2)) + defer server2.Close() + + testPolicy := auditregv1alpha1.Policy{ + Level: auditregv1alpha1.LevelMetadata, + Stages: []auditregv1alpha1.Stage{ + auditregv1alpha1.StageResponseStarted, + }, + } + testEvent := auditinternal.Event{ + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseStarted, + Verb: "get", + RequestURI: "/test/path", + } + testConfig1 := &auditregv1alpha1.AuditSink{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test1", + UID: types.UID("test1"), + }, + Spec: auditregv1alpha1.AuditSinkSpec{ + Policy: testPolicy, + Webhook: auditregv1alpha1.Webhook{ + ClientConfig: auditregv1alpha1.WebhookClientConfig{ + URL: &server1.URL, + }, + }, + }, + } + testConfig2 := &auditregv1alpha1.AuditSink{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test2", + UID: types.UID("test2"), + }, + Spec: auditregv1alpha1.AuditSinkSpec{ + Policy: testPolicy, + Webhook: auditregv1alpha1.Webhook{ + ClientConfig: auditregv1alpha1.WebhookClientConfig{ + URL: &server2.URL, + }, + }, + }, + } + + config, stopChan := defaultTestConfig() + config.BufferedConfig.MaxBatchWait = 10 * time.Millisecond + + b, err := NewBackend(config) + require.NoError(t, err) + d := b.(*backend) + err = b.Run(stopChan) + require.NoError(t, err) + + t.Run("find none", func(t *testing.T) { + require.Len(t, d.GetDelegates(), 0) + }) + + t.Run("find one", func(t *testing.T) { + d.addSink(testConfig1) + delegates := d.GetDelegates() + require.Len(t, delegates, 1) + require.Contains(t, delegates, types.UID("test1")) + require.Equal(t, testConfig1, delegates["test1"].configuration) + + // send event and check that it arrives + b.ProcessEvents(&testEvent) + err := checkForEvent(eventList1, testEvent) + require.NoError(t, err, "unable to find events sent to sink") + }) + + t.Run("find two", func(t *testing.T) { + eventList1.Store(auditinternal.EventList{}) + d.addSink(testConfig2) + delegates := d.GetDelegates() + require.Len(t, delegates, 2) + require.Contains(t, delegates, types.UID("test1")) + require.Contains(t, delegates, types.UID("test2")) + require.Equal(t, testConfig1, delegates["test1"].configuration) + require.Equal(t, testConfig2, delegates["test2"].configuration) + + // send event to both delegates and check that it arrives in both places + b.ProcessEvents(&testEvent) + err := checkForEvent(eventList1, testEvent) + require.NoError(t, err, "unable to find events sent to sink 1") + err = checkForEvent(eventList2, testEvent) + require.NoError(t, err, "unable to find events sent to sink 2") + }) + + t.Run("delete one", func(t *testing.T) { + eventList2.Store(auditinternal.EventList{}) + d.deleteSink(testConfig1) + delegates := d.GetDelegates() + require.Len(t, delegates, 1) + require.Contains(t, delegates, types.UID("test2")) + require.Equal(t, testConfig2, delegates["test2"].configuration) + + // send event and check that it arrives to remaining sink + b.ProcessEvents(&testEvent) + err := checkForEvent(eventList2, testEvent) + require.NoError(t, err, "unable to find events sent to sink") + }) + + t.Run("update one", func(t *testing.T) { + eventList1.Store(auditinternal.EventList{}) + oldConfig := *testConfig2 + testConfig2.Spec.Webhook.ClientConfig.URL = &server1.URL + testConfig2.UID = types.UID("test2.1") + d.updateSink(&oldConfig, testConfig2) + delegates := d.GetDelegates() + require.Len(t, delegates, 1) + require.Contains(t, delegates, types.UID("test2.1")) + require.Equal(t, testConfig2, delegates["test2.1"].configuration) + + // send event and check that it arrives to updated sink + b.ProcessEvents(&testEvent) + err := checkForEvent(eventList1, testEvent) + require.NoError(t, err, "unable to find events sent to sink") + }) + + t.Run("update meta only", func(t *testing.T) { + eventList1.Store(auditinternal.EventList{}) + oldConfig := *testConfig2 + testConfig2.UID = types.UID("test2.2") + testConfig2.Labels = map[string]string{"my": "label"} + d.updateSink(&oldConfig, testConfig2) + delegates := d.GetDelegates() + require.Len(t, delegates, 1) + require.Contains(t, delegates, types.UID("test2.2")) + + // send event and check that it arrives to same sink + b.ProcessEvents(&testEvent) + err := checkForEvent(eventList1, testEvent) + require.NoError(t, err, "unable to find events sent to sink") + }) + + t.Run("shutdown", func(t *testing.T) { + // if the stop signal is not propagated correctly the buffers will not + // close down gracefully, and the shutdown method will hang causing + // the test will timeout. + timeoutChan := make(chan struct{}) + successChan := make(chan struct{}) + go func() { + time.Sleep(1 * time.Second) + timeoutChan <- struct{}{} + }() + go func() { + close(stopChan) + d.Shutdown() + successChan <- struct{}{} + }() + for { + select { + case <-timeoutChan: + t.Error("shutdown timed out") + return + case <-successChan: + return + } + } + }) +} + +// checkForEvent will poll to check for an audit event in an atomic event list +func checkForEvent(a *atomic.Value, evSent auditinternal.Event) error { + return wait.Poll(100*time.Millisecond, 1*time.Second, func() (bool, error) { + el := a.Load().(auditinternal.EventList) + if len(el.Items) != 1 { + return false, nil + } + evFound := el.Items[0] + eq := reflect.DeepEqual(evSent, evFound) + if !eq { + return false, fmt.Errorf("event mismatch -- sent: %+v found: %+v", evSent, evFound) + } + return true, nil + }) +} + +// buildTestHandler returns a handler that will update the atomic value passed in +// with the event list it receives +func buildTestHandler(t *testing.T, a *atomic.Value) http.HandlerFunc { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Fatalf("could not read request body: %v", err) + } + el := auditinternal.EventList{} + decoder := audit.Codecs.UniversalDecoder(auditv1.SchemeGroupVersion) + if err := runtime.DecodeInto(decoder, body, &el); err != nil { + t.Fatalf("failed decoding buf: %b, apiVersion: %s", body, auditv1.SchemeGroupVersion) + } + defer r.Body.Close() + a.Store(el) + w.WriteHeader(200) + }) +} + +// defaultTestConfig returns a Config object suitable for testing along with its +// associated stopChan +func defaultTestConfig() (*Config, chan struct{}) { + authWrapper := webhook.AuthenticationInfoResolverWrapper( + func(a webhook.AuthenticationInfoResolver) webhook.AuthenticationInfoResolver { return a }, + ) + client := fake.NewSimpleClientset() + informerFactory := informers.NewSharedInformerFactory(client, 0) + stop := make(chan struct{}) + + eventSink := &v1core.EventSinkImpl{Interface: client.CoreV1().Events("")} + + informerFactory.Start(stop) + informerFactory.WaitForCacheSync(stop) + informer := informerFactory.Auditregistration().V1alpha1().AuditSinks() + return &Config{ + Informer: informer, + EventConfig: EventConfig{Sink: eventSink}, + BufferedConfig: NewDefaultWebhookBatchConfig(), + WebhookConfig: WebhookConfig{ + AuthInfoResolverWrapper: authWrapper, + ServiceResolver: webhook.NewDefaultServiceResolver(), + }, + }, stop +} diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced/BUILD new file mode 100644 index 00000000000..c9c25ef4bda --- /dev/null +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced/BUILD @@ -0,0 +1,44 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["enforced.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced", + importpath = "k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/audit:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/audit/event:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/audit/policy:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["enforced_test.go"], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/audit/policy:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", + "//staging/src/k8s.io/apiserver/plugin/pkg/audit/fake:go_default_library", + "//vendor/github.com/stretchr/testify/require:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced/enforced.go b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced/enforced.go new file mode 100644 index 00000000000..1263a59ce61 --- /dev/null +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced/enforced.go @@ -0,0 +1,90 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package enforced + +import ( + "fmt" + + auditinternal "k8s.io/apiserver/pkg/apis/audit" + "k8s.io/apiserver/pkg/audit" + ev "k8s.io/apiserver/pkg/audit/event" + "k8s.io/apiserver/pkg/audit/policy" +) + +// PluginName is the name reported in error metrics. +const PluginName = "enforced" + +// Backend filters audit events according to the policy +// trimming them as necessary to match the level +type Backend struct { + policyChecker policy.Checker + delegateBackend audit.Backend +} + +// NewBackend returns an enforced audit backend that wraps delegate backend. +// Enforced backend automatically runs and shuts down the delegate backend. +func NewBackend(delegate audit.Backend, p policy.Checker) audit.Backend { + return &Backend{ + policyChecker: p, + delegateBackend: delegate, + } +} + +// Run the delegate backend +func (b Backend) Run(stopCh <-chan struct{}) error { + return b.delegateBackend.Run(stopCh) +} + +// Shutdown the delegate backend +func (b Backend) Shutdown() { + b.delegateBackend.Shutdown() +} + +// ProcessEvents enforces policy on a shallow copy of the given event +// dropping any sections that don't conform +func (b Backend) ProcessEvents(events ...*auditinternal.Event) { + for _, event := range events { + if event == nil { + continue + } + attr, err := ev.NewAttributes(event) + if err != nil { + audit.HandlePluginError(PluginName, err, event) + continue + } + level, stages := b.policyChecker.LevelAndStages(attr) + if level == auditinternal.LevelNone { + continue + } + // make shallow copy before modifying to satisfy interface definition + ev := *event + e, err := policy.EnforcePolicy(&ev, level, stages) + if err != nil { + audit.HandlePluginError(PluginName, err, event) + continue + } + if e == nil { + continue + } + b.delegateBackend.ProcessEvents(e) + } +} + +// String returns a string representation of the backend +func (b Backend) String() string { + return fmt.Sprintf("%s<%s>", PluginName, b.delegateBackend) +} diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced/enforced_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced/enforced_test.go new file mode 100644 index 00000000000..cbc61327a27 --- /dev/null +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced/enforced_test.go @@ -0,0 +1,117 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package enforced + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "k8s.io/apimachinery/pkg/runtime" + auditinternal "k8s.io/apiserver/pkg/apis/audit" + "k8s.io/apiserver/pkg/audit/policy" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/authorization/authorizer" + fakeplugin "k8s.io/apiserver/plugin/pkg/audit/fake" +) + +func TestEnforced(t *testing.T) { + testCases := []struct { + name string + event *auditinternal.Event + policy auditinternal.Policy + attribs authorizer.Attributes + expected []*auditinternal.Event + }{ + { + name: "enforce level", + event: &auditinternal.Event{ + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: "/apis/extensions/v1beta1", + RequestObject: &runtime.Unknown{Raw: []byte(`test`)}, + ResponseObject: &runtime.Unknown{Raw: []byte(`test`)}, + }, + policy: auditinternal.Policy{ + Rules: []auditinternal.PolicyRule{ + { + Level: auditinternal.LevelMetadata, + }, + }, + }, + expected: []*auditinternal.Event{ + { + Level: auditinternal.LevelMetadata, + Stage: auditinternal.StageResponseComplete, + RequestURI: "/apis/extensions/v1beta1", + }, + }, + }, + { + name: "enforce policy rule", + event: &auditinternal.Event{ + Level: auditinternal.LevelRequestResponse, + Stage: auditinternal.StageResponseComplete, + RequestURI: "/apis/extensions/v1beta1", + User: auditinternal.UserInfo{ + Username: user.Anonymous, + }, + RequestObject: &runtime.Unknown{Raw: []byte(`test`)}, + ResponseObject: &runtime.Unknown{Raw: []byte(`test`)}, + }, + policy: auditinternal.Policy{ + Rules: []auditinternal.PolicyRule{ + { + Level: auditinternal.LevelNone, + Users: []string{user.Anonymous}, + }, + { + Level: auditinternal.LevelMetadata, + }, + }, + }, + expected: []*auditinternal.Event{}, + }, + { + name: "nil event", + event: nil, + policy: auditinternal.Policy{ + Rules: []auditinternal.PolicyRule{ + { + Level: auditinternal.LevelMetadata, + }, + }, + }, + expected: []*auditinternal.Event{}, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ev := []*auditinternal.Event{} + fakeBackend := fakeplugin.Backend{ + OnRequest: func(events []*auditinternal.Event) { + ev = events + }, + } + b := NewBackend(&fakeBackend, policy.NewChecker(&tc.policy)) + defer b.Shutdown() + + b.ProcessEvents(tc.event) + require.Equal(t, tc.expected, ev) + }) + } +} diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/factory.go b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/factory.go new file mode 100644 index 00000000000..f9ce7abf790 --- /dev/null +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/factory.go @@ -0,0 +1,91 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamic + +import ( + "fmt" + "time" + + auditregv1alpha1 "k8s.io/api/auditregistration/v1alpha1" + "k8s.io/apiserver/pkg/audit" + "k8s.io/apiserver/pkg/audit/policy" + auditutil "k8s.io/apiserver/pkg/audit/util" + "k8s.io/apiserver/pkg/util/webhook" + bufferedplugin "k8s.io/apiserver/plugin/pkg/audit/buffered" + enforcedplugin "k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced" + webhookplugin "k8s.io/apiserver/plugin/pkg/audit/webhook" +) + +// TODO: find a common place for all the default retry backoffs +const retryBackoff = 500 * time.Millisecond + +// factory builds a delegate from an AuditSink +type factory struct { + config *Config + webhookClientManager webhook.ClientManager + sink *auditregv1alpha1.AuditSink +} + +// BuildDelegate creates a delegate from the AuditSink object +func (f *factory) BuildDelegate() (*delegate, error) { + backend, err := f.buildWebhookBackend() + if err != nil { + return nil, err + } + backend = f.applyEnforcedOpts(backend) + backend = f.applyBufferedOpts(backend) + ch := make(chan struct{}) + return &delegate{ + Backend: backend, + configuration: f.sink, + stopChan: ch, + }, nil +} + +func (f *factory) buildWebhookBackend() (audit.Backend, error) { + hookClient := auditutil.HookClientConfigForSink(f.sink) + client, err := f.webhookClientManager.HookClient(hookClient) + if err != nil { + return nil, fmt.Errorf("could not create webhook client: %v", err) + } + backend := webhookplugin.NewDynamicBackend(client, retryBackoff) + return backend, nil +} + +func (f *factory) applyEnforcedOpts(delegate audit.Backend) audit.Backend { + pol := policy.ConvertDynamicPolicyToInternal(&f.sink.Spec.Policy) + checker := policy.NewChecker(pol) + eb := enforcedplugin.NewBackend(delegate, checker) + return eb +} + +func (f *factory) applyBufferedOpts(delegate audit.Backend) audit.Backend { + bc := f.config.BufferedConfig + tc := f.sink.Spec.Webhook.Throttle + if tc != nil { + bc.ThrottleEnable = true + if tc.Burst != nil { + bc.ThrottleBurst = int(*tc.Burst) + } + if tc.QPS != nil { + bc.ThrottleQPS = float32(*tc.QPS) + } + } else { + bc.ThrottleEnable = false + } + return bufferedplugin.NewBackend(delegate, *bc) +} diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/factory_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/factory_test.go new file mode 100644 index 00000000000..3caa2c3a5dc --- /dev/null +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic/factory_test.go @@ -0,0 +1,146 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamic + +import ( + "testing" + + "github.com/stretchr/testify/require" + + auditregv1alpha1 "k8s.io/api/auditregistration/v1alpha1" + utilpointer "k8s.io/utils/pointer" +) + +func TestToDelegate(t *testing.T) { + config, _ := defaultTestConfig() + defaultPolicy := auditregv1alpha1.Policy{ + Level: auditregv1alpha1.LevelMetadata, + } + u := "http://localhost:4444" + for _, tc := range []struct { + name string + auditConfig *auditregv1alpha1.AuditSink + throttleConfig *auditregv1alpha1.WebhookThrottleConfig + expectedBackend string + }{ + { + name: "build full", + auditConfig: &auditregv1alpha1.AuditSink{ + Spec: auditregv1alpha1.AuditSinkSpec{ + Policy: defaultPolicy, + Webhook: auditregv1alpha1.Webhook{ + Throttle: &auditregv1alpha1.WebhookThrottleConfig{ + QPS: utilpointer.Int64Ptr(10), + Burst: utilpointer.Int64Ptr(5), + }, + ClientConfig: auditregv1alpha1.WebhookClientConfig{ + URL: &u, + }, + }, + }, + }, + expectedBackend: "buffered>", + }, + { + name: "build no throttle", + auditConfig: &auditregv1alpha1.AuditSink{ + Spec: auditregv1alpha1.AuditSinkSpec{ + Policy: defaultPolicy, + Webhook: auditregv1alpha1.Webhook{ + ClientConfig: auditregv1alpha1.WebhookClientConfig{ + URL: &u, + }, + }, + }, + }, + expectedBackend: "buffered>", + }, + } { + t.Run(tc.name, func(t *testing.T) { + b, err := NewBackend(config) + require.NoError(t, err) + c := factory{ + config: b.(*backend).config, + webhookClientManager: b.(*backend).webhookClientManager, + sink: tc.auditConfig, + } + d, err := c.BuildDelegate() + require.NoError(t, err) + require.Equal(t, tc.expectedBackend, d.String()) + }) + } +} + +func TestBuildWebhookBackend(t *testing.T) { + defaultPolicy := auditregv1alpha1.Policy{ + Level: auditregv1alpha1.LevelMetadata, + } + config, _ := defaultTestConfig() + b, err := NewBackend(config) + require.NoError(t, err) + d := b.(*backend) + u := "http://localhost:4444" + for _, tc := range []struct { + name string + auditConfig *auditregv1alpha1.AuditSink + shouldErr bool + expectedBackend string + }{ + { + name: "build full", + auditConfig: &auditregv1alpha1.AuditSink{ + Spec: auditregv1alpha1.AuditSinkSpec{ + Policy: defaultPolicy, + Webhook: auditregv1alpha1.Webhook{ + ClientConfig: auditregv1alpha1.WebhookClientConfig{ + URL: &u, + }, + }, + }, + }, + expectedBackend: "dynamic_webhook", + shouldErr: false, + }, + { + name: "fail missing url", + auditConfig: &auditregv1alpha1.AuditSink{ + Spec: auditregv1alpha1.AuditSinkSpec{ + Policy: defaultPolicy, + Webhook: auditregv1alpha1.Webhook{ + ClientConfig: auditregv1alpha1.WebhookClientConfig{}, + }, + }, + }, + shouldErr: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + c := &factory{ + config: config, + webhookClientManager: d.webhookClientManager, + sink: tc.auditConfig, + } + ab, err := c.buildWebhookBackend() + if tc.shouldErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.expectedBackend, ab.String()) + }) + } +} diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook.go b/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook.go index 80b4842fd29..5f5f7169dc3 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook.go @@ -18,6 +18,7 @@ limitations under the License. package webhook import ( + "fmt" "time" "k8s.io/apimachinery/pkg/runtime/schema" @@ -47,7 +48,20 @@ func loadWebhook(configFile string, groupVersion schema.GroupVersion, initialBac } type backend struct { - w *webhook.GenericWebhook + w *webhook.GenericWebhook + name string +} + +// NewDynamicBackend returns an audit backend configured from a REST client that +// sends events over HTTP to an external service. +func NewDynamicBackend(rc *rest.RESTClient, initialBackoff time.Duration) audit.Backend { + return &backend{ + w: &webhook.GenericWebhook{ + RestClient: rc, + InitialBackoff: initialBackoff, + }, + name: fmt.Sprintf("dynamic_%s", PluginName), + } } // NewBackend returns an audit backend that sends events over HTTP to an external service. @@ -56,7 +70,7 @@ func NewBackend(kubeConfigFile string, groupVersion schema.GroupVersion, initial if err != nil { return nil, err } - return &backend{w}, nil + return &backend{w: w, name: PluginName}, nil } func (b *backend) Run(stopCh <-chan struct{}) error { @@ -69,7 +83,7 @@ func (b *backend) Shutdown() { func (b *backend) ProcessEvents(ev ...*auditinternal.Event) { if err := b.processEvents(ev...); err != nil { - audit.HandlePluginError(PluginName, err, ev...) + audit.HandlePluginError(b.String(), err, ev...) } } @@ -84,5 +98,5 @@ func (b *backend) processEvents(ev ...*auditinternal.Event) error { } func (b *backend) String() string { - return PluginName + return b.name } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/OWNERS b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/OWNERS new file mode 100644 index 00000000000..c607d2aa8c5 --- /dev/null +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authenticators-approvers +reviewers: +- sig-auth-authenticators-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile/BUILD index bb770700ada..fc5981bd6b5 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile/BUILD +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile/BUILD @@ -21,7 +21,7 @@ go_library( deps = [ "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile/passwordfile.go b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile/passwordfile.go index 8f291d191cc..3c54ed78e13 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile/passwordfile.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile/passwordfile.go @@ -24,7 +24,7 @@ import ( "os" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" @@ -72,7 +72,7 @@ func NewCSV(path string) (*PasswordAuthenticator, error) { } recordNum++ if _, exist := users[obj.info.Name]; exist { - glog.Warningf("duplicate username '%s' has been found in password file '%s', record number '%d'", obj.info.Name, path, recordNum) + klog.Warningf("duplicate username '%s' has been found in password file '%s', record number '%d'", obj.info.Name, path, recordNum) } users[obj.info.Name] = obj } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/BUILD index 3863bb823fd..df9f34382a9 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/BUILD +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/BUILD @@ -13,10 +13,11 @@ go_test( data = glob(["testdata/**"]), embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//vendor/github.com/coreos/go-oidc:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/gopkg.in/square/go-jose.v2:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -32,7 +33,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//vendor/github.com/coreos/go-oidc:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/oidc.go b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/oidc.go index f53fc2ddd4c..1888fa91d6d 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/oidc.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/oidc.go @@ -43,7 +43,7 @@ import ( "time" oidc "github.com/coreos/go-oidc" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/wait" @@ -78,6 +78,12 @@ type Options struct { // See: https://openid.net/specs/openid-connect-core-1_0.html#IDToken ClientID string + // APIAudiences are the audiences that the API server identitifes as. The + // (API audiences unioned with the ClientIDs) should have a non-empty + // intersection with the request's target audience. This preserves the + // behavior of the OIDC authenticator pre-introduction of API audiences. + APIAudiences authenticator.Audiences + // Path to a PEM encoded root certificate of the provider. CAFile string @@ -147,10 +153,10 @@ func newAsyncIDTokenVerifier(ctx context.Context, c *oidc.Config, iss string) *a // Polls indefinitely in an attempt to initialize the distributed claims // verifier, or until context canceled. initFn := func() (done bool, err error) { - glog.V(4).Infof("oidc authenticator: attempting init: iss=%v", iss) + klog.V(4).Infof("oidc authenticator: attempting init: iss=%v", iss) v, err := initVerifier(ctx, c, iss) if err != nil { - glog.Errorf("oidc authenticator: async token verifier for issuer: %q: %v", iss, err) + klog.Errorf("oidc authenticator: async token verifier for issuer: %q: %v", iss, err) return false, nil } t.m.Lock() @@ -188,6 +194,8 @@ type Authenticator struct { groupsClaim string groupsPrefix string requiredClaims map[string]string + clientIDs authenticator.Audiences + apiAudiences authenticator.Audiences // Contains an *oidc.IDTokenVerifier. Do not access directly use the // idTokenVerifier method. @@ -221,7 +229,7 @@ func New(opts Options) (*Authenticator, error) { go wait.PollUntil(time.Second*10, func() (done bool, err error) { provider, err := oidc.NewProvider(ctx, a.issuerURL) if err != nil { - glog.Errorf("oidc authenticator: initializing plugin: %v", err) + klog.Errorf("oidc authenticator: initializing plugin: %v", err) return false, nil } @@ -279,7 +287,7 @@ func newAuthenticator(opts Options, initVerifier func(ctx context.Context, a *Au return nil, fmt.Errorf("Failed to read the CA file: %v", err) } } else { - glog.Info("OIDC: No x509 certificates provided, will use host's root CA set") + klog.Info("OIDC: No x509 certificates provided, will use host's root CA set") } // Copied from http.DefaultTransport. @@ -317,6 +325,8 @@ func newAuthenticator(opts Options, initVerifier func(ctx context.Context, a *Au groupsClaim: opts.GroupsClaim, groupsPrefix: opts.GroupsPrefix, requiredClaims: opts.RequiredClaims, + clientIDs: authenticator.Audiences{opts.ClientID}, + apiAudiences: opts.APIAudiences, cancel: cancel, resolver: resolver, } @@ -532,6 +542,11 @@ func (r *claimResolver) resolve(endpoint endpoint, allClaims claims) error { } func (a *Authenticator) AuthenticateToken(ctx context.Context, token string) (*authenticator.Response, bool, error) { + if reqAuds, ok := authenticator.AudiencesFrom(ctx); ok { + if len(reqAuds.Intersect(a.clientIDs)) == 0 && len(reqAuds.Intersect(a.apiAudiences)) == 0 { + return nil, false, nil + } + } if !hasCorrectIssuer(a.issuerURL, token) { return nil, false, nil } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/oidc_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/oidc_test.go index 0ceb72e83b2..3f5db497e4d 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/oidc_test.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/oidc_test.go @@ -36,9 +36,11 @@ import ( "time" oidc "github.com/coreos/go-oidc" - "github.com/golang/glog" jose "gopkg.in/square/go-jose.v2" + + "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/klog" ) // utilities for loading JOSE keys. @@ -140,6 +142,7 @@ type claimsTest struct { wantInitErr bool claimToResponseMap map[string]string openIDConfig string + reqAudiences authenticator.Audiences } // Replace formats the contents of v into the provided template. @@ -148,7 +151,7 @@ func replace(tmpl string, v interface{}) string { buf := bytes.NewBuffer(nil) t.Execute(buf, &v) ret := buf.String() - glog.V(4).Infof("Replaced: %v into: %v", tmpl, ret) + klog.V(4).Infof("Replaced: %v into: %v", tmpl, ret) return ret } @@ -158,7 +161,7 @@ func replace(tmpl string, v interface{}) string { // responses that the server will return for each claim it is given. func newClaimServer(t *testing.T, keys jose.JSONWebKeySet, signer jose.Signer, claimToResponseMap map[string]string, openIDConfig *string) *httptest.Server { ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - glog.V(5).Infof("request: %+v", *r) + klog.V(5).Infof("request: %+v", *r) switch r.URL.Path { case "/.testing/keys": w.Header().Set("Content-Type", "application/json") @@ -166,12 +169,12 @@ func newClaimServer(t *testing.T, keys jose.JSONWebKeySet, signer jose.Signer, c if err != nil { t.Fatalf("unexpected error while marshaling keys: %v", err) } - glog.V(5).Infof("%v: returning: %+v", r.URL, string(keyBytes)) + klog.V(5).Infof("%v: returning: %+v", r.URL, string(keyBytes)) w.Write(keyBytes) case "/.well-known/openid-configuration": w.Header().Set("Content-Type", "application/json") - glog.V(5).Infof("%v: returning: %+v", r.URL, *openIDConfig) + klog.V(5).Infof("%v: returning: %+v", r.URL, *openIDConfig) w.Write([]byte(*openIDConfig)) // These claims are tested in the unit tests. case "/groups": @@ -200,7 +203,7 @@ func newClaimServer(t *testing.T, keys jose.JSONWebKeySet, signer jose.Signer, c fmt.Fprintf(w, "unexpected URL: %v", r.URL) } })) - glog.V(4).Infof("Serving OIDC at: %v", ts.URL) + klog.V(4).Infof("Serving OIDC at: %v", ts.URL) return ts } @@ -296,7 +299,12 @@ func (c *claimsTest) run(t *testing.T) { t.Fatalf("serialize token: %v", err) } - got, ok, err := a.AuthenticateToken(context.Background(), token) + ctx := context.Background() + if c.reqAudiences != nil { + ctx = authenticator.WithAudiences(ctx, c.reqAudiences) + } + + got, ok, err := a.AuthenticateToken(ctx, token) if err != nil { if !c.wantErr { @@ -1388,6 +1396,148 @@ func TestToken(t *testing.T) { Name: "thomas.jefferson@gmail.com", }, }, + { + name: "good token with api req audience", + options: Options{ + IssuerURL: "https://auth.example.com", + ClientID: "my-client", + APIAudiences: authenticator.Audiences{"api"}, + UsernameClaim: "username", + now: func() time.Time { return now }, + }, + signingKey: loadRSAPrivKey(t, "testdata/rsa_1.pem", jose.RS256), + pubKeys: []*jose.JSONWebKey{ + loadRSAKey(t, "testdata/rsa_1.pem", jose.RS256), + }, + claims: fmt.Sprintf(`{ + "iss": "https://auth.example.com", + "aud": "my-client", + "username": "jane", + "exp": %d + }`, valid.Unix()), + reqAudiences: authenticator.Audiences{"api"}, + want: &user.DefaultInfo{ + Name: "jane", + }, + }, + { + name: "good token with multiple api req audience", + options: Options{ + IssuerURL: "https://auth.example.com", + ClientID: "my-client", + APIAudiences: authenticator.Audiences{"api", "other"}, + UsernameClaim: "username", + now: func() time.Time { return now }, + }, + signingKey: loadRSAPrivKey(t, "testdata/rsa_1.pem", jose.RS256), + pubKeys: []*jose.JSONWebKey{ + loadRSAKey(t, "testdata/rsa_1.pem", jose.RS256), + }, + claims: fmt.Sprintf(`{ + "iss": "https://auth.example.com", + "aud": "my-client", + "username": "jane", + "exp": %d + }`, valid.Unix()), + reqAudiences: authenticator.Audiences{"api"}, + want: &user.DefaultInfo{ + Name: "jane", + }, + }, + { + name: "good token with client_id req audience", + options: Options{ + IssuerURL: "https://auth.example.com", + ClientID: "my-client", + APIAudiences: authenticator.Audiences{"api"}, + UsernameClaim: "username", + now: func() time.Time { return now }, + }, + signingKey: loadRSAPrivKey(t, "testdata/rsa_1.pem", jose.RS256), + pubKeys: []*jose.JSONWebKey{ + loadRSAKey(t, "testdata/rsa_1.pem", jose.RS256), + }, + claims: fmt.Sprintf(`{ + "iss": "https://auth.example.com", + "aud": "my-client", + "username": "jane", + "exp": %d + }`, valid.Unix()), + reqAudiences: authenticator.Audiences{"my-client"}, + want: &user.DefaultInfo{ + Name: "jane", + }, + }, + { + name: "good token with client_id and api req audience", + options: Options{ + IssuerURL: "https://auth.example.com", + ClientID: "my-client", + APIAudiences: authenticator.Audiences{"api"}, + UsernameClaim: "username", + now: func() time.Time { return now }, + }, + signingKey: loadRSAPrivKey(t, "testdata/rsa_1.pem", jose.RS256), + pubKeys: []*jose.JSONWebKey{ + loadRSAKey(t, "testdata/rsa_1.pem", jose.RS256), + }, + claims: fmt.Sprintf(`{ + "iss": "https://auth.example.com", + "aud": "my-client", + "username": "jane", + "exp": %d + }`, valid.Unix()), + reqAudiences: authenticator.Audiences{"my-client", "api"}, + want: &user.DefaultInfo{ + Name: "jane", + }, + }, + { + name: "good token with client_id and api req audience", + options: Options{ + IssuerURL: "https://auth.example.com", + ClientID: "my-client", + APIAudiences: authenticator.Audiences{"api"}, + UsernameClaim: "username", + now: func() time.Time { return now }, + }, + signingKey: loadRSAPrivKey(t, "testdata/rsa_1.pem", jose.RS256), + pubKeys: []*jose.JSONWebKey{ + loadRSAKey(t, "testdata/rsa_1.pem", jose.RS256), + }, + claims: fmt.Sprintf(`{ + "iss": "https://auth.example.com", + "aud": "my-client", + "username": "jane", + "exp": %d + }`, valid.Unix()), + reqAudiences: authenticator.Audiences{"my-client", "api"}, + want: &user.DefaultInfo{ + Name: "jane", + }, + }, + { + name: "good token with client_id and bad req audience", + options: Options{ + IssuerURL: "https://auth.example.com", + ClientID: "my-client", + APIAudiences: authenticator.Audiences{"api"}, + UsernameClaim: "username", + now: func() time.Time { return now }, + }, + signingKey: loadRSAPrivKey(t, "testdata/rsa_1.pem", jose.RS256), + pubKeys: []*jose.JSONWebKey{ + loadRSAKey(t, "testdata/rsa_1.pem", jose.RS256), + }, + claims: fmt.Sprintf(`{ + "iss": "https://auth.example.com", + "aud": "my-client", + "username": "jane", + "exp": %d + }`, valid.Unix()), + reqAudiences: authenticator.Audiences{"other"}, + wantSkip: true, + }, } for _, test := range tests { t.Run(test.name, test.run) diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/BUILD index fd4af99aa55..eb13b669bdf 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/BUILD +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/BUILD @@ -16,6 +16,8 @@ go_test( deps = [ "//staging/src/k8s.io/api/authentication/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authentication/token/cache:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api/v1:go_default_library", ], @@ -30,13 +32,12 @@ go_library( "//staging/src/k8s.io/api/authentication/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/cache:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/authentication/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook.go b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook.go index 90104df5494..cf0a83b5d97 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook.go @@ -21,17 +21,15 @@ import ( "context" "time" - "github.com/golang/glog" - authentication "k8s.io/api/authentication/v1beta1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/cache" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/util/webhook" "k8s.io/client-go/kubernetes/scheme" authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1" + "k8s.io/klog" ) var ( @@ -45,54 +43,82 @@ var _ authenticator.Token = (*WebhookTokenAuthenticator)(nil) type WebhookTokenAuthenticator struct { tokenReview authenticationclient.TokenReviewInterface - responseCache *cache.LRUExpireCache - ttl time.Duration initialBackoff time.Duration + implicitAuds authenticator.Audiences } -// NewFromInterface creates a webhook authenticator using the given tokenReview client -func NewFromInterface(tokenReview authenticationclient.TokenReviewInterface, ttl time.Duration) (*WebhookTokenAuthenticator, error) { - return newWithBackoff(tokenReview, ttl, retryBackoff) +// NewFromInterface creates a webhook authenticator using the given tokenReview +// client. It is recommend to wrap this authenticator with the token cache +// authenticator implemented in +// k8s.io/apiserver/pkg/authentication/token/cache. +func NewFromInterface(tokenReview authenticationclient.TokenReviewInterface, implicitAuds authenticator.Audiences) (*WebhookTokenAuthenticator, error) { + return newWithBackoff(tokenReview, retryBackoff, implicitAuds) } -// New creates a new WebhookTokenAuthenticator from the provided kubeconfig file. -func New(kubeConfigFile string, ttl time.Duration) (*WebhookTokenAuthenticator, error) { +// New creates a new WebhookTokenAuthenticator from the provided kubeconfig +// file. It is recommend to wrap this authenticator with the token cache +// authenticator implemented in +// k8s.io/apiserver/pkg/authentication/token/cache. +func New(kubeConfigFile string, implicitAuds authenticator.Audiences) (*WebhookTokenAuthenticator, error) { tokenReview, err := tokenReviewInterfaceFromKubeconfig(kubeConfigFile) if err != nil { return nil, err } - return newWithBackoff(tokenReview, ttl, retryBackoff) + return newWithBackoff(tokenReview, retryBackoff, implicitAuds) } // newWithBackoff allows tests to skip the sleep. -func newWithBackoff(tokenReview authenticationclient.TokenReviewInterface, ttl, initialBackoff time.Duration) (*WebhookTokenAuthenticator, error) { - return &WebhookTokenAuthenticator{tokenReview, cache.NewLRUExpireCache(1024), ttl, initialBackoff}, nil +func newWithBackoff(tokenReview authenticationclient.TokenReviewInterface, initialBackoff time.Duration, implicitAuds authenticator.Audiences) (*WebhookTokenAuthenticator, error) { + return &WebhookTokenAuthenticator{tokenReview, initialBackoff, implicitAuds}, nil } // AuthenticateToken implements the authenticator.Token interface. func (w *WebhookTokenAuthenticator) AuthenticateToken(ctx context.Context, token string) (*authenticator.Response, bool, error) { + // We take implicit audiences of the API server at WebhookTokenAuthenticator + // construction time. The outline of how we validate audience here is: + // + // * if the ctx is not audience limited, don't do any audience validation. + // * if ctx is audience-limited, add the audiences to the tokenreview spec + // * if the tokenreview returns with audiences in the status that intersect + // with the audiences in the ctx, copy into the response and return success + // * if the tokenreview returns without an audience in the status, ensure + // the ctx audiences intersect with the implicit audiences, and set the + // intersection in the response. + // * otherwise return unauthenticated. + wantAuds, checkAuds := authenticator.AudiencesFrom(ctx) r := &authentication.TokenReview{ - Spec: authentication.TokenReviewSpec{Token: token}, + Spec: authentication.TokenReviewSpec{ + Token: token, + Audiences: wantAuds, + }, } - if entry, ok := w.responseCache.Get(r.Spec); ok { - r.Status = entry.(authentication.TokenReviewStatus) - } else { - var ( - result *authentication.TokenReview - err error - ) - webhook.WithExponentialBackoff(w.initialBackoff, func() error { - result, err = w.tokenReview.Create(r) - return err - }) - if err != nil { - // An error here indicates bad configuration or an outage. Log for debugging. - glog.Errorf("Failed to make webhook authenticator request: %v", err) - return nil, false, err + var ( + result *authentication.TokenReview + err error + auds authenticator.Audiences + ) + webhook.WithExponentialBackoff(w.initialBackoff, func() error { + result, err = w.tokenReview.Create(r) + return err + }) + if err != nil { + // An error here indicates bad configuration or an outage. Log for debugging. + klog.Errorf("Failed to make webhook authenticator request: %v", err) + return nil, false, err + } + + if checkAuds { + gotAuds := w.implicitAuds + if len(result.Status.Audiences) > 0 { + gotAuds = result.Status.Audiences + } + auds = wantAuds.Intersect(gotAuds) + if len(auds) == 0 { + return nil, false, nil } - r.Status = result.Status - w.responseCache.Add(r.Spec, result.Status, w.ttl) } + + r.Status = result.Status if !r.Status.Authenticated { return nil, false, nil } @@ -112,6 +138,7 @@ func (w *WebhookTokenAuthenticator) AuthenticateToken(ctx context.Context, token Groups: r.Status.User.Groups, Extra: extra, }, + Audiences: auds, }, true, nil } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook_test.go index bd2d00bf722..8c8855a894d 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook_test.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook_test.go @@ -33,10 +33,14 @@ import ( "k8s.io/api/authentication/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/token/cache" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/client-go/tools/clientcmd/api/v1" ) +var apiAuds = authenticator.Audiences{"api"} + // Service mocks a remote authentication service. type Service interface { // Review looks at the TokenReviewSpec and provides an authentication @@ -103,6 +107,7 @@ func NewTestServer(s Service, cert, key, caCert []byte) (*httptest.Server, error type status struct { Authenticated bool `json:"authenticated"` User userInfo `json:"user"` + Audiences []string `json:"audiences"` } var extra map[string][]string @@ -128,6 +133,7 @@ func NewTestServer(s Service, cert, key, caCert []byte) (*httptest.Server, error Groups: review.Status.User.Groups, Extra: extra, }, + review.Status.Audiences, }, } w.Header().Set("Content-Type", "application/json") @@ -166,7 +172,7 @@ func (m *mockService) HTTPStatusCode() int { return m.statusCode } // newTokenAuthenticator creates a temporary kubeconfig file from the provided // arguments and attempts to load a new WebhookTokenAuthenticator from it. -func newTokenAuthenticator(serverURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration) (*WebhookTokenAuthenticator, error) { +func newTokenAuthenticator(serverURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration, implicitAuds authenticator.Audiences) (authenticator.Token, error) { tempfile, err := ioutil.TempFile("", "") if err != nil { return nil, err @@ -194,7 +200,12 @@ func newTokenAuthenticator(serverURL string, clientCert, clientKey, ca []byte, c return nil, err } - return newWithBackoff(c, cacheTime, 0) + authn, err := newWithBackoff(c, 0, implicitAuds) + if err != nil { + return nil, err + } + + return cache.New(authn, false, cacheTime, cacheTime), nil } func TestTLSConfig(t *testing.T) { @@ -250,7 +261,7 @@ func TestTLSConfig(t *testing.T) { } defer server.Close() - wh, err := newTokenAuthenticator(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0) + wh, err := newTokenAuthenticator(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0, nil) if err != nil { t.Errorf("%s: failed to create client: %v", tt.test, err) return @@ -305,23 +316,21 @@ func TestWebhookTokenAuthenticator(t *testing.T) { } defer s.Close() - wh, err := newTokenAuthenticator(s.URL, clientCert, clientKey, caCert, 0) - if err != nil { - t.Fatal(err) - } - expTypeMeta := metav1.TypeMeta{ APIVersion: "authentication.k8s.io/v1beta1", Kind: "TokenReview", } tests := []struct { + description string + implicitAuds, reqAuds authenticator.Audiences serverResponse v1beta1.TokenReviewStatus expectedAuthenticated bool expectedUser *user.DefaultInfo + expectedAuds authenticator.Audiences }{ - // Successful response should pass through all user info. { + description: "successful response should pass through all user info.", serverResponse: v1beta1.TokenReviewStatus{ Authenticated: true, User: v1beta1.UserInfo{ @@ -334,6 +343,7 @@ func TestWebhookTokenAuthenticator(t *testing.T) { }, }, { + description: "successful response should pass through all user info.", serverResponse: v1beta1.TokenReviewStatus{ Authenticated: true, User: v1beta1.UserInfo{ @@ -351,8 +361,8 @@ func TestWebhookTokenAuthenticator(t *testing.T) { Extra: map[string][]string{"foo": {"bar", "baz"}}, }, }, - // Unauthenticated shouldn't even include extra provided info. { + description: "unauthenticated shouldn't even include extra provided info.", serverResponse: v1beta1.TokenReviewStatus{ Authenticated: false, User: v1beta1.UserInfo{ @@ -365,37 +375,151 @@ func TestWebhookTokenAuthenticator(t *testing.T) { expectedUser: nil, }, { + description: "unauthenticated shouldn't even include extra provided info.", serverResponse: v1beta1.TokenReviewStatus{ Authenticated: false, }, expectedAuthenticated: false, expectedUser: nil, }, + { + description: "good audience", + implicitAuds: apiAuds, + reqAuds: apiAuds, + serverResponse: v1beta1.TokenReviewStatus{ + Authenticated: true, + User: v1beta1.UserInfo{ + Username: "somebody", + }, + }, + expectedAuthenticated: true, + expectedUser: &user.DefaultInfo{ + Name: "somebody", + }, + expectedAuds: apiAuds, + }, + { + description: "good audience", + implicitAuds: append(apiAuds, "other"), + reqAuds: apiAuds, + serverResponse: v1beta1.TokenReviewStatus{ + Authenticated: true, + User: v1beta1.UserInfo{ + Username: "somebody", + }, + }, + expectedAuthenticated: true, + expectedUser: &user.DefaultInfo{ + Name: "somebody", + }, + expectedAuds: apiAuds, + }, + { + description: "bad audiences", + implicitAuds: apiAuds, + reqAuds: authenticator.Audiences{"other"}, + serverResponse: v1beta1.TokenReviewStatus{ + Authenticated: false, + }, + expectedAuthenticated: false, + }, + { + description: "bad audiences", + implicitAuds: apiAuds, + reqAuds: authenticator.Audiences{"other"}, + // webhook authenticator hasn't been upgraded to support audience. + serverResponse: v1beta1.TokenReviewStatus{ + Authenticated: true, + User: v1beta1.UserInfo{ + Username: "somebody", + }, + }, + expectedAuthenticated: false, + }, + { + description: "audience aware backend", + implicitAuds: apiAuds, + reqAuds: apiAuds, + serverResponse: v1beta1.TokenReviewStatus{ + Authenticated: true, + User: v1beta1.UserInfo{ + Username: "somebody", + }, + Audiences: []string(apiAuds), + }, + expectedAuthenticated: true, + expectedUser: &user.DefaultInfo{ + Name: "somebody", + }, + expectedAuds: apiAuds, + }, + { + description: "audience aware backend", + serverResponse: v1beta1.TokenReviewStatus{ + Authenticated: true, + User: v1beta1.UserInfo{ + Username: "somebody", + }, + Audiences: []string(apiAuds), + }, + expectedAuthenticated: true, + expectedUser: &user.DefaultInfo{ + Name: "somebody", + }, + }, + { + description: "audience aware backend", + implicitAuds: apiAuds, + reqAuds: apiAuds, + serverResponse: v1beta1.TokenReviewStatus{ + Authenticated: true, + User: v1beta1.UserInfo{ + Username: "somebody", + }, + Audiences: []string{"other"}, + }, + expectedAuthenticated: false, + }, } token := "my-s3cr3t-t0ken" - for i, tt := range tests { - serv.response = tt.serverResponse - resp, authenticated, err := wh.AuthenticateToken(context.Background(), token) - if err != nil { - t.Errorf("case %d: authentication failed: %v", i, err) - continue - } - if serv.lastRequest.Spec.Token != token { - t.Errorf("case %d: Server did not see correct token. Got %q, expected %q.", - i, serv.lastRequest.Spec.Token, token) - } - if !reflect.DeepEqual(serv.lastRequest.TypeMeta, expTypeMeta) { - t.Errorf("case %d: Server did not see correct TypeMeta. Got %v, expected %v", - i, serv.lastRequest.TypeMeta, expTypeMeta) - } - if authenticated != tt.expectedAuthenticated { - t.Errorf("case %d: Plugin returned incorrect authentication response. Got %t, expected %t.", - i, authenticated, tt.expectedAuthenticated) - } - if resp != nil && tt.expectedUser != nil && !reflect.DeepEqual(resp.User, tt.expectedUser) { - t.Errorf("case %d: Plugin returned incorrect user. Got %#v, expected %#v", - i, resp.User, tt.expectedUser) - } + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + wh, err := newTokenAuthenticator(s.URL, clientCert, clientKey, caCert, 0, tt.implicitAuds) + if err != nil { + t.Fatal(err) + } + + ctx := context.Background() + if tt.reqAuds != nil { + ctx = authenticator.WithAudiences(ctx, tt.reqAuds) + } + + serv.response = tt.serverResponse + resp, authenticated, err := wh.AuthenticateToken(ctx, token) + if err != nil { + t.Fatalf("authentication failed: %v", err) + } + if serv.lastRequest.Spec.Token != token { + t.Errorf("Server did not see correct token. Got %q, expected %q.", + serv.lastRequest.Spec.Token, token) + } + if !reflect.DeepEqual(serv.lastRequest.TypeMeta, expTypeMeta) { + t.Errorf("Server did not see correct TypeMeta. Got %v, expected %v", + serv.lastRequest.TypeMeta, expTypeMeta) + } + if authenticated != tt.expectedAuthenticated { + t.Errorf("Plugin returned incorrect authentication response. Got %t, expected %t.", + authenticated, tt.expectedAuthenticated) + } + if resp != nil && tt.expectedUser != nil && !reflect.DeepEqual(resp.User, tt.expectedUser) { + t.Errorf("Plugin returned incorrect user. Got %#v, expected %#v", + resp.User, tt.expectedUser) + } + if resp != nil && tt.expectedAuds != nil && !reflect.DeepEqual(resp.Audiences, tt.expectedAuds) { + t.Errorf("Plugin returned incorrect audiences. Got %#v, expected %#v", + resp.Audiences, tt.expectedAuds) + } + }) } } @@ -433,7 +557,7 @@ func TestWebhookCacheAndRetry(t *testing.T) { defer s.Close() // Create an authenticator that caches successful responses "forever" (100 days). - wh, err := newTokenAuthenticator(s.URL, clientCert, clientKey, caCert, 2400*time.Hour) + wh, err := newTokenAuthenticator(s.URL, clientCert, clientKey, caCert, 2400*time.Hour, nil) if err != nil { t.Fatal(err) } @@ -549,15 +673,12 @@ func TestWebhookCacheAndRetry(t *testing.T) { _, ok, err := wh.AuthenticateToken(context.Background(), testcase.token) hasError := err != nil if hasError != testcase.expectError { - t.Log(testcase.description) t.Errorf("Webhook returned HTTP %d, expected error=%v, but got error %v", testcase.code, testcase.expectError, err) } if serv.called != testcase.expectCalls { - t.Log(testcase.description) t.Errorf("Expected %d calls, got %d", testcase.expectCalls, serv.called) } if ok != testcase.expectOk { - t.Log(testcase.description) t.Errorf("Expected ok=%v, got %v", testcase.expectOk, ok) } }) diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/OWNERS b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/OWNERS new file mode 100644 index 00000000000..cd0d70a0f8f --- /dev/null +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authorizers-approvers +reviewers: +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/BUILD index 59a23decbbe..270aed93282 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/BUILD +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/BUILD @@ -38,7 +38,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/authorization/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go index 03b7bda32fe..e05ef503f22 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go @@ -22,7 +22,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" authorization "k8s.io/api/authorization/v1beta1" "k8s.io/apimachinery/pkg/runtime" @@ -189,7 +189,7 @@ func (w *WebhookAuthorizer) Authorize(attr authorizer.Attributes) (decision auth }) if err != nil { // An error here indicates bad configuration or an outage. Log for debugging. - glog.Errorf("Failed to make webhook authorizer request: %v", err) + klog.Errorf("Failed to make webhook authorizer request: %v", err) return w.decisionOnError, "", err } r.Status = result.Status diff --git a/staging/src/k8s.io/cli-runtime/Godeps/Godeps.json b/staging/src/k8s.io/cli-runtime/Godeps/Godeps.json index 280521d2731..aa88de45f6e 100644 --- a/staging/src/k8s.io/cli-runtime/Godeps/Godeps.json +++ b/staging/src/k8s.io/cli-runtime/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/cli-runtime", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -14,10 +14,6 @@ "ImportPath": "github.com/evanphx/json-patch", "Rev": "36442dbdb585210f8d5a1b45e67aa323c197d5c4" }, - { - "ImportPath": "github.com/ghodss/yaml", - "Rev": "c7ce16629ff4cd059ed96ed06419dd3856fd3577" - }, { "ImportPath": "github.com/gogo/protobuf/proto", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" @@ -26,10 +22,6 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/golang/protobuf/proto", "Rev": "b4deda0973fb4c70b50d226b1af49f3da59f5265" @@ -124,23 +116,23 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/oauth2", @@ -574,6 +566,10 @@ "ImportPath": "k8s.io/client-go/util/integer", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/client-go/discovery", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -617,6 +613,10 @@ { "ImportPath": "k8s.io/client-go/util/testing", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" } ] } diff --git a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers/BUILD b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers/BUILD index 60a43ae33cc..2cbb55099d2 100644 --- a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers/BUILD +++ b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers/BUILD @@ -21,7 +21,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/client-go/util/jsonpath:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers/json.go b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers/json.go index 63f5834d418..bb5bec7488c 100644 --- a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers/json.go +++ b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/printers/json.go @@ -25,7 +25,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) // JSONPrinter is an implementation of ResourcePrinter which outputs an object as JSON. diff --git a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/BUILD b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/BUILD index 89259fa1db7..22b34de0087 100644 --- a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/BUILD +++ b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/BUILD @@ -76,8 +76,8 @@ go_test( "//staging/src/k8s.io/client-go/restmapper:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/builder_test.go b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/builder_test.go index add8fbde37a..7fd526b33c6 100644 --- a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/builder_test.go +++ b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/builder_test.go @@ -30,7 +30,7 @@ import ( "testing" "github.com/davecgh/go-spew/spew" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/meta" diff --git a/staging/src/k8s.io/client-go/Godeps/Godeps.json b/staging/src/k8s.io/client-go/Godeps/Godeps.json index efd7a0a6f49..7fbd65b5b5c 100644 --- a/staging/src/k8s.io/client-go/Godeps/Godeps.json +++ b/staging/src/k8s.io/client-go/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/client-go", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -58,10 +58,6 @@ "ImportPath": "github.com/evanphx/json-patch", "Rev": "36442dbdb585210f8d5a1b45e67aa323c197d5c4" }, - { - "ImportPath": "github.com/ghodss/yaml", - "Rev": "c7ce16629ff4cd059ed96ed06419dd3856fd3577" - }, { "ImportPath": "github.com/gogo/protobuf/proto", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" @@ -70,10 +66,6 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/golang/groupcache/lru", "Rev": "02826c3e79038b59d737d3b1c0a1d937f71a4433" @@ -200,27 +192,27 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/context/ctxhttp", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/oauth2", @@ -614,9 +606,17 @@ "ImportPath": "k8s.io/apimachinery/third_party/forked/golang/reflect", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" + }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" } ] } diff --git a/staging/src/k8s.io/client-go/discovery/BUILD b/staging/src/k8s.io/client-go/discovery/BUILD index 1ddb339a4ff..64e75dc511c 100644 --- a/staging/src/k8s.io/client-go/discovery/BUILD +++ b/staging/src/k8s.io/client-go/discovery/BUILD @@ -28,12 +28,12 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/protobuf/proto:go_default_library", "//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library", "//vendor/github.com/gregjones/httpcache:go_default_library", "//vendor/github.com/gregjones/httpcache/diskcache:go_default_library", "//vendor/github.com/peterbourgon/diskv:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/discovery/cached_discovery.go b/staging/src/k8s.io/client-go/discovery/cached_discovery.go index 90cd342017a..df69d6a1930 100644 --- a/staging/src/k8s.io/client-go/discovery/cached_discovery.go +++ b/staging/src/k8s.io/client-go/discovery/cached_discovery.go @@ -25,8 +25,8 @@ import ( "sync" "time" - "github.com/golang/glog" "github.com/googleapis/gnostic/OpenAPIv2" + "k8s.io/klog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -67,23 +67,23 @@ func (d *CachedDiscoveryClient) ServerResourcesForGroupVersion(groupVersion stri if err == nil { cachedResources := &metav1.APIResourceList{} if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), cachedBytes, cachedResources); err == nil { - glog.V(10).Infof("returning cached discovery info from %v", filename) + klog.V(10).Infof("returning cached discovery info from %v", filename) return cachedResources, nil } } liveResources, err := d.delegate.ServerResourcesForGroupVersion(groupVersion) if err != nil { - glog.V(3).Infof("skipped caching discovery info due to %v", err) + klog.V(3).Infof("skipped caching discovery info due to %v", err) return liveResources, err } if liveResources == nil || len(liveResources.APIResources) == 0 { - glog.V(3).Infof("skipped caching discovery info, no resources found") + klog.V(3).Infof("skipped caching discovery info, no resources found") return liveResources, err } if err := d.writeCachedFile(filename, liveResources); err != nil { - glog.V(1).Infof("failed to write cache to %v due to %v", filename, err) + klog.V(1).Infof("failed to write cache to %v due to %v", filename, err) } return liveResources, nil @@ -103,23 +103,23 @@ func (d *CachedDiscoveryClient) ServerGroups() (*metav1.APIGroupList, error) { if err == nil { cachedGroups := &metav1.APIGroupList{} if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), cachedBytes, cachedGroups); err == nil { - glog.V(10).Infof("returning cached discovery info from %v", filename) + klog.V(10).Infof("returning cached discovery info from %v", filename) return cachedGroups, nil } } liveGroups, err := d.delegate.ServerGroups() if err != nil { - glog.V(3).Infof("skipped caching discovery info due to %v", err) + klog.V(3).Infof("skipped caching discovery info due to %v", err) return liveGroups, err } if liveGroups == nil || len(liveGroups.Groups) == 0 { - glog.V(3).Infof("skipped caching discovery info, no groups found") + klog.V(3).Infof("skipped caching discovery info, no groups found") return liveGroups, err } if err := d.writeCachedFile(filename, liveGroups); err != nil { - glog.V(1).Infof("failed to write cache to %v due to %v", filename, err) + klog.V(1).Infof("failed to write cache to %v due to %v", filename, err) } return liveGroups, nil diff --git a/staging/src/k8s.io/client-go/discovery/round_tripper.go b/staging/src/k8s.io/client-go/discovery/round_tripper.go index 75b7f520977..4e2bc24e774 100644 --- a/staging/src/k8s.io/client-go/discovery/round_tripper.go +++ b/staging/src/k8s.io/client-go/discovery/round_tripper.go @@ -20,10 +20,10 @@ import ( "net/http" "path/filepath" - "github.com/golang/glog" "github.com/gregjones/httpcache" "github.com/gregjones/httpcache/diskcache" "github.com/peterbourgon/diskv" + "k8s.io/klog" ) type cacheRoundTripper struct { @@ -55,7 +55,7 @@ func (rt *cacheRoundTripper) CancelRequest(req *http.Request) { if cr, ok := rt.rt.Transport.(canceler); ok { cr.CancelRequest(req) } else { - glog.Errorf("CancelRequest not implemented by %T", rt.rt.Transport) + klog.Errorf("CancelRequest not implemented by %T", rt.rt.Transport) } } diff --git a/staging/src/k8s.io/client-go/examples/workqueue/BUILD b/staging/src/k8s.io/client-go/examples/workqueue/BUILD index 0bf338e8958..e993d2c478b 100644 --- a/staging/src/k8s.io/client-go/examples/workqueue/BUILD +++ b/staging/src/k8s.io/client-go/examples/workqueue/BUILD @@ -26,7 +26,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/examples/workqueue/main.go b/staging/src/k8s.io/client-go/examples/workqueue/main.go index 6768f5d91b2..c306aaae00c 100644 --- a/staging/src/k8s.io/client-go/examples/workqueue/main.go +++ b/staging/src/k8s.io/client-go/examples/workqueue/main.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -72,7 +72,7 @@ func (c *Controller) processNextItem() bool { func (c *Controller) syncToStdout(key string) error { obj, exists, err := c.indexer.GetByKey(key) if err != nil { - glog.Errorf("Fetching object with key %s from store failed with %v", key, err) + klog.Errorf("Fetching object with key %s from store failed with %v", key, err) return err } @@ -99,7 +99,7 @@ func (c *Controller) handleErr(err error, key interface{}) { // This controller retries 5 times if something goes wrong. After that, it stops trying. if c.queue.NumRequeues(key) < 5 { - glog.Infof("Error syncing pod %v: %v", key, err) + klog.Infof("Error syncing pod %v: %v", key, err) // Re-enqueue the key rate limited. Based on the rate limiter on the // queue and the re-enqueue history, the key will be processed later again. @@ -110,7 +110,7 @@ func (c *Controller) handleErr(err error, key interface{}) { c.queue.Forget(key) // Report to an external entity that, even after several retries, we could not successfully process this key runtime.HandleError(err) - glog.Infof("Dropping pod %q out of the queue: %v", key, err) + klog.Infof("Dropping pod %q out of the queue: %v", key, err) } func (c *Controller) Run(threadiness int, stopCh chan struct{}) { @@ -118,7 +118,7 @@ func (c *Controller) Run(threadiness int, stopCh chan struct{}) { // Let the workers stop when we are done defer c.queue.ShutDown() - glog.Info("Starting Pod controller") + klog.Info("Starting Pod controller") go c.informer.Run(stopCh) @@ -133,7 +133,7 @@ func (c *Controller) Run(threadiness int, stopCh chan struct{}) { } <-stopCh - glog.Info("Stopping Pod controller") + klog.Info("Stopping Pod controller") } func (c *Controller) runWorker() { @@ -152,13 +152,13 @@ func main() { // creates the connection config, err := clientcmd.BuildConfigFromFlags(master, kubeconfig) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } // creates the clientset clientset, err := kubernetes.NewForConfig(config) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } // create the pod watcher diff --git a/staging/src/k8s.io/client-go/informers/generic.go b/staging/src/k8s.io/client-go/informers/generic.go index ab9abf87ee3..09a5efe2ec8 100644 --- a/staging/src/k8s.io/client-go/informers/generic.go +++ b/staging/src/k8s.io/client-go/informers/generic.go @@ -262,6 +262,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource // Group=storage.k8s.io, Version=v1 case storagev1.SchemeGroupVersion.WithResource("storageclasses"): return &genericInformer{resource: resource.GroupResource(), informer: f.Storage().V1().StorageClasses().Informer()}, nil + case storagev1.SchemeGroupVersion.WithResource("volumeattachments"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Storage().V1().VolumeAttachments().Informer()}, nil // Group=storage.k8s.io, Version=v1alpha1 case storagev1alpha1.SchemeGroupVersion.WithResource("volumeattachments"): diff --git a/staging/src/k8s.io/client-go/informers/storage/v1/BUILD b/staging/src/k8s.io/client-go/informers/storage/v1/BUILD index 4e3f9966ab0..25f3894667f 100644 --- a/staging/src/k8s.io/client-go/informers/storage/v1/BUILD +++ b/staging/src/k8s.io/client-go/informers/storage/v1/BUILD @@ -10,6 +10,7 @@ go_library( srcs = [ "interface.go", "storageclass.go", + "volumeattachment.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/informers/storage/v1", importpath = "k8s.io/client-go/informers/storage/v1", diff --git a/staging/src/k8s.io/client-go/informers/storage/v1/interface.go b/staging/src/k8s.io/client-go/informers/storage/v1/interface.go index d7e4b5c49ac..64fc2bd8436 100644 --- a/staging/src/k8s.io/client-go/informers/storage/v1/interface.go +++ b/staging/src/k8s.io/client-go/informers/storage/v1/interface.go @@ -26,6 +26,8 @@ import ( type Interface interface { // StorageClasses returns a StorageClassInformer. StorageClasses() StorageClassInformer + // VolumeAttachments returns a VolumeAttachmentInformer. + VolumeAttachments() VolumeAttachmentInformer } type version struct { @@ -43,3 +45,8 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList func (v *version) StorageClasses() StorageClassInformer { return &storageClassInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} } + +// VolumeAttachments returns a VolumeAttachmentInformer. +func (v *version) VolumeAttachments() VolumeAttachmentInformer { + return &volumeAttachmentInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} diff --git a/staging/src/k8s.io/client-go/informers/storage/v1/volumeattachment.go b/staging/src/k8s.io/client-go/informers/storage/v1/volumeattachment.go new file mode 100644 index 00000000000..7ca3b86f22c --- /dev/null +++ b/staging/src/k8s.io/client-go/informers/storage/v1/volumeattachment.go @@ -0,0 +1,88 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + time "time" + + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + internalinterfaces "k8s.io/client-go/informers/internalinterfaces" + kubernetes "k8s.io/client-go/kubernetes" + v1 "k8s.io/client-go/listers/storage/v1" + cache "k8s.io/client-go/tools/cache" +) + +// VolumeAttachmentInformer provides access to a shared informer and lister for +// VolumeAttachments. +type VolumeAttachmentInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1.VolumeAttachmentLister +} + +type volumeAttachmentInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewVolumeAttachmentInformer constructs a new informer for VolumeAttachment type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewVolumeAttachmentInformer(client kubernetes.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredVolumeAttachmentInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredVolumeAttachmentInformer constructs a new informer for VolumeAttachment type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredVolumeAttachmentInformer(client kubernetes.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.StorageV1().VolumeAttachments().List(options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.StorageV1().VolumeAttachments().Watch(options) + }, + }, + &storagev1.VolumeAttachment{}, + resyncPeriod, + indexers, + ) +} + +func (f *volumeAttachmentInformer) defaultInformer(client kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredVolumeAttachmentInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *volumeAttachmentInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&storagev1.VolumeAttachment{}, f.defaultInformer) +} + +func (f *volumeAttachmentInformer) Lister() v1.VolumeAttachmentLister { + return v1.NewVolumeAttachmentLister(f.Informer().GetIndexer()) +} diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1/initializerconfiguration.go b/staging/src/k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1/initializerconfiguration.go index e014ea72b69..7b8acecee9b 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1/initializerconfiguration.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1/initializerconfiguration.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1alpha1 "k8s.io/api/admissionregistration/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *initializerConfigurations) Get(name string, options v1.GetOptions) (res // List takes label and field selectors, and returns the list of InitializerConfigurations that match those selectors. func (c *initializerConfigurations) List(opts v1.ListOptions) (result *v1alpha1.InitializerConfigurationList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.InitializerConfigurationList{} err = c.client.Get(). Resource("initializerconfigurations"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *initializerConfigurations) List(opts v1.ListOptions) (result *v1alpha1. // Watch returns a watch.Interface that watches the requested initializerConfigurations. func (c *initializerConfigurations) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("initializerconfigurations"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *initializerConfigurations) Delete(name string, options *v1.DeleteOption // DeleteCollection deletes a collection of objects. func (c *initializerConfigurations) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("initializerconfigurations"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1/mutatingwebhookconfiguration.go b/staging/src/k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1/mutatingwebhookconfiguration.go index cb015710291..4524896cd6b 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1/mutatingwebhookconfiguration.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1/mutatingwebhookconfiguration.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/admissionregistration/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *mutatingWebhookConfigurations) Get(name string, options v1.GetOptions) // List takes label and field selectors, and returns the list of MutatingWebhookConfigurations that match those selectors. func (c *mutatingWebhookConfigurations) List(opts v1.ListOptions) (result *v1beta1.MutatingWebhookConfigurationList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.MutatingWebhookConfigurationList{} err = c.client.Get(). Resource("mutatingwebhookconfigurations"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *mutatingWebhookConfigurations) List(opts v1.ListOptions) (result *v1bet // Watch returns a watch.Interface that watches the requested mutatingWebhookConfigurations. func (c *mutatingWebhookConfigurations) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("mutatingwebhookconfigurations"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *mutatingWebhookConfigurations) Delete(name string, options *v1.DeleteOp // DeleteCollection deletes a collection of objects. func (c *mutatingWebhookConfigurations) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("mutatingwebhookconfigurations"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1/validatingwebhookconfiguration.go b/staging/src/k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1/validatingwebhookconfiguration.go index 3a9339f6cdc..7e711b3000e 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1/validatingwebhookconfiguration.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1/validatingwebhookconfiguration.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/admissionregistration/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *validatingWebhookConfigurations) Get(name string, options v1.GetOptions // List takes label and field selectors, and returns the list of ValidatingWebhookConfigurations that match those selectors. func (c *validatingWebhookConfigurations) List(opts v1.ListOptions) (result *v1beta1.ValidatingWebhookConfigurationList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.ValidatingWebhookConfigurationList{} err = c.client.Get(). Resource("validatingwebhookconfigurations"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *validatingWebhookConfigurations) List(opts v1.ListOptions) (result *v1b // Watch returns a watch.Interface that watches the requested validatingWebhookConfigurations. func (c *validatingWebhookConfigurations) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("validatingwebhookconfigurations"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *validatingWebhookConfigurations) Delete(name string, options *v1.Delete // DeleteCollection deletes a collection of objects. func (c *validatingWebhookConfigurations) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("validatingwebhookconfigurations"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/controllerrevision.go b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/controllerrevision.go index 1ddaa1a71b7..e28e4d2a3fa 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/controllerrevision.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/controllerrevision.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *controllerRevisions) Get(name string, options metav1.GetOptions) (resul // List takes label and field selectors, and returns the list of ControllerRevisions that match those selectors. func (c *controllerRevisions) List(opts metav1.ListOptions) (result *v1.ControllerRevisionList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.ControllerRevisionList{} err = c.client.Get(). Namespace(c.ns). Resource("controllerrevisions"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *controllerRevisions) List(opts metav1.ListOptions) (result *v1.Controll // Watch returns a watch.Interface that watches the requested controllerRevisions. func (c *controllerRevisions) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("controllerrevisions"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *controllerRevisions) Delete(name string, options *metav1.DeleteOptions) // DeleteCollection deletes a collection of objects. func (c *controllerRevisions) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("controllerrevisions"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/daemonset.go b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/daemonset.go index 03a87069840..a535cdabe64 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/daemonset.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/daemonset.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *daemonSets) Get(name string, options metav1.GetOptions) (result *v1.Dae // List takes label and field selectors, and returns the list of DaemonSets that match those selectors. func (c *daemonSets) List(opts metav1.ListOptions) (result *v1.DaemonSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.DaemonSetList{} err = c.client.Get(). Namespace(c.ns). Resource("daemonsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *daemonSets) List(opts metav1.ListOptions) (result *v1.DaemonSetList, er // Watch returns a watch.Interface that watches the requested daemonSets. func (c *daemonSets) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("daemonsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *daemonSets) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *daemonSets) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("daemonsets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/deployment.go b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/deployment.go index 503eb4b7330..f9799a45395 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/deployment.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/deployment.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -80,11 +82,16 @@ func (c *deployments) Get(name string, options metav1.GetOptions) (result *v1.De // List takes label and field selectors, and returns the list of Deployments that match those selectors. func (c *deployments) List(opts metav1.ListOptions) (result *v1.DeploymentList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.DeploymentList{} err = c.client.Get(). Namespace(c.ns). Resource("deployments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -92,11 +99,16 @@ func (c *deployments) List(opts metav1.ListOptions) (result *v1.DeploymentList, // Watch returns a watch.Interface that watches the requested deployments. func (c *deployments) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("deployments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -154,10 +166,15 @@ func (c *deployments) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *deployments) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("deployments"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/replicaset.go b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/replicaset.go index ea6efb68617..ff3504e78a4 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/replicaset.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/replicaset.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -80,11 +82,16 @@ func (c *replicaSets) Get(name string, options metav1.GetOptions) (result *v1.Re // List takes label and field selectors, and returns the list of ReplicaSets that match those selectors. func (c *replicaSets) List(opts metav1.ListOptions) (result *v1.ReplicaSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.ReplicaSetList{} err = c.client.Get(). Namespace(c.ns). Resource("replicasets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -92,11 +99,16 @@ func (c *replicaSets) List(opts metav1.ListOptions) (result *v1.ReplicaSetList, // Watch returns a watch.Interface that watches the requested replicaSets. func (c *replicaSets) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("replicasets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -154,10 +166,15 @@ func (c *replicaSets) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *replicaSets) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("replicasets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/statefulset.go b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/statefulset.go index 66e522ba124..c12c470bbae 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/statefulset.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1/statefulset.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -80,11 +82,16 @@ func (c *statefulSets) Get(name string, options metav1.GetOptions) (result *v1.S // List takes label and field selectors, and returns the list of StatefulSets that match those selectors. func (c *statefulSets) List(opts metav1.ListOptions) (result *v1.StatefulSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.StatefulSetList{} err = c.client.Get(). Namespace(c.ns). Resource("statefulsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -92,11 +99,16 @@ func (c *statefulSets) List(opts metav1.ListOptions) (result *v1.StatefulSetList // Watch returns a watch.Interface that watches the requested statefulSets. func (c *statefulSets) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("statefulsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -154,10 +166,15 @@ func (c *statefulSets) Delete(name string, options *metav1.DeleteOptions) error // DeleteCollection deletes a collection of objects. func (c *statefulSets) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("statefulsets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta1/controllerrevision.go b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta1/controllerrevision.go index ec8fa9242f5..45ddb91592d 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta1/controllerrevision.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta1/controllerrevision.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/apps/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *controllerRevisions) Get(name string, options v1.GetOptions) (result *v // List takes label and field selectors, and returns the list of ControllerRevisions that match those selectors. func (c *controllerRevisions) List(opts v1.ListOptions) (result *v1beta1.ControllerRevisionList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.ControllerRevisionList{} err = c.client.Get(). Namespace(c.ns). Resource("controllerrevisions"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *controllerRevisions) List(opts v1.ListOptions) (result *v1beta1.Control // Watch returns a watch.Interface that watches the requested controllerRevisions. func (c *controllerRevisions) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("controllerrevisions"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *controllerRevisions) Delete(name string, options *v1.DeleteOptions) err // DeleteCollection deletes a collection of objects. func (c *controllerRevisions) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("controllerrevisions"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta1/deployment.go b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta1/deployment.go index 365e06f3f10..05fdcb7a644 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta1/deployment.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta1/deployment.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/apps/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *deployments) Get(name string, options v1.GetOptions) (result *v1beta1.D // List takes label and field selectors, and returns the list of Deployments that match those selectors. func (c *deployments) List(opts v1.ListOptions) (result *v1beta1.DeploymentList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.DeploymentList{} err = c.client.Get(). Namespace(c.ns). Resource("deployments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *deployments) List(opts v1.ListOptions) (result *v1beta1.DeploymentList, // Watch returns a watch.Interface that watches the requested deployments. func (c *deployments) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("deployments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *deployments) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *deployments) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("deployments"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta1/statefulset.go b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta1/statefulset.go index 651745451d8..c4b35b424c7 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta1/statefulset.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta1/statefulset.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/apps/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *statefulSets) Get(name string, options v1.GetOptions) (result *v1beta1. // List takes label and field selectors, and returns the list of StatefulSets that match those selectors. func (c *statefulSets) List(opts v1.ListOptions) (result *v1beta1.StatefulSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.StatefulSetList{} err = c.client.Get(). Namespace(c.ns). Resource("statefulsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *statefulSets) List(opts v1.ListOptions) (result *v1beta1.StatefulSetLis // Watch returns a watch.Interface that watches the requested statefulSets. func (c *statefulSets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("statefulsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *statefulSets) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *statefulSets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("statefulsets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/controllerrevision.go b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/controllerrevision.go index 1271cc623fe..e1d6025155e 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/controllerrevision.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/controllerrevision.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta2 import ( + "time" + v1beta2 "k8s.io/api/apps/v1beta2" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *controllerRevisions) Get(name string, options v1.GetOptions) (result *v // List takes label and field selectors, and returns the list of ControllerRevisions that match those selectors. func (c *controllerRevisions) List(opts v1.ListOptions) (result *v1beta2.ControllerRevisionList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta2.ControllerRevisionList{} err = c.client.Get(). Namespace(c.ns). Resource("controllerrevisions"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *controllerRevisions) List(opts v1.ListOptions) (result *v1beta2.Control // Watch returns a watch.Interface that watches the requested controllerRevisions. func (c *controllerRevisions) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("controllerrevisions"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *controllerRevisions) Delete(name string, options *v1.DeleteOptions) err // DeleteCollection deletes a collection of objects. func (c *controllerRevisions) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("controllerrevisions"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/daemonset.go b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/daemonset.go index 683c0681216..f8b7ac25973 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/daemonset.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/daemonset.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta2 import ( + "time" + v1beta2 "k8s.io/api/apps/v1beta2" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *daemonSets) Get(name string, options v1.GetOptions) (result *v1beta2.Da // List takes label and field selectors, and returns the list of DaemonSets that match those selectors. func (c *daemonSets) List(opts v1.ListOptions) (result *v1beta2.DaemonSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta2.DaemonSetList{} err = c.client.Get(). Namespace(c.ns). Resource("daemonsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *daemonSets) List(opts v1.ListOptions) (result *v1beta2.DaemonSetList, e // Watch returns a watch.Interface that watches the requested daemonSets. func (c *daemonSets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("daemonsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *daemonSets) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *daemonSets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("daemonsets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/deployment.go b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/deployment.go index 9a04513f1ba..510250b06e1 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/deployment.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/deployment.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta2 import ( + "time" + v1beta2 "k8s.io/api/apps/v1beta2" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *deployments) Get(name string, options v1.GetOptions) (result *v1beta2.D // List takes label and field selectors, and returns the list of Deployments that match those selectors. func (c *deployments) List(opts v1.ListOptions) (result *v1beta2.DeploymentList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta2.DeploymentList{} err = c.client.Get(). Namespace(c.ns). Resource("deployments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *deployments) List(opts v1.ListOptions) (result *v1beta2.DeploymentList, // Watch returns a watch.Interface that watches the requested deployments. func (c *deployments) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("deployments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *deployments) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *deployments) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("deployments"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/replicaset.go b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/replicaset.go index 9fd9de930ba..7b738774b79 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/replicaset.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/replicaset.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta2 import ( + "time" + v1beta2 "k8s.io/api/apps/v1beta2" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *replicaSets) Get(name string, options v1.GetOptions) (result *v1beta2.R // List takes label and field selectors, and returns the list of ReplicaSets that match those selectors. func (c *replicaSets) List(opts v1.ListOptions) (result *v1beta2.ReplicaSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta2.ReplicaSetList{} err = c.client.Get(). Namespace(c.ns). Resource("replicasets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *replicaSets) List(opts v1.ListOptions) (result *v1beta2.ReplicaSetList, // Watch returns a watch.Interface that watches the requested replicaSets. func (c *replicaSets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("replicasets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *replicaSets) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *replicaSets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("replicasets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/statefulset.go b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/statefulset.go index 095601e15a5..de7c3db8b50 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/statefulset.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta2/statefulset.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta2 import ( + "time" + v1beta2 "k8s.io/api/apps/v1beta2" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -79,11 +81,16 @@ func (c *statefulSets) Get(name string, options v1.GetOptions) (result *v1beta2. // List takes label and field selectors, and returns the list of StatefulSets that match those selectors. func (c *statefulSets) List(opts v1.ListOptions) (result *v1beta2.StatefulSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta2.StatefulSetList{} err = c.client.Get(). Namespace(c.ns). Resource("statefulsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -91,11 +98,16 @@ func (c *statefulSets) List(opts v1.ListOptions) (result *v1beta2.StatefulSetLis // Watch returns a watch.Interface that watches the requested statefulSets. func (c *statefulSets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("statefulsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -153,10 +165,15 @@ func (c *statefulSets) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *statefulSets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("statefulsets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1/auditsink.go b/staging/src/k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1/auditsink.go index 4f91f527ef1..414d480062e 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1/auditsink.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1/auditsink.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1alpha1 "k8s.io/api/auditregistration/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *auditSinks) Get(name string, options v1.GetOptions) (result *v1alpha1.A // List takes label and field selectors, and returns the list of AuditSinks that match those selectors. func (c *auditSinks) List(opts v1.ListOptions) (result *v1alpha1.AuditSinkList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.AuditSinkList{} err = c.client.Get(). Resource("auditsinks"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *auditSinks) List(opts v1.ListOptions) (result *v1alpha1.AuditSinkList, // Watch returns a watch.Interface that watches the requested auditSinks. func (c *auditSinks) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("auditsinks"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *auditSinks) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *auditSinks) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("auditsinks"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/authentication/OWNERS b/staging/src/k8s.io/client-go/kubernetes/typed/authentication/OWNERS new file mode 100644 index 00000000000..c607d2aa8c5 --- /dev/null +++ b/staging/src/k8s.io/client-go/kubernetes/typed/authentication/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authenticators-approvers +reviewers: +- sig-auth-authenticators-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/authorization/OWNERS b/staging/src/k8s.io/client-go/kubernetes/typed/authorization/OWNERS new file mode 100644 index 00000000000..cd0d70a0f8f --- /dev/null +++ b/staging/src/k8s.io/client-go/kubernetes/typed/authorization/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authorizers-approvers +reviewers: +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/autoscaling/v1/horizontalpodautoscaler.go b/staging/src/k8s.io/client-go/kubernetes/typed/autoscaling/v1/horizontalpodautoscaler.go index 6891b6b63dd..0e0839fb508 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/autoscaling/v1/horizontalpodautoscaler.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/autoscaling/v1/horizontalpodautoscaler.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/autoscaling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *horizontalPodAutoscalers) Get(name string, options metav1.GetOptions) ( // List takes label and field selectors, and returns the list of HorizontalPodAutoscalers that match those selectors. func (c *horizontalPodAutoscalers) List(opts metav1.ListOptions) (result *v1.HorizontalPodAutoscalerList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.HorizontalPodAutoscalerList{} err = c.client.Get(). Namespace(c.ns). Resource("horizontalpodautoscalers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *horizontalPodAutoscalers) List(opts metav1.ListOptions) (result *v1.Hor // Watch returns a watch.Interface that watches the requested horizontalPodAutoscalers. func (c *horizontalPodAutoscalers) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("horizontalpodautoscalers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *horizontalPodAutoscalers) Delete(name string, options *metav1.DeleteOpt // DeleteCollection deletes a collection of objects. func (c *horizontalPodAutoscalers) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("horizontalpodautoscalers"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1/horizontalpodautoscaler.go b/staging/src/k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1/horizontalpodautoscaler.go index 4ac8cce71bf..02d5cfb9b60 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1/horizontalpodautoscaler.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1/horizontalpodautoscaler.go @@ -19,6 +19,8 @@ limitations under the License. package v2beta1 import ( + "time" + v2beta1 "k8s.io/api/autoscaling/v2beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *horizontalPodAutoscalers) Get(name string, options v1.GetOptions) (resu // List takes label and field selectors, and returns the list of HorizontalPodAutoscalers that match those selectors. func (c *horizontalPodAutoscalers) List(opts v1.ListOptions) (result *v2beta1.HorizontalPodAutoscalerList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v2beta1.HorizontalPodAutoscalerList{} err = c.client.Get(). Namespace(c.ns). Resource("horizontalpodautoscalers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *horizontalPodAutoscalers) List(opts v1.ListOptions) (result *v2beta1.Ho // Watch returns a watch.Interface that watches the requested horizontalPodAutoscalers. func (c *horizontalPodAutoscalers) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("horizontalpodautoscalers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *horizontalPodAutoscalers) Delete(name string, options *v1.DeleteOptions // DeleteCollection deletes a collection of objects. func (c *horizontalPodAutoscalers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("horizontalpodautoscalers"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2/horizontalpodautoscaler.go b/staging/src/k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2/horizontalpodautoscaler.go index ddabda7e702..91a0fa64f9a 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2/horizontalpodautoscaler.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2/horizontalpodautoscaler.go @@ -19,6 +19,8 @@ limitations under the License. package v2beta2 import ( + "time" + v2beta2 "k8s.io/api/autoscaling/v2beta2" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *horizontalPodAutoscalers) Get(name string, options v1.GetOptions) (resu // List takes label and field selectors, and returns the list of HorizontalPodAutoscalers that match those selectors. func (c *horizontalPodAutoscalers) List(opts v1.ListOptions) (result *v2beta2.HorizontalPodAutoscalerList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v2beta2.HorizontalPodAutoscalerList{} err = c.client.Get(). Namespace(c.ns). Resource("horizontalpodautoscalers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *horizontalPodAutoscalers) List(opts v1.ListOptions) (result *v2beta2.Ho // Watch returns a watch.Interface that watches the requested horizontalPodAutoscalers. func (c *horizontalPodAutoscalers) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("horizontalpodautoscalers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *horizontalPodAutoscalers) Delete(name string, options *v1.DeleteOptions // DeleteCollection deletes a collection of objects. func (c *horizontalPodAutoscalers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("horizontalpodautoscalers"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/batch/v1/job.go b/staging/src/k8s.io/client-go/kubernetes/typed/batch/v1/job.go index ba8332a9a25..b55c602b341 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/batch/v1/job.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/batch/v1/job.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/batch/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *jobs) Get(name string, options metav1.GetOptions) (result *v1.Job, err // List takes label and field selectors, and returns the list of Jobs that match those selectors. func (c *jobs) List(opts metav1.ListOptions) (result *v1.JobList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.JobList{} err = c.client.Get(). Namespace(c.ns). Resource("jobs"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *jobs) List(opts metav1.ListOptions) (result *v1.JobList, err error) { // Watch returns a watch.Interface that watches the requested jobs. func (c *jobs) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("jobs"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *jobs) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *jobs) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("jobs"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/batch/v1beta1/cronjob.go b/staging/src/k8s.io/client-go/kubernetes/typed/batch/v1beta1/cronjob.go index 04637c36aa7..d89d2fa21d4 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/batch/v1beta1/cronjob.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/batch/v1beta1/cronjob.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/batch/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *cronJobs) Get(name string, options v1.GetOptions) (result *v1beta1.Cron // List takes label and field selectors, and returns the list of CronJobs that match those selectors. func (c *cronJobs) List(opts v1.ListOptions) (result *v1beta1.CronJobList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.CronJobList{} err = c.client.Get(). Namespace(c.ns). Resource("cronjobs"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *cronJobs) List(opts v1.ListOptions) (result *v1beta1.CronJobList, err e // Watch returns a watch.Interface that watches the requested cronJobs. func (c *cronJobs) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("cronjobs"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *cronJobs) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *cronJobs) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("cronjobs"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/batch/v2alpha1/cronjob.go b/staging/src/k8s.io/client-go/kubernetes/typed/batch/v2alpha1/cronjob.go index 4d922f9ae9e..19123b60411 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/batch/v2alpha1/cronjob.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/batch/v2alpha1/cronjob.go @@ -19,6 +19,8 @@ limitations under the License. package v2alpha1 import ( + "time" + v2alpha1 "k8s.io/api/batch/v2alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *cronJobs) Get(name string, options v1.GetOptions) (result *v2alpha1.Cro // List takes label and field selectors, and returns the list of CronJobs that match those selectors. func (c *cronJobs) List(opts v1.ListOptions) (result *v2alpha1.CronJobList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v2alpha1.CronJobList{} err = c.client.Get(). Namespace(c.ns). Resource("cronjobs"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *cronJobs) List(opts v1.ListOptions) (result *v2alpha1.CronJobList, err // Watch returns a watch.Interface that watches the requested cronJobs. func (c *cronJobs) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("cronjobs"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *cronJobs) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *cronJobs) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("cronjobs"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1/certificatesigningrequest.go b/staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1/certificatesigningrequest.go index b39169a8ff6..712d3a01afa 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1/certificatesigningrequest.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1/certificatesigningrequest.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/certificates/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -73,10 +75,15 @@ func (c *certificateSigningRequests) Get(name string, options v1.GetOptions) (re // List takes label and field selectors, and returns the list of CertificateSigningRequests that match those selectors. func (c *certificateSigningRequests) List(opts v1.ListOptions) (result *v1beta1.CertificateSigningRequestList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.CertificateSigningRequestList{} err = c.client.Get(). Resource("certificatesigningrequests"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *certificateSigningRequests) List(opts v1.ListOptions) (result *v1beta1. // Watch returns a watch.Interface that watches the requested certificateSigningRequests. func (c *certificateSigningRequests) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("certificatesigningrequests"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *certificateSigningRequests) Delete(name string, options *v1.DeleteOptio // DeleteCollection deletes a collection of objects. func (c *certificateSigningRequests) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("certificatesigningrequests"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/coordination/v1beta1/lease.go b/staging/src/k8s.io/client-go/kubernetes/typed/coordination/v1beta1/lease.go index 16277255fa6..490d815aa63 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/coordination/v1beta1/lease.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/coordination/v1beta1/lease.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/coordination/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *leases) Get(name string, options v1.GetOptions) (result *v1beta1.Lease, // List takes label and field selectors, and returns the list of Leases that match those selectors. func (c *leases) List(opts v1.ListOptions) (result *v1beta1.LeaseList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.LeaseList{} err = c.client.Get(). Namespace(c.ns). Resource("leases"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *leases) List(opts v1.ListOptions) (result *v1beta1.LeaseList, err error // Watch returns a watch.Interface that watches the requested leases. func (c *leases) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("leases"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *leases) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *leases) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("leases"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/componentstatus.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/componentstatus.go index e497661cfbe..302b2fdc344 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/componentstatus.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/componentstatus.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *componentStatuses) Get(name string, options metav1.GetOptions) (result // List takes label and field selectors, and returns the list of ComponentStatuses that match those selectors. func (c *componentStatuses) List(opts metav1.ListOptions) (result *v1.ComponentStatusList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.ComponentStatusList{} err = c.client.Get(). Resource("componentstatuses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *componentStatuses) List(opts metav1.ListOptions) (result *v1.ComponentS // Watch returns a watch.Interface that watches the requested componentStatuses. func (c *componentStatuses) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("componentstatuses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *componentStatuses) Delete(name string, options *metav1.DeleteOptions) e // DeleteCollection deletes a collection of objects. func (c *componentStatuses) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("componentstatuses"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/configmap.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/configmap.go index 0984ae70cc1..18ce954ae23 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/configmap.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/configmap.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *configMaps) Get(name string, options metav1.GetOptions) (result *v1.Con // List takes label and field selectors, and returns the list of ConfigMaps that match those selectors. func (c *configMaps) List(opts metav1.ListOptions) (result *v1.ConfigMapList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.ConfigMapList{} err = c.client.Get(). Namespace(c.ns). Resource("configmaps"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *configMaps) List(opts metav1.ListOptions) (result *v1.ConfigMapList, er // Watch returns a watch.Interface that watches the requested configMaps. func (c *configMaps) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("configmaps"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *configMaps) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *configMaps) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("configmaps"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/endpoints.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/endpoints.go index dd8216789bc..978a2a196cc 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/endpoints.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/endpoints.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *endpoints) Get(name string, options metav1.GetOptions) (result *v1.Endp // List takes label and field selectors, and returns the list of Endpoints that match those selectors. func (c *endpoints) List(opts metav1.ListOptions) (result *v1.EndpointsList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.EndpointsList{} err = c.client.Get(). Namespace(c.ns). Resource("endpoints"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *endpoints) List(opts metav1.ListOptions) (result *v1.EndpointsList, err // Watch returns a watch.Interface that watches the requested endpoints. func (c *endpoints) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("endpoints"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *endpoints) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *endpoints) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("endpoints"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/event.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/event.go index 57d30f9fd49..55cfa0901b2 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/event.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/event.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *events) Get(name string, options metav1.GetOptions) (result *v1.Event, // List takes label and field selectors, and returns the list of Events that match those selectors. func (c *events) List(opts metav1.ListOptions) (result *v1.EventList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.EventList{} err = c.client.Get(). Namespace(c.ns). Resource("events"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *events) List(opts metav1.ListOptions) (result *v1.EventList, err error) // Watch returns a watch.Interface that watches the requested events. func (c *events) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("events"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *events) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *events) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("events"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/limitrange.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/limitrange.go index 5b385668b81..2eeae11a837 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/limitrange.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/limitrange.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *limitRanges) Get(name string, options metav1.GetOptions) (result *v1.Li // List takes label and field selectors, and returns the list of LimitRanges that match those selectors. func (c *limitRanges) List(opts metav1.ListOptions) (result *v1.LimitRangeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.LimitRangeList{} err = c.client.Get(). Namespace(c.ns). Resource("limitranges"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *limitRanges) List(opts metav1.ListOptions) (result *v1.LimitRangeList, // Watch returns a watch.Interface that watches the requested limitRanges. func (c *limitRanges) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("limitranges"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *limitRanges) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *limitRanges) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("limitranges"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/namespace.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/namespace.go index e22d07decee..8a81fe85079 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/namespace.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/namespace.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *namespaces) Get(name string, options metav1.GetOptions) (result *v1.Nam // List takes label and field selectors, and returns the list of Namespaces that match those selectors. func (c *namespaces) List(opts metav1.ListOptions) (result *v1.NamespaceList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.NamespaceList{} err = c.client.Get(). Resource("namespaces"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *namespaces) List(opts metav1.ListOptions) (result *v1.NamespaceList, er // Watch returns a watch.Interface that watches the requested namespaces. func (c *namespaces) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("namespaces"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/node.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/node.go index 5c769c11857..d19fab8952f 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/node.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/node.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -73,10 +75,15 @@ func (c *nodes) Get(name string, options metav1.GetOptions) (result *v1.Node, er // List takes label and field selectors, and returns the list of Nodes that match those selectors. func (c *nodes) List(opts metav1.ListOptions) (result *v1.NodeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.NodeList{} err = c.client.Get(). Resource("nodes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *nodes) List(opts metav1.ListOptions) (result *v1.NodeList, err error) { // Watch returns a watch.Interface that watches the requested nodes. func (c *nodes) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("nodes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *nodes) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *nodes) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("nodes"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/persistentvolume.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/persistentvolume.go index d5f19aef522..74514825e98 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/persistentvolume.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/persistentvolume.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -73,10 +75,15 @@ func (c *persistentVolumes) Get(name string, options metav1.GetOptions) (result // List takes label and field selectors, and returns the list of PersistentVolumes that match those selectors. func (c *persistentVolumes) List(opts metav1.ListOptions) (result *v1.PersistentVolumeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.PersistentVolumeList{} err = c.client.Get(). Resource("persistentvolumes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *persistentVolumes) List(opts metav1.ListOptions) (result *v1.Persistent // Watch returns a watch.Interface that watches the requested persistentVolumes. func (c *persistentVolumes) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("persistentvolumes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *persistentVolumes) Delete(name string, options *metav1.DeleteOptions) e // DeleteCollection deletes a collection of objects. func (c *persistentVolumes) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("persistentvolumes"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/persistentvolumeclaim.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/persistentvolumeclaim.go index d32ae5dfd84..410ab37dcba 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/persistentvolumeclaim.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/persistentvolumeclaim.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *persistentVolumeClaims) Get(name string, options metav1.GetOptions) (re // List takes label and field selectors, and returns the list of PersistentVolumeClaims that match those selectors. func (c *persistentVolumeClaims) List(opts metav1.ListOptions) (result *v1.PersistentVolumeClaimList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.PersistentVolumeClaimList{} err = c.client.Get(). Namespace(c.ns). Resource("persistentvolumeclaims"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *persistentVolumeClaims) List(opts metav1.ListOptions) (result *v1.Persi // Watch returns a watch.Interface that watches the requested persistentVolumeClaims. func (c *persistentVolumeClaims) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("persistentvolumeclaims"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *persistentVolumeClaims) Delete(name string, options *metav1.DeleteOptio // DeleteCollection deletes a collection of objects. func (c *persistentVolumeClaims) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("persistentvolumeclaims"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/pod.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/pod.go index b19c5a5c3ea..8d6b6e87963 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/pod.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/pod.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *pods) Get(name string, options metav1.GetOptions) (result *v1.Pod, err // List takes label and field selectors, and returns the list of Pods that match those selectors. func (c *pods) List(opts metav1.ListOptions) (result *v1.PodList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.PodList{} err = c.client.Get(). Namespace(c.ns). Resource("pods"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *pods) List(opts metav1.ListOptions) (result *v1.PodList, err error) { // Watch returns a watch.Interface that watches the requested pods. func (c *pods) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("pods"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *pods) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *pods) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("pods"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/podtemplate.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/podtemplate.go index d644e17d778..84d7c980593 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/podtemplate.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/podtemplate.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *podTemplates) Get(name string, options metav1.GetOptions) (result *v1.P // List takes label and field selectors, and returns the list of PodTemplates that match those selectors. func (c *podTemplates) List(opts metav1.ListOptions) (result *v1.PodTemplateList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.PodTemplateList{} err = c.client.Get(). Namespace(c.ns). Resource("podtemplates"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *podTemplates) List(opts metav1.ListOptions) (result *v1.PodTemplateList // Watch returns a watch.Interface that watches the requested podTemplates. func (c *podTemplates) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("podtemplates"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *podTemplates) Delete(name string, options *metav1.DeleteOptions) error // DeleteCollection deletes a collection of objects. func (c *podTemplates) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("podtemplates"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/replicationcontroller.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/replicationcontroller.go index f3e41612433..dd3182db65b 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/replicationcontroller.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/replicationcontroller.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + autoscalingv1 "k8s.io/api/autoscaling/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -80,11 +82,16 @@ func (c *replicationControllers) Get(name string, options metav1.GetOptions) (re // List takes label and field selectors, and returns the list of ReplicationControllers that match those selectors. func (c *replicationControllers) List(opts metav1.ListOptions) (result *v1.ReplicationControllerList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.ReplicationControllerList{} err = c.client.Get(). Namespace(c.ns). Resource("replicationcontrollers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -92,11 +99,16 @@ func (c *replicationControllers) List(opts metav1.ListOptions) (result *v1.Repli // Watch returns a watch.Interface that watches the requested replicationControllers. func (c *replicationControllers) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("replicationcontrollers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -154,10 +166,15 @@ func (c *replicationControllers) Delete(name string, options *metav1.DeleteOptio // DeleteCollection deletes a collection of objects. func (c *replicationControllers) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("replicationcontrollers"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/resourcequota.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/resourcequota.go index 8b74a4046f0..5a178990ecd 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/resourcequota.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/resourcequota.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *resourceQuotas) Get(name string, options metav1.GetOptions) (result *v1 // List takes label and field selectors, and returns the list of ResourceQuotas that match those selectors. func (c *resourceQuotas) List(opts metav1.ListOptions) (result *v1.ResourceQuotaList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.ResourceQuotaList{} err = c.client.Get(). Namespace(c.ns). Resource("resourcequotas"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *resourceQuotas) List(opts metav1.ListOptions) (result *v1.ResourceQuota // Watch returns a watch.Interface that watches the requested resourceQuotas. func (c *resourceQuotas) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("resourcequotas"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *resourceQuotas) Delete(name string, options *metav1.DeleteOptions) erro // DeleteCollection deletes a collection of objects. func (c *resourceQuotas) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("resourcequotas"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/secret.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/secret.go index 4ea9796b63d..85c143b173d 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/secret.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/secret.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *secrets) Get(name string, options metav1.GetOptions) (result *v1.Secret // List takes label and field selectors, and returns the list of Secrets that match those selectors. func (c *secrets) List(opts metav1.ListOptions) (result *v1.SecretList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.SecretList{} err = c.client.Get(). Namespace(c.ns). Resource("secrets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *secrets) List(opts metav1.ListOptions) (result *v1.SecretList, err erro // Watch returns a watch.Interface that watches the requested secrets. func (c *secrets) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("secrets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *secrets) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *secrets) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("secrets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/service.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/service.go index 6c42ca87a8b..b0e09413efb 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/service.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/service.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *services) Get(name string, options metav1.GetOptions) (result *v1.Servi // List takes label and field selectors, and returns the list of Services that match those selectors. func (c *services) List(opts metav1.ListOptions) (result *v1.ServiceList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.ServiceList{} err = c.client.Get(). Namespace(c.ns). Resource("services"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *services) List(opts metav1.ListOptions) (result *v1.ServiceList, err er // Watch returns a watch.Interface that watches the requested services. func (c *services) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("services"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/serviceaccount.go b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/serviceaccount.go index f3ab7eb878c..50af6a21c9b 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/serviceaccount.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/core/v1/serviceaccount.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *serviceAccounts) Get(name string, options metav1.GetOptions) (result *v // List takes label and field selectors, and returns the list of ServiceAccounts that match those selectors. func (c *serviceAccounts) List(opts metav1.ListOptions) (result *v1.ServiceAccountList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.ServiceAccountList{} err = c.client.Get(). Namespace(c.ns). Resource("serviceaccounts"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *serviceAccounts) List(opts metav1.ListOptions) (result *v1.ServiceAccou // Watch returns a watch.Interface that watches the requested serviceAccounts. func (c *serviceAccounts) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("serviceaccounts"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *serviceAccounts) Delete(name string, options *metav1.DeleteOptions) err // DeleteCollection deletes a collection of objects. func (c *serviceAccounts) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("serviceaccounts"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/events/v1beta1/event.go b/staging/src/k8s.io/client-go/kubernetes/typed/events/v1beta1/event.go index af7d060d591..143281b25cc 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/events/v1beta1/event.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/events/v1beta1/event.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/events/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *events) Get(name string, options v1.GetOptions) (result *v1beta1.Event, // List takes label and field selectors, and returns the list of Events that match those selectors. func (c *events) List(opts v1.ListOptions) (result *v1beta1.EventList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.EventList{} err = c.client.Get(). Namespace(c.ns). Resource("events"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *events) List(opts v1.ListOptions) (result *v1beta1.EventList, err error // Watch returns a watch.Interface that watches the requested events. func (c *events) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("events"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *events) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *events) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("events"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/daemonset.go b/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/daemonset.go index 85294be4b98..93b1ae9b6dd 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/daemonset.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/daemonset.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/extensions/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *daemonSets) Get(name string, options v1.GetOptions) (result *v1beta1.Da // List takes label and field selectors, and returns the list of DaemonSets that match those selectors. func (c *daemonSets) List(opts v1.ListOptions) (result *v1beta1.DaemonSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.DaemonSetList{} err = c.client.Get(). Namespace(c.ns). Resource("daemonsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *daemonSets) List(opts v1.ListOptions) (result *v1beta1.DaemonSetList, e // Watch returns a watch.Interface that watches the requested daemonSets. func (c *daemonSets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("daemonsets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *daemonSets) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *daemonSets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("daemonsets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/deployment.go b/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/deployment.go index 89183d2853e..5557b9f2b1e 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/deployment.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/deployment.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/extensions/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -79,11 +81,16 @@ func (c *deployments) Get(name string, options v1.GetOptions) (result *v1beta1.D // List takes label and field selectors, and returns the list of Deployments that match those selectors. func (c *deployments) List(opts v1.ListOptions) (result *v1beta1.DeploymentList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.DeploymentList{} err = c.client.Get(). Namespace(c.ns). Resource("deployments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -91,11 +98,16 @@ func (c *deployments) List(opts v1.ListOptions) (result *v1beta1.DeploymentList, // Watch returns a watch.Interface that watches the requested deployments. func (c *deployments) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("deployments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -153,10 +165,15 @@ func (c *deployments) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *deployments) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("deployments"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/ingress.go b/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/ingress.go index f8b664cbd1e..4da51c36858 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/ingress.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/ingress.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/extensions/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *ingresses) Get(name string, options v1.GetOptions) (result *v1beta1.Ing // List takes label and field selectors, and returns the list of Ingresses that match those selectors. func (c *ingresses) List(opts v1.ListOptions) (result *v1beta1.IngressList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.IngressList{} err = c.client.Get(). Namespace(c.ns). Resource("ingresses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *ingresses) List(opts v1.ListOptions) (result *v1beta1.IngressList, err // Watch returns a watch.Interface that watches the requested ingresses. func (c *ingresses) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("ingresses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *ingresses) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *ingresses) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("ingresses"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/podsecuritypolicy.go b/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/podsecuritypolicy.go index 8099d77307d..a947a54a6f4 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/podsecuritypolicy.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/podsecuritypolicy.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/extensions/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *podSecurityPolicies) Get(name string, options v1.GetOptions) (result *v // List takes label and field selectors, and returns the list of PodSecurityPolicies that match those selectors. func (c *podSecurityPolicies) List(opts v1.ListOptions) (result *v1beta1.PodSecurityPolicyList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.PodSecurityPolicyList{} err = c.client.Get(). Resource("podsecuritypolicies"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *podSecurityPolicies) List(opts v1.ListOptions) (result *v1beta1.PodSecu // Watch returns a watch.Interface that watches the requested podSecurityPolicies. func (c *podSecurityPolicies) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("podsecuritypolicies"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *podSecurityPolicies) Delete(name string, options *v1.DeleteOptions) err // DeleteCollection deletes a collection of objects. func (c *podSecurityPolicies) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("podsecuritypolicies"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/replicaset.go b/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/replicaset.go index 7e61fa2d12f..444029058b4 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/replicaset.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/extensions/v1beta1/replicaset.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/extensions/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -79,11 +81,16 @@ func (c *replicaSets) Get(name string, options v1.GetOptions) (result *v1beta1.R // List takes label and field selectors, and returns the list of ReplicaSets that match those selectors. func (c *replicaSets) List(opts v1.ListOptions) (result *v1beta1.ReplicaSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.ReplicaSetList{} err = c.client.Get(). Namespace(c.ns). Resource("replicasets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -91,11 +98,16 @@ func (c *replicaSets) List(opts v1.ListOptions) (result *v1beta1.ReplicaSetList, // Watch returns a watch.Interface that watches the requested replicaSets. func (c *replicaSets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("replicasets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -153,10 +165,15 @@ func (c *replicaSets) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *replicaSets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("replicasets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/networking/v1/networkpolicy.go b/staging/src/k8s.io/client-go/kubernetes/typed/networking/v1/networkpolicy.go index d8f0a6b47e9..3f39be957d8 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/networking/v1/networkpolicy.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/networking/v1/networkpolicy.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *networkPolicies) Get(name string, options metav1.GetOptions) (result *v // List takes label and field selectors, and returns the list of NetworkPolicies that match those selectors. func (c *networkPolicies) List(opts metav1.ListOptions) (result *v1.NetworkPolicyList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.NetworkPolicyList{} err = c.client.Get(). Namespace(c.ns). Resource("networkpolicies"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *networkPolicies) List(opts metav1.ListOptions) (result *v1.NetworkPolic // Watch returns a watch.Interface that watches the requested networkPolicies. func (c *networkPolicies) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("networkpolicies"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *networkPolicies) Delete(name string, options *metav1.DeleteOptions) err // DeleteCollection deletes a collection of objects. func (c *networkPolicies) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("networkpolicies"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/policy/v1beta1/poddisruptionbudget.go b/staging/src/k8s.io/client-go/kubernetes/typed/policy/v1beta1/poddisruptionbudget.go index a11f27eb257..864af9a262b 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/policy/v1beta1/poddisruptionbudget.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/policy/v1beta1/poddisruptionbudget.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/policy/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -76,11 +78,16 @@ func (c *podDisruptionBudgets) Get(name string, options v1.GetOptions) (result * // List takes label and field selectors, and returns the list of PodDisruptionBudgets that match those selectors. func (c *podDisruptionBudgets) List(opts v1.ListOptions) (result *v1beta1.PodDisruptionBudgetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.PodDisruptionBudgetList{} err = c.client.Get(). Namespace(c.ns). Resource("poddisruptionbudgets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *podDisruptionBudgets) List(opts v1.ListOptions) (result *v1beta1.PodDis // Watch returns a watch.Interface that watches the requested podDisruptionBudgets. func (c *podDisruptionBudgets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("poddisruptionbudgets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *podDisruptionBudgets) Delete(name string, options *v1.DeleteOptions) er // DeleteCollection deletes a collection of objects. func (c *podDisruptionBudgets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("poddisruptionbudgets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/policy/v1beta1/podsecuritypolicy.go b/staging/src/k8s.io/client-go/kubernetes/typed/policy/v1beta1/podsecuritypolicy.go index 355be1e9c7f..d02096d747a 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/policy/v1beta1/podsecuritypolicy.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/policy/v1beta1/podsecuritypolicy.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/policy/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *podSecurityPolicies) Get(name string, options v1.GetOptions) (result *v // List takes label and field selectors, and returns the list of PodSecurityPolicies that match those selectors. func (c *podSecurityPolicies) List(opts v1.ListOptions) (result *v1beta1.PodSecurityPolicyList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.PodSecurityPolicyList{} err = c.client.Get(). Resource("podsecuritypolicies"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *podSecurityPolicies) List(opts v1.ListOptions) (result *v1beta1.PodSecu // Watch returns a watch.Interface that watches the requested podSecurityPolicies. func (c *podSecurityPolicies) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("podsecuritypolicies"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *podSecurityPolicies) Delete(name string, options *v1.DeleteOptions) err // DeleteCollection deletes a collection of objects. func (c *podSecurityPolicies) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("podsecuritypolicies"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/OWNERS b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/OWNERS new file mode 100644 index 00000000000..cd0d70a0f8f --- /dev/null +++ b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authorizers-approvers +reviewers: +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/clusterrole.go b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/clusterrole.go index c4299d4c68c..0a47c441150 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/clusterrole.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/clusterrole.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *clusterRoles) Get(name string, options metav1.GetOptions) (result *v1.C // List takes label and field selectors, and returns the list of ClusterRoles that match those selectors. func (c *clusterRoles) List(opts metav1.ListOptions) (result *v1.ClusterRoleList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.ClusterRoleList{} err = c.client.Get(). Resource("clusterroles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *clusterRoles) List(opts metav1.ListOptions) (result *v1.ClusterRoleList // Watch returns a watch.Interface that watches the requested clusterRoles. func (c *clusterRoles) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("clusterroles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *clusterRoles) Delete(name string, options *metav1.DeleteOptions) error // DeleteCollection deletes a collection of objects. func (c *clusterRoles) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("clusterroles"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/clusterrolebinding.go b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/clusterrolebinding.go index 30c0469a4fc..c16ebc31222 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/clusterrolebinding.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/clusterrolebinding.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *clusterRoleBindings) Get(name string, options metav1.GetOptions) (resul // List takes label and field selectors, and returns the list of ClusterRoleBindings that match those selectors. func (c *clusterRoleBindings) List(opts metav1.ListOptions) (result *v1.ClusterRoleBindingList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.ClusterRoleBindingList{} err = c.client.Get(). Resource("clusterrolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *clusterRoleBindings) List(opts metav1.ListOptions) (result *v1.ClusterR // Watch returns a watch.Interface that watches the requested clusterRoleBindings. func (c *clusterRoleBindings) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("clusterrolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *clusterRoleBindings) Delete(name string, options *metav1.DeleteOptions) // DeleteCollection deletes a collection of objects. func (c *clusterRoleBindings) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("clusterrolebindings"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/role.go b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/role.go index 81ea12a9ff5..a17d791fff2 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/role.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/role.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *roles) Get(name string, options metav1.GetOptions) (result *v1.Role, er // List takes label and field selectors, and returns the list of Roles that match those selectors. func (c *roles) List(opts metav1.ListOptions) (result *v1.RoleList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.RoleList{} err = c.client.Get(). Namespace(c.ns). Resource("roles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *roles) List(opts metav1.ListOptions) (result *v1.RoleList, err error) { // Watch returns a watch.Interface that watches the requested roles. func (c *roles) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("roles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *roles) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *roles) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("roles"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/rolebinding.go b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/rolebinding.go index 17c6f9913b0..c87e457188e 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/rolebinding.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1/rolebinding.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *roleBindings) Get(name string, options metav1.GetOptions) (result *v1.R // List takes label and field selectors, and returns the list of RoleBindings that match those selectors. func (c *roleBindings) List(opts metav1.ListOptions) (result *v1.RoleBindingList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.RoleBindingList{} err = c.client.Get(). Namespace(c.ns). Resource("rolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *roleBindings) List(opts metav1.ListOptions) (result *v1.RoleBindingList // Watch returns a watch.Interface that watches the requested roleBindings. func (c *roleBindings) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("rolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *roleBindings) Delete(name string, options *metav1.DeleteOptions) error // DeleteCollection deletes a collection of objects. func (c *roleBindings) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("rolebindings"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/clusterrole.go b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/clusterrole.go index 37a54576231..77e66877e72 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/clusterrole.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/clusterrole.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1alpha1 "k8s.io/api/rbac/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *clusterRoles) Get(name string, options v1.GetOptions) (result *v1alpha1 // List takes label and field selectors, and returns the list of ClusterRoles that match those selectors. func (c *clusterRoles) List(opts v1.ListOptions) (result *v1alpha1.ClusterRoleList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.ClusterRoleList{} err = c.client.Get(). Resource("clusterroles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *clusterRoles) List(opts v1.ListOptions) (result *v1alpha1.ClusterRoleLi // Watch returns a watch.Interface that watches the requested clusterRoles. func (c *clusterRoles) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("clusterroles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *clusterRoles) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *clusterRoles) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("clusterroles"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/clusterrolebinding.go b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/clusterrolebinding.go index 6050789066d..0d1b9d2051e 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/clusterrolebinding.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/clusterrolebinding.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1alpha1 "k8s.io/api/rbac/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *clusterRoleBindings) Get(name string, options v1.GetOptions) (result *v // List takes label and field selectors, and returns the list of ClusterRoleBindings that match those selectors. func (c *clusterRoleBindings) List(opts v1.ListOptions) (result *v1alpha1.ClusterRoleBindingList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.ClusterRoleBindingList{} err = c.client.Get(). Resource("clusterrolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *clusterRoleBindings) List(opts v1.ListOptions) (result *v1alpha1.Cluste // Watch returns a watch.Interface that watches the requested clusterRoleBindings. func (c *clusterRoleBindings) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("clusterrolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *clusterRoleBindings) Delete(name string, options *v1.DeleteOptions) err // DeleteCollection deletes a collection of objects. func (c *clusterRoleBindings) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("clusterrolebindings"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/role.go b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/role.go index aa6954bb579..4a4b67240b3 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/role.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/role.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1alpha1 "k8s.io/api/rbac/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *roles) Get(name string, options v1.GetOptions) (result *v1alpha1.Role, // List takes label and field selectors, and returns the list of Roles that match those selectors. func (c *roles) List(opts v1.ListOptions) (result *v1alpha1.RoleList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.RoleList{} err = c.client.Get(). Namespace(c.ns). Resource("roles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *roles) List(opts v1.ListOptions) (result *v1alpha1.RoleList, err error) // Watch returns a watch.Interface that watches the requested roles. func (c *roles) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("roles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *roles) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *roles) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("roles"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/rolebinding.go b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/rolebinding.go index 0941b8e8671..bf4e5a10efb 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/rolebinding.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/rolebinding.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1alpha1 "k8s.io/api/rbac/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *roleBindings) Get(name string, options v1.GetOptions) (result *v1alpha1 // List takes label and field selectors, and returns the list of RoleBindings that match those selectors. func (c *roleBindings) List(opts v1.ListOptions) (result *v1alpha1.RoleBindingList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.RoleBindingList{} err = c.client.Get(). Namespace(c.ns). Resource("rolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *roleBindings) List(opts v1.ListOptions) (result *v1alpha1.RoleBindingLi // Watch returns a watch.Interface that watches the requested roleBindings. func (c *roleBindings) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("rolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *roleBindings) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *roleBindings) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("rolebindings"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/clusterrole.go b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/clusterrole.go index bac951c876c..21d3cab3733 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/clusterrole.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/clusterrole.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/rbac/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *clusterRoles) Get(name string, options v1.GetOptions) (result *v1beta1. // List takes label and field selectors, and returns the list of ClusterRoles that match those selectors. func (c *clusterRoles) List(opts v1.ListOptions) (result *v1beta1.ClusterRoleList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.ClusterRoleList{} err = c.client.Get(). Resource("clusterroles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *clusterRoles) List(opts v1.ListOptions) (result *v1beta1.ClusterRoleLis // Watch returns a watch.Interface that watches the requested clusterRoles. func (c *clusterRoles) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("clusterroles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *clusterRoles) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *clusterRoles) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("clusterroles"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/clusterrolebinding.go b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/clusterrolebinding.go index 96c91de6e2c..47eb9e4e77b 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/clusterrolebinding.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/clusterrolebinding.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/rbac/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *clusterRoleBindings) Get(name string, options v1.GetOptions) (result *v // List takes label and field selectors, and returns the list of ClusterRoleBindings that match those selectors. func (c *clusterRoleBindings) List(opts v1.ListOptions) (result *v1beta1.ClusterRoleBindingList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.ClusterRoleBindingList{} err = c.client.Get(). Resource("clusterrolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *clusterRoleBindings) List(opts v1.ListOptions) (result *v1beta1.Cluster // Watch returns a watch.Interface that watches the requested clusterRoleBindings. func (c *clusterRoleBindings) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("clusterrolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *clusterRoleBindings) Delete(name string, options *v1.DeleteOptions) err // DeleteCollection deletes a collection of objects. func (c *clusterRoleBindings) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("clusterrolebindings"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/role.go b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/role.go index 66f382c07c8..2b61aad5231 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/role.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/role.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/rbac/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *roles) Get(name string, options v1.GetOptions) (result *v1beta1.Role, e // List takes label and field selectors, and returns the list of Roles that match those selectors. func (c *roles) List(opts v1.ListOptions) (result *v1beta1.RoleList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.RoleList{} err = c.client.Get(). Namespace(c.ns). Resource("roles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *roles) List(opts v1.ListOptions) (result *v1beta1.RoleList, err error) // Watch returns a watch.Interface that watches the requested roles. func (c *roles) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("roles"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *roles) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *roles) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("roles"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/rolebinding.go b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/rolebinding.go index 67d3d331bc2..0bd118fdfeb 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/rolebinding.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1beta1/rolebinding.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/rbac/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *roleBindings) Get(name string, options v1.GetOptions) (result *v1beta1. // List takes label and field selectors, and returns the list of RoleBindings that match those selectors. func (c *roleBindings) List(opts v1.ListOptions) (result *v1beta1.RoleBindingList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.RoleBindingList{} err = c.client.Get(). Namespace(c.ns). Resource("rolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *roleBindings) List(opts v1.ListOptions) (result *v1beta1.RoleBindingLis // Watch returns a watch.Interface that watches the requested roleBindings. func (c *roleBindings) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("rolebindings"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *roleBindings) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *roleBindings) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("rolebindings"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1/priorityclass.go b/staging/src/k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1/priorityclass.go index 6845d25c385..29d646fb1f0 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1/priorityclass.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1/priorityclass.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1alpha1 "k8s.io/api/scheduling/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *priorityClasses) Get(name string, options v1.GetOptions) (result *v1alp // List takes label and field selectors, and returns the list of PriorityClasses that match those selectors. func (c *priorityClasses) List(opts v1.ListOptions) (result *v1alpha1.PriorityClassList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.PriorityClassList{} err = c.client.Get(). Resource("priorityclasses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *priorityClasses) List(opts v1.ListOptions) (result *v1alpha1.PriorityCl // Watch returns a watch.Interface that watches the requested priorityClasses. func (c *priorityClasses) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("priorityclasses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *priorityClasses) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *priorityClasses) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("priorityclasses"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/scheduling/v1beta1/priorityclass.go b/staging/src/k8s.io/client-go/kubernetes/typed/scheduling/v1beta1/priorityclass.go index 57b9766e42e..5e402f8e342 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/scheduling/v1beta1/priorityclass.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/scheduling/v1beta1/priorityclass.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/scheduling/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *priorityClasses) Get(name string, options v1.GetOptions) (result *v1bet // List takes label and field selectors, and returns the list of PriorityClasses that match those selectors. func (c *priorityClasses) List(opts v1.ListOptions) (result *v1beta1.PriorityClassList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.PriorityClassList{} err = c.client.Get(). Resource("priorityclasses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *priorityClasses) List(opts v1.ListOptions) (result *v1beta1.PriorityCla // Watch returns a watch.Interface that watches the requested priorityClasses. func (c *priorityClasses) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("priorityclasses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *priorityClasses) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *priorityClasses) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("priorityclasses"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/settings/v1alpha1/podpreset.go b/staging/src/k8s.io/client-go/kubernetes/typed/settings/v1alpha1/podpreset.go index f000ae486cd..8fd6adc56b6 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/settings/v1alpha1/podpreset.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/settings/v1alpha1/podpreset.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1alpha1 "k8s.io/api/settings/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -75,11 +77,16 @@ func (c *podPresets) Get(name string, options v1.GetOptions) (result *v1alpha1.P // List takes label and field selectors, and returns the list of PodPresets that match those selectors. func (c *podPresets) List(opts v1.ListOptions) (result *v1alpha1.PodPresetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.PodPresetList{} err = c.client.Get(). Namespace(c.ns). Resource("podpresets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -87,11 +94,16 @@ func (c *podPresets) List(opts v1.ListOptions) (result *v1alpha1.PodPresetList, // Watch returns a watch.Interface that watches the requested podPresets. func (c *podPresets) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("podpresets"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -133,10 +145,15 @@ func (c *podPresets) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *podPresets) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("podpresets"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/BUILD b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/BUILD index 7948513c080..dee1e799cd9 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/BUILD +++ b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/BUILD @@ -12,6 +12,7 @@ go_library( "generated_expansion.go", "storage_client.go", "storageclass.go", + "volumeattachment.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/kubernetes/typed/storage/v1", importpath = "k8s.io/client-go/kubernetes/typed/storage/v1", diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/fake/BUILD b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/fake/BUILD index c80e0cc9377..9bdd5a4edea 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/fake/BUILD +++ b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/fake/BUILD @@ -11,6 +11,7 @@ go_library( "doc.go", "fake_storage_client.go", "fake_storageclass.go", + "fake_volumeattachment.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/kubernetes/typed/storage/v1/fake", importpath = "k8s.io/client-go/kubernetes/typed/storage/v1/fake", diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/fake/fake_storage_client.go b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/fake/fake_storage_client.go index fc6f98cf6ac..967a528500e 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/fake/fake_storage_client.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/fake/fake_storage_client.go @@ -32,6 +32,10 @@ func (c *FakeStorageV1) StorageClasses() v1.StorageClassInterface { return &FakeStorageClasses{c} } +func (c *FakeStorageV1) VolumeAttachments() v1.VolumeAttachmentInterface { + return &FakeVolumeAttachments{c} +} + // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. func (c *FakeStorageV1) RESTClient() rest.Interface { diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/fake/fake_volumeattachment.go b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/fake/fake_volumeattachment.go new file mode 100644 index 00000000000..58e09da46be --- /dev/null +++ b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/fake/fake_volumeattachment.go @@ -0,0 +1,131 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + storagev1 "k8s.io/api/storage/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeVolumeAttachments implements VolumeAttachmentInterface +type FakeVolumeAttachments struct { + Fake *FakeStorageV1 +} + +var volumeattachmentsResource = schema.GroupVersionResource{Group: "storage.k8s.io", Version: "v1", Resource: "volumeattachments"} + +var volumeattachmentsKind = schema.GroupVersionKind{Group: "storage.k8s.io", Version: "v1", Kind: "VolumeAttachment"} + +// Get takes name of the volumeAttachment, and returns the corresponding volumeAttachment object, and an error if there is any. +func (c *FakeVolumeAttachments) Get(name string, options v1.GetOptions) (result *storagev1.VolumeAttachment, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(volumeattachmentsResource, name), &storagev1.VolumeAttachment{}) + if obj == nil { + return nil, err + } + return obj.(*storagev1.VolumeAttachment), err +} + +// List takes label and field selectors, and returns the list of VolumeAttachments that match those selectors. +func (c *FakeVolumeAttachments) List(opts v1.ListOptions) (result *storagev1.VolumeAttachmentList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(volumeattachmentsResource, volumeattachmentsKind, opts), &storagev1.VolumeAttachmentList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &storagev1.VolumeAttachmentList{ListMeta: obj.(*storagev1.VolumeAttachmentList).ListMeta} + for _, item := range obj.(*storagev1.VolumeAttachmentList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested volumeAttachments. +func (c *FakeVolumeAttachments) Watch(opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(volumeattachmentsResource, opts)) +} + +// Create takes the representation of a volumeAttachment and creates it. Returns the server's representation of the volumeAttachment, and an error, if there is any. +func (c *FakeVolumeAttachments) Create(volumeAttachment *storagev1.VolumeAttachment) (result *storagev1.VolumeAttachment, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(volumeattachmentsResource, volumeAttachment), &storagev1.VolumeAttachment{}) + if obj == nil { + return nil, err + } + return obj.(*storagev1.VolumeAttachment), err +} + +// Update takes the representation of a volumeAttachment and updates it. Returns the server's representation of the volumeAttachment, and an error, if there is any. +func (c *FakeVolumeAttachments) Update(volumeAttachment *storagev1.VolumeAttachment) (result *storagev1.VolumeAttachment, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(volumeattachmentsResource, volumeAttachment), &storagev1.VolumeAttachment{}) + if obj == nil { + return nil, err + } + return obj.(*storagev1.VolumeAttachment), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeVolumeAttachments) UpdateStatus(volumeAttachment *storagev1.VolumeAttachment) (*storagev1.VolumeAttachment, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(volumeattachmentsResource, "status", volumeAttachment), &storagev1.VolumeAttachment{}) + if obj == nil { + return nil, err + } + return obj.(*storagev1.VolumeAttachment), err +} + +// Delete takes name of the volumeAttachment and deletes it. Returns an error if one occurs. +func (c *FakeVolumeAttachments) Delete(name string, options *v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteAction(volumeattachmentsResource, name), &storagev1.VolumeAttachment{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeVolumeAttachments) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(volumeattachmentsResource, listOptions) + + _, err := c.Fake.Invokes(action, &storagev1.VolumeAttachmentList{}) + return err +} + +// Patch applies the patch and returns the patched volumeAttachment. +func (c *FakeVolumeAttachments) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *storagev1.VolumeAttachment, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(volumeattachmentsResource, name, pt, data, subresources...), &storagev1.VolumeAttachment{}) + if obj == nil { + return nil, err + } + return obj.(*storagev1.VolumeAttachment), err +} diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/generated_expansion.go b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/generated_expansion.go index 2bea7ec7fd9..ccac16114c8 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/generated_expansion.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/generated_expansion.go @@ -19,3 +19,5 @@ limitations under the License. package v1 type StorageClassExpansion interface{} + +type VolumeAttachmentExpansion interface{} diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/storage_client.go b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/storage_client.go index ac48f491699..92378cf7f49 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/storage_client.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/storage_client.go @@ -28,6 +28,7 @@ import ( type StorageV1Interface interface { RESTClient() rest.Interface StorageClassesGetter + VolumeAttachmentsGetter } // StorageV1Client is used to interact with features provided by the storage.k8s.io group. @@ -39,6 +40,10 @@ func (c *StorageV1Client) StorageClasses() StorageClassInterface { return newStorageClasses(c) } +func (c *StorageV1Client) VolumeAttachments() VolumeAttachmentInterface { + return newVolumeAttachments(c) +} + // NewForConfig creates a new StorageV1Client for the given config. func NewForConfig(c *rest.Config) (*StorageV1Client, error) { config := *c diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/storageclass.go b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/storageclass.go index 0f7f57f05fb..3f4c48f0a0c 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/storageclass.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/storageclass.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + v1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *storageClasses) Get(name string, options metav1.GetOptions) (result *v1 // List takes label and field selectors, and returns the list of StorageClasses that match those selectors. func (c *storageClasses) List(opts metav1.ListOptions) (result *v1.StorageClassList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.StorageClassList{} err = c.client.Get(). Resource("storageclasses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *storageClasses) List(opts metav1.ListOptions) (result *v1.StorageClassL // Watch returns a watch.Interface that watches the requested storageClasses. func (c *storageClasses) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("storageclasses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *storageClasses) Delete(name string, options *metav1.DeleteOptions) erro // DeleteCollection deletes a collection of objects. func (c *storageClasses) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("storageclasses"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/volumeattachment.go b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/volumeattachment.go new file mode 100644 index 00000000000..0f45097b200 --- /dev/null +++ b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1/volumeattachment.go @@ -0,0 +1,180 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + "time" + + v1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + scheme "k8s.io/client-go/kubernetes/scheme" + rest "k8s.io/client-go/rest" +) + +// VolumeAttachmentsGetter has a method to return a VolumeAttachmentInterface. +// A group's client should implement this interface. +type VolumeAttachmentsGetter interface { + VolumeAttachments() VolumeAttachmentInterface +} + +// VolumeAttachmentInterface has methods to work with VolumeAttachment resources. +type VolumeAttachmentInterface interface { + Create(*v1.VolumeAttachment) (*v1.VolumeAttachment, error) + Update(*v1.VolumeAttachment) (*v1.VolumeAttachment, error) + UpdateStatus(*v1.VolumeAttachment) (*v1.VolumeAttachment, error) + Delete(name string, options *metav1.DeleteOptions) error + DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error + Get(name string, options metav1.GetOptions) (*v1.VolumeAttachment, error) + List(opts metav1.ListOptions) (*v1.VolumeAttachmentList, error) + Watch(opts metav1.ListOptions) (watch.Interface, error) + Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.VolumeAttachment, err error) + VolumeAttachmentExpansion +} + +// volumeAttachments implements VolumeAttachmentInterface +type volumeAttachments struct { + client rest.Interface +} + +// newVolumeAttachments returns a VolumeAttachments +func newVolumeAttachments(c *StorageV1Client) *volumeAttachments { + return &volumeAttachments{ + client: c.RESTClient(), + } +} + +// Get takes name of the volumeAttachment, and returns the corresponding volumeAttachment object, and an error if there is any. +func (c *volumeAttachments) Get(name string, options metav1.GetOptions) (result *v1.VolumeAttachment, err error) { + result = &v1.VolumeAttachment{} + err = c.client.Get(). + Resource("volumeattachments"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of VolumeAttachments that match those selectors. +func (c *volumeAttachments) List(opts metav1.ListOptions) (result *v1.VolumeAttachmentList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1.VolumeAttachmentList{} + err = c.client.Get(). + Resource("volumeattachments"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested volumeAttachments. +func (c *volumeAttachments) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Resource("volumeattachments"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch() +} + +// Create takes the representation of a volumeAttachment and creates it. Returns the server's representation of the volumeAttachment, and an error, if there is any. +func (c *volumeAttachments) Create(volumeAttachment *v1.VolumeAttachment) (result *v1.VolumeAttachment, err error) { + result = &v1.VolumeAttachment{} + err = c.client.Post(). + Resource("volumeattachments"). + Body(volumeAttachment). + Do(). + Into(result) + return +} + +// Update takes the representation of a volumeAttachment and updates it. Returns the server's representation of the volumeAttachment, and an error, if there is any. +func (c *volumeAttachments) Update(volumeAttachment *v1.VolumeAttachment) (result *v1.VolumeAttachment, err error) { + result = &v1.VolumeAttachment{} + err = c.client.Put(). + Resource("volumeattachments"). + Name(volumeAttachment.Name). + Body(volumeAttachment). + Do(). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + +func (c *volumeAttachments) UpdateStatus(volumeAttachment *v1.VolumeAttachment) (result *v1.VolumeAttachment, err error) { + result = &v1.VolumeAttachment{} + err = c.client.Put(). + Resource("volumeattachments"). + Name(volumeAttachment.Name). + SubResource("status"). + Body(volumeAttachment). + Do(). + Into(result) + return +} + +// Delete takes name of the volumeAttachment and deletes it. Returns an error if one occurs. +func (c *volumeAttachments) Delete(name string, options *metav1.DeleteOptions) error { + return c.client.Delete(). + Resource("volumeattachments"). + Name(name). + Body(options). + Do(). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *volumeAttachments) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Resource("volumeattachments"). + VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). + Body(options). + Do(). + Error() +} + +// Patch applies the patch and returns the patched volumeAttachment. +func (c *volumeAttachments) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.VolumeAttachment, err error) { + result = &v1.VolumeAttachment{} + err = c.client.Patch(pt). + Resource("volumeattachments"). + SubResource(subresources...). + Name(name). + Body(data). + Do(). + Into(result) + return +} diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1alpha1/volumeattachment.go b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1alpha1/volumeattachment.go index e6af0018594..7fef94e8d88 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1alpha1/volumeattachment.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1alpha1/volumeattachment.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1alpha1 "k8s.io/api/storage/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -73,10 +75,15 @@ func (c *volumeAttachments) Get(name string, options v1.GetOptions) (result *v1a // List takes label and field selectors, and returns the list of VolumeAttachments that match those selectors. func (c *volumeAttachments) List(opts v1.ListOptions) (result *v1alpha1.VolumeAttachmentList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.VolumeAttachmentList{} err = c.client.Get(). Resource("volumeattachments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *volumeAttachments) List(opts v1.ListOptions) (result *v1alpha1.VolumeAt // Watch returns a watch.Interface that watches the requested volumeAttachments. func (c *volumeAttachments) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("volumeattachments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *volumeAttachments) Delete(name string, options *v1.DeleteOptions) error // DeleteCollection deletes a collection of objects. func (c *volumeAttachments) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("volumeattachments"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1beta1/storageclass.go b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1beta1/storageclass.go index fbe1fd4c215..8a8f3891619 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1beta1/storageclass.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1beta1/storageclass.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/storage/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -72,10 +74,15 @@ func (c *storageClasses) Get(name string, options v1.GetOptions) (result *v1beta // List takes label and field selectors, and returns the list of StorageClasses that match those selectors. func (c *storageClasses) List(opts v1.ListOptions) (result *v1beta1.StorageClassList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.StorageClassList{} err = c.client.Get(). Resource("storageclasses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *storageClasses) List(opts v1.ListOptions) (result *v1beta1.StorageClass // Watch returns a watch.Interface that watches the requested storageClasses. func (c *storageClasses) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("storageclasses"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *storageClasses) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *storageClasses) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("storageclasses"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1beta1/volumeattachment.go b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1beta1/volumeattachment.go index 5cd2d3919f8..d319407f272 100644 --- a/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1beta1/volumeattachment.go +++ b/staging/src/k8s.io/client-go/kubernetes/typed/storage/v1beta1/volumeattachment.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1beta1 "k8s.io/api/storage/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -73,10 +75,15 @@ func (c *volumeAttachments) Get(name string, options v1.GetOptions) (result *v1b // List takes label and field selectors, and returns the list of VolumeAttachments that match those selectors. func (c *volumeAttachments) List(opts v1.ListOptions) (result *v1beta1.VolumeAttachmentList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.VolumeAttachmentList{} err = c.client.Get(). Resource("volumeattachments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *volumeAttachments) List(opts v1.ListOptions) (result *v1beta1.VolumeAtt // Watch returns a watch.Interface that watches the requested volumeAttachments. func (c *volumeAttachments) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("volumeattachments"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *volumeAttachments) Delete(name string, options *v1.DeleteOptions) error // DeleteCollection deletes a collection of objects. func (c *volumeAttachments) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("volumeattachments"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/client-go/kubernetes_test/BUILD b/staging/src/k8s.io/client-go/kubernetes_test/BUILD new file mode 100644 index 00000000000..fcabd03f90c --- /dev/null +++ b/staging/src/k8s.io/client-go/kubernetes_test/BUILD @@ -0,0 +1,29 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +go_test( + name = "go_default_test", + srcs = ["timeout_test.go"], + deps = [ + "//staging/src/k8s.io/api/apps/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/client-go/rest/fake:go_default_library", + "//vendor/github.com/davecgh/go-spew/spew:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/client-go/kubernetes_test/timeout_test.go b/staging/src/k8s.io/client-go/kubernetes_test/timeout_test.go new file mode 100644 index 00000000000..055fb4909df --- /dev/null +++ b/staging/src/k8s.io/client-go/kubernetes_test/timeout_test.go @@ -0,0 +1,60 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes_test + +import ( + "bytes" + "io/ioutil" + "net/http" + "testing" + + "github.com/davecgh/go-spew/spew" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + manualfake "k8s.io/client-go/rest/fake" +) + +func TestListTimeout(t *testing.T) { + fakeClient := &manualfake.RESTClient{ + GroupVersion: appsv1.SchemeGroupVersion, + NegotiatedSerializer: scheme.Codecs, + Client: manualfake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { + if req.URL.Query().Get("timeout") != "21s" { + t.Fatal(spew.Sdump(req.URL.Query())) + } + return &http.Response{StatusCode: http.StatusNotFound, Body: ioutil.NopCloser(&bytes.Buffer{})}, nil + }), + } + clientConfig := &rest.Config{ + APIPath: "/apis", + ContentConfig: rest.ContentConfig{ + NegotiatedSerializer: scheme.Codecs, + GroupVersion: &appsv1.SchemeGroupVersion, + }, + } + restClient, _ := rest.RESTClientFor(clientConfig) + restClient.Client = fakeClient.Client + realClient := kubernetes.New(restClient) + + timeout := int64(21) + realClient.AppsV1().DaemonSets("").List(metav1.ListOptions{TimeoutSeconds: &timeout}) + realClient.AppsV1().DaemonSets("").Watch(metav1.ListOptions{TimeoutSeconds: &timeout}) +} diff --git a/staging/src/k8s.io/client-go/listers/authentication/OWNERS b/staging/src/k8s.io/client-go/listers/authentication/OWNERS new file mode 100644 index 00000000000..c607d2aa8c5 --- /dev/null +++ b/staging/src/k8s.io/client-go/listers/authentication/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authenticators-approvers +reviewers: +- sig-auth-authenticators-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/client-go/listers/authorization/OWNERS b/staging/src/k8s.io/client-go/listers/authorization/OWNERS new file mode 100644 index 00000000000..cd0d70a0f8f --- /dev/null +++ b/staging/src/k8s.io/client-go/listers/authorization/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authorizers-approvers +reviewers: +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/client-go/listers/policy/v1beta1/BUILD b/staging/src/k8s.io/client-go/listers/policy/v1beta1/BUILD index 5c140cb3df0..640f0435cf5 100644 --- a/staging/src/k8s.io/client-go/listers/policy/v1beta1/BUILD +++ b/staging/src/k8s.io/client-go/listers/policy/v1beta1/BUILD @@ -23,7 +23,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/listers/policy/v1beta1/poddisruptionbudget_expansion.go b/staging/src/k8s.io/client-go/listers/policy/v1beta1/poddisruptionbudget_expansion.go index c0ab9d3ed4c..d07d11a98df 100644 --- a/staging/src/k8s.io/client-go/listers/policy/v1beta1/poddisruptionbudget_expansion.go +++ b/staging/src/k8s.io/client-go/listers/policy/v1beta1/poddisruptionbudget_expansion.go @@ -19,11 +19,11 @@ package v1beta1 import ( "fmt" - "github.com/golang/glog" "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/klog" ) // PodDisruptionBudgetListerExpansion allows custom methods to be added to @@ -54,7 +54,7 @@ func (s *podDisruptionBudgetLister) GetPodPodDisruptionBudgets(pod *v1.Pod) ([]* pdb := list[i] selector, err = metav1.LabelSelectorAsSelector(pdb.Spec.Selector) if err != nil { - glog.Warningf("invalid selector: %v", err) + klog.Warningf("invalid selector: %v", err) // TODO(mml): add an event to the PDB continue } diff --git a/staging/src/k8s.io/client-go/listers/rbac/OWNERS b/staging/src/k8s.io/client-go/listers/rbac/OWNERS new file mode 100644 index 00000000000..cd0d70a0f8f --- /dev/null +++ b/staging/src/k8s.io/client-go/listers/rbac/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authorizers-approvers +reviewers: +- sig-auth-authorizers-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/client-go/listers/storage/v1/BUILD b/staging/src/k8s.io/client-go/listers/storage/v1/BUILD index 319290572b5..495e3300fc1 100644 --- a/staging/src/k8s.io/client-go/listers/storage/v1/BUILD +++ b/staging/src/k8s.io/client-go/listers/storage/v1/BUILD @@ -10,6 +10,7 @@ go_library( srcs = [ "expansion_generated.go", "storageclass.go", + "volumeattachment.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/listers/storage/v1", importpath = "k8s.io/client-go/listers/storage/v1", diff --git a/staging/src/k8s.io/client-go/listers/storage/v1/expansion_generated.go b/staging/src/k8s.io/client-go/listers/storage/v1/expansion_generated.go index d932470649c..9d7d8887265 100644 --- a/staging/src/k8s.io/client-go/listers/storage/v1/expansion_generated.go +++ b/staging/src/k8s.io/client-go/listers/storage/v1/expansion_generated.go @@ -21,3 +21,7 @@ package v1 // StorageClassListerExpansion allows custom methods to be added to // StorageClassLister. type StorageClassListerExpansion interface{} + +// VolumeAttachmentListerExpansion allows custom methods to be added to +// VolumeAttachmentLister. +type VolumeAttachmentListerExpansion interface{} diff --git a/staging/src/k8s.io/client-go/listers/storage/v1/volumeattachment.go b/staging/src/k8s.io/client-go/listers/storage/v1/volumeattachment.go new file mode 100644 index 00000000000..14888812ec6 --- /dev/null +++ b/staging/src/k8s.io/client-go/listers/storage/v1/volumeattachment.go @@ -0,0 +1,65 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// VolumeAttachmentLister helps list VolumeAttachments. +type VolumeAttachmentLister interface { + // List lists all VolumeAttachments in the indexer. + List(selector labels.Selector) (ret []*v1.VolumeAttachment, err error) + // Get retrieves the VolumeAttachment from the index for a given name. + Get(name string) (*v1.VolumeAttachment, error) + VolumeAttachmentListerExpansion +} + +// volumeAttachmentLister implements the VolumeAttachmentLister interface. +type volumeAttachmentLister struct { + indexer cache.Indexer +} + +// NewVolumeAttachmentLister returns a new VolumeAttachmentLister. +func NewVolumeAttachmentLister(indexer cache.Indexer) VolumeAttachmentLister { + return &volumeAttachmentLister{indexer: indexer} +} + +// List lists all VolumeAttachments in the indexer. +func (s *volumeAttachmentLister) List(selector labels.Selector) (ret []*v1.VolumeAttachment, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1.VolumeAttachment)) + }) + return ret, err +} + +// Get retrieves the VolumeAttachment from the index for a given name. +func (s *volumeAttachmentLister) Get(name string) (*v1.VolumeAttachment, error) { + obj, exists, err := s.indexer.GetByKey(name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1.Resource("volumeattachment"), name) + } + return obj.(*v1.VolumeAttachment), nil +} diff --git a/staging/src/k8s.io/client-go/pkg/apis/clientauthentication/OWNERS b/staging/src/k8s.io/client-go/pkg/apis/clientauthentication/OWNERS new file mode 100644 index 00000000000..3b7ea1b131f --- /dev/null +++ b/staging/src/k8s.io/client-go/pkg/apis/clientauthentication/OWNERS @@ -0,0 +1,7 @@ +# approval on api packages bubbles to api-approvers +reviewers: +- sig-auth-authenticators-approvers +- sig-auth-authenticators-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/OWNERS b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/OWNERS new file mode 100644 index 00000000000..c607d2aa8c5 --- /dev/null +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authenticators-approvers +reviewers: +- sig-auth-authenticators-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/azure/BUILD b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/azure/BUILD index 264deb2383c..6a28475a76d 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/azure/BUILD +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/azure/BUILD @@ -24,7 +24,7 @@ go_library( "//vendor/github.com/Azure/go-autorest/autorest:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/azure/azure.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/azure/azure.go index 9858963e381..d42449fc257 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/azure/azure.go +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/azure/azure.go @@ -27,7 +27,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/adal" "github.com/Azure/go-autorest/autorest/azure" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/net" restclient "k8s.io/client-go/rest" @@ -50,7 +50,7 @@ const ( func init() { if err := restclient.RegisterAuthProviderPlugin("azure", newAzureAuthProvider); err != nil { - glog.Fatalf("Failed to register azure auth plugin: %v", err) + klog.Fatalf("Failed to register azure auth plugin: %v", err) } } @@ -124,7 +124,7 @@ func (r *azureRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) token, err := r.tokenSource.Token() if err != nil { - glog.Errorf("Failed to acquire a token: %v", err) + klog.Errorf("Failed to acquire a token: %v", err) return nil, fmt.Errorf("acquiring a token for authorization header: %v", err) } diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD index d2459367275..69415598332 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD @@ -18,8 +18,8 @@ go_library( "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//staging/src/k8s.io/client-go/transport:go_default_library", "//staging/src/k8s.io/client-go/util/connrotation:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/crypto/ssh/terminal:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go index cae9d0d618e..4d72526583e 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go @@ -31,7 +31,6 @@ import ( "sync" "time" - "github.com/golang/glog" "golang.org/x/crypto/ssh/terminal" "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -44,6 +43,7 @@ import ( "k8s.io/client-go/tools/clientcmd/api" "k8s.io/client-go/transport" "k8s.io/client-go/util/connrotation" + "k8s.io/klog" ) const execInfoEnv = "KUBERNETES_EXEC_INFO" @@ -228,7 +228,7 @@ func (r *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { Code: int32(res.StatusCode), } if err := r.a.maybeRefreshCreds(creds, resp); err != nil { - glog.Errorf("refreshing credentials: %v", err) + klog.Errorf("refreshing credentials: %v", err) } } return res, nil diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/gcp/BUILD b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/gcp/BUILD index 0f87724c3b7..12549227718 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/gcp/BUILD +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/gcp/BUILD @@ -23,9 +23,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/util/jsonpath:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/oauth2:go_default_library", "//vendor/golang.org/x/oauth2/google:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/gcp/gcp.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/gcp/gcp.go index 2bc6c4474bb..e44c2adabb3 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/gcp/gcp.go +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/gcp/gcp.go @@ -27,18 +27,18 @@ import ( "sync" "time" - "github.com/golang/glog" "golang.org/x/oauth2" "golang.org/x/oauth2/google" "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/yaml" restclient "k8s.io/client-go/rest" "k8s.io/client-go/util/jsonpath" + "k8s.io/klog" ) func init() { if err := restclient.RegisterAuthProviderPlugin("gcp", newGCPAuthProvider); err != nil { - glog.Fatalf("Failed to register gcp auth plugin: %v", err) + klog.Fatalf("Failed to register gcp auth plugin: %v", err) } } @@ -223,7 +223,7 @@ func (t *cachedTokenSource) Token() (*oauth2.Token, error) { cache := t.update(tok) if t.persister != nil { if err := t.persister.Persist(cache); err != nil { - glog.V(4).Infof("Failed to persist token: %v", err) + klog.V(4).Infof("Failed to persist token: %v", err) } } return tok, nil @@ -329,7 +329,7 @@ func (c *commandTokenSource) parseTokenCmdOutput(output []byte) (*oauth2.Token, } var expiry time.Time if t, err := time.Parse(c.timeFmt, expiryStr); err != nil { - glog.V(4).Infof("Failed to parse token expiry from %s (fmt=%s): %v", expiryStr, c.timeFmt, err) + klog.V(4).Infof("Failed to parse token expiry from %s (fmt=%s): %v", expiryStr, c.timeFmt, err) } else { expiry = t } @@ -373,7 +373,7 @@ func (t *conditionalTransport) RoundTrip(req *http.Request) (*http.Response, err } if res.StatusCode == 401 { - glog.V(4).Infof("The credentials that were supplied are invalid for the target cluster") + klog.V(4).Infof("The credentials that were supplied are invalid for the target cluster") t.persister.Persist(t.resetCache) } diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/oidc/BUILD b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/oidc/BUILD index 92674d54135..f87a2d6589a 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/oidc/BUILD +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/oidc/BUILD @@ -20,8 +20,8 @@ go_library( deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/oauth2:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/oidc/oidc.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/oidc/oidc.go index 9c3ea0ab8d5..1383a97c62e 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/oidc/oidc.go +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/oidc/oidc.go @@ -28,10 +28,10 @@ import ( "sync" "time" - "github.com/golang/glog" "golang.org/x/oauth2" "k8s.io/apimachinery/pkg/util/net" restclient "k8s.io/client-go/rest" + "k8s.io/klog" ) const ( @@ -49,7 +49,7 @@ const ( func init() { if err := restclient.RegisterAuthProviderPlugin("oidc", newOIDCAuthProvider); err != nil { - glog.Fatalf("Failed to register oidc auth plugin: %v", err) + klog.Fatalf("Failed to register oidc auth plugin: %v", err) } } @@ -124,7 +124,7 @@ func newOIDCAuthProvider(_ string, cfg map[string]string, persister restclient.A } if len(cfg[cfgExtraScopes]) > 0 { - glog.V(2).Infof("%s auth provider field depricated, refresh request don't send scopes", + klog.V(2).Infof("%s auth provider field depricated, refresh request don't send scopes", cfgExtraScopes) } diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/BUILD b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/BUILD index 15552999ba7..488af00d32b 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/BUILD +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/BUILD @@ -20,9 +20,9 @@ go_library( deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/gophercloud/gophercloud:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/openstack.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/openstack.go index e6d7f04934a..fab5104ef61 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/openstack.go +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/openstack.go @@ -22,9 +22,9 @@ import ( "sync" "time" - "github.com/golang/glog" "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/net" restclient "k8s.io/client-go/rest" @@ -32,7 +32,7 @@ import ( func init() { if err := restclient.RegisterAuthProviderPlugin("openstack", newOpenstackAuthProvider); err != nil { - glog.Fatalf("Failed to register openstack auth plugin: %s", err) + klog.Fatalf("Failed to register openstack auth plugin: %s", err) } } @@ -62,7 +62,7 @@ func (t *tokenGetter) Token() (string, error) { var err error if t.authOpt == nil { // reads the config from the environment - glog.V(4).Info("reading openstack config from the environment variables") + klog.V(4).Info("reading openstack config from the environment variables") options, err = openstack.AuthOptionsFromEnv() if err != nil { return "", fmt.Errorf("failed to read openstack env vars: %s", err) @@ -126,7 +126,7 @@ func (t *tokenRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) if err == nil { req.Header.Set("Authorization", "Bearer "+token) } else { - glog.V(4).Infof("failed to get token: %s", err) + klog.V(4).Infof("failed to get token: %s", err) } return t.RoundTripper.RoundTrip(req) @@ -140,7 +140,7 @@ func newOpenstackAuthProvider(_ string, config map[string]string, persister rest var ttlDuration time.Duration var err error - glog.Warningf("WARNING: in-tree openstack auth plugin is now deprecated. please use the \"client-keystone-auth\" kubectl/client-go credential plugin instead") + klog.Warningf("WARNING: in-tree openstack auth plugin is now deprecated. please use the \"client-keystone-auth\" kubectl/client-go credential plugin instead") ttl, found := config["ttl"] if !found { ttlDuration = DefaultTTLDuration diff --git a/staging/src/k8s.io/client-go/rest/BUILD b/staging/src/k8s.io/client-go/rest/BUILD index 0d95e60cad3..70920303e4e 100644 --- a/staging/src/k8s.io/client-go/rest/BUILD +++ b/staging/src/k8s.io/client-go/rest/BUILD @@ -39,10 +39,10 @@ go_test( "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/gofuzz:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/golang.org/x/oauth2:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -79,9 +79,9 @@ go_library( "//staging/src/k8s.io/client-go/transport:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/net/http2:go_default_library", "//vendor/golang.org/x/oauth2:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/rest/config.go b/staging/src/k8s.io/client-go/rest/config.go index 87e87905523..438eb3bedac 100644 --- a/staging/src/k8s.io/client-go/rest/config.go +++ b/staging/src/k8s.io/client-go/rest/config.go @@ -29,7 +29,6 @@ import ( "strings" "time" - "github.com/golang/glog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -37,6 +36,7 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" certutil "k8s.io/client-go/util/cert" "k8s.io/client-go/util/flowcontrol" + "k8s.io/klog" ) const ( @@ -322,7 +322,7 @@ func InClusterConfig() (*Config, error) { return nil, ErrNotInCluster } - ts := newCachedPathTokenSource(tokenFile) + ts := NewCachedFileTokenSource(tokenFile) if _, err := ts.Token(); err != nil { return nil, err @@ -331,7 +331,7 @@ func InClusterConfig() (*Config, error) { tlsClientConfig := TLSClientConfig{} if _, err := certutil.NewPool(rootCAFile); err != nil { - glog.Errorf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err) + klog.Errorf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err) } else { tlsClientConfig.CAFile = rootCAFile } diff --git a/staging/src/k8s.io/client-go/rest/plugin.go b/staging/src/k8s.io/client-go/rest/plugin.go index cf8fbabfdf1..83ef5ae320f 100644 --- a/staging/src/k8s.io/client-go/rest/plugin.go +++ b/staging/src/k8s.io/client-go/rest/plugin.go @@ -21,7 +21,7 @@ import ( "net/http" "sync" - "github.com/golang/glog" + "k8s.io/klog" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" ) @@ -57,7 +57,7 @@ func RegisterAuthProviderPlugin(name string, plugin Factory) error { if _, found := plugins[name]; found { return fmt.Errorf("Auth Provider Plugin %q was registered twice", name) } - glog.V(4).Infof("Registered Auth Provider Plugin %q", name) + klog.V(4).Infof("Registered Auth Provider Plugin %q", name) plugins[name] = plugin return nil } diff --git a/staging/src/k8s.io/client-go/rest/request.go b/staging/src/k8s.io/client-go/rest/request.go index 9bb311448ab..64901fba20d 100644 --- a/staging/src/k8s.io/client-go/rest/request.go +++ b/staging/src/k8s.io/client-go/rest/request.go @@ -32,7 +32,6 @@ import ( "strings" "time" - "github.com/golang/glog" "golang.org/x/net/http2" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -44,6 +43,7 @@ import ( restclientwatch "k8s.io/client-go/rest/watch" "k8s.io/client-go/tools/metrics" "k8s.io/client-go/util/flowcontrol" + "k8s.io/klog" ) var ( @@ -114,7 +114,7 @@ type Request struct { // NewRequest creates a new request helper object for accessing runtime.Objects on a server. func NewRequest(client HTTPClient, verb string, baseURL *url.URL, versionedAPIPath string, content ContentConfig, serializers Serializers, backoff BackoffManager, throttle flowcontrol.RateLimiter, timeout time.Duration) *Request { if backoff == nil { - glog.V(2).Infof("Not implementing request backoff strategy.") + klog.V(2).Infof("Not implementing request backoff strategy.") backoff = &NoBackoff{} } @@ -527,7 +527,7 @@ func (r *Request) tryThrottle() { r.throttle.Accept() } if latency := time.Since(now); latency > longThrottleLatency { - glog.V(4).Infof("Throttling request took %v, request: %s:%s", latency, r.verb, r.URL().String()) + klog.V(4).Infof("Throttling request took %v, request: %s:%s", latency, r.verb, r.URL().String()) } } @@ -683,7 +683,7 @@ func (r *Request) request(fn func(*http.Request, *http.Response)) error { }() if r.err != nil { - glog.V(4).Infof("Error in request: %v", r.err) + klog.V(4).Infof("Error in request: %v", r.err) return r.err } @@ -770,13 +770,13 @@ func (r *Request) request(fn func(*http.Request, *http.Response)) error { if seeker, ok := r.body.(io.Seeker); ok && r.body != nil { _, err := seeker.Seek(0, 0) if err != nil { - glog.V(4).Infof("Could not retry request, can't Seek() back to beginning of body for %T", r.body) + klog.V(4).Infof("Could not retry request, can't Seek() back to beginning of body for %T", r.body) fn(req, resp) return true } } - glog.V(4).Infof("Got a Retry-After %ds response for attempt %d to %v", seconds, retries, url) + klog.V(4).Infof("Got a Retry-After %ds response for attempt %d to %v", seconds, retries, url) r.backoffMgr.Sleep(time.Duration(seconds) * time.Second) return false } @@ -844,13 +844,13 @@ func (r *Request) transformResponse(resp *http.Response, req *http.Request) Resu // 2. Apiserver sends back the headers and then part of the body // 3. Apiserver closes connection. // 4. client-go should catch this and return an error. - glog.V(2).Infof("Stream error %#v when reading response body, may be caused by closed connection.", err) + klog.V(2).Infof("Stream error %#v when reading response body, may be caused by closed connection.", err) streamErr := fmt.Errorf("Stream error %#v when reading response body, may be caused by closed connection. Please retry.", err) return Result{ err: streamErr, } default: - glog.Errorf("Unexpected error when reading response body: %#v", err) + klog.Errorf("Unexpected error when reading response body: %#v", err) unexpectedErr := fmt.Errorf("Unexpected error %#v when reading response body. Please retry.", err) return Result{ err: unexpectedErr, @@ -914,11 +914,11 @@ func (r *Request) transformResponse(resp *http.Response, req *http.Request) Resu func truncateBody(body string) string { max := 0 switch { - case bool(glog.V(10)): + case bool(klog.V(10)): return body - case bool(glog.V(9)): + case bool(klog.V(9)): max = 10240 - case bool(glog.V(8)): + case bool(klog.V(8)): max = 1024 } @@ -933,13 +933,13 @@ func truncateBody(body string) string { // allocating a new string for the body output unless necessary. Uses a simple heuristic to determine // whether the body is printable. func glogBody(prefix string, body []byte) { - if glog.V(8) { + if klog.V(8) { if bytes.IndexFunc(body, func(r rune) bool { return r < 0x0a }) != -1 { - glog.Infof("%s:\n%s", prefix, truncateBody(hex.Dump(body))) + klog.Infof("%s:\n%s", prefix, truncateBody(hex.Dump(body))) } else { - glog.Infof("%s: %s", prefix, truncateBody(string(body))) + klog.Infof("%s: %s", prefix, truncateBody(string(body))) } } } @@ -1141,7 +1141,7 @@ func (r Result) Error() error { // to be backwards compatible with old servers that do not return a version, default to "v1" out, _, err := r.decoder.Decode(r.body, &schema.GroupVersionKind{Version: "v1"}, nil) if err != nil { - glog.V(5).Infof("body was not decodable (unable to check for Status): %v", err) + klog.V(5).Infof("body was not decodable (unable to check for Status): %v", err) return r.err } switch t := out.(type) { diff --git a/staging/src/k8s.io/client-go/rest/request_test.go b/staging/src/k8s.io/client-go/rest/request_test.go index 2660c0be5d4..a415f60ae79 100755 --- a/staging/src/k8s.io/client-go/rest/request_test.go +++ b/staging/src/k8s.io/client-go/rest/request_test.go @@ -35,7 +35,7 @@ import ( "testing" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -1855,6 +1855,10 @@ func buildString(length int) string { return string(s) } +func init() { + klog.InitFlags(nil) +} + func TestTruncateBody(t *testing.T) { tests := []struct { body string @@ -1904,7 +1908,7 @@ func TestTruncateBody(t *testing.T) { }, } - l := flag.Lookup("v").Value.(flag.Getter).Get().(glog.Level) + l := flag.Lookup("v").Value.(flag.Getter).Get().(klog.Level) for _, test := range tests { flag.Set("v", test.level) got := truncateBody(test.body) diff --git a/staging/src/k8s.io/client-go/rest/token_source.go b/staging/src/k8s.io/client-go/rest/token_source.go index 296b2a0481d..c251b5eb0bb 100644 --- a/staging/src/k8s.io/client-go/rest/token_source.go +++ b/staging/src/k8s.io/client-go/rest/token_source.go @@ -24,8 +24,8 @@ import ( "sync" "time" - "github.com/golang/glog" "golang.org/x/oauth2" + "k8s.io/klog" ) // TokenSourceWrapTransport returns a WrapTransport that injects bearer tokens @@ -42,7 +42,9 @@ func TokenSourceWrapTransport(ts oauth2.TokenSource) func(http.RoundTripper) htt } } -func newCachedPathTokenSource(path string) oauth2.TokenSource { +// NewCachedFileTokenSource returns a oauth2.TokenSource reads a token from a +// file at a specified path and periodically reloads it. +func NewCachedFileTokenSource(path string) oauth2.TokenSource { return &cachingTokenSource{ now: time.Now, leeway: 1 * time.Minute, @@ -129,7 +131,7 @@ func (ts *cachingTokenSource) Token() (*oauth2.Token, error) { if ts.tok == nil { return nil, err } - glog.Errorf("Unable to rotate token: %v", err) + klog.Errorf("Unable to rotate token: %v", err) return ts.tok, nil } diff --git a/staging/src/k8s.io/client-go/rest/urlbackoff.go b/staging/src/k8s.io/client-go/rest/urlbackoff.go index eff848abc12..d00e42f8667 100644 --- a/staging/src/k8s.io/client-go/rest/urlbackoff.go +++ b/staging/src/k8s.io/client-go/rest/urlbackoff.go @@ -20,9 +20,9 @@ import ( "net/url" "time" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/flowcontrol" + "k8s.io/klog" ) // Set of resp. Codes that we backoff for. @@ -64,7 +64,7 @@ func (n *NoBackoff) Sleep(d time.Duration) { // Disable makes the backoff trivial, i.e., sets it to zero. This might be used // by tests which want to run 1000s of mock requests without slowing down. func (b *URLBackoff) Disable() { - glog.V(4).Infof("Disabling backoff strategy") + klog.V(4).Infof("Disabling backoff strategy") b.Backoff = flowcontrol.NewBackOff(0*time.Second, 0*time.Second) } @@ -76,7 +76,7 @@ func (b *URLBackoff) baseUrlKey(rawurl *url.URL) string { // in the future. host, err := url.Parse(rawurl.String()) if err != nil { - glog.V(4).Infof("Error extracting url: %v", rawurl) + klog.V(4).Infof("Error extracting url: %v", rawurl) panic("bad url!") } return host.Host @@ -89,7 +89,7 @@ func (b *URLBackoff) UpdateBackoff(actualUrl *url.URL, err error, responseCode i b.Backoff.Next(b.baseUrlKey(actualUrl), b.Backoff.Clock.Now()) return } else if responseCode >= 300 || err != nil { - glog.V(4).Infof("Client is returning errors: code %v, error %v", responseCode, err) + klog.V(4).Infof("Client is returning errors: code %v, error %v", responseCode, err) } //If we got this far, there is no backoff required for this URL anymore. diff --git a/staging/src/k8s.io/client-go/restmapper/BUILD b/staging/src/k8s.io/client-go/restmapper/BUILD index d13169643ce..ed8006f0318 100644 --- a/staging/src/k8s.io/client-go/restmapper/BUILD +++ b/staging/src/k8s.io/client-go/restmapper/BUILD @@ -15,7 +15,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/client-go/discovery:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/restmapper/discovery.go b/staging/src/k8s.io/client-go/restmapper/discovery.go index aa158626af4..84491f4c5d1 100644 --- a/staging/src/k8s.io/client-go/restmapper/discovery.go +++ b/staging/src/k8s.io/client-go/restmapper/discovery.go @@ -26,7 +26,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/discovery" - "github.com/golang/glog" + "k8s.io/klog" ) // APIGroupResources is an API group with a mapping of versions to @@ -212,7 +212,7 @@ func (d *DeferredDiscoveryRESTMapper) getDelegate() (meta.RESTMapper, error) { // Reset resets the internally cached Discovery information and will // cause the next mapping request to re-discover. func (d *DeferredDiscoveryRESTMapper) Reset() { - glog.V(5).Info("Invalidating discovery information") + klog.V(5).Info("Invalidating discovery information") d.initMu.Lock() defer d.initMu.Unlock() diff --git a/staging/src/k8s.io/client-go/restmapper/shortcut.go b/staging/src/k8s.io/client-go/restmapper/shortcut.go index d9f4be0b6b1..6f3c9d93069 100644 --- a/staging/src/k8s.io/client-go/restmapper/shortcut.go +++ b/staging/src/k8s.io/client-go/restmapper/shortcut.go @@ -19,7 +19,7 @@ package restmapper import ( "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -86,12 +86,12 @@ func (e shortcutExpander) getShortcutMappings() ([]*metav1.APIResourceList, []re // This can return an error *and* the results it was able to find. We don't need to fail on the error. apiResList, err := e.discoveryClient.ServerResources() if err != nil { - glog.V(1).Infof("Error loading discovery information: %v", err) + klog.V(1).Infof("Error loading discovery information: %v", err) } for _, apiResources := range apiResList { gv, err := schema.ParseGroupVersion(apiResources.GroupVersion) if err != nil { - glog.V(1).Infof("Unable to parse groupversion = %s due to = %s", apiResources.GroupVersion, err.Error()) + klog.V(1).Infof("Unable to parse groupversion = %s due to = %s", apiResources.GroupVersion, err.Error()) continue } for _, apiRes := range apiResources.APIResources { diff --git a/staging/src/k8s.io/client-go/tools/auth/OWNERS b/staging/src/k8s.io/client-go/tools/auth/OWNERS new file mode 100644 index 00000000000..c607d2aa8c5 --- /dev/null +++ b/staging/src/k8s.io/client-go/tools/auth/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-authenticators-approvers +reviewers: +- sig-auth-authenticators-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/client-go/tools/cache/BUILD b/staging/src/k8s.io/client-go/tools/cache/BUILD index 0240fa6f700..8b8b9125855 100644 --- a/staging/src/k8s.io/client-go/tools/cache/BUILD +++ b/staging/src/k8s.io/client-go/tools/cache/BUILD @@ -83,7 +83,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/pager:go_default_library", "//staging/src/k8s.io/client-go/util/buffer:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/tools/cache/delta_fifo.go b/staging/src/k8s.io/client-go/tools/cache/delta_fifo.go index 45c3b500d42..f818a293a69 100644 --- a/staging/src/k8s.io/client-go/tools/cache/delta_fifo.go +++ b/staging/src/k8s.io/client-go/tools/cache/delta_fifo.go @@ -23,7 +23,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" - "github.com/golang/glog" + "k8s.io/klog" ) // NewDeltaFIFO returns a Store which can be used process changes to items. @@ -320,17 +320,15 @@ func (f *DeltaFIFO) queueActionLocked(actionType DeltaType, obj interface{}) err newDeltas := append(f.items[id], Delta{actionType, obj}) newDeltas = dedupDeltas(newDeltas) - _, exists := f.items[id] if len(newDeltas) > 0 { - if !exists { + if _, exists := f.items[id]; !exists { f.queue = append(f.queue, id) } f.items[id] = newDeltas f.cond.Broadcast() - } else if exists { - // We need to remove this from our map (extra items - // in the queue are ignored if they are not in the - // map). + } else { + // We need to remove this from our map (extra items in the queue are + // ignored if they are not in the map). delete(f.items, id) } return nil @@ -348,9 +346,6 @@ func (f *DeltaFIFO) List() []interface{} { func (f *DeltaFIFO) listLocked() []interface{} { list := make([]interface{}, 0, len(f.items)) for _, item := range f.items { - // Copy item's slice so operations on this slice - // won't interfere with the object we return. - item = copyDeltas(item) list = append(list, item.Newest().Object) } return list @@ -398,10 +393,7 @@ func (f *DeltaFIFO) GetByKey(key string) (item interface{}, exists bool, err err func (f *DeltaFIFO) IsClosed() bool { f.closedLock.Lock() defer f.closedLock.Unlock() - if f.closed { - return true - } - return false + return f.closed } // Pop blocks until an item is added to the queue, and then returns it. If @@ -432,10 +424,10 @@ func (f *DeltaFIFO) Pop(process PopProcessFunc) (interface{}, error) { } id := f.queue[0] f.queue = f.queue[1:] - item, ok := f.items[id] if f.initialPopulationCount > 0 { f.initialPopulationCount-- } + item, ok := f.items[id] if !ok { // Item may have been deleted subsequently. continue @@ -506,10 +498,10 @@ func (f *DeltaFIFO) Replace(list []interface{}, resourceVersion string) error { deletedObj, exists, err := f.knownObjects.GetByKey(k) if err != nil { deletedObj = nil - glog.Errorf("Unexpected error %v during lookup of key %v, placing DeleteFinalStateUnknown marker without object", err, k) + klog.Errorf("Unexpected error %v during lookup of key %v, placing DeleteFinalStateUnknown marker without object", err, k) } else if !exists { deletedObj = nil - glog.Infof("Key %v does not exist in known objects store, placing DeleteFinalStateUnknown marker without object", k) + klog.Infof("Key %v does not exist in known objects store, placing DeleteFinalStateUnknown marker without object", k) } queuedDeletions++ if err := f.queueActionLocked(Deleted, DeletedFinalStateUnknown{k, deletedObj}); err != nil { @@ -553,10 +545,10 @@ func (f *DeltaFIFO) syncKey(key string) error { func (f *DeltaFIFO) syncKeyLocked(key string) error { obj, exists, err := f.knownObjects.GetByKey(key) if err != nil { - glog.Errorf("Unexpected error %v during lookup of key %v, unable to queue object for sync", err, key) + klog.Errorf("Unexpected error %v during lookup of key %v, unable to queue object for sync", err, key) return nil } else if !exists { - glog.Infof("Key %v does not exist in known objects store, unable to queue object for sync", key) + klog.Infof("Key %v does not exist in known objects store, unable to queue object for sync", key) return nil } diff --git a/staging/src/k8s.io/client-go/tools/cache/expiration_cache.go b/staging/src/k8s.io/client-go/tools/cache/expiration_cache.go index d284453ec43..b38fe70b956 100644 --- a/staging/src/k8s.io/client-go/tools/cache/expiration_cache.go +++ b/staging/src/k8s.io/client-go/tools/cache/expiration_cache.go @@ -20,8 +20,8 @@ import ( "sync" "time" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/clock" + "k8s.io/klog" ) // ExpirationCache implements the store interface @@ -95,7 +95,7 @@ func (c *ExpirationCache) getOrExpire(key string) (interface{}, bool) { return nil, false } if c.expirationPolicy.IsExpired(timestampedItem) { - glog.V(4).Infof("Entry %v: %+v has expired", key, timestampedItem.obj) + klog.V(4).Infof("Entry %v: %+v has expired", key, timestampedItem.obj) c.cacheStorage.Delete(key) return nil, false } diff --git a/staging/src/k8s.io/client-go/tools/cache/listers.go b/staging/src/k8s.io/client-go/tools/cache/listers.go index 27d51a6b387..ce377329c7f 100644 --- a/staging/src/k8s.io/client-go/tools/cache/listers.go +++ b/staging/src/k8s.io/client-go/tools/cache/listers.go @@ -17,7 +17,7 @@ limitations under the License. package cache import ( - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -60,7 +60,7 @@ func ListAllByNamespace(indexer Indexer, namespace string, selector labels.Selec items, err := indexer.Index(NamespaceIndex, &metav1.ObjectMeta{Namespace: namespace}) if err != nil { // Ignore error; do slow search without index. - glog.Warningf("can not retrieve list of objects using index : %v", err) + klog.Warningf("can not retrieve list of objects using index : %v", err) for _, m := range indexer.List() { metadata, err := meta.Accessor(m) if err != nil { diff --git a/staging/src/k8s.io/client-go/tools/cache/mutation_cache.go b/staging/src/k8s.io/client-go/tools/cache/mutation_cache.go index cbb6434ebde..4c6686e918c 100644 --- a/staging/src/k8s.io/client-go/tools/cache/mutation_cache.go +++ b/staging/src/k8s.io/client-go/tools/cache/mutation_cache.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" @@ -156,7 +156,7 @@ func (c *mutationCache) ByIndex(name string, indexKey string) ([]interface{}, er } elements, err := fn(updated) if err != nil { - glog.V(4).Infof("Unable to calculate an index entry for mutation cache entry %s: %v", key, err) + klog.V(4).Infof("Unable to calculate an index entry for mutation cache entry %s: %v", key, err) continue } for _, inIndex := range elements { diff --git a/staging/src/k8s.io/client-go/tools/cache/mutation_detector.go b/staging/src/k8s.io/client-go/tools/cache/mutation_detector.go index e2aa4484840..adb5b8be8af 100644 --- a/staging/src/k8s.io/client-go/tools/cache/mutation_detector.go +++ b/staging/src/k8s.io/client-go/tools/cache/mutation_detector.go @@ -24,7 +24,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/diff" @@ -45,7 +45,7 @@ func NewCacheMutationDetector(name string) CacheMutationDetector { if !mutationDetectionEnabled { return dummyMutationDetector{} } - glog.Warningln("Mutation detector is enabled, this will result in memory leakage.") + klog.Warningln("Mutation detector is enabled, this will result in memory leakage.") return &defaultCacheMutationDetector{name: name, period: 1 * time.Second} } diff --git a/staging/src/k8s.io/client-go/tools/cache/reflector.go b/staging/src/k8s.io/client-go/tools/cache/reflector.go index 9ee7efcbbd8..12e2a334221 100644 --- a/staging/src/k8s.io/client-go/tools/cache/reflector.go +++ b/staging/src/k8s.io/client-go/tools/cache/reflector.go @@ -31,7 +31,6 @@ import ( "syscall" "time" - "github.com/golang/glog" apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -41,6 +40,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" + "k8s.io/klog" ) // Reflector watches a specified resource and causes all changes to be reflected in the given store. @@ -128,7 +128,7 @@ var internalPackages = []string{"client-go/tools/cache/"} // Run starts a watch and handles watch events. Will restart the watch if it is closed. // Run will exit when stopCh is closed. func (r *Reflector) Run(stopCh <-chan struct{}) { - glog.V(3).Infof("Starting reflector %v (%s) from %s", r.expectedType, r.resyncPeriod, r.name) + klog.V(3).Infof("Starting reflector %v (%s) from %s", r.expectedType, r.resyncPeriod, r.name) wait.Until(func() { if err := r.ListAndWatch(stopCh); err != nil { utilruntime.HandleError(err) @@ -166,7 +166,7 @@ func (r *Reflector) resyncChan() (<-chan time.Time, func() bool) { // and then use the resource version to watch. // It returns error if ListAndWatch didn't even try to initialize watch. func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error { - glog.V(3).Infof("Listing and watching %v from %s", r.expectedType, r.name) + klog.V(3).Infof("Listing and watching %v from %s", r.expectedType, r.name) var resourceVersion string // Explicitly set "0" as resource version - it's fine for the List() @@ -212,7 +212,7 @@ func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error { return } if r.ShouldResync == nil || r.ShouldResync() { - glog.V(4).Infof("%s: forcing resync", r.name) + klog.V(4).Infof("%s: forcing resync", r.name) if err := r.store.Resync(); err != nil { resyncerrc <- err return @@ -246,7 +246,7 @@ func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error { case io.EOF: // watch closed normally case io.ErrUnexpectedEOF: - glog.V(1).Infof("%s: Watch for %v closed with unexpected EOF: %v", r.name, r.expectedType, err) + klog.V(1).Infof("%s: Watch for %v closed with unexpected EOF: %v", r.name, r.expectedType, err) default: utilruntime.HandleError(fmt.Errorf("%s: Failed to watch %v: %v", r.name, r.expectedType, err)) } @@ -267,7 +267,7 @@ func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error { if err := r.watchHandler(w, &resourceVersion, resyncerrc, stopCh); err != nil { if err != errorStopRequested { - glog.Warningf("%s: watch of %v ended with: %v", r.name, r.expectedType, err) + klog.Warningf("%s: watch of %v ended with: %v", r.name, r.expectedType, err) } return nil } @@ -354,7 +354,7 @@ loop: r.metrics.numberOfShortWatches.Inc() return fmt.Errorf("very short watch: %s: Unexpected watch close - watch lasted less than a second and no items received", r.name) } - glog.V(4).Infof("%s: Watch close - %v total %v items received", r.name, r.expectedType, eventCount) + klog.V(4).Infof("%s: Watch close - %v total %v items received", r.name, r.expectedType, eventCount) return nil } diff --git a/staging/src/k8s.io/client-go/tools/cache/shared_informer.go b/staging/src/k8s.io/client-go/tools/cache/shared_informer.go index f29a4b33126..e91fc9e9558 100644 --- a/staging/src/k8s.io/client-go/tools/cache/shared_informer.go +++ b/staging/src/k8s.io/client-go/tools/cache/shared_informer.go @@ -28,7 +28,7 @@ import ( "k8s.io/client-go/util/buffer" "k8s.io/client-go/util/retry" - "github.com/golang/glog" + "k8s.io/klog" ) // SharedInformer has a shared data cache and is capable of distributing notifications for changes @@ -116,11 +116,11 @@ func WaitForCacheSync(stopCh <-chan struct{}, cacheSyncs ...InformerSynced) bool }, stopCh) if err != nil { - glog.V(2).Infof("stop requested") + klog.V(2).Infof("stop requested") return false } - glog.V(4).Infof("caches populated") + klog.V(4).Infof("caches populated") return true } @@ -279,11 +279,11 @@ func determineResyncPeriod(desired, check time.Duration) time.Duration { return desired } if check == 0 { - glog.Warningf("The specified resyncPeriod %v is invalid because this shared informer doesn't support resyncing", desired) + klog.Warningf("The specified resyncPeriod %v is invalid because this shared informer doesn't support resyncing", desired) return 0 } if desired < check { - glog.Warningf("The specified resyncPeriod %v is being increased to the minimum resyncCheckPeriod %v", desired, check) + klog.Warningf("The specified resyncPeriod %v is being increased to the minimum resyncCheckPeriod %v", desired, check) return check } return desired @@ -296,19 +296,19 @@ func (s *sharedIndexInformer) AddEventHandlerWithResyncPeriod(handler ResourceEv defer s.startedLock.Unlock() if s.stopped { - glog.V(2).Infof("Handler %v was not added to shared informer because it has stopped already", handler) + klog.V(2).Infof("Handler %v was not added to shared informer because it has stopped already", handler) return } if resyncPeriod > 0 { if resyncPeriod < minimumResyncPeriod { - glog.Warningf("resyncPeriod %d is too small. Changing it to the minimum allowed value of %d", resyncPeriod, minimumResyncPeriod) + klog.Warningf("resyncPeriod %d is too small. Changing it to the minimum allowed value of %d", resyncPeriod, minimumResyncPeriod) resyncPeriod = minimumResyncPeriod } if resyncPeriod < s.resyncCheckPeriod { if s.started { - glog.Warningf("resyncPeriod %d is smaller than resyncCheckPeriod %d and the informer has already started. Changing it to %d", resyncPeriod, s.resyncCheckPeriod, s.resyncCheckPeriod) + klog.Warningf("resyncPeriod %d is smaller than resyncCheckPeriod %d and the informer has already started. Changing it to %d", resyncPeriod, s.resyncCheckPeriod, s.resyncCheckPeriod) resyncPeriod = s.resyncCheckPeriod } else { // if the event handler's resyncPeriod is smaller than the current resyncCheckPeriod, update diff --git a/staging/src/k8s.io/client-go/tools/clientcmd/BUILD b/staging/src/k8s.io/client-go/tools/clientcmd/BUILD index 59a860ffb59..8746085e1a1 100644 --- a/staging/src/k8s.io/client-go/tools/clientcmd/BUILD +++ b/staging/src/k8s.io/client-go/tools/clientcmd/BUILD @@ -22,9 +22,9 @@ go_test( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api/latest:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/imdario/mergo:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) @@ -54,10 +54,10 @@ go_library( "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api/latest:go_default_library", "//staging/src/k8s.io/client-go/util/homedir:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/imdario/mergo:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/golang.org/x/crypto/ssh/terminal:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/tools/clientcmd/api/BUILD b/staging/src/k8s.io/client-go/tools/clientcmd/api/BUILD index 222cec2a545..ce7dac4fb52 100644 --- a/staging/src/k8s.io/client-go/tools/clientcmd/api/BUILD +++ b/staging/src/k8s.io/client-go/tools/clientcmd/api/BUILD @@ -13,7 +13,7 @@ go_test( "types_test.go", ], embed = [":go_default_library"], - deps = ["//vendor/github.com/ghodss/yaml:go_default_library"], + deps = ["//vendor/sigs.k8s.io/yaml:go_default_library"], ) go_library( diff --git a/staging/src/k8s.io/client-go/tools/clientcmd/api/helpers_test.go b/staging/src/k8s.io/client-go/tools/clientcmd/api/helpers_test.go index 7ec30e5da99..48b3cec2861 100644 --- a/staging/src/k8s.io/client-go/tools/clientcmd/api/helpers_test.go +++ b/staging/src/k8s.io/client-go/tools/clientcmd/api/helpers_test.go @@ -23,7 +23,7 @@ import ( "reflect" "testing" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) func newMergedConfig(certFile, certContent, keyFile, keyContent, caFile, caContent string, t *testing.T) Config { diff --git a/staging/src/k8s.io/client-go/tools/clientcmd/api/types_test.go b/staging/src/k8s.io/client-go/tools/clientcmd/api/types_test.go index bd34834521a..f7190d93e1b 100644 --- a/staging/src/k8s.io/client-go/tools/clientcmd/api/types_test.go +++ b/staging/src/k8s.io/client-go/tools/clientcmd/api/types_test.go @@ -19,7 +19,7 @@ package api import ( "fmt" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) func Example_emptyConfig() { diff --git a/staging/src/k8s.io/client-go/tools/clientcmd/client_config.go b/staging/src/k8s.io/client-go/tools/clientcmd/client_config.go index b8927f71087..dea229c9182 100644 --- a/staging/src/k8s.io/client-go/tools/clientcmd/client_config.go +++ b/staging/src/k8s.io/client-go/tools/clientcmd/client_config.go @@ -24,8 +24,8 @@ import ( "os" "strings" - "github.com/golang/glog" "github.com/imdario/mergo" + "k8s.io/klog" restclient "k8s.io/client-go/rest" clientauth "k8s.io/client-go/tools/auth" @@ -229,11 +229,11 @@ func (config *DirectClientConfig) getUserIdentificationPartialConfig(configAuthI if len(configAuthInfo.Token) > 0 { mergedConfig.BearerToken = configAuthInfo.Token } else if len(configAuthInfo.TokenFile) > 0 { - tokenBytes, err := ioutil.ReadFile(configAuthInfo.TokenFile) - if err != nil { + ts := restclient.NewCachedFileTokenSource(configAuthInfo.TokenFile) + if _, err := ts.Token(); err != nil { return nil, err } - mergedConfig.BearerToken = string(tokenBytes) + mergedConfig.WrapTransport = restclient.TokenSourceWrapTransport(ts) } if len(configAuthInfo.Impersonate) > 0 { mergedConfig.Impersonate = restclient.ImpersonationConfig{ @@ -545,12 +545,12 @@ func (config *inClusterClientConfig) Possible() bool { // to the default config. func BuildConfigFromFlags(masterUrl, kubeconfigPath string) (*restclient.Config, error) { if kubeconfigPath == "" && masterUrl == "" { - glog.Warningf("Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work.") + klog.Warningf("Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work.") kubeconfig, err := restclient.InClusterConfig() if err == nil { return kubeconfig, nil } - glog.Warning("error creating inClusterConfig, falling back to default config: ", err) + klog.Warning("error creating inClusterConfig, falling back to default config: ", err) } return NewNonInteractiveDeferredLoadingClientConfig( &ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, diff --git a/staging/src/k8s.io/client-go/tools/clientcmd/client_config_test.go b/staging/src/k8s.io/client-go/tools/clientcmd/client_config_test.go index 798aa3dd89d..6da850ed405 100644 --- a/staging/src/k8s.io/client-go/tools/clientcmd/client_config_test.go +++ b/staging/src/k8s.io/client-go/tools/clientcmd/client_config_test.go @@ -18,12 +18,14 @@ package clientcmd import ( "io/ioutil" + "net/http" "os" "reflect" "strings" "testing" "github.com/imdario/mergo" + restclient "k8s.io/client-go/rest" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" ) @@ -332,7 +334,19 @@ func TestBasicTokenFile(t *testing.T) { t.Fatalf("Unexpected error: %v", err) } - matchStringArg(token, clientConfig.BearerToken, t) + var out *http.Request + clientConfig.WrapTransport(fakeTransport(func(req *http.Request) (*http.Response, error) { + out = req + return &http.Response{}, nil + })).RoundTrip(&http.Request{}) + + matchStringArg(token, strings.TrimPrefix(out.Header.Get("Authorization"), "Bearer "), t) +} + +type fakeTransport func(*http.Request) (*http.Response, error) + +func (ft fakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return ft(req) } func TestPrecedenceTokenFile(t *testing.T) { diff --git a/staging/src/k8s.io/client-go/tools/clientcmd/config.go b/staging/src/k8s.io/client-go/tools/clientcmd/config.go index 9495849b092..b8cc3968821 100644 --- a/staging/src/k8s.io/client-go/tools/clientcmd/config.go +++ b/staging/src/k8s.io/client-go/tools/clientcmd/config.go @@ -24,7 +24,7 @@ import ( "reflect" "sort" - "github.com/golang/glog" + "k8s.io/klog" restclient "k8s.io/client-go/rest" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -483,7 +483,7 @@ func getConfigFromFile(filename string) (*clientcmdapi.Config, error) { func GetConfigFromFileOrDie(filename string) *clientcmdapi.Config { config, err := getConfigFromFile(filename) if err != nil { - glog.FatalDepth(1, err) + klog.FatalDepth(1, err) } return config diff --git a/staging/src/k8s.io/client-go/tools/clientcmd/loader.go b/staging/src/k8s.io/client-go/tools/clientcmd/loader.go index 6038c8d457a..7e928a91856 100644 --- a/staging/src/k8s.io/client-go/tools/clientcmd/loader.go +++ b/staging/src/k8s.io/client-go/tools/clientcmd/loader.go @@ -27,8 +27,8 @@ import ( goruntime "runtime" "strings" - "github.com/golang/glog" "github.com/imdario/mergo" + "k8s.io/klog" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -356,7 +356,7 @@ func LoadFromFile(filename string) (*clientcmdapi.Config, error) { if err != nil { return nil, err } - glog.V(6).Infoln("Config loaded from file", filename) + klog.V(6).Infoln("Config loaded from file", filename) // set LocationOfOrigin on every Cluster, User, and Context for key, obj := range config.AuthInfos { diff --git a/staging/src/k8s.io/client-go/tools/clientcmd/loader_test.go b/staging/src/k8s.io/client-go/tools/clientcmd/loader_test.go index 86eccac45f3..461d970bf11 100644 --- a/staging/src/k8s.io/client-go/tools/clientcmd/loader_test.go +++ b/staging/src/k8s.io/client-go/tools/clientcmd/loader_test.go @@ -26,7 +26,7 @@ import ( "strings" "testing" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" "k8s.io/apimachinery/pkg/runtime" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" diff --git a/staging/src/k8s.io/client-go/tools/clientcmd/merged_client_builder.go b/staging/src/k8s.io/client-go/tools/clientcmd/merged_client_builder.go index 05038133b6b..76380db82ab 100644 --- a/staging/src/k8s.io/client-go/tools/clientcmd/merged_client_builder.go +++ b/staging/src/k8s.io/client-go/tools/clientcmd/merged_client_builder.go @@ -20,7 +20,7 @@ import ( "io" "sync" - "github.com/golang/glog" + "k8s.io/klog" restclient "k8s.io/client-go/rest" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -119,7 +119,7 @@ func (config *DeferredLoadingClientConfig) ClientConfig() (*restclient.Config, e // check for in-cluster configuration and use it if config.icc.Possible() { - glog.V(4).Infof("Using in-cluster configuration") + klog.V(4).Infof("Using in-cluster configuration") return config.icc.ClientConfig() } @@ -156,7 +156,7 @@ func (config *DeferredLoadingClientConfig) Namespace() (string, bool, error) { } } - glog.V(4).Infof("Using in-cluster namespace") + klog.V(4).Infof("Using in-cluster namespace") // allow the namespace from the service account token directory to be used. return config.icc.Namespace() diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/BUILD b/staging/src/k8s.io/client-go/tools/leaderelection/BUILD index 8d53bfe508a..bed0f89fee7 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/BUILD +++ b/staging/src/k8s.io/client-go/tools/leaderelection/BUILD @@ -8,28 +8,36 @@ load( go_library( name = "go_default_library", - srcs = ["leaderelection.go"], + srcs = [ + "healthzadaptor.go", + "leaderelection.go", + ], importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/tools/leaderelection", importpath = "k8s.io/client-go/tools/leaderelection", deps = [ "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) go_test( name = "go_default_test", - srcs = ["leaderelection_test.go"], + srcs = [ + "healthzadaptor_test.go", + "leaderelection_test.go", + ], embed = [":go_default_library"], deps = [ "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", "//staging/src/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library", diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/healthzadaptor.go b/staging/src/k8s.io/client-go/tools/leaderelection/healthzadaptor.go new file mode 100644 index 00000000000..b9353729190 --- /dev/null +++ b/staging/src/k8s.io/client-go/tools/leaderelection/healthzadaptor.go @@ -0,0 +1,69 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package leaderelection + +import ( + "net/http" + "sync" + "time" +) + +// HealthzAdaptor associates the /healthz endpoint with the LeaderElection object. +// It helps deal with the /healthz endpoint being set up prior to the LeaderElection. +// This contains the code needed to act as an adaptor between the leader +// election code the health check code. It allows us to provide health +// status about the leader election. Most specifically about if the leader +// has failed to renew without exiting the process. In that case we should +// report not healthy and rely on the kubelet to take down the process. +type HealthzAdaptor struct { + pointerLock sync.Mutex + le *LeaderElector + timeout time.Duration +} + +// Name returns the name of the health check we are implementing. +func (l *HealthzAdaptor) Name() string { + return "leaderElection" +} + +// Check is called by the healthz endpoint handler. +// It fails (returns an error) if we own the lease but had not been able to renew it. +func (l *HealthzAdaptor) Check(req *http.Request) error { + l.pointerLock.Lock() + defer l.pointerLock.Unlock() + if l.le == nil { + return nil + } + return l.le.Check(l.timeout) +} + +// SetLeaderElection ties a leader election object to a HealthzAdaptor +func (l *HealthzAdaptor) SetLeaderElection(le *LeaderElector) { + l.pointerLock.Lock() + defer l.pointerLock.Unlock() + l.le = le +} + +// NewLeaderHealthzAdaptor creates a basic healthz adaptor to monitor a leader election. +// timeout determines the time beyond the lease expiry to be allowed for timeout. +// checks within the timeout period after the lease expires will still return healthy. +func NewLeaderHealthzAdaptor(timeout time.Duration) *HealthzAdaptor { + result := &HealthzAdaptor{ + timeout: timeout, + } + return result +} diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/healthzadaptor_test.go b/staging/src/k8s.io/client-go/tools/leaderelection/healthzadaptor_test.go new file mode 100644 index 00000000000..746d49130c4 --- /dev/null +++ b/staging/src/k8s.io/client-go/tools/leaderelection/healthzadaptor_test.go @@ -0,0 +1,175 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package leaderelection + +import ( + "fmt" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/clock" + rl "k8s.io/client-go/tools/leaderelection/resourcelock" + "net/http" +) + +type fakeLock struct { + identity string +} + +// Get is a dummy to allow us to have a fakeLock for testing. +func (fl *fakeLock) Get() (ler *rl.LeaderElectionRecord, err error) { + return nil, nil +} + +// Create is a dummy to allow us to have a fakeLock for testing. +func (fl *fakeLock) Create(ler rl.LeaderElectionRecord) error { + return nil +} + +// Update is a dummy to allow us to have a fakeLock for testing. +func (fl *fakeLock) Update(ler rl.LeaderElectionRecord) error { + return nil +} + +// RecordEvent is a dummy to allow us to have a fakeLock for testing. +func (fl *fakeLock) RecordEvent(string) {} + +// Identity is a dummy to allow us to have a fakeLock for testing. +func (fl *fakeLock) Identity() string { + return fl.identity +} + +// Describe is a dummy to allow us to have a fakeLock for testing. +func (fl *fakeLock) Describe() string { + return "Dummy implementation of lock for testing" +} + +// TestLeaderElectionHealthChecker tests that the healthcheck for leader election handles its edge cases. +func TestLeaderElectionHealthChecker(t *testing.T) { + current := time.Now() + req := &http.Request{} + + tests := []struct { + description string + expected error + adaptorTimeout time.Duration + elector *LeaderElector + }{ + { + description: "call check before leader elector initialized", + expected: nil, + adaptorTimeout: time.Second * 20, + elector: nil, + }, + { + description: "call check when the the lease is far expired", + expected: fmt.Errorf("failed election to renew leadership on lease %s", "foo"), + adaptorTimeout: time.Second * 20, + elector: &LeaderElector{ + config: LeaderElectionConfig{ + Lock: &fakeLock{identity: "healthTest"}, + LeaseDuration: time.Minute, + Name: "foo", + }, + observedRecord: rl.LeaderElectionRecord{ + HolderIdentity: "healthTest", + }, + observedTime: current, + clock: clock.NewFakeClock(current.Add(time.Hour)), + }, + }, + { + description: "call check when the the lease is far expired but held by another server", + expected: nil, + adaptorTimeout: time.Second * 20, + elector: &LeaderElector{ + config: LeaderElectionConfig{ + Lock: &fakeLock{identity: "healthTest"}, + LeaseDuration: time.Minute, + Name: "foo", + }, + observedRecord: rl.LeaderElectionRecord{ + HolderIdentity: "otherServer", + }, + observedTime: current, + clock: clock.NewFakeClock(current.Add(time.Hour)), + }, + }, + { + description: "call check when the the lease is not expired", + expected: nil, + adaptorTimeout: time.Second * 20, + elector: &LeaderElector{ + config: LeaderElectionConfig{ + Lock: &fakeLock{identity: "healthTest"}, + LeaseDuration: time.Minute, + Name: "foo", + }, + observedRecord: rl.LeaderElectionRecord{ + HolderIdentity: "healthTest", + }, + observedTime: current, + clock: clock.NewFakeClock(current), + }, + }, + { + description: "call check when the the lease is expired but inside the timeout", + expected: nil, + adaptorTimeout: time.Second * 20, + elector: &LeaderElector{ + config: LeaderElectionConfig{ + Lock: &fakeLock{identity: "healthTest"}, + LeaseDuration: time.Minute, + Name: "foo", + }, + observedRecord: rl.LeaderElectionRecord{ + HolderIdentity: "healthTest", + }, + observedTime: current, + clock: clock.NewFakeClock(current.Add(time.Minute).Add(time.Second)), + }, + }, + } + + for _, test := range tests { + adaptor := NewLeaderHealthzAdaptor(test.adaptorTimeout) + if adaptor.le != nil { + t.Errorf("[%s] leaderChecker started with a LeaderElector %v", test.description, adaptor.le) + } + if test.elector != nil { + test.elector.config.WatchDog = adaptor + adaptor.SetLeaderElection(test.elector) + if adaptor.le == nil { + t.Errorf("[%s] adaptor failed to set the LeaderElector", test.description) + } + } + err := adaptor.Check(req) + if test.expected == nil { + if err == nil { + continue + } + t.Errorf("[%s] called check, expected no error but received \"%v\"", test.description, err) + } else { + if err == nil { + t.Errorf("[%s] called check and failed to received the expected error \"%v\"", test.description, test.expected) + } + if err.Error() != test.expected.Error() { + t.Errorf("[%s] called check, expected %v, received %v", test.description, test.expected, err) + } + } + } +} diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go index be52e85a048..2096a5996bf 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go +++ b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go @@ -56,11 +56,12 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" rl "k8s.io/client-go/tools/leaderelection/resourcelock" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -90,6 +91,7 @@ func NewLeaderElector(lec LeaderElectionConfig) (*LeaderElector, error) { } return &LeaderElector{ config: lec, + clock: clock.RealClock{}, }, nil } @@ -111,6 +113,13 @@ type LeaderElectionConfig struct { // Callbacks are callbacks that are triggered during certain lifecycle // events of the LeaderElector Callbacks LeaderCallbacks + + // WatchDog is the associated health checker + // WatchDog may be null if its not needed/configured. + WatchDog *HealthzAdaptor + + // Name is the name of the resource lock for debugging + Name string } // LeaderCallbacks are callbacks that are triggered during certain @@ -139,6 +148,12 @@ type LeaderElector struct { // value observedRecord.HolderIdentity if the transition has // not yet been reported. reportedLeader string + + // clock is wrapper around time to allow for less flaky testing + clock clock.Clock + + // name is the name of the resource lock for debugging + name string } // Run starts the leader election loop @@ -163,6 +178,9 @@ func RunOrDie(ctx context.Context, lec LeaderElectionConfig) { if err != nil { panic(err) } + if lec.WatchDog != nil { + lec.WatchDog.SetLeaderElection(le) + } le.Run(ctx) } @@ -184,16 +202,16 @@ func (le *LeaderElector) acquire(ctx context.Context) bool { defer cancel() succeeded := false desc := le.config.Lock.Describe() - glog.Infof("attempting to acquire leader lease %v...", desc) + klog.Infof("attempting to acquire leader lease %v...", desc) wait.JitterUntil(func() { succeeded = le.tryAcquireOrRenew() le.maybeReportTransition() if !succeeded { - glog.V(4).Infof("failed to acquire lease %v", desc) + klog.V(4).Infof("failed to acquire lease %v", desc) return } le.config.Lock.RecordEvent("became leader") - glog.Infof("successfully acquired lease %v", desc) + klog.Infof("successfully acquired lease %v", desc) cancel() }, le.config.RetryPeriod, JitterFactor, true, ctx.Done()) return succeeded @@ -224,11 +242,11 @@ func (le *LeaderElector) renew(ctx context.Context) { le.maybeReportTransition() desc := le.config.Lock.Describe() if err == nil { - glog.V(5).Infof("successfully renewed lease %v", desc) + klog.V(5).Infof("successfully renewed lease %v", desc) return } le.config.Lock.RecordEvent("stopped leading") - glog.Infof("failed to renew lease %v: %v", desc, err) + klog.Infof("failed to renew lease %v: %v", desc, err) cancel() }, le.config.RetryPeriod, ctx.Done()) } @@ -249,26 +267,26 @@ func (le *LeaderElector) tryAcquireOrRenew() bool { oldLeaderElectionRecord, err := le.config.Lock.Get() if err != nil { if !errors.IsNotFound(err) { - glog.Errorf("error retrieving resource lock %v: %v", le.config.Lock.Describe(), err) + klog.Errorf("error retrieving resource lock %v: %v", le.config.Lock.Describe(), err) return false } if err = le.config.Lock.Create(leaderElectionRecord); err != nil { - glog.Errorf("error initially creating leader election record: %v", err) + klog.Errorf("error initially creating leader election record: %v", err) return false } le.observedRecord = leaderElectionRecord - le.observedTime = time.Now() + le.observedTime = le.clock.Now() return true } // 2. Record obtained, check the Identity & Time if !reflect.DeepEqual(le.observedRecord, *oldLeaderElectionRecord) { le.observedRecord = *oldLeaderElectionRecord - le.observedTime = time.Now() + le.observedTime = le.clock.Now() } if le.observedTime.Add(le.config.LeaseDuration).After(now.Time) && !le.IsLeader() { - glog.V(4).Infof("lock is held by %v and has not yet expired", oldLeaderElectionRecord.HolderIdentity) + klog.V(4).Infof("lock is held by %v and has not yet expired", oldLeaderElectionRecord.HolderIdentity) return false } @@ -283,11 +301,11 @@ func (le *LeaderElector) tryAcquireOrRenew() bool { // update the lock itself if err = le.config.Lock.Update(leaderElectionRecord); err != nil { - glog.Errorf("Failed to update lock: %v", err) + klog.Errorf("Failed to update lock: %v", err) return false } le.observedRecord = leaderElectionRecord - le.observedTime = time.Now() + le.observedTime = le.clock.Now() return true } @@ -300,3 +318,19 @@ func (le *LeaderElector) maybeReportTransition() { go le.config.Callbacks.OnNewLeader(le.reportedLeader) } } + +// Check will determine if the current lease is expired by more than timeout. +func (le *LeaderElector) Check(maxTolerableExpiredLease time.Duration) error { + if !le.IsLeader() { + // Currently not concerned with the case that we are hot standby + return nil + } + // If we are more than timeout seconds after the lease duration that is past the timeout + // on the lease renew. Time to start reporting ourselves as unhealthy. We should have + // died but conditions like deadlock can prevent this. (See #70819) + if le.clock.Since(le.observedTime) > le.config.LeaseDuration+maxTolerableExpiredLease { + return fmt.Errorf("failed election to renew leadership on lease %s", le.config.Name) + } + + return nil +} diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go index e99cf57afbf..842aebdab2a 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go +++ b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection_test.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/clock" fakecorev1 "k8s.io/client-go/kubernetes/typed/core/v1/fake" core "k8s.io/client-go/testing" rl "k8s.io/client-go/tools/leaderelection/resourcelock" @@ -257,6 +258,7 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) { config: lec, observedRecord: test.observedRecord, observedTime: test.observedTime, + clock: clock.RealClock{}, } if test.expectSuccess != le.tryAcquireOrRenew() { diff --git a/staging/src/k8s.io/client-go/tools/record/BUILD b/staging/src/k8s.io/client-go/tools/record/BUILD index fc1eaf2e671..2a8546138e3 100644 --- a/staging/src/k8s.io/client-go/tools/record/BUILD +++ b/staging/src/k8s.io/client-go/tools/record/BUILD @@ -50,8 +50,8 @@ go_library( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/reference:go_default_library", "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/groupcache/lru:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/tools/record/event.go b/staging/src/k8s.io/client-go/tools/record/event.go index 168dfa80c56..2ee69589c6d 100644 --- a/staging/src/k8s.io/client-go/tools/record/event.go +++ b/staging/src/k8s.io/client-go/tools/record/event.go @@ -33,7 +33,7 @@ import ( "net/http" - "github.com/golang/glog" + "k8s.io/klog" ) const maxTriesPerEvent = 12 @@ -144,7 +144,7 @@ func recordToSink(sink EventSink, event *v1.Event, eventCorrelator *EventCorrela } tries++ if tries >= maxTriesPerEvent { - glog.Errorf("Unable to write event '%#v' (retry limit exceeded!)", event) + klog.Errorf("Unable to write event '%#v' (retry limit exceeded!)", event) break } // Randomize the first sleep so that various clients won't all be @@ -194,13 +194,13 @@ func recordEvent(sink EventSink, event *v1.Event, patch []byte, updateExistingEv switch err.(type) { case *restclient.RequestConstructionError: // We will construct the request the same next time, so don't keep trying. - glog.Errorf("Unable to construct event '%#v': '%v' (will not retry!)", event, err) + klog.Errorf("Unable to construct event '%#v': '%v' (will not retry!)", event, err) return true case *errors.StatusError: if errors.IsAlreadyExists(err) { - glog.V(5).Infof("Server rejected event '%#v': '%v' (will not retry!)", event, err) + klog.V(5).Infof("Server rejected event '%#v': '%v' (will not retry!)", event, err) } else { - glog.Errorf("Server rejected event '%#v': '%v' (will not retry!)", event, err) + klog.Errorf("Server rejected event '%#v': '%v' (will not retry!)", event, err) } return true case *errors.UnexpectedObjectError: @@ -209,7 +209,7 @@ func recordEvent(sink EventSink, event *v1.Event, patch []byte, updateExistingEv default: // This case includes actual http transport errors. Go ahead and retry. } - glog.Errorf("Unable to write event: '%v' (may retry after sleeping)", err) + klog.Errorf("Unable to write event: '%v' (may retry after sleeping)", err) return false } @@ -256,12 +256,12 @@ type recorderImpl struct { func (recorder *recorderImpl) generateEvent(object runtime.Object, annotations map[string]string, timestamp metav1.Time, eventtype, reason, message string) { ref, err := ref.GetReference(recorder.scheme, object) if err != nil { - glog.Errorf("Could not construct reference to: '%#v' due to: '%v'. Will not report event: '%v' '%v' '%v'", object, err, eventtype, reason, message) + klog.Errorf("Could not construct reference to: '%#v' due to: '%v'. Will not report event: '%v' '%v' '%v'", object, err, eventtype, reason, message) return } if !validateEventType(eventtype) { - glog.Errorf("Unsupported event type: '%v'", eventtype) + klog.Errorf("Unsupported event type: '%v'", eventtype) return } diff --git a/staging/src/k8s.io/client-go/tools/remotecommand/BUILD b/staging/src/k8s.io/client-go/tools/remotecommand/BUILD index 9e0aac98770..043217a3ce3 100644 --- a/staging/src/k8s.io/client-go/tools/remotecommand/BUILD +++ b/staging/src/k8s.io/client-go/tools/remotecommand/BUILD @@ -25,6 +25,7 @@ go_library( srcs = [ "doc.go", "errorstream.go", + "reader.go", "remotecommand.go", "resize.go", "v1.go", @@ -43,7 +44,7 @@ go_library( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/transport/spdy:go_default_library", "//staging/src/k8s.io/client-go/util/exec:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/tools/remotecommand/reader.go b/staging/src/k8s.io/client-go/tools/remotecommand/reader.go new file mode 100644 index 00000000000..d1f1be34c9e --- /dev/null +++ b/staging/src/k8s.io/client-go/tools/remotecommand/reader.go @@ -0,0 +1,41 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remotecommand + +import ( + "io" +) + +// readerWrapper delegates to an io.Reader so that only the io.Reader interface is implemented, +// to keep io.Copy from doing things we don't want when copying from the reader to the data stream. +// +// If the Stdin io.Reader provided to remotecommand implements a WriteTo function (like bytes.Buffer does[1]), +// io.Copy calls that method[2] to attempt to write the entire buffer to the stream in one call. +// That results in an oversized call to spdystream.Stream#Write [3], +// which results in a single oversized data frame[4] that is too large. +// +// [1] https://golang.org/pkg/bytes/#Buffer.WriteTo +// [2] https://golang.org/pkg/io/#Copy +// [3] https://github.com/kubernetes/kubernetes/blob/90295640ef87db9daa0144c5617afe889e7992b2/vendor/github.com/docker/spdystream/stream.go#L66-L73 +// [4] https://github.com/kubernetes/kubernetes/blob/90295640ef87db9daa0144c5617afe889e7992b2/vendor/github.com/docker/spdystream/spdy/write.go#L302-L304 +type readerWrapper struct { + reader io.Reader +} + +func (r readerWrapper) Read(p []byte) (int, error) { + return r.reader.Read(p) +} diff --git a/staging/src/k8s.io/client-go/tools/remotecommand/remotecommand.go b/staging/src/k8s.io/client-go/tools/remotecommand/remotecommand.go index d2b29861e6e..892d8d105dc 100644 --- a/staging/src/k8s.io/client-go/tools/remotecommand/remotecommand.go +++ b/staging/src/k8s.io/client-go/tools/remotecommand/remotecommand.go @@ -22,7 +22,7 @@ import ( "net/http" "net/url" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/httpstream" "k8s.io/apimachinery/pkg/util/remotecommand" @@ -132,7 +132,7 @@ func (e *streamExecutor) Stream(options StreamOptions) error { case remotecommand.StreamProtocolV2Name: streamer = newStreamProtocolV2(options) case "": - glog.V(4).Infof("The server did not negotiate a streaming protocol version. Falling back to %s", remotecommand.StreamProtocolV1Name) + klog.V(4).Infof("The server did not negotiate a streaming protocol version. Falling back to %s", remotecommand.StreamProtocolV1Name) fallthrough case remotecommand.StreamProtocolV1Name: streamer = newStreamProtocolV1(options) diff --git a/staging/src/k8s.io/client-go/tools/remotecommand/v1.go b/staging/src/k8s.io/client-go/tools/remotecommand/v1.go index 92dad727f30..4120f1f5f3d 100644 --- a/staging/src/k8s.io/client-go/tools/remotecommand/v1.go +++ b/staging/src/k8s.io/client-go/tools/remotecommand/v1.go @@ -22,9 +22,9 @@ import ( "io/ioutil" "net/http" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/httpstream" + "k8s.io/klog" ) // streamProtocolV1 implements the first version of the streaming exec & attach @@ -53,10 +53,10 @@ func (p *streamProtocolV1) stream(conn streamCreator) error { errorChan := make(chan error) cp := func(s string, dst io.Writer, src io.Reader) { - glog.V(6).Infof("Copying %s", s) - defer glog.V(6).Infof("Done copying %s", s) + klog.V(6).Infof("Copying %s", s) + defer klog.V(6).Infof("Done copying %s", s) if _, err := io.Copy(dst, src); err != nil && err != io.EOF { - glog.Errorf("Error copying %s: %v", s, err) + klog.Errorf("Error copying %s: %v", s, err) } if s == v1.StreamTypeStdout || s == v1.StreamTypeStderr { doneChan <- struct{}{} @@ -127,7 +127,7 @@ func (p *streamProtocolV1) stream(conn streamCreator) error { // because stdin is not closed until the process exits. If we try to call // stdin.Close(), it returns no error but doesn't unblock the copy. It will // exit when the process exits, instead. - go cp(v1.StreamTypeStdin, p.remoteStdin, p.Stdin) + go cp(v1.StreamTypeStdin, p.remoteStdin, readerWrapper{p.Stdin}) } waitCount := 0 diff --git a/staging/src/k8s.io/client-go/tools/remotecommand/v2.go b/staging/src/k8s.io/client-go/tools/remotecommand/v2.go index b74ae8de220..4b0001502a1 100644 --- a/staging/src/k8s.io/client-go/tools/remotecommand/v2.go +++ b/staging/src/k8s.io/client-go/tools/remotecommand/v2.go @@ -101,7 +101,7 @@ func (p *streamProtocolV2) copyStdin() { // the executed command will remain running. defer once.Do(func() { p.remoteStdin.Close() }) - if _, err := io.Copy(p.remoteStdin, p.Stdin); err != nil { + if _, err := io.Copy(p.remoteStdin, readerWrapper{p.Stdin}); err != nil { runtime.HandleError(err) } }() diff --git a/staging/src/k8s.io/client-go/tools/watch/BUILD b/staging/src/k8s.io/client-go/tools/watch/BUILD index d1994ebbbef..9f7a97cd4aa 100644 --- a/staging/src/k8s.io/client-go/tools/watch/BUILD +++ b/staging/src/k8s.io/client-go/tools/watch/BUILD @@ -16,7 +16,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/tools/watch/until.go b/staging/src/k8s.io/client-go/tools/watch/until.go index 93357884397..aa4bbc21169 100644 --- a/staging/src/k8s.io/client-go/tools/watch/until.go +++ b/staging/src/k8s.io/client-go/tools/watch/until.go @@ -22,13 +22,13 @@ import ( "fmt" "time" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/tools/cache" + "k8s.io/klog" ) // PreconditionFunc returns true if the condition has been reached, false if it has not been reached yet, @@ -135,7 +135,7 @@ func UntilWithSync(ctx context.Context, lw cache.ListerWatcher, objType runtime. func ContextWithOptionalTimeout(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { if timeout < 0 { // This should be handled in validation - glog.Errorf("Timeout for context shall not be negative!") + klog.Errorf("Timeout for context shall not be negative!") timeout = 0 } diff --git a/staging/src/k8s.io/client-go/transport/BUILD b/staging/src/k8s.io/client-go/transport/BUILD index 05b0e604ac2..dc1800681d3 100644 --- a/staging/src/k8s.io/client-go/transport/BUILD +++ b/staging/src/k8s.io/client-go/transport/BUILD @@ -28,7 +28,7 @@ go_library( importpath = "k8s.io/client-go/transport", deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/transport/round_trippers.go b/staging/src/k8s.io/client-go/transport/round_trippers.go index 0ebcbbc8037..da417cf96ea 100644 --- a/staging/src/k8s.io/client-go/transport/round_trippers.go +++ b/staging/src/k8s.io/client-go/transport/round_trippers.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" utilnet "k8s.io/apimachinery/pkg/util/net" ) @@ -62,13 +62,13 @@ func HTTPWrappersForConfig(config *Config, rt http.RoundTripper) (http.RoundTrip // DebugWrappers wraps a round tripper and logs based on the current log level. func DebugWrappers(rt http.RoundTripper) http.RoundTripper { switch { - case bool(glog.V(9)): + case bool(klog.V(9)): rt = newDebuggingRoundTripper(rt, debugCurlCommand, debugURLTiming, debugResponseHeaders) - case bool(glog.V(8)): + case bool(klog.V(8)): rt = newDebuggingRoundTripper(rt, debugJustURL, debugRequestHeaders, debugResponseStatus, debugResponseHeaders) - case bool(glog.V(7)): + case bool(klog.V(7)): rt = newDebuggingRoundTripper(rt, debugJustURL, debugRequestHeaders, debugResponseStatus) - case bool(glog.V(6)): + case bool(klog.V(6)): rt = newDebuggingRoundTripper(rt, debugURLTiming) } @@ -138,7 +138,7 @@ func (rt *authProxyRoundTripper) CancelRequest(req *http.Request) { if canceler, ok := rt.rt.(requestCanceler); ok { canceler.CancelRequest(req) } else { - glog.Errorf("CancelRequest not implemented by %T", rt.rt) + klog.Errorf("CancelRequest not implemented by %T", rt.rt) } } @@ -166,7 +166,7 @@ func (rt *userAgentRoundTripper) CancelRequest(req *http.Request) { if canceler, ok := rt.rt.(requestCanceler); ok { canceler.CancelRequest(req) } else { - glog.Errorf("CancelRequest not implemented by %T", rt.rt) + klog.Errorf("CancelRequest not implemented by %T", rt.rt) } } @@ -197,7 +197,7 @@ func (rt *basicAuthRoundTripper) CancelRequest(req *http.Request) { if canceler, ok := rt.rt.(requestCanceler); ok { canceler.CancelRequest(req) } else { - glog.Errorf("CancelRequest not implemented by %T", rt.rt) + klog.Errorf("CancelRequest not implemented by %T", rt.rt) } } @@ -257,7 +257,7 @@ func (rt *impersonatingRoundTripper) CancelRequest(req *http.Request) { if canceler, ok := rt.delegate.(requestCanceler); ok { canceler.CancelRequest(req) } else { - glog.Errorf("CancelRequest not implemented by %T", rt.delegate) + klog.Errorf("CancelRequest not implemented by %T", rt.delegate) } } @@ -288,7 +288,7 @@ func (rt *bearerAuthRoundTripper) CancelRequest(req *http.Request) { if canceler, ok := rt.rt.(requestCanceler); ok { canceler.CancelRequest(req) } else { - glog.Errorf("CancelRequest not implemented by %T", rt.rt) + klog.Errorf("CancelRequest not implemented by %T", rt.rt) } } @@ -372,7 +372,7 @@ func (rt *debuggingRoundTripper) CancelRequest(req *http.Request) { if canceler, ok := rt.delegatedRoundTripper.(requestCanceler); ok { canceler.CancelRequest(req) } else { - glog.Errorf("CancelRequest not implemented by %T", rt.delegatedRoundTripper) + klog.Errorf("CancelRequest not implemented by %T", rt.delegatedRoundTripper) } } @@ -380,17 +380,17 @@ func (rt *debuggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, e reqInfo := newRequestInfo(req) if rt.levels[debugJustURL] { - glog.Infof("%s %s", reqInfo.RequestVerb, reqInfo.RequestURL) + klog.Infof("%s %s", reqInfo.RequestVerb, reqInfo.RequestURL) } if rt.levels[debugCurlCommand] { - glog.Infof("%s", reqInfo.toCurl()) + klog.Infof("%s", reqInfo.toCurl()) } if rt.levels[debugRequestHeaders] { - glog.Infof("Request Headers:") + klog.Infof("Request Headers:") for key, values := range reqInfo.RequestHeaders { for _, value := range values { - glog.Infof(" %s: %s", key, value) + klog.Infof(" %s: %s", key, value) } } } @@ -402,16 +402,16 @@ func (rt *debuggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, e reqInfo.complete(response, err) if rt.levels[debugURLTiming] { - glog.Infof("%s %s %s in %d milliseconds", reqInfo.RequestVerb, reqInfo.RequestURL, reqInfo.ResponseStatus, reqInfo.Duration.Nanoseconds()/int64(time.Millisecond)) + klog.Infof("%s %s %s in %d milliseconds", reqInfo.RequestVerb, reqInfo.RequestURL, reqInfo.ResponseStatus, reqInfo.Duration.Nanoseconds()/int64(time.Millisecond)) } if rt.levels[debugResponseStatus] { - glog.Infof("Response Status: %s in %d milliseconds", reqInfo.ResponseStatus, reqInfo.Duration.Nanoseconds()/int64(time.Millisecond)) + klog.Infof("Response Status: %s in %d milliseconds", reqInfo.ResponseStatus, reqInfo.Duration.Nanoseconds()/int64(time.Millisecond)) } if rt.levels[debugResponseHeaders] { - glog.Infof("Response Headers:") + klog.Infof("Response Headers:") for key, values := range reqInfo.ResponseHeaders { for _, value := range values { - glog.Infof(" %s: %s", key, value) + klog.Infof(" %s: %s", key, value) } } } diff --git a/staging/src/k8s.io/client-go/util/cert/BUILD b/staging/src/k8s.io/client-go/util/cert/BUILD index 4db41c73f29..2a42b716eff 100644 --- a/staging/src/k8s.io/client-go/util/cert/BUILD +++ b/staging/src/k8s.io/client-go/util/cert/BUILD @@ -37,9 +37,6 @@ filegroup( filegroup( name = "all-srcs", - srcs = [ - ":package-srcs", - "//staging/src/k8s.io/client-go/util/cert/triple:all-srcs", - ], + srcs = [":package-srcs"], tags = ["automanaged"], ) diff --git a/staging/src/k8s.io/client-go/util/cert/OWNERS b/staging/src/k8s.io/client-go/util/cert/OWNERS new file mode 100644 index 00000000000..470b7a1c92d --- /dev/null +++ b/staging/src/k8s.io/client-go/util/cert/OWNERS @@ -0,0 +1,7 @@ +approvers: +- sig-auth-certificates-approvers +reviewers: +- sig-auth-certificates-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/client-go/util/cert/cert.go b/staging/src/k8s.io/client-go/util/cert/cert.go index 0e160dabd66..3429c82cdff 100644 --- a/staging/src/k8s.io/client-go/util/cert/cert.go +++ b/staging/src/k8s.io/client-go/util/cert/cert.go @@ -260,34 +260,6 @@ func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, a return certBuffer.Bytes(), keyBuffer.Bytes(), nil } -// FormatBytesCert receives byte array certificate and formats in human-readable format -func FormatBytesCert(cert []byte) (string, error) { - block, _ := pem.Decode(cert) - c, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return "", fmt.Errorf("failed to parse certificate [%v]", err) - } - return FormatCert(c), nil -} - -// FormatCert receives certificate and formats in human-readable format -func FormatCert(c *x509.Certificate) string { - var ips []string - for _, ip := range c.IPAddresses { - ips = append(ips, ip.String()) - } - altNames := append(ips, c.DNSNames...) - res := fmt.Sprintf( - "Issuer: CN=%s | Subject: CN=%s | CA: %t\n", - c.Issuer.CommonName, c.Subject.CommonName, c.IsCA, - ) - res += fmt.Sprintf("Not before: %s Not After: %s", c.NotBefore, c.NotAfter) - if len(altNames) > 0 { - res += fmt.Sprintf("\nAlternate Names: %v", altNames) - } - return res -} - func ipsToStrings(ips []net.IP) []string { ss := make([]string, 0, len(ips)) for _, ip := range ips { diff --git a/staging/src/k8s.io/client-go/util/cert/triple/BUILD b/staging/src/k8s.io/client-go/util/cert/triple/BUILD deleted file mode 100644 index 86eb16f7803..00000000000 --- a/staging/src/k8s.io/client-go/util/cert/triple/BUILD +++ /dev/null @@ -1,27 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) - -go_library( - name = "go_default_library", - srcs = ["triple.go"], - importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/util/cert/triple", - importpath = "k8s.io/client-go/util/cert/triple", - deps = ["//staging/src/k8s.io/client-go/util/cert:go_default_library"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/staging/src/k8s.io/client-go/util/cert/triple/triple.go b/staging/src/k8s.io/client-go/util/cert/triple/triple.go deleted file mode 100644 index b89ef7240f4..00000000000 --- a/staging/src/k8s.io/client-go/util/cert/triple/triple.go +++ /dev/null @@ -1,116 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package triple generates key-certificate pairs for the -// triple (CA, Server, Client). -package triple - -import ( - "crypto/rsa" - "crypto/x509" - "fmt" - "net" - - certutil "k8s.io/client-go/util/cert" -) - -type KeyPair struct { - Key *rsa.PrivateKey - Cert *x509.Certificate -} - -func NewCA(name string) (*KeyPair, error) { - key, err := certutil.NewPrivateKey() - if err != nil { - return nil, fmt.Errorf("unable to create a private key for a new CA: %v", err) - } - - config := certutil.Config{ - CommonName: name, - } - - cert, err := certutil.NewSelfSignedCACert(config, key) - if err != nil { - return nil, fmt.Errorf("unable to create a self-signed certificate for a new CA: %v", err) - } - - return &KeyPair{ - Key: key, - Cert: cert, - }, nil -} - -func NewServerKeyPair(ca *KeyPair, commonName, svcName, svcNamespace, dnsDomain string, ips, hostnames []string) (*KeyPair, error) { - key, err := certutil.NewPrivateKey() - if err != nil { - return nil, fmt.Errorf("unable to create a server private key: %v", err) - } - - namespacedName := fmt.Sprintf("%s.%s", svcName, svcNamespace) - internalAPIServerFQDN := []string{ - svcName, - namespacedName, - fmt.Sprintf("%s.svc", namespacedName), - fmt.Sprintf("%s.svc.%s", namespacedName, dnsDomain), - } - - altNames := certutil.AltNames{} - for _, ipStr := range ips { - ip := net.ParseIP(ipStr) - if ip != nil { - altNames.IPs = append(altNames.IPs, ip) - } - } - altNames.DNSNames = append(altNames.DNSNames, hostnames...) - altNames.DNSNames = append(altNames.DNSNames, internalAPIServerFQDN...) - - config := certutil.Config{ - CommonName: commonName, - AltNames: altNames, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - } - cert, err := certutil.NewSignedCert(config, key, ca.Cert, ca.Key) - if err != nil { - return nil, fmt.Errorf("unable to sign the server certificate: %v", err) - } - - return &KeyPair{ - Key: key, - Cert: cert, - }, nil -} - -func NewClientKeyPair(ca *KeyPair, commonName string, organizations []string) (*KeyPair, error) { - key, err := certutil.NewPrivateKey() - if err != nil { - return nil, fmt.Errorf("unable to create a client private key: %v", err) - } - - config := certutil.Config{ - CommonName: commonName, - Organization: organizations, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - } - cert, err := certutil.NewSignedCert(config, key, ca.Cert, ca.Key) - if err != nil { - return nil, fmt.Errorf("unable to sign the client certificate: %v", err) - } - - return &KeyPair{ - Key: key, - Cert: cert, - }, nil -} diff --git a/staging/src/k8s.io/client-go/util/certificate/BUILD b/staging/src/k8s.io/client-go/util/certificate/BUILD index 609fb2c2834..f204883af91 100644 --- a/staging/src/k8s.io/client-go/util/certificate/BUILD +++ b/staging/src/k8s.io/client-go/util/certificate/BUILD @@ -45,7 +45,7 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/client-go/util/certificate/csr:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/util/certificate/OWNERS b/staging/src/k8s.io/client-go/util/certificate/OWNERS index 2dce803b34d..470b7a1c92d 100644 --- a/staging/src/k8s.io/client-go/util/certificate/OWNERS +++ b/staging/src/k8s.io/client-go/util/certificate/OWNERS @@ -1,8 +1,7 @@ -reviewers: -- mikedanese -- liggit -- smarterclayton approvers: -- mikedanese -- liggit -- smarterclayton +- sig-auth-certificates-approvers +reviewers: +- sig-auth-certificates-reviewers +labels: +- sig/auth + diff --git a/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go b/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go index fbdf4ec7f36..ed74559e203 100644 --- a/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go +++ b/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go @@ -28,7 +28,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" certificates "k8s.io/api/certificates/v1beta1" "k8s.io/apimachinery/pkg/api/errors" @@ -227,17 +227,17 @@ func (m *manager) Start() { // signing API, so don't start the certificate manager if we don't have a // client. if m.certSigningRequestClient == nil { - glog.V(2).Infof("Certificate rotation is not enabled, no connection to the apiserver.") + klog.V(2).Infof("Certificate rotation is not enabled, no connection to the apiserver.") return } - glog.V(2).Infof("Certificate rotation is enabled.") + klog.V(2).Infof("Certificate rotation is enabled.") templateChanged := make(chan struct{}) go wait.Forever(func() { deadline := m.nextRotationDeadline() if sleepInterval := deadline.Sub(time.Now()); sleepInterval > 0 { - glog.V(2).Infof("Waiting %v for next certificate rotation", sleepInterval) + klog.V(2).Infof("Waiting %v for next certificate rotation", sleepInterval) timer := time.NewTimer(sleepInterval) defer timer.Stop() @@ -250,7 +250,7 @@ func (m *manager) Start() { // if the template now matches what we last requested, restart the rotation deadline loop return } - glog.V(2).Infof("Certificate template changed, rotating") + klog.V(2).Infof("Certificate template changed, rotating") } } @@ -321,7 +321,7 @@ func getCurrentCertificateOrBootstrap( if _, err := store.Update(bootstrapCertificatePEM, bootstrapKeyPEM); err != nil { utilruntime.HandleError(fmt.Errorf("Unable to set the cert/key pair to the bootstrap certificate: %v", err)) } else { - glog.V(4).Infof("Updated the store to contain the initial bootstrap certificate") + klog.V(4).Infof("Updated the store to contain the initial bootstrap certificate") } return &bootstrapCert, true, nil @@ -333,7 +333,7 @@ func getCurrentCertificateOrBootstrap( // This method also keeps track of "server health" by interpreting the responses it gets // from the server on the various calls it makes. func (m *manager) rotateCerts() (bool, error) { - glog.V(2).Infof("Rotating certificates") + klog.V(2).Infof("Rotating certificates") template, csrPEM, keyPEM, privateKey, err := m.generateCSR() if err != nil { @@ -403,7 +403,7 @@ func (m *manager) certSatisfiesTemplateLocked() bool { if template := m.getTemplate(); template != nil { if template.Subject.CommonName != m.cert.Leaf.Subject.CommonName { - glog.V(2).Infof("Current certificate CN (%s) does not match requested CN (%s)", m.cert.Leaf.Subject.CommonName, template.Subject.CommonName) + klog.V(2).Infof("Current certificate CN (%s) does not match requested CN (%s)", m.cert.Leaf.Subject.CommonName, template.Subject.CommonName) return false } @@ -411,7 +411,7 @@ func (m *manager) certSatisfiesTemplateLocked() bool { desiredDNSNames := sets.NewString(template.DNSNames...) missingDNSNames := desiredDNSNames.Difference(currentDNSNames) if len(missingDNSNames) > 0 { - glog.V(2).Infof("Current certificate is missing requested DNS names %v", missingDNSNames.List()) + klog.V(2).Infof("Current certificate is missing requested DNS names %v", missingDNSNames.List()) return false } @@ -425,7 +425,7 @@ func (m *manager) certSatisfiesTemplateLocked() bool { } missingIPs := desiredIPs.Difference(currentIPs) if len(missingIPs) > 0 { - glog.V(2).Infof("Current certificate is missing requested IP addresses %v", missingIPs.List()) + klog.V(2).Infof("Current certificate is missing requested IP addresses %v", missingIPs.List()) return false } @@ -433,7 +433,7 @@ func (m *manager) certSatisfiesTemplateLocked() bool { desiredOrgs := sets.NewString(template.Subject.Organization...) missingOrgs := desiredOrgs.Difference(currentOrgs) if len(missingOrgs) > 0 { - glog.V(2).Infof("Current certificate is missing requested orgs %v", missingOrgs.List()) + klog.V(2).Infof("Current certificate is missing requested orgs %v", missingOrgs.List()) return false } } @@ -468,7 +468,7 @@ func (m *manager) nextRotationDeadline() time.Time { totalDuration := float64(notAfter.Sub(m.cert.Leaf.NotBefore)) deadline := m.cert.Leaf.NotBefore.Add(jitteryDuration(totalDuration)) - glog.V(2).Infof("Certificate expiration is %v, rotation deadline is %v", notAfter, deadline) + klog.V(2).Infof("Certificate expiration is %v, rotation deadline is %v", notAfter, deadline) if m.certificateExpiration != nil { m.certificateExpiration.Set(float64(notAfter.Unix())) } diff --git a/staging/src/k8s.io/client-go/util/certificate/certificate_store.go b/staging/src/k8s.io/client-go/util/certificate/certificate_store.go index 81a9e7647c2..d2666615421 100644 --- a/staging/src/k8s.io/client-go/util/certificate/certificate_store.go +++ b/staging/src/k8s.io/client-go/util/certificate/certificate_store.go @@ -26,7 +26,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -127,7 +127,7 @@ func (s *fileStore) Current() (*tls.Certificate, error) { if pairFileExists, err := fileExists(pairFile); err != nil { return nil, err } else if pairFileExists { - glog.Infof("Loading cert/key pair from %q.", pairFile) + klog.Infof("Loading cert/key pair from %q.", pairFile) return loadFile(pairFile) } @@ -140,7 +140,7 @@ func (s *fileStore) Current() (*tls.Certificate, error) { return nil, err } if certFileExists && keyFileExists { - glog.Infof("Loading cert/key pair from (%q, %q).", s.certFile, s.keyFile) + klog.Infof("Loading cert/key pair from (%q, %q).", s.certFile, s.keyFile) return loadX509KeyPair(s.certFile, s.keyFile) } @@ -155,7 +155,7 @@ func (s *fileStore) Current() (*tls.Certificate, error) { return nil, err } if certFileExists && keyFileExists { - glog.Infof("Loading cert/key pair from (%q, %q).", c, k) + klog.Infof("Loading cert/key pair from (%q, %q).", c, k) return loadX509KeyPair(c, k) } diff --git a/staging/src/k8s.io/client-go/util/certificate/csr/BUILD b/staging/src/k8s.io/client-go/util/certificate/csr/BUILD index e160947dbc2..eb28433e05b 100644 --- a/staging/src/k8s.io/client-go/util/certificate/csr/BUILD +++ b/staging/src/k8s.io/client-go/util/certificate/csr/BUILD @@ -1,10 +1,6 @@ package(default_visibility = ["//visibility:public"]) -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", @@ -17,14 +13,13 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -40,16 +35,3 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) - -go_test( - name = "go_default_test", - srcs = ["csr_test.go"], - embed = [":go_default_library"], - deps = [ - "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - ], -) diff --git a/staging/src/k8s.io/client-go/util/certificate/csr/csr.go b/staging/src/k8s.io/client-go/util/certificate/csr/csr.go index 4a53352fee0..6338eef9331 100644 --- a/staging/src/k8s.io/client-go/util/certificate/csr/csr.go +++ b/staging/src/k8s.io/client-go/util/certificate/csr/csr.go @@ -18,23 +18,19 @@ package csr import ( "crypto" - "crypto/sha512" "crypto/x509" - "crypto/x509/pkix" - "encoding/base64" "encoding/pem" "fmt" "reflect" "time" - "github.com/golang/glog" + "k8s.io/klog" certificates "k8s.io/api/certificates/v1beta1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" @@ -43,41 +39,6 @@ import ( certutil "k8s.io/client-go/util/cert" ) -// RequestNodeCertificate will create a certificate signing request for a node -// (Organization and CommonName for the CSR will be set as expected for node -// certificates) and send it to API server, then it will watch the object's -// status, once approved by API server, it will return the API server's issued -// certificate (pem-encoded). If there is any errors, or the watch timeouts, it -// will return an error. This is intended for use on nodes (kubelet and -// kubeadm). -func RequestNodeCertificate(client certificatesclient.CertificateSigningRequestInterface, privateKeyData []byte, nodeName types.NodeName) (certData []byte, err error) { - subject := &pkix.Name{ - Organization: []string{"system:nodes"}, - CommonName: "system:node:" + string(nodeName), - } - - privateKey, err := certutil.ParsePrivateKeyPEM(privateKeyData) - if err != nil { - return nil, fmt.Errorf("invalid private key for certificate request: %v", err) - } - csrData, err := certutil.MakeCSR(privateKey, subject, nil, nil) - if err != nil { - return nil, fmt.Errorf("unable to generate certificate request: %v", err) - } - - usages := []certificates.KeyUsage{ - certificates.UsageDigitalSignature, - certificates.UsageKeyEncipherment, - certificates.UsageClientAuth, - } - name := digestedName(privateKeyData, subject, usages) - req, err := RequestCertificate(client, csrData, name, usages, privateKey) - if err != nil { - return nil, err - } - return WaitForCertificate(client, req, 3600*time.Second) -} - // RequestCertificate will either use an existing (if this process has run // before but not to completion) or create a certificate signing request using the // PEM encoded CSR and send it to API server, then it will watch the object's @@ -104,7 +65,7 @@ func RequestCertificate(client certificatesclient.CertificateSigningRequestInter switch { case err == nil: case errors.IsAlreadyExists(err) && len(name) > 0: - glog.Infof("csr for this node already exists, reusing") + klog.Infof("csr for this node already exists, reusing") req, err = client.Get(name, metav1.GetOptions{}) if err != nil { return nil, formatError("cannot retrieve certificate signing request: %v", err) @@ -112,7 +73,7 @@ func RequestCertificate(client certificatesclient.CertificateSigningRequestInter if err := ensureCompatible(req, csr, privateKey); err != nil { return nil, fmt.Errorf("retrieved csr is not compatible: %v", err) } - glog.Infof("csr for this node is still valid") + klog.Infof("csr for this node is still valid") default: return nil, formatError("cannot create certificate signing request: %v", err) } @@ -168,57 +129,25 @@ func WaitForCertificate(client certificatesclient.CertificateSigningRequestInter return event.Object.(*certificates.CertificateSigningRequest).Status.Certificate, nil } -// This digest should include all the relevant pieces of the CSR we care about. -// We can't direcly hash the serialized CSR because of random padding that we -// regenerate every loop and we include usages which are not contained in the -// CSR. This needs to be kept up to date as we add new fields to the node -// certificates and with ensureCompatible. -func digestedName(privateKeyData []byte, subject *pkix.Name, usages []certificates.KeyUsage) string { - hash := sha512.New512_256() - - // Here we make sure two different inputs can't write the same stream - // to the hash. This delimiter is not in the base64.URLEncoding - // alphabet so there is no way to have spill over collisions. Without - // it 'CN:foo,ORG:bar' hashes to the same value as 'CN:foob,ORG:ar' - const delimiter = '|' - encode := base64.RawURLEncoding.EncodeToString - - write := func(data []byte) { - hash.Write([]byte(encode(data))) - hash.Write([]byte{delimiter}) - } - - write(privateKeyData) - write([]byte(subject.CommonName)) - for _, v := range subject.Organization { - write([]byte(v)) - } - for _, v := range usages { - write([]byte(v)) - } - - return "node-csr-" + encode(hash.Sum(nil)) -} - // ensureCompatible ensures that a CSR object is compatible with an original CSR func ensureCompatible(new, orig *certificates.CertificateSigningRequest, privateKey interface{}) error { - newCsr, err := ParseCSR(new) + newCSR, err := parseCSR(new) if err != nil { return fmt.Errorf("unable to parse new csr: %v", err) } - origCsr, err := ParseCSR(orig) + origCSR, err := parseCSR(orig) if err != nil { return fmt.Errorf("unable to parse original csr: %v", err) } - if !reflect.DeepEqual(newCsr.Subject, origCsr.Subject) { - return fmt.Errorf("csr subjects differ: new: %#v, orig: %#v", newCsr.Subject, origCsr.Subject) + if !reflect.DeepEqual(newCSR.Subject, origCSR.Subject) { + return fmt.Errorf("csr subjects differ: new: %#v, orig: %#v", newCSR.Subject, origCSR.Subject) } signer, ok := privateKey.(crypto.Signer) if !ok { return fmt.Errorf("privateKey is not a signer") } - newCsr.PublicKey = signer.Public() - if err := newCsr.CheckSignature(); err != nil { + newCSR.PublicKey = signer.Public() + if err := newCSR.CheckSignature(); err != nil { return fmt.Errorf("error validating signature new CSR against old key: %v", err) } if len(new.Status.Certificate) > 0 { @@ -247,17 +176,12 @@ func formatError(format string, err error) error { return fmt.Errorf(format, err) } -// ParseCSR extracts the CSR from the API object and decodes it. -func ParseCSR(obj *certificates.CertificateSigningRequest) (*x509.CertificateRequest, error) { +// parseCSR extracts the CSR from the API object and decodes it. +func parseCSR(obj *certificates.CertificateSigningRequest) (*x509.CertificateRequest, error) { // extract PEM from request object - pemBytes := obj.Spec.Request - block, _ := pem.Decode(pemBytes) + block, _ := pem.Decode(obj.Spec.Request) if block == nil || block.Type != "CERTIFICATE REQUEST" { return nil, fmt.Errorf("PEM block type must be CERTIFICATE REQUEST") } - csr, err := x509.ParseCertificateRequest(block.Bytes) - if err != nil { - return nil, err - } - return csr, nil + return x509.ParseCertificateRequest(block.Bytes) } diff --git a/staging/src/k8s.io/client-go/util/certificate/csr/csr_test.go b/staging/src/k8s.io/client-go/util/certificate/csr/csr_test.go deleted file mode 100644 index d0182abe992..00000000000 --- a/staging/src/k8s.io/client-go/util/certificate/csr/csr_test.go +++ /dev/null @@ -1,135 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csr - -import ( - "fmt" - "testing" - - certificates "k8s.io/api/certificates/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - watch "k8s.io/apimachinery/pkg/watch" - certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" - certutil "k8s.io/client-go/util/cert" -) - -func TestRequestNodeCertificateNoKeyData(t *testing.T) { - certData, err := RequestNodeCertificate(&fakeClient{}, []byte{}, "fake-node-name") - if err == nil { - t.Errorf("Got no error, wanted error an error because there was an empty private key passed in.") - } - if certData != nil { - t.Errorf("Got cert data, wanted nothing as there should have been an error.") - } -} - -func TestRequestNodeCertificateErrorCreatingCSR(t *testing.T) { - client := &fakeClient{ - failureType: createError, - } - privateKeyData, err := certutil.MakeEllipticPrivateKeyPEM() - if err != nil { - t.Fatalf("Unable to generate a new private key: %v", err) - } - - certData, err := RequestNodeCertificate(client, privateKeyData, "fake-node-name") - if err == nil { - t.Errorf("Got no error, wanted error an error because client.Create failed.") - } - if certData != nil { - t.Errorf("Got cert data, wanted nothing as there should have been an error.") - } -} - -func TestRequestNodeCertificate(t *testing.T) { - privateKeyData, err := certutil.MakeEllipticPrivateKeyPEM() - if err != nil { - t.Fatalf("Unable to generate a new private key: %v", err) - } - - certData, err := RequestNodeCertificate(&fakeClient{}, privateKeyData, "fake-node-name") - if err != nil { - t.Errorf("Got %v, wanted no error.", err) - } - if certData == nil { - t.Errorf("Got nothing, expected a CSR.") - } -} - -type FailureType int - -const ( - noError FailureType = iota - createError - certificateSigningRequestDenied -) - -type fakeClient struct { - certificatesclient.CertificateSigningRequestInterface - watch *watch.FakeWatcher - failureType FailureType -} - -func (c *fakeClient) Create(*certificates.CertificateSigningRequest) (*certificates.CertificateSigningRequest, error) { - if c.failureType == createError { - return nil, fmt.Errorf("fakeClient failed creating request") - } - csr := certificates.CertificateSigningRequest{ - ObjectMeta: metav1.ObjectMeta{ - UID: "fake-uid", - Name: "fake-certificate-signing-request-name", - }, - } - return &csr, nil -} - -func (c *fakeClient) List(opts metav1.ListOptions) (*certificates.CertificateSigningRequestList, error) { - return &certificates.CertificateSigningRequestList{}, nil -} - -func (c *fakeClient) Watch(opts metav1.ListOptions) (watch.Interface, error) { - c.watch = watch.NewFakeWithChanSize(1, false) - c.watch.Add(c.generateCSR()) - c.watch.Stop() - return c.watch, nil -} - -func (c *fakeClient) generateCSR() *certificates.CertificateSigningRequest { - var condition certificates.CertificateSigningRequestCondition - if c.failureType == certificateSigningRequestDenied { - condition = certificates.CertificateSigningRequestCondition{ - Type: certificates.CertificateDenied, - } - } else { - condition = certificates.CertificateSigningRequestCondition{ - Type: certificates.CertificateApproved, - } - } - - csr := certificates.CertificateSigningRequest{ - ObjectMeta: metav1.ObjectMeta{ - UID: "fake-uid", - }, - Status: certificates.CertificateSigningRequestStatus{ - Conditions: []certificates.CertificateSigningRequestCondition{ - condition, - }, - Certificate: []byte{}, - }, - } - return &csr -} diff --git a/staging/src/k8s.io/client-go/util/workqueue/BUILD b/staging/src/k8s.io/client-go/util/workqueue/BUILD index c139977c1ca..ff0e87a7c75 100644 --- a/staging/src/k8s.io/client-go/util/workqueue/BUILD +++ b/staging/src/k8s.io/client-go/util/workqueue/BUILD @@ -11,6 +11,7 @@ go_test( srcs = [ "default_rate_limiters_test.go", "delaying_queue_test.go", + "metrics_test.go", "queue_test.go", "rate_limitting_queue_test.go", ], diff --git a/staging/src/k8s.io/client-go/util/workqueue/metrics.go b/staging/src/k8s.io/client-go/util/workqueue/metrics.go index a481bdfb266..d4c03d8378f 100644 --- a/staging/src/k8s.io/client-go/util/workqueue/metrics.go +++ b/staging/src/k8s.io/client-go/util/workqueue/metrics.go @@ -19,6 +19,8 @@ package workqueue import ( "sync" "time" + + "k8s.io/apimachinery/pkg/util/clock" ) // This file provides abstractions for setting the provider (e.g., prometheus) @@ -28,6 +30,7 @@ type queueMetrics interface { add(item t) get(item t) done(item t) + updateUnfinishedWork() } // GaugeMetric represents a single numerical value that can arbitrarily go up @@ -37,6 +40,12 @@ type GaugeMetric interface { Dec() } +// SettableGaugeMetric represents a single numerical value that can arbitrarily go up +// and down. (Separate from GaugeMetric to preserve backwards compatibility.) +type SettableGaugeMetric interface { + Set(float64) +} + // CounterMetric represents a single numerical value that only ever // goes up. type CounterMetric interface { @@ -52,9 +61,13 @@ type noopMetric struct{} func (noopMetric) Inc() {} func (noopMetric) Dec() {} +func (noopMetric) Set(float64) {} func (noopMetric) Observe(float64) {} +// defaultQueueMetrics expects the caller to lock before setting any metrics. type defaultQueueMetrics struct { + clock clock.Clock + // current depth of a workqueue depth GaugeMetric // total number of adds handled by a workqueue @@ -65,6 +78,10 @@ type defaultQueueMetrics struct { workDuration SummaryMetric addTimes map[t]time.Time processingStartTimes map[t]time.Time + + // how long have current threads been working? + unfinishedWorkSeconds SettableGaugeMetric + longestRunningProcessor SettableGaugeMetric } func (m *defaultQueueMetrics) add(item t) { @@ -75,7 +92,7 @@ func (m *defaultQueueMetrics) add(item t) { m.adds.Inc() m.depth.Inc() if _, exists := m.addTimes[item]; !exists { - m.addTimes[item] = time.Now() + m.addTimes[item] = m.clock.Now() } } @@ -85,9 +102,9 @@ func (m *defaultQueueMetrics) get(item t) { } m.depth.Dec() - m.processingStartTimes[item] = time.Now() + m.processingStartTimes[item] = m.clock.Now() if startTime, exists := m.addTimes[item]; exists { - m.latency.Observe(sinceInMicroseconds(startTime)) + m.latency.Observe(m.sinceInMicroseconds(startTime)) delete(m.addTimes, item) } } @@ -98,14 +115,39 @@ func (m *defaultQueueMetrics) done(item t) { } if startTime, exists := m.processingStartTimes[item]; exists { - m.workDuration.Observe(sinceInMicroseconds(startTime)) + m.workDuration.Observe(m.sinceInMicroseconds(startTime)) delete(m.processingStartTimes, item) } } +func (m *defaultQueueMetrics) updateUnfinishedWork() { + // Note that a summary metric would be better for this, but prometheus + // doesn't seem to have non-hacky ways to reset the summary metrics. + var total float64 + var oldest float64 + for _, t := range m.processingStartTimes { + age := m.sinceInMicroseconds(t) + total += age + if age > oldest { + oldest = age + } + } + // Convert to seconds; microseconds is unhelpfully granular for this. + total /= 1000000 + m.unfinishedWorkSeconds.Set(total) + m.longestRunningProcessor.Set(oldest) // in microseconds. +} + +type noMetrics struct{} + +func (noMetrics) add(item t) {} +func (noMetrics) get(item t) {} +func (noMetrics) done(item t) {} +func (noMetrics) updateUnfinishedWork() {} + // Gets the time since the specified start in microseconds. -func sinceInMicroseconds(start time.Time) float64 { - return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds()) +func (m *defaultQueueMetrics) sinceInMicroseconds(start time.Time) float64 { + return float64(m.clock.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds()) } type retryMetrics interface { @@ -130,6 +172,8 @@ type MetricsProvider interface { NewAddsMetric(name string) CounterMetric NewLatencyMetric(name string) SummaryMetric NewWorkDurationMetric(name string) SummaryMetric + NewUnfinishedWorkSecondsMetric(name string) SettableGaugeMetric + NewLongestRunningProcessorMicrosecondsMetric(name string) SettableGaugeMetric NewRetriesMetric(name string) CounterMetric } @@ -151,29 +195,49 @@ func (_ noopMetricsProvider) NewWorkDurationMetric(name string) SummaryMetric { return noopMetric{} } +func (_ noopMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) SettableGaugeMetric { + return noopMetric{} +} + +func (_ noopMetricsProvider) NewLongestRunningProcessorMicrosecondsMetric(name string) SettableGaugeMetric { + return noopMetric{} +} + func (_ noopMetricsProvider) NewRetriesMetric(name string) CounterMetric { return noopMetric{} } -var metricsFactory = struct { - metricsProvider MetricsProvider - setProviders sync.Once -}{ +var globalMetricsFactory = queueMetricsFactory{ metricsProvider: noopMetricsProvider{}, } -func newQueueMetrics(name string) queueMetrics { - var ret *defaultQueueMetrics - if len(name) == 0 { - return ret +type queueMetricsFactory struct { + metricsProvider MetricsProvider + + onlyOnce sync.Once +} + +func (f *queueMetricsFactory) setProvider(mp MetricsProvider) { + f.onlyOnce.Do(func() { + f.metricsProvider = mp + }) +} + +func (f *queueMetricsFactory) newQueueMetrics(name string, clock clock.Clock) queueMetrics { + mp := f.metricsProvider + if len(name) == 0 || mp == (noopMetricsProvider{}) { + return noMetrics{} } return &defaultQueueMetrics{ - depth: metricsFactory.metricsProvider.NewDepthMetric(name), - adds: metricsFactory.metricsProvider.NewAddsMetric(name), - latency: metricsFactory.metricsProvider.NewLatencyMetric(name), - workDuration: metricsFactory.metricsProvider.NewWorkDurationMetric(name), - addTimes: map[t]time.Time{}, - processingStartTimes: map[t]time.Time{}, + clock: clock, + depth: mp.NewDepthMetric(name), + adds: mp.NewAddsMetric(name), + latency: mp.NewLatencyMetric(name), + workDuration: mp.NewWorkDurationMetric(name), + unfinishedWorkSeconds: mp.NewUnfinishedWorkSecondsMetric(name), + longestRunningProcessor: mp.NewLongestRunningProcessorMicrosecondsMetric(name), + addTimes: map[t]time.Time{}, + processingStartTimes: map[t]time.Time{}, } } @@ -183,13 +247,12 @@ func newRetryMetrics(name string) retryMetrics { return ret } return &defaultRetryMetrics{ - retries: metricsFactory.metricsProvider.NewRetriesMetric(name), + retries: globalMetricsFactory.metricsProvider.NewRetriesMetric(name), } } -// SetProvider sets the metrics provider of the metricsFactory. +// SetProvider sets the metrics provider for all subsequently created work +// queues. Only the first call has an effect. func SetProvider(metricsProvider MetricsProvider) { - metricsFactory.setProviders.Do(func() { - metricsFactory.metricsProvider = metricsProvider - }) + globalMetricsFactory.setProvider(metricsProvider) } diff --git a/staging/src/k8s.io/client-go/util/workqueue/metrics_test.go b/staging/src/k8s.io/client-go/util/workqueue/metrics_test.go new file mode 100644 index 00000000000..117f90801f8 --- /dev/null +++ b/staging/src/k8s.io/client-go/util/workqueue/metrics_test.go @@ -0,0 +1,293 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workqueue + +import ( + "sync" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/clock" +) + +type testMetrics struct { + added, gotten, finished int64 + + updateCalled chan<- struct{} +} + +func (m *testMetrics) add(item t) { m.added++ } +func (m *testMetrics) get(item t) { m.gotten++ } +func (m *testMetrics) done(item t) { m.finished++ } +func (m *testMetrics) updateUnfinishedWork() { m.updateCalled <- struct{}{} } + +func TestMetricShutdown(t *testing.T) { + ch := make(chan struct{}) + m := &testMetrics{ + updateCalled: ch, + } + c := clock.NewFakeClock(time.Now()) + q := newQueue(c, m, time.Millisecond) + for !c.HasWaiters() { + // Wait for the go routine to call NewTicker() + time.Sleep(time.Millisecond) + } + + c.Step(time.Millisecond) + <-ch + q.ShutDown() + + c.Step(time.Hour) + select { + default: + return + case <-ch: + t.Errorf("Unexpected update after shutdown was called.") + } +} + +type testMetric struct { + inc int64 + dec int64 + set float64 + + observedValue float64 + observedCount int + + notifyCh chan<- struct{} + + lock sync.Mutex +} + +func (m *testMetric) Inc() { + m.lock.Lock() + defer m.lock.Unlock() + m.inc++ + m.notify() +} + +func (m *testMetric) Dec() { + m.lock.Lock() + defer m.lock.Unlock() + m.dec++ + m.notify() +} + +func (m *testMetric) Set(f float64) { + m.lock.Lock() + defer m.lock.Unlock() + m.set = f + m.notify() +} + +func (m *testMetric) Observe(f float64) { + m.lock.Lock() + defer m.lock.Unlock() + m.observedValue = f + m.observedCount++ + m.notify() +} + +func (m *testMetric) gaugeValue() float64 { + m.lock.Lock() + defer m.lock.Unlock() + if m.set != 0 { + return m.set + } + return float64(m.inc - m.dec) +} + +func (m *testMetric) observationValue() float64 { + m.lock.Lock() + defer m.lock.Unlock() + return m.observedValue +} + +func (m *testMetric) observationCount() int { + m.lock.Lock() + defer m.lock.Unlock() + return m.observedCount +} + +func (m *testMetric) notify() { + if m.notifyCh != nil { + m.notifyCh <- struct{}{} + } +} + +type testMetricsProvider struct { + depth testMetric + adds testMetric + latency testMetric + duration testMetric + unfinished testMetric + longest testMetric + retries testMetric +} + +func (m *testMetricsProvider) NewDepthMetric(name string) GaugeMetric { + return &m.depth +} + +func (m *testMetricsProvider) NewAddsMetric(name string) CounterMetric { + return &m.adds +} + +func (m *testMetricsProvider) NewLatencyMetric(name string) SummaryMetric { + return &m.latency +} + +func (m *testMetricsProvider) NewWorkDurationMetric(name string) SummaryMetric { + return &m.duration +} + +func (m *testMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) SettableGaugeMetric { + return &m.unfinished +} + +func (m *testMetricsProvider) NewLongestRunningProcessorMicrosecondsMetric(name string) SettableGaugeMetric { + return &m.longest +} + +func (m *testMetricsProvider) NewRetriesMetric(name string) CounterMetric { + return &m.retries +} + +func TestSinceInMicroseconds(t *testing.T) { + mp := testMetricsProvider{} + c := clock.NewFakeClock(time.Now()) + mf := queueMetricsFactory{metricsProvider: &mp} + m := mf.newQueueMetrics("test", c) + dqm := m.(*defaultQueueMetrics) + + for _, i := range []int{1, 50, 100, 500, 1000, 10000, 100000, 1000000} { + n := c.Now() + c.Step(time.Duration(i) * time.Microsecond) + if e, a := float64(i), dqm.sinceInMicroseconds(n); e != a { + t.Errorf("Expected %v, got %v", e, a) + } + } +} + +func TestMetrics(t *testing.T) { + mp := testMetricsProvider{} + t0 := time.Unix(0, 0) + c := clock.NewFakeClock(t0) + mf := queueMetricsFactory{metricsProvider: &mp} + m := mf.newQueueMetrics("test", c) + q := newQueue(c, m, time.Millisecond) + defer q.ShutDown() + for !c.HasWaiters() { + // Wait for the go routine to call NewTicker() + time.Sleep(time.Millisecond) + } + + q.Add("foo") + if e, a := 1.0, mp.adds.gaugeValue(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + + if e, a := 1.0, mp.depth.gaugeValue(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + + c.Step(50 * time.Microsecond) + + // Start processing + i, _ := q.Get() + if i != "foo" { + t.Errorf("Expected %v, got %v", "foo", i) + } + + if e, a := 50.0, mp.latency.observationValue(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + if e, a := 1, mp.latency.observationCount(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + if e, a := 0.0, mp.depth.gaugeValue(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + + // Add it back while processing; multiple adds of the same item are + // de-duped. + q.Add(i) + q.Add(i) + q.Add(i) + q.Add(i) + q.Add(i) + if e, a := 2.0, mp.adds.gaugeValue(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + // One thing remains in the queue + if e, a := 1.0, mp.depth.gaugeValue(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + + c.Step(25 * time.Microsecond) + + // Finish it up + q.Done(i) + + if e, a := 25.0, mp.duration.observationValue(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + if e, a := 1, mp.duration.observationCount(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + + // One thing remains in the queue + if e, a := 1.0, mp.depth.gaugeValue(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + + // It should be back on the queue + i, _ = q.Get() + if i != "foo" { + t.Errorf("Expected %v, got %v", "foo", i) + } + + if e, a := 25.0, mp.latency.observationValue(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + if e, a := 2, mp.latency.observationCount(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + + // use a channel to ensure we don't look at the metric before it's + // been set. + ch := make(chan struct{}, 1) + mp.unfinished.notifyCh = ch + c.Step(time.Millisecond) + <-ch + mp.unfinished.notifyCh = nil + if e, a := .001, mp.unfinished.gaugeValue(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + if e, a := 1000.0, mp.longest.gaugeValue(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + + // Finish that one up + q.Done(i) + if e, a := 1000.0, mp.duration.observationValue(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + if e, a := 2, mp.duration.observationCount(); e != a { + t.Errorf("expected %v, got %v", e, a) + } +} diff --git a/staging/src/k8s.io/client-go/util/workqueue/queue.go b/staging/src/k8s.io/client-go/util/workqueue/queue.go index dc9a7cc7b7c..39009b8e79a 100644 --- a/staging/src/k8s.io/client-go/util/workqueue/queue.go +++ b/staging/src/k8s.io/client-go/util/workqueue/queue.go @@ -18,6 +18,9 @@ package workqueue import ( "sync" + "time" + + "k8s.io/apimachinery/pkg/util/clock" ) type Interface interface { @@ -35,14 +38,29 @@ func New() *Type { } func NewNamed(name string) *Type { - return &Type{ - dirty: set{}, - processing: set{}, - cond: sync.NewCond(&sync.Mutex{}), - metrics: newQueueMetrics(name), - } + rc := clock.RealClock{} + return newQueue( + rc, + globalMetricsFactory.newQueueMetrics(name, rc), + defaultUnfinishedWorkUpdatePeriod, + ) } +func newQueue(c clock.Clock, metrics queueMetrics, updatePeriod time.Duration) *Type { + t := &Type{ + clock: c, + dirty: set{}, + processing: set{}, + cond: sync.NewCond(&sync.Mutex{}), + metrics: metrics, + unfinishedWorkUpdatePeriod: updatePeriod, + } + go t.updateUnfinishedWorkLoop() + return t +} + +const defaultUnfinishedWorkUpdatePeriod = 500 * time.Millisecond + // Type is a work queue (see the package comment). type Type struct { // queue defines the order in which we will work on items. Every @@ -64,6 +82,9 @@ type Type struct { shuttingDown bool metrics queueMetrics + + unfinishedWorkUpdatePeriod time.Duration + clock clock.Clock } type empty struct{} @@ -170,3 +191,22 @@ func (q *Type) ShuttingDown() bool { return q.shuttingDown } + +func (q *Type) updateUnfinishedWorkLoop() { + t := q.clock.NewTicker(q.unfinishedWorkUpdatePeriod) + defer t.Stop() + for range t.C() { + if !func() bool { + q.cond.L.Lock() + defer q.cond.L.Unlock() + if !q.shuttingDown { + q.metrics.updateUnfinishedWork() + return true + } + return false + + }() { + return + } + } +} diff --git a/staging/src/k8s.io/cloud-provider/BUILD b/staging/src/k8s.io/cloud-provider/BUILD index bbabcabc33a..4c0c21dcbe8 100644 --- a/staging/src/k8s.io/cloud-provider/BUILD +++ b/staging/src/k8s.io/cloud-provider/BUILD @@ -20,7 +20,7 @@ go_library( "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/cloud-provider/Godeps/Godeps.json b/staging/src/k8s.io/cloud-provider/Godeps/Godeps.json index f1f23b35317..bcaec70c09a 100644 --- a/staging/src/k8s.io/cloud-provider/Godeps/Godeps.json +++ b/staging/src/k8s.io/cloud-provider/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/cloud-provider", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -10,10 +10,6 @@ "ImportPath": "github.com/davecgh/go-spew/spew", "Rev": "782f4967f2dc4564575ca782fe2d04090b5faca8" }, - { - "ImportPath": "github.com/ghodss/yaml", - "Rev": "c7ce16629ff4cd059ed96ed06419dd3856fd3577" - }, { "ImportPath": "github.com/gogo/protobuf/proto", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" @@ -22,10 +18,6 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/golang/protobuf/proto", "Rev": "b4deda0973fb4c70b50d226b1af49f3da59f5265" @@ -104,23 +96,23 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/oauth2", @@ -961,6 +953,14 @@ { "ImportPath": "k8s.io/client-go/util/retry", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" } ] } diff --git a/staging/src/k8s.io/cloud-provider/plugins.go b/staging/src/k8s.io/cloud-provider/plugins.go index b9ec938d386..9fc6aff8cd2 100644 --- a/staging/src/k8s.io/cloud-provider/plugins.go +++ b/staging/src/k8s.io/cloud-provider/plugins.go @@ -22,7 +22,7 @@ import ( "os" "sync" - "github.com/golang/glog" + "k8s.io/klog" ) // Factory is a function that returns a cloudprovider.Interface. @@ -59,9 +59,9 @@ func RegisterCloudProvider(name string, cloud Factory) { providersMutex.Lock() defer providersMutex.Unlock() if _, found := providers[name]; found { - glog.Fatalf("Cloud provider %q was registered twice", name) + klog.Fatalf("Cloud provider %q was registered twice", name) } - glog.V(1).Infof("Registered cloud provider %q", name) + klog.V(1).Infof("Registered cloud provider %q", name) providers[name] = cloud } @@ -100,12 +100,12 @@ func InitCloudProvider(name string, configFilePath string) (Interface, error) { var err error if name == "" { - glog.Info("No cloud provider specified.") + klog.Info("No cloud provider specified.") return nil, nil } if IsExternal(name) { - glog.Info("External cloud provider specified") + klog.Info("External cloud provider specified") return nil, nil } @@ -115,7 +115,7 @@ func InitCloudProvider(name string, configFilePath string) (Interface, error) { if provider.external { detail = fmt.Sprintf("Please use 'external' cloud provider for %s: %s", name, provider.detail) } - glog.Warningf("WARNING: %s built-in cloud provider is now deprecated. %s", name, detail) + klog.Warningf("WARNING: %s built-in cloud provider is now deprecated. %s", name, detail) break } @@ -125,7 +125,7 @@ func InitCloudProvider(name string, configFilePath string) (Interface, error) { var config *os.File config, err = os.Open(configFilePath) if err != nil { - glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", + klog.Fatalf("Couldn't open cloud provider configuration %s: %#v", configFilePath, err) } diff --git a/staging/src/k8s.io/cluster-bootstrap/Godeps/Godeps.json b/staging/src/k8s.io/cluster-bootstrap/Godeps/Godeps.json index fc536250ef9..4c402fdc979 100644 --- a/staging/src/k8s.io/cluster-bootstrap/Godeps/Godeps.json +++ b/staging/src/k8s.io/cluster-bootstrap/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/cluster-bootstrap", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -14,29 +14,25 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/google/gofuzz", "Rev": "44d81051d367757e1c7c6a5a86423ece9afcf63c" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/text/secure/bidirule", @@ -145,6 +141,10 @@ { "ImportPath": "k8s.io/apimachinery/third_party/forked/golang/reflect", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" } ] } diff --git a/staging/src/k8s.io/code-generator/Godeps/Godeps.json b/staging/src/k8s.io/code-generator/Godeps/Godeps.json index ad70c9f5a3f..6301566c0e0 100644 --- a/staging/src/k8s.io/code-generator/Godeps/Godeps.json +++ b/staging/src/k8s.io/code-generator/Godeps/Godeps.json @@ -1,43 +1,11 @@ { "ImportPath": "k8s.io/code-generator", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], "Deps": [ - { - "ImportPath": "github.com/PuerkitoBio/purell", - "Rev": "8a290539e2e8629dbc4e6bad948158f790ec31f4" - }, - { - "ImportPath": "github.com/PuerkitoBio/urlesc", - "Rev": "5bd2802263f21d8788851d5305584c82a5c75d7e" - }, - { - "ImportPath": "github.com/emicklei/go-restful", - "Rev": "ff4f55a206334ef123e4f79bbf348980da81ca46" - }, - { - "ImportPath": "github.com/emicklei/go-restful/log", - "Rev": "ff4f55a206334ef123e4f79bbf348980da81ca46" - }, - { - "ImportPath": "github.com/go-openapi/jsonpointer", - "Rev": "ef5f0afec364d3b9396b7b77b43dbe26bf1f8004" - }, - { - "ImportPath": "github.com/go-openapi/jsonreference", - "Rev": "8483a886a90412cd6858df4ea3483dce9c8e35a3" - }, - { - "ImportPath": "github.com/go-openapi/spec", - "Rev": "5bae59e25b21498baea7f9d46e9c147ec106a42e" - }, - { - "ImportPath": "github.com/go-openapi/swag", - "Rev": "5899d5c5e619fda5fa86e14795a835f473ca284c" - }, { "ImportPath": "github.com/gogo/protobuf/gogoproto", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" @@ -138,74 +106,10 @@ "ImportPath": "github.com/gogo/protobuf/vanity/command", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, - { - "ImportPath": "github.com/mailru/easyjson/buffer", - "Rev": "2f5df55504ebc322e4d52d34df6a1f5b503bf26d" - }, - { - "ImportPath": "github.com/mailru/easyjson/jlexer", - "Rev": "2f5df55504ebc322e4d52d34df6a1f5b503bf26d" - }, - { - "ImportPath": "github.com/mailru/easyjson/jwriter", - "Rev": "2f5df55504ebc322e4d52d34df6a1f5b503bf26d" - }, { "ImportPath": "github.com/spf13/pflag", "Rev": "583c0c0531f06d5278b7d917446061adc344b5cd" }, - { - "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" - }, - { - "ImportPath": "golang.org/x/text/cases", - "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" - }, - { - "ImportPath": "golang.org/x/text/internal", - "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" - }, - { - "ImportPath": "golang.org/x/text/internal/tag", - "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" - }, - { - "ImportPath": "golang.org/x/text/language", - "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" - }, - { - "ImportPath": "golang.org/x/text/runes", - "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" - }, - { - "ImportPath": "golang.org/x/text/secure/bidirule", - "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" - }, - { - "ImportPath": "golang.org/x/text/secure/precis", - "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" - }, - { - "ImportPath": "golang.org/x/text/transform", - "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" - }, - { - "ImportPath": "golang.org/x/text/unicode/bidi", - "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" - }, - { - "ImportPath": "golang.org/x/text/unicode/norm", - "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" - }, - { - "ImportPath": "golang.org/x/text/width", - "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" - }, { "ImportPath": "golang.org/x/tools/go/ast/astutil", "Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32" @@ -214,69 +118,49 @@ "ImportPath": "golang.org/x/tools/imports", "Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32" }, - { - "ImportPath": "gopkg.in/yaml.v2", - "Rev": "5420a8b6744d3b0345ab293f6fcba19c978f1183" - }, { "ImportPath": "k8s.io/gengo/args", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/examples/deepcopy-gen/generators", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/examples/defaulter-gen/generators", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/examples/import-boss/generators", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/examples/set-gen/generators", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/examples/set-gen/sets", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/generator", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/namer", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/parser", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { "ImportPath": "k8s.io/gengo/types", - "Rev": "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" + "Rev": "51747d6e00da1fc578d5a333a93bb2abcbce7a95" }, { - "ImportPath": "k8s.io/kube-openapi/cmd/openapi-gen/args", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" - }, - { - "ImportPath": "k8s.io/kube-openapi/pkg/common", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" - }, - { - "ImportPath": "k8s.io/kube-openapi/pkg/generators", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" - }, - { - "ImportPath": "k8s.io/kube-openapi/pkg/generators/rules", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" - }, - { - "ImportPath": "k8s.io/kube-openapi/pkg/util/sets", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" } ] } diff --git a/staging/src/k8s.io/code-generator/_examples/MixedCase/clientset/versioned/typed/example/v1/clustertesttype.go b/staging/src/k8s.io/code-generator/_examples/MixedCase/clientset/versioned/typed/example/v1/clustertesttype.go index b6f023f9444..559ce47eb77 100644 --- a/staging/src/k8s.io/code-generator/_examples/MixedCase/clientset/versioned/typed/example/v1/clustertesttype.go +++ b/staging/src/k8s.io/code-generator/_examples/MixedCase/clientset/versioned/typed/example/v1/clustertesttype.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -77,10 +79,15 @@ func (c *clusterTestTypes) Get(name string, options metav1.GetOptions) (result * // List takes label and field selectors, and returns the list of ClusterTestTypes that match those selectors. func (c *clusterTestTypes) List(opts metav1.ListOptions) (result *v1.ClusterTestTypeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.ClusterTestTypeList{} err = c.client.Get(). Resource("clustertesttypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,10 +95,15 @@ func (c *clusterTestTypes) List(opts metav1.ListOptions) (result *v1.ClusterTest // Watch returns a watch.Interface that watches the requested clusterTestTypes. func (c *clusterTestTypes) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("clustertesttypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -145,9 +157,14 @@ func (c *clusterTestTypes) Delete(name string, options *metav1.DeleteOptions) er // DeleteCollection deletes a collection of objects. func (c *clusterTestTypes) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("clustertesttypes"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/code-generator/_examples/MixedCase/clientset/versioned/typed/example/v1/testtype.go b/staging/src/k8s.io/code-generator/_examples/MixedCase/clientset/versioned/typed/example/v1/testtype.go index 3c7b636e28e..9944e3e7c72 100644 --- a/staging/src/k8s.io/code-generator/_examples/MixedCase/clientset/versioned/typed/example/v1/testtype.go +++ b/staging/src/k8s.io/code-generator/_examples/MixedCase/clientset/versioned/typed/example/v1/testtype.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *testTypes) Get(name string, options metav1.GetOptions) (result *v1.Test // List takes label and field selectors, and returns the list of TestTypes that match those selectors. func (c *testTypes) List(opts metav1.ListOptions) (result *v1.TestTypeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.TestTypeList{} err = c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *testTypes) List(opts metav1.ListOptions) (result *v1.TestTypeList, err // Watch returns a watch.Interface that watches the requested testTypes. func (c *testTypes) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *testTypes) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *testTypes) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example/internalversion/testtype.go b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example/internalversion/testtype.go index d19392e8534..a06b9461692 100644 --- a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example/internalversion/testtype.go +++ b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example/internalversion/testtype.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *testTypes) Get(name string, options v1.GetOptions) (result *example.Tes // List takes label and field selectors, and returns the list of TestTypes that match those selectors. func (c *testTypes) List(opts v1.ListOptions) (result *example.TestTypeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &example.TestTypeList{} err = c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *testTypes) List(opts v1.ListOptions) (result *example.TestTypeList, err // Watch returns a watch.Interface that watches the requested testTypes. func (c *testTypes) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *testTypes) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *testTypes) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example2/internalversion/testtype.go b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example2/internalversion/testtype.go index 1a2ca7891a5..5380b86ce7a 100644 --- a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example2/internalversion/testtype.go +++ b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example2/internalversion/testtype.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *testTypes) Get(name string, options v1.GetOptions) (result *example2.Te // List takes label and field selectors, and returns the list of TestTypes that match those selectors. func (c *testTypes) List(opts v1.ListOptions) (result *example2.TestTypeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &example2.TestTypeList{} err = c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *testTypes) List(opts v1.ListOptions) (result *example2.TestTypeList, er // Watch returns a watch.Interface that watches the requested testTypes. func (c *testTypes) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *testTypes) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *testTypes) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/versioned/typed/example/v1/testtype.go b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/versioned/typed/example/v1/testtype.go index 2a0606512f9..e25fd1fc211 100644 --- a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/versioned/typed/example/v1/testtype.go +++ b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/versioned/typed/example/v1/testtype.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *testTypes) Get(name string, options metav1.GetOptions) (result *v1.Test // List takes label and field selectors, and returns the list of TestTypes that match those selectors. func (c *testTypes) List(opts metav1.ListOptions) (result *v1.TestTypeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.TestTypeList{} err = c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *testTypes) List(opts metav1.ListOptions) (result *v1.TestTypeList, err // Watch returns a watch.Interface that watches the requested testTypes. func (c *testTypes) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *testTypes) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *testTypes) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/versioned/typed/example2/v1/testtype.go b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/versioned/typed/example2/v1/testtype.go index b29bd8118bb..f5afee94ccf 100644 --- a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/versioned/typed/example2/v1/testtype.go +++ b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/versioned/typed/example2/v1/testtype.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *testTypes) Get(name string, options metav1.GetOptions) (result *v1.Test // List takes label and field selectors, and returns the list of TestTypes that match those selectors. func (c *testTypes) List(opts metav1.ListOptions) (result *v1.TestTypeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.TestTypeList{} err = c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *testTypes) List(opts metav1.ListOptions) (result *v1.TestTypeList, err // Watch returns a watch.Interface that watches the requested testTypes. func (c *testTypes) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *testTypes) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *testTypes) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/clustertesttype.go b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/clustertesttype.go index 30e0e749802..1217dd8673d 100644 --- a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/clustertesttype.go +++ b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/clustertesttype.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -77,10 +79,15 @@ func (c *clusterTestTypes) Get(name string, options metav1.GetOptions) (result * // List takes label and field selectors, and returns the list of ClusterTestTypes that match those selectors. func (c *clusterTestTypes) List(opts metav1.ListOptions) (result *v1.ClusterTestTypeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.ClusterTestTypeList{} err = c.client.Get(). Resource("clustertesttypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,10 +95,15 @@ func (c *clusterTestTypes) List(opts metav1.ListOptions) (result *v1.ClusterTest // Watch returns a watch.Interface that watches the requested clusterTestTypes. func (c *clusterTestTypes) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("clustertesttypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -145,9 +157,14 @@ func (c *clusterTestTypes) Delete(name string, options *metav1.DeleteOptions) er // DeleteCollection deletes a collection of objects. func (c *clusterTestTypes) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("clustertesttypes"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/testtype.go b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/testtype.go index 6fafb1e1b15..164b0510e47 100644 --- a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/testtype.go +++ b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/testtype.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *testTypes) Get(name string, options metav1.GetOptions) (result *v1.Test // List takes label and field selectors, and returns the list of TestTypes that match those selectors. func (c *testTypes) List(opts metav1.ListOptions) (result *v1.TestTypeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.TestTypeList{} err = c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *testTypes) List(opts metav1.ListOptions) (result *v1.TestTypeList, err // Watch returns a watch.Interface that watches the requested testTypes. func (c *testTypes) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *testTypes) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *testTypes) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example2/v1/testtype.go b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example2/v1/testtype.go index 69b0e64c528..2e3194e00ab 100644 --- a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example2/v1/testtype.go +++ b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example2/v1/testtype.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *testTypes) Get(name string, options metav1.GetOptions) (result *v1.Test // List takes label and field selectors, and returns the list of TestTypes that match those selectors. func (c *testTypes) List(opts metav1.ListOptions) (result *v1.TestTypeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.TestTypeList{} err = c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *testTypes) List(opts metav1.ListOptions) (result *v1.TestTypeList, err // Watch returns a watch.Interface that watches the requested testTypes. func (c *testTypes) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *testTypes) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *testTypes) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("testtypes"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/code-generator/cmd/client-gen/BUILD b/staging/src/k8s.io/code-generator/cmd/client-gen/BUILD index 8785ee09a7f..5df0eb366d6 100644 --- a/staging/src/k8s.io/code-generator/cmd/client-gen/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/client-gen/BUILD @@ -20,9 +20,9 @@ go_library( "//staging/src/k8s.io/code-generator/cmd/client-gen/args:go_default_library", "//staging/src/k8s.io/code-generator/cmd/client-gen/generators:go_default_library", "//staging/src/k8s.io/code-generator/pkg/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/client-gen/generators/BUILD b/staging/src/k8s.io/code-generator/cmd/client-gen/generators/BUILD index 3bae7189e15..cf386272f1a 100644 --- a/staging/src/k8s.io/code-generator/cmd/client-gen/generators/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/client-gen/generators/BUILD @@ -24,11 +24,11 @@ go_library( "//staging/src/k8s.io/code-generator/cmd/client-gen/generators/util:go_default_library", "//staging/src/k8s.io/code-generator/cmd/client-gen/path:go_default_library", "//staging/src/k8s.io/code-generator/cmd/client-gen/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/generator:go_default_library", "//vendor/k8s.io/gengo/namer:go_default_library", "//vendor/k8s.io/gengo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/client-gen/generators/client_generator.go b/staging/src/k8s.io/code-generator/cmd/client-gen/generators/client_generator.go index bf536878171..ee6ebbcf093 100644 --- a/staging/src/k8s.io/code-generator/cmd/client-gen/generators/client_generator.go +++ b/staging/src/k8s.io/code-generator/cmd/client-gen/generators/client_generator.go @@ -32,7 +32,7 @@ import ( "k8s.io/gengo/namer" "k8s.io/gengo/types" - "github.com/golang/glog" + "k8s.io/klog" ) // NameSystems returns the name system used by the generators in this package. @@ -318,12 +318,12 @@ func applyGroupOverrides(universe types.Universe, customArgs *clientgenargs.Cust func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages { boilerplate, err := arguments.LoadGoBoilerplate() if err != nil { - glog.Fatalf("Failed loading boilerplate: %v", err) + klog.Fatalf("Failed loading boilerplate: %v", err) } customArgs, ok := arguments.CustomArgs.(*clientgenargs.CustomArgs) if !ok { - glog.Fatalf("cannot convert arguments.CustomArgs to clientgenargs.CustomArgs") + klog.Fatalf("cannot convert arguments.CustomArgs to clientgenargs.CustomArgs") } includedTypesOverrides := customArgs.IncludedTypesOverrides diff --git a/staging/src/k8s.io/code-generator/cmd/client-gen/generators/generator_for_type.go b/staging/src/k8s.io/code-generator/cmd/client-gen/generators/generator_for_type.go index 92e2a97f1de..3e8fc7c4c68 100644 --- a/staging/src/k8s.io/code-generator/cmd/client-gen/generators/generator_for_type.go +++ b/staging/src/k8s.io/code-generator/cmd/client-gen/generators/generator_for_type.go @@ -387,11 +387,16 @@ func new$.type|publicPlural$(c *$.GroupGoName$$.Version$Client) *$.type|privateP var listTemplate = ` // List takes label and field selectors, and returns the list of $.resultType|publicPlural$ that match those selectors. func (c *$.type|privatePlural$) List(opts $.ListOptions|raw$) (result *$.resultType|raw$List, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil{ + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &$.resultType|raw$List{} err = c.client.Get(). $if .namespaced$Namespace(c.ns).$end$ Resource("$.type|resource$"). VersionedParams(&opts, $.schemeParameterCodec|raw$). + Timeout(timeout). Do(). Into(result) return @@ -401,6 +406,10 @@ func (c *$.type|privatePlural$) List(opts $.ListOptions|raw$) (result *$.resultT var listSubresourceTemplate = ` // List takes $.type|raw$ name, label and field selectors, and returns the list of $.resultType|publicPlural$ that match those selectors. func (c *$.type|privatePlural$) List($.type|private$Name string, opts $.ListOptions|raw$) (result *$.resultType|raw$List, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil{ + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &$.resultType|raw$List{} err = c.client.Get(). $if .namespaced$Namespace(c.ns).$end$ @@ -408,6 +417,7 @@ func (c *$.type|privatePlural$) List($.type|private$Name string, opts $.ListOpti Name($.type|private$Name). SubResource("$.subresourcePath$"). VersionedParams(&opts, $.schemeParameterCodec|raw$). + Timeout(timeout). Do(). Into(result) return @@ -461,10 +471,15 @@ func (c *$.type|privatePlural$) Delete(name string, options *$.DeleteOptions|raw var deleteCollectionTemplate = ` // DeleteCollection deletes a collection of objects. func (c *$.type|privatePlural$) DeleteCollection(options *$.DeleteOptions|raw$, listOptions $.ListOptions|raw$) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil{ + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). $if .namespaced$Namespace(c.ns).$end$ Resource("$.type|resource$"). VersionedParams(&listOptions, $.schemeParameterCodec|raw$). + Timeout(timeout). Body(options). Do(). Error() @@ -553,11 +568,16 @@ func (c *$.type|privatePlural$) UpdateStatus($.type|private$ *$.type|raw$) (resu var watchTemplate = ` // Watch returns a $.watchInterface|raw$ that watches the requested $.type|privatePlural$. func (c *$.type|privatePlural$) Watch(opts $.ListOptions|raw$) ($.watchInterface|raw$, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil{ + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). $if .namespaced$Namespace(c.ns).$end$ Resource("$.type|resource$"). VersionedParams(&opts, $.schemeParameterCodec|raw$). + Timeout(timeout). Watch() } ` diff --git a/staging/src/k8s.io/code-generator/cmd/client-gen/main.go b/staging/src/k8s.io/code-generator/cmd/client-gen/main.go index 22c28e35f8f..6e0d187f5cb 100644 --- a/staging/src/k8s.io/code-generator/cmd/client-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/client-gen/main.go @@ -21,9 +21,9 @@ import ( "flag" "path/filepath" - "github.com/golang/glog" "github.com/spf13/pflag" "k8s.io/gengo/args" + "k8s.io/klog" generatorargs "k8s.io/code-generator/cmd/client-gen/args" "k8s.io/code-generator/cmd/client-gen/generators" @@ -31,6 +31,7 @@ import ( ) func main() { + klog.InitFlags(nil) genericArgs, customArgs := generatorargs.NewDefaults() // Override defaults. @@ -52,7 +53,7 @@ func main() { } if err := generatorargs.Validate(genericArgs); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } if err := genericArgs.Execute( @@ -60,6 +61,6 @@ func main() { generators.DefaultNameSystem(), generators.Packages, ); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } } diff --git a/staging/src/k8s.io/code-generator/cmd/conversion-gen/BUILD b/staging/src/k8s.io/code-generator/cmd/conversion-gen/BUILD index 1991d175300..fd488f18c25 100644 --- a/staging/src/k8s.io/code-generator/cmd/conversion-gen/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/conversion-gen/BUILD @@ -20,9 +20,9 @@ go_library( "//staging/src/k8s.io/code-generator/cmd/conversion-gen/args:go_default_library", "//staging/src/k8s.io/code-generator/cmd/conversion-gen/generators:go_default_library", "//staging/src/k8s.io/code-generator/pkg/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/conversion-gen/generators/BUILD b/staging/src/k8s.io/code-generator/cmd/conversion-gen/generators/BUILD index 1088b39fcdc..2b650f2405e 100644 --- a/staging/src/k8s.io/code-generator/cmd/conversion-gen/generators/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/conversion-gen/generators/BUILD @@ -12,11 +12,11 @@ go_library( importpath = "k8s.io/code-generator/cmd/conversion-gen/generators", deps = [ "//staging/src/k8s.io/code-generator/cmd/conversion-gen/args:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/generator:go_default_library", "//vendor/k8s.io/gengo/namer:go_default_library", "//vendor/k8s.io/gengo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/conversion-gen/generators/conversion.go b/staging/src/k8s.io/code-generator/cmd/conversion-gen/generators/conversion.go index 422237e1177..775972d1231 100644 --- a/staging/src/k8s.io/code-generator/cmd/conversion-gen/generators/conversion.go +++ b/staging/src/k8s.io/code-generator/cmd/conversion-gen/generators/conversion.go @@ -29,7 +29,7 @@ import ( "k8s.io/gengo/namer" "k8s.io/gengo/types" - "github.com/golang/glog" + "k8s.io/klog" conversionargs "k8s.io/code-generator/cmd/conversion-gen/args" ) @@ -124,10 +124,10 @@ type conversionFuncMap map[conversionPair]*types.Type // Returns all manually-defined conversion functions in the package. func getManualConversionFunctions(context *generator.Context, pkg *types.Package, manualMap conversionFuncMap) { if pkg == nil { - glog.Warningf("Skipping nil package passed to getManualConversionFunctions") + klog.Warningf("Skipping nil package passed to getManualConversionFunctions") return } - glog.V(5).Infof("Scanning for conversion functions in %v", pkg.Name) + klog.V(5).Infof("Scanning for conversion functions in %v", pkg.Name) scopeName := types.Ref(conversionPackagePath, "Scope").Name errorName := types.Ref("", "error").Name @@ -136,34 +136,34 @@ func getManualConversionFunctions(context *generator.Context, pkg *types.Package for _, f := range pkg.Functions { if f.Underlying == nil || f.Underlying.Kind != types.Func { - glog.Errorf("Malformed function: %#v", f) + klog.Errorf("Malformed function: %#v", f) continue } if f.Underlying.Signature == nil { - glog.Errorf("Function without signature: %#v", f) + klog.Errorf("Function without signature: %#v", f) continue } - glog.V(8).Infof("Considering function %s", f.Name) + klog.V(8).Infof("Considering function %s", f.Name) signature := f.Underlying.Signature // Check whether the function is conversion function. // Note that all of them have signature: // func Convert_inType_To_outType(inType, outType, conversion.Scope) error if signature.Receiver != nil { - glog.V(8).Infof("%s has a receiver", f.Name) + klog.V(8).Infof("%s has a receiver", f.Name) continue } if len(signature.Parameters) != 3 || signature.Parameters[2].Name != scopeName { - glog.V(8).Infof("%s has wrong parameters", f.Name) + klog.V(8).Infof("%s has wrong parameters", f.Name) continue } if len(signature.Results) != 1 || signature.Results[0].Name != errorName { - glog.V(8).Infof("%s has wrong results", f.Name) + klog.V(8).Infof("%s has wrong results", f.Name) continue } inType := signature.Parameters[0] outType := signature.Parameters[1] if inType.Kind != types.Pointer || outType.Kind != types.Pointer { - glog.V(8).Infof("%s has wrong parameter types", f.Name) + klog.V(8).Infof("%s has wrong parameter types", f.Name) continue } // Now check if the name satisfies the convention. @@ -171,7 +171,7 @@ func getManualConversionFunctions(context *generator.Context, pkg *types.Package args := argsFromType(inType.Elem, outType.Elem) sw.Do("Convert_$.inType|public$_To_$.outType|public$", args) if f.Name.Name == buffer.String() { - glog.V(4).Infof("Found conversion function %s", f.Name) + klog.V(4).Infof("Found conversion function %s", f.Name) key := conversionPair{inType.Elem, outType.Elem} // We might scan the same package twice, and that's OK. if v, ok := manualMap[key]; ok && v != nil && v.Name.Package != pkg.Path { @@ -181,9 +181,9 @@ func getManualConversionFunctions(context *generator.Context, pkg *types.Package } else { // prevent user error when they don't get the correct conversion signature if strings.HasPrefix(f.Name.Name, "Convert_") { - glog.Errorf("Rename function %s %s -> %s to match expected conversion signature", f.Name.Package, f.Name.Name, buffer.String()) + klog.Errorf("Rename function %s %s -> %s to match expected conversion signature", f.Name.Package, f.Name.Name, buffer.String()) } - glog.V(8).Infof("%s has wrong name", f.Name) + klog.V(8).Infof("%s has wrong name", f.Name) } buffer.Reset() } @@ -192,7 +192,7 @@ func getManualConversionFunctions(context *generator.Context, pkg *types.Package func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages { boilerplate, err := arguments.LoadGoBoilerplate() if err != nil { - glog.Fatalf("Failed loading boilerplate: %v", err) + klog.Fatalf("Failed loading boilerplate: %v", err) } packages := generator.Packages{} @@ -220,7 +220,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat } processed[i] = true - glog.V(5).Infof("considering pkg %q", i) + klog.V(5).Infof("considering pkg %q", i) pkg := context.Universe[i] // typesPkg is where the versioned types are defined. Sometimes it is // different from pkg. For example, kubernetes core/v1 types are defined @@ -239,9 +239,9 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat // in their doc.go file. peerPkgs := extractTag(pkg.Comments) if peerPkgs != nil { - glog.V(5).Infof(" tags: %q", peerPkgs) + klog.V(5).Infof(" tags: %q", peerPkgs) } else { - glog.V(5).Infof(" no tag") + klog.V(5).Infof(" no tag") continue } skipUnsafe := false @@ -255,14 +255,14 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat externalTypesValues := extractExternalTypesTag(pkg.Comments) if externalTypesValues != nil { if len(externalTypesValues) != 1 { - glog.Fatalf(" expect only one value for %q tag, got: %q", externalTypesTagName, externalTypesValues) + klog.Fatalf(" expect only one value for %q tag, got: %q", externalTypesTagName, externalTypesValues) } externalTypes := externalTypesValues[0] - glog.V(5).Infof(" external types tags: %q", externalTypes) + klog.V(5).Infof(" external types tags: %q", externalTypes) var err error typesPkg, err = context.AddDirectory(externalTypes) if err != nil { - glog.Fatalf("cannot import package %s", externalTypes) + klog.Fatalf("cannot import package %s", externalTypes) } // update context.Order to the latest context.Universe orderer := namer.Orderer{Namer: namer.NewPublicNamer(1)} @@ -291,7 +291,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat context.AddDir(pp) p := context.Universe[pp] if nil == p { - glog.Fatalf("failed to find pkg: %s", pp) + klog.Fatalf("failed to find pkg: %s", pp) } getManualConversionFunctions(context, p, manualConversions) } @@ -335,7 +335,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat // from being a candidate for unsafe conversion for k, v := range manualConversions { if isCopyOnly(v.CommentLines) { - glog.V(5).Infof("Conversion function %s will not block memory copy because it is copy-only", v.Name) + klog.V(5).Infof("Conversion function %s will not block memory copy because it is copy-only", v.Name) continue } // this type should be excluded from all equivalence, because the converter must be called. @@ -518,9 +518,9 @@ func (g *genConversion) convertibleOnlyWithinPackage(inType, outType *types.Type tagvals := extractTag(t.CommentLines) if tagvals != nil { if tagvals[0] != "false" { - glog.Fatalf("Type %v: unsupported %s value: %q", t, tagName, tagvals[0]) + klog.Fatalf("Type %v: unsupported %s value: %q", t, tagName, tagvals[0]) } - glog.V(5).Infof("type %v requests no conversion generation, skipping", t) + klog.V(5).Infof("type %v requests no conversion generation, skipping", t) return false } // TODO: Consider generating functions for other kinds too. @@ -582,10 +582,10 @@ func (g *genConversion) preexists(inType, outType *types.Type) (*types.Type, boo } func (g *genConversion) Init(c *generator.Context, w io.Writer) error { - if glog.V(5) { + if klog.V(5) { if m, ok := g.useUnsafe.(equalMemoryTypes); ok { var result []string - glog.Infof("All objects without identical memory layout:") + klog.Infof("All objects without identical memory layout:") for k, v := range m { if v { continue @@ -594,7 +594,7 @@ func (g *genConversion) Init(c *generator.Context, w io.Writer) error { } sort.Strings(result) for _, s := range result { - glog.Infof(s) + klog.Infof(s) } } } @@ -643,7 +643,7 @@ func (g *genConversion) Init(c *generator.Context, w io.Writer) error { } func (g *genConversion) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { - glog.V(5).Infof("generating for type %v", t) + klog.V(5).Infof("generating for type %v", t) peerType := getPeerTypeFor(c, t, g.peerPackages) sw := generator.NewSnippetWriter(w, c, "$", "$") g.generateConversion(t, peerType, sw) @@ -664,10 +664,10 @@ func (g *genConversion) generateConversion(inType, outType *types.Type, sw *gene // There is a public manual Conversion method: use it. } else if skipped := g.skippedFields[inType]; len(skipped) != 0 { // The inType had some fields we could not generate. - glog.Errorf("Warning: could not find nor generate a final Conversion function for %v -> %v", inType, outType) - glog.Errorf(" the following fields need manual conversion:") + klog.Errorf("Warning: could not find nor generate a final Conversion function for %v -> %v", inType, outType) + klog.Errorf(" the following fields need manual conversion:") for _, f := range skipped { - glog.Errorf(" - %v", f) + klog.Errorf(" - %v", f) } } else { // Emit a public conversion function. @@ -682,7 +682,7 @@ func (g *genConversion) generateConversion(inType, outType *types.Type, sw *gene // at any nesting level. This makes the autogenerator easy to understand, and // the compiler shouldn't care. func (g *genConversion) generateFor(inType, outType *types.Type, sw *generator.SnippetWriter) { - glog.V(5).Infof("generating %v -> %v", inType, outType) + klog.V(5).Infof("generating %v -> %v", inType, outType) var f func(*types.Type, *types.Type, *generator.SnippetWriter) switch inType.Kind { @@ -853,7 +853,7 @@ func (g *genConversion) doStruct(inType, outType *types.Type, sw *generator.Snip sw.Do("}\n", nil) continue } - glog.V(5).Infof("Skipped function %s because it is copy-only and we can use direct assignment", function.Name) + klog.V(5).Infof("Skipped function %s because it is copy-only and we can use direct assignment", function.Name) } // If we can't auto-convert, punt before we emit any code. diff --git a/staging/src/k8s.io/code-generator/cmd/conversion-gen/main.go b/staging/src/k8s.io/code-generator/cmd/conversion-gen/main.go index f2b91cc2e29..698baa7db7a 100644 --- a/staging/src/k8s.io/code-generator/cmd/conversion-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/conversion-gen/main.go @@ -38,9 +38,9 @@ import ( "flag" "path/filepath" - "github.com/golang/glog" "github.com/spf13/pflag" "k8s.io/gengo/args" + "k8s.io/klog" generatorargs "k8s.io/code-generator/cmd/conversion-gen/args" "k8s.io/code-generator/cmd/conversion-gen/generators" @@ -48,6 +48,7 @@ import ( ) func main() { + klog.InitFlags(nil) genericArgs, customArgs := generatorargs.NewDefaults() // Override defaults. @@ -61,7 +62,7 @@ func main() { pflag.Parse() if err := generatorargs.Validate(genericArgs); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } // Run it. @@ -70,7 +71,7 @@ func main() { generators.DefaultNameSystem(), generators.Packages, ); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } - glog.V(2).Info("Completed successfully.") + klog.V(2).Info("Completed successfully.") } diff --git a/staging/src/k8s.io/code-generator/cmd/deepcopy-gen/BUILD b/staging/src/k8s.io/code-generator/cmd/deepcopy-gen/BUILD index e6841050a06..a1a48d4d916 100644 --- a/staging/src/k8s.io/code-generator/cmd/deepcopy-gen/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/deepcopy-gen/BUILD @@ -19,10 +19,10 @@ go_library( deps = [ "//staging/src/k8s.io/code-generator/cmd/deepcopy-gen/args:go_default_library", "//staging/src/k8s.io/code-generator/pkg/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/examples/deepcopy-gen/generators:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/deepcopy-gen/main.go b/staging/src/k8s.io/code-generator/cmd/deepcopy-gen/main.go index cce65b772f8..96fb2987341 100644 --- a/staging/src/k8s.io/code-generator/cmd/deepcopy-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/deepcopy-gen/main.go @@ -46,16 +46,17 @@ import ( "flag" "path/filepath" - "github.com/golang/glog" "github.com/spf13/pflag" "k8s.io/gengo/args" "k8s.io/gengo/examples/deepcopy-gen/generators" + "k8s.io/klog" generatorargs "k8s.io/code-generator/cmd/deepcopy-gen/args" "k8s.io/code-generator/pkg/util" ) func main() { + klog.InitFlags(nil) genericArgs, customArgs := generatorargs.NewDefaults() // Override defaults. @@ -69,7 +70,7 @@ func main() { pflag.Parse() if err := generatorargs.Validate(genericArgs); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } // Run it. @@ -78,7 +79,7 @@ func main() { generators.DefaultNameSystem(), generators.Packages, ); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } - glog.V(2).Info("Completed successfully.") + klog.V(2).Info("Completed successfully.") } diff --git a/staging/src/k8s.io/code-generator/cmd/defaulter-gen/BUILD b/staging/src/k8s.io/code-generator/cmd/defaulter-gen/BUILD index 056cd14add8..87a8e852926 100644 --- a/staging/src/k8s.io/code-generator/cmd/defaulter-gen/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/defaulter-gen/BUILD @@ -19,10 +19,10 @@ go_library( deps = [ "//staging/src/k8s.io/code-generator/cmd/defaulter-gen/args:go_default_library", "//staging/src/k8s.io/code-generator/pkg/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/examples/defaulter-gen/generators:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/defaulter-gen/main.go b/staging/src/k8s.io/code-generator/cmd/defaulter-gen/main.go index 9d33f700b33..40bb875e52a 100644 --- a/staging/src/k8s.io/code-generator/cmd/defaulter-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/defaulter-gen/main.go @@ -45,16 +45,17 @@ import ( "flag" "path/filepath" - "github.com/golang/glog" "github.com/spf13/pflag" "k8s.io/gengo/args" "k8s.io/gengo/examples/defaulter-gen/generators" + "k8s.io/klog" generatorargs "k8s.io/code-generator/cmd/defaulter-gen/args" "k8s.io/code-generator/pkg/util" ) func main() { + klog.InitFlags(nil) genericArgs, customArgs := generatorargs.NewDefaults() // Override defaults. @@ -68,7 +69,7 @@ func main() { pflag.Parse() if err := generatorargs.Validate(genericArgs); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } // Run it. @@ -77,7 +78,7 @@ func main() { generators.DefaultNameSystem(), generators.Packages, ); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } - glog.V(2).Info("Completed successfully.") + klog.V(2).Info("Completed successfully.") } diff --git a/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/BUILD b/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/BUILD index e90c1f35e3d..a2718d402b8 100644 --- a/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/BUILD @@ -22,13 +22,13 @@ go_library( deps = [ "//staging/src/k8s.io/code-generator/pkg/util:go_default_library", "//staging/src/k8s.io/code-generator/third_party/forked/golang/reflect:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/generator:go_default_library", "//vendor/k8s.io/gengo/namer:go_default_library", "//vendor/k8s.io/gengo/parser:go_default_library", "//vendor/k8s.io/gengo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/generator.go b/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/generator.go index 49e8297dbd0..1a9803dc88d 100644 --- a/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/generator.go +++ b/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/generator.go @@ -25,7 +25,7 @@ import ( "strconv" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/gengo/generator" "k8s.io/gengo/namer" @@ -85,7 +85,7 @@ func (g *genProtoIDL) Filter(c *generator.Context, t *types.Type) bool { // Type specified "true". return true } - glog.Fatalf(`Comment tag "protobuf" must be true or false, found: %q`, tagVals[0]) + klog.Fatalf(`Comment tag "protobuf" must be true or false, found: %q`, tagVals[0]) } if !g.generateAll { // We're not generating everything. diff --git a/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/tags.go b/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/tags.go index 2dff5b9229d..8e2a1917d04 100644 --- a/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/tags.go +++ b/staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf/tags.go @@ -17,8 +17,8 @@ limitations under the License. package protobuf import ( - "github.com/golang/glog" "k8s.io/gengo/types" + "k8s.io/klog" ) // extractBoolTagOrDie gets the comment-tags for the key and asserts that, if @@ -27,7 +27,7 @@ import ( func extractBoolTagOrDie(key string, lines []string) bool { val, err := types.ExtractSingleBoolCommentTag("+", key, false, lines) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return val } diff --git a/staging/src/k8s.io/code-generator/cmd/import-boss/BUILD b/staging/src/k8s.io/code-generator/cmd/import-boss/BUILD index 1b73d4d9664..6fec2c33388 100644 --- a/staging/src/k8s.io/code-generator/cmd/import-boss/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/import-boss/BUILD @@ -18,9 +18,9 @@ go_library( importpath = "k8s.io/code-generator/cmd/import-boss", deps = [ "//staging/src/k8s.io/code-generator/pkg/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/examples/import-boss/generators:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/import-boss/main.go b/staging/src/k8s.io/code-generator/cmd/import-boss/main.go index d998994415d..c0f10c3a49c 100644 --- a/staging/src/k8s.io/code-generator/cmd/import-boss/main.go +++ b/staging/src/k8s.io/code-generator/cmd/import-boss/main.go @@ -63,10 +63,11 @@ import ( "k8s.io/gengo/args" "k8s.io/gengo/examples/import-boss/generators" - "github.com/golang/glog" + "k8s.io/klog" ) func main() { + klog.InitFlags(nil) arguments := args.Default() // Override defaults. @@ -82,8 +83,8 @@ func main() { generators.DefaultNameSystem(), generators.Packages, ); err != nil { - glog.Errorf("Error: %v", err) + klog.Errorf("Error: %v", err) os.Exit(1) } - glog.V(2).Info("Completed successfully.") + klog.V(2).Info("Completed successfully.") } diff --git a/staging/src/k8s.io/code-generator/cmd/informer-gen/BUILD b/staging/src/k8s.io/code-generator/cmd/informer-gen/BUILD index d27d696d897..5a11f5433fd 100644 --- a/staging/src/k8s.io/code-generator/cmd/informer-gen/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/informer-gen/BUILD @@ -20,9 +20,9 @@ go_library( "//staging/src/k8s.io/code-generator/cmd/informer-gen/args:go_default_library", "//staging/src/k8s.io/code-generator/cmd/informer-gen/generators:go_default_library", "//staging/src/k8s.io/code-generator/pkg/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/BUILD b/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/BUILD index cead45f7aee..9902386d81b 100644 --- a/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/BUILD @@ -24,11 +24,11 @@ go_library( "//staging/src/k8s.io/code-generator/cmd/client-gen/generators/util:go_default_library", "//staging/src/k8s.io/code-generator/cmd/client-gen/types:go_default_library", "//staging/src/k8s.io/code-generator/cmd/informer-gen/args:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/generator:go_default_library", "//vendor/k8s.io/gengo/namer:go_default_library", "//vendor/k8s.io/gengo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/factory.go b/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/factory.go index 5c557db7393..62ae109a4a2 100644 --- a/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/factory.go +++ b/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/factory.go @@ -25,7 +25,7 @@ import ( "k8s.io/gengo/namer" "k8s.io/gengo/types" - "github.com/golang/glog" + "k8s.io/klog" ) // factoryGenerator produces a file of listers for a given GroupVersion and @@ -65,7 +65,7 @@ func (g *factoryGenerator) Imports(c *generator.Context) (imports []string) { func (g *factoryGenerator) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { sw := generator.NewSnippetWriter(w, c, "{{", "}}") - glog.V(5).Infof("processing type %v", t) + klog.V(5).Infof("processing type %v", t) gvInterfaces := make(map[string]*types.Type) gvNewFuncs := make(map[string]*types.Type) diff --git a/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/factoryinterface.go b/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/factoryinterface.go index 92cde1486c8..fc0668c5bed 100644 --- a/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/factoryinterface.go +++ b/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/factoryinterface.go @@ -23,7 +23,7 @@ import ( "k8s.io/gengo/namer" "k8s.io/gengo/types" - "github.com/golang/glog" + "k8s.io/klog" ) // factoryInterfaceGenerator produces a file of interfaces used to break a dependency cycle for @@ -60,7 +60,7 @@ func (g *factoryInterfaceGenerator) Imports(c *generator.Context) (imports []str func (g *factoryInterfaceGenerator) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { sw := generator.NewSnippetWriter(w, c, "{{", "}}") - glog.V(5).Infof("processing type %v", t) + klog.V(5).Infof("processing type %v", t) m := map[string]interface{}{ "cacheSharedIndexInformer": c.Universe.Type(cacheSharedIndexInformer), diff --git a/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/informer.go b/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/informer.go index 88cc08df52f..9204d6215ac 100644 --- a/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/informer.go +++ b/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/informer.go @@ -28,7 +28,7 @@ import ( "k8s.io/code-generator/cmd/client-gen/generators/util" clientgentypes "k8s.io/code-generator/cmd/client-gen/types" - "github.com/golang/glog" + "k8s.io/klog" ) // informerGenerator produces a file of listers for a given GroupVersion and @@ -66,7 +66,7 @@ func (g *informerGenerator) Imports(c *generator.Context) (imports []string) { func (g *informerGenerator) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { sw := generator.NewSnippetWriter(w, c, "$", "$") - glog.V(5).Infof("processing type %v", t) + klog.V(5).Infof("processing type %v", t) listerPackage := fmt.Sprintf("%s/%s/%s", g.listersPackage, g.groupPkgName, strings.ToLower(g.groupVersion.Version.NonEmpty())) clientSetInterface := c.Universe.Type(types.Name{Package: g.clientSetPackage, Name: "Interface"}) diff --git a/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/packages.go b/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/packages.go index 642f9a466ef..cfb91cebac6 100644 --- a/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/packages.go +++ b/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/packages.go @@ -22,11 +22,11 @@ import ( "path/filepath" "strings" - "github.com/golang/glog" "k8s.io/gengo/args" "k8s.io/gengo/generator" "k8s.io/gengo/namer" "k8s.io/gengo/types" + "k8s.io/klog" "k8s.io/code-generator/cmd/client-gen/generators/util" clientgentypes "k8s.io/code-generator/cmd/client-gen/types" @@ -102,12 +102,12 @@ func vendorless(p string) string { func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages { boilerplate, err := arguments.LoadGoBoilerplate() if err != nil { - glog.Fatalf("Failed loading boilerplate: %v", err) + klog.Fatalf("Failed loading boilerplate: %v", err) } customArgs, ok := arguments.CustomArgs.(*informergenargs.CustomArgs) if !ok { - glog.Fatalf("Wrong CustomArgs type: %T", arguments.CustomArgs) + klog.Fatalf("Wrong CustomArgs type: %T", arguments.CustomArgs) } internalVersionPackagePath := filepath.Join(arguments.OutputPackagePath) @@ -128,7 +128,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat objectMeta, internal, err := objectMetaForPackage(p) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } if objectMeta == nil { // no types in this package had genclient @@ -141,7 +141,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat if internal { lastSlash := strings.LastIndex(p.Path, "/") if lastSlash == -1 { - glog.Fatalf("error constructing internal group version for package %q", p.Path) + klog.Fatalf("error constructing internal group version for package %q", p.Path) } gv.Group = clientgentypes.Group(p.Path[lastSlash+1:]) targetGroupVersions = internalGroupVersions diff --git a/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/tags.go b/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/tags.go index afa28781520..d25d5b63049 100644 --- a/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/tags.go +++ b/staging/src/k8s.io/code-generator/cmd/informer-gen/generators/tags.go @@ -17,8 +17,8 @@ limitations under the License. package generators import ( - "github.com/golang/glog" "k8s.io/gengo/types" + "k8s.io/klog" ) // extractBoolTagOrDie gets the comment-tags for the key and asserts that, if @@ -27,7 +27,7 @@ import ( func extractBoolTagOrDie(key string, lines []string) bool { val, err := types.ExtractSingleBoolCommentTag("+", key, false, lines) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return val } diff --git a/staging/src/k8s.io/code-generator/cmd/informer-gen/main.go b/staging/src/k8s.io/code-generator/cmd/informer-gen/main.go index bfe826080cc..14f3e923e6c 100644 --- a/staging/src/k8s.io/code-generator/cmd/informer-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/informer-gen/main.go @@ -20,16 +20,17 @@ import ( "flag" "path/filepath" - "github.com/golang/glog" "github.com/spf13/pflag" "k8s.io/code-generator/cmd/informer-gen/generators" "k8s.io/code-generator/pkg/util" "k8s.io/gengo/args" + "k8s.io/klog" generatorargs "k8s.io/code-generator/cmd/informer-gen/args" ) func main() { + klog.InitFlags(nil) genericArgs, customArgs := generatorargs.NewDefaults() // Override defaults. @@ -47,7 +48,7 @@ func main() { pflag.Parse() if err := generatorargs.Validate(genericArgs); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } // Run it. @@ -56,7 +57,7 @@ func main() { generators.DefaultNameSystem(), generators.Packages, ); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } - glog.V(2).Info("Completed successfully.") + klog.V(2).Info("Completed successfully.") } diff --git a/staging/src/k8s.io/code-generator/cmd/lister-gen/BUILD b/staging/src/k8s.io/code-generator/cmd/lister-gen/BUILD index fec8ca914d2..e2286b8a49e 100644 --- a/staging/src/k8s.io/code-generator/cmd/lister-gen/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/lister-gen/BUILD @@ -20,9 +20,9 @@ go_library( "//staging/src/k8s.io/code-generator/cmd/lister-gen/args:go_default_library", "//staging/src/k8s.io/code-generator/cmd/lister-gen/generators:go_default_library", "//staging/src/k8s.io/code-generator/pkg/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/lister-gen/generators/BUILD b/staging/src/k8s.io/code-generator/cmd/lister-gen/generators/BUILD index 75212ca1da9..ec1a2ec626e 100644 --- a/staging/src/k8s.io/code-generator/cmd/lister-gen/generators/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/lister-gen/generators/BUILD @@ -17,11 +17,11 @@ go_library( deps = [ "//staging/src/k8s.io/code-generator/cmd/client-gen/generators/util:go_default_library", "//staging/src/k8s.io/code-generator/cmd/client-gen/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/generator:go_default_library", "//vendor/k8s.io/gengo/namer:go_default_library", "//vendor/k8s.io/gengo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/lister-gen/generators/lister.go b/staging/src/k8s.io/code-generator/cmd/lister-gen/generators/lister.go index cde6e2f770a..c8ed5ad4d3b 100644 --- a/staging/src/k8s.io/code-generator/cmd/lister-gen/generators/lister.go +++ b/staging/src/k8s.io/code-generator/cmd/lister-gen/generators/lister.go @@ -30,7 +30,7 @@ import ( "k8s.io/code-generator/cmd/client-gen/generators/util" clientgentypes "k8s.io/code-generator/cmd/client-gen/types" - "github.com/golang/glog" + "k8s.io/klog" ) // NameSystems returns the name system used by the generators in this package. @@ -66,7 +66,7 @@ func DefaultNameSystem() string { func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages { boilerplate, err := arguments.LoadGoBoilerplate() if err != nil { - glog.Fatalf("Failed loading boilerplate: %v", err) + klog.Fatalf("Failed loading boilerplate: %v", err) } var packageList generator.Packages @@ -75,7 +75,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat objectMeta, internal, err := objectMetaForPackage(p) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } if objectMeta == nil { // no types in this package had genclient @@ -88,7 +88,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat if internal { lastSlash := strings.LastIndex(p.Path, "/") if lastSlash == -1 { - glog.Fatalf("error constructing internal group version for package %q", p.Path) + klog.Fatalf("error constructing internal group version for package %q", p.Path) } gv.Group = clientgentypes.Group(p.Path[lastSlash+1:]) internalGVPkg = p.Path @@ -223,7 +223,7 @@ func (g *listerGenerator) Imports(c *generator.Context) (imports []string) { func (g *listerGenerator) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { sw := generator.NewSnippetWriter(w, c, "$", "$") - glog.V(5).Infof("processing type %v", t) + klog.V(5).Infof("processing type %v", t) m := map[string]interface{}{ "Resource": c.Universe.Function(types.Name{Package: t.Name.Package, Name: "Resource"}), "type": t, diff --git a/staging/src/k8s.io/code-generator/cmd/lister-gen/generators/tags.go b/staging/src/k8s.io/code-generator/cmd/lister-gen/generators/tags.go index afa28781520..d25d5b63049 100644 --- a/staging/src/k8s.io/code-generator/cmd/lister-gen/generators/tags.go +++ b/staging/src/k8s.io/code-generator/cmd/lister-gen/generators/tags.go @@ -17,8 +17,8 @@ limitations under the License. package generators import ( - "github.com/golang/glog" "k8s.io/gengo/types" + "k8s.io/klog" ) // extractBoolTagOrDie gets the comment-tags for the key and asserts that, if @@ -27,7 +27,7 @@ import ( func extractBoolTagOrDie(key string, lines []string) bool { val, err := types.ExtractSingleBoolCommentTag("+", key, false, lines) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return val } diff --git a/staging/src/k8s.io/code-generator/cmd/lister-gen/main.go b/staging/src/k8s.io/code-generator/cmd/lister-gen/main.go index d5ff8e46ee0..aca16b2bda3 100644 --- a/staging/src/k8s.io/code-generator/cmd/lister-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/lister-gen/main.go @@ -20,16 +20,17 @@ import ( "flag" "path/filepath" - "github.com/golang/glog" "github.com/spf13/pflag" "k8s.io/code-generator/cmd/lister-gen/generators" "k8s.io/code-generator/pkg/util" "k8s.io/gengo/args" + "k8s.io/klog" generatorargs "k8s.io/code-generator/cmd/lister-gen/args" ) func main() { + klog.InitFlags(nil) genericArgs, customArgs := generatorargs.NewDefaults() // Override defaults. @@ -44,7 +45,7 @@ func main() { pflag.Parse() if err := generatorargs.Validate(genericArgs); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } // Run it. @@ -53,7 +54,7 @@ func main() { generators.DefaultNameSystem(), generators.Packages, ); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } - glog.V(2).Info("Completed successfully.") + klog.V(2).Info("Completed successfully.") } diff --git a/staging/src/k8s.io/code-generator/cmd/openapi-gen/BUILD b/staging/src/k8s.io/code-generator/cmd/openapi-gen/BUILD deleted file mode 100644 index 6a8e5f3c115..00000000000 --- a/staging/src/k8s.io/code-generator/cmd/openapi-gen/BUILD +++ /dev/null @@ -1,43 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", -) - -go_binary( - name = "openapi-gen", - embed = [":go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = ["main.go"], - importmap = "k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/openapi-gen", - importpath = "k8s.io/code-generator/cmd/openapi-gen", - deps = [ - "//staging/src/k8s.io/code-generator/cmd/openapi-gen/args:go_default_library", - "//staging/src/k8s.io/code-generator/pkg/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/gengo/args:go_default_library", - "//vendor/k8s.io/kube-openapi/pkg/generators:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//staging/src/k8s.io/code-generator/cmd/openapi-gen/args:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/staging/src/k8s.io/code-generator/cmd/openapi-gen/README b/staging/src/k8s.io/code-generator/cmd/openapi-gen/README deleted file mode 100644 index e6dcc85d0d6..00000000000 --- a/staging/src/k8s.io/code-generator/cmd/openapi-gen/README +++ /dev/null @@ -1,13 +0,0 @@ -# Generate OpenAPI definitions - -- To generate definition for a specific type or package add "+k8s:openapi-gen=true" tag to the type/package comment lines. -- To exclude a type or a member from a tagged package/type, add "+k8s:openapi-gen=false" tag to the comment lines. - -# OpenAPI Extensions -OpenAPI spec can have extensions on types. To define one or more extensions on a type or its member -add "+k8s:openapi-gen=x-kubernetes-$NAME:$VALUE" to the comment lines before type/member. A type/member can -have multiple extensions. The rest of the line in the comment will be used as $VALUE so there is no need to -escape or quote the value string. Extensions can be use to pass more information to client generators or -documentation generators. For example a type my have a friendly name to be displayed in documentation or -being used in a client's fluent interface. - diff --git a/staging/src/k8s.io/code-generator/cmd/openapi-gen/args/args.go b/staging/src/k8s.io/code-generator/cmd/openapi-gen/args/args.go deleted file mode 100644 index f9bb17e1a56..00000000000 --- a/staging/src/k8s.io/code-generator/cmd/openapi-gen/args/args.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package args - -import ( - "fmt" - - "github.com/spf13/pflag" - "k8s.io/gengo/args" -) - -// CustomArgs is used by the gengo framework to pass args specific to this generator. -type CustomArgs struct{} - -// NewDefaults returns default arguments for the generator. -func NewDefaults() (*args.GeneratorArgs, *CustomArgs) { - genericArgs := args.Default().WithoutDefaultFlagParsing() - customArgs := &CustomArgs{} - genericArgs.CustomArgs = customArgs - genericArgs.OutputFileBaseName = "openapi_generated" - return genericArgs, customArgs -} - -// AddFlags add the generator flags to the flag set. -func (ca *CustomArgs) AddFlags(fs *pflag.FlagSet) {} - -// Validate checks the given arguments. -func Validate(genericArgs *args.GeneratorArgs) error { - _ = genericArgs.CustomArgs.(*CustomArgs) - - if len(genericArgs.OutputFileBaseName) == 0 { - return fmt.Errorf("output file base name cannot be empty") - } - if len(genericArgs.OutputPackagePath) == 0 { - return fmt.Errorf("output package cannot be empty") - } - - return nil -} diff --git a/staging/src/k8s.io/code-generator/cmd/openapi-gen/main.go b/staging/src/k8s.io/code-generator/cmd/openapi-gen/main.go deleted file mode 100644 index fbafc502577..00000000000 --- a/staging/src/k8s.io/code-generator/cmd/openapi-gen/main.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This package generates openAPI definition file to be used in open API spec generation on API servers. To generate -// definition for a specific type or package add "+k8s:openapi-gen=true" tag to the type/package comment lines. To -// exclude a type from a tagged package, add "+k8s:openapi-gen=false" tag to the type comment lines. -package main - -import ( - "flag" - "path/filepath" - - "github.com/golang/glog" - "github.com/spf13/pflag" - "k8s.io/gengo/args" - "k8s.io/kube-openapi/pkg/generators" - - generatorargs "k8s.io/code-generator/cmd/openapi-gen/args" - "k8s.io/code-generator/pkg/util" -) - -func main() { - genericArgs, customArgs := generatorargs.NewDefaults() - - // Override defaults. - // TODO: move this out of openapi-gen - genericArgs.GoHeaderFilePath = filepath.Join(args.DefaultSourceTree(), util.BoilerplatePath()) - - genericArgs.AddFlags(pflag.CommandLine) - customArgs.AddFlags(pflag.CommandLine) - flag.Set("logtostderr", "true") - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - pflag.Parse() - - if err := generatorargs.Validate(genericArgs); err != nil { - glog.Fatalf("Error: %v", err) - } - - // Run it. - if err := genericArgs.Execute( - generators.NameSystems(), - generators.DefaultNameSystem(), - generators.Packages, - ); err != nil { - glog.Fatalf("Error: %v", err) - } - glog.V(2).Info("Completed successfully.") -} diff --git a/staging/src/k8s.io/code-generator/cmd/register-gen/BUILD b/staging/src/k8s.io/code-generator/cmd/register-gen/BUILD index bdf68063d9f..a784d2a4bb1 100644 --- a/staging/src/k8s.io/code-generator/cmd/register-gen/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/register-gen/BUILD @@ -10,9 +10,9 @@ go_library( "//staging/src/k8s.io/code-generator/cmd/register-gen/args:go_default_library", "//staging/src/k8s.io/code-generator/cmd/register-gen/generators:go_default_library", "//staging/src/k8s.io/code-generator/pkg/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/register-gen/generators/BUILD b/staging/src/k8s.io/code-generator/cmd/register-gen/generators/BUILD index 810ae444f14..f88391aa5ce 100644 --- a/staging/src/k8s.io/code-generator/cmd/register-gen/generators/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/register-gen/generators/BUILD @@ -11,11 +11,11 @@ go_library( visibility = ["//visibility:public"], deps = [ "//staging/src/k8s.io/code-generator/cmd/client-gen/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/generator:go_default_library", "//vendor/k8s.io/gengo/namer:go_default_library", "//vendor/k8s.io/gengo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/register-gen/generators/packages.go b/staging/src/k8s.io/code-generator/cmd/register-gen/generators/packages.go index ca13ca85798..5186e421f2e 100644 --- a/staging/src/k8s.io/code-generator/cmd/register-gen/generators/packages.go +++ b/staging/src/k8s.io/code-generator/cmd/register-gen/generators/packages.go @@ -22,7 +22,7 @@ import ( "path" "strings" - "github.com/golang/glog" + "k8s.io/klog" clientgentypes "k8s.io/code-generator/cmd/client-gen/types" "k8s.io/gengo/args" @@ -46,7 +46,7 @@ func DefaultNameSystem() string { func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages { boilerplate, err := arguments.LoadGoBoilerplate() if err != nil { - glog.Fatalf("Failed loading boilerplate: %v", err) + klog.Fatalf("Failed loading boilerplate: %v", err) } packages := generator.Packages{} @@ -54,27 +54,27 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat pkg := context.Universe.Package(inputDir) internal, err := isInternal(pkg) if err != nil { - glog.V(5).Infof("skipping the generation of %s file, due to err %v", arguments.OutputFileBaseName, err) + klog.V(5).Infof("skipping the generation of %s file, due to err %v", arguments.OutputFileBaseName, err) continue } if internal { - glog.V(5).Infof("skipping the generation of %s file because %s package contains internal types, note that internal types don't have \"json\" tags", arguments.OutputFileBaseName, pkg.Name) + klog.V(5).Infof("skipping the generation of %s file because %s package contains internal types, note that internal types don't have \"json\" tags", arguments.OutputFileBaseName, pkg.Name) continue } registerFileName := "register.go" searchPath := path.Join(args.DefaultSourceTree(), inputDir, registerFileName) if _, err := os.Stat(path.Join(searchPath)); err == nil { - glog.V(5).Infof("skipping the generation of %s file because %s already exists in the path %s", arguments.OutputFileBaseName, registerFileName, searchPath) + klog.V(5).Infof("skipping the generation of %s file because %s already exists in the path %s", arguments.OutputFileBaseName, registerFileName, searchPath) continue } else if err != nil && !os.IsNotExist(err) { - glog.Fatalf("an error %v has occurred while checking if %s exists", err, registerFileName) + klog.Fatalf("an error %v has occurred while checking if %s exists", err, registerFileName) } gv := clientgentypes.GroupVersion{} { pathParts := strings.Split(pkg.Path, "/") if len(pathParts) < 2 { - glog.Errorf("the path of the package must contain the group name and the version, path = %s", pkg.Path) + klog.Errorf("the path of the package must contain the group name and the version, path = %s", pkg.Path) continue } gv.Group = clientgentypes.Group(pathParts[len(pathParts)-2]) @@ -84,14 +84,14 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat // extract the fully qualified API group name from it and overwrite the group inferred from the package path if override := types.ExtractCommentTags("+", pkg.DocComments)["groupName"]; override != nil { groupName := override[0] - glog.V(5).Infof("overriding the group name with = %s", groupName) + klog.V(5).Infof("overriding the group name with = %s", groupName) gv.Group = clientgentypes.Group(groupName) } } typesToRegister := []*types.Type{} for _, t := range pkg.Types { - glog.V(5).Infof("considering type = %s", t.Name.String()) + klog.V(5).Infof("considering type = %s", t.Name.String()) for _, typeMember := range t.Members { if typeMember.Name == "TypeMeta" && typeMember.Embedded == true { typesToRegister = append(typesToRegister, t) diff --git a/staging/src/k8s.io/code-generator/cmd/register-gen/main.go b/staging/src/k8s.io/code-generator/cmd/register-gen/main.go index db02a4af4b5..30a175d8d62 100644 --- a/staging/src/k8s.io/code-generator/cmd/register-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/register-gen/main.go @@ -20,8 +20,8 @@ import ( "flag" "path/filepath" - "github.com/golang/glog" "github.com/spf13/pflag" + "k8s.io/klog" generatorargs "k8s.io/code-generator/cmd/register-gen/args" "k8s.io/code-generator/cmd/register-gen/generators" @@ -30,6 +30,7 @@ import ( ) func main() { + klog.InitFlags(nil) genericArgs := generatorargs.NewDefaults() genericArgs.GoHeaderFilePath = filepath.Join(args.DefaultSourceTree(), util.BoilerplatePath()) genericArgs.AddFlags(pflag.CommandLine) @@ -38,7 +39,7 @@ func main() { pflag.Parse() if err := generatorargs.Validate(genericArgs); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } if err := genericArgs.Execute( @@ -46,7 +47,7 @@ func main() { generators.DefaultNameSystem(), generators.Packages, ); err != nil { - glog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err) } - glog.V(2).Info("Completed successfully.") + klog.V(2).Info("Completed successfully.") } diff --git a/staging/src/k8s.io/code-generator/cmd/set-gen/BUILD b/staging/src/k8s.io/code-generator/cmd/set-gen/BUILD index 188f8e1ffe1..2b270296406 100644 --- a/staging/src/k8s.io/code-generator/cmd/set-gen/BUILD +++ b/staging/src/k8s.io/code-generator/cmd/set-gen/BUILD @@ -22,9 +22,9 @@ go_library( importpath = "k8s.io/code-generator/cmd/set-gen", deps = [ "//staging/src/k8s.io/code-generator/pkg/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/examples/set-gen/generators:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/code-generator/cmd/set-gen/main.go b/staging/src/k8s.io/code-generator/cmd/set-gen/main.go index cf8f01d89e5..45694d4f330 100644 --- a/staging/src/k8s.io/code-generator/cmd/set-gen/main.go +++ b/staging/src/k8s.io/code-generator/cmd/set-gen/main.go @@ -32,10 +32,11 @@ import ( "k8s.io/gengo/args" "k8s.io/gengo/examples/set-gen/generators" - "github.com/golang/glog" + "k8s.io/klog" ) func main() { + klog.InitFlags(nil) arguments := args.Default() // Override defaults. @@ -48,8 +49,8 @@ func main() { generators.DefaultNameSystem(), generators.Packages, ); err != nil { - glog.Errorf("Error: %v", err) + klog.Errorf("Error: %v", err) os.Exit(1) } - glog.V(2).Info("Completed successfully.") + klog.V(2).Info("Completed successfully.") } diff --git a/staging/src/k8s.io/csi-api/Godeps/Godeps.json b/staging/src/k8s.io/csi-api/Godeps/Godeps.json index 091b3c0e16e..6949a7fa26c 100644 --- a/staging/src/k8s.io/csi-api/Godeps/Godeps.json +++ b/staging/src/k8s.io/csi-api/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/csi-api", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -14,10 +14,6 @@ "ImportPath": "github.com/evanphx/json-patch", "Rev": "36442dbdb585210f8d5a1b45e67aa323c197d5c4" }, - { - "ImportPath": "github.com/ghodss/yaml", - "Rev": "c7ce16629ff4cd059ed96ed06419dd3856fd3577" - }, { "ImportPath": "github.com/gogo/protobuf/proto", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" @@ -26,10 +22,6 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/golang/protobuf/proto", "Rev": "b4deda0973fb4c70b50d226b1af49f3da59f5265" @@ -108,23 +100,23 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/oauth2", @@ -298,18 +290,6 @@ "ImportPath": "k8s.io/api/storage/v1beta1", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - { - "ImportPath": "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, { "ImportPath": "k8s.io/apimachinery/pkg/api/errors", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -566,9 +546,17 @@ "ImportPath": "k8s.io/client-go/util/retry", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" + }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" } ] } diff --git a/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/README.md b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/README.md new file mode 100644 index 00000000000..79a7ec17eed --- /dev/null +++ b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/README.md @@ -0,0 +1,52 @@ +# CSINodeInfo and CSIDriverInfo Usage and Lifecycle + +CSINodeInfo is an API object representing CSI Information at the Node level. +CSINodeInfo contains a Spec and a Status, each containing Drivers represent the +driver spec and status respectively. The CSIDriverInfoSpec represents the +specification of the driver and is generally not changed, whereas the +CSIDriverInfoStatus represents the current status of the driver and can be +updated and reacted to by various components. + +## Who creates it and when + +CSINodeInfo is created by Kubelet when the first CSI Driver is installed to the +cluster and it is registered through the Kubelet device registration mechanism. + +## Who updates CSIDriverInfo Spec and when + +The CSIDriverInfoSpec for a driver is created upon installation of the CSI +Driver to the cluster and it is registered through the Kubelet device +registration mechanism. The spec is populated with information about the driver +through the nodeinfomanager (inside Kubelet) and will remain unchanged from then +on. + +## Who updates Status and when + +The CSIDriverInfoStatus for the driver is created upon installation of the CSI +Driver to the cluster (the same time as the spec) and it is registered through +the Kubelet device registration mechanism. The Status contains information about +installation and the required Volume Plugin Mechanism of the driver. When the +driver is installed/uninstalled through the Kubelet device registration +mechanism the Available flag is flipped from true/false respectively. The +migration status will also be updated when the flags for migration are set to +true/false on the Kubelet for that Driver on that node. + +## Consumers of Status and Spec + +Currently the only consumer of CSINodeInfo/CSIDriverInfo is the +csi-external-provisioner. In the future, the Attach Detach Controller (ADC) will +need to read this object to determine migration status on a per driver per node +basis. The creation of the CSINodeInfo object could possibly race with the +Attach/Detach controller as for CSI Migration the controller depend on the +existence of the API object for the driver but it will not have been created +yet. The ADC is expected to fail (and retry with exponential backoff) the +operation if it is expecting the object and it has not yet been created. + +## Creation of CSINodeInfo object on Kubelet startup + +For CSI Migration Alpha we expect any user who turns on the feature has both +Kubelet and ADC at a version where the CSINodeInfo's are being created on +Kubelet startup. We will not promote the feature to Beta (on by default) until +the CSINodeInfo's are being created on Kubelet startup for a 2 version skew to +prevent the case where the CSINodeInfo does not exist when the ADC depends on +it. This prevents the race described above becoming a permanent bad state. diff --git a/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/types.go b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/types.go index 635f22af8dd..6232b6b4d8e 100644 --- a/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/types.go +++ b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/types.go @@ -18,6 +18,16 @@ package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +// VolumePluginMechanism is the type of mechanism components should use for volume operations +type VolumePluginMechanism string + +const ( + // VolumePluginMechanismInTree means components should use the in-tree plugin for volume operations + VolumePluginMechanismInTree VolumePluginMechanism = "in-tree" + // VolumePluginMechanismCSI means components should use the CSI Driver for volume operations + VolumePluginMechanismCSI VolumePluginMechanism = "csi" +) + // +genclient // +genclient:nonNamespaced // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -91,18 +101,56 @@ type CSINodeInfo struct { // metadata.name must be the Kubernetes node name. metav1.ObjectMeta `json:"metadata,omitempty"` - // List of CSI drivers running on the node and their properties. - // +patchMergeKey=driver - // +patchStrategy=merge - CSIDrivers []CSIDriverInfo `json:"csiDrivers" patchStrategy:"merge" patchMergeKey:"driver"` + // spec is the specification of CSINodeInfo + Spec CSINodeInfoSpec `json:"spec"` + + // status is the current status of CSINodeInfo + Status CSINodeInfoStatus `json:"status"` } -// CSIDriverInfo contains information about one CSI driver installed on a node. -type CSIDriverInfo struct { - // driver is the name of the CSI driver that this object refers to. +// CSINodeInfoSpec holds information about the specification of all CSI drivers installed on a node +type CSINodeInfoSpec struct { + // drivers is a list of specifications of CSIDriverInfo + // +patchMergeKey=name + // +patchStrategy=merge + Drivers []CSIDriverInfoSpec `json:"drivers" patchStrategy:"merge" patchMergeKey:"name"` +} + +// CSINodeInfoStatus holds information about the status of all CSI drivers installed on a node +type CSINodeInfoStatus struct { + // drivers is a list of the statuses of CSIDriverInfo + // +patchMergeKey=name + // +patchStrategy=merge + Drivers []CSIDriverInfoStatus `json:"drivers" patchStrategy:"merge" patchMergeKey:"name"` +} + +// CSIDriverInfoStatus holds information about the status of one CSI driver installed on a node +type CSIDriverInfoStatus struct { + // name is the name of the CSI driver that this object refers to. // This MUST be the same name returned by the CSI GetPluginName() call for // that driver. - Driver string `json:"driver"` + Name string `json:"name"` + + // available is a boolean representing whether the driver has been installed + // on this node or not. + Available bool `json:"available"` + + // volumePluginMechanism announces what mechanism underlies volume plugins. + // It is set by Kubelet. It is used by the attach/detach controller, which + // needs to know how to perform attachments. The allowed values are: + // * "in-tree": the volume operation (e.g., attach/detach) ought to be + // directly performed by the attach/detach controller. + // * "csi-plugin": the attach/detach controller ought to request + // the csi plugin to perform the volume operation rather than perform it directly. + VolumePluginMechanism VolumePluginMechanism `json:"volumePluginMechanism"` +} + +// CSIDriverInfoSpec holds information about the specification of one CSI driver installed on a node +type CSIDriverInfoSpec struct { + // name is the name of the CSI driver that this object refers to. + // This MUST be the same name returned by the CSI GetPluginName() call for + // that driver. + Name string `json:"name"` // nodeID of the node from the driver point of view. // This field enables Kubernetes to communicate with storage systems that do diff --git a/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/zz_generated.deepcopy.go b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/zz_generated.deepcopy.go index aa9ccbbaae8..915ef995a01 100644 --- a/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/zz_generated.deepcopy.go @@ -52,7 +52,7 @@ func (in *CSIDriver) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CSIDriverInfo) DeepCopyInto(out *CSIDriverInfo) { +func (in *CSIDriverInfoSpec) DeepCopyInto(out *CSIDriverInfoSpec) { *out = *in if in.TopologyKeys != nil { in, out := &in.TopologyKeys, &out.TopologyKeys @@ -62,12 +62,28 @@ func (in *CSIDriverInfo) DeepCopyInto(out *CSIDriverInfo) { return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverInfo. -func (in *CSIDriverInfo) DeepCopy() *CSIDriverInfo { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverInfoSpec. +func (in *CSIDriverInfoSpec) DeepCopy() *CSIDriverInfoSpec { if in == nil { return nil } - out := new(CSIDriverInfo) + out := new(CSIDriverInfoSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSIDriverInfoStatus) DeepCopyInto(out *CSIDriverInfoStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverInfoStatus. +func (in *CSIDriverInfoStatus) DeepCopy() *CSIDriverInfoStatus { + if in == nil { + return nil + } + out := new(CSIDriverInfoStatus) in.DeepCopyInto(out) return out } @@ -136,13 +152,8 @@ func (in *CSINodeInfo) DeepCopyInto(out *CSINodeInfo) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - if in.CSIDrivers != nil { - in, out := &in.CSIDrivers, &out.CSIDrivers - *out = make([]CSIDriverInfo, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) return } @@ -196,3 +207,47 @@ func (in *CSINodeInfoList) DeepCopyObject() runtime.Object { } return nil } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSINodeInfoSpec) DeepCopyInto(out *CSINodeInfoSpec) { + *out = *in + if in.Drivers != nil { + in, out := &in.Drivers, &out.Drivers + *out = make([]CSIDriverInfoSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSINodeInfoSpec. +func (in *CSINodeInfoSpec) DeepCopy() *CSINodeInfoSpec { + if in == nil { + return nil + } + out := new(CSINodeInfoSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSINodeInfoStatus) DeepCopyInto(out *CSINodeInfoStatus) { + *out = *in + if in.Drivers != nil { + in, out := &in.Drivers, &out.Drivers + *out = make([]CSIDriverInfoStatus, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSINodeInfoStatus. +func (in *CSINodeInfoStatus) DeepCopy() *CSINodeInfoStatus { + if in == nil { + return nil + } + out := new(CSINodeInfoStatus) + in.DeepCopyInto(out) + return out +} diff --git a/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/csidriver.go b/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/csidriver.go index ffcee6193f1..72c93f6fadc 100644 --- a/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/csidriver.go +++ b/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/csidriver.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -72,10 +74,15 @@ func (c *cSIDrivers) Get(name string, options v1.GetOptions) (result *v1alpha1.C // List takes label and field selectors, and returns the list of CSIDrivers that match those selectors. func (c *cSIDrivers) List(opts v1.ListOptions) (result *v1alpha1.CSIDriverList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.CSIDriverList{} err = c.client.Get(). Resource("csidrivers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *cSIDrivers) List(opts v1.ListOptions) (result *v1alpha1.CSIDriverList, // Watch returns a watch.Interface that watches the requested cSIDrivers. func (c *cSIDrivers) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("csidrivers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *cSIDrivers) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *cSIDrivers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("csidrivers"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/csinodeinfo.go b/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/csinodeinfo.go index 8580127c02f..3e5f73b0462 100644 --- a/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/csinodeinfo.go +++ b/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/csinodeinfo.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -37,6 +39,7 @@ type CSINodeInfosGetter interface { type CSINodeInfoInterface interface { Create(*v1alpha1.CSINodeInfo) (*v1alpha1.CSINodeInfo, error) Update(*v1alpha1.CSINodeInfo) (*v1alpha1.CSINodeInfo, error) + UpdateStatus(*v1alpha1.CSINodeInfo) (*v1alpha1.CSINodeInfo, error) Delete(name string, options *v1.DeleteOptions) error DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error Get(name string, options v1.GetOptions) (*v1alpha1.CSINodeInfo, error) @@ -72,10 +75,15 @@ func (c *cSINodeInfos) Get(name string, options v1.GetOptions) (result *v1alpha1 // List takes label and field selectors, and returns the list of CSINodeInfos that match those selectors. func (c *cSINodeInfos) List(opts v1.ListOptions) (result *v1alpha1.CSINodeInfoList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.CSINodeInfoList{} err = c.client.Get(). Resource("csinodeinfos"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +91,15 @@ func (c *cSINodeInfos) List(opts v1.ListOptions) (result *v1alpha1.CSINodeInfoLi // Watch returns a watch.Interface that watches the requested cSINodeInfos. func (c *cSINodeInfos) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("csinodeinfos"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -113,6 +126,21 @@ func (c *cSINodeInfos) Update(cSINodeInfo *v1alpha1.CSINodeInfo) (result *v1alph return } +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + +func (c *cSINodeInfos) UpdateStatus(cSINodeInfo *v1alpha1.CSINodeInfo) (result *v1alpha1.CSINodeInfo, err error) { + result = &v1alpha1.CSINodeInfo{} + err = c.client.Put(). + Resource("csinodeinfos"). + Name(cSINodeInfo.Name). + SubResource("status"). + Body(cSINodeInfo). + Do(). + Into(result) + return +} + // Delete takes name of the cSINodeInfo and deletes it. Returns an error if one occurs. func (c *cSINodeInfos) Delete(name string, options *v1.DeleteOptions) error { return c.client.Delete(). @@ -125,9 +153,14 @@ func (c *cSINodeInfos) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *cSINodeInfos) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("csinodeinfos"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/fake/fake_csinodeinfo.go b/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/fake/fake_csinodeinfo.go index babda7c171c..6e44b9d3850 100644 --- a/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/fake/fake_csinodeinfo.go +++ b/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/fake/fake_csinodeinfo.go @@ -94,6 +94,17 @@ func (c *FakeCSINodeInfos) Update(cSINodeInfo *v1alpha1.CSINodeInfo) (result *v1 return obj.(*v1alpha1.CSINodeInfo), err } +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeCSINodeInfos) UpdateStatus(cSINodeInfo *v1alpha1.CSINodeInfo) (*v1alpha1.CSINodeInfo, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(csinodeinfosResource, "status", cSINodeInfo), &v1alpha1.CSINodeInfo{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.CSINodeInfo), err +} + // Delete takes name of the cSINodeInfo and deletes it. Returns an error if one occurs. func (c *FakeCSINodeInfos) Delete(name string, options *v1.DeleteOptions) error { _, err := c.Fake. diff --git a/staging/src/k8s.io/csi-api/pkg/crd/BUILD b/staging/src/k8s.io/csi-api/pkg/crd/BUILD index 3a9a64a5c88..71a403cd573 100644 --- a/staging/src/k8s.io/csi-api/pkg/crd/BUILD +++ b/staging/src/k8s.io/csi-api/pkg/crd/BUILD @@ -6,25 +6,17 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/csi-api/pkg/crd", importpath = "k8s.io/csi-api/pkg/crd", visibility = ["//visibility:public"], - deps = [ - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", - ], ) go_test( name = "go_default_test", srcs = ["crd_test.go"], - data = glob(["testdata/**"]), - embed = [":go_default_library"], - deps = [ - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", + data = [ + "//cluster/addons:addon-srcs", + "//staging/src/k8s.io/csi-api/pkg/crd:manifests", ], + embed = [":go_default_library"], + deps = ["//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library"], ) filegroup( @@ -34,6 +26,14 @@ filegroup( visibility = ["//visibility:private"], ) +filegroup( + name = "csi-manifests", + srcs = [ + "//staging/src/k8s.io/csi-api/pkg/crd:manifests", + ], + visibility = ["//visibility:public"], +) + filegroup( name = "all-srcs", srcs = [":package-srcs"], diff --git a/staging/src/k8s.io/csi-api/pkg/crd/crd.go b/staging/src/k8s.io/csi-api/pkg/crd/crd.go index 4a7936e4140..02e5b152811 100644 --- a/staging/src/k8s.io/csi-api/pkg/crd/crd.go +++ b/staging/src/k8s.io/csi-api/pkg/crd/crd.go @@ -14,106 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package crd is only for running tests to verify the manifest files +// in this package and the addons are in sync. +// The manifest file is currently manually generated, in the future, we +// should invest in tooling that will automatically generate the CRD +// manifest from the CR schema. package crd - -import ( - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - csiapiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" - "reflect" -) - -// NOTE: the CRD functions here and the associated unit tests are non-ideal temporary measures in -// release 1.12 in order to aid manual CRD installation. This installation will be automated in -// subsequent releases and as a result this package will be removed. - -// CSIDriverCRD returns the CustomResourceDefinition for CSIDriver object. -func CSIDriverCRD() *apiextensionsv1beta1.CustomResourceDefinition { - return &apiextensionsv1beta1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: csiapiv1alpha1.CsiDriverResourcePlural + "." + csiapiv1alpha1.GroupName, - }, - Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: csiapiv1alpha1.GroupName, - Version: csiapiv1alpha1.SchemeGroupVersion.Version, - Scope: apiextensionsv1beta1.ClusterScoped, - Validation: &apiextensionsv1beta1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "spec": { - Description: "Specification of the CSI Driver.", - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "attachRequired": { - Description: "Indicates this CSI volume driver requires an attach operation," + - " and that Kubernetes should call attach and wait for any attach operation to" + - " complete before proceeding to mount.", - Type: "boolean", - }, - "podInfoOnMountVersion": { - Description: "Indicates this CSI volume driver requires additional pod" + - " information (like podName, podUID, etc.) during mount operations.", - Type: "string", - }, - }, - }, - }, - }, - }, - Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ - Plural: csiapiv1alpha1.CsiDriverResourcePlural, - Kind: reflect.TypeOf(csiapiv1alpha1.CSIDriver{}).Name(), - }, - }, - } -} - -// CSINodeInfoCRD returns the CustomResourceDefinition for CSINodeInfo object. -func CSINodeInfoCRD() *apiextensionsv1beta1.CustomResourceDefinition { - return &apiextensionsv1beta1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: csiapiv1alpha1.CsiNodeInfoResourcePlural + "." + csiapiv1alpha1.GroupName, - }, - Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: csiapiv1alpha1.GroupName, - Version: csiapiv1alpha1.SchemeGroupVersion.Version, - Scope: apiextensionsv1beta1.ClusterScoped, - Validation: &apiextensionsv1beta1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "csiDrivers": { - Description: "List of CSI drivers running on the node and their properties.", - Type: "array", - Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextensionsv1beta1.JSONSchemaProps{ - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "driver": { - Description: "The CSI driver that this object refers to.", - Type: "string", - }, - "nodeID": { - Description: "The node from the driver point of view.", - Type: "string", - }, - "topologyKeys": { - Description: "List of keys supported by the driver.", - Type: "array", - Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextensionsv1beta1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ - Plural: csiapiv1alpha1.CsiNodeInfoResourcePlural, - Kind: reflect.TypeOf(csiapiv1alpha1.CSINodeInfo{}).Name(), - }, - }, - } -} diff --git a/staging/src/k8s.io/csi-api/pkg/crd/crd_test.go b/staging/src/k8s.io/csi-api/pkg/crd/crd_test.go index 369a3b458b5..2bb155baffd 100644 --- a/staging/src/k8s.io/csi-api/pkg/crd/crd_test.go +++ b/staging/src/k8s.io/csi-api/pkg/crd/crd_test.go @@ -14,56 +14,69 @@ See the License for the specific language governing permissions and limitations under the License. */ +// These tests verify the manifest files in this package and the +// addons directory are in sync. package crd_test import ( + "io/ioutil" + "os" "path/filepath" "testing" - "github.com/ghodss/yaml" - "io/ioutil" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextensionsscheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/diff" - "k8s.io/csi-api/pkg/crd" - "os" ) func TestBootstrapCRDs(t *testing.T) { - testObjects(t, crd.CSIDriverCRD(), "csidriver.yaml") - testObjects(t, crd.CSINodeInfoCRD(), "csinodeinfo.yaml") + verifyCopiesAreInSync( + t, + "csidriver.yaml", /* filename */ + "manifests", /* sourceOfTruthDir */ + []string{"../../../../../../cluster/addons/storage-crds"}, /* copyDirs */ + ) + verifyCopiesAreInSync( + t, + "csinodeinfo.yaml", /* filename */ + "manifests", /* sourceOfTruthDir */ + []string{"../../../../../../cluster/addons/storage-crds"}, /* copyDirs */ + ) } -func testObjects(t *testing.T, crd *apiextensionsv1beta1.CustomResourceDefinition, fixtureFilename string) { - filename := filepath.Join("testdata", fixtureFilename) - expectedYAML, err := ioutil.ReadFile(filename) +// verifyCopiesAreInSync fails if any copies are different from source of truth. +func verifyCopiesAreInSync(t *testing.T, filename string, sourceOfTruthDir string, copyDirs []string) { + sourceOfTruthFilename := filepath.Join(sourceOfTruthDir, filename) + + if len(copyDirs) <= 0 { + t.Fatalf("copyDirs is empty. There are no copies to validate.") + } + + expectedYAML, err := ioutil.ReadFile(sourceOfTruthFilename) if err != nil { t.Fatal(err) } - jsonData, err := runtime.Encode(apiextensionsscheme.Codecs.LegacyCodec(apiextensionsv1beta1.SchemeGroupVersion), crd) - if err != nil { - t.Fatal(err) - } - yamlData, err := yaml.JSONToYAML(jsonData) - if err != nil { - t.Fatal(err) - } - if string(yamlData) != string(expectedYAML) { - t.Errorf("Bootstrap CRD data does not match the test fixture in %s", filename) + for _, copyDir := range copyDirs { + copyFilename := filepath.Join(copyDir, filename) + actualYAML, err := ioutil.ReadFile(copyFilename) + if err != nil { + t.Fatal(err) + } - const updateEnvVar = "UPDATE_CSI_CRD_FIXTURE_DATA" - if os.Getenv(updateEnvVar) == "true" { - if err := ioutil.WriteFile(filename, []byte(yamlData), os.FileMode(0755)); err == nil { - t.Logf("Updated data in %s", filename) - t.Logf("Verify the diff, commit changes, and rerun the tests") + if string(actualYAML) != string(expectedYAML) { + t.Errorf("Data in %q does not match source of truth in %q.", copyFilename, sourceOfTruthFilename) + + const updateEnvVar = "UPDATE_CSI_CRD_FIXTURE_DATA" + if os.Getenv(updateEnvVar) == "true" { + if err := ioutil.WriteFile(copyFilename, []byte(expectedYAML), os.FileMode(0755)); err == nil { + t.Logf("Updated data in %s", copyFilename) + t.Logf("Verify the diff, commit changes, and rerun the tests") + } else { + t.Logf("Could not update data in %s: %v", copyFilename, err) + } } else { - t.Logf("Could not update data in %s: %v", filename, err) + t.Logf("Diff between source of truth data and copy data in %s:\n-------------\n%s", copyFilename, diff.StringDiff(string(actualYAML), string(expectedYAML))) + t.Logf("If the change is expected, re-run with %s=true to update the copy data", updateEnvVar) } - } else { - t.Logf("Diff between data in code and fixture data in %s:\n-------------\n%s", filename, diff.StringDiff(string(yamlData), string(expectedYAML))) - t.Logf("If the change is expected, re-run with %s=true to update the fixtures", updateEnvVar) } } } diff --git a/staging/src/k8s.io/csi-api/pkg/crd/manifests/csidriver.yaml b/staging/src/k8s.io/csi-api/pkg/crd/manifests/csidriver.yaml new file mode 100644 index 00000000000..54416b24f93 --- /dev/null +++ b/staging/src/k8s.io/csi-api/pkg/crd/manifests/csidriver.yaml @@ -0,0 +1,28 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: csidrivers.csi.storage.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + group: csi.storage.k8s.io + names: + kind: CSIDriver + plural: csidrivers + scope: Cluster + validation: + openAPIV3Schema: + properties: + spec: + description: Specification of the CSI Driver. + properties: + attachRequired: + description: Indicates this CSI volume driver requires an attach operation, + and that Kubernetes should call attach and wait for any attach operation + to complete before proceeding to mount. + type: boolean + podInfoOnMountVersion: + description: Indicates this CSI volume driver requires additional pod + information (like podName, podUID, etc.) during mount operations. + type: string + version: v1alpha1 diff --git a/staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml b/staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml new file mode 100644 index 00000000000..7b12ba57666 --- /dev/null +++ b/staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml @@ -0,0 +1,54 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: csinodeinfos.csi.storage.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + group: csi.storage.k8s.io + names: + kind: CSINodeInfo + plural: csinodeinfos + scope: Cluster + validation: + openAPIV3Schema: + properties: + spec: + description: Specification of CSINodeInfo + properties: + drivers: + description: List of CSI drivers running on the node and their specs. + type: array + items: + properties: + name: + description: The CSI driver that this object refers to. + type: string + nodeID: + description: The node from the driver point of view. + type: string + topologyKeys: + description: List of keys supported by the driver. + items: + type: string + type: array + status: + description: Status of CSINodeInfo + properties: + drivers: + description: List of CSI drivers running on the node and their statuses. + type: array + items: + properties: + name: + description: The CSI driver that this object refers to. + type: string + available: + description: Whether the CSI driver is installed. + type: boolean + volumePluginMechanism: + description: Indicates to external components the required mechanism + to use for any in-tree plugins replaced by this driver. + pattern: in-tree|csi + type: string + version: v1alpha1 diff --git a/staging/src/k8s.io/csi-api/pkg/crd/testdata/csinodeinfo.yaml b/staging/src/k8s.io/csi-api/pkg/crd/testdata/csinodeinfo.yaml deleted file mode 100644 index 6ea408d5e77..00000000000 --- a/staging/src/k8s.io/csi-api/pkg/crd/testdata/csinodeinfo.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - creationTimestamp: null - name: csinodeinfos.csi.storage.k8s.io -spec: - group: csi.storage.k8s.io - names: - kind: CSINodeInfo - plural: csinodeinfos - scope: Cluster - validation: - openAPIV3Schema: - properties: - csiDrivers: - description: List of CSI drivers running on the node and their properties. - items: - properties: - driver: - description: The CSI driver that this object refers to. - type: string - nodeID: - description: The node from the driver point of view. - type: string - topologyKeys: - description: List of keys supported by the driver. - items: - type: string - type: array - type: array - version: v1alpha1 -status: - acceptedNames: - kind: "" - plural: "" - conditions: null - storedVersions: null diff --git a/staging/src/k8s.io/kube-aggregator/BUILD b/staging/src/k8s.io/kube-aggregator/BUILD index 95cbea4fb90..99bf2540e29 100644 --- a/staging/src/k8s.io/kube-aggregator/BUILD +++ b/staging/src/k8s.io/kube-aggregator/BUILD @@ -28,7 +28,7 @@ go_library( "//staging/src/k8s.io/kube-aggregator/pkg/client/listers/apiregistration/internalversion:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1beta1:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/cmd/server:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json b/staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json index aa618df1cde..a617489e581 100644 --- a/staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json +++ b/staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/kube-aggregator", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -110,10 +110,6 @@ "ImportPath": "github.com/evanphx/json-patch", "Rev": "36442dbdb585210f8d5a1b45e67aa323c197d5c4" }, - { - "ImportPath": "github.com/ghodss/yaml", - "Rev": "c7ce16629ff4cd059ed96ed06419dd3856fd3577" - }, { "ImportPath": "github.com/go-openapi/jsonpointer", "Rev": "ef5f0afec364d3b9396b7b77b43dbe26bf1f8004" @@ -147,8 +143,8 @@ "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" + "ImportPath": "github.com/golang/groupcache/lru", + "Rev": "02826c3e79038b59d737d3b1c0a1d937f71a4433" }, { "ImportPath": "github.com/golang/protobuf/proto", @@ -312,43 +308,43 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/html", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/html/atom", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/internal/timeseries", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/trace", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/websocket", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/oauth2", @@ -420,75 +416,107 @@ }, { "ImportPath": "google.golang.org/grpc", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/balancer", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/balancer/base", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/balancer/roundrobin", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/codes", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/connectivity", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/credentials", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { - "ImportPath": "google.golang.org/grpc/grpclb/grpc_lb_v1/messages", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "ImportPath": "google.golang.org/grpc/encoding", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/encoding/proto", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/grpclog", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/health/grpc_health_v1", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/internal", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/backoff", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/channelz", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/grpcrand", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/keepalive", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/metadata", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/naming", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/peer", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/resolver", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/resolver/dns", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/resolver/passthrough", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/stats", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/status", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/tap", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/transport", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "gopkg.in/inf.v0", @@ -962,10 +990,18 @@ "ImportPath": "k8s.io/apiserver/pkg/audit", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/audit/event", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/audit/policy", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/audit/util", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/authentication/authenticator", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -1006,6 +1042,10 @@ "ImportPath": "k8s.io/apiserver/pkg/authentication/serviceaccount", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/authentication/token/cache", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/authentication/token/tokenfile", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -1206,6 +1246,14 @@ "ImportPath": "k8s.io/apiserver/plugin/pkg/audit/buffered", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/plugin/pkg/audit/dynamic", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "ImportPath": "k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/plugin/pkg/audit/log", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -1730,6 +1778,10 @@ "ImportPath": "k8s.io/client-go/tools/pager", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/client-go/tools/record", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/client-go/tools/reference", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -1770,29 +1822,37 @@ "ImportPath": "k8s.io/client-go/util/workqueue", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/kube-openapi/pkg/aggregator", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/builder", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/common", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/handler", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" + }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" } ] } diff --git a/staging/src/k8s.io/kube-aggregator/main.go b/staging/src/k8s.io/kube-aggregator/main.go index b0d76a545f8..659cb0753ed 100644 --- a/staging/src/k8s.io/kube-aggregator/main.go +++ b/staging/src/k8s.io/kube-aggregator/main.go @@ -20,7 +20,7 @@ import ( "flag" "os" - "github.com/golang/glog" + "k8s.io/klog" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/util/logs" @@ -43,6 +43,6 @@ func main() { cmd := server.NewCommandStartAggregator(options, stopCh) cmd.Flags().AddGoFlagSet(flag.CommandLine) if err := cmd.Execute(); err != nil { - glog.Fatal(err) + klog.Fatal(err) } } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/BUILD b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/BUILD index 9070e6a0894..69af8089830 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/BUILD +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/BUILD @@ -79,7 +79,7 @@ go_library( "//staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/controllers/status:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/registry/apiservice/rest:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiservice_controller.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiservice_controller.go index 8eec79589ce..f0b61a67c30 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiservice_controller.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiservice_controller.go @@ -20,7 +20,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "k8s.io/klog" apierrors "k8s.io/apimachinery/pkg/api/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -87,8 +87,8 @@ func (c *APIServiceRegistrationController) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() - glog.Infof("Starting APIServiceRegistrationController") - defer glog.Infof("Shutting down APIServiceRegistrationController") + klog.Infof("Starting APIServiceRegistrationController") + defer klog.Infof("Shutting down APIServiceRegistrationController") if !controllers.WaitForCacheSync("APIServiceRegistrationController", stopCh, c.apiServiceSynced) { return @@ -129,7 +129,7 @@ func (c *APIServiceRegistrationController) processNextWorkItem() bool { func (c *APIServiceRegistrationController) enqueue(obj *apiregistration.APIService) { key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) if err != nil { - glog.Errorf("Couldn't get key for object %#v: %v", obj, err) + klog.Errorf("Couldn't get key for object %#v: %v", obj, err) return } @@ -138,13 +138,13 @@ func (c *APIServiceRegistrationController) enqueue(obj *apiregistration.APIServi func (c *APIServiceRegistrationController) addAPIService(obj interface{}) { castObj := obj.(*apiregistration.APIService) - glog.V(4).Infof("Adding %s", castObj.Name) + klog.V(4).Infof("Adding %s", castObj.Name) c.enqueue(castObj) } func (c *APIServiceRegistrationController) updateAPIService(obj, _ interface{}) { castObj := obj.(*apiregistration.APIService) - glog.V(4).Infof("Updating %s", castObj.Name) + klog.V(4).Infof("Updating %s", castObj.Name) c.enqueue(castObj) } @@ -153,15 +153,15 @@ func (c *APIServiceRegistrationController) deleteAPIService(obj interface{}) { if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) + klog.Errorf("Couldn't get object from tombstone %#v", obj) return } castObj, ok = tombstone.Obj.(*apiregistration.APIService) if !ok { - glog.Errorf("Tombstone contained object that is not expected %#v", obj) + klog.Errorf("Tombstone contained object that is not expected %#v", obj) return } } - glog.V(4).Infof("Deleting %q", castObj.Name) + klog.V(4).Infof("Deleting %q", castObj.Name) c.enqueue(castObj) } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go index 991d318e13a..e8976f07eff 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go @@ -22,7 +22,7 @@ import ( "net/url" "sync/atomic" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/httpstream" @@ -112,7 +112,7 @@ func (r *proxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { location.Scheme = "https" rloc, err := r.serviceResolver.ResolveEndpoint(handlingInfo.serviceNamespace, handlingInfo.serviceName) if err != nil { - glog.Errorf("error resolving %s/%s: %v", handlingInfo.serviceNamespace, handlingInfo.serviceName, err) + klog.Errorf("error resolving %s/%s: %v", handlingInfo.serviceNamespace, handlingInfo.serviceName, err) http.Error(w, "service unavailable", http.StatusServiceUnavailable) return } @@ -213,7 +213,7 @@ func (r *proxyHandler) updateAPIService(apiService *apiregistrationapi.APIServic } newInfo.proxyRoundTripper, newInfo.transportBuildingError = restclient.TransportFor(newInfo.restConfig) if newInfo.transportBuildingError != nil { - glog.Warning(newInfo.transportBuildingError.Error()) + klog.Warning(newInfo.transportBuildingError.Error()) } r.handlingInfo.Store(newInfo) } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1/apiservice.go b/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1/apiservice.go index 1fa5bf2630c..82652ecaa61 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1/apiservice.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1/apiservice.go @@ -19,6 +19,8 @@ limitations under the License. package v1 import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -73,10 +75,15 @@ func (c *aPIServices) Get(name string, options metav1.GetOptions) (result *v1.AP // List takes label and field selectors, and returns the list of APIServices that match those selectors. func (c *aPIServices) List(opts metav1.ListOptions) (result *v1.APIServiceList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1.APIServiceList{} err = c.client.Get(). Resource("apiservices"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *aPIServices) List(opts metav1.ListOptions) (result *v1.APIServiceList, // Watch returns a watch.Interface that watches the requested aPIServices. func (c *aPIServices) Watch(opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("apiservices"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *aPIServices) Delete(name string, options *metav1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *aPIServices) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("apiservices"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1beta1/apiservice.go b/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1beta1/apiservice.go index e0726441637..f5ff4e2c7ca 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1beta1/apiservice.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1beta1/apiservice.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -73,10 +75,15 @@ func (c *aPIServices) Get(name string, options v1.GetOptions) (result *v1beta1.A // List takes label and field selectors, and returns the list of APIServices that match those selectors. func (c *aPIServices) List(opts v1.ListOptions) (result *v1beta1.APIServiceList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.APIServiceList{} err = c.client.Get(). Resource("apiservices"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *aPIServices) List(opts v1.ListOptions) (result *v1beta1.APIServiceList, // Watch returns a watch.Interface that watches the requested aPIServices. func (c *aPIServices) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("apiservices"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *aPIServices) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *aPIServices) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("apiservices"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/typed/apiregistration/internalversion/apiservice.go b/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/typed/apiregistration/internalversion/apiservice.go index 27d3cfc228b..a1738961e69 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/typed/apiregistration/internalversion/apiservice.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/typed/apiregistration/internalversion/apiservice.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -73,10 +75,15 @@ func (c *aPIServices) Get(name string, options v1.GetOptions) (result *apiregist // List takes label and field selectors, and returns the list of APIServices that match those selectors. func (c *aPIServices) List(opts v1.ListOptions) (result *apiregistration.APIServiceList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &apiregistration.APIServiceList{} err = c.client.Get(). Resource("apiservices"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -84,10 +91,15 @@ func (c *aPIServices) List(opts v1.ListOptions) (result *apiregistration.APIServ // Watch returns a watch.Interface that watches the requested aPIServices. func (c *aPIServices) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("apiservices"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -141,9 +153,14 @@ func (c *aPIServices) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *aPIServices) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("apiservices"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index 40c794972d9..e40f37136d3 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -85,8 +85,12 @@ func (o *AggregatorOptions) AddFlags(fs *pflag.FlagSet) { // NewDefaultOptions builds a "normal" set of options. You wouldn't normally expose this, but hyperkube isn't cobra compatible func NewDefaultOptions(out, err io.Writer) *AggregatorOptions { o := &AggregatorOptions{ - RecommendedOptions: genericoptions.NewRecommendedOptions(defaultEtcdPathPrefix, aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion)), - APIEnablement: genericoptions.NewAPIEnablementOptions(), + RecommendedOptions: genericoptions.NewRecommendedOptions( + defaultEtcdPathPrefix, + aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion), + genericoptions.NewProcessInfo("kube-aggregator", "kube-system"), + ), + APIEnablement: genericoptions.NewAPIEnablementOptions(), StdOut: out, StdErr: err, diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/BUILD b/staging/src/k8s.io/kube-aggregator/pkg/controllers/BUILD index f8a315bf3ff..f5946513bd6 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/BUILD +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/BUILD @@ -13,7 +13,7 @@ go_library( deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister/BUILD b/staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister/BUILD index cd0d7863257..db971bda52d 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister/BUILD +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister/BUILD @@ -38,7 +38,7 @@ go_library( "//staging/src/k8s.io/kube-aggregator/pkg/client/informers/internalversion/apiregistration/internalversion:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/client/listers/apiregistration/internalversion:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/controllers:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister/autoregister_controller.go b/staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister/autoregister_controller.go index c6051136b12..3815c7d4635 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister/autoregister_controller.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister/autoregister_controller.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" @@ -111,12 +111,12 @@ func NewAutoRegisterController(apiServiceInformer informers.APIServiceInformer, if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.V(2).Infof("Couldn't get object from tombstone %#v", obj) + klog.V(2).Infof("Couldn't get object from tombstone %#v", obj) return } cast, ok = tombstone.Obj.(*apiregistration.APIService) if !ok { - glog.V(2).Infof("Tombstone contained unexpected object: %#v", obj) + klog.V(2).Infof("Tombstone contained unexpected object: %#v", obj) return } } @@ -133,8 +133,8 @@ func (c *autoRegisterController) Run(threadiness int, stopCh <-chan struct{}) { // make sure the work queue is shutdown which will trigger workers to end defer c.queue.ShutDown() - glog.Infof("Starting autoregister controller") - defer glog.Infof("Shutting down autoregister controller") + klog.Infof("Starting autoregister controller") + defer klog.Infof("Shutting down autoregister controller") // wait for your secondary caches to fill before starting your work if !controllers.WaitForCacheSync("autoregister", stopCh, c.apiServiceSynced) { diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/cache.go b/staging/src/k8s.io/kube-aggregator/pkg/controllers/cache.go index bd366ec5b87..5a53b90629f 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/cache.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/cache.go @@ -19,7 +19,7 @@ package controllers import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/cache" @@ -29,13 +29,13 @@ import ( // indicating that the controller identified by controllerName is waiting for syncs, followed by // either a successful or failed sync. func WaitForCacheSync(controllerName string, stopCh <-chan struct{}, cacheSyncs ...cache.InformerSynced) bool { - glog.Infof("Waiting for caches to sync for %s controller", controllerName) + klog.Infof("Waiting for caches to sync for %s controller", controllerName) if !cache.WaitForCacheSync(stopCh, cacheSyncs...) { utilruntime.HandleError(fmt.Errorf("Unable to sync caches for %s controller", controllerName)) return false } - glog.Infof("Caches are synced for %s controller", controllerName) + klog.Infof("Caches are synced for %s controller", controllerName) return true } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/BUILD b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/BUILD index a489947c87d..9a4bef259a2 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/BUILD +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/BUILD @@ -20,7 +20,7 @@ go_library( "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/aggregator:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/builder:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/controller.go b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/controller.go index c38350ddbbe..49d190e9020 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/controller.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/controller.go @@ -22,7 +22,7 @@ import ( "time" "github.com/go-openapi/spec" - "github.com/golang/glog" + "k8s.io/klog" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" @@ -81,8 +81,8 @@ func (c *AggregationController) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() - glog.Infof("Starting OpenAPI AggregationController") - defer glog.Infof("Shutting down OpenAPI AggregationController") + klog.Infof("Starting OpenAPI AggregationController") + defer klog.Infof("Shutting down OpenAPI AggregationController") go wait.Until(c.runWorker, time.Second, stopCh) @@ -102,7 +102,7 @@ func (c *AggregationController) processNextWorkItem() bool { return false } - glog.Infof("OpenAPI AggregationController: Processing item %s", key) + klog.Infof("OpenAPI AggregationController: Processing item %s", key) action, err := c.syncHandler(key.(string)) if err == nil { @@ -113,13 +113,13 @@ func (c *AggregationController) processNextWorkItem() bool { switch action { case syncRequeue: - glog.Infof("OpenAPI AggregationController: action for item %s: Requeue.", key) + klog.Infof("OpenAPI AggregationController: action for item %s: Requeue.", key) c.queue.AddAfter(key, successfulUpdateDelay) case syncRequeueRateLimited: - glog.Infof("OpenAPI AggregationController: action for item %s: Rate Limited Requeue.", key) + klog.Infof("OpenAPI AggregationController: action for item %s: Rate Limited Requeue.", key) c.queue.AddRateLimited(key) case syncNothing: - glog.Infof("OpenAPI AggregationController: action for item %s: Nothing (removed from the queue).", key) + klog.Infof("OpenAPI AggregationController: action for item %s: Nothing (removed from the queue).", key) } return true diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/BUILD b/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/BUILD index 5dafa2e178d..3810051bc50 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/BUILD +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/BUILD @@ -29,7 +29,7 @@ go_library( "//staging/src/k8s.io/kube-aggregator/pkg/client/informers/internalversion/apiregistration/internalversion:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/client/listers/apiregistration/internalversion:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/controllers:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/available_controller.go b/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/available_controller.go index 3268f0c701e..66a4d0f2f75 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/available_controller.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/available_controller.go @@ -23,7 +23,7 @@ import ( "net/url" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -280,8 +280,8 @@ func (c *AvailableConditionController) Run(threadiness int, stopCh <-chan struct defer utilruntime.HandleCrash() defer c.queue.ShutDown() - glog.Infof("Starting AvailableConditionController") - defer glog.Infof("Shutting down AvailableConditionController") + klog.Infof("Starting AvailableConditionController") + defer klog.Infof("Shutting down AvailableConditionController") if !controllers.WaitForCacheSync("AvailableConditionController", stopCh, c.apiServiceSynced, c.servicesSynced, c.endpointsSynced) { return @@ -322,7 +322,7 @@ func (c *AvailableConditionController) processNextWorkItem() bool { func (c *AvailableConditionController) enqueue(obj *apiregistration.APIService) { key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) if err != nil { - glog.Errorf("Couldn't get key for object %#v: %v", obj, err) + klog.Errorf("Couldn't get key for object %#v: %v", obj, err) return } @@ -331,13 +331,13 @@ func (c *AvailableConditionController) enqueue(obj *apiregistration.APIService) func (c *AvailableConditionController) addAPIService(obj interface{}) { castObj := obj.(*apiregistration.APIService) - glog.V(4).Infof("Adding %s", castObj.Name) + klog.V(4).Infof("Adding %s", castObj.Name) c.enqueue(castObj) } func (c *AvailableConditionController) updateAPIService(obj, _ interface{}) { castObj := obj.(*apiregistration.APIService) - glog.V(4).Infof("Updating %s", castObj.Name) + klog.V(4).Infof("Updating %s", castObj.Name) c.enqueue(castObj) } @@ -346,16 +346,16 @@ func (c *AvailableConditionController) deleteAPIService(obj interface{}) { if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) + klog.Errorf("Couldn't get object from tombstone %#v", obj) return } castObj, ok = tombstone.Obj.(*apiregistration.APIService) if !ok { - glog.Errorf("Tombstone contained object that is not expected %#v", obj) + klog.Errorf("Tombstone contained object that is not expected %#v", obj) return } } - glog.V(4).Infof("Deleting %q", castObj.Name) + klog.V(4).Infof("Deleting %q", castObj.Name) c.enqueue(castObj) } @@ -400,12 +400,12 @@ func (c *AvailableConditionController) deleteService(obj interface{}) { if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) + klog.Errorf("Couldn't get object from tombstone %#v", obj) return } castObj, ok = tombstone.Obj.(*v1.Service) if !ok { - glog.Errorf("Tombstone contained object that is not expected %#v", obj) + klog.Errorf("Tombstone contained object that is not expected %#v", obj) return } } @@ -431,12 +431,12 @@ func (c *AvailableConditionController) deleteEndpoints(obj interface{}) { if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) + klog.Errorf("Couldn't get object from tombstone %#v", obj) return } castObj, ok = tombstone.Obj.(*v1.Endpoints) if !ok { - glog.Errorf("Tombstone contained object that is not expected %#v", obj) + klog.Errorf("Tombstone contained object that is not expected %#v", obj) return } } diff --git a/staging/src/k8s.io/kube-controller-manager/Godeps/Godeps.json b/staging/src/k8s.io/kube-controller-manager/Godeps/Godeps.json index 0d07751e86f..8927874f37b 100644 --- a/staging/src/k8s.io/kube-controller-manager/Godeps/Godeps.json +++ b/staging/src/k8s.io/kube-controller-manager/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/kube-controller-manager", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -14,29 +14,25 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/google/gofuzz", "Rev": "44d81051d367757e1c7c6a5a86423ece9afcf63c" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/text/secure/bidirule", @@ -158,6 +154,10 @@ "ImportPath": "k8s.io/apiserver/pkg/apis/config/v1alpha1", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/utils/pointer", "Rev": "66066c83e385e385ccc3c964b44fd7dcd413d0ed" diff --git a/staging/src/k8s.io/kube-proxy/Godeps/Godeps.json b/staging/src/k8s.io/kube-proxy/Godeps/Godeps.json index ff3e44acfaf..c0ec78d0391 100644 --- a/staging/src/k8s.io/kube-proxy/Godeps/Godeps.json +++ b/staging/src/k8s.io/kube-proxy/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/kube-proxy", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -14,29 +14,25 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/google/gofuzz", "Rev": "44d81051d367757e1c7c6a5a86423ece9afcf63c" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/text/secure/bidirule", @@ -149,6 +145,10 @@ { "ImportPath": "k8s.io/apimachinery/third_party/forked/golang/reflect", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" } ] } diff --git a/staging/src/k8s.io/kube-scheduler/Godeps/Godeps.json b/staging/src/k8s.io/kube-scheduler/Godeps/Godeps.json index ff61f39417f..99bbabbf4af 100644 --- a/staging/src/k8s.io/kube-scheduler/Godeps/Godeps.json +++ b/staging/src/k8s.io/kube-scheduler/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/kube-scheduler", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -14,29 +14,25 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/google/gofuzz", "Rev": "44d81051d367757e1c7c6a5a86423ece9afcf63c" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/text/secure/bidirule", @@ -158,6 +154,10 @@ "ImportPath": "k8s.io/apiserver/pkg/apis/config/v1alpha1", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/utils/pointer", "Rev": "66066c83e385e385ccc3c964b44fd7dcd413d0ed" diff --git a/staging/src/k8s.io/kubelet/Godeps/Godeps.json b/staging/src/k8s.io/kubelet/Godeps/Godeps.json index dbf0ed89d3d..e2933e6472b 100644 --- a/staging/src/k8s.io/kubelet/Godeps/Godeps.json +++ b/staging/src/k8s.io/kubelet/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/kubelet", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -14,29 +14,25 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/google/gofuzz", "Rev": "44d81051d367757e1c7c6a5a86423ece9afcf63c" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/text/secure/bidirule", @@ -145,6 +141,10 @@ { "ImportPath": "k8s.io/apimachinery/third_party/forked/golang/reflect", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" } ] } diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/types.go b/staging/src/k8s.io/kubelet/config/v1beta1/types.go index 281298f029d..4abe387773d 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/types.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/types.go @@ -292,9 +292,11 @@ type KubeletConfiguration struct { // Default: "4h" // +optional StreamingConnectionIdleTimeout metav1.Duration `json:"streamingConnectionIdleTimeout,omitempty"` - // nodeStatusUpdateFrequency is the frequency that kubelet posts node - // status to master. Note: be cautious when changing the constant, it - // must work with nodeMonitorGracePeriod in nodecontroller. + // nodeStatusUpdateFrequency is the frequency that kubelet computes node + // status. If node lease feature is not enabled, it is also the frequency that + // kubelet posts node status to master. + // Note: When node lease feature is not enabled, be cautious when changing the + // constant, it must work with nodeMonitorGracePeriod in nodecontroller. // Dynamic Kubelet Config (beta): If dynamically updating this field, consider that // it may impact node scalability, and also that the node controller's // nodeMonitorGracePeriod must be set to N*NodeStatusUpdateFrequency, @@ -303,6 +305,16 @@ type KubeletConfiguration struct { // Default: "10s" // +optional NodeStatusUpdateFrequency metav1.Duration `json:"nodeStatusUpdateFrequency,omitempty"` + // nodeStatusReportFrequency is the frequency that kubelet posts node + // status to master if node status does not change. Kubelet will ignore this + // frequency and post node status immediately if any change is detected. It is + // only used when node lease feature is enabled. nodeStatusReportFrequency's + // default value is 1m. But if nodeStatusUpdateFrequency is set explicitly, + // nodeStatusReportFrequency's default value will be set to + // nodeStatusUpdateFrequency for backward compatibility. + // Default: "1m" + // +optional + NodeStatusReportFrequency metav1.Duration `json:"nodeStatusReportFrequency,omitempty"` // nodeLeaseDurationSeconds is the duration the Kubelet will set on its corresponding Lease, // when the NodeLease feature is enabled. This feature provides an indicator of node // health by having the Kublet create and periodically renew a lease, named after the node, diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go index 37be9cb1843..a924fac2ec3 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go @@ -143,6 +143,7 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { } out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + out.NodeStatusReportFrequency = in.NodeStatusReportFrequency out.ImageMinimumGCAge = in.ImageMinimumGCAge if in.ImageGCHighThresholdPercent != nil { in, out := &in.ImageGCHighThresholdPercent, &out.ImageGCHighThresholdPercent diff --git a/staging/src/k8s.io/metrics/Godeps/Godeps.json b/staging/src/k8s.io/metrics/Godeps/Godeps.json index ebffa531823..668ad9186ba 100644 --- a/staging/src/k8s.io/metrics/Godeps/Godeps.json +++ b/staging/src/k8s.io/metrics/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/metrics", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -14,10 +14,6 @@ "ImportPath": "github.com/evanphx/json-patch", "Rev": "36442dbdb585210f8d5a1b45e67aa323c197d5c4" }, - { - "ImportPath": "github.com/ghodss/yaml", - "Rev": "c7ce16629ff4cd059ed96ed06419dd3856fd3577" - }, { "ImportPath": "github.com/gogo/protobuf/proto", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" @@ -26,10 +22,6 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/golang/protobuf/proto", "Rev": "b4deda0973fb4c70b50d226b1af49f3da59f5265" @@ -112,23 +104,23 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/oauth2", @@ -526,9 +518,17 @@ "ImportPath": "k8s.io/client-go/util/integer", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" + }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" } ] } diff --git a/staging/src/k8s.io/metrics/IMPLEMENTATIONS.md b/staging/src/k8s.io/metrics/IMPLEMENTATIONS.md index 7548bb52c96..cde97de5bf1 100644 --- a/staging/src/k8s.io/metrics/IMPLEMENTATIONS.md +++ b/staging/src/k8s.io/metrics/IMPLEMENTATIONS.md @@ -24,3 +24,7 @@ They are listed here for convenience.*** - [Google Stackdriver (coming soon)](https://github.com/GoogleCloudPlatform/k8s-stackdriver) + +- [Datadog Cluster Agent](https://github.com/DataDog/datadog-agent/blob/c4f38af1897bac294d8fed6285098b14aafa6178/docs/cluster-agent/CUSTOM_METRICS_SERVER.md). + Implementation of the external metrics provider, using Datadog as a backend for the metrics. + Coming soon: Implementation of the custom metrics provider to support in-cluster metrics collected by the Datadog Agents. \ No newline at end of file diff --git a/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1/nodemetrics.go b/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1/nodemetrics.go index 0f88e58605a..7b594d73d33 100644 --- a/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1/nodemetrics.go +++ b/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1/nodemetrics.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" @@ -66,10 +68,15 @@ func (c *nodeMetricses) Get(name string, options v1.GetOptions) (result *v1alpha // List takes label and field selectors, and returns the list of NodeMetricses that match those selectors. func (c *nodeMetricses) List(opts v1.ListOptions) (result *v1alpha1.NodeMetricsList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.NodeMetricsList{} err = c.client.Get(). Resource("nodes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -77,9 +84,14 @@ func (c *nodeMetricses) List(opts v1.ListOptions) (result *v1alpha1.NodeMetricsL // Watch returns a watch.Interface that watches the requested nodeMetricses. func (c *nodeMetricses) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("nodes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } diff --git a/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1/podmetrics.go b/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1/podmetrics.go index e11182eb741..e6fb6b5f60f 100644 --- a/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1/podmetrics.go +++ b/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1/podmetrics.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" @@ -69,11 +71,16 @@ func (c *podMetricses) Get(name string, options v1.GetOptions) (result *v1alpha1 // List takes label and field selectors, and returns the list of PodMetricses that match those selectors. func (c *podMetricses) List(opts v1.ListOptions) (result *v1alpha1.PodMetricsList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.PodMetricsList{} err = c.client.Get(). Namespace(c.ns). Resource("pods"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -81,10 +88,15 @@ func (c *podMetricses) List(opts v1.ListOptions) (result *v1alpha1.PodMetricsLis // Watch returns a watch.Interface that watches the requested podMetricses. func (c *podMetricses) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("pods"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } diff --git a/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/nodemetrics.go b/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/nodemetrics.go index 845914133cc..5b1dd89a12c 100644 --- a/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/nodemetrics.go +++ b/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/nodemetrics.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" @@ -66,10 +68,15 @@ func (c *nodeMetricses) Get(name string, options v1.GetOptions) (result *v1beta1 // List takes label and field selectors, and returns the list of NodeMetricses that match those selectors. func (c *nodeMetricses) List(opts v1.ListOptions) (result *v1beta1.NodeMetricsList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.NodeMetricsList{} err = c.client.Get(). Resource("nodes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -77,9 +84,14 @@ func (c *nodeMetricses) List(opts v1.ListOptions) (result *v1beta1.NodeMetricsLi // Watch returns a watch.Interface that watches the requested nodeMetricses. func (c *nodeMetricses) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("nodes"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } diff --git a/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/podmetrics.go b/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/podmetrics.go index 02e4e0a0d7d..cdc2510c7c6 100644 --- a/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/podmetrics.go +++ b/staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/podmetrics.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" @@ -69,11 +71,16 @@ func (c *podMetricses) Get(name string, options v1.GetOptions) (result *v1beta1. // List takes label and field selectors, and returns the list of PodMetricses that match those selectors. func (c *podMetricses) List(opts v1.ListOptions) (result *v1beta1.PodMetricsList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.PodMetricsList{} err = c.client.Get(). Namespace(c.ns). Resource("pods"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -81,10 +88,15 @@ func (c *podMetricses) List(opts v1.ListOptions) (result *v1beta1.PodMetricsList // Watch returns a watch.Interface that watches the requested podMetricses. func (c *podMetricses) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("pods"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } diff --git a/staging/src/k8s.io/sample-apiserver/BUILD b/staging/src/k8s.io/sample-apiserver/BUILD index 6e0d5662844..c2dcbc96442 100644 --- a/staging/src/k8s.io/sample-apiserver/BUILD +++ b/staging/src/k8s.io/sample-apiserver/BUILD @@ -20,7 +20,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library", "//staging/src/k8s.io/sample-apiserver/pkg/cmd/server:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json b/staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json index 15447ca9bd1..50a43f04b7e 100644 --- a/staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json +++ b/staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/sample-apiserver", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -102,10 +102,6 @@ "ImportPath": "github.com/evanphx/json-patch", "Rev": "36442dbdb585210f8d5a1b45e67aa323c197d5c4" }, - { - "ImportPath": "github.com/ghodss/yaml", - "Rev": "c7ce16629ff4cd059ed96ed06419dd3856fd3577" - }, { "ImportPath": "github.com/go-openapi/jsonpointer", "Rev": "ef5f0afec364d3b9396b7b77b43dbe26bf1f8004" @@ -139,8 +135,8 @@ "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" + "ImportPath": "github.com/golang/groupcache/lru", + "Rev": "02826c3e79038b59d737d3b1c0a1d937f71a4433" }, { "ImportPath": "github.com/golang/protobuf/proto", @@ -292,35 +288,35 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/internal/timeseries", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/trace", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/websocket", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/oauth2", @@ -392,75 +388,107 @@ }, { "ImportPath": "google.golang.org/grpc", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/balancer", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/balancer/base", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/balancer/roundrobin", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/codes", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/connectivity", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/credentials", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { - "ImportPath": "google.golang.org/grpc/grpclb/grpc_lb_v1/messages", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "ImportPath": "google.golang.org/grpc/encoding", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/encoding/proto", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/grpclog", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/health/grpc_health_v1", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/internal", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/backoff", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/channelz", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/internal/grpcrand", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/keepalive", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/metadata", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/naming", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/peer", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/resolver", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/resolver/dns", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + }, + { + "ImportPath": "google.golang.org/grpc/resolver/passthrough", + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/stats", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/status", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/tap", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "google.golang.org/grpc/transport", - "Rev": "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + "Rev": "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" }, { "ImportPath": "gopkg.in/inf.v0", @@ -926,10 +954,18 @@ "ImportPath": "k8s.io/apiserver/pkg/audit", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/audit/event", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/audit/policy", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/audit/util", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/authentication/authenticator", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -970,6 +1006,10 @@ "ImportPath": "k8s.io/apiserver/pkg/authentication/serviceaccount", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/authentication/token/cache", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/authentication/token/tokenfile", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -1166,6 +1206,14 @@ "ImportPath": "k8s.io/apiserver/plugin/pkg/audit/buffered", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/plugin/pkg/audit/dynamic", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "ImportPath": "k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/plugin/pkg/audit/log", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -1690,6 +1738,10 @@ "ImportPath": "k8s.io/client-go/tools/pager", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/client-go/tools/record", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/client-go/tools/reference", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -1726,25 +1778,33 @@ "ImportPath": "k8s.io/client-go/util/retry", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/kube-openapi/pkg/builder", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/common", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/handler", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" + }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" } ] } diff --git a/staging/src/k8s.io/sample-apiserver/main.go b/staging/src/k8s.io/sample-apiserver/main.go index 8e1769a36ff..abede29b096 100644 --- a/staging/src/k8s.io/sample-apiserver/main.go +++ b/staging/src/k8s.io/sample-apiserver/main.go @@ -20,7 +20,7 @@ import ( "flag" "os" - "github.com/golang/glog" + "k8s.io/klog" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/util/logs" @@ -36,6 +36,6 @@ func main() { cmd := server.NewCommandStartWardleServer(options, stopCh) cmd.Flags().AddGoFlagSet(flag.CommandLine) if err := cmd.Execute(); err != nil { - glog.Fatal(err) + klog.Fatal(err) } } diff --git a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/typed/wardle/internalversion/fischer.go b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/typed/wardle/internalversion/fischer.go index 9af7ccf61fb..f5d88411fd4 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/typed/wardle/internalversion/fischer.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/typed/wardle/internalversion/fischer.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -72,10 +74,15 @@ func (c *fischers) Get(name string, options v1.GetOptions) (result *wardle.Fisch // List takes label and field selectors, and returns the list of Fischers that match those selectors. func (c *fischers) List(opts v1.ListOptions) (result *wardle.FischerList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &wardle.FischerList{} err = c.client.Get(). Resource("fischers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *fischers) List(opts v1.ListOptions) (result *wardle.FischerList, err er // Watch returns a watch.Interface that watches the requested fischers. func (c *fischers) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("fischers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *fischers) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *fischers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("fischers"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/typed/wardle/internalversion/flunder.go b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/typed/wardle/internalversion/flunder.go index ce9b6064e0c..7d72dbf993a 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/typed/wardle/internalversion/flunder.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/typed/wardle/internalversion/flunder.go @@ -19,6 +19,8 @@ limitations under the License. package internalversion import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *flunders) Get(name string, options v1.GetOptions) (result *wardle.Flund // List takes label and field selectors, and returns the list of Flunders that match those selectors. func (c *flunders) List(opts v1.ListOptions) (result *wardle.FlunderList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &wardle.FlunderList{} err = c.client.Get(). Namespace(c.ns). Resource("flunders"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *flunders) List(opts v1.ListOptions) (result *wardle.FlunderList, err er // Watch returns a watch.Interface that watches the requested flunders. func (c *flunders) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("flunders"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *flunders) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *flunders) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("flunders"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/versioned/typed/wardle/v1alpha1/fischer.go b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/versioned/typed/wardle/v1alpha1/fischer.go index cf32143f87a..b6cb6b7c327 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/versioned/typed/wardle/v1alpha1/fischer.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/versioned/typed/wardle/v1alpha1/fischer.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -72,10 +74,15 @@ func (c *fischers) Get(name string, options v1.GetOptions) (result *v1alpha1.Fis // List takes label and field selectors, and returns the list of Fischers that match those selectors. func (c *fischers) List(opts v1.ListOptions) (result *v1alpha1.FischerList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.FischerList{} err = c.client.Get(). Resource("fischers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -83,10 +90,15 @@ func (c *fischers) List(opts v1.ListOptions) (result *v1alpha1.FischerList, err // Watch returns a watch.Interface that watches the requested fischers. func (c *fischers) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Resource("fischers"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -125,9 +137,14 @@ func (c *fischers) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *fischers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Resource("fischers"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/versioned/typed/wardle/v1alpha1/flunder.go b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/versioned/typed/wardle/v1alpha1/flunder.go index 35545175927..5ff17cb4cc8 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/versioned/typed/wardle/v1alpha1/flunder.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/versioned/typed/wardle/v1alpha1/flunder.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *flunders) Get(name string, options v1.GetOptions) (result *v1alpha1.Flu // List takes label and field selectors, and returns the list of Flunders that match those selectors. func (c *flunders) List(opts v1.ListOptions) (result *v1alpha1.FlunderList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.FlunderList{} err = c.client.Get(). Namespace(c.ns). Resource("flunders"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *flunders) List(opts v1.ListOptions) (result *v1alpha1.FlunderList, err // Watch returns a watch.Interface that watches the requested flunders. func (c *flunders) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("flunders"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *flunders) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *flunders) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("flunders"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/versioned/typed/wardle/v1beta1/flunder.go b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/versioned/typed/wardle/v1beta1/flunder.go index f0fabeb27de..19e60f95c6a 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/versioned/typed/wardle/v1beta1/flunder.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/versioned/typed/wardle/v1beta1/flunder.go @@ -19,6 +19,8 @@ limitations under the License. package v1beta1 import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *flunders) Get(name string, options v1.GetOptions) (result *v1beta1.Flun // List takes label and field selectors, and returns the list of Flunders that match those selectors. func (c *flunders) List(opts v1.ListOptions) (result *v1beta1.FlunderList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1beta1.FlunderList{} err = c.client.Get(). Namespace(c.ns). Resource("flunders"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *flunders) List(opts v1.ListOptions) (result *v1beta1.FlunderList, err e // Watch returns a watch.Interface that watches the requested flunders. func (c *flunders) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("flunders"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *flunders) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *flunders) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("flunders"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index dc2d9290cca..c9fa45850f5 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -47,7 +47,11 @@ type WardleServerOptions struct { func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions { o := &WardleServerOptions{ - RecommendedOptions: genericoptions.NewRecommendedOptions(defaultEtcdPathPrefix, apiserver.Codecs.LegacyCodec(v1alpha1.SchemeGroupVersion)), + RecommendedOptions: genericoptions.NewRecommendedOptions( + defaultEtcdPathPrefix, + apiserver.Codecs.LegacyCodec(v1alpha1.SchemeGroupVersion), + genericoptions.NewProcessInfo("wardle-apiserver", "wardle"), + ), StdOut: out, StdErr: errOut, diff --git a/staging/src/k8s.io/sample-cli-plugin/Godeps/Godeps.json b/staging/src/k8s.io/sample-cli-plugin/Godeps/Godeps.json index abdb8883611..f7550bb7722 100644 --- a/staging/src/k8s.io/sample-cli-plugin/Godeps/Godeps.json +++ b/staging/src/k8s.io/sample-cli-plugin/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/sample-cli-plugin", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -10,10 +10,6 @@ "ImportPath": "github.com/evanphx/json-patch", "Rev": "36442dbdb585210f8d5a1b45e67aa323c197d5c4" }, - { - "ImportPath": "github.com/ghodss/yaml", - "Rev": "c7ce16629ff4cd059ed96ed06419dd3856fd3577" - }, { "ImportPath": "github.com/gogo/protobuf/proto", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" @@ -22,10 +18,6 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/golang/protobuf/proto", "Rev": "b4deda0973fb4c70b50d226b1af49f3da59f5265" @@ -112,23 +104,23 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/oauth2", @@ -577,6 +569,14 @@ { "ImportPath": "k8s.io/client-go/util/jsonpath", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" } ] } diff --git a/staging/src/k8s.io/sample-controller/BUILD b/staging/src/k8s.io/sample-controller/BUILD index 53d8671e0ea..a95c03ffb1e 100644 --- a/staging/src/k8s.io/sample-controller/BUILD +++ b/staging/src/k8s.io/sample-controller/BUILD @@ -34,7 +34,7 @@ go_library( "//staging/src/k8s.io/sample-controller/pkg/client/informers/externalversions/samplecontroller/v1alpha1:go_default_library", "//staging/src/k8s.io/sample-controller/pkg/client/listers/samplecontroller/v1alpha1:go_default_library", "//staging/src/k8s.io/sample-controller/pkg/signals:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/sample-controller/Godeps/Godeps.json b/staging/src/k8s.io/sample-controller/Godeps/Godeps.json index a9ea15b519e..aa95b82db0b 100644 --- a/staging/src/k8s.io/sample-controller/Godeps/Godeps.json +++ b/staging/src/k8s.io/sample-controller/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "k8s.io/sample-controller", "GoVersion": "go1.11", - "GodepVersion": "v80", + "GodepVersion": "v80-k8s-r1", "Packages": [ "./..." ], @@ -14,10 +14,6 @@ "ImportPath": "github.com/evanphx/json-patch", "Rev": "36442dbdb585210f8d5a1b45e67aa323c197d5c4" }, - { - "ImportPath": "github.com/ghodss/yaml", - "Rev": "c7ce16629ff4cd059ed96ed06419dd3856fd3577" - }, { "ImportPath": "github.com/gogo/protobuf/proto", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" @@ -26,10 +22,6 @@ "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "342cbe0a04158f6dcb03ca0079991a51a4248c02" }, - { - "ImportPath": "github.com/golang/glog", - "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" - }, { "ImportPath": "github.com/golang/groupcache/lru", "Rev": "02826c3e79038b59d737d3b1c0a1d937f71a4433" @@ -120,23 +112,23 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" + "Rev": "0ed95abb35c445290478a5348a7b38bb154135fd" }, { "ImportPath": "golang.org/x/oauth2", @@ -1158,9 +1150,17 @@ "ImportPath": "k8s.io/client-go/util/workqueue", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/klog", + "Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" + }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "72693cb1fadd73ae2742f6fe29af77d1aecdd8cd" + "Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d" + }, + { + "ImportPath": "sigs.k8s.io/yaml", + "Rev": "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" } ] } diff --git a/staging/src/k8s.io/sample-controller/controller.go b/staging/src/k8s.io/sample-controller/controller.go index ab8468a0a28..e9d1d8389ee 100644 --- a/staging/src/k8s.io/sample-controller/controller.go +++ b/staging/src/k8s.io/sample-controller/controller.go @@ -20,7 +20,6 @@ import ( "fmt" "time" - "github.com/golang/glog" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -37,6 +36,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" samplev1alpha1 "k8s.io/sample-controller/pkg/apis/samplecontroller/v1alpha1" clientset "k8s.io/sample-controller/pkg/client/clientset/versioned" @@ -96,9 +96,9 @@ func NewController( // Add sample-controller types to the default Kubernetes Scheme so Events can be // logged for sample-controller types. utilruntime.Must(samplescheme.AddToScheme(scheme.Scheme)) - glog.V(4).Info("Creating event broadcaster") + klog.V(4).Info("Creating event broadcaster") eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeclientset.CoreV1().Events("")}) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) @@ -113,7 +113,7 @@ func NewController( recorder: recorder, } - glog.Info("Setting up event handlers") + klog.Info("Setting up event handlers") // Set up an event handler for when Foo resources change fooInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueFoo, @@ -154,23 +154,23 @@ func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error { defer c.workqueue.ShutDown() // Start the informer factories to begin populating the informer caches - glog.Info("Starting Foo controller") + klog.Info("Starting Foo controller") // Wait for the caches to be synced before starting workers - glog.Info("Waiting for informer caches to sync") + klog.Info("Waiting for informer caches to sync") if ok := cache.WaitForCacheSync(stopCh, c.deploymentsSynced, c.foosSynced); !ok { return fmt.Errorf("failed to wait for caches to sync") } - glog.Info("Starting workers") + klog.Info("Starting workers") // Launch two workers to process Foo resources for i := 0; i < threadiness; i++ { go wait.Until(c.runWorker, time.Second, stopCh) } - glog.Info("Started workers") + klog.Info("Started workers") <-stopCh - glog.Info("Shutting down workers") + klog.Info("Shutting down workers") return nil } @@ -226,7 +226,7 @@ func (c *Controller) processNextWorkItem() bool { // Finally, if no error occurs we Forget this item so it does not // get queued again until another change happens. c.workqueue.Forget(obj) - glog.Infof("Successfully synced '%s'", key) + klog.Infof("Successfully synced '%s'", key) return nil }(obj) @@ -297,7 +297,7 @@ func (c *Controller) syncHandler(key string) error { // number does not equal the current desired replicas on the Deployment, we // should update the Deployment resource. if foo.Spec.Replicas != nil && *foo.Spec.Replicas != *deployment.Spec.Replicas { - glog.V(4).Infof("Foo %s replicas: %d, deployment replicas: %d", name, *foo.Spec.Replicas, *deployment.Spec.Replicas) + klog.V(4).Infof("Foo %s replicas: %d, deployment replicas: %d", name, *foo.Spec.Replicas, *deployment.Spec.Replicas) deployment, err = c.kubeclientset.AppsV1().Deployments(foo.Namespace).Update(newDeployment(foo)) } @@ -365,9 +365,9 @@ func (c *Controller) handleObject(obj interface{}) { runtime.HandleError(fmt.Errorf("error decoding object tombstone, invalid type")) return } - glog.V(4).Infof("Recovered deleted object '%s' from tombstone", object.GetName()) + klog.V(4).Infof("Recovered deleted object '%s' from tombstone", object.GetName()) } - glog.V(4).Infof("Processing object: %s", object.GetName()) + klog.V(4).Infof("Processing object: %s", object.GetName()) if ownerRef := metav1.GetControllerOf(object); ownerRef != nil { // If this object is not owned by a Foo, we should not do anything more // with it. @@ -377,7 +377,7 @@ func (c *Controller) handleObject(obj interface{}) { foo, err := c.foosLister.Foos(object.GetNamespace()).Get(ownerRef.Name) if err != nil { - glog.V(4).Infof("ignoring orphaned object '%s' of foo '%s'", object.GetSelfLink(), ownerRef.Name) + klog.V(4).Infof("ignoring orphaned object '%s' of foo '%s'", object.GetSelfLink(), ownerRef.Name) return } diff --git a/staging/src/k8s.io/sample-controller/main.go b/staging/src/k8s.io/sample-controller/main.go index c9d3193df99..d8167715602 100644 --- a/staging/src/k8s.io/sample-controller/main.go +++ b/staging/src/k8s.io/sample-controller/main.go @@ -20,10 +20,10 @@ import ( "flag" "time" - "github.com/golang/glog" kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog" // Uncomment the following line to load the gcp plugin (only required to authenticate against GKE clusters). // _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" @@ -45,17 +45,17 @@ func main() { cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) if err != nil { - glog.Fatalf("Error building kubeconfig: %s", err.Error()) + klog.Fatalf("Error building kubeconfig: %s", err.Error()) } kubeClient, err := kubernetes.NewForConfig(cfg) if err != nil { - glog.Fatalf("Error building kubernetes clientset: %s", err.Error()) + klog.Fatalf("Error building kubernetes clientset: %s", err.Error()) } exampleClient, err := clientset.NewForConfig(cfg) if err != nil { - glog.Fatalf("Error building example clientset: %s", err.Error()) + klog.Fatalf("Error building example clientset: %s", err.Error()) } kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeClient, time.Second*30) @@ -71,7 +71,7 @@ func main() { exampleInformerFactory.Start(stopCh) if err = controller.Run(2, stopCh); err != nil { - glog.Fatalf("Error running controller: %s", err.Error()) + klog.Fatalf("Error running controller: %s", err.Error()) } } diff --git a/staging/src/k8s.io/sample-controller/pkg/client/clientset/versioned/typed/samplecontroller/v1alpha1/foo.go b/staging/src/k8s.io/sample-controller/pkg/client/clientset/versioned/typed/samplecontroller/v1alpha1/foo.go index 667edf527ba..48bc2152f1c 100644 --- a/staging/src/k8s.io/sample-controller/pkg/client/clientset/versioned/typed/samplecontroller/v1alpha1/foo.go +++ b/staging/src/k8s.io/sample-controller/pkg/client/clientset/versioned/typed/samplecontroller/v1alpha1/foo.go @@ -19,6 +19,8 @@ limitations under the License. package v1alpha1 import ( + "time" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" @@ -76,11 +78,16 @@ func (c *foos) Get(name string, options v1.GetOptions) (result *v1alpha1.Foo, er // List takes label and field selectors, and returns the list of Foos that match those selectors. func (c *foos) List(opts v1.ListOptions) (result *v1alpha1.FooList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } result = &v1alpha1.FooList{} err = c.client.Get(). Namespace(c.ns). Resource("foos"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Do(). Into(result) return @@ -88,11 +95,16 @@ func (c *foos) List(opts v1.ListOptions) (result *v1alpha1.FooList, err error) { // Watch returns a watch.Interface that watches the requested foos. func (c *foos) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("foos"). VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). Watch() } @@ -150,10 +162,15 @@ func (c *foos) Delete(name string, options *v1.DeleteOptions) error { // DeleteCollection deletes a collection of objects. func (c *foos) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } return c.client.Delete(). Namespace(c.ns). Resource("foos"). VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). Body(options). Do(). Error() diff --git a/test/cmd/apply.sh b/test/cmd/apply.sh index 58c90902c63..dabc0ff66fc 100755 --- a/test/cmd/apply.sh +++ b/test/cmd/apply.sh @@ -93,6 +93,37 @@ run_kubectl_apply_tests() { # clean-up kubectl delete -f hack/testdata/pod.yaml "${kube_flags[@]}" + ## kubectl apply dry-run on CR + # Create CRD + kubectl "${kube_flags_with_token[@]}" create -f - << __EOF__ +{ + "kind": "CustomResourceDefinition", + "apiVersion": "apiextensions.k8s.io/v1beta1", + "metadata": { + "name": "resources.mygroup.example.com" + }, + "spec": { + "group": "mygroup.example.com", + "version": "v1alpha1", + "scope": "Namespaced", + "names": { + "plural": "resources", + "singular": "resource", + "kind": "Kind", + "listKind": "KindList" + } + } +} +__EOF__ + + # Dry-run create the CR + kubectl "${kube_flags[@]}" apply --server-dry-run -f hack/testdata/CRD/resource.yaml "${kube_flags[@]}" + # Make sure that the CR doesn't exist + ! kubectl "${kube_flags[@]}" get resource/myobj + + # clean-up + kubectl "${kube_flags[@]}" delete customresourcedefinition resources.mygroup.example.com + ## kubectl apply --prune # Pre-Condition: no POD exists kube::test::get_object_assert pods "{{range.items}}{{$id_field}}:{{end}}" '' diff --git a/test/cmd/apps.sh b/test/cmd/apps.sh index 7cabbabe24a..8811ef0a0d8 100755 --- a/test/cmd/apps.sh +++ b/test/cmd/apps.sh @@ -369,7 +369,8 @@ run_deployment_tests() { kubectl create -f hack/testdata/configmap.yaml "${kube_flags[@]}" kubectl create -f hack/testdata/secret.yaml "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx-deployment:' - kube::test::get_object_assert configmap "{{range.items}}{{$id_field}}:{{end}}" 'test-set-env-config:' + #configmap is special here due to controller will create kube-root-ca.crt for each namespace automatically + kube::test::get_object_assert 'configmaps/test-set-env-config' "{{$id_field}}" 'test-set-env-config' kube::test::get_object_assert secret "{{range.items}}{{$id_field}}:{{end}}" 'test-set-env-secret:' # Set env of deployments by configmap from keys kubectl set env deployment nginx-deployment --keys=key-2 --from=configmap/test-set-env-config "${kube_flags[@]}" diff --git a/test/cmd/core.sh b/test/cmd/core.sh index 0300e3bb1c0..656ef330f57 100755 --- a/test/cmd/core.sh +++ b/test/cmd/core.sh @@ -25,7 +25,7 @@ run_configmap_tests() { create_and_use_new_namespace kube::log::status "Testing configmaps" kubectl create -f test/fixtures/doc-yaml/user-guide/configmap/configmap.yaml - kube::test::get_object_assert configmap "{{range.items}}{{$id_field}}{{end}}" 'test-configmap' + kube::test::get_object_assert 'configmap/test-configmap' "{{$id_field}}" 'test-configmap' kubectl delete configmap test-configmap "${kube_flags[@]}" ### Create a new namespace @@ -37,8 +37,10 @@ run_configmap_tests() { kube::test::get_object_assert 'namespaces/test-configmaps' "{{$id_field}}" 'test-configmaps' ### Create a generic configmap in a specific namespace - # Pre-condition: no configmaps namespace exists - kube::test::get_object_assert 'configmaps --namespace=test-configmaps' "{{range.items}}{{$id_field}}:{{end}}" '' + # Pre-condition: configmap test-configmap and test-binary-configmap does not exist + kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"test-configmap\" }}found{{end}}{{end}}:' ':' + kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"test-binary-configmap\" }}found{{end}}{{end}}:' ':' + # Command kubectl create configmap test-configmap --from-literal=key1=value1 --namespace=test-configmaps kubectl create configmap test-binary-configmap --from-file <( head -c 256 /dev/urandom ) --namespace=test-configmaps @@ -222,8 +224,11 @@ run_pod_tests() { kube::test::get_object_assert 'secret/test-secret --namespace=test-kubectl-describe-pod' "{{$secret_type}}" 'test-type' ### Create a generic configmap - # Pre-condition: no CONFIGMAP exists - kube::test::get_object_assert 'configmaps --namespace=test-kubectl-describe-pod' "{{range.items}}{{$id_field}}:{{end}}" '' + # Pre-condition: CONFIGMAP test-configmap does not exist + #kube::test::get_object_assert 'configmap/test-configmap --namespace=test-kubectl-describe-pod' "{{$id_field}}" '' + kube::test::get_object_assert 'configmaps --namespace=test-kubectl-describe-pod' '{{range.items}}{{ if eq $id_field \"test-configmap\" }}found{{end}}{{end}}:' ':' + + #kube::test::get_object_assert 'configmaps --namespace=test-kubectl-describe-pod' "{{range.items}}{{$id_field}}:{{end}}" '' # Command kubectl create configmap test-configmap --from-literal=key-2=value2 --namespace=test-kubectl-describe-pod # Post-condition: configmap exists and has expected values diff --git a/test/cmd/get.sh b/test/cmd/get.sh index a8981748007..26ffcf4dda7 100755 --- a/test/cmd/get.sh +++ b/test/cmd/get.sh @@ -130,8 +130,11 @@ run_kubectl_get_tests() { kube::test::if_has_string "${output_message}" "/clusterroles?limit=500 200 OK" ### Test kubectl get chunk size does not result in a --watch error when resource list is served in multiple chunks - # Pre-condition: no ConfigMaps exist - kube::test::get_object_assert configmap "{{range.items}}{{$id_field}}:{{end}}" '' + # Pre-condition: ConfigMap one two tree does not exist + kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"one\" }}found{{end}}{{end}}:' ':' + kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"two\" }}found{{end}}{{end}}:' ':' + kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"three\" }}found{{end}}{{end}}:' ':' + # Post-condition: Create three configmaps and ensure that we can --watch them with a --chunk-size of 1 kubectl create cm one "${kube_flags[@]}" kubectl create cm two "${kube_flags[@]}" diff --git a/test/cmd/run.sh b/test/cmd/run.sh index bc583b4f4aa..45e7eb56945 100755 --- a/test/cmd/run.sh +++ b/test/cmd/run.sh @@ -45,17 +45,17 @@ run_kubectl_run_tests() { # Post-Condition: Deployment "nginx" is created kube::test::get_object_assert deployment.extensions "{{range.items}}{{$id_field}}:{{end}}" 'nginx-extensions:' # new generator was used - output_message=$(kubectl get deployment.extensions/nginx-extensions -o jsonpath='{.spec.revisionHistoryLimit}') - kube::test::if_has_string "${output_message}" '2' + output_message=$(kubectl get deployment.apps/nginx-extensions -o jsonpath='{.spec.revisionHistoryLimit}') + kube::test::if_has_string "${output_message}" '10' # Clean up kubectl delete deployment nginx-extensions "${kube_flags[@]}" # Command - kubectl run nginx-apps "--image=$IMAGE_NGINX" --generator=deployment/apps.v1beta1 "${kube_flags[@]}" + kubectl run nginx-apps "--image=$IMAGE_NGINX" --generator=deployment/apps.v1 "${kube_flags[@]}" # Post-Condition: Deployment "nginx" is created kube::test::get_object_assert deployment.apps "{{range.items}}{{$id_field}}:{{end}}" 'nginx-apps:' # and new generator was used, iow. new defaults are applied output_message=$(kubectl get deployment/nginx-apps -o jsonpath='{.spec.revisionHistoryLimit}') - kube::test::if_has_string "${output_message}" '2' + kube::test::if_has_string "${output_message}" '10' # Clean up kubectl delete deployment nginx-apps "${kube_flags[@]}" diff --git a/test/e2e/BUILD b/test/e2e/BUILD index a7f1864dc93..51b9455a580 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -68,11 +68,11 @@ go_library( "//test/e2e/framework/testfiles:go_default_library", "//test/e2e/manifest:go_default_library", "//test/utils:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/ginkgo/config:go_default_library", "//vendor/github.com/onsi/ginkgo/reporters:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/e2e/apimachinery/BUILD b/test/e2e/apimachinery/BUILD index bc57dc1698b..ea3fbc2d673 100644 --- a/test/e2e/apimachinery/BUILD +++ b/test/e2e/apimachinery/BUILD @@ -11,6 +11,7 @@ go_library( "aggregator.go", "certs.go", "chunking.go", + "crd_conversion_webhook.go", "crd_watch.go", "custom_resource_definition.go", "etcd_failure.go", @@ -36,9 +37,11 @@ go_library( "//staging/src/k8s.io/api/batch/v1beta1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library", + "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/test/integration:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/test/e2e/apimachinery/crd_conversion_webhook.go b/test/e2e/apimachinery/crd_conversion_webhook.go new file mode 100644 index 00000000000..57614cc8856 --- /dev/null +++ b/test/e2e/apimachinery/crd_conversion_webhook.go @@ -0,0 +1,396 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apimachinery + +import ( + "time" + + apps "k8s.io/api/apps/v1" + "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apiextensions-apiserver/test/integration" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/intstr" + utilversion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/client-go/dynamic" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + imageutils "k8s.io/kubernetes/test/utils/image" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + _ "github.com/stretchr/testify/assert" +) + +const ( + secretCRDName = "sample-custom-resource-conversion-webhook-secret" + deploymentCRDName = "sample-crd-conversion-webhook-deployment" + serviceCRDName = "e2e-test-crd-conversion-webhook" + roleBindingCRDName = "crd-conversion-webhook-auth-reader" +) + +var serverCRDConversionWebhookVersion = utilversion.MustParseSemantic("v1.13.0-alpha") + +var apiVersions = []v1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + }, + { + Name: "v2", + Served: true, + Storage: false, + }, +} + +var alternativeApiVersions = []v1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: false, + }, + { + Name: "v2", + Served: true, + Storage: true, + }, +} + +var _ = SIGDescribe("CustomResourceConversionWebhook [Feature:CustomResourceWebhookConversion]", func() { + var context *certContext + f := framework.NewDefaultFramework("crd-webhook") + + var client clientset.Interface + var namespaceName string + + BeforeEach(func() { + client = f.ClientSet + namespaceName = f.Namespace.Name + + // Make sure the relevant provider supports conversion webhook + framework.SkipUnlessServerVersionGTE(serverCRDConversionWebhookVersion, f.ClientSet.Discovery()) + + By("Setting up server cert") + context = setupServerCert(f.Namespace.Name, serviceCRDName) + createAuthReaderRoleBindingForCRDConversion(f, f.Namespace.Name) + + deployCustomResourceWebhookAndService(f, imageutils.GetE2EImage(imageutils.CRDConversionWebhook), context) + }) + + AfterEach(func() { + cleanCRDWebhookTest(client, namespaceName) + }) + + It("Should be able to convert from CR v1 to CR v2", func() { + testcrd, err := framework.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions, + &v1beta1.WebhookClientConfig{ + CABundle: context.signingCert, + Service: &v1beta1.ServiceReference{ + Namespace: f.Namespace.Name, + Name: serviceCRDName, + Path: strPtr("/crdconvert"), + }}) + if err != nil { + return + } + defer testcrd.CleanUp() + testCustomResourceConversionWebhook(f, testcrd.Crd, testcrd.DynamicClients) + }) + + It("Should be able to convert a non homogeneous list of CRs", func() { + testcrd, err := framework.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions, + &v1beta1.WebhookClientConfig{ + CABundle: context.signingCert, + Service: &v1beta1.ServiceReference{ + Namespace: f.Namespace.Name, + Name: serviceCRDName, + Path: strPtr("/crdconvert"), + }}) + if err != nil { + return + } + defer testcrd.CleanUp() + testCRListConversion(f, testcrd) + }) +}) + +func cleanCRDWebhookTest(client clientset.Interface, namespaceName string) { + _ = client.CoreV1().Services(namespaceName).Delete(serviceCRDName, nil) + _ = client.AppsV1().Deployments(namespaceName).Delete(deploymentCRDName, nil) + _ = client.CoreV1().Secrets(namespaceName).Delete(secretCRDName, nil) + _ = client.RbacV1().RoleBindings("kube-system").Delete(roleBindingCRDName, nil) +} + +func createAuthReaderRoleBindingForCRDConversion(f *framework.Framework, namespace string) { + By("Create role binding to let cr conversion webhook read extension-apiserver-authentication") + client := f.ClientSet + // Create the role binding to allow the webhook read the extension-apiserver-authentication configmap + _, err := client.RbacV1().RoleBindings("kube-system").Create(&rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleBindingCRDName, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "", + Kind: "Role", + Name: "extension-apiserver-authentication-reader", + }, + // Webhook uses the default service account. + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "default", + Namespace: namespace, + }, + }, + }) + if err != nil && errors.IsAlreadyExists(err) { + framework.Logf("role binding %s already exists", roleBindingCRDName) + } else { + framework.ExpectNoError(err, "creating role binding %s:webhook to access configMap", namespace) + } +} + +func deployCustomResourceWebhookAndService(f *framework.Framework, image string, context *certContext) { + By("Deploying the custom resource conversion webhook pod") + client := f.ClientSet + + // Creating the secret that contains the webhook's cert. + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretCRDName, + }, + Type: v1.SecretTypeOpaque, + Data: map[string][]byte{ + "tls.crt": context.cert, + "tls.key": context.key, + }, + } + namespace := f.Namespace.Name + _, err := client.CoreV1().Secrets(namespace).Create(secret) + framework.ExpectNoError(err, "creating secret %q in namespace %q", secretName, namespace) + + // Create the deployment of the webhook + podLabels := map[string]string{"app": "sample-crd-conversion-webhook", "crd-webhook": "true"} + replicas := int32(1) + zero := int64(0) + mounts := []v1.VolumeMount{ + { + Name: "crd-conversion-webhook-certs", + ReadOnly: true, + MountPath: "/webhook.local.config/certificates", + }, + } + volumes := []v1.Volume{ + { + Name: "crd-conversion-webhook-certs", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{SecretName: secretCRDName}, + }, + }, + } + containers := []v1.Container{ + { + Name: "sample-crd-conversion-webhook", + VolumeMounts: mounts, + Args: []string{ + "--tls-cert-file=/webhook.local.config/certificates/tls.crt", + "--tls-private-key-file=/webhook.local.config/certificates/tls.key", + "--alsologtostderr", + "-v=4", + "2>&1", + }, + Image: image, + }, + } + d := &apps.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentCRDName, + Labels: podLabels, + }, + Spec: apps.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: podLabels, + }, + Strategy: apps.DeploymentStrategy{ + Type: apps.RollingUpdateDeploymentStrategyType, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + Spec: v1.PodSpec{ + TerminationGracePeriodSeconds: &zero, + Containers: containers, + Volumes: volumes, + }, + }, + }, + } + deployment, err := client.AppsV1().Deployments(namespace).Create(d) + framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentCRDName, namespace) + By("Wait for the deployment to be ready") + err = framework.WaitForDeploymentRevisionAndImage(client, namespace, deploymentCRDName, "1", image) + framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, namespace) + err = framework.WaitForDeploymentComplete(client, deployment) + framework.ExpectNoError(err, "waiting for the deployment status valid", image, deploymentCRDName, namespace) + + By("Deploying the webhook service") + + serviceLabels := map[string]string{"crd-webhook": "true"} + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: serviceCRDName, + Labels: map[string]string{"test": "crd-webhook"}, + }, + Spec: v1.ServiceSpec{ + Selector: serviceLabels, + Ports: []v1.ServicePort{ + { + Protocol: "TCP", + Port: 443, + TargetPort: intstr.FromInt(443), + }, + }, + }, + } + _, err = client.CoreV1().Services(namespace).Create(service) + framework.ExpectNoError(err, "creating service %s in namespace %s", serviceCRDName, namespace) + + By("Verifying the service has paired with the endpoint") + err = framework.WaitForServiceEndpointsNum(client, namespace, serviceCRDName, 1, 1*time.Second, 30*time.Second) + framework.ExpectNoError(err, "waiting for service %s/%s have %d endpoint", namespace, serviceCRDName, 1) +} + +func verifyV1Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, obj *unstructured.Unstructured) { + Expect(obj.GetAPIVersion()).To(BeEquivalentTo(crd.Spec.Group + "/v1")) + hostPort, exists := obj.Object["hostPort"] + Expect(exists).To(BeTrue()) + Expect(hostPort).To(BeEquivalentTo("localhost:8080")) + _, hostExists := obj.Object["host"] + Expect(hostExists).To(BeFalse()) + _, portExists := obj.Object["port"] + Expect(portExists).To(BeFalse()) +} + +func verifyV2Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, obj *unstructured.Unstructured) { + Expect(obj.GetAPIVersion()).To(BeEquivalentTo(crd.Spec.Group + "/v2")) + _, hostPortExists := obj.Object["hostPort"] + Expect(hostPortExists).To(BeFalse()) + host, hostExists := obj.Object["host"] + Expect(hostExists).To(BeTrue()) + Expect(host).To(BeEquivalentTo("localhost")) + port, portExists := obj.Object["port"] + Expect(portExists).To(BeTrue()) + Expect(port).To(BeEquivalentTo("8080")) +} + +func testCustomResourceConversionWebhook(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, customResourceClients map[string]dynamic.ResourceInterface) { + name := "cr-instance-1" + By("Creating a v1 custom resource") + crInstance := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": crd.Spec.Names.Kind, + "apiVersion": crd.Spec.Group + "/v1", + "metadata": map[string]interface{}{ + "name": name, + "namespace": f.Namespace.Name, + }, + "hostPort": "localhost:8080", + }, + } + _, err := customResourceClients["v1"].Create(crInstance, metav1.CreateOptions{}) + Expect(err).To(BeNil()) + By("v2 custom resource should be converted") + v2crd, err := customResourceClients["v2"].Get(name, metav1.GetOptions{}) + verifyV2Object(f, crd, v2crd) +} + +func testCRListConversion(f *framework.Framework, testCrd *framework.TestCrd) { + crd := testCrd.Crd + customResourceClients := testCrd.DynamicClients + name1 := "cr-instance-1" + name2 := "cr-instance-2" + By("Creating a v1 custom resource") + crInstance := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": crd.Spec.Names.Kind, + "apiVersion": crd.Spec.Group + "/v1", + "metadata": map[string]interface{}{ + "name": name1, + "namespace": f.Namespace.Name, + }, + "hostPort": "localhost:8080", + }, + } + _, err := customResourceClients["v1"].Create(crInstance, metav1.CreateOptions{}) + Expect(err).To(BeNil()) + + // Now cr-instance-1 is stored as v1. lets change storage version + crd, err = integration.UpdateCustomResourceDefinitionWithRetry(testCrd.ApiExtensionClient, crd.Name, func(c *v1beta1.CustomResourceDefinition) { + c.Spec.Versions = alternativeApiVersions + }) + Expect(err).To(BeNil()) + By("Create a v2 custom resource") + crInstance = &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": crd.Spec.Names.Kind, + "apiVersion": crd.Spec.Group + "/v1", + "metadata": map[string]interface{}{ + "name": name2, + "namespace": f.Namespace.Name, + }, + "hostPort": "localhost:8080", + }, + } + + // After changing a CRD, the resources for versions will be re-created that can be result in + // cancelled connection (e.g. "grpc connection closed" or "context canceled"). + // Just retrying fixes that. + for i := 0; i < 5; i++ { + _, err = customResourceClients["v1"].Create(crInstance, metav1.CreateOptions{}) + if err == nil { + break + } + } + Expect(err).To(BeNil()) + + // Now that we have a v1 and v2 object, both list operation in v1 and v2 should work as expected. + + By("List CRs in v1") + list, err := customResourceClients["v1"].List(metav1.ListOptions{}) + Expect(err).To(BeNil()) + Expect(len(list.Items)).To(BeIdenticalTo(2)) + Expect((list.Items[0].GetName() == name1 && list.Items[1].GetName() == name2) || + (list.Items[0].GetName() == name2 && list.Items[1].GetName() == name1)).To(BeTrue()) + verifyV1Object(f, crd, &list.Items[0]) + verifyV1Object(f, crd, &list.Items[1]) + + By("List CRs in v2") + list, err = customResourceClients["v2"].List(metav1.ListOptions{}) + Expect(err).To(BeNil()) + Expect(len(list.Items)).To(BeIdenticalTo(2)) + Expect((list.Items[0].GetName() == name1 && list.Items[1].GetName() == name2) || + (list.Items[0].GetName() == name2 && list.Items[1].GetName() == name1)).To(BeTrue()) + verifyV2Object(f, crd, &list.Items[0]) + verifyV2Object(f, crd, &list.Items[1]) +} diff --git a/test/e2e/apimachinery/webhook.go b/test/e2e/apimachinery/webhook.go index 72c5ea67c09..e2686dccce5 100644 --- a/test/e2e/apimachinery/webhook.go +++ b/test/e2e/apimachinery/webhook.go @@ -136,7 +136,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() { defer testcrd.CleanUp() webhookCleanup := registerWebhookForCustomResource(f, context, testcrd) defer webhookCleanup() - testCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClient) + testCustomResourceWebhook(f, testcrd.Crd, testcrd.GetV1DynamicClient()) }) It("Should unconditionally reject operations on fail closed webhook", func() { @@ -173,7 +173,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() { defer testcrd.CleanUp() webhookCleanup := registerMutatingWebhookForCustomResource(f, context, testcrd) defer webhookCleanup() - testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClient) + testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.GetV1DynamicClient()) }) It("Should deny crd creation", func() { @@ -1157,7 +1157,7 @@ func registerWebhookForCustomResource(f *framework.Framework, context *certConte Operations: []v1beta1.OperationType{v1beta1.Create}, Rule: v1beta1.Rule{ APIGroups: []string{testcrd.ApiGroup}, - APIVersions: []string{testcrd.ApiVersion}, + APIVersions: testcrd.GetAPIVersions(), Resources: []string{testcrd.GetPluralName()}, }, }}, @@ -1198,7 +1198,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c Operations: []v1beta1.OperationType{v1beta1.Create}, Rule: v1beta1.Rule{ APIGroups: []string{testcrd.ApiGroup}, - APIVersions: []string{testcrd.ApiVersion}, + APIVersions: testcrd.GetAPIVersions(), Resources: []string{testcrd.GetPluralName()}, }, }}, @@ -1217,7 +1217,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c Operations: []v1beta1.OperationType{v1beta1.Create}, Rule: v1beta1.Rule{ APIGroups: []string{testcrd.ApiGroup}, - APIVersions: []string{testcrd.ApiVersion}, + APIVersions: testcrd.GetAPIVersions(), Resources: []string{testcrd.GetPluralName()}, }, }}, @@ -1343,12 +1343,18 @@ func testCRDDenyWebhook(f *framework.Framework) { name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, "deny") kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, "deny") group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName) - apiVersion := "v1" + apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + }, + } testcrd := &framework.TestCrd{ - Name: name, - Kind: kind, - ApiGroup: group, - ApiVersion: apiVersion, + Name: name, + Kind: kind, + ApiGroup: group, + Versions: apiVersions, } // Creating a custom resource definition for use by assorted tests. @@ -1370,8 +1376,8 @@ func testCRDDenyWebhook(f *framework.Framework) { }, }, Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: testcrd.ApiGroup, - Version: testcrd.ApiVersion, + Group: testcrd.ApiGroup, + Versions: testcrd.Versions, Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ Plural: testcrd.GetPluralName(), Singular: testcrd.Name, diff --git a/test/e2e/apps/daemon_restart.go b/test/e2e/apps/daemon_restart.go index dfb2c6a1a2c..e37a2bda03f 100644 --- a/test/e2e/apps/daemon_restart.go +++ b/test/e2e/apps/daemon_restart.go @@ -281,7 +281,7 @@ var _ = SIGDescribe("DaemonRestart [Disruptive]", func() { // Requires master ssh access. framework.SkipUnlessProviderIs("gce", "aws") restarter := NewRestartConfig( - framework.GetMasterHost(), "kube-scheduler", ports.SchedulerPort, restartPollInterval, restartTimeout) + framework.GetMasterHost(), "kube-scheduler", ports.InsecureSchedulerPort, restartPollInterval, restartTimeout) // Create pods while the scheduler is down and make sure the scheduler picks them up by // scaling the rc to the same size. diff --git a/test/e2e/apps/network_partition.go b/test/e2e/apps/network_partition.go index 8fbebc96802..7d20149d3ec 100644 --- a/test/e2e/apps/network_partition.go +++ b/test/e2e/apps/network_partition.go @@ -198,10 +198,12 @@ var _ = SIGDescribe("Network Partition [Disruptive] [Slow]", func() { By(fmt.Sprintf("Block traffic from node %s to the master", node.Name)) host, err := framework.GetNodeExternalIP(&node) framework.ExpectNoError(err) - master := framework.GetMasterAddress(c) + masterAddresses := framework.GetAllMasterAddresses(c) defer func() { By(fmt.Sprintf("Unblock traffic from node %s to the master", node.Name)) - framework.UnblockNetwork(host, master) + for _, masterAddress := range masterAddresses { + framework.UnblockNetwork(host, masterAddress) + } if CurrentGinkgoTestDescription().Failed { return @@ -214,7 +216,9 @@ var _ = SIGDescribe("Network Partition [Disruptive] [Slow]", func() { } }() - framework.BlockNetwork(host, master) + for _, masterAddress := range masterAddresses { + framework.BlockNetwork(host, masterAddress) + } By("Expect to observe node and pod status change from Ready to NotReady after network partition") expectNodeReadiness(false, newNode) @@ -576,10 +580,12 @@ var _ = SIGDescribe("Network Partition [Disruptive] [Slow]", func() { By(fmt.Sprintf("Block traffic from node %s to the master", node.Name)) host, err := framework.GetNodeExternalIP(&node) framework.ExpectNoError(err) - master := framework.GetMasterAddress(c) + masterAddresses := framework.GetAllMasterAddresses(c) defer func() { By(fmt.Sprintf("Unblock traffic from node %s to the master", node.Name)) - framework.UnblockNetwork(host, master) + for _, masterAddress := range masterAddresses { + framework.UnblockNetwork(host, masterAddress) + } if CurrentGinkgoTestDescription().Failed { return @@ -589,7 +595,9 @@ var _ = SIGDescribe("Network Partition [Disruptive] [Slow]", func() { expectNodeReadiness(true, newNode) }() - framework.BlockNetwork(host, master) + for _, masterAddress := range masterAddresses { + framework.BlockNetwork(host, masterAddress) + } By("Expect to observe node and pod status change from Ready to NotReady after network partition") expectNodeReadiness(false, newNode) diff --git a/test/e2e/auth/audit.go b/test/e2e/auth/audit.go index 97aaf5c8295..0bf6b9e197a 100644 --- a/test/e2e/auth/audit.go +++ b/test/e2e/auth/audit.go @@ -106,8 +106,7 @@ var _ = SIGDescribe("Advanced Audit", func() { podChan, err := f.PodClient().Watch(watchOptions) framework.ExpectNoError(err, "failed to create watch for pods") - for range podChan.ResultChan() { - } + podChan.Stop() f.PodClient().Update(pod.Name, updatePod) @@ -159,7 +158,7 @@ var _ = SIGDescribe("Advanced Audit", func() { }, { Level: auditinternal.LevelRequest, Stage: auditinternal.StageResponseStarted, - RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout), Verb: "watch", Code: 200, User: auditTestUser, @@ -171,7 +170,7 @@ var _ = SIGDescribe("Advanced Audit", func() { }, { Level: auditinternal.LevelRequest, Stage: auditinternal.StageResponseComplete, - RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout), Verb: "watch", Code: 200, User: auditTestUser, @@ -233,8 +232,7 @@ var _ = SIGDescribe("Advanced Audit", func() { deploymentChan, err := f.ClientSet.AppsV1().Deployments(namespace).Watch(watchOptions) framework.ExpectNoError(err, "failed to create watch for deployments") - for range deploymentChan.ResultChan() { - } + deploymentChan.Stop() _, err = f.ClientSet.AppsV1().Deployments(namespace).Update(d) framework.ExpectNoError(err, "failed to update audit-deployment") @@ -288,7 +286,7 @@ var _ = SIGDescribe("Advanced Audit", func() { }, { Level: auditinternal.LevelRequest, Stage: auditinternal.StageResponseStarted, - RequestURI: fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + RequestURI: fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout), Verb: "watch", Code: 200, User: auditTestUser, @@ -300,7 +298,7 @@ var _ = SIGDescribe("Advanced Audit", func() { }, { Level: auditinternal.LevelRequest, Stage: auditinternal.StageResponseComplete, - RequestURI: fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + RequestURI: fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout), Verb: "watch", Code: 200, User: auditTestUser, @@ -368,8 +366,7 @@ var _ = SIGDescribe("Advanced Audit", func() { configMapChan, err := f.ClientSet.CoreV1().ConfigMaps(namespace).Watch(watchOptions) framework.ExpectNoError(err, "failed to create watch for config maps") - for range configMapChan.ResultChan() { - } + configMapChan.Stop() _, err = f.ClientSet.CoreV1().ConfigMaps(namespace).Update(configMap) framework.ExpectNoError(err, "failed to update audit-configmap") @@ -423,7 +420,7 @@ var _ = SIGDescribe("Advanced Audit", func() { }, { Level: auditinternal.LevelMetadata, Stage: auditinternal.StageResponseStarted, - RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout), Verb: "watch", Code: 200, User: auditTestUser, @@ -435,7 +432,7 @@ var _ = SIGDescribe("Advanced Audit", func() { }, { Level: auditinternal.LevelMetadata, Stage: auditinternal.StageResponseComplete, - RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout), Verb: "watch", Code: 200, User: auditTestUser, @@ -502,8 +499,7 @@ var _ = SIGDescribe("Advanced Audit", func() { secretChan, err := f.ClientSet.CoreV1().Secrets(namespace).Watch(watchOptions) framework.ExpectNoError(err, "failed to create watch for secrets") - for range secretChan.ResultChan() { - } + secretChan.Stop() _, err = f.ClientSet.CoreV1().Secrets(namespace).Update(secret) framework.ExpectNoError(err, "failed to update audit-secret") @@ -557,7 +553,7 @@ var _ = SIGDescribe("Advanced Audit", func() { }, { Level: auditinternal.LevelMetadata, Stage: auditinternal.StageResponseStarted, - RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/secrets?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/secrets?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout), Verb: "watch", Code: 200, User: auditTestUser, @@ -569,7 +565,7 @@ var _ = SIGDescribe("Advanced Audit", func() { }, { Level: auditinternal.LevelMetadata, Stage: auditinternal.StageResponseComplete, - RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/secrets?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/secrets?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout), Verb: "watch", Code: 200, User: auditTestUser, diff --git a/test/e2e/autoscaling/BUILD b/test/e2e/autoscaling/BUILD index df5694b1e23..3cc66811f42 100644 --- a/test/e2e/autoscaling/BUILD +++ b/test/e2e/autoscaling/BUILD @@ -44,11 +44,11 @@ go_library( "//test/e2e/scheduling:go_default_library", "//test/utils:go_default_library", "//test/utils/image:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/golang.org/x/oauth2/google:go_default_library", "//vendor/google.golang.org/api/monitoring/v3:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/e2e/autoscaling/cluster_autoscaler_scalability.go b/test/e2e/autoscaling/cluster_autoscaler_scalability.go index 5ac3f01fb8c..8832cf503f6 100644 --- a/test/e2e/autoscaling/cluster_autoscaler_scalability.go +++ b/test/e2e/autoscaling/cluster_autoscaler_scalability.go @@ -33,9 +33,9 @@ import ( testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" - "github.com/golang/glog" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "k8s.io/klog" ) const ( @@ -132,7 +132,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun } break } - glog.Infof("Made nodes schedulable again in %v", time.Since(s).String()) + klog.Infof("Made nodes schedulable again in %v", time.Since(s).String()) }) It("should scale up at all [Feature:ClusterAutoscalerScalability1]", func() { @@ -170,7 +170,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun replicas1 := additionalNodes1 * replicasPerNode replicas2 := additionalNodes2 * replicasPerNode - glog.Infof("cores per node: %v", coresPerNode) + klog.Infof("cores per node: %v", coresPerNode) // saturate cluster initialReplicas := nodeCount @@ -178,7 +178,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun defer reservationCleanup() framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(f, c)) - glog.Infof("Reserved successfully") + klog.Infof("Reserved successfully") // configure pending pods & expected scale up #1 rcConfig := reserveMemoryRCConfig(f, "extra-pod-1", replicas1, additionalNodes1*perNodeReservation, largeScaleUpTimeout) @@ -191,7 +191,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun testCleanup1 := simpleScaleUpTestWithTolerance(f, config, tolerateUnreadyNodes, tolerateUnreadyPods) defer testCleanup1() - glog.Infof("Scaled up once") + klog.Infof("Scaled up once") // configure pending pods & expected scale up #2 rcConfig2 := reserveMemoryRCConfig(f, "extra-pod-2", replicas2, additionalNodes2*perNodeReservation, largeScaleUpTimeout) @@ -204,7 +204,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun testCleanup2 := simpleScaleUpTestWithTolerance(f, config2, tolerateUnreadyNodes, tolerateUnreadyPods) defer testCleanup2() - glog.Infof("Scaled up twice") + klog.Infof("Scaled up twice") }) It("should scale down empty nodes [Feature:ClusterAutoscalerScalability3]", func() { @@ -327,7 +327,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun By("Checking if the number of nodes is as expected") nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet) - glog.Infof("Nodes: %v, expected: %v", len(nodes.Items), totalNodes) + klog.Infof("Nodes: %v, expected: %v", len(nodes.Items), totalNodes) Expect(len(nodes.Items)).Should(Equal(totalNodes)) }) @@ -390,7 +390,7 @@ func simpleScaleUpTestWithTolerance(f *framework.Framework, config *scaleUpTestC } else { framework.ExpectNoError(framework.WaitForReadyNodes(f.ClientSet, config.expectedResult.nodes, scaleUpTimeout)) } - glog.Infof("cluster is increased") + klog.Infof("cluster is increased") if tolerateMissingPodCount > 0 { framework.ExpectNoError(waitForCaPodsReadyInNamespace(f, f.ClientSet, tolerateMissingPodCount)) } else { @@ -527,5 +527,5 @@ func distributeLoad(f *framework.Framework, namespace string, id string, podDist func timeTrack(start time.Time, name string) { elapsed := time.Since(start) - glog.Infof("%s took %s", name, elapsed) + klog.Infof("%s took %s", name, elapsed) } diff --git a/test/e2e/autoscaling/cluster_size_autoscaling.go b/test/e2e/autoscaling/cluster_size_autoscaling.go index e9e0e542f08..c84d515a02b 100644 --- a/test/e2e/autoscaling/cluster_size_autoscaling.go +++ b/test/e2e/autoscaling/cluster_size_autoscaling.go @@ -48,9 +48,9 @@ import ( testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" - "github.com/golang/glog" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "k8s.io/klog" ) const ( @@ -161,7 +161,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { } break } - glog.Infof("Made nodes schedulable again in %v", time.Since(s).String()) + klog.Infof("Made nodes schedulable again in %v", time.Since(s).String()) }) It("shouldn't increase cluster size if pending pod is too large [Feature:ClusterSizeAutoscalingScaleUp]", func() { @@ -352,7 +352,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { status, err = getScaleUpStatus(c) framework.ExpectNoError(err) if status.target != target { - glog.Warningf("Final number of nodes (%v) does not match initial scale-up target (%v).", status.target, target) + klog.Warningf("Final number of nodes (%v) does not match initial scale-up target (%v).", status.target, target) } Expect(status.timestamp.Add(freshStatusLimit).Before(time.Now())).Should(Equal(false)) Expect(status.status).Should(Equal(caNoScaleUpStatus)) @@ -372,7 +372,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { // We wait for nodes to become schedulable to make sure the new nodes // will be returned by getPoolNodes below. framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, resizeTimeout)) - glog.Infof("Not enabling cluster autoscaler for the node pool (on purpose).") + klog.Infof("Not enabling cluster autoscaler for the node pool (on purpose).") By("Getting memory available on new nodes, so we can account for it when creating RC") nodes := getPoolNodes(f, extraPoolName) @@ -508,7 +508,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(runAntiAffinityPods(f, f.Namespace.Name, pods, "some-pod", labels, labels)) defer func() { framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "some-pod") - glog.Infof("RC and pods not using volume deleted") + klog.Infof("RC and pods not using volume deleted") }() By("waiting for all pods before triggering scale up") @@ -582,14 +582,14 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { newNodesSet.Delete(nodes...) if len(newNodesSet) > 1 { By(fmt.Sprintf("Spotted following new nodes in %s: %v", minMig, newNodesSet)) - glog.Infof("Usually only 1 new node is expected, investigating") - glog.Infof("Kubectl:%s\n", framework.RunKubectlOrDie("get", "nodes", "-o", "json")) + klog.Infof("Usually only 1 new node is expected, investigating") + klog.Infof("Kubectl:%s\n", framework.RunKubectlOrDie("get", "nodes", "-o", "json")) if output, err := exec.Command("gcloud", "compute", "instances", "list", "--project="+framework.TestContext.CloudConfig.ProjectID, "--zone="+framework.TestContext.CloudConfig.Zone).Output(); err == nil { - glog.Infof("Gcloud compute instances list: %s", output) + klog.Infof("Gcloud compute instances list: %s", output) } else { - glog.Errorf("Failed to get instances list: %v", err) + klog.Errorf("Failed to get instances list: %v", err) } for newNode := range newNodesSet { @@ -597,9 +597,9 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { newNode, "--project="+framework.TestContext.CloudConfig.ProjectID, "--zone="+framework.TestContext.CloudConfig.Zone).Output(); err == nil { - glog.Infof("Gcloud compute instances describe: %s", output) + klog.Infof("Gcloud compute instances describe: %s", output) } else { - glog.Errorf("Failed to get instances describe: %v", err) + klog.Errorf("Failed to get instances describe: %v", err) } } @@ -614,7 +614,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { if err == nil && node != nil { registeredNodes.Insert(nodeName) } else { - glog.Errorf("Failed to get node %v: %v", nodeName, err) + klog.Errorf("Failed to get node %v: %v", nodeName, err) } } By(fmt.Sprintf("Setting labels for registered new nodes: %v", registeredNodes.List())) @@ -976,7 +976,7 @@ func installNvidiaDriversDaemonSet() { } func execCmd(args ...string) *exec.Cmd { - glog.Infof("Executing: %s", strings.Join(args, " ")) + klog.Infof("Executing: %s", strings.Join(args, " ")) return exec.Command(args[0], args[1:]...) } @@ -1108,7 +1108,7 @@ func isRegionalCluster() bool { } func enableAutoscaler(nodePool string, minCount, maxCount int) error { - glog.Infof("Using gcloud to enable autoscaling for pool %s", nodePool) + klog.Infof("Using gcloud to enable autoscaling for pool %s", nodePool) args := []string{"container", "clusters", "update", framework.TestContext.CloudConfig.Cluster, "--enable-autoscaling", @@ -1118,10 +1118,10 @@ func enableAutoscaler(nodePool string, minCount, maxCount int) error { output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() if err != nil { - glog.Errorf("Failed config update result: %s", output) + klog.Errorf("Failed config update result: %s", output) return fmt.Errorf("Failed to enable autoscaling: %v", err) } - glog.Infof("Config update result: %s", output) + klog.Infof("Config update result: %s", output) var finalErr error for startTime := time.Now(); startTime.Add(gkeUpdateTimeout).After(time.Now()); time.Sleep(30 * time.Second) { @@ -1135,17 +1135,17 @@ func enableAutoscaler(nodePool string, minCount, maxCount int) error { } func disableAutoscaler(nodePool string, minCount, maxCount int) error { - glog.Infof("Using gcloud to disable autoscaling for pool %s", nodePool) + klog.Infof("Using gcloud to disable autoscaling for pool %s", nodePool) args := []string{"container", "clusters", "update", framework.TestContext.CloudConfig.Cluster, "--no-enable-autoscaling", "--node-pool=" + nodePool} output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() if err != nil { - glog.Errorf("Failed config update result: %s", output) + klog.Errorf("Failed config update result: %s", output) return fmt.Errorf("Failed to disable autoscaling: %v", err) } - glog.Infof("Config update result: %s", output) + klog.Infof("Config update result: %s", output) var finalErr error for startTime := time.Now(); startTime.Add(gkeUpdateTimeout).After(time.Now()); time.Sleep(30 * time.Second) { @@ -1183,7 +1183,7 @@ func addNodePool(name string, machineType string, numNodes int) { "--num-nodes=" + strconv.Itoa(numNodes), "--cluster=" + framework.TestContext.CloudConfig.Cluster} output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - glog.Infof("Creating node-pool %s: %s", name, output) + klog.Infof("Creating node-pool %s: %s", name, output) framework.ExpectNoError(err, string(output)) } @@ -1193,12 +1193,12 @@ func addGpuNodePool(name string, gpuType string, gpuCount int, numNodes int) { "--num-nodes=" + strconv.Itoa(numNodes), "--cluster=" + framework.TestContext.CloudConfig.Cluster} output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - glog.Infof("Creating node-pool %s: %s", name, output) + klog.Infof("Creating node-pool %s: %s", name, output) framework.ExpectNoError(err, string(output)) } func deleteNodePool(name string) { - glog.Infof("Deleting node pool %s", name) + klog.Infof("Deleting node pool %s", name) args := []string{"container", "node-pools", "delete", name, "--quiet", "--cluster=" + framework.TestContext.CloudConfig.Cluster} err := wait.ExponentialBackoff( @@ -1206,10 +1206,10 @@ func deleteNodePool(name string) { func() (bool, error) { output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() if err != nil { - glog.Warningf("Error deleting nodegroup - error:%v, output: %s", err, output) + klog.Warningf("Error deleting nodegroup - error:%v, output: %s", err, output) return false, nil } - glog.Infof("Node-pool deletion output: %s", output) + klog.Infof("Node-pool deletion output: %s", output) return true, nil }) framework.ExpectNoError(err) @@ -1235,7 +1235,7 @@ func getPoolInitialSize(poolName string) int { "--cluster=" + framework.TestContext.CloudConfig.Cluster, "--format=value(initialNodeCount)"} output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - glog.Infof("Node-pool initial size: %s", output) + klog.Infof("Node-pool initial size: %s", output) framework.ExpectNoError(err, string(output)) fields := strings.Fields(string(output)) Expect(len(fields)).Should(Equal(1)) @@ -1302,7 +1302,7 @@ func reserveMemory(f *framework.Framework, id string, replicas, megabytes int, e for start := time.Now(); time.Since(start) < rcCreationRetryTimeout; time.Sleep(rcCreationRetryDelay) { err := framework.RunRC(*config) if err != nil && strings.Contains(err.Error(), "Error creating replication controller") { - glog.Warningf("Failed to create memory reservation: %v", err) + klog.Warningf("Failed to create memory reservation: %v", err) continue } if expectRunning { @@ -1346,7 +1346,7 @@ func WaitForClusterSizeFuncWithUnready(c clientset.Interface, sizeFunc func(int) "spec.unschedulable": "false", }.AsSelector().String()}) if err != nil { - glog.Warningf("Failed to list nodes: %v", err) + klog.Warningf("Failed to list nodes: %v", err) continue } numNodes := len(nodes.Items) @@ -1358,10 +1358,10 @@ func WaitForClusterSizeFuncWithUnready(c clientset.Interface, sizeFunc func(int) numReady := len(nodes.Items) if numNodes == numReady+expectedUnready && sizeFunc(numNodes) { - glog.Infof("Cluster has reached the desired size") + klog.Infof("Cluster has reached the desired size") return nil } - glog.Infof("Waiting for cluster with func, current size %d, not ready nodes %d", numNodes, numNodes-numReady) + klog.Infof("Waiting for cluster with func, current size %d, not ready nodes %d", numNodes, numNodes-numReady) } return fmt.Errorf("timeout waiting %v for appropriate cluster size", timeout) } @@ -1384,21 +1384,21 @@ func waitForCaPodsReadyInNamespace(f *framework.Framework, c clientset.Interface // Failed pods in this context generally mean that they have been // double scheduled onto a node, but then failed a constraint check. if pod.Status.Phase == v1.PodFailed { - glog.Warningf("Pod has failed: %v", pod) + klog.Warningf("Pod has failed: %v", pod) } if !ready && pod.Status.Phase != v1.PodFailed { notready = append(notready, pod.Name) } } if len(notready) <= tolerateUnreadyCount { - glog.Infof("sufficient number of pods ready. Tolerating %d unready", tolerateUnreadyCount) + klog.Infof("sufficient number of pods ready. Tolerating %d unready", tolerateUnreadyCount) return nil } - glog.Infof("Too many pods are not ready yet: %v", notready) + klog.Infof("Too many pods are not ready yet: %v", notready) } - glog.Info("Timeout on waiting for pods being ready") - glog.Info(framework.RunKubectlOrDie("get", "pods", "-o", "json", "--all-namespaces")) - glog.Info(framework.RunKubectlOrDie("get", "nodes", "-o", "json")) + klog.Info("Timeout on waiting for pods being ready") + klog.Info(framework.RunKubectlOrDie("get", "pods", "-o", "json", "--all-namespaces")) + klog.Info(framework.RunKubectlOrDie("get", "nodes", "-o", "json")) // Some pods are still not running. return fmt.Errorf("Too many pods are still not running: %v", notready) @@ -1413,11 +1413,11 @@ func getAnyNode(c clientset.Interface) *v1.Node { "spec.unschedulable": "false", }.AsSelector().String()}) if err != nil { - glog.Errorf("Failed to get node list: %v", err) + klog.Errorf("Failed to get node list: %v", err) return nil } if len(nodes.Items) == 0 { - glog.Errorf("No nodes") + klog.Errorf("No nodes") return nil } return &nodes.Items[0] @@ -1476,7 +1476,7 @@ func makeNodeUnschedulable(c clientset.Interface, node *v1.Node) error { if !errors.IsConflict(err) { return err } - glog.Warningf("Got 409 conflict when trying to taint node, retries left: %v", 3-j) + klog.Warningf("Got 409 conflict when trying to taint node, retries left: %v", 3-j) } return fmt.Errorf("Failed to taint node in allowed number of retries") } @@ -1517,7 +1517,7 @@ func makeNodeSchedulable(c clientset.Interface, node *v1.Node, failOnCriticalAdd if !errors.IsConflict(err) { return err } - glog.Warningf("Got 409 conflict when trying to taint node, retries left: %v", 3-j) + klog.Warningf("Got 409 conflict when trying to taint node, retries left: %v", 3-j) } return fmt.Errorf("Failed to remove taint from node in allowed number of retries") } @@ -1696,7 +1696,7 @@ func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, namespa if !errors.IsConflict(err) { return err } - glog.Warningf("Got 409 conflict when trying to scale RC, retries left: %v", 3-j) + klog.Warningf("Got 409 conflict when trying to scale RC, retries left: %v", 3-j) rc, err = f.ClientSet.CoreV1().ReplicationControllers(namespace).Get(id, metav1.GetOptions{}) if err != nil { return err @@ -1747,7 +1747,7 @@ func manuallyIncreaseClusterSize(f *framework.Framework, originalSizes map[strin } resized := setMigSizes(newSizes) if resized { - glog.Warning("Unexpected node group size while waiting for cluster resize. Setting size to target again.") + klog.Warning("Unexpected node group size while waiting for cluster resize. Setting size to target again.") } return false } @@ -1852,7 +1852,7 @@ func getScaleUpStatus(c clientset.Interface) (*scaleUpStatus, error) { } result.target += newTarget } - glog.Infof("Cluster-Autoscaler scale-up status: %v (%v, %v)", result.status, result.ready, result.target) + klog.Infof("Cluster-Autoscaler scale-up status: %v (%v, %v)", result.status, result.ready, result.target) return &result, nil } @@ -1890,7 +1890,7 @@ func addKubeSystemPdbs(f *framework.Framework) (func(), error) { err := f.ClientSet.PolicyV1beta1().PodDisruptionBudgets("kube-system").Delete(newPdbName, &metav1.DeleteOptions{}) if err != nil { // log error, but attempt to remove other pdbs - glog.Errorf("Failed to delete PodDisruptionBudget %v, err: %v", newPdbName, err) + klog.Errorf("Failed to delete PodDisruptionBudget %v, err: %v", newPdbName, err) finalErr = err } } @@ -1943,7 +1943,7 @@ func createPriorityClasses(f *framework.Framework) func() { for className, priority := range priorityClasses { _, err := f.ClientSet.SchedulingV1beta1().PriorityClasses().Create(&schedulerapi.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: className}, Value: priority}) if err != nil { - glog.Errorf("Error creating priority class: %v", err) + klog.Errorf("Error creating priority class: %v", err) } Expect(err == nil || errors.IsAlreadyExists(err)).To(Equal(true)) } @@ -1952,7 +1952,7 @@ func createPriorityClasses(f *framework.Framework) func() { for className := range priorityClasses { err := f.ClientSet.SchedulingV1beta1().PriorityClasses().Delete(className, nil) if err != nil { - glog.Errorf("Error deleting priority class: %v", err) + klog.Errorf("Error deleting priority class: %v", err) } } } diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling.go b/test/e2e/autoscaling/horizontal_pod_autoscaling.go index 0a250d750e6..bdba3f73923 100644 --- a/test/e2e/autoscaling/horizontal_pod_autoscaling.go +++ b/test/e2e/autoscaling/horizontal_pod_autoscaling.go @@ -120,9 +120,10 @@ func (scaleTest *HPAScaleTest) run(name string, kind schema.GroupVersionKind, rc defer rc.CleanUp() hpa := common.CreateCPUHorizontalPodAutoscaler(rc, scaleTest.targetCPUUtilizationPercent, scaleTest.minPods, scaleTest.maxPods) defer common.DeleteHorizontalPodAutoscaler(rc, hpa.Name) + rc.WaitForReplicas(scaleTest.firstScale, timeToWait) if scaleTest.firstScaleStasis > 0 { - rc.EnsureDesiredReplicasInRange(scaleTest.firstScale, scaleTest.firstScale+1, scaleTest.firstScaleStasis) + rc.EnsureDesiredReplicasInRange(scaleTest.firstScale, scaleTest.firstScale+1, scaleTest.firstScaleStasis, hpa.Name) } if scaleTest.cpuBurst > 0 && scaleTest.secondScale > 0 { rc.ConsumeCPU(scaleTest.cpuBurst) diff --git a/test/e2e/common/BUILD b/test/e2e/common/BUILD index 495e99258a4..040e119a216 100644 --- a/test/e2e/common/BUILD +++ b/test/e2e/common/BUILD @@ -44,6 +44,7 @@ go_library( ], importpath = "k8s.io/kubernetes/test/e2e/common", deps = [ + "//pkg/api/v1/node:go_default_library", "//pkg/api/v1/pod:go_default_library", "//pkg/apis/core:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", @@ -79,11 +80,11 @@ go_library( "//test/e2e/framework:go_default_library", "//test/utils:go_default_library", "//test/utils/image:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/onsi/gomega/types:go_default_library", "//vendor/golang.org/x/net/websocket:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/e2e/common/autoscaling_utils.go b/test/e2e/common/autoscaling_utils.go index 36f5cd8d0da..fe8b5ab97fc 100644 --- a/test/e2e/common/autoscaling_utils.go +++ b/test/e2e/common/autoscaling_utils.go @@ -359,6 +359,10 @@ func (rc *ResourceConsumer) GetReplicas() int { return 0 } +func (rc *ResourceConsumer) GetHpa(name string) (*autoscalingv1.HorizontalPodAutoscaler, error) { + return rc.clientSet.AutoscalingV1().HorizontalPodAutoscalers(rc.nsName).Get(name, metav1.GetOptions{}) +} + func (rc *ResourceConsumer) WaitForReplicas(desiredReplicas int, duration time.Duration) { interval := 20 * time.Second err := wait.PollImmediate(interval, duration, func() (bool, error) { @@ -369,15 +373,21 @@ func (rc *ResourceConsumer) WaitForReplicas(desiredReplicas int, duration time.D framework.ExpectNoErrorWithOffset(1, err, "timeout waiting %v for %d replicas", duration, desiredReplicas) } -func (rc *ResourceConsumer) EnsureDesiredReplicas(desiredReplicas int, duration time.Duration) { - rc.EnsureDesiredReplicasInRange(desiredReplicas, desiredReplicas, duration) +func (rc *ResourceConsumer) EnsureDesiredReplicas(desiredReplicas int, duration time.Duration, hpaName string) { + rc.EnsureDesiredReplicasInRange(desiredReplicas, desiredReplicas, duration, hpaName) } -func (rc *ResourceConsumer) EnsureDesiredReplicasInRange(minDesiredReplicas, maxDesiredReplicas int, duration time.Duration) { +func (rc *ResourceConsumer) EnsureDesiredReplicasInRange(minDesiredReplicas, maxDesiredReplicas int, duration time.Duration, hpaName string) { interval := 10 * time.Second err := wait.PollImmediate(interval, duration, func() (bool, error) { replicas := rc.GetReplicas() framework.Logf("expecting there to be in [%d, %d] replicas (are: %d)", minDesiredReplicas, maxDesiredReplicas, replicas) + as, err := rc.GetHpa(hpaName) + if err != nil { + framework.Logf("Error getting HPA: %s", err) + } else { + framework.Logf("HPA status: %+v", as.Status) + } if replicas < minDesiredReplicas { return false, fmt.Errorf("number of replicas below target") } else if replicas > maxDesiredReplicas { diff --git a/test/e2e/common/configmap.go b/test/e2e/common/configmap.go index 829245fd856..c6479f32d98 100644 --- a/test/e2e/common/configmap.go +++ b/test/e2e/common/configmap.go @@ -20,6 +20,7 @@ import ( "fmt" . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" @@ -27,7 +28,7 @@ import ( imageutils "k8s.io/kubernetes/test/utils/image" ) -var _ = Describe("[sig-api-machinery] ConfigMap", func() { +var _ = Describe("[sig-node] ConfigMap", func() { f := framework.NewDefaultFramework("configmap") /* @@ -122,6 +123,11 @@ var _ = Describe("[sig-api-machinery] ConfigMap", func() { "p_data_1=value-1", "p_data_2=value-2", "p_data_3=value-3", }) }) + + It("should fail to create configMap in volume due to empty configmap key", func() { + configMap, err := newConfigMapWithEmptyKey(f) + Expect(err).To(HaveOccurred(), "created configMap %q with empty key in namespace %q", configMap.Name, f.Namespace.Name) + }) }) func newEnvFromConfigMap(f *framework.Framework, name string) *v1.ConfigMap { @@ -137,3 +143,19 @@ func newEnvFromConfigMap(f *framework.Framework, name string) *v1.ConfigMap { }, } } + +func newConfigMapWithEmptyKey(f *framework.Framework) (*v1.ConfigMap, error) { + name := "configmap-test-emptyKey-" + string(uuid.NewUUID()) + configMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace.Name, + Name: name, + }, + Data: map[string]string{ + "": "value-1", + }, + } + + By(fmt.Sprintf("Creating configMap that has name %s", configMap.Name)) + return f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(configMap) +} diff --git a/test/e2e/common/configmap_volume.go b/test/e2e/common/configmap_volume.go index 32e42a23166..40cdcd5044c 100644 --- a/test/e2e/common/configmap_volume.go +++ b/test/e2e/common/configmap_volume.go @@ -540,6 +540,26 @@ var _ = Describe("[sig-storage] ConfigMap", func() { }) }) + + //The pod is in pending during volume creation until the configMap objects are available + //or until mount the configMap volume times out. There is no configMap object defined for the pod, so it should return timout exception unless it is marked optional. + //Slow (~5 mins) + It("Should fail non-optional pod creation due to configMap object does not exist [Slow]", func() { + volumeMountPath := "/etc/configmap-volumes" + podName := "pod-configmaps-" + string(uuid.NewUUID()) + err := createNonOptionalConfigMapPod(f, volumeMountPath, podName) + Expect(err).To(HaveOccurred(), "created pod %q with non-optional configMap in namespace %q", podName, f.Namespace.Name) + }) + + //ConfigMap object defined for the pod, If a key is specified which is not present in the ConfigMap, + // the volume setup will error unless it is marked optional, during the pod creation. + //Slow (~5 mins) + It("Should fail non-optional pod creation due to the key in the configMap object does not exist [Slow]", func() { + volumeMountPath := "/etc/configmap-volumes" + podName := "pod-configmaps-" + string(uuid.NewUUID()) + err := createNonOptionalConfigMapPodWithConfig(f, volumeMountPath, podName) + Expect(err).To(HaveOccurred(), "created pod %q with non-optional configMap in namespace %q", podName, f.Namespace.Name) + }) }) func newConfigMap(f *framework.Framework, name string) *v1.ConfigMap { @@ -724,3 +744,115 @@ func doConfigMapE2EWithMappings(f *framework.Framework, uid, fsGroup int64, item } f.TestContainerOutput("consume configMaps", pod, 0, output) } + +func createNonOptionalConfigMapPod(f *framework.Framework, volumeMountPath, podName string) error { + podLogTimeout := framework.GetPodSecretUpdateTimeout(f.ClientSet) + containerTimeoutArg := fmt.Sprintf("--retry_time=%v", int(podLogTimeout.Seconds())) + falseValue := false + + createName := "cm-test-opt-create-" + string(uuid.NewUUID()) + createContainerName := "createcm-volume-test" + createVolumeName := "createcm-volume" + + //creating a pod without configMap object created, by mentioning the configMap volume source's local reference name + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: createVolumeName, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: createName, + }, + Optional: &falseValue, + }, + }, + }, + }, + Containers: []v1.Container{ + { + Name: createContainerName, + Image: imageutils.GetE2EImage(imageutils.Mounttest), + Command: []string{"/mounttest", "--break_on_expected_content=false", containerTimeoutArg, "--file_content_in_loop=/etc/configmap-volumes/create/data-1"}, + VolumeMounts: []v1.VolumeMount{ + { + Name: createVolumeName, + MountPath: path.Join(volumeMountPath, "create"), + ReadOnly: true, + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + }, + } + By("Creating the pod") + pod = f.PodClient().Create(pod) + return f.WaitForPodRunning(pod.Name) +} + +func createNonOptionalConfigMapPodWithConfig(f *framework.Framework, volumeMountPath, podName string) error { + podLogTimeout := framework.GetPodSecretUpdateTimeout(f.ClientSet) + containerTimeoutArg := fmt.Sprintf("--retry_time=%v", int(podLogTimeout.Seconds())) + falseValue := false + + createName := "cm-test-opt-create-" + string(uuid.NewUUID()) + createContainerName := "createcm-volume-test" + createVolumeName := "createcm-volume" + configMap := newConfigMap(f, createName) + + By(fmt.Sprintf("Creating configMap with name %s", configMap.Name)) + var err error + if configMap, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(configMap); err != nil { + framework.Failf("unable to create test configMap %s: %v", configMap.Name, err) + } + //creating a pod with configMap object, but with different key which is not present in configMap object. + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: createVolumeName, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: createName, + }, + Items: []v1.KeyToPath{ + { + Key: "data-4", + Path: "path/to/data-4", + }, + }, + Optional: &falseValue, + }, + }, + }, + }, + Containers: []v1.Container{ + { + Name: createContainerName, + Image: imageutils.GetE2EImage(imageutils.Mounttest), + Command: []string{"/mounttest", "--break_on_expected_content=false", containerTimeoutArg, "--file_content_in_loop=/etc/configmap-volumes/create/data-1"}, + VolumeMounts: []v1.VolumeMount{ + { + Name: createVolumeName, + MountPath: path.Join(volumeMountPath, "create"), + ReadOnly: true, + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + }, + } + By("Creating the pod") + pod = f.PodClient().Create(pod) + return f.WaitForPodRunning(pod.Name) +} diff --git a/test/e2e/common/downward_api.go b/test/e2e/common/downward_api.go index 3776dd0bbc7..00a2876a837 100644 --- a/test/e2e/common/downward_api.go +++ b/test/e2e/common/downward_api.go @@ -35,7 +35,7 @@ var ( podUIDVersion = utilversion.MustParseSemantic("v1.8.0") ) -var _ = Describe("[sig-api-machinery] Downward API", func() { +var _ = Describe("[sig-node] Downward API", func() { f := framework.NewDefaultFramework("downward-api") /* diff --git a/test/e2e/common/host_path.go b/test/e2e/common/host_path.go index 942481a7279..2728b6e4c32 100644 --- a/test/e2e/common/host_path.go +++ b/test/e2e/common/host_path.go @@ -116,101 +116,6 @@ var _ = Describe("[sig-storage] HostPath", func() { "content of file \"" + filePathInReader + "\": mount-tester new file", }) }) - - It("should support existing directory subPath", func() { - framework.SkipUnlessSSHKeyPresent() - - subPath := "sub-path" - fileName := "test-file" - retryDuration := 180 - - filePathInWriter := path.Join(volumePath, fileName) - filePathInReader := path.Join(volumePath, subPath, fileName) - - source := &v1.HostPathVolumeSource{ - Path: "/tmp", - } - pod := testPodWithHostVol(volumePath, source) - nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) - pod.Spec.NodeName = nodeList.Items[0].Name - - // Create the subPath directory on the host - existing := path.Join(source.Path, subPath) - nodeIP, err := framework.GetNodeExternalIP(&nodeList.Items[0]) - if err != nil { - nodeIP, err = framework.GetNodeInternalIP(&nodeList.Items[0]) - } - framework.ExpectNoError(err) - result, err := framework.SSH(fmt.Sprintf("mkdir -p %s", existing), nodeIP, framework.TestContext.Provider) - framework.LogSSHResult(result) - framework.ExpectNoError(err) - if result.Code != 0 { - framework.Failf("mkdir returned non-zero") - } - - // Write the file in the subPath from container 0 - container := &pod.Spec.Containers[0] - container.VolumeMounts[0].SubPath = subPath - container.Args = []string{ - fmt.Sprintf("--new_file_0644=%v", filePathInWriter), - fmt.Sprintf("--file_mode=%v", filePathInWriter), - } - - // Read it from outside the subPath from container 1 - pod.Spec.Containers[1].Args = []string{ - fmt.Sprintf("--file_content_in_loop=%v", filePathInReader), - fmt.Sprintf("--retry_time=%d", retryDuration), - } - - f.TestContainerOutput("hostPath subPath", pod, 1, []string{ - "content of file \"" + filePathInReader + "\": mount-tester new file", - }) - }) - - // TODO consolidate common code of this test and above - It("should support existing single file subPath", func() { - framework.SkipUnlessSSHKeyPresent() - - subPath := "sub-path-test-file" - retryDuration := 180 - - filePathInReader := path.Join(volumePath, subPath) - - source := &v1.HostPathVolumeSource{ - Path: "/tmp", - } - pod := testPodWithHostVol(volumePath, source) - nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) - pod.Spec.NodeName = nodeList.Items[0].Name - - // Create the subPath file on the host - existing := path.Join(source.Path, subPath) - nodeIP, err := framework.GetNodeExternalIP(&nodeList.Items[0]) - if err != nil { - nodeIP, err = framework.GetNodeInternalIP(&nodeList.Items[0]) - } - framework.ExpectNoError(err) - result, err := framework.SSH(fmt.Sprintf("echo \"mount-tester new file\" > %s", existing), nodeIP, framework.TestContext.Provider) - framework.LogSSHResult(result) - framework.ExpectNoError(err) - if result.Code != 0 { - framework.Failf("echo returned non-zero") - } - - // Mount the file to the subPath in container 0 - container := &pod.Spec.Containers[0] - container.VolumeMounts[0].SubPath = subPath - - // Read it from outside the subPath from container 1 - pod.Spec.Containers[1].Args = []string{ - fmt.Sprintf("--file_content_in_loop=%v", filePathInReader), - fmt.Sprintf("--retry_time=%d", retryDuration), - } - - f.TestContainerOutput("hostPath subPath", pod, 1, []string{ - "content of file \"" + filePathInReader + "\": mount-tester new file", - }) - }) }) //These constants are borrowed from the other test. diff --git a/test/e2e/common/kubelet.go b/test/e2e/common/kubelet.go index 8038d325aa4..bfe791f1b51 100644 --- a/test/e2e/common/kubelet.go +++ b/test/e2e/common/kubelet.go @@ -175,7 +175,7 @@ var _ = framework.KubeDescribe("Kubelet", func() { buf.ReadFrom(rc) hostsFileContent := buf.String() - if !strings.Contains(hostsFileContent, "123.45.67.89\tfoo") || !strings.Contains(hostsFileContent, "123.45.67.89\tbar") { + if !strings.Contains(hostsFileContent, "123.45.67.89\tfoo\tbar") { return fmt.Errorf("expected hosts file to contain entries from HostAliases. Got:\n%+v", hostsFileContent) } diff --git a/test/e2e/common/kubelet_etc_hosts.go b/test/e2e/common/kubelet_etc_hosts.go index d4e267b7b5a..64d4f6c2b97 100644 --- a/test/e2e/common/kubelet_etc_hosts.go +++ b/test/e2e/common/kubelet_etc_hosts.go @@ -20,10 +20,10 @@ import ( "strings" "time" - "github.com/golang/glog" . "github.com/onsi/ginkgo" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" "k8s.io/kubernetes/test/e2e/framework" imageutils "k8s.io/kubernetes/test/utils/image" ) @@ -126,7 +126,7 @@ func assertManagedStatus( } } - glog.Warningf( + klog.Warningf( "For pod: %s, name: %s, expected %t, (/etc/hosts was %q), (/etc/hosts-original was %q), retryCount: %d", podName, name, expectedIsManaged, etcHostsContent, etcHostsOriginalContent, retryCount) diff --git a/test/e2e/common/node_lease.go b/test/e2e/common/node_lease.go index 66a37c58bb1..633b4e4f4c2 100644 --- a/test/e2e/common/node_lease.go +++ b/test/e2e/common/node_lease.go @@ -23,7 +23,9 @@ import ( coordv1beta1 "k8s.io/api/coordination/v1beta1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + v1node "k8s.io/kubernetes/pkg/api/v1/node" "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo" @@ -31,33 +33,41 @@ import ( ) var _ = framework.KubeDescribe("[Feature:NodeLease][NodeAlphaFeature:NodeLease]", func() { + var nodeName string f := framework.NewDefaultFramework("node-lease-test") + + BeforeEach(func() { + nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet) + Expect(len(nodes.Items)).NotTo(BeZero()) + nodeName = nodes.Items[0].ObjectMeta.Name + }) + Context("when the NodeLease feature is enabled", func() { - It("the Kubelet should create and update a lease in the kube-node-lease namespace", func() { + It("the kubelet should create and update a lease in the kube-node-lease namespace", func() { leaseClient := f.ClientSet.CoordinationV1beta1().Leases(corev1.NamespaceNodeLease) var ( err error lease *coordv1beta1.Lease ) - // check that lease for this Kubelet exists in the kube-node-lease namespace + By("check that lease for this Kubelet exists in the kube-node-lease namespace") Eventually(func() error { - lease, err = leaseClient.Get(framework.TestContext.NodeName, metav1.GetOptions{}) + lease, err = leaseClient.Get(nodeName, metav1.GetOptions{}) if err != nil { return err } return nil }, 5*time.Minute, 5*time.Second).Should(BeNil()) // check basic expectations for the lease - Expect(expectLease(lease)).To(BeNil()) - // ensure that at least one lease renewal happens within the - // lease duration by checking for a change to renew time + Expect(expectLease(lease, nodeName)).To(BeNil()) + + By("check that node lease is updated at least once within the lease duration") Eventually(func() error { - newLease, err := leaseClient.Get(framework.TestContext.NodeName, metav1.GetOptions{}) + newLease, err := leaseClient.Get(nodeName, metav1.GetOptions{}) if err != nil { return err } // check basic expectations for the latest lease - if err := expectLease(newLease); err != nil { + if err := expectLease(newLease, nodeName); err != nil { return err } // check that RenewTime has been updated on the latest lease @@ -68,12 +78,76 @@ var _ = framework.KubeDescribe("[Feature:NodeLease][NodeAlphaFeature:NodeLease]" } return nil }, time.Duration(*lease.Spec.LeaseDurationSeconds)*time.Second, - time.Duration(*lease.Spec.LeaseDurationSeconds/3)*time.Second) + time.Duration(*lease.Spec.LeaseDurationSeconds/4)*time.Second) + }) + + It("the kubelet should report node status infrequently", func() { + By("wait until node is ready") + framework.WaitForNodeToBeReady(f.ClientSet, nodeName, 5*time.Minute) + + By("wait until there is node lease") + var err error + var lease *coordv1beta1.Lease + Eventually(func() error { + lease, err = f.ClientSet.CoordinationV1beta1().Leases(corev1.NamespaceNodeLease).Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + return nil + }, 5*time.Minute, 5*time.Second).Should(BeNil()) + // check basic expectations for the lease + Expect(expectLease(lease, nodeName)).To(BeNil()) + leaseDuration := time.Duration(*lease.Spec.LeaseDurationSeconds) * time.Second + + By("verify NodeStatus report period is longer than lease duration") + // NodeStatus is reported from node to master when there is some change or + // enough time has passed. So for here, keep checking the time diff + // between 2 NodeStatus report, until it is longer than lease duration ( + // the same as nodeMonitorGracePeriod). + heartbeatTime := getNextReadyConditionHeartbeatTime(f.ClientSet, nodeName, metav1.Time{}) + Eventually(func() error { + nextHeartbeatTime := getNextReadyConditionHeartbeatTime(f.ClientSet, nodeName, heartbeatTime) + + if nextHeartbeatTime.Time.After(heartbeatTime.Time.Add(leaseDuration)) { + return nil + } + heartbeatTime = nextHeartbeatTime + return fmt.Errorf("node status report period is shorter than lease duration") + + // Enter next round immediately. + }, 5*time.Minute, time.Nanosecond).Should(BeNil()) + + By("verify node is still in ready status even though node status report is infrequent") + // This check on node status is only meaningful when this e2e test is + // running as cluster e2e test, because node e2e test does not create and + // run controller manager, i.e., no node lifecycle controller. + node, err := f.ClientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + Expect(err).To(BeNil()) + _, readyCondition := v1node.GetNodeCondition(&node.Status, corev1.NodeReady) + Expect(readyCondition.Status).To(Equal(corev1.ConditionTrue)) }) }) }) -func expectLease(lease *coordv1beta1.Lease) error { +func getNextReadyConditionHeartbeatTime(clientSet clientset.Interface, nodeName string, prevHeartbeatTime metav1.Time) metav1.Time { + var newHeartbeatTime metav1.Time + Eventually(func() error { + node, err := clientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + _, readyCondition := v1node.GetNodeCondition(&node.Status, corev1.NodeReady) + Expect(readyCondition.Status).To(Equal(corev1.ConditionTrue)) + newHeartbeatTime = readyCondition.LastHeartbeatTime + if prevHeartbeatTime.Before(&newHeartbeatTime) { + return nil + } + return fmt.Errorf("heartbeat has not changed yet") + }, 5*time.Minute, 5*time.Second).Should(BeNil()) + return newHeartbeatTime +} + +func expectLease(lease *coordv1beta1.Lease, nodeName string) error { // expect values for HolderIdentity, LeaseDurationSeconds, and RenewTime if lease.Spec.HolderIdentity == nil { return fmt.Errorf("Spec.HolderIdentity should not be nil") @@ -85,8 +159,8 @@ func expectLease(lease *coordv1beta1.Lease) error { return fmt.Errorf("Spec.RenewTime should not be nil") } // ensure that the HolderIdentity matches the node name - if *lease.Spec.HolderIdentity != framework.TestContext.NodeName { - return fmt.Errorf("Spec.HolderIdentity (%v) should match the node name (%v)", *lease.Spec.HolderIdentity, framework.TestContext.NodeName) + if *lease.Spec.HolderIdentity != nodeName { + return fmt.Errorf("Spec.HolderIdentity (%v) should match the node name (%v)", *lease.Spec.HolderIdentity, nodeName) } return nil } diff --git a/test/e2e/common/projected_configmap.go b/test/e2e/common/projected_configmap.go index 5a8fda22966..2616189d42b 100644 --- a/test/e2e/common/projected_configmap.go +++ b/test/e2e/common/projected_configmap.go @@ -481,6 +481,26 @@ var _ = Describe("[sig-storage] Projected configMap", func() { }) }) + + //The pod is in pending during volume creation until the configMap objects are available + //or until mount the configMap volume times out. There is no configMap object defined for the pod, so it should return timout exception unless it is marked optional. + //Slow (~5 mins) + It("Should fail non-optional pod creation due to configMap object does not exist [Slow]", func() { + volumeMountPath := "/etc/projected-configmap-volumes" + podName := "pod-projected-configmaps-" + string(uuid.NewUUID()) + err := createNonOptionalConfigMapPod(f, volumeMountPath, podName) + Expect(err).To(HaveOccurred(), "created pod %q with non-optional configMap in namespace %q", podName, f.Namespace.Name) + }) + + //ConfigMap object defined for the pod, If a key is specified which is not present in the ConfigMap, + // the volume setup will error unless it is marked optional, during the pod creation. + //Slow (~5 mins) + It("Should fail non-optional pod creation due to the key in the configMap object does not exist [Slow]", func() { + volumeMountPath := "/etc/configmap-volumes" + podName := "pod-configmaps-" + string(uuid.NewUUID()) + err := createNonOptionalConfigMapPodWithConfig(f, volumeMountPath, podName) + Expect(err).To(HaveOccurred(), "created pod %q with non-optional configMap in namespace %q", podName, f.Namespace.Name) + }) }) func doProjectedConfigMapE2EWithoutMappings(f *framework.Framework, uid, fsGroup int64, defaultMode *int32) { diff --git a/test/e2e/common/projected_secret.go b/test/e2e/common/projected_secret.go index 9b0bb49ec80..4f88da9f3e0 100644 --- a/test/e2e/common/projected_secret.go +++ b/test/e2e/common/projected_secret.go @@ -399,6 +399,26 @@ var _ = Describe("[sig-storage] Projected secret", func() { Eventually(pollUpdateLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("value-3")) Eventually(pollDeleteLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("Error reading file /etc/projected-secret-volumes/delete/data-1")) }) + + //The secret is in pending during volume creation until the secret objects are available + //or until mount the secret volume times out. There is no secret object defined for the pod, so it should return timout exception unless it is marked optional. + //Slow (~5 mins) + It("Should fail non-optional pod creation due to secret object does not exist [Slow]", func() { + volumeMountPath := "/etc/projected-secret-volumes" + podName := "pod-secrets-" + string(uuid.NewUUID()) + err := createNonOptionalSecretPod(f, volumeMountPath, podName) + Expect(err).To(HaveOccurred(), "created pod %q with non-optional secret in namespace %q", podName, f.Namespace.Name) + }) + + //Secret object defined for the pod, If a key is specified which is not present in the secret, + // the volume setup will error unless it is marked optional, during the pod creation. + //Slow (~5 mins) + It("Should fail non-optional pod creation due to the key in the secret object does not exist [Slow]", func() { + volumeMountPath := "/etc/secret-volumes" + podName := "pod-secrets-" + string(uuid.NewUUID()) + err := createNonOptionalSecretPodWithSecret(f, volumeMountPath, podName) + Expect(err).To(HaveOccurred(), "created pod %q with non-optional secret in namespace %q", podName, f.Namespace.Name) + }) }) func doProjectedSecretE2EWithoutMapping(f *framework.Framework, defaultMode *int32, diff --git a/test/e2e/common/secrets.go b/test/e2e/common/secrets.go index 4da60718fd6..ce771c22ebd 100644 --- a/test/e2e/common/secrets.go +++ b/test/e2e/common/secrets.go @@ -26,6 +26,7 @@ import ( imageutils "k8s.io/kubernetes/test/utils/image" . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var _ = Describe("[sig-api-machinery] Secrets", func() { @@ -124,6 +125,11 @@ var _ = Describe("[sig-api-machinery] Secrets", func() { "p_data_1=value-1", "p_data_2=value-2", "p_data_3=value-3", }) }) + + It("should fail to create secret in volume due to empty secret key", func() { + secret, err := createEmptyKeySecretForTest(f) + Expect(err).To(HaveOccurred(), "created secret %q with empty key in namespace %q", secret.Name, f.Namespace.Name) + }) }) func newEnvFromSecret(namespace, name string) *v1.Secret { @@ -139,3 +145,18 @@ func newEnvFromSecret(namespace, name string) *v1.Secret { }, } } + +func createEmptyKeySecretForTest(f *framework.Framework) (*v1.Secret, error) { + secretName := "secret-emptyKey-test-" + string(uuid.NewUUID()) + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace.Name, + Name: secretName, + }, + Data: map[string][]byte{ + "": []byte("value-1\n"), + }, + } + By(fmt.Sprintf("Creating projection with secret that has name %s", secret.Name)) + return f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret) +} diff --git a/test/e2e/common/secrets_volume.go b/test/e2e/common/secrets_volume.go index 432cd21afd5..e700c699e80 100644 --- a/test/e2e/common/secrets_volume.go +++ b/test/e2e/common/secrets_volume.go @@ -364,6 +364,26 @@ var _ = Describe("[sig-storage] Secrets", func() { Eventually(pollUpdateLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("value-3")) Eventually(pollDeleteLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("Error reading file /etc/secret-volumes/delete/data-1")) }) + + //The secret is in pending during volume creation until the secret objects are available + //or until mount the secret volume times out. There is no secret object defined for the pod, so it should return timout exception unless it is marked optional. + //Slow (~5 mins) + It("Should fail non-optional pod creation due to secret object does not exist [Slow]", func() { + volumeMountPath := "/etc/secret-volumes" + podName := "pod-secrets-" + string(uuid.NewUUID()) + err := createNonOptionalSecretPod(f, volumeMountPath, podName) + Expect(err).To(HaveOccurred(), "created pod %q with non-optional secret in namespace %q", podName, f.Namespace.Name) + }) + + //Secret object defined for the pod, If a key is specified which is not present in the secret, + // the volume setup will error unless it is marked optional, during the pod creation. + //Slow (~5 mins) + It("Should fail non-optional pod creation due to the key in the secret object does not exist [Slow]", func() { + volumeMountPath := "/etc/secret-volumes" + podName := "pod-secrets-" + string(uuid.NewUUID()) + err := createNonOptionalSecretPodWithSecret(f, volumeMountPath, podName) + Expect(err).To(HaveOccurred(), "created pod %q with non-optional secret in namespace %q", podName, f.Namespace.Name) + }) }) func secretForTest(namespace, name string) *v1.Secret { @@ -521,3 +541,112 @@ func doSecretE2EWithMapping(f *framework.Framework, mode *int32) { f.TestContainerOutput("consume secrets", pod, 0, expectedOutput) } + +func createNonOptionalSecretPod(f *framework.Framework, volumeMountPath, podName string) error { + podLogTimeout := framework.GetPodSecretUpdateTimeout(f.ClientSet) + containerTimeoutArg := fmt.Sprintf("--retry_time=%v", int(podLogTimeout.Seconds())) + falseValue := false + + createName := "s-test-opt-create-" + string(uuid.NewUUID()) + createContainerName := "creates-volume-test" + createVolumeName := "creates-volume" + + //creating a pod without secret object created, by mentioning the secret volume source reference name + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: createVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: createName, + Optional: &falseValue, + }, + }, + }, + }, + Containers: []v1.Container{ + { + Name: createContainerName, + Image: imageutils.GetE2EImage(imageutils.Mounttest), + Command: []string{"/mounttest", "--break_on_expected_content=false", containerTimeoutArg, "--file_content_in_loop=/etc/secret-volumes/create/data-1"}, + VolumeMounts: []v1.VolumeMount{ + { + Name: createVolumeName, + MountPath: path.Join(volumeMountPath, "create"), + ReadOnly: true, + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + }, + } + By("Creating the pod") + pod = f.PodClient().Create(pod) + return f.WaitForPodRunning(pod.Name) +} + +func createNonOptionalSecretPodWithSecret(f *framework.Framework, volumeMountPath, podName string) error { + podLogTimeout := framework.GetPodSecretUpdateTimeout(f.ClientSet) + containerTimeoutArg := fmt.Sprintf("--retry_time=%v", int(podLogTimeout.Seconds())) + falseValue := false + + createName := "s-test-opt-create-" + string(uuid.NewUUID()) + createContainerName := "creates-volume-test" + createVolumeName := "creates-volume" + + secret := secretForTest(f.Namespace.Name, createName) + + By(fmt.Sprintf("Creating secret with name %s", secret.Name)) + var err error + if secret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret); err != nil { + framework.Failf("unable to create test secret %s: %v", secret.Name, err) + } + //creating a pod with secret object, with the key which is not present in secret object. + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: createVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: createName, + Items: []v1.KeyToPath{ + { + Key: "data_4", + Path: "value-4\n", + }, + }, + Optional: &falseValue, + }, + }, + }, + }, + Containers: []v1.Container{ + { + Name: createContainerName, + Image: imageutils.GetE2EImage(imageutils.Mounttest), + Command: []string{"/mounttest", "--break_on_expected_content=false", containerTimeoutArg, "--file_content_in_loop=/etc/secret-volumes/create/data-1"}, + VolumeMounts: []v1.VolumeMount{ + { + Name: createVolumeName, + MountPath: path.Join(volumeMountPath, "create"), + ReadOnly: true, + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + }, + } + By("Creating the pod") + pod = f.PodClient().Create(pod) + return f.WaitForPodRunning(pod.Name) +} diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 1b06acf97c1..d05483d20c4 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -24,11 +24,11 @@ import ( "testing" "time" - "github.com/golang/glog" "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/config" "github.com/onsi/ginkgo/reporters" "github.com/onsi/gomega" + "k8s.io/klog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtimeutils "k8s.io/apimachinery/pkg/util/runtime" @@ -74,7 +74,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { c, err := framework.LoadClientset() if err != nil { - glog.Fatal("Error loading client: ", err) + klog.Fatal("Error loading client: ", err) } // Delete any namespaces except those created by the system. This ensures no @@ -89,7 +89,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { if err != nil { framework.Failf("Error deleting orphaned namespaces: %v", err) } - glog.Infof("Waiting for deletion of the following namespaces: %v", deleted) + klog.Infof("Waiting for deletion of the following namespaces: %v", deleted) if err := framework.WaitForNamespacesDeleted(c, deleted, framework.NamespaceCleanupTimeout); err != nil { framework.Failf("Failed to delete orphaned namespaces %v: %v", deleted, err) } @@ -216,12 +216,12 @@ func RunE2ETests(t *testing.T) { // TODO: we should probably only be trying to create this directory once // rather than once-per-Ginkgo-node. if err := os.MkdirAll(framework.TestContext.ReportDir, 0755); err != nil { - glog.Errorf("Failed creating report directory: %v", err) + klog.Errorf("Failed creating report directory: %v", err) } else { r = append(r, reporters.NewJUnitReporter(path.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", framework.TestContext.ReportPrefix, config.GinkgoConfig.ParallelNode)))) } } - glog.Infof("Starting e2e run %q on Ginkgo node %d", framework.RunId, config.GinkgoConfig.ParallelNode) + klog.Infof("Starting e2e run %q on Ginkgo node %d", framework.RunId, config.GinkgoConfig.ParallelNode) ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes e2e suite", r) } diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index d24c425b966..6a1be393003 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -119,6 +119,7 @@ go_library( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/restmapper:go_default_library", "//staging/src/k8s.io/client-go/scale:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library", @@ -133,7 +134,6 @@ go_library( "//test/e2e/perftype:go_default_library", "//test/utils:go_default_library", "//test/utils/image:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/ginkgo/config:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", @@ -143,6 +143,7 @@ go_library( "//vendor/github.com/prometheus/common/model:go_default_library", "//vendor/golang.org/x/crypto/ssh:go_default_library", "//vendor/golang.org/x/net/websocket:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/test/e2e/framework/authorizer_util.go b/test/e2e/framework/authorizer_util.go index c77d98e893e..cd355267002 100644 --- a/test/e2e/framework/authorizer_util.go +++ b/test/e2e/framework/authorizer_util.go @@ -17,7 +17,7 @@ limitations under the License. package framework import ( - "github.com/golang/glog" + "k8s.io/klog" "sync" "time" @@ -62,7 +62,7 @@ func WaitForNamedAuthorizationUpdate(c v1beta1authorization.SubjectAccessReviews // GKE doesn't enable the SAR endpoint. Without this endpoint, we cannot determine if the policy engine // has adjusted as expected. In this case, simply wait one second and hope it's up to date if apierrors.IsNotFound(err) { - glog.Info("SubjectAccessReview endpoint is missing") + klog.Info("SubjectAccessReview endpoint is missing") time.Sleep(1 * time.Second) return true, nil } @@ -94,7 +94,7 @@ func BindClusterRole(c v1beta1rbac.ClusterRoleBindingsGetter, clusterRole, ns st // if we failed, don't fail the entire test because it may still work. RBAC may simply be disabled. if err != nil { - glog.Errorf("Error binding clusterrole/%s for %q for %v\n", clusterRole, ns, subjects) + klog.Errorf("Error binding clusterrole/%s for %q for %v\n", clusterRole, ns, subjects) } } @@ -124,7 +124,7 @@ func bindInNamespace(c v1beta1rbac.RoleBindingsGetter, roleType, role, ns string // if we failed, don't fail the entire test because it may still work. RBAC may simply be disabled. if err != nil { - glog.Errorf("Error binding %s/%s into %q for %v\n", roleType, role, ns, subjects) + klog.Errorf("Error binding %s/%s into %q for %v\n", roleType, role, ns, subjects) } } diff --git a/test/e2e/framework/crd_util.go b/test/e2e/framework/crd_util.go index 0421158d4aa..dc48d188d76 100644 --- a/test/e2e/framework/crd_util.go +++ b/test/e2e/framework/crd_util.go @@ -35,25 +35,23 @@ type TestCrd struct { Name string Kind string ApiGroup string - ApiVersion string + Versions []apiextensionsv1beta1.CustomResourceDefinitionVersion ApiExtensionClient *crdclientset.Clientset Crd *apiextensionsv1beta1.CustomResourceDefinition - DynamicClient dynamic.ResourceInterface + DynamicClients map[string]dynamic.ResourceInterface CleanUp CleanCrdFn } // CreateTestCRD creates a new CRD specifically for the calling test. -func CreateTestCRD(f *Framework) (*TestCrd, error) { +func CreateMultiVersionTestCRD(f *Framework, group string, apiVersions []apiextensionsv1beta1.CustomResourceDefinitionVersion, conversionWebhook *apiextensionsv1beta1.WebhookClientConfig) (*TestCrd, error) { suffix := randomSuffix() name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, suffix) kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, suffix) - group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName) - apiVersion := "v1" testcrd := &TestCrd{ - Name: name, - Kind: kind, - ApiGroup: group, - ApiVersion: apiVersion, + Name: name, + Kind: kind, + ApiGroup: group, + Versions: apiVersions, } // Creating a custom resource definition for use by assorted tests. @@ -75,6 +73,13 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) { crd := newCRDForTest(testcrd) + if conversionWebhook != nil { + crd.Spec.Conversion = &apiextensionsv1beta1.CustomResourceConversion{ + Strategy: "Webhook", + WebhookClientConfig: conversionWebhook, + } + } + //create CRD and waits for the resource to be recognized and available. crd, err = fixtures.CreateNewCustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient) if err != nil { @@ -82,12 +87,17 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) { return nil, err } - gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: crd.Spec.Version, Resource: crd.Spec.Names.Plural} - resourceClient := dynamicClient.Resource(gvr).Namespace(f.Namespace.Name) + resourceClients := map[string]dynamic.ResourceInterface{} + for _, v := range crd.Spec.Versions { + if v.Served { + gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: v.Name, Resource: crd.Spec.Names.Plural} + resourceClients[v.Name] = dynamicClient.Resource(gvr).Namespace(f.Namespace.Name) + } + } testcrd.ApiExtensionClient = apiExtensionClient testcrd.Crd = crd - testcrd.DynamicClient = resourceClient + testcrd.DynamicClients = resourceClients testcrd.CleanUp = func() error { err := fixtures.DeleteCustomResourceDefinition(crd, apiExtensionClient) if err != nil { @@ -98,13 +108,26 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) { return testcrd, nil } +// CreateTestCRD creates a new CRD specifically for the calling test. +func CreateTestCRD(f *Framework) (*TestCrd, error) { + group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName) + apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + }, + } + return CreateMultiVersionTestCRD(f, group, apiVersions, nil) +} + // newCRDForTest generates a CRD definition for the test func newCRDForTest(testcrd *TestCrd) *apiextensionsv1beta1.CustomResourceDefinition { return &apiextensionsv1beta1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: testcrd.GetMetaName()}, Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: testcrd.ApiGroup, - Version: testcrd.ApiVersion, + Group: testcrd.ApiGroup, + Versions: testcrd.Versions, Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ Plural: testcrd.GetPluralName(), Singular: testcrd.Name, @@ -130,3 +153,17 @@ func (c *TestCrd) GetPluralName() string { func (c *TestCrd) GetListName() string { return c.Name + "List" } + +func (c *TestCrd) GetAPIVersions() []string { + ret := []string{} + for _, v := range c.Versions { + if v.Served { + ret = append(ret, v.Name) + } + } + return ret +} + +func (c *TestCrd) GetV1DynamicClient() dynamic.ResourceInterface { + return c.DynamicClients["v1"] +} diff --git a/test/e2e/framework/create.go b/test/e2e/framework/create.go index 6a3bc3e5b27..b4d38beda68 100644 --- a/test/e2e/framework/create.go +++ b/test/e2e/framework/create.go @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/test/e2e/framework/testfiles" ) @@ -253,23 +254,16 @@ type ItemFactory interface { // If the item is of an unsupported type, it must return // an error that has ItemNotSupported as cause. Create(f *Framework, item interface{}) (func() error, error) - - // UniqueName returns just the name (for global items) - // or / (for namespaced items) if the - // item has the right type for this factory, otherwise - // the empty string. - UniqueName(item interface{}) string } // DescribeItem always returns a string that describes the item, -// usually by calling out to a factory that supports the item -// type. If that fails, the entire item gets converted to a string. +// usually by calling out to cache.MetaNamespaceKeyFunc which +// concatenates namespace (if set) and name. If that fails, the entire +// item gets converted to a string. func DescribeItem(item interface{}) string { - for _, factory := range Factories { - name := factory.UniqueName(item) - if name != "" { - return fmt.Sprintf("%T: %s", item, name) - } + key, err := cache.MetaNamespaceKeyFunc(item) + if err == nil && key != "" { + return fmt.Sprintf("%T: %s", item, key) } return fmt.Sprintf("%T: %s", item, item) } @@ -279,15 +273,16 @@ func DescribeItem(item interface{}) string { var ItemNotSupported = errors.New("not supported") var Factories = map[What]ItemFactory{ - {"ClusterRole"}: &ClusterRoleFactory{}, - {"ClusterRoleBinding"}: &ClusterRoleBindingFactory{}, - {"DaemonSet"}: &DaemonSetFactory{}, - {"Role"}: &RoleFactory{}, - {"RoleBinding"}: &RoleBindingFactory{}, - {"ServiceAccount"}: &ServiceAccountFactory{}, - {"StatefulSet"}: &StatefulSetFactory{}, - {"StorageClass"}: &StorageClassFactory{}, - {"Service"}: &ServiceFactory{}, + {"ClusterRole"}: &clusterRoleFactory{}, + {"ClusterRoleBinding"}: &clusterRoleBindingFactory{}, + {"DaemonSet"}: &daemonSetFactory{}, + {"Role"}: &roleFactory{}, + {"RoleBinding"}: &roleBindingFactory{}, + {"Secret"}: &secretFactory{}, + {"Service"}: &serviceFactory{}, + {"ServiceAccount"}: &serviceAccountFactory{}, + {"StatefulSet"}: &statefulSetFactory{}, + {"StorageClass"}: &storageClassFactory{}, } // PatchName makes the name of some item unique by appending the @@ -331,6 +326,8 @@ func (f *Framework) patchItemRecursively(item interface{}) error { f.PatchName(&item.Name) case *v1.ServiceAccount: f.PatchNamespace(&item.ObjectMeta.Namespace) + case *v1.Secret: + f.PatchNamespace(&item.ObjectMeta.Namespace) case *rbac.ClusterRoleBinding: f.PatchName(&item.Name) for i := range item.Subjects { @@ -368,13 +365,13 @@ func (f *Framework) patchItemRecursively(item interface{}) error { // looked like the least dirty approach. Perhaps one day Go will have // generics. -type ServiceAccountFactory struct{} +type serviceAccountFactory struct{} -func (f *ServiceAccountFactory) New() runtime.Object { +func (f *serviceAccountFactory) New() runtime.Object { return &v1.ServiceAccount{} } -func (*ServiceAccountFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*serviceAccountFactory) Create(f *Framework, i interface{}) (func() error, error) { item, ok := i.(*v1.ServiceAccount) if !ok { return nil, ItemNotSupported @@ -388,21 +385,13 @@ func (*ServiceAccountFactory) Create(f *Framework, i interface{}) (func() error, }, nil } -func (*ServiceAccountFactory) UniqueName(i interface{}) string { - item, ok := i.(*v1.ServiceAccount) - if !ok { - return "" - } - return fmt.Sprintf("%s/%s", item.GetNamespace(), item.GetName()) -} +type clusterRoleFactory struct{} -type ClusterRoleFactory struct{} - -func (f *ClusterRoleFactory) New() runtime.Object { +func (f *clusterRoleFactory) New() runtime.Object { return &rbac.ClusterRole{} } -func (*ClusterRoleFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*clusterRoleFactory) Create(f *Framework, i interface{}) (func() error, error) { item, ok := i.(*rbac.ClusterRole) if !ok { return nil, ItemNotSupported @@ -436,21 +425,13 @@ func (*ClusterRoleFactory) Create(f *Framework, i interface{}) (func() error, er }, nil } -func (*ClusterRoleFactory) UniqueName(i interface{}) string { - item, ok := i.(*rbac.ClusterRole) - if !ok { - return "" - } - return item.GetName() -} +type clusterRoleBindingFactory struct{} -type ClusterRoleBindingFactory struct{} - -func (f *ClusterRoleBindingFactory) New() runtime.Object { +func (f *clusterRoleBindingFactory) New() runtime.Object { return &rbac.ClusterRoleBinding{} } -func (*ClusterRoleBindingFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*clusterRoleBindingFactory) Create(f *Framework, i interface{}) (func() error, error) { item, ok := i.(*rbac.ClusterRoleBinding) if !ok { return nil, ItemNotSupported @@ -465,21 +446,13 @@ func (*ClusterRoleBindingFactory) Create(f *Framework, i interface{}) (func() er }, nil } -func (*ClusterRoleBindingFactory) UniqueName(i interface{}) string { - item, ok := i.(*rbac.ClusterRoleBinding) - if !ok { - return "" - } - return item.GetName() -} +type roleFactory struct{} -type RoleFactory struct{} - -func (f *RoleFactory) New() runtime.Object { +func (f *roleFactory) New() runtime.Object { return &rbac.Role{} } -func (*RoleFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*roleFactory) Create(f *Framework, i interface{}) (func() error, error) { item, ok := i.(*rbac.Role) if !ok { return nil, ItemNotSupported @@ -494,21 +467,13 @@ func (*RoleFactory) Create(f *Framework, i interface{}) (func() error, error) { }, nil } -func (*RoleFactory) UniqueName(i interface{}) string { - item, ok := i.(*rbac.Role) - if !ok { - return "" - } - return fmt.Sprintf("%s/%s", item.GetNamespace(), item.GetName()) -} +type roleBindingFactory struct{} -type RoleBindingFactory struct{} - -func (f *RoleBindingFactory) New() runtime.Object { +func (f *roleBindingFactory) New() runtime.Object { return &rbac.RoleBinding{} } -func (*RoleBindingFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*roleBindingFactory) Create(f *Framework, i interface{}) (func() error, error) { item, ok := i.(*rbac.RoleBinding) if !ok { return nil, ItemNotSupported @@ -523,21 +488,13 @@ func (*RoleBindingFactory) Create(f *Framework, i interface{}) (func() error, er }, nil } -func (*RoleBindingFactory) UniqueName(i interface{}) string { - item, ok := i.(*rbac.RoleBinding) - if !ok { - return "" - } - return fmt.Sprintf("%s/%s", item.GetNamespace(), item.GetName()) -} +type serviceFactory struct{} -type ServiceFactory struct{} - -func (f *ServiceFactory) New() runtime.Object { +func (f *serviceFactory) New() runtime.Object { return &v1.Service{} } -func (*ServiceFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*serviceFactory) Create(f *Framework, i interface{}) (func() error, error) { item, ok := i.(*v1.Service) if !ok { return nil, ItemNotSupported @@ -552,21 +509,13 @@ func (*ServiceFactory) Create(f *Framework, i interface{}) (func() error, error) }, nil } -func (*ServiceFactory) UniqueName(i interface{}) string { - item, ok := i.(*v1.Service) - if !ok { - return "" - } - return fmt.Sprintf("%s/%s", item.GetNamespace(), item.GetName()) -} +type statefulSetFactory struct{} -type StatefulSetFactory struct{} - -func (f *StatefulSetFactory) New() runtime.Object { +func (f *statefulSetFactory) New() runtime.Object { return &apps.StatefulSet{} } -func (*StatefulSetFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*statefulSetFactory) Create(f *Framework, i interface{}) (func() error, error) { item, ok := i.(*apps.StatefulSet) if !ok { return nil, ItemNotSupported @@ -581,21 +530,13 @@ func (*StatefulSetFactory) Create(f *Framework, i interface{}) (func() error, er }, nil } -func (*StatefulSetFactory) UniqueName(i interface{}) string { - item, ok := i.(*apps.StatefulSet) - if !ok { - return "" - } - return fmt.Sprintf("%s/%s", item.GetNamespace(), item.GetName()) -} +type daemonSetFactory struct{} -type DaemonSetFactory struct{} - -func (f *DaemonSetFactory) New() runtime.Object { +func (f *daemonSetFactory) New() runtime.Object { return &apps.DaemonSet{} } -func (*DaemonSetFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*daemonSetFactory) Create(f *Framework, i interface{}) (func() error, error) { item, ok := i.(*apps.DaemonSet) if !ok { return nil, ItemNotSupported @@ -610,21 +551,13 @@ func (*DaemonSetFactory) Create(f *Framework, i interface{}) (func() error, erro }, nil } -func (*DaemonSetFactory) UniqueName(i interface{}) string { - item, ok := i.(*apps.DaemonSet) - if !ok { - return "" - } - return fmt.Sprintf("%s/%s", item.GetNamespace(), item.GetName()) -} +type storageClassFactory struct{} -type StorageClassFactory struct{} - -func (f *StorageClassFactory) New() runtime.Object { +func (f *storageClassFactory) New() runtime.Object { return &storage.StorageClass{} } -func (*StorageClassFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*storageClassFactory) Create(f *Framework, i interface{}) (func() error, error) { item, ok := i.(*storage.StorageClass) if !ok { return nil, ItemNotSupported @@ -639,12 +572,25 @@ func (*StorageClassFactory) Create(f *Framework, i interface{}) (func() error, e }, nil } -func (*StorageClassFactory) UniqueName(i interface{}) string { - item, ok := i.(*storage.StorageClass) +type secretFactory struct{} + +func (f *secretFactory) New() runtime.Object { + return &v1.Secret{} +} + +func (*secretFactory) Create(f *Framework, i interface{}) (func() error, error) { + item, ok := i.(*v1.Secret) if !ok { - return "" + return nil, ItemNotSupported } - return item.GetName() + + client := f.ClientSet.CoreV1().Secrets(f.Namespace.GetName()) + if _, err := client.Create(item); err != nil { + return nil, errors.Wrap(err, "create Secret") + } + return func() error { + return client.Delete(item.GetName(), &metav1.DeleteOptions{}) + }, nil } // PrettyPrint returns a human-readable representation of an item. diff --git a/test/e2e/framework/gpu_util.go b/test/e2e/framework/gpu_util.go index 80da1aff16c..ff652e8d353 100644 --- a/test/e2e/framework/gpu_util.go +++ b/test/e2e/framework/gpu_util.go @@ -21,8 +21,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" - "github.com/golang/glog" . "github.com/onsi/gomega" + "k8s.io/klog" ) const ( @@ -71,15 +71,15 @@ func NVIDIADevicePlugin() *v1.Pod { func GetGPUDevicePluginImage() string { ds, err := DsFromManifest(GPUDevicePluginDSYAML) if err != nil { - glog.Errorf("Failed to parse the device plugin image: %v", err) + klog.Errorf("Failed to parse the device plugin image: %v", err) return "" } if ds == nil { - glog.Errorf("Failed to parse the device plugin image: the extracted DaemonSet is nil") + klog.Errorf("Failed to parse the device plugin image: the extracted DaemonSet is nil") return "" } if len(ds.Spec.Template.Spec.Containers) < 1 { - glog.Errorf("Failed to parse the device plugin image: cannot extract the container from YAML") + klog.Errorf("Failed to parse the device plugin image: cannot extract the container from YAML") return "" } return ds.Spec.Template.Spec.Containers[0].Image diff --git a/test/e2e/framework/ingress/BUILD b/test/e2e/framework/ingress/BUILD index c5c46a76011..f991757f1c1 100644 --- a/test/e2e/framework/ingress/BUILD +++ b/test/e2e/framework/ingress/BUILD @@ -20,10 +20,10 @@ go_library( "//test/e2e/framework/testfiles:go_default_library", "//test/e2e/manifest:go_default_library", "//test/utils:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/google.golang.org/api/compute/v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/e2e/framework/ingress/ingress_utils.go b/test/e2e/framework/ingress/ingress_utils.go index 219b718dd23..0cb68dc4e9a 100644 --- a/test/e2e/framework/ingress/ingress_utils.go +++ b/test/e2e/framework/ingress/ingress_utils.go @@ -34,7 +34,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" compute "google.golang.org/api/compute/v1" "k8s.io/api/core/v1" @@ -115,11 +115,11 @@ type TestLogger interface { type GLogger struct{} func (l *GLogger) Infof(format string, args ...interface{}) { - glog.Infof(format, args...) + klog.Infof(format, args...) } func (l *GLogger) Errorf(format string, args ...interface{}) { - glog.Errorf(format, args...) + klog.Errorf(format, args...) } type E2ELogger struct{} diff --git a/test/e2e/framework/metrics/BUILD b/test/e2e/framework/metrics/BUILD index 429acdc538d..8bf48c6a93b 100644 --- a/test/e2e/framework/metrics/BUILD +++ b/test/e2e/framework/metrics/BUILD @@ -24,9 +24,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/common/expfmt:go_default_library", "//vendor/github.com/prometheus/common/model:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/e2e/framework/metrics/generic_metrics.go b/test/e2e/framework/metrics/generic_metrics.go index 73558779052..7257c9120f7 100644 --- a/test/e2e/framework/metrics/generic_metrics.go +++ b/test/e2e/framework/metrics/generic_metrics.go @@ -22,9 +22,9 @@ import ( "reflect" "strings" - "github.com/golang/glog" "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" + "k8s.io/klog" ) type Metrics map[string]model.Samples @@ -88,7 +88,7 @@ func parseMetrics(data string, output *Metrics) error { // Expected loop termination condition. return nil } - glog.Warningf("Invalid Decode. Skipping.") + klog.Warningf("Invalid Decode. Skipping.") continue } for _, metric := range v { diff --git a/test/e2e/framework/metrics/metrics_grabber.go b/test/e2e/framework/metrics/metrics_grabber.go index dd95dea3eb6..667ecc29d65 100644 --- a/test/e2e/framework/metrics/metrics_grabber.go +++ b/test/e2e/framework/metrics/metrics_grabber.go @@ -27,7 +27,7 @@ import ( "k8s.io/kubernetes/pkg/master/ports" "k8s.io/kubernetes/pkg/util/system" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -62,7 +62,7 @@ func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets b return nil, err } if len(nodeList.Items) < 1 { - glog.Warning("Can't find any Nodes in the API server to grab metrics from") + klog.Warning("Can't find any Nodes in the API server to grab metrics from") } for _, node := range nodeList.Items { if system.IsMasterNode(node.Name) { @@ -76,9 +76,9 @@ func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets b controllers = false clusterAutoscaler = ec != nil if clusterAutoscaler { - glog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager is disabled.") + klog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager is disabled.") } else { - glog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager and ClusterAutoscaler is disabled.") + klog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager and ClusterAutoscaler is disabled.") } } @@ -127,7 +127,7 @@ func (g *MetricsGrabber) GrabFromScheduler() (SchedulerMetrics, error) { if !g.registeredMaster { return SchedulerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping Scheduler's metrics gathering.") } - output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-scheduler", g.masterName), metav1.NamespaceSystem, ports.SchedulerPort) + output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-scheduler", g.masterName), metav1.NamespaceSystem, ports.InsecureSchedulerPort) if err != nil { return SchedulerMetrics{}, err } diff --git a/test/e2e/framework/metrics_util.go b/test/e2e/framework/metrics_util.go index 1ef6b37268e..9f651df0d37 100644 --- a/test/e2e/framework/metrics_util.go +++ b/test/e2e/framework/metrics_util.go @@ -611,7 +611,7 @@ func sendRestRequestToScheduler(c clientset.Interface, op string) (string, error Context(ctx). Namespace(metav1.NamespaceSystem). Resource("pods"). - Name(fmt.Sprintf("kube-scheduler-%v:%v", TestContext.CloudConfig.MasterName, ports.SchedulerPort)). + Name(fmt.Sprintf("kube-scheduler-%v:%v", TestContext.CloudConfig.MasterName, ports.InsecureSchedulerPort)). SubResource("proxy"). Suffix("metrics"). Do().Raw() diff --git a/test/e2e/framework/networking_utils.go b/test/e2e/framework/networking_utils.go index 7f507a8390f..a8f8831d2c5 100644 --- a/test/e2e/framework/networking_utils.go +++ b/test/e2e/framework/networking_utils.go @@ -952,7 +952,7 @@ func TestUnderTemporaryNetworkFailure(c clientset.Interface, ns string, node *v1 if err != nil { Failf("Error getting node external ip : %v", err) } - master := GetMasterAddress(c) + masterAddresses := GetAllMasterAddresses(c) By(fmt.Sprintf("block network traffic from node %s to the master", node.Name)) defer func() { // This code will execute even if setting the iptables rule failed. @@ -960,14 +960,18 @@ func TestUnderTemporaryNetworkFailure(c clientset.Interface, ns string, node *v1 // had been inserted. (yes, we could look at the error code and ssh error // separately, but I prefer to stay on the safe side). By(fmt.Sprintf("Unblock network traffic from node %s to the master", node.Name)) - UnblockNetwork(host, master) + for _, masterAddress := range masterAddresses { + UnblockNetwork(host, masterAddress) + } }() Logf("Waiting %v to ensure node %s is ready before beginning test...", resizeNodeReadyTimeout, node.Name) if !WaitForNodeToBe(c, node.Name, v1.NodeReady, true, resizeNodeReadyTimeout) { Failf("Node %s did not become ready within %v", node.Name, resizeNodeReadyTimeout) } - BlockNetwork(host, master) + for _, masterAddress := range masterAddresses { + BlockNetwork(host, masterAddress) + } Logf("Waiting %v for node %s to be not ready after simulated network failure", resizeNodeNotReadyTimeout, node.Name) if !WaitForNodeToBe(c, node.Name, v1.NodeReady, false, resizeNodeNotReadyTimeout) { diff --git a/test/e2e/framework/pv_util.go b/test/e2e/framework/pv_util.go index 53483fd193b..bc6e5c97b21 100644 --- a/test/e2e/framework/pv_util.go +++ b/test/e2e/framework/pv_util.go @@ -880,12 +880,13 @@ func CreateSecPod(client clientset.Interface, namespace string, pvclaims []*v1.P // create security pod with given claims func CreateSecPodWithNodeName(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, nodeName string, timeout time.Duration) (*v1.Pod, error) { pod := MakeSecPod(namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup) + // Setting nodeName + pod.Spec.NodeName = nodeName + pod, err := client.CoreV1().Pods(namespace).Create(pod) if err != nil { return nil, fmt.Errorf("pod Create API error: %v", err) } - // Setting nodeName - pod.Spec.NodeName = nodeName // Waiting for pod to be running err = WaitTimeoutForPodRunningInNamespace(client, pod.Name, namespace, timeout) diff --git a/test/e2e/framework/rc_util.go b/test/e2e/framework/rc_util.go index edd88dd0ba4..d7a88937f26 100644 --- a/test/e2e/framework/rc_util.go +++ b/test/e2e/framework/rc_util.go @@ -247,7 +247,7 @@ func ValidateController(c clientset.Interface, containerImage string, replicas i // You can read about the syntax here: http://golang.org/pkg/text/template/. getContainerStateTemplate := fmt.Sprintf(`--template={{if (exists . "status" "containerStatuses")}}{{range .status.containerStatuses}}{{if (and (eq .name "%s") (exists . "state" "running"))}}true{{end}}{{end}}{{end}}`, containername) - getImageTemplate := fmt.Sprintf(`--template={{if (exists . "status" "containerStatuses")}}{{range .status.containerStatuses}}{{if eq .name "%s"}}{{.image}}{{end}}{{end}}{{end}}`, containername) + getImageTemplate := fmt.Sprintf(`--template={{if (exists . "spec" "containers")}}{{range .spec.containers}}{{if eq .name "%s"}}{{.image}}{{end}}{{end}}{{end}}`, containername) By(fmt.Sprintf("waiting for all containers in %s pods to come up.", testname)) //testname should be selector waitLoop: diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index be50ec1c6be..7ac659465a6 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -23,13 +23,13 @@ import ( "os" "time" - "github.com/golang/glog" "github.com/onsi/ginkgo/config" "github.com/pkg/errors" utilflag "k8s.io/apiserver/pkg/util/flag" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/klog" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" ) @@ -355,11 +355,11 @@ func AfterReadingAllFlags(t *TestContextType) { kubeConfig := createKubeConfig(clusterConfig) clientcmd.WriteToFile(*kubeConfig, tempFile.Name()) t.KubeConfig = tempFile.Name() - glog.Infof("Using a temporary kubeconfig file from in-cluster config : %s", tempFile.Name()) + klog.Infof("Using a temporary kubeconfig file from in-cluster config : %s", tempFile.Name()) } } if len(t.KubeConfig) == 0 { - glog.Warningf("Unable to find in-cluster config, using default host : %s", defaultHost) + klog.Warningf("Unable to find in-cluster config, using default host : %s", defaultHost) t.Host = defaultHost } } @@ -382,7 +382,7 @@ func AfterReadingAllFlags(t *TestContextType) { // TODO (https://github.com/kubernetes/kubernetes/issues/70200): // - remove the fallback for unknown providers // - proper error message instead of Failf (which panics) - glog.Warningf("Unknown provider %q, proceeding as for --provider=skeleton.", TestContext.Provider) + klog.Warningf("Unknown provider %q, proceeding as for --provider=skeleton.", TestContext.Provider) TestContext.CloudConfig.Provider, err = SetupProviderConfig("skeleton") if err != nil { Failf("Failed to setup fallback skeleton provider config: %v", err) diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 1e7f09319a8..59d58947f62 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -41,9 +41,9 @@ import ( "text/tabwriter" "time" - "github.com/golang/glog" "golang.org/x/crypto/ssh" "golang.org/x/net/websocket" + "k8s.io/klog" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -1016,22 +1016,40 @@ func WaitForPersistentVolumeDeleted(c clientset.Interface, pvName string, Poll, // WaitForPersistentVolumeClaimPhase waits for a PersistentVolumeClaim to be in a specific phase or until timeout occurs, whichever comes first. func WaitForPersistentVolumeClaimPhase(phase v1.PersistentVolumeClaimPhase, c clientset.Interface, ns string, pvcName string, Poll, timeout time.Duration) error { - Logf("Waiting up to %v for PersistentVolumeClaim %s to have phase %s", timeout, pvcName, phase) + return WaitForPersistentVolumeClaimsPhase(phase, c, ns, []string{pvcName}, Poll, timeout, true) +} + +// WaitForPersistentVolumeClaimPhase waits for any (if matchAny is true) or all (if matchAny is false) PersistentVolumeClaims +// to be in a specific phase or until timeout occurs, whichever comes first. +func WaitForPersistentVolumeClaimsPhase(phase v1.PersistentVolumeClaimPhase, c clientset.Interface, ns string, pvcNames []string, Poll, timeout time.Duration, matchAny bool) error { + if len(pvcNames) == 0 { + return fmt.Errorf("Incorrect parameter: Need at least one PVC to track. Found 0.") + } + Logf("Waiting up to %v for PersistentVolumeClaims %v to have phase %s", timeout, pvcNames, phase) for start := time.Now(); time.Since(start) < timeout; time.Sleep(Poll) { - pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Get(pvcName, metav1.GetOptions{}) - if err != nil { - Logf("Failed to get claim %q, retrying in %v. Error: %v", pvcName, Poll, err) - continue - } else { - if pvc.Status.Phase == phase { - Logf("PersistentVolumeClaim %s found and phase=%s (%v)", pvcName, phase, time.Since(start)) - return nil + phaseFoundInAllClaims := true + for _, pvcName := range pvcNames { + pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Get(pvcName, metav1.GetOptions{}) + if err != nil { + Logf("Failed to get claim %q, retrying in %v. Error: %v", pvcName, Poll, err) + continue } else { - Logf("PersistentVolumeClaim %s found but phase is %s instead of %s.", pvcName, pvc.Status.Phase, phase) + if pvc.Status.Phase == phase { + Logf("PersistentVolumeClaim %s found and phase=%s (%v)", pvcName, phase, time.Since(start)) + if matchAny { + return nil + } + } else { + Logf("PersistentVolumeClaim %s found but phase is %s instead of %s.", pvcName, pvc.Status.Phase, phase) + phaseFoundInAllClaims = false + } } } + if phaseFoundInAllClaims { + return nil + } } - return fmt.Errorf("PersistentVolumeClaim %s not in phase %s within %v", pvcName, phase, timeout) + return fmt.Errorf("PersistentVolumeClaims %v not all in phase %s within %v", pvcNames, phase, timeout) } // CreateTestingNS should be used by every test, note that we append a common prefix to the provided test name. @@ -3161,7 +3179,10 @@ func DeleteResourceAndWaitForGC(c clientset.Interface, kind schema.GroupKind, ns terminatePodTime := time.Since(startTime) - deleteTime Logf("Terminating %v %s pods took: %v", kind, name, terminatePodTime) - err = waitForPodsGone(ps, interval, 10*time.Minute) + // In gce, at any point, small percentage of nodes can disappear for + // ~10 minutes due to hostError. 20 minutes should be long enough to + // restart VM in that case and delete the pod. + err = waitForPodsGone(ps, interval, 20*time.Minute) if err != nil { return fmt.Errorf("error while waiting for pods gone %s: %v", name, err) } @@ -3173,25 +3194,48 @@ func DeleteResourceAndWaitForGC(c clientset.Interface, kind schema.GroupKind, ns // and DeleteRCAndWaitForGC, because the RC controller decreases status.replicas // when the pod is inactvie. func waitForPodsInactive(ps *testutils.PodStore, interval, timeout time.Duration) error { - return wait.PollImmediate(interval, timeout, func() (bool, error) { + var activePods []*v1.Pod + err := wait.PollImmediate(interval, timeout, func() (bool, error) { pods := ps.List() + activePods = nil for _, pod := range pods { if controller.IsPodActive(pod) { - return false, nil + activePods = append(activePods, pod) } } + + if len(activePods) != 0 { + return false, nil + } return true, nil }) + + if err == wait.ErrWaitTimeout { + for _, pod := range activePods { + Logf("ERROR: Pod %q running on %q is still active", pod.Name, pod.Spec.NodeName) + } + return fmt.Errorf("there are %d active pods. E.g. %q on node %q", len(activePods), activePods[0].Name, activePods[0].Spec.NodeName) + } + return err } // waitForPodsGone waits until there are no pods left in the PodStore. func waitForPodsGone(ps *testutils.PodStore, interval, timeout time.Duration) error { - return wait.PollImmediate(interval, timeout, func() (bool, error) { - if pods := ps.List(); len(pods) == 0 { + var pods []*v1.Pod + err := wait.PollImmediate(interval, timeout, func() (bool, error) { + if pods = ps.List(); len(pods) == 0 { return true, nil } return false, nil }) + + if err == wait.ErrWaitTimeout { + for _, pod := range pods { + Logf("ERROR: Pod %q still exists. Node: %q", pod.Name, pod.Spec.NodeName) + } + return fmt.Errorf("there are %d pods left. E.g. %q on node %q", len(pods), pods[0].Name, pods[0].Spec.NodeName) + } + return err } func WaitForPodsReady(c clientset.Interface, ns, name string, minReadySeconds int) error { @@ -3393,8 +3437,8 @@ func IssueSSHCommandWithResult(cmd, provider string, node *v1.Node) (*SSHResult, LogSSHResult(result) if result.Code != 0 || err != nil { - return nil, fmt.Errorf("failed running %q: %v (exit code %d)", - cmd, err, result.Code) + return nil, fmt.Errorf("failed running %q: %v (exit code %d, stderr %v)", + cmd, err, result.Code, result.Stderr) } return &result, nil @@ -4276,13 +4320,13 @@ func (rt *extractRT) RoundTrip(req *http.Request) (*http.Response, error) { // headersForConfig extracts any http client logic necessary for the provided // config. -func headersForConfig(c *restclient.Config) (http.Header, error) { +func headersForConfig(c *restclient.Config, url *url.URL) (http.Header, error) { extract := &extractRT{} rt, err := restclient.HTTPWrappersForConfig(c, extract) if err != nil { return nil, err } - if _, err := rt.RoundTrip(&http.Request{}); err != nil { + if _, err := rt.RoundTrip(&http.Request{URL: url}); err != nil { return nil, err } return extract.Header, nil @@ -4306,7 +4350,7 @@ func OpenWebSocketForURL(url *url.URL, config *restclient.Config, protocols []st url.Host += ":80" } } - headers, err := headersForConfig(config) + headers, err := headersForConfig(config, url) if err != nil { return nil, fmt.Errorf("failed to load http headers: %v", err) } @@ -4839,7 +4883,7 @@ func ListNamespaceEvents(c clientset.Interface, ns string) error { return err } for _, event := range ls.Items { - glog.Infof("Event(%#v): type: '%v' reason: '%v' %v", event.InvolvedObject, event.Type, event.Reason, event.Message) + klog.Infof("Event(%#v): type: '%v' reason: '%v' %v", event.InvolvedObject, event.Type, event.Reason, event.Message) } return nil } @@ -4878,7 +4922,7 @@ func (p *E2ETestNodePreparer) PrepareNodes() error { sum += v.Count for ; index < sum; index++ { if err := testutils.DoPrepareNode(p.client, &nodes.Items[index], v.Strategy); err != nil { - glog.Errorf("Aborting node preparation: %v", err) + klog.Errorf("Aborting node preparation: %v", err) return err } p.nodeToAppliedStrategy[nodes.Items[index].Name] = v.Strategy @@ -4896,7 +4940,7 @@ func (p *E2ETestNodePreparer) CleanupNodes() error { strategy, found := p.nodeToAppliedStrategy[name] if found { if err = testutils.DoCleanupNode(p.client, name, strategy); err != nil { - glog.Errorf("Skipping cleanup of Node: failed update of %v: %v", name, err) + klog.Errorf("Skipping cleanup of Node: failed update of %v: %v", name, err) encounteredError = err } } @@ -4934,19 +4978,28 @@ func getMaster(c clientset.Interface) Address { return master } -// GetMasterAddress returns the hostname/external IP/internal IP as appropriate for e2e tests on a particular provider -// which is the address of the interface used for communication with the kubelet. -func GetMasterAddress(c clientset.Interface) string { +// GetAllMasterAddresses returns all IP addresses on which the kubelet can reach the master. +// It may return internal and external IPs, even if we expect for +// e.g. internal IPs to be used (issue #56787), so that we can be +// sure to block the master fully during tests. +func GetAllMasterAddresses(c clientset.Interface) []string { master := getMaster(c) + + ips := sets.NewString() switch TestContext.Provider { case "gce", "gke": - return master.externalIP + if master.externalIP != "" { + ips.Insert(master.externalIP) + } + if master.internalIP != "" { + ips.Insert(master.internalIP) + } case "aws": - return awsMasterIP + ips.Insert(awsMasterIP) default: Failf("This test is not supported for provider %s and should be disabled", TestContext.Provider) } - return "" + return ips.List() } // GetNodeExternalIP returns node external IP concatenated with port 22 for ssh diff --git a/test/e2e/generated/BUILD b/test/e2e/generated/BUILD index 250b668226f..c5d8f822a81 100644 --- a/test/e2e/generated/BUILD +++ b/test/e2e/generated/BUILD @@ -15,7 +15,7 @@ go_library( ], importpath = "k8s.io/kubernetes/test/e2e/generated", deps = [ - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/e2e/generated/gobindata_util.go b/test/e2e/generated/gobindata_util.go index 62e031d9802..1bb2ed553d6 100644 --- a/test/e2e/generated/gobindata_util.go +++ b/test/e2e/generated/gobindata_util.go @@ -16,7 +16,7 @@ limitations under the License. package generated -import "github.com/golang/glog" +import "k8s.io/klog" /* ReadOrDie reads a file from gobindata. @@ -27,8 +27,8 @@ func ReadOrDie(filePath string) []byte { fileBytes, err := Asset(filePath) if err != nil { gobindataMsg := "An error occurred, possibly gobindata doesn't know about the file you're opening. For questions on maintaining gobindata, contact the sig-testing group." - glog.Infof("Available gobindata files: %v ", AssetNames()) - glog.Fatalf("Failed opening %v , with error %v. %v.", filePath, err, gobindataMsg) + klog.Infof("Available gobindata files: %v ", AssetNames()) + klog.Fatalf("Failed opening %v , with error %v. %v.", filePath, err, gobindataMsg) } return fileBytes } diff --git a/test/e2e/kubectl/BUILD b/test/e2e/kubectl/BUILD index ffb819f481f..971e90cecfa 100644 --- a/test/e2e/kubectl/BUILD +++ b/test/e2e/kubectl/BUILD @@ -35,11 +35,11 @@ go_library( "//test/utils:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/elazarl/goproxy:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/golang.org/x/net/websocket:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/test/e2e/kubectl/kubectl.go b/test/e2e/kubectl/kubectl.go index 333e665072e..3d23c49cbf2 100644 --- a/test/e2e/kubectl/kubectl.go +++ b/test/e2e/kubectl/kubectl.go @@ -41,7 +41,7 @@ import ( "time" "github.com/elazarl/goproxy" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" "k8s.io/api/core/v1" rbacv1beta1 "k8s.io/api/rbac/v1beta1" @@ -1063,7 +1063,7 @@ metadata: /* Release : v1.9 Testname: Kubectl, label update - Description: When a Pod is running, update a Label using ‘kubectl label’ command. The label MUST be created in the Pod. A ‘kubectl get pod’ with -l option on the container MUST verify that the label can be read back. Use ‘kubectl label label-’ to remove the label. Kubetctl get pod’ with -l option SHOULD no list the deleted label as the label is removed. + Description: When a Pod is running, update a Label using ‘kubectl label’ command. The label MUST be created in the Pod. A ‘kubectl get pod’ with -l option on the container MUST verify that the label can be read back. Use ‘kubectl label label-’ to remove the label. ‘kubectl get pod’ with -l option SHOULD not list the deleted label as the label is removed. */ framework.ConformanceIt("should update the label on a resource ", func() { labelName := "testing-label" @@ -1145,10 +1145,10 @@ metadata: Release : v1.9 Testname: Kubectl, logs Description: When a Pod is running then it MUST generate logs. - Starting a Pod should have a log line indicating the server is running and ready to accept connections. Also log command options MUST work as expected and described below. + Starting a Pod should have a log line indicating the server is running and ready to accept connections. Also log command options MUST work as expected and described below. ‘kubectl log -tail=1’ should generate a output of one line, the last line in the log. ‘kubectl --limit-bytes=1’ should generate a single byte output. - ‘kubectl --tail=1 --timestamp should genrate one line with timestamp in RFC3339 format + ‘kubectl --tail=1 --timestamp should generate one line with timestamp in RFC3339 format ‘kubectl --since=1s’ should output logs that are only 1 second older from now ‘kubectl --since=24h’ should output logs that are only 1 day older from now */ @@ -1418,7 +1418,7 @@ metadata: /* Release : v1.9 Testname: Kubectl, run deployment - Description: Command ‘kubectl run’ MUST create a job, with --generator=deployment, when a image name is specified in the run command. After the run command there SHOULD be a deployment that should exist with one container running the specified image. Also there SHOULD be a Pod that is controlled by this deployment, with a container running the specified image. + Description: Command ‘kubectl run’ MUST create a deployment, with --generator=deployment, when a image name is specified in the run command. After the run command there SHOULD be a deployment that should exist with one container running the specified image. Also there SHOULD be a Pod that is controlled by this deployment, with a container running the specified image. */ framework.ConformanceIt("should create a deployment from an image ", func() { By("running the image " + nginxImage) @@ -1463,7 +1463,7 @@ metadata: /* Release : v1.9 Testname: Kubectl, run job - Description: Command ‘kubectl run’ MUST create a deployment, with --generator=job, when a image name is specified in the run command. After the run command there SHOULD be a job that should exist with one container running the specified image. Also there SHOULD be a restart policy on the job spec that SHOULD match the command line. + Description: Command ‘kubectl run’ MUST create a job, with --generator=job, when a image name is specified in the run command. After the run command there SHOULD be a job that should exist with one container running the specified image. Also there SHOULD be a restart policy on the job spec that SHOULD match the command line. */ framework.ConformanceIt("should create a job from an image when restart is OnFailure ", func() { By("running the image " + nginxImage) @@ -1668,7 +1668,7 @@ metadata: /* Release : v1.9 Testname: Kubectl, proxy socket - Description: Start a proxy server on by running ‘kubectl proxy’ with --unix-socket=. Call the proxy server by requesting api versions from http://locahost:0/api. The proxy server MUST provide atleast one version string + Description: Start a proxy server on by running ‘kubectl proxy’ with --unix-socket=. Call the proxy server by requesting api versions from http://locahost:0/api. The proxy server MUST provide at least one version string */ framework.ConformanceIt("should support --unix-socket=/path ", func() { By("Starting the proxy") diff --git a/test/e2e/kubectl/portforward.go b/test/e2e/kubectl/portforward.go index 799147efc91..cc701af20f5 100644 --- a/test/e2e/kubectl/portforward.go +++ b/test/e2e/kubectl/portforward.go @@ -47,6 +47,11 @@ const ( podName = "pfpod" ) +const ( + podCheckInterval = 1 * time.Second + postStartWaitTimeout = 2 * time.Minute +) + // TODO support other ports besides 80 var ( portForwardRegexp = regexp.MustCompile("Forwarding from (127.0.0.1|\\[::1\\]):([0-9]+) -> 80") @@ -202,14 +207,6 @@ func doTestConnectSendDisconnect(bindAddress string, f *framework.Framework) { if err := f.WaitForPodReady(pod.Name); err != nil { framework.Failf("Pod did not start running: %v", err) } - defer func() { - logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester") - if err != nil { - framework.Logf("Error getting pod log: %v", err) - } else { - framework.Logf("Pod log:\n%s", logs) - } - }() By("Running 'kubectl port-forward'") cmd := runPortForward(f.Namespace.Name, pod.Name, 80) @@ -241,12 +238,12 @@ func doTestConnectSendDisconnect(bindAddress string, f *framework.Framework) { } By("Verifying logs") - logOutput, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester") - if err != nil { - framework.Failf("Error retrieving pod logs: %v", err) - } - verifyLogMessage(logOutput, "Accepted client connection") - verifyLogMessage(logOutput, "Done") + Eventually(func() (string, error) { + return framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester") + }, postStartWaitTimeout, podCheckInterval).Should(SatisfyAll( + ContainSubstring("Accepted client connection"), + ContainSubstring("Done"), + )) } func doTestMustConnectSendNothing(bindAddress string, f *framework.Framework) { @@ -258,14 +255,6 @@ func doTestMustConnectSendNothing(bindAddress string, f *framework.Framework) { if err := f.WaitForPodReady(pod.Name); err != nil { framework.Failf("Pod did not start running: %v", err) } - defer func() { - logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester") - if err != nil { - framework.Logf("Error getting pod log: %v", err) - } else { - framework.Logf("Pod log:\n%s", logs) - } - }() By("Running 'kubectl port-forward'") cmd := runPortForward(f.Namespace.Name, pod.Name, 80) @@ -286,12 +275,12 @@ func doTestMustConnectSendNothing(bindAddress string, f *framework.Framework) { } By("Verifying logs") - logOutput, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester") - if err != nil { - framework.Failf("Error retrieving pod logs: %v", err) - } - verifyLogMessage(logOutput, "Accepted client connection") - verifyLogMessage(logOutput, "Expected to read 3 bytes from client, but got 0 instead") + Eventually(func() (string, error) { + return framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester") + }, postStartWaitTimeout, podCheckInterval).Should(SatisfyAll( + ContainSubstring("Accepted client connection"), + ContainSubstring("Expected to read 3 bytes from client, but got 0 instead"), + )) } func doTestMustConnectSendDisconnect(bindAddress string, f *framework.Framework) { @@ -303,14 +292,6 @@ func doTestMustConnectSendDisconnect(bindAddress string, f *framework.Framework) if err := f.WaitForPodReady(pod.Name); err != nil { framework.Failf("Pod did not start running: %v", err) } - defer func() { - logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester") - if err != nil { - framework.Logf("Error getting pod log: %v", err) - } else { - framework.Logf("Pod log:\n%s", logs) - } - }() By("Running 'kubectl port-forward'") cmd := runPortForward(f.Namespace.Name, pod.Name, 80) @@ -352,13 +333,13 @@ func doTestMustConnectSendDisconnect(bindAddress string, f *framework.Framework) } By("Verifying logs") - logOutput, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester") - if err != nil { - framework.Failf("Error retrieving pod logs: %v", err) - } - verifyLogMessage(logOutput, "^Accepted client connection$") - verifyLogMessage(logOutput, "^Received expected client data$") - verifyLogMessage(logOutput, "^Done$") + Eventually(func() (string, error) { + return framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester") + }, postStartWaitTimeout, podCheckInterval).Should(SatisfyAll( + ContainSubstring("Accepted client connection"), + ContainSubstring("Received expected client data"), + ContainSubstring("Done"), + )) } func doTestOverWebSockets(bindAddress string, f *framework.Framework) { @@ -373,14 +354,6 @@ func doTestOverWebSockets(bindAddress string, f *framework.Framework) { if err := f.WaitForPodReady(pod.Name); err != nil { framework.Failf("Pod did not start running: %v", err) } - defer func() { - logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester") - if err != nil { - framework.Logf("Error getting pod log: %v", err) - } else { - framework.Logf("Pod log:\n%s", logs) - } - }() req := f.ClientSet.CoreV1().RESTClient().Get(). Namespace(f.Namespace.Name). @@ -449,12 +422,12 @@ func doTestOverWebSockets(bindAddress string, f *framework.Framework) { }, time.Minute, 10*time.Second).Should(BeNil()) By("Verifying logs") - logOutput, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester") - if err != nil { - framework.Failf("Error retrieving pod logs: %v", err) - } - verifyLogMessage(logOutput, "^Accepted client connection$") - verifyLogMessage(logOutput, "^Received expected client data$") + Eventually(func() (string, error) { + return framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester") + }, postStartWaitTimeout, podCheckInterval).Should(SatisfyAll( + ContainSubstring("Accepted client connection"), + ContainSubstring("Received expected client data"), + )) } var _ = SIGDescribe("Kubectl Port forwarding", func() { @@ -504,17 +477,6 @@ var _ = SIGDescribe("Kubectl Port forwarding", func() { }) }) -func verifyLogMessage(log, expected string) { - re := regexp.MustCompile(expected) - lines := strings.Split(log, "\n") - for i := range lines { - if re.MatchString(lines[i]) { - return - } - } - framework.Failf("Missing %q from log: %s", expected, log) -} - func wsRead(conn *websocket.Conn) (byte, []byte, error) { for { var data []byte diff --git a/test/e2e/lifecycle/BUILD b/test/e2e/lifecycle/BUILD index 89585641a60..24f1c8c9fbd 100644 --- a/test/e2e/lifecycle/BUILD +++ b/test/e2e/lifecycle/BUILD @@ -13,6 +13,7 @@ go_library( "framework.go", "ha_master.go", "kubelet_security.go", + "node_lease.go", "reboot.go", "resize_nodes.go", "restart.go", diff --git a/test/e2e/lifecycle/cluster_upgrade.go b/test/e2e/lifecycle/cluster_upgrade.go index c7744935a75..e6e3a7be19d 100644 --- a/test/e2e/lifecycle/cluster_upgrade.go +++ b/test/e2e/lifecycle/cluster_upgrade.go @@ -58,6 +58,7 @@ var upgradeTests = []upgrades.Test{ &apps.DaemonSetUpgradeTest{}, &upgrades.IngressUpgradeTest{}, &upgrades.AppArmorUpgradeTest{}, + &storage.VolumeModeDowngradeTest{}, } var gpuUpgradeTests = []upgrades.Test{ diff --git a/test/e2e/lifecycle/node_lease.go b/test/e2e/lifecycle/node_lease.go new file mode 100644 index 00000000000..9bd6c6b63f7 --- /dev/null +++ b/test/e2e/lifecycle/node_lease.go @@ -0,0 +1,163 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lifecycle + +import ( + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = SIGDescribe("[Feature:NodeLease][NodeAlphaFeature:NodeLease][Disruptive]", func() { + f := framework.NewDefaultFramework("node-lease-test") + var systemPodsNo int32 + var c clientset.Interface + var ns string + var group string + + BeforeEach(func() { + c = f.ClientSet + ns = f.Namespace.Name + systemPods, err := framework.GetPodsInNamespace(c, ns, map[string]string{}) + Expect(err).To(BeNil()) + systemPodsNo = int32(len(systemPods)) + if strings.Index(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") >= 0 { + framework.Failf("Test dose not support cluster setup with more than one MIG: %s", framework.TestContext.CloudConfig.NodeInstanceGroup) + } else { + group = framework.TestContext.CloudConfig.NodeInstanceGroup + } + }) + + Describe("NodeLease deletion", func() { + var skipped bool + + BeforeEach(func() { + skipped = true + framework.SkipUnlessProviderIs("gce", "gke", "aws") + framework.SkipUnlessNodeCountIsAtLeast(2) + skipped = false + }) + + AfterEach(func() { + if skipped { + return + } + + By("restoring the original node instance group size") + if err := framework.ResizeGroup(group, int32(framework.TestContext.CloudConfig.NumNodes)); err != nil { + framework.Failf("Couldn't restore the original node instance group size: %v", err) + } + // In GKE, our current tunneling setup has the potential to hold on to a broken tunnel (from a + // rebooted/deleted node) for up to 5 minutes before all tunnels are dropped and recreated. + // Most tests make use of some proxy feature to verify functionality. So, if a reboot test runs + // right before a test that tries to get logs, for example, we may get unlucky and try to use a + // closed tunnel to a node that was recently rebooted. There's no good way to framework.Poll for proxies + // being closed, so we sleep. + // + // TODO(cjcullen) reduce this sleep (#19314) + if framework.ProviderIs("gke") { + By("waiting 5 minutes for all dead tunnels to be dropped") + time.Sleep(5 * time.Minute) + } + if err := framework.WaitForGroupSize(group, int32(framework.TestContext.CloudConfig.NumNodes)); err != nil { + framework.Failf("Couldn't restore the original node instance group size: %v", err) + } + + if err := framework.WaitForReadyNodes(c, framework.TestContext.CloudConfig.NumNodes, 10*time.Minute); err != nil { + framework.Failf("Couldn't restore the original cluster size: %v", err) + } + // Many e2e tests assume that the cluster is fully healthy before they start. Wait until + // the cluster is restored to health. + By("waiting for system pods to successfully restart") + err := framework.WaitForPodsRunningReady(c, metav1.NamespaceSystem, systemPodsNo, 0, framework.PodReadyBeforeTimeout, map[string]string{}) + Expect(err).To(BeNil()) + }) + + It("node lease should be deleted when corresponding node is deleted", func() { + leaseClient := c.CoordinationV1beta1().Leases(corev1.NamespaceNodeLease) + err := framework.WaitForReadyNodes(c, framework.TestContext.CloudConfig.NumNodes, 10*time.Minute) + Expect(err).To(BeNil()) + + By("verify node lease exists for every nodes") + originalNodes := framework.GetReadySchedulableNodesOrDie(c) + Expect(len(originalNodes.Items)).To(Equal(framework.TestContext.CloudConfig.NumNodes)) + + Eventually(func() error { + pass := true + for _, node := range originalNodes.Items { + if _, err := leaseClient.Get(node.ObjectMeta.Name, metav1.GetOptions{}); err != nil { + framework.Logf("Try to get lease of node %s, but got error: %v", node.ObjectMeta.Name, err) + pass = false + } + } + if pass { + return nil + } + return fmt.Errorf("some node lease is not ready") + }, 1*time.Minute, 5*time.Second).Should(BeNil()) + + targetNumNodes := int32(framework.TestContext.CloudConfig.NumNodes - 1) + By(fmt.Sprintf("decreasing cluster size to %d", targetNumNodes)) + err = framework.ResizeGroup(group, targetNumNodes) + Expect(err).To(BeNil()) + err = framework.WaitForGroupSize(group, targetNumNodes) + Expect(err).To(BeNil()) + err = framework.WaitForReadyNodes(c, framework.TestContext.CloudConfig.NumNodes-1, 10*time.Minute) + Expect(err).To(BeNil()) + targetNodes := framework.GetReadySchedulableNodesOrDie(c) + Expect(len(targetNodes.Items)).To(Equal(int(targetNumNodes))) + + By("verify node lease is deleted for the deleted node") + var deletedNodeName string + for _, originalNode := range originalNodes.Items { + originalNodeName := originalNode.ObjectMeta.Name + for _, targetNode := range targetNodes.Items { + if originalNodeName == targetNode.ObjectMeta.Name { + continue + } + } + deletedNodeName = originalNodeName + break + } + Expect(deletedNodeName).NotTo(Equal("")) + Eventually(func() error { + if _, err := leaseClient.Get(deletedNodeName, metav1.GetOptions{}); err == nil { + return fmt.Errorf("node lease is not deleted yet for node %q", deletedNodeName) + } + return nil + }, 1*time.Minute, 5*time.Second).Should(BeNil()) + + By("verify node leases still exist for remaining nodes") + Eventually(func() error { + for _, node := range targetNodes.Items { + if _, err := leaseClient.Get(node.ObjectMeta.Name, metav1.GetOptions{}); err != nil { + return err + } + } + return nil + }, 1*time.Minute, 5*time.Second).Should(BeNil()) + }) + }) +}) diff --git a/test/e2e/network/firewall.go b/test/e2e/network/firewall.go index f5c98b70bb2..ee3839f16d2 100644 --- a/test/e2e/network/firewall.go +++ b/test/e2e/network/firewall.go @@ -18,6 +18,7 @@ package network import ( "fmt" + "time" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -172,17 +173,27 @@ var _ = SIGDescribe("Firewall rule", func() { By("Checking well known ports on master and nodes are not exposed externally") nodeAddrs := framework.NodeAddresses(nodes, v1.NodeExternalIP) - Expect(len(nodeAddrs)).NotTo(BeZero()) - masterAddr := framework.GetMasterAddress(cs) - flag, _ := framework.TestNotReachableHTTPTimeout(masterAddr, ports.InsecureKubeControllerManagerPort, gce.FirewallTestTcpTimeout) - Expect(flag).To(BeTrue()) - flag, _ = framework.TestNotReachableHTTPTimeout(masterAddr, ports.SchedulerPort, gce.FirewallTestTcpTimeout) - Expect(flag).To(BeTrue()) - flag, _ = framework.TestNotReachableHTTPTimeout(nodeAddrs[0], ports.KubeletPort, gce.FirewallTestTcpTimeout) - Expect(flag).To(BeTrue()) - flag, _ = framework.TestNotReachableHTTPTimeout(nodeAddrs[0], ports.KubeletReadOnlyPort, gce.FirewallTestTcpTimeout) - Expect(flag).To(BeTrue()) - flag, _ = framework.TestNotReachableHTTPTimeout(nodeAddrs[0], ports.ProxyStatusPort, gce.FirewallTestTcpTimeout) - Expect(flag).To(BeTrue()) + if len(nodeAddrs) == 0 { + framework.Failf("did not find any node addresses") + } + + masterAddresses := framework.GetAllMasterAddresses(cs) + for _, masterAddress := range masterAddresses { + assertNotReachableHTTPTimeout(masterAddress, ports.InsecureKubeControllerManagerPort, gce.FirewallTestTcpTimeout) + assertNotReachableHTTPTimeout(masterAddress, ports.InsecureSchedulerPort, gce.FirewallTestTcpTimeout) + } + assertNotReachableHTTPTimeout(nodeAddrs[0], ports.KubeletPort, gce.FirewallTestTcpTimeout) + assertNotReachableHTTPTimeout(nodeAddrs[0], ports.KubeletReadOnlyPort, gce.FirewallTestTcpTimeout) + assertNotReachableHTTPTimeout(nodeAddrs[0], ports.ProxyStatusPort, gce.FirewallTestTcpTimeout) }) }) + +func assertNotReachableHTTPTimeout(ip string, port int, timeout time.Duration) { + unreachable, err := framework.TestNotReachableHTTPTimeout(ip, port, timeout) + if err != nil { + framework.Failf("Unexpected error checking for reachability of %s:%d: %v", ip, port, err) + } + if !unreachable { + framework.Failf("Was unexpectedly able to reach %s:%d", ip, port) + } +} diff --git a/test/e2e/network/kube_proxy.go b/test/e2e/network/kube_proxy.go index 01a69f90f8e..9cf7acf0399 100644 --- a/test/e2e/network/kube_proxy.go +++ b/test/e2e/network/kube_proxy.go @@ -77,6 +77,16 @@ var _ = SIGDescribe("Network", func() { zero := int64(0) + // Some distributions (Ubuntu 16.04 etc.) don't support the proc file. + _, err := framework.IssueSSHCommandWithResult( + "ls /proc/net/nf_conntrack", + framework.TestContext.Provider, + clientNodeInfo.node) + if err != nil && strings.Contains(err.Error(), "No such file or directory") { + framework.Skipf("The node %s does not support /proc/net/nf_conntrack", clientNodeInfo.name) + } + framework.ExpectNoError(err) + clientPodSpec := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-net-client", diff --git a/test/e2e/network/scale/localrun/BUILD b/test/e2e/network/scale/localrun/BUILD index e2bffae8757..5066276eee6 100644 --- a/test/e2e/network/scale/localrun/BUILD +++ b/test/e2e/network/scale/localrun/BUILD @@ -15,7 +15,7 @@ go_library( "//test/e2e/framework/ingress:go_default_library", "//test/e2e/framework/providers/gce:go_default_library", "//test/e2e/network/scale:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/e2e/network/scale/localrun/ingress_scale.go b/test/e2e/network/scale/localrun/ingress_scale.go index 0fc38675eb5..2cb237ec51b 100644 --- a/test/e2e/network/scale/localrun/ingress_scale.go +++ b/test/e2e/network/scale/localrun/ingress_scale.go @@ -24,7 +24,7 @@ import ( "sort" "strconv" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -91,19 +91,19 @@ func main() { registerFlags() flag.Parse() if err := verifyFlags(); err != nil { - glog.Errorf("Failed to verify flags: %v", err) + klog.Errorf("Failed to verify flags: %v", err) os.Exit(1) } // Initializing a k8s client. config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) if err != nil { - glog.Errorf("Failed to build kubeconfig: %v", err) + klog.Errorf("Failed to build kubeconfig: %v", err) os.Exit(1) } cs, err := clientset.NewForConfig(config) if err != nil { - glog.Errorf("Failed to create kubeclient: %v", err) + klog.Errorf("Failed to create kubeclient: %v", err) os.Exit(1) } @@ -116,7 +116,7 @@ func main() { AlphaFeatureGate: gceAlphaFeatureGate, }) if err != nil { - glog.Errorf("Error building GCE provider: %v", err) + klog.Errorf("Error building GCE provider: %v", err) os.Exit(1) } cloudConfig.Provider = gce.NewProvider(gceCloud) @@ -124,7 +124,7 @@ func main() { testSuccessFlag := true defer func() { if !testSuccessFlag { - glog.Errorf("Ingress scale test failed.") + klog.Errorf("Ingress scale test failed.") os.Exit(1) } }() @@ -134,17 +134,17 @@ func main() { Name: testNamespace, }, } - glog.Infof("Creating namespace %s...", ns.Name) + klog.Infof("Creating namespace %s...", ns.Name) if _, err := cs.CoreV1().Namespaces().Create(ns); err != nil { - glog.Errorf("Failed to create namespace %s: %v", ns.Name, err) + klog.Errorf("Failed to create namespace %s: %v", ns.Name, err) testSuccessFlag = false return } if cleanup { defer func() { - glog.Infof("Deleting namespace %s...", ns.Name) + klog.Infof("Deleting namespace %s...", ns.Name) if err := cs.CoreV1().Namespaces().Delete(ns.Name, nil); err != nil { - glog.Errorf("Failed to delete namespace %s: %v", ns.Name, err) + klog.Errorf("Failed to delete namespace %s: %v", ns.Name, err) testSuccessFlag = false } }() @@ -164,20 +164,20 @@ func main() { if cleanup { defer func() { if errs := f.CleanupScaleTest(); len(errs) != 0 { - glog.Errorf("Failed to cleanup scale test: %v", errs) + klog.Errorf("Failed to cleanup scale test: %v", errs) testSuccessFlag = false } }() } err = f.PrepareScaleTest() if err != nil { - glog.Errorf("Failed to prepare scale test: %v", err) + klog.Errorf("Failed to prepare scale test: %v", err) testSuccessFlag = false return } if errs := f.RunScaleTest(); len(errs) != 0 { - glog.Errorf("Failed while running scale test: %v", errs) + klog.Errorf("Failed while running scale test: %v", errs) testSuccessFlag = false } } diff --git a/test/e2e/scalability/density.go b/test/e2e/scalability/density.go index 4449d69dc2a..3d4d9a6e21e 100644 --- a/test/e2e/scalability/density.go +++ b/test/e2e/scalability/density.go @@ -728,6 +728,8 @@ var _ = SIGDescribe("Density", func() { }) } e2eStartupTime = runDensityTest(dConfig, testPhaseDurations, &scheduleThroughputs) + defer cleanupDensityTest(dConfig, testPhaseDurations) + if itArg.runLatencyTest { // Pick latencyPodsIterations so that: // latencyPodsIterations * nodeCount >= MinPodStartupMeasurements. @@ -969,7 +971,6 @@ var _ = SIGDescribe("Density", func() { framework.LogSuspiciousLatency(startupLag, e2eLag, nodeCount, c) } - cleanupDensityTest(dConfig, testPhaseDurations) }) } }) diff --git a/test/e2e/scheduling/resource_quota.go b/test/e2e/scheduling/resource_quota.go index 40f5b16f3cd..3c33caa3909 100644 --- a/test/e2e/scheduling/resource_quota.go +++ b/test/e2e/scheduling/resource_quota.go @@ -365,6 +365,22 @@ var _ = SIGDescribe("ResourceQuota", func() { }) It("should create a ResourceQuota and capture the life of a configMap.", func() { + found, unchanged := 0, 0 + wait.Poll(1*time.Second, 30*time.Second, func() (bool, error) { + configmaps, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).List(metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + if len(configmaps.Items) == found { + // loop until the number of configmaps has stabilized for 5 seconds + unchanged++ + return unchanged > 4, nil + } + unchanged = 0 + found = len(configmaps.Items) + return false, nil + }) + defaultConfigMaps := fmt.Sprintf("%d", found) + hardConfigMaps := fmt.Sprintf("%d", found+1) + By("Creating a ResourceQuota") quotaName := "test-quota" resourceQuota := newTestResourceQuota(quotaName) @@ -374,6 +390,7 @@ var _ = SIGDescribe("ResourceQuota", func() { By("Ensuring resource quota status is calculated") usedResources := v1.ResourceList{} usedResources[v1.ResourceQuotas] = resource.MustParse("1") + usedResources[v1.ResourceConfigMaps] = resource.MustParse(defaultConfigMaps) err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources) Expect(err).NotTo(HaveOccurred()) @@ -385,7 +402,10 @@ var _ = SIGDescribe("ResourceQuota", func() { By("Ensuring resource quota status captures configMap creation") usedResources = v1.ResourceList{} usedResources[v1.ResourceQuotas] = resource.MustParse("1") - usedResources[v1.ResourceConfigMaps] = resource.MustParse("1") + // we expect there to be two configmaps because each namespace will receive + // a ca.crt configmap by default. + // ref:https://github.com/kubernetes/kubernetes/pull/68812 + usedResources[v1.ResourceConfigMaps] = resource.MustParse(hardConfigMaps) err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources) Expect(err).NotTo(HaveOccurred()) @@ -394,7 +414,7 @@ var _ = SIGDescribe("ResourceQuota", func() { Expect(err).NotTo(HaveOccurred()) By("Ensuring resource quota status released usage") - usedResources[v1.ResourceConfigMaps] = resource.MustParse("0") + usedResources[v1.ResourceConfigMaps] = resource.MustParse(defaultConfigMaps) err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources) Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/e2e/scheduling/taint_based_evictions.go b/test/e2e/scheduling/taint_based_evictions.go index 23de100749b..612a8a77d0c 100644 --- a/test/e2e/scheduling/taint_based_evictions.go +++ b/test/e2e/scheduling/taint_based_evictions.go @@ -128,12 +128,14 @@ var _ = SIGDescribe("TaintBasedEvictions [Serial]", func() { // host, err = framework.GetNodeInternalIP(&node) // } framework.ExpectNoError(err) - master := framework.GetMasterAddress(cs) + masterAddresses := framework.GetAllMasterAddresses(cs) taint := newUnreachableNoExecuteTaint() defer func() { By(fmt.Sprintf("Unblocking traffic from node %s to the master", node.Name)) - framework.UnblockNetwork(host, master) + for _, masterAddress := range masterAddresses { + framework.UnblockNetwork(host, masterAddress) + } if CurrentGinkgoTestDescription().Failed { framework.Failf("Current e2e test has failed, so return from here.") @@ -147,7 +149,9 @@ var _ = SIGDescribe("TaintBasedEvictions [Serial]", func() { framework.ExpectNoError(err) }() - framework.BlockNetwork(host, master) + for _, masterAddress := range masterAddresses { + framework.BlockNetwork(host, masterAddress) + } By(fmt.Sprintf("Expecting to see node %q becomes NotReady", nodeName)) if !framework.WaitForNodeToBeNotReady(cs, nodeName, time.Minute*3) { diff --git a/test/e2e/storage/BUILD b/test/e2e/storage/BUILD index e800048a36f..2d31246c771 100644 --- a/test/e2e/storage/BUILD +++ b/test/e2e/storage/BUILD @@ -3,11 +3,11 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ - "csi_objects.go", "csi_volumes.go", "empty_dir_wrapper.go", "ephemeral_volume.go", "flexvolume.go", + "flexvolume_mounted_volume_resize.go", "flexvolume_online_resize.go", "generic_persistent_volume-disruptive.go", "in_tree_volumes.go", @@ -45,8 +45,6 @@ go_library( "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -68,24 +66,24 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", - "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/framework/podlogs:go_default_library", "//test/e2e/framework/providers/gce:go_default_library", "//test/e2e/framework/testfiles:go_default_library", "//test/e2e/storage/drivers:go_default_library", + "//test/e2e/storage/testpatterns:go_default_library", "//test/e2e/storage/testsuites:go_default_library", "//test/e2e/storage/utils:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/prometheus/common/model:go_default_library", "//vendor/google.golang.org/api/googleapi:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index e93a20f5e12..dcedb65810c 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -19,13 +19,10 @@ package storage import ( "context" "fmt" - "math/rand" "regexp" - "time" "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" @@ -33,6 +30,8 @@ import ( csiclient "k8s.io/csi-api/pkg/client/clientset/versioned" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/podlogs" + "k8s.io/kubernetes/test/e2e/storage/drivers" + "k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" imageutils "k8s.io/kubernetes/test/utils/image" @@ -43,41 +42,56 @@ import ( . "github.com/onsi/gomega" ) -type csiTestDriver interface { - createCSIDriver() - cleanupCSIDriver() - createStorageClassTest() testsuites.StorageClassTest +// List of testDrivers to be executed in below loop +var csiTestDrivers = []func() drivers.TestDriver{ + drivers.InitHostPathCSIDriver, + drivers.InitGcePDCSIDriver, + drivers.InitGcePDExternalCSIDriver, } -var csiTestDrivers = map[string]func(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver{ - "hostPath": initCSIHostpath, - "gcePD": initCSIgcePD, - // TODO(#70258): this is temporary until we can figure out how to make e2e tests a library - "[Feature: gcePD-external]": initCSIgcePDExternal, +// List of testSuites to be executed in below loop +var csiTestSuites = []func() testsuites.TestSuite{ + testsuites.InitVolumesTestSuite, + testsuites.InitVolumeIOTestSuite, + testsuites.InitVolumeModeTestSuite, + testsuites.InitSubPathTestSuite, + testsuites.InitProvisioningTestSuite, } +func csiTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern { + tunedPatterns := []testpatterns.TestPattern{} + + for _, pattern := range patterns { + // Skip inline volume and pre-provsioned PV tests for csi drivers + if pattern.VolType == testpatterns.InlineVolume || pattern.VolType == testpatterns.PreprovisionedPV { + continue + } + tunedPatterns = append(tunedPatterns, pattern) + } + + return tunedPatterns +} + +// This executes testSuites for csi volumes. var _ = utils.SIGDescribe("CSI Volumes", func() { f := framework.NewDefaultFramework("csi-volumes") var ( - cancel context.CancelFunc - cs clientset.Interface - crdclient apiextensionsclient.Interface - csics csiclient.Interface - ns *v1.Namespace - node v1.Node - config framework.VolumeTestConfig + cancel context.CancelFunc + cs clientset.Interface + ns *v1.Namespace + config framework.VolumeTestConfig ) BeforeEach(func() { ctx, c := context.WithCancel(context.Background()) cancel = c - cs = f.ClientSet - crdclient = f.APIExtensionsClientSet - csics = f.CSIClientSet ns = f.Namespace - + config = framework.VolumeTestConfig{ + Namespace: ns.Name, + Prefix: "csi", + } // Debugging of the following tests heavily depends on the log output // of the different containers. Therefore include all of that in log // files (when using --report-dir, as in the CI) or the output stream @@ -103,70 +117,50 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { if framework.TestContext.ReportDir == "" { podlogs.WatchPods(ctx, cs, ns.Name, GinkgoWriter) } - - nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet) - node = nodes.Items[rand.Intn(len(nodes.Items))] - config = framework.VolumeTestConfig{ - Namespace: ns.Name, - Prefix: "csi", - // TODO(#70259): this needs to be parameterized so only hostpath sets node name - ClientNodeName: node.Name, - ServerNodeName: node.Name, - WaitForCompletion: true, - } - createCSICRDs(crdclient) }) AfterEach(func() { cancel() }) - for driverName, initCSIDriver := range csiTestDrivers { - curDriverName := driverName - curInitCSIDriver := initCSIDriver - - Context(fmt.Sprintf("CSI plugin test using CSI driver: %s", curDriverName), func() { - var ( - driver csiTestDriver - ) + for _, initDriver := range csiTestDrivers { + curDriver := initDriver() + Context(drivers.GetDriverNameWithFeatureTags(curDriver), func() { + driver := curDriver BeforeEach(func() { - driver = curInitCSIDriver(f, config) - driver.createCSIDriver() + // setupDriver + drivers.SetCommonDriverParameters(driver, f, config) + driver.CreateDriver() }) AfterEach(func() { - driver.cleanupCSIDriver() + // Cleanup driver + driver.CleanupDriver() }) - It("should provision storage", func() { - t := driver.createStorageClassTest() - claim := newClaim(t, ns.GetName(), "") - var class *storagev1.StorageClass - if t.StorageClassName == "" { - class = newStorageClass(t, ns.GetName(), "") - claim.Spec.StorageClassName = &class.ObjectMeta.Name - } else { - scName := t.StorageClassName - claim.Spec.StorageClassName = &scName - } - testsuites.TestDynamicProvisioning(t, cs, claim, class) - }) + testsuites.RunTestSuite(f, config, driver, csiTestSuites, csiTunePattern) }) } // The CSIDriverRegistry feature gate is needed for this test in Kubernetes 1.12. - Context("CSI attach test using HostPath driver [Feature:CSISkipAttach]", func() { + Context("CSI attach test using HostPath driver [Feature:CSIDriverRegistry]", func() { var ( - driver csiTestDriver + cs clientset.Interface + csics csiclient.Interface + driver drivers.TestDriver ) + BeforeEach(func() { - driver = initCSIHostpath(f, config) - driver.createCSIDriver() + cs = f.ClientSet + csics = f.CSIClientSet + driver = drivers.InitHostPathCSIDriver() + drivers.SetCommonDriverParameters(driver, f, config) + driver.CreateDriver() }) AfterEach(func() { - driver.cleanupCSIDriver() + driver.CleanupDriver() }) tests := []struct { @@ -198,15 +192,27 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { test := t It(test.name, func() { if test.driverExists { - driver := createCSIDriver(csics, "csi-hostpath-"+f.UniqueName, test.driverAttachable) - if driver != nil { - defer csics.CsiV1alpha1().CSIDrivers().Delete(driver.Name, nil) + csiDriver := createCSIDriver(csics, drivers.GetUniqueDriverName(driver), test.driverAttachable) + if csiDriver != nil { + defer csics.CsiV1alpha1().CSIDrivers().Delete(csiDriver.Name, nil) } } By("Creating pod") - t := driver.createStorageClassTest() - class, claim, pod := startPausePod(cs, t, ns.Name) + var sc *storagev1.StorageClass + if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok { + sc = dDriver.GetDynamicProvisionStorageClass("") + } + nodeName := driver.GetDriverInfo().Config.ClientNodeName + scTest := testsuites.StorageClassTest{ + Name: driver.GetDriverInfo().Name, + Provisioner: sc.Provisioner, + Parameters: sc.Parameters, + ClaimSize: "1Gi", + ExpectedSize: "1Gi", + NodeName: nodeName, + } + class, claim, pod := startPausePod(cs, scTest, ns.Name) if class != nil { defer cs.StorageV1().StorageClasses().Delete(class.Name, nil) } @@ -227,7 +233,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { By("Checking if VolumeAttachment was created for the pod") // Check that VolumeAttachment does not exist handle := getVolumeHandle(cs, claim) - attachmentHash := sha256.Sum256([]byte(fmt.Sprintf("%s%s%s", handle, t.Provisioner, node.Name))) + attachmentHash := sha256.Sum256([]byte(fmt.Sprintf("%s%s%s", handle, scTest.Provisioner, nodeName))) attachmentName := fmt.Sprintf("csi-%x", attachmentHash) _, err = cs.StorageV1beta1().VolumeAttachments().Get(attachmentName, metav1.GetOptions{}) if err != nil { @@ -330,164 +336,3 @@ func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, ns str framework.ExpectNoError(err, "Failed to create pod: %v", err) return class, claim, pod } - -type hostpathCSIDriver struct { - f *framework.Framework - config framework.VolumeTestConfig - cleanup func() -} - -func initCSIHostpath(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver { - return &hostpathCSIDriver{ - f: f, - config: config, - } -} - -func (h *hostpathCSIDriver) createStorageClassTest() testsuites.StorageClassTest { - return testsuites.StorageClassTest{ - Name: "csi-hostpath", - Parameters: map[string]string{}, - ClaimSize: "1Gi", - ExpectedSize: "1Gi", - - // The hostpath driver only works when everything runs on a single node. - NodeName: h.config.ServerNodeName, - - // Provisioner and storage class name must match what's used in - // csi-storageclass.yaml, plus the test-specific suffix. - Provisioner: "csi-hostpath-" + h.f.UniqueName, - StorageClassName: "csi-hostpath-sc-" + h.f.UniqueName, - } -} - -func (h *hostpathCSIDriver) createCSIDriver() { - By("deploying csi hostpath driver") - // TODO (?): the storage.csi.image.version and storage.csi.image.registry - // settings are ignored for this test. We could patch the image definitions. - o := utils.PatchCSIOptions{ - OldDriverName: "csi-hostpath", - NewDriverName: "csi-hostpath-" + h.f.UniqueName, - DriverContainerName: "hostpath", - ProvisionerContainerName: "csi-provisioner", - NodeName: h.config.ServerNodeName, - } - cleanup, err := h.f.CreateFromManifests(func(item interface{}) error { - return utils.PatchCSIDeployment(h.f, o, item) - }, - "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/usage/csi-storageclass.yaml", - ) - h.cleanup = cleanup - if err != nil { - framework.Failf("deploying csi hostpath driver: %v", err) - } -} - -func (h *hostpathCSIDriver) cleanupCSIDriver() { - if h.cleanup != nil { - By("uninstalling csi hostpath driver") - h.cleanup() - } -} - -type gcePDCSIDriver struct { - f *framework.Framework - config framework.VolumeTestConfig - cleanup func() -} - -func initCSIgcePD(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver { - cs := f.ClientSet - framework.SkipUnlessProviderIs("gce", "gke") - framework.SkipIfMultizone(cs) - - // TODO(#62561): Use credentials through external pod identity when that goes GA instead of downloading keys. - createGCESecrets(cs, config) - - framework.SkipUnlessSecretExistsAfterWait(cs, "cloud-sa", config.Namespace, 3*time.Minute) - - return &gcePDCSIDriver{ - f: f, - config: config, - } -} - -func (g *gcePDCSIDriver) createStorageClassTest() testsuites.StorageClassTest { - return testsuites.StorageClassTest{ - Name: "com.google.csi.gcepd", - // *Not* renaming the driver, see below. - Provisioner: "com.google.csi.gcepd", - Parameters: map[string]string{"type": "pd-standard"}, - ClaimSize: "5Gi", - ExpectedSize: "5Gi", - } -} - -func (g *gcePDCSIDriver) createCSIDriver() { - By("deploying gce-pd driver") - // It would be safer to rename the gcePD driver, but that - // hasn't been done before either and attempts to do so now led to - // errors during driver registration, therefore it is disabled - // by passing a nil function below. - // - // These are the options which would have to be used: - // o := utils.PatchCSIOptions{ - // OldDriverName: "com.google.csi.gcepd", - // NewDriverName: "com.google.csi.gcepd-" + g.f.UniqueName, - // DriverContainerName: "gce-driver", - // ProvisionerContainerName: "csi-external-provisioner", - // } - cleanup, err := g.f.CreateFromManifests(nil, - "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml", - "test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml", - "test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml", - ) - g.cleanup = cleanup - if err != nil { - framework.Failf("deploying csi hostpath driver: %v", err) - } -} - -func (g *gcePDCSIDriver) cleanupCSIDriver() { - By("uninstalling gce-pd driver") - if g.cleanup != nil { - g.cleanup() - } -} - -type gcePDCSIDriverExternal struct { -} - -func initCSIgcePDExternal(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver { - cs := f.ClientSet - framework.SkipUnlessProviderIs("gce", "gke") - framework.SkipIfMultizone(cs) - - return &gcePDCSIDriverExternal{} -} - -func (g *gcePDCSIDriverExternal) createStorageClassTest() testsuites.StorageClassTest { - return testsuites.StorageClassTest{ - Name: "com.google.csi.gcepd", - Provisioner: "com.google.csi.gcepd", - Parameters: map[string]string{"type": "pd-standard"}, - ClaimSize: "5Gi", - ExpectedSize: "5Gi", - } -} - -func (g *gcePDCSIDriverExternal) createCSIDriver() { -} - -func (g *gcePDCSIDriverExternal) cleanupCSIDriver() { -} diff --git a/test/e2e/storage/drivers/BUILD b/test/e2e/storage/drivers/BUILD index 72beb967aae..487e0f21866 100644 --- a/test/e2e/storage/drivers/BUILD +++ b/test/e2e/storage/drivers/BUILD @@ -4,6 +4,8 @@ go_library( name = "go_default_library", srcs = [ "base.go", + "csi.go", + "csi_objects.go", "in_tree.go", ], importpath = "k8s.io/kubernetes/test/e2e/storage/drivers", @@ -17,7 +19,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/storage/testpatterns:go_default_library", "//test/e2e/storage/utils:go_default_library", diff --git a/test/e2e/storage/drivers/base.go b/test/e2e/storage/drivers/base.go index 5d5ca0e1363..ae8f3ab8340 100644 --- a/test/e2e/storage/drivers/base.go +++ b/test/e2e/storage/drivers/base.go @@ -176,3 +176,8 @@ func getStorageClass( VolumeBindingMode: bindingMode, } } + +// GetUniqueDriverName returns unique driver name that can be used parallelly in tests +func GetUniqueDriverName(driver TestDriver) string { + return fmt.Sprintf("%s-%s", driver.GetDriverInfo().Name, driver.GetDriverInfo().Framework.UniqueName) +} diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go new file mode 100644 index 00000000000..e6153e62cee --- /dev/null +++ b/test/e2e/storage/drivers/csi.go @@ -0,0 +1,281 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + * This file defines various csi volume test drivers for TestSuites. + * + * There are two ways, how to prepare test drivers: + * 1) With containerized server (NFS, Ceph, Gluster, iSCSI, ...) + * It creates a server pod which defines one volume for the tests. + * These tests work only when privileged containers are allowed, exporting + * various filesystems (NFS, GlusterFS, ...) usually needs some mounting or + * other privileged magic in the server pod. + * + * Note that the server containers are for testing purposes only and should not + * be used in production. + * + * 2) With server or cloud provider outside of Kubernetes (Cinder, GCE, AWS, Azure, ...) + * Appropriate server or cloud provider must exist somewhere outside + * the tested Kubernetes cluster. CreateVolume will create a new volume to be + * used in the TestSuites for inlineVolume or DynamicPV tests. + */ + +package drivers + +import ( + "fmt" + "math/rand" + "time" + + . "github.com/onsi/ginkgo" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/testpatterns" + "k8s.io/kubernetes/test/e2e/storage/utils" +) + +// hostpathCSI +type hostpathCSIDriver struct { + cleanup func() + driverInfo DriverInfo +} + +var _ TestDriver = &hostpathCSIDriver{} +var _ DynamicPVTestDriver = &hostpathCSIDriver{} + +// InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface +func InitHostPathCSIDriver() TestDriver { + return &hostpathCSIDriver{ + driverInfo: DriverInfo{ + Name: "csi-hostpath", + FeatureTag: "", + MaxFileSize: testpatterns.FileSizeMedium, + SupportedFsType: sets.NewString( + "", // Default fsType + ), + IsPersistent: true, + IsFsGroupSupported: false, + IsBlockSupported: false, + }, + } +} + +func (h *hostpathCSIDriver) GetDriverInfo() *DriverInfo { + return &h.driverInfo +} + +func (h *hostpathCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { +} + +func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { + provisioner := GetUniqueDriverName(h) + parameters := map[string]string{} + ns := h.driverInfo.Framework.Namespace.Name + suffix := fmt.Sprintf("%s-sc", provisioner) + + return getStorageClass(provisioner, parameters, nil, ns, suffix) +} + +func (h *hostpathCSIDriver) CreateDriver() { + By("deploying csi hostpath driver") + f := h.driverInfo.Framework + cs := f.ClientSet + + // pods should be scheduled on the node + nodes := framework.GetReadySchedulableNodesOrDie(cs) + node := nodes.Items[rand.Intn(len(nodes.Items))] + h.driverInfo.Config.ClientNodeName = node.Name + h.driverInfo.Config.ServerNodeName = node.Name + + // TODO (?): the storage.csi.image.version and storage.csi.image.registry + // settings are ignored for this test. We could patch the image definitions. + o := utils.PatchCSIOptions{ + OldDriverName: h.driverInfo.Name, + NewDriverName: GetUniqueDriverName(h), + DriverContainerName: "hostpath", + ProvisionerContainerName: "csi-provisioner", + NodeName: h.driverInfo.Config.ServerNodeName, + } + cleanup, err := h.driverInfo.Framework.CreateFromManifests(func(item interface{}) error { + return utils.PatchCSIDeployment(h.driverInfo.Framework, o, item) + }, + "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml", + ) + h.cleanup = cleanup + if err != nil { + framework.Failf("deploying csi hostpath driver: %v", err) + } +} + +func (h *hostpathCSIDriver) CleanupDriver() { + if h.cleanup != nil { + By("uninstalling csi hostpath driver") + h.cleanup() + } +} + +// gce-pd +type gcePDCSIDriver struct { + cleanup func() + driverInfo DriverInfo +} + +var _ TestDriver = &gcePDCSIDriver{} +var _ DynamicPVTestDriver = &gcePDCSIDriver{} + +// InitGcePDCSIDriver returns gcePDCSIDriver that implements TestDriver interface +func InitGcePDCSIDriver() TestDriver { + return &gcePDCSIDriver{ + driverInfo: DriverInfo{ + Name: "pd.csi.storage.gke.io", + FeatureTag: "[Serial]", + MaxFileSize: testpatterns.FileSizeMedium, + SupportedFsType: sets.NewString( + "", // Default fsType + "ext2", + "ext3", + "ext4", + "xfs", + ), + IsPersistent: true, + IsFsGroupSupported: true, + IsBlockSupported: false, + }, + } +} + +func (g *gcePDCSIDriver) GetDriverInfo() *DriverInfo { + return &g.driverInfo +} + +func (g *gcePDCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { + f := g.driverInfo.Framework + cs := f.ClientSet + config := g.driverInfo.Config + framework.SkipUnlessProviderIs("gce", "gke") + framework.SkipIfMultizone(cs) + + // TODO(#62561): Use credentials through external pod identity when that goes GA instead of downloading keys. + createGCESecrets(cs, config) + framework.SkipUnlessSecretExistsAfterWait(cs, "cloud-sa", config.Namespace, 3*time.Minute) +} + +func (g *gcePDCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { + ns := g.driverInfo.Framework.Namespace.Name + provisioner := g.driverInfo.Name + suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name) + + parameters := map[string]string{"type": "pd-standard"} + + return getStorageClass(provisioner, parameters, nil, ns, suffix) +} + +func (g *gcePDCSIDriver) CreateDriver() { + By("deploying csi gce-pd driver") + // It would be safer to rename the gcePD driver, but that + // hasn't been done before either and attempts to do so now led to + // errors during driver registration, therefore it is disabled + // by passing a nil function below. + // + // These are the options which would have to be used: + // o := utils.PatchCSIOptions{ + // OldDriverName: g.driverInfo.Name, + // NewDriverName: GetUniqueDriverName(g), + // DriverContainerName: "gce-driver", + // ProvisionerContainerName: "csi-external-provisioner", + // } + cleanup, err := g.driverInfo.Framework.CreateFromManifests(nil, + "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml", + "test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml", + "test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml", + ) + g.cleanup = cleanup + if err != nil { + framework.Failf("deploying csi gce-pd driver: %v", err) + } +} + +func (g *gcePDCSIDriver) CleanupDriver() { + By("uninstalling gce-pd driver") + if g.cleanup != nil { + g.cleanup() + } +} + +// gcePd-external +type gcePDExternalCSIDriver struct { + driverInfo DriverInfo +} + +var _ TestDriver = &gcePDExternalCSIDriver{} +var _ DynamicPVTestDriver = &gcePDExternalCSIDriver{} + +// InitGcePDExternalCSIDriver returns gcePDExternalCSIDriver that implements TestDriver interface +func InitGcePDExternalCSIDriver() TestDriver { + return &gcePDExternalCSIDriver{ + driverInfo: DriverInfo{ + Name: "pd.csi.storage.gke.io", + // TODO(#70258): this is temporary until we can figure out how to make e2e tests a library + FeatureTag: "[Feature: gcePD-external]", + MaxFileSize: testpatterns.FileSizeMedium, + SupportedFsType: sets.NewString( + "", // Default fsType + "ext2", + "ext3", + "ext4", + "xfs", + ), + IsPersistent: true, + IsFsGroupSupported: true, + IsBlockSupported: false, + }, + } +} + +func (g *gcePDExternalCSIDriver) GetDriverInfo() *DriverInfo { + return &g.driverInfo +} + +func (g *gcePDExternalCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { + framework.SkipUnlessProviderIs("gce", "gke") + framework.SkipIfMultizone(g.driverInfo.Framework.ClientSet) +} + +func (g *gcePDExternalCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { + ns := g.driverInfo.Framework.Namespace.Name + provisioner := g.driverInfo.Name + suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name) + + parameters := map[string]string{"type": "pd-standard"} + + return getStorageClass(provisioner, parameters, nil, ns, suffix) +} + +func (g *gcePDExternalCSIDriver) CreateDriver() { +} + +func (g *gcePDExternalCSIDriver) CleanupDriver() { +} diff --git a/test/e2e/storage/csi_objects.go b/test/e2e/storage/drivers/csi_objects.go similarity index 80% rename from test/e2e/storage/csi_objects.go rename to test/e2e/storage/drivers/csi_objects.go index 5907a6f4265..cd3de2fce40 100644 --- a/test/e2e/storage/csi_objects.go +++ b/test/e2e/storage/drivers/csi_objects.go @@ -17,7 +17,7 @@ limitations under the License. // This file is used to deploy the CSI hostPath plugin // More Information: https://github.com/kubernetes-csi/drivers/tree/master/pkg/hostpath -package storage +package drivers import ( "flag" @@ -28,18 +28,11 @@ import ( "path/filepath" "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" - - . "github.com/onsi/ginkgo" - - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - csicrd "k8s.io/csi-api/pkg/crd" ) var ( @@ -64,25 +57,6 @@ func csiContainerImage(image string) string { return fullName } -func createCSICRDs(c apiextensionsclient.Interface) { - By("Creating CSI CRDs") - crds := []*apiextensionsv1beta1.CustomResourceDefinition{ - csicrd.CSIDriverCRD(), - csicrd.CSINodeInfoCRD(), - } - - for _, crd := range crds { - _, err := c.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{}) - if err == nil { - continue - } else if !apierrs.IsNotFound(err) { - framework.ExpectNoError(err, "Failed to check for existing of CSI CRD %q: %v", crd.Name, err) - } - _, err = c.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd) - framework.ExpectNoError(err, "Failed to create CSI CRD %q: %v", crd.Name, err) - } -} - func shredFile(filePath string) { if _, err := os.Stat(filePath); os.IsNotExist(err) { framework.Logf("File %v was not found, skipping shredding", filePath) diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index 553581a8067..c69ae3f42fc 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -274,7 +274,7 @@ func (g *glusterFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string name := gtr.prefix + "-server" return &v1.PersistentVolumeSource{ - Glusterfs: &v1.GlusterfsVolumeSource{ + Glusterfs: &v1.GlusterfsPersistentVolumeSource{ EndpointsName: name, // 'test_vol' comes from test/images/volumes-tester/gluster/run_gluster.sh Path: "test_vol", diff --git a/test/e2e/storage/flexvolume_mounted_volume_resize.go b/test/e2e/storage/flexvolume_mounted_volume_resize.go new file mode 100644 index 00000000000..eb120773028 --- /dev/null +++ b/test/e2e/storage/flexvolume_mounted_volume_resize.go @@ -0,0 +1,176 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "fmt" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/api/core/v1" + storage "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" + "path" +) + +var _ = utils.SIGDescribe("Mounted flexvolume expand[Slow]", func() { + var ( + c clientset.Interface + ns string + err error + pvc *v1.PersistentVolumeClaim + resizableSc *storage.StorageClass + nodeName string + isNodeLabeled bool + nodeKeyValueLabel map[string]string + nodeLabelValue string + nodeKey string + ) + + f := framework.NewDefaultFramework("mounted-flexvolume-expand") + BeforeEach(func() { + framework.SkipUnlessProviderIs("aws", "gce", "local") + framework.SkipUnlessMasterOSDistroIs("debian", "ubuntu", "gci", "custom") + framework.SkipUnlessNodeOSDistroIs("debian", "ubuntu", "gci", "custom") + framework.SkipUnlessSSHKeyPresent() + c = f.ClientSet + ns = f.Namespace.Name + framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout)) + + nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) + if len(nodeList.Items) != 0 { + nodeName = nodeList.Items[0].Name + } else { + framework.Failf("Unable to find ready and schedulable Node") + } + + nodeKey = "mounted_flexvolume_expand" + + if !isNodeLabeled { + nodeLabelValue = ns + nodeKeyValueLabel = make(map[string]string) + nodeKeyValueLabel[nodeKey] = nodeLabelValue + framework.AddOrUpdateLabelOnNode(c, nodeName, nodeKey, nodeLabelValue) + isNodeLabeled = true + } + + resizableSc, err = createStorageClass(ns, c) + if err != nil { + fmt.Printf("storage class creation error: %v\n", err) + } + Expect(err).NotTo(HaveOccurred(), "Error creating resizable storage class") + Expect(*resizableSc.AllowVolumeExpansion).To(BeTrue()) + + pvc = getClaim("2Gi", ns) + pvc.Spec.StorageClassName = &resizableSc.Name + pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc) + Expect(err).NotTo(HaveOccurred(), "Error creating pvc") + }) + + framework.AddCleanupAction(func() { + if len(nodeLabelValue) > 0 { + framework.RemoveLabelOffNode(c, nodeName, nodeKey) + } + }) + + AfterEach(func() { + framework.Logf("AfterEach: Cleaning up resources for mounted volume resize") + + if c != nil { + if errs := framework.PVPVCCleanup(c, ns, nil, pvc); len(errs) > 0 { + framework.Failf("AfterEach: Failed to delete PVC and/or PV. Errors: %v", utilerrors.NewAggregate(errs)) + } + pvc, nodeName, isNodeLabeled, nodeLabelValue = nil, "", false, "" + nodeKeyValueLabel = make(map[string]string) + } + }) + + It("Should verify mounted flex volumes can be resized", func() { + driver := "dummy-attachable" + nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) + node := nodeList.Items[0] + By(fmt.Sprintf("installing flexvolume %s on node %s as %s", path.Join(driverDir, driver), node.Name, driver)) + installFlex(c, &node, "k8s", driver, path.Join(driverDir, driver)) + By(fmt.Sprintf("installing flexvolume %s on (master) node %s as %s", path.Join(driverDir, driver), node.Name, driver)) + installFlex(c, nil, "k8s", driver, path.Join(driverDir, driver)) + + pv := framework.MakePersistentVolume(framework.PersistentVolumeConfig{ + PVSource: v1.PersistentVolumeSource{ + FlexVolume: &v1.FlexPersistentVolumeSource{ + Driver: "k8s/" + driver, + }}, + NamePrefix: "pv-", + StorageClassName: resizableSc.Name, + VolumeMode: pvc.Spec.VolumeMode, + }) + + pv, err = framework.CreatePV(c, pv) + Expect(err).NotTo(HaveOccurred(), "Error creating pv %v", err) + + By("Waiting for PVC to be in bound phase") + pvcClaims := []*v1.PersistentVolumeClaim{pvc} + var pvs []*v1.PersistentVolume + + pvs, err = framework.WaitForPVClaimBoundPhase(c, pvcClaims, framework.ClaimProvisionTimeout) + Expect(err).NotTo(HaveOccurred(), "Failed waiting for PVC to be bound %v", err) + Expect(len(pvs)).To(Equal(1)) + + By("Creating a deployment with the provisioned volume") + deployment, err := framework.CreateDeployment(c, int32(1), map[string]string{"test": "app"}, nodeKeyValueLabel, ns, pvcClaims, "") + Expect(err).NotTo(HaveOccurred(), "Failed creating deployment %v", err) + defer c.AppsV1().Deployments(ns).Delete(deployment.Name, &metav1.DeleteOptions{}) + + By("Expanding current pvc") + newSize := resource.MustParse("6Gi") + pvc, err = expandPVCSize(pvc, newSize, c) + Expect(err).NotTo(HaveOccurred(), "While updating pvc for more size") + Expect(pvc).NotTo(BeNil()) + + pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] + if pvcSize.Cmp(newSize) != 0 { + framework.Failf("error updating pvc size %q", pvc.Name) + } + + By("Waiting for cloudprovider resize to finish") + err = waitForControllerVolumeResize(pvc, c) + Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish") + + By("Getting a pod from deployment") + podList, err := framework.GetPodsForDeployment(c, deployment) + Expect(podList.Items).NotTo(BeEmpty()) + pod := podList.Items[0] + + By("Deleting the pod from deployment") + err = framework.DeletePodWithWait(f, c, &pod) + Expect(err).NotTo(HaveOccurred(), "while deleting pod for resizing") + + By("Waiting for deployment to create new pod") + pod, err = waitForDeploymentToRecreatePod(c, deployment) + Expect(err).NotTo(HaveOccurred(), "While waiting for pod to be recreated") + + By("Waiting for file system resize to finish") + pvc, err = waitForFSResize(pvc, c) + Expect(err).NotTo(HaveOccurred(), "while waiting for fs resize to finish") + + pvcConditions := pvc.Status.Conditions + Expect(len(pvcConditions)).To(Equal(0), "pvc should not have conditions") + }) +}) diff --git a/test/e2e/storage/flexvolume_online_resize.go b/test/e2e/storage/flexvolume_online_resize.go index 4157f164632..f1a915e586b 100644 --- a/test/e2e/storage/flexvolume_online_resize.go +++ b/test/e2e/storage/flexvolume_online_resize.go @@ -59,6 +59,9 @@ var _ = utils.SIGDescribe("Mounted flexvolume volume expand [Slow] [Feature:Expa f := framework.NewDefaultFramework("mounted-flexvolume-expand") BeforeEach(func() { framework.SkipUnlessProviderIs("aws", "gce", "local") + framework.SkipUnlessMasterOSDistroIs("debian", "ubuntu", "gci", "custom") + framework.SkipUnlessNodeOSDistroIs("debian", "ubuntu", "gci", "custom") + framework.SkipUnlessSSHKeyPresent() c = f.ClientSet ns = f.Namespace.Name framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout)) @@ -117,6 +120,8 @@ var _ = utils.SIGDescribe("Mounted flexvolume volume expand [Slow] [Feature:Expa node := nodeList.Items[0] By(fmt.Sprintf("installing flexvolume %s on node %s as %s", path.Join(driverDir, driver), node.Name, driver)) installFlex(c, &node, "k8s", driver, path.Join(driverDir, driver)) + By(fmt.Sprintf("installing flexvolume %s on (master) node %s as %s", path.Join(driverDir, driver), node.Name, driver)) + installFlex(c, nil, "k8s", driver, path.Join(driverDir, driver)) pv := framework.MakePersistentVolume(framework.PersistentVolumeConfig{ PVSource: v1.PersistentVolumeSource{ diff --git a/test/e2e/storage/in_tree_volumes.go b/test/e2e/storage/in_tree_volumes.go index 2906a17cecc..603876150b5 100644 --- a/test/e2e/storage/in_tree_volumes.go +++ b/test/e2e/storage/in_tree_volumes.go @@ -17,12 +17,11 @@ limitations under the License. package storage import ( - "fmt" - . "github.com/onsi/ginkgo" "k8s.io/api/core/v1" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/drivers" + "k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" ) @@ -53,6 +52,10 @@ var testSuites = []func() testsuites.TestSuite{ testsuites.InitProvisioningTestSuite, } +func intreeTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern { + return patterns +} + // This executes testSuites for in-tree volumes. var _ = utils.SIGDescribe("In-tree Volumes", func() { f := framework.NewDefaultFramework("volumes") @@ -72,7 +75,7 @@ var _ = utils.SIGDescribe("In-tree Volumes", func() { for _, initDriver := range testDrivers { curDriver := initDriver() - Context(fmt.Sprintf(drivers.GetDriverNameWithFeatureTags(curDriver)), func() { + Context(drivers.GetDriverNameWithFeatureTags(curDriver), func() { driver := curDriver BeforeEach(func() { @@ -86,7 +89,7 @@ var _ = utils.SIGDescribe("In-tree Volumes", func() { driver.CleanupDriver() }) - testsuites.RunTestSuite(f, config, driver, testSuites) + testsuites.RunTestSuite(f, config, driver, testSuites, intreeTunePattern) }) } }) diff --git a/test/e2e/storage/persistent_volumes-local.go b/test/e2e/storage/persistent_volumes-local.go index f4d066319ae..aa55bf287d6 100644 --- a/test/e2e/storage/persistent_volumes-local.go +++ b/test/e2e/storage/persistent_volumes-local.go @@ -25,9 +25,9 @@ import ( "sync" "time" - "github.com/ghodss/yaml" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "sigs.k8s.io/yaml" appsv1 "k8s.io/api/apps/v1" "k8s.io/api/core/v1" @@ -218,11 +218,7 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { if testVolType == GCELocalSSDVolumeType { serialStr = " [Serial]" } - alphaStr := "" - if testVolType == BlockLocalVolumeType { - alphaStr = " [Feature:BlockVolume]" - } - ctxString := fmt.Sprintf("[Volume type: %s]%v%v", testVolType, serialStr, alphaStr) + ctxString := fmt.Sprintf("[Volume type: %s]%v", testVolType, serialStr) testMode := immediateMode Context(ctxString, func() { @@ -1290,7 +1286,7 @@ func makeLocalPod(config *localTestConfig, volume *localTestVolume, cmd string) return pod } if volume.localVolumeType == BlockLocalVolumeType { - // Block e2e tests require utilities for writing to block devices (e.g. dd), and nginx has this utilites. + // Block e2e tests require utilities for writing to block devices (e.g. dd), and nginx has this utilities. pod.Spec.Containers[0].Image = imageutils.GetE2EImage(imageutils.Nginx) } return pod diff --git a/test/e2e/storage/regional_pd.go b/test/e2e/storage/regional_pd.go index bbd196b1051..428ef8556da 100644 --- a/test/e2e/storage/regional_pd.go +++ b/test/e2e/storage/regional_pd.go @@ -74,7 +74,8 @@ var _ = utils.SIGDescribe("Regional PD", func() { }) It("should provision storage with delayed binding [Slow]", func() { - testRegionalDelayedBinding(c, ns) + testRegionalDelayedBinding(c, ns, 1 /* pvcCount */) + testRegionalDelayedBinding(c, ns, 3 /* pvcCount */) }) It("should provision storage in the allowedTopologies [Slow]", func() { @@ -82,7 +83,8 @@ var _ = utils.SIGDescribe("Regional PD", func() { }) It("should provision storage in the allowedTopologies with delayed binding [Slow]", func() { - testRegionalAllowedTopologiesWithDelayedBinding(c, ns) + testRegionalAllowedTopologiesWithDelayedBinding(c, ns, 1 /* pvcCount */) + testRegionalAllowedTopologiesWithDelayedBinding(c, ns, 3 /* pvcCount */) }) It("should failover to a different zone when all nodes in one zone become unreachable [Slow] [Disruptive]", func() { @@ -297,7 +299,7 @@ func addTaint(c clientset.Interface, ns string, nodes []v1.Node, podZone string) } } -func testRegionalDelayedBinding(c clientset.Interface, ns string) { +func testRegionalDelayedBinding(c clientset.Interface, ns string, pvcCount int) { test := testsuites.StorageClassTest{ Name: "Regional PD storage class with waitForFirstConsumer test on GCE", Provisioner: "kubernetes.io/gce-pd", @@ -311,9 +313,13 @@ func testRegionalDelayedBinding(c clientset.Interface, ns string) { suffix := "delayed-regional" class := newStorageClass(test, ns, suffix) - claim := newClaim(test, ns, suffix) - claim.Spec.StorageClassName = &class.Name - pv, node := testBindingWaitForFirstConsumer(c, claim, class) + var claims []*v1.PersistentVolumeClaim + for i := 0; i < pvcCount; i++ { + claim := newClaim(test, ns, suffix) + claim.Spec.StorageClassName = &class.Name + claims = append(claims, claim) + } + pvs, node := testBindingWaitForFirstConsumerMultiPVC(c, claims, class) if node == nil { framework.Failf("unexpected nil node found") } @@ -321,7 +327,9 @@ func testRegionalDelayedBinding(c clientset.Interface, ns string) { if !ok { framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) } - checkZoneFromLabelAndAffinity(pv, zone, false) + for _, pv := range pvs { + checkZoneFromLabelAndAffinity(pv, zone, false) + } } func testRegionalAllowedTopologies(c clientset.Interface, ns string) { @@ -346,7 +354,7 @@ func testRegionalAllowedTopologies(c clientset.Interface, ns string) { checkZonesFromLabelAndAffinity(pv, sets.NewString(zones...), true) } -func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns string) { +func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns string, pvcCount int) { test := testsuites.StorageClassTest{ Name: "Regional PD storage class with allowedTopologies and waitForFirstConsumer test on GCE", Provisioner: "kubernetes.io/gce-pd", @@ -362,9 +370,13 @@ func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns s class := newStorageClass(test, ns, suffix) topoZones := getTwoRandomZones(c) addAllowedTopologiesToStorageClass(c, class, topoZones) - claim := newClaim(test, ns, suffix) - claim.Spec.StorageClassName = &class.Name - pv, node := testBindingWaitForFirstConsumer(c, claim, class) + var claims []*v1.PersistentVolumeClaim + for i := 0; i < pvcCount; i++ { + claim := newClaim(test, ns, suffix) + claim.Spec.StorageClassName = &class.Name + claims = append(claims, claim) + } + pvs, node := testBindingWaitForFirstConsumerMultiPVC(c, claims, class) if node == nil { framework.Failf("unexpected nil node found") } @@ -382,7 +394,9 @@ func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns s if !zoneFound { framework.Failf("zones specified in AllowedTopologies: %v does not contain zone of node where PV got provisioned: %s", topoZones, nodeZone) } - checkZonesFromLabelAndAffinity(pv, sets.NewString(topoZones...), true) + for _, pv := range pvs { + checkZonesFromLabelAndAffinity(pv, sets.NewString(topoZones...), true) + } } func getPVC(c clientset.Interface, ns string, pvcLabels map[string]string) *v1.PersistentVolumeClaim { diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index fb8bde814f5..8d4f084989f 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -66,12 +66,12 @@ func getTestNameStr(suite TestSuite, pattern testpatterns.TestPattern) string { } // RunTestSuite runs all testpatterns of all testSuites for a driver -func RunTestSuite(f *framework.Framework, config framework.VolumeTestConfig, driver drivers.TestDriver, tsInits []func() TestSuite) { +func RunTestSuite(f *framework.Framework, config framework.VolumeTestConfig, driver drivers.TestDriver, tsInits []func() TestSuite, tunePatternFunc func([]testpatterns.TestPattern) []testpatterns.TestPattern) { for _, testSuiteInit := range tsInits { suite := testSuiteInit() - tsInfo := suite.getTestSuiteInfo() + patterns := tunePatternFunc(suite.getTestSuiteInfo().testPatterns) - for _, pattern := range tsInfo.testPatterns { + for _, pattern := range patterns { suite.execTest(driver, pattern) } } @@ -164,7 +164,7 @@ func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pat case testpatterns.DynamicPV: framework.Logf("Creating resource for dynamic PV") if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok { - claimSize := "2Gi" + claimSize := "5Gi" r.sc = dDriver.GetDynamicProvisionStorageClass(fsType) By("creating a StorageClass " + r.sc.Name) diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index 587b543512b..70c3ad0e3cf 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -151,7 +151,7 @@ func (p *provisioningTestResource) setupResource(driver drivers.TestDriver, patt framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name) } p.driver = driver - p.claimSize = "2Gi" + p.claimSize = "5Gi" p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Framework.Namespace.Name) p.pvc.Spec.StorageClassName = &p.sc.Name framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc) @@ -173,7 +173,7 @@ type provisioningTestInput struct { } func testProvisioning(input *provisioningTestInput) { - It("should provision storage", func() { + It("should provision storage with defaults", func() { TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc) }) @@ -186,7 +186,7 @@ func testProvisioning(input *provisioningTestInput) { TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc) }) - It("should create and delete block persistent volumes [Feature:BlockVolume]", func() { + It("should create and delete block persistent volumes", func() { if !input.dInfo.IsBlockSupported { framework.Skipf("Driver %q does not support BlockVolume - skipping", input.dInfo.Name) } diff --git a/test/e2e/storage/testsuites/subpath.go b/test/e2e/storage/testsuites/subpath.go index b2ccfa29a75..619b1d545a0 100644 --- a/test/e2e/storage/testsuites/subpath.go +++ b/test/e2e/storage/testsuites/subpath.go @@ -19,6 +19,7 @@ package testsuites import ( "fmt" "path/filepath" + "regexp" "strings" "k8s.io/api/core/v1" @@ -315,7 +316,8 @@ func testSubPath(input *subPathTestInput) { }) It("should unmount if pod is force deleted while kubelet is down [Disruptive][Slow]", func() { - if input.volType == "hostPath" || input.volType == "hostPathSymlink" { + if strings.HasPrefix(input.volType, "hostPath") || strings.HasPrefix(input.volType, "csi-hostpath") { + // TODO: This skip should be removed once #61446 is fixed framework.Skipf("%s volume type does not support reconstruction, skipping", input.volType) } testSubpathReconstruction(input.f, input.pod, true) @@ -360,20 +362,6 @@ func testSubPath(input *subPathTestInput) { testReadFile(input.f, input.filePathInSubpath, input.pod, 0) }) - It("should fail for new directories when readOnly specified in the volumeSource [Slow]", func() { - if input.roVol == nil { - framework.Skipf("Volume type %v doesn't support readOnly source", input.volType) - } - - // Format the volume while it's writable - formatVolume(input.f, input.formatPod) - - // Set volume source to read only - input.pod.Spec.Volumes[0].VolumeSource = *input.roVol - // Pod should fail - testPodFailSubpathError(input.f, input.pod, "") - }) - // TODO: add a test case for the same disk with two partitions } @@ -394,10 +382,23 @@ func TestBasicSubpathFile(f *framework.Framework, contents string, pod *v1.Pod, Expect(err).NotTo(HaveOccurred(), "while deleting pod") } +func generateSuffixForPodName(s string) string { + // Pod name must: + // 1. consist of lower case alphanumeric characters or '-', + // 2. start and end with an alphanumeric character. + // (e.g. 'my-name', or '123-abc', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?') + // Therefore, suffix is generated by following steps: + // 1. all strings other than [A-Za-z0-9] is replaced with "-", + // 2. add lower case alphanumeric characters at the end ('-[a-z0-9]{4}' is added), + // 3. convert the entire strings to lower case. + re := regexp.MustCompile("[^A-Za-z0-9]") + return strings.ToLower(fmt.Sprintf("%s-%s", re.ReplaceAllString(s, "-"), rand.String(4))) +} + // SubpathTestPod returns a pod spec for subpath tests func SubpathTestPod(f *framework.Framework, subpath, volumeType string, source *v1.VolumeSource, privilegedSecurityContext bool) *v1.Pod { var ( - suffix = strings.ToLower(fmt.Sprintf("%s-%s", volumeType, rand.String(4))) + suffix = generateSuffixForPodName(volumeType) gracePeriod = int64(1) probeVolumeName = "liveness-probe-volume" ) diff --git a/test/e2e/storage/testsuites/volumemode.go b/test/e2e/storage/testsuites/volumemode.go index cd9992fe153..03468f93329 100644 --- a/test/e2e/storage/testsuites/volumemode.go +++ b/test/e2e/storage/testsuites/volumemode.go @@ -47,8 +47,7 @@ var _ TestSuite = &volumeModeTestSuite{} func InitVolumeModeTestSuite() TestSuite { return &volumeModeTestSuite{ tsInfo: TestSuiteInfo{ - name: "volumeMode", - featureTag: "[Feature:BlockVolume]", + name: "volumeMode", testPatterns: []testpatterns.TestPattern{ testpatterns.FsVolModePreprovisionedPV, testpatterns.FsVolModeDynamicPV, @@ -197,7 +196,7 @@ func (s *volumeModeTestResource) setupResource(driver drivers.TestDriver, patter } s.sc.VolumeBindingMode = &volBindMode - claimSize := "2Gi" + claimSize := "5Gi" s.pvc = getClaim(claimSize, ns.Name) s.pvc.Spec.StorageClassName = &s.sc.Name s.pvc.Spec.VolumeMode = &volMode @@ -304,10 +303,10 @@ func testVolumeModeSuccessForPreprovisionedPV(input *volumeModeTestInput) { Expect(err).NotTo(HaveOccurred()) By("Checking if persistent volume exists as expected volume mode") - checkVolumeModeOfPath(pod, input.volMode, "/mnt/volume1") + utils.CheckVolumeModeOfPath(pod, input.volMode, "/mnt/volume1") By("Checking if read/write to persistent volume works properly") - checkReadWriteToPath(pod, input.volMode, "/mnt/volume1") + utils.CheckReadWriteToPath(pod, input.volMode, "/mnt/volume1") }) // TODO(mkimuram): Add more tests } @@ -366,10 +365,10 @@ func testVolumeModeSuccessForDynamicPV(input *volumeModeTestInput) { Expect(err).NotTo(HaveOccurred()) By("Checking if persistent volume exists as expected volume mode") - checkVolumeModeOfPath(pod, input.volMode, "/mnt/volume1") + utils.CheckVolumeModeOfPath(pod, input.volMode, "/mnt/volume1") By("Checking if read/write to persistent volume works properly") - checkReadWriteToPath(pod, input.volMode, "/mnt/volume1") + utils.CheckReadWriteToPath(pod, input.volMode, "/mnt/volume1") }) // TODO(mkimuram): Add more tests } @@ -401,45 +400,3 @@ func generateConfigsForPreprovisionedPVTest(scName string, volBindMode storagev1 return scConfig, pvConfig, pvcConfig } - -func checkVolumeModeOfPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path string) { - if volMode == v1.PersistentVolumeBlock { - // Check if block exists - utils.VerifyExecInPodSucceed(pod, fmt.Sprintf("test -b %s", path)) - - // Double check that it's not directory - utils.VerifyExecInPodFail(pod, fmt.Sprintf("test -d %s", path), 1) - } else { - // Check if directory exists - utils.VerifyExecInPodSucceed(pod, fmt.Sprintf("test -d %s", path)) - - // Double check that it's not block - utils.VerifyExecInPodFail(pod, fmt.Sprintf("test -b %s", path), 1) - } -} - -func checkReadWriteToPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path string) { - if volMode == v1.PersistentVolumeBlock { - // random -> file1 - utils.VerifyExecInPodSucceed(pod, "dd if=/dev/urandom of=/tmp/file1 bs=64 count=1") - // file1 -> dev (write to dev) - utils.VerifyExecInPodSucceed(pod, fmt.Sprintf("dd if=/tmp/file1 of=%s bs=64 count=1", path)) - // dev -> file2 (read from dev) - utils.VerifyExecInPodSucceed(pod, fmt.Sprintf("dd if=%s of=/tmp/file2 bs=64 count=1", path)) - // file1 == file2 (check contents) - utils.VerifyExecInPodSucceed(pod, "diff /tmp/file1 /tmp/file2") - // Clean up temp files - utils.VerifyExecInPodSucceed(pod, "rm -f /tmp/file1 /tmp/file2") - - // Check that writing file to block volume fails - utils.VerifyExecInPodFail(pod, fmt.Sprintf("echo 'Hello world.' > %s/file1.txt", path), 1) - } else { - // text -> file1 (write to file) - utils.VerifyExecInPodSucceed(pod, fmt.Sprintf("echo 'Hello world.' > %s/file1.txt", path)) - // grep file1 (read from file and check contents) - utils.VerifyExecInPodSucceed(pod, fmt.Sprintf("grep 'Hello world.' %s/file1.txt", path)) - - // Check that writing to directory as block volume fails - utils.VerifyExecInPodFail(pod, fmt.Sprintf("dd if=/dev/urandom of=%s bs=64 count=1", path), 1) - } -} diff --git a/test/e2e/storage/utils/utils.go b/test/e2e/storage/utils/utils.go index 7772132362a..5c212ba3295 100644 --- a/test/e2e/storage/utils/utils.go +++ b/test/e2e/storage/utils/utils.go @@ -441,3 +441,45 @@ func PrivilegedTestPSPClusterRoleBinding(client clientset.Interface, } } + +func CheckVolumeModeOfPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path string) { + if volMode == v1.PersistentVolumeBlock { + // Check if block exists + VerifyExecInPodSucceed(pod, fmt.Sprintf("test -b %s", path)) + + // Double check that it's not directory + VerifyExecInPodFail(pod, fmt.Sprintf("test -d %s", path), 1) + } else { + // Check if directory exists + VerifyExecInPodSucceed(pod, fmt.Sprintf("test -d %s", path)) + + // Double check that it's not block + VerifyExecInPodFail(pod, fmt.Sprintf("test -b %s", path), 1) + } +} + +func CheckReadWriteToPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path string) { + if volMode == v1.PersistentVolumeBlock { + // random -> file1 + VerifyExecInPodSucceed(pod, "dd if=/dev/urandom of=/tmp/file1 bs=64 count=1") + // file1 -> dev (write to dev) + VerifyExecInPodSucceed(pod, fmt.Sprintf("dd if=/tmp/file1 of=%s bs=64 count=1", path)) + // dev -> file2 (read from dev) + VerifyExecInPodSucceed(pod, fmt.Sprintf("dd if=%s of=/tmp/file2 bs=64 count=1", path)) + // file1 == file2 (check contents) + VerifyExecInPodSucceed(pod, "diff /tmp/file1 /tmp/file2") + // Clean up temp files + VerifyExecInPodSucceed(pod, "rm -f /tmp/file1 /tmp/file2") + + // Check that writing file to block volume fails + VerifyExecInPodFail(pod, fmt.Sprintf("echo 'Hello world.' > %s/file1.txt", path), 1) + } else { + // text -> file1 (write to file) + VerifyExecInPodSucceed(pod, fmt.Sprintf("echo 'Hello world.' > %s/file1.txt", path)) + // grep file1 (read from file and check contents) + VerifyExecInPodSucceed(pod, fmt.Sprintf("grep 'Hello world.' %s/file1.txt", path)) + + // Check that writing to directory as block volume fails + VerifyExecInPodFail(pod, fmt.Sprintf("dd if=/dev/urandom of=%s bs=64 count=1", path), 1) + } +} diff --git a/test/e2e/storage/volume_provisioning.go b/test/e2e/storage/volume_provisioning.go index cde6712dc5c..db0e357595d 100644 --- a/test/e2e/storage/volume_provisioning.go +++ b/test/e2e/storage/volume_provisioning.go @@ -54,57 +54,87 @@ import ( const ( // Plugin name of the external provisioner externalPluginName = "example.com/nfs" + // Number of PVCs for multi PVC tests + multiPVCcount = 3 ) func testBindingWaitForFirstConsumer(client clientset.Interface, claim *v1.PersistentVolumeClaim, class *storage.StorageClass) (*v1.PersistentVolume, *v1.Node) { + pvs, node := testBindingWaitForFirstConsumerMultiPVC(client, []*v1.PersistentVolumeClaim{claim}, class) + return pvs[0], node +} + +func testBindingWaitForFirstConsumerMultiPVC(client clientset.Interface, claims []*v1.PersistentVolumeClaim, class *storage.StorageClass) ([]*v1.PersistentVolume, *v1.Node) { var err error + Expect(len(claims)).ToNot(Equal(0)) + namespace := claims[0].Namespace By("creating a storage class " + class.Name) class, err = client.StorageV1().StorageClasses().Create(class) Expect(err).NotTo(HaveOccurred()) defer deleteStorageClass(client, class.Name) - By("creating a claim") - claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim) - Expect(err).NotTo(HaveOccurred()) + By("creating claims") + var claimNames []string + var createdClaims []*v1.PersistentVolumeClaim + for _, claim := range claims { + c, err := client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim) + claimNames = append(claimNames, c.Name) + createdClaims = append(createdClaims, c) + Expect(err).NotTo(HaveOccurred()) + } defer func() { - framework.ExpectNoError(framework.DeletePersistentVolumeClaim(client, claim.Name, claim.Namespace), "Failed to delete PVC ", claim.Name) + var errors map[string]error + for _, claim := range createdClaims { + err := framework.DeletePersistentVolumeClaim(client, claim.Name, claim.Namespace) + if err != nil { + errors[claim.Name] = err + } + } + if len(errors) > 0 { + for claimName, err := range errors { + framework.Logf("Failed to delete PVC: %s due to error: %v", claimName, err) + } + } }() - // Wait for ClaimProvisionTimeout and make sure the phase did not become Bound i.e. the Wait errors out - err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, 2*time.Second, framework.ClaimProvisionShortTimeout) + // Wait for ClaimProvisionTimeout (across all PVCs in parallel) and make sure the phase did not become Bound i.e. the Wait errors out + By("checking the claims are in pending state") + err = framework.WaitForPersistentVolumeClaimsPhase(v1.ClaimBound, client, namespace, claimNames, 2*time.Second, framework.ClaimProvisionShortTimeout, true) Expect(err).To(HaveOccurred()) + for _, claim := range createdClaims { + // Get new copy of the claim + claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(claim.Status.Phase).To(Equal(v1.ClaimPending)) + } - By("checking the claim is in pending state") - // Get new copy of the claim - claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - Expect(claim.Status.Phase).To(Equal(v1.ClaimPending)) - - By("creating a pod referring to the claim") + By("creating a pod referring to the claims") // Create a pod referring to the claim and wait for it to get to running - pod, err := framework.CreateClientPod(client, claim.Namespace, claim) + pod, err := framework.CreatePod(client, namespace, nil /* nodeSelector */, createdClaims, true /* isPrivileged */, "" /* command */) Expect(err).NotTo(HaveOccurred()) defer func() { framework.DeletePodOrFail(client, pod.Namespace, pod.Name) }() - - By("re-checking the claim to see it binded") - // Get new copy of the claim - claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - // make sure claim did bind - err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) - Expect(err).NotTo(HaveOccurred()) - - // collect node and pv details + // collect node details node, err := client.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) - pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) + By("re-checking the claims to see they binded") + var pvs []*v1.PersistentVolume + for _, claim := range createdClaims { + // Get new copy of the claim + claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + // make sure claim did bind + err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) + Expect(err).NotTo(HaveOccurred()) - return pv, node + pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + pvs = append(pvs, pv) + } + Expect(len(pvs)).ToNot(Equal(0)) + return pvs, node } func checkZoneFromLabelAndAffinity(pv *v1.PersistentVolume, zone string, matchZone bool) { @@ -231,6 +261,67 @@ func checkGCEPD(volume *v1.PersistentVolume, volumeType string) error { return nil } +func testZonalDelayedBinding(c clientset.Interface, ns string, specifyAllowedTopology bool, pvcCount int) { + storageClassTestNameFmt := "Delayed binding %s storage class test %s" + storageClassTestNameSuffix := "" + if specifyAllowedTopology { + storageClassTestNameSuffix += " with AllowedTopologies" + } + tests := []testsuites.StorageClassTest{ + { + Name: fmt.Sprintf(storageClassTestNameFmt, "EBS", storageClassTestNameSuffix), + CloudProviders: []string{"aws"}, + Provisioner: "kubernetes.io/aws-ebs", + ClaimSize: "2Gi", + DelayBinding: true, + }, + { + Name: fmt.Sprintf(storageClassTestNameFmt, "GCE PD", storageClassTestNameSuffix), + CloudProviders: []string{"gce", "gke"}, + Provisioner: "kubernetes.io/gce-pd", + ClaimSize: "2Gi", + DelayBinding: true, + }, + } + for _, test := range tests { + if !framework.ProviderIs(test.CloudProviders...) { + framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders) + continue + } + action := "creating claims with class with waitForFirstConsumer" + suffix := "delayed" + var topoZone string + class := newStorageClass(test, ns, suffix) + if specifyAllowedTopology { + action += " and allowedTopologies" + suffix += "-topo" + topoZone = getRandomCloudZone(c) + addSingleZoneAllowedTopologyToStorageClass(c, class, topoZone) + } + By(action) + var claims []*v1.PersistentVolumeClaim + for i := 0; i < pvcCount; i++ { + claim := newClaim(test, ns, suffix) + claim.Spec.StorageClassName = &class.Name + claims = append(claims, claim) + } + pvs, node := testBindingWaitForFirstConsumerMultiPVC(c, claims, class) + if node == nil { + framework.Failf("unexpected nil node found") + } + zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain] + if !ok { + framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) + } + if specifyAllowedTopology && topoZone != zone { + framework.Failf("zone specified in allowedTopologies: %s does not match zone of node where PV got provisioned: %s", topoZone, zone) + } + for _, pv := range pvs { + checkZoneFromLabelAndAffinity(pv, zone, true) + } + } +} + var _ = utils.SIGDescribe("Dynamic Provisioning", func() { f := framework.NewDefaultFramework("volume-provisioning") @@ -846,43 +937,9 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { }) }) Describe("DynamicProvisioner delayed binding [Slow]", func() { - It("should create a persistent volume in the same zone as node after a pod mounting the claim is started", func() { - tests := []testsuites.StorageClassTest{ - { - Name: "Delayed binding EBS storage class test", - CloudProviders: []string{"aws"}, - Provisioner: "kubernetes.io/aws-ebs", - ClaimSize: "2Gi", - DelayBinding: true, - }, - { - Name: "Delayed binding GCE PD storage class test", - CloudProviders: []string{"gce", "gke"}, - Provisioner: "kubernetes.io/gce-pd", - ClaimSize: "2Gi", - DelayBinding: true, - }, - } - for _, test := range tests { - if !framework.ProviderIs(test.CloudProviders...) { - framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders) - continue - } - By("creating a claim with class with waitForFirstConsumer") - suffix := "delayed" - class := newStorageClass(test, ns, suffix) - claim := newClaim(test, ns, suffix) - claim.Spec.StorageClassName = &class.Name - pv, node := testBindingWaitForFirstConsumer(c, claim, class) - if node == nil { - framework.Failf("unexpected nil node found") - } - zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain] - if !ok { - framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) - } - checkZoneFromLabelAndAffinity(pv, zone, true) - } + It("should create persistent volumes in the same zone as node after a pod mounting the claims is started", func() { + testZonalDelayedBinding(c, ns, false /*specifyAllowedTopology*/, 1 /*pvcCount*/) + testZonalDelayedBinding(c, ns, false /*specifyAllowedTopology*/, 3 /*pvcCount*/) }) }) Describe("DynamicProvisioner allowedTopologies", func() { @@ -921,51 +978,11 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { }) }) Describe("DynamicProvisioner delayed binding with allowedTopologies [Slow]", func() { - It("should create persistent volume in the same zone as specified in allowedTopologies after a pod mounting the claim is started", func() { - tests := []testsuites.StorageClassTest{ - { - Name: "AllowedTopologies and delayed binding EBS storage class test", - CloudProviders: []string{"aws"}, - Provisioner: "kubernetes.io/aws-ebs", - ClaimSize: "2Gi", - DelayBinding: true, - }, - { - Name: "AllowedTopologies and delayed binding GCE PD storage class test", - CloudProviders: []string{"gce", "gke"}, - Provisioner: "kubernetes.io/gce-pd", - ClaimSize: "2Gi", - DelayBinding: true, - }, - } - for _, test := range tests { - if !framework.ProviderIs(test.CloudProviders...) { - framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders) - continue - } - By("creating a claim with class with WaitForFirstConsumer and allowedTopologies") - suffix := "delayed-topo" - class := newStorageClass(test, ns, suffix) - topoZone := getRandomCloudZone(c) - addSingleZoneAllowedTopologyToStorageClass(c, class, topoZone) - claim := newClaim(test, ns, suffix) - claim.Spec.StorageClassName = &class.Name - pv, node := testBindingWaitForFirstConsumer(c, claim, class) - if node == nil { - framework.Failf("unexpected nil node found") - } - nodeZone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain] - if !ok { - framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) - } - if topoZone != nodeZone { - framework.Failf("zone specified in allowedTopologies: %s does not match zone of node where PV got provisioned: %s", topoZone, nodeZone) - } - checkZoneFromLabelAndAffinity(pv, topoZone, true) - } + It("should create persistent volumes in the same zone as specified in allowedTopologies after a pod mounting the claims is started", func() { + testZonalDelayedBinding(c, ns, true /*specifyAllowedTopology*/, 1 /*pvcCount*/) + testZonalDelayedBinding(c, ns, true /*specifyAllowedTopology*/, 3 /*pvcCount*/) }) }) - }) func getDefaultStorageClassName(c clientset.Interface) string { diff --git a/test/e2e/storage/vsphere/BUILD b/test/e2e/storage/vsphere/BUILD index 8d0e22c106d..3c23b80976f 100644 --- a/test/e2e/storage/vsphere/BUILD +++ b/test/e2e/storage/vsphere/BUILD @@ -53,7 +53,6 @@ go_library( "//test/e2e/framework:go_default_library", "//test/e2e/storage/utils:go_default_library", "//test/utils/image:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/vmware/govmomi:go_default_library", @@ -65,6 +64,7 @@ go_library( "//vendor/github.com/vmware/govmomi/vim25/soap:go_default_library", "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library", "//vendor/gopkg.in/gcfg.v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/e2e/storage/vsphere/connection.go b/test/e2e/storage/vsphere/connection.go index 60e8e40ac09..bb212ae50f8 100644 --- a/test/e2e/storage/vsphere/connection.go +++ b/test/e2e/storage/vsphere/connection.go @@ -22,10 +22,10 @@ import ( neturl "net/url" "sync" - "github.com/golang/glog" "github.com/vmware/govmomi" "github.com/vmware/govmomi/session" "github.com/vmware/govmomi/vim25" + "k8s.io/klog" ) const ( @@ -46,7 +46,7 @@ func Connect(ctx context.Context, vs *VSphere) error { if vs.Client == nil { vs.Client, err = NewClient(ctx, vs) if err != nil { - glog.Errorf("Failed to create govmomi client. err: %+v", err) + klog.Errorf("Failed to create govmomi client. err: %+v", err) return err } return nil @@ -54,17 +54,17 @@ func Connect(ctx context.Context, vs *VSphere) error { manager := session.NewManager(vs.Client.Client) userSession, err := manager.UserSession(ctx) if err != nil { - glog.Errorf("Error while obtaining user session. err: %+v", err) + klog.Errorf("Error while obtaining user session. err: %+v", err) return err } if userSession != nil { return nil } - glog.Warningf("Creating new client session since the existing session is not valid or not authenticated") + klog.Warningf("Creating new client session since the existing session is not valid or not authenticated") vs.Client.Logout(ctx) vs.Client, err = NewClient(ctx, vs) if err != nil { - glog.Errorf("Failed to create govmomi client. err: %+v", err) + klog.Errorf("Failed to create govmomi client. err: %+v", err) return err } return nil @@ -74,13 +74,13 @@ func Connect(ctx context.Context, vs *VSphere) error { func NewClient(ctx context.Context, vs *VSphere) (*govmomi.Client, error) { url, err := neturl.Parse(fmt.Sprintf("https://%s:%s/sdk", vs.Config.Hostname, vs.Config.Port)) if err != nil { - glog.Errorf("Failed to parse URL: %s. err: %+v", url, err) + klog.Errorf("Failed to parse URL: %s. err: %+v", url, err) return nil, err } url.User = neturl.UserPassword(vs.Config.Username, vs.Config.Password) client, err := govmomi.NewClient(ctx, url, true) if err != nil { - glog.Errorf("Failed to create new client. err: %+v", err) + klog.Errorf("Failed to create new client. err: %+v", err) return nil, err } if vs.Config.RoundTripperCount == 0 { diff --git a/test/e2e/storage/vsphere/vsphere_utils.go b/test/e2e/storage/vsphere/vsphere_utils.go index 99e61e268cf..443c3366d81 100644 --- a/test/e2e/storage/vsphere/vsphere_utils.go +++ b/test/e2e/storage/vsphere/vsphere_utils.go @@ -24,13 +24,13 @@ import ( "strings" "time" - "github.com/golang/glog" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" vim25types "github.com/vmware/govmomi/vim25/types" + "k8s.io/klog" "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" @@ -478,7 +478,7 @@ func getVirtualDiskPage83Data(ctx context.Context, dc *object.Datacenter, diskPa diskUUID, err := vdm.QueryVirtualDiskUuid(ctx, diskPath, dc) if err != nil { - glog.Warningf("QueryVirtualDiskUuid failed for diskPath: %q. err: %+v", diskPath, err) + klog.Warningf("QueryVirtualDiskUuid failed for diskPath: %q. err: %+v", diskPath, err) return "", err } diskUUID = formatVirtualDiskUUID(diskUUID) diff --git a/test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml b/test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml index a4f4f1aacef..cd256039555 100644 --- a/test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml @@ -11,7 +11,7 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: csi-driver-registrar + name: csi-node-sa # replace with non-default namespace name namespace: default @@ -42,7 +42,7 @@ metadata: name: csi-driver-registrar-role subjects: - kind: ServiceAccount - name: csi-driver-registrar + name: csi-node-sa # replace with non-default namespace name namespace: default roleRef: diff --git a/test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml index 707f73a2de7..18f1eff454f 100644 --- a/test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml @@ -79,3 +79,4 @@ subjects: roleRef: kind: Role name: external-attacher-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml index 5d46d34bdc8..65bd2e42e97 100644 --- a/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml @@ -87,3 +87,4 @@ subjects: roleRef: kind: Role name: external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml b/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml index 0cb4476d4be..0cdf4764e77 100644 --- a/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml +++ b/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml @@ -1,54 +1,51 @@ kind: StatefulSet apiVersion: apps/v1 metadata: - name: csi-gce-controller + name: csi-gce-pd-controller spec: serviceName: "csi-gce-pd" replicas: 1 selector: matchLabels: - app: csi-gce-pd-driver + app: gcp-compute-persistent-disk-csi-driver template: metadata: labels: - app: csi-gce-pd-driver + app: gcp-compute-persistent-disk-csi-driver spec: - serviceAccountName: csi-controller + serviceAccountName: csi-controller-sa containers: - - name: csi-external-provisioner - imagePullPolicy: Always - image: gcr.io/gke-release/csi-provisioner:v0.4.1-gke.0 + - name: csi-provisioner + image: gcr.io/gke-release/csi-provisioner:v1.0.0-gke.0 args: - "--v=5" - - "--provisioner=com.google.csi.gcepd" + - "--provisioner=pd.csi.storage.gke.io" - "--csi-address=/csi/csi.sock" volumeMounts: - name: socket-dir mountPath: /csi - name: csi-attacher - imagePullPolicy: Always - image: gcr.io/gke-release/csi-attacher:v0.4.1-gke.0 + image: gcr.io/gke-release/csi-attacher:v1.0.0-gke.0 args: - "--v=5" - "--csi-address=/csi/csi.sock" volumeMounts: - name: socket-dir mountPath: /csi - - name: gce-driver - imagePullPolicy: Always - image: gcr.io/gke-release/gcp-compute-persistent-disk-csi-driver:v0.2.0-gke.0 + - name: gce-pd-driver + image: gcr.io/gke-release/gcp-compute-persistent-disk-csi-driver:v0.3.0-gke.0 args: - "--v=5" - - "--endpoint=unix:///csi/csi.sock" + - "--endpoint=unix:/csi/csi.sock" env: - name: GOOGLE_APPLICATION_CREDENTIALS - value: "/etc/service-account/cloud-sa.json" + value: "/etc/cloud-sa/cloud-sa.json" volumeMounts: - name: socket-dir mountPath: /csi - name: cloud-sa-volume readOnly: true - mountPath: "/etc/service-account" + mountPath: "/etc/cloud-sa" volumes: - name: socket-dir emptyDir: {} diff --git a/test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml b/test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml index 2e143302f9e..55417c4b4be 100644 --- a/test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: csi-controller + name: csi-controller-sa --- kind: ClusterRoleBinding @@ -10,7 +10,7 @@ metadata: name: csi-controller-attacher-role subjects: - kind: ServiceAccount - name: csi-controller + name: csi-controller-sa namespace: default roleRef: kind: ClusterRole @@ -25,7 +25,7 @@ metadata: namespace: default subjects: - kind: ServiceAccount - name: csi-controller + name: csi-controller-sa namespace: default roleRef: kind: Role @@ -38,7 +38,7 @@ metadata: name: csi-controller-provisioner-role subjects: - kind: ServiceAccount - name: csi-controller + name: csi-controller-sa namespace: default roleRef: kind: ClusterRole @@ -53,7 +53,7 @@ metadata: namespace: default subjects: - kind: ServiceAccount - name: csi-controller + name: csi-controller-sa namespace: default roleRef: kind: Role @@ -67,10 +67,10 @@ metadata: name: psp-csi-controller-driver-registrar-role subjects: - kind: ServiceAccount - name: csi-controller + name: csi-controller-sa namespace: default - kind: ServiceAccount - name: csi-driver-registrar + name: csi-node-sa namespace: default roleRef: kind: ClusterRole diff --git a/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml b/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml index 9c0a8b8d3f0..5f2f1ddd330 100644 --- a/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml +++ b/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml @@ -1,26 +1,28 @@ kind: DaemonSet apiVersion: apps/v1 metadata: - name: csi-gce-node + name: csi-gce-pd-node spec: selector: matchLabels: - app: csi-gce-driver - serviceName: csi-gce + app: gcp-compute-persistent-disk-csi-driver template: metadata: labels: - app: csi-gce-driver + app: gcp-compute-persistent-disk-csi-driver spec: - serviceAccountName: csi-driver-registrar + serviceAccountName: csi-node-sa containers: - name: csi-driver-registrar - imagePullPolicy: Always - image: gcr.io/gke-release/csi-driver-registrar:v0.4.1-gke.0 + image: gcr.io/gke-release/csi-driver-registrar:v1.0.0-gke.0 args: - "--v=5" - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/com.google.csi.gcepd/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/pd.csi.storage.gke.io /registration/pd.csi.storage.gke.io-reg.sock"] env: - name: KUBE_NODE_NAME valueFrom: @@ -31,14 +33,13 @@ spec: mountPath: /csi - name: registration-dir mountPath: /registration - - name: gce-driver + - name: gce-pd-driver securityContext: privileged: true - imagePullPolicy: Always - image: gcr.io/gke-release/gcp-compute-persistent-disk-csi-driver:v0.2.0-gke.0 + image: gcr.io/gke-release/gcp-compute-persistent-disk-csi-driver:v0.3.0-gke.0 args: - "--v=5" - - "--endpoint=unix:///csi/csi.sock" + - "--endpoint=unix:/csi/csi.sock" volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet @@ -67,7 +68,7 @@ spec: type: Directory - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/com.google.csi.gcepd/ + path: /var/lib/kubelet/plugins/pd.csi.storage.gke.io/ type: DirectoryOrCreate - name: device-dir hostPath: @@ -90,4 +91,3 @@ spec: hostPath: path: /sys type: Directory - diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml index 6fe198bd3ab..dc2d4c4b7e8 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml @@ -30,7 +30,7 @@ spec: serviceAccountName: csi-attacher containers: - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:v0.4.1 + image: gcr.io/gke-release/csi-attacher:v1.0.0-gke.0 args: - --v=5 - --csi-address=$(ADDRESS) diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml index 84b2c05baad..4f117e11284 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml @@ -30,7 +30,7 @@ spec: serviceAccountName: csi-provisioner containers: - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v0.4.1 + image: gcr.io/gke-release/csi-provisioner:v1.0.0-gke.0 args: - "--provisioner=csi-hostpath" - "--csi-address=$(ADDRESS)" diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml index f7c326b4a87..4df7cea194d 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml @@ -11,11 +11,11 @@ spec: labels: app: csi-hostpathplugin spec: - serviceAccountName: csi-driver-registrar + serviceAccountName: csi-node-sa hostNetwork: true containers: - name: driver-registrar - image: quay.io/k8scsi/driver-registrar:v0.4.1 + image: gcr.io/gke-release/csi-driver-registrar:v1.0.0-gke.0 args: - --v=5 - --csi-address=/csi/csi.sock @@ -33,7 +33,7 @@ spec: - mountPath: /registration name: registration-dir - name: hostpath - image: quay.io/k8scsi/hostpathplugin:v0.4.1 + image: quay.io/k8scsi/hostpathplugin:v1.0.0 args: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml index 1628db1b588..aa008ecac29 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml @@ -8,7 +8,7 @@ subjects: name: csi-attacher namespace: default - kind: ServiceAccount - name: csi-driver-registrar + name: csi-node-sa namespace: default - kind: ServiceAccount name: csi-provisioner diff --git a/test/e2e/upgrades/storage/BUILD b/test/e2e/upgrades/storage/BUILD index ed5351c7a5c..dfe4773f3cb 100644 --- a/test/e2e/upgrades/storage/BUILD +++ b/test/e2e/upgrades/storage/BUILD @@ -7,12 +7,18 @@ load( go_library( name = "go_default_library", - srcs = ["persistent_volumes.go"], + srcs = [ + "persistent_volumes.go", + "volume_mode.go", + ], importpath = "k8s.io/kubernetes/test/e2e/upgrades/storage", deps = [ "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/storage/utils:go_default_library", "//test/e2e/upgrades:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", diff --git a/test/e2e/upgrades/storage/volume_mode.go b/test/e2e/upgrades/storage/volume_mode.go new file mode 100644 index 00000000000..1ef316bb995 --- /dev/null +++ b/test/e2e/upgrades/storage/volume_mode.go @@ -0,0 +1,125 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "fmt" + "time" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" + "k8s.io/kubernetes/test/e2e/upgrades" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const devicePath = "/mnt/volume1" + +// VolumeModeDowngradeTest tests that a VolumeMode Block PV is not mistakenly +// formatted and mounted like a nil/Filesystem PV after a downgrade to a version +// where the BlockVolume feature is disabled +type VolumeModeDowngradeTest struct { + pvSource *v1.PersistentVolumeSource + pv *v1.PersistentVolume + pvc *v1.PersistentVolumeClaim + pod *v1.Pod +} + +func (VolumeModeDowngradeTest) Name() string { + return "[sig-storage] volume-mode-downgrade" +} + +func (t *VolumeModeDowngradeTest) Skip(upgCtx upgrades.UpgradeContext) bool { + if !framework.ProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure") { + return true + } + + // Only run when downgrading from >= 1.13 to < 1.13 + blockVersion := version.MustParseSemantic("1.13.0-alpha.0") + if upgCtx.Versions[0].Version.LessThan(blockVersion) { + return true + } + if !upgCtx.Versions[1].Version.LessThan(blockVersion) { + return true + } + + return false +} + +// Setup creates a block pv and then verifies that a pod can consume it. The pod writes data to the volume. +func (t *VolumeModeDowngradeTest) Setup(f *framework.Framework) { + + var err error + + cs := f.ClientSet + ns := f.Namespace.Name + + By("Creating a PVC") + block := v1.PersistentVolumeBlock + pvcConfig := framework.PersistentVolumeClaimConfig{ + StorageClassName: nil, + VolumeMode: &block, + } + t.pvc = framework.MakePersistentVolumeClaim(pvcConfig, ns) + t.pvc, err = framework.CreatePVC(cs, ns, t.pvc) + Expect(err).NotTo(HaveOccurred()) + + err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, cs, ns, t.pvc.Name, framework.Poll, framework.ClaimProvisionTimeout) + Expect(err).NotTo(HaveOccurred()) + + t.pvc, err = cs.CoreV1().PersistentVolumeClaims(t.pvc.Namespace).Get(t.pvc.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + t.pv, err = cs.CoreV1().PersistentVolumes().Get(t.pvc.Spec.VolumeName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Consuming the PVC before downgrade") + t.pod, err = framework.CreateSecPod(cs, ns, []*v1.PersistentVolumeClaim{t.pvc}, false, "", false, false, framework.SELinuxLabel, nil, framework.PodStartTimeout) + Expect(err).NotTo(HaveOccurred()) + + By("Checking if PV exists as expected volume mode") + utils.CheckVolumeModeOfPath(t.pod, block, devicePath) + + By("Checking if read/write to PV works properly") + utils.CheckReadWriteToPath(t.pod, block, devicePath) +} + +// Test waits for the downgrade to complete, and then verifies that a pod can no +// longer consume the pv as it is not mapped nor mounted into the pod +func (t *VolumeModeDowngradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) { + By("Waiting for downgrade to finish") + <-done + + By("Verifying that nothing exists at the device path in the pod") + utils.VerifyExecInPodFail(t.pod, fmt.Sprintf("test -e %s", devicePath), 1) +} + +// Teardown cleans up any remaining resources. +func (t *VolumeModeDowngradeTest) Teardown(f *framework.Framework) { + By("Deleting the pod") + framework.ExpectNoError(framework.DeletePodWithWait(f, f.ClientSet, t.pod)) + + By("Deleting the PVC") + framework.ExpectNoError(f.ClientSet.CoreV1().PersistentVolumeClaims(t.pvc.Namespace).Delete(t.pvc.Name, nil)) + + By("Waiting for the PV to be deleted") + framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(f.ClientSet, t.pv.Name, 5*time.Second, 20*time.Minute)) +} diff --git a/test/e2e_kubeadm/runner/local/BUILD b/test/e2e_kubeadm/runner/local/BUILD index 412adbe48e3..c0fbdc31655 100644 --- a/test/e2e_kubeadm/runner/local/BUILD +++ b/test/e2e_kubeadm/runner/local/BUILD @@ -7,7 +7,7 @@ go_library( visibility = ["//visibility:private"], deps = [ "//test/utils:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/e2e_kubeadm/runner/local/run_local.go b/test/e2e_kubeadm/runner/local/run_local.go index 9ccdfcb7309..4adcba10948 100644 --- a/test/e2e_kubeadm/runner/local/run_local.go +++ b/test/e2e_kubeadm/runner/local/run_local.go @@ -25,7 +25,7 @@ import ( "runtime" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/test/utils" ) @@ -49,25 +49,25 @@ func main() { if *build { if err := bazelBuild(); err != nil { - glog.Exitf("couldn't build with bazel: %v", err) + klog.Exitf("couldn't build with bazel: %v", err) } } ginkgo, err := getBazelGinkgo() if err != nil { - glog.Fatalf("Failed to get ginkgo binary: %v", err) + klog.Fatalf("Failed to get ginkgo binary: %v", err) } test, err := getBazelTestBin() if err != nil { - glog.Fatalf("Failed to get test file: %v", err) + klog.Fatalf("Failed to get test file: %v", err) } args := append(strings.Split(*ginkgoFlags, " "), test, "--") args = append(args, strings.Split(*testFlags, " ")...) if execCommand(ginkgo, args...); err != nil { - glog.Exitf("Test failed: %v", err) + klog.Exitf("Test failed: %v", err) } } diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index 2670233e2b3..51c902c0e35 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -24,12 +24,15 @@ go_library( "//pkg/kubelet/apis/cri:go_default_library", "//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library", "//pkg/kubelet/apis/deviceplugin/v1beta1:go_default_library", + "//pkg/kubelet/apis/podresources:go_default_library", + "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/cm/devicemanager:go_default_library", "//pkg/kubelet/kubeletconfig/util/codec:go_default_library", "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/remote:go_default_library", + "//pkg/kubelet/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", @@ -46,10 +49,11 @@ go_library( "//vendor/github.com/coreos/go-systemd/util:go_default_library", "//vendor/github.com/docker/docker/api/types:go_default_library", "//vendor/github.com/docker/docker/client:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/prometheus/common/model:go_default_library", + "//vendor/golang.org/x/net/context:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//pkg/api/v1/node:go_default_library", @@ -149,12 +153,12 @@ go_test( "//vendor/github.com/blang/semver:go_default_library", "//vendor/github.com/coreos/go-systemd/util:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/onsi/gomega/gstruct:go_default_library", "//vendor/github.com/onsi/gomega/types:go_default_library", "//vendor/github.com/prometheus/common/model:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//cmd/kubeadm/app/util/system:go_default_library", diff --git a/test/e2e_node/apparmor_test.go b/test/e2e_node/apparmor_test.go index 0bf75efe64a..bda5fe1b73f 100644 --- a/test/e2e_node/apparmor_test.go +++ b/test/e2e_node/apparmor_test.go @@ -37,9 +37,9 @@ import ( "k8s.io/kubernetes/test/e2e/framework" "github.com/davecgh/go-spew/spew" - "github.com/golang/glog" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "k8s.io/klog" ) var _ = framework.KubeDescribe("AppArmor [Feature:AppArmor][NodeFeature:AppArmor]", func() { @@ -132,14 +132,14 @@ func loadTestProfiles() error { // apparmor_parser does not always return an error code, so consider any stderr output an error. if err != nil || stderr.Len() > 0 { if stderr.Len() > 0 { - glog.Warning(stderr.String()) + klog.Warning(stderr.String()) } if len(out) > 0 { - glog.Infof("apparmor_parser: %s", out) + klog.Infof("apparmor_parser: %s", out) } return fmt.Errorf("failed to load profiles: %v", err) } - glog.V(2).Infof("Loaded profiles: %v", out) + klog.V(2).Infof("Loaded profiles: %v", out) return nil } @@ -211,7 +211,7 @@ func isAppArmorEnabled() bool { if len(matches) == 2 { version, err := strconv.Atoi(matches[1]) if err != nil { - glog.Errorf("Error parsing GCI version from NodeName %q: %v", framework.TestContext.NodeName, err) + klog.Errorf("Error parsing GCI version from NodeName %q: %v", framework.TestContext.NodeName, err) return false } return version >= 54 diff --git a/test/e2e_node/builder/BUILD b/test/e2e_node/builder/BUILD index eeda57b85d7..5dd457b5946 100644 --- a/test/e2e_node/builder/BUILD +++ b/test/e2e_node/builder/BUILD @@ -11,7 +11,7 @@ go_library( importpath = "k8s.io/kubernetes/test/e2e_node/builder", deps = [ "//test/utils:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/e2e_node/builder/build.go b/test/e2e_node/builder/build.go index 87ea32a87c7..797de1dc538 100644 --- a/test/e2e_node/builder/build.go +++ b/test/e2e_node/builder/build.go @@ -24,7 +24,7 @@ import ( "path/filepath" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/test/utils" ) @@ -38,7 +38,7 @@ var buildTargets = []string{ } func BuildGo() error { - glog.Infof("Building k8s binaries...") + klog.Infof("Building k8s binaries...") k8sRoot, err := utils.GetK8sRootDir() if err != nil { return fmt.Errorf("failed to locate kubernetes root directory %v.", err) @@ -90,7 +90,7 @@ func getK8sBin(bin string) (string, error) { func GetKubeletServerBin() string { bin, err := getK8sBin("kubelet") if err != nil { - glog.Fatalf("Could not locate kubelet binary %v.", err) + klog.Fatalf("Could not locate kubelet binary %v.", err) } return bin } diff --git a/test/e2e_node/device_plugin.go b/test/e2e_node/device_plugin.go index 62c8fc07497..3951f8c538b 100644 --- a/test/e2e_node/device_plugin.go +++ b/test/e2e_node/device_plugin.go @@ -60,7 +60,11 @@ func testDevicePlugin(f *framework.Framework, enablePluginWatcher bool, pluginSo Context("DevicePlugin", func() { By("Enabling support for Kubelet Plugins Watcher") tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) { + if initialConfig.FeatureGates == nil { + initialConfig.FeatureGates = map[string]bool{} + } initialConfig.FeatureGates[string(features.KubeletPluginsWatcher)] = enablePluginWatcher + initialConfig.FeatureGates[string(features.KubeletPodResources)] = true }) It("Verifies the Kubelet device plugin functionality.", func() { By("Start stub device plugin") @@ -98,6 +102,17 @@ func testDevicePlugin(f *framework.Framework, enablePluginWatcher bool, pluginSo devId1 := parseLog(f, pod1.Name, pod1.Name, deviceIDRE) Expect(devId1).To(Not(Equal(""))) + podResources, err := getNodeDevices() + Expect(err).To(BeNil()) + Expect(len(podResources.PodResources)).To(Equal(1)) + Expect(podResources.PodResources[0].Name).To(Equal(pod1.Name)) + Expect(podResources.PodResources[0].Namespace).To(Equal(pod1.Namespace)) + Expect(len(podResources.PodResources[0].Containers)).To(Equal(1)) + Expect(podResources.PodResources[0].Containers[0].Name).To(Equal(pod1.Spec.Containers[0].Name)) + Expect(len(podResources.PodResources[0].Containers[0].Devices)).To(Equal(1)) + Expect(podResources.PodResources[0].Containers[0].Devices[0].ResourceName).To(Equal(resourceName)) + Expect(len(podResources.PodResources[0].Containers[0].Devices[0].DeviceIds)).To(Equal(1)) + pod1, err = f.PodClient().Get(pod1.Name, metav1.GetOptions{}) framework.ExpectNoError(err) diff --git a/test/e2e_node/dynamic_kubelet_config_test.go b/test/e2e_node/dynamic_kubelet_config_test.go index 60ad4956548..9071dd0913d 100644 --- a/test/e2e_node/dynamic_kubelet_config_test.go +++ b/test/e2e_node/dynamic_kubelet_config_test.go @@ -418,6 +418,70 @@ var _ = framework.KubeDescribe("[Feature:DynamicKubeletConfig][NodeFeature:Dynam }) }) + // previously, we missed a panic because we were not exercising this path + Context("update Node.Spec.ConfigSource: non-nil last-known-good to a new non-nil last-known-good", func() { + It(itDescription, func() { + var err error + // we base the "lkg" configmap off of the configuration from before the test + lkgKC := beforeKC.DeepCopy() + lkgConfigMap1 := newKubeletConfigMap("dynamic-kubelet-config-test-lkg-1", lkgKC) + lkgConfigMap1, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(lkgConfigMap1) + framework.ExpectNoError(err) + + lkgSource1 := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Namespace: lkgConfigMap1.Namespace, + Name: lkgConfigMap1.Name, + KubeletConfigKey: "kubelet", + }} + lkgStatus1 := lkgSource1.DeepCopy() + lkgStatus1.ConfigMap.UID = lkgConfigMap1.UID + lkgStatus1.ConfigMap.ResourceVersion = lkgConfigMap1.ResourceVersion + + lkgConfigMap2 := newKubeletConfigMap("dynamic-kubelet-config-test-lkg-2", lkgKC) + lkgConfigMap2, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(lkgConfigMap2) + framework.ExpectNoError(err) + + lkgSource2 := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Namespace: lkgConfigMap2.Namespace, + Name: lkgConfigMap2.Name, + KubeletConfigKey: "kubelet", + }} + lkgStatus2 := lkgSource2.DeepCopy() + lkgStatus2.ConfigMap.UID = lkgConfigMap2.UID + lkgStatus2.ConfigMap.ResourceVersion = lkgConfigMap2.ResourceVersion + + // cases + first := nodeConfigTestCase{ + desc: "last-known-good-1", + configSource: lkgSource1, + configMap: lkgConfigMap1, + expectConfigStatus: expectNodeConfigStatus{ + lastKnownGood: lkgStatus1, + }, + expectConfig: lkgKC, + event: true, + } + + second := nodeConfigTestCase{ + desc: "last-known-good-2", + configSource: lkgSource2, + configMap: lkgConfigMap2, + expectConfigStatus: expectNodeConfigStatus{ + lastKnownGood: lkgStatus2, + }, + expectConfig: lkgKC, + event: true, + } + + // Manually actuate this to ensure we wait for each case to become the last-known-good + const lkgDuration = 12 * time.Minute + By(fmt.Sprintf("setting initial state %q", first.desc)) + first.run(f, setConfigSourceFunc, true, lkgDuration) + By(fmt.Sprintf("from %q to %q", first.desc, second.desc)) + second.run(f, setConfigSourceFunc, true, lkgDuration) + }) + }) + // exposes resource leaks across config changes Context("update Node.Spec.ConfigSource: 100 update stress test:", func() { It(itDescription, func() { diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index acf3b1fa338..41357cc407d 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -44,13 +44,13 @@ import ( "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e_node/services" - "github.com/golang/glog" "github.com/kardianos/osext" . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/config" morereporters "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/spf13/pflag" + "k8s.io/klog" ) var e2es *services.E2EServices @@ -103,7 +103,7 @@ func TestE2eNode(t *testing.T) { var err error spec, err = loadSystemSpecFromFile(*systemSpecFile) if err != nil { - glog.Exitf("Failed to load system spec: %v", err) + klog.Exitf("Failed to load system spec: %v", err) } } if framework.TestContext.NodeConformance { @@ -112,11 +112,11 @@ func TestE2eNode(t *testing.T) { // TODO(random-liu): Consider to chroot the whole test process to make writing // test easier. if err := syscall.Chroot(rootfs); err != nil { - glog.Exitf("chroot %q failed: %v", rootfs, err) + klog.Exitf("chroot %q failed: %v", rootfs, err) } } if _, err := system.ValidateSpec(*spec, framework.TestContext.ContainerRuntime); err != nil { - glog.Exitf("system validation failed: %v", err) + klog.Exitf("system validation failed: %v", err) } return } @@ -127,7 +127,7 @@ func TestE2eNode(t *testing.T) { if reportDir != "" { // Create the directory if it doesn't already exists if err := os.MkdirAll(reportDir, 0755); err != nil { - glog.Errorf("Failed creating report directory: %v", err) + klog.Errorf("Failed creating report directory: %v", err) } else { // Configure a junit reporter to write to the directory junitFile := fmt.Sprintf("junit_%s_%02d.xml", framework.TestContext.ReportPrefix, config.GinkgoConfig.ParallelNode) @@ -146,7 +146,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { // Pre-pull the images tests depend on so we can fail immediately if there is an image pull issue // This helps with debugging test flakes since it is hard to tell when a test failure is due to image pulling. if framework.TestContext.PrepullImages { - glog.Infof("Pre-pulling images so that they are cached for the tests.") + klog.Infof("Pre-pulling images so that they are cached for the tests.") err := PrePullAllImages() Expect(err).ShouldNot(HaveOccurred()) } @@ -161,12 +161,12 @@ var _ = SynchronizedBeforeSuite(func() []byte { // If the services are expected to keep running after test, they should not monitor the test process. e2es = services.NewE2EServices(*stopServices) Expect(e2es.Start()).To(Succeed(), "should be able to start node services.") - glog.Infof("Node services started. Running tests...") + klog.Infof("Node services started. Running tests...") } else { - glog.Infof("Running tests without starting services.") + klog.Infof("Running tests without starting services.") } - glog.Infof("Wait for the node to be ready") + klog.Infof("Wait for the node to be ready") waitForNodeReady() // Reference common test to make the import valid. @@ -182,12 +182,12 @@ var _ = SynchronizedBeforeSuite(func() []byte { var _ = SynchronizedAfterSuite(func() {}, func() { if e2es != nil { if *startServices && *stopServices { - glog.Infof("Stopping node services...") + klog.Infof("Stopping node services...") e2es.Stop() } } - glog.Infof("Tests Finished") + klog.Infof("Tests Finished") }) // validateSystem runs system validation in a separate process and returns error if validation fails. @@ -210,13 +210,13 @@ func maskLocksmithdOnCoreos() { data, err := ioutil.ReadFile("/etc/os-release") if err != nil { // Not all distros contain this file. - glog.Infof("Could not read /etc/os-release: %v", err) + klog.Infof("Could not read /etc/os-release: %v", err) return } if bytes.Contains(data, []byte("ID=coreos")) { output, err := exec.Command("systemctl", "mask", "--now", "locksmithd").CombinedOutput() Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("should be able to mask locksmithd - output: %q", string(output))) - glog.Infof("Locksmithd is masked successfully") + klog.Infof("Locksmithd is masked successfully") } } diff --git a/test/e2e_node/image_list.go b/test/e2e_node/image_list.go index e7bc3f9baee..dcf62ef2790 100644 --- a/test/e2e_node/image_list.go +++ b/test/e2e_node/image_list.go @@ -22,7 +22,7 @@ import ( "os/user" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/util/sets" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" @@ -128,7 +128,7 @@ func PrePullAllImages() error { return err } images := framework.ImageWhiteList.List() - glog.V(4).Infof("Pre-pulling images with %s %+v", puller.Name(), images) + klog.V(4).Infof("Pre-pulling images with %s %+v", puller.Name(), images) for _, image := range images { var ( err error @@ -141,11 +141,11 @@ func PrePullAllImages() error { if output, err = puller.Pull(image); err == nil { break } - glog.Warningf("Failed to pull %s as user %q, retrying in %s (%d of %d): %v", + klog.Warningf("Failed to pull %s as user %q, retrying in %s (%d of %d): %v", image, usr.Username, imagePullRetryDelay.String(), i+1, maxImagePullRetries, err) } if err != nil { - glog.Warningf("Could not pre-pull image %s %v output: %s", image, err, output) + klog.Warningf("Could not pre-pull image %s %v output: %s", image, err, output) return err } } diff --git a/test/e2e_node/pods_container_manager_test.go b/test/e2e_node/pods_container_manager_test.go index 128860b17c0..f7ba0048a3a 100644 --- a/test/e2e_node/pods_container_manager_test.go +++ b/test/e2e_node/pods_container_manager_test.go @@ -27,9 +27,9 @@ import ( "k8s.io/kubernetes/test/e2e/framework" imageutils "k8s.io/kubernetes/test/utils/image" - "github.com/golang/glog" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "k8s.io/klog" ) // getResourceList returns a ResourceList with the @@ -71,7 +71,7 @@ func makePodToVerifyCgroups(cgroupNames []string) *v1.Pod { cgroupName := cm.NewCgroupName(rootCgroupName, cgroupComponents...) cgroupFsNames = append(cgroupFsNames, toCgroupFsName(cgroupName)) } - glog.Infof("expecting %v cgroups to be found", cgroupFsNames) + klog.Infof("expecting %v cgroups to be found", cgroupFsNames) // build the pod command to either verify cgroups exist command := "" for _, cgroupFsName := range cgroupFsNames { diff --git a/test/e2e_node/remote/BUILD b/test/e2e_node/remote/BUILD index 9a6dcdd50e0..3952cd69cac 100644 --- a/test/e2e_node/remote/BUILD +++ b/test/e2e_node/remote/BUILD @@ -21,7 +21,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//test/e2e_node/builder:go_default_library", "//test/utils:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/e2e_node/remote/cadvisor_e2e.go b/test/e2e_node/remote/cadvisor_e2e.go index 8bdb567d031..6beb2c7c44e 100644 --- a/test/e2e_node/remote/cadvisor_e2e.go +++ b/test/e2e_node/remote/cadvisor_e2e.go @@ -22,7 +22,7 @@ import ( "os/exec" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/test/utils" ) @@ -67,7 +67,7 @@ func (n *CAdvisorE2ERemote) RunTest(host, workspace, results, imageDesc, junitFi // Kill any running node processes cleanupNodeProcesses(host) - glog.V(2).Infof("Starting tests on %q", host) + klog.V(2).Infof("Starting tests on %q", host) return SSH(host, "sh", "-c", getSSHCommand(" && ", fmt.Sprintf("cd %s/cadvisor", workspace), fmt.Sprintf("timeout -k 30s %fs ./build/integration.sh ../results/cadvisor.log", diff --git a/test/e2e_node/remote/node_conformance.go b/test/e2e_node/remote/node_conformance.go index 9c78ae30887..39ef80f5e4b 100644 --- a/test/e2e_node/remote/node_conformance.go +++ b/test/e2e_node/remote/node_conformance.go @@ -25,7 +25,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/test/e2e_node/builder" "k8s.io/kubernetes/test/utils" @@ -206,13 +206,13 @@ func launchKubelet(host, workspace, results, testArgs string) error { ), } } - glog.V(2).Infof("Launch kubelet with command: %v", cmd) + klog.V(2).Infof("Launch kubelet with command: %v", cmd) output, err := SSH(host, cmd...) if err != nil { return fmt.Errorf("failed to launch kubelet with command %v: error - %v output - %q", cmd, err, output) } - glog.Info("Successfully launch kubelet") + klog.Info("Successfully launch kubelet") return nil } @@ -221,12 +221,12 @@ const kubeletStopGracePeriod = 10 * time.Second // stopKubelet stops kubelet launcher and kubelet gracefully. func stopKubelet(host, workspace string) error { - glog.Info("Gracefully stop kubelet launcher") + klog.Info("Gracefully stop kubelet launcher") if output, err := SSH(host, "pkill", conformanceTestBinary); err != nil { return fmt.Errorf("failed to gracefully stop kubelet launcher: error - %v output - %q", err, output) } - glog.Info("Wait for kubelet launcher to stop") + klog.Info("Wait for kubelet launcher to stop") stopped := false for start := time.Now(); time.Since(start) < kubeletStopGracePeriod; time.Sleep(time.Second) { // Check whether the process is still running. @@ -242,13 +242,13 @@ func stopKubelet(host, workspace string) error { } } if !stopped { - glog.Info("Forcibly stop kubelet") + klog.Info("Forcibly stop kubelet") if output, err := SSH(host, "pkill", "-SIGKILL", conformanceTestBinary); err != nil { return fmt.Errorf("failed to forcibly stop kubelet: error - %v output - %q", err, output) } } - glog.Info("Successfully stop kubelet") + klog.Info("Successfully stop kubelet") // Clean up the pod manifest path podManifestPath := getPodPath(workspace) if output, err := SSH(host, "rm", "-f", filepath.Join(workspace, podManifestPath)); err != nil { @@ -286,12 +286,12 @@ func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFi defer func() { if err := stopKubelet(host, workspace); err != nil { // Only log an error if failed to stop kubelet because it is not critical. - glog.Errorf("failed to stop kubelet: %v", err) + klog.Errorf("failed to stop kubelet: %v", err) } }() // Run the tests - glog.V(2).Infof("Starting tests on %q", host) + klog.V(2).Infof("Starting tests on %q", host) podManifestPath := getPodPath(workspace) cmd := fmt.Sprintf("'timeout -k 30s %fs docker run --rm --privileged=true --net=host -v /:/rootfs -v %s:%s -v %s:/var/result -e TEST_ARGS=--report-prefix=%s %s'", timeout.Seconds(), podManifestPath, podManifestPath, results, junitFilePrefix, getConformanceTestImageName(systemSpecName)) diff --git a/test/e2e_node/remote/node_e2e.go b/test/e2e_node/remote/node_e2e.go index 082437975ea..4a45594bfcf 100644 --- a/test/e2e_node/remote/node_e2e.go +++ b/test/e2e_node/remote/node_e2e.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/test/e2e_node/builder" "k8s.io/kubernetes/test/utils" @@ -108,7 +108,7 @@ func tarAddFile(tar, source, dest string) error { // prependCOSMounterFlag prepends the flag for setting the GCI mounter path to // args and returns the result. func prependCOSMounterFlag(args, host, workspace string) (string, error) { - glog.V(2).Infof("GCI/COS node and GCI/COS mounter both detected, modifying --experimental-mounter-path accordingly") + klog.V(2).Infof("GCI/COS node and GCI/COS mounter both detected, modifying --experimental-mounter-path accordingly") mounterPath := filepath.Join(workspace, "mounter") args = fmt.Sprintf("--kubelet-flags=--experimental-mounter-path=%s ", mounterPath) + args return args, nil @@ -164,7 +164,7 @@ func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePr } // Run the tests - glog.V(2).Infof("Starting tests on %q", host) + klog.V(2).Infof("Starting tests on %q", host) cmd := getSSHCommand(" && ", fmt.Sprintf("cd %s", workspace), fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --system-spec-name=%s --system-spec-file=%s --logtostderr --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=\"%s\" %s", diff --git a/test/e2e_node/remote/remote.go b/test/e2e_node/remote/remote.go index 746899f8b57..1a0ff30290c 100644 --- a/test/e2e_node/remote/remote.go +++ b/test/e2e_node/remote/remote.go @@ -27,8 +27,8 @@ import ( "strings" "time" - "github.com/golang/glog" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog" ) var testTimeoutSeconds = flag.Duration("test-timeout", 45*time.Minute, "How long (in golang duration format) to wait for ginkgo tests to complete.") @@ -37,7 +37,7 @@ var resultsDir = flag.String("results-dir", "/tmp/", "Directory to scp test resu const archiveName = "e2e_node_test.tar.gz" func CreateTestArchive(suite TestSuite, systemSpecName string) (string, error) { - glog.V(2).Infof("Building archive...") + klog.V(2).Infof("Building archive...") tardir, err := ioutil.TempDir("", "node-e2e-archive") if err != nil { return "", fmt.Errorf("failed to create temporary directory %v.", err) @@ -67,7 +67,7 @@ func CreateTestArchive(suite TestSuite, systemSpecName string) (string, error) { // TODO(random-liu): junitFilePrefix is not prefix actually, the file name is junit-junitFilePrefix.xml. Change the variable name. func RunRemote(suite TestSuite, archive string, host string, cleanup bool, imageDesc, junitFilePrefix string, testArgs string, ginkgoArgs string, systemSpecName string) (string, bool, error) { // Create the temp staging directory - glog.V(2).Infof("Staging test binaries on %q", host) + klog.V(2).Infof("Staging test binaries on %q", host) workspace := newWorkspaceDir() // Do not sudo here, so that we can use scp to copy test archive to the directdory. if output, err := SSHNoSudo(host, "mkdir", workspace); err != nil { @@ -78,7 +78,7 @@ func RunRemote(suite TestSuite, archive string, host string, cleanup bool, image defer func() { output, err := SSH(host, "rm", "-rf", workspace) if err != nil { - glog.Errorf("failed to cleanup workspace %q on host %q: %v. Output:\n%s", workspace, host, err, output) + klog.Errorf("failed to cleanup workspace %q on host %q: %v. Output:\n%s", workspace, host, err, output) } }() } @@ -94,7 +94,7 @@ func RunRemote(suite TestSuite, archive string, host string, cleanup bool, image fmt.Sprintf("cd %s", workspace), fmt.Sprintf("tar -xzvf ./%s", archiveName), ) - glog.V(2).Infof("Extracting tar on %q", host) + klog.V(2).Infof("Extracting tar on %q", host) // Do not use sudo here, because `sudo tar -x` will recover the file ownership inside the tar ball, but // we want the extracted files to be owned by the current user. if output, err := SSHNoSudo(host, "sh", "-c", cmd); err != nil { @@ -109,7 +109,7 @@ func RunRemote(suite TestSuite, archive string, host string, cleanup bool, image return "", false, fmt.Errorf("failed to create test result directory %q on host %q: %v output: %q", resultDir, host, err, output) } - glog.V(2).Infof("Running test on %q", host) + klog.V(2).Infof("Running test on %q", host) output, err := suite.RunTest(host, workspace, resultDir, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName, *testTimeoutSeconds) aggErrs := []error{} @@ -119,7 +119,7 @@ func RunRemote(suite TestSuite, archive string, host string, cleanup bool, image collectSystemLog(host) } - glog.V(2).Infof("Copying test artifacts from %q", host) + klog.V(2).Infof("Copying test artifacts from %q", host) scpErr := getTestArtifacts(host, workspace) if scpErr != nil { aggErrs = append(aggErrs, scpErr) @@ -194,17 +194,17 @@ func collectSystemLog(host string) { logPath = fmt.Sprintf("/tmp/%s-%s", getTimestamp(), logName) destPath = fmt.Sprintf("%s/%s-%s", *resultsDir, host, logName) ) - glog.V(2).Infof("Test failed unexpectedly. Attempting to retrieving system logs (only works for nodes with journald)") + klog.V(2).Infof("Test failed unexpectedly. Attempting to retrieving system logs (only works for nodes with journald)") // Try getting the system logs from journald and store it to a file. // Don't reuse the original test directory on the remote host because // it could've be been removed if the node was rebooted. if output, err := SSH(host, "sh", "-c", fmt.Sprintf("'journalctl --system --all > %s'", logPath)); err == nil { - glog.V(2).Infof("Got the system logs from journald; copying it back...") + klog.V(2).Infof("Got the system logs from journald; copying it back...") if output, err := runSSHCommand("scp", fmt.Sprintf("%s:%s", GetHostnameOrIp(host), logPath), destPath); err != nil { - glog.V(2).Infof("Failed to copy the log: err: %v, output: %q", err, output) + klog.V(2).Infof("Failed to copy the log: err: %v, output: %q", err, output) } } else { - glog.V(2).Infof("Failed to run journactl (normal if it doesn't exist on the node): %v, output: %q", err, output) + klog.V(2).Infof("Failed to run journactl (normal if it doesn't exist on the node): %v, output: %q", err, output) } } diff --git a/test/e2e_node/remote/ssh.go b/test/e2e_node/remote/ssh.go index fe82a664637..4a4f0d82bd0 100644 --- a/test/e2e_node/remote/ssh.go +++ b/test/e2e_node/remote/ssh.go @@ -24,7 +24,7 @@ import ( "strings" "sync" - "github.com/golang/glog" + "k8s.io/klog" ) var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.") @@ -38,7 +38,7 @@ var sshDefaultKeyMap map[string]string func init() { usr, err := user.Current() if err != nil { - glog.Fatal(err) + klog.Fatal(err) } sshOptionsMap = map[string]string{ "gce": "-o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o CheckHostIP=no -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o LogLevel=ERROR", diff --git a/test/e2e_node/remote/utils.go b/test/e2e_node/remote/utils.go index 28ab74cf52a..fab3fbeb10f 100644 --- a/test/e2e_node/remote/utils.go +++ b/test/e2e_node/remote/utils.go @@ -21,7 +21,7 @@ import ( "path/filepath" "strings" - "github.com/golang/glog" + "k8s.io/klog" ) // utils.go contains functions used across test suites. @@ -52,7 +52,7 @@ const cniConfig = `{ // Install the cni plugin and add basic bridge configuration to the // configuration directory. func setupCNI(host, workspace string) error { - glog.V(2).Infof("Install CNI on %q", host) + klog.V(2).Infof("Install CNI on %q", host) cniPath := filepath.Join(workspace, cniDirectory) cmd := getSSHCommand(" ; ", fmt.Sprintf("mkdir -p %s", cniPath), @@ -65,7 +65,7 @@ func setupCNI(host, workspace string) error { // The added CNI network config is not needed for kubenet. It is only // used when testing the CNI network plugin, but is added in both cases // for consistency and simplicity. - glog.V(2).Infof("Adding CNI configuration on %q", host) + klog.V(2).Infof("Adding CNI configuration on %q", host) cniConfigPath := filepath.Join(workspace, cniConfDirectory) cmd = getSSHCommand(" ; ", fmt.Sprintf("mkdir -p %s", cniConfigPath), @@ -79,7 +79,7 @@ func setupCNI(host, workspace string) error { // configureFirewall configures iptable firewall rules. func configureFirewall(host string) error { - glog.V(2).Infof("Configure iptables firewall rules on %q", host) + klog.V(2).Infof("Configure iptables firewall rules on %q", host) // TODO: consider calling bootstrap script to configure host based on OS output, err := SSH(host, "iptables", "-L", "INPUT") if err != nil { @@ -114,7 +114,7 @@ func configureFirewall(host string) error { // cleanupNodeProcesses kills all running node processes may conflict with the test. func cleanupNodeProcesses(host string) { - glog.V(2).Infof("Killing any existing node processes on %q", host) + klog.V(2).Infof("Killing any existing node processes on %q", host) cmd := getSSHCommand(" ; ", "pkill kubelet", "pkill kube-apiserver", diff --git a/test/e2e_node/runner/local/BUILD b/test/e2e_node/runner/local/BUILD index 8bd0e3b8d1e..50d8969a89d 100644 --- a/test/e2e_node/runner/local/BUILD +++ b/test/e2e_node/runner/local/BUILD @@ -18,7 +18,7 @@ go_library( deps = [ "//test/e2e_node/builder:go_default_library", "//test/utils:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/e2e_node/runner/local/run_local.go b/test/e2e_node/runner/local/run_local.go index 9b011999294..4a69a997b62 100644 --- a/test/e2e_node/runner/local/run_local.go +++ b/test/e2e_node/runner/local/run_local.go @@ -27,7 +27,7 @@ import ( "k8s.io/kubernetes/test/e2e_node/builder" "k8s.io/kubernetes/test/utils" - "github.com/golang/glog" + "k8s.io/klog" ) var buildDependencies = flag.Bool("build-dependencies", true, "If true, build all dependencies.") @@ -40,21 +40,22 @@ const ( ) func main() { + klog.InitFlags(nil) flag.Parse() // Build dependencies - ginkgo, kubelet and apiserver. if *buildDependencies { if err := builder.BuildGo(); err != nil { - glog.Fatalf("Failed to build the dependencies: %v", err) + klog.Fatalf("Failed to build the dependencies: %v", err) } } // Run node e2e test outputDir, err := utils.GetK8sBuildOutputDir() if err != nil { - glog.Fatalf("Failed to get build output directory: %v", err) + klog.Fatalf("Failed to get build output directory: %v", err) } - glog.Infof("Got build output dir: %v", outputDir) + klog.Infof("Got build output dir: %v", outputDir) ginkgo := filepath.Join(outputDir, "ginkgo") test := filepath.Join(outputDir, "e2e_node.test") @@ -62,19 +63,19 @@ func main() { if *systemSpecName != "" { rootDir, err := utils.GetK8sRootDir() if err != nil { - glog.Fatalf("Failed to get k8s root directory: %v", err) + klog.Fatalf("Failed to get k8s root directory: %v", err) } systemSpecFile := filepath.Join(rootDir, systemSpecPath, *systemSpecName+".yaml") args = append(args, fmt.Sprintf("--system-spec-name=%s --system-spec-file=%s", *systemSpecName, systemSpecFile)) } if err := runCommand(ginkgo, args...); err != nil { - glog.Exitf("Test failed: %v", err) + klog.Exitf("Test failed: %v", err) } return } func runCommand(name string, args ...string) error { - glog.Infof("Running command: %v %v", name, strings.Join(args, " ")) + klog.Infof("Running command: %v %v", name, strings.Join(args, " ")) cmd := exec.Command("sudo", "sh", "-c", strings.Join(append([]string{name}, args...), " ")) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/e2e_node/runner/remote/BUILD b/test/e2e_node/runner/remote/BUILD index 36ed1b88153..88c58616d3f 100644 --- a/test/e2e_node/runner/remote/BUILD +++ b/test/e2e_node/runner/remote/BUILD @@ -17,12 +17,12 @@ go_library( importpath = "k8s.io/kubernetes/test/e2e_node/runner/remote", deps = [ "//test/e2e_node/remote:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pborman/uuid:go_default_library", "//vendor/golang.org/x/oauth2:go_default_library", "//vendor/golang.org/x/oauth2/google:go_default_library", "//vendor/google.golang.org/api/compute/v0.beta:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/test/e2e_node/runner/remote/run_remote.go b/test/e2e_node/runner/remote/run_remote.go index 1dca0808c72..c2131da22c7 100644 --- a/test/e2e_node/runner/remote/run_remote.go +++ b/test/e2e_node/runner/remote/run_remote.go @@ -37,12 +37,12 @@ import ( "k8s.io/kubernetes/test/e2e_node/remote" - "github.com/ghodss/yaml" - "github.com/golang/glog" "github.com/pborman/uuid" "golang.org/x/oauth2" "golang.org/x/oauth2/google" compute "google.golang.org/api/compute/v0.beta" + "k8s.io/klog" + "sigs.k8s.io/yaml" ) var testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.") @@ -174,6 +174,7 @@ type internalGCEImage struct { } func main() { + klog.InitFlags(nil) flag.Parse() switch *testSuite { case "conformance": @@ -185,7 +186,7 @@ func main() { // Use node e2e suite by default if no subcommand is specified. suite = remote.InitNodeE2ERemote() default: - glog.Fatalf("--test-suite must be one of default or conformance") + klog.Fatalf("--test-suite must be one of default or conformance") } rand.Seed(time.Now().UnixNano()) @@ -196,12 +197,12 @@ func main() { } if *hosts == "" && *imageConfigFile == "" && *images == "" { - glog.Fatalf("Must specify one of --image-config-file, --hosts, --images.") + klog.Fatalf("Must specify one of --image-config-file, --hosts, --images.") } var err error computeService, err = getComputeClient() if err != nil { - glog.Fatalf("Unable to create gcloud compute service using defaults. Make sure you are authenticated. %v", err) + klog.Fatalf("Unable to create gcloud compute service using defaults. Make sure you are authenticated. %v", err) } gceImages := &internalImageConfig{ @@ -216,12 +217,12 @@ func main() { // parse images imageConfigData, err := ioutil.ReadFile(configPath) if err != nil { - glog.Fatalf("Could not read image config file provided: %v", err) + klog.Fatalf("Could not read image config file provided: %v", err) } externalImageConfig := ImageConfig{Images: make(map[string]GCEImage)} err = yaml.Unmarshal(imageConfigData, &externalImageConfig) if err != nil { - glog.Fatalf("Could not parse image config file: %v", err) + klog.Fatalf("Could not parse image config file: %v", err) } for shortName, imageConfig := range externalImageConfig.Images { var images []string @@ -230,7 +231,7 @@ func main() { isRegex = true images, err = getGCEImages(imageConfig.ImageRegex, imageConfig.Project, imageConfig.PreviousImages) if err != nil { - glog.Fatalf("Could not retrieve list of images based on image prefix %q: %v", imageConfig.ImageRegex, err) + klog.Fatalf("Could not retrieve list of images based on image prefix %q: %v", imageConfig.ImageRegex, err) } } else { images = []string{imageConfig.Image} @@ -265,7 +266,7 @@ func main() { // convenience; merge in with config file if *images != "" { if *imageProject == "" { - glog.Fatal("Must specify --image-project if you specify --images") + klog.Fatal("Must specify --image-project if you specify --images") } cliImages := strings.Split(*images, ",") for _, img := range cliImages { @@ -279,16 +280,16 @@ func main() { } if len(gceImages.images) != 0 && *zone == "" { - glog.Fatal("Must specify --zone flag") + klog.Fatal("Must specify --zone flag") } for shortName, image := range gceImages.images { if image.project == "" { - glog.Fatalf("Invalid config for %v; must specify a project", shortName) + klog.Fatalf("Invalid config for %v; must specify a project", shortName) } } if len(gceImages.images) != 0 { if *project == "" { - glog.Fatal("Must specify --project flag to launch images into") + klog.Fatal("Must specify --project flag to launch images into") } } if *instanceNamePrefix == "" { @@ -394,9 +395,9 @@ func getImageMetadata(input string) *compute.Metadata { if input == "" { return nil } - glog.V(3).Infof("parsing instance metadata: %q", input) + klog.V(3).Infof("parsing instance metadata: %q", input) raw := parseInstanceMetadata(input) - glog.V(4).Infof("parsed instance metadata: %v", raw) + klog.V(4).Infof("parsed instance metadata: %v", raw) metadataItems := []*compute.MetadataItems{} for k, v := range raw { val := v @@ -482,7 +483,7 @@ func getGCEImages(imageRegex, project string, previousImages int) ([]string, err creationTime: creationTime, name: instance.Name, } - glog.V(4).Infof("Found image %q based on regex %q in project %q", io.string(), imageRegex, project) + klog.V(4).Infof("Found image %q based on regex %q in project %q", io.string(), imageRegex, project) imageObjs = append(imageObjs, io) } } @@ -531,12 +532,12 @@ func testImage(imageConfig *internalGCEImage, junitFilePrefix string) *TestResul // TODO(random-liu): Extract out and unify log collection logic with cluste e2e. serialPortOutput, err := computeService.Instances.GetSerialPortOutput(*project, *zone, host).Port(1).Do() if err != nil { - glog.Errorf("Failed to collect serial output from node %q: %v", host, err) + klog.Errorf("Failed to collect serial output from node %q: %v", host, err) } else { logFilename := "serial-1.log" err := remote.WriteLog(host, logFilename, serialPortOutput.Contents) if err != nil { - glog.Errorf("Failed to write serial output from node %q to %q: %v", host, logFilename, err) + klog.Errorf("Failed to write serial output from node %q to %q: %v", host, logFilename, err) } } return result @@ -544,7 +545,7 @@ func testImage(imageConfig *internalGCEImage, junitFilePrefix string) *TestResul // Provision a gce instance using image func createInstance(imageConfig *internalGCEImage) (string, error) { - glog.V(1).Infof("Creating instance %+v", *imageConfig) + klog.V(1).Infof("Creating instance %+v", *imageConfig) name := imageToInstanceName(imageConfig) i := &compute.Instance{ Name: name, @@ -712,10 +713,10 @@ func getComputeClient() (*compute.Service, error) { } func deleteInstance(host string) { - glog.Infof("Deleting instance %q", host) + klog.Infof("Deleting instance %q", host) _, err := computeService.Instances.Delete(*project, *zone, host).Do() if err != nil { - glog.Errorf("Error deleting instance %q: %v", host, err) + klog.Errorf("Error deleting instance %q: %v", host, err) } } @@ -730,7 +731,7 @@ func parseInstanceMetadata(str string) map[string]string { } kp := strings.Split(s, "<") if len(kp) != 2 { - glog.Fatalf("Invalid instance metadata: %q", s) + klog.Fatalf("Invalid instance metadata: %q", s) continue } metaPath := kp[1] @@ -739,7 +740,7 @@ func parseInstanceMetadata(str string) map[string]string { } v, err := ioutil.ReadFile(metaPath) if err != nil { - glog.Fatalf("Failed to read metadata file %q: %v", metaPath, err) + klog.Fatalf("Failed to read metadata file %q: %v", metaPath, err) continue } metadata[kp[0]] = string(v) diff --git a/test/e2e_node/services/BUILD b/test/e2e_node/services/BUILD index 60d41f37d6b..5eb7899fde2 100644 --- a/test/e2e_node/services/BUILD +++ b/test/e2e_node/services/BUILD @@ -40,9 +40,9 @@ go_library( "//test/e2e/framework:go_default_library", "//test/e2e_node/builder:go_default_library", "//test/e2e_node/remote:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/kardianos/osext:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/e2e_node/services/internal_services.go b/test/e2e_node/services/internal_services.go index 356c00ef146..3890ffd2875 100644 --- a/test/e2e_node/services/internal_services.go +++ b/test/e2e_node/services/internal_services.go @@ -24,7 +24,7 @@ import ( "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/kubernetes/test/e2e/framework" - "github.com/golang/glog" + "k8s.io/klog" ) // e2eService manages e2e services in current process. @@ -55,7 +55,7 @@ func (es *e2eServices) run(t *testing.T) error { // start starts the tests embedded services or returns an error. func (es *e2eServices) start(t *testing.T) error { - glog.Info("Starting e2e services...") + klog.Info("Starting e2e services...") err := es.startEtcd(t) if err != nil { return err @@ -68,48 +68,48 @@ func (es *e2eServices) start(t *testing.T) error { if err != nil { return nil } - glog.Info("E2E services started.") + klog.Info("E2E services started.") return nil } // stop stops the embedded e2e services. func (es *e2eServices) stop(t *testing.T) { - glog.Info("Stopping e2e services...") + klog.Info("Stopping e2e services...") // TODO(random-liu): Use a loop to stop all services after introducing // service interface. - glog.Info("Stopping namespace controller") + klog.Info("Stopping namespace controller") if es.nsController != nil { if err := es.nsController.Stop(); err != nil { - glog.Errorf("Failed to stop %q: %v", es.nsController.Name(), err) + klog.Errorf("Failed to stop %q: %v", es.nsController.Name(), err) } } - glog.Info("Stopping API server") + klog.Info("Stopping API server") if es.apiServer != nil { if err := es.apiServer.Stop(); err != nil { - glog.Errorf("Failed to stop %q: %v", es.apiServer.Name(), err) + klog.Errorf("Failed to stop %q: %v", es.apiServer.Name(), err) } } - glog.Info("Stopping etcd") + klog.Info("Stopping etcd") if es.etcdServer != nil { es.etcdServer.Terminate(t) } for _, d := range es.rmDirs { - glog.Infof("Deleting directory %v", d) + klog.Infof("Deleting directory %v", d) err := os.RemoveAll(d) if err != nil { - glog.Errorf("Failed to delete directory %s.\n%v", d, err) + klog.Errorf("Failed to delete directory %s.\n%v", d, err) } } - glog.Info("E2E services stopped.") + klog.Info("E2E services stopped.") } // startEtcd starts the embedded etcd instance or returns an error. func (es *e2eServices) startEtcd(t *testing.T) error { - glog.Info("Starting etcd") + klog.Info("Starting etcd") server, etcdStorage := etcdtesting.NewUnsecuredEtcd3TestClientServer(t) es.etcdServer = server es.etcdStorage = etcdStorage @@ -118,14 +118,14 @@ func (es *e2eServices) startEtcd(t *testing.T) error { // startApiServer starts the embedded API server or returns an error. func (es *e2eServices) startApiServer(etcdStorage *storagebackend.Config) error { - glog.Info("Starting API server") + klog.Info("Starting API server") es.apiServer = NewAPIServer(*etcdStorage) return es.apiServer.Start() } // startNamespaceController starts the embedded namespace controller or returns an error. func (es *e2eServices) startNamespaceController() error { - glog.Info("Starting namespace controller") + klog.Info("Starting namespace controller") es.nsController = NewNamespaceController(framework.TestContext.Host) return es.nsController.Start() } diff --git a/test/e2e_node/services/kubelet.go b/test/e2e_node/services/kubelet.go index 21ef291c141..276dfbe36fa 100644 --- a/test/e2e_node/services/kubelet.go +++ b/test/e2e_node/services/kubelet.go @@ -26,8 +26,8 @@ import ( "strings" "time" - "github.com/golang/glog" "github.com/spf13/pflag" + "k8s.io/klog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -83,7 +83,7 @@ func RunKubelet() { defer e.Stop() e.kubelet, err = e.startKubelet() if err != nil { - glog.Fatalf("Failed to start kubelet: %v", err) + klog.Fatalf("Failed to start kubelet: %v", err) } // Wait until receiving a termination signal. waitForTerminationSignal() @@ -105,7 +105,7 @@ func (e *E2EServices) startKubelet() (*server, error) { return nil, fmt.Errorf("the --hyperkube-image option must be set") } - glog.Info("Starting kubelet") + klog.Info("Starting kubelet") // set feature gates so we can check which features are enabled and pass the appropriate flags utilfeature.DefaultFeatureGate.SetFromMap(framework.TestContext.FeatureGates) diff --git a/test/e2e_node/services/server.go b/test/e2e_node/services/server.go index 999c11da654..22a162f7a25 100644 --- a/test/e2e_node/services/server.go +++ b/test/e2e_node/services/server.go @@ -29,7 +29,7 @@ import ( "syscall" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/test/e2e/framework" ) @@ -99,7 +99,7 @@ func (s *server) String() string { // // Note: restartOnExit == true requires len(s.healthCheckUrls) > 0 to work properly. func (s *server) start() error { - glog.Infof("Starting server %q with command %q", s.name, commandToString(s.startCommand)) + klog.Infof("Starting server %q with command %q", s.name, commandToString(s.startCommand)) errCh := make(chan error) // Set up restart channels if the server is configured for restart on exit. @@ -127,7 +127,7 @@ func (s *server) start() error { errCh <- fmt.Errorf("failed to create file %q for `%s` %v.", outPath, s, err) return } else { - glog.Infof("Output file for server %q: %v", s.name, outfile.Name()) + klog.Infof("Output file for server %q: %v", s.name, outfile.Name()) } defer outfile.Close() defer outfile.Sync() @@ -158,7 +158,7 @@ func (s *server) start() error { return } if !s.restartOnExit { - glog.Infof("Waiting for server %q start command to complete", s.name) + klog.Infof("Waiting for server %q start command to complete", s.name) // If we aren't planning on restarting, ok to Wait() here to release resources. // Otherwise, we Wait() in the restart loop. err = s.startCommand.Wait() @@ -169,18 +169,18 @@ func (s *server) start() error { } else { usedStartCmd := true for { - glog.Infof("Running health check for service %q", s.name) + klog.Infof("Running health check for service %q", s.name) // Wait for an initial health check to pass, so that we are sure the server started. err := readinessCheck(s.name, s.healthCheckUrls, nil) if err != nil { if usedStartCmd { - glog.Infof("Waiting for server %q start command to complete after initial health check failed", s.name) + klog.Infof("Waiting for server %q start command to complete after initial health check failed", s.name) s.startCommand.Wait() // Release resources if necessary. } // This should not happen, immediately stop the e2eService process. - glog.Fatalf("Restart loop readinessCheck failed for %s", s) + klog.Fatalf("Restart loop readinessCheck failed for %s", s) } else { - glog.Infof("Initial health check passed for service %q", s.name) + klog.Infof("Initial health check passed for service %q", s.name) } // Initial health check passed, wait until a health check fails again. @@ -220,11 +220,11 @@ func (s *server) start() error { } // Run and wait for exit. This command is assumed to have // short duration, e.g. systemctl restart - glog.Infof("Restarting server %q with restart command", s.name) + klog.Infof("Restarting server %q with restart command", s.name) err = s.restartCommand.Run() if err != nil { // This should not happen, immediately stop the e2eService process. - glog.Fatalf("Restarting server %s with restartCommand failed. Error: %v.", s, err) + klog.Fatalf("Restarting server %s with restartCommand failed. Error: %v.", s, err) } } else { s.startCommand = &exec.Cmd{ @@ -238,12 +238,12 @@ func (s *server) start() error { ExtraFiles: s.startCommand.ExtraFiles, SysProcAttr: s.startCommand.SysProcAttr, } - glog.Infof("Restarting server %q with start command", s.name) + klog.Infof("Restarting server %q with start command", s.name) err = s.startCommand.Start() usedStartCmd = true if err != nil { // This should not happen, immediately stop the e2eService process. - glog.Fatalf("Restarting server %s with startCommand failed. Error: %v.", s, err) + klog.Fatalf("Restarting server %s with startCommand failed. Error: %v.", s, err) } } } @@ -255,7 +255,7 @@ func (s *server) start() error { // kill runs the server's kill command. func (s *server) kill() error { - glog.Infof("Kill server %q", s.name) + klog.Infof("Kill server %q", s.name) name := s.name cmd := s.startCommand @@ -274,7 +274,7 @@ func (s *server) kill() error { } if cmd.Process == nil { - glog.V(2).Infof("%q not running", name) + klog.V(2).Infof("%q not running", name) return nil } pid := cmd.Process.Pid @@ -292,11 +292,11 @@ func (s *server) kill() error { const timeout = 10 * time.Second for _, signal := range []string{"-TERM", "-KILL"} { - glog.V(2).Infof("Killing process %d (%s) with %s", pid, name, signal) + klog.V(2).Infof("Killing process %d (%s) with %s", pid, name, signal) cmd := exec.Command("kill", signal, strconv.Itoa(pid)) _, err := cmd.Output() if err != nil { - glog.Errorf("Error signaling process %d (%s) with %s: %v", pid, name, signal, err) + klog.Errorf("Error signaling process %d (%s) with %s: %v", pid, name, signal, err) continue } diff --git a/test/e2e_node/services/services.go b/test/e2e_node/services/services.go index b73658ab306..58ac3534ada 100644 --- a/test/e2e_node/services/services.go +++ b/test/e2e_node/services/services.go @@ -24,8 +24,8 @@ import ( "path" "testing" - "github.com/golang/glog" "github.com/kardianos/osext" + "k8s.io/klog" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/test/e2e/framework" @@ -86,19 +86,19 @@ func (e *E2EServices) Stop() { }() if e.services != nil { if err := e.services.kill(); err != nil { - glog.Errorf("Failed to stop services: %v", err) + klog.Errorf("Failed to stop services: %v", err) } } if e.kubelet != nil { if err := e.kubelet.kill(); err != nil { - glog.Errorf("Failed to stop kubelet: %v", err) + klog.Errorf("Failed to stop kubelet: %v", err) } } if e.rmDirs != nil { for _, d := range e.rmDirs { err := os.RemoveAll(d) if err != nil { - glog.Errorf("Failed to delete directory %s: %v", d, err) + klog.Errorf("Failed to delete directory %s: %v", d, err) } } } @@ -112,7 +112,7 @@ func RunE2EServices(t *testing.T) { utilfeature.DefaultFeatureGate.SetFromMap(framework.TestContext.FeatureGates) e := newE2EServices() if err := e.run(t); err != nil { - glog.Fatalf("Failed to run e2e services: %v", err) + klog.Fatalf("Failed to run e2e services: %v", err) } } @@ -143,7 +143,7 @@ func (e *E2EServices) collectLogFiles() { if framework.TestContext.ReportDir == "" { return } - glog.Info("Fetching log files...") + klog.Info("Fetching log files...") journaldFound := isJournaldAvailable() for targetFileName, log := range e.logs { targetLink := path.Join(framework.TestContext.ReportDir, targetFileName) @@ -152,13 +152,13 @@ func (e *E2EServices) collectLogFiles() { if len(log.JournalctlCommand) == 0 { continue } - glog.Infof("Get log file %q with journalctl command %v.", targetFileName, log.JournalctlCommand) + klog.Infof("Get log file %q with journalctl command %v.", targetFileName, log.JournalctlCommand) out, err := exec.Command("journalctl", log.JournalctlCommand...).CombinedOutput() if err != nil { - glog.Errorf("failed to get %q from journald: %v, %v", targetFileName, string(out), err) + klog.Errorf("failed to get %q from journald: %v, %v", targetFileName, string(out), err) } else { if err = ioutil.WriteFile(targetLink, out, 0644); err != nil { - glog.Errorf("failed to write logs to %q: %v", targetLink, err) + klog.Errorf("failed to write logs to %q: %v", targetLink, err) } } continue @@ -169,7 +169,7 @@ func (e *E2EServices) collectLogFiles() { continue } if err := copyLogFile(file, targetLink); err != nil { - glog.Error(err) + klog.Error(err) } else { break } diff --git a/test/e2e_node/services/util.go b/test/e2e_node/services/util.go index a21a483d5f8..0e1f47bb33d 100644 --- a/test/e2e_node/services/util.go +++ b/test/e2e_node/services/util.go @@ -18,7 +18,7 @@ package services import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" "net/http" "os" "os/signal" @@ -41,7 +41,7 @@ func waitForTerminationSignal() { // check URLs. Once there is an error in errCh, the function will stop waiting // and return the error. func readinessCheck(name string, urls []string, errCh <-chan error) error { - glog.Infof("Running readiness check for service %q", name) + klog.Infof("Running readiness check for service %q", name) endTime := time.Now().Add(*serverStartTimeout) blockCh := make(chan error) defer close(blockCh) diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index 14770308663..fd6353e90b2 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -27,7 +27,8 @@ import ( "strings" "time" - "github.com/golang/glog" + "golang.org/x/net/context" + "k8s.io/klog" apiv1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -38,11 +39,14 @@ import ( "k8s.io/kubernetes/pkg/features" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" + "k8s.io/kubernetes/pkg/kubelet/apis/podresources" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cm" kubeletconfigcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec" kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/kubelet/remote" + "k8s.io/kubernetes/pkg/kubelet/util" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/metrics" frameworkmetrics "k8s.io/kubernetes/test/e2e/framework/metrics" @@ -62,6 +66,10 @@ var busyboxImage = imageutils.GetE2EImage(imageutils.BusyBox) const ( // Kubelet internal cgroup name for node allocatable cgroup. defaultNodeAllocatableCgroup = "kubepods" + // defaultPodResourcesPath is the path to the local endpoint serving the podresources GRPC service. + defaultPodResourcesPath = "/var/lib/kubelet/pod-resources" + defaultPodResourcesTimeout = 10 * time.Second + defaultPodResourcesMaxSize = 1024 * 1024 * 16 // 16 Mb ) func getNodeSummary() (*stats.Summary, error) { @@ -92,6 +100,22 @@ func getNodeSummary() (*stats.Summary, error) { return &summary, nil } +func getNodeDevices() (*podresourcesapi.ListPodResourcesResponse, error) { + endpoint := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket) + client, conn, err := podresources.GetClient(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize) + if err != nil { + return nil, fmt.Errorf("Error getting grpc client: %v", err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := client.List(ctx, &podresourcesapi.ListPodResourcesRequest{}) + if err != nil { + return nil, fmt.Errorf("%v.Get(_) = _, %v", client, err) + } + return resp, nil +} + // Returns the current KubeletConfiguration func getCurrentKubeletConfig() (*kubeletconfig.KubeletConfiguration, error) { resp := pollConfigz(5*time.Minute, 5*time.Second) @@ -195,7 +219,7 @@ func setKubeletConfiguration(f *framework.Framework, kubeCfg *kubeletconfig.Kube if !apiequality.Semantic.DeepEqual(*kubeCfg, *newKubeCfg) { return fmt.Errorf("still waiting for new configuration to take effect, will continue to watch /configz") } - glog.Infof("new configuration has taken effect") + klog.Infof("new configuration has taken effect") return nil }, restartGap, pollInterval).Should(BeNil()) @@ -238,11 +262,11 @@ func pollConfigz(timeout time.Duration, pollInterval time.Duration) *http.Respon Eventually(func() bool { resp, err = client.Do(req) if err != nil { - glog.Errorf("Failed to get /configz, retrying. Error: %v", err) + klog.Errorf("Failed to get /configz, retrying. Error: %v", err) return false } if resp.StatusCode != 200 { - glog.Errorf("/configz response status not 200, retrying. Response was: %+v", resp) + klog.Errorf("/configz response status not 200, retrying. Response was: %+v", resp) return false } return true diff --git a/test/fixtures/pkg/kubectl/cmd/apply/cm.yaml b/test/fixtures/pkg/kubectl/cmd/apply/cm.yaml index 571afe8ae56..ef3200581d9 100644 --- a/test/fixtures/pkg/kubectl/cmd/apply/cm.yaml +++ b/test/fixtures/pkg/kubectl/cmd/apply/cm.yaml @@ -3,13 +3,13 @@ items: - kind: ConfigMap apiVersion: v1 metadata: - name: test + name: test0 data: key1: apple - kind: ConfigMap apiVersion: v1 metadata: - name: test2 + name: test1 data: key2: apple kind: ConfigMapList diff --git a/test/images/BUILD b/test/images/BUILD index a797bb178a7..a7f93f6cf3c 100644 --- a/test/images/BUILD +++ b/test/images/BUILD @@ -12,6 +12,7 @@ filegroup( srcs = [ ":package-srcs", "//test/images/apparmor-loader:all-srcs", + "//test/images/crd-conversion-webhook:all-srcs", "//test/images/echoserver:all-srcs", "//test/images/entrypoint-tester:all-srcs", "//test/images/fakegitserver:all-srcs", diff --git a/test/images/Makefile b/test/images/Makefile index 9418f1a9dd2..089c2696de9 100644 --- a/test/images/Makefile +++ b/test/images/Makefile @@ -17,7 +17,7 @@ include ../../hack/make-rules/Makefile.manifest REGISTRY ?= gcr.io/kubernetes-e2e-test-images GOARM=7 QEMUVERSION=v2.9.1 -GOLANG_VERSION=1.11.1 +GOLANG_VERSION=1.11.2 export ifndef WHAT diff --git a/test/images/apparmor-loader/BUILD b/test/images/apparmor-loader/BUILD index f033cc4ad88..8347d5292a1 100644 --- a/test/images/apparmor-loader/BUILD +++ b/test/images/apparmor-loader/BUILD @@ -5,7 +5,7 @@ go_library( srcs = ["loader.go"], importpath = "k8s.io/kubernetes/test/images/apparmor-loader", visibility = ["//visibility:private"], - deps = ["//vendor/github.com/golang/glog:go_default_library"], + deps = ["//vendor/k8s.io/klog:go_default_library"], ) go_binary( diff --git a/test/images/apparmor-loader/loader.go b/test/images/apparmor-loader/loader.go index 5f1e2095e6d..a716bd986d2 100644 --- a/test/images/apparmor-loader/loader.go +++ b/test/images/apparmor-loader/loader.go @@ -29,7 +29,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "k8s.io/klog" ) var ( @@ -53,19 +53,19 @@ func main() { dirs = flag.Args() if len(dirs) == 0 { - glog.Errorf("Must specify at least one directory.") + klog.Errorf("Must specify at least one directory.") flag.Usage() os.Exit(1) } // Check that the required parser binary is found. if _, err := exec.LookPath(parser); err != nil { - glog.Exitf("Required binary %s not found in PATH", parser) + klog.Exitf("Required binary %s not found in PATH", parser) } // Check that loaded profiles can be read. if _, err := getLoadedProfiles(); err != nil { - glog.Exitf("Unable to access apparmor profiles: %v", err) + klog.Exitf("Unable to access apparmor profiles: %v", err) } if *poll < 0 { @@ -79,26 +79,26 @@ func main() { func runOnce() { if success, newProfiles := loadNewProfiles(); !success { if len(newProfiles) > 0 { - glog.Exitf("Not all profiles were successfully loaded. Loaded: %v", newProfiles) + klog.Exitf("Not all profiles were successfully loaded. Loaded: %v", newProfiles) } else { - glog.Exit("Error loading profiles.") + klog.Exit("Error loading profiles.") } } else { if len(newProfiles) > 0 { - glog.Infof("Successfully loaded profiles: %v", newProfiles) + klog.Infof("Successfully loaded profiles: %v", newProfiles) } else { - glog.Warning("No new profiles found.") + klog.Warning("No new profiles found.") } } } // Poll the directories indefinitely. func pollForever() { - glog.V(2).Infof("Polling %s every %s", strings.Join(dirs, ", "), poll.String()) + klog.V(2).Infof("Polling %s every %s", strings.Join(dirs, ", "), poll.String()) pollFn := func() { _, newProfiles := loadNewProfiles() if len(newProfiles) > 0 { - glog.V(2).Infof("Successfully loaded profiles: %v", newProfiles) + klog.V(2).Infof("Successfully loaded profiles: %v", newProfiles) } } pollFn() // Run immediately. @@ -111,7 +111,7 @@ func pollForever() { func loadNewProfiles() (success bool, newProfiles []string) { loadedProfiles, err := getLoadedProfiles() if err != nil { - glog.Errorf("Error reading loaded profiles: %v", err) + klog.Errorf("Error reading loaded profiles: %v", err) return false, nil } @@ -119,7 +119,7 @@ func loadNewProfiles() (success bool, newProfiles []string) { for _, dir := range dirs { infos, err := ioutil.ReadDir(dir) if err != nil { - glog.Warningf("Error reading %s: %v", dir, err) + klog.Warningf("Error reading %s: %v", dir, err) success = false continue } @@ -129,26 +129,26 @@ func loadNewProfiles() (success bool, newProfiles []string) { // If directory, or symlink to a directory, skip it. resolvedInfo, err := resolveSymlink(dir, info) if err != nil { - glog.Warningf("Error resolving symlink: %v", err) + klog.Warningf("Error resolving symlink: %v", err) continue } if resolvedInfo.IsDir() { // Directory listing is shallow. - glog.V(4).Infof("Skipping directory %s", path) + klog.V(4).Infof("Skipping directory %s", path) continue } - glog.V(4).Infof("Scanning %s for new profiles", path) + klog.V(4).Infof("Scanning %s for new profiles", path) profiles, err := getProfileNames(path) if err != nil { - glog.Warningf("Error reading %s: %v", path, err) + klog.Warningf("Error reading %s: %v", path, err) success = false continue } if unloadedProfiles(loadedProfiles, profiles) { if err := loadProfiles(path); err != nil { - glog.Errorf("Could not load profiles: %v", err) + klog.Errorf("Could not load profiles: %v", err) success = false continue } @@ -171,7 +171,7 @@ func getProfileNames(path string) ([]string, error) { out, err := cmd.Output() if err != nil { if stderr.Len() > 0 { - glog.Warning(stderr.String()) + klog.Warning(stderr.String()) } return nil, fmt.Errorf("error reading profiles from %s: %v", path, err) } @@ -194,10 +194,10 @@ func loadProfiles(path string) error { stderr := &bytes.Buffer{} cmd.Stderr = stderr out, err := cmd.Output() - glog.V(2).Infof("Loading profiles from %s:\n%s", path, out) + klog.V(2).Infof("Loading profiles from %s:\n%s", path, out) if err != nil { if stderr.Len() > 0 { - glog.Warning(stderr.String()) + klog.Warning(stderr.String()) } return fmt.Errorf("error loading profiles from %s: %v", path, err) } diff --git a/test/images/crd-conversion-webhook/BASEIMAGE b/test/images/crd-conversion-webhook/BASEIMAGE new file mode 100644 index 00000000000..114844f395e --- /dev/null +++ b/test/images/crd-conversion-webhook/BASEIMAGE @@ -0,0 +1,4 @@ +amd64=alpine:3.6 +arm=arm32v6/alpine:3.6 +arm64=arm64v8/alpine:3.6 +ppc64le=ppc64le/alpine:3.6 diff --git a/test/images/crd-conversion-webhook/BUILD b/test/images/crd-conversion-webhook/BUILD new file mode 100644 index 00000000000..9018d79e77e --- /dev/null +++ b/test/images/crd-conversion-webhook/BUILD @@ -0,0 +1,40 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "config.go", + "main.go", + ], + importpath = "k8s.io/kubernetes/test/images/crd-conversion-webhook", + visibility = ["//visibility:private"], + deps = [ + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//test/images/crd-conversion-webhook/converter:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +go_binary( + name = "crd-conversion-webhook", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//test/images/crd-conversion-webhook/converter:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cluster/addons/python-image/Dockerfile b/test/images/crd-conversion-webhook/Dockerfile similarity index 78% rename from cluster/addons/python-image/Dockerfile rename to test/images/crd-conversion-webhook/Dockerfile index 5205aeb37d4..1743be6bbdb 100644 --- a/cluster/addons/python-image/Dockerfile +++ b/test/images/crd-conversion-webhook/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2016 The Kubernetes Authors. +# Copyright 2018 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:2.7-slim +FROM BASEIMAGE -RUN pip install pyyaml +ADD crd_conversion_webhook /crd_conversion_webhook +ENTRYPOINT ["/crd_conversion_webhook"] diff --git a/cluster/addons/python-image/Makefile b/test/images/crd-conversion-webhook/Makefile similarity index 71% rename from cluster/addons/python-image/Makefile rename to test/images/crd-conversion-webhook/Makefile index a073ac49db7..b0decfb3086 100644 --- a/cluster/addons/python-image/Makefile +++ b/test/images/crd-conversion-webhook/Makefile @@ -1,4 +1,4 @@ -# Copyright 2016 The Kubernetes Authors. +# Copyright 2018 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -IMAGE=staging-k8s.gcr.io/python -VERSION=v1 +SRCS=crd_conversion_webhook +ARCH ?= amd64 +TARGET ?= $(CURDIR) +GOLANG_VERSION ?= latest +SRC_DIR = $(notdir $(shell pwd)) +export -.PHONY: build push - -build: - docker build --pull -t "$(IMAGE):$(VERSION)" . - -push: - docker push "$(IMAGE):$(VERSION)" +bin: + ../image-util.sh bin $(SRCS) +.PHONY: bin diff --git a/test/images/crd-conversion-webhook/README.md b/test/images/crd-conversion-webhook/README.md new file mode 100644 index 00000000000..b04d34fde49 --- /dev/null +++ b/test/images/crd-conversion-webhook/README.md @@ -0,0 +1,11 @@ +# Kubernetes External Admission Webhook Test Image + +The image tests CustomResourceConversionWebhook. After deploying it to kubernetes cluster, +administrator needs to create a CustomResourceConversion.Webhook +in kubernetes cluster to use remote webhook for conversions. + +## Build the code + +```bash +make build +``` diff --git a/test/images/crd-conversion-webhook/VERSION b/test/images/crd-conversion-webhook/VERSION new file mode 100644 index 00000000000..d24646986ce --- /dev/null +++ b/test/images/crd-conversion-webhook/VERSION @@ -0,0 +1 @@ +1.13rev2 diff --git a/test/images/crd-conversion-webhook/config.go b/test/images/crd-conversion-webhook/config.go new file mode 100644 index 00000000000..11ff1b5d98f --- /dev/null +++ b/test/images/crd-conversion-webhook/config.go @@ -0,0 +1,50 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/tls" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog" +) + +// Get a clientset with in-cluster config. +func getClient() *kubernetes.Clientset { + config, err := rest.InClusterConfig() + if err != nil { + klog.Fatal(err) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Fatal(err) + } + return clientset +} + +func configTLS(config Config, clientset *kubernetes.Clientset) *tls.Config { + sCert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile) + if err != nil { + klog.Fatal(err) + } + return &tls.Config{ + Certificates: []tls.Certificate{sCert}, + // TODO: uses mutual tls after we agree on what cert the apiserver should use. + // ClientAuth: tls.RequireAndVerifyClientCert, + } +} diff --git a/test/images/crd-conversion-webhook/converter/BUILD b/test/images/crd-conversion-webhook/converter/BUILD new file mode 100644 index 00000000000..d3abb152edf --- /dev/null +++ b/test/images/crd-conversion-webhook/converter/BUILD @@ -0,0 +1,47 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "example_converter.go", + "framework.go", + ], + importpath = "k8s.io/kubernetes/test/images/crd-conversion-webhook/converter", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library", + "//vendor/bitbucket.org/ww/goautoneg:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["converter_test.go"], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/images/crd-conversion-webhook/converter/converter_test.go b/test/images/crd-conversion-webhook/converter/converter_test.go new file mode 100644 index 00000000000..e8e12391e2b --- /dev/null +++ b/test/images/crd-conversion-webhook/converter/converter_test.go @@ -0,0 +1,97 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package converter + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/json" +) + +func TestConverter(t *testing.T) { + sampleObj := `kind: ConversionReview +apiVersion: apiextensions.k8s.io/v1beta1 +request: + uid: 0000-0000-0000-0000 + desiredAPIVersion: stable.example.com/v2 + objects: + - apiVersion: stable.example.com/v1 + kind: CronTab + metadata: + name: my-new-cron-object + spec: + cronSpec: "* * * * */5" + image: my-awesome-cron-image + hostPort: "localhost:7070" +` + // First try json, it should fail as the data is taml + response := httptest.NewRecorder() + request, err := http.NewRequest("POST", "/convert", strings.NewReader(sampleObj)) + if err != nil { + t.Fatal(err) + } + request.Header.Add("Content-Type", "application/json") + ServeExampleConvert(response, request) + convertReview := v1beta1.ConversionReview{} + scheme := runtime.NewScheme() + jsonSerializer := json.NewSerializer(json.DefaultMetaFactory, scheme, scheme, false) + if _, _, err := jsonSerializer.Decode(response.Body.Bytes(), nil, &convertReview); err != nil { + t.Fatal(err) + } + if convertReview.Response.Result.Status != v1.StatusFailure { + t.Fatalf("expected the operation to fail when yaml is provided with json header") + } else if !strings.Contains(convertReview.Response.Result.Message, "json parse error") { + t.Fatalf("expected to fail on json parser, but it failed with: %v", convertReview.Response.Result.Message) + } + + // Now try yaml, and it should successfully convert + response = httptest.NewRecorder() + request, err = http.NewRequest("POST", "/convert", strings.NewReader(sampleObj)) + if err != nil { + t.Fatal(err) + } + request.Header.Add("Content-Type", "application/yaml") + ServeExampleConvert(response, request) + convertReview = v1beta1.ConversionReview{} + yamlSerializer := json.NewYAMLSerializer(json.DefaultMetaFactory, scheme, scheme) + if _, _, err := yamlSerializer.Decode(response.Body.Bytes(), nil, &convertReview); err != nil { + t.Fatalf("cannot decode data: \n %v\n Error: %v", response.Body, err) + } + if convertReview.Response.Result.Status != v1.StatusSuccess { + t.Fatalf("cr conversion failed: %v", convertReview.Response) + } + convertedObj := unstructured.Unstructured{} + if _, _, err := yamlSerializer.Decode(convertReview.Response.ConvertedObjects[0].Raw, nil, &convertedObj); err != nil { + t.Fatal(err) + } + if e, a := "stable.example.com/v2", convertedObj.GetAPIVersion(); e != a { + t.Errorf("expected= %v, actual= %v", e, a) + } + if e, a := "localhost", convertedObj.Object["host"]; e != a { + t.Errorf("expected= %v, actual= %v", e, a) + } + if e, a := "7070", convertedObj.Object["port"]; e != a { + t.Errorf("expected= %v, actual= %v", e, a) + } +} diff --git a/test/images/crd-conversion-webhook/converter/example_converter.go b/test/images/crd-conversion-webhook/converter/example_converter.go new file mode 100644 index 00000000000..b498f5ef9ff --- /dev/null +++ b/test/images/crd-conversion-webhook/converter/example_converter.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package converter + +import ( + "fmt" + "strings" + + "k8s.io/klog" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func convertExampleCRD(Object *unstructured.Unstructured, toVersion string) (*unstructured.Unstructured, metav1.Status) { + klog.V(2).Info("converting crd") + + convertedObject := Object.DeepCopy() + fromVersion := Object.GetAPIVersion() + + if toVersion == fromVersion { + return nil, statusErrorWithMessage("conversion from a version to itself should not call the webhook: %s", toVersion) + } + + switch Object.GetAPIVersion() { + case "stable.example.com/v1": + switch toVersion { + case "stable.example.com/v2": + hostPort, ok := convertedObject.Object["hostPort"] + if ok { + delete(convertedObject.Object, "hostPort") + parts := strings.Split(hostPort.(string), ":") + if len(parts) != 2 { + return nil, statusErrorWithMessage("invalid hostPort value `%v`", hostPort) + } + convertedObject.Object["host"] = parts[0] + convertedObject.Object["port"] = parts[1] + } + default: + return nil, statusErrorWithMessage("unexpected conversion version %q", toVersion) + } + case "stable.example.com/v2": + switch toVersion { + case "stable.example.com/v1": + host, hasHost := convertedObject.Object["host"] + port, hasPort := convertedObject.Object["port"] + if hasHost || hasPort { + if !hasHost { + host = "" + } + if !hasPort { + port = "" + } + convertedObject.Object["hostPort"] = fmt.Sprintf("%s:%s", host, port) + delete(convertedObject.Object, "host") + delete(convertedObject.Object, "port") + } + default: + return nil, statusErrorWithMessage("unexpected conversion version %q", toVersion) + } + default: + return nil, statusErrorWithMessage("unexpected conversion version %q", fromVersion) + } + return convertedObject, statusSucceed() +} diff --git a/test/images/crd-conversion-webhook/converter/framework.go b/test/images/crd-conversion-webhook/converter/framework.go new file mode 100644 index 00000000000..3e20926b8e4 --- /dev/null +++ b/test/images/crd-conversion-webhook/converter/framework.go @@ -0,0 +1,178 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package converter + +import ( + "bitbucket.org/ww/goautoneg" + "fmt" + "io/ioutil" + "net/http" + "strings" + + "k8s.io/klog" + + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/json" +) + +// convertFunc is the user defined function for any conversion. The code in this file is a +// template that can be use for any CR conversion given this function. +type convertFunc func(Object *unstructured.Unstructured, version string) (*unstructured.Unstructured, metav1.Status) + +// conversionResponseFailureWithMessagef is a helper function to create an AdmissionResponse +// with a formatted embedded error message. +func conversionResponseFailureWithMessagef(msg string, params ...interface{}) *v1beta1.ConversionResponse { + return &v1beta1.ConversionResponse{ + Result: metav1.Status{ + Message: fmt.Sprintf(msg, params...), + Status: metav1.StatusFailure, + }, + } + +} + +func statusErrorWithMessage(msg string, params ...interface{}) metav1.Status { + return metav1.Status{ + Message: fmt.Sprintf(msg, params...), + Status: metav1.StatusFailure, + } +} + +func statusSucceed() metav1.Status { + return metav1.Status{ + Status: metav1.StatusSuccess, + } +} + +// doConversion converts the requested object given the conversion function and returns a conversion response. +// failures will be reported as Reason in the conversion response. +func doConversion(convertRequest *v1beta1.ConversionRequest, convert convertFunc) *v1beta1.ConversionResponse { + var convertedObjects []runtime.RawExtension + for _, obj := range convertRequest.Objects { + cr := unstructured.Unstructured{} + if err := cr.UnmarshalJSON(obj.Raw); err != nil { + klog.Error(err) + return conversionResponseFailureWithMessagef("failed to unmarshall object (%v) with error: %v", string(obj.Raw), err) + } + convertedCR, status := convert(&cr, convertRequest.DesiredAPIVersion) + if status.Status != metav1.StatusSuccess { + klog.Error(status.String()) + return &v1beta1.ConversionResponse{ + Result: status, + } + } + convertedCR.SetAPIVersion(convertRequest.DesiredAPIVersion) + convertedObjects = append(convertedObjects, runtime.RawExtension{Object: convertedCR}) + } + return &v1beta1.ConversionResponse{ + ConvertedObjects: convertedObjects, + Result: statusSucceed(), + } +} + +func serve(w http.ResponseWriter, r *http.Request, convert convertFunc) { + var body []byte + if r.Body != nil { + if data, err := ioutil.ReadAll(r.Body); err == nil { + body = data + } + } + + contentType := r.Header.Get("Content-Type") + serializer := getInputSerializer(contentType) + if serializer == nil { + msg := fmt.Sprintf("invalid Content-Type header `%s`", contentType) + klog.Errorf(msg) + http.Error(w, msg, http.StatusBadRequest) + return + } + + klog.V(2).Infof("handling request: %v", body) + convertReview := v1beta1.ConversionReview{} + if _, _, err := serializer.Decode(body, nil, &convertReview); err != nil { + klog.Error(err) + convertReview.Response = conversionResponseFailureWithMessagef("failed to deserialize body (%v) with error %v", string(body), err) + } else { + convertReview.Response = doConversion(convertReview.Request, convert) + convertReview.Response.UID = convertReview.Request.UID + } + klog.V(2).Info(fmt.Sprintf("sending response: %v", convertReview.Response)) + + // reset the request, it is not needed in a response. + convertReview.Request = &v1beta1.ConversionRequest{} + + accept := r.Header.Get("Accept") + outSerializer := getOutputSerializer(accept) + if outSerializer == nil { + msg := fmt.Sprintf("invalid accept header `%s`", accept) + klog.Errorf(msg) + http.Error(w, msg, http.StatusBadRequest) + return + } + err := outSerializer.Encode(&convertReview, w) + if err != nil { + klog.Error(err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} + +// ServeExampleConvert servers endpoint for the example converter defined as convertExampleCRD function. +func ServeExampleConvert(w http.ResponseWriter, r *http.Request) { + serve(w, r, convertExampleCRD) +} + +type mediaType struct { + Type, SubType string +} + +var scheme = runtime.NewScheme() +var serializers = map[mediaType]runtime.Serializer{ + {"application", "json"}: json.NewSerializer(json.DefaultMetaFactory, scheme, scheme, false), + {"application", "yaml"}: json.NewYAMLSerializer(json.DefaultMetaFactory, scheme, scheme), +} + +func getInputSerializer(contentType string) runtime.Serializer { + parts := strings.SplitN(contentType, "/", 2) + if len(parts) != 2 { + return nil + } + return serializers[mediaType{parts[0], parts[1]}] +} + +func getOutputSerializer(accept string) runtime.Serializer { + if len(accept) == 0 { + return serializers[mediaType{"application", "json"}] + } + + clauses := goautoneg.ParseAccept(accept) + for _, clause := range clauses { + for k, v := range serializers { + switch { + case clause.Type == k.Type && clause.SubType == k.SubType, + clause.Type == k.Type && clause.SubType == "*", + clause.Type == "*" && clause.SubType == "*": + return v + } + } + } + + return nil +} diff --git a/test/images/crd-conversion-webhook/main.go b/test/images/crd-conversion-webhook/main.go new file mode 100644 index 00000000000..6c80074f13c --- /dev/null +++ b/test/images/crd-conversion-webhook/main.go @@ -0,0 +1,52 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "net/http" + + "k8s.io/kubernetes/test/images/crd-conversion-webhook/converter" +) + +// Config contains the server (the webhook) cert and key. +type Config struct { + CertFile string + KeyFile string +} + +func (c *Config) addFlags() { + flag.StringVar(&c.CertFile, "tls-cert-file", c.CertFile, ""+ + "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ + "after server cert).") + flag.StringVar(&c.KeyFile, "tls-private-key-file", c.KeyFile, ""+ + "File containing the default x509 private key matching --tls-cert-file.") +} + +func main() { + var config Config + config.addFlags() + flag.Parse() + + http.HandleFunc("/crdconvert", converter.ServeExampleConvert) + clientset := getClient() + server := &http.Server{ + Addr: ":443", + TLSConfig: configTLS(config, clientset), + } + server.ListenAndServeTLS("", "") +} diff --git a/test/images/image-util.sh b/test/images/image-util.sh index dcfd361963f..61a57e844fc 100755 --- a/test/images/image-util.sh +++ b/test/images/image-util.sh @@ -83,7 +83,12 @@ build() { else ${SED} -i "s|QEMUARCH|${QEMUARCHS[$arch]}|g" Dockerfile # Register qemu-*-static for all supported processors except the current one - docker run --rm --privileged multiarch/qemu-user-static:register --reset + echo "Registering qemu-*-static binaries in the kernel" + local sudo="" + if [[ $(id -u) != 0 ]]; then + sudo=sudo + fi + "${sudo}" "${KUBE_ROOT}/third_party/multiarch/qemu-user-static/register/register.sh" --reset curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/${QEMUVERSION}/x86_64_qemu-${QEMUARCHS[$arch]}-static.tar.gz | tar -xz -C ${temp_dir} # Ensure we don't get surprised by umask settings chmod 0755 "${temp_dir}/qemu-${QEMUARCHS[$arch]}-static" diff --git a/test/images/logs-generator/BUILD b/test/images/logs-generator/BUILD index ce9086efeee..c632d7d1533 100644 --- a/test/images/logs-generator/BUILD +++ b/test/images/logs-generator/BUILD @@ -17,7 +17,7 @@ go_library( importpath = "k8s.io/kubernetes/test/images/logs-generator", deps = [ "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/images/logs-generator/logs_generator.go b/test/images/logs-generator/logs_generator.go index b5ab2bc9533..27a65c3b798 100644 --- a/test/images/logs-generator/logs_generator.go +++ b/test/images/logs-generator/logs_generator.go @@ -21,8 +21,8 @@ import ( "fmt" "time" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/klog" ) var ( @@ -47,11 +47,11 @@ func main() { flag.Parse() if *linesTotal <= 0 { - glog.Fatalf("Invalid total number of lines: %d", *linesTotal) + klog.Fatalf("Invalid total number of lines: %d", *linesTotal) } if *duration <= 0 { - glog.Fatalf("Invalid duration: %v", *duration) + klog.Fatalf("Invalid duration: %v", *duration) } generateLogs(*linesTotal, *duration) @@ -64,7 +64,7 @@ func generateLogs(linesTotal int, duration time.Duration) { ticker := time.NewTicker(delay) defer ticker.Stop() for id := 0; id < linesTotal; id++ { - glog.Info(generateLogLine(id)) + klog.Info(generateLogLine(id)) <-ticker.C } } diff --git a/test/images/webhook/BUILD b/test/images/webhook/BUILD index 5d75f0e94dc..17f9f049256 100644 --- a/test/images/webhook/BUILD +++ b/test/images/webhook/BUILD @@ -24,7 +24,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/images/webhook/addlabel.go b/test/images/webhook/addlabel.go index 48ff86351cb..b6b12fe7f65 100644 --- a/test/images/webhook/addlabel.go +++ b/test/images/webhook/addlabel.go @@ -19,9 +19,9 @@ package main import ( "encoding/json" - "github.com/golang/glog" "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" ) const ( @@ -35,7 +35,7 @@ const ( // Add a label {"added-label": "yes"} to the object func addLabel(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(2).Info("calling add-label") + klog.V(2).Info("calling add-label") obj := struct { metav1.ObjectMeta Data map[string]string @@ -43,7 +43,7 @@ func addLabel(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { raw := ar.Request.Object.Raw err := json.Unmarshal(raw, &obj) if err != nil { - glog.Error(err) + klog.Error(err) return toAdmissionResponse(err) } diff --git a/test/images/webhook/alwaysdeny.go b/test/images/webhook/alwaysdeny.go index 8c8796e18e1..8e417ac020e 100644 --- a/test/images/webhook/alwaysdeny.go +++ b/test/images/webhook/alwaysdeny.go @@ -17,14 +17,14 @@ limitations under the License. package main import ( - "github.com/golang/glog" "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" ) // alwaysDeny all requests made to this function. func alwaysDeny(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(2).Info("calling always-deny") + klog.V(2).Info("calling always-deny") reviewResponse := v1beta1.AdmissionResponse{} reviewResponse.Allowed = false reviewResponse.Result = &metav1.Status{Message: "this webhook denies all requests"} diff --git a/test/images/webhook/config.go b/test/images/webhook/config.go index 5c84a2b8ec7..2aadace809d 100644 --- a/test/images/webhook/config.go +++ b/test/images/webhook/config.go @@ -20,7 +20,7 @@ import ( "crypto/tls" "flag" - "github.com/golang/glog" + "k8s.io/klog" ) // Config contains the server (the webhook) cert and key. @@ -40,7 +40,7 @@ func (c *Config) addFlags() { func configTLS(config Config) *tls.Config { sCert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return &tls.Config{ Certificates: []tls.Certificate{sCert}, diff --git a/test/images/webhook/configmap.go b/test/images/webhook/configmap.go index 147c6e74854..25ba7f4efa3 100644 --- a/test/images/webhook/configmap.go +++ b/test/images/webhook/configmap.go @@ -17,10 +17,10 @@ limitations under the License. package main import ( - "github.com/golang/glog" "k8s.io/api/admission/v1beta1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" ) const ( @@ -34,10 +34,10 @@ const ( // deny configmaps with specific key-value pair. func admitConfigMaps(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(2).Info("admitting configmaps") + klog.V(2).Info("admitting configmaps") configMapResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"} if ar.Request.Resource != configMapResource { - glog.Errorf("expect resource to be %s", configMapResource) + klog.Errorf("expect resource to be %s", configMapResource) return nil } @@ -45,7 +45,7 @@ func admitConfigMaps(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { configmap := corev1.ConfigMap{} deserializer := codecs.UniversalDeserializer() if _, _, err := deserializer.Decode(raw, nil, &configmap); err != nil { - glog.Error(err) + klog.Error(err) return toAdmissionResponse(err) } reviewResponse := v1beta1.AdmissionResponse{} @@ -62,10 +62,10 @@ func admitConfigMaps(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { } func mutateConfigmaps(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(2).Info("mutating configmaps") + klog.V(2).Info("mutating configmaps") configMapResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"} if ar.Request.Resource != configMapResource { - glog.Errorf("expect resource to be %s", configMapResource) + klog.Errorf("expect resource to be %s", configMapResource) return nil } @@ -73,7 +73,7 @@ func mutateConfigmaps(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { configmap := corev1.ConfigMap{} deserializer := codecs.UniversalDeserializer() if _, _, err := deserializer.Decode(raw, nil, &configmap); err != nil { - glog.Error(err) + klog.Error(err) return toAdmissionResponse(err) } reviewResponse := v1beta1.AdmissionResponse{} diff --git a/test/images/webhook/crd.go b/test/images/webhook/crd.go index 1114058c5e2..977a828002d 100644 --- a/test/images/webhook/crd.go +++ b/test/images/webhook/crd.go @@ -22,18 +22,18 @@ import ( apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/golang/glog" "k8s.io/api/admission/v1beta1" + "k8s.io/klog" ) // This function expects all CRDs submitted to it to be apiextensions.k8s.io/v1beta1 // TODO: When apiextensions.k8s.io/v1 is added we will need to update this function. func admitCRD(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(2).Info("admitting crd") + klog.V(2).Info("admitting crd") crdResource := metav1.GroupVersionResource{Group: "apiextensions.k8s.io", Version: "v1beta1", Resource: "customresourcedefinitions"} if ar.Request.Resource != crdResource { err := fmt.Errorf("expect resource to be %s", crdResource) - glog.Error(err) + klog.Error(err) return toAdmissionResponse(err) } @@ -41,7 +41,7 @@ func admitCRD(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { crd := apiextensionsv1beta1.CustomResourceDefinition{} deserializer := codecs.UniversalDeserializer() if _, _, err := deserializer.Decode(raw, nil, &crd); err != nil { - glog.Error(err) + klog.Error(err) return toAdmissionResponse(err) } reviewResponse := v1beta1.AdmissionResponse{} diff --git a/test/images/webhook/customresource.go b/test/images/webhook/customresource.go index 273ae4f08e6..53e235f3fa1 100644 --- a/test/images/webhook/customresource.go +++ b/test/images/webhook/customresource.go @@ -21,8 +21,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/golang/glog" "k8s.io/api/admission/v1beta1" + "k8s.io/klog" ) const ( @@ -35,7 +35,7 @@ const ( ) func mutateCustomResource(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(2).Info("mutating custom resource") + klog.V(2).Info("mutating custom resource") cr := struct { metav1.ObjectMeta Data map[string]string @@ -44,7 +44,7 @@ func mutateCustomResource(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse raw := ar.Request.Object.Raw err := json.Unmarshal(raw, &cr) if err != nil { - glog.Error(err) + klog.Error(err) return toAdmissionResponse(err) } @@ -63,7 +63,7 @@ func mutateCustomResource(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse } func admitCustomResource(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(2).Info("admitting custom resource") + klog.V(2).Info("admitting custom resource") cr := struct { metav1.ObjectMeta Data map[string]string @@ -72,7 +72,7 @@ func admitCustomResource(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse raw := ar.Request.Object.Raw err := json.Unmarshal(raw, &cr) if err != nil { - glog.Error(err) + klog.Error(err) return toAdmissionResponse(err) } diff --git a/test/images/webhook/main.go b/test/images/webhook/main.go index 37c0a1de8a9..0d9460ca8ba 100644 --- a/test/images/webhook/main.go +++ b/test/images/webhook/main.go @@ -23,9 +23,9 @@ import ( "io/ioutil" "net/http" - "github.com/golang/glog" "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" // TODO: try this library to see if it generates correct json patch // https://github.com/mattbaird/jsonpatch ) @@ -56,11 +56,11 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitFunc) { // verify the content type is accurate contentType := r.Header.Get("Content-Type") if contentType != "application/json" { - glog.Errorf("contentType=%s, expect application/json", contentType) + klog.Errorf("contentType=%s, expect application/json", contentType) return } - glog.V(2).Info(fmt.Sprintf("handling request: %s", body)) + klog.V(2).Info(fmt.Sprintf("handling request: %s", body)) // The AdmissionReview that was sent to the webhook requestedAdmissionReview := v1beta1.AdmissionReview{} @@ -70,7 +70,7 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitFunc) { deserializer := codecs.UniversalDeserializer() if _, _, err := deserializer.Decode(body, nil, &requestedAdmissionReview); err != nil { - glog.Error(err) + klog.Error(err) responseAdmissionReview.Response = toAdmissionResponse(err) } else { // pass to admitFunc @@ -80,14 +80,14 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitFunc) { // Return the same UID responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID - glog.V(2).Info(fmt.Sprintf("sending response: %v", responseAdmissionReview.Response)) + klog.V(2).Info(fmt.Sprintf("sending response: %v", responseAdmissionReview.Response)) respBytes, err := json.Marshal(responseAdmissionReview) if err != nil { - glog.Error(err) + klog.Error(err) } if _, err := w.Write(respBytes); err != nil { - glog.Error(err) + klog.Error(err) } } diff --git a/test/images/webhook/pods.go b/test/images/webhook/pods.go index ee0bbb2e1ee..8338551db92 100644 --- a/test/images/webhook/pods.go +++ b/test/images/webhook/pods.go @@ -23,8 +23,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/golang/glog" "k8s.io/api/admission/v1beta1" + "k8s.io/klog" ) const ( @@ -35,11 +35,11 @@ const ( // only allow pods to pull images from specific registry. func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(2).Info("admitting pods") + klog.V(2).Info("admitting pods") podResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} if ar.Request.Resource != podResource { err := fmt.Errorf("expect resource to be %s", podResource) - glog.Error(err) + klog.Error(err) return toAdmissionResponse(err) } @@ -47,7 +47,7 @@ func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { pod := corev1.Pod{} deserializer := codecs.UniversalDeserializer() if _, _, err := deserializer.Decode(raw, nil, &pod); err != nil { - glog.Error(err) + klog.Error(err) return toAdmissionResponse(err) } reviewResponse := v1beta1.AdmissionResponse{} @@ -78,10 +78,10 @@ func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { } func mutatePods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(2).Info("mutating pods") + klog.V(2).Info("mutating pods") podResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} if ar.Request.Resource != podResource { - glog.Errorf("expect resource to be %s", podResource) + klog.Errorf("expect resource to be %s", podResource) return nil } @@ -89,7 +89,7 @@ func mutatePods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { pod := corev1.Pod{} deserializer := codecs.UniversalDeserializer() if _, _, err := deserializer.Decode(raw, nil, &pod); err != nil { - glog.Error(err) + klog.Error(err) return toAdmissionResponse(err) } reviewResponse := v1beta1.AdmissionResponse{} @@ -105,19 +105,19 @@ func mutatePods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { // denySpecificAttachment denies `kubectl attach to-be-attached-pod -i -c=container1" // or equivalent client requests. func denySpecificAttachment(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(2).Info("handling attaching pods") + klog.V(2).Info("handling attaching pods") if ar.Request.Name != "to-be-attached-pod" { return &v1beta1.AdmissionResponse{Allowed: true} } podResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} if e, a := podResource, ar.Request.Resource; e != a { err := fmt.Errorf("expect resource to be %s, got %s", e, a) - glog.Error(err) + klog.Error(err) return toAdmissionResponse(err) } if e, a := "attach", ar.Request.SubResource; e != a { err := fmt.Errorf("expect subresource to be %s, got %s", e, a) - glog.Error(err) + klog.Error(err) return toAdmissionResponse(err) } @@ -125,10 +125,10 @@ func denySpecificAttachment(ar v1beta1.AdmissionReview) *v1beta1.AdmissionRespon podAttachOptions := corev1.PodAttachOptions{} deserializer := codecs.UniversalDeserializer() if _, _, err := deserializer.Decode(raw, nil, &podAttachOptions); err != nil { - glog.Error(err) + klog.Error(err) return toAdmissionResponse(err) } - glog.V(2).Info(fmt.Sprintf("podAttachOptions=%#v\n", podAttachOptions)) + klog.V(2).Info(fmt.Sprintf("podAttachOptions=%#v\n", podAttachOptions)) if !podAttachOptions.Stdin || podAttachOptions.Container != "container1" { return &v1beta1.AdmissionResponse{Allowed: true} } diff --git a/test/integration/BUILD b/test/integration/BUILD index 1fe6fa9d2ce..d024c667e5c 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -41,7 +41,6 @@ filegroup( "//test/integration/benchmark/jsonify:all-srcs", "//test/integration/client:all-srcs", "//test/integration/configmap:all-srcs", - "//test/integration/controllermanager:all-srcs", "//test/integration/cronjob:all-srcs", "//test/integration/daemonset:all-srcs", "//test/integration/defaulttolerationseconds:all-srcs", @@ -66,6 +65,7 @@ filegroup( "//test/integration/scheduler_perf:all-srcs", "//test/integration/secrets:all-srcs", "//test/integration/serviceaccount:all-srcs", + "//test/integration/serving:all-srcs", "//test/integration/statefulset:all-srcs", "//test/integration/storageclasses:all-srcs", "//test/integration/tls:all-srcs", diff --git a/test/integration/apiserver/BUILD b/test/integration/apiserver/BUILD index 8261c405698..ce40df99c5e 100644 --- a/test/integration/apiserver/BUILD +++ b/test/integration/apiserver/BUILD @@ -51,9 +51,9 @@ go_test( "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//staging/src/k8s.io/client-go/tools/pager:go_default_library", "//test/integration/framework:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pborman/uuid:go_default_library", "//vendor/k8s.io/gengo/examples/set-gen/sets:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index 38e604faf84..37606dcd13f 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -26,7 +26,6 @@ import ( "reflect" "testing" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/api/extensions/v1beta1" "k8s.io/apimachinery/pkg/api/meta" @@ -38,6 +37,7 @@ import ( clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/pager" + "k8s.io/klog" "k8s.io/kubernetes/pkg/api/testapi" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/master" @@ -68,7 +68,7 @@ func verifyStatusCode(t *testing.T, verb, URL, body string, expectedStatusCode i t.Fatalf("unexpected error: %v in sending req with verb: %s, URL: %s and body: %s", err, verb, URL, body) } transport := http.DefaultTransport - glog.Infof("Sending request: %v", req) + klog.Infof("Sending request: %v", req) resp, err := transport.RoundTrip(req) if err != nil { t.Fatalf("unexpected error: %v in req: %v", err, req) diff --git a/test/integration/auth/BUILD b/test/integration/auth/BUILD index 6d4d735bdab..af4345306b9 100644 --- a/test/integration/auth/BUILD +++ b/test/integration/auth/BUILD @@ -17,6 +17,9 @@ go_test( "rbac_test.go", "svcaccttoken_test.go", ], + data = [ + "//staging/src/k8s.io/csi-api/pkg/crd:csi-manifests", + ], tags = ["integration"], deps = [ "//cmd/kube-apiserver/app/testing:go_default_library", @@ -55,6 +58,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", @@ -63,6 +67,7 @@ go_test( "//staging/src/k8s.io/apiserver/pkg/authentication/group:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/request/bearertoken:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authentication/token/cache:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/token/tokenfile:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", @@ -81,12 +86,12 @@ go_test( "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", - "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library", "//test/e2e/lifecycle/bootstrap:go_default_library", "//test/integration:go_default_library", + "//test/integration/etcd:go_default_library", "//test/integration/framework:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/gopkg.in/square/go-jose.v2/jwt:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/test/integration/auth/auth_test.go b/test/integration/auth/auth_test.go index 067fb0b6ebe..364f1d264d9 100644 --- a/test/integration/auth/auth_test.go +++ b/test/integration/auth/auth_test.go @@ -40,6 +40,7 @@ import ( "k8s.io/apiserver/pkg/authentication/group" "k8s.io/apiserver/pkg/authentication/request/bearertoken" "k8s.io/apiserver/pkg/authentication/serviceaccount" + "k8s.io/apiserver/pkg/authentication/token/cache" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/authorization/authorizerfactory" @@ -84,11 +85,11 @@ func getTestWebhookTokenAuth(serverURL string) (authenticator.Request, error) { if err := json.NewEncoder(kubecfgFile).Encode(config); err != nil { return nil, err } - webhookTokenAuth, err := webhook.New(kubecfgFile.Name(), 2*time.Minute) + webhookTokenAuth, err := webhook.New(kubecfgFile.Name(), nil) if err != nil { return nil, err } - return bearertoken.New(webhookTokenAuth), nil + return bearertoken.New(cache.New(webhookTokenAuth, false, 2*time.Minute, 2*time.Minute)), nil } func path(resource, namespace, name string) string { diff --git a/test/integration/auth/node_test.go b/test/integration/auth/node_test.go index db871d6d37b..7b701c22cab 100644 --- a/test/integration/auth/node_test.go +++ b/test/integration/auth/node_test.go @@ -18,6 +18,8 @@ package auth import ( "fmt" + "io/ioutil" + "strings" "testing" "time" @@ -26,27 +28,25 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" externalclientset "k8s.io/client-go/kubernetes" csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" + csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/coordination" + "k8s.io/kubernetes/pkg/apis/core" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/policy" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/framework" "k8s.io/utils/pointer" - - "io/ioutil" - apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" - csicrd "k8s.io/csi-api/pkg/crd" - kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" - "k8s.io/kubernetes/pkg/apis/core" - "strings" ) func TestNodeAuthorizer(t *testing.T) { @@ -158,13 +158,12 @@ func TestNodeAuthorizer(t *testing.T) { t.Fatal(err) } - crd, err := superuserCRDClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(csicrd.CSINodeInfoCRD()) + csiNodeInfoCRD, err := crdFromManifest("../../../staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml") if err != nil { t.Fatal(err) } - if err := waitForEstablishedCRD(superuserCRDClient, crd.Name); err != nil { - t.Fatalf("Failed to establish CSINodeInfo CRD: %v", err) - } + + etcd.CreateTestCRDs(t, superuserCRDClient, false, csiNodeInfoCRD) getSecret := func(client clientset.Interface) func() error { return func() error { @@ -421,11 +420,22 @@ func TestNodeAuthorizer(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node1", }, - CSIDrivers: []csiv1alpha1.CSIDriverInfo{ - { - Driver: "com.example.csi/driver1", - NodeID: "com.example.csi/node1", - TopologyKeys: []string{"com.example.csi/zone"}, + Spec: csiv1alpha1.CSINodeInfoSpec{ + Drivers: []csiv1alpha1.CSIDriverInfoSpec{ + { + Name: "com.example.csi/driver1", + NodeID: "com.example.csi/node1", + TopologyKeys: []string{"com.example.csi/zone"}, + }, + }, + }, + Status: csiv1alpha1.CSINodeInfoStatus{ + Drivers: []csiv1alpha1.CSIDriverInfoStatus{ + { + Name: "com.example.csi/driver1", + Available: true, + VolumePluginMechanism: csiv1alpha1.VolumePluginMechanismInTree, + }, }, }, } @@ -439,13 +449,20 @@ func TestNodeAuthorizer(t *testing.T) { if err != nil { return err } - nodeInfo.CSIDrivers = []csiv1alpha1.CSIDriverInfo{ + nodeInfo.Spec.Drivers = []csiv1alpha1.CSIDriverInfoSpec{ { - Driver: "com.example.csi/driver1", + Name: "com.example.csi/driver1", NodeID: "com.example.csi/node1", TopologyKeys: []string{"com.example.csi/rack"}, }, } + nodeInfo.Status.Drivers = []csiv1alpha1.CSIDriverInfoStatus{ + { + Name: "com.example.csi/driver1", + Available: true, + VolumePluginMechanism: csiv1alpha1.VolumePluginMechanismInTree, + }, + } _, err = client.CsiV1alpha1().CSINodeInfos().Update(nodeInfo) return err } @@ -513,7 +530,10 @@ func TestNodeAuthorizer(t *testing.T) { expectAllowed(t, createNode2MirrorPodEviction(node2Client)) expectAllowed(t, createNode2(node2Client)) expectAllowed(t, updateNode2Status(node2Client)) - expectAllowed(t, deleteNode2(node2Client)) + // self deletion is not allowed + expectForbidden(t, deleteNode2(node2Client)) + // clean up node2 + expectAllowed(t, deleteNode2(superuserClient)) // create a pod as an admin to add object references expectAllowed(t, createNode2NormalPod(superuserClient)) @@ -604,7 +624,7 @@ func TestNodeAuthorizer(t *testing.T) { // node2 can no longer get the configmap after it is unassigned as its config source expectForbidden(t, getConfigMapConfigSource(node2Client)) // clean up node2 - expectAllowed(t, deleteNode2(node2Client)) + expectAllowed(t, deleteNode2(superuserClient)) //TODO(mikedanese): integration test node restriction of TokenRequest @@ -673,24 +693,16 @@ func expectAllowed(t *testing.T, f func() error) { } } -func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error { - return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { - crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) - if err != nil { - return false, err - } - for _, cond := range crd.Status.Conditions { - switch cond.Type { - case apiextensionsv1beta1.Established: - if cond.Status == apiextensionsv1beta1.ConditionTrue { - return true, err - } - case apiextensionsv1beta1.NamesAccepted: - if cond.Status == apiextensionsv1beta1.ConditionFalse { - fmt.Printf("Name conflict: %v\n", cond.Reason) - } - } - } - return false, nil - }) +// crdFromManifest reads a .json/yaml file and returns the CRD in it. +func crdFromManifest(filename string) (*apiextensionsv1beta1.CustomResourceDefinition, error) { + var crd apiextensionsv1beta1.CustomResourceDefinition + data, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), data, &crd); err != nil { + return nil, err + } + return &crd, nil } diff --git a/test/integration/auth/rbac_test.go b/test/integration/auth/rbac_test.go index 9c6865c9e9b..effffb12228 100644 --- a/test/integration/auth/rbac_test.go +++ b/test/integration/auth/rbac_test.go @@ -27,7 +27,7 @@ import ( "testing" "time" - "github.com/golang/glog" + "k8s.io/klog" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -562,7 +562,7 @@ func TestRBAC(t *testing.T) { // // go test -v -tags integration -run RBAC -args -v 10 // - glog.V(8).Infof("case %d, req %d: %s\n%s\n", i, j, reqDump, respDump) + klog.V(8).Infof("case %d, req %d: %s\n%s\n", i, j, reqDump, respDump) t.Errorf("case %d, req %d: %s expected %q got %q", i, j, r, statusCode(r.expectedStatus), statusCode(resp.StatusCode)) } diff --git a/test/integration/auth/svcaccttoken_test.go b/test/integration/auth/svcaccttoken_test.go index 6409c475bed..65fcbba58c3 100644 --- a/test/integration/auth/svcaccttoken_test.go +++ b/test/integration/auth/svcaccttoken_test.go @@ -32,6 +32,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/request/bearertoken" apiserverserviceaccount "k8s.io/apiserver/pkg/authentication/serviceaccount" "k8s.io/apiserver/pkg/authorization/authorizerfactory" @@ -63,7 +64,7 @@ func TestServiceAccountTokenCreate(t *testing.T) { pk := sk.(*ecdsa.PrivateKey).PublicKey const iss = "https://foo.bar.example.com" - aud := []string{"api"} + aud := authenticator.Audiences{"api"} maxExpirationSeconds := int64(60 * 60) maxExpirationDuration, err := time.ParseDuration(fmt.Sprintf("%ds", maxExpirationSeconds)) @@ -76,11 +77,13 @@ func TestServiceAccountTokenCreate(t *testing.T) { // Start the server masterConfig := framework.NewIntegrationTestMasterConfig() masterConfig.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() + masterConfig.GenericConfig.Authentication.APIAudiences = aud masterConfig.GenericConfig.Authentication.Authenticator = bearertoken.New( serviceaccount.JWTTokenAuthenticator( iss, []interface{}{&pk}, - serviceaccount.NewValidator(aud, serviceaccountgetter.NewGetterFromClient(gcs)), + aud, + serviceaccount.NewValidator(serviceaccountgetter.NewGetterFromClient(gcs)), ), ) tokenGenerator, err := serviceaccount.JWTTokenGenerator(iss, sk) @@ -89,7 +92,7 @@ func TestServiceAccountTokenCreate(t *testing.T) { } masterConfig.ExtraConfig.ServiceAccountIssuer = tokenGenerator masterConfig.ExtraConfig.ServiceAccountMaxExpiration = maxExpirationDuration - masterConfig.ExtraConfig.APIAudiences = aud + masterConfig.GenericConfig.Authentication.APIAudiences = aud master, _, closeFn := framework.RunAMaster(masterConfig) defer closeFn() diff --git a/test/integration/daemonset/daemonset_test.go b/test/integration/daemonset/daemonset_test.go index 73efd4a14d3..c9450c3c50b 100644 --- a/test/integration/daemonset/daemonset_test.go +++ b/test/integration/daemonset/daemonset_test.go @@ -111,7 +111,7 @@ func setupScheduler( PdbInformer: informerFactory.Policy().V1beta1().PodDisruptionBudgets(), StorageClassInformer: informerFactory.Storage().V1().StorageClasses(), HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight, - EnableEquivalenceClassCache: true, + EnableEquivalenceClassCache: false, DisablePreemption: false, PercentageOfNodesToScore: 100, }) diff --git a/test/integration/etcd/BUILD b/test/integration/etcd/BUILD index d17427684fa..d86e89cc42d 100644 --- a/test/integration/etcd/BUILD +++ b/test/integration/etcd/BUILD @@ -53,6 +53,8 @@ go_library( "//cmd/kube-apiserver/app:go_default_library", "//cmd/kube-apiserver/app/options:go_default_library", "//pkg/master:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", diff --git a/test/integration/etcd/data.go b/test/integration/etcd/data.go index ab495b9dae1..686b704634a 100644 --- a/test/integration/etcd/data.go +++ b/test/integration/etcd/data.go @@ -16,7 +16,11 @@ limitations under the License. package etcd -import "k8s.io/apimachinery/pkg/runtime/schema" +import ( + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) // GetEtcdStorageData returns etcd data for all persisted objects. // It is exported so that it can be reused across multiple tests. @@ -287,6 +291,14 @@ func GetEtcdStorageData() map[schema.GroupVersionResource]StorageData { }, // -- + // k8s.io/kubernetes/pkg/apis/storage/v1 + gvr("storage.k8s.io", "v1", "volumeattachments"): { + Stub: `{"metadata": {"name": "va3"}, "spec": {"attacher": "gce", "nodeName": "localhost", "source": {"persistentVolumeName": "pv3"}}}`, + ExpectedEtcdPath: "/registry/volumeattachments/va3", + ExpectedGVK: gvkP("storage.k8s.io", "v1beta1", "VolumeAttachment"), + }, + // -- + // k8s.io/kubernetes/pkg/apis/storage/v1beta1 gvr("storage.k8s.io", "v1beta1", "storageclasses"): { Stub: `{"metadata": {"name": "sc1"}, "provisioner": "aws"}`, @@ -427,6 +439,30 @@ func GetEtcdStorageData() map[schema.GroupVersionResource]StorageData { Stub: `{"metadata": {"name": "openshiftwebconsoleconfigs.webconsole.operator.openshift.io"},"spec": {"scope": "Cluster","group": "webconsole.operator.openshift.io","version": "v1alpha1","names": {"kind": "OpenShiftWebConsoleConfig","plural": "openshiftwebconsoleconfigs","singular": "openshiftwebconsoleconfig"}}}`, ExpectedEtcdPath: "/registry/apiextensions.k8s.io/customresourcedefinitions/openshiftwebconsoleconfigs.webconsole.operator.openshift.io", }, + gvr("cr.bar.com", "v1", "foos"): { + Stub: `{"kind": "Foo", "apiVersion": "cr.bar.com/v1", "metadata": {"name": "cr1foo"}, "color": "blue"}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper + ExpectedEtcdPath: "/registry/cr.bar.com/foos/etcdstoragepathtestnamespace/cr1foo", + }, + gvr("custom.fancy.com", "v2", "pants"): { + Stub: `{"kind": "Pant", "apiVersion": "custom.fancy.com/v2", "metadata": {"name": "cr2pant"}, "isFancy": true}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper + ExpectedEtcdPath: "/registry/custom.fancy.com/pants/cr2pant", + }, + gvr("awesome.bears.com", "v1", "pandas"): { + Stub: `{"kind": "Panda", "apiVersion": "awesome.bears.com/v1", "metadata": {"name": "cr3panda"}, "weight": 100}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper + ExpectedEtcdPath: "/registry/awesome.bears.com/pandas/cr3panda", + }, + gvr("awesome.bears.com", "v3", "pandas"): { + Stub: `{"kind": "Panda", "apiVersion": "awesome.bears.com/v3", "metadata": {"name": "cr4panda"}, "weight": 300}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper + ExpectedEtcdPath: "/registry/awesome.bears.com/pandas/cr4panda", + ExpectedGVK: gvkP("awesome.bears.com", "v1", "Panda"), + }, + // -- + + // k8s.io/kubernetes/pkg/apis/auditregistration/v1alpha1 + gvr("auditregistration.k8s.io", "v1alpha1", "auditsinks"): { + Stub: `{"metadata":{"name":"sink1"},"spec":{"policy":{"level":"Metadata","stages":["ResponseStarted"]},"webhook":{"clientConfig":{"url":"http://localhost:4444","service":null,"caBundle":null}}}}`, + ExpectedEtcdPath: "/registry/auditsinks/sink1", + }, // -- } } @@ -446,6 +482,74 @@ type Prerequisite struct { Stub string } +// GetCustomResourceDefinitionData returns the resource definitions that back the custom resources +// included in GetEtcdStorageData. They should be created using CreateTestCRDs before running any tests. +func GetCustomResourceDefinitionData() []*apiextensionsv1beta1.CustomResourceDefinition { + return []*apiextensionsv1beta1.CustomResourceDefinition{ + // namespaced with legacy version field + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foos.cr.bar.com", + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "cr.bar.com", + Version: "v1", + Scope: apiextensionsv1beta1.NamespaceScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "foos", + Kind: "Foo", + }, + }, + }, + // cluster scoped with legacy version field + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pants.custom.fancy.com", + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "custom.fancy.com", + Version: "v2", + Scope: apiextensionsv1beta1.ClusterScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "pants", + Kind: "Pant", + }, + }, + }, + // cluster scoped with versions field + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pandas.awesome.bears.com", + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "awesome.bears.com", + Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + }, + { + Name: "v2", + Served: false, + Storage: false, + }, + { + Name: "v3", + Served: true, + Storage: false, + }, + }, + Scope: apiextensionsv1beta1.ClusterScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "pandas", + Kind: "Panda", + }, + }, + }, + } +} + func gvr(g, v, r string) schema.GroupVersionResource { return schema.GroupVersionResource{Group: g, Version: v, Resource: r} } diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index 1ec75a315f7..0335a5030c2 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -99,6 +99,10 @@ func TestEtcdStoragePath(t *testing.T) { if input, err = jsonToMetaObject([]byte(testData.Stub)); err != nil || input.isEmpty() { t.Fatalf("invalid test data for %s: %v", gvResource, err) } + // unset type meta fields - we only set these in the CRD test data and it makes + // any CRD test with an expectedGVK override fail the DeepDerivative test + input.Kind = "" + input.APIVersion = "" } all := &[]cleanupData{} diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index d47465d4ea0..d86534ba292 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -30,6 +30,8 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/clientv3/concurrency" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -153,6 +155,9 @@ func StartRealMasterOrDie(t *testing.T) *Master { t.Fatal(err) } + // create CRDs so we can make sure that custom resources do not get lost + CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(kubeClientConfig), false, GetCustomResourceDefinitionData()...) + // force cached discovery reset discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery()) restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) @@ -169,6 +174,9 @@ func StartRealMasterOrDie(t *testing.T) *Master { } close(stopCh) lock.Unlock() + if err := session.Close(); err != nil { + t.Log(err) + } } return &Master{ @@ -281,3 +289,79 @@ func JSONToUnstructured(stub, namespace string, mapping *meta.RESTMapping, dynam return dynamicClient.Resource(mapping.Resource).Namespace(namespace), &unstructured.Unstructured{Object: typeMetaAdder}, nil } + +// CreateTestCRDs creates the given CRDs, any failure causes the test to Fatal. +// If skipCrdExistsInDiscovery is true, the CRDs are only checked for the Established condition via their Status. +// If skipCrdExistsInDiscovery is false, the CRDs are checked via discovery, see CrdExistsInDiscovery. +func CreateTestCRDs(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crds ...*apiextensionsv1beta1.CustomResourceDefinition) { + for _, crd := range crds { + createTestCRD(t, client, skipCrdExistsInDiscovery, crd) + } +} + +func createTestCRD(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crd *apiextensionsv1beta1.CustomResourceDefinition) { + if _, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil { + t.Fatalf("Failed to create %s CRD; %v", crd.Name, err) + } + if skipCrdExistsInDiscovery { + if err := waitForEstablishedCRD(client, crd.Name); err != nil { + t.Fatalf("Failed to establish %s CRD; %v", crd.Name, err) + } + return + } + if err := wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { + return CrdExistsInDiscovery(client, crd), nil + }); err != nil { + t.Fatalf("Failed to see %s in discovery: %v", crd.Name, err) + } +} + +func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error { + return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { + crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) + if err != nil { + return false, err + } + for _, cond := range crd.Status.Conditions { + switch cond.Type { + case apiextensionsv1beta1.Established: + if cond.Status == apiextensionsv1beta1.ConditionTrue { + return true, nil + } + } + } + return false, nil + }) +} + +// CrdExistsInDiscovery checks to see if the given CRD exists in discovery at all served versions. +func CrdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) bool { + var versions []string + if len(crd.Spec.Version) != 0 { + versions = append(versions, crd.Spec.Version) + } + for _, v := range crd.Spec.Versions { + if v.Served { + versions = append(versions, v.Name) + } + } + for _, v := range versions { + if !crdVersionExistsInDiscovery(client, crd, v) { + return false + } + } + return true +} + +func crdVersionExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition, version string) bool { + resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version) + if err != nil { + return false + } + for _, resource := range resourceList.APIResources { + if resource.Name == crd.Spec.Names.Plural { + return true + } + } + return false +} diff --git a/test/integration/framework/BUILD b/test/integration/framework/BUILD index 9fb021ed33a..dbf6cb4cbb1 100644 --- a/test/integration/framework/BUILD +++ b/test/integration/framework/BUILD @@ -60,8 +60,8 @@ go_library( "//test/e2e/framework:go_default_library", "//test/utils:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pborman/uuid:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/integration/framework/etcd.go b/test/integration/framework/etcd.go index d7f43ec2a38..18ae79904d5 100644 --- a/test/integration/framework/etcd.go +++ b/test/integration/framework/etcd.go @@ -26,7 +26,7 @@ import ( "path/filepath" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/env" ) @@ -69,11 +69,11 @@ func startEtcd() (func(), error) { etcdURL = env.GetEnvAsStringOrFallback("KUBE_INTEGRATION_ETCD_URL", "http://127.0.0.1:2379") conn, err := net.Dial("tcp", strings.TrimPrefix(etcdURL, "http://")) if err == nil { - glog.Infof("etcd already running at %s", etcdURL) + klog.Infof("etcd already running at %s", etcdURL) conn.Close() return func() {}, nil } - glog.V(1).Infof("could not connect to etcd: %v", err) + klog.V(1).Infof("could not connect to etcd: %v", err) // TODO: Check for valid etcd version. etcdPath, err := getEtcdPath() @@ -86,13 +86,13 @@ func startEtcd() (func(), error) { return nil, fmt.Errorf("could not get a port: %v", err) } etcdURL = fmt.Sprintf("http://127.0.0.1:%d", etcdPort) - glog.Infof("starting etcd on %s", etcdURL) + klog.Infof("starting etcd on %s", etcdURL) etcdDataDir, err := ioutil.TempDir(os.TempDir(), "integration_test_etcd_data") if err != nil { return nil, fmt.Errorf("unable to make temp etcd data dir: %v", err) } - glog.Infof("storing etcd data in: %v", etcdDataDir) + klog.Infof("storing etcd data in: %v", etcdDataDir) ctx, cancel := context.WithCancel(context.Background()) cmd := exec.CommandContext( @@ -112,10 +112,10 @@ func startEtcd() (func(), error) { stop := func() { cancel() err := cmd.Wait() - glog.Infof("etcd exit status: %v", err) + klog.Infof("etcd exit status: %v", err) err = os.RemoveAll(etcdDataDir) if err != nil { - glog.Warningf("error during etcd cleanup: %v", err) + klog.Warningf("error during etcd cleanup: %v", err) } } @@ -129,7 +129,7 @@ func startEtcd() (func(), error) { func EtcdMain(tests func() int) { stop, err := startEtcd() if err != nil { - glog.Fatalf("cannot run integration tests: unable to start etcd: %v", err) + klog.Fatalf("cannot run integration tests: unable to start etcd: %v", err) } result := tests() stop() // Don't defer this. See os.Exit documentation. diff --git a/test/integration/framework/master_utils.go b/test/integration/framework/master_utils.go index 52df5c97b44..d5512120b59 100644 --- a/test/integration/framework/master_utils.go +++ b/test/integration/framework/master_utils.go @@ -24,8 +24,8 @@ import ( "time" "github.com/go-openapi/spec" - "github.com/golang/glog" "github.com/pborman/uuid" + "k8s.io/klog" apps "k8s.io/api/apps/v1beta1" auditreg "k8s.io/api/auditregistration/v1alpha1" @@ -178,14 +178,14 @@ func startMasterOrDie(masterConfig *master.Config, incomingServer *httptest.Serv clientset, err := clientset.NewForConfig(masterConfig.GenericConfig.LoopbackClientConfig) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } masterConfig.ExtraConfig.VersionedInformers = informers.NewSharedInformerFactory(clientset, masterConfig.GenericConfig.LoopbackClientConfig.Timeout) m, err = masterConfig.Complete().New(genericapiserver.NewEmptyDelegate()) if err != nil { closeFn() - glog.Fatalf("error in bringing up the master: %v", err) + klog.Fatalf("error in bringing up the master: %v", err) } if masterReceiver != nil { masterReceiver.SetMaster(m) @@ -202,7 +202,7 @@ func startMasterOrDie(masterConfig *master.Config, incomingServer *httptest.Serv privilegedClient, err := restclient.RESTClientFor(&cfg) if err != nil { closeFn() - glog.Fatal(err) + klog.Fatal(err) } var lastHealthContent []byte err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { @@ -217,8 +217,8 @@ func startMasterOrDie(masterConfig *master.Config, incomingServer *httptest.Serv }) if err != nil { closeFn() - glog.Errorf("last health content: %q", string(lastHealthContent)) - glog.Fatal(err) + klog.Errorf("last health content: %q", string(lastHealthContent)) + klog.Fatal(err) } return m, s, closeFn diff --git a/test/integration/framework/perf_utils.go b/test/integration/framework/perf_utils.go index 8897e93f602..77bee423201 100644 --- a/test/integration/framework/perf_utils.go +++ b/test/integration/framework/perf_utils.go @@ -24,7 +24,7 @@ import ( e2eframework "k8s.io/kubernetes/test/e2e/framework" testutils "k8s.io/kubernetes/test/utils" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -51,7 +51,7 @@ func (p *IntegrationTestNodePreparer) PrepareNodes() error { numNodes += v.Count } - glog.Infof("Making %d nodes", numNodes) + klog.Infof("Making %d nodes", numNodes) baseNode := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ GenerateName: p.nodeNamePrefix, @@ -77,7 +77,7 @@ func (p *IntegrationTestNodePreparer) PrepareNodes() error { } } if err != nil { - glog.Fatalf("Error creating node: %v", err) + klog.Fatalf("Error creating node: %v", err) } } @@ -88,7 +88,7 @@ func (p *IntegrationTestNodePreparer) PrepareNodes() error { sum += v.Count for ; index < sum; index++ { if err := testutils.DoPrepareNode(p.client, &nodes.Items[index], v.Strategy); err != nil { - glog.Errorf("Aborting node preparation: %v", err) + klog.Errorf("Aborting node preparation: %v", err) return err } } @@ -100,7 +100,7 @@ func (p *IntegrationTestNodePreparer) CleanupNodes() error { nodes := e2eframework.GetReadySchedulableNodesOrDie(p.client) for i := range nodes.Items { if err := p.client.CoreV1().Nodes().Delete(nodes.Items[i].Name, &metav1.DeleteOptions{}); err != nil { - glog.Errorf("Error while deleting Node: %v", err) + klog.Errorf("Error while deleting Node: %v", err) } } return nil diff --git a/test/integration/ipamperf/BUILD b/test/integration/ipamperf/BUILD index f1113b2310b..52b0889b6a7 100644 --- a/test/integration/ipamperf/BUILD +++ b/test/integration/ipamperf/BUILD @@ -20,7 +20,7 @@ go_test( "//staging/src/k8s.io/client-go/rest:go_default_library", "//test/integration/framework:go_default_library", "//test/integration/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -63,8 +63,8 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//test/integration/util:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/api/compute/v0.beta:go_default_library", "//vendor/google.golang.org/api/compute/v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/integration/ipamperf/ipam_test.go b/test/integration/ipamperf/ipam_test.go index 8c0369bbe0c..afaaa75f812 100644 --- a/test/integration/ipamperf/ipam_test.go +++ b/test/integration/ipamperf/ipam_test.go @@ -25,7 +25,7 @@ import ( "testing" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/informers" @@ -65,7 +65,7 @@ func setupAllocator(apiURL string, config *Config, clusterCIDR, serviceCIDR *net func runTest(t *testing.T, apiURL string, config *Config, clusterCIDR, serviceCIDR *net.IPNet, subnetMaskSize int) (*Results, error) { t.Helper() - glog.Infof("Running test %s", t.Name()) + klog.Infof("Running test %s", t.Name()) defer deleteNodes(apiURL, config) // cleanup nodes on after controller shutdown @@ -85,7 +85,7 @@ func runTest(t *testing.T, apiURL string, config *Config, clusterCIDR, serviceCI } results := o.Results(t.Name(), config) - glog.Infof("Results: %s", results) + klog.Infof("Results: %s", results) if !results.Succeeded { t.Errorf("%s: Not allocations succeeded", t.Name()) } @@ -95,16 +95,16 @@ func runTest(t *testing.T, apiURL string, config *Config, clusterCIDR, serviceCI func logResults(allResults []*Results) { jStr, err := json.MarshalIndent(allResults, "", " ") if err != nil { - glog.Errorf("Error formatting results: %v", err) + klog.Errorf("Error formatting results: %v", err) return } if resultsLogFile != "" { - glog.Infof("Logging results to %s", resultsLogFile) + klog.Infof("Logging results to %s", resultsLogFile) if err := ioutil.WriteFile(resultsLogFile, jStr, os.FileMode(0644)); err != nil { - glog.Errorf("Error logging results to %s: %v", resultsLogFile, err) + klog.Errorf("Error logging results to %s: %v", resultsLogFile, err) } } - glog.Infof("AllResults:\n%s", string(jStr)) + klog.Infof("AllResults:\n%s", string(jStr)) } func TestPerformance(t *testing.T) { diff --git a/test/integration/ipamperf/main_test.go b/test/integration/ipamperf/main_test.go index 401ad452836..0ade6c87575 100644 --- a/test/integration/ipamperf/main_test.go +++ b/test/integration/ipamperf/main_test.go @@ -20,7 +20,7 @@ import ( "flag" "testing" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller/nodeipam/ipam" "k8s.io/kubernetes/test/integration/framework" ) @@ -59,7 +59,7 @@ func TestMain(m *testing.M) { case string(ipam.IPAMFromClusterAllocatorType): customConfig.AllocatorType = ipam.IPAMFromClusterAllocatorType default: - glog.Fatalf("Unknown allocator type: %s", allocator) + klog.Fatalf("Unknown allocator type: %s", allocator) } framework.EtcdMain(m.Run) diff --git a/test/integration/ipamperf/results.go b/test/integration/ipamperf/results.go index 8880477b9b0..cef7942b100 100644 --- a/test/integration/ipamperf/results.go +++ b/test/integration/ipamperf/results.go @@ -23,12 +23,12 @@ import ( "sync" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller/nodeipam/ipam" nodeutil "k8s.io/kubernetes/pkg/controller/util/node" ) @@ -96,7 +96,7 @@ func NewObserver(clientSet *clientset.Clientset, numNodes int) *Observer { // Call Results() to get the test results after starting observer. func (o *Observer) StartObserving() error { o.monitor() - glog.Infof("Test observer started") + klog.Infof("Test observer started") return nil } @@ -174,12 +174,12 @@ func (o *Observer) monitor() { nTime.podCIDR = newNode.Spec.PodCIDR o.numAllocated++ if o.numAllocated%10 == 0 { - glog.Infof("progress: %d/%d - %.2d%%", o.numAllocated, o.numNodes, (o.numAllocated * 100.0 / o.numNodes)) + klog.Infof("progress: %d/%d - %.2d%%", o.numAllocated, o.numNodes, (o.numAllocated * 100.0 / o.numNodes)) } // do following check only if numAllocated is modified, as otherwise, redundant updates // can cause wg.Done() to be called multiple times, causing a panic if o.numAdded == o.numNodes && o.numAllocated == o.numNodes { - glog.Info("All nodes assigned podCIDR") + klog.Info("All nodes assigned podCIDR") o.wg.Done() } } diff --git a/test/integration/ipamperf/util.go b/test/integration/ipamperf/util.go index 34c1c175cce..1b6e25b4874 100644 --- a/test/integration/ipamperf/util.go +++ b/test/integration/ipamperf/util.go @@ -19,7 +19,6 @@ package ipamperf import ( "time" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -27,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" + "k8s.io/klog" ) const ( @@ -54,7 +54,7 @@ var ( ) func deleteNodes(apiURL string, config *Config) { - glog.Info("Deleting nodes") + klog.Info("Deleting nodes") clientSet := clientset.NewForConfigOrDie(&restclient.Config{ Host: apiURL, ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}, @@ -63,7 +63,7 @@ func deleteNodes(apiURL string, config *Config) { }) noGrace := int64(0) if err := clientSet.CoreV1().Nodes().DeleteCollection(&metav1.DeleteOptions{GracePeriodSeconds: &noGrace}, metav1.ListOptions{}); err != nil { - glog.Errorf("Error deleting node: %v", err) + klog.Errorf("Error deleting node: %v", err) } } @@ -74,22 +74,22 @@ func createNodes(apiURL string, config *Config) error { QPS: float32(config.CreateQPS), Burst: config.CreateQPS, }) - glog.Infof("Creating %d nodes", config.NumNodes) + klog.Infof("Creating %d nodes", config.NumNodes) for i := 0; i < config.NumNodes; i++ { var err error for j := 0; j < maxCreateRetries; j++ { if _, err = clientSet.CoreV1().Nodes().Create(baseNodeTemplate); err != nil && errors.IsServerTimeout(err) { - glog.Infof("Server timeout creating nodes, retrying after %v", retryDelay) + klog.Infof("Server timeout creating nodes, retrying after %v", retryDelay) time.Sleep(retryDelay) continue } break } if err != nil { - glog.Errorf("Error creating nodes: %v", err) + klog.Errorf("Error creating nodes: %v", err) return err } } - glog.Infof("%d nodes created", config.NumNodes) + klog.Infof("%d nodes created", config.NumNodes) return nil } diff --git a/test/integration/master/BUILD b/test/integration/master/BUILD index fa012cd25ad..bb7c8cc979b 100644 --- a/test/integration/master/BUILD +++ b/test/integration/master/BUILD @@ -43,6 +43,7 @@ go_test( "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/config/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/group:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/request/bearertoken:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", @@ -50,7 +51,6 @@ go_test( "//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", @@ -62,10 +62,11 @@ go_test( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library", "//test/integration:go_default_library", + "//test/integration/etcd:go_default_library", "//test/integration/framework:go_default_library", "//test/utils:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:android": [ "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", @@ -125,65 +126,65 @@ go_library( "//cmd/kube-apiserver/app/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/config/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//test/integration:go_default_library", "//test/integration/framework:go_default_library", "//vendor/github.com/coreos/etcd/clientv3:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:android": [ "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:darwin": [ "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:dragonfly": [ "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:freebsd": [ "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:linux": [ "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:netbsd": [ "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:openbsd": [ "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:plan9": [ "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "@io_bazel_rules_go//go/platform:solaris": [ "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "//conditions:default": [], }), diff --git a/test/integration/master/audit_test.go b/test/integration/master/audit_test.go index 35c0fccc234..ecdf2d2d0d4 100644 --- a/test/integration/master/audit_test.go +++ b/test/integration/master/audit_test.go @@ -124,11 +124,7 @@ func testAudit(t *testing.T, version string) { configMapChan, err := kubeclient.CoreV1().ConfigMaps(namespace).Watch(watchOptions) expectNoError(t, err, "failed to create watch for config maps") - for range configMapChan.ResultChan() { - // Block until watchOptions.TimeoutSeconds expires. - // If the test finishes before watchOptions.TimeoutSeconds expires, the watch audit - // event at stage ResponseComplete will not be generated. - } + configMapChan.Stop() _, err = kubeclient.CoreV1().ConfigMaps(namespace).Update(configMap) expectNoError(t, err, "failed to update audit-configmap") @@ -183,7 +179,7 @@ func testAudit(t *testing.T, version string) { }, { Level: auditinternal.LevelRequestResponse, Stage: auditinternal.StageResponseStarted, - RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout), Verb: "watch", Code: 200, User: auditTestUser, @@ -195,7 +191,7 @@ func testAudit(t *testing.T, version string) { }, { Level: auditinternal.LevelRequestResponse, Stage: auditinternal.StageResponseComplete, - RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeoutSeconds=%d&watch=true", namespace, watchTestTimeout), + RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout), Verb: "watch", Code: 200, User: auditTestUser, diff --git a/test/integration/master/crd_test.go b/test/integration/master/crd_test.go index 61c11c6a6d4..ac28b037c30 100644 --- a/test/integration/master/crd_test.go +++ b/test/integration/master/crd_test.go @@ -18,7 +18,6 @@ package master import ( "encoding/json" - "fmt" "testing" "time" @@ -37,6 +36,7 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/framework" ) @@ -81,12 +81,8 @@ func TestCRDShadowGroup(t *testing.T) { }, }, } - if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil { - t.Fatalf("Failed to create networking group CRD: %v", err) - } - if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil { - t.Fatalf("Failed to establish networking group CRD: %v", err) - } + etcd.CreateTestCRDs(t, apiextensionsclient, true, crd) + // wait to give aggregator time to update time.Sleep(2 * time.Second) @@ -97,11 +93,7 @@ func TestCRDShadowGroup(t *testing.T) { } t.Logf("Checking that crd resource does not show up in networking group") - found, err := crdExistsInDiscovery(apiextensionsclient, crd) - if err != nil { - t.Fatalf("unexpected discovery error: %v", err) - } - if found { + if etcd.CrdExistsInDiscovery(apiextensionsclient, crd) { t.Errorf("CRD resource shows up in discovery, but shouldn't.") } } @@ -137,17 +129,7 @@ func TestCRD(t *testing.T) { }, }, } - if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil { - t.Fatalf("Failed to create foos.cr.bar.com CRD; %v", err) - } - if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil { - t.Fatalf("Failed to establish foos.cr.bar.com CRD: %v", err) - } - if err := wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) { - return crdExistsInDiscovery(apiextensionsclient, crd) - }); err != nil { - t.Fatalf("Failed to see foos.cr.bar.com in discovery: %v", err) - } + etcd.CreateTestCRDs(t, apiextensionsclient, false, crd) t.Logf("Trying to access foos.cr.bar.com with dynamic client") dynamicClient, err := dynamic.NewForConfig(result.ClientConfig) @@ -306,38 +288,3 @@ func unstructuredFoo(foo *Foo) (*unstructured.Unstructured, error) { } return ret, nil } - -func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error { - return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { - crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) - if err != nil { - return false, err - } - for _, cond := range crd.Status.Conditions { - switch cond.Type { - case apiextensionsv1beta1.Established: - if cond.Status == apiextensionsv1beta1.ConditionTrue { - return true, err - } - case apiextensionsv1beta1.NamesAccepted: - if cond.Status == apiextensionsv1beta1.ConditionFalse { - fmt.Printf("Name conflict: %v\n", cond.Reason) - } - } - } - return false, nil - }) -} - -func crdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) (bool, error) { - resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + crd.Spec.Version) - if err != nil { - return false, nil - } - for _, resource := range resourceList.APIResources { - if resource.Name == crd.Spec.Names.Plural { - return true, nil - } - } - return false, nil -} diff --git a/test/integration/master/kms_plugin_mock.go b/test/integration/master/kms_plugin_mock.go index ad7c73f049c..a731e1ddcb1 100644 --- a/test/integration/master/kms_plugin_mock.go +++ b/test/integration/master/kms_plugin_mock.go @@ -26,8 +26,8 @@ import ( "google.golang.org/grpc" - "github.com/golang/glog" kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" + "k8s.io/klog" ) const ( @@ -51,7 +51,7 @@ func NewBase64Plugin() (*base64Plugin, error) { if err != nil { return nil, fmt.Errorf("failed to listen on the unix socket, error: %v", err) } - glog.Infof("Listening on %s", sockFile) + klog.Infof("Listening on %s", sockFile) server := grpc.NewServer() @@ -78,7 +78,7 @@ func (s *base64Plugin) Version(ctx context.Context, request *kmsapi.VersionReque } func (s *base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) { - glog.Infof("Received Decrypt Request for DEK: %s", string(request.Cipher)) + klog.Infof("Received Decrypt Request for DEK: %s", string(request.Cipher)) buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher))) n, err := base64.StdEncoding.Decode(buf, request.Cipher) @@ -90,7 +90,7 @@ func (s *base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptReque } func (s *base64Plugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) { - glog.Infof("Received Encrypt Request for DEK: %x", request.Plain) + klog.Infof("Received Encrypt Request for DEK: %x", request.Plain) s.encryptRequest <- request buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain))) diff --git a/test/integration/master/kms_transformation_test.go b/test/integration/master/kms_transformation_test.go index df7a70bd314..1b39b436efa 100644 --- a/test/integration/master/kms_transformation_test.go +++ b/test/integration/master/kms_transformation_test.go @@ -39,8 +39,8 @@ const ( dekKeySizeLen = 2 kmsConfigYAML = ` -kind: EncryptionConfig -apiVersion: v1 +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 resources: - resources: - secrets diff --git a/test/integration/master/secrets_transformation_test.go b/test/integration/master/secrets_transformation_test.go index a8d408435d5..98d19ad7a5b 100644 --- a/test/integration/master/secrets_transformation_test.go +++ b/test/integration/master/secrets_transformation_test.go @@ -23,7 +23,7 @@ import ( "fmt" "testing" - "k8s.io/apiserver/pkg/server/options/encryptionconfig" + apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1" "k8s.io/apiserver/pkg/storage/value" aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes" ) @@ -33,8 +33,8 @@ const ( aesCBCPrefix = "k8s:enc:aescbc:v1:key1:" aesGCMConfigYAML = ` -kind: EncryptionConfig -apiVersion: v1 +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 resources: - resources: - secrets @@ -46,8 +46,8 @@ resources: ` aesCBCConfigYAML = ` -kind: EncryptionConfig -apiVersion: v1 +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 resources: - resources: - secrets @@ -59,8 +59,8 @@ resources: ` identityConfigYAML = ` -kind: EncryptionConfig -apiVersion: v1 +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 resources: - resources: - secrets @@ -72,7 +72,7 @@ resources: // TestSecretsShouldBeEnveloped is an integration test between KubeAPI and etcd that checks: // 1. Secrets are encrypted on write // 2. Secrets are decrypted on read -// when EncryptionConfig is passed to KubeAPI server. +// when EncryptionConfiguration is passed to KubeAPI server. func TestSecretsShouldBeTransformed(t *testing.T) { var testCases = []struct { transformerConfigContent string @@ -128,7 +128,7 @@ func runBenchmark(b *testing.B, transformerConfig string) { } func unSealWithGCMTransformer(cipherText []byte, ctx value.Context, - transformerConfig encryptionconfig.ProviderConfig) ([]byte, error) { + transformerConfig apiserverconfigv1.ProviderConfiguration) ([]byte, error) { block, err := newAESCipher(transformerConfig.AESGCM.Keys[0].Secret) if err != nil { @@ -146,7 +146,7 @@ func unSealWithGCMTransformer(cipherText []byte, ctx value.Context, } func unSealWithCBCTransformer(cipherText []byte, ctx value.Context, - transformerConfig encryptionconfig.ProviderConfig) ([]byte, error) { + transformerConfig apiserverconfigv1.ProviderConfiguration) ([]byte, error) { block, err := newAESCipher(transformerConfig.AESCBC.Keys[0].Secret) if err != nil { diff --git a/test/integration/master/synthetic_master_test.go b/test/integration/master/synthetic_master_test.go index a4ef671983a..de3f9f29333 100644 --- a/test/integration/master/synthetic_master_test.go +++ b/test/integration/master/synthetic_master_test.go @@ -30,7 +30,7 @@ import ( "testing" "time" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" appsv1 "k8s.io/api/apps/v1" "k8s.io/api/core/v1" @@ -304,7 +304,7 @@ func TestObjectSizeResponses(t *testing.T) { const DeploymentTwoMegabyteSize = 1000000 expectedMsgFor1MB := `etcdserver: request is too large` - expectedMsgFor2MB := `rpc error: code = ResourceExhausted desc = grpc: trying to send message larger than max` + expectedMsgFor2MB := `rpc error: code = ResourceExhausted desc = trying to send message larger than max` expectedMsgForLargeAnnotation := `metadata.annotations: Too long: must have at most 262144 characters` deployment1 := constructBody("a", DeploymentMegabyteSize, "labels", t) // >1 MB file diff --git a/test/integration/master/transformation_testcase.go b/test/integration/master/transformation_testcase.go index 8f8171b7daf..b5f4eb63179 100644 --- a/test/integration/master/transformation_testcase.go +++ b/test/integration/master/transformation_testcase.go @@ -28,12 +28,12 @@ import ( "testing" "github.com/coreos/etcd/clientv3" - "github.com/ghodss/yaml" "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/yaml" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apiserver/pkg/server/options/encryptionconfig" + apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1" "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/apiserver/pkg/storage/value" "k8s.io/client-go/kubernetes" @@ -51,7 +51,7 @@ const ( metricsPrefix = "apiserver_storage_" ) -type unSealSecret func(cipherText []byte, ctx value.Context, config encryptionconfig.ProviderConfig) ([]byte, error) +type unSealSecret func(cipherText []byte, ctx value.Context, config apiserverconfigv1.ProviderConfiguration) ([]byte, error) type transformTest struct { logger kubeapiservertesting.Logger @@ -186,8 +186,8 @@ func (e *transformTest) createEncryptionConfig() (string, error) { return tempDir, nil } -func (e *transformTest) getEncryptionConfig() (*encryptionconfig.ProviderConfig, error) { - var config encryptionconfig.EncryptionConfig +func (e *transformTest) getEncryptionConfig() (*apiserverconfigv1.ProviderConfiguration, error) { + var config apiserverconfigv1.EncryptionConfiguration err := yaml.Unmarshal([]byte(e.transformerConfig), &config) if err != nil { return nil, fmt.Errorf("failed to extract transformer key: %v", err) diff --git a/test/integration/metrics/BUILD b/test/integration/metrics/BUILD index 55718ed466e..05bd17a9ca2 100644 --- a/test/integration/metrics/BUILD +++ b/test/integration/metrics/BUILD @@ -40,8 +40,8 @@ go_test( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//test/integration/framework:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/protobuf/proto:go_default_library", "//vendor/github.com/prometheus/client_model/go:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/integration/metrics/metrics_test.go b/test/integration/metrics/metrics_test.go index 9269f7ba4e1..6eff0c1908e 100644 --- a/test/integration/metrics/metrics_test.go +++ b/test/integration/metrics/metrics_test.go @@ -30,9 +30,9 @@ import ( restclient "k8s.io/client-go/rest" "k8s.io/kubernetes/test/integration/framework" - "github.com/golang/glog" "github.com/golang/protobuf/proto" prometheuspb "github.com/prometheus/client_model/go" + "k8s.io/klog" ) const scrapeRequestHeader = "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=compact-text" @@ -66,7 +66,7 @@ func scrapeMetrics(s *httptest.Server) ([]*prometheuspb.MetricFamily, error) { if err := proto.UnmarshalText(scanner.Text(), &metric); err != nil { return nil, fmt.Errorf("Failed to unmarshal line of metrics response: %v", err) } - glog.V(4).Infof("Got metric %q", metric.GetName()) + klog.V(4).Infof("Got metric %q", metric.GetName()) metrics = append(metrics, &metric) } return metrics, nil diff --git a/test/integration/scheduler/BUILD b/test/integration/scheduler/BUILD index 0956657c99e..4ef611c9967 100644 --- a/test/integration/scheduler/BUILD +++ b/test/integration/scheduler/BUILD @@ -64,7 +64,7 @@ go_test( "//test/integration/framework:go_default_library", "//test/utils:go_default_library", "//test/utils/image:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/integration/scheduler/extender_test.go b/test/integration/scheduler/extender_test.go index aa6fad56484..2afef4b0d0f 100644 --- a/test/integration/scheduler/extender_test.go +++ b/test/integration/scheduler/extender_test.go @@ -349,7 +349,7 @@ func TestSchedulerExtender(t *testing.T) { } policy.APIVersion = "v1" - context = initTestScheduler(t, context, nil, false, &policy) + context = initTestScheduler(t, context, false, &policy) defer cleanupTest(t, context) DoTestPodScheduling(context.ns, t, clientSet) diff --git a/test/integration/scheduler/preemption_test.go b/test/integration/scheduler/preemption_test.go index 27dd50cdf42..a1fc044a78c 100644 --- a/test/integration/scheduler/preemption_test.go +++ b/test/integration/scheduler/preemption_test.go @@ -36,7 +36,7 @@ import ( _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" testutils "k8s.io/kubernetes/test/utils" - "github.com/golang/glog" + "k8s.io/klog" ) var lowPriority, mediumPriority, highPriority = int32(100), int32(200), int32(300) @@ -481,7 +481,7 @@ func TestPreemptionStarvation(t *testing.T) { t.Errorf("Preemptor pod %v didn't get scheduled: %v", preemptor.Name, err) } // Cleanup - glog.Info("Cleaning up all pods...") + klog.Info("Cleaning up all pods...") allPods := pendingPods allPods = append(allPods, runningPods...) allPods = append(allPods, preemptor) diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index 3151768454a..3f7938bb5ff 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -522,7 +522,10 @@ func TestMultiScheduler(t *testing.T) { informerFactory2 := informers.NewSharedInformerFactory(context.clientSet, 0) podInformer2 := factory.NewPodInformer(context.clientSet, 0) - schedulerConfigFactory2 := createConfiguratorWithPodInformer(fooScheduler, clientSet2, podInformer2, informerFactory2) + stopCh := make(chan struct{}) + defer close(stopCh) + + schedulerConfigFactory2 := createConfiguratorWithPodInformer(fooScheduler, clientSet2, podInformer2, informerFactory2, stopCh) schedulerConfig2, err := schedulerConfigFactory2.Create() if err != nil { t.Errorf("Couldn't create scheduler config: %v", err) @@ -530,12 +533,11 @@ func TestMultiScheduler(t *testing.T) { eventBroadcaster2 := record.NewBroadcaster() schedulerConfig2.Recorder = eventBroadcaster2.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: fooScheduler}) eventBroadcaster2.StartRecordingToSink(&clientv1core.EventSinkImpl{Interface: clientSet2.CoreV1().Events("")}) - go podInformer2.Informer().Run(schedulerConfig2.StopEverything) - informerFactory2.Start(schedulerConfig2.StopEverything) + go podInformer2.Informer().Run(stopCh) + informerFactory2.Start(stopCh) sched2, _ := scheduler.NewFromConfigurator(&scheduler.FakeConfigurator{Config: schedulerConfig2}, nil...) sched2.Run() - defer close(schedulerConfig2.StopEverything) // 6. **check point-2**: // - testPodWithAnnotationFitsFoo should be scheduled diff --git a/test/integration/scheduler/taint_test.go b/test/integration/scheduler/taint_test.go index 89189fc1555..e6964760ec4 100644 --- a/test/integration/scheduler/taint_test.go +++ b/test/integration/scheduler/taint_test.go @@ -85,13 +85,10 @@ func TestTaintNodeByCondition(t *testing.T) { admission.SetExternalKubeClientSet(externalClientset) admission.SetExternalKubeInformerFactory(externalInformers) - controllerCh := make(chan struct{}) - defer close(controllerCh) - // Apply feature gates to enable TaintNodesByCondition algorithmprovider.ApplyFeatureGates() - context = initTestScheduler(t, context, controllerCh, false, nil) + context = initTestScheduler(t, context, false, nil) cs := context.clientSet informers := context.informerFactory nsName := context.ns.Name @@ -120,13 +117,13 @@ func TestTaintNodeByCondition(t *testing.T) { t.Errorf("Failed to create node controller: %v", err) return } - go nc.Run(controllerCh) + go nc.Run(context.stopCh) // Waiting for all controller sync. - externalInformers.Start(controllerCh) - externalInformers.WaitForCacheSync(controllerCh) - informers.Start(controllerCh) - informers.WaitForCacheSync(controllerCh) + externalInformers.Start(context.stopCh) + externalInformers.WaitForCacheSync(context.stopCh) + informers.Start(context.stopCh) + informers.WaitForCacheSync(context.stopCh) // ------------------------------------------- // Test TaintNodeByCondition feature. diff --git a/test/integration/scheduler/util.go b/test/integration/scheduler/util.go index ddc624a050f..71875405ee6 100644 --- a/test/integration/scheduler/util.go +++ b/test/integration/scheduler/util.go @@ -65,6 +65,7 @@ type TestContext struct { schedulerConfigFactory factory.Configurator schedulerConfig *factory.Config scheduler *scheduler.Scheduler + stopCh chan struct{} } // createConfiguratorWithPodInformer creates a configurator for scheduler. @@ -73,6 +74,7 @@ func createConfiguratorWithPodInformer( clientSet clientset.Interface, podInformer coreinformers.PodInformer, informerFactory informers.SharedInformerFactory, + stopCh <-chan struct{}, ) factory.Configurator { return factory.NewConfigFactory(&factory.ConfigFactoryArgs{ SchedulerName: schedulerName, @@ -92,13 +94,16 @@ func createConfiguratorWithPodInformer( DisablePreemption: false, PercentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, BindTimeoutSeconds: 600, + StopCh: stopCh, }) } // initTestMasterAndScheduler initializes a test environment and creates a master with default // configuration. func initTestMaster(t *testing.T, nsPrefix string, admission admission.Interface) *TestContext { - var context TestContext + context := TestContext{ + stopCh: make(chan struct{}), + } // 1. Create master h := &framework.MasterHolder{Initialized: make(chan struct{})} @@ -138,13 +143,12 @@ func initTestMaster(t *testing.T, nsPrefix string, admission admission.Interface func initTestScheduler( t *testing.T, context *TestContext, - controllerCh chan struct{}, setPodInformer bool, policy *schedulerapi.Policy, ) *TestContext { // Pod preemption is enabled by default scheduler configuration, but preemption only happens when PodPriority // feature gate is enabled at the same time. - return initTestSchedulerWithOptions(t, context, controllerCh, setPodInformer, policy, false, false, time.Second) + return initTestSchedulerWithOptions(t, context, setPodInformer, policy, false, true, time.Second) } // initTestSchedulerWithOptions initializes a test environment and creates a scheduler with default @@ -152,7 +156,6 @@ func initTestScheduler( func initTestSchedulerWithOptions( t *testing.T, context *TestContext, - controllerCh chan struct{}, setPodInformer bool, policy *schedulerapi.Policy, disablePreemption bool, @@ -179,7 +182,7 @@ func initTestSchedulerWithOptions( } context.schedulerConfigFactory = createConfiguratorWithPodInformer( - v1.DefaultSchedulerName, context.clientSet, podInformer, context.informerFactory) + v1.DefaultSchedulerName, context.clientSet, podInformer, context.informerFactory, context.stopCh) var err error @@ -193,11 +196,6 @@ func initTestSchedulerWithOptions( t.Fatalf("Couldn't create scheduler config: %v", err) } - // set controllerCh if provided. - if controllerCh != nil { - context.schedulerConfig.StopEverything = controllerCh - } - // set DisablePreemption option context.schedulerConfig.DisablePreemption = disablePreemption @@ -252,21 +250,21 @@ func initDisruptionController(context *TestContext) *disruption.DisruptionContro // initTest initializes a test environment and creates master and scheduler with default // configuration. func initTest(t *testing.T, nsPrefix string) *TestContext { - return initTestScheduler(t, initTestMaster(t, nsPrefix, nil), nil, true, nil) + return initTestScheduler(t, initTestMaster(t, nsPrefix, nil), true, nil) } // initTestDisablePreemption initializes a test environment and creates master and scheduler with default // configuration but with pod preemption disabled. func initTestDisablePreemption(t *testing.T, nsPrefix string) *TestContext { return initTestSchedulerWithOptions( - t, initTestMaster(t, nsPrefix, nil), nil, true, nil, true, false, time.Second) + t, initTestMaster(t, nsPrefix, nil), true, nil, true, true, time.Second) } // cleanupTest deletes the scheduler and the test namespace. It should be called // at the end of a test. func cleanupTest(t *testing.T, context *TestContext) { // Kill the scheduler. - close(context.schedulerConfig.StopEverything) + close(context.stopCh) // Cleanup nodes. context.clientSet.CoreV1().Nodes().DeleteCollection(nil, metav1.ListOptions{}) framework.DeleteTestingNamespace(context.ns, context.httpServer, t) diff --git a/test/integration/scheduler/volume_binding_test.go b/test/integration/scheduler/volume_binding_test.go index e82b3562be2..80cc3d85a45 100644 --- a/test/integration/scheduler/volume_binding_test.go +++ b/test/integration/scheduler/volume_binding_test.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -99,7 +99,7 @@ func TestVolumeBinding(t *testing.T) { "VolumeScheduling": true, "PersistentLocalVolumes": true, } - config := setupCluster(t, "volume-scheduling-", 2, features, 0, 0, false) + config := setupCluster(t, "volume-scheduling-", 2, features, 0, 0, true) defer config.teardown() cases := map[string]struct { @@ -191,7 +191,7 @@ func TestVolumeBinding(t *testing.T) { } for name, test := range cases { - glog.Infof("Running test %v", name) + klog.Infof("Running test %v", name) // Create two StorageClasses suffix := rand.String(4) @@ -272,7 +272,7 @@ func TestVolumeBindingRescheduling(t *testing.T) { "VolumeScheduling": true, "PersistentLocalVolumes": true, } - config := setupCluster(t, "volume-scheduling-", 2, features, 0, 0, false) + config := setupCluster(t, "volume-scheduling-", 2, features, 0, 0, true) defer config.teardown() storageClassName := "local-storage" @@ -335,7 +335,7 @@ func TestVolumeBindingRescheduling(t *testing.T) { } for name, test := range cases { - glog.Infof("Running test %v", name) + klog.Infof("Running test %v", name) if test.pod == nil { t.Fatal("pod is required for this test") @@ -363,7 +363,7 @@ func TestVolumeBindingRescheduling(t *testing.T) { } // Wait for pod is unschedulable. - glog.Infof("Waiting for pod is unschedulable") + klog.Infof("Waiting for pod is unschedulable") if err := waitForPodUnschedulable(config.client, test.pod); err != nil { t.Errorf("Failed as Pod %s was not unschedulable: %v", test.pod.Name, err) } @@ -373,12 +373,12 @@ func TestVolumeBindingRescheduling(t *testing.T) { // Wait for pod is scheduled or unscheduable. if !test.shouldFail { - glog.Infof("Waiting for pod is scheduled") + klog.Infof("Waiting for pod is scheduled") if err := waitForPodToSchedule(config.client, test.pod); err != nil { t.Errorf("Failed to schedule Pod %q: %v", test.pod.Name, err) } } else { - glog.Infof("Waiting for pod is unschedulable") + klog.Infof("Waiting for pod is unschedulable") if err := waitForPodUnschedulable(config.client, test.pod); err != nil { t.Errorf("Failed as Pod %s was not unschedulable: %v", test.pod.Name, err) } @@ -418,7 +418,7 @@ func testVolumeBindingStress(t *testing.T, schedulerResyncPeriod time.Duration, "VolumeScheduling": true, "PersistentLocalVolumes": true, } - config := setupCluster(t, "volume-binding-stress-", 1, features, schedulerResyncPeriod, provisionDelaySeconds, false) + config := setupCluster(t, "volume-binding-stress-", 1, features, schedulerResyncPeriod, provisionDelaySeconds, true) defer config.teardown() // Set max volume limit to the number of PVCs the test will create @@ -625,7 +625,7 @@ func TestPVAffinityConflict(t *testing.T) { "VolumeScheduling": true, "PersistentLocalVolumes": true, } - config := setupCluster(t, "volume-scheduling-", 3, features, 0, 0, false) + config := setupCluster(t, "volume-scheduling-", 3, features, 0, 0, true) defer config.teardown() pv := makePV("local-pv", classImmediate, "", "", node1) @@ -688,7 +688,7 @@ func TestVolumeProvision(t *testing.T) { "VolumeScheduling": true, "PersistentLocalVolumes": true, } - config := setupCluster(t, "volume-scheduling", 1, features, 0, 0, false) + config := setupCluster(t, "volume-scheduling", 1, features, 0, 0, true) defer config.teardown() cases := map[string]struct { @@ -737,7 +737,7 @@ func TestVolumeProvision(t *testing.T) { } for name, test := range cases { - glog.Infof("Running test %v", name) + klog.Infof("Running test %v", name) // Create StorageClasses suffix := rand.String(4) @@ -901,9 +901,7 @@ func setupCluster(t *testing.T, nsName string, numberOfNodes int, features map[s // Set feature gates utilfeature.DefaultFeatureGate.SetFromMap(features) - controllerCh := make(chan struct{}) - - context := initTestSchedulerWithOptions(t, initTestMaster(t, nsName, nil), controllerCh, false, nil, false, disableEquivalenceCache, resyncPeriod) + context := initTestSchedulerWithOptions(t, initTestMaster(t, nsName, nil), false, nil, false, disableEquivalenceCache, resyncPeriod) clientset := context.clientSet ns := context.ns.Name @@ -912,10 +910,10 @@ func setupCluster(t *testing.T, nsName string, numberOfNodes int, features map[s if err != nil { t.Fatalf("Failed to create PV controller: %v", err) } - go ctrl.Run(controllerCh) + go ctrl.Run(context.stopCh) // Start informer factory after all controllers are configured and running. - informerFactory.Start(controllerCh) - informerFactory.WaitForCacheSync(controllerCh) + informerFactory.Start(context.stopCh) + informerFactory.WaitForCacheSync(context.stopCh) // Create shared objects // Create nodes @@ -936,7 +934,7 @@ func setupCluster(t *testing.T, nsName string, numberOfNodes int, features map[s return &testConfig{ client: clientset, ns: ns, - stop: controllerCh, + stop: context.stopCh, teardown: func() { deleteTestObjects(clientset, ns, nil) cleanupTest(t, context) diff --git a/test/integration/scheduler_perf/BUILD b/test/integration/scheduler_perf/BUILD index ab2aeeb0eaa..abaadbe35b7 100644 --- a/test/integration/scheduler_perf/BUILD +++ b/test/integration/scheduler_perf/BUILD @@ -42,7 +42,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//test/integration/framework:go_default_library", "//test/utils:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/integration/scheduler_perf/scheduler_bench_test.go b/test/integration/scheduler_perf/scheduler_bench_test.go index 30403d14080..85af9567234 100644 --- a/test/integration/scheduler_perf/scheduler_bench_test.go +++ b/test/integration/scheduler_perf/scheduler_bench_test.go @@ -28,7 +28,7 @@ import ( "k8s.io/kubernetes/test/integration/framework" testutils "k8s.io/kubernetes/test/utils" - "github.com/golang/glog" + "k8s.io/klog" ) var ( @@ -227,7 +227,7 @@ func benchmarkScheduling(numNodes, numExistingPods, minPods int, "scheduler-perf-", ) if err := nodePreparer.PrepareNodes(); err != nil { - glog.Fatalf("%v", err) + klog.Fatalf("%v", err) } defer nodePreparer.CleanupNodes() @@ -239,7 +239,7 @@ func benchmarkScheduling(numNodes, numExistingPods, minPods int, for { scheduled, err := schedulerConfigFactory.GetScheduledPodLister().List(labels.Everything()) if err != nil { - glog.Fatalf("%v", err) + klog.Fatalf("%v", err) } if len(scheduled) >= numExistingPods { break @@ -257,7 +257,7 @@ func benchmarkScheduling(numNodes, numExistingPods, minPods int, // TODO: Setup watch on apiserver and wait until all pods scheduled. scheduled, err := schedulerConfigFactory.GetScheduledPodLister().List(labels.Everything()) if err != nil { - glog.Fatalf("%v", err) + klog.Fatalf("%v", err) } if len(scheduled) >= numExistingPods+b.N { break diff --git a/test/integration/scheduler_perf/scheduler_test.go b/test/integration/scheduler_perf/scheduler_test.go index e8b232b8e2d..80e97d6f63b 100644 --- a/test/integration/scheduler_perf/scheduler_test.go +++ b/test/integration/scheduler_perf/scheduler_test.go @@ -18,11 +18,11 @@ package benchmark import ( "fmt" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/klog" "k8s.io/kubernetes/pkg/scheduler/factory" testutils "k8s.io/kubernetes/test/utils" "math" @@ -137,7 +137,7 @@ func schedulePods(config *testConfig) int32 { time.Sleep(50 * time.Millisecond) scheduled, err := config.schedulerSupportFunctions.GetScheduledPodLister().List(labels.Everything()) if err != nil { - glog.Fatalf("%v", err) + klog.Fatalf("%v", err) } // 30,000 pods -> wait till @ least 300 are scheduled to start measuring. // TODO Find out why sometimes there may be scheduling blips in the beginning. @@ -155,7 +155,7 @@ func schedulePods(config *testConfig) int32 { // TODO: Setup watch on apiserver and wait until all pods scheduled. scheduled, err := config.schedulerSupportFunctions.GetScheduledPodLister().List(labels.Everything()) if err != nil { - glog.Fatalf("%v", err) + klog.Fatalf("%v", err) } // We will be completed when all pods are done being scheduled. diff --git a/test/integration/serviceaccount/service_account_test.go b/test/integration/serviceaccount/service_account_test.go index c554597a8dc..5139911db66 100644 --- a/test/integration/serviceaccount/service_account_test.go +++ b/test/integration/serviceaccount/service_account_test.go @@ -374,7 +374,7 @@ func startServiceAccountTestServer(t *testing.T) (*clientset.Clientset, restclie }) serviceAccountKey, _ := rsa.GenerateKey(rand.Reader, 2048) serviceAccountTokenGetter := serviceaccountcontroller.NewGetterFromClient(rootClientset) - serviceAccountTokenAuth := serviceaccount.JWTTokenAuthenticator(serviceaccount.LegacyIssuer, []interface{}{&serviceAccountKey.PublicKey}, serviceaccount.NewLegacyValidator(true, serviceAccountTokenGetter)) + serviceAccountTokenAuth := serviceaccount.JWTTokenAuthenticator(serviceaccount.LegacyIssuer, []interface{}{&serviceAccountKey.PublicKey}, nil, serviceaccount.NewLegacyValidator(true, serviceAccountTokenGetter)) authenticator := union.New( bearertoken.New(rootTokenAuth), bearertoken.New(serviceAccountTokenAuth), diff --git a/test/integration/controllermanager/BUILD b/test/integration/serving/BUILD similarity index 95% rename from test/integration/controllermanager/BUILD rename to test/integration/serving/BUILD index 1ebcf099d89..24fe855f3ee 100644 --- a/test/integration/controllermanager/BUILD +++ b/test/integration/serving/BUILD @@ -20,6 +20,7 @@ go_test( "//cmd/cloud-controller-manager/app/testing:go_default_library", "//cmd/kube-apiserver/app/testing:go_default_library", "//cmd/kube-controller-manager/app/testing:go_default_library", + "//cmd/kube-scheduler/app/testing:go_default_library", "//pkg/cloudprovider/providers/fake:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/test/integration/controllermanager/main_test.go b/test/integration/serving/main_test.go similarity index 96% rename from test/integration/controllermanager/main_test.go rename to test/integration/serving/main_test.go index 0ef7244c3c8..5675067c7aa 100644 --- a/test/integration/controllermanager/main_test.go +++ b/test/integration/serving/main_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllermanager +package serving import ( "testing" diff --git a/test/integration/controllermanager/serving_test.go b/test/integration/serving/serving_test.go similarity index 88% rename from test/integration/controllermanager/serving_test.go rename to test/integration/serving/serving_test.go index da315adefff..668e1c26454 100644 --- a/test/integration/controllermanager/serving_test.go +++ b/test/integration/serving/serving_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllermanager +package serving import ( "crypto/tls" @@ -33,15 +33,16 @@ import ( "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/kubernetes" - cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider" cloudctrlmgrtesting "k8s.io/kubernetes/cmd/cloud-controller-manager/app/testing" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" kubectrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing" + kubeschedulertesting "k8s.io/kubernetes/cmd/kube-scheduler/app/testing" "k8s.io/kubernetes/pkg/cloudprovider/providers/fake" "k8s.io/kubernetes/test/integration/framework" ) -type controllerManagerTester interface { +type componentTester interface { StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) } @@ -65,7 +66,17 @@ func (cloudControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err } -func TestControllerManagerServing(t *testing.T) { +type kubeSchedulerTester struct{} + +func (kubeSchedulerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { + gotResult, err := kubeschedulertesting.StartTestServer(t, customFlags) + if err != nil { + return nil, nil, nil, nil, err + } + return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err +} + +func TestComponentSecureServingAndAuth(t *testing.T) { if !cloudprovider.IsCloudProvider("fake") { cloudprovider.RegisterCloudProvider("fake", fakeCloudProviderFactory) } @@ -188,20 +199,21 @@ users: tests := []struct { name string - tester controllerManagerTester + tester componentTester extraFlags []string }{ {"kube-controller-manager", kubeControllerManagerTester{}, nil}, {"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake"}}, + {"kube-scheduler", kubeSchedulerTester{}, nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - testControllerManager(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags) + testComponent(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags) }) } } -func testControllerManager(t *testing.T, tester controllerManagerTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) { +func testComponent(t *testing.T, tester componentTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) { tests := []struct { name string flags []string @@ -228,8 +240,7 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/healthz", true, false, intPtr(http.StatusOK), nil}, - {"/metrics without auhn/z", []string{ - "--kubeconfig", kubeconfig, + {"/metrics without authn/authz", []string{ "--kubeconfig", kubeconfig, "--leader-elect=false", "--port=10253", @@ -297,7 +308,7 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon serverCertPath := path.Join(secureOptions.ServerCert.CertDirectory, secureOptions.ServerCert.PairName+".crt") serverCert, err := ioutil.ReadFile(serverCertPath) if err != nil { - t.Fatalf("Failed to read controller-manager server cert %q: %v", serverCertPath, err) + t.Fatalf("Failed to read component server cert %q: %v", serverCertPath, err) } pool.AppendCertsFromPEM(serverCert) tr := &http.Transport{ @@ -316,13 +327,13 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon } r, err := client.Do(req) if err != nil { - t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err) + t.Fatalf("failed to GET %s from component: %v", tt.path, err) } body, err := ioutil.ReadAll(r.Body) defer r.Body.Close() if got, expected := r.StatusCode, *tt.wantSecureCode; got != expected { - t.Fatalf("expected http %d at %s of controller-manager, got: %d %q", expected, tt.path, got, string(body)) + t.Fatalf("expected http %d at %s of component, got: %d %q", expected, tt.path, got, string(body)) } } @@ -332,12 +343,12 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon url := fmt.Sprintf("http://%s%s", insecureInfo.Listener.Addr().String(), tt.path) r, err := http.Get(url) if err != nil { - t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err) + t.Fatalf("failed to GET %s from component: %v", tt.path, err) } body, err := ioutil.ReadAll(r.Body) defer r.Body.Close() if got, expected := r.StatusCode, *tt.wantInsecureCode; got != expected { - t.Fatalf("expected http %d at %s of controller-manager, got: %d %q", expected, tt.path, got, string(body)) + t.Fatalf("expected http %d at %s of component, got: %d %q", expected, tt.path, got, string(body)) } } }) diff --git a/test/integration/util/BUILD b/test/integration/util/BUILD index 7063c2162ed..5bd8b0bb804 100644 --- a/test/integration/util/BUILD +++ b/test/integration/util/BUILD @@ -27,8 +27,8 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//test/integration/framework:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/oauth2:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/integration/util/util.go b/test/integration/util/util.go index fa39562d513..4137429ec45 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -20,13 +20,13 @@ import ( "net/http" "net/http/httptest" - "github.com/golang/glog" "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" clientv1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" + "k8s.io/klog" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler" @@ -49,9 +49,9 @@ func StartApiserver() (string, ShutdownFunc) { framework.RunAMasterUsingServer(framework.NewIntegrationTestMasterConfig(), s, h) shutdownFunc := func() { - glog.Infof("destroying API server") + klog.Infof("destroying API server") s.Close() - glog.Infof("destroyed API server") + klog.Infof("destroyed API server") } return s.URL, shutdownFunc } @@ -66,26 +66,24 @@ func StartScheduler(clientSet clientset.Interface) (factory.Configurator, Shutdo evtWatch := evtBroadcaster.StartRecordingToSink(&clientv1core.EventSinkImpl{ Interface: clientSet.CoreV1().Events("")}) - schedulerConfigurator := createSchedulerConfigurator(clientSet, informerFactory) + stopCh := make(chan struct{}) + schedulerConfigurator := createSchedulerConfigurator(clientSet, informerFactory, stopCh) sched, err := scheduler.NewFromConfigurator(schedulerConfigurator, func(conf *factory.Config) { conf.Recorder = evtBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "scheduler"}) }) if err != nil { - glog.Fatalf("Error creating scheduler: %v", err) + klog.Fatalf("Error creating scheduler: %v", err) } - stop := make(chan struct{}) - informerFactory.Start(stop) - + informerFactory.Start(stopCh) sched.Run() shutdownFunc := func() { - glog.Infof("destroying scheduler") + klog.Infof("destroying scheduler") evtWatch.Stop() - sched.StopEverything() - close(stop) - glog.Infof("destroyed scheduler") + close(stopCh) + klog.Infof("destroyed scheduler") } return schedulerConfigurator, shutdownFunc } @@ -94,10 +92,8 @@ func StartScheduler(clientSet clientset.Interface) (factory.Configurator, Shutdo func createSchedulerConfigurator( clientSet clientset.Interface, informerFactory informers.SharedInformerFactory, + stopCh <-chan struct{}, ) factory.Configurator { - // Enable EnableEquivalenceClassCache for all integration tests. - utilfeature.DefaultFeatureGate.Set("EnableEquivalenceClassCache=true") - return factory.NewConfigFactory(&factory.ConfigFactoryArgs{ SchedulerName: v1.DefaultSchedulerName, Client: clientSet, @@ -115,5 +111,6 @@ func createSchedulerConfigurator( EnableEquivalenceClassCache: utilfeature.DefaultFeatureGate.Enabled(features.EnableEquivalenceClassCache), DisablePreemption: false, PercentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, + StopCh: stopCh, }) } diff --git a/test/integration/volume/BUILD b/test/integration/volume/BUILD index 0e6231fde6c..27785393155 100644 --- a/test/integration/volume/BUILD +++ b/test/integration/volume/BUILD @@ -37,7 +37,7 @@ go_test( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/reference:go_default_library", "//test/integration/framework:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/integration/volume/persistent_volumes_test.go b/test/integration/volume/persistent_volumes_test.go index 57a49c75fa2..7fdba82ec31 100644 --- a/test/integration/volume/persistent_volumes_test.go +++ b/test/integration/volume/persistent_volumes_test.go @@ -41,8 +41,8 @@ import ( volumetest "k8s.io/kubernetes/pkg/volume/testing" "k8s.io/kubernetes/test/integration/framework" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog" ) // Several tests in this file are configurable by environment variables: @@ -66,10 +66,10 @@ func getObjectCount() int { var err error objectCount, err = strconv.Atoi(s) if err != nil { - glog.Fatalf("cannot parse value of KUBE_INTEGRATION_PV_OBJECTS: %v", err) + klog.Fatalf("cannot parse value of KUBE_INTEGRATION_PV_OBJECTS: %v", err) } } - glog.V(2).Infof("using KUBE_INTEGRATION_PV_OBJECTS=%d", objectCount) + klog.V(2).Infof("using KUBE_INTEGRATION_PV_OBJECTS=%d", objectCount) return objectCount } @@ -79,10 +79,10 @@ func getSyncPeriod(syncPeriod time.Duration) time.Duration { var err error period, err = time.ParseDuration(s) if err != nil { - glog.Fatalf("cannot parse value of KUBE_INTEGRATION_PV_SYNC_PERIOD: %v", err) + klog.Fatalf("cannot parse value of KUBE_INTEGRATION_PV_SYNC_PERIOD: %v", err) } } - glog.V(2).Infof("using KUBE_INTEGRATION_PV_SYNC_PERIOD=%v", period) + klog.V(2).Infof("using KUBE_INTEGRATION_PV_SYNC_PERIOD=%v", period) return period } @@ -92,18 +92,18 @@ func testSleep() { var err error period, err = time.ParseDuration(s) if err != nil { - glog.Fatalf("cannot parse value of KUBE_INTEGRATION_PV_END_SLEEP: %v", err) + klog.Fatalf("cannot parse value of KUBE_INTEGRATION_PV_END_SLEEP: %v", err) } } - glog.V(2).Infof("using KUBE_INTEGRATION_PV_END_SLEEP=%v", period) + klog.V(2).Infof("using KUBE_INTEGRATION_PV_END_SLEEP=%v", period) if period != 0 { time.Sleep(period) - glog.V(2).Infof("sleep finished") + klog.V(2).Infof("sleep finished") } } func TestPersistentVolumeRecycler(t *testing.T) { - glog.V(2).Infof("TestPersistentVolumeRecycler started") + klog.V(2).Infof("TestPersistentVolumeRecycler started") _, s, closeFn := framework.RunAMaster(nil) defer closeFn() @@ -131,34 +131,34 @@ func TestPersistentVolumeRecycler(t *testing.T) { if err != nil { t.Errorf("Failed to create PersistentVolume: %v", err) } - glog.V(2).Infof("TestPersistentVolumeRecycler pvc created") + klog.V(2).Infof("TestPersistentVolumeRecycler pvc created") _, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(pvc) if err != nil { t.Errorf("Failed to create PersistentVolumeClaim: %v", err) } - glog.V(2).Infof("TestPersistentVolumeRecycler pvc created") + klog.V(2).Infof("TestPersistentVolumeRecycler pvc created") // wait until the controller pairs the volume and claim waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeBound) - glog.V(2).Infof("TestPersistentVolumeRecycler pv bound") + klog.V(2).Infof("TestPersistentVolumeRecycler pv bound") waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound) - glog.V(2).Infof("TestPersistentVolumeRecycler pvc bound") + klog.V(2).Infof("TestPersistentVolumeRecycler pvc bound") // deleting a claim releases the volume, after which it can be recycled if err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Delete(pvc.Name, nil); err != nil { t.Errorf("error deleting claim %s", pvc.Name) } - glog.V(2).Infof("TestPersistentVolumeRecycler pvc deleted") + klog.V(2).Infof("TestPersistentVolumeRecycler pvc deleted") waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeReleased) - glog.V(2).Infof("TestPersistentVolumeRecycler pv released") + klog.V(2).Infof("TestPersistentVolumeRecycler pv released") waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeAvailable) - glog.V(2).Infof("TestPersistentVolumeRecycler pv available") + klog.V(2).Infof("TestPersistentVolumeRecycler pv available") } func TestPersistentVolumeDeleter(t *testing.T) { - glog.V(2).Infof("TestPersistentVolumeDeleter started") + klog.V(2).Infof("TestPersistentVolumeDeleter started") _, s, closeFn := framework.RunAMaster(nil) defer closeFn() @@ -186,25 +186,25 @@ func TestPersistentVolumeDeleter(t *testing.T) { if err != nil { t.Errorf("Failed to create PersistentVolume: %v", err) } - glog.V(2).Infof("TestPersistentVolumeDeleter pv created") + klog.V(2).Infof("TestPersistentVolumeDeleter pv created") _, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(pvc) if err != nil { t.Errorf("Failed to create PersistentVolumeClaim: %v", err) } - glog.V(2).Infof("TestPersistentVolumeDeleter pvc created") + klog.V(2).Infof("TestPersistentVolumeDeleter pvc created") waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeBound) - glog.V(2).Infof("TestPersistentVolumeDeleter pv bound") + klog.V(2).Infof("TestPersistentVolumeDeleter pv bound") waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound) - glog.V(2).Infof("TestPersistentVolumeDeleter pvc bound") + klog.V(2).Infof("TestPersistentVolumeDeleter pvc bound") // deleting a claim releases the volume, after which it can be recycled if err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Delete(pvc.Name, nil); err != nil { t.Errorf("error deleting claim %s", pvc.Name) } - glog.V(2).Infof("TestPersistentVolumeDeleter pvc deleted") + klog.V(2).Infof("TestPersistentVolumeDeleter pvc deleted") waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeReleased) - glog.V(2).Infof("TestPersistentVolumeDeleter pv released") + klog.V(2).Infof("TestPersistentVolumeDeleter pv released") for { event := <-watchPV.ResultChan() @@ -212,13 +212,13 @@ func TestPersistentVolumeDeleter(t *testing.T) { break } } - glog.V(2).Infof("TestPersistentVolumeDeleter pv deleted") + klog.V(2).Infof("TestPersistentVolumeDeleter pv deleted") } func TestPersistentVolumeBindRace(t *testing.T) { // Test a race binding many claims to a PV that is pre-bound to a specific // PVC. Only this specific PVC should get bound. - glog.V(2).Infof("TestPersistentVolumeBindRace started") + klog.V(2).Infof("TestPersistentVolumeBindRace started") _, s, closeFn := framework.RunAMaster(nil) defer closeFn() @@ -253,7 +253,7 @@ func TestPersistentVolumeBindRace(t *testing.T) { } claims = append(claims, claim) } - glog.V(2).Infof("TestPersistentVolumeBindRace claims created") + klog.V(2).Infof("TestPersistentVolumeBindRace claims created") // putting a bind manually on a pv should only match the claim it is bound to claim := claims[rand.Intn(maxClaims-1)] @@ -268,12 +268,12 @@ func TestPersistentVolumeBindRace(t *testing.T) { if err != nil { t.Fatalf("Unexpected error creating pv: %v", err) } - glog.V(2).Infof("TestPersistentVolumeBindRace pv created, pre-bound to %s", claim.Name) + klog.V(2).Infof("TestPersistentVolumeBindRace pv created, pre-bound to %s", claim.Name) waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeBound) - glog.V(2).Infof("TestPersistentVolumeBindRace pv bound") + klog.V(2).Infof("TestPersistentVolumeBindRace pv bound") waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound) - glog.V(2).Infof("TestPersistentVolumeBindRace pvc bound") + klog.V(2).Infof("TestPersistentVolumeBindRace pvc bound") pv, err = testClient.CoreV1().PersistentVolumes().Get(pv.Name, metav1.GetOptions{}) if err != nil { @@ -590,7 +590,7 @@ func TestPersistentVolumeMultiPVsPVCs(t *testing.T) { } // Create PVs first - glog.V(2).Infof("TestPersistentVolumeMultiPVsPVCs: start") + klog.V(2).Infof("TestPersistentVolumeMultiPVsPVCs: start") // Create the volumes in a separate goroutine to pop events from // watchPV early - it seems it has limited capacity and it gets stuck @@ -603,9 +603,9 @@ func TestPersistentVolumeMultiPVsPVCs(t *testing.T) { // Wait for them to get Available for i := 0; i < objCount; i++ { waitForAnyPersistentVolumePhase(watchPV, v1.VolumeAvailable) - glog.V(1).Infof("%d volumes available", i+1) + klog.V(1).Infof("%d volumes available", i+1) } - glog.V(2).Infof("TestPersistentVolumeMultiPVsPVCs: volumes are Available") + klog.V(2).Infof("TestPersistentVolumeMultiPVsPVCs: volumes are Available") // Start a separate goroutine that randomly modifies PVs and PVCs while the // binder is working. We test that the binder can bind volumes despite @@ -622,7 +622,7 @@ func TestPersistentVolumeMultiPVsPVCs(t *testing.T) { if err != nil { // Silently ignore error, the PV may have be already deleted // or not exists yet. - glog.V(4).Infof("Failed to read PV %s: %v", name, err) + klog.V(4).Infof("Failed to read PV %s: %v", name, err) continue } if pv.Annotations == nil { @@ -634,10 +634,10 @@ func TestPersistentVolumeMultiPVsPVCs(t *testing.T) { if err != nil { // Silently ignore error, the PV may have been updated by // the controller. - glog.V(4).Infof("Failed to update PV %s: %v", pv.Name, err) + klog.V(4).Infof("Failed to update PV %s: %v", pv.Name, err) continue } - glog.V(4).Infof("Updated PV %s", pv.Name) + klog.V(4).Infof("Updated PV %s", pv.Name) } else { // Modify PVC i := rand.Intn(objCount) @@ -646,7 +646,7 @@ func TestPersistentVolumeMultiPVsPVCs(t *testing.T) { if err != nil { // Silently ignore error, the PVC may have be already // deleted or not exists yet. - glog.V(4).Infof("Failed to read PVC %s: %v", name, err) + klog.V(4).Infof("Failed to read PVC %s: %v", name, err) continue } if pvc.Annotations == nil { @@ -658,10 +658,10 @@ func TestPersistentVolumeMultiPVsPVCs(t *testing.T) { if err != nil { // Silently ignore error, the PVC may have been updated by // the controller. - glog.V(4).Infof("Failed to update PVC %s: %v", pvc.Name, err) + klog.V(4).Infof("Failed to update PVC %s: %v", pvc.Name, err) continue } - glog.V(4).Infof("Updated PVC %s", pvc.Name) + klog.V(4).Infof("Updated PVC %s", pvc.Name) } select { @@ -684,15 +684,15 @@ func TestPersistentVolumeMultiPVsPVCs(t *testing.T) { // wait until the binder pairs all claims for i := 0; i < objCount; i++ { waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound) - glog.V(1).Infof("%d claims bound", i+1) + klog.V(1).Infof("%d claims bound", i+1) } // wait until the binder pairs all volumes for i := 0; i < objCount; i++ { waitForPersistentVolumePhase(testClient, pvs[i].Name, watchPV, v1.VolumeBound) - glog.V(1).Infof("%d claims bound", i+1) + klog.V(1).Infof("%d claims bound", i+1) } - glog.V(2).Infof("TestPersistentVolumeMultiPVsPVCs: claims are bound") + klog.V(2).Infof("TestPersistentVolumeMultiPVsPVCs: claims are bound") stopCh <- struct{}{} // check that everything is bound to something @@ -704,7 +704,7 @@ func TestPersistentVolumeMultiPVsPVCs(t *testing.T) { if pv.Spec.ClaimRef == nil { t.Fatalf("PV %q is not bound", pv.Name) } - glog.V(2).Infof("PV %q is bound to PVC %q", pv.Name, pv.Spec.ClaimRef.Name) + klog.V(2).Infof("PV %q is bound to PVC %q", pv.Name, pv.Spec.ClaimRef.Name) pvc, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Get(pvcs[i].Name, metav1.GetOptions{}) if err != nil { @@ -713,7 +713,7 @@ func TestPersistentVolumeMultiPVsPVCs(t *testing.T) { if pvc.Spec.VolumeName == "" { t.Fatalf("PVC %q is not bound", pvc.Name) } - glog.V(2).Infof("PVC %q is bound to PV %q", pvc.Name, pvc.Spec.VolumeName) + klog.V(2).Infof("PVC %q is bound to PV %q", pvc.Name, pvc.Spec.VolumeName) } testSleep() } @@ -766,7 +766,7 @@ func TestPersistentVolumeControllerStartup(t *testing.T) { []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRetain) claimRef, err := ref.GetReference(legacyscheme.Scheme, newPVC) if err != nil { - glog.V(3).Infof("unexpected error getting claim reference: %v", err) + klog.V(3).Infof("unexpected error getting claim reference: %v", err) return } pv.Spec.ClaimRef = claimRef @@ -820,7 +820,7 @@ func TestPersistentVolumeControllerStartup(t *testing.T) { case <-timer.C: // Wait finished - glog.V(2).Infof("Wait finished") + klog.V(2).Infof("Wait finished") finished = true } } @@ -834,7 +834,7 @@ func TestPersistentVolumeControllerStartup(t *testing.T) { if pv.Spec.ClaimRef == nil { t.Fatalf("PV %q is not bound", pv.Name) } - glog.V(2).Infof("PV %q is bound to PVC %q", pv.Name, pv.Spec.ClaimRef.Name) + klog.V(2).Infof("PV %q is bound to PVC %q", pv.Name, pv.Spec.ClaimRef.Name) pvc, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Get(pvcs[i].Name, metav1.GetOptions{}) if err != nil { @@ -843,7 +843,7 @@ func TestPersistentVolumeControllerStartup(t *testing.T) { if pvc.Spec.VolumeName == "" { t.Fatalf("PVC %q is not bound", pvc.Name) } - glog.V(2).Infof("PVC %q is bound to PV %q", pvc.Name, pvc.Spec.VolumeName) + klog.V(2).Infof("PVC %q is bound to PV %q", pvc.Name, pvc.Spec.VolumeName) } } @@ -888,7 +888,7 @@ func TestPersistentVolumeProvisionMultiPVCs(t *testing.T) { pvcs[i] = pvc } - glog.V(2).Infof("TestPersistentVolumeProvisionMultiPVCs: start") + klog.V(2).Infof("TestPersistentVolumeProvisionMultiPVCs: start") // Create the claims in a separate goroutine to pop events from watchPVC // early. It gets stuck with >3000 claims. go func() { @@ -900,9 +900,9 @@ func TestPersistentVolumeProvisionMultiPVCs(t *testing.T) { // Wait until the controller provisions and binds all of them for i := 0; i < objCount; i++ { waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound) - glog.V(1).Infof("%d claims bound", i+1) + klog.V(1).Infof("%d claims bound", i+1) } - glog.V(2).Infof("TestPersistentVolumeProvisionMultiPVCs: claims are bound") + klog.V(2).Infof("TestPersistentVolumeProvisionMultiPVCs: claims are bound") // check that we have enough bound PVs pvList, err := testClient.CoreV1().PersistentVolumes().List(metav1.ListOptions{}) @@ -917,7 +917,7 @@ func TestPersistentVolumeProvisionMultiPVCs(t *testing.T) { if pv.Status.Phase != v1.VolumeBound { t.Fatalf("Expected volume %s to be bound, is %s instead", pv.Name, pv.Status.Phase) } - glog.V(2).Infof("PV %q is bound to PVC %q", pv.Name, pv.Spec.ClaimRef.Name) + klog.V(2).Infof("PV %q is bound to PVC %q", pv.Name, pv.Spec.ClaimRef.Name) } // Delete the claims @@ -933,13 +933,13 @@ func TestPersistentVolumeProvisionMultiPVCs(t *testing.T) { t.Fatalf("Failed to list volumes: %v", err) } - glog.V(1).Infof("%d volumes remaining", len(volumes.Items)) + klog.V(1).Infof("%d volumes remaining", len(volumes.Items)) if len(volumes.Items) == 0 { break } time.Sleep(time.Second) } - glog.V(2).Infof("TestPersistentVolumeProvisionMultiPVCs: volumes are deleted") + klog.V(2).Infof("TestPersistentVolumeProvisionMultiPVCs: volumes are deleted") } // TestPersistentVolumeMultiPVsDiffAccessModes tests binding of one PVC to two @@ -1038,7 +1038,7 @@ func waitForPersistentVolumePhase(client *clientset.Clientset, pvName string, w continue } if volume.Status.Phase == phase && volume.Name == pvName { - glog.V(2).Infof("volume %q is %s", volume.Name, phase) + klog.V(2).Infof("volume %q is %s", volume.Name, phase) break } } @@ -1059,7 +1059,7 @@ func waitForPersistentVolumeClaimPhase(client *clientset.Clientset, claimName, n continue } if claim.Status.Phase == phase && claim.Name == claimName { - glog.V(2).Infof("claim %q is %s", claim.Name, phase) + klog.V(2).Infof("claim %q is %s", claim.Name, phase) break } } @@ -1073,7 +1073,7 @@ func waitForAnyPersistentVolumePhase(w watch.Interface, phase v1.PersistentVolum continue } if volume.Status.Phase == phase { - glog.V(2).Infof("volume %q is %s", volume.Name, phase) + klog.V(2).Infof("volume %q is %s", volume.Name, phase) break } } @@ -1087,7 +1087,7 @@ func waitForAnyPersistentVolumeClaimPhase(w watch.Interface, phase v1.Persistent continue } if claim.Status.Phase == phase { - glog.V(2).Infof("claim %q is %s", claim.Name, phase) + klog.V(2).Infof("claim %q is %s", claim.Name, phase) break } } diff --git a/test/soak/cauldron/BUILD b/test/soak/cauldron/BUILD index 84a81a97aca..dbf02e64b42 100644 --- a/test/soak/cauldron/BUILD +++ b/test/soak/cauldron/BUILD @@ -23,7 +23,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//test/e2e/framework:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/soak/cauldron/cauldron.go b/test/soak/cauldron/cauldron.go index 867e88c5554..63173667841 100644 --- a/test/soak/cauldron/cauldron.go +++ b/test/soak/cauldron/cauldron.go @@ -29,11 +29,11 @@ import ( "net/http" "time" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" restclient "k8s.io/client-go/rest" + "k8s.io/klog" api "k8s.io/kubernetes/pkg/apis/core" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/test/e2e/framework" @@ -59,17 +59,17 @@ const ( func main() { flag.Parse() - glog.Infof("Starting cauldron soak test with queries=%d podsPerNode=%d upTo=%d maxPar=%d", + klog.Infof("Starting cauldron soak test with queries=%d podsPerNode=%d upTo=%d maxPar=%d", *queriesAverage, *podsPerNode, *upTo, *maxPar) cc, err := restclient.InClusterConfig() if err != nil { - glog.Fatalf("Failed to make client: %v", err) + klog.Fatalf("Failed to make client: %v", err) } client, err := clientset.NewForConfig(cc) if err != nil { - glog.Fatalf("Failed to make client: %v", err) + klog.Fatalf("Failed to make client: %v", err) } var nodes *api.NodeList @@ -78,19 +78,19 @@ func main() { if err == nil { break } - glog.Warningf("Failed to list nodes: %v", err) + klog.Warningf("Failed to list nodes: %v", err) } if err != nil { - glog.Fatalf("Giving up trying to list nodes: %v", err) + klog.Fatalf("Giving up trying to list nodes: %v", err) } if len(nodes.Items) == 0 { - glog.Fatalf("Failed to find any nodes.") + klog.Fatalf("Failed to find any nodes.") } - glog.Infof("Found %d nodes on this cluster:", len(nodes.Items)) + klog.Infof("Found %d nodes on this cluster:", len(nodes.Items)) for i, node := range nodes.Items { - glog.Infof("%d: %s", i, node.Name) + klog.Infof("%d: %s", i, node.Name) } queries := *queriesAverage * len(nodes.Items) * *podsPerNode @@ -98,12 +98,12 @@ func main() { // Create a uniquely named namespace. got, err := client.Core().Namespaces().Create(&api.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "serve-hostnames-"}}) if err != nil { - glog.Fatalf("Failed to create namespace: %v", err) + klog.Fatalf("Failed to create namespace: %v", err) } ns := got.Name defer func(ns string) { if err := client.Core().Namespaces().Delete(ns, nil); err != nil { - glog.Warningf("Failed to delete namespace %s: %v", ns, err) + klog.Warningf("Failed to delete namespace %s: %v", ns, err) } else { // wait until the namespace disappears for i := 0; i < int(namespaceDeleteTimeout/time.Second); i++ { @@ -116,10 +116,10 @@ func main() { } } }(ns) - glog.Infof("Created namespace %s", ns) + klog.Infof("Created namespace %s", ns) // Create a service for these pods. - glog.Infof("Creating service %s/serve-hostnames", ns) + klog.Infof("Creating service %s/serve-hostnames", ns) // Make several attempts to create a service. var svc *api.Service for start := time.Now(); time.Since(start) < serviceCreateTimeout; time.Sleep(2 * time.Second) { @@ -142,25 +142,25 @@ func main() { }, }, }) - glog.V(4).Infof("Service create %s/server-hostnames took %v", ns, time.Since(t)) + klog.V(4).Infof("Service create %s/server-hostnames took %v", ns, time.Since(t)) if err == nil { break } - glog.Warningf("After %v failed to create service %s/serve-hostnames: %v", time.Since(start), ns, err) + klog.Warningf("After %v failed to create service %s/serve-hostnames: %v", time.Since(start), ns, err) } if err != nil { - glog.Warningf("Unable to create service %s/%s: %v", ns, svc.Name, err) + klog.Warningf("Unable to create service %s/%s: %v", ns, svc.Name, err) return } // Clean up service defer func() { - glog.Infof("Cleaning up service %s/serve-hostnames", ns) + klog.Infof("Cleaning up service %s/serve-hostnames", ns) // Make several attempts to delete the service. for start := time.Now(); time.Since(start) < deleteTimeout; time.Sleep(1 * time.Second) { if err := client.Core().Services(ns).Delete(svc.Name, nil); err == nil { return } - glog.Warningf("After %v unable to delete service %s/%s: %v", time.Since(start), ns, svc.Name, err) + klog.Warningf("After %v unable to delete service %s/%s: %v", time.Since(start), ns, svc.Name, err) } }() @@ -172,7 +172,7 @@ func main() { podNames = append(podNames, podName) // Make several attempts for start := time.Now(); time.Since(start) < podCreateTimeout; time.Sleep(2 * time.Second) { - glog.Infof("Creating pod %s/%s on node %s", ns, podName, node.Name) + klog.Infof("Creating pod %s/%s on node %s", ns, podName, node.Name) t := time.Now() _, err = client.Core().Pods(ns).Create(&api.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -192,39 +192,39 @@ func main() { NodeName: node.Name, }, }) - glog.V(4).Infof("Pod create %s/%s request took %v", ns, podName, time.Since(t)) + klog.V(4).Infof("Pod create %s/%s request took %v", ns, podName, time.Since(t)) if err == nil { break } - glog.Warningf("After %s failed to create pod %s/%s: %v", time.Since(start), ns, podName, err) + klog.Warningf("After %s failed to create pod %s/%s: %v", time.Since(start), ns, podName, err) } if err != nil { - glog.Warningf("Failed to create pod %s/%s: %v", ns, podName, err) + klog.Warningf("Failed to create pod %s/%s: %v", ns, podName, err) return } } } // Clean up the pods defer func() { - glog.Info("Cleaning up pods") + klog.Info("Cleaning up pods") // Make several attempts to delete the pods. for _, podName := range podNames { for start := time.Now(); time.Since(start) < deleteTimeout; time.Sleep(1 * time.Second) { if err = client.Core().Pods(ns).Delete(podName, nil); err == nil { break } - glog.Warningf("After %v failed to delete pod %s/%s: %v", time.Since(start), ns, podName, err) + klog.Warningf("After %v failed to delete pod %s/%s: %v", time.Since(start), ns, podName, err) } } }() - glog.Info("Waiting for the serve-hostname pods to be ready") + klog.Info("Waiting for the serve-hostname pods to be ready") for _, podName := range podNames { var pod *api.Pod for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) { pod, err = client.Core().Pods(ns).Get(podName, metav1.GetOptions{}) if err != nil { - glog.Warningf("Get pod %s/%s failed, ignoring for %v: %v", ns, podName, err, podStartTimeout) + klog.Warningf("Get pod %s/%s failed, ignoring for %v: %v", ns, podName, err, podStartTimeout) continue } if pod.Status.Phase == api.PodRunning { @@ -232,9 +232,9 @@ func main() { } } if pod.Status.Phase != api.PodRunning { - glog.Warningf("Gave up waiting on pod %s/%s to be running (saw %v)", ns, podName, pod.Status.Phase) + klog.Warningf("Gave up waiting on pod %s/%s to be running (saw %v)", ns, podName, pod.Status.Phase) } else { - glog.Infof("%s/%s is running", ns, podName) + klog.Infof("%s/%s is running", ns, podName) } } @@ -244,10 +244,10 @@ func main() { if err == nil { break } - glog.Infof("After %v while making a request got error %v", time.Since(start), err) + klog.Infof("After %v while making a request got error %v", time.Since(start), err) } if err != nil { - glog.Errorf("Failed to get a response from service: %v", err) + klog.Errorf("Failed to get a response from service: %v", err) } // Repeatedly make requests. @@ -262,9 +262,9 @@ func main() { inFlight <- struct{}{} t := time.Now() resp, err := http.Get(fmt.Sprintf("http://serve-hostnames.%s:9376", ns)) - glog.V(4).Infof("Call to serve-hostnames in namespace %s took %v", ns, time.Since(t)) + klog.V(4).Infof("Call to serve-hostnames in namespace %s took %v", ns, time.Since(t)) if err != nil { - glog.Warningf("Call failed during iteration %d query %d : %v", i, query, err) + klog.Warningf("Call failed during iteration %d query %d : %v", i, query, err) // If the query failed return a string which starts with a character // that can't be part of a hostname. responseChan <- fmt.Sprintf("!failed in iteration %d to issue query %d: %v", i, query, err) @@ -284,28 +284,28 @@ func main() { missing := 0 for q := 0; q < queries; q++ { r := <-responseChan - glog.V(4).Infof("Got response from %s", r) + klog.V(4).Infof("Got response from %s", r) responses[r]++ // If the returned hostname starts with '!' then it indicates // an error response. if len(r) > 0 && r[0] == '!' { - glog.V(3).Infof("Got response %s", r) + klog.V(3).Infof("Got response %s", r) missing++ } } if missing > 0 { - glog.Warningf("Missing %d responses out of %d", missing, queries) + klog.Warningf("Missing %d responses out of %d", missing, queries) } // Report any nodes that did not respond. for n, node := range nodes.Items { for i := 0; i < *podsPerNode; i++ { name := fmt.Sprintf("serve-hostname-%d-%d", n, i) if _, ok := responses[name]; !ok { - glog.Warningf("No response from pod %s on node %s at iteration %d", name, node.Name, iteration) + klog.Warningf("No response from pod %s on node %s at iteration %d", name, node.Name, iteration) } } } - glog.Infof("Iteration %d took %v for %d queries (%.2f QPS) with %d missing", + klog.Infof("Iteration %d took %v for %d queries (%.2f QPS) with %d missing", iteration, time.Since(start), queries-missing, float64(queries-missing)/time.Since(start).Seconds(), missing) } } diff --git a/test/soak/serve_hostnames/BUILD b/test/soak/serve_hostnames/BUILD index e5bbf5764f9..8fb14520d90 100644 --- a/test/soak/serve_hostnames/BUILD +++ b/test/soak/serve_hostnames/BUILD @@ -26,7 +26,7 @@ go_library( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//test/e2e/framework:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/soak/serve_hostnames/serve_hostnames.go b/test/soak/serve_hostnames/serve_hostnames.go index 92b48bf5512..c3ee9dbc360 100644 --- a/test/soak/serve_hostnames/serve_hostnames.go +++ b/test/soak/serve_hostnames/serve_hostnames.go @@ -40,7 +40,7 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" e2e "k8s.io/kubernetes/test/e2e/framework" - "github.com/golang/glog" + "k8s.io/klog" ) var ( @@ -64,7 +64,7 @@ const ( func main() { flag.Parse() - glog.Infof("Starting serve_hostnames soak test with queries=%d and podsPerNode=%d upTo=%d", + klog.Infof("Starting serve_hostnames soak test with queries=%d and podsPerNode=%d upTo=%d", *queriesAverage, *podsPerNode, *upTo) var spec string @@ -75,19 +75,19 @@ func main() { } settings, err := clientcmd.LoadFromFile(spec) if err != nil { - glog.Fatalf("Error loading configuration: %v", err.Error()) + klog.Fatalf("Error loading configuration: %v", err.Error()) } if *gke != "" { settings.CurrentContext = *gke } config, err := clientcmd.NewDefaultClientConfig(*settings, &clientcmd.ConfigOverrides{}).ClientConfig() if err != nil { - glog.Fatalf("Failed to construct config: %v", err) + klog.Fatalf("Failed to construct config: %v", err) } client, err := clientset.NewForConfig(config) if err != nil { - glog.Fatalf("Failed to make client: %v", err) + klog.Fatalf("Failed to make client: %v", err) } var nodes *v1.NodeList @@ -96,19 +96,19 @@ func main() { if err == nil { break } - glog.Warningf("Failed to list nodes: %v", err) + klog.Warningf("Failed to list nodes: %v", err) } if err != nil { - glog.Fatalf("Giving up trying to list nodes: %v", err) + klog.Fatalf("Giving up trying to list nodes: %v", err) } if len(nodes.Items) == 0 { - glog.Fatalf("Failed to find any nodes.") + klog.Fatalf("Failed to find any nodes.") } - glog.Infof("Found %d nodes on this cluster:", len(nodes.Items)) + klog.Infof("Found %d nodes on this cluster:", len(nodes.Items)) for i, node := range nodes.Items { - glog.Infof("%d: %s", i, node.Name) + klog.Infof("%d: %s", i, node.Name) } queries := *queriesAverage * len(nodes.Items) * *podsPerNode @@ -116,12 +116,12 @@ func main() { // Create the namespace got, err := client.CoreV1().Namespaces().Create(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "serve-hostnames-"}}) if err != nil { - glog.Fatalf("Failed to create namespace: %v", err) + klog.Fatalf("Failed to create namespace: %v", err) } ns := got.Name defer func(ns string) { if err := client.CoreV1().Namespaces().Delete(ns, nil); err != nil { - glog.Warningf("Failed to delete namespace %s: %v", ns, err) + klog.Warningf("Failed to delete namespace %s: %v", ns, err) } else { // wait until the namespace disappears for i := 0; i < int(namespaceDeleteTimeout/time.Second); i++ { @@ -134,10 +134,10 @@ func main() { } } }(ns) - glog.Infof("Created namespace %s", ns) + klog.Infof("Created namespace %s", ns) // Create a service for these pods. - glog.Infof("Creating service %s/serve-hostnames", ns) + klog.Infof("Creating service %s/serve-hostnames", ns) // Make several attempts to create a service. var svc *v1.Service for start := time.Now(); time.Since(start) < serviceCreateTimeout; time.Sleep(2 * time.Second) { @@ -160,25 +160,25 @@ func main() { }, }, }) - glog.V(4).Infof("Service create %s/server-hostnames took %v", ns, time.Since(t)) + klog.V(4).Infof("Service create %s/server-hostnames took %v", ns, time.Since(t)) if err == nil { break } - glog.Warningf("After %v failed to create service %s/serve-hostnames: %v", time.Since(start), ns, err) + klog.Warningf("After %v failed to create service %s/serve-hostnames: %v", time.Since(start), ns, err) } if err != nil { - glog.Warningf("Unable to create service %s/%s: %v", ns, svc.Name, err) + klog.Warningf("Unable to create service %s/%s: %v", ns, svc.Name, err) return } // Clean up service defer func() { - glog.Infof("Cleaning up service %s/serve-hostnames", ns) + klog.Infof("Cleaning up service %s/serve-hostnames", ns) // Make several attempts to delete the service. for start := time.Now(); time.Since(start) < deleteTimeout; time.Sleep(1 * time.Second) { if err := client.CoreV1().Services(ns).Delete(svc.Name, nil); err == nil { return } - glog.Warningf("After %v unable to delete service %s/%s: %v", time.Since(start), ns, svc.Name, err) + klog.Warningf("After %v unable to delete service %s/%s: %v", time.Since(start), ns, svc.Name, err) } }() @@ -190,7 +190,7 @@ func main() { podNames = append(podNames, podName) // Make several attempts for start := time.Now(); time.Since(start) < podCreateTimeout; time.Sleep(2 * time.Second) { - glog.Infof("Creating pod %s/%s on node %s", ns, podName, node.Name) + klog.Infof("Creating pod %s/%s on node %s", ns, podName, node.Name) t := time.Now() _, err = client.CoreV1().Pods(ns).Create(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -210,39 +210,39 @@ func main() { NodeName: node.Name, }, }) - glog.V(4).Infof("Pod create %s/%s request took %v", ns, podName, time.Since(t)) + klog.V(4).Infof("Pod create %s/%s request took %v", ns, podName, time.Since(t)) if err == nil { break } - glog.Warningf("After %s failed to create pod %s/%s: %v", time.Since(start), ns, podName, err) + klog.Warningf("After %s failed to create pod %s/%s: %v", time.Since(start), ns, podName, err) } if err != nil { - glog.Warningf("Failed to create pod %s/%s: %v", ns, podName, err) + klog.Warningf("Failed to create pod %s/%s: %v", ns, podName, err) return } } } // Clean up the pods defer func() { - glog.Info("Cleaning up pods") + klog.Info("Cleaning up pods") // Make several attempts to delete the pods. for _, podName := range podNames { for start := time.Now(); time.Since(start) < deleteTimeout; time.Sleep(1 * time.Second) { if err = client.CoreV1().Pods(ns).Delete(podName, nil); err == nil { break } - glog.Warningf("After %v failed to delete pod %s/%s: %v", time.Since(start), ns, podName, err) + klog.Warningf("After %v failed to delete pod %s/%s: %v", time.Since(start), ns, podName, err) } } }() - glog.Info("Waiting for the serve-hostname pods to be ready") + klog.Info("Waiting for the serve-hostname pods to be ready") for _, podName := range podNames { var pod *v1.Pod for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) { pod, err = client.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{}) if err != nil { - glog.Warningf("Get pod %s/%s failed, ignoring for %v: %v", ns, podName, err, podStartTimeout) + klog.Warningf("Get pod %s/%s failed, ignoring for %v: %v", ns, podName, err, podStartTimeout) continue } if pod.Status.Phase == v1.PodRunning { @@ -250,20 +250,20 @@ func main() { } } if pod.Status.Phase != v1.PodRunning { - glog.Warningf("Gave up waiting on pod %s/%s to be running (saw %v)", ns, podName, pod.Status.Phase) + klog.Warningf("Gave up waiting on pod %s/%s to be running (saw %v)", ns, podName, pod.Status.Phase) } else { - glog.Infof("%s/%s is running", ns, podName) + klog.Infof("%s/%s is running", ns, podName) } } rclient, err := restclient.RESTClientFor(config) if err != nil { - glog.Warningf("Failed to build restclient: %v", err) + klog.Warningf("Failed to build restclient: %v", err) return } proxyRequest, errProxy := e2e.GetServicesProxyRequest(client, rclient.Get()) if errProxy != nil { - glog.Warningf("Get services proxy request failed: %v", errProxy) + klog.Warningf("Get services proxy request failed: %v", errProxy) return } @@ -274,7 +274,7 @@ func main() { Name("serve-hostnames"). DoRaw() if err != nil { - glog.Infof("After %v while making a proxy call got error %v", time.Since(start), err) + klog.Infof("After %v while making a proxy call got error %v", time.Since(start), err) continue } var r metav1.Status @@ -282,7 +282,7 @@ func main() { break } if r.Status == metav1.StatusFailure { - glog.Infof("After %v got status %v", time.Since(start), string(hostname)) + klog.Infof("After %v got status %v", time.Since(start), string(hostname)) continue } break @@ -303,9 +303,9 @@ func main() { Namespace(ns). Name("serve-hostnames"). DoRaw() - glog.V(4).Infof("Proxy call in namespace %s took %v", ns, time.Since(t)) + klog.V(4).Infof("Proxy call in namespace %s took %v", ns, time.Since(t)) if err != nil { - glog.Warningf("Call failed during iteration %d query %d : %v", i, query, err) + klog.Warningf("Call failed during iteration %d query %d : %v", i, query, err) // If the query failed return a string which starts with a character // that can't be part of a hostname. responseChan <- fmt.Sprintf("!failed in iteration %d to issue query %d: %v", i, query, err) @@ -319,28 +319,28 @@ func main() { missing := 0 for q := 0; q < queries; q++ { r := <-responseChan - glog.V(4).Infof("Got response from %s", r) + klog.V(4).Infof("Got response from %s", r) responses[r]++ // If the returned hostname starts with '!' then it indicates // an error response. if len(r) > 0 && r[0] == '!' { - glog.V(3).Infof("Got response %s", r) + klog.V(3).Infof("Got response %s", r) missing++ } } if missing > 0 { - glog.Warningf("Missing %d responses out of %d", missing, queries) + klog.Warningf("Missing %d responses out of %d", missing, queries) } // Report any nodes that did not respond. for n, node := range nodes.Items { for i := 0; i < *podsPerNode; i++ { name := fmt.Sprintf("serve-hostname-%d-%d", n, i) if _, ok := responses[name]; !ok { - glog.Warningf("No response from pod %s on node %s at iteration %d", name, node.Name, iteration) + klog.Warningf("No response from pod %s on node %s at iteration %d", name, node.Name, iteration) } } } - glog.Infof("Iteration %d took %v for %d queries (%.2f QPS) with %d missing", + klog.Infof("Iteration %d took %v for %d queries (%.2f QPS) with %d missing", iteration, time.Since(start), queries-missing, float64(queries-missing)/time.Since(start).Seconds(), missing) } } diff --git a/test/utils/BUILD b/test/utils/BUILD index f594b7ba144..82a0722eeba 100644 --- a/test/utils/BUILD +++ b/test/utils/BUILD @@ -57,7 +57,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/test/utils/density_utils.go b/test/utils/density_utils.go index c607e5c940e..60a32d50c19 100644 --- a/test/utils/density_utils.go +++ b/test/utils/density_utils.go @@ -21,12 +21,12 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" ) const ( @@ -79,7 +79,7 @@ func RemoveLabelOffNode(c clientset.Interface, nodeName string, labelKeys []stri if !apierrs.IsConflict(err) { return err } else { - glog.V(2).Infof("Conflict when trying to remove a labels %v from %v", labelKeys, nodeName) + klog.V(2).Infof("Conflict when trying to remove a labels %v from %v", labelKeys, nodeName) } } else { break diff --git a/test/utils/harness/BUILD b/test/utils/harness/BUILD index 510a8127132..853e519b0bd 100644 --- a/test/utils/harness/BUILD +++ b/test/utils/harness/BUILD @@ -5,7 +5,7 @@ go_library( srcs = ["harness.go"], importpath = "k8s.io/kubernetes/test/utils/harness", visibility = ["//visibility:public"], - deps = ["//vendor/github.com/golang/glog:go_default_library"], + deps = ["//vendor/k8s.io/klog:go_default_library"], ) filegroup( diff --git a/test/utils/harness/harness.go b/test/utils/harness/harness.go index a3c827318f3..46e65da006a 100644 --- a/test/utils/harness/harness.go +++ b/test/utils/harness/harness.go @@ -21,7 +21,7 @@ import ( "os" "testing" - "github.com/golang/glog" + "k8s.io/klog" ) // Harness adds some functionality to testing.T, in particular resource cleanup. @@ -51,7 +51,7 @@ func For(t *testing.T) *Harness { func (h *Harness) Close() { for _, d := range h.defers { if err := d(); err != nil { - glog.Warningf("error closing harness: %v", err) + klog.Warningf("error closing harness: %v", err) } } } diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index 4e83ca47a0e..2e5015bd3a7 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -92,6 +92,7 @@ var ( // Preconfigured image configs var ( + CRDConversionWebhook = Config{e2eRegistry, "crd-conversion-webhook", "1.13rev2"} AdmissionWebhook = Config{e2eRegistry, "webhook", "1.13v1"} APIServer = Config{e2eRegistry, "sample-apiserver", "1.10"} AppArmorLoader = Config{e2eRegistry, "apparmor-loader", "1.0"} diff --git a/test/utils/runners.go b/test/utils/runners.go index 63035325031..4ebfde80e13 100644 --- a/test/utils/runners.go +++ b/test/utils/runners.go @@ -21,6 +21,7 @@ import ( "fmt" "math" "os" + "strings" "sync" "time" @@ -46,7 +47,7 @@ import ( extensionsinternal "k8s.io/kubernetes/pkg/apis/extensions" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -162,7 +163,7 @@ type RCConfig struct { // If set to false starting RC will print progress, otherwise only errors will be printed. Silent bool - // If set this function will be used to print log lines instead of glog. + // If set this function will be used to print log lines instead of klog. LogFunc func(fmt string, args ...interface{}) // If set those functions will be used to gather data from Nodes - in integration tests where no // kubelets are running those variables should be nil. @@ -180,7 +181,7 @@ func (rc *RCConfig) RCConfigLog(fmt string, args ...interface{}) { if rc.LogFunc != nil { rc.LogFunc(fmt, args...) } - glog.Infof(fmt, args...) + klog.Infof(fmt, args...) } type DeploymentConfig struct { @@ -242,6 +243,18 @@ func (p PodDiff) String(ignorePhases sets.String) string { return ret } +// DeletedPods returns a slice of pods that were present at the beginning +// and then disappeared. +func (p PodDiff) DeletedPods() []string { + var deletedPods []string + for podName, podInfo := range p { + if podInfo.hostname == nonExist { + deletedPods = append(deletedPods, podName) + } + } + return deletedPods +} + // Diff computes a PodDiff given 2 lists of pods. func Diff(oldPods []*v1.Pod, curPods []*v1.Pod) PodDiff { podInfoMap := PodDiff{} @@ -765,9 +778,8 @@ func (config *RCConfig) start() error { pods := ps.List() startupStatus := ComputeRCStartupStatus(pods, config.Replicas) - pods = startupStatus.Created if config.CreatedPods != nil { - *config.CreatedPods = pods + *config.CreatedPods = startupStatus.Created } if !config.Silent { config.RCConfigLog(startupStatus.String(config.Name)) @@ -787,16 +799,15 @@ func (config *RCConfig) start() error { } return fmt.Errorf("%d containers failed which is more than allowed %d", startupStatus.FailedContainers, maxContainerFailures) } - if len(pods) < len(oldPods) || len(pods) > config.Replicas { - // This failure mode includes: - // kubelet is dead, so node controller deleted pods and rc creates more - // - diagnose by noting the pod diff below. - // pod is unhealthy, so replication controller creates another to take its place - // - diagnose by comparing the previous "2 Pod states" lines for inactive pods - errorStr := fmt.Sprintf("Number of reported pods for %s changed: %d vs %d", config.Name, len(pods), len(oldPods)) - config.RCConfigLog("%v, pods that changed since the last iteration:", errorStr) - config.RCConfigLog(Diff(oldPods, pods).String(sets.NewString())) - return fmt.Errorf(errorStr) + + diff := Diff(oldPods, pods) + deletedPods := diff.DeletedPods() + if len(deletedPods) != 0 { + // There are some pods that have disappeared. + err := fmt.Errorf("%d pods disappeared for %s: %v", len(deletedPods), config.Name, strings.Join(deletedPods, ", ")) + config.RCConfigLog(err.Error()) + config.RCConfigLog(diff.String(sets.NewString())) + return err } if len(pods) > len(oldPods) || startupStatus.Running > oldRunning { @@ -1134,7 +1145,7 @@ type SecretConfig struct { Client clientset.Interface Name string Namespace string - // If set this function will be used to print log lines instead of glog. + // If set this function will be used to print log lines instead of klog. LogFunc func(fmt string, args ...interface{}) } @@ -1192,7 +1203,7 @@ type ConfigMapConfig struct { Client clientset.Interface Name string Namespace string - // If set this function will be used to print log lines instead of glog. + // If set this function will be used to print log lines instead of klog. LogFunc func(fmt string, args ...interface{}) } @@ -1303,7 +1314,7 @@ type DaemonConfig struct { Name string Namespace string Image string - // If set this function will be used to print log lines instead of glog. + // If set this function will be used to print log lines instead of klog. LogFunc func(fmt string, args ...interface{}) // How long we wait for DaemonSet to become running. Timeout time.Duration diff --git a/test/utils/tmpdir.go b/test/utils/tmpdir.go index 51cf19f1c1e..0c84c74d7ad 100644 --- a/test/utils/tmpdir.go +++ b/test/utils/tmpdir.go @@ -19,7 +19,7 @@ package utils import ( "io/ioutil" - "github.com/golang/glog" + "k8s.io/klog" ) func MakeTempDirOrDie(prefix string, baseDir string) string { @@ -28,7 +28,7 @@ func MakeTempDirOrDie(prefix string, baseDir string) string { } tempDir, err := ioutil.TempDir(baseDir, prefix) if err != nil { - glog.Fatalf("Can't make a temp rootdir: %v", err) + klog.Fatalf("Can't make a temp rootdir: %v", err) } return tempDir } diff --git a/third_party/BUILD b/third_party/BUILD index ce2dc0c079b..3650e5acd40 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -22,6 +22,7 @@ filegroup( "//third_party/forked/etcd221/wal:all-srcs", "//third_party/forked/etcd237/pkg/fileutil:all-srcs", "//third_party/forked/etcd237/wal:all-srcs", + "//third_party/forked/godep:all-srcs", "//third_party/forked/golang/expansion:all-srcs", "//third_party/forked/golang/reflect:all-srcs", "//third_party/forked/golang/template:all-srcs", diff --git a/vendor/github.com/tools/godep/.gitignore b/third_party/forked/godep/.gitignore similarity index 100% rename from vendor/github.com/tools/godep/.gitignore rename to third_party/forked/godep/.gitignore diff --git a/vendor/github.com/tools/godep/.travis.yml b/third_party/forked/godep/.travis.yml similarity index 100% rename from vendor/github.com/tools/godep/.travis.yml rename to third_party/forked/godep/.travis.yml diff --git a/vendor/github.com/tools/godep/BUILD b/third_party/forked/godep/BUILD similarity index 91% rename from vendor/github.com/tools/godep/BUILD rename to third_party/forked/godep/BUILD index 472fcf9e371..04bb53d4c9e 100644 --- a/vendor/github.com/tools/godep/BUILD +++ b/third_party/forked/godep/BUILD @@ -1,5 +1,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +licenses(["notice"]) + go_library( name = "go_default_library", srcs = [ @@ -24,8 +26,7 @@ go_library( "vcs.go", "version.go", ], - importmap = "k8s.io/kubernetes/vendor/github.com/tools/godep", - importpath = "github.com/tools/godep", + importpath = "k8s.io/kubernetes/third_party/forked/godep", visibility = ["//visibility:private"], deps = [ "//vendor/github.com/kr/fs:go_default_library", diff --git a/vendor/github.com/tools/godep/Changelog.md b/third_party/forked/godep/Changelog.md similarity index 100% rename from vendor/github.com/tools/godep/Changelog.md rename to third_party/forked/godep/Changelog.md diff --git a/vendor/github.com/tools/godep/FAQ.md b/third_party/forked/godep/FAQ.md similarity index 100% rename from vendor/github.com/tools/godep/FAQ.md rename to third_party/forked/godep/FAQ.md diff --git a/vendor/github.com/tools/godep/License b/third_party/forked/godep/License similarity index 100% rename from vendor/github.com/tools/godep/License rename to third_party/forked/godep/License diff --git a/vendor/github.com/tools/godep/Readme.md b/third_party/forked/godep/Readme.md similarity index 100% rename from vendor/github.com/tools/godep/Readme.md rename to third_party/forked/godep/Readme.md diff --git a/vendor/github.com/tools/godep/dep.go b/third_party/forked/godep/dep.go similarity index 100% rename from vendor/github.com/tools/godep/dep.go rename to third_party/forked/godep/dep.go diff --git a/vendor/github.com/tools/godep/diff.go b/third_party/forked/godep/diff.go similarity index 100% rename from vendor/github.com/tools/godep/diff.go rename to third_party/forked/godep/diff.go diff --git a/vendor/github.com/tools/godep/doc.go b/third_party/forked/godep/doc.go similarity index 100% rename from vendor/github.com/tools/godep/doc.go rename to third_party/forked/godep/doc.go diff --git a/vendor/github.com/tools/godep/errors.go b/third_party/forked/godep/errors.go similarity index 100% rename from vendor/github.com/tools/godep/errors.go rename to third_party/forked/godep/errors.go diff --git a/vendor/github.com/tools/godep/get.go b/third_party/forked/godep/get.go similarity index 100% rename from vendor/github.com/tools/godep/get.go rename to third_party/forked/godep/get.go diff --git a/vendor/github.com/tools/godep/go.go b/third_party/forked/godep/go.go similarity index 100% rename from vendor/github.com/tools/godep/go.go rename to third_party/forked/godep/go.go diff --git a/vendor/github.com/tools/godep/godepfile.go b/third_party/forked/godep/godepfile.go similarity index 98% rename from vendor/github.com/tools/godep/godepfile.go rename to third_party/forked/godep/godepfile.go index 9d267f53f52..c7689a5d223 100644 --- a/vendor/github.com/tools/godep/godepfile.go +++ b/third_party/forked/godep/godepfile.go @@ -173,7 +173,7 @@ func (g *Godeps) save() (int64, error) { } func (g *Godeps) writeTo(w io.Writer) (int64, error) { - g.GodepVersion = fmt.Sprintf("v%d", version) // godep always writes its current version. + g.GodepVersion = fmt.Sprintf("v%s", version) // godep always writes its current version. b, err := json.MarshalIndent(g, "", "\t") if err != nil { return 0, err diff --git a/vendor/github.com/tools/godep/license.go b/third_party/forked/godep/license.go similarity index 100% rename from vendor/github.com/tools/godep/license.go rename to third_party/forked/godep/license.go diff --git a/vendor/github.com/tools/godep/list.go b/third_party/forked/godep/list.go similarity index 100% rename from vendor/github.com/tools/godep/list.go rename to third_party/forked/godep/list.go diff --git a/vendor/github.com/tools/godep/main.go b/third_party/forked/godep/main.go similarity index 100% rename from vendor/github.com/tools/godep/main.go rename to third_party/forked/godep/main.go diff --git a/vendor/github.com/tools/godep/msg.go b/third_party/forked/godep/msg.go similarity index 100% rename from vendor/github.com/tools/godep/msg.go rename to third_party/forked/godep/msg.go diff --git a/vendor/github.com/tools/godep/path.go b/third_party/forked/godep/path.go similarity index 100% rename from vendor/github.com/tools/godep/path.go rename to third_party/forked/godep/path.go diff --git a/vendor/github.com/tools/godep/pkg.go b/third_party/forked/godep/pkg.go similarity index 100% rename from vendor/github.com/tools/godep/pkg.go rename to third_party/forked/godep/pkg.go diff --git a/vendor/github.com/tools/godep/restore.go b/third_party/forked/godep/restore.go similarity index 100% rename from vendor/github.com/tools/godep/restore.go rename to third_party/forked/godep/restore.go diff --git a/vendor/github.com/tools/godep/rewrite.go b/third_party/forked/godep/rewrite.go similarity index 100% rename from vendor/github.com/tools/godep/rewrite.go rename to third_party/forked/godep/rewrite.go diff --git a/vendor/github.com/tools/godep/save.go b/third_party/forked/godep/save.go similarity index 100% rename from vendor/github.com/tools/godep/save.go rename to third_party/forked/godep/save.go diff --git a/vendor/github.com/tools/godep/update.go b/third_party/forked/godep/update.go similarity index 100% rename from vendor/github.com/tools/godep/update.go rename to third_party/forked/godep/update.go diff --git a/vendor/github.com/tools/godep/util.go b/third_party/forked/godep/util.go similarity index 100% rename from vendor/github.com/tools/godep/util.go rename to third_party/forked/godep/util.go diff --git a/vendor/github.com/tools/godep/vcs.go b/third_party/forked/godep/vcs.go similarity index 99% rename from vendor/github.com/tools/godep/vcs.go rename to third_party/forked/godep/vcs.go index 4d69b1a29ee..83034f27617 100644 --- a/vendor/github.com/tools/godep/vcs.go +++ b/third_party/forked/godep/vcs.go @@ -40,7 +40,7 @@ var vcsGit = &VCS{ vcs: vcs.ByCmd("git"), IdentifyCmd: "rev-parse HEAD", - DescribeCmd: "describe --tags", + DescribeCmd: "describe --tags --abbrev=14", DiffCmd: "diff {rev}", ListCmd: "ls-files --full-name", RootCmd: "rev-parse --show-cdup", diff --git a/vendor/github.com/tools/godep/version.go b/third_party/forked/godep/version.go similarity index 93% rename from vendor/github.com/tools/godep/version.go rename to third_party/forked/godep/version.go index e4a3e830165..6aab102a835 100644 --- a/vendor/github.com/tools/godep/version.go +++ b/third_party/forked/godep/version.go @@ -8,7 +8,7 @@ import ( "strings" ) -const version = 80 +const version = "80-k8s-r1" var cmdVersion = &Command{ Name: "version", @@ -21,7 +21,7 @@ Displays the version of godep as well as the target OS, architecture and go runt } func versionString() string { - return fmt.Sprintf("godep v%d (%s/%s/%s)", version, runtime.GOOS, runtime.GOARCH, runtime.Version()) + return fmt.Sprintf("godep v%s (%s/%s/%s)", version, runtime.GOOS, runtime.GOARCH, runtime.Version()) } func runVersion(cmd *Command, args []string) { diff --git a/third_party/multiarch/qemu-user-static/LICENSE b/third_party/multiarch/qemu-user-static/LICENSE new file mode 100644 index 00000000000..e9658e820b8 --- /dev/null +++ b/third_party/multiarch/qemu-user-static/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015-2016 Manfred Touron + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/multiarch/qemu-user-static/README.kubernetes b/third_party/multiarch/qemu-user-static/README.kubernetes new file mode 100644 index 00000000000..4e046fe6a0b --- /dev/null +++ b/third_party/multiarch/qemu-user-static/README.kubernetes @@ -0,0 +1 @@ +Files copied from https://github.com/multiarch/qemu-user-static at commit 22b0013668d2aed4a2cfd21650e85c664b1f21c6. diff --git a/third_party/multiarch/qemu-user-static/register/qemu-binfmt-conf.sh b/third_party/multiarch/qemu-user-static/register/qemu-binfmt-conf.sh new file mode 100755 index 00000000000..f21f1466029 --- /dev/null +++ b/third_party/multiarch/qemu-user-static/register/qemu-binfmt-conf.sh @@ -0,0 +1,367 @@ +#!/bin/sh +# enable automatic i386/ARM/M68K/MIPS/SPARC/PPC/s390/HPPA/Xtensa/microblaze +# program execution by the kernel +# +# downloaded from https://raw.githubusercontent.com/qemu/qemu/master/scripts/qemu-binfmt-conf.sh + +qemu_target_list="i386 i486 alpha arm armeb sparc32plus ppc ppc64 ppc64le m68k \ +mips mipsel mipsn32 mipsn32el mips64 mips64el \ +sh4 sh4eb s390x aarch64 aarch64_be hppa riscv32 riscv64 xtensa xtensaeb microblaze microblazeel" + +i386_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00' +i386_mask='\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +i386_family=i386 + +i486_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x06\x00' +i486_mask='\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +i486_family=i386 + +alpha_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x26\x90' +alpha_mask='\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +alpha_family=alpha + +arm_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00' +arm_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +arm_family=arm + +armeb_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28' +armeb_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +armeb_family=armeb + +sparc_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x02' +sparc_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +sparc_family=sparc + +sparc32plus_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x12' +sparc32plus_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +sparc32plus_family=sparc + +ppc_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x14' +ppc_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +ppc_family=ppc + +ppc64_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15' +ppc64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +ppc64_family=ppc + +ppc64le_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15\x00' +ppc64le_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\x00' +ppc64le_family=ppcle + +m68k_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x04' +m68k_mask='\xff\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +m68k_family=m68k + +# FIXME: We could use the other endianness on a MIPS host. + +mips_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08' +mips_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +mips_family=mips + +mipsel_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00' +mipsel_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +mipsel_family=mips + +mipsn32_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08' +mipsn32_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +mipsn32_family=mips + +mipsn32el_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00' +mipsn32el_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +mipsn32el_family=mips + +mips64_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08' +mips64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +mips64_family=mips + +mips64el_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00' +mips64el_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +mips64el_family=mips + +sh4_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a\x00' +sh4_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +sh4_family=sh4 + +sh4eb_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a' +sh4eb_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +sh4eb_family=sh4 + +s390x_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x16' +s390x_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +s390x_family=s390x + +aarch64_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00' +aarch64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +aarch64_family=arm + +aarch64_be_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7' +aarch64_be_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +aarch64_be_family=armeb + +hppa_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x0f' +hppa_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +hppa_family=hppa + +riscv32_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xf3\x00' +riscv32_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +riscv32_family=riscv + +riscv64_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xf3\x00' +riscv64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +riscv64_family=riscv + +xtensa_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x5e\x00' +xtensa_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +xtensa_family=xtensa + +xtensaeb_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x5e' +xtensaeb_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff' +xtensaeb_family=xtensaeb + +microblaze_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xba\xab' +microblaze_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +microblaze_family=microblaze + +microblazeel_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xab\xba' +microblazeel_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' +microblazeel_family=microblazeel + +qemu_get_family() { + cpu=${HOST_ARCH:-$(uname -m)} + case "$cpu" in + amd64|i386|i486|i586|i686|i86pc|BePC|x86_64) + echo "i386" + ;; + mips*) + echo "mips" + ;; + "Power Macintosh"|ppc64|powerpc|ppc) + echo "ppc" + ;; + ppc64el|ppc64le) + echo "ppcle" + ;; + arm|armel|armhf|arm64|armv[4-9]*l|aarch64) + echo "arm" + ;; + armeb|armv[4-9]*b|aarch64_be) + echo "armeb" + ;; + sparc*) + echo "sparc" + ;; + riscv*) + echo "riscv" + ;; + *) + echo "$cpu" + ;; + esac +} + +usage() { + cat <&2 + exit 1 + fi +} + +qemu_check_bintfmt_misc() { + # load the binfmt_misc module + if [ ! -d /proc/sys/fs/binfmt_misc ]; then + if ! /sbin/modprobe binfmt_misc ; then + exit 1 + fi + fi + if [ ! -f /proc/sys/fs/binfmt_misc/register ]; then + if ! mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ; then + exit 1 + fi + fi + + qemu_check_access /proc/sys/fs/binfmt_misc/register +} + +installed_dpkg() { + dpkg --status "$1" > /dev/null 2>&1 +} + +qemu_check_debian() { + if [ ! -e /etc/debian_version ] ; then + echo "WARNING: your system is not a Debian based distro" 1>&2 + elif ! installed_dpkg binfmt-support ; then + echo "WARNING: package binfmt-support is needed" 1>&2 + fi + qemu_check_access "$EXPORTDIR" +} + +qemu_check_systemd() { + if ! systemctl -q is-enabled systemd-binfmt.service ; then + echo "WARNING: systemd-binfmt.service is missing or disabled" 1>&2 + fi + qemu_check_access "$EXPORTDIR" +} + +qemu_generate_register() { + echo ":qemu-$cpu:M::$magic:$mask:$qemu:$FLAGS" +} + +qemu_register_interpreter() { + echo "Setting $qemu as binfmt interpreter for $cpu" + qemu_generate_register > /proc/sys/fs/binfmt_misc/register +} + +qemu_generate_systemd() { + echo "Setting $qemu as binfmt interpreter for $cpu for systemd-binfmt.service" + qemu_generate_register > "$EXPORTDIR/qemu-$cpu.conf" +} + +qemu_generate_debian() { + cat > "$EXPORTDIR/qemu-$cpu" <> "$EXPORTDIR/qemu-$cpu" + fi +} + +qemu_set_binfmts() { + # probe cpu type + host_family=$(qemu_get_family) + + # register the interpreter for each cpu except for the native one + + for cpu in ${qemu_target_list} ; do + magic=$(eval echo \$${cpu}_magic) + mask=$(eval echo \$${cpu}_mask) + family=$(eval echo \$${cpu}_family) + + if [ "$magic" = "" ] || [ "$mask" = "" ] || [ "$family" = "" ] ; then + echo "INTERNAL ERROR: unknown cpu $cpu" 1>&2 + continue + fi + + qemu="$QEMU_PATH/qemu-$cpu-static" + if [ "$cpu" = "i486" ] ; then + qemu="$QEMU_PATH/qemu-i386-static" + fi + + if [ "$host_family" != "$family" ] ; then + $BINFMT_SET + fi + done +} + +CHECK=qemu_check_bintfmt_misc +BINFMT_SET=qemu_register_interpreter + +SYSTEMDDIR="/etc/binfmt.d" +DEBIANDIR="/usr/share/binfmts" + +QEMU_PATH=/usr/local/bin +FLAGS="" + +options=$(getopt -o ds:Q:e:hc: -l debian,systemd:,qemu-path:,exportdir:,help,credential: -- "$@") +eval set -- "$options" + +while true ; do + case "$1" in + -d|--debian) + CHECK=qemu_check_debian + BINFMT_SET=qemu_generate_debian + EXPORTDIR=${EXPORTDIR:-$DEBIANDIR} + ;; + -s|--systemd) + CHECK=qemu_check_systemd + BINFMT_SET=qemu_generate_systemd + EXPORTDIR=${EXPORTDIR:-$SYSTEMDDIR} + shift + # check given cpu is in the supported CPU list + if [ "$1" != "ALL" ] ; then + for cpu in ${qemu_target_list} ; do + if [ "$cpu" = "$1" ] ; then + break + fi + done + + if [ "$cpu" = "$1" ] ; then + qemu_target_list="$1" + else + echo "ERROR: unknown CPU \"$1\"" 1>&2 + usage + exit 1 + fi + fi + ;; + -Q|--qemu-path) + shift + QEMU_PATH="$1" + ;; + -e|--exportdir) + shift + EXPORTDIR="$1" + ;; + -h|--help) + usage + exit 1 + ;; + -c|--credential) + shift + if [ "$1" = "yes" ] ; then + FLAGS="OC" + else + FLAGS="" + fi + ;; + *) + break + ;; + esac + shift +done + +$CHECK +qemu_set_binfmts diff --git a/third_party/multiarch/qemu-user-static/register/register.sh b/third_party/multiarch/qemu-user-static/register/register.sh new file mode 100755 index 00000000000..55eafdd4281 --- /dev/null +++ b/third_party/multiarch/qemu-user-static/register/register.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +QEMU_BIN_DIR=${QEMU_BIN_DIR:-/usr/bin} + + +if [ ! -d /proc/sys/fs/binfmt_misc ]; then + echo "No binfmt support in the kernel." + echo " Try: '/sbin/modprobe binfmt_misc' from the host" + exit 1 +fi + + +if [ ! -f /proc/sys/fs/binfmt_misc/register ]; then + mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc +fi + +entries="aarch64 aarch64_be alpha arm armeb hppa m68k microblaze microblazeel mips mips64 mips64el mipsel mipsn32 mipsn32el ppc ppc64 ppc64le riscv32 riscv64 s390x sh4 sh4eb sparc sparc32plus sparc64 xtensa xtensaeb" + +if [ "${1}" = "--reset" ]; then + shift + ( + cd /proc/sys/fs/binfmt_misc + for file in $entries; do + if [ -f qemu-${file} ]; then + echo -1 > qemu-${file} + fi + done + ) +fi + +exec $(dirname "${BASH_SOURCE}")/qemu-binfmt-conf.sh --qemu-path="${QEMU_BIN_DIR}" $@ diff --git a/vendor/BUILD b/vendor/BUILD index 08c874e73f2..e60e2512ce9 100644 --- a/vendor/BUILD +++ b/vendor/BUILD @@ -15,7 +15,6 @@ filegroup( "//vendor/bitbucket.org/ww/goautoneg:all-srcs", "//vendor/cloud.google.com/go/compute/metadata:all-srcs", "//vendor/cloud.google.com/go/internal:all-srcs", - "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-04-01/compute:all-srcs", "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute:all-srcs", "//vendor/github.com/Azure/azure-sdk-for-go/services/containerregistry/mgmt/2017-10-01/containerregistry:all-srcs", "//vendor/github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network:all-srcs", @@ -60,6 +59,7 @@ filegroup( "//vendor/github.com/bazelbuild/bazel-gazelle/internal/repos:all-srcs", "//vendor/github.com/bazelbuild/bazel-gazelle/internal/resolve:all-srcs", "//vendor/github.com/bazelbuild/bazel-gazelle/internal/rule:all-srcs", + "//vendor/github.com/bazelbuild/bazel-gazelle/internal/testtools:all-srcs", "//vendor/github.com/bazelbuild/bazel-gazelle/internal/version:all-srcs", "//vendor/github.com/bazelbuild/bazel-gazelle/internal/walk:all-srcs", "//vendor/github.com/bazelbuild/bazel-gazelle/internal/wspace:all-srcs", @@ -94,7 +94,7 @@ filegroup( "//vendor/github.com/cloudflare/cfssl/ubiquity:all-srcs", "//vendor/github.com/clusterhq/flocker-go:all-srcs", "//vendor/github.com/codedellemc/goscaleio:all-srcs", - "//vendor/github.com/container-storage-interface/spec/lib/go/csi/v0:all-srcs", + "//vendor/github.com/container-storage-interface/spec/lib/go/csi:all-srcs", "//vendor/github.com/containerd/console:all-srcs", "//vendor/github.com/containerd/containerd/api/services/containers/v1:all-srcs", "//vendor/github.com/containerd/containerd/api/services/tasks/v1:all-srcs", @@ -207,6 +207,7 @@ filegroup( "//vendor/github.com/go-openapi/strfmt:all-srcs", "//vendor/github.com/go-openapi/swag:all-srcs", "//vendor/github.com/go-openapi/validate:all-srcs", + "//vendor/github.com/go-ozzo/ozzo-validation:all-srcs", "//vendor/github.com/go-sql-driver/mysql:all-srcs", "//vendor/github.com/godbus/dbus:all-srcs", "//vendor/github.com/gogo/protobuf/gogoproto:all-srcs", @@ -234,11 +235,11 @@ filegroup( "//vendor/github.com/gogo/protobuf/sortkeys:all-srcs", "//vendor/github.com/gogo/protobuf/types:all-srcs", "//vendor/github.com/gogo/protobuf/vanity:all-srcs", - "//vendor/github.com/golang/glog:all-srcs", "//vendor/github.com/golang/groupcache/lru:all-srcs", "//vendor/github.com/golang/mock/gomock:all-srcs", "//vendor/github.com/golang/protobuf/jsonpb:all-srcs", "//vendor/github.com/golang/protobuf/proto:all-srcs", + "//vendor/github.com/golang/protobuf/protoc-gen-go/descriptor:all-srcs", "//vendor/github.com/golang/protobuf/ptypes:all-srcs", "//vendor/github.com/google/btree:all-srcs", "//vendor/github.com/google/cadvisor/accelerators:all-srcs", @@ -286,6 +287,7 @@ filegroup( "//vendor/github.com/json-iterator/go:all-srcs", "//vendor/github.com/jteeuwen/go-bindata:all-srcs", "//vendor/github.com/kardianos/osext:all-srcs", + "//vendor/github.com/karrick/godirwalk:all-srcs", "//vendor/github.com/kisielk/sqlstruct:all-srcs", "//vendor/github.com/kr/fs:all-srcs", "//vendor/github.com/kr/pretty:all-srcs", @@ -296,7 +298,6 @@ filegroup( "//vendor/github.com/libopenstorage/openstorage/pkg/parser:all-srcs", "//vendor/github.com/libopenstorage/openstorage/pkg/units:all-srcs", "//vendor/github.com/libopenstorage/openstorage/volume:all-srcs", - "//vendor/github.com/lpabon/godbc:all-srcs", "//vendor/github.com/magiconair/properties:all-srcs", "//vendor/github.com/mailru/easyjson/buffer:all-srcs", "//vendor/github.com/mailru/easyjson/jlexer:all-srcs", @@ -348,6 +349,7 @@ filegroup( "//vendor/github.com/satori/go.uuid:all-srcs", "//vendor/github.com/seccomp/libseccomp-golang:all-srcs", "//vendor/github.com/shurcooL/sanitized_anchor_name:all-srcs", + "//vendor/github.com/sigma/go-inotify:all-srcs", "//vendor/github.com/sirupsen/logrus:all-srcs", "//vendor/github.com/soheilhy/cmux:all-srcs", "//vendor/github.com/spf13/afero:all-srcs", @@ -363,7 +365,6 @@ filegroup( "//vendor/github.com/stretchr/testify/require:all-srcs", "//vendor/github.com/syndtr/gocapability/capability:all-srcs", "//vendor/github.com/tmc/grpc-websocket-proxy/wsproxy:all-srcs", - "//vendor/github.com/tools/godep:all-srcs", "//vendor/github.com/ugorji/go/codec:all-srcs", "//vendor/github.com/vishvananda/netlink:all-srcs", "//vendor/github.com/vishvananda/netns:all-srcs", @@ -456,6 +457,7 @@ filegroup( "//vendor/k8s.io/gengo/parser:all-srcs", "//vendor/k8s.io/gengo/types:all-srcs", "//vendor/k8s.io/heapster/metrics/api/v1/types:all-srcs", + "//vendor/k8s.io/klog:all-srcs", "//vendor/k8s.io/kube-openapi/cmd/openapi-gen:all-srcs", "//vendor/k8s.io/kube-openapi/pkg/aggregator:all-srcs", "//vendor/k8s.io/kube-openapi/pkg/builder:all-srcs", @@ -466,6 +468,7 @@ filegroup( "//vendor/k8s.io/utils/clock:all-srcs", "//vendor/k8s.io/utils/exec:all-srcs", "//vendor/k8s.io/utils/pointer:all-srcs", + "//vendor/sigs.k8s.io/yaml:all-srcs", "//vendor/vbom.ml/util/sortorder:all-srcs", ], tags = ["automanaged"], diff --git a/vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-04-01/compute/BUILD b/vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-04-01/compute/BUILD deleted file mode 100644 index 6df04e38cd7..00000000000 --- a/vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-04-01/compute/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/BUILD b/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/BUILD index 3d50d941ff8..5ae277b7643 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/BUILD +++ b/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/BUILD @@ -28,6 +28,7 @@ go_library( "//vendor/github.com/bazelbuild/bazel-gazelle/internal/rule:go_default_library", "//vendor/github.com/bazelbuild/bazel-gazelle/internal/version:go_default_library", "//vendor/github.com/bazelbuild/bazel-gazelle/internal/walk:go_default_library", + "//vendor/github.com/pmezard/go-difflib/difflib:go_default_library", ], ) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/diff.go b/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/diff.go index 4e74feafea7..5dbe7934baf 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/diff.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/diff.go @@ -16,37 +16,68 @@ limitations under the License. package main import ( - "bytes" + "fmt" + "io" "io/ioutil" "os" - "os/exec" "path/filepath" + + "github.com/bazelbuild/bazel-gazelle/internal/config" + "github.com/bazelbuild/bazel-gazelle/internal/rule" + "github.com/pmezard/go-difflib/difflib" ) -func diffFile(path string, newContents []byte) error { - oldContents, err := ioutil.ReadFile(path) +func diffFile(c *config.Config, f *rule.File) error { + rel, err := filepath.Rel(c.RepoRoot, f.Path) if err != nil { - oldContents = nil + return fmt.Errorf("error getting old path for file %q: %v", f.Path, err) } - if bytes.Equal(oldContents, newContents) { - return nil + rel = filepath.ToSlash(rel) + + date := "1970-01-01 00:00:00.000000000 +0000" + diff := difflib.UnifiedDiff{ + Context: 3, + FromDate: date, + ToDate: date, } - f, err := ioutil.TempFile("", filepath.Base(path)) - if err != nil { - return err + + if oldContent, err := ioutil.ReadFile(f.Path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error reading original file: %v", err) + } else if err != nil { + diff.FromFile = "/dev/null" + } else if err == nil { + diff.A = difflib.SplitLines(string(oldContent)) + if c.ReadBuildFilesDir == "" { + path, err := filepath.Rel(c.RepoRoot, f.Path) + if err != nil { + return fmt.Errorf("error getting old path for file %q: %v", f.Path, err) + } + diff.FromFile = filepath.ToSlash(path) + } else { + diff.FromFile = f.Path + } } - f.Close() - defer os.Remove(f.Name()) - if err := ioutil.WriteFile(f.Name(), newContents, 0666); err != nil { - return err + + newContent := f.Format() + diff.B = difflib.SplitLines(string(newContent)) + outPath := findOutputPath(c, f) + if c.WriteBuildFilesDir == "" { + path, err := filepath.Rel(c.RepoRoot, f.Path) + if err != nil { + return fmt.Errorf("error getting new path for file %q: %v", f.Path, err) + } + diff.ToFile = filepath.ToSlash(path) + } else { + diff.ToFile = outPath } - cmd := exec.Command("diff", "-u", "--new-file", path, f.Name()) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Run() - if _, ok := err.(*exec.ExitError); ok { - // diff returns non-zero when files are different. This is not an error. - return nil + + uc := getUpdateConfig(c) + var out io.Writer = os.Stdout + if uc.patchPath != "" { + out = &uc.patchBuffer } - return err + if err := difflib.WriteUnifiedDiff(out, diff); err != nil { + return fmt.Errorf("error diffing %s: %v", f.Path, err) + } + return nil } diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/fix-update.go b/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/fix-update.go index a44994ce371..81c60f446ee 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/fix-update.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/fix-update.go @@ -16,6 +16,7 @@ limitations under the License. package main import ( + "bytes" "flag" "fmt" "io/ioutil" @@ -38,11 +39,16 @@ import ( // update commands. This includes everything in config.Config, but it also // includes some additional fields that aren't relevant to other packages. type updateConfig struct { - emit emitFunc - repos []repos.Repo + dirs []string + emit emitFunc + repos []repos.Repo + useIndex bool + walkMode walk.Mode + patchPath string + patchBuffer bytes.Buffer } -type emitFunc func(path string, data []byte) error +type emitFunc func(c *config.Config, f *rule.File) error var modeFromName = map[string]emitFunc{ "print": printFile, @@ -57,7 +63,8 @@ func getUpdateConfig(c *config.Config) *updateConfig { } type updateConfigurer struct { - mode string + mode string + recursive bool } func (ucr *updateConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { @@ -67,33 +74,49 @@ func (ucr *updateConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *conf c.ShouldFix = cmd == "fix" fs.StringVar(&ucr.mode, "mode", "fix", "print: prints all of the updated BUILD files\n\tfix: rewrites all of the BUILD files in place\n\tdiff: computes the rewrite but then just does a diff") + fs.BoolVar(&uc.useIndex, "index", true, "when true, gazelle will build an index of libraries in the workspace for dependency resolution") + fs.BoolVar(&ucr.recursive, "r", true, "when true, gazelle will update subdirectories recursively") + fs.StringVar(&uc.patchPath, "patch", "", "when set with -mode=diff, gazelle will write to a file instead of stdout") } func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error { uc := getUpdateConfig(c) + var ok bool uc.emit, ok = modeFromName[ucr.mode] if !ok { return fmt.Errorf("unrecognized emit mode: %q", ucr.mode) } - - c.Dirs = fs.Args() - if len(c.Dirs) == 0 { - c.Dirs = []string{"."} + if uc.patchPath != "" && ucr.mode != "diff" { + return fmt.Errorf("-patch set but -mode is %s, not diff", ucr.mode) } - for i := range c.Dirs { - dir, err := filepath.Abs(c.Dirs[i]) + + dirs := fs.Args() + if len(dirs) == 0 { + dirs = []string{"."} + } + uc.dirs = make([]string, len(dirs)) + for i := range dirs { + dir, err := filepath.Abs(dirs[i]) if err != nil { - return fmt.Errorf("%s: failed to find absolute path: %v", c.Dirs[i], err) + return fmt.Errorf("%s: failed to find absolute path: %v", dirs[i], err) } dir, err = filepath.EvalSymlinks(dir) if err != nil { - return fmt.Errorf("%s: failed to resolve symlinks: %v", c.Dirs[i], err) + return fmt.Errorf("%s: failed to resolve symlinks: %v", dirs[i], err) } if !isDescendingDir(dir, c.RepoRoot) { return fmt.Errorf("dir %q is not a subdirectory of repo root %q", dir, c.RepoRoot) } - c.Dirs[i] = dir + uc.dirs[i] = dir + } + + if ucr.recursive { + uc.walkMode = walk.VisitAllUpdateSubdirsMode + } else if uc.useIndex { + uc.walkMode = walk.VisitAllUpdateDirsMode + } else { + uc.walkMode = walk.UpdateDirsMode } return nil @@ -134,8 +157,12 @@ var genericLoads = []rule.LoadInfo{ } func runFixUpdate(cmd command, args []string) error { - cexts := make([]config.Configurer, 0, len(languages)+2) - cexts = append(cexts, &config.CommonConfigurer{}, &updateConfigurer{}) + cexts := make([]config.Configurer, 0, len(languages)+3) + cexts = append(cexts, + &config.CommonConfigurer{}, + &updateConfigurer{}, + &walk.Configurer{}, + &resolve.Configurer{}) kindToResolver := make(map[string]resolve.Resolver) kinds := make(map[string]rule.KindInfo) loads := genericLoads @@ -163,11 +190,12 @@ func runFixUpdate(cmd command, args []string) error { // Visit all directories in the repository. var visits []visitRecord - walk.Walk(c, cexts, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) { + uc := getUpdateConfig(c) + walk.Walk(c, cexts, uc.dirs, uc.walkMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) { // If this file is ignored or if Gazelle was not asked to update this // directory, just index the build file and move on. if !update { - if f != nil { + if uc.useIndex && f != nil { for _, r := range f.Rules { ruleIndex.AddRule(c, r, f) } @@ -215,8 +243,6 @@ func runFixUpdate(cmd command, args []string) error { } }) - uc := getUpdateConfig(c) - // Finish building the index for dependency resolution. ruleIndex.Finish() @@ -233,12 +259,16 @@ func runFixUpdate(cmd command, args []string) error { // Emit merged files. for _, v := range visits { merger.FixLoads(v.file, loads) - content := v.file.Format() - outputPath := findOutputPath(c, v.file) - if err := uc.emit(outputPath, content); err != nil { + if err := uc.emit(c, v.file); err != nil { log.Print(err) } } + if uc.patchPath != "" { + if err := ioutil.WriteFile(uc.patchPath, uc.patchBuffer.Bytes(), 0666); err != nil { + return err + } + } + return nil } @@ -338,7 +368,7 @@ func fixWorkspace(c *config.Config, workspace *rule.File, loads []rule.LoadInfo) return nil } shouldFix := false - for _, d := range c.Dirs { + for _, d := range uc.dirs { if d == c.RepoRoot { shouldFix = true } @@ -352,7 +382,7 @@ func fixWorkspace(c *config.Config, workspace *rule.File, loads []rule.LoadInfo) if err := merger.CheckGazelleLoaded(workspace); err != nil { return err } - return uc.emit(workspace.Path, workspace.Format()) + return uc.emit(c, workspace) } func findWorkspaceName(f *rule.File) string { diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/fix.go b/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/fix.go index 7a1c38874a5..9e13578c635 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/fix.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/fix.go @@ -19,11 +19,15 @@ import ( "io/ioutil" "os" "path/filepath" + + "github.com/bazelbuild/bazel-gazelle/internal/config" + "github.com/bazelbuild/bazel-gazelle/internal/rule" ) -func fixFile(path string, data []byte) error { - if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil { +func fixFile(c *config.Config, f *rule.File) error { + outPath := findOutputPath(c, f) + if err := os.MkdirAll(filepath.Dir(outPath), 0777); err != nil { return err } - return ioutil.WriteFile(path, data, 0666) + return ioutil.WriteFile(outPath, f.Format(), 0666) } diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/gazelle.go b/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/gazelle.go index fd44442661c..5248c26f26d 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/gazelle.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/gazelle.go @@ -56,7 +56,7 @@ func main() { log.SetPrefix("gazelle: ") log.SetFlags(0) // don't print timestamps - if err := run(os.Args[1:]); err != nil { + if err := run(os.Args[1:]); err != nil && err != flag.ErrHelp { log.Fatal(err) } } @@ -112,7 +112,7 @@ For example: gazelle update -h -Gazelle is under active delevopment, and its interface may change +Gazelle is under active development, and its interface may change without notice. `) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/print.go b/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/print.go index e7dfe135a0a..06f92e6f658 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/print.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/cmd/gazelle/print.go @@ -17,9 +17,13 @@ package main import ( "os" + + "github.com/bazelbuild/bazel-gazelle/internal/config" + "github.com/bazelbuild/bazel-gazelle/internal/rule" ) -func printFile(_ string, data []byte) error { - _, err := os.Stdout.Write(data) +func printFile(c *config.Config, f *rule.File) error { + content := f.Format() + _, err := os.Stdout.Write(content) return err } diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/config/config.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/config/config.go index d959d782efe..5323e8c7d4a 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/internal/config/config.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/config/config.go @@ -36,10 +36,6 @@ import ( // information is language-specific and is stored in Exts. This information // is modified by extensions that implement Configurer. type Config struct { - // Dirs is a list of absolute, canonical paths to directories where Gazelle - // should run. - Dirs []string - // RepoRoot is the absolute, canonical path to the root directory of the // repository with all symlinks resolved. RepoRoot string diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/go/known_go_imports.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/go/known_go_imports.go index 1443fbe2538..3050208febf 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/go/known_go_imports.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/go/known_go_imports.go @@ -20,117 +20,139 @@ var knownGoProtoImports = map[string]label.Label{ "github.com/golang/protobuf/ptypes/timestamp": label.New("io_bazel_rules_go", "proto/wkt", "timestamp_go_proto"), "google.golang.org/genproto/protobuf/ptype": label.New("io_bazel_rules_go", "proto/wkt", "type_go_proto"), "github.com/golang/protobuf/ptypes/wrappers": label.New("io_bazel_rules_go", "proto/wkt", "wrappers_go_proto"), - "google.golang.org/genproto/googleapis/assistant/embedded/v1alpha2": label.New("go_googleapis", "google/assistant/embedded/v1alpha2", "embedded_go_proto"), - "google.golang.org/genproto/googleapis/assistant/embedded/v1alpha1": label.New("go_googleapis", "google/assistant/embedded/v1alpha1", "embedded_go_proto"), - "google.golang.org/genproto/googleapis/home/graph/v1": label.New("go_googleapis", "google/home/graph/v1", "graph_go_proto"), - "google.golang.org/genproto/googleapis/genomics/v1": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), - "google.golang.org/genproto/googleapis/genomics/v1alpha2": label.New("go_googleapis", "google/genomics/v1alpha2", "genomics_go_proto"), - "google.golang.org/genproto/googleapis/bigtable/v1": label.New("go_googleapis", "google/bigtable/v1", "bigtable_go_proto"), - "google.golang.org/genproto/googleapis/bigtable/admin/cluster/v1": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_go_proto"), - "google.golang.org/genproto/googleapis/bigtable/admin/v2": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_go_proto"), - "google.golang.org/genproto/googleapis/bigtable/admin/table/v1": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_go_proto"), - "google.golang.org/genproto/googleapis/bigtable/v2": label.New("go_googleapis", "google/bigtable/v2", "bigtable_go_proto"), - "google.golang.org/genproto/googleapis/privacy/dlp/v2": label.New("go_googleapis", "google/privacy/dlp/v2", "dlp_go_proto"), - "google.golang.org/genproto/googleapis/watcher/v1": label.New("go_googleapis", "google/watcher/v1", "watcher_go_proto"), - "google.golang.org/genproto/googleapis/firestore/admin/v1beta1": label.New("go_googleapis", "google/firestore/admin/v1beta1", "admin_go_proto"), - "google.golang.org/genproto/googleapis/firestore/v1beta1": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_go_proto"), - "google.golang.org/genproto/googleapis/example/library/v1": label.New("go_googleapis", "google/example/library/v1", "library_go_proto"), - "google.golang.org/genproto/googleapis/appengine/v1": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), + "google.golang.org/genproto/googleapis/api/annotations": label.New("go_googleapis", "google/api", "annotations_go_proto"), + "google.golang.org/genproto/googleapis/api/serviceconfig": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google.golang.org/genproto/googleapis/api/configchange": label.New("go_googleapis", "google/api", "configchange_go_proto"), + "google.golang.org/genproto/googleapis/api/distribution": label.New("go_googleapis", "google/api", "distribution_go_proto"), + "google.golang.org/genproto/googleapis/api": label.New("go_googleapis", "google/api", "api_go_proto"), + "google.golang.org/genproto/googleapis/api/expr/v1alpha1": label.New("go_googleapis", "google/api/expr/v1alpha1", "expr_go_proto"), + "google.golang.org/genproto/googleapis/api/expr/v1beta1": label.New("go_googleapis", "google/api/expr/v1beta1", "expr_go_proto"), + "google.golang.org/genproto/googleapis/api/httpbody": label.New("go_googleapis", "google/api", "httpbody_go_proto"), + "google.golang.org/genproto/googleapis/api/label": label.New("go_googleapis", "google/api", "label_go_proto"), + "google.golang.org/genproto/googleapis/api/metric": label.New("go_googleapis", "google/api", "metric_go_proto"), + "google.golang.org/genproto/googleapis/api/monitoredres": label.New("go_googleapis", "google/api", "monitoredres_go_proto"), + "google.golang.org/genproto/googleapis/api/servicecontrol/v1": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), + "google.golang.org/genproto/googleapis/api/servicemanagement/v1": label.New("go_googleapis", "google/api/servicemanagement/v1", "servicemanagement_go_proto"), "google.golang.org/genproto/googleapis/appengine/legacy": label.New("go_googleapis", "google/appengine/legacy", "legacy_go_proto"), "google.golang.org/genproto/googleapis/appengine/logging/v1": label.New("go_googleapis", "google/appengine/logging/v1", "logging_go_proto"), - "google.golang.org/genproto/googleapis/storagetransfer/v1": label.New("go_googleapis", "google/storagetransfer/v1", "storagetransfer_go_proto"), - "google.golang.org/genproto/googleapis/longrunning": label.New("go_googleapis", "google/longrunning", "longrunning_go_proto"), - "google.golang.org/genproto/googleapis/container/v1": label.New("go_googleapis", "google/container/v1", "container_go_proto"), - "google.golang.org/genproto/googleapis/container/v1beta1": label.New("go_googleapis", "google/container/v1beta1", "container_go_proto"), - "google.golang.org/genproto/googleapis/container/v1alpha1": label.New("go_googleapis", "google/container/v1alpha1", "container_go_proto"), - "google.golang.org/genproto/googleapis/datastore/v1beta3": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_go_proto"), - "google.golang.org/genproto/googleapis/datastore/v1": label.New("go_googleapis", "google/datastore/v1", "datastore_go_proto"), - "google.golang.org/genproto/googleapis/datastore/admin/v1": label.New("go_googleapis", "google/datastore/admin/v1", "admin_go_proto"), - "google.golang.org/genproto/googleapis/datastore/admin/v1beta1": label.New("go_googleapis", "google/datastore/admin/v1beta1", "admin_go_proto"), + "google.golang.org/genproto/googleapis/appengine/v1": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), + "google.golang.org/genproto/googleapis/assistant/embedded/v1alpha1": label.New("go_googleapis", "google/assistant/embedded/v1alpha1", "embedded_go_proto"), + "google.golang.org/genproto/googleapis/assistant/embedded/v1alpha2": label.New("go_googleapis", "google/assistant/embedded/v1alpha2", "embedded_go_proto"), + "google.golang.org/genproto/googleapis/bigtable/admin/cluster/v1": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_go_proto"), + "google.golang.org/genproto/googleapis/bigtable/admin/table/v1": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_go_proto"), + "google.golang.org/genproto/googleapis/bigtable/admin/v2": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_go_proto"), + "google.golang.org/genproto/googleapis/bigtable/v1": label.New("go_googleapis", "google/bigtable/v1", "bigtable_go_proto"), + "google.golang.org/genproto/googleapis/bigtable/v2": label.New("go_googleapis", "google/bigtable/v2", "bigtable_go_proto"), "google.golang.org/genproto/googleapis/bytestream": label.New("go_googleapis", "google/bytestream", "bytestream_go_proto"), - "google.golang.org/genproto/googleapis/iam/v1": label.New("go_googleapis", "google/iam/v1", "iam_go_proto"), - "google.golang.org/genproto/googleapis/iam/v1/logging": label.New("go_googleapis", "google/iam/v1/logging", "logging_go_proto"), - "google.golang.org/genproto/googleapis/iam/admin/v1": label.New("go_googleapis", "google/iam/admin/v1", "admin_go_proto"), - "google.golang.org/genproto/googleapis/type/money": label.New("go_googleapis", "google/type", "money_go_proto"), - "google.golang.org/genproto/googleapis/type/latlng": label.New("go_googleapis", "google/type", "latlng_go_proto"), - "google.golang.org/genproto/googleapis/type/color": label.New("go_googleapis", "google/type", "color_go_proto"), - "google.golang.org/genproto/googleapis/type/timeofday": label.New("go_googleapis", "google/type", "timeofday_go_proto"), - "google.golang.org/genproto/googleapis/type/date": label.New("go_googleapis", "google/type", "date_go_proto"), - "google.golang.org/genproto/googleapis/type/dayofweek": label.New("go_googleapis", "google/type", "dayofweek_go_proto"), - "google.golang.org/genproto/googleapis/type/postaladdress": label.New("go_googleapis", "google/type", "postaladdress_go_proto"), - "google.golang.org/genproto/googleapis/devtools/clouderrorreporting/v1beta1": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_go_proto"), - "google.golang.org/genproto/googleapis/devtools/resultstore/v2": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google.golang.org/genproto/googleapis/devtools/source/v1": label.New("go_googleapis", "google/devtools/source/v1", "source_go_proto"), - "google.golang.org/genproto/googleapis/devtools/remoteexecution/v1test": label.New("go_googleapis", "google/devtools/remoteexecution/v1test", "remoteexecution_go_proto"), - "google.golang.org/genproto/googleapis/devtools/cloudbuild/v1": label.New("go_googleapis", "google/devtools/cloudbuild/v1", "cloudbuild_go_proto"), - "google.golang.org/genproto/googleapis/devtools/sourcerepo/v1": label.New("go_googleapis", "google/devtools/sourcerepo/v1", "sourcerepo_go_proto"), - "google.golang.org/genproto/googleapis/devtools/remoteworkers/v1test2": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_go_proto"), - "google.golang.org/genproto/googleapis/devtools/cloudtrace/v1": label.New("go_googleapis", "google/devtools/cloudtrace/v1", "cloudtrace_go_proto"), - "google.golang.org/genproto/googleapis/devtools/cloudtrace/v2": label.New("go_googleapis", "google/devtools/cloudtrace/v2", "cloudtrace_go_proto"), - "google.golang.org/genproto/googleapis/devtools/cloudprofiler/v2": label.New("go_googleapis", "google/devtools/cloudprofiler/v2", "cloudprofiler_go_proto"), - "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), - "google.golang.org/genproto/googleapis/devtools/build/v1": label.New("go_googleapis", "google/devtools/build/v1", "build_go_proto"), - "google.golang.org/genproto/googleapis/devtools/clouddebugger/v2": label.New("go_googleapis", "google/devtools/clouddebugger/v2", "clouddebugger_go_proto"), - "google.golang.org/genproto/googleapis/cloud/resourcemanager/v2": label.New("go_googleapis", "google/cloud/resourcemanager/v2", "resourcemanager_go_proto"), - "google.golang.org/genproto/googleapis/cloud/kms/v1": label.New("go_googleapis", "google/cloud/kms/v1", "kms_go_proto"), - "google.golang.org/genproto/googleapis/cloud/runtimeconfig/v1beta1": label.New("go_googleapis", "google/cloud/runtimeconfig/v1beta1", "runtimeconfig_go_proto"), - "google.golang.org/genproto/googleapis/cloud/tasks/v2beta2": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_go_proto"), - "google.golang.org/genproto/googleapis/cloud/oslogin/v1": label.New("go_googleapis", "google/cloud/oslogin/v1", "oslogin_go_proto"), - "google.golang.org/genproto/googleapis/cloud/oslogin/v1alpha": label.New("go_googleapis", "google/cloud/oslogin/v1alpha", "oslogin_go_proto"), - "google.golang.org/genproto/googleapis/cloud/oslogin/common": label.New("go_googleapis", "google/cloud/oslogin/common", "common_go_proto"), - "google.golang.org/genproto/googleapis/cloud/oslogin/v1beta": label.New("go_googleapis", "google/cloud/oslogin/v1beta", "oslogin_go_proto"), - "google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), - "google.golang.org/genproto/googleapis/cloud/dialogflow/v2": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), - "google.golang.org/genproto/googleapis/cloud/redis/v1beta1": label.New("go_googleapis", "google/cloud/redis/v1beta1", "redis_go_proto"), - "google.golang.org/genproto/googleapis/cloud/location": label.New("go_googleapis", "google/cloud/location", "location_go_proto"), - "google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), - "google.golang.org/genproto/googleapis/cloud/language/v1": label.New("go_googleapis", "google/cloud/language/v1", "language_go_proto"), - "google.golang.org/genproto/googleapis/cloud/language/v1beta2": label.New("go_googleapis", "google/cloud/language/v1beta2", "language_go_proto"), - "google.golang.org/genproto/googleapis/cloud/language/v1beta1": label.New("go_googleapis", "google/cloud/language/v1beta1", "language_go_proto"), + "google.golang.org/genproto/googleapis/cloud/asset/v1beta1": label.New("go_googleapis", "google/cloud/asset/v1beta1", "asset_go_proto"), + "google.golang.org/genproto/googleapis/cloud/audit": label.New("go_googleapis", "google/cloud/audit", "audit_go_proto"), + "google.golang.org/genproto/googleapis/cloud/automl/v1beta1": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), "google.golang.org/genproto/googleapis/cloud/bigquery/datatransfer/v1": label.New("go_googleapis", "google/cloud/bigquery/datatransfer/v1", "datatransfer_go_proto"), "google.golang.org/genproto/googleapis/cloud/bigquery/logging/v1": label.New("go_googleapis", "google/cloud/bigquery/logging/v1", "logging_go_proto"), - "google.golang.org/genproto/googleapis/cloud/vision/v1": label.New("go_googleapis", "google/cloud/vision/v1", "vision_go_proto"), - "google.golang.org/genproto/googleapis/cloud/vision/v1p2beta1": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_go_proto"), - "google.golang.org/genproto/googleapis/cloud/vision/v1p1beta1": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_go_proto"), - "google.golang.org/genproto/googleapis/cloud/speech/v1": label.New("go_googleapis", "google/cloud/speech/v1", "speech_go_proto"), - "google.golang.org/genproto/googleapis/cloud/speech/v1beta1": label.New("go_googleapis", "google/cloud/speech/v1beta1", "speech_go_proto"), - "google.golang.org/genproto/googleapis/cloud/speech/v1p1beta1": label.New("go_googleapis", "google/cloud/speech/v1p1beta1", "speech_go_proto"), - "google.golang.org/genproto/googleapis/cloud/iot/v1": label.New("go_googleapis", "google/cloud/iot/v1", "iot_go_proto"), - "google.golang.org/genproto/googleapis/cloud/videointelligence/v1": label.New("go_googleapis", "google/cloud/videointelligence/v1", "videointelligence_go_proto"), - "google.golang.org/genproto/googleapis/cloud/videointelligence/v1beta2": label.New("go_googleapis", "google/cloud/videointelligence/v1beta2", "videointelligence_go_proto"), - "google.golang.org/genproto/googleapis/cloud/videointelligence/v1beta1": label.New("go_googleapis", "google/cloud/videointelligence/v1beta1", "videointelligence_go_proto"), - "google.golang.org/genproto/googleapis/cloud/videointelligence/v1p1beta1": label.New("go_googleapis", "google/cloud/videointelligence/v1p1beta1", "videointelligence_go_proto"), - "google.golang.org/genproto/googleapis/cloud/audit": label.New("go_googleapis", "google/cloud/audit", "audit_go_proto"), - "google.golang.org/genproto/googleapis/cloud/support/common": label.New("go_googleapis", "google/cloud/support", "common_go_proto"), - "google.golang.org/genproto/googleapis/cloud/support/v1alpha1": label.New("go_googleapis", "google/cloud/support/v1alpha1", "support_go_proto"), - "google.golang.org/genproto/googleapis/cloud/ml/v1": label.New("go_googleapis", "google/cloud/ml/v1", "ml_go_proto"), - "google.golang.org/genproto/googleapis/cloud/texttospeech/v1": label.New("go_googleapis", "google/cloud/texttospeech/v1", "texttospeech_go_proto"), - "google.golang.org/genproto/googleapis/cloud/texttospeech/v1beta1": label.New("go_googleapis", "google/cloud/texttospeech/v1beta1", "texttospeech_go_proto"), - "google.golang.org/genproto/googleapis/cloud/functions/v1beta2": label.New("go_googleapis", "google/cloud/functions/v1beta2", "functions_go_proto"), + "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta1": label.New("go_googleapis", "google/cloud/bigquery/storage/v1beta1", "storage_go_proto"), "google.golang.org/genproto/googleapis/cloud/billing/v1": label.New("go_googleapis", "google/cloud/billing/v1", "billing_go_proto"), "google.golang.org/genproto/googleapis/cloud/dataproc/v1": label.New("go_googleapis", "google/cloud/dataproc/v1", "dataproc_go_proto"), "google.golang.org/genproto/googleapis/cloud/dataproc/v1beta2": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_go_proto"), - "google.golang.org/genproto/googleapis/api/serviceconfig": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google.golang.org/genproto/googleapis/api/annotations": label.New("go_googleapis", "google/api", "annotations_go_proto"), - "google.golang.org/genproto/googleapis/api/configchange": label.New("go_googleapis", "google/api", "configchange_go_proto"), - "google.golang.org/genproto/googleapis/api/distribution": label.New("go_googleapis", "google/api", "distribution_go_proto"), - "google.golang.org/genproto/googleapis/api/monitoredres": label.New("go_googleapis", "google/api", "monitoredres_go_proto"), - "google.golang.org/genproto/googleapis/api/metric": label.New("go_googleapis", "google/api", "metric_go_proto"), - "google.golang.org/genproto/googleapis/api/label": label.New("go_googleapis", "google/api", "label_go_proto"), - "google.golang.org/genproto/googleapis/api/httpbody": label.New("go_googleapis", "google/api", "httpbody_go_proto"), - "google.golang.org/genproto/googleapis/api": label.New("go_googleapis", "google/api/experimental", "api_go_proto"), - "google.golang.org/genproto/googleapis/api/servicemanagement/v1": label.New("go_googleapis", "google/api/servicemanagement/v1", "servicemanagement_go_proto"), - "google.golang.org/genproto/googleapis/api/servicecontrol/v1": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), + "google.golang.org/genproto/googleapis/cloud/dialogflow/v2": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), + "google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), + "google.golang.org/genproto/googleapis/cloud/functions/v1beta2": label.New("go_googleapis", "google/cloud/functions/v1beta2", "functions_go_proto"), + "google.golang.org/genproto/googleapis/cloud/iot/v1": label.New("go_googleapis", "google/cloud/iot/v1", "iot_go_proto"), + "google.golang.org/genproto/googleapis/cloud/kms/v1": label.New("go_googleapis", "google/cloud/kms/v1", "kms_go_proto"), + "google.golang.org/genproto/googleapis/cloud/language/v1": label.New("go_googleapis", "google/cloud/language/v1", "language_go_proto"), + "google.golang.org/genproto/googleapis/cloud/language/v1beta1": label.New("go_googleapis", "google/cloud/language/v1beta1", "language_go_proto"), + "google.golang.org/genproto/googleapis/cloud/language/v1beta2": label.New("go_googleapis", "google/cloud/language/v1beta2", "language_go_proto"), + "google.golang.org/genproto/googleapis/cloud/location": label.New("go_googleapis", "google/cloud/location", "location_go_proto"), + "google.golang.org/genproto/googleapis/cloud/ml/v1": label.New("go_googleapis", "google/cloud/ml/v1", "ml_go_proto"), + "google.golang.org/genproto/googleapis/cloud/oslogin/common": label.New("go_googleapis", "google/cloud/oslogin/common", "common_go_proto"), + "google.golang.org/genproto/googleapis/cloud/oslogin/v1": label.New("go_googleapis", "google/cloud/oslogin/v1", "oslogin_go_proto"), + "google.golang.org/genproto/googleapis/cloud/oslogin/v1alpha": label.New("go_googleapis", "google/cloud/oslogin/v1alpha", "oslogin_go_proto"), + "google.golang.org/genproto/googleapis/cloud/oslogin/v1beta": label.New("go_googleapis", "google/cloud/oslogin/v1beta", "oslogin_go_proto"), + "google.golang.org/genproto/googleapis/cloud/redis/v1": label.New("go_googleapis", "google/cloud/redis/v1", "redis_go_proto"), + "google.golang.org/genproto/googleapis/cloud/redis/v1beta1": label.New("go_googleapis", "google/cloud/redis/v1beta1", "redis_go_proto"), + "google.golang.org/genproto/googleapis/cloud/resourcemanager/v2": label.New("go_googleapis", "google/cloud/resourcemanager/v2", "resourcemanager_go_proto"), + "google.golang.org/genproto/googleapis/cloud/runtimeconfig/v1beta1": label.New("go_googleapis", "google/cloud/runtimeconfig/v1beta1", "runtimeconfig_go_proto"), + "google.golang.org/genproto/googleapis/cloud/speech/v1": label.New("go_googleapis", "google/cloud/speech/v1", "speech_go_proto"), + "google.golang.org/genproto/googleapis/cloud/speech/v1p1beta1": label.New("go_googleapis", "google/cloud/speech/v1p1beta1", "speech_go_proto"), + "google.golang.org/genproto/googleapis/cloud/support/common": label.New("go_googleapis", "google/cloud/support", "common_go_proto"), + "google.golang.org/genproto/googleapis/cloud/support/v1alpha1": label.New("go_googleapis", "google/cloud/support/v1alpha1", "support_go_proto"), + "google.golang.org/genproto/googleapis/cloud/tasks/v2beta2": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_go_proto"), + "google.golang.org/genproto/googleapis/cloud/tasks/v2beta3": label.New("go_googleapis", "google/cloud/tasks/v2beta3", "tasks_go_proto"), + "google.golang.org/genproto/googleapis/cloud/texttospeech/v1": label.New("go_googleapis", "google/cloud/texttospeech/v1", "texttospeech_go_proto"), + "google.golang.org/genproto/googleapis/cloud/texttospeech/v1beta1": label.New("go_googleapis", "google/cloud/texttospeech/v1beta1", "texttospeech_go_proto"), + "google.golang.org/genproto/googleapis/cloud/videointelligence/v1": label.New("go_googleapis", "google/cloud/videointelligence/v1", "videointelligence_go_proto"), + "google.golang.org/genproto/googleapis/cloud/videointelligence/v1beta1": label.New("go_googleapis", "google/cloud/videointelligence/v1beta1", "videointelligence_go_proto"), + "google.golang.org/genproto/googleapis/cloud/videointelligence/v1beta2": label.New("go_googleapis", "google/cloud/videointelligence/v1beta2", "videointelligence_go_proto"), + "google.golang.org/genproto/googleapis/cloud/videointelligence/v1p1beta1": label.New("go_googleapis", "google/cloud/videointelligence/v1p1beta1", "videointelligence_go_proto"), + "google.golang.org/genproto/googleapis/cloud/videointelligence/v1p2beta1": label.New("go_googleapis", "google/cloud/videointelligence/v1p2beta1", "videointelligence_go_proto"), + "google.golang.org/genproto/googleapis/cloud/vision/v1": label.New("go_googleapis", "google/cloud/vision/v1", "vision_go_proto"), + "google.golang.org/genproto/googleapis/cloud/vision/v1p1beta1": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_go_proto"), + "google.golang.org/genproto/googleapis/cloud/vision/v1p2beta1": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_go_proto"), + "google.golang.org/genproto/googleapis/cloud/vision/v1p3beta1": label.New("go_googleapis", "google/cloud/vision/v1p3beta1", "vision_go_proto"), + "google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), + "google.golang.org/genproto/googleapis/container/v1": label.New("go_googleapis", "google/container/v1", "container_go_proto"), + "google.golang.org/genproto/googleapis/container/v1alpha1": label.New("go_googleapis", "google/container/v1alpha1", "container_go_proto"), + "google.golang.org/genproto/googleapis/container/v1beta1": label.New("go_googleapis", "google/container/v1beta1", "container_go_proto"), + "google.golang.org/genproto/googleapis/datastore/admin/v1": label.New("go_googleapis", "google/datastore/admin/v1", "admin_go_proto"), + "google.golang.org/genproto/googleapis/datastore/admin/v1beta1": label.New("go_googleapis", "google/datastore/admin/v1beta1", "admin_go_proto"), + "google.golang.org/genproto/googleapis/datastore/v1": label.New("go_googleapis", "google/datastore/v1", "datastore_go_proto"), + "google.golang.org/genproto/googleapis/datastore/v1beta3": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_go_proto"), + "google.golang.org/genproto/googleapis/devtools/build/v1": label.New("go_googleapis", "google/devtools/build/v1", "build_go_proto"), + "google.golang.org/genproto/googleapis/devtools/cloudbuild/v1": label.New("go_googleapis", "google/devtools/cloudbuild/v1", "cloudbuild_go_proto"), + "google.golang.org/genproto/googleapis/devtools/clouddebugger/v2": label.New("go_googleapis", "google/devtools/clouddebugger/v2", "clouddebugger_go_proto"), + "google.golang.org/genproto/googleapis/devtools/clouderrorreporting/v1beta1": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_go_proto"), + "google.golang.org/genproto/googleapis/devtools/cloudprofiler/v2": label.New("go_googleapis", "google/devtools/cloudprofiler/v2", "cloudprofiler_go_proto"), + "google.golang.org/genproto/googleapis/devtools/cloudtrace/v1": label.New("go_googleapis", "google/devtools/cloudtrace/v1", "cloudtrace_go_proto"), + "google.golang.org/genproto/googleapis/devtools/cloudtrace/v2": label.New("go_googleapis", "google/devtools/cloudtrace/v2", "cloudtrace_go_proto"), + "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), + "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/attestation": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/attestation", "attestation_go_proto"), + "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/build": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/build", "build_go_proto"), + "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/common": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/common", "common_go_proto"), + "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1", "containeranalysis_go_proto"), + "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/deployment": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/deployment", "deployment_go_proto"), + "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/discovery": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/discovery", "discovery_go_proto"), + "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/grafeas": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/grafeas", "grafeas_go_proto"), + "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/image": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/image", "image_go_proto"), + "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/package": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/package", "package_go_proto"), + "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/provenance": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/provenance", "provenance_go_proto"), + "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/source": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/source", "source_go_proto"), + "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/vulnerability": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/vulnerability", "vulnerability_go_proto"), + "google.golang.org/genproto/googleapis/devtools/remoteexecution/v1test": label.New("go_googleapis", "google/devtools/remoteexecution/v1test", "remoteexecution_go_proto"), + "google.golang.org/genproto/googleapis/devtools/remoteworkers/v1test2": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_go_proto"), + "google.golang.org/genproto/googleapis/devtools/resultstore/v2": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google.golang.org/genproto/googleapis/devtools/source/v1": label.New("go_googleapis", "google/devtools/source/v1", "source_go_proto"), + "google.golang.org/genproto/googleapis/devtools/sourcerepo/v1": label.New("go_googleapis", "google/devtools/sourcerepo/v1", "sourcerepo_go_proto"), + "google.golang.org/genproto/googleapis/example/library/v1": label.New("go_googleapis", "google/example/library/v1", "library_go_proto"), + "google.golang.org/genproto/googleapis/firestore/admin/v1beta1": label.New("go_googleapis", "google/firestore/admin/v1beta1", "admin_go_proto"), + "google.golang.org/genproto/googleapis/firestore/admin/v1beta2": label.New("go_googleapis", "google/firestore/admin/v1beta2", "admin_go_proto"), + "google.golang.org/genproto/googleapis/firestore/v1beta1": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_go_proto"), + "google.golang.org/genproto/googleapis/genomics/v1": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), + "google.golang.org/genproto/googleapis/genomics/v1alpha2": label.New("go_googleapis", "google/genomics/v1alpha2", "genomics_go_proto"), + "google.golang.org/genproto/googleapis/home/graph/v1": label.New("go_googleapis", "google/home/graph/v1", "graph_go_proto"), + "google.golang.org/genproto/googleapis/iam/admin/v1": label.New("go_googleapis", "google/iam/admin/v1", "admin_go_proto"), + "google.golang.org/genproto/googleapis/iam/credentials/v1": label.New("go_googleapis", "google/iam/credentials/v1", "credentials_go_proto"), + "google.golang.org/genproto/googleapis/iam/v1": label.New("go_googleapis", "google/iam/v1", "iam_go_proto"), + "google.golang.org/genproto/googleapis/iam/v1/logging": label.New("go_googleapis", "google/iam/v1/logging", "logging_go_proto"), + "google.golang.org/genproto/googleapis/logging/type": label.New("go_googleapis", "google/logging/type", "ltype_go_proto"), + "google.golang.org/genproto/googleapis/logging/v2": label.New("go_googleapis", "google/logging/v2", "logging_go_proto"), + "google.golang.org/genproto/googleapis/longrunning": label.New("go_googleapis", "google/longrunning", "longrunning_go_proto"), + "google.golang.org/genproto/googleapis/monitoring/v3": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), + "google.golang.org/genproto/googleapis/privacy/dlp/v2": label.New("go_googleapis", "google/privacy/dlp/v2", "dlp_go_proto"), "google.golang.org/genproto/googleapis/pubsub/v1": label.New("go_googleapis", "google/pubsub/v1", "pubsub_go_proto"), "google.golang.org/genproto/googleapis/pubsub/v1beta2": label.New("go_googleapis", "google/pubsub/v1beta2", "pubsub_go_proto"), - "google.golang.org/genproto/googleapis/spanner/v1": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), + "google.golang.org/genproto/googleapis/rpc/code": label.New("go_googleapis", "google/rpc", "code_go_proto"), + "google.golang.org/genproto/googleapis/rpc/errdetails": label.New("go_googleapis", "google/rpc", "errdetails_go_proto"), + "google.golang.org/genproto/googleapis/rpc/status": label.New("go_googleapis", "google/rpc", "status_go_proto"), "google.golang.org/genproto/googleapis/spanner/admin/database/v1": label.New("go_googleapis", "google/spanner/admin/database/v1", "database_go_proto"), "google.golang.org/genproto/googleapis/spanner/admin/instance/v1": label.New("go_googleapis", "google/spanner/admin/instance/v1", "instance_go_proto"), - "google.golang.org/genproto/googleapis/monitoring/v3": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), - "google.golang.org/genproto/googleapis/rpc/code": label.New("go_googleapis", "google/rpc", "code_go_proto"), - "google.golang.org/genproto/googleapis/rpc/status": label.New("go_googleapis", "google/rpc", "status_go_proto"), - "google.golang.org/genproto/googleapis/rpc/errdetails": label.New("go_googleapis", "google/rpc", "errdetails_go_proto"), + "google.golang.org/genproto/googleapis/spanner/v1": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), + "google.golang.org/genproto/googleapis/storagetransfer/v1": label.New("go_googleapis", "google/storagetransfer/v1", "storagetransfer_go_proto"), "google.golang.org/genproto/googleapis/streetview/publish/v1": label.New("go_googleapis", "google/streetview/publish/v1", "publish_go_proto"), - "google.golang.org/genproto/googleapis/logging/v2": label.New("go_googleapis", "google/logging/v2", "logging_go_proto"), - "google.golang.org/genproto/googleapis/logging/type": label.New("go_googleapis", "google/logging/type", "ltype_go_proto"), + "google.golang.org/genproto/googleapis/type/color": label.New("go_googleapis", "google/type", "color_go_proto"), + "google.golang.org/genproto/googleapis/type/date": label.New("go_googleapis", "google/type", "date_go_proto"), + "google.golang.org/genproto/googleapis/type/dayofweek": label.New("go_googleapis", "google/type", "dayofweek_go_proto"), + "google.golang.org/genproto/googleapis/type/latlng": label.New("go_googleapis", "google/type", "latlng_go_proto"), + "google.golang.org/genproto/googleapis/type/money": label.New("go_googleapis", "google/type", "money_go_proto"), + "google.golang.org/genproto/googleapis/type/postaladdress": label.New("go_googleapis", "google/type", "postaladdress_go_proto"), + "google.golang.org/genproto/googleapis/type/timeofday": label.New("go_googleapis", "google/type", "timeofday_go_proto"), + "google.golang.org/genproto/googleapis/watcher/v1": label.New("go_googleapis", "google/watcher/v1", "watcher_go_proto"), } diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/go/known_proto_imports.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/go/known_proto_imports.go index a83178b34ae..cd9dfc3c322 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/go/known_proto_imports.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/go/known_proto_imports.go @@ -20,281 +20,347 @@ var knownProtoImports = map[string]label.Label{ "google/protobuf/timestamp.proto": label.New("io_bazel_rules_go", "proto/wkt", "timestamp_go_proto"), "google/protobuf/type.proto": label.New("io_bazel_rules_go", "proto/wkt", "type_go_proto"), "google/protobuf/wrappers.proto": label.New("io_bazel_rules_go", "proto/wkt", "wrappers_go_proto"), - "google/assistant/embedded/v1alpha2/embedded_assistant.proto": label.New("go_googleapis", "google/assistant/embedded/v1alpha2", "embedded_go_proto"), - "google/assistant/embedded/v1alpha1/embedded_assistant.proto": label.New("go_googleapis", "google/assistant/embedded/v1alpha1", "embedded_go_proto"), - "google/home/graph/v1/device.proto": label.New("go_googleapis", "google/home/graph/v1", "graph_go_proto"), - "google/home/graph/v1/homegraph.proto": label.New("go_googleapis", "google/home/graph/v1", "graph_go_proto"), - "google/genomics/v1/operations.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), - "google/genomics/v1/variants.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), - "google/genomics/v1/position.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), - "google/genomics/v1/references.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), - "google/genomics/v1/cigar.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), - "google/genomics/v1/datasets.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), - "google/genomics/v1/readalignment.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), - "google/genomics/v1/annotations.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), - "google/genomics/v1/reads.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), - "google/genomics/v1/readgroup.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), - "google/genomics/v1/readgroupset.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), - "google/genomics/v1/range.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), - "google/genomics/v1alpha2/pipelines.proto": label.New("go_googleapis", "google/genomics/v1alpha2", "genomics_go_proto"), - "google/bigtable/v1/bigtable_service_messages.proto": label.New("go_googleapis", "google/bigtable/v1", "bigtable_go_proto"), - "google/bigtable/v1/bigtable_service.proto": label.New("go_googleapis", "google/bigtable/v1", "bigtable_go_proto"), - "google/bigtable/v1/bigtable_data.proto": label.New("go_googleapis", "google/bigtable/v1", "bigtable_go_proto"), - "google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_go_proto"), - "google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_go_proto"), - "google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_go_proto"), - "google/bigtable/admin/v2/bigtable_instance_admin.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_go_proto"), - "google/bigtable/admin/v2/instance.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_go_proto"), - "google/bigtable/admin/v2/table.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_go_proto"), - "google/bigtable/admin/v2/bigtable_table_admin.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_go_proto"), - "google/bigtable/admin/v2/common.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_go_proto"), - "google/bigtable/admin/table/v1/bigtable_table_service_messages.proto": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_go_proto"), - "google/bigtable/admin/table/v1/bigtable_table_service.proto": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_go_proto"), - "google/bigtable/admin/table/v1/bigtable_table_data.proto": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_go_proto"), - "google/bigtable/v2/bigtable.proto": label.New("go_googleapis", "google/bigtable/v2", "bigtable_go_proto"), - "google/bigtable/v2/data.proto": label.New("go_googleapis", "google/bigtable/v2", "bigtable_go_proto"), - "google/privacy/dlp/v2/storage.proto": label.New("go_googleapis", "google/privacy/dlp/v2", "dlp_go_proto"), - "google/privacy/dlp/v2/dlp.proto": label.New("go_googleapis", "google/privacy/dlp/v2", "dlp_go_proto"), - "google/watcher/v1/watch.proto": label.New("go_googleapis", "google/watcher/v1", "watcher_go_proto"), - "google/firestore/admin/v1beta1/firestore_admin.proto": label.New("go_googleapis", "google/firestore/admin/v1beta1", "admin_go_proto"), - "google/firestore/admin/v1beta1/index.proto": label.New("go_googleapis", "google/firestore/admin/v1beta1", "admin_go_proto"), - "google/firestore/v1beta1/write.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_go_proto"), - "google/firestore/v1beta1/document.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_go_proto"), - "google/firestore/v1beta1/firestore.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_go_proto"), - "google/firestore/v1beta1/query.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_go_proto"), - "google/firestore/v1beta1/common.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_go_proto"), - "google/example/library/v1/library.proto": label.New("go_googleapis", "google/example/library/v1", "library_go_proto"), - "google/appengine/v1/instance.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), - "google/appengine/v1/audit_data.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), - "google/appengine/v1/appengine.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), - "google/appengine/v1/application.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), - "google/appengine/v1/operation.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), - "google/appengine/v1/app_yaml.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), - "google/appengine/v1/location.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), - "google/appengine/v1/service.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), - "google/appengine/v1/deploy.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), - "google/appengine/v1/version.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), + "google/api/annotations.proto": label.New("go_googleapis", "google/api", "annotations_go_proto"), + "google/api/auth.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/backend.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/billing.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/config_change.proto": label.New("go_googleapis", "google/api", "configchange_go_proto"), + "google/api/consumer.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/context.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/control.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/distribution.proto": label.New("go_googleapis", "google/api", "distribution_go_proto"), + "google/api/documentation.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/endpoint.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/experimental/authorization_config.proto": label.New("go_googleapis", "google/api", "api_go_proto"), + "google/api/experimental/experimental.proto": label.New("go_googleapis", "google/api", "api_go_proto"), + "google/api/expr/v1alpha1/cel_service.proto": label.New("go_googleapis", "google/api/expr/v1alpha1", "expr_go_proto"), + "google/api/expr/v1alpha1/checked.proto": label.New("go_googleapis", "google/api/expr/v1alpha1", "expr_go_proto"), + "google/api/expr/v1alpha1/eval.proto": label.New("go_googleapis", "google/api/expr/v1alpha1", "expr_go_proto"), + "google/api/expr/v1alpha1/explain.proto": label.New("go_googleapis", "google/api/expr/v1alpha1", "expr_go_proto"), + "google/api/expr/v1alpha1/syntax.proto": label.New("go_googleapis", "google/api/expr/v1alpha1", "expr_go_proto"), + "google/api/expr/v1alpha1/value.proto": label.New("go_googleapis", "google/api/expr/v1alpha1", "expr_go_proto"), + "google/api/expr/v1beta1/decl.proto": label.New("go_googleapis", "google/api/expr/v1beta1", "expr_go_proto"), + "google/api/expr/v1beta1/eval.proto": label.New("go_googleapis", "google/api/expr/v1beta1", "expr_go_proto"), + "google/api/expr/v1beta1/expr.proto": label.New("go_googleapis", "google/api/expr/v1beta1", "expr_go_proto"), + "google/api/expr/v1beta1/source.proto": label.New("go_googleapis", "google/api/expr/v1beta1", "expr_go_proto"), + "google/api/expr/v1beta1/value.proto": label.New("go_googleapis", "google/api/expr/v1beta1", "expr_go_proto"), + "google/api/http.proto": label.New("go_googleapis", "google/api", "annotations_go_proto"), + "google/api/httpbody.proto": label.New("go_googleapis", "google/api", "httpbody_go_proto"), + "google/api/label.proto": label.New("go_googleapis", "google/api", "label_go_proto"), + "google/api/launch_stage.proto": label.New("go_googleapis", "google/api", "api_go_proto"), + "google/api/log.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/logging.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/metric.proto": label.New("go_googleapis", "google/api", "metric_go_proto"), + "google/api/monitored_resource.proto": label.New("go_googleapis", "google/api", "monitoredres_go_proto"), + "google/api/monitoring.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/quota.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/service.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/servicecontrol/v1/check_error.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), + "google/api/servicecontrol/v1/distribution.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), + "google/api/servicecontrol/v1/log_entry.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), + "google/api/servicecontrol/v1/metric_value.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), + "google/api/servicecontrol/v1/operation.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), + "google/api/servicecontrol/v1/quota_controller.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), + "google/api/servicecontrol/v1/service_controller.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), + "google/api/servicemanagement/v1/resources.proto": label.New("go_googleapis", "google/api/servicemanagement/v1", "servicemanagement_go_proto"), + "google/api/servicemanagement/v1/servicemanager.proto": label.New("go_googleapis", "google/api/servicemanagement/v1", "servicemanagement_go_proto"), + "google/api/source_info.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/system_parameter.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), + "google/api/usage.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), "google/appengine/legacy/audit_data.proto": label.New("go_googleapis", "google/appengine/legacy", "legacy_go_proto"), "google/appengine/logging/v1/request_log.proto": label.New("go_googleapis", "google/appengine/logging/v1", "logging_go_proto"), - "google/storagetransfer/v1/transfer.proto": label.New("go_googleapis", "google/storagetransfer/v1", "storagetransfer_go_proto"), - "google/storagetransfer/v1/transfer_types.proto": label.New("go_googleapis", "google/storagetransfer/v1", "storagetransfer_go_proto"), - "google/longrunning/operations.proto": label.New("go_googleapis", "google/longrunning", "longrunning_go_proto"), - "google/container/v1/cluster_service.proto": label.New("go_googleapis", "google/container/v1", "container_go_proto"), - "google/container/v1beta1/cluster_service.proto": label.New("go_googleapis", "google/container/v1beta1", "container_go_proto"), - "google/container/v1alpha1/cluster_service.proto": label.New("go_googleapis", "google/container/v1alpha1", "container_go_proto"), - "google/datastore/v1beta3/datastore.proto": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_go_proto"), - "google/datastore/v1beta3/query.proto": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_go_proto"), - "google/datastore/v1beta3/entity.proto": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_go_proto"), - "google/datastore/v1/datastore.proto": label.New("go_googleapis", "google/datastore/v1", "datastore_go_proto"), - "google/datastore/v1/query.proto": label.New("go_googleapis", "google/datastore/v1", "datastore_go_proto"), - "google/datastore/v1/entity.proto": label.New("go_googleapis", "google/datastore/v1", "datastore_go_proto"), - "google/datastore/admin/v1/datastore_admin.proto": label.New("go_googleapis", "google/datastore/admin/v1", "admin_go_proto"), - "google/datastore/admin/v1beta1/datastore_admin.proto": label.New("go_googleapis", "google/datastore/admin/v1beta1", "admin_go_proto"), + "google/appengine/v1/app_yaml.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), + "google/appengine/v1/appengine.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), + "google/appengine/v1/application.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), + "google/appengine/v1/audit_data.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), + "google/appengine/v1/deploy.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), + "google/appengine/v1/instance.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), + "google/appengine/v1/location.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), + "google/appengine/v1/operation.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), + "google/appengine/v1/service.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), + "google/appengine/v1/version.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_go_proto"), + "google/assistant/embedded/v1alpha1/embedded_assistant.proto": label.New("go_googleapis", "google/assistant/embedded/v1alpha1", "embedded_go_proto"), + "google/assistant/embedded/v1alpha2/embedded_assistant.proto": label.New("go_googleapis", "google/assistant/embedded/v1alpha2", "embedded_go_proto"), + "google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_go_proto"), + "google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_go_proto"), + "google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_go_proto"), + "google/bigtable/admin/table/v1/bigtable_table_data.proto": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_go_proto"), + "google/bigtable/admin/table/v1/bigtable_table_service.proto": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_go_proto"), + "google/bigtable/admin/table/v1/bigtable_table_service_messages.proto": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_go_proto"), + "google/bigtable/admin/v2/bigtable_instance_admin.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_go_proto"), + "google/bigtable/admin/v2/bigtable_table_admin.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_go_proto"), + "google/bigtable/admin/v2/common.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_go_proto"), + "google/bigtable/admin/v2/instance.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_go_proto"), + "google/bigtable/admin/v2/table.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_go_proto"), + "google/bigtable/v1/bigtable_data.proto": label.New("go_googleapis", "google/bigtable/v1", "bigtable_go_proto"), + "google/bigtable/v1/bigtable_service.proto": label.New("go_googleapis", "google/bigtable/v1", "bigtable_go_proto"), + "google/bigtable/v1/bigtable_service_messages.proto": label.New("go_googleapis", "google/bigtable/v1", "bigtable_go_proto"), + "google/bigtable/v2/bigtable.proto": label.New("go_googleapis", "google/bigtable/v2", "bigtable_go_proto"), + "google/bigtable/v2/data.proto": label.New("go_googleapis", "google/bigtable/v2", "bigtable_go_proto"), "google/bytestream/bytestream.proto": label.New("go_googleapis", "google/bytestream", "bytestream_go_proto"), - "google/iam/v1/iam_policy.proto": label.New("go_googleapis", "google/iam/v1", "iam_go_proto"), - "google/iam/v1/policy.proto": label.New("go_googleapis", "google/iam/v1", "iam_go_proto"), - "google/iam/v1/logging/audit_data.proto": label.New("go_googleapis", "google/iam/v1/logging", "logging_go_proto"), - "google/iam/admin/v1/iam.proto": label.New("go_googleapis", "google/iam/admin/v1", "admin_go_proto"), - "google/type/money.proto": label.New("go_googleapis", "google/type", "money_go_proto"), - "google/type/latlng.proto": label.New("go_googleapis", "google/type", "latlng_go_proto"), - "google/type/color.proto": label.New("go_googleapis", "google/type", "color_go_proto"), - "google/type/timeofday.proto": label.New("go_googleapis", "google/type", "timeofday_go_proto"), - "google/type/date.proto": label.New("go_googleapis", "google/type", "date_go_proto"), - "google/type/dayofweek.proto": label.New("go_googleapis", "google/type", "dayofweek_go_proto"), - "google/type/postal_address.proto": label.New("go_googleapis", "google/type", "postaladdress_go_proto"), - "google/devtools/clouderrorreporting/v1beta1/report_errors_service.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_go_proto"), - "google/devtools/clouderrorreporting/v1beta1/error_group_service.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_go_proto"), - "google/devtools/clouderrorreporting/v1beta1/error_stats_service.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_go_proto"), - "google/devtools/clouderrorreporting/v1beta1/common.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_go_proto"), - "google/devtools/resultstore/v2/file.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google/devtools/resultstore/v2/resultstore_download.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google/devtools/resultstore/v2/configuration.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google/devtools/resultstore/v2/action.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google/devtools/resultstore/v2/resultstore_file_download.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google/devtools/resultstore/v2/test_suite.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google/devtools/resultstore/v2/file_set.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google/devtools/resultstore/v2/coverage.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google/devtools/resultstore/v2/coverage_summary.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google/devtools/resultstore/v2/configured_target.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google/devtools/resultstore/v2/target.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google/devtools/resultstore/v2/invocation.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google/devtools/resultstore/v2/common.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), - "google/devtools/source/v1/source_context.proto": label.New("go_googleapis", "google/devtools/source/v1", "source_go_proto"), - "google/devtools/remoteexecution/v1test/remote_execution.proto": label.New("go_googleapis", "google/devtools/remoteexecution/v1test", "remoteexecution_go_proto"), - "google/devtools/cloudbuild/v1/cloudbuild.proto": label.New("go_googleapis", "google/devtools/cloudbuild/v1", "cloudbuild_go_proto"), - "google/devtools/sourcerepo/v1/sourcerepo.proto": label.New("go_googleapis", "google/devtools/sourcerepo/v1", "sourcerepo_go_proto"), - "google/devtools/remoteworkers/v1test2/worker.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_go_proto"), - "google/devtools/remoteworkers/v1test2/tasks.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_go_proto"), - "google/devtools/remoteworkers/v1test2/bots.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_go_proto"), - "google/devtools/remoteworkers/v1test2/command.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_go_proto"), - "google/devtools/cloudtrace/v1/trace.proto": label.New("go_googleapis", "google/devtools/cloudtrace/v1", "cloudtrace_go_proto"), - "google/devtools/cloudtrace/v2/trace.proto": label.New("go_googleapis", "google/devtools/cloudtrace/v2", "cloudtrace_go_proto"), - "google/devtools/cloudtrace/v2/tracing.proto": label.New("go_googleapis", "google/devtools/cloudtrace/v2", "cloudtrace_go_proto"), - "google/devtools/cloudprofiler/v2/profiler.proto": label.New("go_googleapis", "google/devtools/cloudprofiler/v2", "cloudprofiler_go_proto"), - "google/devtools/containeranalysis/v1alpha1/containeranalysis.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), - "google/devtools/containeranalysis/v1alpha1/bill_of_materials.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), - "google/devtools/containeranalysis/v1alpha1/provenance.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), - "google/devtools/containeranalysis/v1alpha1/package_vulnerability.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), - "google/devtools/containeranalysis/v1alpha1/source_context.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), - "google/devtools/containeranalysis/v1alpha1/image_basis.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), + "google/cloud/asset/v1beta1/asset_service.proto": label.New("go_googleapis", "google/cloud/asset/v1beta1", "asset_go_proto"), + "google/cloud/asset/v1beta1/assets.proto": label.New("go_googleapis", "google/cloud/asset/v1beta1", "asset_go_proto"), + "google/cloud/audit/audit_log.proto": label.New("go_googleapis", "google/cloud/audit", "audit_go_proto"), + "google/cloud/automl/v1beta1/annotation_payload.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), + "google/cloud/automl/v1beta1/classification.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), + "google/cloud/automl/v1beta1/data_items.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), + "google/cloud/automl/v1beta1/dataset.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), + "google/cloud/automl/v1beta1/image.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), + "google/cloud/automl/v1beta1/io.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), + "google/cloud/automl/v1beta1/model.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), + "google/cloud/automl/v1beta1/model_evaluation.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), + "google/cloud/automl/v1beta1/operations.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), + "google/cloud/automl/v1beta1/prediction_service.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), + "google/cloud/automl/v1beta1/service.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), + "google/cloud/automl/v1beta1/text.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), + "google/cloud/automl/v1beta1/translation.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_go_proto"), + "google/cloud/bigquery/datatransfer/v1/datatransfer.proto": label.New("go_googleapis", "google/cloud/bigquery/datatransfer/v1", "datatransfer_go_proto"), + "google/cloud/bigquery/datatransfer/v1/transfer.proto": label.New("go_googleapis", "google/cloud/bigquery/datatransfer/v1", "datatransfer_go_proto"), + "google/cloud/bigquery/logging/v1/audit_data.proto": label.New("go_googleapis", "google/cloud/bigquery/logging/v1", "logging_go_proto"), + "google/cloud/bigquery/storage/v1beta1/avro.proto": label.New("go_googleapis", "google/cloud/bigquery/storage/v1beta1", "storage_go_proto"), + "google/cloud/bigquery/storage/v1beta1/read_options.proto": label.New("go_googleapis", "google/cloud/bigquery/storage/v1beta1", "storage_go_proto"), + "google/cloud/bigquery/storage/v1beta1/storage.proto": label.New("go_googleapis", "google/cloud/bigquery/storage/v1beta1", "storage_go_proto"), + "google/cloud/bigquery/storage/v1beta1/table_reference.proto": label.New("go_googleapis", "google/cloud/bigquery/storage/v1beta1", "storage_go_proto"), + "google/cloud/billing/v1/cloud_billing.proto": label.New("go_googleapis", "google/cloud/billing/v1", "billing_go_proto"), + "google/cloud/dataproc/v1/clusters.proto": label.New("go_googleapis", "google/cloud/dataproc/v1", "dataproc_go_proto"), + "google/cloud/dataproc/v1/jobs.proto": label.New("go_googleapis", "google/cloud/dataproc/v1", "dataproc_go_proto"), + "google/cloud/dataproc/v1/operations.proto": label.New("go_googleapis", "google/cloud/dataproc/v1", "dataproc_go_proto"), + "google/cloud/dataproc/v1beta2/clusters.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_go_proto"), + "google/cloud/dataproc/v1beta2/jobs.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_go_proto"), + "google/cloud/dataproc/v1beta2/operations.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_go_proto"), + "google/cloud/dataproc/v1beta2/shared.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_go_proto"), + "google/cloud/dataproc/v1beta2/workflow_templates.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_go_proto"), + "google/cloud/dialogflow/v2/agent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2/context.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2/entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2/intent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2/session.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2/session_entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2/webhook.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2beta1/agent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2beta1/audio_config.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2beta1/context.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2beta1/document.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2beta1/entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2beta1/intent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2beta1/knowledge_base.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2beta1/session.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2beta1/session_entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), + "google/cloud/dialogflow/v2beta1/webhook.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), + "google/cloud/functions/v1beta2/functions.proto": label.New("go_googleapis", "google/cloud/functions/v1beta2", "functions_go_proto"), + "google/cloud/functions/v1beta2/operations.proto": label.New("go_googleapis", "google/cloud/functions/v1beta2", "functions_go_proto"), + "google/cloud/iot/v1/device_manager.proto": label.New("go_googleapis", "google/cloud/iot/v1", "iot_go_proto"), + "google/cloud/iot/v1/resources.proto": label.New("go_googleapis", "google/cloud/iot/v1", "iot_go_proto"), + "google/cloud/kms/v1/resources.proto": label.New("go_googleapis", "google/cloud/kms/v1", "kms_go_proto"), + "google/cloud/kms/v1/service.proto": label.New("go_googleapis", "google/cloud/kms/v1", "kms_go_proto"), + "google/cloud/language/v1/language_service.proto": label.New("go_googleapis", "google/cloud/language/v1", "language_go_proto"), + "google/cloud/language/v1beta1/language_service.proto": label.New("go_googleapis", "google/cloud/language/v1beta1", "language_go_proto"), + "google/cloud/language/v1beta2/language_service.proto": label.New("go_googleapis", "google/cloud/language/v1beta2", "language_go_proto"), + "google/cloud/location/locations.proto": label.New("go_googleapis", "google/cloud/location", "location_go_proto"), + "google/cloud/ml/v1/job_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_go_proto"), + "google/cloud/ml/v1/model_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_go_proto"), + "google/cloud/ml/v1/operation_metadata.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_go_proto"), + "google/cloud/ml/v1/prediction_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_go_proto"), + "google/cloud/ml/v1/project_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_go_proto"), + "google/cloud/oslogin/common/common.proto": label.New("go_googleapis", "google/cloud/oslogin/common", "common_go_proto"), + "google/cloud/oslogin/v1/oslogin.proto": label.New("go_googleapis", "google/cloud/oslogin/v1", "oslogin_go_proto"), + "google/cloud/oslogin/v1alpha/oslogin.proto": label.New("go_googleapis", "google/cloud/oslogin/v1alpha", "oslogin_go_proto"), + "google/cloud/oslogin/v1beta/oslogin.proto": label.New("go_googleapis", "google/cloud/oslogin/v1beta", "oslogin_go_proto"), + "google/cloud/redis/v1/cloud_redis.proto": label.New("go_googleapis", "google/cloud/redis/v1", "redis_go_proto"), + "google/cloud/redis/v1beta1/cloud_redis.proto": label.New("go_googleapis", "google/cloud/redis/v1beta1", "redis_go_proto"), + "google/cloud/resourcemanager/v2/folders.proto": label.New("go_googleapis", "google/cloud/resourcemanager/v2", "resourcemanager_go_proto"), + "google/cloud/runtimeconfig/v1beta1/resources.proto": label.New("go_googleapis", "google/cloud/runtimeconfig/v1beta1", "runtimeconfig_go_proto"), + "google/cloud/runtimeconfig/v1beta1/runtimeconfig.proto": label.New("go_googleapis", "google/cloud/runtimeconfig/v1beta1", "runtimeconfig_go_proto"), + "google/cloud/speech/v1/cloud_speech.proto": label.New("go_googleapis", "google/cloud/speech/v1", "speech_go_proto"), + "google/cloud/speech/v1p1beta1/cloud_speech.proto": label.New("go_googleapis", "google/cloud/speech/v1p1beta1", "speech_go_proto"), + "google/cloud/support/common.proto": label.New("go_googleapis", "google/cloud/support", "common_go_proto"), + "google/cloud/support/v1alpha1/cloud_support.proto": label.New("go_googleapis", "google/cloud/support/v1alpha1", "support_go_proto"), + "google/cloud/tasks/v2beta2/cloudtasks.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_go_proto"), + "google/cloud/tasks/v2beta2/queue.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_go_proto"), + "google/cloud/tasks/v2beta2/target.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_go_proto"), + "google/cloud/tasks/v2beta2/task.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_go_proto"), + "google/cloud/tasks/v2beta3/cloudtasks.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta3", "tasks_go_proto"), + "google/cloud/tasks/v2beta3/queue.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta3", "tasks_go_proto"), + "google/cloud/tasks/v2beta3/target.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta3", "tasks_go_proto"), + "google/cloud/tasks/v2beta3/task.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta3", "tasks_go_proto"), + "google/cloud/texttospeech/v1/cloud_tts.proto": label.New("go_googleapis", "google/cloud/texttospeech/v1", "texttospeech_go_proto"), + "google/cloud/texttospeech/v1beta1/cloud_tts.proto": label.New("go_googleapis", "google/cloud/texttospeech/v1beta1", "texttospeech_go_proto"), + "google/cloud/videointelligence/v1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1", "videointelligence_go_proto"), + "google/cloud/videointelligence/v1beta1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1beta1", "videointelligence_go_proto"), + "google/cloud/videointelligence/v1beta2/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1beta2", "videointelligence_go_proto"), + "google/cloud/videointelligence/v1p1beta1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1p1beta1", "videointelligence_go_proto"), + "google/cloud/videointelligence/v1p2beta1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1p2beta1", "videointelligence_go_proto"), + "google/cloud/vision/v1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_go_proto"), + "google/cloud/vision/v1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_go_proto"), + "google/cloud/vision/v1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_go_proto"), + "google/cloud/vision/v1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_go_proto"), + "google/cloud/vision/v1p1beta1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_go_proto"), + "google/cloud/vision/v1p1beta1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_go_proto"), + "google/cloud/vision/v1p1beta1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_go_proto"), + "google/cloud/vision/v1p1beta1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_go_proto"), + "google/cloud/vision/v1p2beta1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_go_proto"), + "google/cloud/vision/v1p2beta1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_go_proto"), + "google/cloud/vision/v1p2beta1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_go_proto"), + "google/cloud/vision/v1p2beta1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_go_proto"), + "google/cloud/vision/v1p3beta1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1p3beta1", "vision_go_proto"), + "google/cloud/vision/v1p3beta1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1p3beta1", "vision_go_proto"), + "google/cloud/vision/v1p3beta1/product_search.proto": label.New("go_googleapis", "google/cloud/vision/v1p3beta1", "vision_go_proto"), + "google/cloud/vision/v1p3beta1/product_search_service.proto": label.New("go_googleapis", "google/cloud/vision/v1p3beta1", "vision_go_proto"), + "google/cloud/vision/v1p3beta1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1p3beta1", "vision_go_proto"), + "google/cloud/vision/v1p3beta1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1p3beta1", "vision_go_proto"), + "google/cloud/websecurityscanner/v1alpha/crawled_url.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), + "google/cloud/websecurityscanner/v1alpha/finding.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), + "google/cloud/websecurityscanner/v1alpha/finding_addon.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), + "google/cloud/websecurityscanner/v1alpha/finding_type_stats.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), + "google/cloud/websecurityscanner/v1alpha/scan_config.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), + "google/cloud/websecurityscanner/v1alpha/scan_run.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), + "google/cloud/websecurityscanner/v1alpha/web_security_scanner.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), + "google/container/v1/cluster_service.proto": label.New("go_googleapis", "google/container/v1", "container_go_proto"), + "google/container/v1alpha1/cluster_service.proto": label.New("go_googleapis", "google/container/v1alpha1", "container_go_proto"), + "google/container/v1beta1/cluster_service.proto": label.New("go_googleapis", "google/container/v1beta1", "container_go_proto"), + "google/datastore/admin/v1/datastore_admin.proto": label.New("go_googleapis", "google/datastore/admin/v1", "admin_go_proto"), + "google/datastore/admin/v1/index.proto": label.New("go_googleapis", "google/datastore/admin/v1", "admin_go_proto"), + "google/datastore/admin/v1beta1/datastore_admin.proto": label.New("go_googleapis", "google/datastore/admin/v1beta1", "admin_go_proto"), + "google/datastore/v1/datastore.proto": label.New("go_googleapis", "google/datastore/v1", "datastore_go_proto"), + "google/datastore/v1/entity.proto": label.New("go_googleapis", "google/datastore/v1", "datastore_go_proto"), + "google/datastore/v1/query.proto": label.New("go_googleapis", "google/datastore/v1", "datastore_go_proto"), + "google/datastore/v1beta3/datastore.proto": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_go_proto"), + "google/datastore/v1beta3/entity.proto": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_go_proto"), + "google/datastore/v1beta3/query.proto": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_go_proto"), "google/devtools/build/v1/build_events.proto": label.New("go_googleapis", "google/devtools/build/v1", "build_go_proto"), "google/devtools/build/v1/build_status.proto": label.New("go_googleapis", "google/devtools/build/v1", "build_go_proto"), "google/devtools/build/v1/publish_build_event.proto": label.New("go_googleapis", "google/devtools/build/v1", "build_go_proto"), - "google/devtools/clouddebugger/v2/debugger.proto": label.New("go_googleapis", "google/devtools/clouddebugger/v2", "clouddebugger_go_proto"), - "google/devtools/clouddebugger/v2/data.proto": label.New("go_googleapis", "google/devtools/clouddebugger/v2", "clouddebugger_go_proto"), + "google/devtools/cloudbuild/v1/cloudbuild.proto": label.New("go_googleapis", "google/devtools/cloudbuild/v1", "cloudbuild_go_proto"), "google/devtools/clouddebugger/v2/controller.proto": label.New("go_googleapis", "google/devtools/clouddebugger/v2", "clouddebugger_go_proto"), - "google/cloud/resourcemanager/v2/folders.proto": label.New("go_googleapis", "google/cloud/resourcemanager/v2", "resourcemanager_go_proto"), - "google/cloud/kms/v1/resources.proto": label.New("go_googleapis", "google/cloud/kms/v1", "kms_go_proto"), - "google/cloud/kms/v1/service.proto": label.New("go_googleapis", "google/cloud/kms/v1", "kms_go_proto"), - "google/cloud/runtimeconfig/v1beta1/resources.proto": label.New("go_googleapis", "google/cloud/runtimeconfig/v1beta1", "runtimeconfig_go_proto"), - "google/cloud/runtimeconfig/v1beta1/runtimeconfig.proto": label.New("go_googleapis", "google/cloud/runtimeconfig/v1beta1", "runtimeconfig_go_proto"), - "google/cloud/tasks/v2beta2/queue.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_go_proto"), - "google/cloud/tasks/v2beta2/task.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_go_proto"), - "google/cloud/tasks/v2beta2/target.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_go_proto"), - "google/cloud/tasks/v2beta2/cloudtasks.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_go_proto"), - "google/cloud/oslogin/v1/oslogin.proto": label.New("go_googleapis", "google/cloud/oslogin/v1", "oslogin_go_proto"), - "google/cloud/oslogin/v1alpha/oslogin.proto": label.New("go_googleapis", "google/cloud/oslogin/v1alpha", "oslogin_go_proto"), - "google/cloud/oslogin/common/common.proto": label.New("go_googleapis", "google/cloud/oslogin/common", "common_go_proto"), - "google/cloud/oslogin/v1beta/oslogin.proto": label.New("go_googleapis", "google/cloud/oslogin/v1beta", "oslogin_go_proto"), - "google/cloud/dialogflow/v2beta1/context.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), - "google/cloud/dialogflow/v2beta1/session_entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), - "google/cloud/dialogflow/v2beta1/intent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), - "google/cloud/dialogflow/v2beta1/entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), - "google/cloud/dialogflow/v2beta1/webhook.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), - "google/cloud/dialogflow/v2beta1/session.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), - "google/cloud/dialogflow/v2beta1/agent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_go_proto"), - "google/cloud/dialogflow/v2/context.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), - "google/cloud/dialogflow/v2/session_entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), - "google/cloud/dialogflow/v2/intent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), - "google/cloud/dialogflow/v2/entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), - "google/cloud/dialogflow/v2/webhook.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), - "google/cloud/dialogflow/v2/session.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), - "google/cloud/dialogflow/v2/agent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_go_proto"), - "google/cloud/redis/v1beta1/cloud_redis.proto": label.New("go_googleapis", "google/cloud/redis/v1beta1", "redis_go_proto"), - "google/cloud/location/locations.proto": label.New("go_googleapis", "google/cloud/location", "location_go_proto"), - "google/cloud/websecurityscanner/v1alpha/finding.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), - "google/cloud/websecurityscanner/v1alpha/finding_type_stats.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), - "google/cloud/websecurityscanner/v1alpha/scan_config.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), - "google/cloud/websecurityscanner/v1alpha/crawled_url.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), - "google/cloud/websecurityscanner/v1alpha/scan_run.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), - "google/cloud/websecurityscanner/v1alpha/web_security_scanner.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), - "google/cloud/websecurityscanner/v1alpha/finding_addon.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_go_proto"), - "google/cloud/language/v1/language_service.proto": label.New("go_googleapis", "google/cloud/language/v1", "language_go_proto"), - "google/cloud/language/v1beta2/language_service.proto": label.New("go_googleapis", "google/cloud/language/v1beta2", "language_go_proto"), - "google/cloud/language/v1beta1/language_service.proto": label.New("go_googleapis", "google/cloud/language/v1beta1", "language_go_proto"), - "google/cloud/bigquery/datatransfer/v1/transfer.proto": label.New("go_googleapis", "google/cloud/bigquery/datatransfer/v1", "datatransfer_go_proto"), - "google/cloud/bigquery/datatransfer/v1/datatransfer.proto": label.New("go_googleapis", "google/cloud/bigquery/datatransfer/v1", "datatransfer_go_proto"), - "google/cloud/bigquery/logging/v1/audit_data.proto": label.New("go_googleapis", "google/cloud/bigquery/logging/v1", "logging_go_proto"), - "google/cloud/vision/v1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_go_proto"), - "google/cloud/vision/v1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_go_proto"), - "google/cloud/vision/v1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_go_proto"), - "google/cloud/vision/v1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_go_proto"), - "google/cloud/vision/v1p2beta1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_go_proto"), - "google/cloud/vision/v1p2beta1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_go_proto"), - "google/cloud/vision/v1p2beta1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_go_proto"), - "google/cloud/vision/v1p2beta1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_go_proto"), - "google/cloud/vision/v1p1beta1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_go_proto"), - "google/cloud/vision/v1p1beta1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_go_proto"), - "google/cloud/vision/v1p1beta1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_go_proto"), - "google/cloud/vision/v1p1beta1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_go_proto"), - "google/cloud/speech/v1/cloud_speech.proto": label.New("go_googleapis", "google/cloud/speech/v1", "speech_go_proto"), - "google/cloud/speech/v1beta1/cloud_speech.proto": label.New("go_googleapis", "google/cloud/speech/v1beta1", "speech_go_proto"), - "google/cloud/speech/v1p1beta1/cloud_speech.proto": label.New("go_googleapis", "google/cloud/speech/v1p1beta1", "speech_go_proto"), - "google/cloud/iot/v1/device_manager.proto": label.New("go_googleapis", "google/cloud/iot/v1", "iot_go_proto"), - "google/cloud/iot/v1/resources.proto": label.New("go_googleapis", "google/cloud/iot/v1", "iot_go_proto"), - "google/cloud/videointelligence/v1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1", "videointelligence_go_proto"), - "google/cloud/videointelligence/v1beta2/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1beta2", "videointelligence_go_proto"), - "google/cloud/videointelligence/v1beta1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1beta1", "videointelligence_go_proto"), - "google/cloud/videointelligence/v1p1beta1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1p1beta1", "videointelligence_go_proto"), - "google/cloud/audit/audit_log.proto": label.New("go_googleapis", "google/cloud/audit", "audit_go_proto"), - "google/cloud/support/common.proto": label.New("go_googleapis", "google/cloud/support", "common_go_proto"), - "google/cloud/support/v1alpha1/cloud_support.proto": label.New("go_googleapis", "google/cloud/support/v1alpha1", "support_go_proto"), - "google/cloud/ml/v1/operation_metadata.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_go_proto"), - "google/cloud/ml/v1/job_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_go_proto"), - "google/cloud/ml/v1/prediction_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_go_proto"), - "google/cloud/ml/v1/model_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_go_proto"), - "google/cloud/ml/v1/project_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_go_proto"), - "google/cloud/texttospeech/v1/cloud_tts.proto": label.New("go_googleapis", "google/cloud/texttospeech/v1", "texttospeech_go_proto"), - "google/cloud/texttospeech/v1beta1/cloud_tts.proto": label.New("go_googleapis", "google/cloud/texttospeech/v1beta1", "texttospeech_go_proto"), - "google/cloud/functions/v1beta2/operations.proto": label.New("go_googleapis", "google/cloud/functions/v1beta2", "functions_go_proto"), - "google/cloud/functions/v1beta2/functions.proto": label.New("go_googleapis", "google/cloud/functions/v1beta2", "functions_go_proto"), - "google/cloud/billing/v1/cloud_billing.proto": label.New("go_googleapis", "google/cloud/billing/v1", "billing_go_proto"), - "google/cloud/dataproc/v1/operations.proto": label.New("go_googleapis", "google/cloud/dataproc/v1", "dataproc_go_proto"), - "google/cloud/dataproc/v1/clusters.proto": label.New("go_googleapis", "google/cloud/dataproc/v1", "dataproc_go_proto"), - "google/cloud/dataproc/v1/jobs.proto": label.New("go_googleapis", "google/cloud/dataproc/v1", "dataproc_go_proto"), - "google/cloud/dataproc/v1beta2/operations.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_go_proto"), - "google/cloud/dataproc/v1beta2/clusters.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_go_proto"), - "google/cloud/dataproc/v1beta2/jobs.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_go_proto"), - "google/cloud/dataproc/v1beta2/workflow_templates.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_go_proto"), - "google/cloud/dataproc/v1beta2/shared.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_go_proto"), - "google/api/context.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/http.proto": label.New("go_googleapis", "google/api", "annotations_go_proto"), - "google/api/config_change.proto": label.New("go_googleapis", "google/api", "configchange_go_proto"), - "google/api/system_parameter.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/monitoring.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/distribution.proto": label.New("go_googleapis", "google/api", "distribution_go_proto"), - "google/api/endpoint.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/usage.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/monitored_resource.proto": label.New("go_googleapis", "google/api", "monitoredres_go_proto"), - "google/api/annotations.proto": label.New("go_googleapis", "google/api", "annotations_go_proto"), - "google/api/control.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/metric.proto": label.New("go_googleapis", "google/api", "metric_go_proto"), - "google/api/label.proto": label.New("go_googleapis", "google/api", "label_go_proto"), - "google/api/consumer.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/log.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/billing.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/service.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/logging.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/documentation.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/quota.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/auth.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/backend.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/source_info.proto": label.New("go_googleapis", "google/api", "serviceconfig_go_proto"), - "google/api/httpbody.proto": label.New("go_googleapis", "google/api", "httpbody_go_proto"), - "google/api/experimental/authorization_config.proto": label.New("go_googleapis", "google/api/experimental", "api_go_proto"), - "google/api/experimental/experimental.proto": label.New("go_googleapis", "google/api/experimental", "api_go_proto"), - "google/api/servicemanagement/v1/servicemanager.proto": label.New("go_googleapis", "google/api/servicemanagement/v1", "servicemanagement_go_proto"), - "google/api/servicemanagement/v1/resources.proto": label.New("go_googleapis", "google/api/servicemanagement/v1", "servicemanagement_go_proto"), - "google/api/servicecontrol/v1/quota_controller.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), - "google/api/servicecontrol/v1/distribution.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), - "google/api/servicecontrol/v1/check_error.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), - "google/api/servicecontrol/v1/operation.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), - "google/api/servicecontrol/v1/metric_value.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), - "google/api/servicecontrol/v1/log_entry.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), - "google/api/servicecontrol/v1/service_controller.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_go_proto"), - "google/pubsub/v1/pubsub.proto": label.New("go_googleapis", "google/pubsub/v1", "pubsub_go_proto"), - "google/pubsub/v1beta2/pubsub.proto": label.New("go_googleapis", "google/pubsub/v1beta2", "pubsub_go_proto"), - "google/spanner/v1/mutation.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), - "google/spanner/v1/spanner.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), - "google/spanner/v1/transaction.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), - "google/spanner/v1/keys.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), - "google/spanner/v1/type.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), - "google/spanner/v1/query_plan.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), - "google/spanner/v1/result_set.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), - "google/spanner/admin/database/v1/spanner_database_admin.proto": label.New("go_googleapis", "google/spanner/admin/database/v1", "database_go_proto"), - "google/spanner/admin/instance/v1/spanner_instance_admin.proto": label.New("go_googleapis", "google/spanner/admin/instance/v1", "instance_go_proto"), + "google/devtools/clouddebugger/v2/data.proto": label.New("go_googleapis", "google/devtools/clouddebugger/v2", "clouddebugger_go_proto"), + "google/devtools/clouddebugger/v2/debugger.proto": label.New("go_googleapis", "google/devtools/clouddebugger/v2", "clouddebugger_go_proto"), + "google/devtools/clouderrorreporting/v1beta1/common.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_go_proto"), + "google/devtools/clouderrorreporting/v1beta1/error_group_service.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_go_proto"), + "google/devtools/clouderrorreporting/v1beta1/error_stats_service.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_go_proto"), + "google/devtools/clouderrorreporting/v1beta1/report_errors_service.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_go_proto"), + "google/devtools/cloudprofiler/v2/profiler.proto": label.New("go_googleapis", "google/devtools/cloudprofiler/v2", "cloudprofiler_go_proto"), + "google/devtools/cloudtrace/v1/trace.proto": label.New("go_googleapis", "google/devtools/cloudtrace/v1", "cloudtrace_go_proto"), + "google/devtools/cloudtrace/v2/trace.proto": label.New("go_googleapis", "google/devtools/cloudtrace/v2", "cloudtrace_go_proto"), + "google/devtools/cloudtrace/v2/tracing.proto": label.New("go_googleapis", "google/devtools/cloudtrace/v2", "cloudtrace_go_proto"), + "google/devtools/containeranalysis/v1alpha1/bill_of_materials.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), + "google/devtools/containeranalysis/v1alpha1/containeranalysis.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), + "google/devtools/containeranalysis/v1alpha1/image_basis.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), + "google/devtools/containeranalysis/v1alpha1/package_vulnerability.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), + "google/devtools/containeranalysis/v1alpha1/provenance.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), + "google/devtools/containeranalysis/v1alpha1/source_context.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_go_proto"), + "google/devtools/containeranalysis/v1beta1/attestation/attestation.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/attestation", "attestation_go_proto"), + "google/devtools/containeranalysis/v1beta1/build/build.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/build", "build_go_proto"), + "google/devtools/containeranalysis/v1beta1/common/common.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/common", "common_go_proto"), + "google/devtools/containeranalysis/v1beta1/containeranalysis.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1", "containeranalysis_go_proto"), + "google/devtools/containeranalysis/v1beta1/deployment/deployment.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/deployment", "deployment_go_proto"), + "google/devtools/containeranalysis/v1beta1/discovery/discovery.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/discovery", "discovery_go_proto"), + "google/devtools/containeranalysis/v1beta1/grafeas/grafeas.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/grafeas", "grafeas_go_proto"), + "google/devtools/containeranalysis/v1beta1/image/image.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/image", "image_go_proto"), + "google/devtools/containeranalysis/v1beta1/package/package.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/package", "package_go_proto"), + "google/devtools/containeranalysis/v1beta1/provenance/provenance.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/provenance", "provenance_go_proto"), + "google/devtools/containeranalysis/v1beta1/source/source.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/source", "source_go_proto"), + "google/devtools/containeranalysis/v1beta1/vulnerability/vulnerability.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/vulnerability", "vulnerability_go_proto"), + "google/devtools/remoteexecution/v1test/remote_execution.proto": label.New("go_googleapis", "google/devtools/remoteexecution/v1test", "remoteexecution_go_proto"), + "google/devtools/remoteworkers/v1test2/bots.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_go_proto"), + "google/devtools/remoteworkers/v1test2/command.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_go_proto"), + "google/devtools/remoteworkers/v1test2/tasks.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_go_proto"), + "google/devtools/remoteworkers/v1test2/worker.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_go_proto"), + "google/devtools/resultstore/v2/action.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google/devtools/resultstore/v2/common.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google/devtools/resultstore/v2/configuration.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google/devtools/resultstore/v2/configured_target.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google/devtools/resultstore/v2/coverage.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google/devtools/resultstore/v2/coverage_summary.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google/devtools/resultstore/v2/file.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google/devtools/resultstore/v2/file_set.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google/devtools/resultstore/v2/invocation.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google/devtools/resultstore/v2/resultstore_download.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google/devtools/resultstore/v2/resultstore_file_download.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google/devtools/resultstore/v2/target.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google/devtools/resultstore/v2/test_suite.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_go_proto"), + "google/devtools/source/v1/source_context.proto": label.New("go_googleapis", "google/devtools/source/v1", "source_go_proto"), + "google/devtools/sourcerepo/v1/sourcerepo.proto": label.New("go_googleapis", "google/devtools/sourcerepo/v1", "sourcerepo_go_proto"), + "google/example/library/v1/library.proto": label.New("go_googleapis", "google/example/library/v1", "library_go_proto"), + "google/firestore/admin/v1beta1/firestore_admin.proto": label.New("go_googleapis", "google/firestore/admin/v1beta1", "admin_go_proto"), + "google/firestore/admin/v1beta1/index.proto": label.New("go_googleapis", "google/firestore/admin/v1beta1", "admin_go_proto"), + "google/firestore/admin/v1beta2/field.proto": label.New("go_googleapis", "google/firestore/admin/v1beta2", "admin_go_proto"), + "google/firestore/admin/v1beta2/firestore_admin.proto": label.New("go_googleapis", "google/firestore/admin/v1beta2", "admin_go_proto"), + "google/firestore/admin/v1beta2/index.proto": label.New("go_googleapis", "google/firestore/admin/v1beta2", "admin_go_proto"), + "google/firestore/admin/v1beta2/operation.proto": label.New("go_googleapis", "google/firestore/admin/v1beta2", "admin_go_proto"), + "google/firestore/v1beta1/common.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_go_proto"), + "google/firestore/v1beta1/document.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_go_proto"), + "google/firestore/v1beta1/firestore.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_go_proto"), + "google/firestore/v1beta1/query.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_go_proto"), + "google/firestore/v1beta1/write.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_go_proto"), + "google/genomics/v1/annotations.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), + "google/genomics/v1/cigar.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), + "google/genomics/v1/datasets.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), + "google/genomics/v1/operations.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), + "google/genomics/v1/position.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), + "google/genomics/v1/range.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), + "google/genomics/v1/readalignment.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), + "google/genomics/v1/readgroup.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), + "google/genomics/v1/readgroupset.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), + "google/genomics/v1/reads.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), + "google/genomics/v1/references.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), + "google/genomics/v1/variants.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_go_proto"), + "google/genomics/v1alpha2/pipelines.proto": label.New("go_googleapis", "google/genomics/v1alpha2", "genomics_go_proto"), + "google/home/graph/v1/device.proto": label.New("go_googleapis", "google/home/graph/v1", "graph_go_proto"), + "google/home/graph/v1/homegraph.proto": label.New("go_googleapis", "google/home/graph/v1", "graph_go_proto"), + "google/iam/admin/v1/iam.proto": label.New("go_googleapis", "google/iam/admin/v1", "admin_go_proto"), + "google/iam/credentials/v1/common.proto": label.New("go_googleapis", "google/iam/credentials/v1", "credentials_go_proto"), + "google/iam/credentials/v1/iamcredentials.proto": label.New("go_googleapis", "google/iam/credentials/v1", "credentials_go_proto"), + "google/iam/v1/iam_policy.proto": label.New("go_googleapis", "google/iam/v1", "iam_go_proto"), + "google/iam/v1/logging/audit_data.proto": label.New("go_googleapis", "google/iam/v1/logging", "logging_go_proto"), + "google/iam/v1/policy.proto": label.New("go_googleapis", "google/iam/v1", "iam_go_proto"), + "google/logging/type/http_request.proto": label.New("go_googleapis", "google/logging/type", "ltype_go_proto"), + "google/logging/type/log_severity.proto": label.New("go_googleapis", "google/logging/type", "ltype_go_proto"), + "google/logging/v2/log_entry.proto": label.New("go_googleapis", "google/logging/v2", "logging_go_proto"), + "google/logging/v2/logging.proto": label.New("go_googleapis", "google/logging/v2", "logging_go_proto"), + "google/logging/v2/logging_config.proto": label.New("go_googleapis", "google/logging/v2", "logging_go_proto"), + "google/logging/v2/logging_metrics.proto": label.New("go_googleapis", "google/logging/v2", "logging_go_proto"), + "google/longrunning/operations.proto": label.New("go_googleapis", "google/longrunning", "longrunning_go_proto"), + "google/monitoring/v3/alert.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), + "google/monitoring/v3/alert_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), + "google/monitoring/v3/common.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), + "google/monitoring/v3/dropped_labels.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), "google/monitoring/v3/group.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), + "google/monitoring/v3/group_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), + "google/monitoring/v3/metric.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), + "google/monitoring/v3/metric_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), "google/monitoring/v3/mutation_record.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), "google/monitoring/v3/notification.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), - "google/monitoring/v3/alert_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), - "google/monitoring/v3/uptime_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), - "google/monitoring/v3/group_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), - "google/monitoring/v3/alert.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), - "google/monitoring/v3/uptime.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), - "google/monitoring/v3/metric.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), "google/monitoring/v3/notification_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), - "google/monitoring/v3/metric_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), - "google/monitoring/v3/common.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), + "google/monitoring/v3/span_context.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), + "google/monitoring/v3/uptime.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), + "google/monitoring/v3/uptime_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_go_proto"), + "google/privacy/dlp/v2/dlp.proto": label.New("go_googleapis", "google/privacy/dlp/v2", "dlp_go_proto"), + "google/privacy/dlp/v2/storage.proto": label.New("go_googleapis", "google/privacy/dlp/v2", "dlp_go_proto"), + "google/pubsub/v1/pubsub.proto": label.New("go_googleapis", "google/pubsub/v1", "pubsub_go_proto"), + "google/pubsub/v1beta2/pubsub.proto": label.New("go_googleapis", "google/pubsub/v1beta2", "pubsub_go_proto"), "google/rpc/code.proto": label.New("go_googleapis", "google/rpc", "code_go_proto"), - "google/rpc/status.proto": label.New("go_googleapis", "google/rpc", "status_go_proto"), "google/rpc/error_details.proto": label.New("go_googleapis", "google/rpc", "errdetails_go_proto"), + "google/rpc/status.proto": label.New("go_googleapis", "google/rpc", "status_go_proto"), + "google/spanner/admin/database/v1/spanner_database_admin.proto": label.New("go_googleapis", "google/spanner/admin/database/v1", "database_go_proto"), + "google/spanner/admin/instance/v1/spanner_instance_admin.proto": label.New("go_googleapis", "google/spanner/admin/instance/v1", "instance_go_proto"), + "google/spanner/v1/keys.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), + "google/spanner/v1/mutation.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), + "google/spanner/v1/query_plan.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), + "google/spanner/v1/result_set.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), + "google/spanner/v1/spanner.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), + "google/spanner/v1/transaction.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), + "google/spanner/v1/type.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_go_proto"), + "google/storagetransfer/v1/transfer.proto": label.New("go_googleapis", "google/storagetransfer/v1", "storagetransfer_go_proto"), + "google/storagetransfer/v1/transfer_types.proto": label.New("go_googleapis", "google/storagetransfer/v1", "storagetransfer_go_proto"), "google/streetview/publish/v1/resources.proto": label.New("go_googleapis", "google/streetview/publish/v1", "publish_go_proto"), "google/streetview/publish/v1/rpcmessages.proto": label.New("go_googleapis", "google/streetview/publish/v1", "publish_go_proto"), "google/streetview/publish/v1/streetview_publish.proto": label.New("go_googleapis", "google/streetview/publish/v1", "publish_go_proto"), - "google/logging/v2/logging_metrics.proto": label.New("go_googleapis", "google/logging/v2", "logging_go_proto"), - "google/logging/v2/logging_config.proto": label.New("go_googleapis", "google/logging/v2", "logging_go_proto"), - "google/logging/v2/log_entry.proto": label.New("go_googleapis", "google/logging/v2", "logging_go_proto"), - "google/logging/v2/logging.proto": label.New("go_googleapis", "google/logging/v2", "logging_go_proto"), - "google/logging/type/log_severity.proto": label.New("go_googleapis", "google/logging/type", "ltype_go_proto"), - "google/logging/type/http_request.proto": label.New("go_googleapis", "google/logging/type", "ltype_go_proto"), + "google/type/color.proto": label.New("go_googleapis", "google/type", "color_go_proto"), + "google/type/date.proto": label.New("go_googleapis", "google/type", "date_go_proto"), + "google/type/dayofweek.proto": label.New("go_googleapis", "google/type", "dayofweek_go_proto"), + "google/type/latlng.proto": label.New("go_googleapis", "google/type", "latlng_go_proto"), + "google/type/money.proto": label.New("go_googleapis", "google/type", "money_go_proto"), + "google/type/postal_address.proto": label.New("go_googleapis", "google/type", "postaladdress_go_proto"), + "google/type/timeofday.proto": label.New("go_googleapis", "google/type", "timeofday_go_proto"), + "google/watcher/v1/watch.proto": label.New("go_googleapis", "google/watcher/v1", "watcher_go_proto"), } diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/go/resolve.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/go/resolve.go index e1d3098d673..283289242e1 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/go/resolve.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/go/resolve.go @@ -91,7 +91,14 @@ func (gl *goLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repos.Rem log.Print(err) } if !deps.IsEmpty() { - r.SetAttr("deps", deps) + if r.Kind() == "go_proto_library" { + // protos may import the same library multiple times by different names, + // so we need to de-duplicate them. Protos are not platform-specific, + // so it's safe to just flatten them. + r.SetAttr("deps", deps.Flat()) + } else { + r.SetAttr("deps", deps) + } } } @@ -115,6 +122,10 @@ func resolveGo(c *config.Config, ix *resolve.RuleIndex, rc *repos.RemoteCache, r return label.NoLabel, skipImportError } + if l, ok := resolve.FindRuleWithOverride(c, resolve.ImportSpec{Lang: "go", Imp: imp}, "go"); ok { + return l, nil + } + if pc.Mode.ShouldUseKnownImports() { // These are commonly used libraries that depend on Well Known Types. // They depend on the generated versions of these protos to avoid conflicts. @@ -129,6 +140,8 @@ func resolveGo(c *config.Config, ix *resolve.RuleIndex, rc *repos.RemoteCache, r return label.New("com_github_golang_protobuf", "descriptor", "go_default_library_gen"), nil case "github.com/golang/protobuf/ptypes": return label.New("com_github_golang_protobuf", "ptypes", "go_default_library_gen"), nil + case "google.golang.org/grpc": + return label.New("org_golang_google_grpc", "", "go_default_library"), nil } if l, ok := knownGoProtoImports[imp]; ok { return l, nil @@ -236,6 +249,10 @@ func resolveProto(c *config.Config, ix *resolve.RuleIndex, rc *repos.RemoteCache return label.NoLabel, skipImportError } + if l, ok := resolve.FindRuleWithOverride(c, resolve.ImportSpec{Lang: "proto", Imp: imp}, "go"); ok { + return l, nil + } + if l, ok := knownProtoImports[imp]; ok && pc.Mode.ShouldUseKnownImports() { if l.Equal(from) { return label.NoLabel, skipImportError diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/proto/known_imports.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/proto/known_imports.go index 525a58dc416..344f911e1d3 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/proto/known_imports.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/proto/known_imports.go @@ -20,281 +20,347 @@ var knownImports = map[string]label.Label{ "google/protobuf/timestamp.proto": label.New("com_google_protobuf", "", "timestamp_proto"), "google/protobuf/type.proto": label.New("com_google_protobuf", "", "type_proto"), "google/protobuf/wrappers.proto": label.New("com_google_protobuf", "", "wrappers_proto"), - "google/assistant/embedded/v1alpha2/embedded_assistant.proto": label.New("go_googleapis", "google/assistant/embedded/v1alpha2", "embedded_proto"), - "google/assistant/embedded/v1alpha1/embedded_assistant.proto": label.New("go_googleapis", "google/assistant/embedded/v1alpha1", "embedded_proto"), - "google/home/graph/v1/device.proto": label.New("go_googleapis", "google/home/graph/v1", "graph_proto"), - "google/home/graph/v1/homegraph.proto": label.New("go_googleapis", "google/home/graph/v1", "graph_proto"), - "google/genomics/v1/operations.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), - "google/genomics/v1/variants.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), - "google/genomics/v1/position.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), - "google/genomics/v1/references.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), - "google/genomics/v1/cigar.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), - "google/genomics/v1/datasets.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), - "google/genomics/v1/readalignment.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), - "google/genomics/v1/annotations.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), - "google/genomics/v1/reads.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), - "google/genomics/v1/readgroup.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), - "google/genomics/v1/readgroupset.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), - "google/genomics/v1/range.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), - "google/genomics/v1alpha2/pipelines.proto": label.New("go_googleapis", "google/genomics/v1alpha2", "genomics_proto"), - "google/bigtable/v1/bigtable_service_messages.proto": label.New("go_googleapis", "google/bigtable/v1", "bigtable_proto"), - "google/bigtable/v1/bigtable_service.proto": label.New("go_googleapis", "google/bigtable/v1", "bigtable_proto"), - "google/bigtable/v1/bigtable_data.proto": label.New("go_googleapis", "google/bigtable/v1", "bigtable_proto"), - "google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_proto"), - "google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_proto"), - "google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_proto"), - "google/bigtable/admin/v2/bigtable_instance_admin.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_proto"), - "google/bigtable/admin/v2/instance.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_proto"), - "google/bigtable/admin/v2/table.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_proto"), - "google/bigtable/admin/v2/bigtable_table_admin.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_proto"), - "google/bigtable/admin/v2/common.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_proto"), - "google/bigtable/admin/table/v1/bigtable_table_service_messages.proto": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_proto"), - "google/bigtable/admin/table/v1/bigtable_table_service.proto": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_proto"), - "google/bigtable/admin/table/v1/bigtable_table_data.proto": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_proto"), - "google/bigtable/v2/bigtable.proto": label.New("go_googleapis", "google/bigtable/v2", "bigtable_proto"), - "google/bigtable/v2/data.proto": label.New("go_googleapis", "google/bigtable/v2", "bigtable_proto"), - "google/privacy/dlp/v2/storage.proto": label.New("go_googleapis", "google/privacy/dlp/v2", "dlp_proto"), - "google/privacy/dlp/v2/dlp.proto": label.New("go_googleapis", "google/privacy/dlp/v2", "dlp_proto"), - "google/watcher/v1/watch.proto": label.New("go_googleapis", "google/watcher/v1", "watcher_proto"), - "google/firestore/admin/v1beta1/firestore_admin.proto": label.New("go_googleapis", "google/firestore/admin/v1beta1", "admin_proto"), - "google/firestore/admin/v1beta1/index.proto": label.New("go_googleapis", "google/firestore/admin/v1beta1", "admin_proto"), - "google/firestore/v1beta1/write.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_proto"), - "google/firestore/v1beta1/document.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_proto"), - "google/firestore/v1beta1/firestore.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_proto"), - "google/firestore/v1beta1/query.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_proto"), - "google/firestore/v1beta1/common.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_proto"), - "google/example/library/v1/library.proto": label.New("go_googleapis", "google/example/library/v1", "library_proto"), - "google/appengine/v1/instance.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), - "google/appengine/v1/audit_data.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), - "google/appengine/v1/appengine.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), - "google/appengine/v1/application.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), - "google/appengine/v1/operation.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), - "google/appengine/v1/app_yaml.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), - "google/appengine/v1/location.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), - "google/appengine/v1/service.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), - "google/appengine/v1/deploy.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), - "google/appengine/v1/version.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), + "google/api/annotations.proto": label.New("go_googleapis", "google/api", "annotations_proto"), + "google/api/auth.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/backend.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/billing.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/config_change.proto": label.New("go_googleapis", "google/api", "configchange_proto"), + "google/api/consumer.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/context.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/control.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/distribution.proto": label.New("go_googleapis", "google/api", "distribution_proto"), + "google/api/documentation.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/endpoint.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/experimental/authorization_config.proto": label.New("go_googleapis", "google/api", "api_proto"), + "google/api/experimental/experimental.proto": label.New("go_googleapis", "google/api", "api_proto"), + "google/api/expr/v1alpha1/cel_service.proto": label.New("go_googleapis", "google/api/expr/v1alpha1", "expr_proto"), + "google/api/expr/v1alpha1/checked.proto": label.New("go_googleapis", "google/api/expr/v1alpha1", "expr_proto"), + "google/api/expr/v1alpha1/eval.proto": label.New("go_googleapis", "google/api/expr/v1alpha1", "expr_proto"), + "google/api/expr/v1alpha1/explain.proto": label.New("go_googleapis", "google/api/expr/v1alpha1", "expr_proto"), + "google/api/expr/v1alpha1/syntax.proto": label.New("go_googleapis", "google/api/expr/v1alpha1", "expr_proto"), + "google/api/expr/v1alpha1/value.proto": label.New("go_googleapis", "google/api/expr/v1alpha1", "expr_proto"), + "google/api/expr/v1beta1/decl.proto": label.New("go_googleapis", "google/api/expr/v1beta1", "expr_proto"), + "google/api/expr/v1beta1/eval.proto": label.New("go_googleapis", "google/api/expr/v1beta1", "expr_proto"), + "google/api/expr/v1beta1/expr.proto": label.New("go_googleapis", "google/api/expr/v1beta1", "expr_proto"), + "google/api/expr/v1beta1/source.proto": label.New("go_googleapis", "google/api/expr/v1beta1", "expr_proto"), + "google/api/expr/v1beta1/value.proto": label.New("go_googleapis", "google/api/expr/v1beta1", "expr_proto"), + "google/api/http.proto": label.New("go_googleapis", "google/api", "annotations_proto"), + "google/api/httpbody.proto": label.New("go_googleapis", "google/api", "httpbody_proto"), + "google/api/label.proto": label.New("go_googleapis", "google/api", "label_proto"), + "google/api/launch_stage.proto": label.New("go_googleapis", "google/api", "api_proto"), + "google/api/log.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/logging.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/metric.proto": label.New("go_googleapis", "google/api", "metric_proto"), + "google/api/monitored_resource.proto": label.New("go_googleapis", "google/api", "monitoredres_proto"), + "google/api/monitoring.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/quota.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/service.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/servicecontrol/v1/check_error.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), + "google/api/servicecontrol/v1/distribution.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), + "google/api/servicecontrol/v1/log_entry.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), + "google/api/servicecontrol/v1/metric_value.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), + "google/api/servicecontrol/v1/operation.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), + "google/api/servicecontrol/v1/quota_controller.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), + "google/api/servicecontrol/v1/service_controller.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), + "google/api/servicemanagement/v1/resources.proto": label.New("go_googleapis", "google/api/servicemanagement/v1", "servicemanagement_proto"), + "google/api/servicemanagement/v1/servicemanager.proto": label.New("go_googleapis", "google/api/servicemanagement/v1", "servicemanagement_proto"), + "google/api/source_info.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/system_parameter.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), + "google/api/usage.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), "google/appengine/legacy/audit_data.proto": label.New("go_googleapis", "google/appengine/legacy", "legacy_proto"), "google/appengine/logging/v1/request_log.proto": label.New("go_googleapis", "google/appengine/logging/v1", "logging_proto"), - "google/storagetransfer/v1/transfer.proto": label.New("go_googleapis", "google/storagetransfer/v1", "storagetransfer_proto"), - "google/storagetransfer/v1/transfer_types.proto": label.New("go_googleapis", "google/storagetransfer/v1", "storagetransfer_proto"), - "google/longrunning/operations.proto": label.New("go_googleapis", "google/longrunning", "longrunning_proto"), - "google/container/v1/cluster_service.proto": label.New("go_googleapis", "google/container/v1", "container_proto"), - "google/container/v1beta1/cluster_service.proto": label.New("go_googleapis", "google/container/v1beta1", "container_proto"), - "google/container/v1alpha1/cluster_service.proto": label.New("go_googleapis", "google/container/v1alpha1", "container_proto"), - "google/datastore/v1beta3/datastore.proto": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_proto"), - "google/datastore/v1beta3/query.proto": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_proto"), - "google/datastore/v1beta3/entity.proto": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_proto"), - "google/datastore/v1/datastore.proto": label.New("go_googleapis", "google/datastore/v1", "datastore_proto"), - "google/datastore/v1/query.proto": label.New("go_googleapis", "google/datastore/v1", "datastore_proto"), - "google/datastore/v1/entity.proto": label.New("go_googleapis", "google/datastore/v1", "datastore_proto"), - "google/datastore/admin/v1/datastore_admin.proto": label.New("go_googleapis", "google/datastore/admin/v1", "admin_proto"), - "google/datastore/admin/v1beta1/datastore_admin.proto": label.New("go_googleapis", "google/datastore/admin/v1beta1", "admin_proto"), + "google/appengine/v1/app_yaml.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), + "google/appengine/v1/appengine.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), + "google/appengine/v1/application.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), + "google/appengine/v1/audit_data.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), + "google/appengine/v1/deploy.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), + "google/appengine/v1/instance.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), + "google/appengine/v1/location.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), + "google/appengine/v1/operation.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), + "google/appengine/v1/service.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), + "google/appengine/v1/version.proto": label.New("go_googleapis", "google/appengine/v1", "appengine_proto"), + "google/assistant/embedded/v1alpha1/embedded_assistant.proto": label.New("go_googleapis", "google/assistant/embedded/v1alpha1", "embedded_proto"), + "google/assistant/embedded/v1alpha2/embedded_assistant.proto": label.New("go_googleapis", "google/assistant/embedded/v1alpha2", "embedded_proto"), + "google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_proto"), + "google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_proto"), + "google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto": label.New("go_googleapis", "google/bigtable/admin/cluster/v1", "cluster_proto"), + "google/bigtable/admin/table/v1/bigtable_table_data.proto": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_proto"), + "google/bigtable/admin/table/v1/bigtable_table_service.proto": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_proto"), + "google/bigtable/admin/table/v1/bigtable_table_service_messages.proto": label.New("go_googleapis", "google/bigtable/admin/table/v1", "table_proto"), + "google/bigtable/admin/v2/bigtable_instance_admin.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_proto"), + "google/bigtable/admin/v2/bigtable_table_admin.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_proto"), + "google/bigtable/admin/v2/common.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_proto"), + "google/bigtable/admin/v2/instance.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_proto"), + "google/bigtable/admin/v2/table.proto": label.New("go_googleapis", "google/bigtable/admin/v2", "admin_proto"), + "google/bigtable/v1/bigtable_data.proto": label.New("go_googleapis", "google/bigtable/v1", "bigtable_proto"), + "google/bigtable/v1/bigtable_service.proto": label.New("go_googleapis", "google/bigtable/v1", "bigtable_proto"), + "google/bigtable/v1/bigtable_service_messages.proto": label.New("go_googleapis", "google/bigtable/v1", "bigtable_proto"), + "google/bigtable/v2/bigtable.proto": label.New("go_googleapis", "google/bigtable/v2", "bigtable_proto"), + "google/bigtable/v2/data.proto": label.New("go_googleapis", "google/bigtable/v2", "bigtable_proto"), "google/bytestream/bytestream.proto": label.New("go_googleapis", "google/bytestream", "bytestream_proto"), - "google/iam/v1/iam_policy.proto": label.New("go_googleapis", "google/iam/v1", "iam_proto"), - "google/iam/v1/policy.proto": label.New("go_googleapis", "google/iam/v1", "iam_proto"), - "google/iam/v1/logging/audit_data.proto": label.New("go_googleapis", "google/iam/v1/logging", "logging_proto"), - "google/iam/admin/v1/iam.proto": label.New("go_googleapis", "google/iam/admin/v1", "admin_proto"), - "google/type/money.proto": label.New("go_googleapis", "google/type", "money_proto"), - "google/type/latlng.proto": label.New("go_googleapis", "google/type", "latlng_proto"), - "google/type/color.proto": label.New("go_googleapis", "google/type", "color_proto"), - "google/type/timeofday.proto": label.New("go_googleapis", "google/type", "timeofday_proto"), - "google/type/date.proto": label.New("go_googleapis", "google/type", "date_proto"), - "google/type/dayofweek.proto": label.New("go_googleapis", "google/type", "dayofweek_proto"), - "google/type/postal_address.proto": label.New("go_googleapis", "google/type", "postaladdress_proto"), - "google/devtools/clouderrorreporting/v1beta1/report_errors_service.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_proto"), - "google/devtools/clouderrorreporting/v1beta1/error_group_service.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_proto"), - "google/devtools/clouderrorreporting/v1beta1/error_stats_service.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_proto"), - "google/devtools/clouderrorreporting/v1beta1/common.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_proto"), - "google/devtools/resultstore/v2/file.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), - "google/devtools/resultstore/v2/resultstore_download.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), - "google/devtools/resultstore/v2/configuration.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), - "google/devtools/resultstore/v2/action.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), - "google/devtools/resultstore/v2/resultstore_file_download.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), - "google/devtools/resultstore/v2/test_suite.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), - "google/devtools/resultstore/v2/file_set.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), - "google/devtools/resultstore/v2/coverage.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), - "google/devtools/resultstore/v2/coverage_summary.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), - "google/devtools/resultstore/v2/configured_target.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), - "google/devtools/resultstore/v2/target.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), - "google/devtools/resultstore/v2/invocation.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), - "google/devtools/resultstore/v2/common.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), - "google/devtools/source/v1/source_context.proto": label.New("go_googleapis", "google/devtools/source/v1", "source_proto"), - "google/devtools/remoteexecution/v1test/remote_execution.proto": label.New("go_googleapis", "google/devtools/remoteexecution/v1test", "remoteexecution_proto"), - "google/devtools/cloudbuild/v1/cloudbuild.proto": label.New("go_googleapis", "google/devtools/cloudbuild/v1", "cloudbuild_proto"), - "google/devtools/sourcerepo/v1/sourcerepo.proto": label.New("go_googleapis", "google/devtools/sourcerepo/v1", "sourcerepo_proto"), - "google/devtools/remoteworkers/v1test2/worker.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_proto"), - "google/devtools/remoteworkers/v1test2/tasks.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_proto"), - "google/devtools/remoteworkers/v1test2/bots.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_proto"), - "google/devtools/remoteworkers/v1test2/command.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_proto"), - "google/devtools/cloudtrace/v1/trace.proto": label.New("go_googleapis", "google/devtools/cloudtrace/v1", "cloudtrace_proto"), - "google/devtools/cloudtrace/v2/trace.proto": label.New("go_googleapis", "google/devtools/cloudtrace/v2", "cloudtrace_proto"), - "google/devtools/cloudtrace/v2/tracing.proto": label.New("go_googleapis", "google/devtools/cloudtrace/v2", "cloudtrace_proto"), - "google/devtools/cloudprofiler/v2/profiler.proto": label.New("go_googleapis", "google/devtools/cloudprofiler/v2", "cloudprofiler_proto"), - "google/devtools/containeranalysis/v1alpha1/containeranalysis.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_proto"), - "google/devtools/containeranalysis/v1alpha1/bill_of_materials.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_proto"), - "google/devtools/containeranalysis/v1alpha1/provenance.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_proto"), - "google/devtools/containeranalysis/v1alpha1/package_vulnerability.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_proto"), - "google/devtools/containeranalysis/v1alpha1/source_context.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_proto"), - "google/devtools/containeranalysis/v1alpha1/image_basis.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_proto"), + "google/cloud/asset/v1beta1/asset_service.proto": label.New("go_googleapis", "google/cloud/asset/v1beta1", "asset_proto"), + "google/cloud/asset/v1beta1/assets.proto": label.New("go_googleapis", "google/cloud/asset/v1beta1", "asset_proto"), + "google/cloud/audit/audit_log.proto": label.New("go_googleapis", "google/cloud/audit", "audit_proto"), + "google/cloud/automl/v1beta1/annotation_payload.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_proto"), + "google/cloud/automl/v1beta1/classification.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_proto"), + "google/cloud/automl/v1beta1/data_items.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_proto"), + "google/cloud/automl/v1beta1/dataset.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_proto"), + "google/cloud/automl/v1beta1/image.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_proto"), + "google/cloud/automl/v1beta1/io.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_proto"), + "google/cloud/automl/v1beta1/model.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_proto"), + "google/cloud/automl/v1beta1/model_evaluation.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_proto"), + "google/cloud/automl/v1beta1/operations.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_proto"), + "google/cloud/automl/v1beta1/prediction_service.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_proto"), + "google/cloud/automl/v1beta1/service.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_proto"), + "google/cloud/automl/v1beta1/text.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_proto"), + "google/cloud/automl/v1beta1/translation.proto": label.New("go_googleapis", "google/cloud/automl/v1beta1", "automl_proto"), + "google/cloud/bigquery/datatransfer/v1/datatransfer.proto": label.New("go_googleapis", "google/cloud/bigquery/datatransfer/v1", "datatransfer_proto"), + "google/cloud/bigquery/datatransfer/v1/transfer.proto": label.New("go_googleapis", "google/cloud/bigquery/datatransfer/v1", "datatransfer_proto"), + "google/cloud/bigquery/logging/v1/audit_data.proto": label.New("go_googleapis", "google/cloud/bigquery/logging/v1", "logging_proto"), + "google/cloud/bigquery/storage/v1beta1/avro.proto": label.New("go_googleapis", "google/cloud/bigquery/storage/v1beta1", "storage_proto"), + "google/cloud/bigquery/storage/v1beta1/read_options.proto": label.New("go_googleapis", "google/cloud/bigquery/storage/v1beta1", "storage_proto"), + "google/cloud/bigquery/storage/v1beta1/storage.proto": label.New("go_googleapis", "google/cloud/bigquery/storage/v1beta1", "storage_proto"), + "google/cloud/bigquery/storage/v1beta1/table_reference.proto": label.New("go_googleapis", "google/cloud/bigquery/storage/v1beta1", "storage_proto"), + "google/cloud/billing/v1/cloud_billing.proto": label.New("go_googleapis", "google/cloud/billing/v1", "billing_proto"), + "google/cloud/dataproc/v1/clusters.proto": label.New("go_googleapis", "google/cloud/dataproc/v1", "dataproc_proto"), + "google/cloud/dataproc/v1/jobs.proto": label.New("go_googleapis", "google/cloud/dataproc/v1", "dataproc_proto"), + "google/cloud/dataproc/v1/operations.proto": label.New("go_googleapis", "google/cloud/dataproc/v1", "dataproc_proto"), + "google/cloud/dataproc/v1beta2/clusters.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_proto"), + "google/cloud/dataproc/v1beta2/jobs.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_proto"), + "google/cloud/dataproc/v1beta2/operations.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_proto"), + "google/cloud/dataproc/v1beta2/shared.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_proto"), + "google/cloud/dataproc/v1beta2/workflow_templates.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_proto"), + "google/cloud/dialogflow/v2/agent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), + "google/cloud/dialogflow/v2/context.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), + "google/cloud/dialogflow/v2/entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), + "google/cloud/dialogflow/v2/intent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), + "google/cloud/dialogflow/v2/session.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), + "google/cloud/dialogflow/v2/session_entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), + "google/cloud/dialogflow/v2/webhook.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), + "google/cloud/dialogflow/v2beta1/agent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), + "google/cloud/dialogflow/v2beta1/audio_config.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), + "google/cloud/dialogflow/v2beta1/context.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), + "google/cloud/dialogflow/v2beta1/document.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), + "google/cloud/dialogflow/v2beta1/entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), + "google/cloud/dialogflow/v2beta1/intent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), + "google/cloud/dialogflow/v2beta1/knowledge_base.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), + "google/cloud/dialogflow/v2beta1/session.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), + "google/cloud/dialogflow/v2beta1/session_entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), + "google/cloud/dialogflow/v2beta1/webhook.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), + "google/cloud/functions/v1beta2/functions.proto": label.New("go_googleapis", "google/cloud/functions/v1beta2", "functions_proto"), + "google/cloud/functions/v1beta2/operations.proto": label.New("go_googleapis", "google/cloud/functions/v1beta2", "functions_proto"), + "google/cloud/iot/v1/device_manager.proto": label.New("go_googleapis", "google/cloud/iot/v1", "iot_proto"), + "google/cloud/iot/v1/resources.proto": label.New("go_googleapis", "google/cloud/iot/v1", "iot_proto"), + "google/cloud/kms/v1/resources.proto": label.New("go_googleapis", "google/cloud/kms/v1", "kms_proto"), + "google/cloud/kms/v1/service.proto": label.New("go_googleapis", "google/cloud/kms/v1", "kms_proto"), + "google/cloud/language/v1/language_service.proto": label.New("go_googleapis", "google/cloud/language/v1", "language_proto"), + "google/cloud/language/v1beta1/language_service.proto": label.New("go_googleapis", "google/cloud/language/v1beta1", "language_proto"), + "google/cloud/language/v1beta2/language_service.proto": label.New("go_googleapis", "google/cloud/language/v1beta2", "language_proto"), + "google/cloud/location/locations.proto": label.New("go_googleapis", "google/cloud/location", "location_proto"), + "google/cloud/ml/v1/job_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_proto"), + "google/cloud/ml/v1/model_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_proto"), + "google/cloud/ml/v1/operation_metadata.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_proto"), + "google/cloud/ml/v1/prediction_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_proto"), + "google/cloud/ml/v1/project_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_proto"), + "google/cloud/oslogin/common/common.proto": label.New("go_googleapis", "google/cloud/oslogin/common", "common_proto"), + "google/cloud/oslogin/v1/oslogin.proto": label.New("go_googleapis", "google/cloud/oslogin/v1", "oslogin_proto"), + "google/cloud/oslogin/v1alpha/oslogin.proto": label.New("go_googleapis", "google/cloud/oslogin/v1alpha", "oslogin_proto"), + "google/cloud/oslogin/v1beta/oslogin.proto": label.New("go_googleapis", "google/cloud/oslogin/v1beta", "oslogin_proto"), + "google/cloud/redis/v1/cloud_redis.proto": label.New("go_googleapis", "google/cloud/redis/v1", "redis_proto"), + "google/cloud/redis/v1beta1/cloud_redis.proto": label.New("go_googleapis", "google/cloud/redis/v1beta1", "redis_proto"), + "google/cloud/resourcemanager/v2/folders.proto": label.New("go_googleapis", "google/cloud/resourcemanager/v2", "resourcemanager_proto"), + "google/cloud/runtimeconfig/v1beta1/resources.proto": label.New("go_googleapis", "google/cloud/runtimeconfig/v1beta1", "runtimeconfig_proto"), + "google/cloud/runtimeconfig/v1beta1/runtimeconfig.proto": label.New("go_googleapis", "google/cloud/runtimeconfig/v1beta1", "runtimeconfig_proto"), + "google/cloud/speech/v1/cloud_speech.proto": label.New("go_googleapis", "google/cloud/speech/v1", "speech_proto"), + "google/cloud/speech/v1p1beta1/cloud_speech.proto": label.New("go_googleapis", "google/cloud/speech/v1p1beta1", "speech_proto"), + "google/cloud/support/common.proto": label.New("go_googleapis", "google/cloud/support", "common_proto"), + "google/cloud/support/v1alpha1/cloud_support.proto": label.New("go_googleapis", "google/cloud/support/v1alpha1", "support_proto"), + "google/cloud/tasks/v2beta2/cloudtasks.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_proto"), + "google/cloud/tasks/v2beta2/queue.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_proto"), + "google/cloud/tasks/v2beta2/target.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_proto"), + "google/cloud/tasks/v2beta2/task.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_proto"), + "google/cloud/tasks/v2beta3/cloudtasks.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta3", "tasks_proto"), + "google/cloud/tasks/v2beta3/queue.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta3", "tasks_proto"), + "google/cloud/tasks/v2beta3/target.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta3", "tasks_proto"), + "google/cloud/tasks/v2beta3/task.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta3", "tasks_proto"), + "google/cloud/texttospeech/v1/cloud_tts.proto": label.New("go_googleapis", "google/cloud/texttospeech/v1", "texttospeech_proto"), + "google/cloud/texttospeech/v1beta1/cloud_tts.proto": label.New("go_googleapis", "google/cloud/texttospeech/v1beta1", "texttospeech_proto"), + "google/cloud/videointelligence/v1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1", "videointelligence_proto"), + "google/cloud/videointelligence/v1beta1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1beta1", "videointelligence_proto"), + "google/cloud/videointelligence/v1beta2/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1beta2", "videointelligence_proto"), + "google/cloud/videointelligence/v1p1beta1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1p1beta1", "videointelligence_proto"), + "google/cloud/videointelligence/v1p2beta1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1p2beta1", "videointelligence_proto"), + "google/cloud/vision/v1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_proto"), + "google/cloud/vision/v1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_proto"), + "google/cloud/vision/v1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_proto"), + "google/cloud/vision/v1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_proto"), + "google/cloud/vision/v1p1beta1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_proto"), + "google/cloud/vision/v1p1beta1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_proto"), + "google/cloud/vision/v1p1beta1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_proto"), + "google/cloud/vision/v1p1beta1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_proto"), + "google/cloud/vision/v1p2beta1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_proto"), + "google/cloud/vision/v1p2beta1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_proto"), + "google/cloud/vision/v1p2beta1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_proto"), + "google/cloud/vision/v1p2beta1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_proto"), + "google/cloud/vision/v1p3beta1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1p3beta1", "vision_proto"), + "google/cloud/vision/v1p3beta1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1p3beta1", "vision_proto"), + "google/cloud/vision/v1p3beta1/product_search.proto": label.New("go_googleapis", "google/cloud/vision/v1p3beta1", "vision_proto"), + "google/cloud/vision/v1p3beta1/product_search_service.proto": label.New("go_googleapis", "google/cloud/vision/v1p3beta1", "vision_proto"), + "google/cloud/vision/v1p3beta1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1p3beta1", "vision_proto"), + "google/cloud/vision/v1p3beta1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1p3beta1", "vision_proto"), + "google/cloud/websecurityscanner/v1alpha/crawled_url.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), + "google/cloud/websecurityscanner/v1alpha/finding.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), + "google/cloud/websecurityscanner/v1alpha/finding_addon.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), + "google/cloud/websecurityscanner/v1alpha/finding_type_stats.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), + "google/cloud/websecurityscanner/v1alpha/scan_config.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), + "google/cloud/websecurityscanner/v1alpha/scan_run.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), + "google/cloud/websecurityscanner/v1alpha/web_security_scanner.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), + "google/container/v1/cluster_service.proto": label.New("go_googleapis", "google/container/v1", "container_proto"), + "google/container/v1alpha1/cluster_service.proto": label.New("go_googleapis", "google/container/v1alpha1", "container_proto"), + "google/container/v1beta1/cluster_service.proto": label.New("go_googleapis", "google/container/v1beta1", "container_proto"), + "google/datastore/admin/v1/datastore_admin.proto": label.New("go_googleapis", "google/datastore/admin/v1", "admin_proto"), + "google/datastore/admin/v1/index.proto": label.New("go_googleapis", "google/datastore/admin/v1", "admin_proto"), + "google/datastore/admin/v1beta1/datastore_admin.proto": label.New("go_googleapis", "google/datastore/admin/v1beta1", "admin_proto"), + "google/datastore/v1/datastore.proto": label.New("go_googleapis", "google/datastore/v1", "datastore_proto"), + "google/datastore/v1/entity.proto": label.New("go_googleapis", "google/datastore/v1", "datastore_proto"), + "google/datastore/v1/query.proto": label.New("go_googleapis", "google/datastore/v1", "datastore_proto"), + "google/datastore/v1beta3/datastore.proto": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_proto"), + "google/datastore/v1beta3/entity.proto": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_proto"), + "google/datastore/v1beta3/query.proto": label.New("go_googleapis", "google/datastore/v1beta3", "datastore_proto"), "google/devtools/build/v1/build_events.proto": label.New("go_googleapis", "google/devtools/build/v1", "build_proto"), "google/devtools/build/v1/build_status.proto": label.New("go_googleapis", "google/devtools/build/v1", "build_proto"), "google/devtools/build/v1/publish_build_event.proto": label.New("go_googleapis", "google/devtools/build/v1", "build_proto"), - "google/devtools/clouddebugger/v2/debugger.proto": label.New("go_googleapis", "google/devtools/clouddebugger/v2", "clouddebugger_proto"), - "google/devtools/clouddebugger/v2/data.proto": label.New("go_googleapis", "google/devtools/clouddebugger/v2", "clouddebugger_proto"), + "google/devtools/cloudbuild/v1/cloudbuild.proto": label.New("go_googleapis", "google/devtools/cloudbuild/v1", "cloudbuild_proto"), "google/devtools/clouddebugger/v2/controller.proto": label.New("go_googleapis", "google/devtools/clouddebugger/v2", "clouddebugger_proto"), - "google/cloud/resourcemanager/v2/folders.proto": label.New("go_googleapis", "google/cloud/resourcemanager/v2", "resourcemanager_proto"), - "google/cloud/kms/v1/resources.proto": label.New("go_googleapis", "google/cloud/kms/v1", "kms_proto"), - "google/cloud/kms/v1/service.proto": label.New("go_googleapis", "google/cloud/kms/v1", "kms_proto"), - "google/cloud/runtimeconfig/v1beta1/resources.proto": label.New("go_googleapis", "google/cloud/runtimeconfig/v1beta1", "runtimeconfig_proto"), - "google/cloud/runtimeconfig/v1beta1/runtimeconfig.proto": label.New("go_googleapis", "google/cloud/runtimeconfig/v1beta1", "runtimeconfig_proto"), - "google/cloud/tasks/v2beta2/queue.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_proto"), - "google/cloud/tasks/v2beta2/task.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_proto"), - "google/cloud/tasks/v2beta2/target.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_proto"), - "google/cloud/tasks/v2beta2/cloudtasks.proto": label.New("go_googleapis", "google/cloud/tasks/v2beta2", "tasks_proto"), - "google/cloud/oslogin/v1/oslogin.proto": label.New("go_googleapis", "google/cloud/oslogin/v1", "oslogin_proto"), - "google/cloud/oslogin/v1alpha/oslogin.proto": label.New("go_googleapis", "google/cloud/oslogin/v1alpha", "oslogin_proto"), - "google/cloud/oslogin/common/common.proto": label.New("go_googleapis", "google/cloud/oslogin/common", "common_proto"), - "google/cloud/oslogin/v1beta/oslogin.proto": label.New("go_googleapis", "google/cloud/oslogin/v1beta", "oslogin_proto"), - "google/cloud/dialogflow/v2beta1/context.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), - "google/cloud/dialogflow/v2beta1/session_entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), - "google/cloud/dialogflow/v2beta1/intent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), - "google/cloud/dialogflow/v2beta1/entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), - "google/cloud/dialogflow/v2beta1/webhook.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), - "google/cloud/dialogflow/v2beta1/session.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), - "google/cloud/dialogflow/v2beta1/agent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2beta1", "dialogflow_proto"), - "google/cloud/dialogflow/v2/context.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), - "google/cloud/dialogflow/v2/session_entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), - "google/cloud/dialogflow/v2/intent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), - "google/cloud/dialogflow/v2/entity_type.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), - "google/cloud/dialogflow/v2/webhook.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), - "google/cloud/dialogflow/v2/session.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), - "google/cloud/dialogflow/v2/agent.proto": label.New("go_googleapis", "google/cloud/dialogflow/v2", "dialogflow_proto"), - "google/cloud/redis/v1beta1/cloud_redis.proto": label.New("go_googleapis", "google/cloud/redis/v1beta1", "redis_proto"), - "google/cloud/location/locations.proto": label.New("go_googleapis", "google/cloud/location", "location_proto"), - "google/cloud/websecurityscanner/v1alpha/finding.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), - "google/cloud/websecurityscanner/v1alpha/finding_type_stats.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), - "google/cloud/websecurityscanner/v1alpha/scan_config.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), - "google/cloud/websecurityscanner/v1alpha/crawled_url.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), - "google/cloud/websecurityscanner/v1alpha/scan_run.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), - "google/cloud/websecurityscanner/v1alpha/web_security_scanner.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), - "google/cloud/websecurityscanner/v1alpha/finding_addon.proto": label.New("go_googleapis", "google/cloud/websecurityscanner/v1alpha", "websecurityscanner_proto"), - "google/cloud/language/v1/language_service.proto": label.New("go_googleapis", "google/cloud/language/v1", "language_proto"), - "google/cloud/language/v1beta2/language_service.proto": label.New("go_googleapis", "google/cloud/language/v1beta2", "language_proto"), - "google/cloud/language/v1beta1/language_service.proto": label.New("go_googleapis", "google/cloud/language/v1beta1", "language_proto"), - "google/cloud/bigquery/datatransfer/v1/transfer.proto": label.New("go_googleapis", "google/cloud/bigquery/datatransfer/v1", "datatransfer_proto"), - "google/cloud/bigquery/datatransfer/v1/datatransfer.proto": label.New("go_googleapis", "google/cloud/bigquery/datatransfer/v1", "datatransfer_proto"), - "google/cloud/bigquery/logging/v1/audit_data.proto": label.New("go_googleapis", "google/cloud/bigquery/logging/v1", "logging_proto"), - "google/cloud/vision/v1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_proto"), - "google/cloud/vision/v1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_proto"), - "google/cloud/vision/v1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_proto"), - "google/cloud/vision/v1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1", "vision_proto"), - "google/cloud/vision/v1p2beta1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_proto"), - "google/cloud/vision/v1p2beta1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_proto"), - "google/cloud/vision/v1p2beta1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_proto"), - "google/cloud/vision/v1p2beta1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1p2beta1", "vision_proto"), - "google/cloud/vision/v1p1beta1/image_annotator.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_proto"), - "google/cloud/vision/v1p1beta1/geometry.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_proto"), - "google/cloud/vision/v1p1beta1/web_detection.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_proto"), - "google/cloud/vision/v1p1beta1/text_annotation.proto": label.New("go_googleapis", "google/cloud/vision/v1p1beta1", "vision_proto"), - "google/cloud/speech/v1/cloud_speech.proto": label.New("go_googleapis", "google/cloud/speech/v1", "speech_proto"), - "google/cloud/speech/v1beta1/cloud_speech.proto": label.New("go_googleapis", "google/cloud/speech/v1beta1", "speech_proto"), - "google/cloud/speech/v1p1beta1/cloud_speech.proto": label.New("go_googleapis", "google/cloud/speech/v1p1beta1", "speech_proto"), - "google/cloud/iot/v1/device_manager.proto": label.New("go_googleapis", "google/cloud/iot/v1", "iot_proto"), - "google/cloud/iot/v1/resources.proto": label.New("go_googleapis", "google/cloud/iot/v1", "iot_proto"), - "google/cloud/videointelligence/v1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1", "videointelligence_proto"), - "google/cloud/videointelligence/v1beta2/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1beta2", "videointelligence_proto"), - "google/cloud/videointelligence/v1beta1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1beta1", "videointelligence_proto"), - "google/cloud/videointelligence/v1p1beta1/video_intelligence.proto": label.New("go_googleapis", "google/cloud/videointelligence/v1p1beta1", "videointelligence_proto"), - "google/cloud/audit/audit_log.proto": label.New("go_googleapis", "google/cloud/audit", "audit_proto"), - "google/cloud/support/common.proto": label.New("go_googleapis", "google/cloud/support", "common_proto"), - "google/cloud/support/v1alpha1/cloud_support.proto": label.New("go_googleapis", "google/cloud/support/v1alpha1", "support_proto"), - "google/cloud/ml/v1/operation_metadata.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_proto"), - "google/cloud/ml/v1/job_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_proto"), - "google/cloud/ml/v1/prediction_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_proto"), - "google/cloud/ml/v1/model_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_proto"), - "google/cloud/ml/v1/project_service.proto": label.New("go_googleapis", "google/cloud/ml/v1", "ml_proto"), - "google/cloud/texttospeech/v1/cloud_tts.proto": label.New("go_googleapis", "google/cloud/texttospeech/v1", "texttospeech_proto"), - "google/cloud/texttospeech/v1beta1/cloud_tts.proto": label.New("go_googleapis", "google/cloud/texttospeech/v1beta1", "texttospeech_proto"), - "google/cloud/functions/v1beta2/operations.proto": label.New("go_googleapis", "google/cloud/functions/v1beta2", "functions_proto"), - "google/cloud/functions/v1beta2/functions.proto": label.New("go_googleapis", "google/cloud/functions/v1beta2", "functions_proto"), - "google/cloud/billing/v1/cloud_billing.proto": label.New("go_googleapis", "google/cloud/billing/v1", "billing_proto"), - "google/cloud/dataproc/v1/operations.proto": label.New("go_googleapis", "google/cloud/dataproc/v1", "dataproc_proto"), - "google/cloud/dataproc/v1/clusters.proto": label.New("go_googleapis", "google/cloud/dataproc/v1", "dataproc_proto"), - "google/cloud/dataproc/v1/jobs.proto": label.New("go_googleapis", "google/cloud/dataproc/v1", "dataproc_proto"), - "google/cloud/dataproc/v1beta2/operations.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_proto"), - "google/cloud/dataproc/v1beta2/clusters.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_proto"), - "google/cloud/dataproc/v1beta2/jobs.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_proto"), - "google/cloud/dataproc/v1beta2/workflow_templates.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_proto"), - "google/cloud/dataproc/v1beta2/shared.proto": label.New("go_googleapis", "google/cloud/dataproc/v1beta2", "dataproc_proto"), - "google/api/context.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/http.proto": label.New("go_googleapis", "google/api", "annotations_proto"), - "google/api/config_change.proto": label.New("go_googleapis", "google/api", "configchange_proto"), - "google/api/system_parameter.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/monitoring.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/distribution.proto": label.New("go_googleapis", "google/api", "distribution_proto"), - "google/api/endpoint.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/usage.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/monitored_resource.proto": label.New("go_googleapis", "google/api", "monitoredres_proto"), - "google/api/annotations.proto": label.New("go_googleapis", "google/api", "annotations_proto"), - "google/api/control.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/metric.proto": label.New("go_googleapis", "google/api", "metric_proto"), - "google/api/label.proto": label.New("go_googleapis", "google/api", "label_proto"), - "google/api/consumer.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/log.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/billing.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/service.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/logging.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/documentation.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/quota.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/auth.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/backend.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/source_info.proto": label.New("go_googleapis", "google/api", "serviceconfig_proto"), - "google/api/httpbody.proto": label.New("go_googleapis", "google/api", "httpbody_proto"), - "google/api/experimental/authorization_config.proto": label.New("go_googleapis", "google/api/experimental", "api_proto"), - "google/api/experimental/experimental.proto": label.New("go_googleapis", "google/api/experimental", "api_proto"), - "google/api/servicemanagement/v1/servicemanager.proto": label.New("go_googleapis", "google/api/servicemanagement/v1", "servicemanagement_proto"), - "google/api/servicemanagement/v1/resources.proto": label.New("go_googleapis", "google/api/servicemanagement/v1", "servicemanagement_proto"), - "google/api/servicecontrol/v1/quota_controller.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), - "google/api/servicecontrol/v1/distribution.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), - "google/api/servicecontrol/v1/check_error.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), - "google/api/servicecontrol/v1/operation.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), - "google/api/servicecontrol/v1/metric_value.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), - "google/api/servicecontrol/v1/log_entry.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), - "google/api/servicecontrol/v1/service_controller.proto": label.New("go_googleapis", "google/api/servicecontrol/v1", "servicecontrol_proto"), - "google/pubsub/v1/pubsub.proto": label.New("go_googleapis", "google/pubsub/v1", "pubsub_proto"), - "google/pubsub/v1beta2/pubsub.proto": label.New("go_googleapis", "google/pubsub/v1beta2", "pubsub_proto"), - "google/spanner/v1/mutation.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), - "google/spanner/v1/spanner.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), - "google/spanner/v1/transaction.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), - "google/spanner/v1/keys.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), - "google/spanner/v1/type.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), - "google/spanner/v1/query_plan.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), - "google/spanner/v1/result_set.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), - "google/spanner/admin/database/v1/spanner_database_admin.proto": label.New("go_googleapis", "google/spanner/admin/database/v1", "database_proto"), - "google/spanner/admin/instance/v1/spanner_instance_admin.proto": label.New("go_googleapis", "google/spanner/admin/instance/v1", "instance_proto"), + "google/devtools/clouddebugger/v2/data.proto": label.New("go_googleapis", "google/devtools/clouddebugger/v2", "clouddebugger_proto"), + "google/devtools/clouddebugger/v2/debugger.proto": label.New("go_googleapis", "google/devtools/clouddebugger/v2", "clouddebugger_proto"), + "google/devtools/clouderrorreporting/v1beta1/common.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_proto"), + "google/devtools/clouderrorreporting/v1beta1/error_group_service.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_proto"), + "google/devtools/clouderrorreporting/v1beta1/error_stats_service.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_proto"), + "google/devtools/clouderrorreporting/v1beta1/report_errors_service.proto": label.New("go_googleapis", "google/devtools/clouderrorreporting/v1beta1", "clouderrorreporting_proto"), + "google/devtools/cloudprofiler/v2/profiler.proto": label.New("go_googleapis", "google/devtools/cloudprofiler/v2", "cloudprofiler_proto"), + "google/devtools/cloudtrace/v1/trace.proto": label.New("go_googleapis", "google/devtools/cloudtrace/v1", "cloudtrace_proto"), + "google/devtools/cloudtrace/v2/trace.proto": label.New("go_googleapis", "google/devtools/cloudtrace/v2", "cloudtrace_proto"), + "google/devtools/cloudtrace/v2/tracing.proto": label.New("go_googleapis", "google/devtools/cloudtrace/v2", "cloudtrace_proto"), + "google/devtools/containeranalysis/v1alpha1/bill_of_materials.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_proto"), + "google/devtools/containeranalysis/v1alpha1/containeranalysis.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_proto"), + "google/devtools/containeranalysis/v1alpha1/image_basis.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_proto"), + "google/devtools/containeranalysis/v1alpha1/package_vulnerability.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_proto"), + "google/devtools/containeranalysis/v1alpha1/provenance.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_proto"), + "google/devtools/containeranalysis/v1alpha1/source_context.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1alpha1", "containeranalysis_proto"), + "google/devtools/containeranalysis/v1beta1/attestation/attestation.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/attestation", "attestation_proto"), + "google/devtools/containeranalysis/v1beta1/build/build.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/build", "build_proto"), + "google/devtools/containeranalysis/v1beta1/common/common.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/common", "common_proto"), + "google/devtools/containeranalysis/v1beta1/containeranalysis.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1", "containeranalysis_proto"), + "google/devtools/containeranalysis/v1beta1/deployment/deployment.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/deployment", "deployment_proto"), + "google/devtools/containeranalysis/v1beta1/discovery/discovery.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/discovery", "discovery_proto"), + "google/devtools/containeranalysis/v1beta1/grafeas/grafeas.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/grafeas", "grafeas_proto"), + "google/devtools/containeranalysis/v1beta1/image/image.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/image", "image_proto"), + "google/devtools/containeranalysis/v1beta1/package/package.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/package", "package_proto"), + "google/devtools/containeranalysis/v1beta1/provenance/provenance.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/provenance", "provenance_proto"), + "google/devtools/containeranalysis/v1beta1/source/source.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/source", "source_proto"), + "google/devtools/containeranalysis/v1beta1/vulnerability/vulnerability.proto": label.New("go_googleapis", "google/devtools/containeranalysis/v1beta1/vulnerability", "vulnerability_proto"), + "google/devtools/remoteexecution/v1test/remote_execution.proto": label.New("go_googleapis", "google/devtools/remoteexecution/v1test", "remoteexecution_proto"), + "google/devtools/remoteworkers/v1test2/bots.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_proto"), + "google/devtools/remoteworkers/v1test2/command.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_proto"), + "google/devtools/remoteworkers/v1test2/tasks.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_proto"), + "google/devtools/remoteworkers/v1test2/worker.proto": label.New("go_googleapis", "google/devtools/remoteworkers/v1test2", "remoteworkers_proto"), + "google/devtools/resultstore/v2/action.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), + "google/devtools/resultstore/v2/common.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), + "google/devtools/resultstore/v2/configuration.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), + "google/devtools/resultstore/v2/configured_target.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), + "google/devtools/resultstore/v2/coverage.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), + "google/devtools/resultstore/v2/coverage_summary.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), + "google/devtools/resultstore/v2/file.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), + "google/devtools/resultstore/v2/file_set.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), + "google/devtools/resultstore/v2/invocation.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), + "google/devtools/resultstore/v2/resultstore_download.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), + "google/devtools/resultstore/v2/resultstore_file_download.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), + "google/devtools/resultstore/v2/target.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), + "google/devtools/resultstore/v2/test_suite.proto": label.New("go_googleapis", "google/devtools/resultstore/v2", "resultstore_proto"), + "google/devtools/source/v1/source_context.proto": label.New("go_googleapis", "google/devtools/source/v1", "source_proto"), + "google/devtools/sourcerepo/v1/sourcerepo.proto": label.New("go_googleapis", "google/devtools/sourcerepo/v1", "sourcerepo_proto"), + "google/example/library/v1/library.proto": label.New("go_googleapis", "google/example/library/v1", "library_proto"), + "google/firestore/admin/v1beta1/firestore_admin.proto": label.New("go_googleapis", "google/firestore/admin/v1beta1", "admin_proto"), + "google/firestore/admin/v1beta1/index.proto": label.New("go_googleapis", "google/firestore/admin/v1beta1", "admin_proto"), + "google/firestore/admin/v1beta2/field.proto": label.New("go_googleapis", "google/firestore/admin/v1beta2", "admin_proto"), + "google/firestore/admin/v1beta2/firestore_admin.proto": label.New("go_googleapis", "google/firestore/admin/v1beta2", "admin_proto"), + "google/firestore/admin/v1beta2/index.proto": label.New("go_googleapis", "google/firestore/admin/v1beta2", "admin_proto"), + "google/firestore/admin/v1beta2/operation.proto": label.New("go_googleapis", "google/firestore/admin/v1beta2", "admin_proto"), + "google/firestore/v1beta1/common.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_proto"), + "google/firestore/v1beta1/document.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_proto"), + "google/firestore/v1beta1/firestore.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_proto"), + "google/firestore/v1beta1/query.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_proto"), + "google/firestore/v1beta1/write.proto": label.New("go_googleapis", "google/firestore/v1beta1", "firestore_proto"), + "google/genomics/v1/annotations.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), + "google/genomics/v1/cigar.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), + "google/genomics/v1/datasets.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), + "google/genomics/v1/operations.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), + "google/genomics/v1/position.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), + "google/genomics/v1/range.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), + "google/genomics/v1/readalignment.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), + "google/genomics/v1/readgroup.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), + "google/genomics/v1/readgroupset.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), + "google/genomics/v1/reads.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), + "google/genomics/v1/references.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), + "google/genomics/v1/variants.proto": label.New("go_googleapis", "google/genomics/v1", "genomics_proto"), + "google/genomics/v1alpha2/pipelines.proto": label.New("go_googleapis", "google/genomics/v1alpha2", "genomics_proto"), + "google/home/graph/v1/device.proto": label.New("go_googleapis", "google/home/graph/v1", "graph_proto"), + "google/home/graph/v1/homegraph.proto": label.New("go_googleapis", "google/home/graph/v1", "graph_proto"), + "google/iam/admin/v1/iam.proto": label.New("go_googleapis", "google/iam/admin/v1", "admin_proto"), + "google/iam/credentials/v1/common.proto": label.New("go_googleapis", "google/iam/credentials/v1", "credentials_proto"), + "google/iam/credentials/v1/iamcredentials.proto": label.New("go_googleapis", "google/iam/credentials/v1", "credentials_proto"), + "google/iam/v1/iam_policy.proto": label.New("go_googleapis", "google/iam/v1", "iam_proto"), + "google/iam/v1/logging/audit_data.proto": label.New("go_googleapis", "google/iam/v1/logging", "logging_proto"), + "google/iam/v1/policy.proto": label.New("go_googleapis", "google/iam/v1", "iam_proto"), + "google/logging/type/http_request.proto": label.New("go_googleapis", "google/logging/type", "ltype_proto"), + "google/logging/type/log_severity.proto": label.New("go_googleapis", "google/logging/type", "ltype_proto"), + "google/logging/v2/log_entry.proto": label.New("go_googleapis", "google/logging/v2", "logging_proto"), + "google/logging/v2/logging.proto": label.New("go_googleapis", "google/logging/v2", "logging_proto"), + "google/logging/v2/logging_config.proto": label.New("go_googleapis", "google/logging/v2", "logging_proto"), + "google/logging/v2/logging_metrics.proto": label.New("go_googleapis", "google/logging/v2", "logging_proto"), + "google/longrunning/operations.proto": label.New("go_googleapis", "google/longrunning", "longrunning_proto"), + "google/monitoring/v3/alert.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), + "google/monitoring/v3/alert_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), + "google/monitoring/v3/common.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), + "google/monitoring/v3/dropped_labels.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), "google/monitoring/v3/group.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), + "google/monitoring/v3/group_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), + "google/monitoring/v3/metric.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), + "google/monitoring/v3/metric_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), "google/monitoring/v3/mutation_record.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), "google/monitoring/v3/notification.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), - "google/monitoring/v3/alert_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), - "google/monitoring/v3/uptime_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), - "google/monitoring/v3/group_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), - "google/monitoring/v3/alert.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), - "google/monitoring/v3/uptime.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), - "google/monitoring/v3/metric.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), "google/monitoring/v3/notification_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), - "google/monitoring/v3/metric_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), - "google/monitoring/v3/common.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), + "google/monitoring/v3/span_context.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), + "google/monitoring/v3/uptime.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), + "google/monitoring/v3/uptime_service.proto": label.New("go_googleapis", "google/monitoring/v3", "monitoring_proto"), + "google/privacy/dlp/v2/dlp.proto": label.New("go_googleapis", "google/privacy/dlp/v2", "dlp_proto"), + "google/privacy/dlp/v2/storage.proto": label.New("go_googleapis", "google/privacy/dlp/v2", "dlp_proto"), + "google/pubsub/v1/pubsub.proto": label.New("go_googleapis", "google/pubsub/v1", "pubsub_proto"), + "google/pubsub/v1beta2/pubsub.proto": label.New("go_googleapis", "google/pubsub/v1beta2", "pubsub_proto"), "google/rpc/code.proto": label.New("go_googleapis", "google/rpc", "code_proto"), - "google/rpc/status.proto": label.New("go_googleapis", "google/rpc", "status_proto"), "google/rpc/error_details.proto": label.New("go_googleapis", "google/rpc", "errdetails_proto"), + "google/rpc/status.proto": label.New("go_googleapis", "google/rpc", "status_proto"), + "google/spanner/admin/database/v1/spanner_database_admin.proto": label.New("go_googleapis", "google/spanner/admin/database/v1", "database_proto"), + "google/spanner/admin/instance/v1/spanner_instance_admin.proto": label.New("go_googleapis", "google/spanner/admin/instance/v1", "instance_proto"), + "google/spanner/v1/keys.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), + "google/spanner/v1/mutation.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), + "google/spanner/v1/query_plan.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), + "google/spanner/v1/result_set.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), + "google/spanner/v1/spanner.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), + "google/spanner/v1/transaction.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), + "google/spanner/v1/type.proto": label.New("go_googleapis", "google/spanner/v1", "spanner_proto"), + "google/storagetransfer/v1/transfer.proto": label.New("go_googleapis", "google/storagetransfer/v1", "storagetransfer_proto"), + "google/storagetransfer/v1/transfer_types.proto": label.New("go_googleapis", "google/storagetransfer/v1", "storagetransfer_proto"), "google/streetview/publish/v1/resources.proto": label.New("go_googleapis", "google/streetview/publish/v1", "publish_proto"), "google/streetview/publish/v1/rpcmessages.proto": label.New("go_googleapis", "google/streetview/publish/v1", "publish_proto"), "google/streetview/publish/v1/streetview_publish.proto": label.New("go_googleapis", "google/streetview/publish/v1", "publish_proto"), - "google/logging/v2/logging_metrics.proto": label.New("go_googleapis", "google/logging/v2", "logging_proto"), - "google/logging/v2/logging_config.proto": label.New("go_googleapis", "google/logging/v2", "logging_proto"), - "google/logging/v2/log_entry.proto": label.New("go_googleapis", "google/logging/v2", "logging_proto"), - "google/logging/v2/logging.proto": label.New("go_googleapis", "google/logging/v2", "logging_proto"), - "google/logging/type/log_severity.proto": label.New("go_googleapis", "google/logging/type", "ltype_proto"), - "google/logging/type/http_request.proto": label.New("go_googleapis", "google/logging/type", "ltype_proto"), + "google/type/color.proto": label.New("go_googleapis", "google/type", "color_proto"), + "google/type/date.proto": label.New("go_googleapis", "google/type", "date_proto"), + "google/type/dayofweek.proto": label.New("go_googleapis", "google/type", "dayofweek_proto"), + "google/type/latlng.proto": label.New("go_googleapis", "google/type", "latlng_proto"), + "google/type/money.proto": label.New("go_googleapis", "google/type", "money_proto"), + "google/type/postal_address.proto": label.New("go_googleapis", "google/type", "postaladdress_proto"), + "google/type/timeofday.proto": label.New("go_googleapis", "google/type", "timeofday_proto"), + "google/watcher/v1/watch.proto": label.New("go_googleapis", "google/watcher/v1", "watcher_proto"), } diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/proto/proto.csv b/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/proto/proto.csv index bbad8e8da8b..e004b9c4c86 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/proto/proto.csv +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/proto/proto.csv @@ -1,6 +1,9 @@ # This file lists special protos that Gazelle knows how to import. This is used to generate # code for proto and Go resolvers. # +# Generated by internal/language/proto/gen/update_proto_csv.go +# Do not edit directly. +# # proto name,proto label,go import path,go proto label google/protobuf/any.proto,@com_google_protobuf//:any_proto,github.com/golang/protobuf/ptypes/any,@io_bazel_rules_go//proto/wkt:any_go_proto google/protobuf/api.proto,@com_google_protobuf//:api_proto,google.golang.org/genproto/protobuf/api,@io_bazel_rules_go//proto/wkt:api_go_proto @@ -14,280 +17,346 @@ google/protobuf/struct.proto,@com_google_protobuf//:struct_proto,github.com/gola google/protobuf/timestamp.proto,@com_google_protobuf//:timestamp_proto,github.com/golang/protobuf/ptypes/timestamp,@io_bazel_rules_go//proto/wkt:timestamp_go_proto google/protobuf/type.proto,@com_google_protobuf//:type_proto,google.golang.org/genproto/protobuf/ptype,@io_bazel_rules_go//proto/wkt:type_go_proto google/protobuf/wrappers.proto,@com_google_protobuf//:wrappers_proto,github.com/golang/protobuf/ptypes/wrappers,@io_bazel_rules_go//proto/wkt:wrappers_go_proto -google/assistant/embedded/v1alpha2/embedded_assistant.proto,@go_googleapis//google/assistant/embedded/v1alpha2:embedded_proto,google.golang.org/genproto/googleapis/assistant/embedded/v1alpha2,@go_googleapis//google/assistant/embedded/v1alpha2:embedded_go_proto -google/assistant/embedded/v1alpha1/embedded_assistant.proto,@go_googleapis//google/assistant/embedded/v1alpha1:embedded_proto,google.golang.org/genproto/googleapis/assistant/embedded/v1alpha1,@go_googleapis//google/assistant/embedded/v1alpha1:embedded_go_proto -google/home/graph/v1/device.proto,@go_googleapis//google/home/graph/v1:graph_proto,google.golang.org/genproto/googleapis/home/graph/v1,@go_googleapis//google/home/graph/v1:graph_go_proto -google/home/graph/v1/homegraph.proto,@go_googleapis//google/home/graph/v1:graph_proto,google.golang.org/genproto/googleapis/home/graph/v1,@go_googleapis//google/home/graph/v1:graph_go_proto -google/genomics/v1/operations.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto -google/genomics/v1/variants.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto -google/genomics/v1/position.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto -google/genomics/v1/references.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto -google/genomics/v1/cigar.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto -google/genomics/v1/datasets.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto -google/genomics/v1/readalignment.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto -google/genomics/v1/annotations.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto -google/genomics/v1/reads.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto -google/genomics/v1/readgroup.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto -google/genomics/v1/readgroupset.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto -google/genomics/v1/range.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto -google/genomics/v1alpha2/pipelines.proto,@go_googleapis//google/genomics/v1alpha2:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1alpha2,@go_googleapis//google/genomics/v1alpha2:genomics_go_proto -google/bigtable/v1/bigtable_service_messages.proto,@go_googleapis//google/bigtable/v1:bigtable_proto,google.golang.org/genproto/googleapis/bigtable/v1,@go_googleapis//google/bigtable/v1:bigtable_go_proto -google/bigtable/v1/bigtable_service.proto,@go_googleapis//google/bigtable/v1:bigtable_proto,google.golang.org/genproto/googleapis/bigtable/v1,@go_googleapis//google/bigtable/v1:bigtable_go_proto -google/bigtable/v1/bigtable_data.proto,@go_googleapis//google/bigtable/v1:bigtable_proto,google.golang.org/genproto/googleapis/bigtable/v1,@go_googleapis//google/bigtable/v1:bigtable_go_proto -google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto,@go_googleapis//google/bigtable/admin/cluster/v1:cluster_proto,google.golang.org/genproto/googleapis/bigtable/admin/cluster/v1,@go_googleapis//google/bigtable/admin/cluster/v1:cluster_go_proto -google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto,@go_googleapis//google/bigtable/admin/cluster/v1:cluster_proto,google.golang.org/genproto/googleapis/bigtable/admin/cluster/v1,@go_googleapis//google/bigtable/admin/cluster/v1:cluster_go_proto -google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto,@go_googleapis//google/bigtable/admin/cluster/v1:cluster_proto,google.golang.org/genproto/googleapis/bigtable/admin/cluster/v1,@go_googleapis//google/bigtable/admin/cluster/v1:cluster_go_proto -google/bigtable/admin/v2/bigtable_instance_admin.proto,@go_googleapis//google/bigtable/admin/v2:admin_proto,google.golang.org/genproto/googleapis/bigtable/admin/v2,@go_googleapis//google/bigtable/admin/v2:admin_go_proto -google/bigtable/admin/v2/instance.proto,@go_googleapis//google/bigtable/admin/v2:admin_proto,google.golang.org/genproto/googleapis/bigtable/admin/v2,@go_googleapis//google/bigtable/admin/v2:admin_go_proto -google/bigtable/admin/v2/table.proto,@go_googleapis//google/bigtable/admin/v2:admin_proto,google.golang.org/genproto/googleapis/bigtable/admin/v2,@go_googleapis//google/bigtable/admin/v2:admin_go_proto -google/bigtable/admin/v2/bigtable_table_admin.proto,@go_googleapis//google/bigtable/admin/v2:admin_proto,google.golang.org/genproto/googleapis/bigtable/admin/v2,@go_googleapis//google/bigtable/admin/v2:admin_go_proto -google/bigtable/admin/v2/common.proto,@go_googleapis//google/bigtable/admin/v2:admin_proto,google.golang.org/genproto/googleapis/bigtable/admin/v2,@go_googleapis//google/bigtable/admin/v2:admin_go_proto -google/bigtable/admin/table/v1/bigtable_table_service_messages.proto,@go_googleapis//google/bigtable/admin/table/v1:table_proto,google.golang.org/genproto/googleapis/bigtable/admin/table/v1,@go_googleapis//google/bigtable/admin/table/v1:table_go_proto -google/bigtable/admin/table/v1/bigtable_table_service.proto,@go_googleapis//google/bigtable/admin/table/v1:table_proto,google.golang.org/genproto/googleapis/bigtable/admin/table/v1,@go_googleapis//google/bigtable/admin/table/v1:table_go_proto -google/bigtable/admin/table/v1/bigtable_table_data.proto,@go_googleapis//google/bigtable/admin/table/v1:table_proto,google.golang.org/genproto/googleapis/bigtable/admin/table/v1,@go_googleapis//google/bigtable/admin/table/v1:table_go_proto -google/bigtable/v2/bigtable.proto,@go_googleapis//google/bigtable/v2:bigtable_proto,google.golang.org/genproto/googleapis/bigtable/v2,@go_googleapis//google/bigtable/v2:bigtable_go_proto -google/bigtable/v2/data.proto,@go_googleapis//google/bigtable/v2:bigtable_proto,google.golang.org/genproto/googleapis/bigtable/v2,@go_googleapis//google/bigtable/v2:bigtable_go_proto -google/privacy/dlp/v2/storage.proto,@go_googleapis//google/privacy/dlp/v2:dlp_proto,google.golang.org/genproto/googleapis/privacy/dlp/v2,@go_googleapis//google/privacy/dlp/v2:dlp_go_proto -google/privacy/dlp/v2/dlp.proto,@go_googleapis//google/privacy/dlp/v2:dlp_proto,google.golang.org/genproto/googleapis/privacy/dlp/v2,@go_googleapis//google/privacy/dlp/v2:dlp_go_proto -google/watcher/v1/watch.proto,@go_googleapis//google/watcher/v1:watcher_proto,google.golang.org/genproto/googleapis/watcher/v1,@go_googleapis//google/watcher/v1:watcher_go_proto -google/firestore/admin/v1beta1/firestore_admin.proto,@go_googleapis//google/firestore/admin/v1beta1:admin_proto,google.golang.org/genproto/googleapis/firestore/admin/v1beta1,@go_googleapis//google/firestore/admin/v1beta1:admin_go_proto -google/firestore/admin/v1beta1/index.proto,@go_googleapis//google/firestore/admin/v1beta1:admin_proto,google.golang.org/genproto/googleapis/firestore/admin/v1beta1,@go_googleapis//google/firestore/admin/v1beta1:admin_go_proto -google/firestore/v1beta1/write.proto,@go_googleapis//google/firestore/v1beta1:firestore_proto,google.golang.org/genproto/googleapis/firestore/v1beta1,@go_googleapis//google/firestore/v1beta1:firestore_go_proto -google/firestore/v1beta1/document.proto,@go_googleapis//google/firestore/v1beta1:firestore_proto,google.golang.org/genproto/googleapis/firestore/v1beta1,@go_googleapis//google/firestore/v1beta1:firestore_go_proto -google/firestore/v1beta1/firestore.proto,@go_googleapis//google/firestore/v1beta1:firestore_proto,google.golang.org/genproto/googleapis/firestore/v1beta1,@go_googleapis//google/firestore/v1beta1:firestore_go_proto -google/firestore/v1beta1/query.proto,@go_googleapis//google/firestore/v1beta1:firestore_proto,google.golang.org/genproto/googleapis/firestore/v1beta1,@go_googleapis//google/firestore/v1beta1:firestore_go_proto -google/firestore/v1beta1/common.proto,@go_googleapis//google/firestore/v1beta1:firestore_proto,google.golang.org/genproto/googleapis/firestore/v1beta1,@go_googleapis//google/firestore/v1beta1:firestore_go_proto -google/example/library/v1/library.proto,@go_googleapis//google/example/library/v1:library_proto,google.golang.org/genproto/googleapis/example/library/v1,@go_googleapis//google/example/library/v1:library_go_proto -google/appengine/v1/instance.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto -google/appengine/v1/audit_data.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto -google/appengine/v1/appengine.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto -google/appengine/v1/application.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto -google/appengine/v1/operation.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto -google/appengine/v1/app_yaml.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto -google/appengine/v1/location.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto -google/appengine/v1/service.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto -google/appengine/v1/deploy.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto -google/appengine/v1/version.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto +google/api/annotations.proto,@go_googleapis//google/api:annotations_proto,google.golang.org/genproto/googleapis/api/annotations,@go_googleapis//google/api:annotations_go_proto +google/api/auth.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/backend.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/billing.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/config_change.proto,@go_googleapis//google/api:configchange_proto,google.golang.org/genproto/googleapis/api/configchange,@go_googleapis//google/api:configchange_go_proto +google/api/consumer.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/context.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/control.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/distribution.proto,@go_googleapis//google/api:distribution_proto,google.golang.org/genproto/googleapis/api/distribution,@go_googleapis//google/api:distribution_go_proto +google/api/documentation.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/endpoint.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/experimental/authorization_config.proto,@go_googleapis//google/api:api_proto,google.golang.org/genproto/googleapis/api,@go_googleapis//google/api:api_go_proto +google/api/experimental/experimental.proto,@go_googleapis//google/api:api_proto,google.golang.org/genproto/googleapis/api,@go_googleapis//google/api:api_go_proto +google/api/expr/v1alpha1/cel_service.proto,@go_googleapis//google/api/expr/v1alpha1:expr_proto,google.golang.org/genproto/googleapis/api/expr/v1alpha1,@go_googleapis//google/api/expr/v1alpha1:expr_go_proto +google/api/expr/v1alpha1/checked.proto,@go_googleapis//google/api/expr/v1alpha1:expr_proto,google.golang.org/genproto/googleapis/api/expr/v1alpha1,@go_googleapis//google/api/expr/v1alpha1:expr_go_proto +google/api/expr/v1alpha1/eval.proto,@go_googleapis//google/api/expr/v1alpha1:expr_proto,google.golang.org/genproto/googleapis/api/expr/v1alpha1,@go_googleapis//google/api/expr/v1alpha1:expr_go_proto +google/api/expr/v1alpha1/explain.proto,@go_googleapis//google/api/expr/v1alpha1:expr_proto,google.golang.org/genproto/googleapis/api/expr/v1alpha1,@go_googleapis//google/api/expr/v1alpha1:expr_go_proto +google/api/expr/v1alpha1/syntax.proto,@go_googleapis//google/api/expr/v1alpha1:expr_proto,google.golang.org/genproto/googleapis/api/expr/v1alpha1,@go_googleapis//google/api/expr/v1alpha1:expr_go_proto +google/api/expr/v1alpha1/value.proto,@go_googleapis//google/api/expr/v1alpha1:expr_proto,google.golang.org/genproto/googleapis/api/expr/v1alpha1,@go_googleapis//google/api/expr/v1alpha1:expr_go_proto +google/api/expr/v1beta1/decl.proto,@go_googleapis//google/api/expr/v1beta1:expr_proto,google.golang.org/genproto/googleapis/api/expr/v1beta1,@go_googleapis//google/api/expr/v1beta1:expr_go_proto +google/api/expr/v1beta1/eval.proto,@go_googleapis//google/api/expr/v1beta1:expr_proto,google.golang.org/genproto/googleapis/api/expr/v1beta1,@go_googleapis//google/api/expr/v1beta1:expr_go_proto +google/api/expr/v1beta1/expr.proto,@go_googleapis//google/api/expr/v1beta1:expr_proto,google.golang.org/genproto/googleapis/api/expr/v1beta1,@go_googleapis//google/api/expr/v1beta1:expr_go_proto +google/api/expr/v1beta1/source.proto,@go_googleapis//google/api/expr/v1beta1:expr_proto,google.golang.org/genproto/googleapis/api/expr/v1beta1,@go_googleapis//google/api/expr/v1beta1:expr_go_proto +google/api/expr/v1beta1/value.proto,@go_googleapis//google/api/expr/v1beta1:expr_proto,google.golang.org/genproto/googleapis/api/expr/v1beta1,@go_googleapis//google/api/expr/v1beta1:expr_go_proto +google/api/http.proto,@go_googleapis//google/api:annotations_proto,google.golang.org/genproto/googleapis/api/annotations,@go_googleapis//google/api:annotations_go_proto +google/api/httpbody.proto,@go_googleapis//google/api:httpbody_proto,google.golang.org/genproto/googleapis/api/httpbody,@go_googleapis//google/api:httpbody_go_proto +google/api/label.proto,@go_googleapis//google/api:label_proto,google.golang.org/genproto/googleapis/api/label,@go_googleapis//google/api:label_go_proto +google/api/launch_stage.proto,@go_googleapis//google/api:api_proto,google.golang.org/genproto/googleapis/api,@go_googleapis//google/api:api_go_proto +google/api/log.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/logging.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/metric.proto,@go_googleapis//google/api:metric_proto,google.golang.org/genproto/googleapis/api/metric,@go_googleapis//google/api:metric_go_proto +google/api/monitored_resource.proto,@go_googleapis//google/api:monitoredres_proto,google.golang.org/genproto/googleapis/api/monitoredres,@go_googleapis//google/api:monitoredres_go_proto +google/api/monitoring.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/quota.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/service.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/servicecontrol/v1/check_error.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto +google/api/servicecontrol/v1/distribution.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto +google/api/servicecontrol/v1/log_entry.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto +google/api/servicecontrol/v1/metric_value.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto +google/api/servicecontrol/v1/operation.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto +google/api/servicecontrol/v1/quota_controller.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto +google/api/servicecontrol/v1/service_controller.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto +google/api/servicemanagement/v1/resources.proto,@go_googleapis//google/api/servicemanagement/v1:servicemanagement_proto,google.golang.org/genproto/googleapis/api/servicemanagement/v1,@go_googleapis//google/api/servicemanagement/v1:servicemanagement_go_proto +google/api/servicemanagement/v1/servicemanager.proto,@go_googleapis//google/api/servicemanagement/v1:servicemanagement_proto,google.golang.org/genproto/googleapis/api/servicemanagement/v1,@go_googleapis//google/api/servicemanagement/v1:servicemanagement_go_proto +google/api/source_info.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/system_parameter.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto +google/api/usage.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto google/appengine/legacy/audit_data.proto,@go_googleapis//google/appengine/legacy:legacy_proto,google.golang.org/genproto/googleapis/appengine/legacy,@go_googleapis//google/appengine/legacy:legacy_go_proto google/appengine/logging/v1/request_log.proto,@go_googleapis//google/appengine/logging/v1:logging_proto,google.golang.org/genproto/googleapis/appengine/logging/v1,@go_googleapis//google/appengine/logging/v1:logging_go_proto -google/storagetransfer/v1/transfer.proto,@go_googleapis//google/storagetransfer/v1:storagetransfer_proto,google.golang.org/genproto/googleapis/storagetransfer/v1,@go_googleapis//google/storagetransfer/v1:storagetransfer_go_proto -google/storagetransfer/v1/transfer_types.proto,@go_googleapis//google/storagetransfer/v1:storagetransfer_proto,google.golang.org/genproto/googleapis/storagetransfer/v1,@go_googleapis//google/storagetransfer/v1:storagetransfer_go_proto -google/longrunning/operations.proto,@go_googleapis//google/longrunning:longrunning_proto,google.golang.org/genproto/googleapis/longrunning,@go_googleapis//google/longrunning:longrunning_go_proto -google/container/v1/cluster_service.proto,@go_googleapis//google/container/v1:container_proto,google.golang.org/genproto/googleapis/container/v1,@go_googleapis//google/container/v1:container_go_proto -google/container/v1beta1/cluster_service.proto,@go_googleapis//google/container/v1beta1:container_proto,google.golang.org/genproto/googleapis/container/v1beta1,@go_googleapis//google/container/v1beta1:container_go_proto -google/container/v1alpha1/cluster_service.proto,@go_googleapis//google/container/v1alpha1:container_proto,google.golang.org/genproto/googleapis/container/v1alpha1,@go_googleapis//google/container/v1alpha1:container_go_proto -google/datastore/v1beta3/datastore.proto,@go_googleapis//google/datastore/v1beta3:datastore_proto,google.golang.org/genproto/googleapis/datastore/v1beta3,@go_googleapis//google/datastore/v1beta3:datastore_go_proto -google/datastore/v1beta3/query.proto,@go_googleapis//google/datastore/v1beta3:datastore_proto,google.golang.org/genproto/googleapis/datastore/v1beta3,@go_googleapis//google/datastore/v1beta3:datastore_go_proto -google/datastore/v1beta3/entity.proto,@go_googleapis//google/datastore/v1beta3:datastore_proto,google.golang.org/genproto/googleapis/datastore/v1beta3,@go_googleapis//google/datastore/v1beta3:datastore_go_proto -google/datastore/v1/datastore.proto,@go_googleapis//google/datastore/v1:datastore_proto,google.golang.org/genproto/googleapis/datastore/v1,@go_googleapis//google/datastore/v1:datastore_go_proto -google/datastore/v1/query.proto,@go_googleapis//google/datastore/v1:datastore_proto,google.golang.org/genproto/googleapis/datastore/v1,@go_googleapis//google/datastore/v1:datastore_go_proto -google/datastore/v1/entity.proto,@go_googleapis//google/datastore/v1:datastore_proto,google.golang.org/genproto/googleapis/datastore/v1,@go_googleapis//google/datastore/v1:datastore_go_proto -google/datastore/admin/v1/datastore_admin.proto,@go_googleapis//google/datastore/admin/v1:admin_proto,google.golang.org/genproto/googleapis/datastore/admin/v1,@go_googleapis//google/datastore/admin/v1:admin_go_proto -google/datastore/admin/v1beta1/datastore_admin.proto,@go_googleapis//google/datastore/admin/v1beta1:admin_proto,google.golang.org/genproto/googleapis/datastore/admin/v1beta1,@go_googleapis//google/datastore/admin/v1beta1:admin_go_proto +google/appengine/v1/app_yaml.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto +google/appengine/v1/appengine.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto +google/appengine/v1/application.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto +google/appengine/v1/audit_data.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto +google/appengine/v1/deploy.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto +google/appengine/v1/instance.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto +google/appengine/v1/location.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto +google/appengine/v1/operation.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto +google/appengine/v1/service.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto +google/appengine/v1/version.proto,@go_googleapis//google/appengine/v1:appengine_proto,google.golang.org/genproto/googleapis/appengine/v1,@go_googleapis//google/appengine/v1:appengine_go_proto +google/assistant/embedded/v1alpha1/embedded_assistant.proto,@go_googleapis//google/assistant/embedded/v1alpha1:embedded_proto,google.golang.org/genproto/googleapis/assistant/embedded/v1alpha1,@go_googleapis//google/assistant/embedded/v1alpha1:embedded_go_proto +google/assistant/embedded/v1alpha2/embedded_assistant.proto,@go_googleapis//google/assistant/embedded/v1alpha2:embedded_proto,google.golang.org/genproto/googleapis/assistant/embedded/v1alpha2,@go_googleapis//google/assistant/embedded/v1alpha2:embedded_go_proto +google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto,@go_googleapis//google/bigtable/admin/cluster/v1:cluster_proto,google.golang.org/genproto/googleapis/bigtable/admin/cluster/v1,@go_googleapis//google/bigtable/admin/cluster/v1:cluster_go_proto +google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto,@go_googleapis//google/bigtable/admin/cluster/v1:cluster_proto,google.golang.org/genproto/googleapis/bigtable/admin/cluster/v1,@go_googleapis//google/bigtable/admin/cluster/v1:cluster_go_proto +google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto,@go_googleapis//google/bigtable/admin/cluster/v1:cluster_proto,google.golang.org/genproto/googleapis/bigtable/admin/cluster/v1,@go_googleapis//google/bigtable/admin/cluster/v1:cluster_go_proto +google/bigtable/admin/table/v1/bigtable_table_data.proto,@go_googleapis//google/bigtable/admin/table/v1:table_proto,google.golang.org/genproto/googleapis/bigtable/admin/table/v1,@go_googleapis//google/bigtable/admin/table/v1:table_go_proto +google/bigtable/admin/table/v1/bigtable_table_service.proto,@go_googleapis//google/bigtable/admin/table/v1:table_proto,google.golang.org/genproto/googleapis/bigtable/admin/table/v1,@go_googleapis//google/bigtable/admin/table/v1:table_go_proto +google/bigtable/admin/table/v1/bigtable_table_service_messages.proto,@go_googleapis//google/bigtable/admin/table/v1:table_proto,google.golang.org/genproto/googleapis/bigtable/admin/table/v1,@go_googleapis//google/bigtable/admin/table/v1:table_go_proto +google/bigtable/admin/v2/bigtable_instance_admin.proto,@go_googleapis//google/bigtable/admin/v2:admin_proto,google.golang.org/genproto/googleapis/bigtable/admin/v2,@go_googleapis//google/bigtable/admin/v2:admin_go_proto +google/bigtable/admin/v2/bigtable_table_admin.proto,@go_googleapis//google/bigtable/admin/v2:admin_proto,google.golang.org/genproto/googleapis/bigtable/admin/v2,@go_googleapis//google/bigtable/admin/v2:admin_go_proto +google/bigtable/admin/v2/common.proto,@go_googleapis//google/bigtable/admin/v2:admin_proto,google.golang.org/genproto/googleapis/bigtable/admin/v2,@go_googleapis//google/bigtable/admin/v2:admin_go_proto +google/bigtable/admin/v2/instance.proto,@go_googleapis//google/bigtable/admin/v2:admin_proto,google.golang.org/genproto/googleapis/bigtable/admin/v2,@go_googleapis//google/bigtable/admin/v2:admin_go_proto +google/bigtable/admin/v2/table.proto,@go_googleapis//google/bigtable/admin/v2:admin_proto,google.golang.org/genproto/googleapis/bigtable/admin/v2,@go_googleapis//google/bigtable/admin/v2:admin_go_proto +google/bigtable/v1/bigtable_data.proto,@go_googleapis//google/bigtable/v1:bigtable_proto,google.golang.org/genproto/googleapis/bigtable/v1,@go_googleapis//google/bigtable/v1:bigtable_go_proto +google/bigtable/v1/bigtable_service.proto,@go_googleapis//google/bigtable/v1:bigtable_proto,google.golang.org/genproto/googleapis/bigtable/v1,@go_googleapis//google/bigtable/v1:bigtable_go_proto +google/bigtable/v1/bigtable_service_messages.proto,@go_googleapis//google/bigtable/v1:bigtable_proto,google.golang.org/genproto/googleapis/bigtable/v1,@go_googleapis//google/bigtable/v1:bigtable_go_proto +google/bigtable/v2/bigtable.proto,@go_googleapis//google/bigtable/v2:bigtable_proto,google.golang.org/genproto/googleapis/bigtable/v2,@go_googleapis//google/bigtable/v2:bigtable_go_proto +google/bigtable/v2/data.proto,@go_googleapis//google/bigtable/v2:bigtable_proto,google.golang.org/genproto/googleapis/bigtable/v2,@go_googleapis//google/bigtable/v2:bigtable_go_proto google/bytestream/bytestream.proto,@go_googleapis//google/bytestream:bytestream_proto,google.golang.org/genproto/googleapis/bytestream,@go_googleapis//google/bytestream:bytestream_go_proto -google/iam/v1/iam_policy.proto,@go_googleapis//google/iam/v1:iam_proto,google.golang.org/genproto/googleapis/iam/v1,@go_googleapis//google/iam/v1:iam_go_proto -google/iam/v1/policy.proto,@go_googleapis//google/iam/v1:iam_proto,google.golang.org/genproto/googleapis/iam/v1,@go_googleapis//google/iam/v1:iam_go_proto -google/iam/v1/logging/audit_data.proto,@go_googleapis//google/iam/v1/logging:logging_proto,google.golang.org/genproto/googleapis/iam/v1/logging,@go_googleapis//google/iam/v1/logging:logging_go_proto -google/iam/admin/v1/iam.proto,@go_googleapis//google/iam/admin/v1:admin_proto,google.golang.org/genproto/googleapis/iam/admin/v1,@go_googleapis//google/iam/admin/v1:admin_go_proto -google/type/money.proto,@go_googleapis//google/type:money_proto,google.golang.org/genproto/googleapis/type/money,@go_googleapis//google/type:money_go_proto -google/type/latlng.proto,@go_googleapis//google/type:latlng_proto,google.golang.org/genproto/googleapis/type/latlng,@go_googleapis//google/type:latlng_go_proto -google/type/color.proto,@go_googleapis//google/type:color_proto,google.golang.org/genproto/googleapis/type/color,@go_googleapis//google/type:color_go_proto -google/type/timeofday.proto,@go_googleapis//google/type:timeofday_proto,google.golang.org/genproto/googleapis/type/timeofday,@go_googleapis//google/type:timeofday_go_proto -google/type/date.proto,@go_googleapis//google/type:date_proto,google.golang.org/genproto/googleapis/type/date,@go_googleapis//google/type:date_go_proto -google/type/dayofweek.proto,@go_googleapis//google/type:dayofweek_proto,google.golang.org/genproto/googleapis/type/dayofweek,@go_googleapis//google/type:dayofweek_go_proto -google/type/postal_address.proto,@go_googleapis//google/type:postaladdress_proto,google.golang.org/genproto/googleapis/type/postaladdress,@go_googleapis//google/type:postaladdress_go_proto -google/devtools/clouderrorreporting/v1beta1/report_errors_service.proto,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_proto,google.golang.org/genproto/googleapis/devtools/clouderrorreporting/v1beta1,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_go_proto -google/devtools/clouderrorreporting/v1beta1/error_group_service.proto,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_proto,google.golang.org/genproto/googleapis/devtools/clouderrorreporting/v1beta1,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_go_proto -google/devtools/clouderrorreporting/v1beta1/error_stats_service.proto,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_proto,google.golang.org/genproto/googleapis/devtools/clouderrorreporting/v1beta1,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_go_proto -google/devtools/clouderrorreporting/v1beta1/common.proto,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_proto,google.golang.org/genproto/googleapis/devtools/clouderrorreporting/v1beta1,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_go_proto -google/devtools/resultstore/v2/file.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto -google/devtools/resultstore/v2/resultstore_download.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto -google/devtools/resultstore/v2/configuration.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto -google/devtools/resultstore/v2/action.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto -google/devtools/resultstore/v2/resultstore_file_download.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto -google/devtools/resultstore/v2/test_suite.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto -google/devtools/resultstore/v2/file_set.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto -google/devtools/resultstore/v2/coverage.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto -google/devtools/resultstore/v2/coverage_summary.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto -google/devtools/resultstore/v2/configured_target.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto -google/devtools/resultstore/v2/target.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto -google/devtools/resultstore/v2/invocation.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto -google/devtools/resultstore/v2/common.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto -google/devtools/source/v1/source_context.proto,@go_googleapis//google/devtools/source/v1:source_proto,google.golang.org/genproto/googleapis/devtools/source/v1,@go_googleapis//google/devtools/source/v1:source_go_proto -google/devtools/remoteexecution/v1test/remote_execution.proto,@go_googleapis//google/devtools/remoteexecution/v1test:remoteexecution_proto,google.golang.org/genproto/googleapis/devtools/remoteexecution/v1test,@go_googleapis//google/devtools/remoteexecution/v1test:remoteexecution_go_proto -google/devtools/cloudbuild/v1/cloudbuild.proto,@go_googleapis//google/devtools/cloudbuild/v1:cloudbuild_proto,google.golang.org/genproto/googleapis/devtools/cloudbuild/v1,@go_googleapis//google/devtools/cloudbuild/v1:cloudbuild_go_proto -google/devtools/sourcerepo/v1/sourcerepo.proto,@go_googleapis//google/devtools/sourcerepo/v1:sourcerepo_proto,google.golang.org/genproto/googleapis/devtools/sourcerepo/v1,@go_googleapis//google/devtools/sourcerepo/v1:sourcerepo_go_proto -google/devtools/remoteworkers/v1test2/worker.proto,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_proto,google.golang.org/genproto/googleapis/devtools/remoteworkers/v1test2,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_go_proto -google/devtools/remoteworkers/v1test2/tasks.proto,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_proto,google.golang.org/genproto/googleapis/devtools/remoteworkers/v1test2,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_go_proto -google/devtools/remoteworkers/v1test2/bots.proto,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_proto,google.golang.org/genproto/googleapis/devtools/remoteworkers/v1test2,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_go_proto -google/devtools/remoteworkers/v1test2/command.proto,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_proto,google.golang.org/genproto/googleapis/devtools/remoteworkers/v1test2,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_go_proto -google/devtools/cloudtrace/v1/trace.proto,@go_googleapis//google/devtools/cloudtrace/v1:cloudtrace_proto,google.golang.org/genproto/googleapis/devtools/cloudtrace/v1,@go_googleapis//google/devtools/cloudtrace/v1:cloudtrace_go_proto -google/devtools/cloudtrace/v2/trace.proto,@go_googleapis//google/devtools/cloudtrace/v2:cloudtrace_proto,google.golang.org/genproto/googleapis/devtools/cloudtrace/v2,@go_googleapis//google/devtools/cloudtrace/v2:cloudtrace_go_proto -google/devtools/cloudtrace/v2/tracing.proto,@go_googleapis//google/devtools/cloudtrace/v2:cloudtrace_proto,google.golang.org/genproto/googleapis/devtools/cloudtrace/v2,@go_googleapis//google/devtools/cloudtrace/v2:cloudtrace_go_proto -google/devtools/cloudprofiler/v2/profiler.proto,@go_googleapis//google/devtools/cloudprofiler/v2:cloudprofiler_proto,google.golang.org/genproto/googleapis/devtools/cloudprofiler/v2,@go_googleapis//google/devtools/cloudprofiler/v2:cloudprofiler_go_proto -google/devtools/containeranalysis/v1alpha1/containeranalysis.proto,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_go_proto -google/devtools/containeranalysis/v1alpha1/bill_of_materials.proto,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_go_proto -google/devtools/containeranalysis/v1alpha1/provenance.proto,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_go_proto -google/devtools/containeranalysis/v1alpha1/package_vulnerability.proto,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_go_proto -google/devtools/containeranalysis/v1alpha1/source_context.proto,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_go_proto -google/devtools/containeranalysis/v1alpha1/image_basis.proto,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_go_proto +google/cloud/asset/v1beta1/asset_service.proto,@go_googleapis//google/cloud/asset/v1beta1:asset_proto,google.golang.org/genproto/googleapis/cloud/asset/v1beta1,@go_googleapis//google/cloud/asset/v1beta1:asset_go_proto +google/cloud/asset/v1beta1/assets.proto,@go_googleapis//google/cloud/asset/v1beta1:asset_proto,google.golang.org/genproto/googleapis/cloud/asset/v1beta1,@go_googleapis//google/cloud/asset/v1beta1:asset_go_proto +google/cloud/audit/audit_log.proto,@go_googleapis//google/cloud/audit:audit_proto,google.golang.org/genproto/googleapis/cloud/audit,@go_googleapis//google/cloud/audit:audit_go_proto +google/cloud/automl/v1beta1/annotation_payload.proto,@go_googleapis//google/cloud/automl/v1beta1:automl_proto,google.golang.org/genproto/googleapis/cloud/automl/v1beta1,@go_googleapis//google/cloud/automl/v1beta1:automl_go_proto +google/cloud/automl/v1beta1/classification.proto,@go_googleapis//google/cloud/automl/v1beta1:automl_proto,google.golang.org/genproto/googleapis/cloud/automl/v1beta1,@go_googleapis//google/cloud/automl/v1beta1:automl_go_proto +google/cloud/automl/v1beta1/data_items.proto,@go_googleapis//google/cloud/automl/v1beta1:automl_proto,google.golang.org/genproto/googleapis/cloud/automl/v1beta1,@go_googleapis//google/cloud/automl/v1beta1:automl_go_proto +google/cloud/automl/v1beta1/dataset.proto,@go_googleapis//google/cloud/automl/v1beta1:automl_proto,google.golang.org/genproto/googleapis/cloud/automl/v1beta1,@go_googleapis//google/cloud/automl/v1beta1:automl_go_proto +google/cloud/automl/v1beta1/image.proto,@go_googleapis//google/cloud/automl/v1beta1:automl_proto,google.golang.org/genproto/googleapis/cloud/automl/v1beta1,@go_googleapis//google/cloud/automl/v1beta1:automl_go_proto +google/cloud/automl/v1beta1/io.proto,@go_googleapis//google/cloud/automl/v1beta1:automl_proto,google.golang.org/genproto/googleapis/cloud/automl/v1beta1,@go_googleapis//google/cloud/automl/v1beta1:automl_go_proto +google/cloud/automl/v1beta1/model.proto,@go_googleapis//google/cloud/automl/v1beta1:automl_proto,google.golang.org/genproto/googleapis/cloud/automl/v1beta1,@go_googleapis//google/cloud/automl/v1beta1:automl_go_proto +google/cloud/automl/v1beta1/model_evaluation.proto,@go_googleapis//google/cloud/automl/v1beta1:automl_proto,google.golang.org/genproto/googleapis/cloud/automl/v1beta1,@go_googleapis//google/cloud/automl/v1beta1:automl_go_proto +google/cloud/automl/v1beta1/operations.proto,@go_googleapis//google/cloud/automl/v1beta1:automl_proto,google.golang.org/genproto/googleapis/cloud/automl/v1beta1,@go_googleapis//google/cloud/automl/v1beta1:automl_go_proto +google/cloud/automl/v1beta1/prediction_service.proto,@go_googleapis//google/cloud/automl/v1beta1:automl_proto,google.golang.org/genproto/googleapis/cloud/automl/v1beta1,@go_googleapis//google/cloud/automl/v1beta1:automl_go_proto +google/cloud/automl/v1beta1/service.proto,@go_googleapis//google/cloud/automl/v1beta1:automl_proto,google.golang.org/genproto/googleapis/cloud/automl/v1beta1,@go_googleapis//google/cloud/automl/v1beta1:automl_go_proto +google/cloud/automl/v1beta1/text.proto,@go_googleapis//google/cloud/automl/v1beta1:automl_proto,google.golang.org/genproto/googleapis/cloud/automl/v1beta1,@go_googleapis//google/cloud/automl/v1beta1:automl_go_proto +google/cloud/automl/v1beta1/translation.proto,@go_googleapis//google/cloud/automl/v1beta1:automl_proto,google.golang.org/genproto/googleapis/cloud/automl/v1beta1,@go_googleapis//google/cloud/automl/v1beta1:automl_go_proto +google/cloud/bigquery/datatransfer/v1/datatransfer.proto,@go_googleapis//google/cloud/bigquery/datatransfer/v1:datatransfer_proto,google.golang.org/genproto/googleapis/cloud/bigquery/datatransfer/v1,@go_googleapis//google/cloud/bigquery/datatransfer/v1:datatransfer_go_proto +google/cloud/bigquery/datatransfer/v1/transfer.proto,@go_googleapis//google/cloud/bigquery/datatransfer/v1:datatransfer_proto,google.golang.org/genproto/googleapis/cloud/bigquery/datatransfer/v1,@go_googleapis//google/cloud/bigquery/datatransfer/v1:datatransfer_go_proto +google/cloud/bigquery/logging/v1/audit_data.proto,@go_googleapis//google/cloud/bigquery/logging/v1:logging_proto,google.golang.org/genproto/googleapis/cloud/bigquery/logging/v1,@go_googleapis//google/cloud/bigquery/logging/v1:logging_go_proto +google/cloud/bigquery/storage/v1beta1/avro.proto,@go_googleapis//google/cloud/bigquery/storage/v1beta1:storage_proto,google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta1,@go_googleapis//google/cloud/bigquery/storage/v1beta1:storage_go_proto +google/cloud/bigquery/storage/v1beta1/read_options.proto,@go_googleapis//google/cloud/bigquery/storage/v1beta1:storage_proto,google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta1,@go_googleapis//google/cloud/bigquery/storage/v1beta1:storage_go_proto +google/cloud/bigquery/storage/v1beta1/storage.proto,@go_googleapis//google/cloud/bigquery/storage/v1beta1:storage_proto,google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta1,@go_googleapis//google/cloud/bigquery/storage/v1beta1:storage_go_proto +google/cloud/bigquery/storage/v1beta1/table_reference.proto,@go_googleapis//google/cloud/bigquery/storage/v1beta1:storage_proto,google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta1,@go_googleapis//google/cloud/bigquery/storage/v1beta1:storage_go_proto +google/cloud/billing/v1/cloud_billing.proto,@go_googleapis//google/cloud/billing/v1:billing_proto,google.golang.org/genproto/googleapis/cloud/billing/v1,@go_googleapis//google/cloud/billing/v1:billing_go_proto +google/cloud/dataproc/v1/clusters.proto,@go_googleapis//google/cloud/dataproc/v1:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1,@go_googleapis//google/cloud/dataproc/v1:dataproc_go_proto +google/cloud/dataproc/v1/jobs.proto,@go_googleapis//google/cloud/dataproc/v1:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1,@go_googleapis//google/cloud/dataproc/v1:dataproc_go_proto +google/cloud/dataproc/v1/operations.proto,@go_googleapis//google/cloud/dataproc/v1:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1,@go_googleapis//google/cloud/dataproc/v1:dataproc_go_proto +google/cloud/dataproc/v1beta2/clusters.proto,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1beta2,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_go_proto +google/cloud/dataproc/v1beta2/jobs.proto,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1beta2,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_go_proto +google/cloud/dataproc/v1beta2/operations.proto,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1beta2,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_go_proto +google/cloud/dataproc/v1beta2/shared.proto,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1beta2,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_go_proto +google/cloud/dataproc/v1beta2/workflow_templates.proto,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1beta2,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_go_proto +google/cloud/dialogflow/v2/agent.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto +google/cloud/dialogflow/v2/context.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto +google/cloud/dialogflow/v2/entity_type.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto +google/cloud/dialogflow/v2/intent.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto +google/cloud/dialogflow/v2/session.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto +google/cloud/dialogflow/v2/session_entity_type.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto +google/cloud/dialogflow/v2/webhook.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto +google/cloud/dialogflow/v2beta1/agent.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto +google/cloud/dialogflow/v2beta1/audio_config.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto +google/cloud/dialogflow/v2beta1/context.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto +google/cloud/dialogflow/v2beta1/document.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto +google/cloud/dialogflow/v2beta1/entity_type.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto +google/cloud/dialogflow/v2beta1/intent.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto +google/cloud/dialogflow/v2beta1/knowledge_base.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto +google/cloud/dialogflow/v2beta1/session.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto +google/cloud/dialogflow/v2beta1/session_entity_type.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto +google/cloud/dialogflow/v2beta1/webhook.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto +google/cloud/functions/v1beta2/functions.proto,@go_googleapis//google/cloud/functions/v1beta2:functions_proto,google.golang.org/genproto/googleapis/cloud/functions/v1beta2,@go_googleapis//google/cloud/functions/v1beta2:functions_go_proto +google/cloud/functions/v1beta2/operations.proto,@go_googleapis//google/cloud/functions/v1beta2:functions_proto,google.golang.org/genproto/googleapis/cloud/functions/v1beta2,@go_googleapis//google/cloud/functions/v1beta2:functions_go_proto +google/cloud/iot/v1/device_manager.proto,@go_googleapis//google/cloud/iot/v1:iot_proto,google.golang.org/genproto/googleapis/cloud/iot/v1,@go_googleapis//google/cloud/iot/v1:iot_go_proto +google/cloud/iot/v1/resources.proto,@go_googleapis//google/cloud/iot/v1:iot_proto,google.golang.org/genproto/googleapis/cloud/iot/v1,@go_googleapis//google/cloud/iot/v1:iot_go_proto +google/cloud/kms/v1/resources.proto,@go_googleapis//google/cloud/kms/v1:kms_proto,google.golang.org/genproto/googleapis/cloud/kms/v1,@go_googleapis//google/cloud/kms/v1:kms_go_proto +google/cloud/kms/v1/service.proto,@go_googleapis//google/cloud/kms/v1:kms_proto,google.golang.org/genproto/googleapis/cloud/kms/v1,@go_googleapis//google/cloud/kms/v1:kms_go_proto +google/cloud/language/v1/language_service.proto,@go_googleapis//google/cloud/language/v1:language_proto,google.golang.org/genproto/googleapis/cloud/language/v1,@go_googleapis//google/cloud/language/v1:language_go_proto +google/cloud/language/v1beta1/language_service.proto,@go_googleapis//google/cloud/language/v1beta1:language_proto,google.golang.org/genproto/googleapis/cloud/language/v1beta1,@go_googleapis//google/cloud/language/v1beta1:language_go_proto +google/cloud/language/v1beta2/language_service.proto,@go_googleapis//google/cloud/language/v1beta2:language_proto,google.golang.org/genproto/googleapis/cloud/language/v1beta2,@go_googleapis//google/cloud/language/v1beta2:language_go_proto +google/cloud/location/locations.proto,@go_googleapis//google/cloud/location:location_proto,google.golang.org/genproto/googleapis/cloud/location,@go_googleapis//google/cloud/location:location_go_proto +google/cloud/ml/v1/job_service.proto,@go_googleapis//google/cloud/ml/v1:ml_proto,google.golang.org/genproto/googleapis/cloud/ml/v1,@go_googleapis//google/cloud/ml/v1:ml_go_proto +google/cloud/ml/v1/model_service.proto,@go_googleapis//google/cloud/ml/v1:ml_proto,google.golang.org/genproto/googleapis/cloud/ml/v1,@go_googleapis//google/cloud/ml/v1:ml_go_proto +google/cloud/ml/v1/operation_metadata.proto,@go_googleapis//google/cloud/ml/v1:ml_proto,google.golang.org/genproto/googleapis/cloud/ml/v1,@go_googleapis//google/cloud/ml/v1:ml_go_proto +google/cloud/ml/v1/prediction_service.proto,@go_googleapis//google/cloud/ml/v1:ml_proto,google.golang.org/genproto/googleapis/cloud/ml/v1,@go_googleapis//google/cloud/ml/v1:ml_go_proto +google/cloud/ml/v1/project_service.proto,@go_googleapis//google/cloud/ml/v1:ml_proto,google.golang.org/genproto/googleapis/cloud/ml/v1,@go_googleapis//google/cloud/ml/v1:ml_go_proto +google/cloud/oslogin/common/common.proto,@go_googleapis//google/cloud/oslogin/common:common_proto,google.golang.org/genproto/googleapis/cloud/oslogin/common,@go_googleapis//google/cloud/oslogin/common:common_go_proto +google/cloud/oslogin/v1/oslogin.proto,@go_googleapis//google/cloud/oslogin/v1:oslogin_proto,google.golang.org/genproto/googleapis/cloud/oslogin/v1,@go_googleapis//google/cloud/oslogin/v1:oslogin_go_proto +google/cloud/oslogin/v1alpha/oslogin.proto,@go_googleapis//google/cloud/oslogin/v1alpha:oslogin_proto,google.golang.org/genproto/googleapis/cloud/oslogin/v1alpha,@go_googleapis//google/cloud/oslogin/v1alpha:oslogin_go_proto +google/cloud/oslogin/v1beta/oslogin.proto,@go_googleapis//google/cloud/oslogin/v1beta:oslogin_proto,google.golang.org/genproto/googleapis/cloud/oslogin/v1beta,@go_googleapis//google/cloud/oslogin/v1beta:oslogin_go_proto +google/cloud/redis/v1/cloud_redis.proto,@go_googleapis//google/cloud/redis/v1:redis_proto,google.golang.org/genproto/googleapis/cloud/redis/v1,@go_googleapis//google/cloud/redis/v1:redis_go_proto +google/cloud/redis/v1beta1/cloud_redis.proto,@go_googleapis//google/cloud/redis/v1beta1:redis_proto,google.golang.org/genproto/googleapis/cloud/redis/v1beta1,@go_googleapis//google/cloud/redis/v1beta1:redis_go_proto +google/cloud/resourcemanager/v2/folders.proto,@go_googleapis//google/cloud/resourcemanager/v2:resourcemanager_proto,google.golang.org/genproto/googleapis/cloud/resourcemanager/v2,@go_googleapis//google/cloud/resourcemanager/v2:resourcemanager_go_proto +google/cloud/runtimeconfig/v1beta1/resources.proto,@go_googleapis//google/cloud/runtimeconfig/v1beta1:runtimeconfig_proto,google.golang.org/genproto/googleapis/cloud/runtimeconfig/v1beta1,@go_googleapis//google/cloud/runtimeconfig/v1beta1:runtimeconfig_go_proto +google/cloud/runtimeconfig/v1beta1/runtimeconfig.proto,@go_googleapis//google/cloud/runtimeconfig/v1beta1:runtimeconfig_proto,google.golang.org/genproto/googleapis/cloud/runtimeconfig/v1beta1,@go_googleapis//google/cloud/runtimeconfig/v1beta1:runtimeconfig_go_proto +google/cloud/speech/v1/cloud_speech.proto,@go_googleapis//google/cloud/speech/v1:speech_proto,google.golang.org/genproto/googleapis/cloud/speech/v1,@go_googleapis//google/cloud/speech/v1:speech_go_proto +google/cloud/speech/v1p1beta1/cloud_speech.proto,@go_googleapis//google/cloud/speech/v1p1beta1:speech_proto,google.golang.org/genproto/googleapis/cloud/speech/v1p1beta1,@go_googleapis//google/cloud/speech/v1p1beta1:speech_go_proto +google/cloud/support/common.proto,@go_googleapis//google/cloud/support:common_proto,google.golang.org/genproto/googleapis/cloud/support/common,@go_googleapis//google/cloud/support:common_go_proto +google/cloud/support/v1alpha1/cloud_support.proto,@go_googleapis//google/cloud/support/v1alpha1:support_proto,google.golang.org/genproto/googleapis/cloud/support/v1alpha1,@go_googleapis//google/cloud/support/v1alpha1:support_go_proto +google/cloud/tasks/v2beta2/cloudtasks.proto,@go_googleapis//google/cloud/tasks/v2beta2:tasks_proto,google.golang.org/genproto/googleapis/cloud/tasks/v2beta2,@go_googleapis//google/cloud/tasks/v2beta2:tasks_go_proto +google/cloud/tasks/v2beta2/queue.proto,@go_googleapis//google/cloud/tasks/v2beta2:tasks_proto,google.golang.org/genproto/googleapis/cloud/tasks/v2beta2,@go_googleapis//google/cloud/tasks/v2beta2:tasks_go_proto +google/cloud/tasks/v2beta2/target.proto,@go_googleapis//google/cloud/tasks/v2beta2:tasks_proto,google.golang.org/genproto/googleapis/cloud/tasks/v2beta2,@go_googleapis//google/cloud/tasks/v2beta2:tasks_go_proto +google/cloud/tasks/v2beta2/task.proto,@go_googleapis//google/cloud/tasks/v2beta2:tasks_proto,google.golang.org/genproto/googleapis/cloud/tasks/v2beta2,@go_googleapis//google/cloud/tasks/v2beta2:tasks_go_proto +google/cloud/tasks/v2beta3/cloudtasks.proto,@go_googleapis//google/cloud/tasks/v2beta3:tasks_proto,google.golang.org/genproto/googleapis/cloud/tasks/v2beta3,@go_googleapis//google/cloud/tasks/v2beta3:tasks_go_proto +google/cloud/tasks/v2beta3/queue.proto,@go_googleapis//google/cloud/tasks/v2beta3:tasks_proto,google.golang.org/genproto/googleapis/cloud/tasks/v2beta3,@go_googleapis//google/cloud/tasks/v2beta3:tasks_go_proto +google/cloud/tasks/v2beta3/target.proto,@go_googleapis//google/cloud/tasks/v2beta3:tasks_proto,google.golang.org/genproto/googleapis/cloud/tasks/v2beta3,@go_googleapis//google/cloud/tasks/v2beta3:tasks_go_proto +google/cloud/tasks/v2beta3/task.proto,@go_googleapis//google/cloud/tasks/v2beta3:tasks_proto,google.golang.org/genproto/googleapis/cloud/tasks/v2beta3,@go_googleapis//google/cloud/tasks/v2beta3:tasks_go_proto +google/cloud/texttospeech/v1/cloud_tts.proto,@go_googleapis//google/cloud/texttospeech/v1:texttospeech_proto,google.golang.org/genproto/googleapis/cloud/texttospeech/v1,@go_googleapis//google/cloud/texttospeech/v1:texttospeech_go_proto +google/cloud/texttospeech/v1beta1/cloud_tts.proto,@go_googleapis//google/cloud/texttospeech/v1beta1:texttospeech_proto,google.golang.org/genproto/googleapis/cloud/texttospeech/v1beta1,@go_googleapis//google/cloud/texttospeech/v1beta1:texttospeech_go_proto +google/cloud/videointelligence/v1/video_intelligence.proto,@go_googleapis//google/cloud/videointelligence/v1:videointelligence_proto,google.golang.org/genproto/googleapis/cloud/videointelligence/v1,@go_googleapis//google/cloud/videointelligence/v1:videointelligence_go_proto +google/cloud/videointelligence/v1beta1/video_intelligence.proto,@go_googleapis//google/cloud/videointelligence/v1beta1:videointelligence_proto,google.golang.org/genproto/googleapis/cloud/videointelligence/v1beta1,@go_googleapis//google/cloud/videointelligence/v1beta1:videointelligence_go_proto +google/cloud/videointelligence/v1beta2/video_intelligence.proto,@go_googleapis//google/cloud/videointelligence/v1beta2:videointelligence_proto,google.golang.org/genproto/googleapis/cloud/videointelligence/v1beta2,@go_googleapis//google/cloud/videointelligence/v1beta2:videointelligence_go_proto +google/cloud/videointelligence/v1p1beta1/video_intelligence.proto,@go_googleapis//google/cloud/videointelligence/v1p1beta1:videointelligence_proto,google.golang.org/genproto/googleapis/cloud/videointelligence/v1p1beta1,@go_googleapis//google/cloud/videointelligence/v1p1beta1:videointelligence_go_proto +google/cloud/videointelligence/v1p2beta1/video_intelligence.proto,@go_googleapis//google/cloud/videointelligence/v1p2beta1:videointelligence_proto,google.golang.org/genproto/googleapis/cloud/videointelligence/v1p2beta1,@go_googleapis//google/cloud/videointelligence/v1p2beta1:videointelligence_go_proto +google/cloud/vision/v1/geometry.proto,@go_googleapis//google/cloud/vision/v1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1,@go_googleapis//google/cloud/vision/v1:vision_go_proto +google/cloud/vision/v1/image_annotator.proto,@go_googleapis//google/cloud/vision/v1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1,@go_googleapis//google/cloud/vision/v1:vision_go_proto +google/cloud/vision/v1/text_annotation.proto,@go_googleapis//google/cloud/vision/v1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1,@go_googleapis//google/cloud/vision/v1:vision_go_proto +google/cloud/vision/v1/web_detection.proto,@go_googleapis//google/cloud/vision/v1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1,@go_googleapis//google/cloud/vision/v1:vision_go_proto +google/cloud/vision/v1p1beta1/geometry.proto,@go_googleapis//google/cloud/vision/v1p1beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p1beta1,@go_googleapis//google/cloud/vision/v1p1beta1:vision_go_proto +google/cloud/vision/v1p1beta1/image_annotator.proto,@go_googleapis//google/cloud/vision/v1p1beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p1beta1,@go_googleapis//google/cloud/vision/v1p1beta1:vision_go_proto +google/cloud/vision/v1p1beta1/text_annotation.proto,@go_googleapis//google/cloud/vision/v1p1beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p1beta1,@go_googleapis//google/cloud/vision/v1p1beta1:vision_go_proto +google/cloud/vision/v1p1beta1/web_detection.proto,@go_googleapis//google/cloud/vision/v1p1beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p1beta1,@go_googleapis//google/cloud/vision/v1p1beta1:vision_go_proto +google/cloud/vision/v1p2beta1/geometry.proto,@go_googleapis//google/cloud/vision/v1p2beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p2beta1,@go_googleapis//google/cloud/vision/v1p2beta1:vision_go_proto +google/cloud/vision/v1p2beta1/image_annotator.proto,@go_googleapis//google/cloud/vision/v1p2beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p2beta1,@go_googleapis//google/cloud/vision/v1p2beta1:vision_go_proto +google/cloud/vision/v1p2beta1/text_annotation.proto,@go_googleapis//google/cloud/vision/v1p2beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p2beta1,@go_googleapis//google/cloud/vision/v1p2beta1:vision_go_proto +google/cloud/vision/v1p2beta1/web_detection.proto,@go_googleapis//google/cloud/vision/v1p2beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p2beta1,@go_googleapis//google/cloud/vision/v1p2beta1:vision_go_proto +google/cloud/vision/v1p3beta1/geometry.proto,@go_googleapis//google/cloud/vision/v1p3beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p3beta1,@go_googleapis//google/cloud/vision/v1p3beta1:vision_go_proto +google/cloud/vision/v1p3beta1/image_annotator.proto,@go_googleapis//google/cloud/vision/v1p3beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p3beta1,@go_googleapis//google/cloud/vision/v1p3beta1:vision_go_proto +google/cloud/vision/v1p3beta1/product_search.proto,@go_googleapis//google/cloud/vision/v1p3beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p3beta1,@go_googleapis//google/cloud/vision/v1p3beta1:vision_go_proto +google/cloud/vision/v1p3beta1/product_search_service.proto,@go_googleapis//google/cloud/vision/v1p3beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p3beta1,@go_googleapis//google/cloud/vision/v1p3beta1:vision_go_proto +google/cloud/vision/v1p3beta1/text_annotation.proto,@go_googleapis//google/cloud/vision/v1p3beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p3beta1,@go_googleapis//google/cloud/vision/v1p3beta1:vision_go_proto +google/cloud/vision/v1p3beta1/web_detection.proto,@go_googleapis//google/cloud/vision/v1p3beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p3beta1,@go_googleapis//google/cloud/vision/v1p3beta1:vision_go_proto +google/cloud/websecurityscanner/v1alpha/crawled_url.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto +google/cloud/websecurityscanner/v1alpha/finding.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto +google/cloud/websecurityscanner/v1alpha/finding_addon.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto +google/cloud/websecurityscanner/v1alpha/finding_type_stats.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto +google/cloud/websecurityscanner/v1alpha/scan_config.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto +google/cloud/websecurityscanner/v1alpha/scan_run.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto +google/cloud/websecurityscanner/v1alpha/web_security_scanner.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto +google/container/v1/cluster_service.proto,@go_googleapis//google/container/v1:container_proto,google.golang.org/genproto/googleapis/container/v1,@go_googleapis//google/container/v1:container_go_proto +google/container/v1alpha1/cluster_service.proto,@go_googleapis//google/container/v1alpha1:container_proto,google.golang.org/genproto/googleapis/container/v1alpha1,@go_googleapis//google/container/v1alpha1:container_go_proto +google/container/v1beta1/cluster_service.proto,@go_googleapis//google/container/v1beta1:container_proto,google.golang.org/genproto/googleapis/container/v1beta1,@go_googleapis//google/container/v1beta1:container_go_proto +google/datastore/admin/v1/datastore_admin.proto,@go_googleapis//google/datastore/admin/v1:admin_proto,google.golang.org/genproto/googleapis/datastore/admin/v1,@go_googleapis//google/datastore/admin/v1:admin_go_proto +google/datastore/admin/v1/index.proto,@go_googleapis//google/datastore/admin/v1:admin_proto,google.golang.org/genproto/googleapis/datastore/admin/v1,@go_googleapis//google/datastore/admin/v1:admin_go_proto +google/datastore/admin/v1beta1/datastore_admin.proto,@go_googleapis//google/datastore/admin/v1beta1:admin_proto,google.golang.org/genproto/googleapis/datastore/admin/v1beta1,@go_googleapis//google/datastore/admin/v1beta1:admin_go_proto +google/datastore/v1/datastore.proto,@go_googleapis//google/datastore/v1:datastore_proto,google.golang.org/genproto/googleapis/datastore/v1,@go_googleapis//google/datastore/v1:datastore_go_proto +google/datastore/v1/entity.proto,@go_googleapis//google/datastore/v1:datastore_proto,google.golang.org/genproto/googleapis/datastore/v1,@go_googleapis//google/datastore/v1:datastore_go_proto +google/datastore/v1/query.proto,@go_googleapis//google/datastore/v1:datastore_proto,google.golang.org/genproto/googleapis/datastore/v1,@go_googleapis//google/datastore/v1:datastore_go_proto +google/datastore/v1beta3/datastore.proto,@go_googleapis//google/datastore/v1beta3:datastore_proto,google.golang.org/genproto/googleapis/datastore/v1beta3,@go_googleapis//google/datastore/v1beta3:datastore_go_proto +google/datastore/v1beta3/entity.proto,@go_googleapis//google/datastore/v1beta3:datastore_proto,google.golang.org/genproto/googleapis/datastore/v1beta3,@go_googleapis//google/datastore/v1beta3:datastore_go_proto +google/datastore/v1beta3/query.proto,@go_googleapis//google/datastore/v1beta3:datastore_proto,google.golang.org/genproto/googleapis/datastore/v1beta3,@go_googleapis//google/datastore/v1beta3:datastore_go_proto google/devtools/build/v1/build_events.proto,@go_googleapis//google/devtools/build/v1:build_proto,google.golang.org/genproto/googleapis/devtools/build/v1,@go_googleapis//google/devtools/build/v1:build_go_proto google/devtools/build/v1/build_status.proto,@go_googleapis//google/devtools/build/v1:build_proto,google.golang.org/genproto/googleapis/devtools/build/v1,@go_googleapis//google/devtools/build/v1:build_go_proto google/devtools/build/v1/publish_build_event.proto,@go_googleapis//google/devtools/build/v1:build_proto,google.golang.org/genproto/googleapis/devtools/build/v1,@go_googleapis//google/devtools/build/v1:build_go_proto -google/devtools/clouddebugger/v2/debugger.proto,@go_googleapis//google/devtools/clouddebugger/v2:clouddebugger_proto,google.golang.org/genproto/googleapis/devtools/clouddebugger/v2,@go_googleapis//google/devtools/clouddebugger/v2:clouddebugger_go_proto -google/devtools/clouddebugger/v2/data.proto,@go_googleapis//google/devtools/clouddebugger/v2:clouddebugger_proto,google.golang.org/genproto/googleapis/devtools/clouddebugger/v2,@go_googleapis//google/devtools/clouddebugger/v2:clouddebugger_go_proto +google/devtools/cloudbuild/v1/cloudbuild.proto,@go_googleapis//google/devtools/cloudbuild/v1:cloudbuild_proto,google.golang.org/genproto/googleapis/devtools/cloudbuild/v1,@go_googleapis//google/devtools/cloudbuild/v1:cloudbuild_go_proto google/devtools/clouddebugger/v2/controller.proto,@go_googleapis//google/devtools/clouddebugger/v2:clouddebugger_proto,google.golang.org/genproto/googleapis/devtools/clouddebugger/v2,@go_googleapis//google/devtools/clouddebugger/v2:clouddebugger_go_proto -google/cloud/resourcemanager/v2/folders.proto,@go_googleapis//google/cloud/resourcemanager/v2:resourcemanager_proto,google.golang.org/genproto/googleapis/cloud/resourcemanager/v2,@go_googleapis//google/cloud/resourcemanager/v2:resourcemanager_go_proto -google/cloud/kms/v1/resources.proto,@go_googleapis//google/cloud/kms/v1:kms_proto,google.golang.org/genproto/googleapis/cloud/kms/v1,@go_googleapis//google/cloud/kms/v1:kms_go_proto -google/cloud/kms/v1/service.proto,@go_googleapis//google/cloud/kms/v1:kms_proto,google.golang.org/genproto/googleapis/cloud/kms/v1,@go_googleapis//google/cloud/kms/v1:kms_go_proto -google/cloud/runtimeconfig/v1beta1/resources.proto,@go_googleapis//google/cloud/runtimeconfig/v1beta1:runtimeconfig_proto,google.golang.org/genproto/googleapis/cloud/runtimeconfig/v1beta1,@go_googleapis//google/cloud/runtimeconfig/v1beta1:runtimeconfig_go_proto -google/cloud/runtimeconfig/v1beta1/runtimeconfig.proto,@go_googleapis//google/cloud/runtimeconfig/v1beta1:runtimeconfig_proto,google.golang.org/genproto/googleapis/cloud/runtimeconfig/v1beta1,@go_googleapis//google/cloud/runtimeconfig/v1beta1:runtimeconfig_go_proto -google/cloud/tasks/v2beta2/queue.proto,@go_googleapis//google/cloud/tasks/v2beta2:tasks_proto,google.golang.org/genproto/googleapis/cloud/tasks/v2beta2,@go_googleapis//google/cloud/tasks/v2beta2:tasks_go_proto -google/cloud/tasks/v2beta2/task.proto,@go_googleapis//google/cloud/tasks/v2beta2:tasks_proto,google.golang.org/genproto/googleapis/cloud/tasks/v2beta2,@go_googleapis//google/cloud/tasks/v2beta2:tasks_go_proto -google/cloud/tasks/v2beta2/target.proto,@go_googleapis//google/cloud/tasks/v2beta2:tasks_proto,google.golang.org/genproto/googleapis/cloud/tasks/v2beta2,@go_googleapis//google/cloud/tasks/v2beta2:tasks_go_proto -google/cloud/tasks/v2beta2/cloudtasks.proto,@go_googleapis//google/cloud/tasks/v2beta2:tasks_proto,google.golang.org/genproto/googleapis/cloud/tasks/v2beta2,@go_googleapis//google/cloud/tasks/v2beta2:tasks_go_proto -google/cloud/oslogin/v1/oslogin.proto,@go_googleapis//google/cloud/oslogin/v1:oslogin_proto,google.golang.org/genproto/googleapis/cloud/oslogin/v1,@go_googleapis//google/cloud/oslogin/v1:oslogin_go_proto -google/cloud/oslogin/v1alpha/oslogin.proto,@go_googleapis//google/cloud/oslogin/v1alpha:oslogin_proto,google.golang.org/genproto/googleapis/cloud/oslogin/v1alpha,@go_googleapis//google/cloud/oslogin/v1alpha:oslogin_go_proto -google/cloud/oslogin/common/common.proto,@go_googleapis//google/cloud/oslogin/common:common_proto,google.golang.org/genproto/googleapis/cloud/oslogin/common,@go_googleapis//google/cloud/oslogin/common:common_go_proto -google/cloud/oslogin/v1beta/oslogin.proto,@go_googleapis//google/cloud/oslogin/v1beta:oslogin_proto,google.golang.org/genproto/googleapis/cloud/oslogin/v1beta,@go_googleapis//google/cloud/oslogin/v1beta:oslogin_go_proto -google/cloud/dialogflow/v2beta1/context.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto -google/cloud/dialogflow/v2beta1/session_entity_type.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto -google/cloud/dialogflow/v2beta1/intent.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto -google/cloud/dialogflow/v2beta1/entity_type.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto -google/cloud/dialogflow/v2beta1/webhook.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto -google/cloud/dialogflow/v2beta1/session.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto -google/cloud/dialogflow/v2beta1/agent.proto,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2beta1,@go_googleapis//google/cloud/dialogflow/v2beta1:dialogflow_go_proto -google/cloud/dialogflow/v2/context.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto -google/cloud/dialogflow/v2/session_entity_type.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto -google/cloud/dialogflow/v2/intent.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto -google/cloud/dialogflow/v2/entity_type.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto -google/cloud/dialogflow/v2/webhook.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto -google/cloud/dialogflow/v2/session.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto -google/cloud/dialogflow/v2/agent.proto,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_proto,google.golang.org/genproto/googleapis/cloud/dialogflow/v2,@go_googleapis//google/cloud/dialogflow/v2:dialogflow_go_proto -google/cloud/redis/v1beta1/cloud_redis.proto,@go_googleapis//google/cloud/redis/v1beta1:redis_proto,google.golang.org/genproto/googleapis/cloud/redis/v1beta1,@go_googleapis//google/cloud/redis/v1beta1:redis_go_proto -google/cloud/location/locations.proto,@go_googleapis//google/cloud/location:location_proto,google.golang.org/genproto/googleapis/cloud/location,@go_googleapis//google/cloud/location:location_go_proto -google/cloud/websecurityscanner/v1alpha/finding.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto -google/cloud/websecurityscanner/v1alpha/finding_type_stats.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto -google/cloud/websecurityscanner/v1alpha/scan_config.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto -google/cloud/websecurityscanner/v1alpha/crawled_url.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto -google/cloud/websecurityscanner/v1alpha/scan_run.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto -google/cloud/websecurityscanner/v1alpha/web_security_scanner.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto -google/cloud/websecurityscanner/v1alpha/finding_addon.proto,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_proto,google.golang.org/genproto/googleapis/cloud/websecurityscanner/v1alpha,@go_googleapis//google/cloud/websecurityscanner/v1alpha:websecurityscanner_go_proto -google/cloud/language/v1/language_service.proto,@go_googleapis//google/cloud/language/v1:language_proto,google.golang.org/genproto/googleapis/cloud/language/v1,@go_googleapis//google/cloud/language/v1:language_go_proto -google/cloud/language/v1beta2/language_service.proto,@go_googleapis//google/cloud/language/v1beta2:language_proto,google.golang.org/genproto/googleapis/cloud/language/v1beta2,@go_googleapis//google/cloud/language/v1beta2:language_go_proto -google/cloud/language/v1beta1/language_service.proto,@go_googleapis//google/cloud/language/v1beta1:language_proto,google.golang.org/genproto/googleapis/cloud/language/v1beta1,@go_googleapis//google/cloud/language/v1beta1:language_go_proto -google/cloud/bigquery/datatransfer/v1/transfer.proto,@go_googleapis//google/cloud/bigquery/datatransfer/v1:datatransfer_proto,google.golang.org/genproto/googleapis/cloud/bigquery/datatransfer/v1,@go_googleapis//google/cloud/bigquery/datatransfer/v1:datatransfer_go_proto -google/cloud/bigquery/datatransfer/v1/datatransfer.proto,@go_googleapis//google/cloud/bigquery/datatransfer/v1:datatransfer_proto,google.golang.org/genproto/googleapis/cloud/bigquery/datatransfer/v1,@go_googleapis//google/cloud/bigquery/datatransfer/v1:datatransfer_go_proto -google/cloud/bigquery/logging/v1/audit_data.proto,@go_googleapis//google/cloud/bigquery/logging/v1:logging_proto,google.golang.org/genproto/googleapis/cloud/bigquery/logging/v1,@go_googleapis//google/cloud/bigquery/logging/v1:logging_go_proto -google/cloud/vision/v1/image_annotator.proto,@go_googleapis//google/cloud/vision/v1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1,@go_googleapis//google/cloud/vision/v1:vision_go_proto -google/cloud/vision/v1/geometry.proto,@go_googleapis//google/cloud/vision/v1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1,@go_googleapis//google/cloud/vision/v1:vision_go_proto -google/cloud/vision/v1/web_detection.proto,@go_googleapis//google/cloud/vision/v1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1,@go_googleapis//google/cloud/vision/v1:vision_go_proto -google/cloud/vision/v1/text_annotation.proto,@go_googleapis//google/cloud/vision/v1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1,@go_googleapis//google/cloud/vision/v1:vision_go_proto -google/cloud/vision/v1p2beta1/image_annotator.proto,@go_googleapis//google/cloud/vision/v1p2beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p2beta1,@go_googleapis//google/cloud/vision/v1p2beta1:vision_go_proto -google/cloud/vision/v1p2beta1/geometry.proto,@go_googleapis//google/cloud/vision/v1p2beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p2beta1,@go_googleapis//google/cloud/vision/v1p2beta1:vision_go_proto -google/cloud/vision/v1p2beta1/web_detection.proto,@go_googleapis//google/cloud/vision/v1p2beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p2beta1,@go_googleapis//google/cloud/vision/v1p2beta1:vision_go_proto -google/cloud/vision/v1p2beta1/text_annotation.proto,@go_googleapis//google/cloud/vision/v1p2beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p2beta1,@go_googleapis//google/cloud/vision/v1p2beta1:vision_go_proto -google/cloud/vision/v1p1beta1/image_annotator.proto,@go_googleapis//google/cloud/vision/v1p1beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p1beta1,@go_googleapis//google/cloud/vision/v1p1beta1:vision_go_proto -google/cloud/vision/v1p1beta1/geometry.proto,@go_googleapis//google/cloud/vision/v1p1beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p1beta1,@go_googleapis//google/cloud/vision/v1p1beta1:vision_go_proto -google/cloud/vision/v1p1beta1/web_detection.proto,@go_googleapis//google/cloud/vision/v1p1beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p1beta1,@go_googleapis//google/cloud/vision/v1p1beta1:vision_go_proto -google/cloud/vision/v1p1beta1/text_annotation.proto,@go_googleapis//google/cloud/vision/v1p1beta1:vision_proto,google.golang.org/genproto/googleapis/cloud/vision/v1p1beta1,@go_googleapis//google/cloud/vision/v1p1beta1:vision_go_proto -google/cloud/speech/v1/cloud_speech.proto,@go_googleapis//google/cloud/speech/v1:speech_proto,google.golang.org/genproto/googleapis/cloud/speech/v1,@go_googleapis//google/cloud/speech/v1:speech_go_proto -google/cloud/speech/v1beta1/cloud_speech.proto,@go_googleapis//google/cloud/speech/v1beta1:speech_proto,google.golang.org/genproto/googleapis/cloud/speech/v1beta1,@go_googleapis//google/cloud/speech/v1beta1:speech_go_proto -google/cloud/speech/v1p1beta1/cloud_speech.proto,@go_googleapis//google/cloud/speech/v1p1beta1:speech_proto,google.golang.org/genproto/googleapis/cloud/speech/v1p1beta1,@go_googleapis//google/cloud/speech/v1p1beta1:speech_go_proto -google/cloud/iot/v1/device_manager.proto,@go_googleapis//google/cloud/iot/v1:iot_proto,google.golang.org/genproto/googleapis/cloud/iot/v1,@go_googleapis//google/cloud/iot/v1:iot_go_proto -google/cloud/iot/v1/resources.proto,@go_googleapis//google/cloud/iot/v1:iot_proto,google.golang.org/genproto/googleapis/cloud/iot/v1,@go_googleapis//google/cloud/iot/v1:iot_go_proto -google/cloud/videointelligence/v1/video_intelligence.proto,@go_googleapis//google/cloud/videointelligence/v1:videointelligence_proto,google.golang.org/genproto/googleapis/cloud/videointelligence/v1,@go_googleapis//google/cloud/videointelligence/v1:videointelligence_go_proto -google/cloud/videointelligence/v1beta2/video_intelligence.proto,@go_googleapis//google/cloud/videointelligence/v1beta2:videointelligence_proto,google.golang.org/genproto/googleapis/cloud/videointelligence/v1beta2,@go_googleapis//google/cloud/videointelligence/v1beta2:videointelligence_go_proto -google/cloud/videointelligence/v1beta1/video_intelligence.proto,@go_googleapis//google/cloud/videointelligence/v1beta1:videointelligence_proto,google.golang.org/genproto/googleapis/cloud/videointelligence/v1beta1,@go_googleapis//google/cloud/videointelligence/v1beta1:videointelligence_go_proto -google/cloud/videointelligence/v1p1beta1/video_intelligence.proto,@go_googleapis//google/cloud/videointelligence/v1p1beta1:videointelligence_proto,google.golang.org/genproto/googleapis/cloud/videointelligence/v1p1beta1,@go_googleapis//google/cloud/videointelligence/v1p1beta1:videointelligence_go_proto -google/cloud/audit/audit_log.proto,@go_googleapis//google/cloud/audit:audit_proto,google.golang.org/genproto/googleapis/cloud/audit,@go_googleapis//google/cloud/audit:audit_go_proto -google/cloud/support/common.proto,@go_googleapis//google/cloud/support:common_proto,google.golang.org/genproto/googleapis/cloud/support/common,@go_googleapis//google/cloud/support:common_go_proto -google/cloud/support/v1alpha1/cloud_support.proto,@go_googleapis//google/cloud/support/v1alpha1:support_proto,google.golang.org/genproto/googleapis/cloud/support/v1alpha1,@go_googleapis//google/cloud/support/v1alpha1:support_go_proto -google/cloud/ml/v1/operation_metadata.proto,@go_googleapis//google/cloud/ml/v1:ml_proto,google.golang.org/genproto/googleapis/cloud/ml/v1,@go_googleapis//google/cloud/ml/v1:ml_go_proto -google/cloud/ml/v1/job_service.proto,@go_googleapis//google/cloud/ml/v1:ml_proto,google.golang.org/genproto/googleapis/cloud/ml/v1,@go_googleapis//google/cloud/ml/v1:ml_go_proto -google/cloud/ml/v1/prediction_service.proto,@go_googleapis//google/cloud/ml/v1:ml_proto,google.golang.org/genproto/googleapis/cloud/ml/v1,@go_googleapis//google/cloud/ml/v1:ml_go_proto -google/cloud/ml/v1/model_service.proto,@go_googleapis//google/cloud/ml/v1:ml_proto,google.golang.org/genproto/googleapis/cloud/ml/v1,@go_googleapis//google/cloud/ml/v1:ml_go_proto -google/cloud/ml/v1/project_service.proto,@go_googleapis//google/cloud/ml/v1:ml_proto,google.golang.org/genproto/googleapis/cloud/ml/v1,@go_googleapis//google/cloud/ml/v1:ml_go_proto -google/cloud/texttospeech/v1/cloud_tts.proto,@go_googleapis//google/cloud/texttospeech/v1:texttospeech_proto,google.golang.org/genproto/googleapis/cloud/texttospeech/v1,@go_googleapis//google/cloud/texttospeech/v1:texttospeech_go_proto -google/cloud/texttospeech/v1beta1/cloud_tts.proto,@go_googleapis//google/cloud/texttospeech/v1beta1:texttospeech_proto,google.golang.org/genproto/googleapis/cloud/texttospeech/v1beta1,@go_googleapis//google/cloud/texttospeech/v1beta1:texttospeech_go_proto -google/cloud/functions/v1beta2/operations.proto,@go_googleapis//google/cloud/functions/v1beta2:functions_proto,google.golang.org/genproto/googleapis/cloud/functions/v1beta2,@go_googleapis//google/cloud/functions/v1beta2:functions_go_proto -google/cloud/functions/v1beta2/functions.proto,@go_googleapis//google/cloud/functions/v1beta2:functions_proto,google.golang.org/genproto/googleapis/cloud/functions/v1beta2,@go_googleapis//google/cloud/functions/v1beta2:functions_go_proto -google/cloud/billing/v1/cloud_billing.proto,@go_googleapis//google/cloud/billing/v1:billing_proto,google.golang.org/genproto/googleapis/cloud/billing/v1,@go_googleapis//google/cloud/billing/v1:billing_go_proto -google/cloud/dataproc/v1/operations.proto,@go_googleapis//google/cloud/dataproc/v1:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1,@go_googleapis//google/cloud/dataproc/v1:dataproc_go_proto -google/cloud/dataproc/v1/clusters.proto,@go_googleapis//google/cloud/dataproc/v1:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1,@go_googleapis//google/cloud/dataproc/v1:dataproc_go_proto -google/cloud/dataproc/v1/jobs.proto,@go_googleapis//google/cloud/dataproc/v1:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1,@go_googleapis//google/cloud/dataproc/v1:dataproc_go_proto -google/cloud/dataproc/v1beta2/operations.proto,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1beta2,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_go_proto -google/cloud/dataproc/v1beta2/clusters.proto,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1beta2,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_go_proto -google/cloud/dataproc/v1beta2/jobs.proto,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1beta2,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_go_proto -google/cloud/dataproc/v1beta2/workflow_templates.proto,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1beta2,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_go_proto -google/cloud/dataproc/v1beta2/shared.proto,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_proto,google.golang.org/genproto/googleapis/cloud/dataproc/v1beta2,@go_googleapis//google/cloud/dataproc/v1beta2:dataproc_go_proto -google/api/context.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/http.proto,@go_googleapis//google/api:annotations_proto,google.golang.org/genproto/googleapis/api/annotations,@go_googleapis//google/api:annotations_go_proto -google/api/config_change.proto,@go_googleapis//google/api:configchange_proto,google.golang.org/genproto/googleapis/api/configchange,@go_googleapis//google/api:configchange_go_proto -google/api/system_parameter.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/monitoring.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/distribution.proto,@go_googleapis//google/api:distribution_proto,google.golang.org/genproto/googleapis/api/distribution,@go_googleapis//google/api:distribution_go_proto -google/api/endpoint.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/usage.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/monitored_resource.proto,@go_googleapis//google/api:monitoredres_proto,google.golang.org/genproto/googleapis/api/monitoredres,@go_googleapis//google/api:monitoredres_go_proto -google/api/annotations.proto,@go_googleapis//google/api:annotations_proto,google.golang.org/genproto/googleapis/api/annotations,@go_googleapis//google/api:annotations_go_proto -google/api/control.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/metric.proto,@go_googleapis//google/api:metric_proto,google.golang.org/genproto/googleapis/api/metric,@go_googleapis//google/api:metric_go_proto -google/api/label.proto,@go_googleapis//google/api:label_proto,google.golang.org/genproto/googleapis/api/label,@go_googleapis//google/api:label_go_proto -google/api/consumer.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/log.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/billing.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/service.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/logging.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/documentation.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/quota.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/auth.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/backend.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/source_info.proto,@go_googleapis//google/api:serviceconfig_proto,google.golang.org/genproto/googleapis/api/serviceconfig,@go_googleapis//google/api:serviceconfig_go_proto -google/api/httpbody.proto,@go_googleapis//google/api:httpbody_proto,google.golang.org/genproto/googleapis/api/httpbody,@go_googleapis//google/api:httpbody_go_proto -google/api/experimental/authorization_config.proto,@go_googleapis//google/api/experimental:api_proto,google.golang.org/genproto/googleapis/api,@go_googleapis//google/api/experimental:api_go_proto -google/api/experimental/experimental.proto,@go_googleapis//google/api/experimental:api_proto,google.golang.org/genproto/googleapis/api,@go_googleapis//google/api/experimental:api_go_proto -google/api/servicemanagement/v1/servicemanager.proto,@go_googleapis//google/api/servicemanagement/v1:servicemanagement_proto,google.golang.org/genproto/googleapis/api/servicemanagement/v1,@go_googleapis//google/api/servicemanagement/v1:servicemanagement_go_proto -google/api/servicemanagement/v1/resources.proto,@go_googleapis//google/api/servicemanagement/v1:servicemanagement_proto,google.golang.org/genproto/googleapis/api/servicemanagement/v1,@go_googleapis//google/api/servicemanagement/v1:servicemanagement_go_proto -google/api/servicecontrol/v1/quota_controller.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto -google/api/servicecontrol/v1/distribution.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto -google/api/servicecontrol/v1/check_error.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto -google/api/servicecontrol/v1/operation.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto -google/api/servicecontrol/v1/metric_value.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto -google/api/servicecontrol/v1/log_entry.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto -google/api/servicecontrol/v1/service_controller.proto,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_proto,google.golang.org/genproto/googleapis/api/servicecontrol/v1,@go_googleapis//google/api/servicecontrol/v1:servicecontrol_go_proto -google/pubsub/v1/pubsub.proto,@go_googleapis//google/pubsub/v1:pubsub_proto,google.golang.org/genproto/googleapis/pubsub/v1,@go_googleapis//google/pubsub/v1:pubsub_go_proto -google/pubsub/v1beta2/pubsub.proto,@go_googleapis//google/pubsub/v1beta2:pubsub_proto,google.golang.org/genproto/googleapis/pubsub/v1beta2,@go_googleapis//google/pubsub/v1beta2:pubsub_go_proto -google/spanner/v1/mutation.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto -google/spanner/v1/spanner.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto -google/spanner/v1/transaction.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto -google/spanner/v1/keys.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto -google/spanner/v1/type.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto -google/spanner/v1/query_plan.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto -google/spanner/v1/result_set.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto -google/spanner/admin/database/v1/spanner_database_admin.proto,@go_googleapis//google/spanner/admin/database/v1:database_proto,google.golang.org/genproto/googleapis/spanner/admin/database/v1,@go_googleapis//google/spanner/admin/database/v1:database_go_proto -google/spanner/admin/instance/v1/spanner_instance_admin.proto,@go_googleapis//google/spanner/admin/instance/v1:instance_proto,google.golang.org/genproto/googleapis/spanner/admin/instance/v1,@go_googleapis//google/spanner/admin/instance/v1:instance_go_proto +google/devtools/clouddebugger/v2/data.proto,@go_googleapis//google/devtools/clouddebugger/v2:clouddebugger_proto,google.golang.org/genproto/googleapis/devtools/clouddebugger/v2,@go_googleapis//google/devtools/clouddebugger/v2:clouddebugger_go_proto +google/devtools/clouddebugger/v2/debugger.proto,@go_googleapis//google/devtools/clouddebugger/v2:clouddebugger_proto,google.golang.org/genproto/googleapis/devtools/clouddebugger/v2,@go_googleapis//google/devtools/clouddebugger/v2:clouddebugger_go_proto +google/devtools/clouderrorreporting/v1beta1/common.proto,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_proto,google.golang.org/genproto/googleapis/devtools/clouderrorreporting/v1beta1,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_go_proto +google/devtools/clouderrorreporting/v1beta1/error_group_service.proto,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_proto,google.golang.org/genproto/googleapis/devtools/clouderrorreporting/v1beta1,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_go_proto +google/devtools/clouderrorreporting/v1beta1/error_stats_service.proto,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_proto,google.golang.org/genproto/googleapis/devtools/clouderrorreporting/v1beta1,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_go_proto +google/devtools/clouderrorreporting/v1beta1/report_errors_service.proto,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_proto,google.golang.org/genproto/googleapis/devtools/clouderrorreporting/v1beta1,@go_googleapis//google/devtools/clouderrorreporting/v1beta1:clouderrorreporting_go_proto +google/devtools/cloudprofiler/v2/profiler.proto,@go_googleapis//google/devtools/cloudprofiler/v2:cloudprofiler_proto,google.golang.org/genproto/googleapis/devtools/cloudprofiler/v2,@go_googleapis//google/devtools/cloudprofiler/v2:cloudprofiler_go_proto +google/devtools/cloudtrace/v1/trace.proto,@go_googleapis//google/devtools/cloudtrace/v1:cloudtrace_proto,google.golang.org/genproto/googleapis/devtools/cloudtrace/v1,@go_googleapis//google/devtools/cloudtrace/v1:cloudtrace_go_proto +google/devtools/cloudtrace/v2/trace.proto,@go_googleapis//google/devtools/cloudtrace/v2:cloudtrace_proto,google.golang.org/genproto/googleapis/devtools/cloudtrace/v2,@go_googleapis//google/devtools/cloudtrace/v2:cloudtrace_go_proto +google/devtools/cloudtrace/v2/tracing.proto,@go_googleapis//google/devtools/cloudtrace/v2:cloudtrace_proto,google.golang.org/genproto/googleapis/devtools/cloudtrace/v2,@go_googleapis//google/devtools/cloudtrace/v2:cloudtrace_go_proto +google/devtools/containeranalysis/v1alpha1/bill_of_materials.proto,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_go_proto +google/devtools/containeranalysis/v1alpha1/containeranalysis.proto,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_go_proto +google/devtools/containeranalysis/v1alpha1/image_basis.proto,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_go_proto +google/devtools/containeranalysis/v1alpha1/package_vulnerability.proto,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_go_proto +google/devtools/containeranalysis/v1alpha1/provenance.proto,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_go_proto +google/devtools/containeranalysis/v1alpha1/source_context.proto,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1alpha1,@go_googleapis//google/devtools/containeranalysis/v1alpha1:containeranalysis_go_proto +google/devtools/containeranalysis/v1beta1/attestation/attestation.proto,@go_googleapis//google/devtools/containeranalysis/v1beta1/attestation:attestation_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/attestation,@go_googleapis//google/devtools/containeranalysis/v1beta1/attestation:attestation_go_proto +google/devtools/containeranalysis/v1beta1/build/build.proto,@go_googleapis//google/devtools/containeranalysis/v1beta1/build:build_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/build,@go_googleapis//google/devtools/containeranalysis/v1beta1/build:build_go_proto +google/devtools/containeranalysis/v1beta1/common/common.proto,@go_googleapis//google/devtools/containeranalysis/v1beta1/common:common_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/common,@go_googleapis//google/devtools/containeranalysis/v1beta1/common:common_go_proto +google/devtools/containeranalysis/v1beta1/containeranalysis.proto,@go_googleapis//google/devtools/containeranalysis/v1beta1:containeranalysis_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1,@go_googleapis//google/devtools/containeranalysis/v1beta1:containeranalysis_go_proto +google/devtools/containeranalysis/v1beta1/deployment/deployment.proto,@go_googleapis//google/devtools/containeranalysis/v1beta1/deployment:deployment_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/deployment,@go_googleapis//google/devtools/containeranalysis/v1beta1/deployment:deployment_go_proto +google/devtools/containeranalysis/v1beta1/discovery/discovery.proto,@go_googleapis//google/devtools/containeranalysis/v1beta1/discovery:discovery_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/discovery,@go_googleapis//google/devtools/containeranalysis/v1beta1/discovery:discovery_go_proto +google/devtools/containeranalysis/v1beta1/grafeas/grafeas.proto,@go_googleapis//google/devtools/containeranalysis/v1beta1/grafeas:grafeas_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/grafeas,@go_googleapis//google/devtools/containeranalysis/v1beta1/grafeas:grafeas_go_proto +google/devtools/containeranalysis/v1beta1/image/image.proto,@go_googleapis//google/devtools/containeranalysis/v1beta1/image:image_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/image,@go_googleapis//google/devtools/containeranalysis/v1beta1/image:image_go_proto +google/devtools/containeranalysis/v1beta1/package/package.proto,@go_googleapis//google/devtools/containeranalysis/v1beta1/package:package_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/package,@go_googleapis//google/devtools/containeranalysis/v1beta1/package:package_go_proto +google/devtools/containeranalysis/v1beta1/provenance/provenance.proto,@go_googleapis//google/devtools/containeranalysis/v1beta1/provenance:provenance_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/provenance,@go_googleapis//google/devtools/containeranalysis/v1beta1/provenance:provenance_go_proto +google/devtools/containeranalysis/v1beta1/source/source.proto,@go_googleapis//google/devtools/containeranalysis/v1beta1/source:source_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/source,@go_googleapis//google/devtools/containeranalysis/v1beta1/source:source_go_proto +google/devtools/containeranalysis/v1beta1/vulnerability/vulnerability.proto,@go_googleapis//google/devtools/containeranalysis/v1beta1/vulnerability:vulnerability_proto,google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/vulnerability,@go_googleapis//google/devtools/containeranalysis/v1beta1/vulnerability:vulnerability_go_proto +google/devtools/remoteexecution/v1test/remote_execution.proto,@go_googleapis//google/devtools/remoteexecution/v1test:remoteexecution_proto,google.golang.org/genproto/googleapis/devtools/remoteexecution/v1test,@go_googleapis//google/devtools/remoteexecution/v1test:remoteexecution_go_proto +google/devtools/remoteworkers/v1test2/bots.proto,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_proto,google.golang.org/genproto/googleapis/devtools/remoteworkers/v1test2,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_go_proto +google/devtools/remoteworkers/v1test2/command.proto,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_proto,google.golang.org/genproto/googleapis/devtools/remoteworkers/v1test2,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_go_proto +google/devtools/remoteworkers/v1test2/tasks.proto,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_proto,google.golang.org/genproto/googleapis/devtools/remoteworkers/v1test2,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_go_proto +google/devtools/remoteworkers/v1test2/worker.proto,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_proto,google.golang.org/genproto/googleapis/devtools/remoteworkers/v1test2,@go_googleapis//google/devtools/remoteworkers/v1test2:remoteworkers_go_proto +google/devtools/resultstore/v2/action.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto +google/devtools/resultstore/v2/common.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto +google/devtools/resultstore/v2/configuration.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto +google/devtools/resultstore/v2/configured_target.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto +google/devtools/resultstore/v2/coverage.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto +google/devtools/resultstore/v2/coverage_summary.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto +google/devtools/resultstore/v2/file.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto +google/devtools/resultstore/v2/file_set.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto +google/devtools/resultstore/v2/invocation.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto +google/devtools/resultstore/v2/resultstore_download.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto +google/devtools/resultstore/v2/resultstore_file_download.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto +google/devtools/resultstore/v2/target.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto +google/devtools/resultstore/v2/test_suite.proto,@go_googleapis//google/devtools/resultstore/v2:resultstore_proto,google.golang.org/genproto/googleapis/devtools/resultstore/v2,@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto +google/devtools/source/v1/source_context.proto,@go_googleapis//google/devtools/source/v1:source_proto,google.golang.org/genproto/googleapis/devtools/source/v1,@go_googleapis//google/devtools/source/v1:source_go_proto +google/devtools/sourcerepo/v1/sourcerepo.proto,@go_googleapis//google/devtools/sourcerepo/v1:sourcerepo_proto,google.golang.org/genproto/googleapis/devtools/sourcerepo/v1,@go_googleapis//google/devtools/sourcerepo/v1:sourcerepo_go_proto +google/example/library/v1/library.proto,@go_googleapis//google/example/library/v1:library_proto,google.golang.org/genproto/googleapis/example/library/v1,@go_googleapis//google/example/library/v1:library_go_proto +google/firestore/admin/v1beta1/firestore_admin.proto,@go_googleapis//google/firestore/admin/v1beta1:admin_proto,google.golang.org/genproto/googleapis/firestore/admin/v1beta1,@go_googleapis//google/firestore/admin/v1beta1:admin_go_proto +google/firestore/admin/v1beta1/index.proto,@go_googleapis//google/firestore/admin/v1beta1:admin_proto,google.golang.org/genproto/googleapis/firestore/admin/v1beta1,@go_googleapis//google/firestore/admin/v1beta1:admin_go_proto +google/firestore/admin/v1beta2/field.proto,@go_googleapis//google/firestore/admin/v1beta2:admin_proto,google.golang.org/genproto/googleapis/firestore/admin/v1beta2,@go_googleapis//google/firestore/admin/v1beta2:admin_go_proto +google/firestore/admin/v1beta2/firestore_admin.proto,@go_googleapis//google/firestore/admin/v1beta2:admin_proto,google.golang.org/genproto/googleapis/firestore/admin/v1beta2,@go_googleapis//google/firestore/admin/v1beta2:admin_go_proto +google/firestore/admin/v1beta2/index.proto,@go_googleapis//google/firestore/admin/v1beta2:admin_proto,google.golang.org/genproto/googleapis/firestore/admin/v1beta2,@go_googleapis//google/firestore/admin/v1beta2:admin_go_proto +google/firestore/admin/v1beta2/operation.proto,@go_googleapis//google/firestore/admin/v1beta2:admin_proto,google.golang.org/genproto/googleapis/firestore/admin/v1beta2,@go_googleapis//google/firestore/admin/v1beta2:admin_go_proto +google/firestore/v1beta1/common.proto,@go_googleapis//google/firestore/v1beta1:firestore_proto,google.golang.org/genproto/googleapis/firestore/v1beta1,@go_googleapis//google/firestore/v1beta1:firestore_go_proto +google/firestore/v1beta1/document.proto,@go_googleapis//google/firestore/v1beta1:firestore_proto,google.golang.org/genproto/googleapis/firestore/v1beta1,@go_googleapis//google/firestore/v1beta1:firestore_go_proto +google/firestore/v1beta1/firestore.proto,@go_googleapis//google/firestore/v1beta1:firestore_proto,google.golang.org/genproto/googleapis/firestore/v1beta1,@go_googleapis//google/firestore/v1beta1:firestore_go_proto +google/firestore/v1beta1/query.proto,@go_googleapis//google/firestore/v1beta1:firestore_proto,google.golang.org/genproto/googleapis/firestore/v1beta1,@go_googleapis//google/firestore/v1beta1:firestore_go_proto +google/firestore/v1beta1/write.proto,@go_googleapis//google/firestore/v1beta1:firestore_proto,google.golang.org/genproto/googleapis/firestore/v1beta1,@go_googleapis//google/firestore/v1beta1:firestore_go_proto +google/genomics/v1/annotations.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto +google/genomics/v1/cigar.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto +google/genomics/v1/datasets.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto +google/genomics/v1/operations.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto +google/genomics/v1/position.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto +google/genomics/v1/range.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto +google/genomics/v1/readalignment.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto +google/genomics/v1/readgroup.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto +google/genomics/v1/readgroupset.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto +google/genomics/v1/reads.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto +google/genomics/v1/references.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto +google/genomics/v1/variants.proto,@go_googleapis//google/genomics/v1:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1,@go_googleapis//google/genomics/v1:genomics_go_proto +google/genomics/v1alpha2/pipelines.proto,@go_googleapis//google/genomics/v1alpha2:genomics_proto,google.golang.org/genproto/googleapis/genomics/v1alpha2,@go_googleapis//google/genomics/v1alpha2:genomics_go_proto +google/home/graph/v1/device.proto,@go_googleapis//google/home/graph/v1:graph_proto,google.golang.org/genproto/googleapis/home/graph/v1,@go_googleapis//google/home/graph/v1:graph_go_proto +google/home/graph/v1/homegraph.proto,@go_googleapis//google/home/graph/v1:graph_proto,google.golang.org/genproto/googleapis/home/graph/v1,@go_googleapis//google/home/graph/v1:graph_go_proto +google/iam/admin/v1/iam.proto,@go_googleapis//google/iam/admin/v1:admin_proto,google.golang.org/genproto/googleapis/iam/admin/v1,@go_googleapis//google/iam/admin/v1:admin_go_proto +google/iam/credentials/v1/common.proto,@go_googleapis//google/iam/credentials/v1:credentials_proto,google.golang.org/genproto/googleapis/iam/credentials/v1,@go_googleapis//google/iam/credentials/v1:credentials_go_proto +google/iam/credentials/v1/iamcredentials.proto,@go_googleapis//google/iam/credentials/v1:credentials_proto,google.golang.org/genproto/googleapis/iam/credentials/v1,@go_googleapis//google/iam/credentials/v1:credentials_go_proto +google/iam/v1/iam_policy.proto,@go_googleapis//google/iam/v1:iam_proto,google.golang.org/genproto/googleapis/iam/v1,@go_googleapis//google/iam/v1:iam_go_proto +google/iam/v1/logging/audit_data.proto,@go_googleapis//google/iam/v1/logging:logging_proto,google.golang.org/genproto/googleapis/iam/v1/logging,@go_googleapis//google/iam/v1/logging:logging_go_proto +google/iam/v1/policy.proto,@go_googleapis//google/iam/v1:iam_proto,google.golang.org/genproto/googleapis/iam/v1,@go_googleapis//google/iam/v1:iam_go_proto +google/logging/type/http_request.proto,@go_googleapis//google/logging/type:ltype_proto,google.golang.org/genproto/googleapis/logging/type,@go_googleapis//google/logging/type:ltype_go_proto +google/logging/type/log_severity.proto,@go_googleapis//google/logging/type:ltype_proto,google.golang.org/genproto/googleapis/logging/type,@go_googleapis//google/logging/type:ltype_go_proto +google/logging/v2/log_entry.proto,@go_googleapis//google/logging/v2:logging_proto,google.golang.org/genproto/googleapis/logging/v2,@go_googleapis//google/logging/v2:logging_go_proto +google/logging/v2/logging.proto,@go_googleapis//google/logging/v2:logging_proto,google.golang.org/genproto/googleapis/logging/v2,@go_googleapis//google/logging/v2:logging_go_proto +google/logging/v2/logging_config.proto,@go_googleapis//google/logging/v2:logging_proto,google.golang.org/genproto/googleapis/logging/v2,@go_googleapis//google/logging/v2:logging_go_proto +google/logging/v2/logging_metrics.proto,@go_googleapis//google/logging/v2:logging_proto,google.golang.org/genproto/googleapis/logging/v2,@go_googleapis//google/logging/v2:logging_go_proto +google/longrunning/operations.proto,@go_googleapis//google/longrunning:longrunning_proto,google.golang.org/genproto/googleapis/longrunning,@go_googleapis//google/longrunning:longrunning_go_proto +google/monitoring/v3/alert.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto +google/monitoring/v3/alert_service.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto +google/monitoring/v3/common.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto +google/monitoring/v3/dropped_labels.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto google/monitoring/v3/group.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto +google/monitoring/v3/group_service.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto +google/monitoring/v3/metric.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto +google/monitoring/v3/metric_service.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto google/monitoring/v3/mutation_record.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto google/monitoring/v3/notification.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto -google/monitoring/v3/alert_service.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto -google/monitoring/v3/uptime_service.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto -google/monitoring/v3/group_service.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto -google/monitoring/v3/alert.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto -google/monitoring/v3/uptime.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto -google/monitoring/v3/metric.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto google/monitoring/v3/notification_service.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto -google/monitoring/v3/metric_service.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto -google/monitoring/v3/common.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto +google/monitoring/v3/span_context.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto +google/monitoring/v3/uptime.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto +google/monitoring/v3/uptime_service.proto,@go_googleapis//google/monitoring/v3:monitoring_proto,google.golang.org/genproto/googleapis/monitoring/v3,@go_googleapis//google/monitoring/v3:monitoring_go_proto +google/privacy/dlp/v2/dlp.proto,@go_googleapis//google/privacy/dlp/v2:dlp_proto,google.golang.org/genproto/googleapis/privacy/dlp/v2,@go_googleapis//google/privacy/dlp/v2:dlp_go_proto +google/privacy/dlp/v2/storage.proto,@go_googleapis//google/privacy/dlp/v2:dlp_proto,google.golang.org/genproto/googleapis/privacy/dlp/v2,@go_googleapis//google/privacy/dlp/v2:dlp_go_proto +google/pubsub/v1/pubsub.proto,@go_googleapis//google/pubsub/v1:pubsub_proto,google.golang.org/genproto/googleapis/pubsub/v1,@go_googleapis//google/pubsub/v1:pubsub_go_proto +google/pubsub/v1beta2/pubsub.proto,@go_googleapis//google/pubsub/v1beta2:pubsub_proto,google.golang.org/genproto/googleapis/pubsub/v1beta2,@go_googleapis//google/pubsub/v1beta2:pubsub_go_proto google/rpc/code.proto,@go_googleapis//google/rpc:code_proto,google.golang.org/genproto/googleapis/rpc/code,@go_googleapis//google/rpc:code_go_proto -google/rpc/status.proto,@go_googleapis//google/rpc:status_proto,google.golang.org/genproto/googleapis/rpc/status,@go_googleapis//google/rpc:status_go_proto google/rpc/error_details.proto,@go_googleapis//google/rpc:errdetails_proto,google.golang.org/genproto/googleapis/rpc/errdetails,@go_googleapis//google/rpc:errdetails_go_proto +google/rpc/status.proto,@go_googleapis//google/rpc:status_proto,google.golang.org/genproto/googleapis/rpc/status,@go_googleapis//google/rpc:status_go_proto +google/spanner/admin/database/v1/spanner_database_admin.proto,@go_googleapis//google/spanner/admin/database/v1:database_proto,google.golang.org/genproto/googleapis/spanner/admin/database/v1,@go_googleapis//google/spanner/admin/database/v1:database_go_proto +google/spanner/admin/instance/v1/spanner_instance_admin.proto,@go_googleapis//google/spanner/admin/instance/v1:instance_proto,google.golang.org/genproto/googleapis/spanner/admin/instance/v1,@go_googleapis//google/spanner/admin/instance/v1:instance_go_proto +google/spanner/v1/keys.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto +google/spanner/v1/mutation.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto +google/spanner/v1/query_plan.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto +google/spanner/v1/result_set.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto +google/spanner/v1/spanner.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto +google/spanner/v1/transaction.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto +google/spanner/v1/type.proto,@go_googleapis//google/spanner/v1:spanner_proto,google.golang.org/genproto/googleapis/spanner/v1,@go_googleapis//google/spanner/v1:spanner_go_proto +google/storagetransfer/v1/transfer.proto,@go_googleapis//google/storagetransfer/v1:storagetransfer_proto,google.golang.org/genproto/googleapis/storagetransfer/v1,@go_googleapis//google/storagetransfer/v1:storagetransfer_go_proto +google/storagetransfer/v1/transfer_types.proto,@go_googleapis//google/storagetransfer/v1:storagetransfer_proto,google.golang.org/genproto/googleapis/storagetransfer/v1,@go_googleapis//google/storagetransfer/v1:storagetransfer_go_proto google/streetview/publish/v1/resources.proto,@go_googleapis//google/streetview/publish/v1:publish_proto,google.golang.org/genproto/googleapis/streetview/publish/v1,@go_googleapis//google/streetview/publish/v1:publish_go_proto google/streetview/publish/v1/rpcmessages.proto,@go_googleapis//google/streetview/publish/v1:publish_proto,google.golang.org/genproto/googleapis/streetview/publish/v1,@go_googleapis//google/streetview/publish/v1:publish_go_proto google/streetview/publish/v1/streetview_publish.proto,@go_googleapis//google/streetview/publish/v1:publish_proto,google.golang.org/genproto/googleapis/streetview/publish/v1,@go_googleapis//google/streetview/publish/v1:publish_go_proto -google/logging/v2/logging_metrics.proto,@go_googleapis//google/logging/v2:logging_proto,google.golang.org/genproto/googleapis/logging/v2,@go_googleapis//google/logging/v2:logging_go_proto -google/logging/v2/logging_config.proto,@go_googleapis//google/logging/v2:logging_proto,google.golang.org/genproto/googleapis/logging/v2,@go_googleapis//google/logging/v2:logging_go_proto -google/logging/v2/log_entry.proto,@go_googleapis//google/logging/v2:logging_proto,google.golang.org/genproto/googleapis/logging/v2,@go_googleapis//google/logging/v2:logging_go_proto -google/logging/v2/logging.proto,@go_googleapis//google/logging/v2:logging_proto,google.golang.org/genproto/googleapis/logging/v2,@go_googleapis//google/logging/v2:logging_go_proto -google/logging/type/log_severity.proto,@go_googleapis//google/logging/type:ltype_proto,google.golang.org/genproto/googleapis/logging/type,@go_googleapis//google/logging/type:ltype_go_proto -google/logging/type/http_request.proto,@go_googleapis//google/logging/type:ltype_proto,google.golang.org/genproto/googleapis/logging/type,@go_googleapis//google/logging/type:ltype_go_proto +google/type/color.proto,@go_googleapis//google/type:color_proto,google.golang.org/genproto/googleapis/type/color,@go_googleapis//google/type:color_go_proto +google/type/date.proto,@go_googleapis//google/type:date_proto,google.golang.org/genproto/googleapis/type/date,@go_googleapis//google/type:date_go_proto +google/type/dayofweek.proto,@go_googleapis//google/type:dayofweek_proto,google.golang.org/genproto/googleapis/type/dayofweek,@go_googleapis//google/type:dayofweek_go_proto +google/type/latlng.proto,@go_googleapis//google/type:latlng_proto,google.golang.org/genproto/googleapis/type/latlng,@go_googleapis//google/type:latlng_go_proto +google/type/money.proto,@go_googleapis//google/type:money_proto,google.golang.org/genproto/googleapis/type/money,@go_googleapis//google/type:money_go_proto +google/type/postal_address.proto,@go_googleapis//google/type:postaladdress_proto,google.golang.org/genproto/googleapis/type/postaladdress,@go_googleapis//google/type:postaladdress_go_proto +google/type/timeofday.proto,@go_googleapis//google/type:timeofday_proto,google.golang.org/genproto/googleapis/type/timeofday,@go_googleapis//google/type:timeofday_go_proto +google/watcher/v1/watch.proto,@go_googleapis//google/watcher/v1:watcher_proto,google.golang.org/genproto/googleapis/watcher/v1,@go_googleapis//google/watcher/v1:watcher_go_proto diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/proto/resolve.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/proto/resolve.go index 0d25ad3e1e2..b153f8c026e 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/proto/resolve.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/language/proto/resolve.go @@ -20,6 +20,7 @@ import ( "fmt" "log" "path" + "sort" "strings" "github.com/bazelbuild/bazel-gazelle/internal/config" @@ -44,7 +45,6 @@ func (_ *protoLang) Embeds(r *rule.Rule, from label.Label) []label.Label { } func (_ *protoLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repos.RemoteCache, r *rule.Rule, from label.Label) { - pc := GetProtoConfig(c) importsRaw := r.PrivateAttr(config.GazelleImportsKey) if importsRaw == nil { // may not be set in tests. @@ -52,19 +52,24 @@ func (_ *protoLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repos.R } imports := importsRaw.([]string) r.DelAttr("deps") - deps := make([]string, 0, len(imports)) + depSet := make(map[string]bool) for _, imp := range imports { - l, err := resolveProto(pc, ix, r, imp, from) + l, err := resolveProto(c, ix, r, imp, from) if err == skipImportError { continue } else if err != nil { log.Print(err) } else { l = l.Rel(from.Repo, from.Pkg) - deps = append(deps, l.String()) + depSet[l.String()] = true } } - if len(deps) > 0 { + if len(depSet) > 0 { + deps := make([]string, 0, len(depSet)) + for dep := range depSet { + deps = append(deps, dep) + } + sort.Strings(deps) r.SetAttr("deps", deps) } } @@ -74,11 +79,16 @@ var ( notFoundError = errors.New("not found") ) -func resolveProto(pc *ProtoConfig, ix *resolve.RuleIndex, r *rule.Rule, imp string, from label.Label) (label.Label, error) { +func resolveProto(c *config.Config, ix *resolve.RuleIndex, r *rule.Rule, imp string, from label.Label) (label.Label, error) { + pc := GetProtoConfig(c) if !strings.HasSuffix(imp, ".proto") { return label.NoLabel, fmt.Errorf("can't import non-proto: %q", imp) } + if l, ok := resolve.FindRuleWithOverride(c, resolve.ImportSpec{Imp: imp, Lang: "proto"}, "proto"); ok { + return l, nil + } + if l, ok := knownImports[imp]; ok && pc.Mode.ShouldUseKnownImports() { if l.Equal(from) { return label.NoLabel, skipImportError diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/repos/BUILD b/vendor/github.com/bazelbuild/bazel-gazelle/internal/repos/BUILD index aa8395f08fa..d4c641df89d 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/internal/repos/BUILD +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/repos/BUILD @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "dep.go", + "modules.go", "remote.go", "repo.go", ], diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/repos/modules.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/repos/modules.go new file mode 100644 index 00000000000..f70c7c90ee2 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/repos/modules.go @@ -0,0 +1,145 @@ +/* Copyright 2018 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repos + +import ( + "bytes" + "encoding/json" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + + "github.com/bazelbuild/bazel-gazelle/internal/label" +) + +type module struct { + Path, Version string + Main bool +} + +var regexMixedVersioning = regexp.MustCompile(`^(.*?)-([0-9]{14})-([a-fA-F0-9]{12})$`) + +func toRepoRule(mod module) Repo { + var tag, commit string + + if gr := regexMixedVersioning.FindStringSubmatch(mod.Version); gr != nil { + commit = gr[3] + } else { + tag = strings.TrimSuffix(mod.Version, "+incompatible") + } + + return Repo{ + Name: label.ImportPathToBazelRepoName(mod.Path), + GoPrefix: mod.Path, + Commit: commit, + Tag: tag, + } +} + +func importRepoRulesModules(filename string) (repos []Repo, err error) { + tempDir, err := copyGoModToTemp(filename) + if err != nil { + return nil, err + } + defer os.RemoveAll(tempDir) + + data, err := goListModulesFn(tempDir) + if err != nil { + return nil, err + } + + dec := json.NewDecoder(bytes.NewReader(data)) + for dec.More() { + var mod module + if err := dec.Decode(&mod); err != nil { + return nil, err + } + if mod.Main { + continue + } + + repos = append(repos, toRepoRule(mod)) + } + + return repos, nil +} + +// goListModulesFn may be overridden by tests. +var goListModulesFn = goListModules + +// goListModules invokes "go list" in a directory containing a go.mod file. +func goListModules(dir string) ([]byte, error) { + goTool := findGoTool() + cmd := exec.Command(goTool, "list", "-m", "-json", "all") + cmd.Stderr = os.Stderr + cmd.Dir = dir + data, err := cmd.Output() + return data, err +} + +// copyGoModToTemp copies to given go.mod file to a temporary directory. +// go list tends to mutate go.mod files, but gazelle shouldn't do that. +func copyGoModToTemp(filename string) (tempDir string, err error) { + goModOrig, err := os.Open(filename) + if err != nil { + return "", err + } + defer goModOrig.Close() + + tempDir, err = ioutil.TempDir("", "gazelle-temp-gomod") + if err != nil { + return "", err + } + + goModCopy, err := os.Create(filepath.Join(tempDir, "go.mod")) + if err != nil { + os.Remove(tempDir) + return "", err + } + defer func() { + if cerr := goModCopy.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + _, err = io.Copy(goModCopy, goModOrig) + if err != nil { + os.RemoveAll(tempDir) + return "", err + } + return tempDir, err +} + +// findGoTool attempts to locate the go executable. If GOROOT is set, we'll +// prefer the one in there; otherwise, we'll rely on PATH. If the wrapper +// script generated by the gazelle rule is invoked by Bazel, it will set +// GOROOT to the configured SDK. We don't want to rely on the host SDK in +// that situation. +func findGoTool() string { + path := "go" // rely on PATH by default + if goroot, ok := os.LookupEnv("GOROOT"); ok { + path = filepath.Join(goroot, "bin", "go") + } + if runtime.GOOS == "windows" { + path += ".exe" + } + return path +} diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/repos/repo.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/repos/repo.go index ba5e91a1a67..262595e19d7 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/internal/repos/repo.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/repos/repo.go @@ -61,10 +61,12 @@ type lockFileFormat int const ( unknownFormat lockFileFormat = iota depFormat + moduleFormat ) var lockFileParsers = map[lockFileFormat]func(string) ([]Repo, error){ - depFormat: importRepoRulesDep, + depFormat: importRepoRulesDep, + moduleFormat: importRepoRulesModules, } // ImportRepoRules reads the lock file of a vendoring tool and returns @@ -94,6 +96,8 @@ func getLockFileFormat(filename string) lockFileFormat { switch filepath.Base(filename) { case "Gopkg.lock": return depFormat + case "go.mod": + return moduleFormat default: return unknownFormat } @@ -103,7 +107,12 @@ func getLockFileFormat(filename string) lockFileFormat { // be written in a WORKSPACE file. func GenerateRule(repo Repo) *rule.Rule { r := rule.NewRule("go_repository", repo.Name) - r.SetAttr("commit", repo.Commit) + if repo.Commit != "" { + r.SetAttr("commit", repo.Commit) + } + if repo.Tag != "" { + r.SetAttr("tag", repo.Tag) + } r.SetAttr("importpath", repo.GoPrefix) if repo.Remote != "" { r.SetAttr("remote", repo.Remote) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/resolve/BUILD b/vendor/github.com/bazelbuild/bazel-gazelle/internal/resolve/BUILD index 30ddf5b40b3..7aae6f7a1e0 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/internal/resolve/BUILD +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/resolve/BUILD @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["index.go"], + srcs = [ + "config.go", + "index.go", + ], importmap = "k8s.io/kubernetes/vendor/github.com/bazelbuild/bazel-gazelle/internal/resolve", importpath = "github.com/bazelbuild/bazel-gazelle/internal/resolve", visibility = ["//vendor/github.com/bazelbuild/bazel-gazelle:__subpackages__"], diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/resolve/config.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/resolve/config.go new file mode 100644 index 00000000000..339f4c9e529 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/resolve/config.go @@ -0,0 +1,115 @@ +/* Copyright 2018 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resolve + +import ( + "flag" + "log" + "strings" + + "github.com/bazelbuild/bazel-gazelle/internal/config" + "github.com/bazelbuild/bazel-gazelle/internal/label" + "github.com/bazelbuild/bazel-gazelle/internal/rule" +) + +// FindRuleWithOverride searches the current configuration for user-specified +// dependency resolution overrides. Overrides specified later (in configuration +// files in deeper directories, or closer to the end of the file) are +// returned first. If no override is found, label.NoLabel is returned. +func FindRuleWithOverride(c *config.Config, imp ImportSpec, lang string) (label.Label, bool) { + rc := getResolveConfig(c) + for i := len(rc.overrides) - 1; i >= 0; i-- { + o := rc.overrides[i] + if o.matches(imp, lang) { + return o.dep, true + } + } + return label.NoLabel, false +} + +type overrideSpec struct { + imp ImportSpec + lang string + dep label.Label +} + +func (o overrideSpec) matches(imp ImportSpec, lang string) bool { + return imp.Lang == o.imp.Lang && + imp.Imp == o.imp.Imp && + (o.lang == "" || o.lang == lang) +} + +type resolveConfig struct { + overrides []overrideSpec +} + +const resolveName = "_resolve" + +func getResolveConfig(c *config.Config) *resolveConfig { + return c.Exts[resolveName].(*resolveConfig) +} + +type Configurer struct{} + +func (_ *Configurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { + c.Exts[resolveName] = &resolveConfig{} +} + +func (_ *Configurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error { return nil } + +func (_ *Configurer) KnownDirectives() []string { + return []string{"resolve"} +} + +func (_ *Configurer) Configure(c *config.Config, rel string, f *rule.File) { + rc := getResolveConfig(c) + rcCopy := &resolveConfig{ + overrides: rc.overrides[:], + } + + if f != nil { + for _, d := range f.Directives { + if d.Key == "resolve" { + parts := strings.Fields(d.Value) + o := overrideSpec{} + var lbl string + if len(parts) == 3 { + o.imp.Lang = parts[0] + o.imp.Imp = parts[1] + lbl = parts[2] + } else if len(parts) == 4 { + o.imp.Lang = parts[0] + o.lang = parts[1] + o.imp.Imp = parts[2] + lbl = parts[3] + } else { + log.Printf("could not parse directive: %s\n\texpected gazelle:resolve source-language [import-language] import-string label", d.Value) + continue + } + var err error + o.dep, err = label.Parse(lbl) + if err != nil { + log.Printf("gazelle:resolve %s: %v", d.Value, err) + continue + } + o.dep = o.dep.Abs("", rel) + rcCopy.overrides = append(rcCopy.overrides, o) + } + } + } + + c.Exts[resolveName] = rcCopy +} diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/resolve/index.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/resolve/index.go index 211eba6b0c4..dba4eeac9c8 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/internal/resolve/index.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/resolve/index.go @@ -195,8 +195,6 @@ type FindResult struct { // a matched rule. Label label.Label - Rule *rule.Rule - // Embeds is the transitive closure of labels for rules that the matched // rule embeds. It may contains duplicates and does not include the label // for the rule itself. @@ -222,7 +220,6 @@ func (ix *RuleIndex) FindRulesByImport(imp ImportSpec, lang string) []FindResult } results = append(results, FindResult{ Label: m.label, - Rule: m.rule, Embeds: m.embeds, }) } diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/testtools/BUILD b/vendor/github.com/bazelbuild/bazel-gazelle/internal/testtools/BUILD new file mode 100644 index 00000000000..5848a6a99b5 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/testtools/BUILD @@ -0,0 +1,30 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "config.go", + "files.go", + ], + importmap = "k8s.io/kubernetes/vendor/github.com/bazelbuild/bazel-gazelle/internal/testtools", + importpath = "github.com/bazelbuild/bazel-gazelle/internal/testtools", + visibility = ["//vendor/github.com/bazelbuild/bazel-gazelle:__subpackages__"], + deps = [ + "//vendor/github.com/bazelbuild/bazel-gazelle/internal/config:go_default_library", + "//vendor/github.com/bazelbuild/bazel-gazelle/internal/language:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/testtools/config.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/testtools/config.go new file mode 100644 index 00000000000..15be3bdc52c --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/testtools/config.go @@ -0,0 +1,53 @@ +/* Copyright 2018 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testtools + +import ( + "flag" + "testing" + + "github.com/bazelbuild/bazel-gazelle/internal/config" + "github.com/bazelbuild/bazel-gazelle/internal/language" +) + +// NewTestConfig returns a Config used for tests in any language extension. +// cexts is a list of configuration extensions to use. langs is a list of +// language extensions to use (languages are also configuration extensions, +// but it may be convenient to keep them separate). args is a list of +// command line arguments to apply. NewTestConfig calls t.Fatal if any +// error is encountered while processing flags. +func NewTestConfig(t *testing.T, cexts []config.Configurer, langs []language.Language, args []string) *config.Config { + c := config.New() + fs := flag.NewFlagSet("test", flag.ContinueOnError) + + for _, lang := range langs { + cexts = append(cexts, lang) + } + for _, cext := range cexts { + cext.RegisterFlags(fs, "update", c) + } + + if err := fs.Parse(args); err != nil { + t.Fatal(err) + } + for _, cext := range cexts { + if err := cext.CheckFlags(fs, c); err != nil { + t.Fatal(err) + } + } + + return c +} diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/testtools/files.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/testtools/files.go new file mode 100644 index 00000000000..ca6fcda428c --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/testtools/files.go @@ -0,0 +1,109 @@ +/* Copyright 2018 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testtools + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" +) + +// FileSpec specifies the content of a test file. +type FileSpec struct { + // Path is a slash-separated path relative to the test directory. If Path + // ends with a slash, it indicates a directory should be created + // instead of a file. + Path string + + // Symlink is a slash-separated path relative to the test directory. If set, + // it indicates a symbolic link should be created with this path instead of a + // file. + Symlink string + + // Content is the content of the test file. + Content string +} + +// CreateFiles creates a directory of test files. This is a more compact +// alternative to testdata directories. CreateFiles returns a canonical path +// to the directory and a function to call to clean up the directory +// after the test. +func CreateFiles(t *testing.T, files []FileSpec) (dir string, cleanup func()) { + dir, err := ioutil.TempDir(os.Getenv("TEST_TEMPDIR"), "gazelle_test") + if err != nil { + t.Fatal(err) + } + dir, err = filepath.EvalSymlinks(dir) + if err != nil { + t.Fatal(err) + } + + for _, f := range files { + path := filepath.Join(dir, filepath.FromSlash(f.Path)) + if strings.HasSuffix(f.Path, "/") { + if err := os.MkdirAll(path, 0700); err != nil { + os.RemoveAll(dir) + t.Fatal(err) + } + continue + } + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + os.RemoveAll(dir) + t.Fatal(err) + } + if f.Symlink != "" { + if err := os.Symlink(f.Symlink, path); err != nil { + t.Fatal(err) + } + continue + } + if err := ioutil.WriteFile(path, []byte(f.Content), 0600); err != nil { + os.RemoveAll(dir) + t.Fatal(err) + } + } + + return dir, func() { os.RemoveAll(dir) } +} + +// CheckFiles checks that files in "dir" exist and have the content specified +// in "files". Files not listed in "files" are not tested, so extra files +// are allowed. +func CheckFiles(t *testing.T, dir string, files []FileSpec) { + for _, f := range files { + path := filepath.Join(dir, f.Path) + if strings.HasSuffix(f.Path, "/") { + if st, err := os.Stat(path); err != nil { + t.Errorf("could not stat %s: %v", f.Path, err) + } else if !st.IsDir() { + t.Errorf("not a directory: %s", f.Path) + } + } else { + want := strings.TrimSpace(f.Content) + gotBytes, err := ioutil.ReadFile(filepath.Join(dir, f.Path)) + if err != nil { + t.Errorf("could not read %s: %v", f.Path, err) + continue + } + got := strings.TrimSpace(string(gotBytes)) + if got != want { + t.Errorf("%s: got:\n%s\nwant:\n %s", f.Path, gotBytes, f.Content) + } + } + } +} diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/walk/config.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/walk/config.go index fd38828e19d..f5da70bbb9f 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/internal/walk/config.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/walk/config.go @@ -18,7 +18,6 @@ package walk import ( "flag" "path" - "strings" "github.com/bazelbuild/bazel-gazelle/internal/config" "github.com/bazelbuild/bazel-gazelle/internal/rule" @@ -27,60 +26,55 @@ import ( type walkConfig struct { excludes []string ignore bool + follow []string } const walkName = "_walk" -func getWalkConfig(c *config.Config) walkConfig { - return c.Exts[walkName].(walkConfig) +func getWalkConfig(c *config.Config) *walkConfig { + return c.Exts[walkName].(*walkConfig) } -func (wc *walkConfig) isExcluded(base string) bool { +func (wc *walkConfig) isExcluded(rel, base string) bool { + f := path.Join(rel, base) for _, x := range wc.excludes { - if base == x { + if f == x { return true } } return false } -type walkConfigurer struct{} +type Configurer struct{} -func (_ *walkConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) {} - -func (_ *walkConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error { return nil } - -func (_ *walkConfigurer) KnownDirectives() []string { - return []string{"exclude", "ignore"} +func (_ *Configurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { + c.Exts[walkName] = &walkConfig{} } -func (_ *walkConfigurer) Configure(c *config.Config, rel string, f *rule.File) { - var wc walkConfig - if raw, ok := c.Exts[walkName]; ok { - wc = raw.(walkConfig) - wc.ignore = false - if rel != "" { - prefix := path.Base(rel) + "/" - excludes := make([]string, 0, len(wc.excludes)) - for _, x := range wc.excludes { - if strings.HasPrefix(x, prefix) { - excludes = append(excludes, x[len(prefix):]) - } - } - wc.excludes = excludes - } - } +func (_ *Configurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error { return nil } + +func (_ *Configurer) KnownDirectives() []string { + return []string{"exclude", "follow", "ignore"} +} + +func (_ *Configurer) Configure(c *config.Config, rel string, f *rule.File) { + wc := getWalkConfig(c) + wcCopy := &walkConfig{} + *wcCopy = *wc + wcCopy.ignore = false if f != nil { for _, d := range f.Directives { switch d.Key { case "exclude": - wc.excludes = append(wc.excludes, d.Value) + wcCopy.excludes = append(wcCopy.excludes, path.Join(rel, d.Value)) + case "follow": + wcCopy.follow = append(wcCopy.follow, path.Join(rel, d.Value)) case "ignore": - wc.ignore = true + wcCopy.ignore = true } } } - c.Exts[walkName] = wc + c.Exts[walkName] = wcCopy } diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/walk/walk.go b/vendor/github.com/bazelbuild/bazel-gazelle/internal/walk/walk.go index 896a7935d54..d93bc1e289f 100644 --- a/vendor/github.com/bazelbuild/bazel-gazelle/internal/walk/walk.go +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/walk/walk.go @@ -28,6 +28,26 @@ import ( "github.com/bazelbuild/bazel-gazelle/internal/rule" ) +// Mode determines which directories Walk visits and which directories +// should be updated. +type Mode int + +const ( + // In VisitAllUpdateSubdirsMode, Walk visits every directory in the + // repository. The directories given to Walk and their subdirectories are + // updated. + VisitAllUpdateSubdirsMode Mode = iota + + // In VisitAllUpdateDirsMode, Walk visits every directory in the repository. + // Only the directories given to Walk are updated (not their subdirectories). + VisitAllUpdateDirsMode + + // In UpdateDirsMode, Walk only visits and updates directories given to Walk. + // Build files in parent directories are read in order to produce a complete + // configuration, but the callback is not called for parent directories. + UpdateDirsMode +) + // WalkFunc is a callback called by Walk in each visited directory. // // dir is the absolute file system path to the directory being visited. @@ -57,11 +77,10 @@ type WalkFunc func(dir, rel string, c *config.Config, update bool, f *rule.File, // // Walk calls the Configure method on each configuration extension in cexts // in each directory in pre-order, whether a build file is present in the -// directory or not. +// directory or not. cexts must contain a walk.Configurer. // // Walk calls the callback wf in post-order. -func Walk(c *config.Config, cexts []config.Configurer, wf WalkFunc) { - cexts = append(cexts, &walkConfigurer{}) +func Walk(c *config.Config, cexts []config.Configurer, dirs []string, mode Mode, wf WalkFunc) { knownDirectives := make(map[string]bool) for _, cext := range cexts { for _, d := range cext.KnownDirectives() { @@ -69,17 +88,14 @@ func Walk(c *config.Config, cexts []config.Configurer, wf WalkFunc) { } } - updateRels := buildUpdateRels(c.RepoRoot, c.Dirs) - symlinks := symlinkResolver{root: c.RepoRoot, visited: []string{c.RepoRoot}} + symlinks := symlinkResolver{visited: []string{c.RepoRoot}} + + updateRels := buildUpdateRelMap(c.RepoRoot, dirs) var visit func(*config.Config, string, string, bool) - visit = func(c *config.Config, dir, rel string, isUpdateDir bool) { + visit = func(c *config.Config, dir, rel string, updateParent bool) { haveError := false - if !isUpdateDir { - isUpdateDir = shouldUpdateDir(rel, updateRels) - } - // TODO: OPT: ReadDir stats all the files, which is slow. We just care about // names and modes, so we should use something like // golang.org/x/tools/internal/fastwalk to speed this up. @@ -102,10 +118,10 @@ func Walk(c *config.Config, cexts []config.Configurer, wf WalkFunc) { for _, fi := range files { base := fi.Name() switch { - case base == "" || base[0] == '.' || base[0] == '_' || wc.isExcluded(base): + case base == "" || base[0] == '.' || base[0] == '_' || wc.isExcluded(rel, base): continue - case fi.IsDir() || fi.Mode()&os.ModeSymlink != 0 && symlinks.follow(dir, base): + case fi.IsDir() || fi.Mode()&os.ModeSymlink != 0 && symlinks.follow(c, dir, rel, base): subdirs = append(subdirs, base) default: @@ -113,45 +129,75 @@ func Walk(c *config.Config, cexts []config.Configurer, wf WalkFunc) { } } + shouldUpdate := shouldUpdate(rel, mode, updateParent, updateRels) for _, sub := range subdirs { - visit(c, filepath.Join(dir, sub), path.Join(rel, sub), isUpdateDir) + if subRel := path.Join(rel, sub); shouldVisit(subRel, mode, updateRels) { + visit(c, filepath.Join(dir, sub), subRel, shouldUpdate) + } } - genFiles := findGenFiles(wc, f) - update := !haveError && isUpdateDir && !wc.ignore - wf(dir, rel, c, update, f, subdirs, regularFiles, genFiles) + update := !haveError && !wc.ignore && shouldUpdate + if shouldCall(rel, mode, updateRels) { + genFiles := findGenFiles(wc, f) + wf(dir, rel, c, update, f, subdirs, regularFiles, genFiles) + } } visit(c, c.RepoRoot, "", false) } -// buildUpdateRels builds a list of relative paths from the repository root -// directory (passed as an absolute file path) to directories that Gazelle -// may update. The relative paths are slash-separated. "" represents the -// root directory itself. -func buildUpdateRels(root string, dirs []string) []string { - var updateRels []string +// buildUpdateRelMap builds a table of prefixes, used to determine which +// directories to update and visit. +// +// root and dirs must be absolute, canonical file paths. Each entry in dirs +// must be a subdirectory of root. The caller is responsible for checking this. +// +// buildUpdateRelMap returns a map from slash-separated paths relative to the +// root directory ("" for the root itself) to a boolean indicating whether +// the directory should be updated. +func buildUpdateRelMap(root string, dirs []string) map[string]bool { + relMap := make(map[string]bool) for _, dir := range dirs { - rel, err := filepath.Rel(root, dir) - if err != nil { - // This should have been verified when c was built. - log.Panicf("%s: not a subdirectory of repository root %q", dir, root) - } + rel, _ := filepath.Rel(root, dir) rel = filepath.ToSlash(rel) - if rel == "." || rel == "/" { + if rel == "." { rel = "" } - updateRels = append(updateRels, rel) - } - return updateRels -} -func shouldUpdateDir(rel string, updateRels []string) bool { - for _, r := range updateRels { - if pathtools.HasPrefix(rel, r) { - return true + i := 0 + for { + next := strings.IndexByte(rel[i:], '/') + i + if next-i < 0 { + relMap[rel] = true + break + } + prefix := rel[:next] + relMap[prefix] = relMap[prefix] // set to false if not present + i = next + 1 } } - return false + return relMap +} + +// shouldCall returns true if Walk should call the callback in the +// directory rel. +func shouldCall(rel string, mode Mode, updateRels map[string]bool) bool { + return mode != UpdateDirsMode || updateRels[rel] +} + +// shouldUpdate returns true if Walk should pass true to the callback's update +// parameter in the directory rel. This indicates the build file should be +// updated. +func shouldUpdate(rel string, mode Mode, updateParent bool, updateRels map[string]bool) bool { + return mode == VisitAllUpdateSubdirsMode && updateParent || updateRels[rel] +} + +// shouldVisit returns true if Walk should visit the subdirectory rel. +func shouldVisit(rel string, mode Mode, updateRels map[string]bool) bool { + if mode != UpdateDirsMode { + return true + } + _, ok := updateRels[rel] + return ok } func loadBuildFile(c *config.Config, pkg, dir string, files []os.FileInfo) (*rule.File, error) { @@ -189,7 +235,7 @@ func configure(cexts []config.Configurer, knownDirectives map[string]bool, c *co return c } -func findGenFiles(wc walkConfig, f *rule.File) []string { +func findGenFiles(wc *walkConfig, f *rule.File) []string { if f == nil { return nil } @@ -206,7 +252,7 @@ func findGenFiles(wc walkConfig, f *rule.File) []string { var genFiles []string for _, s := range strs { - if !wc.isExcluded(s) { + if !wc.isExcluded(f.Pkg, s) { genFiles = append(genFiles, s) } } @@ -214,17 +260,26 @@ func findGenFiles(wc walkConfig, f *rule.File) []string { } type symlinkResolver struct { - root string visited []string } // Decide if symlink dir/base should be followed. -func (r *symlinkResolver) follow(dir, base string) bool { - if dir == r.root && strings.HasPrefix(base, "bazel-") { +func (r *symlinkResolver) follow(c *config.Config, dir, rel, base string) bool { + if dir == c.RepoRoot && strings.HasPrefix(base, "bazel-") { // Links such as bazel-, bazel-out, bazel-genfiles are created by // Bazel to point to internal build directories. return false } + + // See if the user has explicitly directed us to follow the link. + wc := getWalkConfig(c) + linkRel := path.Join(rel, base) + for _, follow := range wc.follow { + if linkRel == follow { + return true + } + } + // See if the symlink points to a tree that has been already visited. fullpath := filepath.Join(dir, base) dest, err := filepath.EvalSymlinks(fullpath) diff --git a/vendor/github.com/container-storage-interface/spec/lib/go/csi/v0/BUILD b/vendor/github.com/container-storage-interface/spec/lib/go/csi/BUILD similarity index 79% rename from vendor/github.com/container-storage-interface/spec/lib/go/csi/v0/BUILD rename to vendor/github.com/container-storage-interface/spec/lib/go/csi/BUILD index ca66165f6d0..88af8cdba59 100644 --- a/vendor/github.com/container-storage-interface/spec/lib/go/csi/v0/BUILD +++ b/vendor/github.com/container-storage-interface/spec/lib/go/csi/BUILD @@ -3,11 +3,13 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = ["csi.pb.go"], - importmap = "k8s.io/kubernetes/vendor/github.com/container-storage-interface/spec/lib/go/csi/v0", - importpath = "github.com/container-storage-interface/spec/lib/go/csi/v0", + importmap = "k8s.io/kubernetes/vendor/github.com/container-storage-interface/spec/lib/go/csi", + importpath = "github.com/container-storage-interface/spec/lib/go/csi", visibility = ["//visibility:public"], deps = [ "//vendor/github.com/golang/protobuf/proto:go_default_library", + "//vendor/github.com/golang/protobuf/protoc-gen-go/descriptor:go_default_library", + "//vendor/github.com/golang/protobuf/ptypes/timestamp:go_default_library", "//vendor/github.com/golang/protobuf/ptypes/wrappers:go_default_library", "//vendor/golang.org/x/net/context:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", diff --git a/vendor/github.com/container-storage-interface/spec/lib/go/csi/v0/csi.pb.go b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go similarity index 69% rename from vendor/github.com/container-storage-interface/spec/lib/go/csi/v0/csi.pb.go rename to vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go index 612cfdd71dc..0652b822f77 100644 --- a/vendor/github.com/container-storage-interface/spec/lib/go/csi/v0/csi.pb.go +++ b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go @@ -1,11 +1,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: csi.proto +// source: github.com/container-storage-interface/spec/csi.proto package csi import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" +import timestamp "github.com/golang/protobuf/ptypes/timestamp" import wrappers "github.com/golang/protobuf/ptypes/wrappers" import ( @@ -30,38 +32,38 @@ const ( PluginCapability_Service_UNKNOWN PluginCapability_Service_Type = 0 // CONTROLLER_SERVICE indicates that the Plugin provides RPCs for // the ControllerService. Plugins SHOULD provide this capability. - // In rare cases certain plugins may wish to omit the + // In rare cases certain plugins MAY wish to omit the // ControllerService entirely from their implementation, but such // SHOULD NOT be the common case. // The presence of this capability determines whether the CO will // attempt to invoke the REQUIRED ControllerService RPCs, as well // as specific RPCs as indicated by ControllerGetCapabilities. PluginCapability_Service_CONTROLLER_SERVICE PluginCapability_Service_Type = 1 - // ACCESSIBILITY_CONSTRAINTS indicates that the volumes for this - // plugin may not be equally accessible by all nodes in the + // VOLUME_ACCESSIBILITY_CONSTRAINTS indicates that the volumes for + // this plugin MAY NOT be equally accessible by all nodes in the // cluster. The CO MUST use the topology information returned by // CreateVolumeRequest along with the topology information // returned by NodeGetInfo to ensure that a given volume is // accessible from a given node when scheduling workloads. - PluginCapability_Service_ACCESSIBILITY_CONSTRAINTS PluginCapability_Service_Type = 2 + PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS PluginCapability_Service_Type = 2 ) var PluginCapability_Service_Type_name = map[int32]string{ 0: "UNKNOWN", 1: "CONTROLLER_SERVICE", - 2: "ACCESSIBILITY_CONSTRAINTS", + 2: "VOLUME_ACCESSIBILITY_CONSTRAINTS", } var PluginCapability_Service_Type_value = map[string]int32{ - "UNKNOWN": 0, - "CONTROLLER_SERVICE": 1, - "ACCESSIBILITY_CONSTRAINTS": 2, + "UNKNOWN": 0, + "CONTROLLER_SERVICE": 1, + "VOLUME_ACCESSIBILITY_CONSTRAINTS": 2, } func (x PluginCapability_Service_Type) String() string { return proto.EnumName(PluginCapability_Service_Type_name, int32(x)) } func (PluginCapability_Service_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{4, 0, 0} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{4, 0, 0} } type VolumeCapability_AccessMode_Mode int32 @@ -105,7 +107,7 @@ func (x VolumeCapability_AccessMode_Mode) String() string { return proto.EnumName(VolumeCapability_AccessMode_Mode_name, int32(x)) } func (VolumeCapability_AccessMode_Mode) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{10, 2, 0} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{10, 2, 0} } type ControllerServiceCapability_RPC_Type int32 @@ -121,11 +123,15 @@ const ( // CREATE_DELETE_SNAPSHOT MUST support creating volume from // snapshot. ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT ControllerServiceCapability_RPC_Type = 5 - // LIST_SNAPSHOTS is NOT REQUIRED. For plugins that need to upload - // a snapshot after it is being cut, LIST_SNAPSHOTS COULD be used - // with the snapshot_id as the filter to query whether the - // uploading process is complete or not. - ControllerServiceCapability_RPC_LIST_SNAPSHOTS ControllerServiceCapability_RPC_Type = 6 + ControllerServiceCapability_RPC_LIST_SNAPSHOTS ControllerServiceCapability_RPC_Type = 6 + // Plugins supporting volume cloning at the storage level MAY + // report this capability. The source volume MUST be managed by + // the same plugin. Not all volume sources and parameters + // combinations MAY work. + ControllerServiceCapability_RPC_CLONE_VOLUME ControllerServiceCapability_RPC_Type = 7 + // Indicates the SP supports ControllerPublishVolume.readonly + // field. + ControllerServiceCapability_RPC_PUBLISH_READONLY ControllerServiceCapability_RPC_Type = 8 ) var ControllerServiceCapability_RPC_Type_name = map[int32]string{ @@ -136,6 +142,8 @@ var ControllerServiceCapability_RPC_Type_name = map[int32]string{ 4: "GET_CAPACITY", 5: "CREATE_DELETE_SNAPSHOT", 6: "LIST_SNAPSHOTS", + 7: "CLONE_VOLUME", + 8: "PUBLISH_READONLY", } var ControllerServiceCapability_RPC_Type_value = map[string]int32{ "UNKNOWN": 0, @@ -145,55 +153,41 @@ var ControllerServiceCapability_RPC_Type_value = map[string]int32{ "GET_CAPACITY": 4, "CREATE_DELETE_SNAPSHOT": 5, "LIST_SNAPSHOTS": 6, + "CLONE_VOLUME": 7, + "PUBLISH_READONLY": 8, } func (x ControllerServiceCapability_RPC_Type) String() string { return proto.EnumName(ControllerServiceCapability_RPC_Type_name, int32(x)) } func (ControllerServiceCapability_RPC_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{29, 0, 0} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{29, 0, 0} } -type SnapshotStatus_Type int32 +type VolumeUsage_Unit int32 const ( - SnapshotStatus_UNKNOWN SnapshotStatus_Type = 0 - // A snapshot is ready for use. - SnapshotStatus_READY SnapshotStatus_Type = 1 - // A snapshot is cut and is now being uploaded. - // Some cloud providers and storage systems uploads the snapshot - // to the cloud after the snapshot is cut. During this phase, - // `thaw` can be done so the application can be running again if - // `freeze` was done before taking the snapshot. - SnapshotStatus_UPLOADING SnapshotStatus_Type = 2 - // An error occurred during the snapshot uploading process. - // This error status is specific for uploading because - // `CreateSnaphot` is a blocking call before the snapshot is - // cut and therefore it SHOULD NOT come back with an error - // status when an error occurs. Instead a gRPC error code SHALL - // be returned by `CreateSnapshot` when an error occurs before - // a snapshot is cut. - SnapshotStatus_ERROR_UPLOADING SnapshotStatus_Type = 3 + VolumeUsage_UNKNOWN VolumeUsage_Unit = 0 + VolumeUsage_BYTES VolumeUsage_Unit = 1 + VolumeUsage_INODES VolumeUsage_Unit = 2 ) -var SnapshotStatus_Type_name = map[int32]string{ +var VolumeUsage_Unit_name = map[int32]string{ 0: "UNKNOWN", - 1: "READY", - 2: "UPLOADING", - 3: "ERROR_UPLOADING", + 1: "BYTES", + 2: "INODES", } -var SnapshotStatus_Type_value = map[string]int32{ - "UNKNOWN": 0, - "READY": 1, - "UPLOADING": 2, - "ERROR_UPLOADING": 3, +var VolumeUsage_Unit_value = map[string]int32{ + "UNKNOWN": 0, + "BYTES": 1, + "INODES": 2, } -func (x SnapshotStatus_Type) String() string { - return proto.EnumName(SnapshotStatus_Type_name, int32(x)) +func (x VolumeUsage_Unit) String() string { + return proto.EnumName(VolumeUsage_Unit_name, int32(x)) } -func (SnapshotStatus_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{33, 0} +func (VolumeUsage_Unit) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{47, 0} } type NodeServiceCapability_RPC_Type int32 @@ -201,22 +195,28 @@ type NodeServiceCapability_RPC_Type int32 const ( NodeServiceCapability_RPC_UNKNOWN NodeServiceCapability_RPC_Type = 0 NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME NodeServiceCapability_RPC_Type = 1 + // If Plugin implements GET_VOLUME_STATS capability + // then it MUST implement NodeGetVolumeStats RPC + // call for fetching volume statistics. + NodeServiceCapability_RPC_GET_VOLUME_STATS NodeServiceCapability_RPC_Type = 2 ) var NodeServiceCapability_RPC_Type_name = map[int32]string{ 0: "UNKNOWN", 1: "STAGE_UNSTAGE_VOLUME", + 2: "GET_VOLUME_STATS", } var NodeServiceCapability_RPC_Type_value = map[string]int32{ "UNKNOWN": 0, "STAGE_UNSTAGE_VOLUME": 1, + "GET_VOLUME_STATS": 2, } func (x NodeServiceCapability_RPC_Type) String() string { return proto.EnumName(NodeServiceCapability_RPC_Type_name, int32(x)) } func (NodeServiceCapability_RPC_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{50, 0, 0} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{50, 0, 0} } type GetPluginInfoRequest struct { @@ -229,7 +229,7 @@ func (m *GetPluginInfoRequest) Reset() { *m = GetPluginInfoRequest{} } func (m *GetPluginInfoRequest) String() string { return proto.CompactTextString(m) } func (*GetPluginInfoRequest) ProtoMessage() {} func (*GetPluginInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{0} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{0} } func (m *GetPluginInfoRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetPluginInfoRequest.Unmarshal(m, b) @@ -250,18 +250,18 @@ func (m *GetPluginInfoRequest) XXX_DiscardUnknown() { var xxx_messageInfo_GetPluginInfoRequest proto.InternalMessageInfo type GetPluginInfoResponse struct { - // The name MUST follow reverse domain name notation format - // (https://en.wikipedia.org/wiki/Reverse_domain_name_notation). - // It SHOULD include the plugin's host company name and the plugin - // name, to minimize the possibility of collisions. It MUST be 63 + // The name MUST follow domain name notation format + // (https://tools.ietf.org/html/rfc1035#section-2.3.1). It SHOULD + // include the plugin's host company name and the plugin name, + // to minimize the possibility of collisions. It MUST be 63 // characters or less, beginning and ending with an alphanumeric - // character ([a-z0-9A-Z]) with dashes (-), underscores (_), - // dots (.), and alphanumerics between. This field is REQUIRED. - Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // character ([a-z0-9A-Z]) with dashes (-), dots (.), and + // alphanumerics between. This field is REQUIRED. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // This field is REQUIRED. Value of this field is opaque to the CO. - VendorVersion string `protobuf:"bytes,2,opt,name=vendor_version,json=vendorVersion" json:"vendor_version,omitempty"` + VendorVersion string `protobuf:"bytes,2,opt,name=vendor_version,json=vendorVersion,proto3" json:"vendor_version,omitempty"` // This field is OPTIONAL. Values are opaque to the CO. - Manifest map[string]string `protobuf:"bytes,3,rep,name=manifest" json:"manifest,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Manifest map[string]string `protobuf:"bytes,3,rep,name=manifest,proto3" json:"manifest,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -271,7 +271,7 @@ func (m *GetPluginInfoResponse) Reset() { *m = GetPluginInfoResponse{} } func (m *GetPluginInfoResponse) String() string { return proto.CompactTextString(m) } func (*GetPluginInfoResponse) ProtoMessage() {} func (*GetPluginInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{1} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{1} } func (m *GetPluginInfoResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetPluginInfoResponse.Unmarshal(m, b) @@ -322,7 +322,7 @@ func (m *GetPluginCapabilitiesRequest) Reset() { *m = GetPluginCapabilit func (m *GetPluginCapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*GetPluginCapabilitiesRequest) ProtoMessage() {} func (*GetPluginCapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{2} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{2} } func (m *GetPluginCapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetPluginCapabilitiesRequest.Unmarshal(m, b) @@ -345,7 +345,7 @@ var xxx_messageInfo_GetPluginCapabilitiesRequest proto.InternalMessageInfo type GetPluginCapabilitiesResponse struct { // All the capabilities that the controller service supports. This // field is OPTIONAL. - Capabilities []*PluginCapability `protobuf:"bytes,2,rep,name=capabilities" json:"capabilities,omitempty"` + Capabilities []*PluginCapability `protobuf:"bytes,1,rep,name=capabilities,proto3" json:"capabilities,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -355,7 +355,7 @@ func (m *GetPluginCapabilitiesResponse) Reset() { *m = GetPluginCapabili func (m *GetPluginCapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*GetPluginCapabilitiesResponse) ProtoMessage() {} func (*GetPluginCapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{3} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{3} } func (m *GetPluginCapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetPluginCapabilitiesResponse.Unmarshal(m, b) @@ -396,7 +396,7 @@ func (m *PluginCapability) Reset() { *m = PluginCapability{} } func (m *PluginCapability) String() string { return proto.CompactTextString(m) } func (*PluginCapability) ProtoMessage() {} func (*PluginCapability) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{4} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{4} } func (m *PluginCapability) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PluginCapability.Unmarshal(m, b) @@ -421,7 +421,7 @@ type isPluginCapability_Type interface { } type PluginCapability_Service_ struct { - Service *PluginCapability_Service `protobuf:"bytes,1,opt,name=service,oneof"` + Service *PluginCapability_Service `protobuf:"bytes,1,opt,name=service,proto3,oneof"` } func (*PluginCapability_Service_) isPluginCapability_Type() {} @@ -496,7 +496,7 @@ func _PluginCapability_OneofSizer(msg proto.Message) (n int) { } type PluginCapability_Service struct { - Type PluginCapability_Service_Type `protobuf:"varint,1,opt,name=type,enum=csi.v0.PluginCapability_Service_Type" json:"type,omitempty"` + Type PluginCapability_Service_Type `protobuf:"varint,1,opt,name=type,proto3,enum=csi.v1.PluginCapability_Service_Type" json:"type,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -506,7 +506,7 @@ func (m *PluginCapability_Service) Reset() { *m = PluginCapability_Servi func (m *PluginCapability_Service) String() string { return proto.CompactTextString(m) } func (*PluginCapability_Service) ProtoMessage() {} func (*PluginCapability_Service) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{4, 0} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{4, 0} } func (m *PluginCapability_Service) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PluginCapability_Service.Unmarshal(m, b) @@ -543,7 +543,7 @@ func (m *ProbeRequest) Reset() { *m = ProbeRequest{} } func (m *ProbeRequest) String() string { return proto.CompactTextString(m) } func (*ProbeRequest) ProtoMessage() {} func (*ProbeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{5} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{5} } func (m *ProbeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ProbeRequest.Unmarshal(m, b) @@ -584,7 +584,7 @@ type ProbeResponse struct { // that the plugin is in a ready state and is accepting calls to its // Controller and/or Node services (according to the plugin's reported // capabilities). - Ready *wrappers.BoolValue `protobuf:"bytes,1,opt,name=ready" json:"ready,omitempty"` + Ready *wrappers.BoolValue `protobuf:"bytes,1,opt,name=ready,proto3" json:"ready,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -594,7 +594,7 @@ func (m *ProbeResponse) Reset() { *m = ProbeResponse{} } func (m *ProbeResponse) String() string { return proto.CompactTextString(m) } func (*ProbeResponse) ProtoMessage() {} func (*ProbeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{6} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{6} } func (m *ProbeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ProbeResponse.Unmarshal(m, b) @@ -625,40 +625,63 @@ type CreateVolumeRequest struct { // The suggested name for the storage space. This field is REQUIRED. // It serves two purposes: // 1) Idempotency - This name is generated by the CO to achieve - // idempotency. If `CreateVolume` fails, the volume may or may not - // be provisioned. In this case, the CO may call `CreateVolume` - // again, with the same name, to ensure the volume exists. The - // Plugin should ensure that multiple `CreateVolume` calls for the - // same name do not result in more than one piece of storage - // provisioned corresponding to that name. If a Plugin is unable to - // enforce idempotency, the CO's error recovery logic could result - // in multiple (unused) volumes being provisioned. + // idempotency. The Plugin SHOULD ensure that multiple + // `CreateVolume` calls for the same name do not result in more + // than one piece of storage provisioned corresponding to that + // name. If a Plugin is unable to enforce idempotency, the CO's + // error recovery logic could result in multiple (unused) volumes + // being provisioned. + // In the case of error, the CO MUST handle the gRPC error codes + // per the recovery behavior defined in the "CreateVolume Errors" + // section below. + // The CO is responsible for cleaning up volumes it provisioned + // that it no longer needs. If the CO is uncertain whether a volume + // was provisioned or not when a `CreateVolume` call fails, the CO + // MAY call `CreateVolume` again, with the same name, to ensure the + // volume exists and to retrieve the volume's `volume_id` (unless + // otherwise prohibited by "CreateVolume Errors"). // 2) Suggested name - Some storage systems allow callers to specify // an identifier by which to refer to the newly provisioned // storage. If a storage system supports this, it can optionally // use this name as the identifier for the new volume. - Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - CapacityRange *CapacityRange `protobuf:"bytes,2,opt,name=capacity_range,json=capacityRange" json:"capacity_range,omitempty"` - // The capabilities that the provisioned volume MUST have: the Plugin - // MUST provision a volume that could satisfy ALL of the - // capabilities specified in this list. The Plugin MUST assume that - // the CO MAY use the provisioned volume later with ANY of the - // capabilities specified in this list. This also enables the CO to do - // early validation: if ANY of the specified volume capabilities are - // not supported by the Plugin, the call SHALL fail. This field is - // REQUIRED. - VolumeCapabilities []*VolumeCapability `protobuf:"bytes,3,rep,name=volume_capabilities,json=volumeCapabilities" json:"volume_capabilities,omitempty"` + // Any Unicode string that conforms to the length limit is allowed + // except those containing the following banned characters: + // U+0000-U+0008, U+000B, U+000C, U+000E-U+001F, U+007F-U+009F. + // (These are control characters other than commonly used whitespace.) + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // This field is OPTIONAL. This allows the CO to specify the capacity + // requirement of the volume to be provisioned. If not specified, the + // Plugin MAY choose an implementation-defined capacity range. If + // specified it MUST always be honored, even when creating volumes + // from a source; which MAY force some backends to internally extend + // the volume after creating it. + CapacityRange *CapacityRange `protobuf:"bytes,2,opt,name=capacity_range,json=capacityRange,proto3" json:"capacity_range,omitempty"` + // The capabilities that the provisioned volume MUST have. SP MUST + // provision a volume that will satisfy ALL of the capabilities + // specified in this list. Otherwise SP MUST return the appropriate + // gRPC error code. + // The Plugin MUST assume that the CO MAY use the provisioned volume + // with ANY of the capabilities specified in this list. + // For example, a CO MAY specify two volume capabilities: one with + // access mode SINGLE_NODE_WRITER and another with access mode + // MULTI_NODE_READER_ONLY. In this case, the SP MUST verify that the + // provisioned volume can be used in either mode. + // This also enables the CO to do early validation: If ANY of the + // specified volume capabilities are not supported by the SP, the call + // MUST return the appropriate gRPC error code. + // This field is REQUIRED. + VolumeCapabilities []*VolumeCapability `protobuf:"bytes,3,rep,name=volume_capabilities,json=volumeCapabilities,proto3" json:"volume_capabilities,omitempty"` // Plugin specific parameters passed in as opaque key-value pairs. // This field is OPTIONAL. The Plugin is responsible for parsing and // validating these parameters. COs will treat these as opaque. - Parameters map[string]string `protobuf:"bytes,4,rep,name=parameters" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Parameters map[string]string `protobuf:"bytes,4,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // Secrets required by plugin to complete volume creation request. // This field is OPTIONAL. Refer to the `Secrets Requirements` // section on how to use this field. - ControllerCreateSecrets map[string]string `protobuf:"bytes,5,rep,name=controller_create_secrets,json=controllerCreateSecrets" json:"controller_create_secrets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Secrets map[string]string `protobuf:"bytes,5,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // If specified, the new volume will be pre-populated with data from // this source. This field is OPTIONAL. - VolumeContentSource *VolumeContentSource `protobuf:"bytes,6,opt,name=volume_content_source,json=volumeContentSource" json:"volume_content_source,omitempty"` + VolumeContentSource *VolumeContentSource `protobuf:"bytes,6,opt,name=volume_content_source,json=volumeContentSource,proto3" json:"volume_content_source,omitempty"` // Specifies where (regions, zones, racks, etc.) the provisioned // volume MUST be accessible from. // An SP SHALL advertise the requirements for topological @@ -666,11 +689,11 @@ type CreateVolumeRequest struct { // topological accessibility information supported by the SP. // This field is OPTIONAL. // This field SHALL NOT be specified unless the SP has the - // ACCESSIBILITY_CONSTRAINTS plugin capability. + // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability. // If this field is not specified and the SP has the - // ACCESSIBILITY_CONSTRAINTS plugin capability, the SP MAY choose - // where the provisioned volume is accessible from. - AccessibilityRequirements *TopologyRequirement `protobuf:"bytes,7,opt,name=accessibility_requirements,json=accessibilityRequirements" json:"accessibility_requirements,omitempty"` + // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability, the SP MAY + // choose where the provisioned volume is accessible from. + AccessibilityRequirements *TopologyRequirement `protobuf:"bytes,7,opt,name=accessibility_requirements,json=accessibilityRequirements,proto3" json:"accessibility_requirements,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -680,7 +703,7 @@ func (m *CreateVolumeRequest) Reset() { *m = CreateVolumeRequest{} } func (m *CreateVolumeRequest) String() string { return proto.CompactTextString(m) } func (*CreateVolumeRequest) ProtoMessage() {} func (*CreateVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{7} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{7} } func (m *CreateVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateVolumeRequest.Unmarshal(m, b) @@ -728,9 +751,9 @@ func (m *CreateVolumeRequest) GetParameters() map[string]string { return nil } -func (m *CreateVolumeRequest) GetControllerCreateSecrets() map[string]string { +func (m *CreateVolumeRequest) GetSecrets() map[string]string { if m != nil { - return m.ControllerCreateSecrets + return m.Secrets } return nil } @@ -754,6 +777,7 @@ func (m *CreateVolumeRequest) GetAccessibilityRequirements() *TopologyRequiremen type VolumeContentSource struct { // Types that are valid to be assigned to Type: // *VolumeContentSource_Snapshot + // *VolumeContentSource_Volume Type isVolumeContentSource_Type `protobuf_oneof:"type"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -764,7 +788,7 @@ func (m *VolumeContentSource) Reset() { *m = VolumeContentSource{} } func (m *VolumeContentSource) String() string { return proto.CompactTextString(m) } func (*VolumeContentSource) ProtoMessage() {} func (*VolumeContentSource) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{8} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{8} } func (m *VolumeContentSource) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeContentSource.Unmarshal(m, b) @@ -789,11 +813,17 @@ type isVolumeContentSource_Type interface { } type VolumeContentSource_Snapshot struct { - Snapshot *VolumeContentSource_SnapshotSource `protobuf:"bytes,1,opt,name=snapshot,oneof"` + Snapshot *VolumeContentSource_SnapshotSource `protobuf:"bytes,1,opt,name=snapshot,proto3,oneof"` +} + +type VolumeContentSource_Volume struct { + Volume *VolumeContentSource_VolumeSource `protobuf:"bytes,2,opt,name=volume,proto3,oneof"` } func (*VolumeContentSource_Snapshot) isVolumeContentSource_Type() {} +func (*VolumeContentSource_Volume) isVolumeContentSource_Type() {} + func (m *VolumeContentSource) GetType() isVolumeContentSource_Type { if m != nil { return m.Type @@ -808,10 +838,18 @@ func (m *VolumeContentSource) GetSnapshot() *VolumeContentSource_SnapshotSource return nil } +func (m *VolumeContentSource) GetVolume() *VolumeContentSource_VolumeSource { + if x, ok := m.GetType().(*VolumeContentSource_Volume); ok { + return x.Volume + } + return nil +} + // XXX_OneofFuncs is for the internal use of the proto package. func (*VolumeContentSource) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { return _VolumeContentSource_OneofMarshaler, _VolumeContentSource_OneofUnmarshaler, _VolumeContentSource_OneofSizer, []interface{}{ (*VolumeContentSource_Snapshot)(nil), + (*VolumeContentSource_Volume)(nil), } } @@ -824,6 +862,11 @@ func _VolumeContentSource_OneofMarshaler(msg proto.Message, b *proto.Buffer) err if err := b.EncodeMessage(x.Snapshot); err != nil { return err } + case *VolumeContentSource_Volume: + b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Volume); err != nil { + return err + } case nil: default: return fmt.Errorf("VolumeContentSource.Type has unexpected type %T", x) @@ -842,6 +885,14 @@ func _VolumeContentSource_OneofUnmarshaler(msg proto.Message, tag, wire int, b * err := b.DecodeMessage(msg) m.Type = &VolumeContentSource_Snapshot{msg} return true, err + case 2: // type.volume + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(VolumeContentSource_VolumeSource) + err := b.DecodeMessage(msg) + m.Type = &VolumeContentSource_Volume{msg} + return true, err default: return false, nil } @@ -856,6 +907,11 @@ func _VolumeContentSource_OneofSizer(msg proto.Message) (n int) { n += 1 // tag and wire n += proto.SizeVarint(uint64(s)) n += s + case *VolumeContentSource_Volume: + s := proto.Size(x.Volume) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s case nil: default: panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) @@ -868,7 +924,7 @@ type VolumeContentSource_SnapshotSource struct { // This field is REQUIRED. Plugin is REQUIRED to support creating // volume from snapshot if it supports the capability // CREATE_DELETE_SNAPSHOT. - Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"` + SnapshotId string `protobuf:"bytes,1,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -878,7 +934,7 @@ func (m *VolumeContentSource_SnapshotSource) Reset() { *m = VolumeConten func (m *VolumeContentSource_SnapshotSource) String() string { return proto.CompactTextString(m) } func (*VolumeContentSource_SnapshotSource) ProtoMessage() {} func (*VolumeContentSource_SnapshotSource) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{8, 0} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{8, 0} } func (m *VolumeContentSource_SnapshotSource) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeContentSource_SnapshotSource.Unmarshal(m, b) @@ -898,9 +954,50 @@ func (m *VolumeContentSource_SnapshotSource) XXX_DiscardUnknown() { var xxx_messageInfo_VolumeContentSource_SnapshotSource proto.InternalMessageInfo -func (m *VolumeContentSource_SnapshotSource) GetId() string { +func (m *VolumeContentSource_SnapshotSource) GetSnapshotId() string { if m != nil { - return m.Id + return m.SnapshotId + } + return "" +} + +type VolumeContentSource_VolumeSource struct { + // Contains identity information for the existing source volume. + // This field is REQUIRED. Plugins reporting CLONE_VOLUME + // capability MUST support creating a volume from another volume. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeContentSource_VolumeSource) Reset() { *m = VolumeContentSource_VolumeSource{} } +func (m *VolumeContentSource_VolumeSource) String() string { return proto.CompactTextString(m) } +func (*VolumeContentSource_VolumeSource) ProtoMessage() {} +func (*VolumeContentSource_VolumeSource) Descriptor() ([]byte, []int) { + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{8, 1} +} +func (m *VolumeContentSource_VolumeSource) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VolumeContentSource_VolumeSource.Unmarshal(m, b) +} +func (m *VolumeContentSource_VolumeSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VolumeContentSource_VolumeSource.Marshal(b, m, deterministic) +} +func (dst *VolumeContentSource_VolumeSource) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeContentSource_VolumeSource.Merge(dst, src) +} +func (m *VolumeContentSource_VolumeSource) XXX_Size() int { + return xxx_messageInfo_VolumeContentSource_VolumeSource.Size(m) +} +func (m *VolumeContentSource_VolumeSource) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeContentSource_VolumeSource.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeContentSource_VolumeSource proto.InternalMessageInfo + +func (m *VolumeContentSource_VolumeSource) GetVolumeId() string { + if m != nil { + return m.VolumeId } return "" } @@ -909,7 +1006,7 @@ type CreateVolumeResponse struct { // Contains all attributes of the newly created volume that are // relevant to the CO along with information required by the Plugin // to uniquely identify the volume. This field is REQUIRED. - Volume *Volume `protobuf:"bytes,1,opt,name=volume" json:"volume,omitempty"` + Volume *Volume `protobuf:"bytes,1,opt,name=volume,proto3" json:"volume,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -919,7 +1016,7 @@ func (m *CreateVolumeResponse) Reset() { *m = CreateVolumeResponse{} } func (m *CreateVolumeResponse) String() string { return proto.CompactTextString(m) } func (*CreateVolumeResponse) ProtoMessage() {} func (*CreateVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{9} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{9} } func (m *CreateVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateVolumeResponse.Unmarshal(m, b) @@ -956,7 +1053,7 @@ type VolumeCapability struct { // *VolumeCapability_Mount AccessType isVolumeCapability_AccessType `protobuf_oneof:"access_type"` // This is a REQUIRED field. - AccessMode *VolumeCapability_AccessMode `protobuf:"bytes,3,opt,name=access_mode,json=accessMode" json:"access_mode,omitempty"` + AccessMode *VolumeCapability_AccessMode `protobuf:"bytes,3,opt,name=access_mode,json=accessMode,proto3" json:"access_mode,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -966,7 +1063,7 @@ func (m *VolumeCapability) Reset() { *m = VolumeCapability{} } func (m *VolumeCapability) String() string { return proto.CompactTextString(m) } func (*VolumeCapability) ProtoMessage() {} func (*VolumeCapability) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{10} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{10} } func (m *VolumeCapability) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeCapability.Unmarshal(m, b) @@ -991,13 +1088,15 @@ type isVolumeCapability_AccessType interface { } type VolumeCapability_Block struct { - Block *VolumeCapability_BlockVolume `protobuf:"bytes,1,opt,name=block,oneof"` + Block *VolumeCapability_BlockVolume `protobuf:"bytes,1,opt,name=block,proto3,oneof"` } + type VolumeCapability_Mount struct { - Mount *VolumeCapability_MountVolume `protobuf:"bytes,2,opt,name=mount,oneof"` + Mount *VolumeCapability_MountVolume `protobuf:"bytes,2,opt,name=mount,proto3,oneof"` } func (*VolumeCapability_Block) isVolumeCapability_AccessType() {} + func (*VolumeCapability_Mount) isVolumeCapability_AccessType() {} func (m *VolumeCapability) GetAccessType() isVolumeCapability_AccessType { @@ -1113,7 +1212,7 @@ func (m *VolumeCapability_BlockVolume) Reset() { *m = VolumeCapability_B func (m *VolumeCapability_BlockVolume) String() string { return proto.CompactTextString(m) } func (*VolumeCapability_BlockVolume) ProtoMessage() {} func (*VolumeCapability_BlockVolume) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{10, 0} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{10, 0} } func (m *VolumeCapability_BlockVolume) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeCapability_BlockVolume.Unmarshal(m, b) @@ -1137,13 +1236,13 @@ var xxx_messageInfo_VolumeCapability_BlockVolume proto.InternalMessageInfo type VolumeCapability_MountVolume struct { // The filesystem type. This field is OPTIONAL. // An empty string is equal to an unspecified field value. - FsType string `protobuf:"bytes,1,opt,name=fs_type,json=fsType" json:"fs_type,omitempty"` + FsType string `protobuf:"bytes,1,opt,name=fs_type,json=fsType,proto3" json:"fs_type,omitempty"` // The mount options that can be used for the volume. This field is // OPTIONAL. `mount_flags` MAY contain sensitive information. // Therefore, the CO and the Plugin MUST NOT leak this information // to untrusted entities. The total size of this repeated field // SHALL NOT exceed 4 KiB. - MountFlags []string `protobuf:"bytes,2,rep,name=mount_flags,json=mountFlags" json:"mount_flags,omitempty"` + MountFlags []string `protobuf:"bytes,2,rep,name=mount_flags,json=mountFlags,proto3" json:"mount_flags,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1153,7 +1252,7 @@ func (m *VolumeCapability_MountVolume) Reset() { *m = VolumeCapability_M func (m *VolumeCapability_MountVolume) String() string { return proto.CompactTextString(m) } func (*VolumeCapability_MountVolume) ProtoMessage() {} func (*VolumeCapability_MountVolume) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{10, 1} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{10, 1} } func (m *VolumeCapability_MountVolume) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeCapability_MountVolume.Unmarshal(m, b) @@ -1190,7 +1289,7 @@ func (m *VolumeCapability_MountVolume) GetMountFlags() []string { // Specify how a volume can be accessed. type VolumeCapability_AccessMode struct { // This field is REQUIRED. - Mode VolumeCapability_AccessMode_Mode `protobuf:"varint,1,opt,name=mode,enum=csi.v0.VolumeCapability_AccessMode_Mode" json:"mode,omitempty"` + Mode VolumeCapability_AccessMode_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=csi.v1.VolumeCapability_AccessMode_Mode" json:"mode,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1200,7 +1299,7 @@ func (m *VolumeCapability_AccessMode) Reset() { *m = VolumeCapability_Ac func (m *VolumeCapability_AccessMode) String() string { return proto.CompactTextString(m) } func (*VolumeCapability_AccessMode) ProtoMessage() {} func (*VolumeCapability_AccessMode) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{10, 2} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{10, 2} } func (m *VolumeCapability_AccessMode) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeCapability_AccessMode.Unmarshal(m, b) @@ -1234,11 +1333,11 @@ type CapacityRange struct { // Volume MUST be at least this big. This field is OPTIONAL. // A value of 0 is equal to an unspecified field value. // The value of this field MUST NOT be negative. - RequiredBytes int64 `protobuf:"varint,1,opt,name=required_bytes,json=requiredBytes" json:"required_bytes,omitempty"` + RequiredBytes int64 `protobuf:"varint,1,opt,name=required_bytes,json=requiredBytes,proto3" json:"required_bytes,omitempty"` // Volume MUST not be bigger than this. This field is OPTIONAL. // A value of 0 is equal to an unspecified field value. // The value of this field MUST NOT be negative. - LimitBytes int64 `protobuf:"varint,2,opt,name=limit_bytes,json=limitBytes" json:"limit_bytes,omitempty"` + LimitBytes int64 `protobuf:"varint,2,opt,name=limit_bytes,json=limitBytes,proto3" json:"limit_bytes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1248,7 +1347,7 @@ func (m *CapacityRange) Reset() { *m = CapacityRange{} } func (m *CapacityRange) String() string { return proto.CompactTextString(m) } func (*CapacityRange) ProtoMessage() {} func (*CapacityRange) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{11} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{11} } func (m *CapacityRange) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CapacityRange.Unmarshal(m, b) @@ -1282,34 +1381,46 @@ func (m *CapacityRange) GetLimitBytes() int64 { return 0 } -// The information about a provisioned volume. +// Information about a specific volume. type Volume struct { // The capacity of the volume in bytes. This field is OPTIONAL. If not // set (value of 0), it indicates that the capacity of the volume is // unknown (e.g., NFS share). // The value of this field MUST NOT be negative. - CapacityBytes int64 `protobuf:"varint,1,opt,name=capacity_bytes,json=capacityBytes" json:"capacity_bytes,omitempty"` - // Contains identity information for the created volume. This field is - // REQUIRED. The identity information will be used by the CO in - // subsequent calls to refer to the provisioned volume. - Id string `protobuf:"bytes,2,opt,name=id" json:"id,omitempty"` - // Attributes reflect static properties of a volume and MUST be passed - // to volume validation and publishing calls. - // Attributes SHALL be opaque to a CO. Attributes SHALL NOT be mutable - // and SHALL be safe for the CO to cache. Attributes SHOULD NOT - // contain sensitive information. Attributes MAY NOT uniquely identify - // a volume. A volume uniquely identified by `id` SHALL always report - // the same attributes. This field is OPTIONAL and when present MUST - // be passed to volume validation and publishing calls. - Attributes map[string]string `protobuf:"bytes,3,rep,name=attributes" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + CapacityBytes int64 `protobuf:"varint,1,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` + // The identifier for this volume, generated by the plugin. + // This field is REQUIRED. + // This field MUST contain enough information to uniquely identify + // this specific volume vs all other volumes supported by this plugin. + // This field SHALL be used by the CO in subsequent calls to refer to + // this volume. + // The SP is NOT responsible for global uniqueness of volume_id across + // multiple SPs. + VolumeId string `protobuf:"bytes,2,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // Opaque static properties of the volume. SP MAY use this field to + // ensure subsequent volume validation and publishing calls have + // contextual information. + // The contents of this field SHALL be opaque to a CO. + // The contents of this field SHALL NOT be mutable. + // The contents of this field SHALL be safe for the CO to cache. + // The contents of this field SHOULD NOT contain sensitive + // information. + // The contents of this field SHOULD NOT be used for uniquely + // identifying a volume. The `volume_id` alone SHOULD be sufficient to + // identify the volume. + // A volume uniquely identified by `volume_id` SHALL always report the + // same volume_context. + // This field is OPTIONAL and when present MUST be passed to volume + // validation and publishing calls. + VolumeContext map[string]string `protobuf:"bytes,3,rep,name=volume_context,json=volumeContext,proto3" json:"volume_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // If specified, indicates that the volume is not empty and is // pre-populated with data from the specified source. // This field is OPTIONAL. - ContentSource *VolumeContentSource `protobuf:"bytes,4,opt,name=content_source,json=contentSource" json:"content_source,omitempty"` + ContentSource *VolumeContentSource `protobuf:"bytes,4,opt,name=content_source,json=contentSource,proto3" json:"content_source,omitempty"` // Specifies where (regions, zones, racks, etc.) the provisioned // volume is accessible from. // A plugin that returns this field MUST also set the - // ACCESSIBILITY_CONSTRAINTS plugin capability. + // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability. // An SP MAY specify multiple topologies to indicate the volume is // accessible from multiple locations. // COs MAY use this information along with the topology information @@ -1317,7 +1428,7 @@ type Volume struct { // from a given node when scheduling workloads. // This field is OPTIONAL. If it is not specified, the CO MAY assume // the volume is equally accessible from all nodes in the cluster and - // may schedule workloads referencing the volume on any available + // MAY schedule workloads referencing the volume on any available // node. // // Example 1: @@ -1331,7 +1442,7 @@ type Volume struct { // {"region": "R1", "zone": "Z3"} // Indicates a volume accessible from both "zone" "Z2" and "zone" "Z3" // in the "region" "R1". - AccessibleTopology []*Topology `protobuf:"bytes,5,rep,name=accessible_topology,json=accessibleTopology" json:"accessible_topology,omitempty"` + AccessibleTopology []*Topology `protobuf:"bytes,5,rep,name=accessible_topology,json=accessibleTopology,proto3" json:"accessible_topology,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1341,7 +1452,7 @@ func (m *Volume) Reset() { *m = Volume{} } func (m *Volume) String() string { return proto.CompactTextString(m) } func (*Volume) ProtoMessage() {} func (*Volume) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{12} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{12} } func (m *Volume) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Volume.Unmarshal(m, b) @@ -1368,16 +1479,16 @@ func (m *Volume) GetCapacityBytes() int64 { return 0 } -func (m *Volume) GetId() string { +func (m *Volume) GetVolumeId() string { if m != nil { - return m.Id + return m.VolumeId } return "" } -func (m *Volume) GetAttributes() map[string]string { +func (m *Volume) GetVolumeContext() map[string]string { if m != nil { - return m.Attributes + return m.VolumeContext } return nil } @@ -1453,7 +1564,7 @@ type TopologyRequirement struct { // then the provisioned volume MUST be accessible from the "region" // "R1" and the "zone" "Z2" and the SP may select the second zone // independently, e.g. "R1/Z4". - Requisite []*Topology `protobuf:"bytes,1,rep,name=requisite" json:"requisite,omitempty"` + Requisite []*Topology `protobuf:"bytes,1,rep,name=requisite,proto3" json:"requisite,omitempty"` // Specifies the list of topologies the CO would prefer the volume to // be provisioned in. // @@ -1523,7 +1634,7 @@ type TopologyRequirement struct { // combination of "Z3" and other possibilities from the list of // requisite. If that's not possible, it should fall back to a // combination of other possibilities from the list of requisite. - Preferred []*Topology `protobuf:"bytes,2,rep,name=preferred" json:"preferred,omitempty"` + Preferred []*Topology `protobuf:"bytes,2,rep,name=preferred,proto3" json:"preferred,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1533,7 +1644,7 @@ func (m *TopologyRequirement) Reset() { *m = TopologyRequirement{} } func (m *TopologyRequirement) String() string { return proto.CompactTextString(m) } func (*TopologyRequirement) ProtoMessage() {} func (*TopologyRequirement) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{13} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{13} } func (m *TopologyRequirement) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TopologyRequirement.Unmarshal(m, b) @@ -1573,15 +1684,18 @@ func (m *TopologyRequirement) GetPreferred() []*Topology { // A topological segment is a specific instance of a topological domain, // like "zone3", "rack3", etc. // For example {"com.company/zone": "Z1", "com.company/rack": "R3"} -// Valid keys have two segments: an optional prefix and name, separated +// Valid keys have two segments: an OPTIONAL prefix and name, separated // by a slash (/), for example: "com.company.example/zone". -// The key name segment is required. The prefix is optional. -// Both the key name and the prefix MUST each be 63 characters or less, -// begin and end with an alphanumeric character ([a-z0-9A-Z]) and -// contain only dashes (-), underscores (_), dots (.), or alphanumerics -// in between, for example "zone". -// The key prefix MUST follow reverse domain name notation format -// (https://en.wikipedia.org/wiki/Reverse_domain_name_notation). +// The key name segment is REQUIRED. The prefix is OPTIONAL. +// The key name MUST be 63 characters or less, begin and end with an +// alphanumeric character ([a-z0-9A-Z]), and contain only dashes (-), +// underscores (_), dots (.), or alphanumerics in between, for example +// "zone". +// The key prefix MUST be 63 characters or less, begin and end with a +// lower-case alphanumeric character ([a-z0-9]), contain only +// dashes (-), dots (.), or lower-case alphanumerics in between, and +// follow domain name notation format +// (https://tools.ietf.org/html/rfc1035#section-2.3.1). // The key prefix SHOULD include the plugin's host company name and/or // the plugin name, to minimize the possibility of collisions with keys // from other plugins. @@ -1594,7 +1708,7 @@ func (m *TopologyRequirement) GetPreferred() []*Topology { // alphanumeric character with '-', '_', '.', or alphanumerics in // between. type Topology struct { - Segments map[string]string `protobuf:"bytes,1,rep,name=segments" json:"segments,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Segments map[string]string `protobuf:"bytes,1,rep,name=segments,proto3" json:"segments,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1604,7 +1718,7 @@ func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} func (*Topology) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{14} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{14} } func (m *Topology) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Topology.Unmarshal(m, b) @@ -1634,21 +1748,21 @@ func (m *Topology) GetSegments() map[string]string { type DeleteVolumeRequest struct { // The ID of the volume to be deprovisioned. // This field is REQUIRED. - VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` // Secrets required by plugin to complete volume deletion request. // This field is OPTIONAL. Refer to the `Secrets Requirements` // section on how to use this field. - ControllerDeleteSecrets map[string]string `protobuf:"bytes,2,rep,name=controller_delete_secrets,json=controllerDeleteSecrets" json:"controller_delete_secrets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Secrets map[string]string `protobuf:"bytes,2,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *DeleteVolumeRequest) Reset() { *m = DeleteVolumeRequest{} } func (m *DeleteVolumeRequest) String() string { return proto.CompactTextString(m) } func (*DeleteVolumeRequest) ProtoMessage() {} func (*DeleteVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{15} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{15} } func (m *DeleteVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteVolumeRequest.Unmarshal(m, b) @@ -1675,9 +1789,9 @@ func (m *DeleteVolumeRequest) GetVolumeId() string { return "" } -func (m *DeleteVolumeRequest) GetControllerDeleteSecrets() map[string]string { +func (m *DeleteVolumeRequest) GetSecrets() map[string]string { if m != nil { - return m.ControllerDeleteSecrets + return m.Secrets } return nil } @@ -1692,7 +1806,7 @@ func (m *DeleteVolumeResponse) Reset() { *m = DeleteVolumeResponse{} } func (m *DeleteVolumeResponse) String() string { return proto.CompactTextString(m) } func (*DeleteVolumeResponse) ProtoMessage() {} func (*DeleteVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{16} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{16} } func (m *DeleteVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteVolumeResponse.Unmarshal(m, b) @@ -1715,24 +1829,28 @@ var xxx_messageInfo_DeleteVolumeResponse proto.InternalMessageInfo type ControllerPublishVolumeRequest struct { // The ID of the volume to be used on a node. // This field is REQUIRED. - VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` // The ID of the node. This field is REQUIRED. The CO SHALL set this // field to match the node ID returned by `NodeGetInfo`. - NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId" json:"node_id,omitempty"` - // The capability of the volume the CO expects the volume to have. + NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` + // Volume capability describing how the CO intends to use this volume. + // SP MUST ensure the CO can use the published volume as described. + // Otherwise SP MUST return the appropriate gRPC error code. // This is a REQUIRED field. - VolumeCapability *VolumeCapability `protobuf:"bytes,3,opt,name=volume_capability,json=volumeCapability" json:"volume_capability,omitempty"` - // Whether to publish the volume in readonly mode. This field is - // REQUIRED. - Readonly bool `protobuf:"varint,4,opt,name=readonly" json:"readonly,omitempty"` + VolumeCapability *VolumeCapability `protobuf:"bytes,3,opt,name=volume_capability,json=volumeCapability,proto3" json:"volume_capability,omitempty"` + // Indicates SP MUST publish the volume in readonly mode. + // CO MUST set this field to false if SP does not have the + // PUBLISH_READONLY controller capability. + // This is a REQUIRED field. + Readonly bool `protobuf:"varint,4,opt,name=readonly,proto3" json:"readonly,omitempty"` // Secrets required by plugin to complete controller publish volume // request. This field is OPTIONAL. Refer to the // `Secrets Requirements` section on how to use this field. - ControllerPublishSecrets map[string]string `protobuf:"bytes,5,rep,name=controller_publish_secrets,json=controllerPublishSecrets" json:"controller_publish_secrets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // Attributes of the volume to be used on a node. This field is - // OPTIONAL and MUST match the attributes of the Volume identified - // by `volume_id`. - VolumeAttributes map[string]string `protobuf:"bytes,6,rep,name=volume_attributes,json=volumeAttributes" json:"volume_attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Secrets map[string]string `protobuf:"bytes,5,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Volume context as returned by CO in CreateVolumeRequest. This field + // is OPTIONAL and MUST match the volume_context of the volume + // identified by `volume_id`. + VolumeContext map[string]string `protobuf:"bytes,6,rep,name=volume_context,json=volumeContext,proto3" json:"volume_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1742,7 +1860,7 @@ func (m *ControllerPublishVolumeRequest) Reset() { *m = ControllerPublis func (m *ControllerPublishVolumeRequest) String() string { return proto.CompactTextString(m) } func (*ControllerPublishVolumeRequest) ProtoMessage() {} func (*ControllerPublishVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{17} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{17} } func (m *ControllerPublishVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerPublishVolumeRequest.Unmarshal(m, b) @@ -1790,26 +1908,35 @@ func (m *ControllerPublishVolumeRequest) GetReadonly() bool { return false } -func (m *ControllerPublishVolumeRequest) GetControllerPublishSecrets() map[string]string { +func (m *ControllerPublishVolumeRequest) GetSecrets() map[string]string { if m != nil { - return m.ControllerPublishSecrets + return m.Secrets } return nil } -func (m *ControllerPublishVolumeRequest) GetVolumeAttributes() map[string]string { +func (m *ControllerPublishVolumeRequest) GetVolumeContext() map[string]string { if m != nil { - return m.VolumeAttributes + return m.VolumeContext } return nil } type ControllerPublishVolumeResponse struct { - // The SP specific information that will be passed to the Plugin in - // the subsequent `NodeStageVolume` or `NodePublishVolume` calls - // for the given volume. - // This information is opaque to the CO. This field is OPTIONAL. - PublishInfo map[string]string `protobuf:"bytes,1,rep,name=publish_info,json=publishInfo" json:"publish_info,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // Opaque static publish properties of the volume. SP MAY use this + // field to ensure subsequent `NodeStageVolume` or `NodePublishVolume` + // calls calls have contextual information. + // The contents of this field SHALL be opaque to a CO. + // The contents of this field SHALL NOT be mutable. + // The contents of this field SHALL be safe for the CO to cache. + // The contents of this field SHOULD NOT contain sensitive + // information. + // The contents of this field SHOULD NOT be used for uniquely + // identifying a volume. The `volume_id` alone SHOULD be sufficient to + // identify the volume. + // This field is OPTIONAL and when present MUST be passed to + // subsequent `NodeStageVolume` or `NodePublishVolume` calls + PublishContext map[string]string `protobuf:"bytes,1,rep,name=publish_context,json=publishContext,proto3" json:"publish_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1819,7 +1946,7 @@ func (m *ControllerPublishVolumeResponse) Reset() { *m = ControllerPubli func (m *ControllerPublishVolumeResponse) String() string { return proto.CompactTextString(m) } func (*ControllerPublishVolumeResponse) ProtoMessage() {} func (*ControllerPublishVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{18} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{18} } func (m *ControllerPublishVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerPublishVolumeResponse.Unmarshal(m, b) @@ -1839,38 +1966,38 @@ func (m *ControllerPublishVolumeResponse) XXX_DiscardUnknown() { var xxx_messageInfo_ControllerPublishVolumeResponse proto.InternalMessageInfo -func (m *ControllerPublishVolumeResponse) GetPublishInfo() map[string]string { +func (m *ControllerPublishVolumeResponse) GetPublishContext() map[string]string { if m != nil { - return m.PublishInfo + return m.PublishContext } return nil } type ControllerUnpublishVolumeRequest struct { // The ID of the volume. This field is REQUIRED. - VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` // The ID of the node. This field is OPTIONAL. The CO SHOULD set this // field to match the node ID returned by `NodeGetInfo` or leave it // unset. If the value is set, the SP MUST unpublish the volume from // the specified node. If the value is unset, the SP MUST unpublish // the volume from all nodes it is published to. - NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId" json:"node_id,omitempty"` + NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` // Secrets required by plugin to complete controller unpublish volume // request. This SHOULD be the same secrets passed to the // ControllerPublishVolume call for the specified volume. // This field is OPTIONAL. Refer to the `Secrets Requirements` // section on how to use this field. - ControllerUnpublishSecrets map[string]string `protobuf:"bytes,3,rep,name=controller_unpublish_secrets,json=controllerUnpublishSecrets" json:"controller_unpublish_secrets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Secrets map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ControllerUnpublishVolumeRequest) Reset() { *m = ControllerUnpublishVolumeRequest{} } func (m *ControllerUnpublishVolumeRequest) String() string { return proto.CompactTextString(m) } func (*ControllerUnpublishVolumeRequest) ProtoMessage() {} func (*ControllerUnpublishVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{19} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{19} } func (m *ControllerUnpublishVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerUnpublishVolumeRequest.Unmarshal(m, b) @@ -1904,9 +2031,9 @@ func (m *ControllerUnpublishVolumeRequest) GetNodeId() string { return "" } -func (m *ControllerUnpublishVolumeRequest) GetControllerUnpublishSecrets() map[string]string { +func (m *ControllerUnpublishVolumeRequest) GetSecrets() map[string]string { if m != nil { - return m.ControllerUnpublishSecrets + return m.Secrets } return nil } @@ -1921,7 +2048,7 @@ func (m *ControllerUnpublishVolumeResponse) Reset() { *m = ControllerUnp func (m *ControllerUnpublishVolumeResponse) String() string { return proto.CompactTextString(m) } func (*ControllerUnpublishVolumeResponse) ProtoMessage() {} func (*ControllerUnpublishVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{20} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{20} } func (m *ControllerUnpublishVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerUnpublishVolumeResponse.Unmarshal(m, b) @@ -1943,31 +2070,32 @@ var xxx_messageInfo_ControllerUnpublishVolumeResponse proto.InternalMessageInfo type ValidateVolumeCapabilitiesRequest struct { // The ID of the volume to check. This field is REQUIRED. - VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // Volume context as returned by CO in CreateVolumeRequest. This field + // is OPTIONAL and MUST match the volume_context of the volume + // identified by `volume_id`. + VolumeContext map[string]string `protobuf:"bytes,2,rep,name=volume_context,json=volumeContext,proto3" json:"volume_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // The capabilities that the CO wants to check for the volume. This - // call SHALL return "supported" only if all the volume capabilities + // call SHALL return "confirmed" only if all the volume capabilities // specified below are supported. This field is REQUIRED. - VolumeCapabilities []*VolumeCapability `protobuf:"bytes,2,rep,name=volume_capabilities,json=volumeCapabilities" json:"volume_capabilities,omitempty"` - // Attributes of the volume to check. This field is OPTIONAL and MUST - // match the attributes of the Volume identified by `volume_id`. - VolumeAttributes map[string]string `protobuf:"bytes,3,rep,name=volume_attributes,json=volumeAttributes" json:"volume_attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // Specifies where (regions, zones, racks, etc.) the caller believes - // the volume is accessible from. - // A caller MAY specify multiple topologies to indicate they believe - // the volume to be accessible from multiple locations. - // This field is OPTIONAL. This field SHALL NOT be set unless the - // plugin advertises the ACCESSIBILITY_CONSTRAINTS capability. - AccessibleTopology []*Topology `protobuf:"bytes,4,rep,name=accessible_topology,json=accessibleTopology" json:"accessible_topology,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + VolumeCapabilities []*VolumeCapability `protobuf:"bytes,3,rep,name=volume_capabilities,json=volumeCapabilities,proto3" json:"volume_capabilities,omitempty"` + // See CreateVolumeRequest.parameters. + // This field is OPTIONAL. + Parameters map[string]string `protobuf:"bytes,4,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Secrets required by plugin to complete volume validation request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + Secrets map[string]string `protobuf:"bytes,5,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ValidateVolumeCapabilitiesRequest) Reset() { *m = ValidateVolumeCapabilitiesRequest{} } func (m *ValidateVolumeCapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*ValidateVolumeCapabilitiesRequest) ProtoMessage() {} func (*ValidateVolumeCapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{21} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{21} } func (m *ValidateVolumeCapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValidateVolumeCapabilitiesRequest.Unmarshal(m, b) @@ -1994,6 +2122,13 @@ func (m *ValidateVolumeCapabilitiesRequest) GetVolumeId() string { return "" } +func (m *ValidateVolumeCapabilitiesRequest) GetVolumeContext() map[string]string { + if m != nil { + return m.VolumeContext + } + return nil +} + func (m *ValidateVolumeCapabilitiesRequest) GetVolumeCapabilities() []*VolumeCapability { if m != nil { return m.VolumeCapabilities @@ -2001,28 +2136,34 @@ func (m *ValidateVolumeCapabilitiesRequest) GetVolumeCapabilities() []*VolumeCap return nil } -func (m *ValidateVolumeCapabilitiesRequest) GetVolumeAttributes() map[string]string { +func (m *ValidateVolumeCapabilitiesRequest) GetParameters() map[string]string { if m != nil { - return m.VolumeAttributes + return m.Parameters } return nil } -func (m *ValidateVolumeCapabilitiesRequest) GetAccessibleTopology() []*Topology { +func (m *ValidateVolumeCapabilitiesRequest) GetSecrets() map[string]string { if m != nil { - return m.AccessibleTopology + return m.Secrets } return nil } type ValidateVolumeCapabilitiesResponse struct { - // True if the Plugin supports the specified capabilities for the - // given volume. This field is REQUIRED. - Supported bool `protobuf:"varint,1,opt,name=supported" json:"supported,omitempty"` - // Message to the CO if `supported` above is false. This field is + // Confirmed indicates to the CO the set of capabilities that the + // plugin has validated. This field SHALL only be set to a non-empty + // value for successful validation responses. + // For successful validation responses, the CO SHALL compare the + // fields of this message to the originally requested capabilities in + // order to guard against an older plugin reporting "valid" for newer + // capability fields that it does not yet understand. + // This field is OPTIONAL. + Confirmed *ValidateVolumeCapabilitiesResponse_Confirmed `protobuf:"bytes,1,opt,name=confirmed,proto3" json:"confirmed,omitempty"` + // Message to the CO if `confirmed` above is empty. This field is // OPTIONAL. // An empty string is equal to an unspecified field value. - Message string `protobuf:"bytes,2,opt,name=message" json:"message,omitempty"` + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2032,7 +2173,7 @@ func (m *ValidateVolumeCapabilitiesResponse) Reset() { *m = ValidateVolu func (m *ValidateVolumeCapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*ValidateVolumeCapabilitiesResponse) ProtoMessage() {} func (*ValidateVolumeCapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{22} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{22} } func (m *ValidateVolumeCapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValidateVolumeCapabilitiesResponse.Unmarshal(m, b) @@ -2052,11 +2193,11 @@ func (m *ValidateVolumeCapabilitiesResponse) XXX_DiscardUnknown() { var xxx_messageInfo_ValidateVolumeCapabilitiesResponse proto.InternalMessageInfo -func (m *ValidateVolumeCapabilitiesResponse) GetSupported() bool { +func (m *ValidateVolumeCapabilitiesResponse) GetConfirmed() *ValidateVolumeCapabilitiesResponse_Confirmed { if m != nil { - return m.Supported + return m.Confirmed } - return false + return nil } func (m *ValidateVolumeCapabilitiesResponse) GetMessage() string { @@ -2066,6 +2207,70 @@ func (m *ValidateVolumeCapabilitiesResponse) GetMessage() string { return "" } +type ValidateVolumeCapabilitiesResponse_Confirmed struct { + // Volume context validated by the plugin. + // This field is OPTIONAL. + VolumeContext map[string]string `protobuf:"bytes,1,rep,name=volume_context,json=volumeContext,proto3" json:"volume_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Volume capabilities supported by the plugin. + // This field is REQUIRED. + VolumeCapabilities []*VolumeCapability `protobuf:"bytes,2,rep,name=volume_capabilities,json=volumeCapabilities,proto3" json:"volume_capabilities,omitempty"` + // The volume creation parameters validated by the plugin. + // This field is OPTIONAL. + Parameters map[string]string `protobuf:"bytes,3,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) Reset() { + *m = ValidateVolumeCapabilitiesResponse_Confirmed{} +} +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) String() string { + return proto.CompactTextString(m) +} +func (*ValidateVolumeCapabilitiesResponse_Confirmed) ProtoMessage() {} +func (*ValidateVolumeCapabilitiesResponse_Confirmed) Descriptor() ([]byte, []int) { + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{22, 0} +} +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Unmarshal(m, b) +} +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Marshal(b, m, deterministic) +} +func (dst *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Merge(dst, src) +} +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Size() int { + return xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Size(m) +} +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_DiscardUnknown() { + xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.DiscardUnknown(m) +} + +var xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed proto.InternalMessageInfo + +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) GetVolumeContext() map[string]string { + if m != nil { + return m.VolumeContext + } + return nil +} + +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) GetVolumeCapabilities() []*VolumeCapability { + if m != nil { + return m.VolumeCapabilities + } + return nil +} + +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) GetParameters() map[string]string { + if m != nil { + return m.Parameters + } + return nil +} + type ListVolumesRequest struct { // If specified (non-zero value), the Plugin MUST NOT return more // entries than this number in the response. If the actual number of @@ -2075,12 +2280,12 @@ type ListVolumesRequest struct { // not specified (zero value), it means there is no restriction on the // number of entries that can be returned. // The value of this field MUST NOT be negative. - MaxEntries int32 `protobuf:"varint,1,opt,name=max_entries,json=maxEntries" json:"max_entries,omitempty"` + MaxEntries int32 `protobuf:"varint,1,opt,name=max_entries,json=maxEntries,proto3" json:"max_entries,omitempty"` // A token to specify where to start paginating. Set this field to // `next_token` returned by a previous `ListVolumes` call to get the // next page of entries. This field is OPTIONAL. // An empty string is equal to an unspecified field value. - StartingToken string `protobuf:"bytes,2,opt,name=starting_token,json=startingToken" json:"starting_token,omitempty"` + StartingToken string `protobuf:"bytes,2,opt,name=starting_token,json=startingToken,proto3" json:"starting_token,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2090,7 +2295,7 @@ func (m *ListVolumesRequest) Reset() { *m = ListVolumesRequest{} } func (m *ListVolumesRequest) String() string { return proto.CompactTextString(m) } func (*ListVolumesRequest) ProtoMessage() {} func (*ListVolumesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{23} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{23} } func (m *ListVolumesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListVolumesRequest.Unmarshal(m, b) @@ -2125,14 +2330,14 @@ func (m *ListVolumesRequest) GetStartingToken() string { } type ListVolumesResponse struct { - Entries []*ListVolumesResponse_Entry `protobuf:"bytes,1,rep,name=entries" json:"entries,omitempty"` + Entries []*ListVolumesResponse_Entry `protobuf:"bytes,1,rep,name=entries,proto3" json:"entries,omitempty"` // This token allows you to get the next page of entries for // `ListVolumes` request. If the number of entries is larger than // `max_entries`, use the `next_token` as a value for the // `starting_token` field in the next `ListVolumes` request. This // field is OPTIONAL. // An empty string is equal to an unspecified field value. - NextToken string `protobuf:"bytes,2,opt,name=next_token,json=nextToken" json:"next_token,omitempty"` + NextToken string `protobuf:"bytes,2,opt,name=next_token,json=nextToken,proto3" json:"next_token,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2142,7 +2347,7 @@ func (m *ListVolumesResponse) Reset() { *m = ListVolumesResponse{} } func (m *ListVolumesResponse) String() string { return proto.CompactTextString(m) } func (*ListVolumesResponse) ProtoMessage() {} func (*ListVolumesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{24} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{24} } func (m *ListVolumesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListVolumesResponse.Unmarshal(m, b) @@ -2177,7 +2382,7 @@ func (m *ListVolumesResponse) GetNextToken() string { } type ListVolumesResponse_Entry struct { - Volume *Volume `protobuf:"bytes,1,opt,name=volume" json:"volume,omitempty"` + Volume *Volume `protobuf:"bytes,1,opt,name=volume,proto3" json:"volume,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2187,7 +2392,7 @@ func (m *ListVolumesResponse_Entry) Reset() { *m = ListVolumesResponse_E func (m *ListVolumesResponse_Entry) String() string { return proto.CompactTextString(m) } func (*ListVolumesResponse_Entry) ProtoMessage() {} func (*ListVolumesResponse_Entry) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{24, 0} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{24, 0} } func (m *ListVolumesResponse_Entry) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListVolumesResponse_Entry.Unmarshal(m, b) @@ -2220,19 +2425,19 @@ type GetCapacityRequest struct { // specified `volume_capabilities`. These are the same // `volume_capabilities` the CO will use in `CreateVolumeRequest`. // This field is OPTIONAL. - VolumeCapabilities []*VolumeCapability `protobuf:"bytes,1,rep,name=volume_capabilities,json=volumeCapabilities" json:"volume_capabilities,omitempty"` + VolumeCapabilities []*VolumeCapability `protobuf:"bytes,1,rep,name=volume_capabilities,json=volumeCapabilities,proto3" json:"volume_capabilities,omitempty"` // If specified, the Plugin SHALL report the capacity of the storage // that can be used to provision volumes with the given Plugin // specific `parameters`. These are the same `parameters` the CO will // use in `CreateVolumeRequest`. This field is OPTIONAL. - Parameters map[string]string `protobuf:"bytes,2,rep,name=parameters" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Parameters map[string]string `protobuf:"bytes,2,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // If specified, the Plugin SHALL report the capacity of the storage // that can be used to provision volumes that in the specified // `accessible_topology`. This is the same as the // `accessible_topology` the CO returns in a `CreateVolumeResponse`. // This field is OPTIONAL. This field SHALL NOT be set unless the - // plugin advertises the ACCESSIBILITY_CONSTRAINTS capability. - AccessibleTopology *Topology `protobuf:"bytes,3,opt,name=accessible_topology,json=accessibleTopology" json:"accessible_topology,omitempty"` + // plugin advertises the VOLUME_ACCESSIBILITY_CONSTRAINTS capability. + AccessibleTopology *Topology `protobuf:"bytes,3,opt,name=accessible_topology,json=accessibleTopology,proto3" json:"accessible_topology,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2242,7 +2447,7 @@ func (m *GetCapacityRequest) Reset() { *m = GetCapacityRequest{} } func (m *GetCapacityRequest) String() string { return proto.CompactTextString(m) } func (*GetCapacityRequest) ProtoMessage() {} func (*GetCapacityRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{25} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{25} } func (m *GetCapacityRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetCapacityRequest.Unmarshal(m, b) @@ -2290,7 +2495,7 @@ type GetCapacityResponse struct { // consideration when calculating the available capacity of the // storage. This field is REQUIRED. // The value of this field MUST NOT be negative. - AvailableCapacity int64 `protobuf:"varint,1,opt,name=available_capacity,json=availableCapacity" json:"available_capacity,omitempty"` + AvailableCapacity int64 `protobuf:"varint,1,opt,name=available_capacity,json=availableCapacity,proto3" json:"available_capacity,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2300,7 +2505,7 @@ func (m *GetCapacityResponse) Reset() { *m = GetCapacityResponse{} } func (m *GetCapacityResponse) String() string { return proto.CompactTextString(m) } func (*GetCapacityResponse) ProtoMessage() {} func (*GetCapacityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{26} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{26} } func (m *GetCapacityResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetCapacityResponse.Unmarshal(m, b) @@ -2337,7 +2542,7 @@ func (m *ControllerGetCapabilitiesRequest) Reset() { *m = ControllerGetC func (m *ControllerGetCapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*ControllerGetCapabilitiesRequest) ProtoMessage() {} func (*ControllerGetCapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{27} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{27} } func (m *ControllerGetCapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerGetCapabilitiesRequest.Unmarshal(m, b) @@ -2360,7 +2565,7 @@ var xxx_messageInfo_ControllerGetCapabilitiesRequest proto.InternalMessageInfo type ControllerGetCapabilitiesResponse struct { // All the capabilities that the controller service supports. This // field is OPTIONAL. - Capabilities []*ControllerServiceCapability `protobuf:"bytes,2,rep,name=capabilities" json:"capabilities,omitempty"` + Capabilities []*ControllerServiceCapability `protobuf:"bytes,1,rep,name=capabilities,proto3" json:"capabilities,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2370,7 +2575,7 @@ func (m *ControllerGetCapabilitiesResponse) Reset() { *m = ControllerGet func (m *ControllerGetCapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*ControllerGetCapabilitiesResponse) ProtoMessage() {} func (*ControllerGetCapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{28} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{28} } func (m *ControllerGetCapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerGetCapabilitiesResponse.Unmarshal(m, b) @@ -2411,7 +2616,7 @@ func (m *ControllerServiceCapability) Reset() { *m = ControllerServiceCa func (m *ControllerServiceCapability) String() string { return proto.CompactTextString(m) } func (*ControllerServiceCapability) ProtoMessage() {} func (*ControllerServiceCapability) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{29} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{29} } func (m *ControllerServiceCapability) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerServiceCapability.Unmarshal(m, b) @@ -2436,7 +2641,7 @@ type isControllerServiceCapability_Type interface { } type ControllerServiceCapability_Rpc struct { - Rpc *ControllerServiceCapability_RPC `protobuf:"bytes,1,opt,name=rpc,oneof"` + Rpc *ControllerServiceCapability_RPC `protobuf:"bytes,1,opt,name=rpc,proto3,oneof"` } func (*ControllerServiceCapability_Rpc) isControllerServiceCapability_Type() {} @@ -2511,7 +2716,7 @@ func _ControllerServiceCapability_OneofSizer(msg proto.Message) (n int) { } type ControllerServiceCapability_RPC struct { - Type ControllerServiceCapability_RPC_Type `protobuf:"varint,1,opt,name=type,enum=csi.v0.ControllerServiceCapability_RPC_Type" json:"type,omitempty"` + Type ControllerServiceCapability_RPC_Type `protobuf:"varint,1,opt,name=type,proto3,enum=csi.v1.ControllerServiceCapability_RPC_Type" json:"type,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2521,7 +2726,7 @@ func (m *ControllerServiceCapability_RPC) Reset() { *m = ControllerServi func (m *ControllerServiceCapability_RPC) String() string { return proto.CompactTextString(m) } func (*ControllerServiceCapability_RPC) ProtoMessage() {} func (*ControllerServiceCapability_RPC) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{29, 0} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{29, 0} } func (m *ControllerServiceCapability_RPC) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerServiceCapability_RPC.Unmarshal(m, b) @@ -2551,14 +2756,18 @@ func (m *ControllerServiceCapability_RPC) GetType() ControllerServiceCapability_ type CreateSnapshotRequest struct { // The ID of the source volume to be snapshotted. // This field is REQUIRED. - SourceVolumeId string `protobuf:"bytes,1,opt,name=source_volume_id,json=sourceVolumeId" json:"source_volume_id,omitempty"` + SourceVolumeId string `protobuf:"bytes,1,opt,name=source_volume_id,json=sourceVolumeId,proto3" json:"source_volume_id,omitempty"` // The suggested name for the snapshot. This field is REQUIRED for // idempotency. - Name string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"` + // Any Unicode string that conforms to the length limit is allowed + // except those containing the following banned characters: + // U+0000-U+0008, U+000B, U+000C, U+000E-U+001F, U+007F-U+009F. + // (These are control characters other than commonly used whitespace.) + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` // Secrets required by plugin to complete snapshot creation request. // This field is OPTIONAL. Refer to the `Secrets Requirements` // section on how to use this field. - CreateSnapshotSecrets map[string]string `protobuf:"bytes,3,rep,name=create_snapshot_secrets,json=createSnapshotSecrets" json:"create_snapshot_secrets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Secrets map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // Plugin specific parameters passed in as opaque key-value pairs. // This field is OPTIONAL. The Plugin is responsible for parsing and // validating these parameters. COs will treat these as opaque. @@ -2569,7 +2778,7 @@ type CreateSnapshotRequest struct { // - Specify if the snapshot should be replicated to some place. // - Specify primary or secondary for replication systems that // support snapshotting only on primary. - Parameters map[string]string `protobuf:"bytes,4,rep,name=parameters" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Parameters map[string]string `protobuf:"bytes,4,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2579,7 +2788,7 @@ func (m *CreateSnapshotRequest) Reset() { *m = CreateSnapshotRequest{} } func (m *CreateSnapshotRequest) String() string { return proto.CompactTextString(m) } func (*CreateSnapshotRequest) ProtoMessage() {} func (*CreateSnapshotRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{30} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{30} } func (m *CreateSnapshotRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateSnapshotRequest.Unmarshal(m, b) @@ -2613,9 +2822,9 @@ func (m *CreateSnapshotRequest) GetName() string { return "" } -func (m *CreateSnapshotRequest) GetCreateSnapshotSecrets() map[string]string { +func (m *CreateSnapshotRequest) GetSecrets() map[string]string { if m != nil { - return m.CreateSnapshotSecrets + return m.Secrets } return nil } @@ -2631,7 +2840,7 @@ type CreateSnapshotResponse struct { // Contains all attributes of the newly created snapshot that are // relevant to the CO along with information required by the Plugin // to uniquely identify the snapshot. This field is REQUIRED. - Snapshot *Snapshot `protobuf:"bytes,1,opt,name=snapshot" json:"snapshot,omitempty"` + Snapshot *Snapshot `protobuf:"bytes,1,opt,name=snapshot,proto3" json:"snapshot,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2641,7 +2850,7 @@ func (m *CreateSnapshotResponse) Reset() { *m = CreateSnapshotResponse{} func (m *CreateSnapshotResponse) String() string { return proto.CompactTextString(m) } func (*CreateSnapshotResponse) ProtoMessage() {} func (*CreateSnapshotResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{31} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{31} } func (m *CreateSnapshotResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateSnapshotResponse.Unmarshal(m, b) @@ -2668,7 +2877,7 @@ func (m *CreateSnapshotResponse) GetSnapshot() *Snapshot { return nil } -// The information about a provisioned snapshot. +// Information about a specific snapshot. type Snapshot struct { // This is the complete size of the snapshot in bytes. The purpose of // this field is to give CO guidance on how much space is needed to @@ -2677,34 +2886,38 @@ type Snapshot struct { // OPTIONAL. If this field is not set, it indicates that this size is // unknown. The value of this field MUST NOT be negative and a size of // zero means it is unspecified. - SizeBytes int64 `protobuf:"varint,1,opt,name=size_bytes,json=sizeBytes" json:"size_bytes,omitempty"` - // Uniquely identifies a snapshot and is generated by the plugin. It - // will not change over time. This field is REQUIRED. The identity - // information will be used by the CO in subsequent calls to refer to - // the provisioned snapshot. - Id string `protobuf:"bytes,2,opt,name=id" json:"id,omitempty"` + SizeBytes int64 `protobuf:"varint,1,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + // The identifier for this snapshot, generated by the plugin. + // This field is REQUIRED. + // This field MUST contain enough information to uniquely identify + // this specific snapshot vs all other snapshots supported by this + // plugin. + // This field SHALL be used by the CO in subsequent calls to refer to + // this snapshot. + // The SP is NOT responsible for global uniqueness of snapshot_id + // across multiple SPs. + SnapshotId string `protobuf:"bytes,2,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` // Identity information for the source volume. Note that creating a // snapshot from a snapshot is not supported here so the source has to // be a volume. This field is REQUIRED. - SourceVolumeId string `protobuf:"bytes,3,opt,name=source_volume_id,json=sourceVolumeId" json:"source_volume_id,omitempty"` + SourceVolumeId string `protobuf:"bytes,3,opt,name=source_volume_id,json=sourceVolumeId,proto3" json:"source_volume_id,omitempty"` // Timestamp when the point-in-time snapshot is taken on the storage - // system. The format of this field should be a Unix nanoseconds time - // encoded as an int64. On Unix, the command `date +%s%N` returns the - // current time in nanoseconds since 1970-01-01 00:00:00 UTC. This - // field is REQUIRED. - CreatedAt int64 `protobuf:"varint,4,opt,name=created_at,json=createdAt" json:"created_at,omitempty"` - // The status of a snapshot. - Status *SnapshotStatus `protobuf:"bytes,5,opt,name=status" json:"status,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // system. This field is REQUIRED. + CreationTime *timestamp.Timestamp `protobuf:"bytes,4,opt,name=creation_time,json=creationTime,proto3" json:"creation_time,omitempty"` + // Indicates if a snapshot is ready to use as a + // `volume_content_source` in a `CreateVolumeRequest`. The default + // value is false. This field is REQUIRED. + ReadyToUse bool `protobuf:"varint,5,opt,name=ready_to_use,json=readyToUse,proto3" json:"ready_to_use,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Snapshot) Reset() { *m = Snapshot{} } func (m *Snapshot) String() string { return proto.CompactTextString(m) } func (*Snapshot) ProtoMessage() {} func (*Snapshot) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{32} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{32} } func (m *Snapshot) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Snapshot.Unmarshal(m, b) @@ -2731,9 +2944,9 @@ func (m *Snapshot) GetSizeBytes() int64 { return 0 } -func (m *Snapshot) GetId() string { +func (m *Snapshot) GetSnapshotId() string { if m != nil { - return m.Id + return m.SnapshotId } return "" } @@ -2745,88 +2958,38 @@ func (m *Snapshot) GetSourceVolumeId() string { return "" } -func (m *Snapshot) GetCreatedAt() int64 { +func (m *Snapshot) GetCreationTime() *timestamp.Timestamp { if m != nil { - return m.CreatedAt - } - return 0 -} - -func (m *Snapshot) GetStatus() *SnapshotStatus { - if m != nil { - return m.Status + return m.CreationTime } return nil } -// The status of a snapshot. -type SnapshotStatus struct { - // This field is REQUIRED. - Type SnapshotStatus_Type `protobuf:"varint,1,opt,name=type,enum=csi.v0.SnapshotStatus_Type" json:"type,omitempty"` - // Additional information to describe why a snapshot ended up in the - // `ERROR_UPLOADING` status. This field is OPTIONAL. - Details string `protobuf:"bytes,2,opt,name=details" json:"details,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SnapshotStatus) Reset() { *m = SnapshotStatus{} } -func (m *SnapshotStatus) String() string { return proto.CompactTextString(m) } -func (*SnapshotStatus) ProtoMessage() {} -func (*SnapshotStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{33} -} -func (m *SnapshotStatus) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SnapshotStatus.Unmarshal(m, b) -} -func (m *SnapshotStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SnapshotStatus.Marshal(b, m, deterministic) -} -func (dst *SnapshotStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_SnapshotStatus.Merge(dst, src) -} -func (m *SnapshotStatus) XXX_Size() int { - return xxx_messageInfo_SnapshotStatus.Size(m) -} -func (m *SnapshotStatus) XXX_DiscardUnknown() { - xxx_messageInfo_SnapshotStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_SnapshotStatus proto.InternalMessageInfo - -func (m *SnapshotStatus) GetType() SnapshotStatus_Type { +func (m *Snapshot) GetReadyToUse() bool { if m != nil { - return m.Type + return m.ReadyToUse } - return SnapshotStatus_UNKNOWN -} - -func (m *SnapshotStatus) GetDetails() string { - if m != nil { - return m.Details - } - return "" + return false } type DeleteSnapshotRequest struct { // The ID of the snapshot to be deleted. // This field is REQUIRED. - SnapshotId string `protobuf:"bytes,1,opt,name=snapshot_id,json=snapshotId" json:"snapshot_id,omitempty"` + SnapshotId string `protobuf:"bytes,1,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` // Secrets required by plugin to complete snapshot deletion request. // This field is OPTIONAL. Refer to the `Secrets Requirements` // section on how to use this field. - DeleteSnapshotSecrets map[string]string `protobuf:"bytes,2,rep,name=delete_snapshot_secrets,json=deleteSnapshotSecrets" json:"delete_snapshot_secrets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Secrets map[string]string `protobuf:"bytes,2,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *DeleteSnapshotRequest) Reset() { *m = DeleteSnapshotRequest{} } func (m *DeleteSnapshotRequest) String() string { return proto.CompactTextString(m) } func (*DeleteSnapshotRequest) ProtoMessage() {} func (*DeleteSnapshotRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{34} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{33} } func (m *DeleteSnapshotRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteSnapshotRequest.Unmarshal(m, b) @@ -2853,9 +3016,9 @@ func (m *DeleteSnapshotRequest) GetSnapshotId() string { return "" } -func (m *DeleteSnapshotRequest) GetDeleteSnapshotSecrets() map[string]string { +func (m *DeleteSnapshotRequest) GetSecrets() map[string]string { if m != nil { - return m.DeleteSnapshotSecrets + return m.Secrets } return nil } @@ -2870,7 +3033,7 @@ func (m *DeleteSnapshotResponse) Reset() { *m = DeleteSnapshotResponse{} func (m *DeleteSnapshotResponse) String() string { return proto.CompactTextString(m) } func (*DeleteSnapshotResponse) ProtoMessage() {} func (*DeleteSnapshotResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{35} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{34} } func (m *DeleteSnapshotResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteSnapshotResponse.Unmarshal(m, b) @@ -2901,20 +3064,21 @@ type ListSnapshotsRequest struct { // not specified (zero value), it means there is no restriction on the // number of entries that can be returned. // The value of this field MUST NOT be negative. - MaxEntries int32 `protobuf:"varint,1,opt,name=max_entries,json=maxEntries" json:"max_entries,omitempty"` + MaxEntries int32 `protobuf:"varint,1,opt,name=max_entries,json=maxEntries,proto3" json:"max_entries,omitempty"` // A token to specify where to start paginating. Set this field to // `next_token` returned by a previous `ListSnapshots` call to get the // next page of entries. This field is OPTIONAL. // An empty string is equal to an unspecified field value. - StartingToken string `protobuf:"bytes,2,opt,name=starting_token,json=startingToken" json:"starting_token,omitempty"` + StartingToken string `protobuf:"bytes,2,opt,name=starting_token,json=startingToken,proto3" json:"starting_token,omitempty"` // Identity information for the source volume. This field is OPTIONAL. // It can be used to list snapshots by volume. - SourceVolumeId string `protobuf:"bytes,3,opt,name=source_volume_id,json=sourceVolumeId" json:"source_volume_id,omitempty"` + SourceVolumeId string `protobuf:"bytes,3,opt,name=source_volume_id,json=sourceVolumeId,proto3" json:"source_volume_id,omitempty"` // Identity information for a specific snapshot. This field is // OPTIONAL. It can be used to list only a specific snapshot. // ListSnapshots will return with current snapshot information - // and will not block if the snapshot is being uploaded. - SnapshotId string `protobuf:"bytes,4,opt,name=snapshot_id,json=snapshotId" json:"snapshot_id,omitempty"` + // and will not block if the snapshot is being processed after + // it is cut. + SnapshotId string `protobuf:"bytes,4,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2924,7 +3088,7 @@ func (m *ListSnapshotsRequest) Reset() { *m = ListSnapshotsRequest{} } func (m *ListSnapshotsRequest) String() string { return proto.CompactTextString(m) } func (*ListSnapshotsRequest) ProtoMessage() {} func (*ListSnapshotsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{36} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{35} } func (m *ListSnapshotsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListSnapshotsRequest.Unmarshal(m, b) @@ -2973,14 +3137,14 @@ func (m *ListSnapshotsRequest) GetSnapshotId() string { } type ListSnapshotsResponse struct { - Entries []*ListSnapshotsResponse_Entry `protobuf:"bytes,1,rep,name=entries" json:"entries,omitempty"` + Entries []*ListSnapshotsResponse_Entry `protobuf:"bytes,1,rep,name=entries,proto3" json:"entries,omitempty"` // This token allows you to get the next page of entries for // `ListSnapshots` request. If the number of entries is larger than // `max_entries`, use the `next_token` as a value for the // `starting_token` field in the next `ListSnapshots` request. This // field is OPTIONAL. // An empty string is equal to an unspecified field value. - NextToken string `protobuf:"bytes,2,opt,name=next_token,json=nextToken" json:"next_token,omitempty"` + NextToken string `protobuf:"bytes,2,opt,name=next_token,json=nextToken,proto3" json:"next_token,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2990,7 +3154,7 @@ func (m *ListSnapshotsResponse) Reset() { *m = ListSnapshotsResponse{} } func (m *ListSnapshotsResponse) String() string { return proto.CompactTextString(m) } func (*ListSnapshotsResponse) ProtoMessage() {} func (*ListSnapshotsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{37} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{36} } func (m *ListSnapshotsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListSnapshotsResponse.Unmarshal(m, b) @@ -3025,7 +3189,7 @@ func (m *ListSnapshotsResponse) GetNextToken() string { } type ListSnapshotsResponse_Entry struct { - Snapshot *Snapshot `protobuf:"bytes,1,opt,name=snapshot" json:"snapshot,omitempty"` + Snapshot *Snapshot `protobuf:"bytes,1,opt,name=snapshot,proto3" json:"snapshot,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3035,7 +3199,7 @@ func (m *ListSnapshotsResponse_Entry) Reset() { *m = ListSnapshotsRespon func (m *ListSnapshotsResponse_Entry) String() string { return proto.CompactTextString(m) } func (*ListSnapshotsResponse_Entry) ProtoMessage() {} func (*ListSnapshotsResponse_Entry) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{37, 0} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{36, 0} } func (m *ListSnapshotsResponse_Entry) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListSnapshotsResponse_Entry.Unmarshal(m, b) @@ -3064,30 +3228,36 @@ func (m *ListSnapshotsResponse_Entry) GetSnapshot() *Snapshot { type NodeStageVolumeRequest struct { // The ID of the volume to publish. This field is REQUIRED. - VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` // The CO SHALL set this field to the value returned by // `ControllerPublishVolume` if the corresponding Controller Plugin // has `PUBLISH_UNPUBLISH_VOLUME` controller capability, and SHALL be // left unset if the corresponding Controller Plugin does not have // this capability. This is an OPTIONAL field. - PublishInfo map[string]string `protobuf:"bytes,2,rep,name=publish_info,json=publishInfo" json:"publish_info,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // The path to which the volume will be published. It MUST be an + PublishContext map[string]string `protobuf:"bytes,2,rep,name=publish_context,json=publishContext,proto3" json:"publish_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // The path to which the volume MAY be staged. It MUST be an // absolute path in the root filesystem of the process serving this - // request. The CO SHALL ensure that there is only one - // staging_target_path per volume. + // request, and MUST be a directory. The CO SHALL ensure that there + // is only one `staging_target_path` per volume. The CO SHALL ensure + // that the path is directory and that the process serving the + // request has `read` and `write` permission to that directory. The + // CO SHALL be responsible for creating the directory if it does not + // exist. // This is a REQUIRED field. - StagingTargetPath string `protobuf:"bytes,3,opt,name=staging_target_path,json=stagingTargetPath" json:"staging_target_path,omitempty"` - // The capability of the volume the CO expects the volume to have. + StagingTargetPath string `protobuf:"bytes,3,opt,name=staging_target_path,json=stagingTargetPath,proto3" json:"staging_target_path,omitempty"` + // Volume capability describing how the CO intends to use this volume. + // SP MUST ensure the CO can use the staged volume as described. + // Otherwise SP MUST return the appropriate gRPC error code. // This is a REQUIRED field. - VolumeCapability *VolumeCapability `protobuf:"bytes,4,opt,name=volume_capability,json=volumeCapability" json:"volume_capability,omitempty"` + VolumeCapability *VolumeCapability `protobuf:"bytes,4,opt,name=volume_capability,json=volumeCapability,proto3" json:"volume_capability,omitempty"` // Secrets required by plugin to complete node stage volume request. // This field is OPTIONAL. Refer to the `Secrets Requirements` // section on how to use this field. - NodeStageSecrets map[string]string `protobuf:"bytes,5,rep,name=node_stage_secrets,json=nodeStageSecrets" json:"node_stage_secrets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // Attributes of the volume to publish. This field is OPTIONAL and - // MUST match the attributes of the `Volume` identified by - // `volume_id`. - VolumeAttributes map[string]string `protobuf:"bytes,6,rep,name=volume_attributes,json=volumeAttributes" json:"volume_attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Secrets map[string]string `protobuf:"bytes,5,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Volume context as returned by CO in CreateVolumeRequest. This field + // is OPTIONAL and MUST match the volume_context of the volume + // identified by `volume_id`. + VolumeContext map[string]string `protobuf:"bytes,6,rep,name=volume_context,json=volumeContext,proto3" json:"volume_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3097,7 +3267,7 @@ func (m *NodeStageVolumeRequest) Reset() { *m = NodeStageVolumeRequest{} func (m *NodeStageVolumeRequest) String() string { return proto.CompactTextString(m) } func (*NodeStageVolumeRequest) ProtoMessage() {} func (*NodeStageVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{38} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{37} } func (m *NodeStageVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeStageVolumeRequest.Unmarshal(m, b) @@ -3124,9 +3294,9 @@ func (m *NodeStageVolumeRequest) GetVolumeId() string { return "" } -func (m *NodeStageVolumeRequest) GetPublishInfo() map[string]string { +func (m *NodeStageVolumeRequest) GetPublishContext() map[string]string { if m != nil { - return m.PublishInfo + return m.PublishContext } return nil } @@ -3145,16 +3315,16 @@ func (m *NodeStageVolumeRequest) GetVolumeCapability() *VolumeCapability { return nil } -func (m *NodeStageVolumeRequest) GetNodeStageSecrets() map[string]string { +func (m *NodeStageVolumeRequest) GetSecrets() map[string]string { if m != nil { - return m.NodeStageSecrets + return m.Secrets } return nil } -func (m *NodeStageVolumeRequest) GetVolumeAttributes() map[string]string { +func (m *NodeStageVolumeRequest) GetVolumeContext() map[string]string { if m != nil { - return m.VolumeAttributes + return m.VolumeContext } return nil } @@ -3169,7 +3339,7 @@ func (m *NodeStageVolumeResponse) Reset() { *m = NodeStageVolumeResponse func (m *NodeStageVolumeResponse) String() string { return proto.CompactTextString(m) } func (*NodeStageVolumeResponse) ProtoMessage() {} func (*NodeStageVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{39} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{38} } func (m *NodeStageVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeStageVolumeResponse.Unmarshal(m, b) @@ -3191,11 +3361,11 @@ var xxx_messageInfo_NodeStageVolumeResponse proto.InternalMessageInfo type NodeUnstageVolumeRequest struct { // The ID of the volume. This field is REQUIRED. - VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` - // The path at which the volume was published. It MUST be an absolute + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // The path at which the volume was staged. It MUST be an absolute // path in the root filesystem of the process serving this request. // This is a REQUIRED field. - StagingTargetPath string `protobuf:"bytes,2,opt,name=staging_target_path,json=stagingTargetPath" json:"staging_target_path,omitempty"` + StagingTargetPath string `protobuf:"bytes,2,opt,name=staging_target_path,json=stagingTargetPath,proto3" json:"staging_target_path,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3205,7 +3375,7 @@ func (m *NodeUnstageVolumeRequest) Reset() { *m = NodeUnstageVolumeReque func (m *NodeUnstageVolumeRequest) String() string { return proto.CompactTextString(m) } func (*NodeUnstageVolumeRequest) ProtoMessage() {} func (*NodeUnstageVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{40} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{39} } func (m *NodeUnstageVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeUnstageVolumeRequest.Unmarshal(m, b) @@ -3249,7 +3419,7 @@ func (m *NodeUnstageVolumeResponse) Reset() { *m = NodeUnstageVolumeResp func (m *NodeUnstageVolumeResponse) String() string { return proto.CompactTextString(m) } func (*NodeUnstageVolumeResponse) ProtoMessage() {} func (*NodeUnstageVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{41} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{40} } func (m *NodeUnstageVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeUnstageVolumeResponse.Unmarshal(m, b) @@ -3271,41 +3441,49 @@ var xxx_messageInfo_NodeUnstageVolumeResponse proto.InternalMessageInfo type NodePublishVolumeRequest struct { // The ID of the volume to publish. This field is REQUIRED. - VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` // The CO SHALL set this field to the value returned by // `ControllerPublishVolume` if the corresponding Controller Plugin // has `PUBLISH_UNPUBLISH_VOLUME` controller capability, and SHALL be // left unset if the corresponding Controller Plugin does not have // this capability. This is an OPTIONAL field. - PublishInfo map[string]string `protobuf:"bytes,2,rep,name=publish_info,json=publishInfo" json:"publish_info,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // The path to which the device was mounted by `NodeStageVolume`. + PublishContext map[string]string `protobuf:"bytes,2,rep,name=publish_context,json=publishContext,proto3" json:"publish_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // The path to which the volume was staged by `NodeStageVolume`. // It MUST be an absolute path in the root filesystem of the process // serving this request. // It MUST be set if the Node Plugin implements the // `STAGE_UNSTAGE_VOLUME` node capability. // This is an OPTIONAL field. - StagingTargetPath string `protobuf:"bytes,3,opt,name=staging_target_path,json=stagingTargetPath" json:"staging_target_path,omitempty"` + StagingTargetPath string `protobuf:"bytes,3,opt,name=staging_target_path,json=stagingTargetPath,proto3" json:"staging_target_path,omitempty"` // The path to which the volume will be published. It MUST be an // absolute path in the root filesystem of the process serving this // request. The CO SHALL ensure uniqueness of target_path per volume. - // The CO SHALL ensure that the path exists, and that the process - // serving the request has `read` and `write` permissions to the path. + // The CO SHALL ensure that the parent directory of this path exists + // and that the process serving the request has `read` and `write` + // permissions to that parent directory. + // For volumes with an access type of block, the SP SHALL place the + // block device at target_path. + // For volumes with an access type of mount, the SP SHALL place the + // mounted directory at target_path. + // Creation of target_path is the responsibility of the SP. // This is a REQUIRED field. - TargetPath string `protobuf:"bytes,4,opt,name=target_path,json=targetPath" json:"target_path,omitempty"` - // The capability of the volume the CO expects the volume to have. + TargetPath string `protobuf:"bytes,4,opt,name=target_path,json=targetPath,proto3" json:"target_path,omitempty"` + // Volume capability describing how the CO intends to use this volume. + // SP MUST ensure the CO can use the published volume as described. + // Otherwise SP MUST return the appropriate gRPC error code. // This is a REQUIRED field. - VolumeCapability *VolumeCapability `protobuf:"bytes,5,opt,name=volume_capability,json=volumeCapability" json:"volume_capability,omitempty"` - // Whether to publish the volume in readonly mode. This field is - // REQUIRED. - Readonly bool `protobuf:"varint,6,opt,name=readonly" json:"readonly,omitempty"` + VolumeCapability *VolumeCapability `protobuf:"bytes,5,opt,name=volume_capability,json=volumeCapability,proto3" json:"volume_capability,omitempty"` + // Indicates SP MUST publish the volume in readonly mode. + // This field is REQUIRED. + Readonly bool `protobuf:"varint,6,opt,name=readonly,proto3" json:"readonly,omitempty"` // Secrets required by plugin to complete node publish volume request. // This field is OPTIONAL. Refer to the `Secrets Requirements` // section on how to use this field. - NodePublishSecrets map[string]string `protobuf:"bytes,7,rep,name=node_publish_secrets,json=nodePublishSecrets" json:"node_publish_secrets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // Attributes of the volume to publish. This field is OPTIONAL and - // MUST match the attributes of the Volume identified by - // `volume_id`. - VolumeAttributes map[string]string `protobuf:"bytes,8,rep,name=volume_attributes,json=volumeAttributes" json:"volume_attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Secrets map[string]string `protobuf:"bytes,7,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Volume context as returned by CO in CreateVolumeRequest. This field + // is OPTIONAL and MUST match the volume_context of the volume + // identified by `volume_id`. + VolumeContext map[string]string `protobuf:"bytes,8,rep,name=volume_context,json=volumeContext,proto3" json:"volume_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3315,7 +3493,7 @@ func (m *NodePublishVolumeRequest) Reset() { *m = NodePublishVolumeReque func (m *NodePublishVolumeRequest) String() string { return proto.CompactTextString(m) } func (*NodePublishVolumeRequest) ProtoMessage() {} func (*NodePublishVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{42} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{41} } func (m *NodePublishVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodePublishVolumeRequest.Unmarshal(m, b) @@ -3342,9 +3520,9 @@ func (m *NodePublishVolumeRequest) GetVolumeId() string { return "" } -func (m *NodePublishVolumeRequest) GetPublishInfo() map[string]string { +func (m *NodePublishVolumeRequest) GetPublishContext() map[string]string { if m != nil { - return m.PublishInfo + return m.PublishContext } return nil } @@ -3377,16 +3555,16 @@ func (m *NodePublishVolumeRequest) GetReadonly() bool { return false } -func (m *NodePublishVolumeRequest) GetNodePublishSecrets() map[string]string { +func (m *NodePublishVolumeRequest) GetSecrets() map[string]string { if m != nil { - return m.NodePublishSecrets + return m.Secrets } return nil } -func (m *NodePublishVolumeRequest) GetVolumeAttributes() map[string]string { +func (m *NodePublishVolumeRequest) GetVolumeContext() map[string]string { if m != nil { - return m.VolumeAttributes + return m.VolumeContext } return nil } @@ -3401,7 +3579,7 @@ func (m *NodePublishVolumeResponse) Reset() { *m = NodePublishVolumeResp func (m *NodePublishVolumeResponse) String() string { return proto.CompactTextString(m) } func (*NodePublishVolumeResponse) ProtoMessage() {} func (*NodePublishVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{43} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{42} } func (m *NodePublishVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodePublishVolumeResponse.Unmarshal(m, b) @@ -3423,11 +3601,12 @@ var xxx_messageInfo_NodePublishVolumeResponse proto.InternalMessageInfo type NodeUnpublishVolumeRequest struct { // The ID of the volume. This field is REQUIRED. - VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` // The path at which the volume was published. It MUST be an absolute // path in the root filesystem of the process serving this request. + // The SP MUST delete the file or directory it created at this path. // This is a REQUIRED field. - TargetPath string `protobuf:"bytes,2,opt,name=target_path,json=targetPath" json:"target_path,omitempty"` + TargetPath string `protobuf:"bytes,2,opt,name=target_path,json=targetPath,proto3" json:"target_path,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3437,7 +3616,7 @@ func (m *NodeUnpublishVolumeRequest) Reset() { *m = NodeUnpublishVolumeR func (m *NodeUnpublishVolumeRequest) String() string { return proto.CompactTextString(m) } func (*NodeUnpublishVolumeRequest) ProtoMessage() {} func (*NodeUnpublishVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{44} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{43} } func (m *NodeUnpublishVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeUnpublishVolumeRequest.Unmarshal(m, b) @@ -3481,7 +3660,7 @@ func (m *NodeUnpublishVolumeResponse) Reset() { *m = NodeUnpublishVolume func (m *NodeUnpublishVolumeResponse) String() string { return proto.CompactTextString(m) } func (*NodeUnpublishVolumeResponse) ProtoMessage() {} func (*NodeUnpublishVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{45} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{44} } func (m *NodeUnpublishVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeUnpublishVolumeResponse.Unmarshal(m, b) @@ -3501,77 +3680,166 @@ func (m *NodeUnpublishVolumeResponse) XXX_DiscardUnknown() { var xxx_messageInfo_NodeUnpublishVolumeResponse proto.InternalMessageInfo -type NodeGetIdRequest struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *NodeGetIdRequest) Reset() { *m = NodeGetIdRequest{} } -func (m *NodeGetIdRequest) String() string { return proto.CompactTextString(m) } -func (*NodeGetIdRequest) ProtoMessage() {} -func (*NodeGetIdRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{46} -} -func (m *NodeGetIdRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_NodeGetIdRequest.Unmarshal(m, b) -} -func (m *NodeGetIdRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_NodeGetIdRequest.Marshal(b, m, deterministic) -} -func (dst *NodeGetIdRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeGetIdRequest.Merge(dst, src) -} -func (m *NodeGetIdRequest) XXX_Size() int { - return xxx_messageInfo_NodeGetIdRequest.Size(m) -} -func (m *NodeGetIdRequest) XXX_DiscardUnknown() { - xxx_messageInfo_NodeGetIdRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_NodeGetIdRequest proto.InternalMessageInfo - -type NodeGetIdResponse struct { - // The ID of the node as understood by the SP which SHALL be used by - // CO in subsequent `ControllerPublishVolume`. +type NodeGetVolumeStatsRequest struct { + // The ID of the volume. This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // It can be any valid path where volume was previously + // staged or published. + // It MUST be an absolute path in the root filesystem of + // the process serving this request. // This is a REQUIRED field. - NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId" json:"node_id,omitempty"` + VolumePath string `protobuf:"bytes,2,opt,name=volume_path,json=volumePath,proto3" json:"volume_path,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *NodeGetIdResponse) Reset() { *m = NodeGetIdResponse{} } -func (m *NodeGetIdResponse) String() string { return proto.CompactTextString(m) } -func (*NodeGetIdResponse) ProtoMessage() {} -func (*NodeGetIdResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{47} +func (m *NodeGetVolumeStatsRequest) Reset() { *m = NodeGetVolumeStatsRequest{} } +func (m *NodeGetVolumeStatsRequest) String() string { return proto.CompactTextString(m) } +func (*NodeGetVolumeStatsRequest) ProtoMessage() {} +func (*NodeGetVolumeStatsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{45} } -func (m *NodeGetIdResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_NodeGetIdResponse.Unmarshal(m, b) +func (m *NodeGetVolumeStatsRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeGetVolumeStatsRequest.Unmarshal(m, b) } -func (m *NodeGetIdResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_NodeGetIdResponse.Marshal(b, m, deterministic) +func (m *NodeGetVolumeStatsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeGetVolumeStatsRequest.Marshal(b, m, deterministic) } -func (dst *NodeGetIdResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeGetIdResponse.Merge(dst, src) +func (dst *NodeGetVolumeStatsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetVolumeStatsRequest.Merge(dst, src) } -func (m *NodeGetIdResponse) XXX_Size() int { - return xxx_messageInfo_NodeGetIdResponse.Size(m) +func (m *NodeGetVolumeStatsRequest) XXX_Size() int { + return xxx_messageInfo_NodeGetVolumeStatsRequest.Size(m) } -func (m *NodeGetIdResponse) XXX_DiscardUnknown() { - xxx_messageInfo_NodeGetIdResponse.DiscardUnknown(m) +func (m *NodeGetVolumeStatsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_NodeGetVolumeStatsRequest.DiscardUnknown(m) } -var xxx_messageInfo_NodeGetIdResponse proto.InternalMessageInfo +var xxx_messageInfo_NodeGetVolumeStatsRequest proto.InternalMessageInfo -func (m *NodeGetIdResponse) GetNodeId() string { +func (m *NodeGetVolumeStatsRequest) GetVolumeId() string { if m != nil { - return m.NodeId + return m.VolumeId } return "" } +func (m *NodeGetVolumeStatsRequest) GetVolumePath() string { + if m != nil { + return m.VolumePath + } + return "" +} + +type NodeGetVolumeStatsResponse struct { + // This field is OPTIONAL. + Usage []*VolumeUsage `protobuf:"bytes,1,rep,name=usage,proto3" json:"usage,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeGetVolumeStatsResponse) Reset() { *m = NodeGetVolumeStatsResponse{} } +func (m *NodeGetVolumeStatsResponse) String() string { return proto.CompactTextString(m) } +func (*NodeGetVolumeStatsResponse) ProtoMessage() {} +func (*NodeGetVolumeStatsResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{46} +} +func (m *NodeGetVolumeStatsResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeGetVolumeStatsResponse.Unmarshal(m, b) +} +func (m *NodeGetVolumeStatsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeGetVolumeStatsResponse.Marshal(b, m, deterministic) +} +func (dst *NodeGetVolumeStatsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetVolumeStatsResponse.Merge(dst, src) +} +func (m *NodeGetVolumeStatsResponse) XXX_Size() int { + return xxx_messageInfo_NodeGetVolumeStatsResponse.Size(m) +} +func (m *NodeGetVolumeStatsResponse) XXX_DiscardUnknown() { + xxx_messageInfo_NodeGetVolumeStatsResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeGetVolumeStatsResponse proto.InternalMessageInfo + +func (m *NodeGetVolumeStatsResponse) GetUsage() []*VolumeUsage { + if m != nil { + return m.Usage + } + return nil +} + +type VolumeUsage struct { + // The available capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + Available int64 `protobuf:"varint,1,opt,name=available,proto3" json:"available,omitempty"` + // The total capacity in specified Unit. This field is REQUIRED. + // The value of this field MUST NOT be negative. + Total int64 `protobuf:"varint,2,opt,name=total,proto3" json:"total,omitempty"` + // The used capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + Used int64 `protobuf:"varint,3,opt,name=used,proto3" json:"used,omitempty"` + // Units by which values are measured. This field is REQUIRED. + Unit VolumeUsage_Unit `protobuf:"varint,4,opt,name=unit,proto3,enum=csi.v1.VolumeUsage_Unit" json:"unit,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeUsage) Reset() { *m = VolumeUsage{} } +func (m *VolumeUsage) String() string { return proto.CompactTextString(m) } +func (*VolumeUsage) ProtoMessage() {} +func (*VolumeUsage) Descriptor() ([]byte, []int) { + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{47} +} +func (m *VolumeUsage) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VolumeUsage.Unmarshal(m, b) +} +func (m *VolumeUsage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VolumeUsage.Marshal(b, m, deterministic) +} +func (dst *VolumeUsage) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeUsage.Merge(dst, src) +} +func (m *VolumeUsage) XXX_Size() int { + return xxx_messageInfo_VolumeUsage.Size(m) +} +func (m *VolumeUsage) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeUsage.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeUsage proto.InternalMessageInfo + +func (m *VolumeUsage) GetAvailable() int64 { + if m != nil { + return m.Available + } + return 0 +} + +func (m *VolumeUsage) GetTotal() int64 { + if m != nil { + return m.Total + } + return 0 +} + +func (m *VolumeUsage) GetUsed() int64 { + if m != nil { + return m.Used + } + return 0 +} + +func (m *VolumeUsage) GetUnit() VolumeUsage_Unit { + if m != nil { + return m.Unit + } + return VolumeUsage_UNKNOWN +} + type NodeGetCapabilitiesRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -3582,7 +3850,7 @@ func (m *NodeGetCapabilitiesRequest) Reset() { *m = NodeGetCapabilitiesR func (m *NodeGetCapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*NodeGetCapabilitiesRequest) ProtoMessage() {} func (*NodeGetCapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{48} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{48} } func (m *NodeGetCapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetCapabilitiesRequest.Unmarshal(m, b) @@ -3605,7 +3873,7 @@ var xxx_messageInfo_NodeGetCapabilitiesRequest proto.InternalMessageInfo type NodeGetCapabilitiesResponse struct { // All the capabilities that the node service supports. This field // is OPTIONAL. - Capabilities []*NodeServiceCapability `protobuf:"bytes,1,rep,name=capabilities" json:"capabilities,omitempty"` + Capabilities []*NodeServiceCapability `protobuf:"bytes,1,rep,name=capabilities,proto3" json:"capabilities,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3615,7 +3883,7 @@ func (m *NodeGetCapabilitiesResponse) Reset() { *m = NodeGetCapabilities func (m *NodeGetCapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*NodeGetCapabilitiesResponse) ProtoMessage() {} func (*NodeGetCapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{49} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{49} } func (m *NodeGetCapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetCapabilitiesResponse.Unmarshal(m, b) @@ -3656,7 +3924,7 @@ func (m *NodeServiceCapability) Reset() { *m = NodeServiceCapability{} } func (m *NodeServiceCapability) String() string { return proto.CompactTextString(m) } func (*NodeServiceCapability) ProtoMessage() {} func (*NodeServiceCapability) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{50} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{50} } func (m *NodeServiceCapability) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeServiceCapability.Unmarshal(m, b) @@ -3681,7 +3949,7 @@ type isNodeServiceCapability_Type interface { } type NodeServiceCapability_Rpc struct { - Rpc *NodeServiceCapability_RPC `protobuf:"bytes,1,opt,name=rpc,oneof"` + Rpc *NodeServiceCapability_RPC `protobuf:"bytes,1,opt,name=rpc,proto3,oneof"` } func (*NodeServiceCapability_Rpc) isNodeServiceCapability_Type() {} @@ -3756,7 +4024,7 @@ func _NodeServiceCapability_OneofSizer(msg proto.Message) (n int) { } type NodeServiceCapability_RPC struct { - Type NodeServiceCapability_RPC_Type `protobuf:"varint,1,opt,name=type,enum=csi.v0.NodeServiceCapability_RPC_Type" json:"type,omitempty"` + Type NodeServiceCapability_RPC_Type `protobuf:"varint,1,opt,name=type,proto3,enum=csi.v1.NodeServiceCapability_RPC_Type" json:"type,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3766,7 +4034,7 @@ func (m *NodeServiceCapability_RPC) Reset() { *m = NodeServiceCapability func (m *NodeServiceCapability_RPC) String() string { return proto.CompactTextString(m) } func (*NodeServiceCapability_RPC) ProtoMessage() {} func (*NodeServiceCapability_RPC) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{50, 0} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{50, 0} } func (m *NodeServiceCapability_RPC) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeServiceCapability_RPC.Unmarshal(m, b) @@ -3803,7 +4071,7 @@ func (m *NodeGetInfoRequest) Reset() { *m = NodeGetInfoRequest{} } func (m *NodeGetInfoRequest) String() string { return proto.CompactTextString(m) } func (*NodeGetInfoRequest) ProtoMessage() {} func (*NodeGetInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{51} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{51} } func (m *NodeGetInfoRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetInfoRequest.Unmarshal(m, b) @@ -3824,20 +4092,25 @@ func (m *NodeGetInfoRequest) XXX_DiscardUnknown() { var xxx_messageInfo_NodeGetInfoRequest proto.InternalMessageInfo type NodeGetInfoResponse struct { - // The ID of the node as understood by the SP which SHALL be used by - // CO in subsequent calls to `ControllerPublishVolume`. - // This is a REQUIRED field. - NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId" json:"node_id,omitempty"` + // The identifier of the node as understood by the SP. + // This field is REQUIRED. + // This field MUST contain enough information to uniquely identify + // this specific node vs all other nodes supported by this plugin. + // This field SHALL be used by the CO in subsequent calls, including + // `ControllerPublishVolume`, to refer to this node. + // The SP is NOT responsible for global uniqueness of node_id across + // multiple SPs. + NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` // Maximum number of volumes that controller can publish to the node. // If value is not set or zero CO SHALL decide how many volumes of // this type can be published by the controller to the node. The // plugin MUST NOT set negative values here. // This field is OPTIONAL. - MaxVolumesPerNode int64 `protobuf:"varint,2,opt,name=max_volumes_per_node,json=maxVolumesPerNode" json:"max_volumes_per_node,omitempty"` + MaxVolumesPerNode int64 `protobuf:"varint,2,opt,name=max_volumes_per_node,json=maxVolumesPerNode,proto3" json:"max_volumes_per_node,omitempty"` // Specifies where (regions, zones, racks, etc.) the node is // accessible from. // A plugin that returns this field MUST also set the - // ACCESSIBILITY_CONSTRAINTS plugin capability. + // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability. // COs MAY use this information along with the topology information // returned in CreateVolumeResponse to ensure that a given volume is // accessible from a given node when scheduling workloads. @@ -3851,7 +4124,7 @@ type NodeGetInfoResponse struct { // {"region": "R1", "zone": "R2"} // Indicates the node exists within the "region" "R1" and the "zone" // "Z2". - AccessibleTopology *Topology `protobuf:"bytes,3,opt,name=accessible_topology,json=accessibleTopology" json:"accessible_topology,omitempty"` + AccessibleTopology *Topology `protobuf:"bytes,3,opt,name=accessible_topology,json=accessibleTopology,proto3" json:"accessible_topology,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3861,7 +4134,7 @@ func (m *NodeGetInfoResponse) Reset() { *m = NodeGetInfoResponse{} } func (m *NodeGetInfoResponse) String() string { return proto.CompactTextString(m) } func (*NodeGetInfoResponse) ProtoMessage() {} func (*NodeGetInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_31237507707d37ec, []int{52} + return fileDescriptor_csi_1092d4f3f3c8dc30, []int{52} } func (m *NodeGetInfoResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetInfoResponse.Unmarshal(m, b) @@ -3902,95 +4175,111 @@ func (m *NodeGetInfoResponse) GetAccessibleTopology() *Topology { return nil } +var E_CsiSecret = &proto.ExtensionDesc{ + ExtendedType: (*descriptor.FieldOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 1059, + Name: "csi.v1.csi_secret", + Tag: "varint,1059,opt,name=csi_secret,json=csiSecret", + Filename: "github.com/container-storage-interface/spec/csi.proto", +} + func init() { - proto.RegisterType((*GetPluginInfoRequest)(nil), "csi.v0.GetPluginInfoRequest") - proto.RegisterType((*GetPluginInfoResponse)(nil), "csi.v0.GetPluginInfoResponse") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.GetPluginInfoResponse.ManifestEntry") - proto.RegisterType((*GetPluginCapabilitiesRequest)(nil), "csi.v0.GetPluginCapabilitiesRequest") - proto.RegisterType((*GetPluginCapabilitiesResponse)(nil), "csi.v0.GetPluginCapabilitiesResponse") - proto.RegisterType((*PluginCapability)(nil), "csi.v0.PluginCapability") - proto.RegisterType((*PluginCapability_Service)(nil), "csi.v0.PluginCapability.Service") - proto.RegisterType((*ProbeRequest)(nil), "csi.v0.ProbeRequest") - proto.RegisterType((*ProbeResponse)(nil), "csi.v0.ProbeResponse") - proto.RegisterType((*CreateVolumeRequest)(nil), "csi.v0.CreateVolumeRequest") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.CreateVolumeRequest.ControllerCreateSecretsEntry") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.CreateVolumeRequest.ParametersEntry") - proto.RegisterType((*VolumeContentSource)(nil), "csi.v0.VolumeContentSource") - proto.RegisterType((*VolumeContentSource_SnapshotSource)(nil), "csi.v0.VolumeContentSource.SnapshotSource") - proto.RegisterType((*CreateVolumeResponse)(nil), "csi.v0.CreateVolumeResponse") - proto.RegisterType((*VolumeCapability)(nil), "csi.v0.VolumeCapability") - proto.RegisterType((*VolumeCapability_BlockVolume)(nil), "csi.v0.VolumeCapability.BlockVolume") - proto.RegisterType((*VolumeCapability_MountVolume)(nil), "csi.v0.VolumeCapability.MountVolume") - proto.RegisterType((*VolumeCapability_AccessMode)(nil), "csi.v0.VolumeCapability.AccessMode") - proto.RegisterType((*CapacityRange)(nil), "csi.v0.CapacityRange") - proto.RegisterType((*Volume)(nil), "csi.v0.Volume") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.Volume.AttributesEntry") - proto.RegisterType((*TopologyRequirement)(nil), "csi.v0.TopologyRequirement") - proto.RegisterType((*Topology)(nil), "csi.v0.Topology") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.Topology.SegmentsEntry") - proto.RegisterType((*DeleteVolumeRequest)(nil), "csi.v0.DeleteVolumeRequest") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.DeleteVolumeRequest.ControllerDeleteSecretsEntry") - proto.RegisterType((*DeleteVolumeResponse)(nil), "csi.v0.DeleteVolumeResponse") - proto.RegisterType((*ControllerPublishVolumeRequest)(nil), "csi.v0.ControllerPublishVolumeRequest") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.ControllerPublishVolumeRequest.ControllerPublishSecretsEntry") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.ControllerPublishVolumeRequest.VolumeAttributesEntry") - proto.RegisterType((*ControllerPublishVolumeResponse)(nil), "csi.v0.ControllerPublishVolumeResponse") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.ControllerPublishVolumeResponse.PublishInfoEntry") - proto.RegisterType((*ControllerUnpublishVolumeRequest)(nil), "csi.v0.ControllerUnpublishVolumeRequest") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.ControllerUnpublishVolumeRequest.ControllerUnpublishSecretsEntry") - proto.RegisterType((*ControllerUnpublishVolumeResponse)(nil), "csi.v0.ControllerUnpublishVolumeResponse") - proto.RegisterType((*ValidateVolumeCapabilitiesRequest)(nil), "csi.v0.ValidateVolumeCapabilitiesRequest") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.ValidateVolumeCapabilitiesRequest.VolumeAttributesEntry") - proto.RegisterType((*ValidateVolumeCapabilitiesResponse)(nil), "csi.v0.ValidateVolumeCapabilitiesResponse") - proto.RegisterType((*ListVolumesRequest)(nil), "csi.v0.ListVolumesRequest") - proto.RegisterType((*ListVolumesResponse)(nil), "csi.v0.ListVolumesResponse") - proto.RegisterType((*ListVolumesResponse_Entry)(nil), "csi.v0.ListVolumesResponse.Entry") - proto.RegisterType((*GetCapacityRequest)(nil), "csi.v0.GetCapacityRequest") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.GetCapacityRequest.ParametersEntry") - proto.RegisterType((*GetCapacityResponse)(nil), "csi.v0.GetCapacityResponse") - proto.RegisterType((*ControllerGetCapabilitiesRequest)(nil), "csi.v0.ControllerGetCapabilitiesRequest") - proto.RegisterType((*ControllerGetCapabilitiesResponse)(nil), "csi.v0.ControllerGetCapabilitiesResponse") - proto.RegisterType((*ControllerServiceCapability)(nil), "csi.v0.ControllerServiceCapability") - proto.RegisterType((*ControllerServiceCapability_RPC)(nil), "csi.v0.ControllerServiceCapability.RPC") - proto.RegisterType((*CreateSnapshotRequest)(nil), "csi.v0.CreateSnapshotRequest") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.CreateSnapshotRequest.CreateSnapshotSecretsEntry") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.CreateSnapshotRequest.ParametersEntry") - proto.RegisterType((*CreateSnapshotResponse)(nil), "csi.v0.CreateSnapshotResponse") - proto.RegisterType((*Snapshot)(nil), "csi.v0.Snapshot") - proto.RegisterType((*SnapshotStatus)(nil), "csi.v0.SnapshotStatus") - proto.RegisterType((*DeleteSnapshotRequest)(nil), "csi.v0.DeleteSnapshotRequest") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.DeleteSnapshotRequest.DeleteSnapshotSecretsEntry") - proto.RegisterType((*DeleteSnapshotResponse)(nil), "csi.v0.DeleteSnapshotResponse") - proto.RegisterType((*ListSnapshotsRequest)(nil), "csi.v0.ListSnapshotsRequest") - proto.RegisterType((*ListSnapshotsResponse)(nil), "csi.v0.ListSnapshotsResponse") - proto.RegisterType((*ListSnapshotsResponse_Entry)(nil), "csi.v0.ListSnapshotsResponse.Entry") - proto.RegisterType((*NodeStageVolumeRequest)(nil), "csi.v0.NodeStageVolumeRequest") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.NodeStageVolumeRequest.NodeStageSecretsEntry") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.NodeStageVolumeRequest.PublishInfoEntry") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.NodeStageVolumeRequest.VolumeAttributesEntry") - proto.RegisterType((*NodeStageVolumeResponse)(nil), "csi.v0.NodeStageVolumeResponse") - proto.RegisterType((*NodeUnstageVolumeRequest)(nil), "csi.v0.NodeUnstageVolumeRequest") - proto.RegisterType((*NodeUnstageVolumeResponse)(nil), "csi.v0.NodeUnstageVolumeResponse") - proto.RegisterType((*NodePublishVolumeRequest)(nil), "csi.v0.NodePublishVolumeRequest") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.NodePublishVolumeRequest.NodePublishSecretsEntry") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.NodePublishVolumeRequest.PublishInfoEntry") - proto.RegisterMapType((map[string]string)(nil), "csi.v0.NodePublishVolumeRequest.VolumeAttributesEntry") - proto.RegisterType((*NodePublishVolumeResponse)(nil), "csi.v0.NodePublishVolumeResponse") - proto.RegisterType((*NodeUnpublishVolumeRequest)(nil), "csi.v0.NodeUnpublishVolumeRequest") - proto.RegisterType((*NodeUnpublishVolumeResponse)(nil), "csi.v0.NodeUnpublishVolumeResponse") - proto.RegisterType((*NodeGetIdRequest)(nil), "csi.v0.NodeGetIdRequest") - proto.RegisterType((*NodeGetIdResponse)(nil), "csi.v0.NodeGetIdResponse") - proto.RegisterType((*NodeGetCapabilitiesRequest)(nil), "csi.v0.NodeGetCapabilitiesRequest") - proto.RegisterType((*NodeGetCapabilitiesResponse)(nil), "csi.v0.NodeGetCapabilitiesResponse") - proto.RegisterType((*NodeServiceCapability)(nil), "csi.v0.NodeServiceCapability") - proto.RegisterType((*NodeServiceCapability_RPC)(nil), "csi.v0.NodeServiceCapability.RPC") - proto.RegisterType((*NodeGetInfoRequest)(nil), "csi.v0.NodeGetInfoRequest") - proto.RegisterType((*NodeGetInfoResponse)(nil), "csi.v0.NodeGetInfoResponse") - proto.RegisterEnum("csi.v0.PluginCapability_Service_Type", PluginCapability_Service_Type_name, PluginCapability_Service_Type_value) - proto.RegisterEnum("csi.v0.VolumeCapability_AccessMode_Mode", VolumeCapability_AccessMode_Mode_name, VolumeCapability_AccessMode_Mode_value) - proto.RegisterEnum("csi.v0.ControllerServiceCapability_RPC_Type", ControllerServiceCapability_RPC_Type_name, ControllerServiceCapability_RPC_Type_value) - proto.RegisterEnum("csi.v0.SnapshotStatus_Type", SnapshotStatus_Type_name, SnapshotStatus_Type_value) - proto.RegisterEnum("csi.v0.NodeServiceCapability_RPC_Type", NodeServiceCapability_RPC_Type_name, NodeServiceCapability_RPC_Type_value) + proto.RegisterType((*GetPluginInfoRequest)(nil), "csi.v1.GetPluginInfoRequest") + proto.RegisterType((*GetPluginInfoResponse)(nil), "csi.v1.GetPluginInfoResponse") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.GetPluginInfoResponse.ManifestEntry") + proto.RegisterType((*GetPluginCapabilitiesRequest)(nil), "csi.v1.GetPluginCapabilitiesRequest") + proto.RegisterType((*GetPluginCapabilitiesResponse)(nil), "csi.v1.GetPluginCapabilitiesResponse") + proto.RegisterType((*PluginCapability)(nil), "csi.v1.PluginCapability") + proto.RegisterType((*PluginCapability_Service)(nil), "csi.v1.PluginCapability.Service") + proto.RegisterType((*ProbeRequest)(nil), "csi.v1.ProbeRequest") + proto.RegisterType((*ProbeResponse)(nil), "csi.v1.ProbeResponse") + proto.RegisterType((*CreateVolumeRequest)(nil), "csi.v1.CreateVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateVolumeRequest.ParametersEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateVolumeRequest.SecretsEntry") + proto.RegisterType((*VolumeContentSource)(nil), "csi.v1.VolumeContentSource") + proto.RegisterType((*VolumeContentSource_SnapshotSource)(nil), "csi.v1.VolumeContentSource.SnapshotSource") + proto.RegisterType((*VolumeContentSource_VolumeSource)(nil), "csi.v1.VolumeContentSource.VolumeSource") + proto.RegisterType((*CreateVolumeResponse)(nil), "csi.v1.CreateVolumeResponse") + proto.RegisterType((*VolumeCapability)(nil), "csi.v1.VolumeCapability") + proto.RegisterType((*VolumeCapability_BlockVolume)(nil), "csi.v1.VolumeCapability.BlockVolume") + proto.RegisterType((*VolumeCapability_MountVolume)(nil), "csi.v1.VolumeCapability.MountVolume") + proto.RegisterType((*VolumeCapability_AccessMode)(nil), "csi.v1.VolumeCapability.AccessMode") + proto.RegisterType((*CapacityRange)(nil), "csi.v1.CapacityRange") + proto.RegisterType((*Volume)(nil), "csi.v1.Volume") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.Volume.VolumeContextEntry") + proto.RegisterType((*TopologyRequirement)(nil), "csi.v1.TopologyRequirement") + proto.RegisterType((*Topology)(nil), "csi.v1.Topology") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.Topology.SegmentsEntry") + proto.RegisterType((*DeleteVolumeRequest)(nil), "csi.v1.DeleteVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.DeleteVolumeRequest.SecretsEntry") + proto.RegisterType((*DeleteVolumeResponse)(nil), "csi.v1.DeleteVolumeResponse") + proto.RegisterType((*ControllerPublishVolumeRequest)(nil), "csi.v1.ControllerPublishVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerPublishVolumeRequest.SecretsEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerPublishVolumeRequest.VolumeContextEntry") + proto.RegisterType((*ControllerPublishVolumeResponse)(nil), "csi.v1.ControllerPublishVolumeResponse") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerPublishVolumeResponse.PublishContextEntry") + proto.RegisterType((*ControllerUnpublishVolumeRequest)(nil), "csi.v1.ControllerUnpublishVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerUnpublishVolumeRequest.SecretsEntry") + proto.RegisterType((*ControllerUnpublishVolumeResponse)(nil), "csi.v1.ControllerUnpublishVolumeResponse") + proto.RegisterType((*ValidateVolumeCapabilitiesRequest)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest.ParametersEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest.SecretsEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest.VolumeContextEntry") + proto.RegisterType((*ValidateVolumeCapabilitiesResponse)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse") + proto.RegisterType((*ValidateVolumeCapabilitiesResponse_Confirmed)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse.Confirmed") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse.Confirmed.ParametersEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse.Confirmed.VolumeContextEntry") + proto.RegisterType((*ListVolumesRequest)(nil), "csi.v1.ListVolumesRequest") + proto.RegisterType((*ListVolumesResponse)(nil), "csi.v1.ListVolumesResponse") + proto.RegisterType((*ListVolumesResponse_Entry)(nil), "csi.v1.ListVolumesResponse.Entry") + proto.RegisterType((*GetCapacityRequest)(nil), "csi.v1.GetCapacityRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.GetCapacityRequest.ParametersEntry") + proto.RegisterType((*GetCapacityResponse)(nil), "csi.v1.GetCapacityResponse") + proto.RegisterType((*ControllerGetCapabilitiesRequest)(nil), "csi.v1.ControllerGetCapabilitiesRequest") + proto.RegisterType((*ControllerGetCapabilitiesResponse)(nil), "csi.v1.ControllerGetCapabilitiesResponse") + proto.RegisterType((*ControllerServiceCapability)(nil), "csi.v1.ControllerServiceCapability") + proto.RegisterType((*ControllerServiceCapability_RPC)(nil), "csi.v1.ControllerServiceCapability.RPC") + proto.RegisterType((*CreateSnapshotRequest)(nil), "csi.v1.CreateSnapshotRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateSnapshotRequest.ParametersEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateSnapshotRequest.SecretsEntry") + proto.RegisterType((*CreateSnapshotResponse)(nil), "csi.v1.CreateSnapshotResponse") + proto.RegisterType((*Snapshot)(nil), "csi.v1.Snapshot") + proto.RegisterType((*DeleteSnapshotRequest)(nil), "csi.v1.DeleteSnapshotRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.DeleteSnapshotRequest.SecretsEntry") + proto.RegisterType((*DeleteSnapshotResponse)(nil), "csi.v1.DeleteSnapshotResponse") + proto.RegisterType((*ListSnapshotsRequest)(nil), "csi.v1.ListSnapshotsRequest") + proto.RegisterType((*ListSnapshotsResponse)(nil), "csi.v1.ListSnapshotsResponse") + proto.RegisterType((*ListSnapshotsResponse_Entry)(nil), "csi.v1.ListSnapshotsResponse.Entry") + proto.RegisterType((*NodeStageVolumeRequest)(nil), "csi.v1.NodeStageVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodeStageVolumeRequest.PublishContextEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodeStageVolumeRequest.SecretsEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodeStageVolumeRequest.VolumeContextEntry") + proto.RegisterType((*NodeStageVolumeResponse)(nil), "csi.v1.NodeStageVolumeResponse") + proto.RegisterType((*NodeUnstageVolumeRequest)(nil), "csi.v1.NodeUnstageVolumeRequest") + proto.RegisterType((*NodeUnstageVolumeResponse)(nil), "csi.v1.NodeUnstageVolumeResponse") + proto.RegisterType((*NodePublishVolumeRequest)(nil), "csi.v1.NodePublishVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodePublishVolumeRequest.PublishContextEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodePublishVolumeRequest.SecretsEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodePublishVolumeRequest.VolumeContextEntry") + proto.RegisterType((*NodePublishVolumeResponse)(nil), "csi.v1.NodePublishVolumeResponse") + proto.RegisterType((*NodeUnpublishVolumeRequest)(nil), "csi.v1.NodeUnpublishVolumeRequest") + proto.RegisterType((*NodeUnpublishVolumeResponse)(nil), "csi.v1.NodeUnpublishVolumeResponse") + proto.RegisterType((*NodeGetVolumeStatsRequest)(nil), "csi.v1.NodeGetVolumeStatsRequest") + proto.RegisterType((*NodeGetVolumeStatsResponse)(nil), "csi.v1.NodeGetVolumeStatsResponse") + proto.RegisterType((*VolumeUsage)(nil), "csi.v1.VolumeUsage") + proto.RegisterType((*NodeGetCapabilitiesRequest)(nil), "csi.v1.NodeGetCapabilitiesRequest") + proto.RegisterType((*NodeGetCapabilitiesResponse)(nil), "csi.v1.NodeGetCapabilitiesResponse") + proto.RegisterType((*NodeServiceCapability)(nil), "csi.v1.NodeServiceCapability") + proto.RegisterType((*NodeServiceCapability_RPC)(nil), "csi.v1.NodeServiceCapability.RPC") + proto.RegisterType((*NodeGetInfoRequest)(nil), "csi.v1.NodeGetInfoRequest") + proto.RegisterType((*NodeGetInfoResponse)(nil), "csi.v1.NodeGetInfoResponse") + proto.RegisterEnum("csi.v1.PluginCapability_Service_Type", PluginCapability_Service_Type_name, PluginCapability_Service_Type_value) + proto.RegisterEnum("csi.v1.VolumeCapability_AccessMode_Mode", VolumeCapability_AccessMode_Mode_name, VolumeCapability_AccessMode_Mode_value) + proto.RegisterEnum("csi.v1.ControllerServiceCapability_RPC_Type", ControllerServiceCapability_RPC_Type_name, ControllerServiceCapability_RPC_Type_value) + proto.RegisterEnum("csi.v1.VolumeUsage_Unit", VolumeUsage_Unit_name, VolumeUsage_Unit_value) + proto.RegisterEnum("csi.v1.NodeServiceCapability_RPC_Type", NodeServiceCapability_RPC_Type_name, NodeServiceCapability_RPC_Type_value) + proto.RegisterExtension(E_CsiSecret) } // Reference imports to suppress errors if they are not otherwise used. @@ -4001,8 +4290,9 @@ var _ grpc.ClientConn // is compatible with the grpc package it is being compiled against. const _ = grpc.SupportPackageIsVersion4 -// Client API for Identity service - +// IdentityClient is the client API for Identity service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type IdentityClient interface { GetPluginInfo(ctx context.Context, in *GetPluginInfoRequest, opts ...grpc.CallOption) (*GetPluginInfoResponse, error) GetPluginCapabilities(ctx context.Context, in *GetPluginCapabilitiesRequest, opts ...grpc.CallOption) (*GetPluginCapabilitiesResponse, error) @@ -4019,7 +4309,7 @@ func NewIdentityClient(cc *grpc.ClientConn) IdentityClient { func (c *identityClient) GetPluginInfo(ctx context.Context, in *GetPluginInfoRequest, opts ...grpc.CallOption) (*GetPluginInfoResponse, error) { out := new(GetPluginInfoResponse) - err := grpc.Invoke(ctx, "/csi.v0.Identity/GetPluginInfo", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Identity/GetPluginInfo", in, out, opts...) if err != nil { return nil, err } @@ -4028,7 +4318,7 @@ func (c *identityClient) GetPluginInfo(ctx context.Context, in *GetPluginInfoReq func (c *identityClient) GetPluginCapabilities(ctx context.Context, in *GetPluginCapabilitiesRequest, opts ...grpc.CallOption) (*GetPluginCapabilitiesResponse, error) { out := new(GetPluginCapabilitiesResponse) - err := grpc.Invoke(ctx, "/csi.v0.Identity/GetPluginCapabilities", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Identity/GetPluginCapabilities", in, out, opts...) if err != nil { return nil, err } @@ -4037,15 +4327,14 @@ func (c *identityClient) GetPluginCapabilities(ctx context.Context, in *GetPlugi func (c *identityClient) Probe(ctx context.Context, in *ProbeRequest, opts ...grpc.CallOption) (*ProbeResponse, error) { out := new(ProbeResponse) - err := grpc.Invoke(ctx, "/csi.v0.Identity/Probe", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Identity/Probe", in, out, opts...) if err != nil { return nil, err } return out, nil } -// Server API for Identity service - +// IdentityServer is the server API for Identity service. type IdentityServer interface { GetPluginInfo(context.Context, *GetPluginInfoRequest) (*GetPluginInfoResponse, error) GetPluginCapabilities(context.Context, *GetPluginCapabilitiesRequest) (*GetPluginCapabilitiesResponse, error) @@ -4066,7 +4355,7 @@ func _Identity_GetPluginInfo_Handler(srv interface{}, ctx context.Context, dec f } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Identity/GetPluginInfo", + FullMethod: "/csi.v1.Identity/GetPluginInfo", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(IdentityServer).GetPluginInfo(ctx, req.(*GetPluginInfoRequest)) @@ -4084,7 +4373,7 @@ func _Identity_GetPluginCapabilities_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Identity/GetPluginCapabilities", + FullMethod: "/csi.v1.Identity/GetPluginCapabilities", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(IdentityServer).GetPluginCapabilities(ctx, req.(*GetPluginCapabilitiesRequest)) @@ -4102,7 +4391,7 @@ func _Identity_Probe_Handler(srv interface{}, ctx context.Context, dec func(inte } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Identity/Probe", + FullMethod: "/csi.v1.Identity/Probe", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(IdentityServer).Probe(ctx, req.(*ProbeRequest)) @@ -4111,7 +4400,7 @@ func _Identity_Probe_Handler(srv interface{}, ctx context.Context, dec func(inte } var _Identity_serviceDesc = grpc.ServiceDesc{ - ServiceName: "csi.v0.Identity", + ServiceName: "csi.v1.Identity", HandlerType: (*IdentityServer)(nil), Methods: []grpc.MethodDesc{ { @@ -4128,11 +4417,12 @@ var _Identity_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "csi.proto", + Metadata: "github.com/container-storage-interface/spec/csi.proto", } -// Client API for Controller service - +// ControllerClient is the client API for Controller service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type ControllerClient interface { CreateVolume(ctx context.Context, in *CreateVolumeRequest, opts ...grpc.CallOption) (*CreateVolumeResponse, error) DeleteVolume(ctx context.Context, in *DeleteVolumeRequest, opts ...grpc.CallOption) (*DeleteVolumeResponse, error) @@ -4157,7 +4447,7 @@ func NewControllerClient(cc *grpc.ClientConn) ControllerClient { func (c *controllerClient) CreateVolume(ctx context.Context, in *CreateVolumeRequest, opts ...grpc.CallOption) (*CreateVolumeResponse, error) { out := new(CreateVolumeResponse) - err := grpc.Invoke(ctx, "/csi.v0.Controller/CreateVolume", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/CreateVolume", in, out, opts...) if err != nil { return nil, err } @@ -4166,7 +4456,7 @@ func (c *controllerClient) CreateVolume(ctx context.Context, in *CreateVolumeReq func (c *controllerClient) DeleteVolume(ctx context.Context, in *DeleteVolumeRequest, opts ...grpc.CallOption) (*DeleteVolumeResponse, error) { out := new(DeleteVolumeResponse) - err := grpc.Invoke(ctx, "/csi.v0.Controller/DeleteVolume", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/DeleteVolume", in, out, opts...) if err != nil { return nil, err } @@ -4175,7 +4465,7 @@ func (c *controllerClient) DeleteVolume(ctx context.Context, in *DeleteVolumeReq func (c *controllerClient) ControllerPublishVolume(ctx context.Context, in *ControllerPublishVolumeRequest, opts ...grpc.CallOption) (*ControllerPublishVolumeResponse, error) { out := new(ControllerPublishVolumeResponse) - err := grpc.Invoke(ctx, "/csi.v0.Controller/ControllerPublishVolume", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ControllerPublishVolume", in, out, opts...) if err != nil { return nil, err } @@ -4184,7 +4474,7 @@ func (c *controllerClient) ControllerPublishVolume(ctx context.Context, in *Cont func (c *controllerClient) ControllerUnpublishVolume(ctx context.Context, in *ControllerUnpublishVolumeRequest, opts ...grpc.CallOption) (*ControllerUnpublishVolumeResponse, error) { out := new(ControllerUnpublishVolumeResponse) - err := grpc.Invoke(ctx, "/csi.v0.Controller/ControllerUnpublishVolume", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ControllerUnpublishVolume", in, out, opts...) if err != nil { return nil, err } @@ -4193,7 +4483,7 @@ func (c *controllerClient) ControllerUnpublishVolume(ctx context.Context, in *Co func (c *controllerClient) ValidateVolumeCapabilities(ctx context.Context, in *ValidateVolumeCapabilitiesRequest, opts ...grpc.CallOption) (*ValidateVolumeCapabilitiesResponse, error) { out := new(ValidateVolumeCapabilitiesResponse) - err := grpc.Invoke(ctx, "/csi.v0.Controller/ValidateVolumeCapabilities", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ValidateVolumeCapabilities", in, out, opts...) if err != nil { return nil, err } @@ -4202,7 +4492,7 @@ func (c *controllerClient) ValidateVolumeCapabilities(ctx context.Context, in *V func (c *controllerClient) ListVolumes(ctx context.Context, in *ListVolumesRequest, opts ...grpc.CallOption) (*ListVolumesResponse, error) { out := new(ListVolumesResponse) - err := grpc.Invoke(ctx, "/csi.v0.Controller/ListVolumes", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ListVolumes", in, out, opts...) if err != nil { return nil, err } @@ -4211,7 +4501,7 @@ func (c *controllerClient) ListVolumes(ctx context.Context, in *ListVolumesReque func (c *controllerClient) GetCapacity(ctx context.Context, in *GetCapacityRequest, opts ...grpc.CallOption) (*GetCapacityResponse, error) { out := new(GetCapacityResponse) - err := grpc.Invoke(ctx, "/csi.v0.Controller/GetCapacity", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/GetCapacity", in, out, opts...) if err != nil { return nil, err } @@ -4220,7 +4510,7 @@ func (c *controllerClient) GetCapacity(ctx context.Context, in *GetCapacityReque func (c *controllerClient) ControllerGetCapabilities(ctx context.Context, in *ControllerGetCapabilitiesRequest, opts ...grpc.CallOption) (*ControllerGetCapabilitiesResponse, error) { out := new(ControllerGetCapabilitiesResponse) - err := grpc.Invoke(ctx, "/csi.v0.Controller/ControllerGetCapabilities", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ControllerGetCapabilities", in, out, opts...) if err != nil { return nil, err } @@ -4229,7 +4519,7 @@ func (c *controllerClient) ControllerGetCapabilities(ctx context.Context, in *Co func (c *controllerClient) CreateSnapshot(ctx context.Context, in *CreateSnapshotRequest, opts ...grpc.CallOption) (*CreateSnapshotResponse, error) { out := new(CreateSnapshotResponse) - err := grpc.Invoke(ctx, "/csi.v0.Controller/CreateSnapshot", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/CreateSnapshot", in, out, opts...) if err != nil { return nil, err } @@ -4238,7 +4528,7 @@ func (c *controllerClient) CreateSnapshot(ctx context.Context, in *CreateSnapsho func (c *controllerClient) DeleteSnapshot(ctx context.Context, in *DeleteSnapshotRequest, opts ...grpc.CallOption) (*DeleteSnapshotResponse, error) { out := new(DeleteSnapshotResponse) - err := grpc.Invoke(ctx, "/csi.v0.Controller/DeleteSnapshot", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/DeleteSnapshot", in, out, opts...) if err != nil { return nil, err } @@ -4247,15 +4537,14 @@ func (c *controllerClient) DeleteSnapshot(ctx context.Context, in *DeleteSnapsho func (c *controllerClient) ListSnapshots(ctx context.Context, in *ListSnapshotsRequest, opts ...grpc.CallOption) (*ListSnapshotsResponse, error) { out := new(ListSnapshotsResponse) - err := grpc.Invoke(ctx, "/csi.v0.Controller/ListSnapshots", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ListSnapshots", in, out, opts...) if err != nil { return nil, err } return out, nil } -// Server API for Controller service - +// ControllerServer is the server API for Controller service. type ControllerServer interface { CreateVolume(context.Context, *CreateVolumeRequest) (*CreateVolumeResponse, error) DeleteVolume(context.Context, *DeleteVolumeRequest) (*DeleteVolumeResponse, error) @@ -4284,7 +4573,7 @@ func _Controller_CreateVolume_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Controller/CreateVolume", + FullMethod: "/csi.v1.Controller/CreateVolume", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControllerServer).CreateVolume(ctx, req.(*CreateVolumeRequest)) @@ -4302,7 +4591,7 @@ func _Controller_DeleteVolume_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Controller/DeleteVolume", + FullMethod: "/csi.v1.Controller/DeleteVolume", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControllerServer).DeleteVolume(ctx, req.(*DeleteVolumeRequest)) @@ -4320,7 +4609,7 @@ func _Controller_ControllerPublishVolume_Handler(srv interface{}, ctx context.Co } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Controller/ControllerPublishVolume", + FullMethod: "/csi.v1.Controller/ControllerPublishVolume", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControllerServer).ControllerPublishVolume(ctx, req.(*ControllerPublishVolumeRequest)) @@ -4338,7 +4627,7 @@ func _Controller_ControllerUnpublishVolume_Handler(srv interface{}, ctx context. } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Controller/ControllerUnpublishVolume", + FullMethod: "/csi.v1.Controller/ControllerUnpublishVolume", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControllerServer).ControllerUnpublishVolume(ctx, req.(*ControllerUnpublishVolumeRequest)) @@ -4356,7 +4645,7 @@ func _Controller_ValidateVolumeCapabilities_Handler(srv interface{}, ctx context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Controller/ValidateVolumeCapabilities", + FullMethod: "/csi.v1.Controller/ValidateVolumeCapabilities", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControllerServer).ValidateVolumeCapabilities(ctx, req.(*ValidateVolumeCapabilitiesRequest)) @@ -4374,7 +4663,7 @@ func _Controller_ListVolumes_Handler(srv interface{}, ctx context.Context, dec f } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Controller/ListVolumes", + FullMethod: "/csi.v1.Controller/ListVolumes", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControllerServer).ListVolumes(ctx, req.(*ListVolumesRequest)) @@ -4392,7 +4681,7 @@ func _Controller_GetCapacity_Handler(srv interface{}, ctx context.Context, dec f } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Controller/GetCapacity", + FullMethod: "/csi.v1.Controller/GetCapacity", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControllerServer).GetCapacity(ctx, req.(*GetCapacityRequest)) @@ -4410,7 +4699,7 @@ func _Controller_ControllerGetCapabilities_Handler(srv interface{}, ctx context. } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Controller/ControllerGetCapabilities", + FullMethod: "/csi.v1.Controller/ControllerGetCapabilities", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControllerServer).ControllerGetCapabilities(ctx, req.(*ControllerGetCapabilitiesRequest)) @@ -4428,7 +4717,7 @@ func _Controller_CreateSnapshot_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Controller/CreateSnapshot", + FullMethod: "/csi.v1.Controller/CreateSnapshot", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControllerServer).CreateSnapshot(ctx, req.(*CreateSnapshotRequest)) @@ -4446,7 +4735,7 @@ func _Controller_DeleteSnapshot_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Controller/DeleteSnapshot", + FullMethod: "/csi.v1.Controller/DeleteSnapshot", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControllerServer).DeleteSnapshot(ctx, req.(*DeleteSnapshotRequest)) @@ -4464,7 +4753,7 @@ func _Controller_ListSnapshots_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Controller/ListSnapshots", + FullMethod: "/csi.v1.Controller/ListSnapshots", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControllerServer).ListSnapshots(ctx, req.(*ListSnapshotsRequest)) @@ -4473,7 +4762,7 @@ func _Controller_ListSnapshots_Handler(srv interface{}, ctx context.Context, dec } var _Controller_serviceDesc = grpc.ServiceDesc{ - ServiceName: "csi.v0.Controller", + ServiceName: "csi.v1.Controller", HandlerType: (*ControllerServer)(nil), Methods: []grpc.MethodDesc{ { @@ -4522,24 +4811,19 @@ var _Controller_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "csi.proto", + Metadata: "github.com/container-storage-interface/spec/csi.proto", } -// Client API for Node service - +// NodeClient is the client API for Node service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type NodeClient interface { NodeStageVolume(ctx context.Context, in *NodeStageVolumeRequest, opts ...grpc.CallOption) (*NodeStageVolumeResponse, error) NodeUnstageVolume(ctx context.Context, in *NodeUnstageVolumeRequest, opts ...grpc.CallOption) (*NodeUnstageVolumeResponse, error) NodePublishVolume(ctx context.Context, in *NodePublishVolumeRequest, opts ...grpc.CallOption) (*NodePublishVolumeResponse, error) NodeUnpublishVolume(ctx context.Context, in *NodeUnpublishVolumeRequest, opts ...grpc.CallOption) (*NodeUnpublishVolumeResponse, error) - // NodeGetId is being deprecated in favor of NodeGetInfo and will be - // removed in CSI 1.0. Existing drivers, however, may depend on this - // RPC call and hence this RPC call MUST be implemented by the CSI - // plugin prior to v1.0. - NodeGetId(ctx context.Context, in *NodeGetIdRequest, opts ...grpc.CallOption) (*NodeGetIdResponse, error) + NodeGetVolumeStats(ctx context.Context, in *NodeGetVolumeStatsRequest, opts ...grpc.CallOption) (*NodeGetVolumeStatsResponse, error) NodeGetCapabilities(ctx context.Context, in *NodeGetCapabilitiesRequest, opts ...grpc.CallOption) (*NodeGetCapabilitiesResponse, error) - // Prior to CSI 1.0 - CSI plugins MUST implement both NodeGetId and - // NodeGetInfo RPC calls. NodeGetInfo(ctx context.Context, in *NodeGetInfoRequest, opts ...grpc.CallOption) (*NodeGetInfoResponse, error) } @@ -4553,7 +4837,7 @@ func NewNodeClient(cc *grpc.ClientConn) NodeClient { func (c *nodeClient) NodeStageVolume(ctx context.Context, in *NodeStageVolumeRequest, opts ...grpc.CallOption) (*NodeStageVolumeResponse, error) { out := new(NodeStageVolumeResponse) - err := grpc.Invoke(ctx, "/csi.v0.Node/NodeStageVolume", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeStageVolume", in, out, opts...) if err != nil { return nil, err } @@ -4562,7 +4846,7 @@ func (c *nodeClient) NodeStageVolume(ctx context.Context, in *NodeStageVolumeReq func (c *nodeClient) NodeUnstageVolume(ctx context.Context, in *NodeUnstageVolumeRequest, opts ...grpc.CallOption) (*NodeUnstageVolumeResponse, error) { out := new(NodeUnstageVolumeResponse) - err := grpc.Invoke(ctx, "/csi.v0.Node/NodeUnstageVolume", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeUnstageVolume", in, out, opts...) if err != nil { return nil, err } @@ -4571,7 +4855,7 @@ func (c *nodeClient) NodeUnstageVolume(ctx context.Context, in *NodeUnstageVolum func (c *nodeClient) NodePublishVolume(ctx context.Context, in *NodePublishVolumeRequest, opts ...grpc.CallOption) (*NodePublishVolumeResponse, error) { out := new(NodePublishVolumeResponse) - err := grpc.Invoke(ctx, "/csi.v0.Node/NodePublishVolume", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodePublishVolume", in, out, opts...) if err != nil { return nil, err } @@ -4580,17 +4864,16 @@ func (c *nodeClient) NodePublishVolume(ctx context.Context, in *NodePublishVolum func (c *nodeClient) NodeUnpublishVolume(ctx context.Context, in *NodeUnpublishVolumeRequest, opts ...grpc.CallOption) (*NodeUnpublishVolumeResponse, error) { out := new(NodeUnpublishVolumeResponse) - err := grpc.Invoke(ctx, "/csi.v0.Node/NodeUnpublishVolume", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeUnpublishVolume", in, out, opts...) if err != nil { return nil, err } return out, nil } -// Deprecated: Do not use. -func (c *nodeClient) NodeGetId(ctx context.Context, in *NodeGetIdRequest, opts ...grpc.CallOption) (*NodeGetIdResponse, error) { - out := new(NodeGetIdResponse) - err := grpc.Invoke(ctx, "/csi.v0.Node/NodeGetId", in, out, c.cc, opts...) +func (c *nodeClient) NodeGetVolumeStats(ctx context.Context, in *NodeGetVolumeStatsRequest, opts ...grpc.CallOption) (*NodeGetVolumeStatsResponse, error) { + out := new(NodeGetVolumeStatsResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeGetVolumeStats", in, out, opts...) if err != nil { return nil, err } @@ -4599,7 +4882,7 @@ func (c *nodeClient) NodeGetId(ctx context.Context, in *NodeGetIdRequest, opts . func (c *nodeClient) NodeGetCapabilities(ctx context.Context, in *NodeGetCapabilitiesRequest, opts ...grpc.CallOption) (*NodeGetCapabilitiesResponse, error) { out := new(NodeGetCapabilitiesResponse) - err := grpc.Invoke(ctx, "/csi.v0.Node/NodeGetCapabilities", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeGetCapabilities", in, out, opts...) if err != nil { return nil, err } @@ -4608,28 +4891,21 @@ func (c *nodeClient) NodeGetCapabilities(ctx context.Context, in *NodeGetCapabil func (c *nodeClient) NodeGetInfo(ctx context.Context, in *NodeGetInfoRequest, opts ...grpc.CallOption) (*NodeGetInfoResponse, error) { out := new(NodeGetInfoResponse) - err := grpc.Invoke(ctx, "/csi.v0.Node/NodeGetInfo", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeGetInfo", in, out, opts...) if err != nil { return nil, err } return out, nil } -// Server API for Node service - +// NodeServer is the server API for Node service. type NodeServer interface { NodeStageVolume(context.Context, *NodeStageVolumeRequest) (*NodeStageVolumeResponse, error) NodeUnstageVolume(context.Context, *NodeUnstageVolumeRequest) (*NodeUnstageVolumeResponse, error) NodePublishVolume(context.Context, *NodePublishVolumeRequest) (*NodePublishVolumeResponse, error) NodeUnpublishVolume(context.Context, *NodeUnpublishVolumeRequest) (*NodeUnpublishVolumeResponse, error) - // NodeGetId is being deprecated in favor of NodeGetInfo and will be - // removed in CSI 1.0. Existing drivers, however, may depend on this - // RPC call and hence this RPC call MUST be implemented by the CSI - // plugin prior to v1.0. - NodeGetId(context.Context, *NodeGetIdRequest) (*NodeGetIdResponse, error) + NodeGetVolumeStats(context.Context, *NodeGetVolumeStatsRequest) (*NodeGetVolumeStatsResponse, error) NodeGetCapabilities(context.Context, *NodeGetCapabilitiesRequest) (*NodeGetCapabilitiesResponse, error) - // Prior to CSI 1.0 - CSI plugins MUST implement both NodeGetId and - // NodeGetInfo RPC calls. NodeGetInfo(context.Context, *NodeGetInfoRequest) (*NodeGetInfoResponse, error) } @@ -4647,7 +4923,7 @@ func _Node_NodeStageVolume_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Node/NodeStageVolume", + FullMethod: "/csi.v1.Node/NodeStageVolume", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServer).NodeStageVolume(ctx, req.(*NodeStageVolumeRequest)) @@ -4665,7 +4941,7 @@ func _Node_NodeUnstageVolume_Handler(srv interface{}, ctx context.Context, dec f } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Node/NodeUnstageVolume", + FullMethod: "/csi.v1.Node/NodeUnstageVolume", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServer).NodeUnstageVolume(ctx, req.(*NodeUnstageVolumeRequest)) @@ -4683,7 +4959,7 @@ func _Node_NodePublishVolume_Handler(srv interface{}, ctx context.Context, dec f } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Node/NodePublishVolume", + FullMethod: "/csi.v1.Node/NodePublishVolume", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServer).NodePublishVolume(ctx, req.(*NodePublishVolumeRequest)) @@ -4701,7 +4977,7 @@ func _Node_NodeUnpublishVolume_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Node/NodeUnpublishVolume", + FullMethod: "/csi.v1.Node/NodeUnpublishVolume", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServer).NodeUnpublishVolume(ctx, req.(*NodeUnpublishVolumeRequest)) @@ -4709,20 +4985,20 @@ func _Node_NodeUnpublishVolume_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } -func _Node_NodeGetId_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(NodeGetIdRequest) +func _Node_NodeGetVolumeStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeGetVolumeStatsRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(NodeServer).NodeGetId(ctx, in) + return srv.(NodeServer).NodeGetVolumeStats(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Node/NodeGetId", + FullMethod: "/csi.v1.Node/NodeGetVolumeStats", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(NodeServer).NodeGetId(ctx, req.(*NodeGetIdRequest)) + return srv.(NodeServer).NodeGetVolumeStats(ctx, req.(*NodeGetVolumeStatsRequest)) } return interceptor(ctx, in, info, handler) } @@ -4737,7 +5013,7 @@ func _Node_NodeGetCapabilities_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Node/NodeGetCapabilities", + FullMethod: "/csi.v1.Node/NodeGetCapabilities", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServer).NodeGetCapabilities(ctx, req.(*NodeGetCapabilitiesRequest)) @@ -4755,7 +5031,7 @@ func _Node_NodeGetInfo_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/csi.v0.Node/NodeGetInfo", + FullMethod: "/csi.v1.Node/NodeGetInfo", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServer).NodeGetInfo(ctx, req.(*NodeGetInfoRequest)) @@ -4764,7 +5040,7 @@ func _Node_NodeGetInfo_Handler(srv interface{}, ctx context.Context, dec func(in } var _Node_serviceDesc = grpc.ServiceDesc{ - ServiceName: "csi.v0.Node", + ServiceName: "csi.v1.Node", HandlerType: (*NodeServer)(nil), Methods: []grpc.MethodDesc{ { @@ -4784,8 +5060,8 @@ var _Node_serviceDesc = grpc.ServiceDesc{ Handler: _Node_NodeUnpublishVolume_Handler, }, { - MethodName: "NodeGetId", - Handler: _Node_NodeGetId_Handler, + MethodName: "NodeGetVolumeStats", + Handler: _Node_NodeGetVolumeStats_Handler, }, { MethodName: "NodeGetCapabilities", @@ -4797,195 +5073,205 @@ var _Node_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "csi.proto", + Metadata: "github.com/container-storage-interface/spec/csi.proto", } -func init() { proto.RegisterFile("csi.proto", fileDescriptor_csi_31237507707d37ec) } - -var fileDescriptor_csi_31237507707d37ec = []byte{ - // 2932 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x1a, 0x4d, 0x73, 0x23, 0x47, - 0xd5, 0xa3, 0x0f, 0xdb, 0x7a, 0x5e, 0x3b, 0xda, 0xf6, 0x97, 0x3c, 0xb6, 0x77, 0xbd, 0xb3, 0xd9, - 0x64, 0x13, 0x12, 0x6d, 0x30, 0x24, 0x15, 0x92, 0x4d, 0x40, 0x96, 0x15, 0x5b, 0x59, 0x5b, 0x36, - 0x23, 0xd9, 0xa9, 0x5d, 0x42, 0x4d, 0xc6, 0x52, 0x5b, 0x3b, 0xac, 0x3c, 0xa3, 0xcc, 0x8c, 0xcc, - 0x9a, 0x1b, 0x70, 0x01, 0x4e, 0xf0, 0x0b, 0x52, 0x95, 0x1b, 0x14, 0xb9, 0x50, 0xdc, 0xa8, 0xe2, - 0x46, 0x15, 0x27, 0xce, 0x9c, 0xb8, 0xa7, 0xe0, 0xc8, 0x89, 0x2a, 0xaa, 0xa8, 0x9e, 0xee, 0x19, - 0x4d, 0xb7, 0x7a, 0xf4, 0x91, 0xdd, 0x4a, 0x71, 0x92, 0xe6, 0x7d, 0xf5, 0xeb, 0xd7, 0xef, 0xbd, - 0x7e, 0xef, 0xcd, 0x40, 0xae, 0xe9, 0x59, 0xc5, 0xae, 0xeb, 0xf8, 0x0e, 0x9a, 0x26, 0x7f, 0x2f, - 0xdf, 0x50, 0x6f, 0xb4, 0x1d, 0xa7, 0xdd, 0xc1, 0xf7, 0x02, 0xe8, 0x59, 0xef, 0xfc, 0xde, 0x8f, - 0x5d, 0xb3, 0xdb, 0xc5, 0xae, 0x47, 0xe9, 0xb4, 0x15, 0x58, 0xda, 0xc3, 0xfe, 0x71, 0xa7, 0xd7, - 0xb6, 0xec, 0xaa, 0x7d, 0xee, 0xe8, 0xf8, 0xd3, 0x1e, 0xf6, 0x7c, 0xed, 0xef, 0x0a, 0x2c, 0x0b, - 0x08, 0xaf, 0xeb, 0xd8, 0x1e, 0x46, 0x08, 0x32, 0xb6, 0x79, 0x81, 0x0b, 0xca, 0x96, 0x72, 0x37, - 0xa7, 0x07, 0xff, 0xd1, 0x1d, 0x58, 0xb8, 0xc4, 0x76, 0xcb, 0x71, 0x8d, 0x4b, 0xec, 0x7a, 0x96, - 0x63, 0x17, 0x52, 0x01, 0x76, 0x9e, 0x42, 0x4f, 0x29, 0x10, 0xed, 0xc1, 0xec, 0x85, 0x69, 0x5b, - 0xe7, 0xd8, 0xf3, 0x0b, 0xe9, 0xad, 0xf4, 0xdd, 0xb9, 0xed, 0x6f, 0x14, 0xa9, 0x9e, 0x45, 0xe9, - 0x5a, 0xc5, 0x43, 0x46, 0x5d, 0xb1, 0x7d, 0xf7, 0x4a, 0x8f, 0x98, 0xd5, 0x77, 0x61, 0x9e, 0x43, - 0xa1, 0x3c, 0xa4, 0x9f, 0xe0, 0x2b, 0xa6, 0x13, 0xf9, 0x8b, 0x96, 0x20, 0x7b, 0x69, 0x76, 0x7a, - 0x98, 0x69, 0x42, 0x1f, 0xde, 0x49, 0xbd, 0xad, 0x68, 0x37, 0x60, 0x23, 0x5a, 0xad, 0x6c, 0x76, - 0xcd, 0x33, 0xab, 0x63, 0xf9, 0x16, 0xf6, 0xc2, 0xad, 0xff, 0x10, 0x36, 0x13, 0xf0, 0xcc, 0x02, - 0xf7, 0xe1, 0x5a, 0x33, 0x06, 0x2f, 0xa4, 0x82, 0xad, 0x14, 0xc2, 0xad, 0x08, 0x9c, 0x57, 0x3a, - 0x47, 0xad, 0xfd, 0x53, 0x81, 0xbc, 0x48, 0x82, 0xee, 0xc3, 0x8c, 0x87, 0xdd, 0x4b, 0xab, 0x49, - 0xed, 0x3a, 0xb7, 0xbd, 0x95, 0x24, 0xad, 0x58, 0xa7, 0x74, 0xfb, 0x53, 0x7a, 0xc8, 0xa2, 0xfe, - 0x5a, 0x81, 0x19, 0x06, 0x46, 0xdf, 0x81, 0x8c, 0x7f, 0xd5, 0xa5, 0x62, 0x16, 0xb6, 0xef, 0x8c, - 0x12, 0x53, 0x6c, 0x5c, 0x75, 0xb1, 0x1e, 0xb0, 0x68, 0x1f, 0x42, 0x86, 0x3c, 0xa1, 0x39, 0x98, - 0x39, 0xa9, 0x3d, 0xa8, 0x1d, 0x7d, 0x54, 0xcb, 0x4f, 0xa1, 0x15, 0x40, 0xe5, 0xa3, 0x5a, 0x43, - 0x3f, 0x3a, 0x38, 0xa8, 0xe8, 0x46, 0xbd, 0xa2, 0x9f, 0x56, 0xcb, 0x95, 0xbc, 0x82, 0x36, 0x61, - 0xad, 0x54, 0x2e, 0x57, 0xea, 0xf5, 0xea, 0x4e, 0xf5, 0xa0, 0xda, 0x78, 0x68, 0x94, 0x8f, 0x6a, - 0xf5, 0x86, 0x5e, 0xaa, 0xd6, 0x1a, 0xf5, 0x7c, 0x6a, 0x67, 0x9a, 0xaa, 0xa1, 0x2d, 0xc0, 0xb5, - 0x63, 0xd7, 0x39, 0xc3, 0xa1, 0x71, 0x4b, 0x30, 0xcf, 0x9e, 0x99, 0x31, 0xdf, 0x80, 0xac, 0x8b, - 0xcd, 0xd6, 0x15, 0xdb, 0xb7, 0x5a, 0xa4, 0x0e, 0x5b, 0x0c, 0x1d, 0xb6, 0xb8, 0xe3, 0x38, 0x9d, - 0x53, 0x72, 0x78, 0x3a, 0x25, 0xd4, 0xbe, 0xc8, 0xc2, 0x62, 0xd9, 0xc5, 0xa6, 0x8f, 0x4f, 0x9d, - 0x4e, 0xef, 0x22, 0x14, 0x2d, 0x75, 0xcc, 0xfb, 0xb0, 0x40, 0x8c, 0xdf, 0xb4, 0xfc, 0x2b, 0xc3, - 0x35, 0xed, 0x36, 0x75, 0x87, 0xb9, 0xed, 0xe5, 0xd0, 0x2e, 0x65, 0x86, 0xd5, 0x09, 0x52, 0x9f, - 0x6f, 0xc6, 0x1f, 0x51, 0x15, 0x16, 0x2f, 0x83, 0x25, 0x0c, 0xee, 0xbc, 0xd3, 0xfc, 0x79, 0x53, - 0x2d, 0x62, 0xe7, 0x8d, 0x2e, 0x79, 0x88, 0x85, 0x3d, 0xf4, 0x00, 0xa0, 0x6b, 0xba, 0xe6, 0x05, - 0xf6, 0xb1, 0xeb, 0x15, 0x32, 0xbc, 0xf3, 0x4b, 0x76, 0x53, 0x3c, 0x8e, 0xa8, 0xa9, 0xf3, 0xc7, - 0xd8, 0x91, 0x0f, 0x6b, 0x4d, 0xc7, 0xf6, 0x5d, 0xa7, 0xd3, 0xc1, 0xae, 0xd1, 0x0c, 0xb8, 0x0d, - 0x0f, 0x37, 0x5d, 0xec, 0x7b, 0x85, 0x6c, 0x20, 0xfb, 0xed, 0x61, 0xb2, 0xcb, 0x11, 0x33, 0xc5, - 0xd6, 0x29, 0x2b, 0x5d, 0x68, 0xb5, 0x29, 0xc7, 0xa2, 0x23, 0x58, 0x0e, 0xad, 0xe1, 0xd8, 0x3e, - 0xb6, 0x7d, 0xc3, 0x73, 0x7a, 0x6e, 0x13, 0x17, 0xa6, 0x03, 0x93, 0xae, 0x0b, 0xf6, 0xa0, 0x34, - 0xf5, 0x80, 0x44, 0x67, 0x76, 0xe4, 0x80, 0xe8, 0x11, 0xa8, 0x66, 0xb3, 0x89, 0x3d, 0xcf, 0xa2, - 0x86, 0x33, 0x5c, 0xfc, 0x69, 0xcf, 0x72, 0xf1, 0x05, 0xb6, 0x7d, 0xaf, 0x30, 0xc3, 0x4b, 0x6d, - 0x38, 0x5d, 0xa7, 0xe3, 0xb4, 0xaf, 0xf4, 0x3e, 0x8d, 0xbe, 0xc6, 0xb1, 0xc7, 0x30, 0x9e, 0xfa, - 0x1e, 0xbc, 0x20, 0x58, 0x70, 0x92, 0x1c, 0xa1, 0x7e, 0x08, 0x1b, 0xc3, 0x8c, 0x34, 0x51, 0xbe, - 0xf9, 0xa5, 0x02, 0x8b, 0x12, 0x9b, 0xa0, 0x7d, 0x98, 0xf5, 0x6c, 0xb3, 0xeb, 0x3d, 0x76, 0x7c, - 0xe6, 0xfc, 0xaf, 0x0e, 0x31, 0x61, 0xb1, 0xce, 0x68, 0xe9, 0xe3, 0xfe, 0x94, 0x1e, 0x71, 0xab, - 0x5b, 0xb0, 0xc0, 0x63, 0xd1, 0x02, 0xa4, 0xac, 0x16, 0x53, 0x2f, 0x65, 0xb5, 0xa2, 0x70, 0x7c, - 0x1f, 0x96, 0x78, 0x87, 0x60, 0x51, 0xf8, 0x12, 0x4c, 0xd3, 0x13, 0x62, 0x9a, 0x2c, 0xf0, 0x9a, - 0xe8, 0x0c, 0xab, 0xfd, 0x2e, 0x03, 0x79, 0xd1, 0xdf, 0xd1, 0x7d, 0xc8, 0x9e, 0x75, 0x9c, 0xe6, - 0x13, 0xc6, 0xfb, 0x62, 0x52, 0x60, 0x14, 0x77, 0x08, 0x15, 0x85, 0xee, 0x4f, 0xe9, 0x94, 0x89, - 0x70, 0x5f, 0x38, 0x3d, 0xdb, 0x67, 0x91, 0x99, 0xcc, 0x7d, 0x48, 0xa8, 0xfa, 0xdc, 0x01, 0x13, - 0xda, 0x85, 0x39, 0xea, 0x04, 0xc6, 0x85, 0xd3, 0xc2, 0x85, 0x74, 0x20, 0xe3, 0x76, 0xa2, 0x8c, - 0x52, 0x40, 0x7b, 0xe8, 0xb4, 0xb0, 0x0e, 0x66, 0xf4, 0x5f, 0x9d, 0x87, 0xb9, 0x98, 0x6e, 0xea, - 0x1e, 0xcc, 0xc5, 0x16, 0x43, 0xab, 0x30, 0x73, 0xee, 0x19, 0x51, 0x56, 0xcd, 0xe9, 0xd3, 0xe7, - 0x5e, 0x90, 0x28, 0x6f, 0xc2, 0x5c, 0xa0, 0x85, 0x71, 0xde, 0x31, 0xdb, 0xf4, 0x1e, 0xc8, 0xe9, - 0x10, 0x80, 0x3e, 0x20, 0x10, 0xf5, 0x5f, 0x0a, 0x40, 0x7f, 0x49, 0x74, 0x1f, 0x32, 0x81, 0x96, - 0x34, 0x37, 0xdf, 0x1d, 0x43, 0xcb, 0x62, 0xa0, 0x6a, 0xc0, 0xa5, 0x7d, 0xa6, 0x40, 0x26, 0x10, - 0x23, 0xe6, 0xe7, 0x7a, 0xb5, 0xb6, 0x77, 0x50, 0x31, 0x6a, 0x47, 0xbb, 0x15, 0xe3, 0x23, 0xbd, - 0xda, 0xa8, 0xe8, 0x79, 0x05, 0xad, 0xc3, 0x6a, 0x1c, 0xae, 0x57, 0x4a, 0xbb, 0x15, 0xdd, 0x38, - 0xaa, 0x1d, 0x3c, 0xcc, 0xa7, 0x90, 0x0a, 0x2b, 0x87, 0x27, 0x07, 0x8d, 0xea, 0x20, 0x2e, 0x8d, - 0x36, 0xa0, 0x10, 0xc3, 0x31, 0x19, 0x4c, 0x6c, 0x86, 0x88, 0x8d, 0x61, 0xe9, 0x5f, 0x86, 0xcc, - 0xee, 0xcc, 0x47, 0x87, 0x11, 0x38, 0xdb, 0x47, 0x30, 0xcf, 0xa5, 0x57, 0x52, 0x26, 0xb0, 0x10, - 0x6f, 0x19, 0x67, 0x57, 0x3e, 0xf6, 0x02, 0x4b, 0xa4, 0xf5, 0xf9, 0x10, 0xba, 0x43, 0x80, 0xc4, - 0xac, 0x1d, 0xeb, 0xc2, 0xf2, 0x19, 0x4d, 0x2a, 0xa0, 0x81, 0x00, 0x14, 0x10, 0x68, 0x7f, 0x49, - 0xc1, 0x34, 0x3b, 0x9b, 0x3b, 0xb1, 0x04, 0xcf, 0x89, 0x0c, 0xa1, 0x54, 0x24, 0x8d, 0x87, 0x54, - 0x18, 0x0f, 0xe8, 0x7d, 0x00, 0xd3, 0xf7, 0x5d, 0xeb, 0xac, 0xe7, 0x47, 0x09, 0xfd, 0x06, 0x7f, - 0x1e, 0xc5, 0x52, 0x44, 0xc0, 0x32, 0x70, 0x9f, 0x03, 0xed, 0xc0, 0x82, 0x90, 0x04, 0x33, 0xa3, - 0x93, 0xe0, 0x7c, 0x93, 0x8b, 0xff, 0x12, 0x2c, 0x86, 0xf9, 0xab, 0x83, 0x0d, 0x9f, 0xe5, 0x37, - 0x96, 0xbf, 0xf3, 0x03, 0x79, 0x0f, 0xf5, 0x89, 0x43, 0x18, 0xc9, 0x72, 0x82, 0x96, 0x13, 0x65, - 0xa6, 0x1e, 0x2c, 0x4a, 0xd2, 0x2a, 0x2a, 0x42, 0x2e, 0x38, 0x10, 0xcf, 0xf2, 0x89, 0xaf, 0xca, - 0xd5, 0xe9, 0x93, 0x10, 0xfa, 0xae, 0x8b, 0xcf, 0xb1, 0xeb, 0xe2, 0x16, 0x2b, 0x86, 0x24, 0xf4, - 0x11, 0x89, 0xf6, 0x73, 0x05, 0x66, 0x43, 0x38, 0x7a, 0x07, 0x66, 0x3d, 0xdc, 0xa6, 0x29, 0x5f, - 0xe1, 0xcf, 0x21, 0xa4, 0x29, 0xd6, 0x19, 0x01, 0x2b, 0x03, 0x43, 0x7a, 0x52, 0x06, 0x72, 0xa8, - 0x89, 0x36, 0xff, 0x6f, 0x05, 0x16, 0x77, 0x71, 0x07, 0x8b, 0x65, 0xc4, 0x3a, 0xe4, 0xd8, 0x35, - 0x17, 0x65, 0xd0, 0x59, 0x0a, 0xa8, 0xb6, 0x84, 0x9b, 0xb7, 0x15, 0xb0, 0x47, 0x37, 0x6f, 0x8a, - 0xbf, 0x79, 0x25, 0xc2, 0x63, 0x37, 0x2f, 0xc5, 0x26, 0xdd, 0xbc, 0x1c, 0x96, 0xbf, 0x8d, 0x06, - 0x19, 0x27, 0xda, 0xf6, 0x0a, 0x2c, 0xf1, 0x8a, 0xd1, 0x1b, 0x40, 0xfb, 0x53, 0x06, 0x6e, 0xf4, - 0x17, 0x39, 0xee, 0x9d, 0x75, 0x2c, 0xef, 0xf1, 0x04, 0x96, 0x59, 0x85, 0x19, 0xdb, 0x69, 0x05, - 0x28, 0xba, 0xe6, 0x34, 0x79, 0xac, 0xb6, 0x50, 0x05, 0xae, 0x8b, 0x45, 0xd4, 0x15, 0xcb, 0xd3, - 0xc9, 0x25, 0x54, 0xfe, 0x52, 0xbc, 0x64, 0x54, 0x98, 0x25, 0xe5, 0x9f, 0x63, 0x77, 0xae, 0x82, - 0x58, 0x9b, 0xd5, 0xa3, 0x67, 0xf4, 0x33, 0x05, 0xd4, 0xd8, 0xb1, 0x74, 0xa9, 0xf2, 0x42, 0x45, - 0xb4, 0x1b, 0x55, 0x44, 0x43, 0x77, 0x39, 0x88, 0xe6, 0xce, 0xa8, 0xd0, 0x4c, 0x40, 0x23, 0x2b, - 0xda, 0x67, 0x2c, 0xb3, 0x4c, 0x07, 0x4b, 0xdf, 0x1f, 0x73, 0x69, 0xfa, 0x24, 0xe6, 0x1d, 0x66, - 0x8b, 0x3e, 0x58, 0x7d, 0x00, 0x9b, 0x43, 0xb5, 0x9c, 0xa8, 0xd4, 0x29, 0xc3, 0xb2, 0x74, 0xdd, - 0x89, 0xbc, 0xea, 0xcf, 0x0a, 0xdc, 0x4c, 0xdc, 0x1c, 0xab, 0x31, 0x7e, 0x00, 0xd7, 0xc2, 0x93, - 0xb1, 0xec, 0x73, 0x87, 0x45, 0xfb, 0xdb, 0x23, 0x6d, 0xc3, 0x7a, 0x41, 0x06, 0x25, 0xfd, 0x21, - 0xb5, 0xcb, 0x5c, 0xb7, 0x0f, 0x51, 0xdf, 0x87, 0xbc, 0x48, 0x30, 0xd1, 0x06, 0xfe, 0x98, 0x82, - 0xad, 0xbe, 0x06, 0x27, 0x76, 0xf7, 0xf9, 0x05, 0xc0, 0xaf, 0x14, 0xd8, 0x88, 0x79, 0x67, 0xcf, - 0x16, 0xfd, 0x93, 0x5e, 0x3f, 0xfb, 0x83, 0x86, 0x90, 0xab, 0x21, 0x23, 0xe0, 0x7c, 0x34, 0x16, - 0x0b, 0x22, 0x81, 0x7a, 0x18, 0x3f, 0x27, 0x29, 0xfb, 0x44, 0x66, 0xbb, 0x0d, 0xb7, 0x86, 0xa8, - 0xcb, 0x52, 0xcb, 0x4f, 0xd3, 0x70, 0xeb, 0xd4, 0xec, 0x58, 0xad, 0xa8, 0xee, 0x94, 0xb4, 0xdd, - 0xc3, 0x8d, 0x9b, 0xd0, 0x89, 0xa5, 0xbe, 0x42, 0x27, 0xd6, 0x91, 0xc5, 0x29, 0x3d, 0x82, 0xef, - 0x46, 0x82, 0x46, 0x69, 0x3b, 0x6e, 0xa8, 0x26, 0x5d, 0xf2, 0x99, 0x09, 0x2e, 0xf9, 0xe7, 0x12, - 0xa0, 0x1f, 0x83, 0x36, 0x6c, 0x53, 0x2c, 0x44, 0x37, 0x20, 0xe7, 0xf5, 0xba, 0x5d, 0xc7, 0xf5, - 0x31, 0x3d, 0x83, 0x59, 0xbd, 0x0f, 0x40, 0x05, 0x98, 0xb9, 0xc0, 0x9e, 0x67, 0xb6, 0x43, 0xf9, - 0xe1, 0xa3, 0xf6, 0x31, 0xa0, 0x03, 0xcb, 0x63, 0xf5, 0x72, 0x74, 0xa2, 0xa4, 0x3c, 0x36, 0x9f, - 0x1a, 0xd8, 0xf6, 0x5d, 0x8b, 0x15, 0x66, 0x59, 0x1d, 0x2e, 0xcc, 0xa7, 0x15, 0x0a, 0x21, 0xc5, - 0x9b, 0xe7, 0x9b, 0xae, 0x6f, 0xd9, 0x6d, 0xc3, 0x77, 0x9e, 0xe0, 0x68, 0x6c, 0x14, 0x42, 0x1b, - 0x04, 0xa8, 0x7d, 0xae, 0xc0, 0x22, 0x27, 0x9e, 0x69, 0xfb, 0x2e, 0xcc, 0xf4, 0x65, 0x13, 0x7b, - 0xde, 0x0a, 0xed, 0x29, 0xa1, 0x2e, 0xd2, 0x13, 0x0a, 0x39, 0xd0, 0x26, 0x80, 0x8d, 0x9f, 0xfa, - 0xdc, 0xba, 0x39, 0x02, 0x09, 0xd6, 0x54, 0xef, 0x41, 0x96, 0x1a, 0x79, 0xdc, 0xce, 0xe8, 0x8b, - 0x14, 0xa0, 0x3d, 0xec, 0x47, 0x05, 0x2f, 0xb3, 0x41, 0x82, 0xe3, 0x2a, 0x5f, 0xc1, 0x71, 0x3f, - 0xe4, 0x46, 0x08, 0xd4, 0xf5, 0x5f, 0x8d, 0xcd, 0xcf, 0x84, 0xa5, 0x87, 0x4e, 0x10, 0x12, 0xdc, - 0x92, 0x5e, 0xcb, 0x63, 0xd7, 0x9e, 0xcf, 0xd0, 0x61, 0x6b, 0xbb, 0xb0, 0xc8, 0xe9, 0xcc, 0xce, - 0xf4, 0x75, 0x40, 0xe6, 0xa5, 0x69, 0x75, 0x4c, 0xa2, 0x57, 0x58, 0xc3, 0xb3, 0x9a, 0xfe, 0x7a, - 0x84, 0x09, 0xd9, 0x34, 0x2d, 0x9e, 0xb5, 0x99, 0x3c, 0x71, 0x9e, 0xd7, 0x89, 0xe7, 0xa8, 0x01, - 0x1a, 0xb6, 0xee, 0x9e, 0x74, 0xa6, 0x77, 0x7b, 0x30, 0x27, 0xb3, 0xb9, 0x59, 0xe2, 0x78, 0xef, - 0x6f, 0x29, 0x58, 0x1f, 0x42, 0x8d, 0xde, 0x85, 0xb4, 0xdb, 0x6d, 0x32, 0x67, 0x7a, 0x79, 0x0c, - 0xf9, 0x45, 0xfd, 0xb8, 0xbc, 0x3f, 0xa5, 0x13, 0x2e, 0xf5, 0x4b, 0x05, 0xd2, 0xfa, 0x71, 0x19, - 0x7d, 0x8f, 0x1b, 0xf2, 0xbd, 0x36, 0xa6, 0x94, 0xf8, 0xac, 0x8f, 0x34, 0x93, 0x83, 0xc3, 0xbe, - 0x02, 0x2c, 0x95, 0xf5, 0x4a, 0xa9, 0x51, 0x31, 0x76, 0x2b, 0x07, 0x95, 0x46, 0xc5, 0x38, 0x3d, - 0x3a, 0x38, 0x39, 0xac, 0xe4, 0x15, 0xd2, 0x15, 0x1e, 0x9f, 0xec, 0x1c, 0x54, 0xeb, 0xfb, 0xc6, - 0x49, 0x2d, 0xfc, 0xc7, 0xb0, 0x29, 0x94, 0x87, 0x6b, 0x07, 0xd5, 0x7a, 0x83, 0x01, 0xea, 0xf9, - 0x34, 0x81, 0xec, 0x55, 0x1a, 0x46, 0xb9, 0x74, 0x5c, 0x2a, 0x57, 0x1b, 0x0f, 0xf3, 0x19, 0xd2, - 0x73, 0xf2, 0xb2, 0xeb, 0xb5, 0xd2, 0x71, 0x7d, 0xff, 0xa8, 0x91, 0xcf, 0x22, 0x04, 0x0b, 0x01, - 0x7f, 0x08, 0xaa, 0xe7, 0xa7, 0xa3, 0x91, 0xc5, 0x67, 0x69, 0x58, 0x66, 0x13, 0x18, 0x36, 0xe3, - 0x08, 0x63, 0xeb, 0x2e, 0xe4, 0x69, 0xf3, 0x65, 0x88, 0x17, 0xc7, 0x02, 0x85, 0x9f, 0x86, 0xd7, - 0x47, 0x38, 0x1a, 0x4c, 0xc5, 0x46, 0x83, 0x5d, 0x58, 0x0d, 0x27, 0x67, 0x4c, 0xae, 0x70, 0x21, - 0x0b, 0x23, 0x34, 0x61, 0x75, 0x01, 0xca, 0x5d, 0xc0, 0xcb, 0x4d, 0x19, 0x0e, 0x1d, 0x4a, 0x66, - 0x80, 0xaf, 0x0f, 0x5f, 0x64, 0x48, 0x0c, 0xab, 0xfb, 0xa0, 0x26, 0xeb, 0x30, 0x51, 0x09, 0xf8, - 0x8c, 0xa1, 0xfc, 0x01, 0xac, 0x88, 0xda, 0xb3, 0xa8, 0x7a, 0x6d, 0x60, 0xc4, 0x15, 0xe5, 0x96, - 0x88, 0x36, 0xa2, 0xd0, 0xfe, 0xa0, 0xc0, 0x6c, 0x08, 0x26, 0xf9, 0xd9, 0xb3, 0x7e, 0x82, 0xb9, - 0xa6, 0x3e, 0x47, 0x20, 0xf2, 0x86, 0x5e, 0xe6, 0x0b, 0x69, 0xa9, 0x2f, 0x6c, 0x02, 0xd0, 0xe3, - 0x69, 0x19, 0xa6, 0x1f, 0xb4, 0x12, 0x69, 0x3d, 0xc7, 0x20, 0x25, 0xd2, 0xfc, 0x4e, 0x7b, 0xbe, - 0xe9, 0xf7, 0x48, 0xdb, 0x40, 0x14, 0x5e, 0x11, 0x15, 0xae, 0x07, 0x58, 0x9d, 0x51, 0x91, 0x40, - 0x5a, 0xe0, 0x51, 0xe8, 0x1e, 0x17, 0x9d, 0xeb, 0x72, 0x01, 0xb1, 0x60, 0x24, 0x17, 0x6b, 0x0b, - 0xfb, 0xa6, 0xd5, 0xf1, 0xc2, 0x8b, 0x95, 0x3d, 0x6a, 0x3b, 0xb2, 0x28, 0xcd, 0x41, 0x56, 0xaf, - 0x94, 0x76, 0x1f, 0xe6, 0x15, 0x34, 0x0f, 0xb9, 0x93, 0xe3, 0x83, 0xa3, 0xd2, 0x6e, 0xb5, 0xb6, - 0x97, 0x4f, 0xa1, 0x45, 0x78, 0xa1, 0xa2, 0xeb, 0x47, 0xba, 0xd1, 0x07, 0xa6, 0x49, 0xa3, 0xbb, - 0xcc, 0x9a, 0x46, 0x21, 0x80, 0x6e, 0xc2, 0x5c, 0xe4, 0xfb, 0x51, 0xec, 0x40, 0x08, 0xaa, 0xb6, - 0x48, 0x8c, 0x84, 0x3d, 0xae, 0x18, 0x23, 0xd2, 0x66, 0x57, 0x74, 0x5f, 0x1e, 0xca, 0xc7, 0x48, - 0x4b, 0x86, 0x23, 0x4e, 0x9d, 0xcc, 0x34, 0x91, 0x57, 0x16, 0x60, 0x45, 0x54, 0x8a, 0xd5, 0xa3, - 0xbf, 0x55, 0x60, 0x89, 0x54, 0x08, 0x21, 0xe2, 0x79, 0x17, 0x2c, 0x13, 0x38, 0xa3, 0x70, 0x02, - 0x19, 0xf1, 0x04, 0xb4, 0xdf, 0x2b, 0xb0, 0x2c, 0xe8, 0xca, 0x62, 0xeb, 0x3d, 0xb1, 0xfa, 0xb9, - 0x1d, 0xaf, 0x7e, 0x06, 0xe8, 0x27, 0xac, 0x7f, 0xde, 0x0c, 0xeb, 0x9f, 0xc9, 0x42, 0xf8, 0x37, - 0x59, 0x58, 0xa9, 0x39, 0x2d, 0x5c, 0xf7, 0xcd, 0xf6, 0x24, 0x73, 0x15, 0x5d, 0xe8, 0x0d, 0xa9, - 0x77, 0xdd, 0x0b, 0x57, 0x92, 0x8b, 0x1c, 0xde, 0x12, 0xa2, 0x22, 0x2c, 0x7a, 0xbe, 0xd9, 0x0e, - 0xce, 0xca, 0x74, 0xdb, 0xd8, 0x37, 0xba, 0xa6, 0xff, 0x98, 0x1d, 0xc4, 0x75, 0x86, 0x6a, 0x04, - 0x98, 0x63, 0xd3, 0x7f, 0x2c, 0x1f, 0x54, 0x64, 0x26, 0x1e, 0x54, 0x9c, 0x01, 0x0a, 0xfa, 0x40, - 0xb2, 0x80, 0xf8, 0x56, 0xe6, 0xdb, 0x23, 0x36, 0x14, 0x81, 0xb9, 0x50, 0xc9, 0xdb, 0x02, 0x18, - 0x99, 0xc9, 0xb3, 0x86, 0x51, 0x4b, 0x8c, 0x3b, 0x63, 0x78, 0xc6, 0x86, 0x9a, 0x74, 0x2d, 0xd2, - 0xdd, 0x7c, 0xfd, 0xb3, 0x89, 0x35, 0x58, 0x1d, 0xb0, 0x05, 0xcb, 0x04, 0x6d, 0x28, 0x10, 0xd4, - 0x89, 0xed, 0x4d, 0xe8, 0xaf, 0x09, 0xbe, 0x95, 0x4a, 0xf0, 0x2d, 0x6d, 0x1d, 0xd6, 0x24, 0x0b, - 0x31, 0x2d, 0xfe, 0x91, 0xa5, 0x6a, 0x4c, 0x3e, 0x74, 0x6b, 0x48, 0xc3, 0xe6, 0x9b, 0x71, 0x17, - 0x90, 0x0e, 0x9a, 0x9e, 0x6f, 0xe0, 0xdc, 0x84, 0xb9, 0x38, 0x1d, 0x4b, 0x62, 0xfe, 0x88, 0xc8, - 0xca, 0x3e, 0xd3, 0x08, 0x70, 0x5a, 0x18, 0x01, 0xfe, 0x08, 0x96, 0x82, 0xa8, 0x13, 0x67, 0x2b, - 0x33, 0xfc, 0x35, 0x95, 0x68, 0x91, 0x18, 0x82, 0x8b, 0xbd, 0x20, 0x96, 0x85, 0x49, 0x5f, 0x53, - 0x16, 0x7d, 0xb3, 0xc1, 0x42, 0x6f, 0x8d, 0x5c, 0xe8, 0xeb, 0x8a, 0xbf, 0x0a, 0xf5, 0xfa, 0xff, - 0x8b, 0xe9, 0x20, 0xf3, 0x7e, 0xe9, 0x5c, 0x4f, 0x7b, 0x04, 0x2a, 0x0d, 0x8d, 0xc9, 0x47, 0x6e, - 0x82, 0xe3, 0xa5, 0x44, 0xc7, 0xd3, 0x36, 0x61, 0x5d, 0x2a, 0x9b, 0x2d, 0x8d, 0x20, 0x4f, 0xd0, - 0x7b, 0xd8, 0xaf, 0xb6, 0xc2, 0x6e, 0xf1, 0x35, 0xb8, 0x1e, 0x83, 0xb1, 0xbb, 0x36, 0x36, 0xdb, - 0x53, 0xe2, 0xb3, 0x3d, 0x6d, 0x83, 0x2a, 0x9f, 0xd0, 0x79, 0x7e, 0x42, 0x97, 0x4f, 0xea, 0x39, - 0x4b, 0x42, 0xcf, 0x49, 0xaf, 0xf1, 0x4d, 0x2e, 0x81, 0x8f, 0xe8, 0x36, 0xff, 0xaa, 0xb0, 0x34, - 0x3b, 0xd0, 0x67, 0xbe, 0x19, 0xef, 0x33, 0x6f, 0x0d, 0x95, 0x19, 0xef, 0x30, 0xbb, 0xb4, 0xc1, - 0x7c, 0x87, 0x2b, 0x61, 0x5f, 0x1a, 0xc9, 0x1e, 0x6f, 0x2d, 0x5f, 0x4f, 0xe8, 0x2c, 0xeb, 0x8d, - 0xd2, 0x5e, 0xc5, 0x38, 0xa9, 0xd1, 0xdf, 0xb0, 0xb3, 0x8c, 0xfa, 0xbc, 0x25, 0x40, 0xa1, 0xe1, - 0x63, 0xdf, 0x21, 0x7d, 0xae, 0xc0, 0x22, 0x07, 0x1e, 0x71, 0x22, 0xe8, 0x1e, 0x2c, 0x91, 0x1a, - 0x8e, 0xfa, 0x88, 0x67, 0x74, 0xb1, 0x6b, 0x10, 0x0c, 0x7b, 0x8b, 0x78, 0xfd, 0xc2, 0x7c, 0xca, - 0x06, 0x43, 0xc7, 0xd8, 0x25, 0x82, 0x9f, 0xc3, 0x28, 0x64, 0xfb, 0x3f, 0x0a, 0xcc, 0x56, 0x5b, - 0xd8, 0xf6, 0x89, 0xe1, 0x6b, 0x30, 0xcf, 0x7d, 0xcc, 0x84, 0x36, 0x12, 0xbe, 0x71, 0x0a, 0x36, - 0xa8, 0x6e, 0x0e, 0xfd, 0x02, 0x4a, 0x9b, 0x42, 0xe7, 0xb1, 0x0f, 0xb1, 0xb8, 0x79, 0xd0, 0x8b, - 0x03, 0x9c, 0x12, 0x1f, 0x54, 0xef, 0x8c, 0xa0, 0x8a, 0xd6, 0x79, 0x0b, 0xb2, 0xc1, 0x97, 0x39, - 0x68, 0x29, 0xfa, 0x66, 0x28, 0xf6, 0xe1, 0x8e, 0xba, 0x2c, 0x40, 0x43, 0xbe, 0xed, 0xff, 0xce, - 0x00, 0xf4, 0x07, 0x0f, 0xe8, 0x01, 0x5c, 0x8b, 0x7f, 0x61, 0x80, 0xd6, 0x87, 0x7c, 0x88, 0xa2, - 0x6e, 0xc8, 0x91, 0x91, 0x4e, 0x0f, 0xe0, 0x5a, 0xfc, 0x65, 0x55, 0x5f, 0x98, 0xe4, 0xdd, 0x5a, - 0x5f, 0x98, 0xf4, 0xfd, 0xd6, 0x14, 0xea, 0xc0, 0x6a, 0xc2, 0x3b, 0x06, 0xf4, 0xd2, 0x78, 0x2f, - 0x68, 0xd4, 0x97, 0xc7, 0x7c, 0x59, 0xa1, 0x4d, 0x21, 0x17, 0xd6, 0x12, 0x27, 0xe3, 0xe8, 0xee, - 0xb8, 0xb3, 0x7e, 0xf5, 0x95, 0x31, 0x28, 0xa3, 0x35, 0x7b, 0xa0, 0x26, 0x0f, 0x79, 0xd1, 0x2b, - 0x63, 0x4f, 0xb7, 0xd5, 0x57, 0xc7, 0x21, 0x8d, 0x96, 0xdd, 0x87, 0xb9, 0xd8, 0xc0, 0x15, 0xa9, - 0xd2, 0x29, 0x2c, 0x15, 0xbc, 0x3e, 0x64, 0x42, 0x4b, 0x25, 0xc5, 0x86, 0x82, 0x7d, 0x49, 0x83, - 0xd3, 0xcd, 0xbe, 0x24, 0xc9, 0x14, 0x51, 0x34, 0xbf, 0x90, 0x80, 0x65, 0xe6, 0x97, 0x67, 0x70, - 0x99, 0xf9, 0x13, 0xb2, 0xb9, 0x36, 0x85, 0xbe, 0x0f, 0x0b, 0xfc, 0x1c, 0x04, 0x6d, 0x0e, 0x9d, - 0xee, 0xa8, 0x37, 0x92, 0xd0, 0x71, 0x91, 0x7c, 0x13, 0xdb, 0x17, 0x29, 0xed, 0xb8, 0xfb, 0x22, - 0x13, 0x7a, 0xdf, 0x29, 0x92, 0x9f, 0xb8, 0x06, 0xb1, 0x9f, 0x9f, 0x64, 0x3d, 0x71, 0x3f, 0x3f, - 0x49, 0xbb, 0x4a, 0x6d, 0x6a, 0xfb, 0xcb, 0x0c, 0x64, 0x82, 0x44, 0xda, 0x80, 0x17, 0x84, 0x3a, - 0x1b, 0xdd, 0x18, 0xde, 0x8c, 0xa8, 0x37, 0x13, 0xf1, 0x91, 0xba, 0x8f, 0xe8, 0x7d, 0xcc, 0x55, - 0xce, 0x68, 0x2b, 0xce, 0x27, 0xab, 0xde, 0xd5, 0x5b, 0x43, 0x28, 0x44, 0xd9, 0x7c, 0x2e, 0xd8, - 0x1a, 0x55, 0xc2, 0xf1, 0xb2, 0x93, 0xe2, 0xff, 0x13, 0x7a, 0x6f, 0x89, 0x91, 0xaf, 0xf1, 0x7a, - 0x49, 0x63, 0xfe, 0xf6, 0x50, 0x9a, 0x68, 0x85, 0x0a, 0xe4, 0xa2, 0x4a, 0x05, 0x15, 0xe2, 0x3c, - 0xf1, 0x82, 0x46, 0x5d, 0x93, 0x60, 0x98, 0x8c, 0xf4, 0x2f, 0x52, 0x4a, 0xa8, 0xa8, 0x18, 0x23, - 0x9a, 0xc0, 0x26, 0x8b, 0x8e, 0xdb, 0x43, 0x69, 0xe2, 0x51, 0x1d, 0xbb, 0xc2, 0xfb, 0x51, 0x3d, - 0x78, 0xdd, 0xf7, 0xa3, 0x5a, 0x72, 0xe7, 0x6b, 0x53, 0x3b, 0xd9, 0x47, 0xe9, 0xa6, 0x67, 0x9d, - 0x4d, 0x07, 0x1f, 0x87, 0x7e, 0xeb, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x9c, 0x3b, 0x5a, 0x51, - 0xf0, 0x2c, 0x00, 0x00, +func init() { + proto.RegisterFile("github.com/container-storage-interface/spec/csi.proto", fileDescriptor_csi_1092d4f3f3c8dc30) +} + +var fileDescriptor_csi_1092d4f3f3c8dc30 = []byte{ + // 3070 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x1a, 0x4d, 0x6f, 0xe3, 0xc6, + 0xd5, 0xd4, 0x87, 0x3f, 0x9e, 0x6d, 0x45, 0x3b, 0xfe, 0x58, 0x2d, 0x6d, 0xef, 0x7a, 0xb9, 0xd9, + 0xc4, 0xd9, 0x64, 0xe5, 0xc6, 0xc9, 0x06, 0xcd, 0xee, 0xa6, 0x8d, 0x24, 0x2b, 0xb6, 0xb2, 0x5a, + 0xd9, 0xa1, 0x64, 0xa7, 0xbb, 0x6d, 0xc0, 0xd0, 0xd2, 0x58, 0x4b, 0x44, 0x22, 0x15, 0x72, 0xe4, + 0xae, 0x7b, 0x2b, 0x0a, 0xf4, 0xd2, 0x53, 0x4f, 0xed, 0xad, 0x40, 0x7a, 0x6a, 0xd1, 0xa0, 0xa7, + 0xa2, 0xc7, 0x02, 0xbd, 0x14, 0xe8, 0x1f, 0x68, 0x6f, 0xb9, 0x07, 0x2d, 0x10, 0xf4, 0xd0, 0x43, + 0x81, 0x02, 0x05, 0x39, 0x43, 0x8a, 0x43, 0x91, 0x94, 0xb4, 0xde, 0x20, 0x87, 0x9e, 0x24, 0xbe, + 0x79, 0x5f, 0xf3, 0xe6, 0xbd, 0x37, 0xef, 0x3d, 0x12, 0xee, 0xb4, 0x35, 0xf2, 0xa4, 0x7f, 0x92, + 0x6f, 0x1a, 0xdd, 0xed, 0xa6, 0xa1, 0x13, 0x55, 0xd3, 0xb1, 0x79, 0xdb, 0x22, 0x86, 0xa9, 0xb6, + 0xf1, 0x6d, 0x4d, 0x27, 0xd8, 0x3c, 0x55, 0x9b, 0x78, 0xdb, 0xea, 0xe1, 0xe6, 0x76, 0xd3, 0xd2, + 0xf2, 0x3d, 0xd3, 0x20, 0x06, 0x9a, 0xb6, 0xff, 0x9e, 0xbd, 0x2e, 0x6e, 0xb6, 0x0d, 0xa3, 0xdd, + 0xc1, 0xdb, 0x0e, 0xf4, 0xa4, 0x7f, 0xba, 0xdd, 0xc2, 0x56, 0xd3, 0xd4, 0x7a, 0xc4, 0x30, 0x29, + 0xa6, 0x78, 0x2d, 0x88, 0x41, 0xb4, 0x2e, 0xb6, 0x88, 0xda, 0xed, 0x31, 0x84, 0xab, 0x41, 0x84, + 0x1f, 0x9a, 0x6a, 0xaf, 0x87, 0x4d, 0x8b, 0xae, 0x4b, 0xab, 0xb0, 0xbc, 0x87, 0xc9, 0x61, 0xa7, + 0xdf, 0xd6, 0xf4, 0x8a, 0x7e, 0x6a, 0xc8, 0xf8, 0xd3, 0x3e, 0xb6, 0x88, 0xf4, 0x77, 0x01, 0x56, + 0x02, 0x0b, 0x56, 0xcf, 0xd0, 0x2d, 0x8c, 0x10, 0xa4, 0x74, 0xb5, 0x8b, 0x73, 0xc2, 0xa6, 0xb0, + 0x35, 0x27, 0x3b, 0xff, 0xd1, 0x4d, 0xc8, 0x9c, 0x61, 0xbd, 0x65, 0x98, 0xca, 0x19, 0x36, 0x2d, + 0xcd, 0xd0, 0x73, 0x09, 0x67, 0x75, 0x91, 0x42, 0x8f, 0x29, 0x10, 0xed, 0xc1, 0x6c, 0x57, 0xd5, + 0xb5, 0x53, 0x6c, 0x91, 0x5c, 0x72, 0x33, 0xb9, 0x35, 0xbf, 0xf3, 0x6a, 0x9e, 0x6e, 0x35, 0x1f, + 0x2a, 0x2b, 0xff, 0x90, 0x61, 0x97, 0x75, 0x62, 0x9e, 0xcb, 0x1e, 0xb1, 0x78, 0x0f, 0x16, 0xb9, + 0x25, 0x94, 0x85, 0xe4, 0x27, 0xf8, 0x9c, 0xe9, 0x64, 0xff, 0x45, 0xcb, 0x90, 0x3e, 0x53, 0x3b, + 0x7d, 0xcc, 0x34, 0xa1, 0x0f, 0x77, 0x13, 0xdf, 0x16, 0xa4, 0xab, 0xb0, 0xee, 0x49, 0x2b, 0xa9, + 0x3d, 0xf5, 0x44, 0xeb, 0x68, 0x44, 0xc3, 0x96, 0xbb, 0xf5, 0x8f, 0x60, 0x23, 0x62, 0x9d, 0x59, + 0xe0, 0x3e, 0x2c, 0x34, 0x7d, 0xf0, 0x9c, 0xe0, 0x6c, 0x25, 0xe7, 0x6e, 0x25, 0x40, 0x79, 0x2e, + 0x73, 0xd8, 0xd2, 0xbf, 0x04, 0xc8, 0x06, 0x51, 0xd0, 0x7d, 0x98, 0xb1, 0xb0, 0x79, 0xa6, 0x35, + 0xa9, 0x5d, 0xe7, 0x77, 0x36, 0xa3, 0xb8, 0xe5, 0xeb, 0x14, 0x6f, 0x7f, 0x4a, 0x76, 0x49, 0xc4, + 0x5f, 0x08, 0x30, 0xc3, 0xc0, 0xe8, 0x6d, 0x48, 0x91, 0xf3, 0x1e, 0x65, 0x93, 0xd9, 0xb9, 0x39, + 0x8a, 0x4d, 0xbe, 0x71, 0xde, 0xc3, 0xb2, 0x43, 0x22, 0x7d, 0x00, 0x29, 0xfb, 0x09, 0xcd, 0xc3, + 0xcc, 0x51, 0xed, 0x41, 0xed, 0xe0, 0xc3, 0x5a, 0x76, 0x0a, 0xad, 0x02, 0x2a, 0x1d, 0xd4, 0x1a, + 0xf2, 0x41, 0xb5, 0x5a, 0x96, 0x95, 0x7a, 0x59, 0x3e, 0xae, 0x94, 0xca, 0x59, 0x01, 0xbd, 0x08, + 0x9b, 0xc7, 0x07, 0xd5, 0xa3, 0x87, 0x65, 0xa5, 0x50, 0x2a, 0x95, 0xeb, 0xf5, 0x4a, 0xb1, 0x52, + 0xad, 0x34, 0x1e, 0x29, 0xa5, 0x83, 0x5a, 0xbd, 0x21, 0x17, 0x2a, 0xb5, 0x46, 0x3d, 0x9b, 0x28, + 0x4e, 0x53, 0x6d, 0xa4, 0x0c, 0x2c, 0x1c, 0x9a, 0xc6, 0x09, 0x76, 0x6d, 0x5c, 0x80, 0x45, 0xf6, + 0xcc, 0x6c, 0xfa, 0x2d, 0x48, 0x9b, 0x58, 0x6d, 0x9d, 0xb3, 0xed, 0x8b, 0x79, 0xea, 0xb7, 0x79, + 0xd7, 0x6f, 0xf3, 0x45, 0xc3, 0xe8, 0x1c, 0xdb, 0x67, 0x28, 0x53, 0x44, 0xe9, 0xab, 0x14, 0x2c, + 0x95, 0x4c, 0xac, 0x12, 0x7c, 0x6c, 0x74, 0xfa, 0x5d, 0x97, 0x75, 0xa8, 0x7f, 0xde, 0x87, 0x8c, + 0x7d, 0x06, 0x4d, 0x8d, 0x9c, 0x2b, 0xa6, 0xaa, 0xb7, 0xa9, 0x57, 0xcc, 0xef, 0xac, 0xb8, 0xe6, + 0x29, 0xb1, 0x55, 0xd9, 0x5e, 0x94, 0x17, 0x9b, 0xfe, 0x47, 0x54, 0x81, 0xa5, 0x33, 0x47, 0x84, + 0xc2, 0x1d, 0x7b, 0x92, 0x3f, 0x76, 0xaa, 0x85, 0xef, 0xd8, 0xd1, 0x19, 0x0f, 0xd1, 0xb0, 0x85, + 0x1e, 0x00, 0xf4, 0x54, 0x53, 0xed, 0x62, 0x82, 0x4d, 0x2b, 0x97, 0xe2, 0x63, 0x20, 0x64, 0x37, + 0xf9, 0x43, 0x0f, 0x9b, 0xc6, 0x80, 0x8f, 0x1c, 0xed, 0xd9, 0x4e, 0xd3, 0x34, 0x31, 0xb1, 0x72, + 0x69, 0x87, 0xd3, 0x56, 0x1c, 0xa7, 0x3a, 0x45, 0x75, 0xd8, 0x14, 0x93, 0xbf, 0x2c, 0x0a, 0xb2, + 0x4b, 0x8d, 0x0e, 0x60, 0xc5, 0xdd, 0xa0, 0xa1, 0x13, 0xac, 0x13, 0xc5, 0x32, 0xfa, 0x66, 0x13, + 0xe7, 0xa6, 0x1d, 0x2b, 0xad, 0x05, 0xb6, 0x48, 0x71, 0xea, 0x0e, 0x8a, 0xcc, 0x4c, 0xc3, 0x01, + 0xd1, 0x63, 0x10, 0xd5, 0x66, 0x13, 0x5b, 0x96, 0x46, 0x6d, 0xa1, 0x98, 0xf8, 0xd3, 0xbe, 0x66, + 0xe2, 0x2e, 0xd6, 0x89, 0x95, 0x9b, 0xe1, 0xb9, 0x36, 0x8c, 0x9e, 0xd1, 0x31, 0xda, 0xe7, 0xf2, + 0x00, 0x47, 0xbe, 0xc2, 0x91, 0xfb, 0x56, 0x2c, 0xf1, 0x1d, 0x78, 0x21, 0x60, 0x94, 0x49, 0xa2, + 0x5f, 0xbc, 0x0b, 0x0b, 0x7e, 0x4b, 0x4c, 0x94, 0x39, 0x7e, 0x96, 0x80, 0xa5, 0x10, 0x1b, 0xa0, + 0x7d, 0x98, 0xb5, 0x74, 0xb5, 0x67, 0x3d, 0x31, 0x08, 0xf3, 0xdf, 0x5b, 0x31, 0x26, 0xcb, 0xd7, + 0x19, 0x2e, 0x7d, 0xdc, 0x9f, 0x92, 0x3d, 0x6a, 0x54, 0x84, 0x69, 0x6a, 0x4f, 0xe6, 0xa0, 0x5b, + 0x71, 0x7c, 0x28, 0xcc, 0xe3, 0xc2, 0x28, 0xc5, 0xd7, 0x21, 0xc3, 0x4b, 0x40, 0xd7, 0x60, 0xde, + 0x95, 0xa0, 0x68, 0x2d, 0xb6, 0x57, 0x70, 0x41, 0x95, 0x96, 0xf8, 0x2a, 0x2c, 0xf8, 0x99, 0xa1, + 0x35, 0x98, 0x63, 0x0e, 0xe1, 0xa1, 0xcf, 0x52, 0x40, 0xa5, 0xe5, 0xc5, 0xf4, 0x77, 0x60, 0x99, + 0xf7, 0x33, 0x16, 0xca, 0x2f, 0x79, 0x7b, 0xa0, 0xb6, 0xc8, 0xf0, 0x7b, 0x70, 0xf5, 0x94, 0x7e, + 0x9b, 0x82, 0x6c, 0x30, 0x68, 0xd0, 0x7d, 0x48, 0x9f, 0x74, 0x8c, 0xe6, 0x27, 0x8c, 0xf6, 0xc5, + 0xa8, 0xe8, 0xca, 0x17, 0x6d, 0x2c, 0x0a, 0xdd, 0x9f, 0x92, 0x29, 0x91, 0x4d, 0xdd, 0x35, 0xfa, + 0x3a, 0x61, 0xd6, 0x8b, 0xa6, 0x7e, 0x68, 0x63, 0x0d, 0xa8, 0x1d, 0x22, 0xb4, 0x0b, 0xf3, 0xd4, + 0xed, 0x94, 0xae, 0xd1, 0xc2, 0xb9, 0xa4, 0xc3, 0xe3, 0x46, 0x24, 0x8f, 0x82, 0x83, 0xfb, 0xd0, + 0x68, 0x61, 0x19, 0x54, 0xef, 0xbf, 0xb8, 0x08, 0xf3, 0x3e, 0xdd, 0xc4, 0x3d, 0x98, 0xf7, 0x09, + 0x43, 0x97, 0x61, 0xe6, 0xd4, 0x52, 0xbc, 0x0c, 0x3d, 0x27, 0x4f, 0x9f, 0x5a, 0x4e, 0xd2, 0xbd, + 0x06, 0xf3, 0x8e, 0x16, 0xca, 0x69, 0x47, 0x6d, 0x5b, 0xb9, 0xc4, 0x66, 0xd2, 0x3e, 0x23, 0x07, + 0xf4, 0x9e, 0x0d, 0x11, 0xff, 0x21, 0x00, 0x0c, 0x44, 0xa2, 0xfb, 0x90, 0x72, 0xb4, 0xa4, 0x79, + 0x7e, 0x6b, 0x0c, 0x2d, 0xf3, 0x8e, 0xaa, 0x0e, 0x95, 0xf4, 0x2b, 0x01, 0x52, 0x0e, 0x9b, 0x60, + 0xae, 0xaf, 0x57, 0x6a, 0x7b, 0xd5, 0xb2, 0x52, 0x3b, 0xd8, 0x2d, 0x2b, 0x1f, 0xca, 0x95, 0x46, + 0x59, 0xce, 0x0a, 0x68, 0x0d, 0x2e, 0xfb, 0xe1, 0x72, 0xb9, 0xb0, 0x5b, 0x96, 0x95, 0x83, 0x5a, + 0xf5, 0x51, 0x36, 0x81, 0x44, 0x58, 0x7d, 0x78, 0x54, 0x6d, 0x54, 0x86, 0xd7, 0x92, 0x68, 0x1d, + 0x72, 0xbe, 0x35, 0xc6, 0x83, 0xb1, 0x4d, 0xd9, 0x6c, 0x7d, 0xab, 0xf4, 0x2f, 0x5b, 0x4c, 0x17, + 0x17, 0xbd, 0xc3, 0x70, 0x9c, 0xed, 0x43, 0x58, 0xe4, 0x72, 0xb4, 0x5d, 0x72, 0xb0, 0xa4, 0xd2, + 0x52, 0x4e, 0xce, 0x89, 0x73, 0x0d, 0x0b, 0x5b, 0x49, 0x79, 0xd1, 0x85, 0x16, 0x6d, 0xa0, 0x6d, + 0xd6, 0x8e, 0xd6, 0xd5, 0x08, 0xc3, 0x49, 0x38, 0x38, 0xe0, 0x80, 0x1c, 0x04, 0xe9, 0x8b, 0x04, + 0x4c, 0xb3, 0xb3, 0xb9, 0xe9, 0xbb, 0x25, 0x38, 0x96, 0x2e, 0x94, 0xb2, 0xe4, 0x82, 0x23, 0xc1, + 0x07, 0x07, 0xda, 0x87, 0x8c, 0x3f, 0x95, 0x3e, 0x75, 0x0b, 0x9d, 0xeb, 0xfc, 0x01, 0xf9, 0xe3, + 0xf9, 0x29, 0x2b, 0x6f, 0x16, 0xcf, 0xfc, 0x30, 0x54, 0x84, 0x4c, 0x20, 0x1b, 0xa7, 0x46, 0x67, + 0xe3, 0xc5, 0x26, 0x97, 0x98, 0x0a, 0xb0, 0xe4, 0x26, 0xd2, 0x0e, 0x56, 0x08, 0x4b, 0xb4, 0xec, + 0xb6, 0xc8, 0x0e, 0x25, 0x60, 0x34, 0x40, 0x76, 0x61, 0xe2, 0xbb, 0x80, 0x86, 0x75, 0x9d, 0x28, + 0x6b, 0xf6, 0x61, 0x29, 0x24, 0xc5, 0xa3, 0x3c, 0xcc, 0x39, 0x47, 0x65, 0x69, 0x04, 0xb3, 0x12, + 0x6a, 0x58, 0xa3, 0x01, 0x8a, 0x8d, 0xdf, 0x33, 0xf1, 0x29, 0x36, 0x4d, 0xdc, 0x72, 0xc2, 0x23, + 0x14, 0xdf, 0x43, 0x91, 0x7e, 0x22, 0xc0, 0xac, 0x0b, 0x47, 0x77, 0x61, 0xd6, 0xc2, 0x6d, 0x7a, + 0xfd, 0x50, 0x59, 0x57, 0x83, 0xb4, 0xf9, 0x3a, 0x43, 0x60, 0xc5, 0xa6, 0x8b, 0x6f, 0x17, 0x9b, + 0xdc, 0xd2, 0x44, 0x9b, 0xff, 0xa3, 0x00, 0x4b, 0xbb, 0xb8, 0x83, 0x83, 0x55, 0x4a, 0x5c, 0x86, + 0xf5, 0x5f, 0xec, 0x09, 0xfe, 0x62, 0x0f, 0x61, 0x15, 0x73, 0xb1, 0x5f, 0xe8, 0xb2, 0x5b, 0x85, + 0x65, 0x5e, 0x1a, 0x4d, 0xef, 0xd2, 0x3f, 0x93, 0x70, 0xd5, 0xf6, 0x05, 0xd3, 0xe8, 0x74, 0xb0, + 0x79, 0xd8, 0x3f, 0xe9, 0x68, 0xd6, 0x93, 0x09, 0x36, 0x77, 0x19, 0x66, 0x74, 0xa3, 0xe5, 0x0b, + 0x9e, 0x69, 0xfb, 0xb1, 0xd2, 0x42, 0x65, 0xb8, 0x14, 0x2c, 0xb3, 0xce, 0x59, 0x12, 0x8e, 0x2e, + 0xb2, 0xb2, 0x67, 0xc1, 0x1b, 0x44, 0x84, 0x59, 0xbb, 0x40, 0x34, 0xf4, 0xce, 0xb9, 0x13, 0x31, + 0xb3, 0xb2, 0xf7, 0x8c, 0xe4, 0x60, 0xc5, 0xf4, 0x86, 0x57, 0x31, 0xc5, 0xee, 0x28, 0xae, 0x78, + 0xfa, 0x78, 0x28, 0xe2, 0xa7, 0x1d, 0xd6, 0x6f, 0x8f, 0xc9, 0x7a, 0x64, 0x26, 0xb8, 0xc8, 0x29, + 0x3e, 0x87, 0xf0, 0xfd, 0xab, 0x00, 0xd7, 0x22, 0xb7, 0xc0, 0xae, 0xfc, 0x16, 0xbc, 0xd0, 0xa3, + 0x0b, 0x9e, 0x11, 0x68, 0x94, 0xdd, 0x1b, 0x69, 0x04, 0xd6, 0xe9, 0x31, 0x28, 0x67, 0x86, 0x4c, + 0x8f, 0x03, 0x8a, 0x05, 0x58, 0x0a, 0x41, 0x9b, 0x68, 0x33, 0x5f, 0x0a, 0xb0, 0x39, 0x50, 0xe5, + 0x48, 0xef, 0x3d, 0x3f, 0xf7, 0x6d, 0x0c, 0x7c, 0x8b, 0xa6, 0xfc, 0x3b, 0xc3, 0x7b, 0x0f, 0x17, + 0xf8, 0x75, 0x45, 0xf0, 0x0d, 0xb8, 0x1e, 0x23, 0x9a, 0x85, 0xf3, 0x17, 0x29, 0xb8, 0x7e, 0xac, + 0x76, 0xb4, 0x96, 0x57, 0xc8, 0x85, 0xf4, 0xc4, 0xf1, 0x26, 0x69, 0x0e, 0x45, 0x00, 0xcd, 0x5a, + 0xf7, 0xbd, 0xa8, 0x1d, 0xc5, 0x7f, 0x8c, 0xeb, 0xf0, 0x39, 0x36, 0x61, 0x8f, 0x42, 0x9a, 0xb0, + 0xb7, 0xc7, 0xd7, 0x35, 0xae, 0x25, 0x3b, 0x0a, 0x26, 0x98, 0xb7, 0xc6, 0xe7, 0x1b, 0xe3, 0x05, + 0x17, 0x8e, 0xe2, 0x6f, 0xb2, 0x6b, 0xfa, 0x73, 0x0a, 0xa4, 0xb8, 0xdd, 0xb3, 0x1c, 0x22, 0xc3, + 0x5c, 0xd3, 0xd0, 0x4f, 0x35, 0xb3, 0x8b, 0x5b, 0xac, 0xfa, 0x7f, 0x73, 0x1c, 0xe3, 0xb1, 0x04, + 0x52, 0x72, 0x69, 0xe5, 0x01, 0x1b, 0x94, 0x83, 0x99, 0x2e, 0xb6, 0x2c, 0xb5, 0xed, 0xaa, 0xe5, + 0x3e, 0x8a, 0x9f, 0x27, 0x61, 0xce, 0x23, 0x41, 0xfa, 0x90, 0x07, 0xd3, 0xf4, 0xb5, 0xf7, 0x2c, + 0x0a, 0x3c, 0xbb, 0x33, 0x27, 0x9e, 0xc1, 0x99, 0x5b, 0x9c, 0x33, 0xd3, 0x70, 0xd8, 0x7d, 0x26, + 0xb5, 0x63, 0xfc, 0xfa, 0x1b, 0x77, 0x40, 0xe9, 0x07, 0x80, 0xaa, 0x9a, 0xc5, 0xba, 0x28, 0x2f, + 0x2d, 0xd9, 0x4d, 0x93, 0xfa, 0x54, 0xc1, 0x3a, 0x31, 0x35, 0x56, 0xae, 0xa7, 0x65, 0xe8, 0xaa, + 0x4f, 0xcb, 0x14, 0x62, 0x97, 0xf4, 0x16, 0x51, 0x4d, 0xa2, 0xe9, 0x6d, 0x85, 0x18, 0x9f, 0x60, + 0x6f, 0x30, 0xe9, 0x42, 0x1b, 0x36, 0x50, 0xfa, 0x4c, 0x80, 0x25, 0x8e, 0x3d, 0xf3, 0xc9, 0x7b, + 0x30, 0x33, 0xe0, 0xcd, 0x95, 0xf1, 0x21, 0xd8, 0x79, 0x6a, 0x36, 0x97, 0x02, 0x6d, 0x00, 0xe8, + 0xf8, 0x29, 0xe1, 0xe4, 0xce, 0xd9, 0x10, 0x47, 0xa6, 0xb8, 0x0d, 0x69, 0x6a, 0x86, 0x71, 0xfb, + 0xe5, 0xcf, 0x13, 0x80, 0xf6, 0x30, 0xf1, 0xda, 0x20, 0x66, 0x83, 0x08, 0x5f, 0x12, 0x9e, 0xc1, + 0x97, 0xde, 0xe7, 0x7c, 0x89, 0x7a, 0xe3, 0x2d, 0xdf, 0x84, 0x36, 0x20, 0x3a, 0x36, 0x13, 0x46, + 0xb4, 0x1e, 0xb4, 0x9e, 0x1b, 0xaf, 0xf5, 0xb8, 0xa0, 0xcb, 0xec, 0xc2, 0x12, 0xa7, 0x33, 0x3b, + 0xd3, 0xdb, 0x80, 0xd4, 0x33, 0x55, 0xeb, 0xa8, 0xb6, 0x5e, 0x6e, 0x67, 0xc7, 0x3a, 0xbd, 0x4b, + 0xde, 0x8a, 0x4b, 0x26, 0x49, 0xfe, 0x82, 0x81, 0xf1, 0x0b, 0x4e, 0x8c, 0x3b, 0xfe, 0x8b, 0x76, + 0x08, 0x87, 0xc9, 0xdd, 0x0b, 0x9d, 0x1a, 0xdf, 0x18, 0x2e, 0x12, 0xd8, 0x64, 0x36, 0x72, 0x80, + 0xfc, 0xef, 0x04, 0xac, 0xc5, 0x60, 0xa3, 0x7b, 0x90, 0x34, 0x7b, 0x4d, 0xe6, 0x4c, 0x2f, 0x8f, + 0xc1, 0x3f, 0x2f, 0x1f, 0x96, 0xf6, 0xa7, 0x64, 0x9b, 0x4a, 0xfc, 0x79, 0x02, 0x92, 0xf2, 0x61, + 0x09, 0xbd, 0xcb, 0x8d, 0x91, 0x5f, 0x1b, 0x93, 0x8b, 0x7f, 0x9a, 0xfc, 0x17, 0x21, 0x6c, 0x9c, + 0x9c, 0x83, 0xe5, 0x92, 0x5c, 0x2e, 0x34, 0xca, 0xca, 0x6e, 0xb9, 0x5a, 0x6e, 0x94, 0x15, 0x3a, + 0x44, 0xce, 0x0a, 0x68, 0x1d, 0x72, 0x87, 0x47, 0xc5, 0x6a, 0xa5, 0xbe, 0xaf, 0x1c, 0xd5, 0xdc, + 0x7f, 0x6c, 0x35, 0x81, 0xb2, 0xb0, 0x50, 0xad, 0xd4, 0x1b, 0x0c, 0x50, 0xcf, 0x26, 0x6d, 0xc8, + 0x5e, 0xb9, 0xa1, 0x94, 0x0a, 0x87, 0x85, 0x52, 0xa5, 0xf1, 0x28, 0x9b, 0x42, 0x22, 0xac, 0xf2, + 0xbc, 0xeb, 0xb5, 0xc2, 0x61, 0x7d, 0xff, 0xa0, 0x91, 0x4d, 0x23, 0x04, 0x19, 0x87, 0xde, 0x05, + 0xd5, 0xb3, 0xd3, 0x36, 0x87, 0x52, 0xf5, 0xa0, 0xe6, 0xe9, 0x30, 0x83, 0x96, 0x21, 0xeb, 0x4a, + 0x96, 0xcb, 0x85, 0x5d, 0x67, 0x8a, 0x31, 0xeb, 0x0d, 0xbc, 0xbe, 0x4c, 0xc0, 0x0a, 0x9d, 0x78, + 0xb9, 0xf3, 0x35, 0x37, 0x06, 0xb7, 0x20, 0x4b, 0x7b, 0x74, 0x25, 0x58, 0x25, 0x65, 0x28, 0xfc, + 0xd8, 0xad, 0x95, 0xdc, 0xe9, 0x74, 0xc2, 0x37, 0x9d, 0xae, 0x04, 0x2b, 0xc7, 0x5b, 0xfc, 0x1c, + 0x37, 0x20, 0x2d, 0xae, 0x19, 0x79, 0x18, 0x52, 0xda, 0xdc, 0x8e, 0xe7, 0x16, 0x97, 0xf6, 0x2f, + 0xd2, 0x79, 0x5c, 0x30, 0x7a, 0xdf, 0x83, 0xd5, 0xa0, 0xbe, 0x2c, 0x90, 0x5e, 0x1b, 0x9a, 0xb6, + 0x7a, 0xe9, 0xc4, 0xc3, 0xf5, 0x30, 0xa4, 0xbf, 0x09, 0x30, 0xeb, 0x82, 0xed, 0x94, 0x6c, 0x69, + 0x3f, 0xc2, 0xdc, 0x74, 0x67, 0xce, 0x86, 0x78, 0xc3, 0x22, 0xff, 0x9c, 0x34, 0x11, 0x9c, 0x93, + 0x86, 0x9e, 0x73, 0x32, 0xf4, 0x9c, 0xbf, 0x0b, 0x8b, 0x4d, 0x5b, 0x7d, 0xcd, 0xd0, 0x15, 0xa2, + 0x75, 0xdd, 0xe1, 0xcd, 0xf0, 0x7b, 0x8d, 0x86, 0xfb, 0xc2, 0x4e, 0x5e, 0x70, 0x09, 0x6c, 0x10, + 0xda, 0x84, 0x05, 0xe7, 0x3d, 0x87, 0x42, 0x0c, 0xa5, 0x6f, 0xe1, 0x5c, 0xda, 0x69, 0x65, 0xc1, + 0x81, 0x35, 0x8c, 0x23, 0x0b, 0x4b, 0x7f, 0x12, 0x60, 0x85, 0x76, 0xe8, 0x41, 0x77, 0x1c, 0x35, + 0xef, 0xf5, 0x7b, 0x5c, 0x20, 0xcb, 0x87, 0x32, 0xfc, 0xba, 0x1a, 0x94, 0x1c, 0xac, 0x06, 0xe5, + 0xb1, 0xae, 0xe4, 0x37, 0x02, 0x2c, 0xdb, 0x57, 0xac, 0xbb, 0xf0, 0xbc, 0x6f, 0xfc, 0x09, 0x4e, + 0x32, 0x60, 0xcc, 0x54, 0xd0, 0x98, 0xd2, 0xef, 0x04, 0x58, 0x09, 0xe8, 0xca, 0x3c, 0xf5, 0x9d, + 0x60, 0xf9, 0x70, 0xc3, 0x5f, 0x3e, 0x0c, 0xe1, 0x4f, 0x58, 0x40, 0xdc, 0x71, 0x0b, 0x88, 0xc9, + 0x02, 0xe2, 0xab, 0x14, 0xac, 0xd6, 0x8c, 0x16, 0xae, 0x13, 0xb5, 0x3d, 0xc9, 0x50, 0xea, 0xfb, + 0xc3, 0x3d, 0x3e, 0xf5, 0x9d, 0x1d, 0x57, 0x58, 0x38, 0xd7, 0x71, 0x5a, 0x7b, 0x94, 0x87, 0x25, + 0x8b, 0xa8, 0x6d, 0xe7, 0xd0, 0x54, 0xb3, 0x8d, 0x89, 0xd2, 0x53, 0xc9, 0x13, 0x76, 0x22, 0x97, + 0xd8, 0x52, 0xc3, 0x59, 0x39, 0x54, 0xc9, 0x93, 0xf0, 0x59, 0x51, 0x6a, 0xe2, 0x59, 0xd1, 0xfb, + 0xc1, 0x76, 0xed, 0xd5, 0x11, 0x7b, 0x89, 0x49, 0xbd, 0xdf, 0x8b, 0x98, 0x03, 0xbd, 0x3e, 0x82, + 0xe5, 0xe8, 0xf9, 0xcf, 0xc5, 0xe7, 0x1e, 0xdf, 0xf0, 0x08, 0xe9, 0x0a, 0x5c, 0x1e, 0xda, 0x3c, + 0x0b, 0xf4, 0x36, 0xe4, 0xec, 0xa5, 0x23, 0xdd, 0x9a, 0xd0, 0x1d, 0x23, 0x3c, 0x26, 0x11, 0xe1, + 0x31, 0xd2, 0x1a, 0x5c, 0x09, 0x11, 0xc4, 0xb4, 0xf8, 0x43, 0x9a, 0xaa, 0x31, 0xf9, 0x34, 0xf3, + 0xa3, 0xa8, 0xa8, 0x78, 0xd3, 0x7f, 0xec, 0xa1, 0x83, 0xbf, 0xaf, 0x23, 0x2e, 0xae, 0xc1, 0xbc, + 0x1f, 0x8f, 0x25, 0x2b, 0x32, 0x22, 0x70, 0xd2, 0x17, 0x1a, 0xb2, 0x4e, 0x07, 0x86, 0xac, 0xd5, + 0x41, 0x50, 0xcd, 0xf0, 0x05, 0x48, 0xa4, 0x29, 0x62, 0xc2, 0xea, 0xf1, 0x50, 0x58, 0xcd, 0xf2, + 0x93, 0xdb, 0x48, 0xa6, 0xff, 0x07, 0x81, 0xc5, 0x9c, 0x3a, 0x74, 0xa4, 0x2a, 0x3d, 0x06, 0x91, + 0x7a, 0xfc, 0xe4, 0x43, 0xce, 0x80, 0x1b, 0x25, 0x82, 0x6e, 0x24, 0x6d, 0xc0, 0x5a, 0x28, 0x6f, + 0x26, 0xfa, 0x11, 0xd5, 0x6b, 0x0f, 0xb3, 0x1e, 0xb9, 0x4e, 0x54, 0x62, 0x8d, 0x2b, 0x99, 0x2d, + 0xfa, 0x25, 0x53, 0x90, 0x23, 0x79, 0x8f, 0xee, 0x2a, 0xc8, 0x9a, 0xdd, 0xb8, 0xaf, 0x40, 0xba, + 0xef, 0x8c, 0x7b, 0xe8, 0x7d, 0xbb, 0xc4, 0xbb, 0xf4, 0x91, 0xbd, 0x24, 0x53, 0x0c, 0xe9, 0xf7, + 0x02, 0xcc, 0xfb, 0xc0, 0x68, 0x1d, 0xe6, 0xbc, 0xee, 0xcf, 0x2d, 0x0d, 0x3d, 0x80, 0x7d, 0x06, + 0xc4, 0x20, 0x6a, 0x87, 0xbd, 0x41, 0xa4, 0x0f, 0x76, 0x35, 0xdf, 0xb7, 0x30, 0xad, 0x1c, 0x92, + 0xb2, 0xf3, 0x1f, 0xbd, 0x06, 0xa9, 0xbe, 0xae, 0x11, 0x27, 0xf6, 0x32, 0xc1, 0xa0, 0x72, 0x44, + 0xe5, 0x8f, 0x74, 0x8d, 0xc8, 0x0e, 0x96, 0x74, 0x0b, 0x52, 0xf6, 0x13, 0xdf, 0x24, 0xcd, 0x41, + 0xba, 0xf8, 0xa8, 0x51, 0xae, 0x67, 0x05, 0x04, 0x30, 0x5d, 0xa9, 0x1d, 0xec, 0x96, 0xeb, 0xd9, + 0x84, 0xb4, 0xee, 0x6d, 0x3d, 0xac, 0x09, 0xfd, 0x98, 0x1e, 0x49, 0x54, 0xfb, 0x59, 0x08, 0x6d, + 0x3f, 0x37, 0xb8, 0xcb, 0x69, 0x44, 0xe3, 0xf9, 0x85, 0x00, 0x2b, 0xa1, 0x78, 0xe8, 0x8e, 0xbf, + 0xe5, 0xbc, 0x1e, 0xcb, 0xd3, 0xdf, 0x6c, 0xfe, 0x54, 0xa0, 0xcd, 0xe6, 0x5d, 0xae, 0xd9, 0x7c, + 0x69, 0x24, 0xbd, 0xbf, 0xcd, 0x2c, 0x45, 0x74, 0x99, 0xf5, 0x46, 0x61, 0xaf, 0xac, 0x1c, 0xd5, + 0xe8, 0xaf, 0xd7, 0x65, 0x2e, 0x43, 0xd6, 0xee, 0x1a, 0xd9, 0xa7, 0x4b, 0xf5, 0x46, 0x81, 0xfb, + 0x4c, 0x69, 0x19, 0x10, 0xb3, 0xa1, 0xff, 0x5b, 0xb8, 0xcf, 0x04, 0x58, 0xe2, 0xc0, 0xcc, 0xa4, + 0xbe, 0x57, 0x01, 0x02, 0xf7, 0x2a, 0x60, 0x1b, 0x96, 0xed, 0x22, 0x95, 0x7a, 0xad, 0xa5, 0xf4, + 0xb0, 0xa9, 0xd8, 0x2b, 0xcc, 0x77, 0x2e, 0x75, 0xd5, 0xa7, 0x6c, 0x74, 0x74, 0x88, 0x4d, 0x9b, + 0xf1, 0x73, 0x18, 0x96, 0xec, 0xfc, 0x47, 0x80, 0xd9, 0x4a, 0x0b, 0xeb, 0xc4, 0x3e, 0x8f, 0x1a, + 0x2c, 0x72, 0x1f, 0xd4, 0xa1, 0xf5, 0x88, 0xef, 0xec, 0x9c, 0x0d, 0x8a, 0x1b, 0xb1, 0x5f, 0xe1, + 0x49, 0x53, 0xe8, 0xd4, 0xf7, 0x31, 0x20, 0x37, 0x31, 0x7a, 0x71, 0x88, 0x32, 0xc4, 0x35, 0xc5, + 0x9b, 0x23, 0xb0, 0x3c, 0x39, 0x6f, 0x41, 0xda, 0xf9, 0x2c, 0x0c, 0x2d, 0x7b, 0xdf, 0xad, 0xf9, + 0xbe, 0x1a, 0x13, 0x57, 0x02, 0x50, 0x97, 0x6e, 0xe7, 0xbf, 0x33, 0x00, 0x83, 0xd1, 0x04, 0x7a, + 0x00, 0x0b, 0xfe, 0x2f, 0x53, 0xd0, 0x5a, 0xcc, 0x77, 0x51, 0xe2, 0x7a, 0xf8, 0xa2, 0xa7, 0xd3, + 0x03, 0x58, 0xf0, 0xbf, 0x07, 0x1d, 0x30, 0x0b, 0x79, 0x17, 0x3b, 0x60, 0x16, 0xfa, 0xea, 0x74, + 0x0a, 0x75, 0xe0, 0x72, 0xc4, 0x9b, 0x30, 0xf4, 0xd2, 0x78, 0xef, 0x0b, 0xc5, 0x97, 0xc7, 0x7c, + 0xa5, 0x26, 0x4d, 0x21, 0x13, 0xae, 0x44, 0xbe, 0x00, 0x42, 0x5b, 0xe3, 0xbe, 0x9e, 0x12, 0x5f, + 0x19, 0x03, 0xd3, 0x93, 0xd9, 0x07, 0x31, 0x7a, 0xea, 0x8c, 0x5e, 0x19, 0xfb, 0x75, 0x88, 0x78, + 0x6b, 0xfc, 0x21, 0xb6, 0x34, 0x85, 0xf6, 0x61, 0xde, 0x37, 0x92, 0x45, 0x62, 0xe8, 0x9c, 0x96, + 0x32, 0x5e, 0x8b, 0x99, 0xe1, 0x52, 0x4e, 0xbe, 0xb1, 0xe1, 0x80, 0xd3, 0xf0, 0xfc, 0x73, 0xc0, + 0x29, 0x64, 0xce, 0x18, 0x34, 0x7f, 0x20, 0x2f, 0x87, 0x99, 0x3f, 0x3c, 0xb1, 0x87, 0x99, 0x3f, + 0x22, 0xc9, 0x4b, 0x53, 0xe8, 0x03, 0xc8, 0xf0, 0x63, 0x13, 0xb4, 0x11, 0x3b, 0xfe, 0x11, 0xaf, + 0x46, 0x2d, 0xfb, 0x59, 0xf2, 0x5d, 0xfa, 0x80, 0x65, 0xe8, 0xb4, 0x60, 0xc0, 0x32, 0xa2, 0xb9, + 0x9f, 0xb2, 0xf3, 0x13, 0xd7, 0x01, 0x0f, 0xf2, 0x53, 0x58, 0xd3, 0x3f, 0xc8, 0x4f, 0xa1, 0x6d, + 0xb3, 0x34, 0xb5, 0xf3, 0xe3, 0x34, 0xa4, 0x9c, 0x44, 0xda, 0x80, 0x17, 0x02, 0x9d, 0x06, 0xba, + 0x1a, 0xdf, 0x7f, 0x89, 0xd7, 0x22, 0xd7, 0x3d, 0x75, 0x1f, 0xc3, 0xa5, 0xa1, 0xde, 0x01, 0x6d, + 0xfa, 0xe9, 0xc2, 0xfa, 0x17, 0xf1, 0x7a, 0x0c, 0x46, 0x90, 0x37, 0x9f, 0x0b, 0x36, 0x47, 0x15, + 0xb7, 0x3c, 0xef, 0xa8, 0xf8, 0xff, 0x98, 0xde, 0x5b, 0xc1, 0xc8, 0x97, 0x78, 0xbd, 0x42, 0x63, + 0xfe, 0x46, 0x2c, 0x8e, 0x27, 0xe1, 0x23, 0xef, 0xc2, 0xf4, 0x55, 0x63, 0x88, 0x53, 0x2e, 0xb4, + 0x08, 0x14, 0xa5, 0x38, 0x94, 0xe0, 0x06, 0x82, 0xb1, 0x13, 0x24, 0x0e, 0x8b, 0x9a, 0x1b, 0xb1, + 0x38, 0xfe, 0x68, 0xf7, 0x5d, 0xed, 0x83, 0x68, 0x1f, 0x2e, 0x03, 0x06, 0xd1, 0x1e, 0x52, 0x0b, + 0x48, 0x53, 0x77, 0xdf, 0x01, 0x68, 0x5a, 0x9a, 0x42, 0xfb, 0x16, 0xb4, 0x31, 0x34, 0xe8, 0x7b, + 0x4f, 0xc3, 0x9d, 0xd6, 0x41, 0x8f, 0x68, 0x86, 0x6e, 0xe5, 0x7e, 0x3d, 0xeb, 0x34, 0x4d, 0x73, + 0x4d, 0x4b, 0xa3, 0xed, 0x43, 0x31, 0xfd, 0x38, 0xd9, 0xb4, 0xb4, 0x93, 0x69, 0x07, 0xff, 0x8d, + 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x47, 0x5d, 0xbf, 0x76, 0x3a, 0x30, 0x00, 0x00, } diff --git a/vendor/github.com/lpabon/godbc/.gitignore b/vendor/github.com/go-ozzo/ozzo-validation/.gitignore similarity index 93% rename from vendor/github.com/lpabon/godbc/.gitignore rename to vendor/github.com/go-ozzo/ozzo-validation/.gitignore index 836562412fe..5a3865c74f7 100644 --- a/vendor/github.com/lpabon/godbc/.gitignore +++ b/vendor/github.com/go-ozzo/ozzo-validation/.gitignore @@ -21,3 +21,5 @@ _testmain.go *.exe *.test +*.prof +.DS_Store diff --git a/vendor/github.com/go-ozzo/ozzo-validation/.travis.yml b/vendor/github.com/go-ozzo/ozzo-validation/.travis.yml new file mode 100644 index 00000000000..fc2af598618 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/.travis.yml @@ -0,0 +1,16 @@ +language: go + +go: + - 1.8 + - 1.9 + - tip + +install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v + - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v + +script: + - go test -v -covermode=count -coverprofile=coverage.out + - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci diff --git a/vendor/github.com/go-ozzo/ozzo-validation/BUILD b/vendor/github.com/go-ozzo/ozzo-validation/BUILD new file mode 100644 index 00000000000..96ca75ad183 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/BUILD @@ -0,0 +1,41 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "date.go", + "error.go", + "in.go", + "length.go", + "match.go", + "minmax.go", + "multipleof.go", + "not_in.go", + "not_nil.go", + "required.go", + "string.go", + "struct.go", + "util.go", + "validation.go", + ], + importmap = "k8s.io/kubernetes/vendor/github.com/go-ozzo/ozzo-validation", + importpath = "github.com/go-ozzo/ozzo-validation", + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//vendor/github.com/go-ozzo/ozzo-validation/is:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/go-ozzo/ozzo-validation/LICENSE b/vendor/github.com/go-ozzo/ozzo-validation/LICENSE new file mode 100644 index 00000000000..d235be9cc6d --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/LICENSE @@ -0,0 +1,17 @@ +The MIT License (MIT) +Copyright (c) 2016, Qiang Xue + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software +and associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/go-ozzo/ozzo-validation/README.md b/vendor/github.com/go-ozzo/ozzo-validation/README.md new file mode 100644 index 00000000000..ca9445d2cf8 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/README.md @@ -0,0 +1,534 @@ +# ozzo-validation + +[![GoDoc](https://godoc.org/github.com/go-ozzo/ozzo-validation?status.png)](http://godoc.org/github.com/go-ozzo/ozzo-validation) +[![Build Status](https://travis-ci.org/go-ozzo/ozzo-validation.svg?branch=master)](https://travis-ci.org/go-ozzo/ozzo-validation) +[![Coverage Status](https://coveralls.io/repos/github/go-ozzo/ozzo-validation/badge.svg?branch=master)](https://coveralls.io/github/go-ozzo/ozzo-validation?branch=master) +[![Go Report](https://goreportcard.com/badge/github.com/go-ozzo/ozzo-validation)](https://goreportcard.com/report/github.com/go-ozzo/ozzo-validation) + +## Description + +ozzo-validation is a Go package that provides configurable and extensible data validation capabilities. +It has the following features: + +* use normal programming constructs rather than error-prone struct tags to specify how data should be validated. +* can validate data of different types, e.g., structs, strings, byte slices, slices, maps, arrays. +* can validate custom data types as long as they implement the `Validatable` interface. +* can validate data types that implement the `sql.Valuer` interface (e.g. `sql.NullString`). +* customizable and well-formatted validation errors. +* provide a rich set of validation rules right out of box. +* extremely easy to create and use custom validation rules. + + +## Requirements + +Go 1.8 or above. + + +## Getting Started + +The ozzo-validation package mainly includes a set of validation rules and two validation methods. You use +validation rules to describe how a value should be considered valid, and you call either `validation.Validate()` +or `validation.ValidateStruct()` to validate the value. + + +### Installation + +Run the following command to install the package: + +``` +go get github.com/go-ozzo/ozzo-validation +go get github.com/go-ozzo/ozzo-validation/is +``` + +### Validating a Simple Value + +For a simple value, such as a string or an integer, you may use `validation.Validate()` to validate it. For example, + +```go +package main + +import ( + "fmt" + + "github.com/go-ozzo/ozzo-validation" + "github.com/go-ozzo/ozzo-validation/is" +) + +func main() { + data := "example" + err := validation.Validate(data, + validation.Required, // not empty + validation.Length(5, 100), // length between 5 and 100 + is.URL, // is a valid URL + ) + fmt.Println(err) + // Output: + // must be a valid URL +} +``` + +The method `validation.Validate()` will run through the rules in the order that they are listed. If a rule fails +the validation, the method will return the corresponding error and skip the rest of the rules. The method will +return nil if the value passes all validation rules. + + +### Validating a Struct + +For a struct value, you usually want to check if its fields are valid. For example, in a RESTful application, you +may unmarshal the request payload into a struct and then validate the struct fields. If one or multiple fields +are invalid, you may want to get an error describing which fields are invalid. You can use `validation.ValidateStruct()` +to achieve this purpose. A single struct can have rules for multiple fields, and a field can be associated with multiple +rules. For example, + +```go +package main + +import ( + "fmt" + "regexp" + + "github.com/go-ozzo/ozzo-validation" + "github.com/go-ozzo/ozzo-validation/is" +) + +type Address struct { + Street string + City string + State string + Zip string +} + +func (a Address) Validate() error { + return validation.ValidateStruct(&a, + // Street cannot be empty, and the length must between 5 and 50 + validation.Field(&a.Street, validation.Required, validation.Length(5, 50)), + // City cannot be empty, and the length must between 5 and 50 + validation.Field(&a.City, validation.Required, validation.Length(5, 50)), + // State cannot be empty, and must be a string consisting of two letters in upper case + validation.Field(&a.State, validation.Required, validation.Match(regexp.MustCompile("^[A-Z]{2}$"))), + // State cannot be empty, and must be a string consisting of five digits + validation.Field(&a.Zip, validation.Required, validation.Match(regexp.MustCompile("^[0-9]{5}$"))), + ) +} + +func main() { + a := Address{ + Street: "123", + City: "Unknown", + State: "Virginia", + Zip: "12345", + } + + err := a.Validate() + fmt.Println(err) + // Output: + // Street: the length must be between 5 and 50; State: must be in a valid format. +} +``` + +Note that when calling `validation.ValidateStruct` to validate a struct, you should pass to the method a pointer +to the struct instead of the struct itself. Similarly, when calling `validation.Field` to specify the rules +for a struct field, you should use a pointer to the struct field. + +When the struct validation is performed, the fields are validated in the order they are specified in `ValidateStruct`. +And when each field is validated, its rules are also evaluated in the order they are associated with the field. +If a rule fails, an error is recorded for that field, and the validation will continue with the next field. + + +### Validation Errors + +The `validation.ValidateStruct` method returns validation errors found in struct fields in terms of `validation.Errors` +which is a map of fields and their corresponding errors. Nil is returned if validation passes. + +By default, `validation.Errors` uses the struct tags named `json` to determine what names should be used to +represent the invalid fields. The type also implements the `json.Marshaler` interface so that it can be marshaled +into a proper JSON object. For example, + +```go +type Address struct { + Street string `json:"street"` + City string `json:"city"` + State string `json:"state"` + Zip string `json:"zip"` +} + +// ...perform validation here... + +err := a.Validate() +b, _ := json.Marshal(err) +fmt.Println(string(b)) +// Output: +// {"street":"the length must be between 5 and 50","state":"must be in a valid format"} +``` + +You may modify `validation.ErrorTag` to use a different struct tag name. + +If you do not like the magic that `ValidateStruct` determines error keys based on struct field names or corresponding +tag values, you may use the following alternative approach: + +```go +c := Customer{ + Name: "Qiang Xue", + Email: "q", + Address: Address{ + State: "Virginia", + }, +} + +err := validation.Errors{ + "name": validation.Validate(c.Name, validation.Required, validation.Length(5, 20)), + "email": validation.Validate(c.Name, validation.Required, is.Email), + "zip": validation.Validate(c.Address.Zip, validation.Required, validation.Match(regexp.MustCompile("^[0-9]{5}$"))), +}.Filter() +fmt.Println(err) +// Output: +// email: must be a valid email address; zip: cannot be blank. +``` + +In the above example, we build a `validation.Errors` by a list of names and the corresponding validation results. +At the end we call `Errors.Filter()` to remove from `Errors` all nils which correspond to those successful validation +results. The method will return nil if `Errors` is empty. + +The above approach is very flexible as it allows you to freely build up your validation error structure. You can use +it to validate both struct and non-struct values. Compared to using `ValidateStruct` to validate a struct, +it has the drawback that you have to redundantly specify the error keys while `ValidateStruct` can automatically +find them out. + + +### Internal Errors + +Internal errors are different from validation errors in that internal errors are caused by malfunctioning code (e.g. +a validator making a remote call to validate some data when the remote service is down) rather +than the data being validated. When an internal error happens during data validation, you may allow the user to resubmit +the same data to perform validation again, hoping the program resumes functioning. On the other hand, if data validation +fails due to data error, the user should generally not resubmit the same data again. + +To differentiate internal errors from validation errors, when an internal error occurs in a validator, wrap it +into `validation.InternalError` by calling `validation.NewInternalError()`. The user of the validator can then check +if a returned error is an internal error or not. For example, + +```go +if err := a.Validate(); err != nil { + if e, ok := err.(validation.InternalError); ok { + // an internal error happened + fmt.Println(e.InternalError()) + } +} +``` + + +## Validatable Types + +A type is validatable if it implements the `validation.Validatable` interface. + +When `validation.Validate` is used to validate a validatable value, if it does not find any error with the +given validation rules, it will further call the value's `Validate()` method. + +Similarly, when `validation.ValidateStruct` is validating a struct field whose type is validatable, it will call +the field's `Validate` method after it passes the listed rules. + +In the following example, the `Address` field of `Customer` is validatable because `Address` implements +`validation.Validatable`. Therefore, when validating a `Customer` struct with `validation.ValidateStruct`, +validation will "dive" into the `Address` field. + +```go +type Customer struct { + Name string + Gender string + Email string + Address Address +} + +func (c Customer) Validate() error { + return validation.ValidateStruct(&c, + // Name cannot be empty, and the length must be between 5 and 20. + validation.Field(&c.Name, validation.Required, validation.Length(5, 20)), + // Gender is optional, and should be either "Female" or "Male". + validation.Field(&c.Gender, validation.In("Female", "Male")), + // Email cannot be empty and should be in a valid email format. + validation.Field(&c.Email, validation.Required, is.Email), + // Validate Address using its own validation rules + validation.Field(&c.Address), + ) +} + +c := Customer{ + Name: "Qiang Xue", + Email: "q", + Address: Address{ + Street: "123 Main Street", + City: "Unknown", + State: "Virginia", + Zip: "12345", + }, +} + +err := c.Validate() +fmt.Println(err) +// Output: +// Address: (State: must be in a valid format.); Email: must be a valid email address. +``` + +Sometimes, you may want to skip the invocation of a type's `Validate` method. To do so, simply associate +a `validation.Skip` rule with the value being validated. + + +### Maps/Slices/Arrays of Validatables + +When validating a map, slice, or array, whose element type implements the `validation.Validatable` interface, +the `validation.Validate` method will call the `Validate` method of every non-nil element. +The validation errors of the elements will be returned as `validation.Errors` which maps the keys of the +invalid elements to their corresponding validation errors. For example, + +```go +addresses := []Address{ + Address{State: "MD", Zip: "12345"}, + Address{Street: "123 Main St", City: "Vienna", State: "VA", Zip: "12345"}, + Address{City: "Unknown", State: "NC", Zip: "123"}, +} +err := validation.Validate(addresses) +fmt.Println(err) +// Output: +// 0: (City: cannot be blank; Street: cannot be blank.); 2: (Street: cannot be blank; Zip: must be in a valid format.). +``` + +When using `validation.ValidateStruct` to validate a struct, the above validation procedure also applies to those struct +fields which are map/slices/arrays of validatables. + + +### Pointers + +When a value being validated is a pointer, most validation rules will validate the actual value pointed to by the pointer. +If the pointer is nil, these rules will skip the validation. + +An exception is the `validation.Required` and `validation.NotNil` rules. When a pointer is nil, they +will report a validation error. + + +### Types Implementing `sql.Valuer` + +If a data type implements the `sql.Valuer` interface (e.g. `sql.NullString`), the built-in validation rules will handle +it properly. In particular, when a rule is validating such data, it will call the `Value()` method and validate +the returned value instead. + + +### Required vs. Not Nil + +When validating input values, there are two different scenarios about checking if input values are provided or not. + +In the first scenario, an input value is considered missing if it is not entered or it is entered as a zero value +(e.g. an empty string, a zero integer). You can use the `validation.Required` rule in this case. + +In the second scenario, an input value is considered missing only if it is not entered. A pointer field is usually +used in this case so that you can detect if a value is entered or not by checking if the pointer is nil or not. +You can use the `validation.NotNil` rule to ensure a value is entered (even if it is a zero value). + + +### Embedded Structs + +The `validation.ValidateStruct` method will properly validate a struct that contains embedded structs. In particular, +the fields of an embedded struct are treated as if they belong directly to the containing struct. For example, + +```go +type Employee struct { + Name string +} + +func () + +type Manager struct { + Employee + Level int +} + +m := Manager{} +err := validation.ValidateStruct(&m, + validation.Field(&m.Name, validation.Required), + validation.Field(&m.Level, validation.Required), +) +fmt.Println(err) +// Output: +// Level: cannot be blank; Name: cannot be blank. +``` + +In the above code, we use `&m.Name` to specify the validation of the `Name` field of the embedded struct `Employee`. +And the validation error uses `Name` as the key for the error associated with the `Name` field as if `Name` a field +directly belonging to `Manager`. + +If `Employee` implements the `validation.Validatable` interface, we can also use the following code to validate +`Manager`, which generates the same validation result: + +```go +func (e Employee) Validate() error { + return validation.ValidateStruct(&e, + validation.Field(&e.Name, validation.Required), + ) +} + +err := validation.ValidateStruct(&m, + validation.Field(&m.Employee), + validation.Field(&m.Level, validation.Required), +) +fmt.Println(err) +// Output: +// Level: cannot be blank; Name: cannot be blank. +``` + + +## Built-in Validation Rules + +The following rules are provided in the `validation` package: + +* `In(...interface{})`: checks if a value can be found in the given list of values. +* `Length(min, max int)`: checks if the length of a value is within the specified range. + This rule should only be used for validating strings, slices, maps, and arrays. +* `RuneLength(min, max int)`: checks if the length of a string is within the specified range. + This rule is similar as `Length` except that when the value being validated is a string, it checks + its rune length instead of byte length. +* `Min(min interface{})` and `Max(max interface{})`: checks if a value is within the specified range. + These two rules should only be used for validating int, uint, float and time.Time types. +* `Match(*regexp.Regexp)`: checks if a value matches the specified regular expression. + This rule should only be used for strings and byte slices. +* `Date(layout string)`: checks if a string value is a date whose format is specified by the layout. + By calling `Min()` and/or `Max()`, you can check additionally if the date is within the specified range. +* `Required`: checks if a value is not empty (neither nil nor zero). +* `NotNil`: checks if a pointer value is not nil. Non-pointer values are considered valid. +* `NilOrNotEmpty`: checks if a value is a nil pointer or a non-empty value. This differs from `Required` in that it treats a nil pointer as valid. +* `Skip`: this is a special rule used to indicate that all rules following it should be skipped (including the nested ones). +* `MultipleOf`: checks if the value is a multiple of the specified range. + +The `is` sub-package provides a list of commonly used string validation rules that can be used to check if the format +of a value satisfies certain requirements. Note that these rules only handle strings and byte slices and if a string + or byte slice is empty, it is considered valid. You may use a `Required` rule to ensure a value is not empty. +Below is the whole list of the rules provided by the `is` package: + +* `Email`: validates if a string is an email or not +* `URL`: validates if a string is a valid URL +* `RequestURL`: validates if a string is a valid request URL +* `RequestURI`: validates if a string is a valid request URI +* `Alpha`: validates if a string contains English letters only (a-zA-Z) +* `Digit`: validates if a string contains digits only (0-9) +* `Alphanumeric`: validates if a string contains English letters and digits only (a-zA-Z0-9) +* `UTFLetter`: validates if a string contains unicode letters only +* `UTFDigit`: validates if a string contains unicode decimal digits only +* `UTFLetterNumeric`: validates if a string contains unicode letters and numbers only +* `UTFNumeric`: validates if a string contains unicode number characters (category N) only +* `LowerCase`: validates if a string contains lower case unicode letters only +* `UpperCase`: validates if a string contains upper case unicode letters only +* `Hexadecimal`: validates if a string is a valid hexadecimal number +* `HexColor`: validates if a string is a valid hexadecimal color code +* `RGBColor`: validates if a string is a valid RGB color in the form of rgb(R, G, B) +* `Int`: validates if a string is a valid integer number +* `Float`: validates if a string is a floating point number +* `UUIDv3`: validates if a string is a valid version 3 UUID +* `UUIDv4`: validates if a string is a valid version 4 UUID +* `UUIDv5`: validates if a string is a valid version 5 UUID +* `UUID`: validates if a string is a valid UUID +* `CreditCard`: validates if a string is a valid credit card number +* `ISBN10`: validates if a string is an ISBN version 10 +* `ISBN13`: validates if a string is an ISBN version 13 +* `ISBN`: validates if a string is an ISBN (either version 10 or 13) +* `JSON`: validates if a string is in valid JSON format +* `ASCII`: validates if a string contains ASCII characters only +* `PrintableASCII`: validates if a string contains printable ASCII characters only +* `Multibyte`: validates if a string contains multibyte characters +* `FullWidth`: validates if a string contains full-width characters +* `HalfWidth`: validates if a string contains half-width characters +* `VariableWidth`: validates if a string contains both full-width and half-width characters +* `Base64`: validates if a string is encoded in Base64 +* `DataURI`: validates if a string is a valid base64-encoded data URI +* `E164`: validates if a string is a valid E164 phone number (+19251232233) +* `CountryCode2`: validates if a string is a valid ISO3166 Alpha 2 country code +* `CountryCode3`: validates if a string is a valid ISO3166 Alpha 3 country code +* `DialString`: validates if a string is a valid dial string that can be passed to Dial() +* `MAC`: validates if a string is a MAC address +* `IP`: validates if a string is a valid IP address (either version 4 or 6) +* `IPv4`: validates if a string is a valid version 4 IP address +* `IPv6`: validates if a string is a valid version 6 IP address +* `Subdomain`: validates if a string is valid subdomain +* `Domain`: validates if a string is valid domain +* `DNSName`: validates if a string is valid DNS name +* `Host`: validates if a string is a valid IP (both v4 and v6) or a valid DNS name +* `Port`: validates if a string is a valid port number +* `MongoID`: validates if a string is a valid Mongo ID +* `Latitude`: validates if a string is a valid latitude +* `Longitude`: validates if a string is a valid longitude +* `SSN`: validates if a string is a social security number (SSN) +* `Semver`: validates if a string is a valid semantic version + +### Customizing Error Messages + +All built-in validation rules allow you to customize error messages. To do so, simply call the `Error()` method +of the rules. For example, + +```go +data := "2123" +err := validation.Validate(data, + validation.Required.Error("is required"), + validation.Match(regexp.MustCompile("^[0-9]{5}$")).Error("must be a string with five digits"), +) +fmt.Println(err) +// Output: +// must be a string with five digits +``` + + +## Creating Custom Rules + +Creating a custom rule is as simple as implementing the `validation.Rule` interface. The interface contains a single +method as shown below, which should validate the value and return the validation error, if any: + +```go +// Validate validates a value and returns an error if validation fails. +Validate(value interface{}) error +``` + +If you already have a function with the same signature as shown above, you can call `validation.By()` to turn +it into a validation rule. For example, + +```go +func checkAbc(value interface{}) error { + s, _ := value.(string) + if s != "abc" { + return errors.New("must be abc") + } + return nil +} + +err := validation.Validate("xyz", validation.By(checkAbc)) +fmt.Println(err) +// Output: must be abc +``` + + +### Rule Groups + +When a combination of several rules are used in multiple places, you may use the following trick to create a +rule group so that your code is more maintainable. + +```go +var NameRule = []validation.Rule{ + validation.Required, + validation.Length(5, 20), +} + +type User struct { + FirstName string + LastName string +} + +func (u User) Validate() error { + return validation.ValidateStruct(&u, + validation.Field(&u.FirstName, NameRule...), + validation.Field(&u.LastName, NameRule...), + ) +} +``` + +In the above example, we create a rule group `NameRule` which consists of two validation rules. We then use this rule +group to validate both `FirstName` and `LastName`. + + +## Credits + +The `is` sub-package wraps the excellent validators provided by the [govalidator](https://github.com/asaskevich/govalidator) package. diff --git a/vendor/github.com/go-ozzo/ozzo-validation/UPGRADE.md b/vendor/github.com/go-ozzo/ozzo-validation/UPGRADE.md new file mode 100644 index 00000000000..8f11d03eaa9 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/UPGRADE.md @@ -0,0 +1,46 @@ +# Upgrade Instructions + +## Upgrade from 2.x to 3.x + +* Instead of using `StructRules` to define struct validation rules, use `ValidateStruct()` to declare and perform + struct validation. The following code snippet shows how to modify your code: +```go +// 2.x usage +err := validation.StructRules{}. + Add("Street", validation.Required, validation.Length(5, 50)). + Add("City", validation.Required, validation.Length(5, 50)). + Add("State", validation.Required, validation.Match(regexp.MustCompile("^[A-Z]{2}$"))). + Add("Zip", validation.Required, validation.Match(regexp.MustCompile("^[0-9]{5}$"))). + Validate(a) + +// 3.x usage +err := validation.ValidateStruct(&a, + validation.Field(&a.Street, validation.Required, validation.Length(5, 50)), + validation.Field(&a.City, validation.Required, validation.Length(5, 50)), + validation.Field(&a.State, validation.Required, validation.Match(regexp.MustCompile("^[A-Z]{2}$"))), + validation.Field(&a.Zip, validation.Required, validation.Match(regexp.MustCompile("^[0-9]{5}$"))), +) +``` + +* Instead of using `Rules` to declare a rule list and use it to validate a value, call `Validate()` with the rules directly. +```go +data := "example" + +// 2.x usage +rules := validation.Rules{ + validation.Required, + validation.Length(5, 100), + is.URL, +} +err := rules.Validate(data) + +// 3.x usage +err := validation.Validate(data, + validation.Required, + validation.Length(5, 100), + is.URL, +) +``` + +* The default struct tags used for determining error keys is changed from `validation` to `json`. You may modify + `validation.ErrorTag` to change it back. \ No newline at end of file diff --git a/vendor/github.com/go-ozzo/ozzo-validation/date.go b/vendor/github.com/go-ozzo/ozzo-validation/date.go new file mode 100644 index 00000000000..432e035155e --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/date.go @@ -0,0 +1,84 @@ +// Copyright 2016 Qiang Xue. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package validation + +import ( + "errors" + "time" +) + +type DateRule struct { + layout string + min, max time.Time + message string + rangeMessage string +} + +// Date returns a validation rule that checks if a string value is in a format that can be parsed into a date. +// The format of the date should be specified as the layout parameter which accepts the same value as that for time.Parse. +// For example, +// validation.Date(time.ANSIC) +// validation.Date("02 Jan 06 15:04 MST") +// validation.Date("2006-01-02") +// +// By calling Min() and/or Max(), you can let the Date rule to check if a parsed date value is within +// the specified date range. +// +// An empty value is considered valid. Use the Required rule to make sure a value is not empty. +func Date(layout string) *DateRule { + return &DateRule{ + layout: layout, + message: "must be a valid date", + rangeMessage: "the data is out of range", + } +} + +// Error sets the error message that is used when the value being validated is not a valid date. +func (r *DateRule) Error(message string) *DateRule { + r.message = message + return r +} + +// RangeError sets the error message that is used when the value being validated is out of the specified Min/Max date range. +func (r *DateRule) RangeError(message string) *DateRule { + r.rangeMessage = message + return r +} + +// Min sets the minimum date range. A zero value means skipping the minimum range validation. +func (r *DateRule) Min(min time.Time) *DateRule { + r.min = min + return r +} + +// Max sets the maximum date range. A zero value means skipping the maximum range validation. +func (r *DateRule) Max(max time.Time) *DateRule { + r.max = max + return r +} + +// Validate checks if the given value is a valid date. +func (r *DateRule) Validate(value interface{}) error { + value, isNil := Indirect(value) + if isNil || IsEmpty(value) { + return nil + } + + str, err := EnsureString(value) + if err != nil { + return err + } + + date, err := time.Parse(r.layout, str) + if err != nil { + return errors.New(r.message) + } + + if !r.min.IsZero() && r.min.After(date) || !r.max.IsZero() && date.After(r.max) { + return errors.New(r.rangeMessage) + } + + return nil +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/error.go b/vendor/github.com/go-ozzo/ozzo-validation/error.go new file mode 100644 index 00000000000..d89d6285737 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/error.go @@ -0,0 +1,89 @@ +// Copyright 2016 Qiang Xue. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package validation + +import ( + "encoding/json" + "fmt" + "sort" +) + +type ( + // Errors represents the validation errors that are indexed by struct field names, map or slice keys. + Errors map[string]error + + // InternalError represents an error that should NOT be treated as a validation error. + InternalError interface { + error + InternalError() error + } + + internalError struct { + error + } +) + +// NewInternalError wraps a given error into an InternalError. +func NewInternalError(err error) InternalError { + return &internalError{error: err} +} + +// InternalError returns the actual error that it wraps around. +func (e *internalError) InternalError() error { + return e.error +} + +// Error returns the error string of Errors. +func (es Errors) Error() string { + if len(es) == 0 { + return "" + } + + keys := []string{} + for key := range es { + keys = append(keys, key) + } + sort.Strings(keys) + + s := "" + for i, key := range keys { + if i > 0 { + s += "; " + } + if errs, ok := es[key].(Errors); ok { + s += fmt.Sprintf("%v: (%v)", key, errs) + } else { + s += fmt.Sprintf("%v: %v", key, es[key].Error()) + } + } + return s + "." +} + +// MarshalJSON converts the Errors into a valid JSON. +func (es Errors) MarshalJSON() ([]byte, error) { + errs := map[string]interface{}{} + for key, err := range es { + if ms, ok := err.(json.Marshaler); ok { + errs[key] = ms + } else { + errs[key] = err.Error() + } + } + return json.Marshal(errs) +} + +// Filter removes all nils from Errors and returns back the updated Errors as an error. +// If the length of Errors becomes 0, it will return nil. +func (es Errors) Filter() error { + for key, value := range es { + if value == nil { + delete(es, key) + } + } + if len(es) == 0 { + return nil + } + return es +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/in.go b/vendor/github.com/go-ozzo/ozzo-validation/in.go new file mode 100644 index 00000000000..33ee5e53442 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/in.go @@ -0,0 +1,43 @@ +// Copyright 2016 Qiang Xue. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package validation + +import "errors" + +// In returns a validation rule that checks if a value can be found in the given list of values. +// Note that the value being checked and the possible range of values must be of the same type. +// An empty value is considered valid. Use the Required rule to make sure a value is not empty. +func In(values ...interface{}) *InRule { + return &InRule{ + elements: values, + message: "must be a valid value", + } +} + +type InRule struct { + elements []interface{} + message string +} + +// Validate checks if the given value is valid or not. +func (r *InRule) Validate(value interface{}) error { + value, isNil := Indirect(value) + if isNil || IsEmpty(value) { + return nil + } + + for _, e := range r.elements { + if e == value { + return nil + } + } + return errors.New(r.message) +} + +// Error sets the error message for the rule. +func (r *InRule) Error(message string) *InRule { + r.message = message + return r +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/is/BUILD b/vendor/github.com/go-ozzo/ozzo-validation/is/BUILD new file mode 100644 index 00000000000..95b6767cce5 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/is/BUILD @@ -0,0 +1,27 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["rules.go"], + importmap = "k8s.io/kubernetes/vendor/github.com/go-ozzo/ozzo-validation/is", + importpath = "github.com/go-ozzo/ozzo-validation/is", + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/asaskevich/govalidator:go_default_library", + "//vendor/github.com/go-ozzo/ozzo-validation:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/go-ozzo/ozzo-validation/is/rules.go b/vendor/github.com/go-ozzo/ozzo-validation/is/rules.go new file mode 100644 index 00000000000..73cfb7e4de7 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/is/rules.go @@ -0,0 +1,171 @@ +// Copyright 2016 Qiang Xue. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package is provides a list of commonly used string validation rules. +package is + +import ( + "regexp" + "unicode" + + "github.com/asaskevich/govalidator" + "github.com/go-ozzo/ozzo-validation" +) + +var ( + // Email validates if a string is an email or not. + Email = validation.NewStringRule(govalidator.IsEmail, "must be a valid email address") + // URL validates if a string is a valid URL + URL = validation.NewStringRule(govalidator.IsURL, "must be a valid URL") + // RequestURL validates if a string is a valid request URL + RequestURL = validation.NewStringRule(govalidator.IsRequestURL, "must be a valid request URL") + // RequestURI validates if a string is a valid request URI + RequestURI = validation.NewStringRule(govalidator.IsRequestURI, "must be a valid request URI") + // Alpha validates if a string contains English letters only (a-zA-Z) + Alpha = validation.NewStringRule(govalidator.IsAlpha, "must contain English letters only") + // Digit validates if a string contains digits only (0-9) + Digit = validation.NewStringRule(isDigit, "must contain digits only") + // Alphanumeric validates if a string contains English letters and digits only (a-zA-Z0-9) + Alphanumeric = validation.NewStringRule(govalidator.IsAlphanumeric, "must contain English letters and digits only") + // UTFLetter validates if a string contains unicode letters only + UTFLetter = validation.NewStringRule(govalidator.IsUTFLetter, "must contain unicode letter characters only") + // UTFDigit validates if a string contains unicode decimal digits only + UTFDigit = validation.NewStringRule(govalidator.IsUTFDigit, "must contain unicode decimal digits only") + // UTFLetterNumeric validates if a string contains unicode letters and numbers only + UTFLetterNumeric = validation.NewStringRule(govalidator.IsUTFLetterNumeric, "must contain unicode letters and numbers only") + // UTFNumeric validates if a string contains unicode number characters (category N) only + UTFNumeric = validation.NewStringRule(isUTFNumeric, "must contain unicode number characters only") + // LowerCase validates if a string contains lower case unicode letters only + LowerCase = validation.NewStringRule(govalidator.IsLowerCase, "must be in lower case") + // UpperCase validates if a string contains upper case unicode letters only + UpperCase = validation.NewStringRule(govalidator.IsUpperCase, "must be in upper case") + // Hexadecimal validates if a string is a valid hexadecimal number + Hexadecimal = validation.NewStringRule(govalidator.IsHexadecimal, "must be a valid hexadecimal number") + // HexColor validates if a string is a valid hexadecimal color code + HexColor = validation.NewStringRule(govalidator.IsHexcolor, "must be a valid hexadecimal color code") + // RGBColor validates if a string is a valid RGB color in the form of rgb(R, G, B) + RGBColor = validation.NewStringRule(govalidator.IsRGBcolor, "must be a valid RGB color code") + // Int validates if a string is a valid integer number + Int = validation.NewStringRule(govalidator.IsInt, "must be an integer number") + // Float validates if a string is a floating point number + Float = validation.NewStringRule(govalidator.IsFloat, "must be a floating point number") + // UUIDv3 validates if a string is a valid version 3 UUID + UUIDv3 = validation.NewStringRule(govalidator.IsUUIDv3, "must be a valid UUID v3") + // UUIDv4 validates if a string is a valid version 4 UUID + UUIDv4 = validation.NewStringRule(govalidator.IsUUIDv4, "must be a valid UUID v4") + // UUIDv5 validates if a string is a valid version 5 UUID + UUIDv5 = validation.NewStringRule(govalidator.IsUUIDv5, "must be a valid UUID v5") + // UUID validates if a string is a valid UUID + UUID = validation.NewStringRule(govalidator.IsUUID, "must be a valid UUID") + // CreditCard validates if a string is a valid credit card number + CreditCard = validation.NewStringRule(govalidator.IsCreditCard, "must be a valid credit card number") + // ISBN10 validates if a string is an ISBN version 10 + ISBN10 = validation.NewStringRule(govalidator.IsISBN10, "must be a valid ISBN-10") + // ISBN13 validates if a string is an ISBN version 13 + ISBN13 = validation.NewStringRule(govalidator.IsISBN13, "must be a valid ISBN-13") + // ISBN validates if a string is an ISBN (either version 10 or 13) + ISBN = validation.NewStringRule(isISBN, "must be a valid ISBN") + // JSON validates if a string is in valid JSON format + JSON = validation.NewStringRule(govalidator.IsJSON, "must be in valid JSON format") + // ASCII validates if a string contains ASCII characters only + ASCII = validation.NewStringRule(govalidator.IsASCII, "must contain ASCII characters only") + // PrintableASCII validates if a string contains printable ASCII characters only + PrintableASCII = validation.NewStringRule(govalidator.IsPrintableASCII, "must contain printable ASCII characters only") + // Multibyte validates if a string contains multibyte characters + Multibyte = validation.NewStringRule(govalidator.IsMultibyte, "must contain multibyte characters") + // FullWidth validates if a string contains full-width characters + FullWidth = validation.NewStringRule(govalidator.IsFullWidth, "must contain full-width characters") + // HalfWidth validates if a string contains half-width characters + HalfWidth = validation.NewStringRule(govalidator.IsHalfWidth, "must contain half-width characters") + // VariableWidth validates if a string contains both full-width and half-width characters + VariableWidth = validation.NewStringRule(govalidator.IsVariableWidth, "must contain both full-width and half-width characters") + // Base64 validates if a string is encoded in Base64 + Base64 = validation.NewStringRule(govalidator.IsBase64, "must be encoded in Base64") + // DataURI validates if a string is a valid base64-encoded data URI + DataURI = validation.NewStringRule(govalidator.IsDataURI, "must be a Base64-encoded data URI") + // E164 validates if a string is a valid ISO3166 Alpha 2 country code + E164 = validation.NewStringRule(isE164Number, "must be a valid E164 number") + // CountryCode2 validates if a string is a valid ISO3166 Alpha 2 country code + CountryCode2 = validation.NewStringRule(govalidator.IsISO3166Alpha2, "must be a valid two-letter country code") + // CountryCode3 validates if a string is a valid ISO3166 Alpha 3 country code + CountryCode3 = validation.NewStringRule(govalidator.IsISO3166Alpha3, "must be a valid three-letter country code") + // DialString validates if a string is a valid dial string that can be passed to Dial() + DialString = validation.NewStringRule(govalidator.IsDialString, "must be a valid dial string") + // MAC validates if a string is a MAC address + MAC = validation.NewStringRule(govalidator.IsMAC, "must be a valid MAC address") + // IP validates if a string is a valid IP address (either version 4 or 6) + IP = validation.NewStringRule(govalidator.IsIP, "must be a valid IP address") + // IPv4 validates if a string is a valid version 4 IP address + IPv4 = validation.NewStringRule(govalidator.IsIPv4, "must be a valid IPv4 address") + // IPv6 validates if a string is a valid version 6 IP address + IPv6 = validation.NewStringRule(govalidator.IsIPv6, "must be a valid IPv6 address") + // Subdomain validates if a string is valid subdomain + Subdomain = validation.NewStringRule(isSubdomain, "must be a valid subdomain") + // Domain validates if a string is valid domain + Domain = validation.NewStringRule(isDomain, "must be a valid domain") + // DNSName validates if a string is valid DNS name + DNSName = validation.NewStringRule(govalidator.IsDNSName, "must be a valid DNS name") + // Host validates if a string is a valid IP (both v4 and v6) or a valid DNS name + Host = validation.NewStringRule(govalidator.IsHost, "must be a valid IP address or DNS name") + // Port validates if a string is a valid port number + Port = validation.NewStringRule(govalidator.IsPort, "must be a valid port number") + // MongoID validates if a string is a valid Mongo ID + MongoID = validation.NewStringRule(govalidator.IsMongoID, "must be a valid hex-encoded MongoDB ObjectId") + // Latitude validates if a string is a valid latitude + Latitude = validation.NewStringRule(govalidator.IsLatitude, "must be a valid latitude") + // Longitude validates if a string is a valid longitude + Longitude = validation.NewStringRule(govalidator.IsLongitude, "must be a valid longitude") + // SSN validates if a string is a social security number (SSN) + SSN = validation.NewStringRule(govalidator.IsSSN, "must be a valid social security number") + // Semver validates if a string is a valid semantic version + Semver = validation.NewStringRule(govalidator.IsSemver, "must be a valid semantic version") +) + +var ( + reDigit = regexp.MustCompile("^[0-9]+$") + // Subdomain regex source: https://stackoverflow.com/a/7933253 + reSubdomain = regexp.MustCompile(`^[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?$`) +) + +func isISBN(value string) bool { + return govalidator.IsISBN(value, 10) || govalidator.IsISBN(value, 13) +} + +func isDigit(value string) bool { + return reDigit.MatchString(value) +} + +func isE164Number(value string) bool { + // E164 regex source: https://stackoverflow.com/a/23299989 + reE164 := regexp.MustCompile(`^\+?[1-9]\d{1,14}$`) + return reE164.MatchString(value) +} + +func isSubdomain(value string) bool { + // Subdomain regex source: https://stackoverflow.com/a/7933253 + reSubdomain := regexp.MustCompile(`^[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?$`) + return reSubdomain.MatchString(value) +} + +func isDomain(value string) bool { + if len(value) > 255 { + return false + } + + // Domain regex source: https://stackoverflow.com/a/7933253 + // Slightly modified: Removed 255 max length validation since Go regex does not + // support lookarounds. More info: https://stackoverflow.com/a/38935027 + reDomain := regexp.MustCompile(`^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+(?:[a-z]{1,63}| xn--[a-z0-9]{1,59})$`) + + return reDomain.MatchString(value) +} + +func isUTFNumeric(value string) bool { + for _, c := range value { + if unicode.IsNumber(c) == false { + return false + } + } + return true +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/length.go b/vendor/github.com/go-ozzo/ozzo-validation/length.go new file mode 100644 index 00000000000..d5effdf0f72 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/length.go @@ -0,0 +1,81 @@ +// Copyright 2016 Qiang Xue. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package validation + +import ( + "errors" + "fmt" + "unicode/utf8" +) + +// Length returns a validation rule that checks if a value's length is within the specified range. +// If max is 0, it means there is no upper bound for the length. +// This rule should only be used for validating strings, slices, maps, and arrays. +// An empty value is considered valid. Use the Required rule to make sure a value is not empty. +func Length(min, max int) *LengthRule { + message := "the value must be empty" + if min == 0 && max > 0 { + message = fmt.Sprintf("the length must be no more than %v", max) + } else if min > 0 && max == 0 { + message = fmt.Sprintf("the length must be no less than %v", min) + } else if min > 0 && max > 0 { + if min == max { + message = fmt.Sprintf("the length must be exactly %v", min) + } else { + message = fmt.Sprintf("the length must be between %v and %v", min, max) + } + } + return &LengthRule{ + min: min, + max: max, + message: message, + } +} + +// RuneLength returns a validation rule that checks if a string's rune length is within the specified range. +// If max is 0, it means there is no upper bound for the length. +// This rule should only be used for validating strings, slices, maps, and arrays. +// An empty value is considered valid. Use the Required rule to make sure a value is not empty. +// If the value being validated is not a string, the rule works the same as Length. +func RuneLength(min, max int) *LengthRule { + r := Length(min, max) + r.rune = true + return r +} + +type LengthRule struct { + min, max int + message string + rune bool +} + +// Validate checks if the given value is valid or not. +func (v *LengthRule) Validate(value interface{}) error { + value, isNil := Indirect(value) + if isNil || IsEmpty(value) { + return nil + } + + var ( + l int + err error + ) + if s, ok := value.(string); ok && v.rune { + l = utf8.RuneCountInString(s) + } else if l, err = LengthOfValue(value); err != nil { + return err + } + + if v.min > 0 && l < v.min || v.max > 0 && l > v.max { + return errors.New(v.message) + } + return nil +} + +// Error sets the error message for the rule. +func (v *LengthRule) Error(message string) *LengthRule { + v.message = message + return v +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/match.go b/vendor/github.com/go-ozzo/ozzo-validation/match.go new file mode 100644 index 00000000000..4a842bedecb --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/match.go @@ -0,0 +1,47 @@ +// Copyright 2016 Qiang Xue. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package validation + +import ( + "errors" + "regexp" +) + +// Match returns a validation rule that checks if a value matches the specified regular expression. +// This rule should only be used for validating strings and byte slices, or a validation error will be reported. +// An empty value is considered valid. Use the Required rule to make sure a value is not empty. +func Match(re *regexp.Regexp) *MatchRule { + return &MatchRule{ + re: re, + message: "must be in a valid format", + } +} + +type MatchRule struct { + re *regexp.Regexp + message string +} + +// Validate checks if the given value is valid or not. +func (v *MatchRule) Validate(value interface{}) error { + value, isNil := Indirect(value) + if isNil { + return nil + } + + isString, str, isBytes, bs := StringOrBytes(value) + if isString && (str == "" || v.re.MatchString(str)) { + return nil + } else if isBytes && (len(bs) == 0 || v.re.Match(bs)) { + return nil + } + return errors.New(v.message) +} + +// Error sets the error message for the rule. +func (v *MatchRule) Error(message string) *MatchRule { + v.message = message + return v +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/minmax.go b/vendor/github.com/go-ozzo/ozzo-validation/minmax.go new file mode 100644 index 00000000000..5eded0a54b6 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/minmax.go @@ -0,0 +1,177 @@ +// Copyright 2016 Qiang Xue. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package validation + +import ( + "errors" + "fmt" + "reflect" + "time" +) + +type ThresholdRule struct { + threshold interface{} + operator int + message string +} + +const ( + greaterThan = iota + greaterEqualThan + lessThan + lessEqualThan +) + +// Min is a validation rule that checks if a value is greater or equal than the specified value. +// By calling Exclusive, the rule will check if the value is strictly greater than the specified value. +// Note that the value being checked and the threshold value must be of the same type. +// Only int, uint, float and time.Time types are supported. +// An empty value is considered valid. Please use the Required rule to make sure a value is not empty. +func Min(min interface{}) *ThresholdRule { + return &ThresholdRule{ + threshold: min, + operator: greaterEqualThan, + message: fmt.Sprintf("must be no less than %v", min), + } +} + +// Max is a validation rule that checks if a value is less or equal than the specified value. +// By calling Exclusive, the rule will check if the value is strictly less than the specified value. +// Note that the value being checked and the threshold value must be of the same type. +// Only int, uint, float and time.Time types are supported. +// An empty value is considered valid. Please use the Required rule to make sure a value is not empty. +func Max(max interface{}) *ThresholdRule { + return &ThresholdRule{ + threshold: max, + operator: lessEqualThan, + message: fmt.Sprintf("must be no greater than %v", max), + } +} + +// Exclusive sets the comparison to exclude the boundary value. +func (r *ThresholdRule) Exclusive() *ThresholdRule { + if r.operator == greaterEqualThan { + r.operator = greaterThan + r.message = fmt.Sprintf("must be greater than %v", r.threshold) + } else if r.operator == lessEqualThan { + r.operator = lessThan + r.message = fmt.Sprintf("must be less than %v", r.threshold) + } + return r +} + +// Validate checks if the given value is valid or not. +func (r *ThresholdRule) Validate(value interface{}) error { + value, isNil := Indirect(value) + if isNil || IsEmpty(value) { + return nil + } + + rv := reflect.ValueOf(r.threshold) + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + v, err := ToInt(value) + if err != nil { + return err + } + if r.compareInt(rv.Int(), v) { + return nil + } + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + v, err := ToUint(value) + if err != nil { + return err + } + if r.compareUint(rv.Uint(), v) { + return nil + } + + case reflect.Float32, reflect.Float64: + v, err := ToFloat(value) + if err != nil { + return err + } + if r.compareFloat(rv.Float(), v) { + return nil + } + + case reflect.Struct: + t, ok := r.threshold.(time.Time) + if !ok { + return fmt.Errorf("type not supported: %v", rv.Type()) + } + v, ok := value.(time.Time) + if !ok { + return fmt.Errorf("cannot convert %v to time.Time", reflect.TypeOf(value)) + } + if v.IsZero() || r.compareTime(t, v) { + return nil + } + + default: + return fmt.Errorf("type not supported: %v", rv.Type()) + } + + return errors.New(r.message) +} + +// Error sets the error message for the rule. +func (r *ThresholdRule) Error(message string) *ThresholdRule { + r.message = message + return r +} + +func (r *ThresholdRule) compareInt(threshold, value int64) bool { + switch r.operator { + case greaterThan: + return value > threshold + case greaterEqualThan: + return value >= threshold + case lessThan: + return value < threshold + default: + return value <= threshold + } +} + +func (r *ThresholdRule) compareUint(threshold, value uint64) bool { + switch r.operator { + case greaterThan: + return value > threshold + case greaterEqualThan: + return value >= threshold + case lessThan: + return value < threshold + default: + return value <= threshold + } +} + +func (r *ThresholdRule) compareFloat(threshold, value float64) bool { + switch r.operator { + case greaterThan: + return value > threshold + case greaterEqualThan: + return value >= threshold + case lessThan: + return value < threshold + default: + return value <= threshold + } +} + +func (r *ThresholdRule) compareTime(threshold, value time.Time) bool { + switch r.operator { + case greaterThan: + return value.After(threshold) + case greaterEqualThan: + return value.After(threshold) || value.Equal(threshold) + case lessThan: + return value.Before(threshold) + default: + return value.Before(threshold) || value.Equal(threshold) + } +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/multipleof.go b/vendor/github.com/go-ozzo/ozzo-validation/multipleof.go new file mode 100644 index 00000000000..c40fcfa72c8 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/multipleof.go @@ -0,0 +1,55 @@ +package validation + +import ( + "errors" + "fmt" + "reflect" +) + +func MultipleOf(threshold interface{}) *multipleOfRule { + return &multipleOfRule{ + threshold, + fmt.Sprintf("must be multiple of %v", threshold), + } +} + +type multipleOfRule struct { + threshold interface{} + message string +} + +// Error sets the error message for the rule. +func (r *multipleOfRule) Error(message string) *multipleOfRule { + r.message = message + return r +} + + +func (r *multipleOfRule) Validate(value interface{}) error { + + rv := reflect.ValueOf(r.threshold) + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + v, err := ToInt(value) + if err != nil { + return err + } + if v%rv.Int() == 0 { + return nil + } + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + v, err := ToUint(value) + if err != nil { + return err + } + + if v%rv.Uint() == 0 { + return nil + } + default: + return fmt.Errorf("type not supported: %v", rv.Type()) + } + + return errors.New(r.message) +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/not_in.go b/vendor/github.com/go-ozzo/ozzo-validation/not_in.go new file mode 100644 index 00000000000..18cf4a0ffc4 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/not_in.go @@ -0,0 +1,45 @@ +// Copyright 2018 Qiang Xue, Google LLC. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package validation + +import ( + "errors" +) + +// NotIn returns a validation rule that checks if a value os absent from, the given list of values. +// Note that the value being checked and the possible range of values must be of the same type. +// An empty value is considered valid. Use the Required rule to make sure a value is not empty. +func NotIn(values ...interface{}) *NotInRule { + return &NotInRule{ + elements: values, + message: "must not be in list", + } +} + +type NotInRule struct { + elements []interface{} + message string +} + +// Validate checks if the given value is valid or not. +func (r *NotInRule) Validate(value interface{}) error { + value, isNil := Indirect(value) + if isNil || IsEmpty(value) { + return nil + } + + for _, e := range r.elements { + if e == value { + return errors.New(r.message) + } + } + return nil +} + +// Error sets the error message for the rule. +func (r *NotInRule) Error(message string) *NotInRule { + r.message = message + return r +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/not_nil.go b/vendor/github.com/go-ozzo/ozzo-validation/not_nil.go new file mode 100644 index 00000000000..6cfca1204af --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/not_nil.go @@ -0,0 +1,32 @@ +// Copyright 2016 Qiang Xue. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package validation + +import "errors" + +// NotNil is a validation rule that checks if a value is not nil. +// NotNil only handles types including interface, pointer, slice, and map. +// All other types are considered valid. +var NotNil = ¬NilRule{message: "is required"} + +type notNilRule struct { + message string +} + +// Validate checks if the given value is valid or not. +func (r *notNilRule) Validate(value interface{}) error { + _, isNil := Indirect(value) + if isNil { + return errors.New(r.message) + } + return nil +} + +// Error sets the error message for the rule. +func (r *notNilRule) Error(message string) *notNilRule { + return ¬NilRule{ + message: message, + } +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/required.go b/vendor/github.com/go-ozzo/ozzo-validation/required.go new file mode 100644 index 00000000000..ef9558e0254 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/required.go @@ -0,0 +1,42 @@ +// Copyright 2016 Qiang Xue. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package validation + +import "errors" + +// Required is a validation rule that checks if a value is not empty. +// A value is considered not empty if +// - integer, float: not zero +// - bool: true +// - string, array, slice, map: len() > 0 +// - interface, pointer: not nil and the referenced value is not empty +// - any other types +var Required = &requiredRule{message: "cannot be blank", skipNil: false} + +// NilOrNotEmpty checks if a value is a nil pointer or a value that is not empty. +// NilOrNotEmpty differs from Required in that it treats a nil pointer as valid. +var NilOrNotEmpty = &requiredRule{message: "cannot be blank", skipNil: true} + +type requiredRule struct { + message string + skipNil bool +} + +// Validate checks if the given value is valid or not. +func (v *requiredRule) Validate(value interface{}) error { + value, isNil := Indirect(value) + if v.skipNil && !isNil && IsEmpty(value) || !v.skipNil && (isNil || IsEmpty(value)) { + return errors.New(v.message) + } + return nil +} + +// Error sets the error message for the rule. +func (v *requiredRule) Error(message string) *requiredRule { + return &requiredRule{ + message: message, + skipNil: v.skipNil, + } +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/string.go b/vendor/github.com/go-ozzo/ozzo-validation/string.go new file mode 100644 index 00000000000..e8f2a5e749a --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/string.go @@ -0,0 +1,48 @@ +// Copyright 2016 Qiang Xue. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package validation + +import "errors" + +type stringValidator func(string) bool + +// StringRule is a rule that checks a string variable using a specified stringValidator. +type StringRule struct { + validate stringValidator + message string +} + +// NewStringRule creates a new validation rule using a function that takes a string value and returns a bool. +// The rule returned will use the function to check if a given string or byte slice is valid or not. +// An empty value is considered to be valid. Please use the Required rule to make sure a value is not empty. +func NewStringRule(validator stringValidator, message string) *StringRule { + return &StringRule{ + validate: validator, + message: message, + } +} + +// Error sets the error message for the rule. +func (v *StringRule) Error(message string) *StringRule { + return NewStringRule(v.validate, message) +} + +// Validate checks if the given value is valid or not. +func (v *StringRule) Validate(value interface{}) error { + value, isNil := Indirect(value) + if isNil || IsEmpty(value) { + return nil + } + + str, err := EnsureString(value) + if err != nil { + return err + } + + if v.validate(str) { + return nil + } + return errors.New(v.message) +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/struct.go b/vendor/github.com/go-ozzo/ozzo-validation/struct.go new file mode 100644 index 00000000000..2ff852ad5dc --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/struct.go @@ -0,0 +1,154 @@ +// Copyright 2016 Qiang Xue. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package validation + +import ( + "errors" + "fmt" + "reflect" + "strings" +) + +var ( + // ErrStructPointer is the error that a struct being validated is not specified as a pointer. + ErrStructPointer = errors.New("only a pointer to a struct can be validated") +) + +type ( + // ErrFieldPointer is the error that a field is not specified as a pointer. + ErrFieldPointer int + + // ErrFieldNotFound is the error that a field cannot be found in the struct. + ErrFieldNotFound int + + // FieldRules represents a rule set associated with a struct field. + FieldRules struct { + fieldPtr interface{} + rules []Rule + } +) + +// Error returns the error string of ErrFieldPointer. +func (e ErrFieldPointer) Error() string { + return fmt.Sprintf("field #%v must be specified as a pointer", int(e)) +} + +// Error returns the error string of ErrFieldNotFound. +func (e ErrFieldNotFound) Error() string { + return fmt.Sprintf("field #%v cannot be found in the struct", int(e)) +} + +// ValidateStruct validates a struct by checking the specified struct fields against the corresponding validation rules. +// Note that the struct being validated must be specified as a pointer to it. If the pointer is nil, it is considered valid. +// Use Field() to specify struct fields that need to be validated. Each Field() call specifies a single field which +// should be specified as a pointer to the field. A field can be associated with multiple rules. +// For example, +// +// value := struct { +// Name string +// Value string +// }{"name", "demo"} +// err := validation.ValidateStruct(&value, +// validation.Field(&a.Name, validation.Required), +// validation.Field(&a.Value, validation.Required, validation.Length(5, 10)), +// ) +// fmt.Println(err) +// // Value: the length must be between 5 and 10. +// +// An error will be returned if validation fails. +func ValidateStruct(structPtr interface{}, fields ...*FieldRules) error { + value := reflect.ValueOf(structPtr) + if value.Kind() != reflect.Ptr || !value.IsNil() && value.Elem().Kind() != reflect.Struct { + // must be a pointer to a struct + return NewInternalError(ErrStructPointer) + } + if value.IsNil() { + // treat a nil struct pointer as valid + return nil + } + value = value.Elem() + + errs := Errors{} + + for i, fr := range fields { + fv := reflect.ValueOf(fr.fieldPtr) + if fv.Kind() != reflect.Ptr { + return NewInternalError(ErrFieldPointer(i)) + } + ft := findStructField(value, fv) + if ft == nil { + return NewInternalError(ErrFieldNotFound(i)) + } + if err := Validate(fv.Elem().Interface(), fr.rules...); err != nil { + if ie, ok := err.(InternalError); ok && ie.InternalError() != nil { + return err + } + if ft.Anonymous { + // merge errors from anonymous struct field + if es, ok := err.(Errors); ok { + for name, value := range es { + errs[name] = value + } + continue + } + } + errs[getErrorFieldName(ft)] = err + } + } + + if len(errs) > 0 { + return errs + } + return nil +} + +// Field specifies a struct field and the corresponding validation rules. +// The struct field must be specified as a pointer to it. +func Field(fieldPtr interface{}, rules ...Rule) *FieldRules { + return &FieldRules{ + fieldPtr: fieldPtr, + rules: rules, + } +} + +// findStructField looks for a field in the given struct. +// The field being looked for should be a pointer to the actual struct field. +// If found, the field info will be returned. Otherwise, nil will be returned. +func findStructField(structValue reflect.Value, fieldValue reflect.Value) *reflect.StructField { + ptr := fieldValue.Pointer() + for i := structValue.NumField() - 1; i >= 0; i-- { + sf := structValue.Type().Field(i) + if ptr == structValue.Field(i).UnsafeAddr() { + // do additional type comparison because it's possible that the address of + // an embedded struct is the same as the first field of the embedded struct + if sf.Type == fieldValue.Elem().Type() { + return &sf + } + } + if sf.Anonymous { + // delve into anonymous struct to look for the field + fi := structValue.Field(i) + if sf.Type.Kind() == reflect.Ptr { + fi = fi.Elem() + } + if fi.Kind() == reflect.Struct { + if f := findStructField(fi, fieldValue); f != nil { + return f + } + } + } + } + return nil +} + +// getErrorFieldName returns the name that should be used to represent the validation error of a struct field. +func getErrorFieldName(f *reflect.StructField) string { + if tag := f.Tag.Get(ErrorTag); tag != "" { + if cps := strings.SplitN(tag, ",", 2); cps[0] != "" { + return cps[0] + } + } + return f.Name +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/util.go b/vendor/github.com/go-ozzo/ozzo-validation/util.go new file mode 100644 index 00000000000..b15fd9a2979 --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/util.go @@ -0,0 +1,163 @@ +// Copyright 2016 Qiang Xue. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package validation + +import ( + "database/sql/driver" + "errors" + "fmt" + "reflect" + "time" +) + +var ( + bytesType = reflect.TypeOf([]byte(nil)) + valuerType = reflect.TypeOf((*driver.Valuer)(nil)).Elem() +) + +// EnsureString ensures the given value is a string. +// If the value is a byte slice, it will be typecast into a string. +// An error is returned otherwise. +func EnsureString(value interface{}) (string, error) { + v := reflect.ValueOf(value) + if v.Kind() == reflect.String { + return v.String(), nil + } + if v.Type() == bytesType { + return string(v.Interface().([]byte)), nil + } + return "", errors.New("must be either a string or byte slice") +} + +// StringOrBytes typecasts a value into a string or byte slice. +// Boolean flags are returned to indicate if the typecasting succeeds or not. +func StringOrBytes(value interface{}) (isString bool, str string, isBytes bool, bs []byte) { + v := reflect.ValueOf(value) + if v.Kind() == reflect.String { + str = v.String() + isString = true + } else if v.Kind() == reflect.Slice && v.Type() == bytesType { + bs = v.Interface().([]byte) + isBytes = true + } + return +} + +// LengthOfValue returns the length of a value that is a string, slice, map, or array. +// An error is returned for all other types. +func LengthOfValue(value interface{}) (int, error) { + v := reflect.ValueOf(value) + switch v.Kind() { + case reflect.String, reflect.Slice, reflect.Map, reflect.Array: + return v.Len(), nil + } + return 0, fmt.Errorf("cannot get the length of %v", v.Kind()) +} + +// ToInt converts the given value to an int64. +// An error is returned for all incompatible types. +func ToInt(value interface{}) (int64, error) { + v := reflect.ValueOf(value) + switch v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int(), nil + } + return 0, fmt.Errorf("cannot convert %v to int64", v.Kind()) +} + +// ToUint converts the given value to an uint64. +// An error is returned for all incompatible types. +func ToUint(value interface{}) (uint64, error) { + v := reflect.ValueOf(value) + switch v.Kind() { + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint(), nil + } + return 0, fmt.Errorf("cannot convert %v to uint64", v.Kind()) +} + +// ToFloat converts the given value to a float64. +// An error is returned for all incompatible types. +func ToFloat(value interface{}) (float64, error) { + v := reflect.ValueOf(value) + switch v.Kind() { + case reflect.Float32, reflect.Float64: + return v.Float(), nil + } + return 0, fmt.Errorf("cannot convert %v to float64", v.Kind()) +} + +// IsEmpty checks if a value is empty or not. +// A value is considered empty if +// - integer, float: zero +// - bool: false +// - string, array: len() == 0 +// - slice, map: nil or len() == 0 +// - interface, pointer: nil or the referenced value is empty +func IsEmpty(value interface{}) bool { + v := reflect.ValueOf(value) + switch v.Kind() { + case reflect.String, reflect.Array, reflect.Map, reflect.Slice: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Invalid: + return true + case reflect.Interface, reflect.Ptr: + if v.IsNil() { + return true + } + return IsEmpty(v.Elem().Interface()) + case reflect.Struct: + v, ok := value.(time.Time) + if ok && v.IsZero() { + return true + } + } + + return false +} + +// Indirect returns the value that the given interface or pointer references to. +// If the value implements driver.Valuer, it will deal with the value returned by +// the Value() method instead. A boolean value is also returned to indicate if +// the value is nil or not (only applicable to interface, pointer, map, and slice). +// If the value is neither an interface nor a pointer, it will be returned back. +func Indirect(value interface{}) (interface{}, bool) { + rv := reflect.ValueOf(value) + kind := rv.Kind() + switch kind { + case reflect.Invalid: + return nil, true + case reflect.Ptr, reflect.Interface: + if rv.IsNil() { + return nil, true + } + return Indirect(rv.Elem().Interface()) + case reflect.Slice, reflect.Map, reflect.Func, reflect.Chan: + if rv.IsNil() { + return nil, true + } + } + + if rv.Type().Implements(valuerType) { + return indirectValuer(value.(driver.Valuer)) + } + + return value, false +} + +func indirectValuer(valuer driver.Valuer) (interface{}, bool) { + if value, err := valuer.Value(); value != nil && err == nil { + return Indirect(value) + } + return nil, true +} diff --git a/vendor/github.com/go-ozzo/ozzo-validation/validation.go b/vendor/github.com/go-ozzo/ozzo-validation/validation.go new file mode 100644 index 00000000000..1633258178d --- /dev/null +++ b/vendor/github.com/go-ozzo/ozzo-validation/validation.go @@ -0,0 +1,133 @@ +// Copyright 2016 Qiang Xue. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package validation provides configurable and extensible rules for validating data of various types. +package validation + +import ( + "fmt" + "reflect" + "strconv" +) + +type ( + // Validatable is the interface indicating the type implementing it supports data validation. + Validatable interface { + // Validate validates the data and returns an error if validation fails. + Validate() error + } + + // Rule represents a validation rule. + Rule interface { + // Validate validates a value and returns a value if validation fails. + Validate(value interface{}) error + } + + // RuleFunc represents a validator function. + // You may wrap it as a Rule by calling By(). + RuleFunc func(value interface{}) error +) + +var ( + // ErrorTag is the struct tag name used to customize the error field name for a struct field. + ErrorTag = "json" + + // Skip is a special validation rule that indicates all rules following it should be skipped. + Skip = &skipRule{} + + validatableType = reflect.TypeOf((*Validatable)(nil)).Elem() +) + +// Validate validates the given value and returns the validation error, if any. +// +// Validate performs validation using the following steps: +// - validate the value against the rules passed in as parameters +// - if the value is a map and the map values implement `Validatable`, call `Validate` of every map value +// - if the value is a slice or array whose values implement `Validatable`, call `Validate` of every element +func Validate(value interface{}, rules ...Rule) error { + for _, rule := range rules { + if _, ok := rule.(*skipRule); ok { + return nil + } + if err := rule.Validate(value); err != nil { + return err + } + } + + rv := reflect.ValueOf(value) + if (rv.Kind() == reflect.Ptr || rv.Kind() == reflect.Interface) && rv.IsNil() { + return nil + } + + if v, ok := value.(Validatable); ok { + return v.Validate() + } + + switch rv.Kind() { + case reflect.Map: + if rv.Type().Elem().Implements(validatableType) { + return validateMap(rv) + } + case reflect.Slice, reflect.Array: + if rv.Type().Elem().Implements(validatableType) { + return validateSlice(rv) + } + case reflect.Ptr, reflect.Interface: + return Validate(rv.Elem().Interface()) + } + + return nil +} + +// validateMap validates a map of validatable elements +func validateMap(rv reflect.Value) error { + errs := Errors{} + for _, key := range rv.MapKeys() { + if mv := rv.MapIndex(key).Interface(); mv != nil { + if err := mv.(Validatable).Validate(); err != nil { + errs[fmt.Sprintf("%v", key.Interface())] = err + } + } + } + if len(errs) > 0 { + return errs + } + return nil +} + +// validateMap validates a slice/array of validatable elements +func validateSlice(rv reflect.Value) error { + errs := Errors{} + l := rv.Len() + for i := 0; i < l; i++ { + if ev := rv.Index(i).Interface(); ev != nil { + if err := ev.(Validatable).Validate(); err != nil { + errs[strconv.Itoa(i)] = err + } + } + } + if len(errs) > 0 { + return errs + } + return nil +} + +type skipRule struct{} + +func (r *skipRule) Validate(interface{}) error { + return nil +} + +type inlineRule struct { + f RuleFunc +} + +func (r *inlineRule) Validate(value interface{}) error { + return r.f(value) +} + +// By wraps a RuleFunc into a Rule. +func By(f RuleFunc) Rule { + return &inlineRule{f} +} diff --git a/vendor/github.com/golang/protobuf/protoc-gen-go/descriptor/BUILD b/vendor/github.com/golang/protobuf/protoc-gen-go/descriptor/BUILD new file mode 100644 index 00000000000..0263d174a47 --- /dev/null +++ b/vendor/github.com/golang/protobuf/protoc-gen-go/descriptor/BUILD @@ -0,0 +1,24 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["descriptor.pb.go"], + importmap = "k8s.io/kubernetes/vendor/github.com/golang/protobuf/protoc-gen-go/descriptor", + importpath = "github.com/golang/protobuf/protoc-gen-go/descriptor", + visibility = ["//visibility:public"], + deps = ["//vendor/github.com/golang/protobuf/proto:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/golang/protobuf/protoc-gen-go/descriptor/descriptor.pb.go b/vendor/github.com/golang/protobuf/protoc-gen-go/descriptor/descriptor.pb.go new file mode 100644 index 00000000000..9e029e8cb44 --- /dev/null +++ b/vendor/github.com/golang/protobuf/protoc-gen-go/descriptor/descriptor.pb.go @@ -0,0 +1,2812 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: google/protobuf/descriptor.proto + +package descriptor + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type FieldDescriptorProto_Type int32 + +const ( + // 0 is reserved for errors. + // Order is weird for historical reasons. + FieldDescriptorProto_TYPE_DOUBLE FieldDescriptorProto_Type = 1 + FieldDescriptorProto_TYPE_FLOAT FieldDescriptorProto_Type = 2 + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if + // negative values are likely. + FieldDescriptorProto_TYPE_INT64 FieldDescriptorProto_Type = 3 + FieldDescriptorProto_TYPE_UINT64 FieldDescriptorProto_Type = 4 + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if + // negative values are likely. + FieldDescriptorProto_TYPE_INT32 FieldDescriptorProto_Type = 5 + FieldDescriptorProto_TYPE_FIXED64 FieldDescriptorProto_Type = 6 + FieldDescriptorProto_TYPE_FIXED32 FieldDescriptorProto_Type = 7 + FieldDescriptorProto_TYPE_BOOL FieldDescriptorProto_Type = 8 + FieldDescriptorProto_TYPE_STRING FieldDescriptorProto_Type = 9 + // Tag-delimited aggregate. + // Group type is deprecated and not supported in proto3. However, Proto3 + // implementations should still be able to parse the group wire format and + // treat group fields as unknown fields. + FieldDescriptorProto_TYPE_GROUP FieldDescriptorProto_Type = 10 + FieldDescriptorProto_TYPE_MESSAGE FieldDescriptorProto_Type = 11 + // New in version 2. + FieldDescriptorProto_TYPE_BYTES FieldDescriptorProto_Type = 12 + FieldDescriptorProto_TYPE_UINT32 FieldDescriptorProto_Type = 13 + FieldDescriptorProto_TYPE_ENUM FieldDescriptorProto_Type = 14 + FieldDescriptorProto_TYPE_SFIXED32 FieldDescriptorProto_Type = 15 + FieldDescriptorProto_TYPE_SFIXED64 FieldDescriptorProto_Type = 16 + FieldDescriptorProto_TYPE_SINT32 FieldDescriptorProto_Type = 17 + FieldDescriptorProto_TYPE_SINT64 FieldDescriptorProto_Type = 18 +) + +var FieldDescriptorProto_Type_name = map[int32]string{ + 1: "TYPE_DOUBLE", + 2: "TYPE_FLOAT", + 3: "TYPE_INT64", + 4: "TYPE_UINT64", + 5: "TYPE_INT32", + 6: "TYPE_FIXED64", + 7: "TYPE_FIXED32", + 8: "TYPE_BOOL", + 9: "TYPE_STRING", + 10: "TYPE_GROUP", + 11: "TYPE_MESSAGE", + 12: "TYPE_BYTES", + 13: "TYPE_UINT32", + 14: "TYPE_ENUM", + 15: "TYPE_SFIXED32", + 16: "TYPE_SFIXED64", + 17: "TYPE_SINT32", + 18: "TYPE_SINT64", +} +var FieldDescriptorProto_Type_value = map[string]int32{ + "TYPE_DOUBLE": 1, + "TYPE_FLOAT": 2, + "TYPE_INT64": 3, + "TYPE_UINT64": 4, + "TYPE_INT32": 5, + "TYPE_FIXED64": 6, + "TYPE_FIXED32": 7, + "TYPE_BOOL": 8, + "TYPE_STRING": 9, + "TYPE_GROUP": 10, + "TYPE_MESSAGE": 11, + "TYPE_BYTES": 12, + "TYPE_UINT32": 13, + "TYPE_ENUM": 14, + "TYPE_SFIXED32": 15, + "TYPE_SFIXED64": 16, + "TYPE_SINT32": 17, + "TYPE_SINT64": 18, +} + +func (x FieldDescriptorProto_Type) Enum() *FieldDescriptorProto_Type { + p := new(FieldDescriptorProto_Type) + *p = x + return p +} +func (x FieldDescriptorProto_Type) String() string { + return proto.EnumName(FieldDescriptorProto_Type_name, int32(x)) +} +func (x *FieldDescriptorProto_Type) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(FieldDescriptorProto_Type_value, data, "FieldDescriptorProto_Type") + if err != nil { + return err + } + *x = FieldDescriptorProto_Type(value) + return nil +} +func (FieldDescriptorProto_Type) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{4, 0} +} + +type FieldDescriptorProto_Label int32 + +const ( + // 0 is reserved for errors + FieldDescriptorProto_LABEL_OPTIONAL FieldDescriptorProto_Label = 1 + FieldDescriptorProto_LABEL_REQUIRED FieldDescriptorProto_Label = 2 + FieldDescriptorProto_LABEL_REPEATED FieldDescriptorProto_Label = 3 +) + +var FieldDescriptorProto_Label_name = map[int32]string{ + 1: "LABEL_OPTIONAL", + 2: "LABEL_REQUIRED", + 3: "LABEL_REPEATED", +} +var FieldDescriptorProto_Label_value = map[string]int32{ + "LABEL_OPTIONAL": 1, + "LABEL_REQUIRED": 2, + "LABEL_REPEATED": 3, +} + +func (x FieldDescriptorProto_Label) Enum() *FieldDescriptorProto_Label { + p := new(FieldDescriptorProto_Label) + *p = x + return p +} +func (x FieldDescriptorProto_Label) String() string { + return proto.EnumName(FieldDescriptorProto_Label_name, int32(x)) +} +func (x *FieldDescriptorProto_Label) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(FieldDescriptorProto_Label_value, data, "FieldDescriptorProto_Label") + if err != nil { + return err + } + *x = FieldDescriptorProto_Label(value) + return nil +} +func (FieldDescriptorProto_Label) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{4, 1} +} + +// Generated classes can be optimized for speed or code size. +type FileOptions_OptimizeMode int32 + +const ( + FileOptions_SPEED FileOptions_OptimizeMode = 1 + // etc. + FileOptions_CODE_SIZE FileOptions_OptimizeMode = 2 + FileOptions_LITE_RUNTIME FileOptions_OptimizeMode = 3 +) + +var FileOptions_OptimizeMode_name = map[int32]string{ + 1: "SPEED", + 2: "CODE_SIZE", + 3: "LITE_RUNTIME", +} +var FileOptions_OptimizeMode_value = map[string]int32{ + "SPEED": 1, + "CODE_SIZE": 2, + "LITE_RUNTIME": 3, +} + +func (x FileOptions_OptimizeMode) Enum() *FileOptions_OptimizeMode { + p := new(FileOptions_OptimizeMode) + *p = x + return p +} +func (x FileOptions_OptimizeMode) String() string { + return proto.EnumName(FileOptions_OptimizeMode_name, int32(x)) +} +func (x *FileOptions_OptimizeMode) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(FileOptions_OptimizeMode_value, data, "FileOptions_OptimizeMode") + if err != nil { + return err + } + *x = FileOptions_OptimizeMode(value) + return nil +} +func (FileOptions_OptimizeMode) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{10, 0} +} + +type FieldOptions_CType int32 + +const ( + // Default mode. + FieldOptions_STRING FieldOptions_CType = 0 + FieldOptions_CORD FieldOptions_CType = 1 + FieldOptions_STRING_PIECE FieldOptions_CType = 2 +) + +var FieldOptions_CType_name = map[int32]string{ + 0: "STRING", + 1: "CORD", + 2: "STRING_PIECE", +} +var FieldOptions_CType_value = map[string]int32{ + "STRING": 0, + "CORD": 1, + "STRING_PIECE": 2, +} + +func (x FieldOptions_CType) Enum() *FieldOptions_CType { + p := new(FieldOptions_CType) + *p = x + return p +} +func (x FieldOptions_CType) String() string { + return proto.EnumName(FieldOptions_CType_name, int32(x)) +} +func (x *FieldOptions_CType) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(FieldOptions_CType_value, data, "FieldOptions_CType") + if err != nil { + return err + } + *x = FieldOptions_CType(value) + return nil +} +func (FieldOptions_CType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{12, 0} +} + +type FieldOptions_JSType int32 + +const ( + // Use the default type. + FieldOptions_JS_NORMAL FieldOptions_JSType = 0 + // Use JavaScript strings. + FieldOptions_JS_STRING FieldOptions_JSType = 1 + // Use JavaScript numbers. + FieldOptions_JS_NUMBER FieldOptions_JSType = 2 +) + +var FieldOptions_JSType_name = map[int32]string{ + 0: "JS_NORMAL", + 1: "JS_STRING", + 2: "JS_NUMBER", +} +var FieldOptions_JSType_value = map[string]int32{ + "JS_NORMAL": 0, + "JS_STRING": 1, + "JS_NUMBER": 2, +} + +func (x FieldOptions_JSType) Enum() *FieldOptions_JSType { + p := new(FieldOptions_JSType) + *p = x + return p +} +func (x FieldOptions_JSType) String() string { + return proto.EnumName(FieldOptions_JSType_name, int32(x)) +} +func (x *FieldOptions_JSType) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(FieldOptions_JSType_value, data, "FieldOptions_JSType") + if err != nil { + return err + } + *x = FieldOptions_JSType(value) + return nil +} +func (FieldOptions_JSType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{12, 1} +} + +// Is this method side-effect-free (or safe in HTTP parlance), or idempotent, +// or neither? HTTP based RPC implementation may choose GET verb for safe +// methods, and PUT verb for idempotent methods instead of the default POST. +type MethodOptions_IdempotencyLevel int32 + +const ( + MethodOptions_IDEMPOTENCY_UNKNOWN MethodOptions_IdempotencyLevel = 0 + MethodOptions_NO_SIDE_EFFECTS MethodOptions_IdempotencyLevel = 1 + MethodOptions_IDEMPOTENT MethodOptions_IdempotencyLevel = 2 +) + +var MethodOptions_IdempotencyLevel_name = map[int32]string{ + 0: "IDEMPOTENCY_UNKNOWN", + 1: "NO_SIDE_EFFECTS", + 2: "IDEMPOTENT", +} +var MethodOptions_IdempotencyLevel_value = map[string]int32{ + "IDEMPOTENCY_UNKNOWN": 0, + "NO_SIDE_EFFECTS": 1, + "IDEMPOTENT": 2, +} + +func (x MethodOptions_IdempotencyLevel) Enum() *MethodOptions_IdempotencyLevel { + p := new(MethodOptions_IdempotencyLevel) + *p = x + return p +} +func (x MethodOptions_IdempotencyLevel) String() string { + return proto.EnumName(MethodOptions_IdempotencyLevel_name, int32(x)) +} +func (x *MethodOptions_IdempotencyLevel) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(MethodOptions_IdempotencyLevel_value, data, "MethodOptions_IdempotencyLevel") + if err != nil { + return err + } + *x = MethodOptions_IdempotencyLevel(value) + return nil +} +func (MethodOptions_IdempotencyLevel) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{17, 0} +} + +// The protocol compiler can output a FileDescriptorSet containing the .proto +// files it parses. +type FileDescriptorSet struct { + File []*FileDescriptorProto `protobuf:"bytes,1,rep,name=file" json:"file,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FileDescriptorSet) Reset() { *m = FileDescriptorSet{} } +func (m *FileDescriptorSet) String() string { return proto.CompactTextString(m) } +func (*FileDescriptorSet) ProtoMessage() {} +func (*FileDescriptorSet) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{0} +} +func (m *FileDescriptorSet) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FileDescriptorSet.Unmarshal(m, b) +} +func (m *FileDescriptorSet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FileDescriptorSet.Marshal(b, m, deterministic) +} +func (dst *FileDescriptorSet) XXX_Merge(src proto.Message) { + xxx_messageInfo_FileDescriptorSet.Merge(dst, src) +} +func (m *FileDescriptorSet) XXX_Size() int { + return xxx_messageInfo_FileDescriptorSet.Size(m) +} +func (m *FileDescriptorSet) XXX_DiscardUnknown() { + xxx_messageInfo_FileDescriptorSet.DiscardUnknown(m) +} + +var xxx_messageInfo_FileDescriptorSet proto.InternalMessageInfo + +func (m *FileDescriptorSet) GetFile() []*FileDescriptorProto { + if m != nil { + return m.File + } + return nil +} + +// Describes a complete .proto file. +type FileDescriptorProto struct { + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Package *string `protobuf:"bytes,2,opt,name=package" json:"package,omitempty"` + // Names of files imported by this file. + Dependency []string `protobuf:"bytes,3,rep,name=dependency" json:"dependency,omitempty"` + // Indexes of the public imported files in the dependency list above. + PublicDependency []int32 `protobuf:"varint,10,rep,name=public_dependency,json=publicDependency" json:"public_dependency,omitempty"` + // Indexes of the weak imported files in the dependency list. + // For Google-internal migration only. Do not use. + WeakDependency []int32 `protobuf:"varint,11,rep,name=weak_dependency,json=weakDependency" json:"weak_dependency,omitempty"` + // All top-level definitions in this file. + MessageType []*DescriptorProto `protobuf:"bytes,4,rep,name=message_type,json=messageType" json:"message_type,omitempty"` + EnumType []*EnumDescriptorProto `protobuf:"bytes,5,rep,name=enum_type,json=enumType" json:"enum_type,omitempty"` + Service []*ServiceDescriptorProto `protobuf:"bytes,6,rep,name=service" json:"service,omitempty"` + Extension []*FieldDescriptorProto `protobuf:"bytes,7,rep,name=extension" json:"extension,omitempty"` + Options *FileOptions `protobuf:"bytes,8,opt,name=options" json:"options,omitempty"` + // This field contains optional information about the original source code. + // You may safely remove this entire field without harming runtime + // functionality of the descriptors -- the information is needed only by + // development tools. + SourceCodeInfo *SourceCodeInfo `protobuf:"bytes,9,opt,name=source_code_info,json=sourceCodeInfo" json:"source_code_info,omitempty"` + // The syntax of the proto file. + // The supported values are "proto2" and "proto3". + Syntax *string `protobuf:"bytes,12,opt,name=syntax" json:"syntax,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FileDescriptorProto) Reset() { *m = FileDescriptorProto{} } +func (m *FileDescriptorProto) String() string { return proto.CompactTextString(m) } +func (*FileDescriptorProto) ProtoMessage() {} +func (*FileDescriptorProto) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{1} +} +func (m *FileDescriptorProto) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FileDescriptorProto.Unmarshal(m, b) +} +func (m *FileDescriptorProto) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FileDescriptorProto.Marshal(b, m, deterministic) +} +func (dst *FileDescriptorProto) XXX_Merge(src proto.Message) { + xxx_messageInfo_FileDescriptorProto.Merge(dst, src) +} +func (m *FileDescriptorProto) XXX_Size() int { + return xxx_messageInfo_FileDescriptorProto.Size(m) +} +func (m *FileDescriptorProto) XXX_DiscardUnknown() { + xxx_messageInfo_FileDescriptorProto.DiscardUnknown(m) +} + +var xxx_messageInfo_FileDescriptorProto proto.InternalMessageInfo + +func (m *FileDescriptorProto) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +func (m *FileDescriptorProto) GetPackage() string { + if m != nil && m.Package != nil { + return *m.Package + } + return "" +} + +func (m *FileDescriptorProto) GetDependency() []string { + if m != nil { + return m.Dependency + } + return nil +} + +func (m *FileDescriptorProto) GetPublicDependency() []int32 { + if m != nil { + return m.PublicDependency + } + return nil +} + +func (m *FileDescriptorProto) GetWeakDependency() []int32 { + if m != nil { + return m.WeakDependency + } + return nil +} + +func (m *FileDescriptorProto) GetMessageType() []*DescriptorProto { + if m != nil { + return m.MessageType + } + return nil +} + +func (m *FileDescriptorProto) GetEnumType() []*EnumDescriptorProto { + if m != nil { + return m.EnumType + } + return nil +} + +func (m *FileDescriptorProto) GetService() []*ServiceDescriptorProto { + if m != nil { + return m.Service + } + return nil +} + +func (m *FileDescriptorProto) GetExtension() []*FieldDescriptorProto { + if m != nil { + return m.Extension + } + return nil +} + +func (m *FileDescriptorProto) GetOptions() *FileOptions { + if m != nil { + return m.Options + } + return nil +} + +func (m *FileDescriptorProto) GetSourceCodeInfo() *SourceCodeInfo { + if m != nil { + return m.SourceCodeInfo + } + return nil +} + +func (m *FileDescriptorProto) GetSyntax() string { + if m != nil && m.Syntax != nil { + return *m.Syntax + } + return "" +} + +// Describes a message type. +type DescriptorProto struct { + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Field []*FieldDescriptorProto `protobuf:"bytes,2,rep,name=field" json:"field,omitempty"` + Extension []*FieldDescriptorProto `protobuf:"bytes,6,rep,name=extension" json:"extension,omitempty"` + NestedType []*DescriptorProto `protobuf:"bytes,3,rep,name=nested_type,json=nestedType" json:"nested_type,omitempty"` + EnumType []*EnumDescriptorProto `protobuf:"bytes,4,rep,name=enum_type,json=enumType" json:"enum_type,omitempty"` + ExtensionRange []*DescriptorProto_ExtensionRange `protobuf:"bytes,5,rep,name=extension_range,json=extensionRange" json:"extension_range,omitempty"` + OneofDecl []*OneofDescriptorProto `protobuf:"bytes,8,rep,name=oneof_decl,json=oneofDecl" json:"oneof_decl,omitempty"` + Options *MessageOptions `protobuf:"bytes,7,opt,name=options" json:"options,omitempty"` + ReservedRange []*DescriptorProto_ReservedRange `protobuf:"bytes,9,rep,name=reserved_range,json=reservedRange" json:"reserved_range,omitempty"` + // Reserved field names, which may not be used by fields in the same message. + // A given name may only be reserved once. + ReservedName []string `protobuf:"bytes,10,rep,name=reserved_name,json=reservedName" json:"reserved_name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DescriptorProto) Reset() { *m = DescriptorProto{} } +func (m *DescriptorProto) String() string { return proto.CompactTextString(m) } +func (*DescriptorProto) ProtoMessage() {} +func (*DescriptorProto) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{2} +} +func (m *DescriptorProto) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DescriptorProto.Unmarshal(m, b) +} +func (m *DescriptorProto) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DescriptorProto.Marshal(b, m, deterministic) +} +func (dst *DescriptorProto) XXX_Merge(src proto.Message) { + xxx_messageInfo_DescriptorProto.Merge(dst, src) +} +func (m *DescriptorProto) XXX_Size() int { + return xxx_messageInfo_DescriptorProto.Size(m) +} +func (m *DescriptorProto) XXX_DiscardUnknown() { + xxx_messageInfo_DescriptorProto.DiscardUnknown(m) +} + +var xxx_messageInfo_DescriptorProto proto.InternalMessageInfo + +func (m *DescriptorProto) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +func (m *DescriptorProto) GetField() []*FieldDescriptorProto { + if m != nil { + return m.Field + } + return nil +} + +func (m *DescriptorProto) GetExtension() []*FieldDescriptorProto { + if m != nil { + return m.Extension + } + return nil +} + +func (m *DescriptorProto) GetNestedType() []*DescriptorProto { + if m != nil { + return m.NestedType + } + return nil +} + +func (m *DescriptorProto) GetEnumType() []*EnumDescriptorProto { + if m != nil { + return m.EnumType + } + return nil +} + +func (m *DescriptorProto) GetExtensionRange() []*DescriptorProto_ExtensionRange { + if m != nil { + return m.ExtensionRange + } + return nil +} + +func (m *DescriptorProto) GetOneofDecl() []*OneofDescriptorProto { + if m != nil { + return m.OneofDecl + } + return nil +} + +func (m *DescriptorProto) GetOptions() *MessageOptions { + if m != nil { + return m.Options + } + return nil +} + +func (m *DescriptorProto) GetReservedRange() []*DescriptorProto_ReservedRange { + if m != nil { + return m.ReservedRange + } + return nil +} + +func (m *DescriptorProto) GetReservedName() []string { + if m != nil { + return m.ReservedName + } + return nil +} + +type DescriptorProto_ExtensionRange struct { + Start *int32 `protobuf:"varint,1,opt,name=start" json:"start,omitempty"` + End *int32 `protobuf:"varint,2,opt,name=end" json:"end,omitempty"` + Options *ExtensionRangeOptions `protobuf:"bytes,3,opt,name=options" json:"options,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DescriptorProto_ExtensionRange) Reset() { *m = DescriptorProto_ExtensionRange{} } +func (m *DescriptorProto_ExtensionRange) String() string { return proto.CompactTextString(m) } +func (*DescriptorProto_ExtensionRange) ProtoMessage() {} +func (*DescriptorProto_ExtensionRange) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{2, 0} +} +func (m *DescriptorProto_ExtensionRange) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DescriptorProto_ExtensionRange.Unmarshal(m, b) +} +func (m *DescriptorProto_ExtensionRange) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DescriptorProto_ExtensionRange.Marshal(b, m, deterministic) +} +func (dst *DescriptorProto_ExtensionRange) XXX_Merge(src proto.Message) { + xxx_messageInfo_DescriptorProto_ExtensionRange.Merge(dst, src) +} +func (m *DescriptorProto_ExtensionRange) XXX_Size() int { + return xxx_messageInfo_DescriptorProto_ExtensionRange.Size(m) +} +func (m *DescriptorProto_ExtensionRange) XXX_DiscardUnknown() { + xxx_messageInfo_DescriptorProto_ExtensionRange.DiscardUnknown(m) +} + +var xxx_messageInfo_DescriptorProto_ExtensionRange proto.InternalMessageInfo + +func (m *DescriptorProto_ExtensionRange) GetStart() int32 { + if m != nil && m.Start != nil { + return *m.Start + } + return 0 +} + +func (m *DescriptorProto_ExtensionRange) GetEnd() int32 { + if m != nil && m.End != nil { + return *m.End + } + return 0 +} + +func (m *DescriptorProto_ExtensionRange) GetOptions() *ExtensionRangeOptions { + if m != nil { + return m.Options + } + return nil +} + +// Range of reserved tag numbers. Reserved tag numbers may not be used by +// fields or extension ranges in the same message. Reserved ranges may +// not overlap. +type DescriptorProto_ReservedRange struct { + Start *int32 `protobuf:"varint,1,opt,name=start" json:"start,omitempty"` + End *int32 `protobuf:"varint,2,opt,name=end" json:"end,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DescriptorProto_ReservedRange) Reset() { *m = DescriptorProto_ReservedRange{} } +func (m *DescriptorProto_ReservedRange) String() string { return proto.CompactTextString(m) } +func (*DescriptorProto_ReservedRange) ProtoMessage() {} +func (*DescriptorProto_ReservedRange) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{2, 1} +} +func (m *DescriptorProto_ReservedRange) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DescriptorProto_ReservedRange.Unmarshal(m, b) +} +func (m *DescriptorProto_ReservedRange) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DescriptorProto_ReservedRange.Marshal(b, m, deterministic) +} +func (dst *DescriptorProto_ReservedRange) XXX_Merge(src proto.Message) { + xxx_messageInfo_DescriptorProto_ReservedRange.Merge(dst, src) +} +func (m *DescriptorProto_ReservedRange) XXX_Size() int { + return xxx_messageInfo_DescriptorProto_ReservedRange.Size(m) +} +func (m *DescriptorProto_ReservedRange) XXX_DiscardUnknown() { + xxx_messageInfo_DescriptorProto_ReservedRange.DiscardUnknown(m) +} + +var xxx_messageInfo_DescriptorProto_ReservedRange proto.InternalMessageInfo + +func (m *DescriptorProto_ReservedRange) GetStart() int32 { + if m != nil && m.Start != nil { + return *m.Start + } + return 0 +} + +func (m *DescriptorProto_ReservedRange) GetEnd() int32 { + if m != nil && m.End != nil { + return *m.End + } + return 0 +} + +type ExtensionRangeOptions struct { + // The parser stores options it doesn't recognize here. See above. + UninterpretedOption []*UninterpretedOption `protobuf:"bytes,999,rep,name=uninterpreted_option,json=uninterpretedOption" json:"uninterpreted_option,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + proto.XXX_InternalExtensions `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ExtensionRangeOptions) Reset() { *m = ExtensionRangeOptions{} } +func (m *ExtensionRangeOptions) String() string { return proto.CompactTextString(m) } +func (*ExtensionRangeOptions) ProtoMessage() {} +func (*ExtensionRangeOptions) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{3} +} + +var extRange_ExtensionRangeOptions = []proto.ExtensionRange{ + {Start: 1000, End: 536870911}, +} + +func (*ExtensionRangeOptions) ExtensionRangeArray() []proto.ExtensionRange { + return extRange_ExtensionRangeOptions +} +func (m *ExtensionRangeOptions) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ExtensionRangeOptions.Unmarshal(m, b) +} +func (m *ExtensionRangeOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ExtensionRangeOptions.Marshal(b, m, deterministic) +} +func (dst *ExtensionRangeOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_ExtensionRangeOptions.Merge(dst, src) +} +func (m *ExtensionRangeOptions) XXX_Size() int { + return xxx_messageInfo_ExtensionRangeOptions.Size(m) +} +func (m *ExtensionRangeOptions) XXX_DiscardUnknown() { + xxx_messageInfo_ExtensionRangeOptions.DiscardUnknown(m) +} + +var xxx_messageInfo_ExtensionRangeOptions proto.InternalMessageInfo + +func (m *ExtensionRangeOptions) GetUninterpretedOption() []*UninterpretedOption { + if m != nil { + return m.UninterpretedOption + } + return nil +} + +// Describes a field within a message. +type FieldDescriptorProto struct { + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Number *int32 `protobuf:"varint,3,opt,name=number" json:"number,omitempty"` + Label *FieldDescriptorProto_Label `protobuf:"varint,4,opt,name=label,enum=google.protobuf.FieldDescriptorProto_Label" json:"label,omitempty"` + // If type_name is set, this need not be set. If both this and type_name + // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. + Type *FieldDescriptorProto_Type `protobuf:"varint,5,opt,name=type,enum=google.protobuf.FieldDescriptorProto_Type" json:"type,omitempty"` + // For message and enum types, this is the name of the type. If the name + // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping + // rules are used to find the type (i.e. first the nested types within this + // message are searched, then within the parent, on up to the root + // namespace). + TypeName *string `protobuf:"bytes,6,opt,name=type_name,json=typeName" json:"type_name,omitempty"` + // For extensions, this is the name of the type being extended. It is + // resolved in the same manner as type_name. + Extendee *string `protobuf:"bytes,2,opt,name=extendee" json:"extendee,omitempty"` + // For numeric types, contains the original text representation of the value. + // For booleans, "true" or "false". + // For strings, contains the default text contents (not escaped in any way). + // For bytes, contains the C escaped value. All bytes >= 128 are escaped. + // TODO(kenton): Base-64 encode? + DefaultValue *string `protobuf:"bytes,7,opt,name=default_value,json=defaultValue" json:"default_value,omitempty"` + // If set, gives the index of a oneof in the containing type's oneof_decl + // list. This field is a member of that oneof. + OneofIndex *int32 `protobuf:"varint,9,opt,name=oneof_index,json=oneofIndex" json:"oneof_index,omitempty"` + // JSON name of this field. The value is set by protocol compiler. If the + // user has set a "json_name" option on this field, that option's value + // will be used. Otherwise, it's deduced from the field's name by converting + // it to camelCase. + JsonName *string `protobuf:"bytes,10,opt,name=json_name,json=jsonName" json:"json_name,omitempty"` + Options *FieldOptions `protobuf:"bytes,8,opt,name=options" json:"options,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FieldDescriptorProto) Reset() { *m = FieldDescriptorProto{} } +func (m *FieldDescriptorProto) String() string { return proto.CompactTextString(m) } +func (*FieldDescriptorProto) ProtoMessage() {} +func (*FieldDescriptorProto) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{4} +} +func (m *FieldDescriptorProto) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FieldDescriptorProto.Unmarshal(m, b) +} +func (m *FieldDescriptorProto) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FieldDescriptorProto.Marshal(b, m, deterministic) +} +func (dst *FieldDescriptorProto) XXX_Merge(src proto.Message) { + xxx_messageInfo_FieldDescriptorProto.Merge(dst, src) +} +func (m *FieldDescriptorProto) XXX_Size() int { + return xxx_messageInfo_FieldDescriptorProto.Size(m) +} +func (m *FieldDescriptorProto) XXX_DiscardUnknown() { + xxx_messageInfo_FieldDescriptorProto.DiscardUnknown(m) +} + +var xxx_messageInfo_FieldDescriptorProto proto.InternalMessageInfo + +func (m *FieldDescriptorProto) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +func (m *FieldDescriptorProto) GetNumber() int32 { + if m != nil && m.Number != nil { + return *m.Number + } + return 0 +} + +func (m *FieldDescriptorProto) GetLabel() FieldDescriptorProto_Label { + if m != nil && m.Label != nil { + return *m.Label + } + return FieldDescriptorProto_LABEL_OPTIONAL +} + +func (m *FieldDescriptorProto) GetType() FieldDescriptorProto_Type { + if m != nil && m.Type != nil { + return *m.Type + } + return FieldDescriptorProto_TYPE_DOUBLE +} + +func (m *FieldDescriptorProto) GetTypeName() string { + if m != nil && m.TypeName != nil { + return *m.TypeName + } + return "" +} + +func (m *FieldDescriptorProto) GetExtendee() string { + if m != nil && m.Extendee != nil { + return *m.Extendee + } + return "" +} + +func (m *FieldDescriptorProto) GetDefaultValue() string { + if m != nil && m.DefaultValue != nil { + return *m.DefaultValue + } + return "" +} + +func (m *FieldDescriptorProto) GetOneofIndex() int32 { + if m != nil && m.OneofIndex != nil { + return *m.OneofIndex + } + return 0 +} + +func (m *FieldDescriptorProto) GetJsonName() string { + if m != nil && m.JsonName != nil { + return *m.JsonName + } + return "" +} + +func (m *FieldDescriptorProto) GetOptions() *FieldOptions { + if m != nil { + return m.Options + } + return nil +} + +// Describes a oneof. +type OneofDescriptorProto struct { + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Options *OneofOptions `protobuf:"bytes,2,opt,name=options" json:"options,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *OneofDescriptorProto) Reset() { *m = OneofDescriptorProto{} } +func (m *OneofDescriptorProto) String() string { return proto.CompactTextString(m) } +func (*OneofDescriptorProto) ProtoMessage() {} +func (*OneofDescriptorProto) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{5} +} +func (m *OneofDescriptorProto) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_OneofDescriptorProto.Unmarshal(m, b) +} +func (m *OneofDescriptorProto) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_OneofDescriptorProto.Marshal(b, m, deterministic) +} +func (dst *OneofDescriptorProto) XXX_Merge(src proto.Message) { + xxx_messageInfo_OneofDescriptorProto.Merge(dst, src) +} +func (m *OneofDescriptorProto) XXX_Size() int { + return xxx_messageInfo_OneofDescriptorProto.Size(m) +} +func (m *OneofDescriptorProto) XXX_DiscardUnknown() { + xxx_messageInfo_OneofDescriptorProto.DiscardUnknown(m) +} + +var xxx_messageInfo_OneofDescriptorProto proto.InternalMessageInfo + +func (m *OneofDescriptorProto) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +func (m *OneofDescriptorProto) GetOptions() *OneofOptions { + if m != nil { + return m.Options + } + return nil +} + +// Describes an enum type. +type EnumDescriptorProto struct { + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Value []*EnumValueDescriptorProto `protobuf:"bytes,2,rep,name=value" json:"value,omitempty"` + Options *EnumOptions `protobuf:"bytes,3,opt,name=options" json:"options,omitempty"` + // Range of reserved numeric values. Reserved numeric values may not be used + // by enum values in the same enum declaration. Reserved ranges may not + // overlap. + ReservedRange []*EnumDescriptorProto_EnumReservedRange `protobuf:"bytes,4,rep,name=reserved_range,json=reservedRange" json:"reserved_range,omitempty"` + // Reserved enum value names, which may not be reused. A given name may only + // be reserved once. + ReservedName []string `protobuf:"bytes,5,rep,name=reserved_name,json=reservedName" json:"reserved_name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *EnumDescriptorProto) Reset() { *m = EnumDescriptorProto{} } +func (m *EnumDescriptorProto) String() string { return proto.CompactTextString(m) } +func (*EnumDescriptorProto) ProtoMessage() {} +func (*EnumDescriptorProto) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{6} +} +func (m *EnumDescriptorProto) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_EnumDescriptorProto.Unmarshal(m, b) +} +func (m *EnumDescriptorProto) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_EnumDescriptorProto.Marshal(b, m, deterministic) +} +func (dst *EnumDescriptorProto) XXX_Merge(src proto.Message) { + xxx_messageInfo_EnumDescriptorProto.Merge(dst, src) +} +func (m *EnumDescriptorProto) XXX_Size() int { + return xxx_messageInfo_EnumDescriptorProto.Size(m) +} +func (m *EnumDescriptorProto) XXX_DiscardUnknown() { + xxx_messageInfo_EnumDescriptorProto.DiscardUnknown(m) +} + +var xxx_messageInfo_EnumDescriptorProto proto.InternalMessageInfo + +func (m *EnumDescriptorProto) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +func (m *EnumDescriptorProto) GetValue() []*EnumValueDescriptorProto { + if m != nil { + return m.Value + } + return nil +} + +func (m *EnumDescriptorProto) GetOptions() *EnumOptions { + if m != nil { + return m.Options + } + return nil +} + +func (m *EnumDescriptorProto) GetReservedRange() []*EnumDescriptorProto_EnumReservedRange { + if m != nil { + return m.ReservedRange + } + return nil +} + +func (m *EnumDescriptorProto) GetReservedName() []string { + if m != nil { + return m.ReservedName + } + return nil +} + +// Range of reserved numeric values. Reserved values may not be used by +// entries in the same enum. Reserved ranges may not overlap. +// +// Note that this is distinct from DescriptorProto.ReservedRange in that it +// is inclusive such that it can appropriately represent the entire int32 +// domain. +type EnumDescriptorProto_EnumReservedRange struct { + Start *int32 `protobuf:"varint,1,opt,name=start" json:"start,omitempty"` + End *int32 `protobuf:"varint,2,opt,name=end" json:"end,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *EnumDescriptorProto_EnumReservedRange) Reset() { *m = EnumDescriptorProto_EnumReservedRange{} } +func (m *EnumDescriptorProto_EnumReservedRange) String() string { return proto.CompactTextString(m) } +func (*EnumDescriptorProto_EnumReservedRange) ProtoMessage() {} +func (*EnumDescriptorProto_EnumReservedRange) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{6, 0} +} +func (m *EnumDescriptorProto_EnumReservedRange) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_EnumDescriptorProto_EnumReservedRange.Unmarshal(m, b) +} +func (m *EnumDescriptorProto_EnumReservedRange) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_EnumDescriptorProto_EnumReservedRange.Marshal(b, m, deterministic) +} +func (dst *EnumDescriptorProto_EnumReservedRange) XXX_Merge(src proto.Message) { + xxx_messageInfo_EnumDescriptorProto_EnumReservedRange.Merge(dst, src) +} +func (m *EnumDescriptorProto_EnumReservedRange) XXX_Size() int { + return xxx_messageInfo_EnumDescriptorProto_EnumReservedRange.Size(m) +} +func (m *EnumDescriptorProto_EnumReservedRange) XXX_DiscardUnknown() { + xxx_messageInfo_EnumDescriptorProto_EnumReservedRange.DiscardUnknown(m) +} + +var xxx_messageInfo_EnumDescriptorProto_EnumReservedRange proto.InternalMessageInfo + +func (m *EnumDescriptorProto_EnumReservedRange) GetStart() int32 { + if m != nil && m.Start != nil { + return *m.Start + } + return 0 +} + +func (m *EnumDescriptorProto_EnumReservedRange) GetEnd() int32 { + if m != nil && m.End != nil { + return *m.End + } + return 0 +} + +// Describes a value within an enum. +type EnumValueDescriptorProto struct { + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Number *int32 `protobuf:"varint,2,opt,name=number" json:"number,omitempty"` + Options *EnumValueOptions `protobuf:"bytes,3,opt,name=options" json:"options,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *EnumValueDescriptorProto) Reset() { *m = EnumValueDescriptorProto{} } +func (m *EnumValueDescriptorProto) String() string { return proto.CompactTextString(m) } +func (*EnumValueDescriptorProto) ProtoMessage() {} +func (*EnumValueDescriptorProto) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{7} +} +func (m *EnumValueDescriptorProto) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_EnumValueDescriptorProto.Unmarshal(m, b) +} +func (m *EnumValueDescriptorProto) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_EnumValueDescriptorProto.Marshal(b, m, deterministic) +} +func (dst *EnumValueDescriptorProto) XXX_Merge(src proto.Message) { + xxx_messageInfo_EnumValueDescriptorProto.Merge(dst, src) +} +func (m *EnumValueDescriptorProto) XXX_Size() int { + return xxx_messageInfo_EnumValueDescriptorProto.Size(m) +} +func (m *EnumValueDescriptorProto) XXX_DiscardUnknown() { + xxx_messageInfo_EnumValueDescriptorProto.DiscardUnknown(m) +} + +var xxx_messageInfo_EnumValueDescriptorProto proto.InternalMessageInfo + +func (m *EnumValueDescriptorProto) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +func (m *EnumValueDescriptorProto) GetNumber() int32 { + if m != nil && m.Number != nil { + return *m.Number + } + return 0 +} + +func (m *EnumValueDescriptorProto) GetOptions() *EnumValueOptions { + if m != nil { + return m.Options + } + return nil +} + +// Describes a service. +type ServiceDescriptorProto struct { + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Method []*MethodDescriptorProto `protobuf:"bytes,2,rep,name=method" json:"method,omitempty"` + Options *ServiceOptions `protobuf:"bytes,3,opt,name=options" json:"options,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ServiceDescriptorProto) Reset() { *m = ServiceDescriptorProto{} } +func (m *ServiceDescriptorProto) String() string { return proto.CompactTextString(m) } +func (*ServiceDescriptorProto) ProtoMessage() {} +func (*ServiceDescriptorProto) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{8} +} +func (m *ServiceDescriptorProto) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ServiceDescriptorProto.Unmarshal(m, b) +} +func (m *ServiceDescriptorProto) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ServiceDescriptorProto.Marshal(b, m, deterministic) +} +func (dst *ServiceDescriptorProto) XXX_Merge(src proto.Message) { + xxx_messageInfo_ServiceDescriptorProto.Merge(dst, src) +} +func (m *ServiceDescriptorProto) XXX_Size() int { + return xxx_messageInfo_ServiceDescriptorProto.Size(m) +} +func (m *ServiceDescriptorProto) XXX_DiscardUnknown() { + xxx_messageInfo_ServiceDescriptorProto.DiscardUnknown(m) +} + +var xxx_messageInfo_ServiceDescriptorProto proto.InternalMessageInfo + +func (m *ServiceDescriptorProto) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +func (m *ServiceDescriptorProto) GetMethod() []*MethodDescriptorProto { + if m != nil { + return m.Method + } + return nil +} + +func (m *ServiceDescriptorProto) GetOptions() *ServiceOptions { + if m != nil { + return m.Options + } + return nil +} + +// Describes a method of a service. +type MethodDescriptorProto struct { + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // Input and output type names. These are resolved in the same way as + // FieldDescriptorProto.type_name, but must refer to a message type. + InputType *string `protobuf:"bytes,2,opt,name=input_type,json=inputType" json:"input_type,omitempty"` + OutputType *string `protobuf:"bytes,3,opt,name=output_type,json=outputType" json:"output_type,omitempty"` + Options *MethodOptions `protobuf:"bytes,4,opt,name=options" json:"options,omitempty"` + // Identifies if client streams multiple client messages + ClientStreaming *bool `protobuf:"varint,5,opt,name=client_streaming,json=clientStreaming,def=0" json:"client_streaming,omitempty"` + // Identifies if server streams multiple server messages + ServerStreaming *bool `protobuf:"varint,6,opt,name=server_streaming,json=serverStreaming,def=0" json:"server_streaming,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *MethodDescriptorProto) Reset() { *m = MethodDescriptorProto{} } +func (m *MethodDescriptorProto) String() string { return proto.CompactTextString(m) } +func (*MethodDescriptorProto) ProtoMessage() {} +func (*MethodDescriptorProto) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{9} +} +func (m *MethodDescriptorProto) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_MethodDescriptorProto.Unmarshal(m, b) +} +func (m *MethodDescriptorProto) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_MethodDescriptorProto.Marshal(b, m, deterministic) +} +func (dst *MethodDescriptorProto) XXX_Merge(src proto.Message) { + xxx_messageInfo_MethodDescriptorProto.Merge(dst, src) +} +func (m *MethodDescriptorProto) XXX_Size() int { + return xxx_messageInfo_MethodDescriptorProto.Size(m) +} +func (m *MethodDescriptorProto) XXX_DiscardUnknown() { + xxx_messageInfo_MethodDescriptorProto.DiscardUnknown(m) +} + +var xxx_messageInfo_MethodDescriptorProto proto.InternalMessageInfo + +const Default_MethodDescriptorProto_ClientStreaming bool = false +const Default_MethodDescriptorProto_ServerStreaming bool = false + +func (m *MethodDescriptorProto) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +func (m *MethodDescriptorProto) GetInputType() string { + if m != nil && m.InputType != nil { + return *m.InputType + } + return "" +} + +func (m *MethodDescriptorProto) GetOutputType() string { + if m != nil && m.OutputType != nil { + return *m.OutputType + } + return "" +} + +func (m *MethodDescriptorProto) GetOptions() *MethodOptions { + if m != nil { + return m.Options + } + return nil +} + +func (m *MethodDescriptorProto) GetClientStreaming() bool { + if m != nil && m.ClientStreaming != nil { + return *m.ClientStreaming + } + return Default_MethodDescriptorProto_ClientStreaming +} + +func (m *MethodDescriptorProto) GetServerStreaming() bool { + if m != nil && m.ServerStreaming != nil { + return *m.ServerStreaming + } + return Default_MethodDescriptorProto_ServerStreaming +} + +type FileOptions struct { + // Sets the Java package where classes generated from this .proto will be + // placed. By default, the proto package is used, but this is often + // inappropriate because proto packages do not normally start with backwards + // domain names. + JavaPackage *string `protobuf:"bytes,1,opt,name=java_package,json=javaPackage" json:"java_package,omitempty"` + // If set, all the classes from the .proto file are wrapped in a single + // outer class with the given name. This applies to both Proto1 + // (equivalent to the old "--one_java_file" option) and Proto2 (where + // a .proto always translates to a single class, but you may want to + // explicitly choose the class name). + JavaOuterClassname *string `protobuf:"bytes,8,opt,name=java_outer_classname,json=javaOuterClassname" json:"java_outer_classname,omitempty"` + // If set true, then the Java code generator will generate a separate .java + // file for each top-level message, enum, and service defined in the .proto + // file. Thus, these types will *not* be nested inside the outer class + // named by java_outer_classname. However, the outer class will still be + // generated to contain the file's getDescriptor() method as well as any + // top-level extensions defined in the file. + JavaMultipleFiles *bool `protobuf:"varint,10,opt,name=java_multiple_files,json=javaMultipleFiles,def=0" json:"java_multiple_files,omitempty"` + // This option does nothing. + JavaGenerateEqualsAndHash *bool `protobuf:"varint,20,opt,name=java_generate_equals_and_hash,json=javaGenerateEqualsAndHash" json:"java_generate_equals_and_hash,omitempty"` // Deprecated: Do not use. + // If set true, then the Java2 code generator will generate code that + // throws an exception whenever an attempt is made to assign a non-UTF-8 + // byte sequence to a string field. + // Message reflection will do the same. + // However, an extension field still accepts non-UTF-8 byte sequences. + // This option has no effect on when used with the lite runtime. + JavaStringCheckUtf8 *bool `protobuf:"varint,27,opt,name=java_string_check_utf8,json=javaStringCheckUtf8,def=0" json:"java_string_check_utf8,omitempty"` + OptimizeFor *FileOptions_OptimizeMode `protobuf:"varint,9,opt,name=optimize_for,json=optimizeFor,enum=google.protobuf.FileOptions_OptimizeMode,def=1" json:"optimize_for,omitempty"` + // Sets the Go package where structs generated from this .proto will be + // placed. If omitted, the Go package will be derived from the following: + // - The basename of the package import path, if provided. + // - Otherwise, the package statement in the .proto file, if present. + // - Otherwise, the basename of the .proto file, without extension. + GoPackage *string `protobuf:"bytes,11,opt,name=go_package,json=goPackage" json:"go_package,omitempty"` + // Should generic services be generated in each language? "Generic" services + // are not specific to any particular RPC system. They are generated by the + // main code generators in each language (without additional plugins). + // Generic services were the only kind of service generation supported by + // early versions of google.protobuf. + // + // Generic services are now considered deprecated in favor of using plugins + // that generate code specific to your particular RPC system. Therefore, + // these default to false. Old code which depends on generic services should + // explicitly set them to true. + CcGenericServices *bool `protobuf:"varint,16,opt,name=cc_generic_services,json=ccGenericServices,def=0" json:"cc_generic_services,omitempty"` + JavaGenericServices *bool `protobuf:"varint,17,opt,name=java_generic_services,json=javaGenericServices,def=0" json:"java_generic_services,omitempty"` + PyGenericServices *bool `protobuf:"varint,18,opt,name=py_generic_services,json=pyGenericServices,def=0" json:"py_generic_services,omitempty"` + PhpGenericServices *bool `protobuf:"varint,42,opt,name=php_generic_services,json=phpGenericServices,def=0" json:"php_generic_services,omitempty"` + // Is this file deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for everything in the file, or it will be completely ignored; in the very + // least, this is a formalization for deprecating files. + Deprecated *bool `protobuf:"varint,23,opt,name=deprecated,def=0" json:"deprecated,omitempty"` + // Enables the use of arenas for the proto messages in this file. This applies + // only to generated classes for C++. + CcEnableArenas *bool `protobuf:"varint,31,opt,name=cc_enable_arenas,json=ccEnableArenas,def=0" json:"cc_enable_arenas,omitempty"` + // Sets the objective c class prefix which is prepended to all objective c + // generated classes from this .proto. There is no default. + ObjcClassPrefix *string `protobuf:"bytes,36,opt,name=objc_class_prefix,json=objcClassPrefix" json:"objc_class_prefix,omitempty"` + // Namespace for generated classes; defaults to the package. + CsharpNamespace *string `protobuf:"bytes,37,opt,name=csharp_namespace,json=csharpNamespace" json:"csharp_namespace,omitempty"` + // By default Swift generators will take the proto package and CamelCase it + // replacing '.' with underscore and use that to prefix the types/symbols + // defined. When this options is provided, they will use this value instead + // to prefix the types/symbols defined. + SwiftPrefix *string `protobuf:"bytes,39,opt,name=swift_prefix,json=swiftPrefix" json:"swift_prefix,omitempty"` + // Sets the php class prefix which is prepended to all php generated classes + // from this .proto. Default is empty. + PhpClassPrefix *string `protobuf:"bytes,40,opt,name=php_class_prefix,json=phpClassPrefix" json:"php_class_prefix,omitempty"` + // Use this option to change the namespace of php generated classes. Default + // is empty. When this option is empty, the package name will be used for + // determining the namespace. + PhpNamespace *string `protobuf:"bytes,41,opt,name=php_namespace,json=phpNamespace" json:"php_namespace,omitempty"` + // The parser stores options it doesn't recognize here. + // See the documentation for the "Options" section above. + UninterpretedOption []*UninterpretedOption `protobuf:"bytes,999,rep,name=uninterpreted_option,json=uninterpretedOption" json:"uninterpreted_option,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + proto.XXX_InternalExtensions `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FileOptions) Reset() { *m = FileOptions{} } +func (m *FileOptions) String() string { return proto.CompactTextString(m) } +func (*FileOptions) ProtoMessage() {} +func (*FileOptions) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{10} +} + +var extRange_FileOptions = []proto.ExtensionRange{ + {Start: 1000, End: 536870911}, +} + +func (*FileOptions) ExtensionRangeArray() []proto.ExtensionRange { + return extRange_FileOptions +} +func (m *FileOptions) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FileOptions.Unmarshal(m, b) +} +func (m *FileOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FileOptions.Marshal(b, m, deterministic) +} +func (dst *FileOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_FileOptions.Merge(dst, src) +} +func (m *FileOptions) XXX_Size() int { + return xxx_messageInfo_FileOptions.Size(m) +} +func (m *FileOptions) XXX_DiscardUnknown() { + xxx_messageInfo_FileOptions.DiscardUnknown(m) +} + +var xxx_messageInfo_FileOptions proto.InternalMessageInfo + +const Default_FileOptions_JavaMultipleFiles bool = false +const Default_FileOptions_JavaStringCheckUtf8 bool = false +const Default_FileOptions_OptimizeFor FileOptions_OptimizeMode = FileOptions_SPEED +const Default_FileOptions_CcGenericServices bool = false +const Default_FileOptions_JavaGenericServices bool = false +const Default_FileOptions_PyGenericServices bool = false +const Default_FileOptions_PhpGenericServices bool = false +const Default_FileOptions_Deprecated bool = false +const Default_FileOptions_CcEnableArenas bool = false + +func (m *FileOptions) GetJavaPackage() string { + if m != nil && m.JavaPackage != nil { + return *m.JavaPackage + } + return "" +} + +func (m *FileOptions) GetJavaOuterClassname() string { + if m != nil && m.JavaOuterClassname != nil { + return *m.JavaOuterClassname + } + return "" +} + +func (m *FileOptions) GetJavaMultipleFiles() bool { + if m != nil && m.JavaMultipleFiles != nil { + return *m.JavaMultipleFiles + } + return Default_FileOptions_JavaMultipleFiles +} + +// Deprecated: Do not use. +func (m *FileOptions) GetJavaGenerateEqualsAndHash() bool { + if m != nil && m.JavaGenerateEqualsAndHash != nil { + return *m.JavaGenerateEqualsAndHash + } + return false +} + +func (m *FileOptions) GetJavaStringCheckUtf8() bool { + if m != nil && m.JavaStringCheckUtf8 != nil { + return *m.JavaStringCheckUtf8 + } + return Default_FileOptions_JavaStringCheckUtf8 +} + +func (m *FileOptions) GetOptimizeFor() FileOptions_OptimizeMode { + if m != nil && m.OptimizeFor != nil { + return *m.OptimizeFor + } + return Default_FileOptions_OptimizeFor +} + +func (m *FileOptions) GetGoPackage() string { + if m != nil && m.GoPackage != nil { + return *m.GoPackage + } + return "" +} + +func (m *FileOptions) GetCcGenericServices() bool { + if m != nil && m.CcGenericServices != nil { + return *m.CcGenericServices + } + return Default_FileOptions_CcGenericServices +} + +func (m *FileOptions) GetJavaGenericServices() bool { + if m != nil && m.JavaGenericServices != nil { + return *m.JavaGenericServices + } + return Default_FileOptions_JavaGenericServices +} + +func (m *FileOptions) GetPyGenericServices() bool { + if m != nil && m.PyGenericServices != nil { + return *m.PyGenericServices + } + return Default_FileOptions_PyGenericServices +} + +func (m *FileOptions) GetPhpGenericServices() bool { + if m != nil && m.PhpGenericServices != nil { + return *m.PhpGenericServices + } + return Default_FileOptions_PhpGenericServices +} + +func (m *FileOptions) GetDeprecated() bool { + if m != nil && m.Deprecated != nil { + return *m.Deprecated + } + return Default_FileOptions_Deprecated +} + +func (m *FileOptions) GetCcEnableArenas() bool { + if m != nil && m.CcEnableArenas != nil { + return *m.CcEnableArenas + } + return Default_FileOptions_CcEnableArenas +} + +func (m *FileOptions) GetObjcClassPrefix() string { + if m != nil && m.ObjcClassPrefix != nil { + return *m.ObjcClassPrefix + } + return "" +} + +func (m *FileOptions) GetCsharpNamespace() string { + if m != nil && m.CsharpNamespace != nil { + return *m.CsharpNamespace + } + return "" +} + +func (m *FileOptions) GetSwiftPrefix() string { + if m != nil && m.SwiftPrefix != nil { + return *m.SwiftPrefix + } + return "" +} + +func (m *FileOptions) GetPhpClassPrefix() string { + if m != nil && m.PhpClassPrefix != nil { + return *m.PhpClassPrefix + } + return "" +} + +func (m *FileOptions) GetPhpNamespace() string { + if m != nil && m.PhpNamespace != nil { + return *m.PhpNamespace + } + return "" +} + +func (m *FileOptions) GetUninterpretedOption() []*UninterpretedOption { + if m != nil { + return m.UninterpretedOption + } + return nil +} + +type MessageOptions struct { + // Set true to use the old proto1 MessageSet wire format for extensions. + // This is provided for backwards-compatibility with the MessageSet wire + // format. You should not use this for any other reason: It's less + // efficient, has fewer features, and is more complicated. + // + // The message must be defined exactly as follows: + // message Foo { + // option message_set_wire_format = true; + // extensions 4 to max; + // } + // Note that the message cannot have any defined fields; MessageSets only + // have extensions. + // + // All extensions of your type must be singular messages; e.g. they cannot + // be int32s, enums, or repeated messages. + // + // Because this is an option, the above two restrictions are not enforced by + // the protocol compiler. + MessageSetWireFormat *bool `protobuf:"varint,1,opt,name=message_set_wire_format,json=messageSetWireFormat,def=0" json:"message_set_wire_format,omitempty"` + // Disables the generation of the standard "descriptor()" accessor, which can + // conflict with a field of the same name. This is meant to make migration + // from proto1 easier; new code should avoid fields named "descriptor". + NoStandardDescriptorAccessor *bool `protobuf:"varint,2,opt,name=no_standard_descriptor_accessor,json=noStandardDescriptorAccessor,def=0" json:"no_standard_descriptor_accessor,omitempty"` + // Is this message deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the message, or it will be completely ignored; in the very least, + // this is a formalization for deprecating messages. + Deprecated *bool `protobuf:"varint,3,opt,name=deprecated,def=0" json:"deprecated,omitempty"` + // Whether the message is an automatically generated map entry type for the + // maps field. + // + // For maps fields: + // map map_field = 1; + // The parsed descriptor looks like: + // message MapFieldEntry { + // option map_entry = true; + // optional KeyType key = 1; + // optional ValueType value = 2; + // } + // repeated MapFieldEntry map_field = 1; + // + // Implementations may choose not to generate the map_entry=true message, but + // use a native map in the target language to hold the keys and values. + // The reflection APIs in such implementions still need to work as + // if the field is a repeated message field. + // + // NOTE: Do not set the option in .proto files. Always use the maps syntax + // instead. The option should only be implicitly set by the proto compiler + // parser. + MapEntry *bool `protobuf:"varint,7,opt,name=map_entry,json=mapEntry" json:"map_entry,omitempty"` + // The parser stores options it doesn't recognize here. See above. + UninterpretedOption []*UninterpretedOption `protobuf:"bytes,999,rep,name=uninterpreted_option,json=uninterpretedOption" json:"uninterpreted_option,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + proto.XXX_InternalExtensions `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *MessageOptions) Reset() { *m = MessageOptions{} } +func (m *MessageOptions) String() string { return proto.CompactTextString(m) } +func (*MessageOptions) ProtoMessage() {} +func (*MessageOptions) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{11} +} + +var extRange_MessageOptions = []proto.ExtensionRange{ + {Start: 1000, End: 536870911}, +} + +func (*MessageOptions) ExtensionRangeArray() []proto.ExtensionRange { + return extRange_MessageOptions +} +func (m *MessageOptions) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_MessageOptions.Unmarshal(m, b) +} +func (m *MessageOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_MessageOptions.Marshal(b, m, deterministic) +} +func (dst *MessageOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_MessageOptions.Merge(dst, src) +} +func (m *MessageOptions) XXX_Size() int { + return xxx_messageInfo_MessageOptions.Size(m) +} +func (m *MessageOptions) XXX_DiscardUnknown() { + xxx_messageInfo_MessageOptions.DiscardUnknown(m) +} + +var xxx_messageInfo_MessageOptions proto.InternalMessageInfo + +const Default_MessageOptions_MessageSetWireFormat bool = false +const Default_MessageOptions_NoStandardDescriptorAccessor bool = false +const Default_MessageOptions_Deprecated bool = false + +func (m *MessageOptions) GetMessageSetWireFormat() bool { + if m != nil && m.MessageSetWireFormat != nil { + return *m.MessageSetWireFormat + } + return Default_MessageOptions_MessageSetWireFormat +} + +func (m *MessageOptions) GetNoStandardDescriptorAccessor() bool { + if m != nil && m.NoStandardDescriptorAccessor != nil { + return *m.NoStandardDescriptorAccessor + } + return Default_MessageOptions_NoStandardDescriptorAccessor +} + +func (m *MessageOptions) GetDeprecated() bool { + if m != nil && m.Deprecated != nil { + return *m.Deprecated + } + return Default_MessageOptions_Deprecated +} + +func (m *MessageOptions) GetMapEntry() bool { + if m != nil && m.MapEntry != nil { + return *m.MapEntry + } + return false +} + +func (m *MessageOptions) GetUninterpretedOption() []*UninterpretedOption { + if m != nil { + return m.UninterpretedOption + } + return nil +} + +type FieldOptions struct { + // The ctype option instructs the C++ code generator to use a different + // representation of the field than it normally would. See the specific + // options below. This option is not yet implemented in the open source + // release -- sorry, we'll try to include it in a future version! + Ctype *FieldOptions_CType `protobuf:"varint,1,opt,name=ctype,enum=google.protobuf.FieldOptions_CType,def=0" json:"ctype,omitempty"` + // The packed option can be enabled for repeated primitive fields to enable + // a more efficient representation on the wire. Rather than repeatedly + // writing the tag and type for each element, the entire array is encoded as + // a single length-delimited blob. In proto3, only explicit setting it to + // false will avoid using packed encoding. + Packed *bool `protobuf:"varint,2,opt,name=packed" json:"packed,omitempty"` + // The jstype option determines the JavaScript type used for values of the + // field. The option is permitted only for 64 bit integral and fixed types + // (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING + // is represented as JavaScript string, which avoids loss of precision that + // can happen when a large value is converted to a floating point JavaScript. + // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to + // use the JavaScript "number" type. The behavior of the default option + // JS_NORMAL is implementation dependent. + // + // This option is an enum to permit additional types to be added, e.g. + // goog.math.Integer. + Jstype *FieldOptions_JSType `protobuf:"varint,6,opt,name=jstype,enum=google.protobuf.FieldOptions_JSType,def=0" json:"jstype,omitempty"` + // Should this field be parsed lazily? Lazy applies only to message-type + // fields. It means that when the outer message is initially parsed, the + // inner message's contents will not be parsed but instead stored in encoded + // form. The inner message will actually be parsed when it is first accessed. + // + // This is only a hint. Implementations are free to choose whether to use + // eager or lazy parsing regardless of the value of this option. However, + // setting this option true suggests that the protocol author believes that + // using lazy parsing on this field is worth the additional bookkeeping + // overhead typically needed to implement it. + // + // This option does not affect the public interface of any generated code; + // all method signatures remain the same. Furthermore, thread-safety of the + // interface is not affected by this option; const methods remain safe to + // call from multiple threads concurrently, while non-const methods continue + // to require exclusive access. + // + // + // Note that implementations may choose not to check required fields within + // a lazy sub-message. That is, calling IsInitialized() on the outer message + // may return true even if the inner message has missing required fields. + // This is necessary because otherwise the inner message would have to be + // parsed in order to perform the check, defeating the purpose of lazy + // parsing. An implementation which chooses not to check required fields + // must be consistent about it. That is, for any particular sub-message, the + // implementation must either *always* check its required fields, or *never* + // check its required fields, regardless of whether or not the message has + // been parsed. + Lazy *bool `protobuf:"varint,5,opt,name=lazy,def=0" json:"lazy,omitempty"` + // Is this field deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for accessors, or it will be completely ignored; in the very least, this + // is a formalization for deprecating fields. + Deprecated *bool `protobuf:"varint,3,opt,name=deprecated,def=0" json:"deprecated,omitempty"` + // For Google-internal migration only. Do not use. + Weak *bool `protobuf:"varint,10,opt,name=weak,def=0" json:"weak,omitempty"` + // The parser stores options it doesn't recognize here. See above. + UninterpretedOption []*UninterpretedOption `protobuf:"bytes,999,rep,name=uninterpreted_option,json=uninterpretedOption" json:"uninterpreted_option,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + proto.XXX_InternalExtensions `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FieldOptions) Reset() { *m = FieldOptions{} } +func (m *FieldOptions) String() string { return proto.CompactTextString(m) } +func (*FieldOptions) ProtoMessage() {} +func (*FieldOptions) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{12} +} + +var extRange_FieldOptions = []proto.ExtensionRange{ + {Start: 1000, End: 536870911}, +} + +func (*FieldOptions) ExtensionRangeArray() []proto.ExtensionRange { + return extRange_FieldOptions +} +func (m *FieldOptions) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FieldOptions.Unmarshal(m, b) +} +func (m *FieldOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FieldOptions.Marshal(b, m, deterministic) +} +func (dst *FieldOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_FieldOptions.Merge(dst, src) +} +func (m *FieldOptions) XXX_Size() int { + return xxx_messageInfo_FieldOptions.Size(m) +} +func (m *FieldOptions) XXX_DiscardUnknown() { + xxx_messageInfo_FieldOptions.DiscardUnknown(m) +} + +var xxx_messageInfo_FieldOptions proto.InternalMessageInfo + +const Default_FieldOptions_Ctype FieldOptions_CType = FieldOptions_STRING +const Default_FieldOptions_Jstype FieldOptions_JSType = FieldOptions_JS_NORMAL +const Default_FieldOptions_Lazy bool = false +const Default_FieldOptions_Deprecated bool = false +const Default_FieldOptions_Weak bool = false + +func (m *FieldOptions) GetCtype() FieldOptions_CType { + if m != nil && m.Ctype != nil { + return *m.Ctype + } + return Default_FieldOptions_Ctype +} + +func (m *FieldOptions) GetPacked() bool { + if m != nil && m.Packed != nil { + return *m.Packed + } + return false +} + +func (m *FieldOptions) GetJstype() FieldOptions_JSType { + if m != nil && m.Jstype != nil { + return *m.Jstype + } + return Default_FieldOptions_Jstype +} + +func (m *FieldOptions) GetLazy() bool { + if m != nil && m.Lazy != nil { + return *m.Lazy + } + return Default_FieldOptions_Lazy +} + +func (m *FieldOptions) GetDeprecated() bool { + if m != nil && m.Deprecated != nil { + return *m.Deprecated + } + return Default_FieldOptions_Deprecated +} + +func (m *FieldOptions) GetWeak() bool { + if m != nil && m.Weak != nil { + return *m.Weak + } + return Default_FieldOptions_Weak +} + +func (m *FieldOptions) GetUninterpretedOption() []*UninterpretedOption { + if m != nil { + return m.UninterpretedOption + } + return nil +} + +type OneofOptions struct { + // The parser stores options it doesn't recognize here. See above. + UninterpretedOption []*UninterpretedOption `protobuf:"bytes,999,rep,name=uninterpreted_option,json=uninterpretedOption" json:"uninterpreted_option,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + proto.XXX_InternalExtensions `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *OneofOptions) Reset() { *m = OneofOptions{} } +func (m *OneofOptions) String() string { return proto.CompactTextString(m) } +func (*OneofOptions) ProtoMessage() {} +func (*OneofOptions) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{13} +} + +var extRange_OneofOptions = []proto.ExtensionRange{ + {Start: 1000, End: 536870911}, +} + +func (*OneofOptions) ExtensionRangeArray() []proto.ExtensionRange { + return extRange_OneofOptions +} +func (m *OneofOptions) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_OneofOptions.Unmarshal(m, b) +} +func (m *OneofOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_OneofOptions.Marshal(b, m, deterministic) +} +func (dst *OneofOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_OneofOptions.Merge(dst, src) +} +func (m *OneofOptions) XXX_Size() int { + return xxx_messageInfo_OneofOptions.Size(m) +} +func (m *OneofOptions) XXX_DiscardUnknown() { + xxx_messageInfo_OneofOptions.DiscardUnknown(m) +} + +var xxx_messageInfo_OneofOptions proto.InternalMessageInfo + +func (m *OneofOptions) GetUninterpretedOption() []*UninterpretedOption { + if m != nil { + return m.UninterpretedOption + } + return nil +} + +type EnumOptions struct { + // Set this option to true to allow mapping different tag names to the same + // value. + AllowAlias *bool `protobuf:"varint,2,opt,name=allow_alias,json=allowAlias" json:"allow_alias,omitempty"` + // Is this enum deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum, or it will be completely ignored; in the very least, this + // is a formalization for deprecating enums. + Deprecated *bool `protobuf:"varint,3,opt,name=deprecated,def=0" json:"deprecated,omitempty"` + // The parser stores options it doesn't recognize here. See above. + UninterpretedOption []*UninterpretedOption `protobuf:"bytes,999,rep,name=uninterpreted_option,json=uninterpretedOption" json:"uninterpreted_option,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + proto.XXX_InternalExtensions `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *EnumOptions) Reset() { *m = EnumOptions{} } +func (m *EnumOptions) String() string { return proto.CompactTextString(m) } +func (*EnumOptions) ProtoMessage() {} +func (*EnumOptions) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{14} +} + +var extRange_EnumOptions = []proto.ExtensionRange{ + {Start: 1000, End: 536870911}, +} + +func (*EnumOptions) ExtensionRangeArray() []proto.ExtensionRange { + return extRange_EnumOptions +} +func (m *EnumOptions) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_EnumOptions.Unmarshal(m, b) +} +func (m *EnumOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_EnumOptions.Marshal(b, m, deterministic) +} +func (dst *EnumOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_EnumOptions.Merge(dst, src) +} +func (m *EnumOptions) XXX_Size() int { + return xxx_messageInfo_EnumOptions.Size(m) +} +func (m *EnumOptions) XXX_DiscardUnknown() { + xxx_messageInfo_EnumOptions.DiscardUnknown(m) +} + +var xxx_messageInfo_EnumOptions proto.InternalMessageInfo + +const Default_EnumOptions_Deprecated bool = false + +func (m *EnumOptions) GetAllowAlias() bool { + if m != nil && m.AllowAlias != nil { + return *m.AllowAlias + } + return false +} + +func (m *EnumOptions) GetDeprecated() bool { + if m != nil && m.Deprecated != nil { + return *m.Deprecated + } + return Default_EnumOptions_Deprecated +} + +func (m *EnumOptions) GetUninterpretedOption() []*UninterpretedOption { + if m != nil { + return m.UninterpretedOption + } + return nil +} + +type EnumValueOptions struct { + // Is this enum value deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum value, or it will be completely ignored; in the very least, + // this is a formalization for deprecating enum values. + Deprecated *bool `protobuf:"varint,1,opt,name=deprecated,def=0" json:"deprecated,omitempty"` + // The parser stores options it doesn't recognize here. See above. + UninterpretedOption []*UninterpretedOption `protobuf:"bytes,999,rep,name=uninterpreted_option,json=uninterpretedOption" json:"uninterpreted_option,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + proto.XXX_InternalExtensions `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *EnumValueOptions) Reset() { *m = EnumValueOptions{} } +func (m *EnumValueOptions) String() string { return proto.CompactTextString(m) } +func (*EnumValueOptions) ProtoMessage() {} +func (*EnumValueOptions) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{15} +} + +var extRange_EnumValueOptions = []proto.ExtensionRange{ + {Start: 1000, End: 536870911}, +} + +func (*EnumValueOptions) ExtensionRangeArray() []proto.ExtensionRange { + return extRange_EnumValueOptions +} +func (m *EnumValueOptions) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_EnumValueOptions.Unmarshal(m, b) +} +func (m *EnumValueOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_EnumValueOptions.Marshal(b, m, deterministic) +} +func (dst *EnumValueOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_EnumValueOptions.Merge(dst, src) +} +func (m *EnumValueOptions) XXX_Size() int { + return xxx_messageInfo_EnumValueOptions.Size(m) +} +func (m *EnumValueOptions) XXX_DiscardUnknown() { + xxx_messageInfo_EnumValueOptions.DiscardUnknown(m) +} + +var xxx_messageInfo_EnumValueOptions proto.InternalMessageInfo + +const Default_EnumValueOptions_Deprecated bool = false + +func (m *EnumValueOptions) GetDeprecated() bool { + if m != nil && m.Deprecated != nil { + return *m.Deprecated + } + return Default_EnumValueOptions_Deprecated +} + +func (m *EnumValueOptions) GetUninterpretedOption() []*UninterpretedOption { + if m != nil { + return m.UninterpretedOption + } + return nil +} + +type ServiceOptions struct { + // Is this service deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the service, or it will be completely ignored; in the very least, + // this is a formalization for deprecating services. + Deprecated *bool `protobuf:"varint,33,opt,name=deprecated,def=0" json:"deprecated,omitempty"` + // The parser stores options it doesn't recognize here. See above. + UninterpretedOption []*UninterpretedOption `protobuf:"bytes,999,rep,name=uninterpreted_option,json=uninterpretedOption" json:"uninterpreted_option,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + proto.XXX_InternalExtensions `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ServiceOptions) Reset() { *m = ServiceOptions{} } +func (m *ServiceOptions) String() string { return proto.CompactTextString(m) } +func (*ServiceOptions) ProtoMessage() {} +func (*ServiceOptions) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{16} +} + +var extRange_ServiceOptions = []proto.ExtensionRange{ + {Start: 1000, End: 536870911}, +} + +func (*ServiceOptions) ExtensionRangeArray() []proto.ExtensionRange { + return extRange_ServiceOptions +} +func (m *ServiceOptions) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ServiceOptions.Unmarshal(m, b) +} +func (m *ServiceOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ServiceOptions.Marshal(b, m, deterministic) +} +func (dst *ServiceOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_ServiceOptions.Merge(dst, src) +} +func (m *ServiceOptions) XXX_Size() int { + return xxx_messageInfo_ServiceOptions.Size(m) +} +func (m *ServiceOptions) XXX_DiscardUnknown() { + xxx_messageInfo_ServiceOptions.DiscardUnknown(m) +} + +var xxx_messageInfo_ServiceOptions proto.InternalMessageInfo + +const Default_ServiceOptions_Deprecated bool = false + +func (m *ServiceOptions) GetDeprecated() bool { + if m != nil && m.Deprecated != nil { + return *m.Deprecated + } + return Default_ServiceOptions_Deprecated +} + +func (m *ServiceOptions) GetUninterpretedOption() []*UninterpretedOption { + if m != nil { + return m.UninterpretedOption + } + return nil +} + +type MethodOptions struct { + // Is this method deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the method, or it will be completely ignored; in the very least, + // this is a formalization for deprecating methods. + Deprecated *bool `protobuf:"varint,33,opt,name=deprecated,def=0" json:"deprecated,omitempty"` + IdempotencyLevel *MethodOptions_IdempotencyLevel `protobuf:"varint,34,opt,name=idempotency_level,json=idempotencyLevel,enum=google.protobuf.MethodOptions_IdempotencyLevel,def=0" json:"idempotency_level,omitempty"` + // The parser stores options it doesn't recognize here. See above. + UninterpretedOption []*UninterpretedOption `protobuf:"bytes,999,rep,name=uninterpreted_option,json=uninterpretedOption" json:"uninterpreted_option,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + proto.XXX_InternalExtensions `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *MethodOptions) Reset() { *m = MethodOptions{} } +func (m *MethodOptions) String() string { return proto.CompactTextString(m) } +func (*MethodOptions) ProtoMessage() {} +func (*MethodOptions) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{17} +} + +var extRange_MethodOptions = []proto.ExtensionRange{ + {Start: 1000, End: 536870911}, +} + +func (*MethodOptions) ExtensionRangeArray() []proto.ExtensionRange { + return extRange_MethodOptions +} +func (m *MethodOptions) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_MethodOptions.Unmarshal(m, b) +} +func (m *MethodOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_MethodOptions.Marshal(b, m, deterministic) +} +func (dst *MethodOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_MethodOptions.Merge(dst, src) +} +func (m *MethodOptions) XXX_Size() int { + return xxx_messageInfo_MethodOptions.Size(m) +} +func (m *MethodOptions) XXX_DiscardUnknown() { + xxx_messageInfo_MethodOptions.DiscardUnknown(m) +} + +var xxx_messageInfo_MethodOptions proto.InternalMessageInfo + +const Default_MethodOptions_Deprecated bool = false +const Default_MethodOptions_IdempotencyLevel MethodOptions_IdempotencyLevel = MethodOptions_IDEMPOTENCY_UNKNOWN + +func (m *MethodOptions) GetDeprecated() bool { + if m != nil && m.Deprecated != nil { + return *m.Deprecated + } + return Default_MethodOptions_Deprecated +} + +func (m *MethodOptions) GetIdempotencyLevel() MethodOptions_IdempotencyLevel { + if m != nil && m.IdempotencyLevel != nil { + return *m.IdempotencyLevel + } + return Default_MethodOptions_IdempotencyLevel +} + +func (m *MethodOptions) GetUninterpretedOption() []*UninterpretedOption { + if m != nil { + return m.UninterpretedOption + } + return nil +} + +// A message representing a option the parser does not recognize. This only +// appears in options protos created by the compiler::Parser class. +// DescriptorPool resolves these when building Descriptor objects. Therefore, +// options protos in descriptor objects (e.g. returned by Descriptor::options(), +// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions +// in them. +type UninterpretedOption struct { + Name []*UninterpretedOption_NamePart `protobuf:"bytes,2,rep,name=name" json:"name,omitempty"` + // The value of the uninterpreted option, in whatever type the tokenizer + // identified it as during parsing. Exactly one of these should be set. + IdentifierValue *string `protobuf:"bytes,3,opt,name=identifier_value,json=identifierValue" json:"identifier_value,omitempty"` + PositiveIntValue *uint64 `protobuf:"varint,4,opt,name=positive_int_value,json=positiveIntValue" json:"positive_int_value,omitempty"` + NegativeIntValue *int64 `protobuf:"varint,5,opt,name=negative_int_value,json=negativeIntValue" json:"negative_int_value,omitempty"` + DoubleValue *float64 `protobuf:"fixed64,6,opt,name=double_value,json=doubleValue" json:"double_value,omitempty"` + StringValue []byte `protobuf:"bytes,7,opt,name=string_value,json=stringValue" json:"string_value,omitempty"` + AggregateValue *string `protobuf:"bytes,8,opt,name=aggregate_value,json=aggregateValue" json:"aggregate_value,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *UninterpretedOption) Reset() { *m = UninterpretedOption{} } +func (m *UninterpretedOption) String() string { return proto.CompactTextString(m) } +func (*UninterpretedOption) ProtoMessage() {} +func (*UninterpretedOption) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{18} +} +func (m *UninterpretedOption) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_UninterpretedOption.Unmarshal(m, b) +} +func (m *UninterpretedOption) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_UninterpretedOption.Marshal(b, m, deterministic) +} +func (dst *UninterpretedOption) XXX_Merge(src proto.Message) { + xxx_messageInfo_UninterpretedOption.Merge(dst, src) +} +func (m *UninterpretedOption) XXX_Size() int { + return xxx_messageInfo_UninterpretedOption.Size(m) +} +func (m *UninterpretedOption) XXX_DiscardUnknown() { + xxx_messageInfo_UninterpretedOption.DiscardUnknown(m) +} + +var xxx_messageInfo_UninterpretedOption proto.InternalMessageInfo + +func (m *UninterpretedOption) GetName() []*UninterpretedOption_NamePart { + if m != nil { + return m.Name + } + return nil +} + +func (m *UninterpretedOption) GetIdentifierValue() string { + if m != nil && m.IdentifierValue != nil { + return *m.IdentifierValue + } + return "" +} + +func (m *UninterpretedOption) GetPositiveIntValue() uint64 { + if m != nil && m.PositiveIntValue != nil { + return *m.PositiveIntValue + } + return 0 +} + +func (m *UninterpretedOption) GetNegativeIntValue() int64 { + if m != nil && m.NegativeIntValue != nil { + return *m.NegativeIntValue + } + return 0 +} + +func (m *UninterpretedOption) GetDoubleValue() float64 { + if m != nil && m.DoubleValue != nil { + return *m.DoubleValue + } + return 0 +} + +func (m *UninterpretedOption) GetStringValue() []byte { + if m != nil { + return m.StringValue + } + return nil +} + +func (m *UninterpretedOption) GetAggregateValue() string { + if m != nil && m.AggregateValue != nil { + return *m.AggregateValue + } + return "" +} + +// The name of the uninterpreted option. Each string represents a segment in +// a dot-separated name. is_extension is true iff a segment represents an +// extension (denoted with parentheses in options specs in .proto files). +// E.g.,{ ["foo", false], ["bar.baz", true], ["qux", false] } represents +// "foo.(bar.baz).qux". +type UninterpretedOption_NamePart struct { + NamePart *string `protobuf:"bytes,1,req,name=name_part,json=namePart" json:"name_part,omitempty"` + IsExtension *bool `protobuf:"varint,2,req,name=is_extension,json=isExtension" json:"is_extension,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *UninterpretedOption_NamePart) Reset() { *m = UninterpretedOption_NamePart{} } +func (m *UninterpretedOption_NamePart) String() string { return proto.CompactTextString(m) } +func (*UninterpretedOption_NamePart) ProtoMessage() {} +func (*UninterpretedOption_NamePart) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{18, 0} +} +func (m *UninterpretedOption_NamePart) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_UninterpretedOption_NamePart.Unmarshal(m, b) +} +func (m *UninterpretedOption_NamePart) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_UninterpretedOption_NamePart.Marshal(b, m, deterministic) +} +func (dst *UninterpretedOption_NamePart) XXX_Merge(src proto.Message) { + xxx_messageInfo_UninterpretedOption_NamePart.Merge(dst, src) +} +func (m *UninterpretedOption_NamePart) XXX_Size() int { + return xxx_messageInfo_UninterpretedOption_NamePart.Size(m) +} +func (m *UninterpretedOption_NamePart) XXX_DiscardUnknown() { + xxx_messageInfo_UninterpretedOption_NamePart.DiscardUnknown(m) +} + +var xxx_messageInfo_UninterpretedOption_NamePart proto.InternalMessageInfo + +func (m *UninterpretedOption_NamePart) GetNamePart() string { + if m != nil && m.NamePart != nil { + return *m.NamePart + } + return "" +} + +func (m *UninterpretedOption_NamePart) GetIsExtension() bool { + if m != nil && m.IsExtension != nil { + return *m.IsExtension + } + return false +} + +// Encapsulates information about the original source file from which a +// FileDescriptorProto was generated. +type SourceCodeInfo struct { + // A Location identifies a piece of source code in a .proto file which + // corresponds to a particular definition. This information is intended + // to be useful to IDEs, code indexers, documentation generators, and similar + // tools. + // + // For example, say we have a file like: + // message Foo { + // optional string foo = 1; + // } + // Let's look at just the field definition: + // optional string foo = 1; + // ^ ^^ ^^ ^ ^^^ + // a bc de f ghi + // We have the following locations: + // span path represents + // [a,i) [ 4, 0, 2, 0 ] The whole field definition. + // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). + // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). + // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). + // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). + // + // Notes: + // - A location may refer to a repeated field itself (i.e. not to any + // particular index within it). This is used whenever a set of elements are + // logically enclosed in a single code segment. For example, an entire + // extend block (possibly containing multiple extension definitions) will + // have an outer location whose path refers to the "extensions" repeated + // field without an index. + // - Multiple locations may have the same path. This happens when a single + // logical declaration is spread out across multiple places. The most + // obvious example is the "extend" block again -- there may be multiple + // extend blocks in the same scope, each of which will have the same path. + // - A location's span is not always a subset of its parent's span. For + // example, the "extendee" of an extension declaration appears at the + // beginning of the "extend" block and is shared by all extensions within + // the block. + // - Just because a location's span is a subset of some other location's span + // does not mean that it is a descendent. For example, a "group" defines + // both a type and a field in a single declaration. Thus, the locations + // corresponding to the type and field and their components will overlap. + // - Code which tries to interpret locations should probably be designed to + // ignore those that it doesn't understand, as more types of locations could + // be recorded in the future. + Location []*SourceCodeInfo_Location `protobuf:"bytes,1,rep,name=location" json:"location,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SourceCodeInfo) Reset() { *m = SourceCodeInfo{} } +func (m *SourceCodeInfo) String() string { return proto.CompactTextString(m) } +func (*SourceCodeInfo) ProtoMessage() {} +func (*SourceCodeInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{19} +} +func (m *SourceCodeInfo) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SourceCodeInfo.Unmarshal(m, b) +} +func (m *SourceCodeInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SourceCodeInfo.Marshal(b, m, deterministic) +} +func (dst *SourceCodeInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_SourceCodeInfo.Merge(dst, src) +} +func (m *SourceCodeInfo) XXX_Size() int { + return xxx_messageInfo_SourceCodeInfo.Size(m) +} +func (m *SourceCodeInfo) XXX_DiscardUnknown() { + xxx_messageInfo_SourceCodeInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_SourceCodeInfo proto.InternalMessageInfo + +func (m *SourceCodeInfo) GetLocation() []*SourceCodeInfo_Location { + if m != nil { + return m.Location + } + return nil +} + +type SourceCodeInfo_Location struct { + // Identifies which part of the FileDescriptorProto was defined at this + // location. + // + // Each element is a field number or an index. They form a path from + // the root FileDescriptorProto to the place where the definition. For + // example, this path: + // [ 4, 3, 2, 7, 1 ] + // refers to: + // file.message_type(3) // 4, 3 + // .field(7) // 2, 7 + // .name() // 1 + // This is because FileDescriptorProto.message_type has field number 4: + // repeated DescriptorProto message_type = 4; + // and DescriptorProto.field has field number 2: + // repeated FieldDescriptorProto field = 2; + // and FieldDescriptorProto.name has field number 1: + // optional string name = 1; + // + // Thus, the above path gives the location of a field name. If we removed + // the last element: + // [ 4, 3, 2, 7 ] + // this path refers to the whole field declaration (from the beginning + // of the label to the terminating semicolon). + Path []int32 `protobuf:"varint,1,rep,packed,name=path" json:"path,omitempty"` + // Always has exactly three or four elements: start line, start column, + // end line (optional, otherwise assumed same as start line), end column. + // These are packed into a single field for efficiency. Note that line + // and column numbers are zero-based -- typically you will want to add + // 1 to each before displaying to a user. + Span []int32 `protobuf:"varint,2,rep,packed,name=span" json:"span,omitempty"` + // If this SourceCodeInfo represents a complete declaration, these are any + // comments appearing before and after the declaration which appear to be + // attached to the declaration. + // + // A series of line comments appearing on consecutive lines, with no other + // tokens appearing on those lines, will be treated as a single comment. + // + // leading_detached_comments will keep paragraphs of comments that appear + // before (but not connected to) the current element. Each paragraph, + // separated by empty lines, will be one comment element in the repeated + // field. + // + // Only the comment content is provided; comment markers (e.g. //) are + // stripped out. For block comments, leading whitespace and an asterisk + // will be stripped from the beginning of each line other than the first. + // Newlines are included in the output. + // + // Examples: + // + // optional int32 foo = 1; // Comment attached to foo. + // // Comment attached to bar. + // optional int32 bar = 2; + // + // optional string baz = 3; + // // Comment attached to baz. + // // Another line attached to baz. + // + // // Comment attached to qux. + // // + // // Another line attached to qux. + // optional double qux = 4; + // + // // Detached comment for corge. This is not leading or trailing comments + // // to qux or corge because there are blank lines separating it from + // // both. + // + // // Detached comment for corge paragraph 2. + // + // optional string corge = 5; + // /* Block comment attached + // * to corge. Leading asterisks + // * will be removed. */ + // /* Block comment attached to + // * grault. */ + // optional int32 grault = 6; + // + // // ignored detached comments. + LeadingComments *string `protobuf:"bytes,3,opt,name=leading_comments,json=leadingComments" json:"leading_comments,omitempty"` + TrailingComments *string `protobuf:"bytes,4,opt,name=trailing_comments,json=trailingComments" json:"trailing_comments,omitempty"` + LeadingDetachedComments []string `protobuf:"bytes,6,rep,name=leading_detached_comments,json=leadingDetachedComments" json:"leading_detached_comments,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SourceCodeInfo_Location) Reset() { *m = SourceCodeInfo_Location{} } +func (m *SourceCodeInfo_Location) String() string { return proto.CompactTextString(m) } +func (*SourceCodeInfo_Location) ProtoMessage() {} +func (*SourceCodeInfo_Location) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{19, 0} +} +func (m *SourceCodeInfo_Location) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SourceCodeInfo_Location.Unmarshal(m, b) +} +func (m *SourceCodeInfo_Location) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SourceCodeInfo_Location.Marshal(b, m, deterministic) +} +func (dst *SourceCodeInfo_Location) XXX_Merge(src proto.Message) { + xxx_messageInfo_SourceCodeInfo_Location.Merge(dst, src) +} +func (m *SourceCodeInfo_Location) XXX_Size() int { + return xxx_messageInfo_SourceCodeInfo_Location.Size(m) +} +func (m *SourceCodeInfo_Location) XXX_DiscardUnknown() { + xxx_messageInfo_SourceCodeInfo_Location.DiscardUnknown(m) +} + +var xxx_messageInfo_SourceCodeInfo_Location proto.InternalMessageInfo + +func (m *SourceCodeInfo_Location) GetPath() []int32 { + if m != nil { + return m.Path + } + return nil +} + +func (m *SourceCodeInfo_Location) GetSpan() []int32 { + if m != nil { + return m.Span + } + return nil +} + +func (m *SourceCodeInfo_Location) GetLeadingComments() string { + if m != nil && m.LeadingComments != nil { + return *m.LeadingComments + } + return "" +} + +func (m *SourceCodeInfo_Location) GetTrailingComments() string { + if m != nil && m.TrailingComments != nil { + return *m.TrailingComments + } + return "" +} + +func (m *SourceCodeInfo_Location) GetLeadingDetachedComments() []string { + if m != nil { + return m.LeadingDetachedComments + } + return nil +} + +// Describes the relationship between generated code and its original source +// file. A GeneratedCodeInfo message is associated with only one generated +// source file, but may contain references to different source .proto files. +type GeneratedCodeInfo struct { + // An Annotation connects some span of text in generated code to an element + // of its generating .proto file. + Annotation []*GeneratedCodeInfo_Annotation `protobuf:"bytes,1,rep,name=annotation" json:"annotation,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GeneratedCodeInfo) Reset() { *m = GeneratedCodeInfo{} } +func (m *GeneratedCodeInfo) String() string { return proto.CompactTextString(m) } +func (*GeneratedCodeInfo) ProtoMessage() {} +func (*GeneratedCodeInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{20} +} +func (m *GeneratedCodeInfo) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GeneratedCodeInfo.Unmarshal(m, b) +} +func (m *GeneratedCodeInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GeneratedCodeInfo.Marshal(b, m, deterministic) +} +func (dst *GeneratedCodeInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_GeneratedCodeInfo.Merge(dst, src) +} +func (m *GeneratedCodeInfo) XXX_Size() int { + return xxx_messageInfo_GeneratedCodeInfo.Size(m) +} +func (m *GeneratedCodeInfo) XXX_DiscardUnknown() { + xxx_messageInfo_GeneratedCodeInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_GeneratedCodeInfo proto.InternalMessageInfo + +func (m *GeneratedCodeInfo) GetAnnotation() []*GeneratedCodeInfo_Annotation { + if m != nil { + return m.Annotation + } + return nil +} + +type GeneratedCodeInfo_Annotation struct { + // Identifies the element in the original source .proto file. This field + // is formatted the same as SourceCodeInfo.Location.path. + Path []int32 `protobuf:"varint,1,rep,packed,name=path" json:"path,omitempty"` + // Identifies the filesystem path to the original source .proto. + SourceFile *string `protobuf:"bytes,2,opt,name=source_file,json=sourceFile" json:"source_file,omitempty"` + // Identifies the starting offset in bytes in the generated code + // that relates to the identified object. + Begin *int32 `protobuf:"varint,3,opt,name=begin" json:"begin,omitempty"` + // Identifies the ending offset in bytes in the generated code that + // relates to the identified offset. The end offset should be one past + // the last relevant byte (so the length of the text = end - begin). + End *int32 `protobuf:"varint,4,opt,name=end" json:"end,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GeneratedCodeInfo_Annotation) Reset() { *m = GeneratedCodeInfo_Annotation{} } +func (m *GeneratedCodeInfo_Annotation) String() string { return proto.CompactTextString(m) } +func (*GeneratedCodeInfo_Annotation) ProtoMessage() {} +func (*GeneratedCodeInfo_Annotation) Descriptor() ([]byte, []int) { + return fileDescriptor_descriptor_4df4cb5f42392df6, []int{20, 0} +} +func (m *GeneratedCodeInfo_Annotation) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GeneratedCodeInfo_Annotation.Unmarshal(m, b) +} +func (m *GeneratedCodeInfo_Annotation) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GeneratedCodeInfo_Annotation.Marshal(b, m, deterministic) +} +func (dst *GeneratedCodeInfo_Annotation) XXX_Merge(src proto.Message) { + xxx_messageInfo_GeneratedCodeInfo_Annotation.Merge(dst, src) +} +func (m *GeneratedCodeInfo_Annotation) XXX_Size() int { + return xxx_messageInfo_GeneratedCodeInfo_Annotation.Size(m) +} +func (m *GeneratedCodeInfo_Annotation) XXX_DiscardUnknown() { + xxx_messageInfo_GeneratedCodeInfo_Annotation.DiscardUnknown(m) +} + +var xxx_messageInfo_GeneratedCodeInfo_Annotation proto.InternalMessageInfo + +func (m *GeneratedCodeInfo_Annotation) GetPath() []int32 { + if m != nil { + return m.Path + } + return nil +} + +func (m *GeneratedCodeInfo_Annotation) GetSourceFile() string { + if m != nil && m.SourceFile != nil { + return *m.SourceFile + } + return "" +} + +func (m *GeneratedCodeInfo_Annotation) GetBegin() int32 { + if m != nil && m.Begin != nil { + return *m.Begin + } + return 0 +} + +func (m *GeneratedCodeInfo_Annotation) GetEnd() int32 { + if m != nil && m.End != nil { + return *m.End + } + return 0 +} + +func init() { + proto.RegisterType((*FileDescriptorSet)(nil), "google.protobuf.FileDescriptorSet") + proto.RegisterType((*FileDescriptorProto)(nil), "google.protobuf.FileDescriptorProto") + proto.RegisterType((*DescriptorProto)(nil), "google.protobuf.DescriptorProto") + proto.RegisterType((*DescriptorProto_ExtensionRange)(nil), "google.protobuf.DescriptorProto.ExtensionRange") + proto.RegisterType((*DescriptorProto_ReservedRange)(nil), "google.protobuf.DescriptorProto.ReservedRange") + proto.RegisterType((*ExtensionRangeOptions)(nil), "google.protobuf.ExtensionRangeOptions") + proto.RegisterType((*FieldDescriptorProto)(nil), "google.protobuf.FieldDescriptorProto") + proto.RegisterType((*OneofDescriptorProto)(nil), "google.protobuf.OneofDescriptorProto") + proto.RegisterType((*EnumDescriptorProto)(nil), "google.protobuf.EnumDescriptorProto") + proto.RegisterType((*EnumDescriptorProto_EnumReservedRange)(nil), "google.protobuf.EnumDescriptorProto.EnumReservedRange") + proto.RegisterType((*EnumValueDescriptorProto)(nil), "google.protobuf.EnumValueDescriptorProto") + proto.RegisterType((*ServiceDescriptorProto)(nil), "google.protobuf.ServiceDescriptorProto") + proto.RegisterType((*MethodDescriptorProto)(nil), "google.protobuf.MethodDescriptorProto") + proto.RegisterType((*FileOptions)(nil), "google.protobuf.FileOptions") + proto.RegisterType((*MessageOptions)(nil), "google.protobuf.MessageOptions") + proto.RegisterType((*FieldOptions)(nil), "google.protobuf.FieldOptions") + proto.RegisterType((*OneofOptions)(nil), "google.protobuf.OneofOptions") + proto.RegisterType((*EnumOptions)(nil), "google.protobuf.EnumOptions") + proto.RegisterType((*EnumValueOptions)(nil), "google.protobuf.EnumValueOptions") + proto.RegisterType((*ServiceOptions)(nil), "google.protobuf.ServiceOptions") + proto.RegisterType((*MethodOptions)(nil), "google.protobuf.MethodOptions") + proto.RegisterType((*UninterpretedOption)(nil), "google.protobuf.UninterpretedOption") + proto.RegisterType((*UninterpretedOption_NamePart)(nil), "google.protobuf.UninterpretedOption.NamePart") + proto.RegisterType((*SourceCodeInfo)(nil), "google.protobuf.SourceCodeInfo") + proto.RegisterType((*SourceCodeInfo_Location)(nil), "google.protobuf.SourceCodeInfo.Location") + proto.RegisterType((*GeneratedCodeInfo)(nil), "google.protobuf.GeneratedCodeInfo") + proto.RegisterType((*GeneratedCodeInfo_Annotation)(nil), "google.protobuf.GeneratedCodeInfo.Annotation") + proto.RegisterEnum("google.protobuf.FieldDescriptorProto_Type", FieldDescriptorProto_Type_name, FieldDescriptorProto_Type_value) + proto.RegisterEnum("google.protobuf.FieldDescriptorProto_Label", FieldDescriptorProto_Label_name, FieldDescriptorProto_Label_value) + proto.RegisterEnum("google.protobuf.FileOptions_OptimizeMode", FileOptions_OptimizeMode_name, FileOptions_OptimizeMode_value) + proto.RegisterEnum("google.protobuf.FieldOptions_CType", FieldOptions_CType_name, FieldOptions_CType_value) + proto.RegisterEnum("google.protobuf.FieldOptions_JSType", FieldOptions_JSType_name, FieldOptions_JSType_value) + proto.RegisterEnum("google.protobuf.MethodOptions_IdempotencyLevel", MethodOptions_IdempotencyLevel_name, MethodOptions_IdempotencyLevel_value) +} + +func init() { + proto.RegisterFile("google/protobuf/descriptor.proto", fileDescriptor_descriptor_4df4cb5f42392df6) +} + +var fileDescriptor_descriptor_4df4cb5f42392df6 = []byte{ + // 2555 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x59, 0xdd, 0x6e, 0x1b, 0xc7, + 0xf5, 0xcf, 0xf2, 0x4b, 0xe4, 0x21, 0x45, 0x8d, 0x46, 0x8a, 0xbd, 0x56, 0x3e, 0x2c, 0x33, 0x1f, + 0x96, 0x9d, 0x7f, 0xa8, 0xc0, 0xb1, 0x1d, 0x47, 0xfe, 0x23, 0x2d, 0x45, 0xae, 0x15, 0xaa, 0x12, + 0xc9, 0x2e, 0xa9, 0xe6, 0x03, 0x28, 0x16, 0xa3, 0xdd, 0x21, 0xb9, 0xf6, 0x72, 0x77, 0xb3, 0xbb, + 0xb4, 0xad, 0xa0, 0x17, 0x06, 0x7a, 0xd5, 0xab, 0xde, 0x16, 0x45, 0xd1, 0x8b, 0xde, 0x04, 0xe8, + 0x03, 0x14, 0xc8, 0x5d, 0x9f, 0xa0, 0x40, 0xde, 0xa0, 0x68, 0x0b, 0xb4, 0x8f, 0xd0, 0xcb, 0x62, + 0x66, 0x76, 0x97, 0xbb, 0x24, 0x15, 0x2b, 0x01, 0xe2, 0x5c, 0x91, 0xf3, 0x9b, 0xdf, 0x39, 0x73, + 0xe6, 0xcc, 0x99, 0x33, 0x67, 0x66, 0x61, 0x7b, 0xe4, 0x38, 0x23, 0x8b, 0xee, 0xba, 0x9e, 0x13, + 0x38, 0xa7, 0xd3, 0xe1, 0xae, 0x41, 0x7d, 0xdd, 0x33, 0xdd, 0xc0, 0xf1, 0xea, 0x1c, 0xc3, 0x6b, + 0x82, 0x51, 0x8f, 0x18, 0xb5, 0x63, 0x58, 0x7f, 0x60, 0x5a, 0xb4, 0x15, 0x13, 0xfb, 0x34, 0xc0, + 0xf7, 0x20, 0x37, 0x34, 0x2d, 0x2a, 0x4b, 0xdb, 0xd9, 0x9d, 0xf2, 0xad, 0x37, 0xeb, 0x73, 0x42, + 0xf5, 0xb4, 0x44, 0x8f, 0xc1, 0x2a, 0x97, 0xa8, 0xfd, 0x2b, 0x07, 0x1b, 0x4b, 0x7a, 0x31, 0x86, + 0x9c, 0x4d, 0x26, 0x4c, 0xa3, 0xb4, 0x53, 0x52, 0xf9, 0x7f, 0x2c, 0xc3, 0x8a, 0x4b, 0xf4, 0x47, + 0x64, 0x44, 0xe5, 0x0c, 0x87, 0xa3, 0x26, 0x7e, 0x1d, 0xc0, 0xa0, 0x2e, 0xb5, 0x0d, 0x6a, 0xeb, + 0x67, 0x72, 0x76, 0x3b, 0xbb, 0x53, 0x52, 0x13, 0x08, 0x7e, 0x07, 0xd6, 0xdd, 0xe9, 0xa9, 0x65, + 0xea, 0x5a, 0x82, 0x06, 0xdb, 0xd9, 0x9d, 0xbc, 0x8a, 0x44, 0x47, 0x6b, 0x46, 0xbe, 0x0e, 0x6b, + 0x4f, 0x28, 0x79, 0x94, 0xa4, 0x96, 0x39, 0xb5, 0xca, 0xe0, 0x04, 0xb1, 0x09, 0x95, 0x09, 0xf5, + 0x7d, 0x32, 0xa2, 0x5a, 0x70, 0xe6, 0x52, 0x39, 0xc7, 0x67, 0xbf, 0xbd, 0x30, 0xfb, 0xf9, 0x99, + 0x97, 0x43, 0xa9, 0xc1, 0x99, 0x4b, 0x71, 0x03, 0x4a, 0xd4, 0x9e, 0x4e, 0x84, 0x86, 0xfc, 0x39, + 0xfe, 0x53, 0xec, 0xe9, 0x64, 0x5e, 0x4b, 0x91, 0x89, 0x85, 0x2a, 0x56, 0x7c, 0xea, 0x3d, 0x36, + 0x75, 0x2a, 0x17, 0xb8, 0x82, 0xeb, 0x0b, 0x0a, 0xfa, 0xa2, 0x7f, 0x5e, 0x47, 0x24, 0x87, 0x9b, + 0x50, 0xa2, 0x4f, 0x03, 0x6a, 0xfb, 0xa6, 0x63, 0xcb, 0x2b, 0x5c, 0xc9, 0x5b, 0x4b, 0x56, 0x91, + 0x5a, 0xc6, 0xbc, 0x8a, 0x99, 0x1c, 0xbe, 0x0b, 0x2b, 0x8e, 0x1b, 0x98, 0x8e, 0xed, 0xcb, 0xc5, + 0x6d, 0x69, 0xa7, 0x7c, 0xeb, 0xd5, 0xa5, 0x81, 0xd0, 0x15, 0x1c, 0x35, 0x22, 0xe3, 0x36, 0x20, + 0xdf, 0x99, 0x7a, 0x3a, 0xd5, 0x74, 0xc7, 0xa0, 0x9a, 0x69, 0x0f, 0x1d, 0xb9, 0xc4, 0x15, 0x5c, + 0x5d, 0x9c, 0x08, 0x27, 0x36, 0x1d, 0x83, 0xb6, 0xed, 0xa1, 0xa3, 0x56, 0xfd, 0x54, 0x1b, 0x5f, + 0x82, 0x82, 0x7f, 0x66, 0x07, 0xe4, 0xa9, 0x5c, 0xe1, 0x11, 0x12, 0xb6, 0x6a, 0x5f, 0x17, 0x60, + 0xed, 0x22, 0x21, 0x76, 0x1f, 0xf2, 0x43, 0x36, 0x4b, 0x39, 0xf3, 0x5d, 0x7c, 0x20, 0x64, 0xd2, + 0x4e, 0x2c, 0x7c, 0x4f, 0x27, 0x36, 0xa0, 0x6c, 0x53, 0x3f, 0xa0, 0x86, 0x88, 0x88, 0xec, 0x05, + 0x63, 0x0a, 0x84, 0xd0, 0x62, 0x48, 0xe5, 0xbe, 0x57, 0x48, 0x7d, 0x0a, 0x6b, 0xb1, 0x49, 0x9a, + 0x47, 0xec, 0x51, 0x14, 0x9b, 0xbb, 0xcf, 0xb3, 0xa4, 0xae, 0x44, 0x72, 0x2a, 0x13, 0x53, 0xab, + 0x34, 0xd5, 0xc6, 0x2d, 0x00, 0xc7, 0xa6, 0xce, 0x50, 0x33, 0xa8, 0x6e, 0xc9, 0xc5, 0x73, 0xbc, + 0xd4, 0x65, 0x94, 0x05, 0x2f, 0x39, 0x02, 0xd5, 0x2d, 0xfc, 0xe1, 0x2c, 0xd4, 0x56, 0xce, 0x89, + 0x94, 0x63, 0xb1, 0xc9, 0x16, 0xa2, 0xed, 0x04, 0xaa, 0x1e, 0x65, 0x71, 0x4f, 0x8d, 0x70, 0x66, + 0x25, 0x6e, 0x44, 0xfd, 0xb9, 0x33, 0x53, 0x43, 0x31, 0x31, 0xb1, 0x55, 0x2f, 0xd9, 0xc4, 0x6f, + 0x40, 0x0c, 0x68, 0x3c, 0xac, 0x80, 0x67, 0xa1, 0x4a, 0x04, 0x76, 0xc8, 0x84, 0x6e, 0x7d, 0x09, + 0xd5, 0xb4, 0x7b, 0xf0, 0x26, 0xe4, 0xfd, 0x80, 0x78, 0x01, 0x8f, 0xc2, 0xbc, 0x2a, 0x1a, 0x18, + 0x41, 0x96, 0xda, 0x06, 0xcf, 0x72, 0x79, 0x95, 0xfd, 0xc5, 0x3f, 0x9d, 0x4d, 0x38, 0xcb, 0x27, + 0xfc, 0xf6, 0xe2, 0x8a, 0xa6, 0x34, 0xcf, 0xcf, 0x7b, 0xeb, 0x03, 0x58, 0x4d, 0x4d, 0xe0, 0xa2, + 0x43, 0xd7, 0x7e, 0x05, 0x2f, 0x2f, 0x55, 0x8d, 0x3f, 0x85, 0xcd, 0xa9, 0x6d, 0xda, 0x01, 0xf5, + 0x5c, 0x8f, 0xb2, 0x88, 0x15, 0x43, 0xc9, 0xff, 0x5e, 0x39, 0x27, 0xe6, 0x4e, 0x92, 0x6c, 0xa1, + 0x45, 0xdd, 0x98, 0x2e, 0x82, 0x37, 0x4b, 0xc5, 0xff, 0xac, 0xa0, 0x67, 0xcf, 0x9e, 0x3d, 0xcb, + 0xd4, 0x7e, 0x57, 0x80, 0xcd, 0x65, 0x7b, 0x66, 0xe9, 0xf6, 0xbd, 0x04, 0x05, 0x7b, 0x3a, 0x39, + 0xa5, 0x1e, 0x77, 0x52, 0x5e, 0x0d, 0x5b, 0xb8, 0x01, 0x79, 0x8b, 0x9c, 0x52, 0x4b, 0xce, 0x6d, + 0x4b, 0x3b, 0xd5, 0x5b, 0xef, 0x5c, 0x68, 0x57, 0xd6, 0x8f, 0x98, 0x88, 0x2a, 0x24, 0xf1, 0x47, + 0x90, 0x0b, 0x53, 0x34, 0xd3, 0x70, 0xf3, 0x62, 0x1a, 0xd8, 0x5e, 0x52, 0xb9, 0x1c, 0x7e, 0x05, + 0x4a, 0xec, 0x57, 0xc4, 0x46, 0x81, 0xdb, 0x5c, 0x64, 0x00, 0x8b, 0x0b, 0xbc, 0x05, 0x45, 0xbe, + 0x4d, 0x0c, 0x1a, 0x1d, 0x6d, 0x71, 0x9b, 0x05, 0x96, 0x41, 0x87, 0x64, 0x6a, 0x05, 0xda, 0x63, + 0x62, 0x4d, 0x29, 0x0f, 0xf8, 0x92, 0x5a, 0x09, 0xc1, 0x5f, 0x30, 0x0c, 0x5f, 0x85, 0xb2, 0xd8, + 0x55, 0xa6, 0x6d, 0xd0, 0xa7, 0x3c, 0x7b, 0xe6, 0x55, 0xb1, 0xd1, 0xda, 0x0c, 0x61, 0xc3, 0x3f, + 0xf4, 0x1d, 0x3b, 0x0a, 0x4d, 0x3e, 0x04, 0x03, 0xf8, 0xf0, 0x1f, 0xcc, 0x27, 0xee, 0xd7, 0x96, + 0x4f, 0x6f, 0x3e, 0xa6, 0x6a, 0x7f, 0xc9, 0x40, 0x8e, 0xe7, 0x8b, 0x35, 0x28, 0x0f, 0x3e, 0xeb, + 0x29, 0x5a, 0xab, 0x7b, 0xb2, 0x7f, 0xa4, 0x20, 0x09, 0x57, 0x01, 0x38, 0xf0, 0xe0, 0xa8, 0xdb, + 0x18, 0xa0, 0x4c, 0xdc, 0x6e, 0x77, 0x06, 0x77, 0x6f, 0xa3, 0x6c, 0x2c, 0x70, 0x22, 0x80, 0x5c, + 0x92, 0xf0, 0xfe, 0x2d, 0x94, 0xc7, 0x08, 0x2a, 0x42, 0x41, 0xfb, 0x53, 0xa5, 0x75, 0xf7, 0x36, + 0x2a, 0xa4, 0x91, 0xf7, 0x6f, 0xa1, 0x15, 0xbc, 0x0a, 0x25, 0x8e, 0xec, 0x77, 0xbb, 0x47, 0xa8, + 0x18, 0xeb, 0xec, 0x0f, 0xd4, 0x76, 0xe7, 0x00, 0x95, 0x62, 0x9d, 0x07, 0x6a, 0xf7, 0xa4, 0x87, + 0x20, 0xd6, 0x70, 0xac, 0xf4, 0xfb, 0x8d, 0x03, 0x05, 0x95, 0x63, 0xc6, 0xfe, 0x67, 0x03, 0xa5, + 0x8f, 0x2a, 0x29, 0xb3, 0xde, 0xbf, 0x85, 0x56, 0xe3, 0x21, 0x94, 0xce, 0xc9, 0x31, 0xaa, 0xe2, + 0x75, 0x58, 0x15, 0x43, 0x44, 0x46, 0xac, 0xcd, 0x41, 0x77, 0x6f, 0x23, 0x34, 0x33, 0x44, 0x68, + 0x59, 0x4f, 0x01, 0x77, 0x6f, 0x23, 0x5c, 0x6b, 0x42, 0x9e, 0x47, 0x17, 0xc6, 0x50, 0x3d, 0x6a, + 0xec, 0x2b, 0x47, 0x5a, 0xb7, 0x37, 0x68, 0x77, 0x3b, 0x8d, 0x23, 0x24, 0xcd, 0x30, 0x55, 0xf9, + 0xf9, 0x49, 0x5b, 0x55, 0x5a, 0x28, 0x93, 0xc4, 0x7a, 0x4a, 0x63, 0xa0, 0xb4, 0x50, 0xb6, 0xa6, + 0xc3, 0xe6, 0xb2, 0x3c, 0xb9, 0x74, 0x67, 0x24, 0x96, 0x38, 0x73, 0xce, 0x12, 0x73, 0x5d, 0x0b, + 0x4b, 0xfc, 0xcf, 0x0c, 0x6c, 0x2c, 0x39, 0x2b, 0x96, 0x0e, 0xf2, 0x13, 0xc8, 0x8b, 0x10, 0x15, + 0xa7, 0xe7, 0x8d, 0xa5, 0x87, 0x0e, 0x0f, 0xd8, 0x85, 0x13, 0x94, 0xcb, 0x25, 0x2b, 0x88, 0xec, + 0x39, 0x15, 0x04, 0x53, 0xb1, 0x90, 0xd3, 0x7f, 0xb9, 0x90, 0xd3, 0xc5, 0xb1, 0x77, 0xf7, 0x22, + 0xc7, 0x1e, 0xc7, 0xbe, 0x5b, 0x6e, 0xcf, 0x2f, 0xc9, 0xed, 0xf7, 0x61, 0x7d, 0x41, 0xd1, 0x85, + 0x73, 0xec, 0xaf, 0x25, 0x90, 0xcf, 0x73, 0xce, 0x73, 0x32, 0x5d, 0x26, 0x95, 0xe9, 0xee, 0xcf, + 0x7b, 0xf0, 0xda, 0xf9, 0x8b, 0xb0, 0xb0, 0xd6, 0x5f, 0x49, 0x70, 0x69, 0x79, 0xa5, 0xb8, 0xd4, + 0x86, 0x8f, 0xa0, 0x30, 0xa1, 0xc1, 0xd8, 0x89, 0xaa, 0xa5, 0xb7, 0x97, 0x9c, 0xc1, 0xac, 0x7b, + 0x7e, 0xb1, 0x43, 0xa9, 0xe4, 0x21, 0x9e, 0x3d, 0xaf, 0xdc, 0x13, 0xd6, 0x2c, 0x58, 0xfa, 0x9b, + 0x0c, 0xbc, 0xbc, 0x54, 0xf9, 0x52, 0x43, 0x5f, 0x03, 0x30, 0x6d, 0x77, 0x1a, 0x88, 0x8a, 0x48, + 0x24, 0xd8, 0x12, 0x47, 0x78, 0xf2, 0x62, 0xc9, 0x73, 0x1a, 0xc4, 0xfd, 0x59, 0xde, 0x0f, 0x02, + 0xe2, 0x84, 0x7b, 0x33, 0x43, 0x73, 0xdc, 0xd0, 0xd7, 0xcf, 0x99, 0xe9, 0x42, 0x60, 0xbe, 0x07, + 0x48, 0xb7, 0x4c, 0x6a, 0x07, 0x9a, 0x1f, 0x78, 0x94, 0x4c, 0x4c, 0x7b, 0xc4, 0x4f, 0x90, 0xe2, + 0x5e, 0x7e, 0x48, 0x2c, 0x9f, 0xaa, 0x6b, 0xa2, 0xbb, 0x1f, 0xf5, 0x32, 0x09, 0x1e, 0x40, 0x5e, + 0x42, 0xa2, 0x90, 0x92, 0x10, 0xdd, 0xb1, 0x44, 0xed, 0xeb, 0x22, 0x94, 0x13, 0x75, 0x35, 0xbe, + 0x06, 0x95, 0x87, 0xe4, 0x31, 0xd1, 0xa2, 0xbb, 0x92, 0xf0, 0x44, 0x99, 0x61, 0xbd, 0xf0, 0xbe, + 0xf4, 0x1e, 0x6c, 0x72, 0x8a, 0x33, 0x0d, 0xa8, 0xa7, 0xe9, 0x16, 0xf1, 0x7d, 0xee, 0xb4, 0x22, + 0xa7, 0x62, 0xd6, 0xd7, 0x65, 0x5d, 0xcd, 0xa8, 0x07, 0xdf, 0x81, 0x0d, 0x2e, 0x31, 0x99, 0x5a, + 0x81, 0xe9, 0x5a, 0x54, 0x63, 0xb7, 0x37, 0x9f, 0x9f, 0x24, 0xb1, 0x65, 0xeb, 0x8c, 0x71, 0x1c, + 0x12, 0x98, 0x45, 0x3e, 0x6e, 0xc1, 0x6b, 0x5c, 0x6c, 0x44, 0x6d, 0xea, 0x91, 0x80, 0x6a, 0xf4, + 0x8b, 0x29, 0xb1, 0x7c, 0x8d, 0xd8, 0x86, 0x36, 0x26, 0xfe, 0x58, 0xde, 0x64, 0x0a, 0xf6, 0x33, + 0xb2, 0xa4, 0x5e, 0x61, 0xc4, 0x83, 0x90, 0xa7, 0x70, 0x5a, 0xc3, 0x36, 0x3e, 0x26, 0xfe, 0x18, + 0xef, 0xc1, 0x25, 0xae, 0xc5, 0x0f, 0x3c, 0xd3, 0x1e, 0x69, 0xfa, 0x98, 0xea, 0x8f, 0xb4, 0x69, + 0x30, 0xbc, 0x27, 0xbf, 0x92, 0x1c, 0x9f, 0x5b, 0xd8, 0xe7, 0x9c, 0x26, 0xa3, 0x9c, 0x04, 0xc3, + 0x7b, 0xb8, 0x0f, 0x15, 0xb6, 0x18, 0x13, 0xf3, 0x4b, 0xaa, 0x0d, 0x1d, 0x8f, 0x1f, 0x8d, 0xd5, + 0x25, 0xa9, 0x29, 0xe1, 0xc1, 0x7a, 0x37, 0x14, 0x38, 0x76, 0x0c, 0xba, 0x97, 0xef, 0xf7, 0x14, + 0xa5, 0xa5, 0x96, 0x23, 0x2d, 0x0f, 0x1c, 0x8f, 0x05, 0xd4, 0xc8, 0x89, 0x1d, 0x5c, 0x16, 0x01, + 0x35, 0x72, 0x22, 0xf7, 0xde, 0x81, 0x0d, 0x5d, 0x17, 0x73, 0x36, 0x75, 0x2d, 0xbc, 0x63, 0xf9, + 0x32, 0x4a, 0x39, 0x4b, 0xd7, 0x0f, 0x04, 0x21, 0x8c, 0x71, 0x1f, 0x7f, 0x08, 0x2f, 0xcf, 0x9c, + 0x95, 0x14, 0x5c, 0x5f, 0x98, 0xe5, 0xbc, 0xe8, 0x1d, 0xd8, 0x70, 0xcf, 0x16, 0x05, 0x71, 0x6a, + 0x44, 0xf7, 0x6c, 0x5e, 0xec, 0x03, 0xd8, 0x74, 0xc7, 0xee, 0xa2, 0xdc, 0xcd, 0xa4, 0x1c, 0x76, + 0xc7, 0xee, 0xbc, 0xe0, 0x5b, 0xfc, 0xc2, 0xed, 0x51, 0x9d, 0x04, 0xd4, 0x90, 0x2f, 0x27, 0xe9, + 0x89, 0x0e, 0xbc, 0x0b, 0x48, 0xd7, 0x35, 0x6a, 0x93, 0x53, 0x8b, 0x6a, 0xc4, 0xa3, 0x36, 0xf1, + 0xe5, 0xab, 0x49, 0x72, 0x55, 0xd7, 0x15, 0xde, 0xdb, 0xe0, 0x9d, 0xf8, 0x26, 0xac, 0x3b, 0xa7, + 0x0f, 0x75, 0x11, 0x92, 0x9a, 0xeb, 0xd1, 0xa1, 0xf9, 0x54, 0x7e, 0x93, 0xfb, 0x77, 0x8d, 0x75, + 0xf0, 0x80, 0xec, 0x71, 0x18, 0xdf, 0x00, 0xa4, 0xfb, 0x63, 0xe2, 0xb9, 0x3c, 0x27, 0xfb, 0x2e, + 0xd1, 0xa9, 0xfc, 0x96, 0xa0, 0x0a, 0xbc, 0x13, 0xc1, 0x6c, 0x4b, 0xf8, 0x4f, 0xcc, 0x61, 0x10, + 0x69, 0xbc, 0x2e, 0xb6, 0x04, 0xc7, 0x42, 0x6d, 0x3b, 0x80, 0x98, 0x2b, 0x52, 0x03, 0xef, 0x70, + 0x5a, 0xd5, 0x1d, 0xbb, 0xc9, 0x71, 0xdf, 0x80, 0x55, 0xc6, 0x9c, 0x0d, 0x7a, 0x43, 0x14, 0x64, + 0xee, 0x38, 0x31, 0xe2, 0x0f, 0x56, 0x1b, 0xd7, 0xf6, 0xa0, 0x92, 0x8c, 0x4f, 0x5c, 0x02, 0x11, + 0xa1, 0x48, 0x62, 0xc5, 0x4a, 0xb3, 0xdb, 0x62, 0x65, 0xc6, 0xe7, 0x0a, 0xca, 0xb0, 0x72, 0xe7, + 0xa8, 0x3d, 0x50, 0x34, 0xf5, 0xa4, 0x33, 0x68, 0x1f, 0x2b, 0x28, 0x9b, 0xa8, 0xab, 0x0f, 0x73, + 0xc5, 0xb7, 0xd1, 0xf5, 0xda, 0x37, 0x19, 0xa8, 0xa6, 0x2f, 0x4a, 0xf8, 0xff, 0xe1, 0x72, 0xf4, + 0xaa, 0xe1, 0xd3, 0x40, 0x7b, 0x62, 0x7a, 0x7c, 0xe3, 0x4c, 0x88, 0x38, 0xc4, 0xe2, 0xa5, 0xdb, + 0x0c, 0x59, 0x7d, 0x1a, 0x7c, 0x62, 0x7a, 0x6c, 0x5b, 0x4c, 0x48, 0x80, 0x8f, 0xe0, 0xaa, 0xed, + 0x68, 0x7e, 0x40, 0x6c, 0x83, 0x78, 0x86, 0x36, 0x7b, 0x4f, 0xd2, 0x88, 0xae, 0x53, 0xdf, 0x77, + 0xc4, 0x81, 0x15, 0x6b, 0x79, 0xd5, 0x76, 0xfa, 0x21, 0x79, 0x96, 0xc9, 0x1b, 0x21, 0x75, 0x2e, + 0xcc, 0xb2, 0xe7, 0x85, 0xd9, 0x2b, 0x50, 0x9a, 0x10, 0x57, 0xa3, 0x76, 0xe0, 0x9d, 0xf1, 0xf2, + 0xb8, 0xa8, 0x16, 0x27, 0xc4, 0x55, 0x58, 0xfb, 0x85, 0xdc, 0x52, 0x0e, 0x73, 0xc5, 0x22, 0x2a, + 0x1d, 0xe6, 0x8a, 0x25, 0x04, 0xb5, 0x7f, 0x64, 0xa1, 0x92, 0x2c, 0x97, 0xd9, 0xed, 0x43, 0xe7, + 0x27, 0x8b, 0xc4, 0x73, 0xcf, 0x1b, 0xdf, 0x5a, 0x5c, 0xd7, 0x9b, 0xec, 0xc8, 0xd9, 0x2b, 0x88, + 0x22, 0x56, 0x15, 0x92, 0xec, 0xb8, 0x67, 0xd9, 0x86, 0x8a, 0xa2, 0xa1, 0xa8, 0x86, 0x2d, 0x7c, + 0x00, 0x85, 0x87, 0x3e, 0xd7, 0x5d, 0xe0, 0xba, 0xdf, 0xfc, 0x76, 0xdd, 0x87, 0x7d, 0xae, 0xbc, + 0x74, 0xd8, 0xd7, 0x3a, 0x5d, 0xf5, 0xb8, 0x71, 0xa4, 0x86, 0xe2, 0xf8, 0x0a, 0xe4, 0x2c, 0xf2, + 0xe5, 0x59, 0xfa, 0x70, 0xe2, 0xd0, 0x45, 0x17, 0xe1, 0x0a, 0xe4, 0x9e, 0x50, 0xf2, 0x28, 0x7d, + 0x24, 0x70, 0xe8, 0x07, 0xdc, 0x0c, 0xbb, 0x90, 0xe7, 0xfe, 0xc2, 0x00, 0xa1, 0xc7, 0xd0, 0x4b, + 0xb8, 0x08, 0xb9, 0x66, 0x57, 0x65, 0x1b, 0x02, 0x41, 0x45, 0xa0, 0x5a, 0xaf, 0xad, 0x34, 0x15, + 0x94, 0xa9, 0xdd, 0x81, 0x82, 0x70, 0x02, 0xdb, 0x2c, 0xb1, 0x1b, 0xd0, 0x4b, 0x61, 0x33, 0xd4, + 0x21, 0x45, 0xbd, 0x27, 0xc7, 0xfb, 0x8a, 0x8a, 0x32, 0xe9, 0xa5, 0xce, 0xa1, 0x7c, 0xcd, 0x87, + 0x4a, 0xb2, 0x5e, 0x7e, 0x31, 0x77, 0xe1, 0xbf, 0x4a, 0x50, 0x4e, 0xd4, 0xbf, 0xac, 0x70, 0x21, + 0x96, 0xe5, 0x3c, 0xd1, 0x88, 0x65, 0x12, 0x3f, 0x0c, 0x0d, 0xe0, 0x50, 0x83, 0x21, 0x17, 0x5d, + 0xba, 0x17, 0xb4, 0x45, 0xf2, 0xa8, 0x50, 0xfb, 0xa3, 0x04, 0x68, 0xbe, 0x00, 0x9d, 0x33, 0x53, + 0xfa, 0x31, 0xcd, 0xac, 0xfd, 0x41, 0x82, 0x6a, 0xba, 0xea, 0x9c, 0x33, 0xef, 0xda, 0x8f, 0x6a, + 0xde, 0xdf, 0x33, 0xb0, 0x9a, 0xaa, 0x35, 0x2f, 0x6a, 0xdd, 0x17, 0xb0, 0x6e, 0x1a, 0x74, 0xe2, + 0x3a, 0x01, 0xb5, 0xf5, 0x33, 0xcd, 0xa2, 0x8f, 0xa9, 0x25, 0xd7, 0x78, 0xd2, 0xd8, 0xfd, 0xf6, + 0x6a, 0xb6, 0xde, 0x9e, 0xc9, 0x1d, 0x31, 0xb1, 0xbd, 0x8d, 0x76, 0x4b, 0x39, 0xee, 0x75, 0x07, + 0x4a, 0xa7, 0xf9, 0x99, 0x76, 0xd2, 0xf9, 0x59, 0xa7, 0xfb, 0x49, 0x47, 0x45, 0xe6, 0x1c, 0xed, + 0x07, 0xdc, 0xf6, 0x3d, 0x40, 0xf3, 0x46, 0xe1, 0xcb, 0xb0, 0xcc, 0x2c, 0xf4, 0x12, 0xde, 0x80, + 0xb5, 0x4e, 0x57, 0xeb, 0xb7, 0x5b, 0x8a, 0xa6, 0x3c, 0x78, 0xa0, 0x34, 0x07, 0x7d, 0xf1, 0x3e, + 0x11, 0xb3, 0x07, 0xa9, 0x0d, 0x5e, 0xfb, 0x7d, 0x16, 0x36, 0x96, 0x58, 0x82, 0x1b, 0xe1, 0xcd, + 0x42, 0x5c, 0x76, 0xde, 0xbd, 0x88, 0xf5, 0x75, 0x56, 0x10, 0xf4, 0x88, 0x17, 0x84, 0x17, 0x91, + 0x1b, 0xc0, 0xbc, 0x64, 0x07, 0xe6, 0xd0, 0xa4, 0x5e, 0xf8, 0x9c, 0x23, 0xae, 0x1b, 0x6b, 0x33, + 0x5c, 0xbc, 0xe8, 0xfc, 0x1f, 0x60, 0xd7, 0xf1, 0xcd, 0xc0, 0x7c, 0x4c, 0x35, 0xd3, 0x8e, 0xde, + 0x7e, 0xd8, 0xf5, 0x23, 0xa7, 0xa2, 0xa8, 0xa7, 0x6d, 0x07, 0x31, 0xdb, 0xa6, 0x23, 0x32, 0xc7, + 0x66, 0xc9, 0x3c, 0xab, 0xa2, 0xa8, 0x27, 0x66, 0x5f, 0x83, 0x8a, 0xe1, 0x4c, 0x59, 0x4d, 0x26, + 0x78, 0xec, 0xec, 0x90, 0xd4, 0xb2, 0xc0, 0x62, 0x4a, 0x58, 0x6d, 0xcf, 0x1e, 0x9d, 0x2a, 0x6a, + 0x59, 0x60, 0x82, 0x72, 0x1d, 0xd6, 0xc8, 0x68, 0xe4, 0x31, 0xe5, 0x91, 0x22, 0x71, 0x7f, 0xa8, + 0xc6, 0x30, 0x27, 0x6e, 0x1d, 0x42, 0x31, 0xf2, 0x03, 0x3b, 0xaa, 0x99, 0x27, 0x34, 0x57, 0x5c, + 0x8a, 0x33, 0x3b, 0x25, 0xb5, 0x68, 0x47, 0x9d, 0xd7, 0xa0, 0x62, 0xfa, 0xda, 0xec, 0x0d, 0x3d, + 0xb3, 0x9d, 0xd9, 0x29, 0xaa, 0x65, 0xd3, 0x8f, 0xdf, 0x1f, 0x6b, 0x5f, 0x65, 0xa0, 0x9a, 0xfe, + 0x06, 0x80, 0x5b, 0x50, 0xb4, 0x1c, 0x9d, 0xf0, 0xd0, 0x12, 0x1f, 0xa0, 0x76, 0x9e, 0xf3, 0xd9, + 0xa0, 0x7e, 0x14, 0xf2, 0xd5, 0x58, 0x72, 0xeb, 0x6f, 0x12, 0x14, 0x23, 0x18, 0x5f, 0x82, 0x9c, + 0x4b, 0x82, 0x31, 0x57, 0x97, 0xdf, 0xcf, 0x20, 0x49, 0xe5, 0x6d, 0x86, 0xfb, 0x2e, 0xb1, 0x79, + 0x08, 0x84, 0x38, 0x6b, 0xb3, 0x75, 0xb5, 0x28, 0x31, 0xf8, 0xe5, 0xc4, 0x99, 0x4c, 0xa8, 0x1d, + 0xf8, 0xd1, 0xba, 0x86, 0x78, 0x33, 0x84, 0xf1, 0x3b, 0xb0, 0x1e, 0x78, 0xc4, 0xb4, 0x52, 0xdc, + 0x1c, 0xe7, 0xa2, 0xa8, 0x23, 0x26, 0xef, 0xc1, 0x95, 0x48, 0xaf, 0x41, 0x03, 0xa2, 0x8f, 0xa9, + 0x31, 0x13, 0x2a, 0xf0, 0x47, 0x88, 0xcb, 0x21, 0xa1, 0x15, 0xf6, 0x47, 0xb2, 0xb5, 0x6f, 0x24, + 0x58, 0x8f, 0xae, 0x53, 0x46, 0xec, 0xac, 0x63, 0x00, 0x62, 0xdb, 0x4e, 0x90, 0x74, 0xd7, 0x62, + 0x28, 0x2f, 0xc8, 0xd5, 0x1b, 0xb1, 0x90, 0x9a, 0x50, 0xb0, 0x35, 0x01, 0x98, 0xf5, 0x9c, 0xeb, + 0xb6, 0xab, 0x50, 0x0e, 0x3f, 0xf0, 0xf0, 0xaf, 0x84, 0xe2, 0x02, 0x0e, 0x02, 0x62, 0xf7, 0x2e, + 0xbc, 0x09, 0xf9, 0x53, 0x3a, 0x32, 0xed, 0xf0, 0xd9, 0x56, 0x34, 0xa2, 0x67, 0x92, 0x5c, 0xfc, + 0x4c, 0xb2, 0xff, 0x5b, 0x09, 0x36, 0x74, 0x67, 0x32, 0x6f, 0xef, 0x3e, 0x9a, 0x7b, 0x05, 0xf0, + 0x3f, 0x96, 0x3e, 0xff, 0x68, 0x64, 0x06, 0xe3, 0xe9, 0x69, 0x5d, 0x77, 0x26, 0xbb, 0x23, 0xc7, + 0x22, 0xf6, 0x68, 0xf6, 0x99, 0x93, 0xff, 0xd1, 0xdf, 0x1d, 0x51, 0xfb, 0xdd, 0x91, 0x93, 0xf8, + 0xe8, 0x79, 0x7f, 0xf6, 0xf7, 0xbf, 0x92, 0xf4, 0xa7, 0x4c, 0xf6, 0xa0, 0xb7, 0xff, 0xe7, 0xcc, + 0xd6, 0x81, 0x18, 0xae, 0x17, 0xb9, 0x47, 0xa5, 0x43, 0x8b, 0xea, 0x6c, 0xca, 0xff, 0x0b, 0x00, + 0x00, 0xff, 0xff, 0x1a, 0x28, 0x25, 0x79, 0x42, 0x1d, 0x00, 0x00, +} diff --git a/vendor/github.com/golang/protobuf/protoc-gen-go/descriptor/descriptor.proto b/vendor/github.com/golang/protobuf/protoc-gen-go/descriptor/descriptor.proto new file mode 100644 index 00000000000..8697a50de4a --- /dev/null +++ b/vendor/github.com/golang/protobuf/protoc-gen-go/descriptor/descriptor.proto @@ -0,0 +1,872 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. +// +// The messages in this file describe the definitions found in .proto files. +// A valid .proto file can be translated directly to a FileDescriptorProto +// without any other information (e.g. without reading its imports). + + +syntax = "proto2"; + +package google.protobuf; +option go_package = "github.com/golang/protobuf/protoc-gen-go/descriptor;descriptor"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DescriptorProtos"; +option csharp_namespace = "Google.Protobuf.Reflection"; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// descriptor.proto must be optimized for speed because reflection-based +// algorithms don't work during bootstrapping. +option optimize_for = SPEED; + +// The protocol compiler can output a FileDescriptorSet containing the .proto +// files it parses. +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; +} + +// Describes a complete .proto file. +message FileDescriptorProto { + optional string name = 1; // file name, relative to root of source tree + optional string package = 2; // e.g. "foo", "foo.bar", etc. + + // Names of files imported by this file. + repeated string dependency = 3; + // Indexes of the public imported files in the dependency list above. + repeated int32 public_dependency = 10; + // Indexes of the weak imported files in the dependency list. + // For Google-internal migration only. Do not use. + repeated int32 weak_dependency = 11; + + // All top-level definitions in this file. + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + + optional FileOptions options = 8; + + // This field contains optional information about the original source code. + // You may safely remove this entire field without harming runtime + // functionality of the descriptors -- the information is needed only by + // development tools. + optional SourceCodeInfo source_code_info = 9; + + // The syntax of the proto file. + // The supported values are "proto2" and "proto3". + optional string syntax = 12; +} + +// Describes a message type. +message DescriptorProto { + optional string name = 1; + + repeated FieldDescriptorProto field = 2; + repeated FieldDescriptorProto extension = 6; + + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + + optional ExtensionRangeOptions options = 3; + } + repeated ExtensionRange extension_range = 5; + + repeated OneofDescriptorProto oneof_decl = 8; + + optional MessageOptions options = 7; + + // Range of reserved tag numbers. Reserved tag numbers may not be used by + // fields or extension ranges in the same message. Reserved ranges may + // not overlap. + message ReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + } + repeated ReservedRange reserved_range = 9; + // Reserved field names, which may not be used by fields in the same message. + // A given name may only be reserved once. + repeated string reserved_name = 10; +} + +message ExtensionRangeOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +// Describes a field within a message. +message FieldDescriptorProto { + enum Type { + // 0 is reserved for errors. + // Order is weird for historical reasons. + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if + // negative values are likely. + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if + // negative values are likely. + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + // Tag-delimited aggregate. + // Group type is deprecated and not supported in proto3. However, Proto3 + // implementations should still be able to parse the group wire format and + // treat group fields as unknown fields. + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; // Length-delimited aggregate. + + // New in version 2. + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; // Uses ZigZag encoding. + TYPE_SINT64 = 18; // Uses ZigZag encoding. + }; + + enum Label { + // 0 is reserved for errors + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + }; + + optional string name = 1; + optional int32 number = 3; + optional Label label = 4; + + // If type_name is set, this need not be set. If both this and type_name + // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. + optional Type type = 5; + + // For message and enum types, this is the name of the type. If the name + // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping + // rules are used to find the type (i.e. first the nested types within this + // message are searched, then within the parent, on up to the root + // namespace). + optional string type_name = 6; + + // For extensions, this is the name of the type being extended. It is + // resolved in the same manner as type_name. + optional string extendee = 2; + + // For numeric types, contains the original text representation of the value. + // For booleans, "true" or "false". + // For strings, contains the default text contents (not escaped in any way). + // For bytes, contains the C escaped value. All bytes >= 128 are escaped. + // TODO(kenton): Base-64 encode? + optional string default_value = 7; + + // If set, gives the index of a oneof in the containing type's oneof_decl + // list. This field is a member of that oneof. + optional int32 oneof_index = 9; + + // JSON name of this field. The value is set by protocol compiler. If the + // user has set a "json_name" option on this field, that option's value + // will be used. Otherwise, it's deduced from the field's name by converting + // it to camelCase. + optional string json_name = 10; + + optional FieldOptions options = 8; +} + +// Describes a oneof. +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} + +// Describes an enum type. +message EnumDescriptorProto { + optional string name = 1; + + repeated EnumValueDescriptorProto value = 2; + + optional EnumOptions options = 3; + + // Range of reserved numeric values. Reserved values may not be used by + // entries in the same enum. Reserved ranges may not overlap. + // + // Note that this is distinct from DescriptorProto.ReservedRange in that it + // is inclusive such that it can appropriately represent the entire int32 + // domain. + message EnumReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Inclusive. + } + + // Range of reserved numeric values. Reserved numeric values may not be used + // by enum values in the same enum declaration. Reserved ranges may not + // overlap. + repeated EnumReservedRange reserved_range = 4; + + // Reserved enum value names, which may not be reused. A given name may only + // be reserved once. + repeated string reserved_name = 5; +} + +// Describes a value within an enum. +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + + optional EnumValueOptions options = 3; +} + +// Describes a service. +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + + optional ServiceOptions options = 3; +} + +// Describes a method of a service. +message MethodDescriptorProto { + optional string name = 1; + + // Input and output type names. These are resolved in the same way as + // FieldDescriptorProto.type_name, but must refer to a message type. + optional string input_type = 2; + optional string output_type = 3; + + optional MethodOptions options = 4; + + // Identifies if client streams multiple client messages + optional bool client_streaming = 5 [default=false]; + // Identifies if server streams multiple server messages + optional bool server_streaming = 6 [default=false]; +} + + +// =================================================================== +// Options + +// Each of the definitions above may have "options" attached. These are +// just annotations which may cause code to be generated slightly differently +// or may contain hints for code that manipulates protocol messages. +// +// Clients may define custom options as extensions of the *Options messages. +// These extensions may not yet be known at parsing time, so the parser cannot +// store the values in them. Instead it stores them in a field in the *Options +// message called uninterpreted_option. This field must have the same name +// across all *Options messages. We then use this field to populate the +// extensions when we build a descriptor, at which point all protos have been +// parsed and so all extensions are known. +// +// Extension numbers for custom options may be chosen as follows: +// * For options which will only be used within a single application or +// organization, or for experimental options, use field numbers 50000 +// through 99999. It is up to you to ensure that you do not use the +// same number for multiple options. +// * For options which will be published and used publicly by multiple +// independent entities, e-mail protobuf-global-extension-registry@google.com +// to reserve extension numbers. Simply provide your project name (e.g. +// Objective-C plugin) and your project website (if available) -- there's no +// need to explain how you intend to use them. Usually you only need one +// extension number. You can declare multiple options with only one extension +// number by putting them in a sub-message. See the Custom Options section of +// the docs for examples: +// https://developers.google.com/protocol-buffers/docs/proto#options +// If this turns out to be popular, a web service will be set up +// to automatically assign option numbers. + + +message FileOptions { + + // Sets the Java package where classes generated from this .proto will be + // placed. By default, the proto package is used, but this is often + // inappropriate because proto packages do not normally start with backwards + // domain names. + optional string java_package = 1; + + + // If set, all the classes from the .proto file are wrapped in a single + // outer class with the given name. This applies to both Proto1 + // (equivalent to the old "--one_java_file" option) and Proto2 (where + // a .proto always translates to a single class, but you may want to + // explicitly choose the class name). + optional string java_outer_classname = 8; + + // If set true, then the Java code generator will generate a separate .java + // file for each top-level message, enum, and service defined in the .proto + // file. Thus, these types will *not* be nested inside the outer class + // named by java_outer_classname. However, the outer class will still be + // generated to contain the file's getDescriptor() method as well as any + // top-level extensions defined in the file. + optional bool java_multiple_files = 10 [default=false]; + + // This option does nothing. + optional bool java_generate_equals_and_hash = 20 [deprecated=true]; + + // If set true, then the Java2 code generator will generate code that + // throws an exception whenever an attempt is made to assign a non-UTF-8 + // byte sequence to a string field. + // Message reflection will do the same. + // However, an extension field still accepts non-UTF-8 byte sequences. + // This option has no effect on when used with the lite runtime. + optional bool java_string_check_utf8 = 27 [default=false]; + + + // Generated classes can be optimized for speed or code size. + enum OptimizeMode { + SPEED = 1; // Generate complete code for parsing, serialization, + // etc. + CODE_SIZE = 2; // Use ReflectionOps to implement these methods. + LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime. + } + optional OptimizeMode optimize_for = 9 [default=SPEED]; + + // Sets the Go package where structs generated from this .proto will be + // placed. If omitted, the Go package will be derived from the following: + // - The basename of the package import path, if provided. + // - Otherwise, the package statement in the .proto file, if present. + // - Otherwise, the basename of the .proto file, without extension. + optional string go_package = 11; + + + + // Should generic services be generated in each language? "Generic" services + // are not specific to any particular RPC system. They are generated by the + // main code generators in each language (without additional plugins). + // Generic services were the only kind of service generation supported by + // early versions of google.protobuf. + // + // Generic services are now considered deprecated in favor of using plugins + // that generate code specific to your particular RPC system. Therefore, + // these default to false. Old code which depends on generic services should + // explicitly set them to true. + optional bool cc_generic_services = 16 [default=false]; + optional bool java_generic_services = 17 [default=false]; + optional bool py_generic_services = 18 [default=false]; + optional bool php_generic_services = 42 [default=false]; + + // Is this file deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for everything in the file, or it will be completely ignored; in the very + // least, this is a formalization for deprecating files. + optional bool deprecated = 23 [default=false]; + + // Enables the use of arenas for the proto messages in this file. This applies + // only to generated classes for C++. + optional bool cc_enable_arenas = 31 [default=false]; + + + // Sets the objective c class prefix which is prepended to all objective c + // generated classes from this .proto. There is no default. + optional string objc_class_prefix = 36; + + // Namespace for generated classes; defaults to the package. + optional string csharp_namespace = 37; + + // By default Swift generators will take the proto package and CamelCase it + // replacing '.' with underscore and use that to prefix the types/symbols + // defined. When this options is provided, they will use this value instead + // to prefix the types/symbols defined. + optional string swift_prefix = 39; + + // Sets the php class prefix which is prepended to all php generated classes + // from this .proto. Default is empty. + optional string php_class_prefix = 40; + + // Use this option to change the namespace of php generated classes. Default + // is empty. When this option is empty, the package name will be used for + // determining the namespace. + optional string php_namespace = 41; + + // The parser stores options it doesn't recognize here. + // See the documentation for the "Options" section above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. + // See the documentation for the "Options" section above. + extensions 1000 to max; + + reserved 38; +} + +message MessageOptions { + // Set true to use the old proto1 MessageSet wire format for extensions. + // This is provided for backwards-compatibility with the MessageSet wire + // format. You should not use this for any other reason: It's less + // efficient, has fewer features, and is more complicated. + // + // The message must be defined exactly as follows: + // message Foo { + // option message_set_wire_format = true; + // extensions 4 to max; + // } + // Note that the message cannot have any defined fields; MessageSets only + // have extensions. + // + // All extensions of your type must be singular messages; e.g. they cannot + // be int32s, enums, or repeated messages. + // + // Because this is an option, the above two restrictions are not enforced by + // the protocol compiler. + optional bool message_set_wire_format = 1 [default=false]; + + // Disables the generation of the standard "descriptor()" accessor, which can + // conflict with a field of the same name. This is meant to make migration + // from proto1 easier; new code should avoid fields named "descriptor". + optional bool no_standard_descriptor_accessor = 2 [default=false]; + + // Is this message deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the message, or it will be completely ignored; in the very least, + // this is a formalization for deprecating messages. + optional bool deprecated = 3 [default=false]; + + // Whether the message is an automatically generated map entry type for the + // maps field. + // + // For maps fields: + // map map_field = 1; + // The parsed descriptor looks like: + // message MapFieldEntry { + // option map_entry = true; + // optional KeyType key = 1; + // optional ValueType value = 2; + // } + // repeated MapFieldEntry map_field = 1; + // + // Implementations may choose not to generate the map_entry=true message, but + // use a native map in the target language to hold the keys and values. + // The reflection APIs in such implementions still need to work as + // if the field is a repeated message field. + // + // NOTE: Do not set the option in .proto files. Always use the maps syntax + // instead. The option should only be implicitly set by the proto compiler + // parser. + optional bool map_entry = 7; + + reserved 8; // javalite_serializable + reserved 9; // javanano_as_lite + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message FieldOptions { + // The ctype option instructs the C++ code generator to use a different + // representation of the field than it normally would. See the specific + // options below. This option is not yet implemented in the open source + // release -- sorry, we'll try to include it in a future version! + optional CType ctype = 1 [default = STRING]; + enum CType { + // Default mode. + STRING = 0; + + CORD = 1; + + STRING_PIECE = 2; + } + // The packed option can be enabled for repeated primitive fields to enable + // a more efficient representation on the wire. Rather than repeatedly + // writing the tag and type for each element, the entire array is encoded as + // a single length-delimited blob. In proto3, only explicit setting it to + // false will avoid using packed encoding. + optional bool packed = 2; + + // The jstype option determines the JavaScript type used for values of the + // field. The option is permitted only for 64 bit integral and fixed types + // (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING + // is represented as JavaScript string, which avoids loss of precision that + // can happen when a large value is converted to a floating point JavaScript. + // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to + // use the JavaScript "number" type. The behavior of the default option + // JS_NORMAL is implementation dependent. + // + // This option is an enum to permit additional types to be added, e.g. + // goog.math.Integer. + optional JSType jstype = 6 [default = JS_NORMAL]; + enum JSType { + // Use the default type. + JS_NORMAL = 0; + + // Use JavaScript strings. + JS_STRING = 1; + + // Use JavaScript numbers. + JS_NUMBER = 2; + } + + // Should this field be parsed lazily? Lazy applies only to message-type + // fields. It means that when the outer message is initially parsed, the + // inner message's contents will not be parsed but instead stored in encoded + // form. The inner message will actually be parsed when it is first accessed. + // + // This is only a hint. Implementations are free to choose whether to use + // eager or lazy parsing regardless of the value of this option. However, + // setting this option true suggests that the protocol author believes that + // using lazy parsing on this field is worth the additional bookkeeping + // overhead typically needed to implement it. + // + // This option does not affect the public interface of any generated code; + // all method signatures remain the same. Furthermore, thread-safety of the + // interface is not affected by this option; const methods remain safe to + // call from multiple threads concurrently, while non-const methods continue + // to require exclusive access. + // + // + // Note that implementations may choose not to check required fields within + // a lazy sub-message. That is, calling IsInitialized() on the outer message + // may return true even if the inner message has missing required fields. + // This is necessary because otherwise the inner message would have to be + // parsed in order to perform the check, defeating the purpose of lazy + // parsing. An implementation which chooses not to check required fields + // must be consistent about it. That is, for any particular sub-message, the + // implementation must either *always* check its required fields, or *never* + // check its required fields, regardless of whether or not the message has + // been parsed. + optional bool lazy = 5 [default=false]; + + // Is this field deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for accessors, or it will be completely ignored; in the very least, this + // is a formalization for deprecating fields. + optional bool deprecated = 3 [default=false]; + + // For Google-internal migration only. Do not use. + optional bool weak = 10 [default=false]; + + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; + + reserved 4; // removed jtype +} + +message OneofOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumOptions { + + // Set this option to true to allow mapping different tag names to the same + // value. + optional bool allow_alias = 2; + + // Is this enum deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum, or it will be completely ignored; in the very least, this + // is a formalization for deprecating enums. + optional bool deprecated = 3 [default=false]; + + reserved 5; // javanano_as_lite + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumValueOptions { + // Is this enum value deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum value, or it will be completely ignored; in the very least, + // this is a formalization for deprecating enum values. + optional bool deprecated = 1 [default=false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message ServiceOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this service deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the service, or it will be completely ignored; in the very least, + // this is a formalization for deprecating services. + optional bool deprecated = 33 [default=false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message MethodOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this method deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the method, or it will be completely ignored; in the very least, + // this is a formalization for deprecating methods. + optional bool deprecated = 33 [default=false]; + + // Is this method side-effect-free (or safe in HTTP parlance), or idempotent, + // or neither? HTTP based RPC implementation may choose GET verb for safe + // methods, and PUT verb for idempotent methods instead of the default POST. + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; // implies idempotent + IDEMPOTENT = 2; // idempotent, but may have side effects + } + optional IdempotencyLevel idempotency_level = + 34 [default=IDEMPOTENCY_UNKNOWN]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + + +// A message representing a option the parser does not recognize. This only +// appears in options protos created by the compiler::Parser class. +// DescriptorPool resolves these when building Descriptor objects. Therefore, +// options protos in descriptor objects (e.g. returned by Descriptor::options(), +// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions +// in them. +message UninterpretedOption { + // The name of the uninterpreted option. Each string represents a segment in + // a dot-separated name. is_extension is true iff a segment represents an + // extension (denoted with parentheses in options specs in .proto files). + // E.g.,{ ["foo", false], ["bar.baz", true], ["qux", false] } represents + // "foo.(bar.baz).qux". + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } + repeated NamePart name = 2; + + // The value of the uninterpreted option, in whatever type the tokenizer + // identified it as during parsing. Exactly one of these should be set. + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; +} + +// =================================================================== +// Optional source code info + +// Encapsulates information about the original source file from which a +// FileDescriptorProto was generated. +message SourceCodeInfo { + // A Location identifies a piece of source code in a .proto file which + // corresponds to a particular definition. This information is intended + // to be useful to IDEs, code indexers, documentation generators, and similar + // tools. + // + // For example, say we have a file like: + // message Foo { + // optional string foo = 1; + // } + // Let's look at just the field definition: + // optional string foo = 1; + // ^ ^^ ^^ ^ ^^^ + // a bc de f ghi + // We have the following locations: + // span path represents + // [a,i) [ 4, 0, 2, 0 ] The whole field definition. + // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). + // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). + // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). + // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). + // + // Notes: + // - A location may refer to a repeated field itself (i.e. not to any + // particular index within it). This is used whenever a set of elements are + // logically enclosed in a single code segment. For example, an entire + // extend block (possibly containing multiple extension definitions) will + // have an outer location whose path refers to the "extensions" repeated + // field without an index. + // - Multiple locations may have the same path. This happens when a single + // logical declaration is spread out across multiple places. The most + // obvious example is the "extend" block again -- there may be multiple + // extend blocks in the same scope, each of which will have the same path. + // - A location's span is not always a subset of its parent's span. For + // example, the "extendee" of an extension declaration appears at the + // beginning of the "extend" block and is shared by all extensions within + // the block. + // - Just because a location's span is a subset of some other location's span + // does not mean that it is a descendent. For example, a "group" defines + // both a type and a field in a single declaration. Thus, the locations + // corresponding to the type and field and their components will overlap. + // - Code which tries to interpret locations should probably be designed to + // ignore those that it doesn't understand, as more types of locations could + // be recorded in the future. + repeated Location location = 1; + message Location { + // Identifies which part of the FileDescriptorProto was defined at this + // location. + // + // Each element is a field number or an index. They form a path from + // the root FileDescriptorProto to the place where the definition. For + // example, this path: + // [ 4, 3, 2, 7, 1 ] + // refers to: + // file.message_type(3) // 4, 3 + // .field(7) // 2, 7 + // .name() // 1 + // This is because FileDescriptorProto.message_type has field number 4: + // repeated DescriptorProto message_type = 4; + // and DescriptorProto.field has field number 2: + // repeated FieldDescriptorProto field = 2; + // and FieldDescriptorProto.name has field number 1: + // optional string name = 1; + // + // Thus, the above path gives the location of a field name. If we removed + // the last element: + // [ 4, 3, 2, 7 ] + // this path refers to the whole field declaration (from the beginning + // of the label to the terminating semicolon). + repeated int32 path = 1 [packed=true]; + + // Always has exactly three or four elements: start line, start column, + // end line (optional, otherwise assumed same as start line), end column. + // These are packed into a single field for efficiency. Note that line + // and column numbers are zero-based -- typically you will want to add + // 1 to each before displaying to a user. + repeated int32 span = 2 [packed=true]; + + // If this SourceCodeInfo represents a complete declaration, these are any + // comments appearing before and after the declaration which appear to be + // attached to the declaration. + // + // A series of line comments appearing on consecutive lines, with no other + // tokens appearing on those lines, will be treated as a single comment. + // + // leading_detached_comments will keep paragraphs of comments that appear + // before (but not connected to) the current element. Each paragraph, + // separated by empty lines, will be one comment element in the repeated + // field. + // + // Only the comment content is provided; comment markers (e.g. //) are + // stripped out. For block comments, leading whitespace and an asterisk + // will be stripped from the beginning of each line other than the first. + // Newlines are included in the output. + // + // Examples: + // + // optional int32 foo = 1; // Comment attached to foo. + // // Comment attached to bar. + // optional int32 bar = 2; + // + // optional string baz = 3; + // // Comment attached to baz. + // // Another line attached to baz. + // + // // Comment attached to qux. + // // + // // Another line attached to qux. + // optional double qux = 4; + // + // // Detached comment for corge. This is not leading or trailing comments + // // to qux or corge because there are blank lines separating it from + // // both. + // + // // Detached comment for corge paragraph 2. + // + // optional string corge = 5; + // /* Block comment attached + // * to corge. Leading asterisks + // * will be removed. */ + // /* Block comment attached to + // * grault. */ + // optional int32 grault = 6; + // + // // ignored detached comments. + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } +} + +// Describes the relationship between generated code and its original source +// file. A GeneratedCodeInfo message is associated with only one generated +// source file, but may contain references to different source .proto files. +message GeneratedCodeInfo { + // An Annotation connects some span of text in generated code to an element + // of its generating .proto file. + repeated Annotation annotation = 1; + message Annotation { + // Identifies the element in the original source .proto file. This field + // is formatted the same as SourceCodeInfo.Location.path. + repeated int32 path = 1 [packed=true]; + + // Identifies the filesystem path to the original source .proto. + optional string source_file = 2; + + // Identifies the starting offset in bytes in the generated code + // that relates to the identified object. + optional int32 begin = 3; + + // Identifies the ending offset in bytes in the generated code that + // relates to the identified offset. The end offset should be one past + // the last relevant byte (so the length of the text = end - begin). + optional int32 end = 4; + } +} diff --git a/vendor/github.com/google/cadvisor/accelerators/BUILD b/vendor/github.com/google/cadvisor/accelerators/BUILD index 9936faecd06..2c802b820f6 100644 --- a/vendor/github.com/google/cadvisor/accelerators/BUILD +++ b/vendor/github.com/google/cadvisor/accelerators/BUILD @@ -10,9 +10,9 @@ go_library( importpath = "github.com/google/cadvisor/accelerators", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/mindprince/gonvml:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/accelerators/nvidia.go b/vendor/github.com/google/cadvisor/accelerators/nvidia.go index 496feba5eed..b0c3c0c5af9 100644 --- a/vendor/github.com/google/cadvisor/accelerators/nvidia.go +++ b/vendor/github.com/google/cadvisor/accelerators/nvidia.go @@ -26,8 +26,8 @@ import ( info "github.com/google/cadvisor/info/v1" - "github.com/golang/glog" "github.com/mindprince/gonvml" + "k8s.io/klog" ) type NvidiaManager struct { @@ -50,7 +50,7 @@ const nvidiaVendorId = "0x10de" // Setup initializes NVML if nvidia devices are present on the node. func (nm *NvidiaManager) Setup() { if !detectDevices(nvidiaVendorId) { - glog.V(4).Info("No NVIDIA devices found.") + klog.V(4).Info("No NVIDIA devices found.") return } @@ -63,7 +63,7 @@ func (nm *NvidiaManager) Setup() { func detectDevices(vendorId string) bool { devices, err := ioutil.ReadDir(sysFsPCIDevicesPath) if err != nil { - glog.Warningf("Error reading %q: %v", sysFsPCIDevicesPath, err) + klog.Warningf("Error reading %q: %v", sysFsPCIDevicesPath, err) return false } @@ -71,11 +71,11 @@ func detectDevices(vendorId string) bool { vendorPath := filepath.Join(sysFsPCIDevicesPath, device.Name(), "vendor") content, err := ioutil.ReadFile(vendorPath) if err != nil { - glog.V(4).Infof("Error while reading %q: %v", vendorPath, err) + klog.V(4).Infof("Error while reading %q: %v", vendorPath, err) continue } if strings.EqualFold(strings.TrimSpace(string(content)), vendorId) { - glog.V(3).Infof("Found device with vendorId %q", vendorId) + klog.V(3).Infof("Found device with vendorId %q", vendorId) return true } } @@ -88,26 +88,26 @@ var initializeNVML = func(nm *NvidiaManager) { if err := gonvml.Initialize(); err != nil { // This is under a logging level because otherwise we may cause // log spam if the drivers/nvml is not installed on the system. - glog.V(4).Infof("Could not initialize NVML: %v", err) + klog.V(4).Infof("Could not initialize NVML: %v", err) return } nm.nvmlInitialized = true numDevices, err := gonvml.DeviceCount() if err != nil { - glog.Warningf("GPU metrics would not be available. Failed to get the number of nvidia devices: %v", err) + klog.Warningf("GPU metrics would not be available. Failed to get the number of nvidia devices: %v", err) return } - glog.V(1).Infof("NVML initialized. Number of nvidia devices: %v", numDevices) + klog.V(1).Infof("NVML initialized. Number of nvidia devices: %v", numDevices) nm.nvidiaDevices = make(map[int]gonvml.Device, numDevices) for i := 0; i < int(numDevices); i++ { device, err := gonvml.DeviceHandleByIndex(uint(i)) if err != nil { - glog.Warningf("Failed to get nvidia device handle %d: %v", i, err) + klog.Warningf("Failed to get nvidia device handle %d: %v", i, err) continue } minorNumber, err := device.MinorNumber() if err != nil { - glog.Warningf("Failed to get nvidia device minor number: %v", err) + klog.Warningf("Failed to get nvidia device minor number: %v", err) continue } nm.nvidiaDevices[int(minorNumber)] = device diff --git a/vendor/github.com/google/cadvisor/cache/memory/BUILD b/vendor/github.com/google/cadvisor/cache/memory/BUILD index ba68ce4cf5b..1d88524fa6f 100644 --- a/vendor/github.com/google/cadvisor/cache/memory/BUILD +++ b/vendor/github.com/google/cadvisor/cache/memory/BUILD @@ -7,10 +7,10 @@ go_library( importpath = "github.com/google/cadvisor/cache/memory", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/storage:go_default_library", "//vendor/github.com/google/cadvisor/utils:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/cache/memory/memory.go b/vendor/github.com/google/cadvisor/cache/memory/memory.go index 4152692e4d9..fbeb2070d84 100644 --- a/vendor/github.com/google/cadvisor/cache/memory/memory.go +++ b/vendor/github.com/google/cadvisor/cache/memory/memory.go @@ -23,13 +23,13 @@ import ( "github.com/google/cadvisor/storage" "github.com/google/cadvisor/utils" - "github.com/golang/glog" + "k8s.io/klog" ) // ErrDataNotFound is the error resulting if failed to find a container in memory cache. var ErrDataNotFound = errors.New("unable to find data in memory cache") -// TODO(vmarmol): See about refactoring this class, we have an unecessary redirection of containerCache and InMemoryCache. +// TODO(vmarmol): See about refactoring this class, we have an unnecessary redirection of containerCache and InMemoryCache. // containerCache is used to store per-container information type containerCache struct { ref info.ContainerReference @@ -91,7 +91,7 @@ func (self *InMemoryCache) AddStats(cInfo *info.ContainerInfo, stats *info.Conta // may want to start a pool of goroutines to do write // operations. if err := self.backend.AddStats(cInfo, stats); err != nil { - glog.Error(err) + klog.Error(err) } } return cstore.AddStats(stats) diff --git a/vendor/github.com/google/cadvisor/container/BUILD b/vendor/github.com/google/cadvisor/container/BUILD index 2627a6c16f4..3f8fc8f74f6 100644 --- a/vendor/github.com/google/cadvisor/container/BUILD +++ b/vendor/github.com/google/cadvisor/container/BUILD @@ -10,9 +10,9 @@ go_library( importpath = "github.com/google/cadvisor/container", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/manager/watcher:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/container/common/BUILD b/vendor/github.com/google/cadvisor/container/common/BUILD index 0d0600913a7..dcfca84dc6e 100644 --- a/vendor/github.com/google/cadvisor/container/common/BUILD +++ b/vendor/github.com/google/cadvisor/container/common/BUILD @@ -12,12 +12,14 @@ go_library( importpath = "github.com/google/cadvisor/container/common", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/container:go_default_library", "//vendor/github.com/google/cadvisor/fs:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/utils:go_default_library", - "//vendor/golang.org/x/exp/inotify:go_default_library", + "//vendor/github.com/karrick/godirwalk:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/github.com/sigma/go-inotify:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/container/common/fsHandler.go b/vendor/github.com/google/cadvisor/container/common/fsHandler.go index d261c983a60..2a14358eb55 100644 --- a/vendor/github.com/google/cadvisor/container/common/fsHandler.go +++ b/vendor/github.com/google/cadvisor/container/common/fsHandler.go @@ -22,7 +22,7 @@ import ( "github.com/google/cadvisor/fs" - "github.com/golang/glog" + "k8s.io/klog" ) type FsHandler interface { @@ -118,7 +118,7 @@ func (fh *realFsHandler) trackUsage() { case <-time.After(fh.period): start := time.Now() if err := fh.update(); err != nil { - glog.Errorf("failed to collect filesystem stats - %v", err) + klog.Errorf("failed to collect filesystem stats - %v", err) fh.period = fh.period * 2 if fh.period > maxBackoffFactor*fh.minPeriod { fh.period = maxBackoffFactor * fh.minPeriod @@ -132,7 +132,7 @@ func (fh *realFsHandler) trackUsage() { // if the long duration is persistent either because of slow // disk or lots of containers. longOp = longOp + time.Second - glog.V(2).Infof("du and find on following dirs took %v: %v; will not log again for this container unless duration exceeds %v", duration, []string{fh.rootfs, fh.extraDir}, longOp) + klog.V(2).Infof("du and find on following dirs took %v: %v; will not log again for this container unless duration exceeds %v", duration, []string{fh.rootfs, fh.extraDir}, longOp) } } } diff --git a/vendor/github.com/google/cadvisor/container/common/helpers.go b/vendor/github.com/google/cadvisor/container/common/helpers.go index f5539b5d1ac..d38777f93bf 100644 --- a/vendor/github.com/google/cadvisor/container/common/helpers.go +++ b/vendor/github.com/google/cadvisor/container/common/helpers.go @@ -26,8 +26,10 @@ import ( "github.com/google/cadvisor/container" info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/utils" + "github.com/karrick/godirwalk" + "github.com/pkg/errors" - "github.com/golang/glog" + "k8s.io/klog" ) func DebugInfo(watches map[string][]string) map[string][]string { @@ -85,7 +87,7 @@ func GetSpec(cgroupPaths map[string]string, machineInfoFactory info.MachineInfoF if quota != "" && quota != "-1" { val, err := strconv.ParseUint(quota, 10, 64) if err != nil { - glog.Errorf("GetSpec: Failed to parse CPUQuota from %q: %s", path.Join(cpuRoot, "cpu.cfs_quota_us"), err) + klog.Errorf("GetSpec: Failed to parse CPUQuota from %q: %s", path.Join(cpuRoot, "cpu.cfs_quota_us"), err) } spec.Cpu.Quota = val } @@ -132,7 +134,7 @@ func readString(dirpath string, file string) string { if err != nil { // Ignore non-existent files if !os.IsNotExist(err) { - glog.Errorf("readString: Failed to read %q: %s", cgroupFile, err) + klog.Errorf("readString: Failed to read %q: %s", cgroupFile, err) } return "" } @@ -147,7 +149,7 @@ func readUInt64(dirpath string, file string) uint64 { val, err := strconv.ParseUint(out, 10, 64) if err != nil { - glog.Errorf("readUInt64: Failed to parse int %q from file %q: %s", out, path.Join(dirpath, file), err) + klog.Errorf("readUInt64: Failed to parse int %q from file %q: %s", out, path.Join(dirpath, file), err) return 0 } @@ -156,26 +158,34 @@ func readUInt64(dirpath string, file string) uint64 { // Lists all directories under "path" and outputs the results as children of "parent". func ListDirectories(dirpath string, parent string, recursive bool, output map[string]struct{}) error { - entries, err := ioutil.ReadDir(dirpath) + buf := make([]byte, godirwalk.DefaultScratchBufferSize) + return listDirectories(dirpath, parent, recursive, output, buf) +} + +func listDirectories(dirpath string, parent string, recursive bool, output map[string]struct{}, buf []byte) error { + dirents, err := godirwalk.ReadDirents(dirpath, buf) if err != nil { // Ignore if this hierarchy does not exist. - if os.IsNotExist(err) { + if os.IsNotExist(errors.Cause(err)) { err = nil } return err } - for _, entry := range entries { + for _, dirent := range dirents { // We only grab directories. - if entry.IsDir() { - name := path.Join(parent, entry.Name()) - output[name] = struct{}{} + if !dirent.IsDir() { + continue + } + dirname := dirent.Name() - // List subcontainers if asked to. - if recursive { - err := ListDirectories(path.Join(dirpath, entry.Name()), name, true, output) - if err != nil { - return err - } + name := path.Join(parent, dirname) + output[name] = struct{}{} + + // List subcontainers if asked to. + if recursive { + err := listDirectories(path.Join(dirpath, dirname), name, true, output, buf) + if err != nil { + return err } } } diff --git a/vendor/github.com/google/cadvisor/container/common/inotify_watcher.go b/vendor/github.com/google/cadvisor/container/common/inotify_watcher.go index 16e2f2c9b0f..787f599a22b 100644 --- a/vendor/github.com/google/cadvisor/container/common/inotify_watcher.go +++ b/vendor/github.com/google/cadvisor/container/common/inotify_watcher.go @@ -17,7 +17,7 @@ package common import ( "sync" - "golang.org/x/exp/inotify" + inotify "github.com/sigma/go-inotify" ) // Watcher for container-related inotify events in the cgroup hierarchy. @@ -78,7 +78,7 @@ func (iw *InotifyWatcher) RemoveWatch(containerName, dir string) (bool, error) { iw.lock.Lock() defer iw.lock.Unlock() - // If we don't have a watch registed for this, just return. + // If we don't have a watch registered for this, just return. cgroupsWatched, ok := iw.containersWatched[containerName] if !ok { return false, nil diff --git a/vendor/github.com/google/cadvisor/container/containerd/BUILD b/vendor/github.com/google/cadvisor/container/containerd/BUILD index c89f813ef27..01685971866 100644 --- a/vendor/github.com/google/cadvisor/container/containerd/BUILD +++ b/vendor/github.com/google/cadvisor/container/containerd/BUILD @@ -20,7 +20,6 @@ go_library( "//vendor/github.com/containerd/containerd/errdefs:go_default_library", "//vendor/github.com/containerd/containerd/namespaces:go_default_library", "//vendor/github.com/gogo/protobuf/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/container:go_default_library", "//vendor/github.com/google/cadvisor/container/common:go_default_library", "//vendor/github.com/google/cadvisor/container/libcontainer:go_default_library", @@ -32,6 +31,7 @@ go_library( "//vendor/github.com/opencontainers/runtime-spec/specs-go:go_default_library", "//vendor/golang.org/x/net/context:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/container/containerd/factory.go b/vendor/github.com/google/cadvisor/container/containerd/factory.go index 18d6070e6d4..2e2bb33346a 100644 --- a/vendor/github.com/google/cadvisor/container/containerd/factory.go +++ b/vendor/github.com/google/cadvisor/container/containerd/factory.go @@ -21,8 +21,8 @@ import ( "regexp" "strings" - "github.com/golang/glog" "golang.org/x/net/context" + "k8s.io/klog" "github.com/google/cadvisor/container" "github.com/google/cadvisor/container/libcontainer" @@ -133,7 +133,7 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics return fmt.Errorf("failed to get cgroup subsystems: %v", err) } - glog.V(1).Infof("Registering containerd factory") + klog.V(1).Infof("Registering containerd factory") f := &containerdFactory{ cgroupSubsystems: cgroupSubsystems, client: client, diff --git a/vendor/github.com/google/cadvisor/container/crio/BUILD b/vendor/github.com/google/cadvisor/container/crio/BUILD index aa78a582d9a..a216be2c112 100644 --- a/vendor/github.com/google/cadvisor/container/crio/BUILD +++ b/vendor/github.com/google/cadvisor/container/crio/BUILD @@ -11,7 +11,6 @@ go_library( importpath = "github.com/google/cadvisor/container/crio", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/container:go_default_library", "//vendor/github.com/google/cadvisor/container/common:go_default_library", "//vendor/github.com/google/cadvisor/container/libcontainer:go_default_library", @@ -20,6 +19,7 @@ go_library( "//vendor/github.com/google/cadvisor/manager/watcher:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/configs:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/container/crio/factory.go b/vendor/github.com/google/cadvisor/container/crio/factory.go index e106c0ba363..510b66f91c5 100644 --- a/vendor/github.com/google/cadvisor/container/crio/factory.go +++ b/vendor/github.com/google/cadvisor/container/crio/factory.go @@ -26,7 +26,7 @@ import ( info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/manager/watcher" - "github.com/golang/glog" + "k8s.io/klog" ) // The namespace under which crio aliases are unique. @@ -154,7 +154,7 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics return fmt.Errorf("failed to get cgroup subsystems: %v", err) } - glog.V(1).Infof("Registering CRI-O factory") + klog.V(1).Infof("Registering CRI-O factory") f := &crioFactory{ client: client, cgroupSubsystems: cgroupSubsystems, diff --git a/vendor/github.com/google/cadvisor/container/crio/handler.go b/vendor/github.com/google/cadvisor/container/crio/handler.go index b9748b8ccf3..d17ba6d932f 100644 --- a/vendor/github.com/google/cadvisor/container/crio/handler.go +++ b/vendor/github.com/google/cadvisor/container/crio/handler.go @@ -176,7 +176,7 @@ func newCrioContainerHandler( } // TODO for env vars we wanted to show from container.Config.Env from whitelist //for _, exposedEnv := range metadataEnvs { - //glog.V(4).Infof("TODO env whitelist: %v", exposedEnv) + //klog.V(4).Infof("TODO env whitelist: %v", exposedEnv) //} return handler, nil diff --git a/vendor/github.com/google/cadvisor/container/docker/BUILD b/vendor/github.com/google/cadvisor/container/docker/BUILD index 4753ea5a77f..030b14a9657 100644 --- a/vendor/github.com/google/cadvisor/container/docker/BUILD +++ b/vendor/github.com/google/cadvisor/container/docker/BUILD @@ -17,7 +17,6 @@ go_library( "//vendor/github.com/docker/docker/api/types/container:go_default_library", "//vendor/github.com/docker/docker/client:go_default_library", "//vendor/github.com/docker/go-connections/tlsconfig:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/container:go_default_library", "//vendor/github.com/google/cadvisor/container/common:go_default_library", "//vendor/github.com/google/cadvisor/container/libcontainer:go_default_library", @@ -31,6 +30,7 @@ go_library( "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/configs:go_default_library", "//vendor/golang.org/x/net/context:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/container/docker/factory.go b/vendor/github.com/google/cadvisor/container/docker/factory.go index 9f544ee8cc5..7502781c43c 100644 --- a/vendor/github.com/google/cadvisor/container/docker/factory.go +++ b/vendor/github.com/google/cadvisor/container/docker/factory.go @@ -36,8 +36,8 @@ import ( "github.com/google/cadvisor/zfs" docker "github.com/docker/docker/client" - "github.com/golang/glog" "golang.org/x/net/context" + "k8s.io/klog" ) var ArgDockerEndpoint = flag.String("docker", "unix:///var/run/docker.sock", "docker endpoint") @@ -337,7 +337,7 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics if storageDriver(dockerInfo.Driver) == devicemapperStorageDriver { thinPoolWatcher, err = startThinPoolWatcher(dockerInfo) if err != nil { - glog.Errorf("devicemapper filesystem stats will not be reported: %v", err) + klog.Errorf("devicemapper filesystem stats will not be reported: %v", err) } // Safe to ignore error - driver status should always be populated. @@ -349,11 +349,11 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics if storageDriver(dockerInfo.Driver) == zfsStorageDriver { zfsWatcher, err = startZfsWatcher(dockerInfo) if err != nil { - glog.Errorf("zfs filesystem stats will not be reported: %v", err) + klog.Errorf("zfs filesystem stats will not be reported: %v", err) } } - glog.V(1).Infof("Registering Docker factory") + klog.V(1).Infof("Registering Docker factory") f := &dockerFactory{ cgroupSubsystems: cgroupSubsystems, client: client, diff --git a/vendor/github.com/google/cadvisor/container/docker/handler.go b/vendor/github.com/google/cadvisor/container/docker/handler.go index 69f84faf47e..638350c0929 100644 --- a/vendor/github.com/google/cadvisor/container/docker/handler.go +++ b/vendor/github.com/google/cadvisor/container/docker/handler.go @@ -34,10 +34,10 @@ import ( dockercontainer "github.com/docker/docker/api/types/container" docker "github.com/docker/docker/client" - "github.com/golang/glog" cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs" libcontainerconfigs "github.com/opencontainers/runc/libcontainer/configs" "golang.org/x/net/context" + "k8s.io/klog" ) const ( @@ -228,7 +228,7 @@ func newDockerContainerHandler( handler.labels["restartcount"] = strconv.Itoa(ctnr.RestartCount) } - // Obtain the IP address for the contianer. + // Obtain the IP address for the container. // If the NetworkMode starts with 'container:' then we need to use the IP address of the container specified. // This happens in cases such as kubernetes where the containers doesn't have an IP address itself and we need to use the pod's address ipAddress := ctnr.NetworkSettings.IPAddress @@ -309,7 +309,7 @@ func (h *dockerFsHandler) Usage() common.FsUsage { // TODO: ideally we should keep track of how many times we failed to get the usage for this // device vs how many refreshes of the cache there have been, and display an error e.g. if we've // had at least 1 refresh and we still can't find the device. - glog.V(5).Infof("unable to get fs usage from thin pool for device %s: %v", h.deviceID, err) + klog.V(5).Infof("unable to get fs usage from thin pool for device %s: %v", h.deviceID, err) } else { usage.BaseUsageBytes = thinPoolUsage usage.TotalUsageBytes += thinPoolUsage @@ -319,7 +319,7 @@ func (h *dockerFsHandler) Usage() common.FsUsage { if h.zfsWatcher != nil { zfsUsage, err := h.zfsWatcher.GetUsage(h.zfsFilesystem) if err != nil { - glog.V(5).Infof("unable to get fs usage from zfs for filesystem %s: %v", h.zfsFilesystem, err) + klog.V(5).Infof("unable to get fs usage from zfs for filesystem %s: %v", h.zfsFilesystem, err) } else { usage.BaseUsageBytes = zfsUsage usage.TotalUsageBytes += zfsUsage diff --git a/vendor/github.com/google/cadvisor/container/factory.go b/vendor/github.com/google/cadvisor/container/factory.go index 47847057e03..ae03960ea09 100644 --- a/vendor/github.com/google/cadvisor/container/factory.go +++ b/vendor/github.com/google/cadvisor/container/factory.go @@ -20,7 +20,7 @@ import ( "github.com/google/cadvisor/manager/watcher" - "github.com/golang/glog" + "k8s.io/klog" ) type ContainerHandlerFactory interface { @@ -53,6 +53,7 @@ const ( NetworkUdpUsageMetrics MetricKind = "udp" AcceleratorUsageMetrics MetricKind = "accelerator" AppMetrics MetricKind = "app" + ProcessMetrics MetricKind = "process" ) func (mk MetricKind) String() string { @@ -105,18 +106,18 @@ func NewContainerHandler(name string, watchType watcher.ContainerWatchSource, in for _, factory := range factories[watchType] { canHandle, canAccept, err := factory.CanHandleAndAccept(name) if err != nil { - glog.V(4).Infof("Error trying to work out if we can handle %s: %v", name, err) + klog.V(4).Infof("Error trying to work out if we can handle %s: %v", name, err) } if canHandle { if !canAccept { - glog.V(3).Infof("Factory %q can handle container %q, but ignoring.", factory, name) + klog.V(3).Infof("Factory %q can handle container %q, but ignoring.", factory, name) return nil, false, nil } - glog.V(3).Infof("Using factory %q for container %q", factory, name) + klog.V(3).Infof("Using factory %q for container %q", factory, name) handle, err := factory.NewContainerHandler(name, inHostNamespace) return handle, canAccept, err } else { - glog.V(4).Infof("Factory %q was unable to handle container %q", factory, name) + klog.V(4).Infof("Factory %q was unable to handle container %q", factory, name) } } diff --git a/vendor/github.com/google/cadvisor/container/libcontainer/BUILD b/vendor/github.com/google/cadvisor/container/libcontainer/BUILD index a4a7ffab76c..bd6bb9783c4 100644 --- a/vendor/github.com/google/cadvisor/container/libcontainer/BUILD +++ b/vendor/github.com/google/cadvisor/container/libcontainer/BUILD @@ -11,11 +11,11 @@ go_library( importpath = "github.com/google/cadvisor/container/libcontainer", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/container:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/container/libcontainer/handler.go b/vendor/github.com/google/cadvisor/container/libcontainer/handler.go index 18c465f2200..5b44fb7437b 100644 --- a/vendor/github.com/google/cadvisor/container/libcontainer/handler.go +++ b/vendor/github.com/google/cadvisor/container/libcontainer/handler.go @@ -29,9 +29,9 @@ import ( info "github.com/google/cadvisor/info/v1" "bytes" - "github.com/golang/glog" "github.com/opencontainers/runc/libcontainer" "github.com/opencontainers/runc/libcontainer/cgroups" + "k8s.io/klog" ) /* @@ -72,11 +72,11 @@ func (h *Handler) GetStats() (*info.ContainerStats, error) { if h.includedMetrics.Has(container.ProcessSchedulerMetrics) { pids, err := h.cgroupManager.GetAllPids() if err != nil { - glog.V(4).Infof("Could not get PIDs for container %d: %v", h.pid, err) + klog.V(4).Infof("Could not get PIDs for container %d: %v", h.pid, err) } else { stats.Cpu.Schedstat, err = schedulerStatsFromProcs(h.rootFs, pids, h.pidMetricsCache) if err != nil { - glog.V(4).Infof("Unable to get Process Scheduler Stats: %v", err) + klog.V(4).Infof("Unable to get Process Scheduler Stats: %v", err) } } } @@ -88,7 +88,7 @@ func (h *Handler) GetStats() (*info.ContainerStats, error) { if h.includedMetrics.Has(container.NetworkUsageMetrics) { netStats, err := networkStatsFromProc(h.rootFs, h.pid) if err != nil { - glog.V(4).Infof("Unable to get network stats from pid %d: %v", h.pid, err) + klog.V(4).Infof("Unable to get network stats from pid %d: %v", h.pid, err) } else { stats.Network.Interfaces = append(stats.Network.Interfaces, netStats...) } @@ -96,14 +96,14 @@ func (h *Handler) GetStats() (*info.ContainerStats, error) { if h.includedMetrics.Has(container.NetworkTcpUsageMetrics) { t, err := tcpStatsFromProc(h.rootFs, h.pid, "net/tcp") if err != nil { - glog.V(4).Infof("Unable to get tcp stats from pid %d: %v", h.pid, err) + klog.V(4).Infof("Unable to get tcp stats from pid %d: %v", h.pid, err) } else { stats.Network.Tcp = t } t6, err := tcpStatsFromProc(h.rootFs, h.pid, "net/tcp6") if err != nil { - glog.V(4).Infof("Unable to get tcp6 stats from pid %d: %v", h.pid, err) + klog.V(4).Infof("Unable to get tcp6 stats from pid %d: %v", h.pid, err) } else { stats.Network.Tcp6 = t6 } @@ -111,18 +111,30 @@ func (h *Handler) GetStats() (*info.ContainerStats, error) { if h.includedMetrics.Has(container.NetworkUdpUsageMetrics) { u, err := udpStatsFromProc(h.rootFs, h.pid, "net/udp") if err != nil { - glog.V(4).Infof("Unable to get udp stats from pid %d: %v", h.pid, err) + klog.V(4).Infof("Unable to get udp stats from pid %d: %v", h.pid, err) } else { stats.Network.Udp = u } u6, err := udpStatsFromProc(h.rootFs, h.pid, "net/udp6") if err != nil { - glog.V(4).Infof("Unable to get udp6 stats from pid %d: %v", h.pid, err) + klog.V(4).Infof("Unable to get udp6 stats from pid %d: %v", h.pid, err) } else { stats.Network.Udp6 = u6 } } + if h.includedMetrics.Has(container.ProcessMetrics) { + paths := h.cgroupManager.GetPaths() + path, ok := paths["cpu"] + if !ok { + klog.V(4).Infof("Could not find cgroups CPU for container %d", h.pid) + } else { + stats.Processes, err = processStatsFromProcs(h.rootFs, path) + if err != nil { + klog.V(4).Infof("Unable to get Process Stats: %v", err) + } + } + } // For backwards compatibility. if len(stats.Network.Interfaces) > 0 { @@ -132,6 +144,41 @@ func (h *Handler) GetStats() (*info.ContainerStats, error) { return stats, nil } +func processStatsFromProcs(rootFs string, cgroupPath string) (info.ProcessStats, error) { + var fdCount uint64 + filePath := path.Join(cgroupPath, "cgroup.procs") + out, err := ioutil.ReadFile(filePath) + if err != nil { + return info.ProcessStats{}, fmt.Errorf("couldn't open cpu cgroup procs file %v : %v", filePath, err) + } + + pids := strings.Split(string(out), "\n") + + // EOL is also treated as a new line while reading "cgroup.procs" file with ioutil.ReadFile. + // The last value is an empty string "". Ex: pids = ["22", "1223", ""] + // Trim the last value + if len(pids) != 0 && pids[len(pids)-1] == "" { + pids = pids[:len(pids)-1] + } + + for _, pid := range pids { + dirPath := path.Join(rootFs, "/proc", pid, "fd") + fds, err := ioutil.ReadDir(dirPath) + if err != nil { + klog.V(4).Infof("error while listing directory %q to measure fd count: %v", dirPath, err) + continue + } + fdCount += uint64(len(fds)) + } + + processStats := info.ProcessStats{ + ProcessCount: uint64(len(pids)), + FdCount: fdCount, + } + + return processStats, nil +} + func schedulerStatsFromProcs(rootFs string, pids []int, pidMetricsCache map[int]*info.CpuSchedstat) (info.CpuSchedstat, error) { for _, pid := range pids { f, err := os.Open(path.Join(rootFs, "proc", strconv.Itoa(pid), "schedstat")) @@ -451,13 +498,13 @@ func setCpuStats(s *cgroups.Stats, ret *info.ContainerStats, withPerCPU bool) { // We intentionally ignore these extra zeroes. numActual, err := numCpusFunc() if err != nil { - glog.Errorf("unable to determine number of actual cpus; defaulting to maximum possible number: errno %v", err) + klog.Errorf("unable to determine number of actual cpus; defaulting to maximum possible number: errno %v", err) numActual = numPossible } if numActual > numPossible { // The real number of cores should never be greater than the number of // datapoints reported in cpu usage. - glog.Errorf("PercpuUsage had %v cpus, but the actual number is %v; ignoring extra CPUs", numPossible, numActual) + klog.Errorf("PercpuUsage had %v cpus, but the actual number is %v; ignoring extra CPUs", numPossible, numActual) } numActual = minUint32(numPossible, numActual) ret.Cpu.Usage.PerCpu = make([]uint64, numActual) diff --git a/vendor/github.com/google/cadvisor/container/libcontainer/helpers.go b/vendor/github.com/google/cadvisor/container/libcontainer/helpers.go index d033f7bfc50..05554465caa 100644 --- a/vendor/github.com/google/cadvisor/container/libcontainer/helpers.go +++ b/vendor/github.com/google/cadvisor/container/libcontainer/helpers.go @@ -19,8 +19,8 @@ import ( info "github.com/google/cadvisor/info/v1" - "github.com/golang/glog" "github.com/opencontainers/runc/libcontainer/cgroups" + "k8s.io/klog" ) type CgroupSubsystems struct { @@ -61,7 +61,7 @@ func getCgroupSubsystemsHelper(allCgroups []cgroups.Mount) (CgroupSubsystems, er } if _, ok := mountPoints[subsystem]; ok { // duplicate mount for this subsystem; use the first one we saw - glog.V(5).Infof("skipping %s, already using mount at %s", mount.Mountpoint, mountPoints[subsystem]) + klog.V(5).Infof("skipping %s, already using mount at %s", mount.Mountpoint, mountPoints[subsystem]) continue } if _, ok := recordedMountpoints[mount.Mountpoint]; !ok { diff --git a/vendor/github.com/google/cadvisor/container/mesos/BUILD b/vendor/github.com/google/cadvisor/container/mesos/BUILD index 9ee0d41cd82..09c97edf1c7 100644 --- a/vendor/github.com/google/cadvisor/container/mesos/BUILD +++ b/vendor/github.com/google/cadvisor/container/mesos/BUILD @@ -14,7 +14,6 @@ go_library( deps = [ "//vendor/github.com/Rican7/retry:go_default_library", "//vendor/github.com/Rican7/retry/strategy:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/container:go_default_library", "//vendor/github.com/google/cadvisor/container/common:go_default_library", "//vendor/github.com/google/cadvisor/container/libcontainer:go_default_library", @@ -29,6 +28,7 @@ go_library( "//vendor/github.com/mesos/mesos-go/api/v1/lib/httpcli:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/configs:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/container/mesos/client.go b/vendor/github.com/google/cadvisor/container/mesos/client.go index ce0c8ef16d9..fa1beb90b41 100644 --- a/vendor/github.com/google/cadvisor/container/mesos/client.go +++ b/vendor/github.com/google/cadvisor/container/mesos/client.go @@ -70,6 +70,11 @@ func Client() (mesosAgentClient, error) { ), } }) + + _, err := mesosClient.getVersion() + if err != nil { + return nil, fmt.Errorf("failed to get version") + } return mesosClient, nil } @@ -134,6 +139,20 @@ func (self *client) getContainer(id string) (*mContainer, error) { return nil, fmt.Errorf("can't locate container %s", id) } +func (self *client) getVersion() (string, error) { + req := calls.NonStreaming(calls.GetVersion()) + result, err := self.fetchAndDecode(req) + if err != nil { + return "", fmt.Errorf("failed to get mesos version: %v", err) + } + version := result.GetVersion + + if version == nil { + return "", fmt.Errorf("failed to get mesos version") + } + return version.VersionInfo.Version, nil +} + func (self *client) getContainers() (mContainers, error) { req := calls.NonStreaming(calls.GetContainers()) result, err := self.fetchAndDecode(req) @@ -141,6 +160,10 @@ func (self *client) getContainers() (mContainers, error) { return nil, fmt.Errorf("failed to get mesos containers: %v", err) } cntrs := result.GetContainers + + if cntrs == nil { + return nil, fmt.Errorf("failed to get mesos containers") + } return cntrs, nil } diff --git a/vendor/github.com/google/cadvisor/container/mesos/factory.go b/vendor/github.com/google/cadvisor/container/mesos/factory.go index 66cdfe830c3..b2c61bb3fa9 100644 --- a/vendor/github.com/google/cadvisor/container/mesos/factory.go +++ b/vendor/github.com/google/cadvisor/container/mesos/factory.go @@ -22,12 +22,12 @@ import ( "strings" "time" - "github.com/golang/glog" "github.com/google/cadvisor/container" "github.com/google/cadvisor/container/libcontainer" "github.com/google/cadvisor/fs" info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/manager/watcher" + "k8s.io/klog" ) var MesosAgentAddress = flag.String("mesos_agent", "127.0.0.1:5051", "Mesos agent address") @@ -135,7 +135,7 @@ func Register( return fmt.Errorf("failed to get cgroup subsystems: %v", err) } - glog.V(1).Infof("Registering mesos factory") + klog.V(1).Infof("Registering mesos factory") factory := &mesosFactory{ machineInfoFactory: machineInfoFactory, cgroupSubsystems: cgroupSubsystems, diff --git a/vendor/github.com/google/cadvisor/container/raw/BUILD b/vendor/github.com/google/cadvisor/container/raw/BUILD index f5db9fcc5c9..f7c87c12dc8 100644 --- a/vendor/github.com/google/cadvisor/container/raw/BUILD +++ b/vendor/github.com/google/cadvisor/container/raw/BUILD @@ -10,7 +10,6 @@ go_library( importpath = "github.com/google/cadvisor/container/raw", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/container:go_default_library", "//vendor/github.com/google/cadvisor/container/common:go_default_library", "//vendor/github.com/google/cadvisor/container/libcontainer:go_default_library", @@ -20,6 +19,7 @@ go_library( "//vendor/github.com/google/cadvisor/manager/watcher:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/configs:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/container/raw/factory.go b/vendor/github.com/google/cadvisor/container/raw/factory.go index dd277c32317..afc852e345b 100644 --- a/vendor/github.com/google/cadvisor/container/raw/factory.go +++ b/vendor/github.com/google/cadvisor/container/raw/factory.go @@ -26,7 +26,7 @@ import ( info "github.com/google/cadvisor/info/v1" watch "github.com/google/cadvisor/manager/watcher" - "github.com/golang/glog" + "k8s.io/klog" ) var dockerOnly = flag.Bool("docker_only", false, "Only report docker containers in addition to root stats") @@ -94,7 +94,7 @@ func Register(machineInfoFactory info.MachineInfoFactory, fsInfo fs.FsInfo, incl return err } - glog.V(1).Infof("Registering Raw factory") + klog.V(1).Infof("Registering Raw factory") factory := &rawFactory{ machineInfoFactory: machineInfoFactory, fsInfo: fsInfo, diff --git a/vendor/github.com/google/cadvisor/container/raw/handler.go b/vendor/github.com/google/cadvisor/container/raw/handler.go index 0a3f2a9c550..7e7504d933b 100644 --- a/vendor/github.com/google/cadvisor/container/raw/handler.go +++ b/vendor/github.com/google/cadvisor/container/raw/handler.go @@ -25,9 +25,9 @@ import ( info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/machine" - "github.com/golang/glog" cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs" "github.com/opencontainers/runc/libcontainer/configs" + "k8s.io/klog" ) type rawContainerHandler struct { @@ -134,7 +134,7 @@ func (self *rawContainerHandler) GetSpec() (info.ContainerSpec, error) { // Get memory and swap limits of the running machine memLimit, err := machine.GetMachineMemoryCapacity() if err != nil { - glog.Warningf("failed to obtain memory limit for machine container") + klog.Warningf("failed to obtain memory limit for machine container") spec.HasMemory = false } else { spec.Memory.Limit = uint64(memLimit) @@ -144,7 +144,7 @@ func (self *rawContainerHandler) GetSpec() (info.ContainerSpec, error) { swapLimit, err := machine.GetMachineSwapCapacity() if err != nil { - glog.Warningf("failed to obtain swap limit for machine container") + klog.Warningf("failed to obtain swap limit for machine container") } else { spec.Memory.SwapLimit = uint64(swapLimit) } diff --git a/vendor/github.com/google/cadvisor/container/rkt/BUILD b/vendor/github.com/google/cadvisor/container/rkt/BUILD index 8dfe0c1e0a2..3875c8f676e 100644 --- a/vendor/github.com/google/cadvisor/container/rkt/BUILD +++ b/vendor/github.com/google/cadvisor/container/rkt/BUILD @@ -14,7 +14,6 @@ go_library( deps = [ "//vendor/github.com/blang/semver:go_default_library", "//vendor/github.com/coreos/rkt/api/v1alpha:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/container:go_default_library", "//vendor/github.com/google/cadvisor/container/common:go_default_library", "//vendor/github.com/google/cadvisor/container/libcontainer:go_default_library", @@ -25,6 +24,7 @@ go_library( "//vendor/github.com/opencontainers/runc/libcontainer/configs:go_default_library", "//vendor/golang.org/x/net/context:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/container/rkt/factory.go b/vendor/github.com/google/cadvisor/container/rkt/factory.go index e80cbd554ac..cf806032423 100644 --- a/vendor/github.com/google/cadvisor/container/rkt/factory.go +++ b/vendor/github.com/google/cadvisor/container/rkt/factory.go @@ -23,7 +23,7 @@ import ( info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/manager/watcher" - "github.com/golang/glog" + "k8s.io/klog" ) const RktNamespace = "rkt" @@ -86,7 +86,7 @@ func Register(machineInfoFactory info.MachineInfoFactory, fsInfo fs.FsInfo, incl return fmt.Errorf("failed to find supported cgroup mounts for the raw factory") } - glog.V(1).Infof("Registering Rkt factory") + klog.V(1).Infof("Registering Rkt factory") factory := &rktFactory{ machineInfoFactory: machineInfoFactory, fsInfo: fsInfo, diff --git a/vendor/github.com/google/cadvisor/container/rkt/handler.go b/vendor/github.com/google/cadvisor/container/rkt/handler.go index afd48b4de2e..1421e6ceeab 100644 --- a/vendor/github.com/google/cadvisor/container/rkt/handler.go +++ b/vendor/github.com/google/cadvisor/container/rkt/handler.go @@ -27,9 +27,9 @@ import ( info "github.com/google/cadvisor/info/v1" "golang.org/x/net/context" - "github.com/golang/glog" cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs" "github.com/opencontainers/runc/libcontainer/configs" + "k8s.io/klog" ) type rktContainerHandler struct { @@ -89,7 +89,7 @@ func newRktContainerHandler(name string, rktClient rktapi.PublicAPIClient, rktPa annotations := resp.Pod.Annotations if parsed.Container != "" { // As not empty string, an App container if contAnnotations, ok := findAnnotations(resp.Pod.Apps, parsed.Container); !ok { - glog.Warningf("couldn't find app %v in pod", parsed.Container) + klog.Warningf("couldn't find app %v in pod", parsed.Container) } else { annotations = append(annotations, contAnnotations...) } diff --git a/vendor/github.com/google/cadvisor/container/rkt/helpers.go b/vendor/github.com/google/cadvisor/container/rkt/helpers.go index a4fb8da829a..94bbdecc407 100644 --- a/vendor/github.com/google/cadvisor/container/rkt/helpers.go +++ b/vendor/github.com/google/cadvisor/container/rkt/helpers.go @@ -21,8 +21,8 @@ import ( "strings" rktapi "github.com/coreos/rkt/api/v1alpha" - "github.com/golang/glog" "golang.org/x/net/context" + "k8s.io/klog" ) type parsedName struct { @@ -128,7 +128,7 @@ func getRootFs(root string, parsed *parsedName) string { bytes, err := ioutil.ReadFile(tree) if err != nil { - glog.Errorf("ReadFile failed, couldn't read %v to get upper dir: %v", tree, err) + klog.Errorf("ReadFile failed, couldn't read %v to get upper dir: %v", tree, err) return "" } diff --git a/vendor/github.com/google/cadvisor/container/systemd/BUILD b/vendor/github.com/google/cadvisor/container/systemd/BUILD index 640a4b7d512..1b8a35ac9a6 100644 --- a/vendor/github.com/google/cadvisor/container/systemd/BUILD +++ b/vendor/github.com/google/cadvisor/container/systemd/BUILD @@ -7,11 +7,11 @@ go_library( importpath = "github.com/google/cadvisor/container/systemd", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/container:go_default_library", "//vendor/github.com/google/cadvisor/fs:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/manager/watcher:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/container/systemd/factory.go b/vendor/github.com/google/cadvisor/container/systemd/factory.go index dfe751a4d2f..100c79e1390 100644 --- a/vendor/github.com/google/cadvisor/container/systemd/factory.go +++ b/vendor/github.com/google/cadvisor/container/systemd/factory.go @@ -23,7 +23,7 @@ import ( info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/manager/watcher" - "github.com/golang/glog" + "k8s.io/klog" ) type systemdFactory struct{} @@ -51,7 +51,7 @@ func (f *systemdFactory) DebugInfo() map[string][]string { // Register registers the systemd container factory. func Register(machineInfoFactory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics container.MetricSet) error { - glog.V(1).Infof("Registering systemd factory") + klog.V(1).Infof("Registering systemd factory") factory := &systemdFactory{} container.RegisterContainerHandlerFactory(factory, []watcher.ContainerWatchSource{watcher.Raw}) return nil diff --git a/vendor/github.com/google/cadvisor/devicemapper/BUILD b/vendor/github.com/google/cadvisor/devicemapper/BUILD index d3e813a0e17..53217411195 100644 --- a/vendor/github.com/google/cadvisor/devicemapper/BUILD +++ b/vendor/github.com/google/cadvisor/devicemapper/BUILD @@ -12,7 +12,7 @@ go_library( importmap = "k8s.io/kubernetes/vendor/github.com/google/cadvisor/devicemapper", importpath = "github.com/google/cadvisor/devicemapper", visibility = ["//visibility:public"], - deps = ["//vendor/github.com/golang/glog:go_default_library"], + deps = ["//vendor/k8s.io/klog:go_default_library"], ) filegroup( diff --git a/vendor/github.com/google/cadvisor/devicemapper/dmsetup_client.go b/vendor/github.com/google/cadvisor/devicemapper/dmsetup_client.go index c0936da9a75..3a37b560953 100644 --- a/vendor/github.com/google/cadvisor/devicemapper/dmsetup_client.go +++ b/vendor/github.com/google/cadvisor/devicemapper/dmsetup_client.go @@ -18,7 +18,7 @@ import ( "strconv" "strings" - "github.com/golang/glog" + "k8s.io/klog" ) // DmsetupClient is a low-level client for interacting with device mapper via @@ -58,6 +58,6 @@ func (c *defaultDmsetupClient) Status(deviceName string) ([]byte, error) { } func (*defaultDmsetupClient) dmsetup(args ...string) ([]byte, error) { - glog.V(5).Infof("running dmsetup %v", strings.Join(args, " ")) + klog.V(5).Infof("running dmsetup %v", strings.Join(args, " ")) return exec.Command("dmsetup", args...).Output() } diff --git a/vendor/github.com/google/cadvisor/devicemapper/thin_ls_client.go b/vendor/github.com/google/cadvisor/devicemapper/thin_ls_client.go index 29737434bfd..1bbc360ff4d 100644 --- a/vendor/github.com/google/cadvisor/devicemapper/thin_ls_client.go +++ b/vendor/github.com/google/cadvisor/devicemapper/thin_ls_client.go @@ -21,7 +21,7 @@ import ( "strconv" "strings" - "github.com/golang/glog" + "k8s.io/klog" ) // thinLsClient knows how to run a thin_ls very specific to CoW usage for @@ -53,7 +53,7 @@ var _ thinLsClient = &defaultThinLsClient{} func (c *defaultThinLsClient) ThinLs(deviceName string) (map[string]uint64, error) { args := []string{"--no-headers", "-m", "-o", "DEV,EXCLUSIVE_BYTES", deviceName} - glog.V(4).Infof("running command: thin_ls %v", strings.Join(args, " ")) + klog.V(4).Infof("running command: thin_ls %v", strings.Join(args, " ")) output, err := exec.Command(c.thinLsPath, args...).Output() if err != nil { @@ -80,7 +80,7 @@ func parseThinLsOutput(output []byte) map[string]uint64 { deviceID := fields[0] usage, err := strconv.ParseUint(fields[1], 10, 64) if err != nil { - glog.Warningf("unexpected error parsing thin_ls output: %v", err) + klog.Warningf("unexpected error parsing thin_ls output: %v", err) continue } diff --git a/vendor/github.com/google/cadvisor/devicemapper/thin_pool_watcher.go b/vendor/github.com/google/cadvisor/devicemapper/thin_pool_watcher.go index 6f5666a02fe..2eb8e002c85 100644 --- a/vendor/github.com/google/cadvisor/devicemapper/thin_pool_watcher.go +++ b/vendor/github.com/google/cadvisor/devicemapper/thin_pool_watcher.go @@ -19,7 +19,7 @@ import ( "sync" "time" - "github.com/golang/glog" + "k8s.io/klog" ) // ThinPoolWatcher maintains a cache of device name -> usage stats for a @@ -58,7 +58,7 @@ func NewThinPoolWatcher(poolName, metadataDevice string) (*ThinPoolWatcher, erro func (w *ThinPoolWatcher) Start() { err := w.Refresh() if err != nil { - glog.Errorf("encountered error refreshing thin pool watcher: %v", err) + klog.Errorf("encountered error refreshing thin pool watcher: %v", err) } for { @@ -69,12 +69,12 @@ func (w *ThinPoolWatcher) Start() { start := time.Now() err = w.Refresh() if err != nil { - glog.Errorf("encountered error refreshing thin pool watcher: %v", err) + klog.Errorf("encountered error refreshing thin pool watcher: %v", err) } // print latency for refresh duration := time.Since(start) - glog.V(5).Infof("thin_ls(%d) took %s", start.Unix(), duration) + klog.V(5).Infof("thin_ls(%d) took %s", start.Unix(), duration) } } } @@ -115,7 +115,7 @@ func (w *ThinPoolWatcher) Refresh() error { } if currentlyReserved { - glog.V(5).Infof("metadata for %v is currently reserved; releasing", w.poolName) + klog.V(5).Infof("metadata for %v is currently reserved; releasing", w.poolName) _, err = w.dmsetup.Message(w.poolName, 0, releaseMetadataMessage) if err != nil { err = fmt.Errorf("error releasing metadata snapshot for %v: %v", w.poolName, err) @@ -123,22 +123,22 @@ func (w *ThinPoolWatcher) Refresh() error { } } - glog.V(5).Infof("reserving metadata snapshot for thin-pool %v", w.poolName) + klog.V(5).Infof("reserving metadata snapshot for thin-pool %v", w.poolName) // NOTE: "0" in the call below is for the 'sector' argument to 'dmsetup // message'. It's not needed for thin pools. if output, err := w.dmsetup.Message(w.poolName, 0, reserveMetadataMessage); err != nil { err = fmt.Errorf("error reserving metadata for thin-pool %v: %v output: %v", w.poolName, err, string(output)) return err } else { - glog.V(5).Infof("reserved metadata snapshot for thin-pool %v", w.poolName) + klog.V(5).Infof("reserved metadata snapshot for thin-pool %v", w.poolName) } defer func() { - glog.V(5).Infof("releasing metadata snapshot for thin-pool %v", w.poolName) + klog.V(5).Infof("releasing metadata snapshot for thin-pool %v", w.poolName) w.dmsetup.Message(w.poolName, 0, releaseMetadataMessage) }() - glog.V(5).Infof("running thin_ls on metadata device %v", w.metadataDevice) + klog.V(5).Infof("running thin_ls on metadata device %v", w.metadataDevice) newCache, err := w.thinLsClient.ThinLs(w.metadataDevice) if err != nil { err = fmt.Errorf("error performing thin_ls on metadata device %v: %v", w.metadataDevice, err) @@ -157,7 +157,7 @@ const ( // checkReservation checks to see whether the thin device is currently holding // userspace metadata. func (w *ThinPoolWatcher) checkReservation(poolName string) (bool, error) { - glog.V(5).Infof("checking whether the thin-pool is holding a metadata snapshot") + klog.V(5).Infof("checking whether the thin-pool is holding a metadata snapshot") output, err := w.dmsetup.Status(poolName) if err != nil { return false, err diff --git a/vendor/github.com/google/cadvisor/events/BUILD b/vendor/github.com/google/cadvisor/events/BUILD index f5b785479be..ab8a454e8d7 100644 --- a/vendor/github.com/google/cadvisor/events/BUILD +++ b/vendor/github.com/google/cadvisor/events/BUILD @@ -7,9 +7,9 @@ go_library( importpath = "github.com/google/cadvisor/events", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/utils:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/events/handler.go b/vendor/github.com/google/cadvisor/events/handler.go index d920d7d6b7a..28a67addb00 100644 --- a/vendor/github.com/google/cadvisor/events/handler.go +++ b/vendor/github.com/google/cadvisor/events/handler.go @@ -24,7 +24,7 @@ import ( info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/utils" - "github.com/golang/glog" + "k8s.io/klog" ) type byTimestamp []*info.Event @@ -322,7 +322,7 @@ func (self *events) AddEvent(e *info.Event) error { for _, watchObject := range watchesToSend { watchObject.eventChannel.GetChannel() <- e } - glog.V(4).Infof("Added event %v", e) + klog.V(4).Infof("Added event %v", e) return nil } @@ -332,7 +332,7 @@ func (self *events) StopWatch(watchId int) { defer self.watcherLock.Unlock() _, ok := self.watchers[watchId] if !ok { - glog.Errorf("Could not find watcher instance %v", watchId) + klog.Errorf("Could not find watcher instance %v", watchId) } close(self.watchers[watchId].eventChannel.GetChannel()) delete(self.watchers, watchId) diff --git a/vendor/github.com/google/cadvisor/fs/BUILD b/vendor/github.com/google/cadvisor/fs/BUILD index 4af2e93854e..a4f229a42f9 100644 --- a/vendor/github.com/google/cadvisor/fs/BUILD +++ b/vendor/github.com/google/cadvisor/fs/BUILD @@ -12,11 +12,11 @@ go_library( deps = select({ "@io_bazel_rules_go//go/platform:linux": [ "//vendor/github.com/docker/docker/pkg/mount:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/devicemapper:go_default_library", "//vendor/github.com/google/cadvisor/utils:go_default_library", "//vendor/github.com/google/cadvisor/utils/docker:go_default_library", "//vendor/github.com/mistifyio/go-zfs:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], "//conditions:default": [], }), diff --git a/vendor/github.com/google/cadvisor/fs/fs.go b/vendor/github.com/google/cadvisor/fs/fs.go index 7d0ead97660..e628b29ffd3 100644 --- a/vendor/github.com/google/cadvisor/fs/fs.go +++ b/vendor/github.com/google/cadvisor/fs/fs.go @@ -33,11 +33,11 @@ import ( "time" "github.com/docker/docker/pkg/mount" - "github.com/golang/glog" "github.com/google/cadvisor/devicemapper" "github.com/google/cadvisor/utils" dockerutil "github.com/google/cadvisor/utils/docker" zfs "github.com/mistifyio/go-zfs" + "k8s.io/klog" ) const ( @@ -114,9 +114,9 @@ func NewFsInfo(context Context) (FsInfo, error) { fsUUIDToDeviceName, err := getFsUUIDToDeviceNameMap() if err != nil { - // UUID is not always avaiable across different OS distributions. + // UUID is not always available across different OS distributions. // Do not fail if there is an error. - glog.Warningf("Failed to get disk UUID mapping, getting disk info by uuid will not work: %v", err) + klog.Warningf("Failed to get disk UUID mapping, getting disk info by uuid will not work: %v", err) } // Avoid devicemapper container mounts - these are tracked by the ThinPoolWatcher @@ -139,8 +139,8 @@ func NewFsInfo(context Context) (FsInfo, error) { fsInfo.addDockerImagesLabel(context, mounts) fsInfo.addCrioImagesLabel(context, mounts) - glog.V(1).Infof("Filesystem UUIDs: %+v", fsInfo.fsUUIDToDeviceName) - glog.V(1).Infof("Filesystem partitions: %+v", fsInfo.partitions) + klog.V(1).Infof("Filesystem UUIDs: %+v", fsInfo.fsUUIDToDeviceName) + klog.V(1).Infof("Filesystem partitions: %+v", fsInfo.partitions) fsInfo.addSystemRootLabel(mounts) return fsInfo, nil } @@ -165,7 +165,7 @@ func getFsUUIDToDeviceNameMap() (map[string]string, error) { path := filepath.Join(dir, file.Name()) target, err := os.Readlink(path) if err != nil { - glog.Warningf("Failed to resolve symlink for %q", path) + klog.Warningf("Failed to resolve symlink for %q", path) continue } device, err := filepath.Abs(filepath.Join(dir, target)) @@ -213,7 +213,7 @@ func processMounts(mounts []*mount.Info, excludedMountpointPrefixes []string) ma if mount.Fstype == "btrfs" && mount.Major == 0 && strings.HasPrefix(mount.Source, "/dev/") { major, minor, err := getBtrfsMajorMinorIds(mount) if err != nil { - glog.Warningf("%s", err) + klog.Warningf("%s", err) } else { mount.Major = major mount.Minor = minor @@ -278,7 +278,7 @@ func (self *RealFsInfo) addSystemRootLabel(mounts []*mount.Info) { func (self *RealFsInfo) addDockerImagesLabel(context Context, mounts []*mount.Info) { dockerDev, dockerPartition, err := self.getDockerDeviceMapperInfo(context.Docker) if err != nil { - glog.Warningf("Could not get Docker devicemapper device: %v", err) + klog.Warningf("Could not get Docker devicemapper device: %v", err) } if len(dockerDev) > 0 && dockerPartition != nil { self.partitions[dockerDev] = *dockerPartition @@ -405,7 +405,7 @@ func (self *RealFsInfo) GetFsInfoForPath(mountSet map[string]struct{}) ([]Fs, er switch partition.fsType { case DeviceMapper.String(): fs.Capacity, fs.Free, fs.Available, err = getDMStats(device, partition.blockSize) - glog.V(5).Infof("got devicemapper fs capacity stats: capacity: %v free: %v available: %v:", fs.Capacity, fs.Free, fs.Available) + klog.V(5).Infof("got devicemapper fs capacity stats: capacity: %v free: %v available: %v:", fs.Capacity, fs.Free, fs.Available) fs.Type = DeviceMapper case ZFS.String(): fs.Capacity, fs.Free, fs.Available, err = getZfstats(device) @@ -418,11 +418,11 @@ func (self *RealFsInfo) GetFsInfoForPath(mountSet map[string]struct{}) ([]Fs, er fs.InodesFree = &inodesFree fs.Type = VFS } else { - glog.V(4).Infof("unable to determine file system type, partition mountpoint does not exist: %v", partition.mountpoint) + klog.V(4).Infof("unable to determine file system type, partition mountpoint does not exist: %v", partition.mountpoint) } } if err != nil { - glog.V(4).Infof("Stat fs failed. Error: %v", err) + klog.V(4).Infof("Stat fs failed. Error: %v", err) } else { deviceSet[device] = struct{}{} fs.DeviceInfo = DeviceInfo{ @@ -445,7 +445,7 @@ func getDiskStatsMap(diskStatsFile string) (map[string]DiskStats, error) { file, err := os.Open(diskStatsFile) if err != nil { if os.IsNotExist(err) { - glog.Warningf("Not collecting filesystem statistics because file %q was not found", diskStatsFile) + klog.Warningf("Not collecting filesystem statistics because file %q was not found", diskStatsFile) return diskStatsMap, nil } return nil, err @@ -551,7 +551,7 @@ func (self *RealFsInfo) GetDirFsDevice(dir string) (*DeviceInfo, error) { if found && mount.Fstype == "btrfs" && mount.Major == 0 && strings.HasPrefix(mount.Source, "/dev/") { major, minor, err := getBtrfsMajorMinorIds(mount) if err != nil { - glog.Warningf("%s", err) + klog.Warningf("%s", err) } else { return &DeviceInfo{mount.Source, uint(major), uint(minor)}, nil } @@ -583,12 +583,12 @@ func GetDirDiskUsage(dir string, timeout time.Duration) (uint64, error) { return 0, fmt.Errorf("failed to exec du - %v", err) } timer := time.AfterFunc(timeout, func() { - glog.Warningf("Killing cmd %v due to timeout(%s)", cmd.Args, timeout.String()) + klog.Warningf("Killing cmd %v due to timeout(%s)", cmd.Args, timeout.String()) cmd.Process.Kill() }) stdoutb, souterr := ioutil.ReadAll(stdoutp) if souterr != nil { - glog.Errorf("Failed to read from stdout for cmd %v - %v", cmd.Args, souterr) + klog.Errorf("Failed to read from stdout for cmd %v - %v", cmd.Args, souterr) } stderrb, _ := ioutil.ReadAll(stderrp) err = cmd.Wait() @@ -622,7 +622,7 @@ func GetDirInodeUsage(dir string, timeout time.Duration) (uint64, error) { return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr: %v", findCmd.Args, err, stderr.String()) } timer := time.AfterFunc(timeout, func() { - glog.Warningf("Killing cmd %v due to timeout(%s)", findCmd.Args, timeout.String()) + klog.Warningf("Killing cmd %v due to timeout(%s)", findCmd.Args, timeout.String()) findCmd.Process.Kill() }) err := findCmd.Wait() @@ -763,7 +763,7 @@ func getBtrfsMajorMinorIds(mount *mount.Info) (int, int, error) { return 0, 0, err } - glog.V(4).Infof("btrfs mount %#v", mount) + klog.V(4).Infof("btrfs mount %#v", mount) if buf.Mode&syscall.S_IFMT == syscall.S_IFBLK { err := syscall.Stat(mount.Mountpoint, buf) if err != nil { @@ -771,8 +771,8 @@ func getBtrfsMajorMinorIds(mount *mount.Info) (int, int, error) { return 0, 0, err } - glog.V(4).Infof("btrfs dev major:minor %d:%d\n", int(major(buf.Dev)), int(minor(buf.Dev))) - glog.V(4).Infof("btrfs rdev major:minor %d:%d\n", int(major(buf.Rdev)), int(minor(buf.Rdev))) + klog.V(4).Infof("btrfs dev major:minor %d:%d\n", int(major(buf.Dev)), int(minor(buf.Dev))) + klog.V(4).Infof("btrfs rdev major:minor %d:%d\n", int(major(buf.Rdev)), int(minor(buf.Rdev))) return int(major(buf.Dev)), int(minor(buf.Dev)), nil } else { diff --git a/vendor/github.com/google/cadvisor/info/v1/container.go b/vendor/github.com/google/cadvisor/info/v1/container.go index 41f7ae536ba..b54233d535c 100644 --- a/vendor/github.com/google/cadvisor/info/v1/container.go +++ b/vendor/github.com/google/cadvisor/info/v1/container.go @@ -102,11 +102,11 @@ type ContainerInfoRequest struct { NumStats int `json:"num_stats,omitempty"` // Start time for which to query information. - // If ommitted, the beginning of time is assumed. + // If omitted, the beginning of time is assumed. Start time.Time `json:"start,omitempty"` // End time for which to query information. - // If ommitted, current time is assumed. + // If omitted, current time is assumed. End time.Time `json:"end,omitempty"` } @@ -557,6 +557,14 @@ type AcceleratorStats struct { DutyCycle uint64 `json:"duty_cycle"` } +type ProcessStats struct { + // Number of processes + ProcessCount uint64 `json:"process_count"` + + // Number of open file descriptors + FdCount uint64 `json:"fd_count"` +} + type ContainerStats struct { // The time of this stat point. Timestamp time.Time `json:"timestamp"` @@ -574,6 +582,9 @@ type ContainerStats struct { // Metrics for Accelerators. Each Accelerator corresponds to one element in the array. Accelerators []AcceleratorStats `json:"accelerators,omitempty"` + // ProcessStats for Containers + Processes ProcessStats `json:"processes,omitempty"` + // Custom metrics from all collectors CustomMetrics map[string][]MetricVal `json:"custom_metrics,omitempty"` } diff --git a/vendor/github.com/google/cadvisor/info/v2/BUILD b/vendor/github.com/google/cadvisor/info/v2/BUILD index 4a6f117534f..b5b3a0f3ae0 100644 --- a/vendor/github.com/google/cadvisor/info/v2/BUILD +++ b/vendor/github.com/google/cadvisor/info/v2/BUILD @@ -11,8 +11,8 @@ go_library( importpath = "github.com/google/cadvisor/info/v2", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/info/v2/container.go b/vendor/github.com/google/cadvisor/info/v2/container.go index d32f571cb67..4288d003dbf 100644 --- a/vendor/github.com/google/cadvisor/info/v2/container.go +++ b/vendor/github.com/google/cadvisor/info/v2/container.go @@ -38,7 +38,7 @@ type CpuSpec struct { Mask string `json:"mask,omitempty"` // CPUQuota Default is disabled Quota uint64 `json:"quota,omitempty"` - // Period is the CPU reference time in ns e.g the quota is compared aginst this. + // Period is the CPU reference time in ns e.g the quota is compared against this. Period uint64 `json:"period,omitempty"` } @@ -254,6 +254,7 @@ type ProcessInfo struct { RunningTime string `json:"running_time"` CgroupPath string `json:"cgroup_path"` Cmd string `json:"cmd"` + FdCount int `json:"fd_count"` } type TcpStat struct { diff --git a/vendor/github.com/google/cadvisor/info/v2/conversion.go b/vendor/github.com/google/cadvisor/info/v2/conversion.go index 97fb463b9fa..74279a20a13 100644 --- a/vendor/github.com/google/cadvisor/info/v2/conversion.go +++ b/vendor/github.com/google/cadvisor/info/v2/conversion.go @@ -18,8 +18,8 @@ import ( "fmt" "time" - "github.com/golang/glog" "github.com/google/cadvisor/info/v1" + "k8s.io/klog" ) func machineFsStatsFromV1(fsStats []v1.FsStats) []MachineFsStats { @@ -70,7 +70,7 @@ func MachineStatsFromV1(cont *v1.ContainerInfo) []MachineStats { stat.Cpu = &val.Cpu cpuInst, err := InstCpuStats(last, val) if err != nil { - glog.Warningf("Could not get instant cpu stats: %v", err) + klog.Warningf("Could not get instant cpu stats: %v", err) } else { stat.CpuInst = cpuInst } @@ -107,7 +107,7 @@ func ContainerStatsFromV1(containerName string, spec *v1.ContainerSpec, stats [] stat.Cpu = &val.Cpu cpuInst, err := InstCpuStats(last, val) if err != nil { - glog.Warningf("Could not get instant cpu stats: %v", err) + klog.Warningf("Could not get instant cpu stats: %v", err) } else { stat.CpuInst = cpuInst } @@ -133,7 +133,7 @@ func ContainerStatsFromV1(containerName string, spec *v1.ContainerSpec, stats [] } } else if len(val.Filesystem) > 1 && containerName != "/" { // Cannot handle multiple devices per container. - glog.V(4).Infof("failed to handle multiple devices for container %s. Skipping Filesystem stats", containerName) + klog.V(4).Infof("failed to handle multiple devices for container %s. Skipping Filesystem stats", containerName) } } if spec.HasDiskIo { @@ -168,7 +168,7 @@ func DeprecatedStatsFromV1(cont *v1.ContainerInfo) []DeprecatedContainerStats { stat.Cpu = val.Cpu cpuInst, err := InstCpuStats(last, val) if err != nil { - glog.Warningf("Could not get instant cpu stats: %v", err) + klog.Warningf("Could not get instant cpu stats: %v", err) } else { stat.CpuInst = cpuInst } diff --git a/vendor/github.com/google/cadvisor/machine/BUILD b/vendor/github.com/google/cadvisor/machine/BUILD index 2e6d3c5932a..dc376cbf534 100644 --- a/vendor/github.com/google/cadvisor/machine/BUILD +++ b/vendor/github.com/google/cadvisor/machine/BUILD @@ -11,7 +11,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//vendor/github.com/docker/docker/pkg/parsers/operatingsystem:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/fs:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/utils:go_default_library", @@ -19,6 +18,7 @@ go_library( "//vendor/github.com/google/cadvisor/utils/sysfs:go_default_library", "//vendor/github.com/google/cadvisor/utils/sysinfo:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/machine/info.go b/vendor/github.com/google/cadvisor/machine/info.go index bf4f595ab21..03a65792c3e 100644 --- a/vendor/github.com/google/cadvisor/machine/info.go +++ b/vendor/github.com/google/cadvisor/machine/info.go @@ -30,7 +30,7 @@ import ( "github.com/google/cadvisor/utils/sysfs" "github.com/google/cadvisor/utils/sysinfo" - "github.com/golang/glog" + "k8s.io/klog" "golang.org/x/sys/unix" ) @@ -50,7 +50,7 @@ func getInfoFromFiles(filePaths string) string { return strings.TrimSpace(string(id)) } } - glog.Warningf("Couldn't collect info from any of the files in %q", filePaths) + klog.Warningf("Couldn't collect info from any of the files in %q", filePaths) return "" } @@ -117,27 +117,27 @@ func Info(sysFs sysfs.SysFs, fsInfo fs.FsInfo, inHostNamespace bool) (*info.Mach filesystems, err := fsInfo.GetGlobalFsInfo() if err != nil { - glog.Errorf("Failed to get global filesystem information: %v", err) + klog.Errorf("Failed to get global filesystem information: %v", err) } diskMap, err := sysinfo.GetBlockDeviceInfo(sysFs) if err != nil { - glog.Errorf("Failed to get disk map: %v", err) + klog.Errorf("Failed to get disk map: %v", err) } netDevices, err := sysinfo.GetNetworkDevices(sysFs) if err != nil { - glog.Errorf("Failed to get network devices: %v", err) + klog.Errorf("Failed to get network devices: %v", err) } topology, numCores, err := GetTopology(sysFs, string(cpuinfo)) if err != nil { - glog.Errorf("Failed to get topology information: %v", err) + klog.Errorf("Failed to get topology information: %v", err) } systemUUID, err := sysinfo.GetSystemUUID(sysFs) if err != nil { - glog.Errorf("Failed to get system UUID: %v", err) + klog.Errorf("Failed to get system UUID: %v", err) } realCloudInfo := cloudinfo.NewRealCloudInfo() diff --git a/vendor/github.com/google/cadvisor/machine/machine.go b/vendor/github.com/google/cadvisor/machine/machine.go index bd07414456c..3a60e2e5c52 100644 --- a/vendor/github.com/google/cadvisor/machine/machine.go +++ b/vendor/github.com/google/cadvisor/machine/machine.go @@ -30,7 +30,7 @@ import ( "github.com/google/cadvisor/utils/sysfs" "github.com/google/cadvisor/utils/sysinfo" - "github.com/golang/glog" + "k8s.io/klog" "golang.org/x/sys/unix" ) @@ -191,7 +191,7 @@ func GetTopology(sysFs sysfs.SysFs, cpuinfo string) ([]info.Node, int, error) { for idx, node := range nodes { caches, err := sysinfo.GetCacheInfo(sysFs, node.Cores[0].Threads[0]) if err != nil { - glog.Errorf("failed to get cache information for node %d: %v", node.Id, err) + klog.Errorf("failed to get cache information for node %d: %v", node.Id, err) continue } numThreadsPerCore := len(node.Cores[0].Threads) diff --git a/vendor/github.com/google/cadvisor/manager/BUILD b/vendor/github.com/google/cadvisor/manager/BUILD index 52dd2bac784..f397303590c 100644 --- a/vendor/github.com/google/cadvisor/manager/BUILD +++ b/vendor/github.com/google/cadvisor/manager/BUILD @@ -11,7 +11,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//vendor/github.com/docker/go-units:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/accelerators:go_default_library", "//vendor/github.com/google/cadvisor/cache/memory:go_default_library", "//vendor/github.com/google/cadvisor/collector:go_default_library", @@ -38,6 +37,7 @@ go_library( "//vendor/github.com/google/cadvisor/version:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups:go_default_library", "//vendor/golang.org/x/net/context:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/clock:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/manager/container.go b/vendor/github.com/google/cadvisor/manager/container.go index 295479f092d..dae4cf8fd5a 100644 --- a/vendor/github.com/google/cadvisor/manager/container.go +++ b/vendor/github.com/google/cadvisor/manager/container.go @@ -39,7 +39,7 @@ import ( "github.com/google/cadvisor/utils/cpuload" units "github.com/docker/go-units" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/utils/clock" ) @@ -47,7 +47,9 @@ import ( var enableLoadReader = flag.Bool("enable_load_reader", false, "Whether to enable cpu load reader") var HousekeepingInterval = flag.Duration("housekeeping_interval", 1*time.Second, "Interval between container housekeepings") -var cgroupPathRegExp = regexp.MustCompile(`devices[^:]*:(.*?)[,;$]`) +// cgroup type chosen to fetch the cgroup path of a process. +// Memory has been chosen, as it is one of the default cgroups that is enabled for most containers. +var cgroupPathRegExp = regexp.MustCompile(`memory[^:]*:(.*?)[,;$]`) type containerInfo struct { info.ContainerReference @@ -185,8 +187,8 @@ func (c *containerData) getCgroupPath(cgroups string) (string, error) { } matches := cgroupPathRegExp.FindSubmatch([]byte(cgroups)) if len(matches) != 2 { - glog.V(3).Infof("failed to get devices cgroup path from %q", cgroups) - // return root in case of failures - devices hierarchy might not be enabled. + klog.V(3).Infof("failed to get memory cgroup path from %q", cgroups) + // return root in case of failures - memory hierarchy might not be enabled. return "/", nil } return string(matches[1]), nil @@ -206,7 +208,7 @@ func (c *containerData) ReadFile(filepath string, inHostNamespace bool) ([]byte, } for _, pid := range pids { filePath := path.Join(rootfs, "/proc", pid, "/root", filepath) - glog.V(3).Infof("Trying path %q", filePath) + klog.V(3).Infof("Trying path %q", filePath) data, err := ioutil.ReadFile(filePath) if err == nil { return data, err @@ -266,6 +268,10 @@ func (c *containerData) getContainerPids(inHostNamespace bool) ([]string, error) func (c *containerData) GetProcessList(cadvisorContainer string, inHostNamespace bool) ([]v2.ProcessInfo, error) { // report all processes for root. isRoot := c.info.Name == "/" + rootfs := "/" + if !inHostNamespace { + rootfs = "/rootfs" + } format := "user,pid,ppid,stime,pcpu,pmem,rss,vsz,stat,time,comm,cgroup" out, err := c.getPsOutput(inHostNamespace, format) if err != nil { @@ -324,6 +330,15 @@ func (c *containerData) GetProcessList(cadvisorContainer string, inHostNamespace cgroupPath = cgroup } + var fdCount int + dirPath := path.Join(rootfs, "/proc", strconv.Itoa(pid), "fd") + fds, err := ioutil.ReadDir(dirPath) + if err != nil { + klog.V(4).Infof("error while listing directory %q to measure fd count: %v", dirPath, err) + continue + } + fdCount = len(fds) + if isRoot || c.info.Name == cgroup { processes = append(processes, v2.ProcessInfo{ User: fields[0], @@ -338,6 +353,7 @@ func (c *containerData) GetProcessList(cadvisorContainer string, inHostNamespace RunningTime: fields[9], Cmd: fields[10], CgroupPath: cgroupPath, + FdCount: fdCount, }) } } @@ -377,7 +393,7 @@ func newContainerData(containerName string, memoryCache *memory.InMemoryCache, h // Create cpu load reader. loadReader, err := cpuload.New() if err != nil { - glog.Warningf("Could not initialize cpu load reader for %q: %s", ref.Name, err) + klog.Warningf("Could not initialize cpu load reader for %q: %s", ref.Name, err) } else { cont.loadReader = loadReader } @@ -390,7 +406,7 @@ func newContainerData(containerName string, memoryCache *memory.InMemoryCache, h cont.summaryReader, err = summary.New(cont.info.Spec) if err != nil { cont.summaryReader = nil - glog.Warningf("Failed to create summary reader for %q: %v", ref.Name, err) + klog.Warningf("Failed to create summary reader for %q: %v", ref.Name, err) } return cont, nil @@ -403,7 +419,7 @@ func (self *containerData) nextHousekeepingInterval() time.Duration { stats, err := self.memoryCache.RecentStats(self.info.Name, empty, empty, 2) if err != nil { if self.allowErrorLogging() { - glog.Warningf("Failed to get RecentStats(%q) while determining the next housekeeping: %v", self.info.Name, err) + klog.Warningf("Failed to get RecentStats(%q) while determining the next housekeeping: %v", self.info.Name, err) } } else if len(stats) == 2 { // TODO(vishnuk): Use no processes as a signal. @@ -433,7 +449,7 @@ func (c *containerData) housekeeping() { if c.loadReader != nil { err := c.loadReader.Start() if err != nil { - glog.Warningf("Could not start cpu load stat collector for %q: %s", c.info.Name, err) + klog.Warningf("Could not start cpu load stat collector for %q: %s", c.info.Name, err) } defer c.loadReader.Stop() } @@ -445,7 +461,7 @@ func (c *containerData) housekeeping() { } // Housekeep every second. - glog.V(3).Infof("Start housekeeping for container %q\n", c.info.Name) + klog.V(3).Infof("Start housekeeping for container %q\n", c.info.Name) houseKeepingTimer := c.clock.NewTimer(0 * time.Second) defer houseKeepingTimer.Stop() for { @@ -466,7 +482,7 @@ func (c *containerData) housekeeping() { stats, err := c.memoryCache.RecentStats(c.info.Name, empty, empty, numSamples) if err != nil { if c.allowErrorLogging() { - glog.Warningf("[%s] Failed to get recent stats for logging usage: %v", c.info.Name, err) + klog.Warningf("[%s] Failed to get recent stats for logging usage: %v", c.info.Name, err) } } else if len(stats) < numSamples { // Ignore, not enough stats yet. @@ -483,7 +499,7 @@ func (c *containerData) housekeeping() { usageInCores := float64(usageCpuNs) / float64(stats[numSamples-1].Timestamp.Sub(stats[0].Timestamp).Nanoseconds()) usageInHuman := units.HumanSize(float64(usageMemory)) // Don't set verbosity since this is already protected by the logUsage flag. - glog.Infof("[%s] %.3f cores (average: %.3f cores), %s of memory", c.info.Name, instantUsageInCores, usageInCores, usageInHuman) + klog.Infof("[%s] %.3f cores (average: %.3f cores), %s of memory", c.info.Name, instantUsageInCores, usageInCores, usageInHuman) } } houseKeepingTimer.Reset(c.nextHousekeepingInterval()) @@ -504,13 +520,13 @@ func (c *containerData) housekeepingTick(timer <-chan time.Time, longHousekeepin err := c.updateStats() if err != nil { if c.allowErrorLogging() { - glog.Warningf("Failed to update stats for container \"%s\": %s", c.info.Name, err) + klog.Warningf("Failed to update stats for container \"%s\": %s", c.info.Name, err) } } // Log if housekeeping took too long. duration := c.clock.Since(start) if duration >= longHousekeeping { - glog.V(3).Infof("[%s] Housekeeping took %s", c.info.Name, duration) + klog.V(3).Infof("[%s] Housekeeping took %s", c.info.Name, duration) } c.notifyOnDemand() c.statsLastUpdatedTime = c.clock.Now() @@ -584,7 +600,7 @@ func (c *containerData) updateStats() error { err := c.summaryReader.AddSample(*stats) if err != nil { // Ignore summary errors for now. - glog.V(2).Infof("Failed to add summary stats for %q: %v", c.info.Name, err) + klog.V(2).Infof("Failed to add summary stats for %q: %v", c.info.Name, err) } } var customStatsErr error diff --git a/vendor/github.com/google/cadvisor/manager/manager.go b/vendor/github.com/google/cadvisor/manager/manager.go index 0fb6d5dced8..5c62bd66ebe 100644 --- a/vendor/github.com/google/cadvisor/manager/manager.go +++ b/vendor/github.com/google/cadvisor/manager/manager.go @@ -49,9 +49,9 @@ import ( "github.com/google/cadvisor/utils/sysfs" "github.com/google/cadvisor/version" - "github.com/golang/glog" "github.com/opencontainers/runc/libcontainer/cgroups" "golang.org/x/net/context" + "k8s.io/klog" "k8s.io/utils/clock" ) @@ -152,7 +152,7 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn if err != nil { return nil, err } - glog.V(2).Infof("cAdvisor running in container: %q", selfContainer) + klog.V(2).Infof("cAdvisor running in container: %q", selfContainer) var ( dockerStatus info.DockerStatus @@ -163,7 +163,7 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn dockerStatus = retryDockerStatus() if tmpRktPath, err := rkt.RktPath(); err != nil { - glog.V(5).Infof("Rkt not connected: %v", err) + klog.V(5).Infof("Rkt not connected: %v", err) } else { rktPath = tmpRktPath } @@ -174,7 +174,7 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn } crioInfo, err := crioClient.Info() if err != nil { - glog.V(5).Infof("CRI-O not connected: %v", err) + klog.V(5).Infof("CRI-O not connected: %v", err) } context := fs.Context{ @@ -226,13 +226,13 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn return nil, err } newManager.machineInfo = *machineInfo - glog.V(1).Infof("Machine: %+v", newManager.machineInfo) + klog.V(1).Infof("Machine: %+v", newManager.machineInfo) versionInfo, err := getVersionInfo() if err != nil { return nil, err } - glog.V(1).Infof("Version: %+v", *versionInfo) + klog.V(1).Infof("Version: %+v", *versionInfo) newManager.eventHandler = events.NewEventManager(parseEventsStoragePolicy()) return newManager, nil @@ -250,9 +250,9 @@ func retryDockerStatus() info.DockerStatus { switch err { case context.DeadlineExceeded: - glog.Warningf("Timeout trying to communicate with docker during initialization, will retry") + klog.Warningf("Timeout trying to communicate with docker during initialization, will retry") default: - glog.V(5).Infof("Docker not connected: %v", err) + klog.V(5).Infof("Docker not connected: %v", err) return info.DockerStatus{} } @@ -298,12 +298,12 @@ type manager struct { func (self *manager) Start() error { err := docker.Register(self, self.fsInfo, self.includedMetrics) if err != nil { - glog.V(5).Infof("Registration of the Docker container factory failed: %v.", err) + klog.V(5).Infof("Registration of the Docker container factory failed: %v.", err) } err = rkt.Register(self, self.fsInfo, self.includedMetrics) if err != nil { - glog.V(5).Infof("Registration of the rkt container factory failed: %v", err) + klog.V(5).Infof("Registration of the rkt container factory failed: %v", err) } else { watcher, err := rktwatcher.NewRktContainerWatcher() if err != nil { @@ -314,27 +314,27 @@ func (self *manager) Start() error { err = containerd.Register(self, self.fsInfo, self.includedMetrics) if err != nil { - glog.V(5).Infof("Registration of the containerd container factory failed: %v", err) + klog.V(5).Infof("Registration of the containerd container factory failed: %v", err) } err = crio.Register(self, self.fsInfo, self.includedMetrics) if err != nil { - glog.V(5).Infof("Registration of the crio container factory failed: %v", err) + klog.V(5).Infof("Registration of the crio container factory failed: %v", err) } err = mesos.Register(self, self.fsInfo, self.includedMetrics) if err != nil { - glog.V(5).Infof("Registration of the mesos container factory failed: %v", err) + klog.V(5).Infof("Registration of the mesos container factory failed: %v", err) } err = systemd.Register(self, self.fsInfo, self.includedMetrics) if err != nil { - glog.V(5).Infof("Registration of the systemd container factory failed: %v", err) + klog.V(5).Infof("Registration of the systemd container factory failed: %v", err) } err = raw.Register(self, self.fsInfo, self.includedMetrics, self.rawContainerCgroupPathPrefixWhiteList) if err != nil { - glog.Errorf("Registration of the raw container factory failed: %v", err) + klog.Errorf("Registration of the raw container factory failed: %v", err) } rawWatcher, err := rawwatcher.NewRawContainerWatcher() @@ -346,7 +346,7 @@ func (self *manager) Start() error { // Watch for OOMs. err = self.watchForNewOoms() if err != nil { - glog.Warningf("Could not configure a source for OOM detection, disabling OOM events: %v", err) + klog.Warningf("Could not configure a source for OOM detection, disabling OOM events: %v", err) } // If there are no factories, don't start any housekeeping and serve the information we do have. @@ -362,12 +362,12 @@ func (self *manager) Start() error { if err != nil { return err } - glog.V(2).Infof("Starting recovery of all containers") + klog.V(2).Infof("Starting recovery of all containers") err = self.detectSubcontainers("/") if err != nil { return err } - glog.V(2).Infof("Recovery completed") + klog.V(2).Infof("Recovery completed") // Watch for new container. quitWatcher := make(chan error) @@ -418,18 +418,18 @@ func (self *manager) globalHousekeeping(quit chan error) { // Check for new containers. err := self.detectSubcontainers("/") if err != nil { - glog.Errorf("Failed to detect containers: %s", err) + klog.Errorf("Failed to detect containers: %s", err) } // Log if housekeeping took too long. duration := time.Since(start) if duration >= longHousekeeping { - glog.V(3).Infof("Global Housekeeping(%d) took %s", t.Unix(), duration) + klog.V(3).Infof("Global Housekeeping(%d) took %s", t.Unix(), duration) } case <-quit: // Quit if asked to do so. quit <- nil - glog.Infof("Exiting global housekeeping thread") + klog.Infof("Exiting global housekeeping thread") return } } @@ -630,7 +630,7 @@ func (self *manager) AllDockerContainers(query *info.ContainerInfoRequest) (map[ if err != nil { // Ignore the error because of race condition and return best-effort result. if err == memory.ErrDataNotFound { - glog.Warningf("Error getting data for container %s because of race condition", name) + klog.Warningf("Error getting data for container %s because of race condition", name) continue } return nil, err @@ -890,7 +890,7 @@ func (m *manager) registerCollectors(collectorConfigs map[string]string, cont *c if err != nil { return fmt.Errorf("failed to read config file %q for config %q, container %q: %v", k, v, cont.info.Name, err) } - glog.V(4).Infof("Got config from %q: %q", v, configFile) + klog.V(4).Infof("Got config from %q: %q", v, configFile) if strings.HasPrefix(k, "prometheus") || strings.HasPrefix(k, "Prometheus") { newCollector, err := collector.NewPrometheusCollector(k, configFile, *applicationMetricsCountLimit, cont.handler, m.collectorHttpClient) @@ -968,7 +968,7 @@ func (m *manager) createContainerLocked(containerName string, watchSource watche } if !accept { // ignoring this container. - glog.V(4).Infof("ignoring container %q", containerName) + klog.V(4).Infof("ignoring container %q", containerName) return nil } collectorManager, err := collector.NewCollectorManager() @@ -983,11 +983,11 @@ func (m *manager) createContainerLocked(containerName string, watchSource watche } devicesCgroupPath, err := handler.GetCgroupPath("devices") if err != nil { - glog.Warningf("Error getting devices cgroup path: %v", err) + klog.Warningf("Error getting devices cgroup path: %v", err) } else { cont.nvidiaCollector, err = m.nvidiaManager.GetCollector(devicesCgroupPath) if err != nil { - glog.V(4).Infof("GPU metrics may be unavailable/incomplete for container %q: %v", cont.info.Name, err) + klog.V(4).Infof("GPU metrics may be unavailable/incomplete for container %q: %v", cont.info.Name, err) } } @@ -996,7 +996,7 @@ func (m *manager) createContainerLocked(containerName string, watchSource watche collectorConfigs := collector.GetCollectorConfigs(labels) err = m.registerCollectors(collectorConfigs, cont) if err != nil { - glog.Warningf("Failed to register collectors for %q: %v", containerName, err) + klog.Warningf("Failed to register collectors for %q: %v", containerName, err) } // Add the container name and all its aliases. The aliases must be within the namespace of the factory. @@ -1008,7 +1008,7 @@ func (m *manager) createContainerLocked(containerName string, watchSource watche }] = cont } - glog.V(3).Infof("Added container: %q (aliases: %v, namespace: %q)", containerName, cont.info.Aliases, cont.info.Namespace) + klog.V(3).Infof("Added container: %q (aliases: %v, namespace: %q)", containerName, cont.info.Aliases, cont.info.Namespace) contSpec, err := cont.handler.GetSpec() if err != nil { @@ -1065,7 +1065,7 @@ func (m *manager) destroyContainerLocked(containerName string) error { Name: alias, }) } - glog.V(3).Infof("Destroyed container: %q (aliases: %v, namespace: %q)", containerName, cont.info.Aliases, cont.info.Namespace) + klog.V(3).Infof("Destroyed container: %q (aliases: %v, namespace: %q)", containerName, cont.info.Aliases, cont.info.Namespace) contRef, err := cont.handler.ContainerReference() if err != nil { @@ -1144,7 +1144,7 @@ func (m *manager) detectSubcontainers(containerName string) error { for _, cont := range added { err = m.createContainer(cont.Name, watcher.Raw) if err != nil { - glog.Errorf("Failed to create existing container: %s: %s", cont.Name, err) + klog.Errorf("Failed to create existing container: %s: %s", cont.Name, err) } } @@ -1152,7 +1152,7 @@ func (m *manager) detectSubcontainers(containerName string) error { for _, cont := range removed { err = m.destroyContainer(cont.Name) if err != nil { - glog.Errorf("Failed to destroy existing container: %s: %s", cont.Name, err) + klog.Errorf("Failed to destroy existing container: %s: %s", cont.Name, err) } } @@ -1192,7 +1192,7 @@ func (self *manager) watchForNewContainers(quit chan error) error { err = self.destroyContainer(event.Name) } if err != nil { - glog.Warningf("Failed to process watch event %+v: %v", event, err) + klog.Warningf("Failed to process watch event %+v: %v", event, err) } case <-quit: var errs partialFailure @@ -1209,7 +1209,7 @@ func (self *manager) watchForNewContainers(quit chan error) error { quit <- errs } else { quit <- nil - glog.Infof("Exiting thread watching subcontainers") + klog.Infof("Exiting thread watching subcontainers") return } } @@ -1219,7 +1219,7 @@ func (self *manager) watchForNewContainers(quit chan error) error { } func (self *manager) watchForNewOoms() error { - glog.V(2).Infof("Started watching for new ooms in manager") + klog.V(2).Infof("Started watching for new ooms in manager") outStream := make(chan *oomparser.OomInstance, 10) oomLog, err := oomparser.New() if err != nil { @@ -1237,9 +1237,9 @@ func (self *manager) watchForNewOoms() error { } err := self.eventHandler.AddEvent(newEvent) if err != nil { - glog.Errorf("failed to add OOM event for %q: %v", oomInstance.ContainerName, err) + klog.Errorf("failed to add OOM event for %q: %v", oomInstance.ContainerName, err) } - glog.V(3).Infof("Created an OOM event in container %q at %v", oomInstance.ContainerName, oomInstance.TimeOfDeath) + klog.V(3).Infof("Created an OOM event in container %q at %v", oomInstance.ContainerName, oomInstance.TimeOfDeath) newEvent = &info.Event{ ContainerName: oomInstance.VictimContainerName, @@ -1254,7 +1254,7 @@ func (self *manager) watchForNewOoms() error { } err = self.eventHandler.AddEvent(newEvent) if err != nil { - glog.Errorf("failed to add OOM kill event for %q: %v", oomInstance.ContainerName, err) + klog.Errorf("failed to add OOM kill event for %q: %v", oomInstance.ContainerName, err) } } }() @@ -1285,12 +1285,12 @@ func parseEventsStoragePolicy() events.StoragePolicy { for _, part := range parts { items := strings.Split(part, "=") if len(items) != 2 { - glog.Warningf("Unknown event storage policy %q when parsing max age", part) + klog.Warningf("Unknown event storage policy %q when parsing max age", part) continue } dur, err := time.ParseDuration(items[1]) if err != nil { - glog.Warningf("Unable to parse event max age duration %q: %v", items[1], err) + klog.Warningf("Unable to parse event max age duration %q: %v", items[1], err) continue } if items[0] == "default" { @@ -1305,12 +1305,12 @@ func parseEventsStoragePolicy() events.StoragePolicy { for _, part := range parts { items := strings.Split(part, "=") if len(items) != 2 { - glog.Warningf("Unknown event storage policy %q when parsing max event limit", part) + klog.Warningf("Unknown event storage policy %q when parsing max event limit", part) continue } val, err := strconv.Atoi(items[1]) if err != nil { - glog.Warningf("Unable to parse integer from %q: %v", items[1], err) + klog.Warningf("Unable to parse integer from %q: %v", items[1], err) continue } if items[0] == "default" { diff --git a/vendor/github.com/google/cadvisor/manager/watcher/raw/BUILD b/vendor/github.com/google/cadvisor/manager/watcher/raw/BUILD index 613a56934f1..30ff579c0f7 100644 --- a/vendor/github.com/google/cadvisor/manager/watcher/raw/BUILD +++ b/vendor/github.com/google/cadvisor/manager/watcher/raw/BUILD @@ -7,11 +7,11 @@ go_library( importpath = "github.com/google/cadvisor/manager/watcher/raw", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/container/common:go_default_library", "//vendor/github.com/google/cadvisor/container/libcontainer:go_default_library", "//vendor/github.com/google/cadvisor/manager/watcher:go_default_library", - "//vendor/golang.org/x/exp/inotify:go_default_library", + "//vendor/github.com/sigma/go-inotify:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/manager/watcher/raw/raw.go b/vendor/github.com/google/cadvisor/manager/watcher/raw/raw.go index 5a8b79d39e6..76983cbe00f 100644 --- a/vendor/github.com/google/cadvisor/manager/watcher/raw/raw.go +++ b/vendor/github.com/google/cadvisor/manager/watcher/raw/raw.go @@ -26,9 +26,9 @@ import ( "github.com/google/cadvisor/container/common" "github.com/google/cadvisor/container/libcontainer" "github.com/google/cadvisor/manager/watcher" + inotify "github.com/sigma/go-inotify" - "github.com/golang/glog" - "golang.org/x/exp/inotify" + "k8s.io/klog" ) type rawContainerWatcher struct { @@ -84,10 +84,10 @@ func (self *rawContainerWatcher) Start(events chan watcher.ContainerEvent) error case event := <-self.watcher.Event(): err := self.processEvent(event, events) if err != nil { - glog.Warningf("Error while processing event (%+v): %v", event, err) + klog.Warningf("Error while processing event (%+v): %v", event, err) } case err := <-self.watcher.Error(): - glog.Warningf("Error while watching %q:", "/", err) + klog.Warningf("Error while watching %q: %v", "/", err) case <-self.stopWatcher: err := self.watcher.Close() if err == nil { @@ -126,7 +126,7 @@ func (self *rawContainerWatcher) watchDirectory(events chan watcher.ContainerEve if cleanup { _, err := self.watcher.RemoveWatch(containerName, dir) if err != nil { - glog.Warningf("Failed to remove inotify watch for %q: %v", dir, err) + klog.Warningf("Failed to remove inotify watch for %q: %v", dir, err) } } }() @@ -143,7 +143,7 @@ func (self *rawContainerWatcher) watchDirectory(events chan watcher.ContainerEve subcontainerName := path.Join(containerName, entry.Name()) alreadyWatchingSubDir, err := self.watchDirectory(events, entryPath, subcontainerName) if err != nil { - glog.Errorf("Failed to watch directory %q: %v", entryPath, err) + klog.Errorf("Failed to watch directory %q: %v", entryPath, err) if os.IsNotExist(err) { // The directory may have been removed before watching. Try to watch the other // subdirectories. (https://github.com/kubernetes/kubernetes/issues/28997) diff --git a/vendor/github.com/google/cadvisor/manager/watcher/rkt/BUILD b/vendor/github.com/google/cadvisor/manager/watcher/rkt/BUILD index a638993c4a6..90db2dd310e 100644 --- a/vendor/github.com/google/cadvisor/manager/watcher/rkt/BUILD +++ b/vendor/github.com/google/cadvisor/manager/watcher/rkt/BUILD @@ -8,10 +8,10 @@ go_library( visibility = ["//visibility:public"], deps = [ "//vendor/github.com/coreos/rkt/api/v1alpha:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/container/rkt:go_default_library", "//vendor/github.com/google/cadvisor/manager/watcher:go_default_library", "//vendor/golang.org/x/net/context:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/manager/watcher/rkt/rkt.go b/vendor/github.com/google/cadvisor/manager/watcher/rkt/rkt.go index 4c54d9b94e0..a2d910f89e0 100644 --- a/vendor/github.com/google/cadvisor/manager/watcher/rkt/rkt.go +++ b/vendor/github.com/google/cadvisor/manager/watcher/rkt/rkt.go @@ -23,8 +23,8 @@ import ( "github.com/google/cadvisor/manager/watcher" rktapi "github.com/coreos/rkt/api/v1alpha" - "github.com/golang/glog" "golang.org/x/net/context" + "k8s.io/klog" ) type rktContainerWatcher struct { @@ -53,7 +53,7 @@ func (self *rktContainerWatcher) Stop() error { } func (self *rktContainerWatcher) detectRktContainers(events chan watcher.ContainerEvent) { - glog.V(1).Infof("Starting detectRktContainers thread") + klog.V(1).Infof("Starting detectRktContainers thread") ticker := time.Tick(10 * time.Second) curpods := make(map[string]*rktapi.Pod) @@ -62,13 +62,13 @@ func (self *rktContainerWatcher) detectRktContainers(events chan watcher.Contain case <-ticker: pods, err := listRunningPods() if err != nil { - glog.Errorf("detectRktContainers: listRunningPods failed: %v", err) + klog.Errorf("detectRktContainers: listRunningPods failed: %v", err) continue } curpods = self.syncRunningPods(pods, events, curpods) case <-self.stopWatcher: - glog.Infof("Exiting rktContainer Thread") + klog.Infof("Exiting rktContainer Thread") return } } @@ -92,7 +92,7 @@ func (self *rktContainerWatcher) syncRunningPods(pods []*rktapi.Pod, events chan for id, pod := range curpods { if _, ok := newpods[id]; !ok { for _, cgroup := range podToCgroup(pod) { - glog.V(2).Infof("cgroup to delete = %v", cgroup) + klog.V(2).Infof("cgroup to delete = %v", cgroup) self.sendDestroyEvent(cgroup, events) } } diff --git a/vendor/github.com/google/cadvisor/metrics/BUILD b/vendor/github.com/google/cadvisor/metrics/BUILD index 3644234e8ac..73b18fee2ed 100644 --- a/vendor/github.com/google/cadvisor/metrics/BUILD +++ b/vendor/github.com/google/cadvisor/metrics/BUILD @@ -7,10 +7,10 @@ go_library( importpath = "github.com/google/cadvisor/metrics", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/container:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/metrics/prometheus.go b/vendor/github.com/google/cadvisor/metrics/prometheus.go index 9ff94502299..38502743edc 100644 --- a/vendor/github.com/google/cadvisor/metrics/prometheus.go +++ b/vendor/github.com/google/cadvisor/metrics/prometheus.go @@ -22,8 +22,8 @@ import ( "github.com/google/cadvisor/container" info "github.com/google/cadvisor/info/v1" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" + "k8s.io/klog" ) // infoProvider will usually be manager.Manager, but can be swapped out for testing. @@ -109,6 +109,7 @@ type PrometheusCollector struct { errors prometheus.Gauge containerMetrics []containerMetric containerLabelsFunc ContainerLabelsFunc + includedMetrics container.MetricSet } // NewPrometheusCollector returns a new PrometheusCollector. The passed @@ -137,6 +138,7 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri }, }, }, + includedMetrics: includedMetrics, } if includedMetrics.Has(container.CpuUsageMetrics) { c.containerMetrics = append(c.containerMetrics, []containerMetric{ @@ -336,7 +338,7 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri name: "container_memory_failures_total", help: "Cumulative count of memory allocation failures.", valueType: prometheus.CounterValue, - extraLabels: []string{"type", "scope"}, + extraLabels: []string{"failure_type", "scope"}, getValues: func(s *info.ContainerStats) metricValues { return metricValues{ { @@ -835,6 +837,26 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri }, }...) } + if includedMetrics.Has(container.ProcessMetrics) { + c.containerMetrics = append(c.containerMetrics, []containerMetric{ + { + name: "container_processes", + help: "Number of processes running inside the container.", + valueType: prometheus.GaugeValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Processes.ProcessCount)}} + }, + }, + { + name: "container_file_descriptors", + help: "Number of open file descriptors for the container.", + valueType: prometheus.GaugeValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.Processes.FdCount)}} + }, + }, + }...) + } return c } @@ -917,7 +939,7 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) containers, err := c.infoProvider.SubcontainersInfo("/", &info.ContainerInfoRequest{NumStats: 1}) if err != nil { c.errors.Set(1) - glog.Warningf("Couldn't get containers: %s", err) + klog.Warningf("Couldn't get containers: %s", err) return } rawLabels := map[string]struct{}{} @@ -926,10 +948,11 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) rawLabels[l] = struct{}{} } } - for _, container := range containers { + + for _, cont := range containers { values := make([]string, 0, len(rawLabels)) labels := make([]string, 0, len(rawLabels)) - containerLabels := c.containerLabelsFunc(container) + containerLabels := c.containerLabelsFunc(cont) for l := range rawLabels { labels = append(labels, sanitizeLabelName(l)) values = append(values, containerLabels[l]) @@ -937,35 +960,35 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) // Container spec desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", labels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), values...) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(cont.Spec.CreationTime.Unix()), values...) - if container.Spec.HasCpu { + if cont.Spec.HasCpu { desc = prometheus.NewDesc("container_spec_cpu_period", "CPU period of the container.", labels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Period), values...) - if container.Spec.Cpu.Quota != 0 { + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(cont.Spec.Cpu.Period), values...) + if cont.Spec.Cpu.Quota != 0 { desc = prometheus.NewDesc("container_spec_cpu_quota", "CPU quota of the container.", labels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Quota), values...) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(cont.Spec.Cpu.Quota), values...) } desc := prometheus.NewDesc("container_spec_cpu_shares", "CPU share of the container.", labels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Limit), values...) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(cont.Spec.Cpu.Limit), values...) } - if container.Spec.HasMemory { + if cont.Spec.HasMemory { desc := prometheus.NewDesc("container_spec_memory_limit_bytes", "Memory limit for the container.", labels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.Limit), values...) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(cont.Spec.Memory.Limit), values...) desc = prometheus.NewDesc("container_spec_memory_swap_limit_bytes", "Memory swap limit for the container.", labels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.SwapLimit), values...) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(cont.Spec.Memory.SwapLimit), values...) desc = prometheus.NewDesc("container_spec_memory_reservation_limit_bytes", "Memory reservation limit for the container.", labels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.Reservation), values...) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(cont.Spec.Memory.Reservation), values...) } // Now for the actual metrics - if len(container.Stats) == 0 { + if len(cont.Stats) == 0 { continue } - stats := container.Stats[0] + stats := cont.Stats[0] for _, cm := range c.containerMetrics { - if cm.condition != nil && !cm.condition(container.Spec) { + if cm.condition != nil && !cm.condition(cont.Spec) { continue } desc := cm.desc(labels) @@ -980,7 +1003,7 @@ func (c *PrometheusCollector) collectVersionInfo(ch chan<- prometheus.Metric) { versionInfo, err := c.infoProvider.GetVersionInfo() if err != nil { c.errors.Set(1) - glog.Warningf("Couldn't get version info: %s", err) + klog.Warningf("Couldn't get version info: %s", err) return } ch <- prometheus.MustNewConstMetric(versionInfoDesc, prometheus.GaugeValue, 1, []string{versionInfo.KernelVersion, versionInfo.ContainerOsVersion, versionInfo.DockerVersion, versionInfo.CadvisorVersion, versionInfo.CadvisorRevision}...) @@ -990,7 +1013,7 @@ func (c *PrometheusCollector) collectMachineInfo(ch chan<- prometheus.Metric) { machineInfo, err := c.infoProvider.GetMachineInfo() if err != nil { c.errors.Set(1) - glog.Warningf("Couldn't get machine info: %s", err) + klog.Warningf("Couldn't get machine info: %s", err) return } ch <- prometheus.MustNewConstMetric(machineInfoCoresDesc, prometheus.GaugeValue, float64(machineInfo.NumCores)) diff --git a/vendor/github.com/google/cadvisor/utils/cloudinfo/BUILD b/vendor/github.com/google/cadvisor/utils/cloudinfo/BUILD index 99ba21238f3..d660894a1b5 100644 --- a/vendor/github.com/google/cadvisor/utils/cloudinfo/BUILD +++ b/vendor/github.com/google/cadvisor/utils/cloudinfo/BUILD @@ -16,8 +16,8 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/ec2metadata:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/utils/cloudinfo/gce.go b/vendor/github.com/google/cadvisor/utils/cloudinfo/gce.go index b525451c05d..7e4856a47d6 100644 --- a/vendor/github.com/google/cadvisor/utils/cloudinfo/gce.go +++ b/vendor/github.com/google/cadvisor/utils/cloudinfo/gce.go @@ -21,7 +21,7 @@ import ( info "github.com/google/cadvisor/info/v1" "cloud.google.com/go/compute/metadata" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -32,7 +32,7 @@ const ( func onGCE() bool { data, err := ioutil.ReadFile(gceProductName) if err != nil { - glog.V(2).Infof("Error while reading product_name: %v", err) + klog.V(2).Infof("Error while reading product_name: %v", err) return false } return strings.Contains(string(data), google) diff --git a/vendor/github.com/google/cadvisor/utils/cpuload/BUILD b/vendor/github.com/google/cadvisor/utils/cpuload/BUILD index c3bb700a77f..ed6b9f77506 100644 --- a/vendor/github.com/google/cadvisor/utils/cpuload/BUILD +++ b/vendor/github.com/google/cadvisor/utils/cpuload/BUILD @@ -7,9 +7,9 @@ go_library( importpath = "github.com/google/cadvisor/utils/cpuload", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/utils/cpuload/netlink:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/utils/cpuload/cpuload.go b/vendor/github.com/google/cadvisor/utils/cpuload/cpuload.go index f3d29b8dd05..ca46ea1e674 100644 --- a/vendor/github.com/google/cadvisor/utils/cpuload/cpuload.go +++ b/vendor/github.com/google/cadvisor/utils/cpuload/cpuload.go @@ -19,8 +19,8 @@ import ( info "github.com/google/cadvisor/info/v1" - "github.com/golang/glog" "github.com/google/cadvisor/utils/cpuload/netlink" + "k8s.io/klog" ) type CpuLoadReader interface { @@ -41,6 +41,6 @@ func New() (CpuLoadReader, error) { if err != nil { return nil, fmt.Errorf("failed to create a netlink based cpuload reader: %v", err) } - glog.V(4).Info("Using a netlink-based load reader") + klog.V(4).Info("Using a netlink-based load reader") return reader, nil } diff --git a/vendor/github.com/google/cadvisor/utils/cpuload/netlink/BUILD b/vendor/github.com/google/cadvisor/utils/cpuload/netlink/BUILD index 80040cab7f6..fd9b4d5acf9 100644 --- a/vendor/github.com/google/cadvisor/utils/cpuload/netlink/BUILD +++ b/vendor/github.com/google/cadvisor/utils/cpuload/netlink/BUILD @@ -13,8 +13,8 @@ go_library( importpath = "github.com/google/cadvisor/utils/cpuload/netlink", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/utils/cpuload/netlink/reader.go b/vendor/github.com/google/cadvisor/utils/cpuload/netlink/reader.go index 1f43d755e17..7236955215d 100644 --- a/vendor/github.com/google/cadvisor/utils/cpuload/netlink/reader.go +++ b/vendor/github.com/google/cadvisor/utils/cpuload/netlink/reader.go @@ -20,7 +20,7 @@ import ( info "github.com/google/cadvisor/info/v1" - "github.com/golang/glog" + "k8s.io/klog" ) type NetlinkReader struct { @@ -38,7 +38,7 @@ func New() (*NetlinkReader, error) { if err != nil { return nil, fmt.Errorf("failed to get netlink family id for task stats: %s", err) } - glog.V(4).Infof("Family id for taskstats: %d", id) + klog.V(4).Infof("Family id for taskstats: %d", id) return &NetlinkReader{ familyId: id, conn: conn, @@ -75,6 +75,6 @@ func (self *NetlinkReader) GetCpuLoad(name string, path string) (info.LoadStats, if err != nil { return info.LoadStats{}, err } - glog.V(4).Infof("Task stats for %q: %+v", path, stats) + klog.V(4).Infof("Task stats for %q: %+v", path, stats) return stats, nil } diff --git a/vendor/github.com/google/cadvisor/utils/oomparser/BUILD b/vendor/github.com/google/cadvisor/utils/oomparser/BUILD index fbd22a9c00b..f83fbc5a938 100644 --- a/vendor/github.com/google/cadvisor/utils/oomparser/BUILD +++ b/vendor/github.com/google/cadvisor/utils/oomparser/BUILD @@ -8,7 +8,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//vendor/github.com/euank/go-kmsg-parser/kmsgparser:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/utils/oomparser/oomparser.go b/vendor/github.com/google/cadvisor/utils/oomparser/oomparser.go index a73243f2e3e..039baa30bee 100644 --- a/vendor/github.com/google/cadvisor/utils/oomparser/oomparser.go +++ b/vendor/github.com/google/cadvisor/utils/oomparser/oomparser.go @@ -22,7 +22,7 @@ import ( "github.com/euank/go-kmsg-parser/kmsgparser" - "github.com/golang/glog" + "k8s.io/klog" ) var ( @@ -107,11 +107,11 @@ func (self *OomParser) StreamOoms(outStream chan<- *OomInstance) { for msg := range kmsgEntries { err := getContainerName(msg.Message, oomCurrentInstance) if err != nil { - glog.Errorf("%v", err) + klog.Errorf("%v", err) } finished, err := getProcessNamePid(msg.Message, oomCurrentInstance) if err != nil { - glog.Errorf("%v", err) + klog.Errorf("%v", err) } if finished { oomCurrentInstance.TimeOfDeath = msg.Timestamp @@ -122,7 +122,7 @@ func (self *OomParser) StreamOoms(outStream chan<- *OomInstance) { } } // Should not happen - glog.Errorf("exiting analyzeLines. OOM events will not be reported.") + klog.Errorf("exiting analyzeLines. OOM events will not be reported.") } // initializes an OomParser object. Returns an OomParser object and an error. @@ -140,11 +140,11 @@ type glogAdapter struct{} var _ kmsgparser.Logger = glogAdapter{} func (glogAdapter) Infof(format string, args ...interface{}) { - glog.V(4).Infof(format, args) + klog.V(4).Infof(format, args...) } func (glogAdapter) Warningf(format string, args ...interface{}) { - glog.V(2).Infof(format, args) + klog.V(2).Infof(format, args...) } func (glogAdapter) Errorf(format string, args ...interface{}) { - glog.Warningf(format, args) + klog.Warningf(format, args...) } diff --git a/vendor/github.com/google/cadvisor/zfs/BUILD b/vendor/github.com/google/cadvisor/zfs/BUILD index 20eb5cd40d4..4ce002ed99f 100644 --- a/vendor/github.com/google/cadvisor/zfs/BUILD +++ b/vendor/github.com/google/cadvisor/zfs/BUILD @@ -7,8 +7,8 @@ go_library( importpath = "github.com/google/cadvisor/zfs", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/mistifyio/go-zfs:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/google/cadvisor/zfs/watcher.go b/vendor/github.com/google/cadvisor/zfs/watcher.go index 1bc3fb741d3..844f52f2e70 100644 --- a/vendor/github.com/google/cadvisor/zfs/watcher.go +++ b/vendor/github.com/google/cadvisor/zfs/watcher.go @@ -18,8 +18,8 @@ import ( "sync" "time" - "github.com/golang/glog" zfs "github.com/mistifyio/go-zfs" + "k8s.io/klog" ) // zfsWatcher maintains a cache of filesystem -> usage stats for a @@ -49,7 +49,7 @@ func NewZfsWatcher(filesystem string) (*ZfsWatcher, error) { func (w *ZfsWatcher) Start() { err := w.Refresh() if err != nil { - glog.Errorf("encountered error refreshing zfs watcher: %v", err) + klog.Errorf("encountered error refreshing zfs watcher: %v", err) } for { @@ -60,12 +60,12 @@ func (w *ZfsWatcher) Start() { start := time.Now() err = w.Refresh() if err != nil { - glog.Errorf("encountered error refreshing zfs watcher: %v", err) + klog.Errorf("encountered error refreshing zfs watcher: %v", err) } // print latency for refresh duration := time.Since(start) - glog.V(5).Infof("zfs(%d) took %s", start.Unix(), duration) + klog.V(5).Infof("zfs(%d) took %s", start.Unix(), duration) } } } @@ -95,12 +95,12 @@ func (w *ZfsWatcher) Refresh() error { newCache := make(map[string]uint64) parent, err := zfs.GetDataset(w.filesystem) if err != nil { - glog.Errorf("encountered error getting zfs filesystem: %s: %v", w.filesystem, err) + klog.Errorf("encountered error getting zfs filesystem: %s: %v", w.filesystem, err) return err } children, err := parent.Children(0) if err != nil { - glog.Errorf("encountered error getting children of zfs filesystem: %s: %v", w.filesystem, err) + klog.Errorf("encountered error getting children of zfs filesystem: %s: %v", w.filesystem, err) return err } diff --git a/vendor/github.com/heketi/heketi/client/api/go-client/BUILD b/vendor/github.com/heketi/heketi/client/api/go-client/BUILD index f83f7de7253..4cc7cec05c8 100644 --- a/vendor/github.com/heketi/heketi/client/api/go-client/BUILD +++ b/vendor/github.com/heketi/heketi/client/api/go-client/BUILD @@ -3,11 +3,16 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ + "admin.go", "backup.go", + "block_volume.go", "client.go", "cluster.go", + "db.go", "device.go", + "logging.go", "node.go", + "operations.go", "topology.go", "volume.go", ], diff --git a/vendor/github.com/heketi/heketi/client/api/go-client/admin.go b/vendor/github.com/heketi/heketi/client/api/go-client/admin.go new file mode 100644 index 00000000000..f47cf236109 --- /dev/null +++ b/vendor/github.com/heketi/heketi/client/api/go-client/admin.go @@ -0,0 +1,78 @@ +// +// Copyright (c) 2018 The heketi Authors +// +// This file is licensed to you under your choice of the GNU Lesser +// General Public License, version 3 or any later version (LGPLv3 or +// later), as published by the Free Software Foundation, +// or under the Apache License, Version 2.0 . +// +// You may not use this file except in compliance with those terms. +// + +package client + +import ( + "bytes" + "encoding/json" + "net/http" + + "github.com/heketi/heketi/pkg/glusterfs/api" + "github.com/heketi/heketi/pkg/utils" +) + +func (c *Client) AdminStatusGet() (*api.AdminStatus, error) { + req, err := http.NewRequest("GET", c.host+"/admin", nil) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + // Set token + err = c.setToken(req) + if err != nil { + return nil, err + } + + // Send request + r, err := c.do(req) + if err != nil { + return nil, err + } + var as api.AdminStatus + err = utils.GetJsonFromResponse(r, &as) + if err != nil { + return nil, err + } + return &as, nil +} + +func (c *Client) AdminStatusSet(request *api.AdminStatus) error { + // Marshal request to JSON + buffer, err := json.Marshal(request) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", c.host+"/admin", bytes.NewBuffer(buffer)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + + // Set token + err = c.setToken(req) + if err != nil { + return err + } + + // Send request + r, err := c.do(req) + if err != nil { + return err + } + if r.StatusCode == http.StatusOK || r.StatusCode == http.StatusNoContent { + return nil + } + return utils.GetErrorFromResponse(r) +} diff --git a/vendor/github.com/heketi/heketi/client/api/go-client/backup.go b/vendor/github.com/heketi/heketi/client/api/go-client/backup.go index 4b04d12de6d..0690b3f4173 100644 --- a/vendor/github.com/heketi/heketi/client/api/go-client/backup.go +++ b/vendor/github.com/heketi/heketi/client/api/go-client/backup.go @@ -37,12 +37,12 @@ func (c *Client) BackupDb(w io.Writer) error { if err != nil { return err } + defer r.Body.Close() if r.StatusCode != http.StatusOK { return utils.GetErrorFromResponse(r) } // Read data from response - defer r.Body.Close() _, err = io.Copy(w, r.Body) return err diff --git a/vendor/github.com/heketi/heketi/client/api/go-client/block_volume.go b/vendor/github.com/heketi/heketi/client/api/go-client/block_volume.go new file mode 100644 index 00000000000..c84187f533e --- /dev/null +++ b/vendor/github.com/heketi/heketi/client/api/go-client/block_volume.go @@ -0,0 +1,160 @@ +// +// Copyright (c) 2015 The heketi Authors +// +// This file is licensed to you under your choice of the GNU Lesser +// General Public License, version 3 or any later version (LGPLv3 or +// later), as published by the Free Software Foundation, +// or under the Apache License, Version 2.0 . +// +// You may not use this file except in compliance with those terms. +// + +package client + +import ( + "bytes" + "encoding/json" + "net/http" + "time" + + "github.com/heketi/heketi/pkg/glusterfs/api" + "github.com/heketi/heketi/pkg/utils" +) + +func (c *Client) BlockVolumeCreate(request *api.BlockVolumeCreateRequest) ( + *api.BlockVolumeInfoResponse, error) { + + buffer, err := json.Marshal(request) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", + c.host+"/blockvolumes", + bytes.NewBuffer(buffer)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + err = c.setToken(req) + if err != nil { + return nil, err + } + + r, err := c.do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + if r.StatusCode != http.StatusAccepted { + return nil, utils.GetErrorFromResponse(r) + } + + r, err = c.waitForResponseWithTimer(r, time.Second) + if err != nil { + return nil, err + } + if r.StatusCode != http.StatusOK { + return nil, utils.GetErrorFromResponse(r) + } + + var blockvolume api.BlockVolumeInfoResponse + err = utils.GetJsonFromResponse(r, &blockvolume) + if err != nil { + return nil, err + } + + return &blockvolume, nil + +} + +func (c *Client) BlockVolumeList() (*api.BlockVolumeListResponse, error) { + req, err := http.NewRequest("GET", c.host+"/blockvolumes", nil) + if err != nil { + return nil, err + } + + err = c.setToken(req) + if err != nil { + return nil, err + } + + r, err := c.do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + return nil, utils.GetErrorFromResponse(r) + } + + var blockvolumes api.BlockVolumeListResponse + err = utils.GetJsonFromResponse(r, &blockvolumes) + if err != nil { + return nil, err + } + + return &blockvolumes, nil +} + +func (c *Client) BlockVolumeInfo(id string) (*api.BlockVolumeInfoResponse, error) { + req, err := http.NewRequest("GET", c.host+"/blockvolumes/"+id, nil) + if err != nil { + return nil, err + } + + err = c.setToken(req) + if err != nil { + return nil, err + } + + r, err := c.do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + return nil, utils.GetErrorFromResponse(r) + } + + var blockvolume api.BlockVolumeInfoResponse + err = utils.GetJsonFromResponse(r, &blockvolume) + if err != nil { + return nil, err + } + + return &blockvolume, nil +} + +func (c *Client) BlockVolumeDelete(id string) error { + req, err := http.NewRequest("DELETE", c.host+"/blockvolumes/"+id, nil) + if err != nil { + return err + } + + err = c.setToken(req) + if err != nil { + return err + } + + r, err := c.do(req) + if err != nil { + return err + } + defer r.Body.Close() + if r.StatusCode != http.StatusAccepted { + return utils.GetErrorFromResponse(r) + } + + r, err = c.waitForResponseWithTimer(r, time.Second) + if err != nil { + return err + } + if r.StatusCode != http.StatusNoContent { + return utils.GetErrorFromResponse(r) + } + + return nil +} diff --git a/vendor/github.com/heketi/heketi/client/api/go-client/client.go b/vendor/github.com/heketi/heketi/client/api/go-client/client.go index e29abfbfb88..74d6fe2739c 100644 --- a/vendor/github.com/heketi/heketi/client/api/go-client/client.go +++ b/vendor/github.com/heketi/heketi/client/api/go-client/client.go @@ -13,9 +13,16 @@ package client import ( + "bytes" "crypto/sha256" + "crypto/tls" + "crypto/x509" "encoding/hex" + "fmt" + "io/ioutil" + "math/rand" "net/http" + "strconv" "time" jwt "github.com/dgrijalva/jwt-go" @@ -24,35 +31,118 @@ import ( const ( MAX_CONCURRENT_REQUESTS = 32 + RETRY_COUNT = 6 + + // default delay values + MIN_DELAY = 10 + MAX_DELAY = 30 ) +type ClientTLSOptions struct { + // directly borrow the field names from crypto/tls + InsecureSkipVerify bool + // one or more cert file paths (best for self-signed certs) + VerifyCerts []string +} + +// Client configuration options +type ClientOptions struct { + RetryEnabled bool + RetryCount int + // control waits between retries + RetryMinDelay, RetryMaxDelay int +} + // Client object type Client struct { host string key string user string throttle chan bool + + // configuration for TLS support + tlsClientConfig *tls.Config + + // general behavioral options + opts ClientOptions + + // allow plugging in custom do wrappers + do func(*http.Request) (*http.Response, error) } -// Creates a new client to access a Heketi server +var defaultClientOptions = ClientOptions{ + RetryEnabled: true, + RetryCount: RETRY_COUNT, + RetryMinDelay: MIN_DELAY, + RetryMaxDelay: MAX_DELAY, +} + +// NewClient creates a new client to access a Heketi server func NewClient(host, user, key string) *Client { + return NewClientWithOptions(host, user, key, defaultClientOptions) +} + +// NewClientWithOptions creates a new client to access a Heketi server +// with a user specified suite of options. +func NewClientWithOptions(host, user, key string, opts ClientOptions) *Client { c := &Client{} c.key = key c.host = host c.user = user - + c.opts = opts // Maximum concurrent requests c.throttle = make(chan bool, MAX_CONCURRENT_REQUESTS) + if opts.RetryEnabled { + c.do = c.retryOperationDo + } else { + c.do = c.doBasic + } return c } +func NewClientTLS(host, user, key string, tlsOpts *ClientTLSOptions) (*Client, error) { + c := NewClient(host, user, key) + if err := c.SetTLSOptions(tlsOpts); err != nil { + return nil, err + } + return c, nil +} + // Create a client to access a Heketi server without authentication enabled func NewClientNoAuth(host string) *Client { return NewClient(host, "", "") } +// SetTLSOptions configures an existing heketi client for +// TLS support based on the ClientTLSOptions. +func (c *Client) SetTLSOptions(o *ClientTLSOptions) error { + if o == nil { + c.tlsClientConfig = nil + return nil + } + + tlsConfig := &tls.Config{} + tlsConfig.InsecureSkipVerify = o.InsecureSkipVerify + if len(o.VerifyCerts) > 0 { + tlsConfig.RootCAs = x509.NewCertPool() + for _, path := range o.VerifyCerts { + pem, err := ioutil.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read cert file %v: %v", + path, err) + } + if ok := tlsConfig.RootCAs.AppendCertsFromPEM(pem); !ok { + return fmt.Errorf("failed to load PEM encoded cert from %s", + path) + } + } + } + c.tlsClientConfig = tlsConfig + return nil +} + // Simple Hello test to check if the server is up func (c *Client) Hello() error { // Create request @@ -72,6 +162,7 @@ func (c *Client) Hello() error { if err != nil { return err } + defer r.Body.Close() if r.StatusCode != http.StatusOK { return utils.GetErrorFromResponse(r) } @@ -79,14 +170,20 @@ func (c *Client) Hello() error { return nil } +// doBasic performs the core http transaction. // Make sure we do not run out of fds by throttling the requests -func (c *Client) do(req *http.Request) (*http.Response, error) { +func (c *Client) doBasic(req *http.Request) (*http.Response, error) { c.throttle <- true defer func() { <-c.throttle }() httpClient := &http.Client{} + if c.tlsClientConfig != nil { + httpClient.Transport = &http.Transport{ + TLSClientConfig: c.tlsClientConfig, + } + } httpClient.CheckRedirect = c.checkRedirect return httpClient.Do(req) } @@ -122,7 +219,7 @@ func (c *Client) waitForResponseWithTimer(r *http.Response, } // Wait for response - r, err = c.do(req) + r, err = c.doBasic(req) if err != nil { return nil, err } @@ -132,6 +229,11 @@ func (c *Client) waitForResponseWithTimer(r *http.Response, if r.StatusCode != http.StatusOK { return nil, utils.GetErrorFromResponse(r) } + if r != nil { + //Read Response Body + ioutil.ReadAll(r.Body) + r.Body.Close() + } time.Sleep(waitTime) } else { return r, nil @@ -174,3 +276,66 @@ func (c *Client) setToken(r *http.Request) error { return nil } + +// retryOperationDo performs the http request and internally +// handles http 429 codes up to the number of retries specified +// by the Client. +func (c *Client) retryOperationDo(req *http.Request) (*http.Response, error) { + var ( + requestBody []byte + err error + ) + if req.Body != nil { + requestBody, err = ioutil.ReadAll(req.Body) + if err != nil { + return nil, err + } + } + + // Send request + var r *http.Response + for i := 0; i <= c.opts.RetryCount; i++ { + req.Body = ioutil.NopCloser(bytes.NewReader(requestBody)) + r, err = c.doBasic(req) + if err != nil { + return nil, err + } + switch r.StatusCode { + case http.StatusTooManyRequests: + if r != nil { + //Read Response Body + // I don't like discarding error here, but I cant + // think of something better atm + b, _ := ioutil.ReadAll(r.Body) + r.Body.Close() + r.Body = ioutil.NopCloser(bytes.NewReader(b)) + } + //sleep before continue + time.Sleep(c.opts.retryDelay(r)) + continue + + default: + return r, err + + } + } + return r, err +} + +// retryDelay returns a duration for which a retry should wait +// (after failure) before continuing. +func (c *ClientOptions) retryDelay(r *http.Response) time.Duration { + var ( + min = c.RetryMinDelay + max = c.RetryMaxDelay + ) + if ra := r.Header.Get("Retry-After"); ra != "" { + // TODO: support http date + if i, err := strconv.Atoi(ra); err == nil { + s := rand.Intn(min) + i + return time.Second * time.Duration(s) + } + } + s := rand.Intn(max-min) + min + return time.Second * time.Duration(s) +} diff --git a/vendor/github.com/heketi/heketi/client/api/go-client/cluster.go b/vendor/github.com/heketi/heketi/client/api/go-client/cluster.go index ddab70dac54..b68a9e43729 100644 --- a/vendor/github.com/heketi/heketi/client/api/go-client/cluster.go +++ b/vendor/github.com/heketi/heketi/client/api/go-client/cluster.go @@ -14,16 +14,23 @@ package client import ( "bytes" + "encoding/json" "net/http" "github.com/heketi/heketi/pkg/glusterfs/api" "github.com/heketi/heketi/pkg/utils" ) -func (c *Client) ClusterCreate() (*api.ClusterInfoResponse, error) { +func (c *Client) ClusterCreate(request *api.ClusterCreateRequest) (*api.ClusterInfoResponse, error) { + + buffer, err := json.Marshal(request) + if err != nil { + return nil, err + } // Create a request - req, err := http.NewRequest("POST", c.host+"/clusters", bytes.NewBuffer([]byte(`{}`))) + req, err := http.NewRequest("POST", c.host+"/clusters", + bytes.NewBuffer(buffer)) if err != nil { return nil, err } @@ -40,6 +47,7 @@ func (c *Client) ClusterCreate() (*api.ClusterInfoResponse, error) { if err != nil { return nil, err } + defer r.Body.Close() if r.StatusCode != http.StatusCreated { return nil, utils.GetErrorFromResponse(r) } @@ -47,7 +55,6 @@ func (c *Client) ClusterCreate() (*api.ClusterInfoResponse, error) { // Read JSON response var cluster api.ClusterInfoResponse err = utils.GetJsonFromResponse(r, &cluster) - r.Body.Close() if err != nil { return nil, err } @@ -55,6 +62,40 @@ func (c *Client) ClusterCreate() (*api.ClusterInfoResponse, error) { return &cluster, nil } +func (c *Client) ClusterSetFlags(id string, request *api.ClusterSetFlagsRequest) error { + + buffer, err := json.Marshal(request) + if err != nil { + return err + } + + // Create a request + req, err := http.NewRequest("POST", c.host+"/clusters/"+id+"/flags", + bytes.NewBuffer(buffer)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + + // Set token + err = c.setToken(req) + if err != nil { + return err + } + + // Send request + r, err := c.do(req) + if err != nil { + return err + } + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + return utils.GetErrorFromResponse(r) + } + + return nil +} + func (c *Client) ClusterInfo(id string) (*api.ClusterInfoResponse, error) { // Create request @@ -74,6 +115,7 @@ func (c *Client) ClusterInfo(id string) (*api.ClusterInfoResponse, error) { if err != nil { return nil, err } + defer r.Body.Close() if r.StatusCode != http.StatusOK { return nil, utils.GetErrorFromResponse(r) } @@ -81,7 +123,6 @@ func (c *Client) ClusterInfo(id string) (*api.ClusterInfoResponse, error) { // Read JSON response var cluster api.ClusterInfoResponse err = utils.GetJsonFromResponse(r, &cluster) - r.Body.Close() if err != nil { return nil, err } @@ -108,6 +149,7 @@ func (c *Client) ClusterList() (*api.ClusterListResponse, error) { if err != nil { return nil, err } + defer r.Body.Close() if r.StatusCode != http.StatusOK { return nil, utils.GetErrorFromResponse(r) } @@ -141,6 +183,7 @@ func (c *Client) ClusterDelete(id string) error { if err != nil { return err } + defer r.Body.Close() if r.StatusCode != http.StatusOK { return utils.GetErrorFromResponse(r) } diff --git a/vendor/github.com/heketi/heketi/client/api/go-client/db.go b/vendor/github.com/heketi/heketi/client/api/go-client/db.go new file mode 100644 index 00000000000..72ed129b002 --- /dev/null +++ b/vendor/github.com/heketi/heketi/client/api/go-client/db.go @@ -0,0 +1,52 @@ +// +// Copyright (c) 2018 The heketi Authors +// +// This file is licensed to you under your choice of the GNU Lesser +// General Public License, version 3 or any later version (LGPLv3 or +// later), as published by the Free Software Foundation, +// or under the Apache License, Version 2.0 . +// +// You may not use this file except in compliance with those terms. +// + +package client + +import ( + "io/ioutil" + "net/http" + + "github.com/heketi/heketi/pkg/utils" +) + +// DbDump provides a JSON representation of current state of DB +func (c *Client) DbDump() (string, error) { + req, err := http.NewRequest("GET", c.host+"/db/dump", nil) + if err != nil { + return "", err + } + + // Set token + err = c.setToken(req) + if err != nil { + return "", err + } + + // Send request + r, err := c.do(req) + if err != nil { + return "", err + } + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + return "", utils.GetErrorFromResponse(r) + } + + respBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + return "", err + } + + respJSON := string(respBytes) + return respJSON, nil +} diff --git a/vendor/github.com/heketi/heketi/client/api/go-client/device.go b/vendor/github.com/heketi/heketi/client/api/go-client/device.go index 50412521a56..fd856d8b476 100644 --- a/vendor/github.com/heketi/heketi/client/api/go-client/device.go +++ b/vendor/github.com/heketi/heketi/client/api/go-client/device.go @@ -15,6 +15,7 @@ package client import ( "bytes" "encoding/json" + "io" "net/http" "time" @@ -47,6 +48,7 @@ func (c *Client) DeviceAdd(request *api.DeviceAddRequest) error { if err != nil { return err } + defer r.Body.Close() if r.StatusCode != http.StatusAccepted { return utils.GetErrorFromResponse(r) } @@ -82,6 +84,7 @@ func (c *Client) DeviceInfo(id string) (*api.DeviceInfoResponse, error) { if err != nil { return nil, err } + defer r.Body.Close() if r.StatusCode != http.StatusOK { return nil, utils.GetErrorFromResponse(r) } @@ -89,7 +92,6 @@ func (c *Client) DeviceInfo(id string) (*api.DeviceInfoResponse, error) { // Read JSON response var device api.DeviceInfoResponse err = utils.GetJsonFromResponse(r, &device) - r.Body.Close() if err != nil { return nil, err } @@ -98,9 +100,23 @@ func (c *Client) DeviceInfo(id string) (*api.DeviceInfoResponse, error) { } func (c *Client) DeviceDelete(id string) error { + return c.DeviceDeleteWithOptions(id, nil) +} + +func (c *Client) DeviceDeleteWithOptions( + id string, request *api.DeviceDeleteOptions) error { + + var buf io.Reader + if request != nil { + b, err := json.Marshal(request) + if err != nil { + return err + } + buf = bytes.NewBuffer(b) + } // Create a request - req, err := http.NewRequest("DELETE", c.host+"/devices/"+id, nil) + req, err := http.NewRequest("DELETE", c.host+"/devices/"+id, buf) if err != nil { return err } @@ -116,6 +132,7 @@ func (c *Client) DeviceDelete(id string) error { if err != nil { return err } + defer r.Body.Close() if r.StatusCode != http.StatusAccepted { return utils.GetErrorFromResponse(r) } @@ -161,6 +178,7 @@ func (c *Client) DeviceState(id string, if err != nil { return err } + defer r.Body.Close() if r.StatusCode != http.StatusAccepted { return utils.GetErrorFromResponse(r) } @@ -176,3 +194,71 @@ func (c *Client) DeviceState(id string, return nil } + +func (c *Client) DeviceResync(id string) error { + + // Create a request + req, err := http.NewRequest("GET", c.host+"/devices/"+id+"/resync", nil) + if err != nil { + return err + } + + // Set token + err = c.setToken(req) + if err != nil { + return err + } + + // Send request + r, err := c.do(req) + if err != nil { + return err + } + defer r.Body.Close() + if r.StatusCode != http.StatusAccepted { + return utils.GetErrorFromResponse(r) + } + + // Wait for response + r, err = c.waitForResponseWithTimer(r, time.Millisecond*250) + if err != nil { + return err + } + if r.StatusCode != http.StatusNoContent { + return utils.GetErrorFromResponse(r) + } + + return nil +} + +func (c *Client) DeviceSetTags(id string, request *api.TagsChangeRequest) error { + buffer, err := json.Marshal(request) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", + c.host+"/devices/"+id+"/tags", + bytes.NewBuffer(buffer)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + + // Set token + err = c.setToken(req) + if err != nil { + return err + } + + // Get info + r, err := c.do(req) + if err != nil { + return err + } + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + return utils.GetErrorFromResponse(r) + } + return nil +} diff --git a/vendor/github.com/heketi/heketi/client/api/go-client/logging.go b/vendor/github.com/heketi/heketi/client/api/go-client/logging.go new file mode 100644 index 00000000000..96a29d2ec75 --- /dev/null +++ b/vendor/github.com/heketi/heketi/client/api/go-client/logging.go @@ -0,0 +1,78 @@ +// +// Copyright (c) 2018 The heketi Authors +// +// This file is licensed to you under your choice of the GNU Lesser +// General Public License, version 3 or any later version (LGPLv3 or +// later), as published by the Free Software Foundation, +// or under the Apache License, Version 2.0 . +// +// You may not use this file except in compliance with those terms. +// + +package client + +import ( + "bytes" + "encoding/json" + "net/http" + + "github.com/heketi/heketi/pkg/glusterfs/api" + "github.com/heketi/heketi/pkg/utils" +) + +func (c *Client) LogLevelGet() (*api.LogLevelInfo, error) { + req, err := http.NewRequest("GET", c.host+"/internal/logging", nil) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + // Set token + err = c.setToken(req) + if err != nil { + return nil, err + } + + // Send request + r, err := c.do(req) + if err != nil { + return nil, err + } + var lli api.LogLevelInfo + err = utils.GetJsonFromResponse(r, &lli) + if err != nil { + return nil, err + } + return &lli, nil +} + +func (c *Client) LogLevelSet(request *api.LogLevelInfo) error { + // Marshal request to JSON + buffer, err := json.Marshal(request) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", c.host+"/internal/logging", bytes.NewBuffer(buffer)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + + // Set token + err = c.setToken(req) + if err != nil { + return err + } + + // Send request + r, err := c.do(req) + if err != nil { + return err + } + if r.StatusCode != http.StatusOK { + return utils.GetErrorFromResponse(r) + } + return nil +} diff --git a/vendor/github.com/heketi/heketi/client/api/go-client/node.go b/vendor/github.com/heketi/heketi/client/api/go-client/node.go index 20039c1ef43..da31379d199 100644 --- a/vendor/github.com/heketi/heketi/client/api/go-client/node.go +++ b/vendor/github.com/heketi/heketi/client/api/go-client/node.go @@ -48,6 +48,7 @@ func (c *Client) NodeAdd(request *api.NodeAddRequest) (*api.NodeInfoResponse, er if err != nil { return nil, err } + defer r.Body.Close() if r.StatusCode != http.StatusAccepted { return nil, utils.GetErrorFromResponse(r) } @@ -64,7 +65,6 @@ func (c *Client) NodeAdd(request *api.NodeAddRequest) (*api.NodeInfoResponse, er // Read JSON response var node api.NodeInfoResponse err = utils.GetJsonFromResponse(r, &node) - r.Body.Close() if err != nil { return nil, err } @@ -91,6 +91,7 @@ func (c *Client) NodeInfo(id string) (*api.NodeInfoResponse, error) { if err != nil { return nil, err } + defer r.Body.Close() if r.StatusCode != http.StatusOK { return nil, utils.GetErrorFromResponse(r) } @@ -98,7 +99,6 @@ func (c *Client) NodeInfo(id string) (*api.NodeInfoResponse, error) { // Read JSON response var node api.NodeInfoResponse err = utils.GetJsonFromResponse(r, &node) - r.Body.Close() if err != nil { return nil, err } @@ -125,6 +125,7 @@ func (c *Client) NodeDelete(id string) error { if err != nil { return err } + defer r.Body.Close() if r.StatusCode != http.StatusAccepted { return utils.GetErrorFromResponse(r) } @@ -168,6 +169,7 @@ func (c *Client) NodeState(id string, request *api.StateRequest) error { if err != nil { return err } + defer r.Body.Close() if r.StatusCode != http.StatusAccepted { return utils.GetErrorFromResponse(r) } @@ -183,3 +185,35 @@ func (c *Client) NodeState(id string, request *api.StateRequest) error { return nil } + +func (c *Client) NodeSetTags(id string, request *api.TagsChangeRequest) error { + buffer, err := json.Marshal(request) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", + c.host+"/nodes/"+id+"/tags", + bytes.NewBuffer(buffer)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + + // Set token + err = c.setToken(req) + if err != nil { + return err + } + + // Get info + r, err := c.do(req) + if err != nil { + return err + } + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + return utils.GetErrorFromResponse(r) + } + return nil +} diff --git a/vendor/github.com/heketi/heketi/client/api/go-client/operations.go b/vendor/github.com/heketi/heketi/client/api/go-client/operations.go new file mode 100644 index 00000000000..d641aea5007 --- /dev/null +++ b/vendor/github.com/heketi/heketi/client/api/go-client/operations.go @@ -0,0 +1,46 @@ +// +// Copyright (c) 2018 The heketi Authors +// +// This file is licensed to you under your choice of the GNU Lesser +// General Public License, version 3 or any later version (LGPLv3 or +// later), as published by the Free Software Foundation, +// or under the Apache License, Version 2.0 . +// +// You may not use this file except in compliance with those terms. +// + +package client + +import ( + "net/http" + + "github.com/heketi/heketi/pkg/glusterfs/api" + "github.com/heketi/heketi/pkg/utils" +) + +func (c *Client) OperationsInfo() (*api.OperationsInfo, error) { + req, err := http.NewRequest("GET", c.host+"/operations", nil) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + // Set token + err = c.setToken(req) + if err != nil { + return nil, err + } + + // Send request + r, err := c.do(req) + if err != nil { + return nil, err + } + var oi api.OperationsInfo + err = utils.GetJsonFromResponse(r, &oi) + if err != nil { + return nil, err + } + return &oi, nil +} diff --git a/vendor/github.com/heketi/heketi/client/api/go-client/topology.go b/vendor/github.com/heketi/heketi/client/api/go-client/topology.go index 6c9ffe28c88..549aa7c75c4 100644 --- a/vendor/github.com/heketi/heketi/client/api/go-client/topology.go +++ b/vendor/github.com/heketi/heketi/client/api/go-client/topology.go @@ -33,6 +33,10 @@ func (c *Client) TopologyInfo() (*api.TopologyInfoResponse, error) { Id: clusteri.Id, Volumes: make([]api.VolumeInfoResponse, 0), Nodes: make([]api.NodeInfoResponse, 0), + ClusterFlags: api.ClusterFlags{ + Block: clusteri.Block, + File: clusteri.File, + }, } cluster.Id = clusteri.Id diff --git a/vendor/github.com/heketi/heketi/client/api/go-client/volume.go b/vendor/github.com/heketi/heketi/client/api/go-client/volume.go index 821cfa80837..5b5d919ac5f 100644 --- a/vendor/github.com/heketi/heketi/client/api/go-client/volume.go +++ b/vendor/github.com/heketi/heketi/client/api/go-client/volume.go @@ -51,6 +51,61 @@ func (c *Client) VolumeCreate(request *api.VolumeCreateRequest) ( if err != nil { return nil, err } + defer r.Body.Close() + if r.StatusCode != http.StatusAccepted { + return nil, utils.GetErrorFromResponse(r) + } + + // Wait for response + r, err = c.waitForResponseWithTimer(r, time.Second) + if err != nil { + return nil, err + } + if r.StatusCode != http.StatusOK { + return nil, utils.GetErrorFromResponse(r) + } + + // Read JSON response + var volume api.VolumeInfoResponse + err = utils.GetJsonFromResponse(r, &volume) + if err != nil { + return nil, err + } + + return &volume, nil + +} + +func (c *Client) VolumeSetBlockRestriction(id string, request *api.VolumeBlockRestrictionRequest) ( + *api.VolumeInfoResponse, error) { + + // Marshal request to JSON + buffer, err := json.Marshal(request) + if err != nil { + return nil, err + } + + // Create a request + req, err := http.NewRequest("POST", + c.host+"/volumes/"+id+"/block-restriction", + bytes.NewBuffer(buffer)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + // Set token + err = c.setToken(req) + if err != nil { + return nil, err + } + + // Send request + r, err := c.do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() if r.StatusCode != http.StatusAccepted { return nil, utils.GetErrorFromResponse(r) } @@ -67,7 +122,6 @@ func (c *Client) VolumeCreate(request *api.VolumeCreateRequest) ( // Read JSON response var volume api.VolumeInfoResponse err = utils.GetJsonFromResponse(r, &volume) - r.Body.Close() if err != nil { return nil, err } @@ -105,6 +159,7 @@ func (c *Client) VolumeExpand(id string, request *api.VolumeExpandRequest) ( if err != nil { return nil, err } + defer r.Body.Close() if r.StatusCode != http.StatusAccepted { return nil, utils.GetErrorFromResponse(r) } @@ -121,7 +176,6 @@ func (c *Client) VolumeExpand(id string, request *api.VolumeExpandRequest) ( // Read JSON response var volume api.VolumeInfoResponse err = utils.GetJsonFromResponse(r, &volume) - r.Body.Close() if err != nil { return nil, err } @@ -149,6 +203,7 @@ func (c *Client) VolumeList() (*api.VolumeListResponse, error) { if err != nil { return nil, err } + defer r.Body.Close() if r.StatusCode != http.StatusOK { return nil, utils.GetErrorFromResponse(r) } @@ -182,6 +237,7 @@ func (c *Client) VolumeInfo(id string) (*api.VolumeInfoResponse, error) { if err != nil { return nil, err } + defer r.Body.Close() if r.StatusCode != http.StatusOK { return nil, utils.GetErrorFromResponse(r) } @@ -189,7 +245,6 @@ func (c *Client) VolumeInfo(id string) (*api.VolumeInfoResponse, error) { // Read JSON response var volume api.VolumeInfoResponse err = utils.GetJsonFromResponse(r, &volume) - r.Body.Close() if err != nil { return nil, err } @@ -216,6 +271,7 @@ func (c *Client) VolumeDelete(id string) error { if err != nil { return err } + defer r.Body.Close() if r.StatusCode != http.StatusAccepted { return utils.GetErrorFromResponse(r) } @@ -231,3 +287,52 @@ func (c *Client) VolumeDelete(id string) error { return nil } + +func (c *Client) VolumeClone(id string, request *api.VolumeCloneRequest) (*api.VolumeInfoResponse, error) { + // Marshal request to JSON + buffer, err := json.Marshal(request) + if err != nil { + return nil, err + } + + // Create a request + req, err := http.NewRequest("POST", c.host+"/volumes/"+id+"/clone", bytes.NewBuffer(buffer)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + // Set token + err = c.setToken(req) + if err != nil { + return nil, err + } + + // Send request + r, err := c.do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + if r.StatusCode != http.StatusAccepted { + return nil, utils.GetErrorFromResponse(r) + } + + // Wait for response + r, err = c.waitForResponseWithTimer(r, time.Second) + if err != nil { + return nil, err + } + if r.StatusCode != http.StatusOK { + return nil, utils.GetErrorFromResponse(r) + } + + // Read JSON response + var volume api.VolumeInfoResponse + err = utils.GetJsonFromResponse(r, &volume) + if err != nil { + return nil, err + } + + return &volume, nil +} diff --git a/vendor/github.com/heketi/heketi/pkg/glusterfs/api/BUILD b/vendor/github.com/heketi/heketi/pkg/glusterfs/api/BUILD index 4adc57ab969..53936fa0ad9 100644 --- a/vendor/github.com/heketi/heketi/pkg/glusterfs/api/BUILD +++ b/vendor/github.com/heketi/heketi/pkg/glusterfs/api/BUILD @@ -6,6 +6,10 @@ go_library( importmap = "k8s.io/kubernetes/vendor/github.com/heketi/heketi/pkg/glusterfs/api", importpath = "github.com/heketi/heketi/pkg/glusterfs/api", visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/go-ozzo/ozzo-validation:go_default_library", + "//vendor/github.com/go-ozzo/ozzo-validation/is:go_default_library", + ], ) filegroup( diff --git a/vendor/github.com/heketi/heketi/pkg/glusterfs/api/types.go b/vendor/github.com/heketi/heketi/pkg/glusterfs/api/types.go index c244f5af9b9..759fb9bda3a 100644 --- a/vendor/github.com/heketi/heketi/pkg/glusterfs/api/types.go +++ b/vendor/github.com/heketi/heketi/pkg/glusterfs/api/types.go @@ -18,9 +18,38 @@ package api import ( "fmt" + "regexp" "sort" + + "github.com/go-ozzo/ozzo-validation" + "github.com/go-ozzo/ozzo-validation/is" ) +var ( + // Restricting the deviceName to much smaller subset of Unix Path + // as unix path takes almost everything except NULL + deviceNameRe = regexp.MustCompile("^/[a-zA-Z0-9_.:/-]+$") + + // Volume name constraints decided by looking at + // "cli_validate_volname" function in cli-cmd-parser.c of gluster code + volumeNameRe = regexp.MustCompile("^[a-zA-Z0-9_-]+$") + + blockVolNameRe = regexp.MustCompile("^[a-zA-Z0-9_-]+$") + + tagNameRe = regexp.MustCompile("^[a-zA-Z0-9_.-]+$") +) + +// ValidateUUID is written this way because heketi UUID does not +// conform to neither UUID v4 nor v5. +func ValidateUUID(value interface{}) error { + s, _ := value.(string) + err := validation.Validate(s, validation.RuneLength(32, 32), is.Hexadecimal) + if err != nil { + return fmt.Errorf("%v is not a valid UUID", s) + } + return nil +} + // State type EntryState string @@ -31,6 +60,15 @@ const ( EntryStateFailed EntryState = "failed" ) +func ValidateEntryState(value interface{}) error { + s, _ := value.(EntryState) + err := validation.Validate(s, validation.Required, validation.In(EntryStateOnline, EntryStateOffline, EntryStateFailed)) + if err != nil { + return fmt.Errorf("%v is not valid state", s) + } + return nil +} + type DurabilityType string const ( @@ -39,11 +77,26 @@ const ( DurabilityEC DurabilityType = "disperse" ) +func ValidateDurabilityType(value interface{}) error { + s, _ := value.(DurabilityType) + err := validation.Validate(s, validation.Required, validation.In(DurabilityReplicate, DurabilityDistributeOnly, DurabilityEC)) + if err != nil { + return fmt.Errorf("%v is not a valid durability type", s) + } + return nil +} + // Common type StateRequest struct { State EntryState `json:"state"` } +func (statereq StateRequest) Validate() error { + return validation.ValidateStruct(&statereq, + validation.Field(&statereq.State, validation.Required, validation.By(ValidateEntryState)), + ) +} + // Storage values in KB type StorageSize struct { Total uint64 `json:"total"` @@ -56,6 +109,35 @@ type HostAddresses struct { Storage sort.StringSlice `json:"storage"` } +func ValidateManagementHostname(value interface{}) error { + s, _ := value.(sort.StringSlice) + for _, fqdn := range s { + err := validation.Validate(fqdn, validation.Required, is.Host) + if err != nil { + return fmt.Errorf("%v is not a valid manage hostname", s) + } + } + return nil +} + +func ValidateStorageHostname(value interface{}) error { + s, _ := value.(sort.StringSlice) + for _, ip := range s { + err := validation.Validate(ip, validation.Required, is.Host) + if err != nil { + return fmt.Errorf("%v is not a valid storage hostname", s) + } + } + return nil +} + +func (hostadd HostAddresses) Validate() error { + return validation.ValidateStruct(&hostadd, + validation.Field(&hostadd.Manage, validation.Required, validation.By(ValidateManagementHostname)), + validation.Field(&hostadd.Storage, validation.Required, validation.By(ValidateStorageHostname)), + ) +} + // Brick type BrickInfo struct { Id string `json:"id"` @@ -70,12 +152,29 @@ type BrickInfo struct { // Device type Device struct { - Name string `json:"name"` + Name string `json:"name"` + Tags map[string]string `json:"tags,omitempty"` +} + +func (dev Device) Validate() error { + return validation.ValidateStruct(&dev, + validation.Field(&dev.Name, validation.Required, validation.Match(deviceNameRe)), + validation.Field(&dev.Tags, validation.By(ValidateTags)), + ) } type DeviceAddRequest struct { Device - NodeId string `json:"node"` + NodeId string `json:"node"` + DestroyData bool `json:"destroydata,omitempty"` +} + +func (devAddReq DeviceAddRequest) Validate() error { + return validation.ValidateStruct(&devAddReq, + validation.Field(&devAddReq.Device, validation.Required), + validation.Field(&devAddReq.NodeId, validation.Required, validation.By(ValidateUUID)), + validation.Field(&devAddReq.DestroyData, validation.In(true, false)), + ) } type DeviceInfo struct { @@ -92,9 +191,19 @@ type DeviceInfoResponse struct { // Node type NodeAddRequest struct { - Zone int `json:"zone"` - Hostnames HostAddresses `json:"hostnames"` - ClusterId string `json:"cluster"` + Zone int `json:"zone"` + Hostnames HostAddresses `json:"hostnames"` + ClusterId string `json:"cluster"` + Tags map[string]string `json:"tags,omitempty"` +} + +func (req NodeAddRequest) Validate() error { + return validation.ValidateStruct(&req, + validation.Field(&req.Zone, validation.Required, validation.Min(1)), + validation.Field(&req.Hostnames, validation.Required), + validation.Field(&req.ClusterId, validation.Required, validation.By(ValidateUUID)), + validation.Field(&req.Tags, validation.By(ValidateTags)), + ) } type NodeInfo struct { @@ -109,20 +218,37 @@ type NodeInfoResponse struct { } // Cluster + +type ClusterFlags struct { + Block bool `json:"block"` + File bool `json:"file"` +} + type Cluster struct { Volumes []VolumeInfoResponse `json:"volumes"` Nodes []NodeInfoResponse `json:"nodes"` Id string `json:"id"` + ClusterFlags } type TopologyInfoResponse struct { ClusterList []Cluster `json:"clusters"` } +type ClusterCreateRequest struct { + ClusterFlags +} + +type ClusterSetFlagsRequest struct { + ClusterFlags +} + type ClusterInfoResponse struct { Id string `json:"id"` Nodes sort.StringSlice `json:"nodes"` Volumes sort.StringSlice `json:"volumes"` + ClusterFlags + BlockVolumes sort.StringSlice `json:"blockvolumes"` } type ClusterListResponse struct { @@ -147,19 +273,57 @@ type VolumeDurabilityInfo struct { } type VolumeCreateRequest struct { - // Size in GB + // Size in GiB Size int `json:"size"` Clusters []string `json:"clusters,omitempty"` Name string `json:"name"` Durability VolumeDurabilityInfo `json:"durability,omitempty"` Gid int64 `json:"gid,omitempty"` GlusterVolumeOptions []string `json:"glustervolumeoptions,omitempty"` + Block bool `json:"block,omitempty"` Snapshot struct { Enable bool `json:"enable"` Factor float32 `json:"factor"` } `json:"snapshot"` } +func (volCreateRequest VolumeCreateRequest) Validate() error { + return validation.ValidateStruct(&volCreateRequest, + validation.Field(&volCreateRequest.Size, validation.Required, validation.Min(1)), + validation.Field(&volCreateRequest.Clusters, validation.By(ValidateUUID)), + validation.Field(&volCreateRequest.Name, validation.Match(volumeNameRe)), + validation.Field(&volCreateRequest.Durability, validation.Skip), + validation.Field(&volCreateRequest.Gid, validation.Skip), + validation.Field(&volCreateRequest.GlusterVolumeOptions, validation.Skip), + validation.Field(&volCreateRequest.Block, validation.In(true, false)), + // This is possibly a bug in validation lib, ignore next two lines for now + // validation.Field(&volCreateRequest.Snapshot.Enable, validation.In(true, false)), + // validation.Field(&volCreateRequest.Snapshot.Factor, validation.Min(1.0)), + ) +} + +type BlockRestriction string + +const ( + Unrestricted BlockRestriction = "" + Locked BlockRestriction = "locked" + LockedByUpdate BlockRestriction = "locked-by-update" +) + +func (br BlockRestriction) String() string { + switch br { + case Unrestricted: + return "(none)" + case Locked: + return "locked" + case LockedByUpdate: + return "locked-by-update" + default: + return "unknown" + + } +} + type VolumeInfo struct { VolumeCreateRequest Id string `json:"id"` @@ -171,6 +335,12 @@ type VolumeInfo struct { Options map[string]string `json:"options"` } `json:"glusterfs"` } `json:"mount"` + BlockInfo struct { + FreeSize int `json:"freesize,omitempty"` + ReservedSize int `json:"reservedsize,omitempty"` + BlockVolumes sort.StringSlice `json:"blockvolume,omitempty"` + Restriction BlockRestriction `json:"restriction,omitempty"` + } `json:"blockinfo,omitempty"` } type VolumeInfoResponse struct { @@ -186,6 +356,132 @@ type VolumeExpandRequest struct { Size int `json:"expand_size"` } +func (volExpandReq VolumeExpandRequest) Validate() error { + return validation.ValidateStruct(&volExpandReq, + validation.Field(&volExpandReq.Size, validation.Required, validation.Min(1)), + ) +} + +type VolumeCloneRequest struct { + Name string `json:"name,omitempty"` +} + +func (vcr VolumeCloneRequest) Validate() error { + return validation.ValidateStruct(&vcr, + validation.Field(&vcr.Name, validation.Match(volumeNameRe)), + ) +} + +type VolumeBlockRestrictionRequest struct { + Restriction BlockRestriction `json:"restriction"` +} + +func (vbrr VolumeBlockRestrictionRequest) Validate() error { + return validation.ValidateStruct(&vbrr, + validation.Field(&vbrr.Restriction, + validation.In(Unrestricted, Locked))) +} + +// BlockVolume + +type BlockVolumeCreateRequest struct { + // Size in GiB + Size int `json:"size"` + Clusters []string `json:"clusters,omitempty"` + Name string `json:"name"` + Hacount int `json:"hacount,omitempty"` + Auth bool `json:"auth,omitempty"` +} + +func (blockVolCreateReq BlockVolumeCreateRequest) Validate() error { + return validation.ValidateStruct(&blockVolCreateReq, + validation.Field(&blockVolCreateReq.Size, validation.Required, validation.Min(1)), + validation.Field(&blockVolCreateReq.Clusters, validation.By(ValidateUUID)), + validation.Field(&blockVolCreateReq.Name, validation.Match(blockVolNameRe)), + validation.Field(&blockVolCreateReq.Hacount, validation.Min(1)), + validation.Field(&blockVolCreateReq.Auth, validation.Skip), + ) +} + +type BlockVolumeInfo struct { + BlockVolumeCreateRequest + Id string `json:"id"` + BlockVolume struct { + Hosts []string `json:"hosts"` + Iqn string `json:"iqn"` + Lun int `json:"lun"` + Username string `json:"username"` + Password string `json:"password"` + /* + Options map[string]string `json:"options"` // needed?... + */ + } `json:"blockvolume"` + Cluster string `json:"cluster,omitempty"` + BlockHostingVolume string `json:"blockhostingvolume,omitempty"` +} + +type BlockVolumeInfoResponse struct { + BlockVolumeInfo +} + +type BlockVolumeListResponse struct { + BlockVolumes []string `json:"blockvolumes"` +} + +type LogLevelInfo struct { + // should contain one or more logger to log-level-name mapping + LogLevel map[string]string `json:"loglevel"` +} + +type TagsChangeType string + +const ( + UnknownTagsChangeType TagsChangeType = "" + SetTags TagsChangeType = "set" + UpdateTags TagsChangeType = "update" + DeleteTags TagsChangeType = "delete" +) + +// Common tag post body +type TagsChangeRequest struct { + Tags map[string]string `json:"tags"` + Change TagsChangeType `json:"change_type"` +} + +func (tcr TagsChangeRequest) Validate() error { + return validation.ValidateStruct(&tcr, + validation.Field(&tcr.Tags, validation.By(ValidateTags)), + validation.Field(&tcr.Change, + validation.Required, + validation.In(SetTags, UpdateTags, DeleteTags))) +} + +func ValidateTags(v interface{}) error { + t, ok := v.(map[string]string) + if !ok { + return fmt.Errorf("tags must be a map of strings to strings") + } + if len(t) > 32 { + return fmt.Errorf("too many tags specified (%v), up to %v supported", + len(t), 32) + } + for k, v := range t { + if len(k) == 0 { + return fmt.Errorf("tag names may not be empty") + } + if err := validation.Validate(k, validation.RuneLength(1, 32)); err != nil { + return fmt.Errorf("tag name %v: %v", k, err) + } + if err := validation.Validate(v, validation.RuneLength(0, 64)); err != nil { + return fmt.Errorf("value of tag %v: %v", k, err) + } + if !tagNameRe.MatchString(k) { + return fmt.Errorf("invalid characters in tag name %+v", k) + } + } + return nil +} + // Constructors func NewVolumeInfoResponse() *VolumeInfoResponse { @@ -205,6 +501,11 @@ func (v *VolumeInfoResponse) String() string { "Cluster Id: %v\n"+ "Mount: %v\n"+ "Mount Options: backup-volfile-servers=%v\n"+ + "Block: %v\n"+ + "Free Size: %v\n"+ + "Reserved Size: %v\n"+ + "Block Hosting Restriction: %v\n"+ + "Block Volumes: %v\n"+ "Durability Type: %v\n", v.Name, v.Size, @@ -212,6 +513,11 @@ func (v *VolumeInfoResponse) String() string { v.Cluster, v.Mount.GlusterFS.MountPoint, v.Mount.GlusterFS.Options["backup-volfile-servers"], + v.Block, + v.BlockInfo.FreeSize, + v.BlockInfo.ReservedSize, + v.BlockInfo.Restriction, + v.BlockInfo.BlockVolumes, v.Durability.Type) switch v.Durability.Type { @@ -248,3 +554,90 @@ func (v *VolumeInfoResponse) String() string { return s } + +func NewBlockVolumeInfoResponse() *BlockVolumeInfoResponse { + + info := &BlockVolumeInfoResponse{} + // Nothing to Construct now maybe for future + + return info +} + +// String functions +func (v *BlockVolumeInfoResponse) String() string { + s := fmt.Sprintf("Name: %v\n"+ + "Size: %v\n"+ + "Volume Id: %v\n"+ + "Cluster Id: %v\n"+ + "Hosts: %v\n"+ + "IQN: %v\n"+ + "LUN: %v\n"+ + "Hacount: %v\n"+ + "Username: %v\n"+ + "Password: %v\n"+ + "Block Hosting Volume: %v\n", + v.Name, + v.Size, + v.Id, + v.Cluster, + v.BlockVolume.Hosts, + v.BlockVolume.Iqn, + v.BlockVolume.Lun, + v.Hacount, + v.BlockVolume.Username, + v.BlockVolume.Password, + v.BlockHostingVolume) + + /* + s += "\nBricks:\n" + for _, b := range v.Bricks { + s += fmt.Sprintf("Id: %v\n"+ + "Path: %v\n"+ + "Size (GiB): %v\n"+ + "Node: %v\n"+ + "Device: %v\n\n", + b.Id, + b.Path, + b.Size/(1024*1024), + b.NodeId, + b.DeviceId) + } + */ + + return s +} + +type OperationsInfo struct { + Total uint64 `json:"total"` + InFlight uint64 `json:"in_flight"` + // state based counts: + Stale uint64 `json:"stale"` + New uint64 `json:"new"` +} + +type AdminState string + +const ( + AdminStateNormal AdminState = "normal" + AdminStateReadOnly AdminState = "read-only" + AdminStateLocal AdminState = "local-client" +) + +type AdminStatus struct { + State AdminState `json:"state"` +} + +func (as AdminStatus) Validate() error { + return validation.ValidateStruct(&as, + validation.Field(&as.State, + validation.Required, + validation.In(AdminStateNormal, AdminStateReadOnly, AdminStateLocal))) +} + +// DeviceDeleteOptions is used to specify additional behavior for device +// deletes. +type DeviceDeleteOptions struct { + // force heketi to forget about a device, possibly + // orphaning metadata on the node + ForceForget bool `json:"forceforget"` +} diff --git a/vendor/github.com/heketi/heketi/pkg/utils/BUILD b/vendor/github.com/heketi/heketi/pkg/utils/BUILD index e7cbd912983..bb83291af8a 100644 --- a/vendor/github.com/heketi/heketi/pkg/utils/BUILD +++ b/vendor/github.com/heketi/heketi/pkg/utils/BUILD @@ -5,17 +5,11 @@ go_library( srcs = [ "bodystring.go", "jsonutils.go", - "log.go", - "sortedstrings.go", "statusgroup.go", - "stringset.go", - "stringstack.go", - "uuid.go", ], importmap = "k8s.io/kubernetes/vendor/github.com/heketi/heketi/pkg/utils", importpath = "github.com/heketi/heketi/pkg/utils", visibility = ["//visibility:public"], - deps = ["//vendor/github.com/lpabon/godbc:go_default_library"], ) filegroup( diff --git a/vendor/github.com/heketi/heketi/pkg/utils/bodystring.go b/vendor/github.com/heketi/heketi/pkg/utils/bodystring.go index 468db04e454..8a7b4aca1bc 100644 --- a/vendor/github.com/heketi/heketi/pkg/utils/bodystring.go +++ b/vendor/github.com/heketi/heketi/pkg/utils/bodystring.go @@ -3,14 +3,18 @@ // // This file is licensed to you under your choice of the GNU Lesser // General Public License, version 3 or any later version (LGPLv3 or -// later), or the GNU General Public License, version 2 (GPLv2), in all -// cases as published by the Free Software Foundation. +// later), as published by the Free Software Foundation, +// or under the Apache License, Version 2.0 . +// +// You may not use this file except in compliance with those terms. // package utils import ( "errors" + "fmt" "io" "io/ioutil" "net/http" @@ -20,10 +24,10 @@ import ( // Return the body from a response as a string func GetStringFromResponse(r *http.Response) (string, error) { body, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength)) + defer r.Body.Close() if err != nil { return "", err } - r.Body.Close() return string(body), nil } @@ -33,5 +37,10 @@ func GetErrorFromResponse(r *http.Response) error { if err != nil { return err } - return errors.New(strings.TrimSpace(s)) + + s = strings.TrimSpace(s) + if len(s) == 0 { + return fmt.Errorf("server did not provide a message (status %v: %v)", r.StatusCode, http.StatusText(r.StatusCode)) + } + return errors.New(s) } diff --git a/vendor/github.com/heketi/heketi/pkg/utils/jsonutils.go b/vendor/github.com/heketi/heketi/pkg/utils/jsonutils.go index 77781d9c938..6fb958d8ff4 100644 --- a/vendor/github.com/heketi/heketi/pkg/utils/jsonutils.go +++ b/vendor/github.com/heketi/heketi/pkg/utils/jsonutils.go @@ -3,8 +3,11 @@ // // This file is licensed to you under your choice of the GNU Lesser // General Public License, version 3 or any later version (LGPLv3 or -// later), or the GNU General Public License, version 2 (GPLv2), in all -// cases as published by the Free Software Foundation. +// later), as published by the Free Software Foundation, +// or under the Apache License, Version 2.0 . +// +// You may not use this file except in compliance with those terms. // package utils diff --git a/vendor/github.com/heketi/heketi/pkg/utils/log.go b/vendor/github.com/heketi/heketi/pkg/utils/log.go deleted file mode 100644 index a9451ed7e57..00000000000 --- a/vendor/github.com/heketi/heketi/pkg/utils/log.go +++ /dev/null @@ -1,155 +0,0 @@ -// -// Copyright (c) 2015 The heketi Authors -// -// This file is licensed to you under your choice of the GNU Lesser -// General Public License, version 3 or any later version (LGPLv3 or -// later), or the GNU General Public License, version 2 (GPLv2), in all -// cases as published by the Free Software Foundation. -// - -package utils - -import ( - "fmt" - "io" - "log" - "os" - "runtime" - "strings" - - "github.com/lpabon/godbc" -) - -type LogLevel int - -// Log levels -const ( - LEVEL_NOLOG LogLevel = iota - LEVEL_CRITICAL - LEVEL_ERROR - LEVEL_WARNING - LEVEL_INFO - LEVEL_DEBUG -) - -var ( - stderr io.Writer = os.Stderr - stdout io.Writer = os.Stdout -) - -type Logger struct { - critlog, errorlog, infolog *log.Logger - debuglog, warninglog *log.Logger - - level LogLevel -} - -func logWithLongFile(l *log.Logger, format string, v ...interface{}) { - _, file, line, _ := runtime.Caller(2) - - // Shorten the path. - // From - // /builddir/build/BUILD/heketi-3f4a5b1b6edff87232e8b24533c53b4151ebd9c7/src/github.com/heketi/heketi/apps/glusterfs/volume_entry.go - // to - // src/github.com/heketi/heketi/apps/glusterfs/volume_entry.go - i := strings.Index(file, "/src/") - if i == -1 { - i = 0 - } - - l.Print(fmt.Sprintf("%v:%v: ", file[i:], line) + - fmt.Sprintf(format, v...)) -} - -// Create a new logger -func NewLogger(prefix string, level LogLevel) *Logger { - godbc.Require(level >= 0, level) - godbc.Require(level <= LEVEL_DEBUG, level) - - l := &Logger{} - - if level == LEVEL_NOLOG { - l.level = LEVEL_DEBUG - } else { - l.level = level - } - - l.critlog = log.New(stderr, prefix+" CRITICAL ", log.LstdFlags) - l.errorlog = log.New(stderr, prefix+" ERROR ", log.LstdFlags) - l.warninglog = log.New(stdout, prefix+" WARNING ", log.LstdFlags) - l.infolog = log.New(stdout, prefix+" INFO ", log.LstdFlags) - l.debuglog = log.New(stdout, prefix+" DEBUG ", log.LstdFlags) - - godbc.Ensure(l.critlog != nil) - godbc.Ensure(l.errorlog != nil) - godbc.Ensure(l.warninglog != nil) - godbc.Ensure(l.infolog != nil) - godbc.Ensure(l.debuglog != nil) - - return l -} - -// Return current level -func (l *Logger) Level() LogLevel { - return l.level -} - -// Set level -func (l *Logger) SetLevel(level LogLevel) { - l.level = level -} - -// Log critical information -func (l *Logger) Critical(format string, v ...interface{}) { - if l.level >= LEVEL_CRITICAL { - logWithLongFile(l.critlog, format, v...) - } -} - -// Log error string -func (l *Logger) LogError(format string, v ...interface{}) error { - if l.level >= LEVEL_ERROR { - logWithLongFile(l.errorlog, format, v...) - } - - return fmt.Errorf(format, v...) -} - -// Log error variable -func (l *Logger) Err(err error) error { - if l.level >= LEVEL_ERROR { - logWithLongFile(l.errorlog, "%v", err) - } - - return err -} - -// Log warning information -func (l *Logger) Warning(format string, v ...interface{}) { - if l.level >= LEVEL_WARNING { - l.warninglog.Printf(format, v...) - } -} - -// Log error variable as a warning -func (l *Logger) WarnErr(err error) error { - if l.level >= LEVEL_WARNING { - logWithLongFile(l.warninglog, "%v", err) - } - - return err -} - -// Log string -func (l *Logger) Info(format string, v ...interface{}) { - if l.level >= LEVEL_INFO { - l.infolog.Printf(format, v...) - } -} - -// Log string as debug -func (l *Logger) Debug(format string, v ...interface{}) { - if l.level >= LEVEL_DEBUG { - logWithLongFile(l.debuglog, format, v...) - } -} diff --git a/vendor/github.com/heketi/heketi/pkg/utils/sortedstrings.go b/vendor/github.com/heketi/heketi/pkg/utils/sortedstrings.go deleted file mode 100644 index 985230d9f32..00000000000 --- a/vendor/github.com/heketi/heketi/pkg/utils/sortedstrings.go +++ /dev/null @@ -1,33 +0,0 @@ -// -// Copyright (c) 2015 The heketi Authors -// -// This file is licensed to you under your choice of the GNU Lesser -// General Public License, version 3 or any later version (LGPLv3 or -// later), or the GNU General Public License, version 2 (GPLv2), in all -// cases as published by the Free Software Foundation. -// - -package utils - -import ( - "sort" -) - -// Check if a sorted string list has a string -func SortedStringHas(s sort.StringSlice, x string) bool { - index := s.Search(x) - if index == len(s) { - return false - } - return s[s.Search(x)] == x -} - -// Delete a string from a sorted string list -func SortedStringsDelete(s sort.StringSlice, x string) sort.StringSlice { - index := s.Search(x) - if len(s) != index && s[index] == x { - s = append(s[:index], s[index+1:]...) - } - - return s -} diff --git a/vendor/github.com/heketi/heketi/pkg/utils/statusgroup.go b/vendor/github.com/heketi/heketi/pkg/utils/statusgroup.go index 7c678a5031b..c0354c964c4 100644 --- a/vendor/github.com/heketi/heketi/pkg/utils/statusgroup.go +++ b/vendor/github.com/heketi/heketi/pkg/utils/statusgroup.go @@ -3,8 +3,11 @@ // // This file is licensed to you under your choice of the GNU Lesser // General Public License, version 3 or any later version (LGPLv3 or -// later), or the GNU General Public License, version 2 (GPLv2), in all -// cases as published by the Free Software Foundation. +// later), as published by the Free Software Foundation, +// or under the Apache License, Version 2.0 . +// +// You may not use this file except in compliance with those terms. // package utils diff --git a/vendor/github.com/heketi/heketi/pkg/utils/stringset.go b/vendor/github.com/heketi/heketi/pkg/utils/stringset.go deleted file mode 100644 index 8d7f3ca780d..00000000000 --- a/vendor/github.com/heketi/heketi/pkg/utils/stringset.go +++ /dev/null @@ -1,44 +0,0 @@ -// -// Copyright (c) 2015 The heketi Authors -// -// This file is licensed to you under your choice of the GNU Lesser -// General Public License, version 3 or any later version (LGPLv3 or -// later), or the GNU General Public License, version 2 (GPLv2), in all -// cases as published by the Free Software Foundation. -// - -package utils - -import ( - "sort" -) - -type StringSet struct { - Set sort.StringSlice -} - -// Create a string set. -// -// A string set is a list where each element appears only once -func NewStringSet() *StringSet { - return &StringSet{ - Set: make(sort.StringSlice, 0), - } -} - -// Add a string to the string set -func (s *StringSet) Add(v string) { - if !SortedStringHas(s.Set, v) { - s.Set = append(s.Set, v) - s.Set.Sort() - } -} - -// Return string list -func (s *StringSet) Strings() []string { - return s.Set -} - -func (s *StringSet) Len() int { - return len(s.Set) -} diff --git a/vendor/github.com/heketi/heketi/pkg/utils/stringstack.go b/vendor/github.com/heketi/heketi/pkg/utils/stringstack.go deleted file mode 100644 index c1b48e219aa..00000000000 --- a/vendor/github.com/heketi/heketi/pkg/utils/stringstack.go +++ /dev/null @@ -1,33 +0,0 @@ -// -// Copyright (c) 2015 The heketi Authors -// -// This file is licensed to you under your choice of the GNU Lesser -// General Public License, version 3 or any later version (LGPLv3 or -// later), or the GNU General Public License, version 2 (GPLv2), in all -// cases as published by the Free Software Foundation. -// - -package utils - -type StringStack struct { - list []string -} - -func NewStringStack() *StringStack { - a := &StringStack{} - a.list = make([]string, 0) - return a -} - -func (a *StringStack) IsEmpty() bool { - return len(a.list) == 0 -} - -func (a *StringStack) Pop() (x string) { - x, a.list = a.list[0], a.list[1:len(a.list)] - return -} - -func (a *StringStack) Push(x string) { - a.list = append(a.list, x) -} diff --git a/vendor/github.com/heketi/heketi/pkg/utils/uuid.go b/vendor/github.com/heketi/heketi/pkg/utils/uuid.go deleted file mode 100644 index 39e743d4dad..00000000000 --- a/vendor/github.com/heketi/heketi/pkg/utils/uuid.go +++ /dev/null @@ -1,27 +0,0 @@ -// -// Copyright (c) 2015 The heketi Authors -// -// This file is licensed to you under your choice of the GNU Lesser -// General Public License, version 3 or any later version (LGPLv3 or -// later), or the GNU General Public License, version 2 (GPLv2), in all -// cases as published by the Free Software Foundation. - -package utils - -// From http://www.ashishbanerjee.com/home/go/go-generate-uuid - -import ( - "crypto/rand" - "encoding/hex" - "github.com/lpabon/godbc" -) - -// Return a 16-byte uuid -func GenUUID() string { - uuid := make([]byte, 16) - n, err := rand.Read(uuid) - godbc.Check(n == len(uuid), n, len(uuid)) - godbc.Check(err == nil, err) - - return hex.EncodeToString(uuid) -} diff --git a/vendor/github.com/karrick/godirwalk/.gitignore b/vendor/github.com/karrick/godirwalk/.gitignore new file mode 100644 index 00000000000..a1338d68517 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/.gitignore @@ -0,0 +1,14 @@ +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ diff --git a/vendor/github.com/karrick/godirwalk/BUILD b/vendor/github.com/karrick/godirwalk/BUILD new file mode 100644 index 00000000000..d126c13c867 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/BUILD @@ -0,0 +1,35 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "dirent.go", + "doc.go", + "readdir.go", + "readdir_unix.go", + "readdir_windows.go", + "walk.go", + "withFileno.go", + "withIno.go", + "withNamlen.go", + "withoutNamlen.go", + ], + importmap = "k8s.io/kubernetes/vendor/github.com/karrick/godirwalk", + importpath = "github.com/karrick/godirwalk", + visibility = ["//visibility:public"], + deps = ["//vendor/github.com/pkg/errors:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/karrick/godirwalk/LICENSE b/vendor/github.com/karrick/godirwalk/LICENSE new file mode 100644 index 00000000000..01ce194c80d --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/LICENSE @@ -0,0 +1,25 @@ +BSD 2-Clause License + +Copyright (c) 2017, Karrick McDermott +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/karrick/godirwalk/README.md b/vendor/github.com/karrick/godirwalk/README.md new file mode 100644 index 00000000000..4f9922fefb0 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/README.md @@ -0,0 +1,208 @@ +# godirwalk + +`godirwalk` is a library for traversing a directory tree on a file +system. + +In short, why do I use this library? + +1. It's faster than `filepath.Walk`. +1. It's more correct on Windows than `filepath.Walk`. +1. It's more easy to use than `filepath.Walk`. +1. It's more flexible than `filepath.Walk`. + +## Usage Example + +Additional examples are provided in the `examples/` subdirectory. + +This library will normalize the provided top level directory name +based on the os-specific path separator by calling `filepath.Clean` on +its first argument. However it always provides the pathname created by +using the correct os-specific path separator when invoking the +provided callback function. + +```Go + dirname := "some/directory/root" + err := godirwalk.Walk(dirname, &godirwalk.Options{ + Callback: func(osPathname string, de *godirwalk.Dirent) error { + fmt.Printf("%s %s\n", de.ModeType(), osPathname) + return nil + }, + Unsorted: true, // (optional) set true for faster yet non-deterministic enumeration (see godoc) + }) +``` + +This library not only provides functions for traversing a file system +directory tree, but also for obtaining a list of immediate descendants +of a particular directory, typically much more quickly than using +`os.ReadDir` or `os.ReadDirnames`. + +Documentation is available via +[![GoDoc](https://godoc.org/github.com/karrick/godirwalk?status.svg)](https://godoc.org/github.com/karrick/godirwalk). + +## Description + +Here's why I use `godirwalk` in preference to `filepath.Walk`, +`os.ReadDir`, and `os.ReadDirnames`. + +### It's faster than `filepath.Walk` + +When compared against `filepath.Walk` in benchmarks, it has been +observed to run between five and ten times the speed on darwin, at +speeds comparable to the that of the unix `find` utility; about twice +the speed on linux; and about four times the speed on Windows. + +How does it obtain this performance boost? It does less work to give +you nearly the same output. This library calls the same `syscall` +functions to do the work, but it makes fewer calls, does not throw +away information that it might need, and creates less memory churn +along the way by reusing the same scratch buffer rather than +reallocating a new buffer every time it reads data from the operating +system. + +While traversing a file system directory tree, `filepath.Walk` obtains +the list of immediate descendants of a directory, and throws away the +file system node type information provided by the operating system +that comes with the node's name. Then, immediately prior to invoking +the callback function, `filepath.Walk` invokes `os.Stat` for each +node, and passes the returned `os.FileInfo` information to the +callback. + +While the `os.FileInfo` information provided by `os.Stat` is extremely +helpful--and even includes the `os.FileMode` data--providing it +requires an additional system call for each node. + +Because most callbacks only care about what the node type is, this +library does not throw the type information away, but rather provides +that information to the callback function in the form of a +`os.FileMode` value. Note that the provided `os.FileMode` value that +this library provides only has the node type information, and does not +have the permission bits, sticky bits, or other information from the +file's mode. If the callback does care about a particular node's +entire `os.FileInfo` data structure, the callback can easiy invoke +`os.Stat` when needed, and only when needed. + +#### Benchmarks + +##### macOS + +```Bash +go test -bench=. +goos: darwin +goarch: amd64 +pkg: github.com/karrick/godirwalk +BenchmarkFilepathWalk-8 1 3001274570 ns/op +BenchmarkGoDirWalk-8 3 465573172 ns/op +BenchmarkFlameGraphFilepathWalk-8 1 6957916936 ns/op +BenchmarkFlameGraphGoDirWalk-8 1 4210582571 ns/op +PASS +ok github.com/karrick/godirwalk 16.822s +``` + +##### Linux + +```Bash +go test -bench=. +goos: linux +goarch: amd64 +pkg: github.com/karrick/godirwalk +BenchmarkFilepathWalk-12 1 1609189170 ns/op +BenchmarkGoDirWalk-12 5 211336628 ns/op +BenchmarkFlameGraphFilepathWalk-12 1 3968119932 ns/op +BenchmarkFlameGraphGoDirWalk-12 1 2139598998 ns/op +PASS +ok github.com/karrick/godirwalk 9.007s +``` + +### It's more correct on Windows than `filepath.Walk` + +I did not previously care about this either, but humor me. We all love +how we can write once and run everywhere. It is essential for the +language's adoption, growth, and success, that the software we create +can run unmodified on all architectures and operating systems +supported by Go. + +When the traversed file system has a logical loop caused by symbolic +links to directories, on unix `filepath.Walk` ignores symbolic links +and traverses the entire directory tree without error. On Windows +however, `filepath.Walk` will continue following directory symbolic +links, even though it is not supposed to, eventually causing +`filepath.Walk` to terminate early and return an error when the +pathname gets too long from concatenating endless loops of symbolic +links onto the pathname. This error comes from Windows, passes through +`filepath.Walk`, and to the upstream client running `filepath.Walk`. + +The takeaway is that behavior is different based on which platform +`filepath.Walk` is running. While this is clearly not intentional, +until it is fixed in the standard library, it presents a compatibility +problem. + +This library correctly identifies symbolic links that point to +directories and will only follow them when `FollowSymbolicLinks` is +set to true. Behavior on Windows and other operating systems is +identical. + +### It's more easy to use than `filepath.Walk` + +Since this library does not invoke `os.Stat` on every file system node +it encounters, there is no possible error event for the callback +function to filter on. The third argument in the `filepath.WalkFunc` +function signature to pass the error from `os.Stat` to the callback +function is no longer necessary, and thus eliminated from signature of +the callback function from this library. + +Also, `filepath.Walk` invokes the callback function with a solidus +delimited pathname regardless of the os-specific path separator. This +library invokes the callback function with the os-specific pathname +separator, obviating a call to `filepath.Clean` in the callback +function for each node prior to actually using the provided pathname. + +In other words, even on Windows, `filepath.Walk` will invoke the +callback with `some/path/to/foo.txt`, requiring well written clients +to perform pathname normalization for every file prior to working with +the specified file. In truth, many clients developed on unix and not +tested on Windows neglect this subtlety, and will result in software +bugs when running on Windows. This library would invoke the callback +function with `some\path\to\foo.txt` for the same file when running on +Windows, eliminating the need to normalize the pathname by the client, +and lessen the likelyhood that a client will work on unix but not on +Windows. + +### It's more flexible than `filepath.Walk` + +#### Configurable Handling of Symbolic Links + +The default behavior of this library is to ignore symbolic links to +directories when walking a directory tree, just like `filepath.Walk` +does. However, it does invoke the callback function with each node it +finds, including symbolic links. If a particular use case exists to +follow symbolic links when traversing a directory tree, this library +can be invoked in manner to do so, by setting the +`FollowSymbolicLinks` parameter to true. + +#### Configurable Sorting of Directory Children + +The default behavior of this library is to always sort the immediate +descendants of a directory prior to visiting each node, just like +`filepath.Walk` does. This is usually the desired behavior. However, +this does come at a performance penalty to sort the names when a +directory node has many entries. If a particular use case exists that +does not require sorting the directory's immediate descendants prior +to visiting its nodes, this library will skip the sorting step when +the `Unsorted` parameter is set to true. + +#### Configurable Post Children Callback + +This library provides upstream code with the ability to specify a +callback to be invoked for each directory after its children are +processed. This has been used to recursively delete empty directories +after traversing the file system in a more efficient manner. See the +`examples/clean-empties` directory for an example of this usage. + +#### Configurable Error Callback + +This library provides upstream code with the ability to specify a +callback to be invoked for errors that the operating system returns, +allowing the upstream code to determine the next course of action to +take, whether to halt walking the hierarchy, as it would do were no +error callback provided, or skip the node that caused the error. See +the `examples/walk-fast` directory for an example of this usage. diff --git a/vendor/github.com/karrick/godirwalk/dirent.go b/vendor/github.com/karrick/godirwalk/dirent.go new file mode 100644 index 00000000000..5a277224802 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/dirent.go @@ -0,0 +1,74 @@ +package godirwalk + +import ( + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// Dirent stores the name and file system mode type of discovered file system +// entries. +type Dirent struct { + name string + modeType os.FileMode +} + +// NewDirent returns a newly initialized Dirent structure, or an error. This +// function does not follow symbolic links. +// +// This function is rarely used, as Dirent structures are provided by other +// functions in this library that read and walk directories. +func NewDirent(osPathname string) (*Dirent, error) { + fi, err := os.Lstat(osPathname) + if err != nil { + return nil, errors.Wrap(err, "cannot lstat") + } + return &Dirent{ + name: filepath.Base(osPathname), + modeType: fi.Mode() & os.ModeType, + }, nil +} + +// Name returns the basename of the file system entry. +func (de Dirent) Name() string { return de.name } + +// ModeType returns the mode bits that specify the file system node type. We +// could make our own enum-like data type for encoding the file type, but Go's +// runtime already gives us architecture independent file modes, as discussed in +// `os/types.go`: +// +// Go's runtime FileMode type has same definition on all systems, so that +// information about files can be moved from one system to another portably. +func (de Dirent) ModeType() os.FileMode { return de.modeType } + +// IsDir returns true if and only if the Dirent represents a file system +// directory. Note that on some operating systems, more than one file mode bit +// may be set for a node. For instance, on Windows, a symbolic link that points +// to a directory will have both the directory and the symbolic link bits set. +func (de Dirent) IsDir() bool { return de.modeType&os.ModeDir != 0 } + +// IsRegular returns true if and only if the Dirent represents a regular +// file. That is, it ensures that no mode type bits are set. +func (de Dirent) IsRegular() bool { return de.modeType&os.ModeType == 0 } + +// IsSymlink returns true if and only if the Dirent represents a file system +// symbolic link. Note that on some operating systems, more than one file mode +// bit may be set for a node. For instance, on Windows, a symbolic link that +// points to a directory will have both the directory and the symbolic link bits +// set. +func (de Dirent) IsSymlink() bool { return de.modeType&os.ModeSymlink != 0 } + +// Dirents represents a slice of Dirent pointers, which are sortable by +// name. This type satisfies the `sort.Interface` interface. +type Dirents []*Dirent + +// Len returns the count of Dirent structures in the slice. +func (l Dirents) Len() int { return len(l) } + +// Less returns true if and only if the Name of the element specified by the +// first index is lexicographically less than that of the second index. +func (l Dirents) Less(i, j int) bool { return l[i].name < l[j].name } + +// Swap exchanges the two Dirent entries specified by the two provided indexes. +func (l Dirents) Swap(i, j int) { l[i], l[j] = l[j], l[i] } diff --git a/vendor/github.com/karrick/godirwalk/doc.go b/vendor/github.com/karrick/godirwalk/doc.go new file mode 100644 index 00000000000..0dfdabd4884 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/doc.go @@ -0,0 +1,34 @@ +/* +Package godirwalk provides functions to read and traverse directory trees. + +In short, why do I use this library? + +* It's faster than `filepath.Walk`. + +* It's more correct on Windows than `filepath.Walk`. + +* It's more easy to use than `filepath.Walk`. + +* It's more flexible than `filepath.Walk`. + +USAGE + +This library will normalize the provided top level directory name based on the +os-specific path separator by calling `filepath.Clean` on its first +argument. However it always provides the pathname created by using the correct +os-specific path separator when invoking the provided callback function. + + dirname := "some/directory/root" + err := godirwalk.Walk(dirname, &godirwalk.Options{ + Callback: func(osPathname string, de *godirwalk.Dirent) error { + fmt.Printf("%s %s\n", de.ModeType(), osPathname) + return nil + }, + }) + +This library not only provides functions for traversing a file system directory +tree, but also for obtaining a list of immediate descendants of a particular +directory, typically much more quickly than using `os.ReadDir` or +`os.ReadDirnames`. +*/ +package godirwalk diff --git a/vendor/github.com/karrick/godirwalk/go.mod b/vendor/github.com/karrick/godirwalk/go.mod new file mode 100644 index 00000000000..6b467a9e153 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/go.mod @@ -0,0 +1,3 @@ +module github.com/karrick/godirwalk + +require github.com/pkg/errors v0.8.0 diff --git a/vendor/github.com/karrick/godirwalk/go.sum b/vendor/github.com/karrick/godirwalk/go.sum new file mode 100644 index 00000000000..a31014d5f0b --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/go.sum @@ -0,0 +1 @@ +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/vendor/github.com/karrick/godirwalk/readdir.go b/vendor/github.com/karrick/godirwalk/readdir.go new file mode 100644 index 00000000000..2bba689751f --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/readdir.go @@ -0,0 +1,47 @@ +package godirwalk + +// ReadDirents returns a sortable slice of pointers to Dirent structures, each +// representing the file system name and mode type for one of the immediate +// descendant of the specified directory. If the specified directory is a +// symbolic link, it will be resolved. +// +// If an optional scratch buffer is provided that is at least one page of +// memory, it will be used when reading directory entries from the file system. +// +// children, err := godirwalk.ReadDirents(osDirname, nil) +// if err != nil { +// return nil, errors.Wrap(err, "cannot get list of directory children") +// } +// sort.Sort(children) +// for _, child := range children { +// fmt.Printf("%s %s\n", child.ModeType, child.Name) +// } +func ReadDirents(osDirname string, scratchBuffer []byte) (Dirents, error) { + return readdirents(osDirname, scratchBuffer) +} + +// ReadDirnames returns a slice of strings, representing the immediate +// descendants of the specified directory. If the specified directory is a +// symbolic link, it will be resolved. +// +// If an optional scratch buffer is provided that is at least one page of +// memory, it will be used when reading directory entries from the file system. +// +// Note that this function, depending on operating system, may or may not invoke +// the ReadDirents function, in order to prepare the list of immediate +// descendants. Therefore, if your program needs both the names and the file +// system mode types of descendants, it will always be faster to invoke +// ReadDirents directly, rather than calling this function, then looping over +// the results and calling os.Stat for each child. +// +// children, err := godirwalk.ReadDirnames(osDirname, nil) +// if err != nil { +// return nil, errors.Wrap(err, "cannot get list of directory children") +// } +// sort.Strings(children) +// for _, child := range children { +// fmt.Printf("%s\n", child) +// } +func ReadDirnames(osDirname string, scratchBuffer []byte) ([]string, error) { + return readdirnames(osDirname, scratchBuffer) +} diff --git a/vendor/github.com/karrick/godirwalk/readdir_unix.go b/vendor/github.com/karrick/godirwalk/readdir_unix.go new file mode 100644 index 00000000000..04a628f7fc1 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/readdir_unix.go @@ -0,0 +1,109 @@ +// +build darwin freebsd linux netbsd openbsd + +package godirwalk + +import ( + "os" + "path/filepath" + "syscall" + "unsafe" + + "github.com/pkg/errors" +) + +func readdirents(osDirname string, scratchBuffer []byte) (Dirents, error) { + dh, err := os.Open(osDirname) + if err != nil { + return nil, errors.Wrap(err, "cannot Open") + } + + var entries Dirents + + fd := int(dh.Fd()) + + if len(scratchBuffer) < MinimumScratchBufferSize { + scratchBuffer = make([]byte, DefaultScratchBufferSize) + } + + var de *syscall.Dirent + + for { + n, err := syscall.ReadDirent(fd, scratchBuffer) + if err != nil { + _ = dh.Close() // ignore potential error returned by Close + return nil, errors.Wrap(err, "cannot ReadDirent") + } + if n <= 0 { + break // end of directory reached + } + // Loop over the bytes returned by reading the directory entries. + buf := scratchBuffer[:n] + for len(buf) > 0 { + de = (*syscall.Dirent)(unsafe.Pointer(&buf[0])) // point entry to first syscall.Dirent in buffer + buf = buf[de.Reclen:] // advance buffer + + if inoFromDirent(de) == 0 { + continue // this item has been deleted, but not yet removed from directory + } + + nameSlice := nameFromDirent(de) + namlen := len(nameSlice) + if (namlen == 0) || (namlen == 1 && nameSlice[0] == '.') || (namlen == 2 && nameSlice[0] == '.' && nameSlice[1] == '.') { + continue // skip unimportant entries + } + osChildname := string(nameSlice) + + // Convert syscall constant, which is in purview of OS, to a + // constant defined by Go, assumed by this project to be stable. + var mode os.FileMode + switch de.Type { + case syscall.DT_REG: + // regular file + case syscall.DT_DIR: + mode = os.ModeDir + case syscall.DT_LNK: + mode = os.ModeSymlink + case syscall.DT_CHR: + mode = os.ModeDevice | os.ModeCharDevice + case syscall.DT_BLK: + mode = os.ModeDevice + case syscall.DT_FIFO: + mode = os.ModeNamedPipe + case syscall.DT_SOCK: + mode = os.ModeSocket + default: + // If syscall returned unknown type (e.g., DT_UNKNOWN, DT_WHT), + // then resolve actual mode by getting stat. + fi, err := os.Lstat(filepath.Join(osDirname, osChildname)) + if err != nil { + _ = dh.Close() // ignore potential error returned by Close + return nil, errors.Wrap(err, "cannot Stat") + } + // We only care about the bits that identify the type of a file + // system node, and can ignore append, exclusive, temporary, + // setuid, setgid, permission bits, and sticky bits, which are + // coincident to the bits that declare type of the file system + // node. + mode = fi.Mode() & os.ModeType + } + + entries = append(entries, &Dirent{name: osChildname, modeType: mode}) + } + } + if err = dh.Close(); err != nil { + return nil, err + } + return entries, nil +} + +func readdirnames(osDirname string, scratchBuffer []byte) ([]string, error) { + des, err := readdirents(osDirname, scratchBuffer) + if err != nil { + return nil, err + } + names := make([]string, len(des)) + for i, v := range des { + names[i] = v.name + } + return names, nil +} diff --git a/vendor/github.com/karrick/godirwalk/readdir_windows.go b/vendor/github.com/karrick/godirwalk/readdir_windows.go new file mode 100644 index 00000000000..885a067a4bc --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/readdir_windows.go @@ -0,0 +1,54 @@ +package godirwalk + +import ( + "os" + + "github.com/pkg/errors" +) + +// The functions in this file are mere wrappers of what is already provided by +// standard library, in order to provide the same API as this library provides. +// +// The scratch buffer argument is ignored by this architecture. +// +// Please send PR or link to article if you know of a more performant way of +// enumerating directory contents and mode types on Windows. + +func readdirents(osDirname string, _ []byte) (Dirents, error) { + dh, err := os.Open(osDirname) + if err != nil { + return nil, errors.Wrap(err, "cannot Open") + } + + fileinfos, err := dh.Readdir(0) + if er := dh.Close(); err == nil { + err = er + } + if err != nil { + return nil, errors.Wrap(err, "cannot Readdir") + } + + entries := make(Dirents, len(fileinfos)) + for i, info := range fileinfos { + entries[i] = &Dirent{name: info.Name(), modeType: info.Mode() & os.ModeType} + } + + return entries, nil +} + +func readdirnames(osDirname string, _ []byte) ([]string, error) { + dh, err := os.Open(osDirname) + if err != nil { + return nil, errors.Wrap(err, "cannot Open") + } + + entries, err := dh.Readdirnames(0) + if er := dh.Close(); err == nil { + err = er + } + if err != nil { + return nil, errors.Wrap(err, "cannot Readdirnames") + } + + return entries, nil +} diff --git a/vendor/github.com/karrick/godirwalk/walk.go b/vendor/github.com/karrick/godirwalk/walk.go new file mode 100644 index 00000000000..4c184ab87e6 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/walk.go @@ -0,0 +1,367 @@ +package godirwalk + +import ( + "os" + "path/filepath" + "sort" + + "github.com/pkg/errors" +) + +// DefaultScratchBufferSize specifies the size of the scratch buffer that will +// be allocated by Walk, ReadDirents, or ReadDirnames when a scratch buffer is +// not provided or the scratch buffer that is provided is smaller than +// MinimumScratchBufferSize bytes. This may seem like a large value; however, +// when a program intends to enumerate large directories, having a larger +// scratch buffer results in fewer operating system calls. +const DefaultScratchBufferSize = 64 * 1024 + +// MinimumScratchBufferSize specifies the minimum size of the scratch buffer +// that Walk, ReadDirents, and ReadDirnames will use when reading file entries +// from the operating system. It is initialized to the result from calling +// `os.Getpagesize()` during program startup. +var MinimumScratchBufferSize int + +func init() { + MinimumScratchBufferSize = os.Getpagesize() +} + +// Options provide parameters for how the Walk function operates. +type Options struct { + // ErrorCallback specifies a function to be invoked in the case of an error + // that could potentially be ignored while walking a file system + // hierarchy. When set to nil or left as its zero-value, any error condition + // causes Walk to immediately return the error describing what took + // place. When non-nil, this user supplied function is invoked with the OS + // pathname of the file system object that caused the error along with the + // error that took place. The return value of the supplied ErrorCallback + // function determines whether the error will cause Walk to halt immediately + // as it would were no ErrorCallback value provided, or skip this file + // system node yet continue on with the remaining nodes in the file system + // hierarchy. + // + // ErrorCallback is invoked both for errors that are returned by the + // runtime, and for errors returned by other user supplied callback + // functions. + ErrorCallback func(string, error) ErrorAction + + // FollowSymbolicLinks specifies whether Walk will follow symbolic links + // that refer to directories. When set to false or left as its zero-value, + // Walk will still invoke the callback function with symbolic link nodes, + // but if the symbolic link refers to a directory, it will not recurse on + // that directory. When set to true, Walk will recurse on symbolic links + // that refer to a directory. + FollowSymbolicLinks bool + + // Unsorted controls whether or not Walk will sort the immediate descendants + // of a directory by their relative names prior to visiting each of those + // entries. + // + // When set to false or left at its zero-value, Walk will get the list of + // immediate descendants of a particular directory, sort that list by + // lexical order of their names, and then visit each node in the list in + // sorted order. This will cause Walk to always traverse the same directory + // tree in the same order, however may be inefficient for directories with + // many immediate descendants. + // + // When set to true, Walk skips sorting the list of immediate descendants + // for a directory, and simply visits each node in the order the operating + // system enumerated them. This will be more fast, but with the side effect + // that the traversal order may be different from one invocation to the + // next. + Unsorted bool + + // Callback is a required function that Walk will invoke for every file + // system node it encounters. + Callback WalkFunc + + // PostChildrenCallback is an option function that Walk will invoke for + // every file system directory it encounters after its children have been + // processed. + PostChildrenCallback WalkFunc + + // ScratchBuffer is an optional byte slice to use as a scratch buffer for + // Walk to use when reading directory entries, to reduce amount of garbage + // generation. Not all architectures take advantage of the scratch + // buffer. If omitted or the provided buffer has fewer bytes than + // MinimumScratchBufferSize, then a buffer with DefaultScratchBufferSize + // bytes will be created and used once per Walk invocation. + ScratchBuffer []byte +} + +// ErrorAction defines a set of actions the Walk function could take based on +// the occurrence of an error while walking the file system. See the +// documentation for the ErrorCallback field of the Options structure for more +// information. +type ErrorAction int + +const ( + // Halt is the ErrorAction return value when the upstream code wants to halt + // the walk process when a runtime error takes place. It matches the default + // action the Walk function would take were no ErrorCallback provided. + Halt ErrorAction = iota + + // SkipNode is the ErrorAction return value when the upstream code wants to + // ignore the runtime error for the current file system node, skip + // processing of the node that caused the error, and continue walking the + // file system hierarchy with the remaining nodes. + SkipNode +) + +// WalkFunc is the type of the function called for each file system node visited +// by Walk. The pathname argument will contain the argument to Walk as a prefix; +// that is, if Walk is called with "dir", which is a directory containing the +// file "a", the provided WalkFunc will be invoked with the argument "dir/a", +// using the correct os.PathSeparator for the Go Operating System architecture, +// GOOS. The directory entry argument is a pointer to a Dirent for the node, +// providing access to both the basename and the mode type of the file system +// node. +// +// If an error is returned by the Callback or PostChildrenCallback functions, +// and no ErrorCallback function is provided, processing stops. If an +// ErrorCallback function is provided, then it is invoked with the OS pathname +// of the node that caused the error along along with the error. The return +// value of the ErrorCallback function determines whether to halt processing, or +// skip this node and continue processing remaining file system nodes. +// +// The exception is when the function returns the special value +// filepath.SkipDir. If the function returns filepath.SkipDir when invoked on a +// directory, Walk skips the directory's contents entirely. If the function +// returns filepath.SkipDir when invoked on a non-directory file system node, +// Walk skips the remaining files in the containing directory. Note that any +// supplied ErrorCallback function is not invoked with filepath.SkipDir when the +// Callback or PostChildrenCallback functions return that special value. +type WalkFunc func(osPathname string, directoryEntry *Dirent) error + +// Walk walks the file tree rooted at the specified directory, calling the +// specified callback function for each file system node in the tree, including +// root, symbolic links, and other node types. The nodes are walked in lexical +// order, which makes the output deterministic but means that for very large +// directories this function can be inefficient. +// +// This function is often much faster than filepath.Walk because it does not +// invoke os.Stat for every node it encounters, but rather obtains the file +// system node type when it reads the parent directory. +// +// If a runtime error occurs, either from the operating system or from the +// upstream Callback or PostChildrenCallback functions, processing typically +// halts. However, when an ErrorCallback function is provided in the provided +// Options structure, that function is invoked with the error along with the OS +// pathname of the file system node that caused the error. The ErrorCallback +// function's return value determines the action that Walk will then take. +// +// func main() { +// dirname := "." +// if len(os.Args) > 1 { +// dirname = os.Args[1] +// } +// err := godirwalk.Walk(dirname, &godirwalk.Options{ +// Callback: func(osPathname string, de *godirwalk.Dirent) error { +// fmt.Printf("%s %s\n", de.ModeType(), osPathname) +// return nil +// }, +// ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { +// // Your program may want to log the error somehow. +// fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) +// +// // For the purposes of this example, a simple SkipNode will suffice, +// // although in reality perhaps additional logic might be called for. +// return godirwalk.SkipNode +// }, +// }) +// if err != nil { +// fmt.Fprintf(os.Stderr, "%s\n", err) +// os.Exit(1) +// } +// } +func Walk(pathname string, options *Options) error { + pathname = filepath.Clean(pathname) + + var fi os.FileInfo + var err error + + if options.FollowSymbolicLinks { + fi, err = os.Stat(pathname) + if err != nil { + return errors.Wrap(err, "cannot Stat") + } + } else { + fi, err = os.Lstat(pathname) + if err != nil { + return errors.Wrap(err, "cannot Lstat") + } + } + + mode := fi.Mode() + if mode&os.ModeDir == 0 { + return errors.Errorf("cannot Walk non-directory: %s", pathname) + } + + dirent := &Dirent{ + name: filepath.Base(pathname), + modeType: mode & os.ModeType, + } + + // If ErrorCallback is nil, set to a default value that halts the walk + // process on all operating system errors. This is done to allow error + // handling to be more succinct in the walk code. + if options.ErrorCallback == nil { + options.ErrorCallback = defaultErrorCallback + } + + if len(options.ScratchBuffer) < MinimumScratchBufferSize { + options.ScratchBuffer = make([]byte, DefaultScratchBufferSize) + } + + err = walk(pathname, dirent, options) + if err == filepath.SkipDir { + return nil // silence SkipDir for top level + } + return err +} + +// defaultErrorCallback always returns Halt because if the upstream code did not +// provide an ErrorCallback function, walking the file system hierarchy ought to +// halt upon any operating system error. +func defaultErrorCallback(_ string, _ error) ErrorAction { return Halt } + +// walk recursively traverses the file system node specified by pathname and the +// Dirent. +func walk(osPathname string, dirent *Dirent, options *Options) error { + err := options.Callback(osPathname, dirent) + if err != nil { + if err == filepath.SkipDir { + return err + } + err = errors.Wrap(err, "Callback") // wrap potential errors returned by callback + if action := options.ErrorCallback(osPathname, err); action == SkipNode { + return nil + } + return err + } + + // On some platforms, an entry can have more than one mode type bit set. + // For instance, it could have both the symlink bit and the directory bit + // set indicating it's a symlink to a directory. + if dirent.IsSymlink() { + if !options.FollowSymbolicLinks { + return nil + } + // Only need to Stat entry if platform did not already have os.ModeDir + // set, such as would be the case for unix like operating systems. (This + // guard eliminates extra os.Stat check on Windows.) + if !dirent.IsDir() { + referent, err := os.Readlink(osPathname) + if err != nil { + err = errors.Wrap(err, "cannot Readlink") + if action := options.ErrorCallback(osPathname, err); action == SkipNode { + return nil + } + return err + } + + var osp string + if filepath.IsAbs(referent) { + osp = referent + } else { + osp = filepath.Join(filepath.Dir(osPathname), referent) + } + + fi, err := os.Stat(osp) + if err != nil { + err = errors.Wrap(err, "cannot Stat") + if action := options.ErrorCallback(osp, err); action == SkipNode { + return nil + } + return err + } + dirent.modeType = fi.Mode() & os.ModeType + } + } + + if !dirent.IsDir() { + return nil + } + + // If get here, then specified pathname refers to a directory. + deChildren, err := ReadDirents(osPathname, options.ScratchBuffer) + if err != nil { + err = errors.Wrap(err, "cannot ReadDirents") + if action := options.ErrorCallback(osPathname, err); action == SkipNode { + return nil + } + return err + } + + if !options.Unsorted { + sort.Sort(deChildren) // sort children entries unless upstream says to leave unsorted + } + + for _, deChild := range deChildren { + osChildname := filepath.Join(osPathname, deChild.name) + err = walk(osChildname, deChild, options) + if err != nil { + if err != filepath.SkipDir { + return err + } + // If received skipdir on a directory, stop processing that + // directory, but continue to its siblings. If received skipdir on a + // non-directory, stop processing remaining siblings. + if deChild.IsSymlink() { + // Only need to Stat entry if platform did not already have + // os.ModeDir set, such as would be the case for unix like + // operating systems. (This guard eliminates extra os.Stat check + // on Windows.) + if !deChild.IsDir() { + // Resolve symbolic link referent to determine whether node + // is directory or not. + referent, err := os.Readlink(osChildname) + if err != nil { + err = errors.Wrap(err, "cannot Readlink") + if action := options.ErrorCallback(osChildname, err); action == SkipNode { + continue // with next child + } + return err + } + + var osp string + if filepath.IsAbs(referent) { + osp = referent + } else { + osp = filepath.Join(osPathname, referent) + } + + fi, err := os.Stat(osp) + if err != nil { + err = errors.Wrap(err, "cannot Stat") + if action := options.ErrorCallback(osp, err); action == SkipNode { + continue // with next child + } + return err + } + deChild.modeType = fi.Mode() & os.ModeType + } + } + if !deChild.IsDir() { + // If not directory, return immediately, thus skipping remainder + // of siblings. + return nil + } + } + } + + if options.PostChildrenCallback == nil { + return nil + } + + err = options.PostChildrenCallback(osPathname, dirent) + if err == nil || err == filepath.SkipDir { + return err + } + + err = errors.Wrap(err, "PostChildrenCallback") // wrap potential errors returned by callback + if action := options.ErrorCallback(osPathname, err); action == SkipNode { + return nil + } + return err +} diff --git a/vendor/github.com/karrick/godirwalk/withFileno.go b/vendor/github.com/karrick/godirwalk/withFileno.go new file mode 100644 index 00000000000..1dc04a717a7 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/withFileno.go @@ -0,0 +1,9 @@ +// +build dragonfly freebsd openbsd netbsd + +package godirwalk + +import "syscall" + +func inoFromDirent(de *syscall.Dirent) uint64 { + return uint64(de.Fileno) +} diff --git a/vendor/github.com/karrick/godirwalk/withIno.go b/vendor/github.com/karrick/godirwalk/withIno.go new file mode 100644 index 00000000000..47fc12540f3 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/withIno.go @@ -0,0 +1,9 @@ +// +build darwin linux + +package godirwalk + +import "syscall" + +func inoFromDirent(de *syscall.Dirent) uint64 { + return de.Ino +} diff --git a/vendor/github.com/karrick/godirwalk/withNamlen.go b/vendor/github.com/karrick/godirwalk/withNamlen.go new file mode 100644 index 00000000000..46a4af50045 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/withNamlen.go @@ -0,0 +1,29 @@ +// +build darwin dragonfly freebsd netbsd openbsd + +package godirwalk + +import ( + "reflect" + "syscall" + "unsafe" +) + +func nameFromDirent(de *syscall.Dirent) []byte { + // Because this GOOS' syscall.Dirent provides a Namlen field that says how + // long the name is, this function does not need to search for the NULL + // byte. + ml := int(de.Namlen) + + // Convert syscall.Dirent.Name, which is array of int8, to []byte, by + // overwriting Cap, Len, and Data slice header fields to values from + // syscall.Dirent fields. Setting the Cap, Len, and Data field values for + // the slice header modifies what the slice header points to, and in this + // case, the name buffer. + var name []byte + sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) + sh.Cap = ml + sh.Len = ml + sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) + + return name +} diff --git a/vendor/github.com/karrick/godirwalk/withoutNamlen.go b/vendor/github.com/karrick/godirwalk/withoutNamlen.go new file mode 100644 index 00000000000..dcf9f3a9722 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/withoutNamlen.go @@ -0,0 +1,36 @@ +// +build nacl linux solaris + +package godirwalk + +import ( + "bytes" + "reflect" + "syscall" + "unsafe" +) + +func nameFromDirent(de *syscall.Dirent) []byte { + // Because this GOOS' syscall.Dirent does not provide a field that specifies + // the name length, this function must first calculate the max possible name + // length, and then search for the NULL byte. + ml := int(uint64(de.Reclen) - uint64(unsafe.Offsetof(syscall.Dirent{}.Name))) + + // Convert syscall.Dirent.Name, which is array of int8, to []byte, by + // overwriting Cap, Len, and Data slice header fields to values from + // syscall.Dirent fields. Setting the Cap, Len, and Data field values for + // the slice header modifies what the slice header points to, and in this + // case, the name buffer. + var name []byte + sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) + sh.Cap = ml + sh.Len = ml + sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) + + if index := bytes.IndexByte(name, 0); index >= 0 { + // Found NULL byte; set slice's cap and len accordingly. + sh.Cap = index + sh.Len = index + } + + return name +} diff --git a/vendor/github.com/kubernetes/repo-infra/kazel/BUILD b/vendor/github.com/kubernetes/repo-infra/kazel/BUILD index 1b722abc7b4..a00a2951633 100644 --- a/vendor/github.com/kubernetes/repo-infra/kazel/BUILD +++ b/vendor/github.com/kubernetes/repo-infra/kazel/BUILD @@ -14,7 +14,7 @@ go_library( visibility = ["//visibility:private"], deps = [ "//vendor/github.com/bazelbuild/buildtools/build:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/github.com/kubernetes/repo-infra/kazel/config.go b/vendor/github.com/kubernetes/repo-infra/kazel/config.go index a2c6ed0af75..19e390f3302 100644 --- a/vendor/github.com/kubernetes/repo-infra/kazel/config.go +++ b/vendor/github.com/kubernetes/repo-infra/kazel/config.go @@ -28,6 +28,9 @@ type Cfg struct { SrcDirs []string // regexps that match packages to skip SkippedPaths []string + // regexps that match packages to skip for K8SOpenAPIGen. + // note that this skips anything matched by SkippedPaths as well. + SkippedOpenAPIGenPaths []string // whether to add "pkg-srcs" and "all-srcs" filegroups // note that this operates on the entire tree (not just SrcsDirs) but skips anything matching SkippedPaths AddSourcesRules bool diff --git a/vendor/github.com/kubernetes/repo-infra/kazel/generator.go b/vendor/github.com/kubernetes/repo-infra/kazel/generator.go index fa53daef8fc..b1e38ad8089 100644 --- a/vendor/github.com/kubernetes/repo-infra/kazel/generator.go +++ b/vendor/github.com/kubernetes/repo-infra/kazel/generator.go @@ -55,7 +55,7 @@ func (v *Vendorer) walkGenerated() error { // findOpenAPI searches for all packages under root that request OpenAPI. It // returns the go import paths. It does not follow symlinks. func (v *Vendorer) findOpenAPI(root string) ([]string, error) { - for _, r := range v.skippedPaths { + for _, r := range v.skippedOpenAPIPaths { if r.MatchString(root) { return nil, nil } diff --git a/vendor/github.com/kubernetes/repo-infra/kazel/kazel.go b/vendor/github.com/kubernetes/repo-infra/kazel/kazel.go index b1b4e4aad29..fc0caff9e36 100644 --- a/vendor/github.com/kubernetes/repo-infra/kazel/kazel.go +++ b/vendor/github.com/kubernetes/repo-infra/kazel/kazel.go @@ -31,7 +31,8 @@ import ( "strings" bzl "github.com/bazelbuild/buildtools/build" - "github.com/golang/glog" + + "k8s.io/klog" ) const ( @@ -51,36 +52,36 @@ func main() { flag.Parse() flag.Set("alsologtostderr", "true") if *root == "" { - glog.Fatalf("-root argument is required") + klog.Fatalf("-root argument is required") } if *validate { *dryRun = true } v, err := newVendorer(*root, *cfgPath, *dryRun) if err != nil { - glog.Fatalf("unable to build vendorer: %v", err) + klog.Fatalf("unable to build vendorer: %v", err) } if err = os.Chdir(v.root); err != nil { - glog.Fatalf("cannot chdir into root %q: %v", v.root, err) + klog.Fatalf("cannot chdir into root %q: %v", v.root, err) } if v.cfg.ManageGoRules { if err = v.walkVendor(); err != nil { - glog.Fatalf("err walking vendor: %v", err) + klog.Fatalf("err walking vendor: %v", err) } if err = v.walkRepo(); err != nil { - glog.Fatalf("err walking repo: %v", err) + klog.Fatalf("err walking repo: %v", err) } } if err = v.walkGenerated(); err != nil { - glog.Fatalf("err walking generated: %v", err) + klog.Fatalf("err walking generated: %v", err) } if _, err = v.walkSource("."); err != nil { - glog.Fatalf("err walking source: %v", err) + klog.Fatalf("err walking source: %v", err) } written := 0 if written, err = v.reconcileAllRules(); err != nil { - glog.Fatalf("err reconciling rules: %v", err) + klog.Fatalf("err reconciling rules: %v", err) } if *validate && written > 0 { fmt.Fprintf(os.Stderr, "\n%d BUILD files not up-to-date.\n", written) @@ -90,14 +91,15 @@ func main() { // Vendorer collects context, configuration, and cache while walking the tree. type Vendorer struct { - ctx *build.Context - icache map[icacheKey]icacheVal - skippedPaths []*regexp.Regexp - dryRun bool - root string - cfg *Cfg - newRules map[string][]*bzl.Rule // package path -> list of rules to add or update - managedAttrs []string + ctx *build.Context + icache map[icacheKey]icacheVal + skippedPaths []*regexp.Regexp + skippedOpenAPIPaths []*regexp.Regexp + dryRun bool + root string + cfg *Cfg + newRules map[string][]*bzl.Rule // package path -> list of rules to add or update + managedAttrs []string } func newVendorer(root, cfgPath string, dryRun bool) (*Vendorer, error) { @@ -123,19 +125,23 @@ func newVendorer(root, cfgPath string, dryRun bool) (*Vendorer, error) { managedAttrs: []string{"srcs", "deps", "library"}, } - for _, sp := range cfg.SkippedPaths { - r, err := regexp.Compile(sp) - if err != nil { - return nil, err - } - v.skippedPaths = append(v.skippedPaths, r) + builtIn, err := compileSkippedPaths([]string{"^\\.git", "^bazel-*"}) + if err != nil { + return nil, err } - for _, builtinSkip := range []string{ - "^\\.git", - "^bazel-*", - } { - v.skippedPaths = append(v.skippedPaths, regexp.MustCompile(builtinSkip)) + + sp, err := compileSkippedPaths(cfg.SkippedPaths) + if err != nil { + return nil, err } + sp = append(builtIn, sp...) + v.skippedPaths = sp + + sop, err := compileSkippedPaths(cfg.SkippedOpenAPIGenPaths) + if err != nil { + return nil, err + } + v.skippedOpenAPIPaths = append(sop, sp...) return &v, nil @@ -541,7 +547,7 @@ func asExpr(e interface{}) bzl.Expr { } return &bzl.ListExpr{List: list} default: - glog.Fatalf("Uh oh") + klog.Fatalf("Uh oh") return nil } } @@ -782,3 +788,16 @@ func context() *build.Context { func walk(root string, walkFn filepath.WalkFunc) error { return nil } + +func compileSkippedPaths(skippedPaths []string) ([]*regexp.Regexp, error) { + regexPaths := []*regexp.Regexp{} + + for _, sp := range skippedPaths { + r, err := regexp.Compile(sp) + if err != nil { + return nil, err + } + regexPaths = append(regexPaths, r) + } + return regexPaths, nil +} diff --git a/vendor/github.com/lpabon/godbc/.travis.yml b/vendor/github.com/lpabon/godbc/.travis.yml deleted file mode 100644 index 43b7cb86537..00000000000 --- a/vendor/github.com/lpabon/godbc/.travis.yml +++ /dev/null @@ -1,13 +0,0 @@ -language: go - -go: - - 1.1 - - 1.2 - - tip - -install: - - go get github.com/stretchr/testify - -script: - - go test - - go test -tags 'prod' diff --git a/vendor/github.com/lpabon/godbc/AUTHORS b/vendor/github.com/lpabon/godbc/AUTHORS deleted file mode 100644 index d4ddde5dd01..00000000000 --- a/vendor/github.com/lpabon/godbc/AUTHORS +++ /dev/null @@ -1 +0,0 @@ -lpabon@redhat.com \ No newline at end of file diff --git a/vendor/github.com/lpabon/godbc/LICENSE b/vendor/github.com/lpabon/godbc/LICENSE deleted file mode 100644 index ad410e11302..00000000000 --- a/vendor/github.com/lpabon/godbc/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/vendor/github.com/lpabon/godbc/README.md b/vendor/github.com/lpabon/godbc/README.md deleted file mode 100644 index 2df53d42373..00000000000 --- a/vendor/github.com/lpabon/godbc/README.md +++ /dev/null @@ -1,21 +0,0 @@ -[![Build Status](https://travis-ci.org/lpabon/godbc.svg?branch=master)](https://travis-ci.org/lpabon/godbc) - -# godbc - -Design by contract for Go - -# Installation - -To install godbc, use `go get`: - - go get github.com/lpabon/godbc - -Import the `godbc` package into your code using this template: - - import ( - "github.com/lpabon/godbc" - ) - -# Documentation - -Documentation is available at https://godoc.org/github.com/lpabon/godbc diff --git a/vendor/github.com/lpabon/godbc/godbc.go b/vendor/github.com/lpabon/godbc/godbc.go deleted file mode 100644 index 7c44a9154ea..00000000000 --- a/vendor/github.com/lpabon/godbc/godbc.go +++ /dev/null @@ -1,146 +0,0 @@ -//+build !prod - -// -// Copyright (c) 2014 The godbc Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Design-by-Contract for Go -// -// Design by Contract is a programming methodology -// which binds the caller and the function called to a -// contract. The contract is represented using Hoare Triple: -// {P} C {Q} -// where {P} is the precondition before executing command C, -// and {Q} is the postcondition. -// -// See Also -// -// * http://en.wikipedia.org/wiki/Design_by_contract -// * http://en.wikipedia.org/wiki/Hoare_logic -// * http://dlang.org/dbc.html -// -// Usage -// -// Godbc is enabled by default, but can be disabled for production -// builds by using the tag 'prod' in builds and tests as follows: -// go build -tags 'prod' -// or -// go test -tags 'prod' -// -package godbc - -import ( - "errors" - "fmt" - "runtime" -) - -// InvariantSimpleTester is an interface which provides a receiver to -// test the object -type InvariantSimpleTester interface { - Invariant() bool -} - -// InvariantTester is an interface which provides not only an Invariant(), -// but also a receiver to print the structure -type InvariantTester interface { - InvariantSimpleTester - String() string -} - -// dbc_panic prints to the screen information of the failure followed -// by a call to panic() -func dbc_panic(dbc_func_name string, b bool, message ...interface{}) { - if !b { - - // Get caller information which is the caller - // of the caller of this function - pc, file, line, _ := runtime.Caller(2) - caller_func_info := runtime.FuncForPC(pc) - - error_string := fmt.Sprintf("%s:\n\r\tfunc (%s) 0x%x\n\r\tFile %s:%d", - dbc_func_name, - caller_func_info.Name(), - pc, - file, - line) - - if len(message) > 0 { - error_string += fmt.Sprintf("\n\r\tInfo: %+v", message) - } - err := errors.New(error_string) - - // Finally panic - panic(err) - } -} - -// Require checks that the preconditions are satisfied before -// executing the function -// -// Example Code -// -// func Divide(a, b int) int { -// godbc.Require(b != 0) -// return a/b -// } -// -func Require(b bool, message ...interface{}) { - dbc_panic("REQUIRE", b, message...) -} - -// Ensure checks the postconditions are satisfied before returning -// to the caller. -// -// Example Code -// -// type Data struct { -// a int -// } -// -// func (*d Data) Set(a int) { -// d.a = a -// godbc.Ensure(d.a == a) -// } -// -func Ensure(b bool, message ...interface{}) { - dbc_panic("ENSURE", b, message...) -} - -// Check provides a simple assert -func Check(b bool, message ...interface{}) { - dbc_panic("CHECK", b, message...) -} - -// InvariantSimple calls the objects Invariant() receiver to test -// the object for correctness. -// -// The caller object must provide an object that supports the -// interface InvariantSimpleTester and does not need to provide -// a String() receiver -func InvariantSimple(obj InvariantSimpleTester, message ...interface{}) { - dbc_panic("INVARIANT", obj.Invariant(), message...) -} - -// Invariant calls the objects Invariant() receiver to test -// the object for correctness. -// -// The caller object must provide an object that supports the -// interface InvariantTester -// -// To see an example, please take a look at the godbc_test.go -func Invariant(obj InvariantTester, message ...interface{}) { - m := append(message, obj) - dbc_panic("INVARIANT", obj.Invariant(), m) -} diff --git a/vendor/github.com/lpabon/godbc/godbc_prod.go b/vendor/github.com/lpabon/godbc/godbc_prod.go deleted file mode 100644 index 1cb2fffb807..00000000000 --- a/vendor/github.com/lpabon/godbc/godbc_prod.go +++ /dev/null @@ -1,42 +0,0 @@ -//+build prod - -// -// Copyright (c) 2014 The godbc Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package godbc - -type InvariantSimpleTester interface { - Invariant() bool -} - -type InvariantTester interface { - InvariantSimpleTester - String() string -} - -func Require(b bool, message ...interface{}) { -} - -func Ensure(b bool, message ...interface{}) { -} - -func Check(b bool, message ...interface{}) { -} - -func InvariantSimple(obj InvariantSimpleTester, message ...interface{}) { -} - -func Invariant(obj InvariantTester, message ...interface{}) { -} diff --git a/vendor/github.com/sigma/go-inotify/BUILD b/vendor/github.com/sigma/go-inotify/BUILD new file mode 100644 index 00000000000..ffe8ab90bcf --- /dev/null +++ b/vendor/github.com/sigma/go-inotify/BUILD @@ -0,0 +1,23 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["inotify_linux.go"], + importmap = "k8s.io/kubernetes/vendor/github.com/sigma/go-inotify", + importpath = "github.com/sigma/go-inotify", + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/sigma/go-inotify/LICENSE b/vendor/github.com/sigma/go-inotify/LICENSE new file mode 100644 index 00000000000..6a66aea5eaf --- /dev/null +++ b/vendor/github.com/sigma/go-inotify/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/sigma/go-inotify/PATENTS b/vendor/github.com/sigma/go-inotify/PATENTS new file mode 100644 index 00000000000..733099041f8 --- /dev/null +++ b/vendor/github.com/sigma/go-inotify/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/github.com/sigma/go-inotify/README.md b/vendor/github.com/sigma/go-inotify/README.md new file mode 100644 index 00000000000..0c723a8d2b4 --- /dev/null +++ b/vendor/github.com/sigma/go-inotify/README.md @@ -0,0 +1,5 @@ +This is a fork of golang.org/x/exp/inotify before it was deleted. + +Please use gopkg.in/fsnotify.v0 instead. + +For updates, see: https://fsnotify.org/ diff --git a/vendor/github.com/sigma/go-inotify/inotify_linux.go b/vendor/github.com/sigma/go-inotify/inotify_linux.go new file mode 100644 index 00000000000..901f308d84a --- /dev/null +++ b/vendor/github.com/sigma/go-inotify/inotify_linux.go @@ -0,0 +1,306 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package inotify implements a wrapper for the Linux inotify system. + +Example: + watcher, err := inotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + err = watcher.Watch("/tmp") + if err != nil { + log.Fatal(err) + } + for { + select { + case ev := <-watcher.Event: + log.Println("event:", ev) + case err := <-watcher.Error: + log.Println("error:", err) + } + } + +*/ +package inotify + +import ( + "errors" + "fmt" + "os" + "strings" + "sync" + "syscall" + "unsafe" +) + +type Event struct { + Mask uint32 // Mask of events + Cookie uint32 // Unique cookie associating related events (for rename(2)) + Name string // File name (optional) +} + +type watch struct { + wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) + flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) +} + +type Watcher struct { + mu sync.Mutex + fd int // File descriptor (as returned by the inotify_init() syscall) + watches map[string]*watch // Map of inotify watches (key: path) + paths map[int]string // Map of watched paths (key: watch descriptor) + Error chan error // Errors are sent on this channel + Event chan *Event // Events are returned on this channel + done chan bool // Channel for sending a "quit message" to the reader goroutine + isClosed bool // Set to true when Close() is first called +} + +// NewWatcher creates and returns a new inotify instance using inotify_init(2) +func NewWatcher() (*Watcher, error) { + fd, errno := syscall.InotifyInit() + if fd == -1 { + return nil, os.NewSyscallError("inotify_init", errno) + } + w := &Watcher{ + fd: fd, + watches: make(map[string]*watch), + paths: make(map[int]string), + Event: make(chan *Event), + Error: make(chan error), + done: make(chan bool, 1), + } + + go w.readEvents() + return w, nil +} + +// Close closes an inotify watcher instance +// It sends a message to the reader goroutine to quit and removes all watches +// associated with the inotify instance +func (w *Watcher) Close() error { + if w.isClosed { + return nil + } + w.isClosed = true + + // Send "quit" message to the reader goroutine + w.done <- true + for path := range w.watches { + w.RemoveWatch(path) + } + + return nil +} + +// AddWatch adds path to the watched file set. +// The flags are interpreted as described in inotify_add_watch(2). +func (w *Watcher) AddWatch(path string, flags uint32) error { + if w.isClosed { + return errors.New("inotify instance already closed") + } + + watchEntry, found := w.watches[path] + if found { + watchEntry.flags |= flags + flags |= syscall.IN_MASK_ADD + } + + w.mu.Lock() // synchronize with readEvents goroutine + + wd, err := syscall.InotifyAddWatch(w.fd, path, flags) + if err != nil { + w.mu.Unlock() + return &os.PathError{ + Op: "inotify_add_watch", + Path: path, + Err: err, + } + } + + if !found { + w.watches[path] = &watch{wd: uint32(wd), flags: flags} + w.paths[wd] = path + } + w.mu.Unlock() + return nil +} + +// Watch adds path to the watched file set, watching all events. +func (w *Watcher) Watch(path string) error { + return w.AddWatch(path, IN_ALL_EVENTS) +} + +// RemoveWatch removes path from the watched file set. +func (w *Watcher) RemoveWatch(path string) error { + watch, ok := w.watches[path] + if !ok { + return errors.New(fmt.Sprintf("can't remove non-existent inotify watch for: %s", path)) + } + success, errno := syscall.InotifyRmWatch(w.fd, watch.wd) + if success == -1 { + return os.NewSyscallError("inotify_rm_watch", errno) + } + delete(w.watches, path) + // Locking here to protect the read from paths in readEvents. + w.mu.Lock() + delete(w.paths, int(watch.wd)) + w.mu.Unlock() + return nil +} + +// readEvents reads from the inotify file descriptor, converts the +// received events into Event objects and sends them via the Event channel +func (w *Watcher) readEvents() { + var buf [syscall.SizeofInotifyEvent * 4096]byte + + for { + n, err := syscall.Read(w.fd, buf[:]) + // See if there is a message on the "done" channel + var done bool + select { + case done = <-w.done: + default: + } + + // If EOF or a "done" message is received + if n == 0 || done { + // The syscall.Close can be slow. Close + // w.Event first. + close(w.Event) + err := syscall.Close(w.fd) + if err != nil { + w.Error <- os.NewSyscallError("close", err) + } + close(w.Error) + return + } + if n < 0 { + w.Error <- os.NewSyscallError("read", err) + continue + } + if n < syscall.SizeofInotifyEvent { + w.Error <- errors.New("inotify: short read in readEvents()") + continue + } + + var offset uint32 = 0 + // We don't know how many events we just read into the buffer + // While the offset points to at least one whole event... + for offset <= uint32(n-syscall.SizeofInotifyEvent) { + // Point "raw" to the event in the buffer + raw := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[offset])) + event := new(Event) + event.Mask = uint32(raw.Mask) + event.Cookie = uint32(raw.Cookie) + nameLen := uint32(raw.Len) + // If the event happened to the watched directory or the watched file, the kernel + // doesn't append the filename to the event, but we would like to always fill the + // the "Name" field with a valid filename. We retrieve the path of the watch from + // the "paths" map. + w.mu.Lock() + name, ok := w.paths[int(raw.Wd)] + w.mu.Unlock() + if ok { + event.Name = name + if nameLen > 0 { + // Point "bytes" at the first byte of the filename + bytes := (*[syscall.PathMax]byte)(unsafe.Pointer(&buf[offset+syscall.SizeofInotifyEvent])) + // The filename is padded with NUL bytes. TrimRight() gets rid of those. + event.Name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") + } + // Send the event on the events channel + w.Event <- event + } + // Move to the next event in the buffer + offset += syscall.SizeofInotifyEvent + nameLen + } + } +} + +// String formats the event e in the form +// "filename: 0xEventMask = IN_ACCESS|IN_ATTRIB_|..." +func (e *Event) String() string { + var events string = "" + + m := e.Mask + for _, b := range eventBits { + if m&b.Value == b.Value { + m &^= b.Value + events += "|" + b.Name + } + } + + if m != 0 { + events += fmt.Sprintf("|%#x", m) + } + if len(events) > 0 { + events = " == " + events[1:] + } + + return fmt.Sprintf("%q: %#x%s", e.Name, e.Mask, events) +} + +const ( + // Options for inotify_init() are not exported + // IN_CLOEXEC uint32 = syscall.IN_CLOEXEC + // IN_NONBLOCK uint32 = syscall.IN_NONBLOCK + + // Options for AddWatch + IN_DONT_FOLLOW uint32 = syscall.IN_DONT_FOLLOW + IN_ONESHOT uint32 = syscall.IN_ONESHOT + IN_ONLYDIR uint32 = syscall.IN_ONLYDIR + + // The "IN_MASK_ADD" option is not exported, as AddWatch + // adds it automatically, if there is already a watch for the given path + // IN_MASK_ADD uint32 = syscall.IN_MASK_ADD + + // Events + IN_ACCESS uint32 = syscall.IN_ACCESS + IN_ALL_EVENTS uint32 = syscall.IN_ALL_EVENTS + IN_ATTRIB uint32 = syscall.IN_ATTRIB + IN_CLOSE uint32 = syscall.IN_CLOSE + IN_CLOSE_NOWRITE uint32 = syscall.IN_CLOSE_NOWRITE + IN_CLOSE_WRITE uint32 = syscall.IN_CLOSE_WRITE + IN_CREATE uint32 = syscall.IN_CREATE + IN_DELETE uint32 = syscall.IN_DELETE + IN_DELETE_SELF uint32 = syscall.IN_DELETE_SELF + IN_MODIFY uint32 = syscall.IN_MODIFY + IN_MOVE uint32 = syscall.IN_MOVE + IN_MOVED_FROM uint32 = syscall.IN_MOVED_FROM + IN_MOVED_TO uint32 = syscall.IN_MOVED_TO + IN_MOVE_SELF uint32 = syscall.IN_MOVE_SELF + IN_OPEN uint32 = syscall.IN_OPEN + + // Special events + IN_ISDIR uint32 = syscall.IN_ISDIR + IN_IGNORED uint32 = syscall.IN_IGNORED + IN_Q_OVERFLOW uint32 = syscall.IN_Q_OVERFLOW + IN_UNMOUNT uint32 = syscall.IN_UNMOUNT +) + +var eventBits = []struct { + Value uint32 + Name string +}{ + {IN_ACCESS, "IN_ACCESS"}, + {IN_ATTRIB, "IN_ATTRIB"}, + {IN_CLOSE, "IN_CLOSE"}, + {IN_CLOSE_NOWRITE, "IN_CLOSE_NOWRITE"}, + {IN_CLOSE_WRITE, "IN_CLOSE_WRITE"}, + {IN_CREATE, "IN_CREATE"}, + {IN_DELETE, "IN_DELETE"}, + {IN_DELETE_SELF, "IN_DELETE_SELF"}, + {IN_MODIFY, "IN_MODIFY"}, + {IN_MOVE, "IN_MOVE"}, + {IN_MOVED_FROM, "IN_MOVED_FROM"}, + {IN_MOVED_TO, "IN_MOVED_TO"}, + {IN_MOVE_SELF, "IN_MOVE_SELF"}, + {IN_OPEN, "IN_OPEN"}, + {IN_ISDIR, "IN_ISDIR"}, + {IN_IGNORED, "IN_IGNORED"}, + {IN_Q_OVERFLOW, "IN_Q_OVERFLOW"}, + {IN_UNMOUNT, "IN_UNMOUNT"}, +} diff --git a/vendor/golang.org/x/net/context/context.go b/vendor/golang.org/x/net/context/context.go index a25301a76fa..839e3a64b62 100644 --- a/vendor/golang.org/x/net/context/context.go +++ b/vendor/golang.org/x/net/context/context.go @@ -5,6 +5,8 @@ // Package context defines the Context type, which carries deadlines, // cancelation signals, and other request-scoped values across API boundaries // and between processes. +// As of Go 1.7 this package is available in the standard library under the +// name context. https://golang.org/pkg/context. // // Incoming requests to a server should create a Context, and outgoing calls to // servers should accept a Context. The chain of function calls between must diff --git a/vendor/golang.org/x/net/html/atom/gen.go b/vendor/golang.org/x/net/html/atom/gen.go index 6bfa8660198..cc5dc5dbcee 100644 --- a/vendor/golang.org/x/net/html/atom/gen.go +++ b/vendor/golang.org/x/net/html/atom/gen.go @@ -4,17 +4,17 @@ // +build ignore +//go:generate go run gen.go +//go:generate go run gen.go -test + package main -// This program generates table.go and table_test.go. -// Invoke as -// -// go run gen.go |gofmt >table.go -// go run gen.go -test |gofmt >table_test.go - import ( + "bytes" "flag" "fmt" + "go/format" + "io/ioutil" "math/rand" "os" "sort" @@ -42,6 +42,18 @@ func identifier(s string) string { var test = flag.Bool("test", false, "generate table_test.go") +func genFile(name string, buf *bytes.Buffer) { + b, err := format.Source(buf.Bytes()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := ioutil.WriteFile(name, b, 0644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + func main() { flag.Parse() @@ -52,32 +64,31 @@ func main() { all = append(all, extra...) sort.Strings(all) - if *test { - fmt.Printf("// generated by go run gen.go -test; DO NOT EDIT\n\n") - fmt.Printf("package atom\n\n") - fmt.Printf("var testAtomList = []string{\n") - for _, s := range all { - fmt.Printf("\t%q,\n", s) - } - fmt.Printf("}\n") - return - } - // uniq - lists have dups - // compute max len too - maxLen := 0 w := 0 for _, s := range all { if w == 0 || all[w-1] != s { - if maxLen < len(s) { - maxLen = len(s) - } all[w] = s w++ } } all = all[:w] + if *test { + var buf bytes.Buffer + fmt.Fprintln(&buf, "// Code generated by go generate gen.go; DO NOT EDIT.\n") + fmt.Fprintln(&buf, "//go:generate go run gen.go -test\n") + fmt.Fprintln(&buf, "package atom\n") + fmt.Fprintln(&buf, "var testAtomList = []string{") + for _, s := range all { + fmt.Fprintf(&buf, "\t%q,\n", s) + } + fmt.Fprintln(&buf, "}") + + genFile("table_test.go", &buf) + return + } + // Find hash that minimizes table size. var best *table for i := 0; i < 1000000; i++ { @@ -163,36 +174,46 @@ func main() { atom[s] = uint32(off<<8 | len(s)) } + var buf bytes.Buffer // Generate the Go code. - fmt.Printf("// generated by go run gen.go; DO NOT EDIT\n\n") - fmt.Printf("package atom\n\nconst (\n") + fmt.Fprintln(&buf, "// Code generated by go generate gen.go; DO NOT EDIT.\n") + fmt.Fprintln(&buf, "//go:generate go run gen.go\n") + fmt.Fprintln(&buf, "package atom\n\nconst (") + + // compute max len + maxLen := 0 for _, s := range all { - fmt.Printf("\t%s Atom = %#x\n", identifier(s), atom[s]) + if maxLen < len(s) { + maxLen = len(s) + } + fmt.Fprintf(&buf, "\t%s Atom = %#x\n", identifier(s), atom[s]) } - fmt.Printf(")\n\n") + fmt.Fprintln(&buf, ")\n") - fmt.Printf("const hash0 = %#x\n\n", best.h0) - fmt.Printf("const maxAtomLen = %d\n\n", maxLen) + fmt.Fprintf(&buf, "const hash0 = %#x\n\n", best.h0) + fmt.Fprintf(&buf, "const maxAtomLen = %d\n\n", maxLen) - fmt.Printf("var table = [1<<%d]Atom{\n", best.k) + fmt.Fprintf(&buf, "var table = [1<<%d]Atom{\n", best.k) for i, s := range best.tab { if s == "" { continue } - fmt.Printf("\t%#x: %#x, // %s\n", i, atom[s], s) + fmt.Fprintf(&buf, "\t%#x: %#x, // %s\n", i, atom[s], s) } - fmt.Printf("}\n") + fmt.Fprintf(&buf, "}\n") datasize := (1 << best.k) * 4 - fmt.Printf("const atomText =\n") + fmt.Fprintln(&buf, "const atomText =") textsize := len(text) for len(text) > 60 { - fmt.Printf("\t%q +\n", text[:60]) + fmt.Fprintf(&buf, "\t%q +\n", text[:60]) text = text[60:] } - fmt.Printf("\t%q\n\n", text) + fmt.Fprintf(&buf, "\t%q\n\n", text) - fmt.Fprintf(os.Stderr, "%d atoms; %d string bytes + %d tables = %d total data\n", len(all), textsize, datasize, textsize+datasize) + genFile("table.go", &buf) + + fmt.Fprintf(os.Stdout, "%d atoms; %d string bytes + %d tables = %d total data\n", len(all), textsize, datasize, textsize+datasize) } type byLen []string @@ -285,8 +306,10 @@ func (t *table) push(i uint32, depth int) bool { // The lists of element names and attribute keys were taken from // https://html.spec.whatwg.org/multipage/indices.html#index -// as of the "HTML Living Standard - Last Updated 21 February 2015" version. +// as of the "HTML Living Standard - Last Updated 18 September 2017" version. +// "command", "keygen" and "menuitem" have been removed from the spec, +// but are kept here for backwards compatibility. var elements = []string{ "a", "abbr", @@ -349,6 +372,7 @@ var elements = []string{ "legend", "li", "link", + "main", "map", "mark", "menu", @@ -364,6 +388,7 @@ var elements = []string{ "output", "p", "param", + "picture", "pre", "progress", "q", @@ -375,6 +400,7 @@ var elements = []string{ "script", "section", "select", + "slot", "small", "source", "span", @@ -403,14 +429,21 @@ var elements = []string{ } // https://html.spec.whatwg.org/multipage/indices.html#attributes-3 - +// +// "challenge", "command", "contextmenu", "dropzone", "icon", "keytype", "mediagroup", +// "radiogroup", "spellcheck", "scoped", "seamless", "sortable" and "sorted" have been removed from the spec, +// but are kept here for backwards compatibility. var attributes = []string{ "abbr", "accept", "accept-charset", "accesskey", "action", + "allowfullscreen", + "allowpaymentrequest", + "allowusermedia", "alt", + "as", "async", "autocomplete", "autofocus", @@ -420,6 +453,7 @@ var attributes = []string{ "checked", "cite", "class", + "color", "cols", "colspan", "command", @@ -457,6 +491,8 @@ var attributes = []string{ "icon", "id", "inputmode", + "integrity", + "is", "ismap", "itemid", "itemprop", @@ -481,16 +517,20 @@ var attributes = []string{ "multiple", "muted", "name", + "nomodule", + "nonce", "novalidate", "open", "optimum", "pattern", "ping", "placeholder", + "playsinline", "poster", "preload", "radiogroup", "readonly", + "referrerpolicy", "rel", "required", "reversed", @@ -507,10 +547,13 @@ var attributes = []string{ "sizes", "sortable", "sorted", + "slot", "span", + "spellcheck", "src", "srcdoc", "srclang", + "srcset", "start", "step", "style", @@ -520,16 +563,22 @@ var attributes = []string{ "translate", "type", "typemustmatch", + "updateviacache", "usemap", "value", "width", + "workertype", "wrap", } +// "onautocomplete", "onautocompleteerror", "onmousewheel", +// "onshow" and "onsort" have been removed from the spec, +// but are kept here for backwards compatibility. var eventHandlers = []string{ "onabort", "onautocomplete", "onautocompleteerror", + "onauxclick", "onafterprint", "onbeforeprint", "onbeforeunload", @@ -541,11 +590,14 @@ var eventHandlers = []string{ "onclick", "onclose", "oncontextmenu", + "oncopy", "oncuechange", + "oncut", "ondblclick", "ondrag", "ondragend", "ondragenter", + "ondragexit", "ondragleave", "ondragover", "ondragstart", @@ -565,18 +617,24 @@ var eventHandlers = []string{ "onload", "onloadeddata", "onloadedmetadata", + "onloadend", "onloadstart", "onmessage", + "onmessageerror", "onmousedown", + "onmouseenter", + "onmouseleave", "onmousemove", "onmouseout", "onmouseover", "onmouseup", "onmousewheel", + "onwheel", "onoffline", "ononline", "onpagehide", "onpageshow", + "onpaste", "onpause", "onplay", "onplaying", @@ -585,7 +643,9 @@ var eventHandlers = []string{ "onratechange", "onreset", "onresize", + "onrejectionhandled", "onscroll", + "onsecuritypolicyviolation", "onseeked", "onseeking", "onselect", @@ -597,6 +657,7 @@ var eventHandlers = []string{ "onsuspend", "ontimeupdate", "ontoggle", + "onunhandledrejection", "onunload", "onvolumechange", "onwaiting", diff --git a/vendor/golang.org/x/net/html/atom/table.go b/vendor/golang.org/x/net/html/atom/table.go index 2605ba3102f..f74018ecea1 100644 --- a/vendor/golang.org/x/net/html/atom/table.go +++ b/vendor/golang.org/x/net/html/atom/table.go @@ -1,713 +1,777 @@ -// generated by go run gen.go; DO NOT EDIT +// Code generated by go generate gen.go; DO NOT EDIT. + +//go:generate go run gen.go package atom const ( - A Atom = 0x1 - Abbr Atom = 0x4 - Accept Atom = 0x2106 - AcceptCharset Atom = 0x210e - Accesskey Atom = 0x3309 - Action Atom = 0x1f606 - Address Atom = 0x4f307 - Align Atom = 0x1105 - Alt Atom = 0x4503 - Annotation Atom = 0x1670a - AnnotationXml Atom = 0x1670e - Applet Atom = 0x2b306 - Area Atom = 0x2fa04 - Article Atom = 0x38807 - Aside Atom = 0x8305 - Async Atom = 0x7b05 - Audio Atom = 0xa605 - Autocomplete Atom = 0x1fc0c - Autofocus Atom = 0xb309 - Autoplay Atom = 0xce08 - B Atom = 0x101 - Base Atom = 0xd604 - Basefont Atom = 0xd608 - Bdi Atom = 0x1a03 - Bdo Atom = 0xe703 - Bgsound Atom = 0x11807 - Big Atom = 0x12403 - Blink Atom = 0x12705 - Blockquote Atom = 0x12c0a - Body Atom = 0x2f04 - Br Atom = 0x202 - Button Atom = 0x13606 - Canvas Atom = 0x7f06 - Caption Atom = 0x1bb07 - Center Atom = 0x5b506 - Challenge Atom = 0x21f09 - Charset Atom = 0x2807 - Checked Atom = 0x32807 - Cite Atom = 0x3c804 - Class Atom = 0x4de05 - Code Atom = 0x14904 - Col Atom = 0x15003 - Colgroup Atom = 0x15008 - Color Atom = 0x15d05 - Cols Atom = 0x16204 - Colspan Atom = 0x16207 - Command Atom = 0x17507 - Content Atom = 0x42307 - Contenteditable Atom = 0x4230f - Contextmenu Atom = 0x3310b - Controls Atom = 0x18808 - Coords Atom = 0x19406 - Crossorigin Atom = 0x19f0b - Data Atom = 0x44a04 - Datalist Atom = 0x44a08 - Datetime Atom = 0x23c08 - Dd Atom = 0x26702 - Default Atom = 0x8607 - Defer Atom = 0x14b05 - Del Atom = 0x3ef03 - Desc Atom = 0x4db04 - Details Atom = 0x4807 - Dfn Atom = 0x6103 - Dialog Atom = 0x1b06 - Dir Atom = 0x6903 - Dirname Atom = 0x6907 - Disabled Atom = 0x10c08 - Div Atom = 0x11303 - Dl Atom = 0x11e02 - Download Atom = 0x40008 - Draggable Atom = 0x17b09 - Dropzone Atom = 0x39108 - Dt Atom = 0x50902 - Em Atom = 0x6502 - Embed Atom = 0x6505 - Enctype Atom = 0x21107 - Face Atom = 0x5b304 - Fieldset Atom = 0x1b008 - Figcaption Atom = 0x1b80a - Figure Atom = 0x1cc06 - Font Atom = 0xda04 - Footer Atom = 0x8d06 - For Atom = 0x1d803 - ForeignObject Atom = 0x1d80d - Foreignobject Atom = 0x1e50d - Form Atom = 0x1f204 - Formaction Atom = 0x1f20a - Formenctype Atom = 0x20d0b - Formmethod Atom = 0x2280a - Formnovalidate Atom = 0x2320e - Formtarget Atom = 0x2470a - Frame Atom = 0x9a05 - Frameset Atom = 0x9a08 - H1 Atom = 0x26e02 - H2 Atom = 0x29402 - H3 Atom = 0x2a702 - H4 Atom = 0x2e902 - H5 Atom = 0x2f302 - H6 Atom = 0x50b02 - Head Atom = 0x2d504 - Header Atom = 0x2d506 - Headers Atom = 0x2d507 - Height Atom = 0x25106 - Hgroup Atom = 0x25906 - Hidden Atom = 0x26506 - High Atom = 0x26b04 - Hr Atom = 0x27002 - Href Atom = 0x27004 - Hreflang Atom = 0x27008 - Html Atom = 0x25504 - HttpEquiv Atom = 0x2780a - I Atom = 0x601 - Icon Atom = 0x42204 - Id Atom = 0x8502 - Iframe Atom = 0x29606 - Image Atom = 0x29c05 - Img Atom = 0x2a103 - Input Atom = 0x3e805 - Inputmode Atom = 0x3e809 - Ins Atom = 0x1a803 - Isindex Atom = 0x2a907 - Ismap Atom = 0x2b005 - Itemid Atom = 0x33c06 - Itemprop Atom = 0x3c908 - Itemref Atom = 0x5ad07 - Itemscope Atom = 0x2b909 - Itemtype Atom = 0x2c308 - Kbd Atom = 0x1903 - Keygen Atom = 0x3906 - Keytype Atom = 0x53707 - Kind Atom = 0x10904 - Label Atom = 0xf005 - Lang Atom = 0x27404 - Legend Atom = 0x18206 - Li Atom = 0x1202 - Link Atom = 0x12804 - List Atom = 0x44e04 - Listing Atom = 0x44e07 - Loop Atom = 0xf404 - Low Atom = 0x11f03 - Malignmark Atom = 0x100a - Manifest Atom = 0x5f108 - Map Atom = 0x2b203 - Mark Atom = 0x1604 - Marquee Atom = 0x2cb07 - Math Atom = 0x2d204 - Max Atom = 0x2e103 - Maxlength Atom = 0x2e109 - Media Atom = 0x6e05 - Mediagroup Atom = 0x6e0a - Menu Atom = 0x33804 - Menuitem Atom = 0x33808 - Meta Atom = 0x45d04 - Meter Atom = 0x24205 - Method Atom = 0x22c06 - Mglyph Atom = 0x2a206 - Mi Atom = 0x2eb02 - Min Atom = 0x2eb03 - Minlength Atom = 0x2eb09 - Mn Atom = 0x23502 - Mo Atom = 0x3ed02 - Ms Atom = 0x2bc02 - Mtext Atom = 0x2f505 - Multiple Atom = 0x30308 - Muted Atom = 0x30b05 - Name Atom = 0x6c04 - Nav Atom = 0x3e03 - Nobr Atom = 0x5704 - Noembed Atom = 0x6307 - Noframes Atom = 0x9808 - Noscript Atom = 0x3d208 - Novalidate Atom = 0x2360a - Object Atom = 0x1ec06 - Ol Atom = 0xc902 - Onabort Atom = 0x13a07 - Onafterprint Atom = 0x1c00c - Onautocomplete Atom = 0x1fa0e - Onautocompleteerror Atom = 0x1fa13 - Onbeforeprint Atom = 0x6040d - Onbeforeunload Atom = 0x4e70e - Onblur Atom = 0xaa06 - Oncancel Atom = 0xe908 - Oncanplay Atom = 0x28509 - Oncanplaythrough Atom = 0x28510 - Onchange Atom = 0x3a708 - Onclick Atom = 0x31007 - Onclose Atom = 0x31707 - Oncontextmenu Atom = 0x32f0d - Oncuechange Atom = 0x3420b - Ondblclick Atom = 0x34d0a - Ondrag Atom = 0x35706 - Ondragend Atom = 0x35709 - Ondragenter Atom = 0x3600b - Ondragleave Atom = 0x36b0b - Ondragover Atom = 0x3760a - Ondragstart Atom = 0x3800b - Ondrop Atom = 0x38f06 - Ondurationchange Atom = 0x39f10 - Onemptied Atom = 0x39609 - Onended Atom = 0x3af07 - Onerror Atom = 0x3b607 - Onfocus Atom = 0x3bd07 - Onhashchange Atom = 0x3da0c - Oninput Atom = 0x3e607 - Oninvalid Atom = 0x3f209 - Onkeydown Atom = 0x3fb09 - Onkeypress Atom = 0x4080a - Onkeyup Atom = 0x41807 - Onlanguagechange Atom = 0x43210 - Onload Atom = 0x44206 - Onloadeddata Atom = 0x4420c - Onloadedmetadata Atom = 0x45510 - Onloadstart Atom = 0x46b0b - Onmessage Atom = 0x47609 - Onmousedown Atom = 0x47f0b - Onmousemove Atom = 0x48a0b - Onmouseout Atom = 0x4950a - Onmouseover Atom = 0x4a20b - Onmouseup Atom = 0x4ad09 - Onmousewheel Atom = 0x4b60c - Onoffline Atom = 0x4c209 - Ononline Atom = 0x4cb08 - Onpagehide Atom = 0x4d30a - Onpageshow Atom = 0x4fe0a - Onpause Atom = 0x50d07 - Onplay Atom = 0x51706 - Onplaying Atom = 0x51709 - Onpopstate Atom = 0x5200a - Onprogress Atom = 0x52a0a - Onratechange Atom = 0x53e0c - Onreset Atom = 0x54a07 - Onresize Atom = 0x55108 - Onscroll Atom = 0x55f08 - Onseeked Atom = 0x56708 - Onseeking Atom = 0x56f09 - Onselect Atom = 0x57808 - Onshow Atom = 0x58206 - Onsort Atom = 0x58b06 - Onstalled Atom = 0x59509 - Onstorage Atom = 0x59e09 - Onsubmit Atom = 0x5a708 - Onsuspend Atom = 0x5bb09 - Ontimeupdate Atom = 0xdb0c - Ontoggle Atom = 0x5c408 - Onunload Atom = 0x5cc08 - Onvolumechange Atom = 0x5d40e - Onwaiting Atom = 0x5e209 - Open Atom = 0x3cf04 - Optgroup Atom = 0xf608 - Optimum Atom = 0x5eb07 - Option Atom = 0x60006 - Output Atom = 0x49c06 - P Atom = 0xc01 - Param Atom = 0xc05 - Pattern Atom = 0x5107 - Ping Atom = 0x7704 - Placeholder Atom = 0xc30b - Plaintext Atom = 0xfd09 - Poster Atom = 0x15706 - Pre Atom = 0x25e03 - Preload Atom = 0x25e07 - Progress Atom = 0x52c08 - Prompt Atom = 0x5fa06 - Public Atom = 0x41e06 - Q Atom = 0x13101 - Radiogroup Atom = 0x30a - Readonly Atom = 0x2fb08 - Rel Atom = 0x25f03 - Required Atom = 0x1d008 - Reversed Atom = 0x5a08 - Rows Atom = 0x9204 - Rowspan Atom = 0x9207 - Rp Atom = 0x1c602 - Rt Atom = 0x13f02 - Ruby Atom = 0xaf04 - S Atom = 0x2c01 - Samp Atom = 0x4e04 - Sandbox Atom = 0xbb07 - Scope Atom = 0x2bd05 - Scoped Atom = 0x2bd06 - Script Atom = 0x3d406 - Seamless Atom = 0x31c08 - Section Atom = 0x4e207 - Select Atom = 0x57a06 - Selected Atom = 0x57a08 - Shape Atom = 0x4f905 - Size Atom = 0x55504 - Sizes Atom = 0x55505 - Small Atom = 0x18f05 - Sortable Atom = 0x58d08 - Sorted Atom = 0x19906 - Source Atom = 0x1aa06 - Spacer Atom = 0x2db06 - Span Atom = 0x9504 - Spellcheck Atom = 0x3230a - Src Atom = 0x3c303 - Srcdoc Atom = 0x3c306 - Srclang Atom = 0x41107 - Start Atom = 0x38605 - Step Atom = 0x5f704 - Strike Atom = 0x53306 - Strong Atom = 0x55906 - Style Atom = 0x61105 - Sub Atom = 0x5a903 - Summary Atom = 0x61607 - Sup Atom = 0x61d03 - Svg Atom = 0x62003 - System Atom = 0x62306 - Tabindex Atom = 0x46308 - Table Atom = 0x42d05 - Target Atom = 0x24b06 - Tbody Atom = 0x2e05 - Td Atom = 0x4702 - Template Atom = 0x62608 - Textarea Atom = 0x2f608 - Tfoot Atom = 0x8c05 - Th Atom = 0x22e02 - Thead Atom = 0x2d405 - Time Atom = 0xdd04 - Title Atom = 0xa105 - Tr Atom = 0x10502 - Track Atom = 0x10505 - Translate Atom = 0x14009 - Tt Atom = 0x5302 - Type Atom = 0x21404 - Typemustmatch Atom = 0x2140d - U Atom = 0xb01 - Ul Atom = 0x8a02 - Usemap Atom = 0x51106 - Value Atom = 0x4005 - Var Atom = 0x11503 - Video Atom = 0x28105 - Wbr Atom = 0x12103 - Width Atom = 0x50705 - Wrap Atom = 0x58704 - Xmp Atom = 0xc103 + A Atom = 0x1 + Abbr Atom = 0x4 + Accept Atom = 0x1a06 + AcceptCharset Atom = 0x1a0e + Accesskey Atom = 0x2c09 + Action Atom = 0x25a06 + Address Atom = 0x6ed07 + Align Atom = 0x6d405 + Allowfullscreen Atom = 0x1f00f + Allowpaymentrequest Atom = 0x6913 + Allowusermedia Atom = 0x850e + Alt Atom = 0xb003 + Annotation Atom = 0x1b90a + AnnotationXml Atom = 0x1b90e + Applet Atom = 0x30106 + Area Atom = 0x34a04 + Article Atom = 0x3f007 + As Atom = 0xb902 + Aside Atom = 0xc105 + Async Atom = 0xb905 + Audio Atom = 0xcf05 + Autocomplete Atom = 0x2600c + Autofocus Atom = 0xeb09 + Autoplay Atom = 0x10608 + B Atom = 0x101 + Base Atom = 0x11504 + Basefont Atom = 0x11508 + Bdi Atom = 0x16103 + Bdo Atom = 0x13403 + Bgsound Atom = 0x14707 + Big Atom = 0x15903 + Blink Atom = 0x15c05 + Blockquote Atom = 0x1680a + Body Atom = 0x2804 + Br Atom = 0x202 + Button Atom = 0x17206 + Canvas Atom = 0xbd06 + Caption Atom = 0x21907 + Center Atom = 0x20806 + Challenge Atom = 0x28309 + Charset Atom = 0x2107 + Checked Atom = 0x46d07 + Cite Atom = 0x55804 + Class Atom = 0x5b905 + Code Atom = 0x19004 + Col Atom = 0x19703 + Colgroup Atom = 0x19708 + Color Atom = 0x1af05 + Cols Atom = 0x1b404 + Colspan Atom = 0x1b407 + Command Atom = 0x1c707 + Content Atom = 0x57f07 + Contenteditable Atom = 0x57f0f + Contextmenu Atom = 0x3740b + Controls Atom = 0x1ce08 + Coords Atom = 0x1da06 + Crossorigin Atom = 0x1e30b + Data Atom = 0x49904 + Datalist Atom = 0x49908 + Datetime Atom = 0x2a008 + Dd Atom = 0x2bf02 + Default Atom = 0xc407 + Defer Atom = 0x19205 + Del Atom = 0x44603 + Desc Atom = 0x55504 + Details Atom = 0x4607 + Dfn Atom = 0x5f03 + Dialog Atom = 0x16206 + Dir Atom = 0xa303 + Dirname Atom = 0xa307 + Disabled Atom = 0x14d08 + Div Atom = 0x15403 + Dl Atom = 0x5e202 + Download Atom = 0x45708 + Draggable Atom = 0x18309 + Dropzone Atom = 0x3f908 + Dt Atom = 0x64702 + Em Atom = 0x4202 + Embed Atom = 0x4205 + Enctype Atom = 0x27507 + Face Atom = 0x20604 + Fieldset Atom = 0x20e08 + Figcaption Atom = 0x2160a + Figure Atom = 0x23006 + Font Atom = 0x11904 + Footer Atom = 0xb306 + For Atom = 0x23c03 + ForeignObject Atom = 0x23c0d + Foreignobject Atom = 0x2490d + Form Atom = 0x25604 + Formaction Atom = 0x2560a + Formenctype Atom = 0x2710b + Formmethod Atom = 0x28c0a + Formnovalidate Atom = 0x2960e + Formtarget Atom = 0x2a80a + Frame Atom = 0x5705 + Frameset Atom = 0x5708 + H1 Atom = 0x14502 + H2 Atom = 0x2c602 + H3 Atom = 0x2f502 + H4 Atom = 0x33902 + H5 Atom = 0x34302 + H6 Atom = 0x64902 + Head Atom = 0x32504 + Header Atom = 0x32506 + Headers Atom = 0x32507 + Height Atom = 0x12c06 + Hgroup Atom = 0x2b206 + Hidden Atom = 0x2bd06 + High Atom = 0x2c304 + Hr Atom = 0x14002 + Href Atom = 0x2c804 + Hreflang Atom = 0x2c808 + Html Atom = 0x13004 + HttpEquiv Atom = 0x2d00a + I Atom = 0x601 + Icon Atom = 0x57e04 + Id Atom = 0xc302 + Iframe Atom = 0x2e406 + Image Atom = 0x2ea05 + Img Atom = 0x2ef03 + Input Atom = 0x43f05 + Inputmode Atom = 0x43f09 + Ins Atom = 0x1ec03 + Integrity Atom = 0x22709 + Is Atom = 0x14e02 + Isindex Atom = 0x2f707 + Ismap Atom = 0x2fe05 + Itemid Atom = 0x37f06 + Itemprop Atom = 0x55908 + Itemref Atom = 0x3c107 + Itemscope Atom = 0x66d09 + Itemtype Atom = 0x30708 + Kbd Atom = 0x16003 + Keygen Atom = 0x3206 + Keytype Atom = 0x7e07 + Kind Atom = 0x18004 + Label Atom = 0xda05 + Lang Atom = 0x2cc04 + Legend Atom = 0x18a06 + Li Atom = 0x11102 + Link Atom = 0x15d04 + List Atom = 0x49d04 + Listing Atom = 0x49d07 + Loop Atom = 0xde04 + Low Atom = 0x6b03 + Main Atom = 0x1004 + Malignmark Atom = 0x6d30a + Manifest Atom = 0x30f08 + Map Atom = 0x30003 + Mark Atom = 0x6d904 + Marquee Atom = 0x31b07 + Math Atom = 0x32204 + Max Atom = 0x33103 + Maxlength Atom = 0x33109 + Media Atom = 0x8e05 + Mediagroup Atom = 0x8e0a + Menu Atom = 0x37b04 + Menuitem Atom = 0x37b08 + Meta Atom = 0x4ac04 + Meter Atom = 0xa805 + Method Atom = 0x29006 + Mglyph Atom = 0x2f006 + Mi Atom = 0x33b02 + Min Atom = 0x33b03 + Minlength Atom = 0x33b09 + Mn Atom = 0x29902 + Mo Atom = 0x6302 + Ms Atom = 0x67002 + Mtext Atom = 0x34505 + Multiple Atom = 0x35308 + Muted Atom = 0x35b05 + Name Atom = 0xa604 + Nav Atom = 0x1303 + Nobr Atom = 0x3704 + Noembed Atom = 0x4007 + Noframes Atom = 0x5508 + Nomodule Atom = 0x6108 + Nonce Atom = 0x56205 + Noscript Atom = 0x1fe08 + Novalidate Atom = 0x29a0a + Object Atom = 0x25006 + Ol Atom = 0x10102 + Onabort Atom = 0x17607 + Onafterprint Atom = 0x21e0c + Onautocomplete Atom = 0x25e0e + Onautocompleteerror Atom = 0x25e13 + Onauxclick Atom = 0x61b0a + Onbeforeprint Atom = 0x69a0d + Onbeforeunload Atom = 0x6e10e + Onblur Atom = 0x5c206 + Oncancel Atom = 0xd308 + Oncanplay Atom = 0x13609 + Oncanplaythrough Atom = 0x13610 + Onchange Atom = 0x40f08 + Onclick Atom = 0x2dd07 + Onclose Atom = 0x36007 + Oncontextmenu Atom = 0x3720d + Oncopy Atom = 0x38506 + Oncuechange Atom = 0x38b0b + Oncut Atom = 0x39605 + Ondblclick Atom = 0x39b0a + Ondrag Atom = 0x3a506 + Ondragend Atom = 0x3a509 + Ondragenter Atom = 0x3ae0b + Ondragexit Atom = 0x3b90a + Ondragleave Atom = 0x3d30b + Ondragover Atom = 0x3de0a + Ondragstart Atom = 0x3e80b + Ondrop Atom = 0x3f706 + Ondurationchange Atom = 0x40710 + Onemptied Atom = 0x3fe09 + Onended Atom = 0x41707 + Onerror Atom = 0x41e07 + Onfocus Atom = 0x42507 + Onhashchange Atom = 0x4310c + Oninput Atom = 0x43d07 + Oninvalid Atom = 0x44909 + Onkeydown Atom = 0x45209 + Onkeypress Atom = 0x45f0a + Onkeyup Atom = 0x47407 + Onlanguagechange Atom = 0x48110 + Onload Atom = 0x49106 + Onloadeddata Atom = 0x4910c + Onloadedmetadata Atom = 0x4a410 + Onloadend Atom = 0x4ba09 + Onloadstart Atom = 0x4c30b + Onmessage Atom = 0x4ce09 + Onmessageerror Atom = 0x4ce0e + Onmousedown Atom = 0x4dc0b + Onmouseenter Atom = 0x4e70c + Onmouseleave Atom = 0x4f30c + Onmousemove Atom = 0x4ff0b + Onmouseout Atom = 0x50a0a + Onmouseover Atom = 0x5170b + Onmouseup Atom = 0x52209 + Onmousewheel Atom = 0x5300c + Onoffline Atom = 0x53c09 + Ononline Atom = 0x54508 + Onpagehide Atom = 0x54d0a + Onpageshow Atom = 0x5670a + Onpaste Atom = 0x57307 + Onpause Atom = 0x58e07 + Onplay Atom = 0x59806 + Onplaying Atom = 0x59809 + Onpopstate Atom = 0x5a10a + Onprogress Atom = 0x5ab0a + Onratechange Atom = 0x5c80c + Onrejectionhandled Atom = 0x5d412 + Onreset Atom = 0x5e607 + Onresize Atom = 0x5ed08 + Onscroll Atom = 0x5fc08 + Onsecuritypolicyviolation Atom = 0x60419 + Onseeked Atom = 0x62508 + Onseeking Atom = 0x62d09 + Onselect Atom = 0x63608 + Onshow Atom = 0x64006 + Onsort Atom = 0x64b06 + Onstalled Atom = 0x65509 + Onstorage Atom = 0x65e09 + Onsubmit Atom = 0x66708 + Onsuspend Atom = 0x67709 + Ontimeupdate Atom = 0x11a0c + Ontoggle Atom = 0x68008 + Onunhandledrejection Atom = 0x68814 + Onunload Atom = 0x6a708 + Onvolumechange Atom = 0x6af0e + Onwaiting Atom = 0x6bd09 + Onwheel Atom = 0x6c607 + Open Atom = 0x55f04 + Optgroup Atom = 0xe008 + Optimum Atom = 0x6cd07 + Option Atom = 0x6dd06 + Output Atom = 0x51106 + P Atom = 0xc01 + Param Atom = 0xc05 + Pattern Atom = 0x4f07 + Picture Atom = 0x9707 + Ping Atom = 0xe704 + Placeholder Atom = 0xfb0b + Plaintext Atom = 0x19e09 + Playsinline Atom = 0x10a0b + Poster Atom = 0x2b706 + Pre Atom = 0x46403 + Preload Atom = 0x47a07 + Progress Atom = 0x5ad08 + Prompt Atom = 0x52a06 + Public Atom = 0x57a06 + Q Atom = 0x7701 + Radiogroup Atom = 0x30a + Readonly Atom = 0x34b08 + Referrerpolicy Atom = 0x3c50e + Rel Atom = 0x47b03 + Required Atom = 0x23408 + Reversed Atom = 0x9c08 + Rows Atom = 0x3a04 + Rowspan Atom = 0x3a07 + Rp Atom = 0x22402 + Rt Atom = 0x17b02 + Ruby Atom = 0xac04 + S Atom = 0x2501 + Samp Atom = 0x4c04 + Sandbox Atom = 0xf307 + Scope Atom = 0x67105 + Scoped Atom = 0x67106 + Script Atom = 0x20006 + Seamless Atom = 0x36508 + Section Atom = 0x5bd07 + Select Atom = 0x63806 + Selected Atom = 0x63808 + Shape Atom = 0x1d505 + Size Atom = 0x5f104 + Sizes Atom = 0x5f105 + Slot Atom = 0x1df04 + Small Atom = 0x1ee05 + Sortable Atom = 0x64d08 + Sorted Atom = 0x32b06 + Source Atom = 0x36c06 + Spacer Atom = 0x42b06 + Span Atom = 0x3d04 + Spellcheck Atom = 0x4680a + Src Atom = 0x5b403 + Srcdoc Atom = 0x5b406 + Srclang Atom = 0x5f507 + Srcset Atom = 0x6f306 + Start Atom = 0x3ee05 + Step Atom = 0x57704 + Strike Atom = 0x7a06 + Strong Atom = 0x31506 + Style Atom = 0x6f905 + Sub Atom = 0x66903 + Summary Atom = 0x6fe07 + Sup Atom = 0x70503 + Svg Atom = 0x70803 + System Atom = 0x70b06 + Tabindex Atom = 0x4b208 + Table Atom = 0x58905 + Target Atom = 0x2ac06 + Tbody Atom = 0x2705 + Td Atom = 0x5e02 + Template Atom = 0x70e08 + Textarea Atom = 0x34608 + Tfoot Atom = 0xb205 + Th Atom = 0x13f02 + Thead Atom = 0x32405 + Time Atom = 0x11c04 + Title Atom = 0xca05 + Tr Atom = 0x7402 + Track Atom = 0x17c05 + Translate Atom = 0x1a609 + Tt Atom = 0x5102 + Type Atom = 0x8104 + Typemustmatch Atom = 0x2780d + U Atom = 0xb01 + Ul Atom = 0x6602 + Updateviacache Atom = 0x1200e + Usemap Atom = 0x59206 + Value Atom = 0x1505 + Var Atom = 0x15603 + Video Atom = 0x2d905 + Wbr Atom = 0x57003 + Width Atom = 0x64505 + Workertype Atom = 0x7160a + Wrap Atom = 0x72004 + Xmp Atom = 0xf903 ) -const hash0 = 0xc17da63e +const hash0 = 0x81cdf10e -const maxAtomLen = 19 +const maxAtomLen = 25 var table = [1 << 9]Atom{ - 0x1: 0x48a0b, // onmousemove - 0x2: 0x5e209, // onwaiting - 0x3: 0x1fa13, // onautocompleteerror - 0x4: 0x5fa06, // prompt - 0x7: 0x5eb07, // optimum - 0x8: 0x1604, // mark - 0xa: 0x5ad07, // itemref - 0xb: 0x4fe0a, // onpageshow - 0xc: 0x57a06, // select - 0xd: 0x17b09, // draggable - 0xe: 0x3e03, // nav - 0xf: 0x17507, // command - 0x11: 0xb01, // u - 0x14: 0x2d507, // headers - 0x15: 0x44a08, // datalist - 0x17: 0x4e04, // samp - 0x1a: 0x3fb09, // onkeydown - 0x1b: 0x55f08, // onscroll - 0x1c: 0x15003, // col - 0x20: 0x3c908, // itemprop - 0x21: 0x2780a, // http-equiv - 0x22: 0x61d03, // sup - 0x24: 0x1d008, // required - 0x2b: 0x25e07, // preload - 0x2c: 0x6040d, // onbeforeprint - 0x2d: 0x3600b, // ondragenter - 0x2e: 0x50902, // dt - 0x2f: 0x5a708, // onsubmit - 0x30: 0x27002, // hr - 0x31: 0x32f0d, // oncontextmenu - 0x33: 0x29c05, // image - 0x34: 0x50d07, // onpause - 0x35: 0x25906, // hgroup - 0x36: 0x7704, // ping - 0x37: 0x57808, // onselect - 0x3a: 0x11303, // div - 0x3b: 0x1fa0e, // onautocomplete - 0x40: 0x2eb02, // mi - 0x41: 0x31c08, // seamless - 0x42: 0x2807, // charset - 0x43: 0x8502, // id - 0x44: 0x5200a, // onpopstate - 0x45: 0x3ef03, // del - 0x46: 0x2cb07, // marquee - 0x47: 0x3309, // accesskey - 0x49: 0x8d06, // footer - 0x4a: 0x44e04, // list - 0x4b: 0x2b005, // ismap - 0x51: 0x33804, // menu - 0x52: 0x2f04, // body - 0x55: 0x9a08, // frameset - 0x56: 0x54a07, // onreset - 0x57: 0x12705, // blink - 0x58: 0xa105, // title - 0x59: 0x38807, // article - 0x5b: 0x22e02, // th - 0x5d: 0x13101, // q - 0x5e: 0x3cf04, // open - 0x5f: 0x2fa04, // area - 0x61: 0x44206, // onload - 0x62: 0xda04, // font - 0x63: 0xd604, // base - 0x64: 0x16207, // colspan - 0x65: 0x53707, // keytype - 0x66: 0x11e02, // dl - 0x68: 0x1b008, // fieldset - 0x6a: 0x2eb03, // min - 0x6b: 0x11503, // var - 0x6f: 0x2d506, // header - 0x70: 0x13f02, // rt - 0x71: 0x15008, // colgroup - 0x72: 0x23502, // mn - 0x74: 0x13a07, // onabort - 0x75: 0x3906, // keygen - 0x76: 0x4c209, // onoffline - 0x77: 0x21f09, // challenge - 0x78: 0x2b203, // map - 0x7a: 0x2e902, // h4 - 0x7b: 0x3b607, // onerror - 0x7c: 0x2e109, // maxlength - 0x7d: 0x2f505, // mtext - 0x7e: 0xbb07, // sandbox - 0x7f: 0x58b06, // onsort - 0x80: 0x100a, // malignmark - 0x81: 0x45d04, // meta - 0x82: 0x7b05, // async - 0x83: 0x2a702, // h3 - 0x84: 0x26702, // dd - 0x85: 0x27004, // href - 0x86: 0x6e0a, // mediagroup - 0x87: 0x19406, // coords - 0x88: 0x41107, // srclang - 0x89: 0x34d0a, // ondblclick - 0x8a: 0x4005, // value - 0x8c: 0xe908, // oncancel - 0x8e: 0x3230a, // spellcheck - 0x8f: 0x9a05, // frame - 0x91: 0x12403, // big - 0x94: 0x1f606, // action - 0x95: 0x6903, // dir - 0x97: 0x2fb08, // readonly - 0x99: 0x42d05, // table - 0x9a: 0x61607, // summary - 0x9b: 0x12103, // wbr - 0x9c: 0x30a, // radiogroup - 0x9d: 0x6c04, // name - 0x9f: 0x62306, // system - 0xa1: 0x15d05, // color - 0xa2: 0x7f06, // canvas - 0xa3: 0x25504, // html - 0xa5: 0x56f09, // onseeking - 0xac: 0x4f905, // shape - 0xad: 0x25f03, // rel - 0xae: 0x28510, // oncanplaythrough - 0xaf: 0x3760a, // ondragover - 0xb0: 0x62608, // template - 0xb1: 0x1d80d, // foreignObject - 0xb3: 0x9204, // rows - 0xb6: 0x44e07, // listing - 0xb7: 0x49c06, // output - 0xb9: 0x3310b, // contextmenu - 0xbb: 0x11f03, // low - 0xbc: 0x1c602, // rp - 0xbd: 0x5bb09, // onsuspend - 0xbe: 0x13606, // button - 0xbf: 0x4db04, // desc - 0xc1: 0x4e207, // section - 0xc2: 0x52a0a, // onprogress - 0xc3: 0x59e09, // onstorage - 0xc4: 0x2d204, // math - 0xc5: 0x4503, // alt - 0xc7: 0x8a02, // ul - 0xc8: 0x5107, // pattern - 0xc9: 0x4b60c, // onmousewheel - 0xca: 0x35709, // ondragend - 0xcb: 0xaf04, // ruby - 0xcc: 0xc01, // p - 0xcd: 0x31707, // onclose - 0xce: 0x24205, // meter - 0xcf: 0x11807, // bgsound - 0xd2: 0x25106, // height - 0xd4: 0x101, // b - 0xd5: 0x2c308, // itemtype - 0xd8: 0x1bb07, // caption - 0xd9: 0x10c08, // disabled - 0xdb: 0x33808, // menuitem - 0xdc: 0x62003, // svg - 0xdd: 0x18f05, // small - 0xde: 0x44a04, // data - 0xe0: 0x4cb08, // ononline - 0xe1: 0x2a206, // mglyph - 0xe3: 0x6505, // embed - 0xe4: 0x10502, // tr - 0xe5: 0x46b0b, // onloadstart - 0xe7: 0x3c306, // srcdoc - 0xeb: 0x5c408, // ontoggle - 0xed: 0xe703, // bdo - 0xee: 0x4702, // td - 0xef: 0x8305, // aside - 0xf0: 0x29402, // h2 - 0xf1: 0x52c08, // progress - 0xf2: 0x12c0a, // blockquote - 0xf4: 0xf005, // label - 0xf5: 0x601, // i - 0xf7: 0x9207, // rowspan - 0xfb: 0x51709, // onplaying - 0xfd: 0x2a103, // img - 0xfe: 0xf608, // optgroup - 0xff: 0x42307, // content - 0x101: 0x53e0c, // onratechange - 0x103: 0x3da0c, // onhashchange - 0x104: 0x4807, // details - 0x106: 0x40008, // download - 0x109: 0x14009, // translate - 0x10b: 0x4230f, // contenteditable - 0x10d: 0x36b0b, // ondragleave - 0x10e: 0x2106, // accept - 0x10f: 0x57a08, // selected - 0x112: 0x1f20a, // formaction - 0x113: 0x5b506, // center - 0x115: 0x45510, // onloadedmetadata - 0x116: 0x12804, // link - 0x117: 0xdd04, // time - 0x118: 0x19f0b, // crossorigin - 0x119: 0x3bd07, // onfocus - 0x11a: 0x58704, // wrap - 0x11b: 0x42204, // icon - 0x11d: 0x28105, // video - 0x11e: 0x4de05, // class - 0x121: 0x5d40e, // onvolumechange - 0x122: 0xaa06, // onblur - 0x123: 0x2b909, // itemscope - 0x124: 0x61105, // style - 0x127: 0x41e06, // public - 0x129: 0x2320e, // formnovalidate - 0x12a: 0x58206, // onshow - 0x12c: 0x51706, // onplay - 0x12d: 0x3c804, // cite - 0x12e: 0x2bc02, // ms - 0x12f: 0xdb0c, // ontimeupdate - 0x130: 0x10904, // kind - 0x131: 0x2470a, // formtarget - 0x135: 0x3af07, // onended - 0x136: 0x26506, // hidden - 0x137: 0x2c01, // s - 0x139: 0x2280a, // formmethod - 0x13a: 0x3e805, // input - 0x13c: 0x50b02, // h6 - 0x13d: 0xc902, // ol - 0x13e: 0x3420b, // oncuechange - 0x13f: 0x1e50d, // foreignobject - 0x143: 0x4e70e, // onbeforeunload - 0x144: 0x2bd05, // scope - 0x145: 0x39609, // onemptied - 0x146: 0x14b05, // defer - 0x147: 0xc103, // xmp - 0x148: 0x39f10, // ondurationchange - 0x149: 0x1903, // kbd - 0x14c: 0x47609, // onmessage - 0x14d: 0x60006, // option - 0x14e: 0x2eb09, // minlength - 0x14f: 0x32807, // checked - 0x150: 0xce08, // autoplay - 0x152: 0x202, // br - 0x153: 0x2360a, // novalidate - 0x156: 0x6307, // noembed - 0x159: 0x31007, // onclick - 0x15a: 0x47f0b, // onmousedown - 0x15b: 0x3a708, // onchange - 0x15e: 0x3f209, // oninvalid - 0x15f: 0x2bd06, // scoped - 0x160: 0x18808, // controls - 0x161: 0x30b05, // muted - 0x162: 0x58d08, // sortable - 0x163: 0x51106, // usemap - 0x164: 0x1b80a, // figcaption - 0x165: 0x35706, // ondrag - 0x166: 0x26b04, // high - 0x168: 0x3c303, // src - 0x169: 0x15706, // poster - 0x16b: 0x1670e, // annotation-xml - 0x16c: 0x5f704, // step - 0x16d: 0x4, // abbr - 0x16e: 0x1b06, // dialog - 0x170: 0x1202, // li - 0x172: 0x3ed02, // mo - 0x175: 0x1d803, // for - 0x176: 0x1a803, // ins - 0x178: 0x55504, // size - 0x179: 0x43210, // onlanguagechange - 0x17a: 0x8607, // default - 0x17b: 0x1a03, // bdi - 0x17c: 0x4d30a, // onpagehide - 0x17d: 0x6907, // dirname - 0x17e: 0x21404, // type - 0x17f: 0x1f204, // form - 0x181: 0x28509, // oncanplay - 0x182: 0x6103, // dfn - 0x183: 0x46308, // tabindex - 0x186: 0x6502, // em - 0x187: 0x27404, // lang - 0x189: 0x39108, // dropzone - 0x18a: 0x4080a, // onkeypress - 0x18b: 0x23c08, // datetime - 0x18c: 0x16204, // cols - 0x18d: 0x1, // a - 0x18e: 0x4420c, // onloadeddata - 0x190: 0xa605, // audio - 0x192: 0x2e05, // tbody - 0x193: 0x22c06, // method - 0x195: 0xf404, // loop - 0x196: 0x29606, // iframe - 0x198: 0x2d504, // head - 0x19e: 0x5f108, // manifest - 0x19f: 0xb309, // autofocus - 0x1a0: 0x14904, // code - 0x1a1: 0x55906, // strong - 0x1a2: 0x30308, // multiple - 0x1a3: 0xc05, // param - 0x1a6: 0x21107, // enctype - 0x1a7: 0x5b304, // face - 0x1a8: 0xfd09, // plaintext - 0x1a9: 0x26e02, // h1 - 0x1aa: 0x59509, // onstalled - 0x1ad: 0x3d406, // script - 0x1ae: 0x2db06, // spacer - 0x1af: 0x55108, // onresize - 0x1b0: 0x4a20b, // onmouseover - 0x1b1: 0x5cc08, // onunload - 0x1b2: 0x56708, // onseeked - 0x1b4: 0x2140d, // typemustmatch - 0x1b5: 0x1cc06, // figure - 0x1b6: 0x4950a, // onmouseout - 0x1b7: 0x25e03, // pre - 0x1b8: 0x50705, // width - 0x1b9: 0x19906, // sorted - 0x1bb: 0x5704, // nobr - 0x1be: 0x5302, // tt - 0x1bf: 0x1105, // align - 0x1c0: 0x3e607, // oninput - 0x1c3: 0x41807, // onkeyup - 0x1c6: 0x1c00c, // onafterprint - 0x1c7: 0x210e, // accept-charset - 0x1c8: 0x33c06, // itemid - 0x1c9: 0x3e809, // inputmode - 0x1cb: 0x53306, // strike - 0x1cc: 0x5a903, // sub - 0x1cd: 0x10505, // track - 0x1ce: 0x38605, // start - 0x1d0: 0xd608, // basefont - 0x1d6: 0x1aa06, // source - 0x1d7: 0x18206, // legend - 0x1d8: 0x2d405, // thead - 0x1da: 0x8c05, // tfoot - 0x1dd: 0x1ec06, // object - 0x1de: 0x6e05, // media - 0x1df: 0x1670a, // annotation - 0x1e0: 0x20d0b, // formenctype - 0x1e2: 0x3d208, // noscript - 0x1e4: 0x55505, // sizes - 0x1e5: 0x1fc0c, // autocomplete - 0x1e6: 0x9504, // span - 0x1e7: 0x9808, // noframes - 0x1e8: 0x24b06, // target - 0x1e9: 0x38f06, // ondrop - 0x1ea: 0x2b306, // applet - 0x1ec: 0x5a08, // reversed - 0x1f0: 0x2a907, // isindex - 0x1f3: 0x27008, // hreflang - 0x1f5: 0x2f302, // h5 - 0x1f6: 0x4f307, // address - 0x1fa: 0x2e103, // max - 0x1fb: 0xc30b, // placeholder - 0x1fc: 0x2f608, // textarea - 0x1fe: 0x4ad09, // onmouseup - 0x1ff: 0x3800b, // ondragstart + 0x1: 0x8e0a, // mediagroup + 0x2: 0x2cc04, // lang + 0x4: 0x2c09, // accesskey + 0x5: 0x5708, // frameset + 0x7: 0x63608, // onselect + 0x8: 0x70b06, // system + 0xa: 0x64505, // width + 0xc: 0x2710b, // formenctype + 0xd: 0x10102, // ol + 0xe: 0x38b0b, // oncuechange + 0x10: 0x13403, // bdo + 0x11: 0xcf05, // audio + 0x12: 0x18309, // draggable + 0x14: 0x2d905, // video + 0x15: 0x29902, // mn + 0x16: 0x37b04, // menu + 0x17: 0x2b706, // poster + 0x19: 0xb306, // footer + 0x1a: 0x29006, // method + 0x1b: 0x2a008, // datetime + 0x1c: 0x17607, // onabort + 0x1d: 0x1200e, // updateviacache + 0x1e: 0xb905, // async + 0x1f: 0x49106, // onload + 0x21: 0xd308, // oncancel + 0x22: 0x62508, // onseeked + 0x23: 0x2ea05, // image + 0x24: 0x5d412, // onrejectionhandled + 0x26: 0x15d04, // link + 0x27: 0x51106, // output + 0x28: 0x32504, // head + 0x29: 0x4f30c, // onmouseleave + 0x2a: 0x57307, // onpaste + 0x2b: 0x59809, // onplaying + 0x2c: 0x1b407, // colspan + 0x2f: 0x1af05, // color + 0x30: 0x5f104, // size + 0x31: 0x2d00a, // http-equiv + 0x33: 0x601, // i + 0x34: 0x54d0a, // onpagehide + 0x35: 0x68814, // onunhandledrejection + 0x37: 0x41e07, // onerror + 0x3a: 0x11508, // basefont + 0x3f: 0x1303, // nav + 0x40: 0x18004, // kind + 0x41: 0x34b08, // readonly + 0x42: 0x2f006, // mglyph + 0x44: 0x11102, // li + 0x46: 0x2bd06, // hidden + 0x47: 0x70803, // svg + 0x48: 0x57704, // step + 0x49: 0x22709, // integrity + 0x4a: 0x57a06, // public + 0x4c: 0x19703, // col + 0x4d: 0x1680a, // blockquote + 0x4e: 0x34302, // h5 + 0x50: 0x5ad08, // progress + 0x51: 0x5f105, // sizes + 0x52: 0x33902, // h4 + 0x56: 0x32405, // thead + 0x57: 0x7e07, // keytype + 0x58: 0x5ab0a, // onprogress + 0x59: 0x43f09, // inputmode + 0x5a: 0x3a509, // ondragend + 0x5d: 0x39605, // oncut + 0x5e: 0x42b06, // spacer + 0x5f: 0x19708, // colgroup + 0x62: 0x14e02, // is + 0x65: 0xb902, // as + 0x66: 0x53c09, // onoffline + 0x67: 0x32b06, // sorted + 0x69: 0x48110, // onlanguagechange + 0x6c: 0x4310c, // onhashchange + 0x6d: 0xa604, // name + 0x6e: 0xb205, // tfoot + 0x6f: 0x55504, // desc + 0x70: 0x33103, // max + 0x72: 0x1da06, // coords + 0x73: 0x2f502, // h3 + 0x74: 0x6e10e, // onbeforeunload + 0x75: 0x3a04, // rows + 0x76: 0x63806, // select + 0x77: 0xa805, // meter + 0x78: 0x37f06, // itemid + 0x79: 0x5300c, // onmousewheel + 0x7a: 0x5b406, // srcdoc + 0x7d: 0x17c05, // track + 0x7f: 0x30708, // itemtype + 0x82: 0x6302, // mo + 0x83: 0x40f08, // onchange + 0x84: 0x32507, // headers + 0x85: 0x5c80c, // onratechange + 0x86: 0x60419, // onsecuritypolicyviolation + 0x88: 0x49908, // datalist + 0x89: 0x4dc0b, // onmousedown + 0x8a: 0x1df04, // slot + 0x8b: 0x4a410, // onloadedmetadata + 0x8c: 0x1a06, // accept + 0x8d: 0x25006, // object + 0x91: 0x6af0e, // onvolumechange + 0x92: 0x2107, // charset + 0x93: 0x25e13, // onautocompleteerror + 0x94: 0x6913, // allowpaymentrequest + 0x95: 0x2804, // body + 0x96: 0xc407, // default + 0x97: 0x63808, // selected + 0x98: 0x20604, // face + 0x99: 0x1d505, // shape + 0x9b: 0x68008, // ontoggle + 0x9e: 0x64702, // dt + 0x9f: 0x6d904, // mark + 0xa1: 0xb01, // u + 0xa4: 0x6a708, // onunload + 0xa5: 0xde04, // loop + 0xa6: 0x14d08, // disabled + 0xaa: 0x41707, // onended + 0xab: 0x6d30a, // malignmark + 0xad: 0x67709, // onsuspend + 0xae: 0x34505, // mtext + 0xaf: 0x64b06, // onsort + 0xb0: 0x55908, // itemprop + 0xb3: 0x66d09, // itemscope + 0xb4: 0x15c05, // blink + 0xb6: 0x3a506, // ondrag + 0xb7: 0x6602, // ul + 0xb8: 0x25604, // form + 0xb9: 0xf307, // sandbox + 0xba: 0x5705, // frame + 0xbb: 0x1505, // value + 0xbc: 0x65e09, // onstorage + 0xc0: 0x17b02, // rt + 0xc2: 0x202, // br + 0xc3: 0x20e08, // fieldset + 0xc4: 0x2780d, // typemustmatch + 0xc5: 0x6108, // nomodule + 0xc6: 0x4007, // noembed + 0xc7: 0x69a0d, // onbeforeprint + 0xc8: 0x17206, // button + 0xc9: 0x2dd07, // onclick + 0xca: 0x6fe07, // summary + 0xcd: 0xac04, // ruby + 0xce: 0x5b905, // class + 0xcf: 0x3e80b, // ondragstart + 0xd0: 0x21907, // caption + 0xd4: 0x850e, // allowusermedia + 0xd5: 0x4c30b, // onloadstart + 0xd9: 0x15403, // div + 0xda: 0x49d04, // list + 0xdb: 0x32204, // math + 0xdc: 0x43f05, // input + 0xdf: 0x3de0a, // ondragover + 0xe0: 0x2c602, // h2 + 0xe2: 0x19e09, // plaintext + 0xe4: 0x4e70c, // onmouseenter + 0xe7: 0x46d07, // checked + 0xe8: 0x46403, // pre + 0xea: 0x35308, // multiple + 0xeb: 0x16103, // bdi + 0xec: 0x33109, // maxlength + 0xed: 0x7701, // q + 0xee: 0x61b0a, // onauxclick + 0xf0: 0x57003, // wbr + 0xf2: 0x11504, // base + 0xf3: 0x6dd06, // option + 0xf5: 0x40710, // ondurationchange + 0xf7: 0x5508, // noframes + 0xf9: 0x3f908, // dropzone + 0xfb: 0x67105, // scope + 0xfc: 0x9c08, // reversed + 0xfd: 0x3ae0b, // ondragenter + 0xfe: 0x3ee05, // start + 0xff: 0xf903, // xmp + 0x100: 0x5f507, // srclang + 0x101: 0x2ef03, // img + 0x104: 0x101, // b + 0x105: 0x23c03, // for + 0x106: 0xc105, // aside + 0x107: 0x43d07, // oninput + 0x108: 0x34a04, // area + 0x109: 0x28c0a, // formmethod + 0x10a: 0x72004, // wrap + 0x10c: 0x22402, // rp + 0x10d: 0x45f0a, // onkeypress + 0x10e: 0x5102, // tt + 0x110: 0x33b02, // mi + 0x111: 0x35b05, // muted + 0x112: 0xb003, // alt + 0x113: 0x19004, // code + 0x114: 0x4202, // em + 0x115: 0x3b90a, // ondragexit + 0x117: 0x3d04, // span + 0x119: 0x30f08, // manifest + 0x11a: 0x37b08, // menuitem + 0x11b: 0x57f07, // content + 0x11d: 0x6bd09, // onwaiting + 0x11f: 0x4ba09, // onloadend + 0x121: 0x3720d, // oncontextmenu + 0x123: 0x5c206, // onblur + 0x124: 0x3f007, // article + 0x125: 0xa303, // dir + 0x126: 0xe704, // ping + 0x127: 0x23408, // required + 0x128: 0x44909, // oninvalid + 0x129: 0x6d405, // align + 0x12b: 0x57e04, // icon + 0x12c: 0x64902, // h6 + 0x12d: 0x1b404, // cols + 0x12e: 0x2160a, // figcaption + 0x12f: 0x45209, // onkeydown + 0x130: 0x66708, // onsubmit + 0x131: 0x13609, // oncanplay + 0x132: 0x70503, // sup + 0x133: 0xc01, // p + 0x135: 0x3fe09, // onemptied + 0x136: 0x38506, // oncopy + 0x137: 0x55804, // cite + 0x138: 0x39b0a, // ondblclick + 0x13a: 0x4ff0b, // onmousemove + 0x13c: 0x66903, // sub + 0x13d: 0x47b03, // rel + 0x13e: 0xe008, // optgroup + 0x142: 0x3a07, // rowspan + 0x143: 0x36c06, // source + 0x144: 0x1fe08, // noscript + 0x145: 0x55f04, // open + 0x146: 0x1ec03, // ins + 0x147: 0x23c0d, // foreignObject + 0x148: 0x5a10a, // onpopstate + 0x14a: 0x27507, // enctype + 0x14b: 0x25e0e, // onautocomplete + 0x14c: 0x34608, // textarea + 0x14e: 0x2600c, // autocomplete + 0x14f: 0x14002, // hr + 0x150: 0x1ce08, // controls + 0x151: 0xc302, // id + 0x153: 0x21e0c, // onafterprint + 0x155: 0x2490d, // foreignobject + 0x156: 0x31b07, // marquee + 0x157: 0x58e07, // onpause + 0x158: 0x5e202, // dl + 0x159: 0x12c06, // height + 0x15a: 0x33b03, // min + 0x15b: 0xa307, // dirname + 0x15c: 0x1a609, // translate + 0x15d: 0x13004, // html + 0x15e: 0x33b09, // minlength + 0x15f: 0x47a07, // preload + 0x160: 0x70e08, // template + 0x161: 0x3d30b, // ondragleave + 0x164: 0x5b403, // src + 0x165: 0x31506, // strong + 0x167: 0x4c04, // samp + 0x168: 0x6ed07, // address + 0x169: 0x54508, // ononline + 0x16b: 0xfb0b, // placeholder + 0x16c: 0x2ac06, // target + 0x16d: 0x1ee05, // small + 0x16e: 0x6c607, // onwheel + 0x16f: 0x1b90a, // annotation + 0x170: 0x4680a, // spellcheck + 0x171: 0x4607, // details + 0x172: 0xbd06, // canvas + 0x173: 0xeb09, // autofocus + 0x174: 0xc05, // param + 0x176: 0x45708, // download + 0x177: 0x44603, // del + 0x178: 0x36007, // onclose + 0x179: 0x16003, // kbd + 0x17a: 0x30106, // applet + 0x17b: 0x2c804, // href + 0x17c: 0x5ed08, // onresize + 0x17e: 0x4910c, // onloadeddata + 0x180: 0x7402, // tr + 0x181: 0x2a80a, // formtarget + 0x182: 0xca05, // title + 0x183: 0x6f905, // style + 0x184: 0x7a06, // strike + 0x185: 0x59206, // usemap + 0x186: 0x2e406, // iframe + 0x187: 0x1004, // main + 0x189: 0x9707, // picture + 0x18c: 0x2fe05, // ismap + 0x18e: 0x49904, // data + 0x18f: 0xda05, // label + 0x191: 0x3c50e, // referrerpolicy + 0x192: 0x13f02, // th + 0x194: 0x52a06, // prompt + 0x195: 0x5bd07, // section + 0x197: 0x6cd07, // optimum + 0x198: 0x2c304, // high + 0x199: 0x14502, // h1 + 0x19a: 0x65509, // onstalled + 0x19b: 0x15603, // var + 0x19c: 0x11c04, // time + 0x19e: 0x67002, // ms + 0x19f: 0x32506, // header + 0x1a0: 0x4ce09, // onmessage + 0x1a1: 0x56205, // nonce + 0x1a2: 0x2560a, // formaction + 0x1a3: 0x20806, // center + 0x1a4: 0x3704, // nobr + 0x1a5: 0x58905, // table + 0x1a6: 0x49d07, // listing + 0x1a7: 0x18a06, // legend + 0x1a9: 0x28309, // challenge + 0x1aa: 0x23006, // figure + 0x1ab: 0x8e05, // media + 0x1ae: 0x8104, // type + 0x1af: 0x11904, // font + 0x1b0: 0x4ce0e, // onmessageerror + 0x1b1: 0x36508, // seamless + 0x1b2: 0x5f03, // dfn + 0x1b3: 0x19205, // defer + 0x1b4: 0x6b03, // low + 0x1b5: 0x62d09, // onseeking + 0x1b6: 0x5170b, // onmouseover + 0x1b7: 0x29a0a, // novalidate + 0x1b8: 0x7160a, // workertype + 0x1ba: 0x3c107, // itemref + 0x1bd: 0x1, // a + 0x1be: 0x30003, // map + 0x1bf: 0x11a0c, // ontimeupdate + 0x1c0: 0x14707, // bgsound + 0x1c1: 0x3206, // keygen + 0x1c2: 0x2705, // tbody + 0x1c5: 0x64006, // onshow + 0x1c7: 0x2501, // s + 0x1c8: 0x4f07, // pattern + 0x1cc: 0x13610, // oncanplaythrough + 0x1ce: 0x2bf02, // dd + 0x1cf: 0x6f306, // srcset + 0x1d0: 0x15903, // big + 0x1d2: 0x64d08, // sortable + 0x1d3: 0x47407, // onkeyup + 0x1d5: 0x59806, // onplay + 0x1d7: 0x4ac04, // meta + 0x1d8: 0x3f706, // ondrop + 0x1da: 0x5fc08, // onscroll + 0x1db: 0x1e30b, // crossorigin + 0x1dc: 0x5670a, // onpageshow + 0x1dd: 0x4, // abbr + 0x1de: 0x5e02, // td + 0x1df: 0x57f0f, // contenteditable + 0x1e0: 0x25a06, // action + 0x1e1: 0x10a0b, // playsinline + 0x1e2: 0x42507, // onfocus + 0x1e3: 0x2c808, // hreflang + 0x1e5: 0x50a0a, // onmouseout + 0x1e6: 0x5e607, // onreset + 0x1e7: 0x10608, // autoplay + 0x1ea: 0x67106, // scoped + 0x1ec: 0x30a, // radiogroup + 0x1ee: 0x3740b, // contextmenu + 0x1ef: 0x52209, // onmouseup + 0x1f1: 0x2b206, // hgroup + 0x1f2: 0x1f00f, // allowfullscreen + 0x1f3: 0x4b208, // tabindex + 0x1f6: 0x2f707, // isindex + 0x1f7: 0x1a0e, // accept-charset + 0x1f8: 0x2960e, // formnovalidate + 0x1fb: 0x1b90e, // annotation-xml + 0x1fc: 0x4205, // embed + 0x1fd: 0x20006, // script + 0x1fe: 0x16206, // dialog + 0x1ff: 0x1c707, // command } -const atomText = "abbradiogrouparamalignmarkbdialogaccept-charsetbodyaccesskey" + - "genavaluealtdetailsampatternobreversedfnoembedirnamediagroup" + - "ingasyncanvasidefaultfooterowspanoframesetitleaudionblurubya" + - "utofocusandboxmplaceholderautoplaybasefontimeupdatebdoncance" + - "labelooptgrouplaintextrackindisabledivarbgsoundlowbrbigblink" + - "blockquotebuttonabortranslatecodefercolgroupostercolorcolspa" + - "nnotation-xmlcommandraggablegendcontrolsmallcoordsortedcross" + - "originsourcefieldsetfigcaptionafterprintfigurequiredforeignO" + - "bjectforeignobjectformactionautocompleteerrorformenctypemust" + - "matchallengeformmethodformnovalidatetimeterformtargetheightm" + - "lhgroupreloadhiddenhigh1hreflanghttp-equivideoncanplaythroug" + - "h2iframeimageimglyph3isindexismappletitemscopeditemtypemarqu" + - "eematheaderspacermaxlength4minlength5mtextareadonlymultiplem" + - "utedonclickoncloseamlesspellcheckedoncontextmenuitemidoncuec" + - "hangeondblclickondragendondragenterondragleaveondragoverondr" + - "agstarticleondropzonemptiedondurationchangeonendedonerroronf" + - "ocusrcdocitempropenoscriptonhashchangeoninputmodeloninvalido" + - "nkeydownloadonkeypressrclangonkeyupublicontenteditableonlang" + - "uagechangeonloadeddatalistingonloadedmetadatabindexonloadsta" + - "rtonmessageonmousedownonmousemoveonmouseoutputonmouseoveronm" + - "ouseuponmousewheelonofflineononlineonpagehidesclassectionbef" + - "oreunloaddresshapeonpageshowidth6onpausemaponplayingonpopsta" + - "teonprogresstrikeytypeonratechangeonresetonresizestrongonscr" + - "ollonseekedonseekingonselectedonshowraponsortableonstalledon" + - "storageonsubmitemrefacenteronsuspendontoggleonunloadonvolume" + - "changeonwaitingoptimumanifestepromptoptionbeforeprintstylesu" + - "mmarysupsvgsystemplate" +const atomText = "abbradiogrouparamainavalueaccept-charsetbodyaccesskeygenobro" + + "wspanoembedetailsampatternoframesetdfnomoduleallowpaymentreq" + + "uestrikeytypeallowusermediagroupictureversedirnameterubyaltf" + + "ooterasyncanvasidefaultitleaudioncancelabelooptgroupingautof" + + "ocusandboxmplaceholderautoplaysinlinebasefontimeupdateviacac" + + "heightmlbdoncanplaythrough1bgsoundisabledivarbigblinkbdialog" + + "blockquotebuttonabortrackindraggablegendcodefercolgrouplaint" + + "extranslatecolorcolspannotation-xmlcommandcontrolshapecoords" + + "lotcrossoriginsmallowfullscreenoscriptfacenterfieldsetfigcap" + + "tionafterprintegrityfigurequiredforeignObjectforeignobjectfo" + + "rmactionautocompleteerrorformenctypemustmatchallengeformmeth" + + "odformnovalidatetimeformtargethgrouposterhiddenhigh2hreflang" + + "http-equivideonclickiframeimageimglyph3isindexismappletitemt" + + "ypemanifestrongmarqueematheadersortedmaxlength4minlength5mte" + + "xtareadonlymultiplemutedoncloseamlessourceoncontextmenuitemi" + + "doncopyoncuechangeoncutondblclickondragendondragenterondrage" + + "xitemreferrerpolicyondragleaveondragoverondragstarticleondro" + + "pzonemptiedondurationchangeonendedonerroronfocuspaceronhashc" + + "hangeoninputmodeloninvalidonkeydownloadonkeypresspellchecked" + + "onkeyupreloadonlanguagechangeonloadeddatalistingonloadedmeta" + + "databindexonloadendonloadstartonmessageerroronmousedownonmou" + + "seenteronmouseleaveonmousemoveonmouseoutputonmouseoveronmous" + + "eupromptonmousewheelonofflineononlineonpagehidescitempropeno" + + "nceonpageshowbronpastepublicontenteditableonpausemaponplayin" + + "gonpopstateonprogressrcdoclassectionbluronratechangeonreject" + + "ionhandledonresetonresizesrclangonscrollonsecuritypolicyviol" + + "ationauxclickonseekedonseekingonselectedonshowidth6onsortabl" + + "eonstalledonstorageonsubmitemscopedonsuspendontoggleonunhand" + + "ledrejectionbeforeprintonunloadonvolumechangeonwaitingonwhee" + + "loptimumalignmarkoptionbeforeunloaddressrcsetstylesummarysup" + + "svgsystemplateworkertypewrap" diff --git a/vendor/golang.org/x/net/html/const.go b/vendor/golang.org/x/net/html/const.go index 52f651ff6db..b37e6212471 100644 --- a/vendor/golang.org/x/net/html/const.go +++ b/vendor/golang.org/x/net/html/const.go @@ -52,10 +52,12 @@ var isSpecialElementMap = map[string]bool{ "iframe": true, "img": true, "input": true, - "isindex": true, + "isindex": true, // The 'isindex' element has been removed, but keep it for backwards compatibility. + "keygen": true, "li": true, "link": true, "listing": true, + "main": true, "marquee": true, "menu": true, "meta": true, diff --git a/vendor/golang.org/x/net/html/doc.go b/vendor/golang.org/x/net/html/doc.go index b453fe1e4c2..e3ec457d325 100644 --- a/vendor/golang.org/x/net/html/doc.go +++ b/vendor/golang.org/x/net/html/doc.go @@ -49,18 +49,18 @@ call to Next. For example, to extract an HTML page's anchor text: for { tt := z.Next() switch tt { - case ErrorToken: + case html.ErrorToken: return z.Err() - case TextToken: + case html.TextToken: if depth > 0 { // emitBytes should copy the []byte it receives, // if it doesn't process it immediately. emitBytes(z.Text()) } - case StartTagToken, EndTagToken: + case html.StartTagToken, html.EndTagToken: tn, _ := z.TagName() if len(tn) == 1 && tn[0] == 'a' { - if tt == StartTagToken { + if tt == html.StartTagToken { depth++ } else { depth-- diff --git a/vendor/golang.org/x/net/html/token.go b/vendor/golang.org/x/net/html/token.go index 893e272a9e8..e3c01d7c906 100644 --- a/vendor/golang.org/x/net/html/token.go +++ b/vendor/golang.org/x/net/html/token.go @@ -1161,8 +1161,8 @@ func (z *Tokenizer) TagAttr() (key, val []byte, moreAttr bool) { return nil, nil, false } -// Token returns the next Token. The result's Data and Attr values remain valid -// after subsequent Next calls. +// Token returns the current Token. The result's Data and Attr values remain +// valid after subsequent Next calls. func (z *Tokenizer) Token() Token { t := Token{Type: z.tt} switch z.tt { diff --git a/vendor/golang.org/x/net/http2/ciphers.go b/vendor/golang.org/x/net/http2/ciphers.go index 698860b7775..c9a0cf3b422 100644 --- a/vendor/golang.org/x/net/http2/ciphers.go +++ b/vendor/golang.org/x/net/http2/ciphers.go @@ -5,7 +5,7 @@ package http2 // A list of the possible cipher suite ids. Taken from -// http://www.iana.org/assignments/tls-parameters/tls-parameters.txt +// https://www.iana.org/assignments/tls-parameters/tls-parameters.txt const ( cipher_TLS_NULL_WITH_NULL_NULL uint16 = 0x0000 diff --git a/vendor/golang.org/x/net/http2/configure_transport.go b/vendor/golang.org/x/net/http2/configure_transport.go index b65fc6d423d..088d6e2bdb3 100644 --- a/vendor/golang.org/x/net/http2/configure_transport.go +++ b/vendor/golang.org/x/net/http2/configure_transport.go @@ -73,7 +73,7 @@ type noDialH2RoundTripper struct{ t *Transport } func (rt noDialH2RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { res, err := rt.t.RoundTrip(req) - if err == ErrNoCachedConn { + if isNoCachedConnError(err) { return nil, http.ErrSkipAltProtocol } return res, err diff --git a/vendor/golang.org/x/net/http2/server.go b/vendor/golang.org/x/net/http2/server.go index eae143ddff1..460ede03b17 100644 --- a/vendor/golang.org/x/net/http2/server.go +++ b/vendor/golang.org/x/net/http2/server.go @@ -220,12 +220,15 @@ func ConfigureServer(s *http.Server, conf *Server) error { } else if s.TLSConfig.CipherSuites != nil { // If they already provided a CipherSuite list, return // an error if it has a bad order or is missing - // ECDHE_RSA_WITH_AES_128_GCM_SHA256. - const requiredCipher = tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + // ECDHE_RSA_WITH_AES_128_GCM_SHA256 or ECDHE_ECDSA_WITH_AES_128_GCM_SHA256. haveRequired := false sawBad := false for i, cs := range s.TLSConfig.CipherSuites { - if cs == requiredCipher { + switch cs { + case tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + // Alternative MTI cipher to not discourage ECDSA-only servers. + // See http://golang.org/cl/30721 for further information. + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256: haveRequired = true } if isBadCipher(cs) { @@ -235,7 +238,7 @@ func ConfigureServer(s *http.Server, conf *Server) error { } } if !haveRequired { - return fmt.Errorf("http2: TLSConfig.CipherSuites is missing HTTP/2-required TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256") + return fmt.Errorf("http2: TLSConfig.CipherSuites is missing an HTTP/2-required AES_128_GCM_SHA256 cipher.") } } @@ -649,7 +652,7 @@ func (sc *serverConn) condlogf(err error, format string, args ...interface{}) { if err == nil { return } - if err == io.EOF || err == io.ErrUnexpectedEOF || isClosedConnError(err) { + if err == io.EOF || err == io.ErrUnexpectedEOF || isClosedConnError(err) || err == errPrefaceTimeout { // Boring, expected errors. sc.vlogf(format, args...) } else { @@ -853,8 +856,13 @@ func (sc *serverConn) serve() { } } - if sc.inGoAway && sc.curOpenStreams() == 0 && !sc.needToSendGoAway && !sc.writingFrame { - return + // Start the shutdown timer after sending a GOAWAY. When sending GOAWAY + // with no error code (graceful shutdown), don't start the timer until + // all open streams have been completed. + sentGoAway := sc.inGoAway && !sc.needToSendGoAway && !sc.writingFrame + gracefulShutdownComplete := sc.goAwayCode == ErrCodeNo && sc.curOpenStreams() == 0 + if sentGoAway && sc.shutdownTimer == nil && (sc.goAwayCode != ErrCodeNo || gracefulShutdownComplete) { + sc.shutDownIn(goAwayTimeout) } } } @@ -889,8 +897,11 @@ func (sc *serverConn) sendServeMsg(msg interface{}) { } } -// readPreface reads the ClientPreface greeting from the peer -// or returns an error on timeout or an invalid greeting. +var errPrefaceTimeout = errors.New("timeout waiting for client preface") + +// readPreface reads the ClientPreface greeting from the peer or +// returns errPrefaceTimeout on timeout, or an error if the greeting +// is invalid. func (sc *serverConn) readPreface() error { errc := make(chan error, 1) go func() { @@ -908,7 +919,7 @@ func (sc *serverConn) readPreface() error { defer timer.Stop() select { case <-timer.C: - return errors.New("timeout waiting for client preface") + return errPrefaceTimeout case err := <-errc: if err == nil { if VerboseLogs { @@ -1218,30 +1229,31 @@ func (sc *serverConn) startGracefulShutdown() { sc.shutdownOnce.Do(func() { sc.sendServeMsg(gracefulShutdownMsg) }) } +// After sending GOAWAY, the connection will close after goAwayTimeout. +// If we close the connection immediately after sending GOAWAY, there may +// be unsent data in our kernel receive buffer, which will cause the kernel +// to send a TCP RST on close() instead of a FIN. This RST will abort the +// connection immediately, whether or not the client had received the GOAWAY. +// +// Ideally we should delay for at least 1 RTT + epsilon so the client has +// a chance to read the GOAWAY and stop sending messages. Measuring RTT +// is hard, so we approximate with 1 second. See golang.org/issue/18701. +// +// This is a var so it can be shorter in tests, where all requests uses the +// loopback interface making the expected RTT very small. +// +// TODO: configurable? +var goAwayTimeout = 1 * time.Second + func (sc *serverConn) startGracefulShutdownInternal() { - sc.goAwayIn(ErrCodeNo, 0) + sc.goAway(ErrCodeNo) } func (sc *serverConn) goAway(code ErrCode) { - sc.serveG.check() - var forceCloseIn time.Duration - if code != ErrCodeNo { - forceCloseIn = 250 * time.Millisecond - } else { - // TODO: configurable - forceCloseIn = 1 * time.Second - } - sc.goAwayIn(code, forceCloseIn) -} - -func (sc *serverConn) goAwayIn(code ErrCode, forceCloseIn time.Duration) { sc.serveG.check() if sc.inGoAway { return } - if forceCloseIn != 0 { - sc.shutDownIn(forceCloseIn) - } sc.inGoAway = true sc.needToSendGoAway = true sc.goAwayCode = code @@ -2310,7 +2322,7 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) { clen = strconv.Itoa(len(p)) } _, hasContentType := rws.snapHeader["Content-Type"] - if !hasContentType && bodyAllowedForStatus(rws.status) { + if !hasContentType && bodyAllowedForStatus(rws.status) && len(p) > 0 { ctype = http.DetectContentType(p) } var date string @@ -2478,6 +2490,24 @@ func (w *responseWriter) Header() http.Header { return rws.handlerHeader } +// checkWriteHeaderCode is a copy of net/http's checkWriteHeaderCode. +func checkWriteHeaderCode(code int) { + // Issue 22880: require valid WriteHeader status codes. + // For now we only enforce that it's three digits. + // In the future we might block things over 599 (600 and above aren't defined + // at http://httpwg.org/specs/rfc7231.html#status.codes) + // and we might block under 200 (once we have more mature 1xx support). + // But for now any three digits. + // + // We used to send "HTTP/1.1 000 0" on the wire in responses but there's + // no equivalent bogus thing we can realistically send in HTTP/2, + // so we'll consistently panic instead and help people find their bugs + // early. (We can't return an error from WriteHeader even if we wanted to.) + if code < 100 || code > 999 { + panic(fmt.Sprintf("invalid WriteHeader code %v", code)) + } +} + func (w *responseWriter) WriteHeader(code int) { rws := w.rws if rws == nil { @@ -2488,6 +2518,7 @@ func (w *responseWriter) WriteHeader(code int) { func (rws *responseWriterState) writeHeader(code int) { if !rws.wroteHeader { + checkWriteHeaderCode(code) rws.wroteHeader = true rws.status = code if len(rws.handlerHeader) > 0 { diff --git a/vendor/golang.org/x/net/http2/transport.go b/vendor/golang.org/x/net/http2/transport.go index e0dfe9f6a69..e6b321f4bb6 100644 --- a/vendor/golang.org/x/net/http2/transport.go +++ b/vendor/golang.org/x/net/http2/transport.go @@ -87,7 +87,7 @@ type Transport struct { // MaxHeaderListSize is the http2 SETTINGS_MAX_HEADER_LIST_SIZE to // send in the initial settings frame. It is how many bytes - // of response headers are allow. Unlike the http2 spec, zero here + // of response headers are allowed. Unlike the http2 spec, zero here // means to use a default limit (currently 10MB). If you actually // want to advertise an ulimited value to the peer, Transport // interprets the highest possible value here (0xffffffff or 1<<32-1) @@ -172,9 +172,10 @@ type ClientConn struct { fr *Framer lastActive time.Time // Settings from peer: (also guarded by mu) - maxFrameSize uint32 - maxConcurrentStreams uint32 - initialWindowSize uint32 + maxFrameSize uint32 + maxConcurrentStreams uint32 + peerMaxHeaderListSize uint64 + initialWindowSize uint32 hbuf bytes.Buffer // HPACK encoder writes into this henc *hpack.Encoder @@ -273,6 +274,13 @@ func (cs *clientStream) checkResetOrDone() error { } } +func (cs *clientStream) getStartedWrite() bool { + cc := cs.cc + cc.mu.Lock() + defer cc.mu.Unlock() + return cs.startedWrite +} + func (cs *clientStream) abortRequestBodyWrite(err error) { if err == nil { panic("nil error") @@ -298,7 +306,26 @@ func (sew stickyErrWriter) Write(p []byte) (n int, err error) { return } -var ErrNoCachedConn = errors.New("http2: no cached connection was available") +// noCachedConnError is the concrete type of ErrNoCachedConn, which +// needs to be detected by net/http regardless of whether it's its +// bundled version (in h2_bundle.go with a rewritten type name) or +// from a user's x/net/http2. As such, as it has a unique method name +// (IsHTTP2NoCachedConnError) that net/http sniffs for via func +// isNoCachedConnError. +type noCachedConnError struct{} + +func (noCachedConnError) IsHTTP2NoCachedConnError() {} +func (noCachedConnError) Error() string { return "http2: no cached connection was available" } + +// isNoCachedConnError reports whether err is of type noCachedConnError +// or its equivalent renamed type in net/http2's h2_bundle.go. Both types +// may coexist in the same running program. +func isNoCachedConnError(err error) bool { + _, ok := err.(interface{ IsHTTP2NoCachedConnError() }) + return ok +} + +var ErrNoCachedConn error = noCachedConnError{} // RoundTripOpt are options for the Transport.RoundTripOpt method. type RoundTripOpt struct { @@ -348,14 +375,9 @@ func (t *Transport) RoundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Res return nil, err } traceGotConn(req, cc) - res, err := cc.RoundTrip(req) + res, gotErrAfterReqBodyWrite, err := cc.roundTrip(req) if err != nil && retry <= 6 { - afterBodyWrite := false - if e, ok := err.(afterReqBodyWriteError); ok { - err = e - afterBodyWrite = true - } - if req, err = shouldRetryRequest(req, err, afterBodyWrite); err == nil { + if req, err = shouldRetryRequest(req, err, gotErrAfterReqBodyWrite); err == nil { // After the first retry, do exponential backoff with 10% jitter. if retry == 0 { continue @@ -393,16 +415,6 @@ var ( errClientConnGotGoAway = errors.New("http2: Transport received Server's graceful shutdown GOAWAY") ) -// afterReqBodyWriteError is a wrapper around errors returned by ClientConn.RoundTrip. -// It is used to signal that err happened after part of Request.Body was sent to the server. -type afterReqBodyWriteError struct { - err error -} - -func (e afterReqBodyWriteError) Error() string { - return e.err.Error() + "; some request body already written" -} - // shouldRetryRequest is called by RoundTrip when a request fails to get // response headers. It is always called with a non-nil error. // It returns either a request to retry (either the same request, or a @@ -519,17 +531,18 @@ func (t *Transport) NewClientConn(c net.Conn) (*ClientConn, error) { func (t *Transport) newClientConn(c net.Conn, singleUse bool) (*ClientConn, error) { cc := &ClientConn{ - t: t, - tconn: c, - readerDone: make(chan struct{}), - nextStreamID: 1, - maxFrameSize: 16 << 10, // spec default - initialWindowSize: 65535, // spec default - maxConcurrentStreams: 1000, // "infinite", per spec. 1000 seems good enough. - streams: make(map[uint32]*clientStream), - singleUse: singleUse, - wantSettingsAck: true, - pings: make(map[[8]byte]chan struct{}), + t: t, + tconn: c, + readerDone: make(chan struct{}), + nextStreamID: 1, + maxFrameSize: 16 << 10, // spec default + initialWindowSize: 65535, // spec default + maxConcurrentStreams: 1000, // "infinite", per spec. 1000 seems good enough. + peerMaxHeaderListSize: 0xffffffffffffffff, // "infinite", per spec. Use 2^64-1 instead. + streams: make(map[uint32]*clientStream), + singleUse: singleUse, + wantSettingsAck: true, + pings: make(map[[8]byte]chan struct{}), } if d := t.idleConnTimeout(); d != 0 { cc.idleTimeout = d @@ -750,8 +763,13 @@ func actualContentLength(req *http.Request) int64 { } func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { + resp, _, err := cc.roundTrip(req) + return resp, err +} + +func (cc *ClientConn) roundTrip(req *http.Request) (res *http.Response, gotErrAfterReqBodyWrite bool, err error) { if err := checkConnHeaders(req); err != nil { - return nil, err + return nil, false, err } if cc.idleTimer != nil { cc.idleTimer.Stop() @@ -759,14 +777,14 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { trailers, err := commaSeparatedTrailers(req) if err != nil { - return nil, err + return nil, false, err } hasTrailers := trailers != "" cc.mu.Lock() if err := cc.awaitOpenSlotForRequest(req); err != nil { cc.mu.Unlock() - return nil, err + return nil, false, err } body := req.Body @@ -800,7 +818,7 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { hdrs, err := cc.encodeHeaders(req, requestedGzip, trailers, contentLen) if err != nil { cc.mu.Unlock() - return nil, err + return nil, false, err } cs := cc.newStream() @@ -812,7 +830,7 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { cc.wmu.Lock() endStream := !hasBody && !hasTrailers - werr := cc.writeHeaders(cs.ID, endStream, hdrs) + werr := cc.writeHeaders(cs.ID, endStream, int(cc.maxFrameSize), hdrs) cc.wmu.Unlock() traceWroteHeaders(cs.trace) cc.mu.Unlock() @@ -826,7 +844,7 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { // Don't bother sending a RST_STREAM (our write already failed; // no need to keep writing) traceWroteRequest(cs.trace, werr) - return nil, werr + return nil, false, werr } var respHeaderTimer <-chan time.Time @@ -845,7 +863,7 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { bodyWritten := false ctx := reqContext(req) - handleReadLoopResponse := func(re resAndError) (*http.Response, error) { + handleReadLoopResponse := func(re resAndError) (*http.Response, bool, error) { res := re.res if re.err != nil || res.StatusCode > 299 { // On error or status code 3xx, 4xx, 5xx, etc abort any @@ -861,18 +879,12 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { cs.abortRequestBodyWrite(errStopReqBodyWrite) } if re.err != nil { - cc.mu.Lock() - afterBodyWrite := cs.startedWrite - cc.mu.Unlock() cc.forgetStreamID(cs.ID) - if afterBodyWrite { - return nil, afterReqBodyWriteError{re.err} - } - return nil, re.err + return nil, cs.getStartedWrite(), re.err } res.Request = req res.TLS = cc.tlsState - return res, nil + return res, false, nil } for { @@ -887,7 +899,7 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { cs.abortRequestBodyWrite(errStopReqBodyWriteAndCancel) } cc.forgetStreamID(cs.ID) - return nil, errTimeout + return nil, cs.getStartedWrite(), errTimeout case <-ctx.Done(): if !hasBody || bodyWritten { cc.writeStreamReset(cs.ID, ErrCodeCancel, nil) @@ -896,7 +908,7 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { cs.abortRequestBodyWrite(errStopReqBodyWriteAndCancel) } cc.forgetStreamID(cs.ID) - return nil, ctx.Err() + return nil, cs.getStartedWrite(), ctx.Err() case <-req.Cancel: if !hasBody || bodyWritten { cc.writeStreamReset(cs.ID, ErrCodeCancel, nil) @@ -905,12 +917,12 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { cs.abortRequestBodyWrite(errStopReqBodyWriteAndCancel) } cc.forgetStreamID(cs.ID) - return nil, errRequestCanceled + return nil, cs.getStartedWrite(), errRequestCanceled case <-cs.peerReset: // processResetStream already removed the // stream from the streams map; no need for // forgetStreamID. - return nil, cs.resetErr + return nil, cs.getStartedWrite(), cs.resetErr case err := <-bodyWriter.resc: // Prefer the read loop's response, if available. Issue 16102. select { @@ -919,7 +931,7 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { default: } if err != nil { - return nil, err + return nil, cs.getStartedWrite(), err } bodyWritten = true if d := cc.responseHeaderTimeout(); d != 0 { @@ -971,13 +983,12 @@ func (cc *ClientConn) awaitOpenSlotForRequest(req *http.Request) error { } // requires cc.wmu be held -func (cc *ClientConn) writeHeaders(streamID uint32, endStream bool, hdrs []byte) error { +func (cc *ClientConn) writeHeaders(streamID uint32, endStream bool, maxFrameSize int, hdrs []byte) error { first := true // first frame written (HEADERS is first, then CONTINUATION) - frameSize := int(cc.maxFrameSize) for len(hdrs) > 0 && cc.werr == nil { chunk := hdrs - if len(chunk) > frameSize { - chunk = chunk[:frameSize] + if len(chunk) > maxFrameSize { + chunk = chunk[:maxFrameSize] } hdrs = hdrs[len(chunk):] endHeaders := len(hdrs) == 0 @@ -1085,17 +1096,26 @@ func (cs *clientStream) writeRequestBody(body io.Reader, bodyCloser io.Closer) ( var trls []byte if hasTrailers { cc.mu.Lock() - defer cc.mu.Unlock() - trls = cc.encodeTrailers(req) + trls, err = cc.encodeTrailers(req) + cc.mu.Unlock() + if err != nil { + cc.writeStreamReset(cs.ID, ErrCodeInternal, err) + cc.forgetStreamID(cs.ID) + return err + } } + cc.mu.Lock() + maxFrameSize := int(cc.maxFrameSize) + cc.mu.Unlock() + cc.wmu.Lock() defer cc.wmu.Unlock() // Two ways to send END_STREAM: either with trailers, or // with an empty DATA frame. if len(trls) > 0 { - err = cc.writeHeaders(cs.ID, true, trls) + err = cc.writeHeaders(cs.ID, true, maxFrameSize, trls) } else { err = cc.fr.WriteData(cs.ID, true, nil) } @@ -1189,62 +1209,86 @@ func (cc *ClientConn) encodeHeaders(req *http.Request, addGzipHeader bool, trail } } - // 8.1.2.3 Request Pseudo-Header Fields - // The :path pseudo-header field includes the path and query parts of the - // target URI (the path-absolute production and optionally a '?' character - // followed by the query production (see Sections 3.3 and 3.4 of - // [RFC3986]). - cc.writeHeader(":authority", host) - cc.writeHeader(":method", req.Method) - if req.Method != "CONNECT" { - cc.writeHeader(":path", path) - cc.writeHeader(":scheme", req.URL.Scheme) - } - if trailers != "" { - cc.writeHeader("trailer", trailers) + enumerateHeaders := func(f func(name, value string)) { + // 8.1.2.3 Request Pseudo-Header Fields + // The :path pseudo-header field includes the path and query parts of the + // target URI (the path-absolute production and optionally a '?' character + // followed by the query production (see Sections 3.3 and 3.4 of + // [RFC3986]). + f(":authority", host) + f(":method", req.Method) + if req.Method != "CONNECT" { + f(":path", path) + f(":scheme", req.URL.Scheme) + } + if trailers != "" { + f("trailer", trailers) + } + + var didUA bool + for k, vv := range req.Header { + if strings.EqualFold(k, "host") || strings.EqualFold(k, "content-length") { + // Host is :authority, already sent. + // Content-Length is automatic, set below. + continue + } else if strings.EqualFold(k, "connection") || strings.EqualFold(k, "proxy-connection") || + strings.EqualFold(k, "transfer-encoding") || strings.EqualFold(k, "upgrade") || + strings.EqualFold(k, "keep-alive") { + // Per 8.1.2.2 Connection-Specific Header + // Fields, don't send connection-specific + // fields. We have already checked if any + // are error-worthy so just ignore the rest. + continue + } else if strings.EqualFold(k, "user-agent") { + // Match Go's http1 behavior: at most one + // User-Agent. If set to nil or empty string, + // then omit it. Otherwise if not mentioned, + // include the default (below). + didUA = true + if len(vv) < 1 { + continue + } + vv = vv[:1] + if vv[0] == "" { + continue + } + + } + + for _, v := range vv { + f(k, v) + } + } + if shouldSendReqContentLength(req.Method, contentLength) { + f("content-length", strconv.FormatInt(contentLength, 10)) + } + if addGzipHeader { + f("accept-encoding", "gzip") + } + if !didUA { + f("user-agent", defaultUserAgent) + } } - var didUA bool - for k, vv := range req.Header { - lowKey := strings.ToLower(k) - switch lowKey { - case "host", "content-length": - // Host is :authority, already sent. - // Content-Length is automatic, set below. - continue - case "connection", "proxy-connection", "transfer-encoding", "upgrade", "keep-alive": - // Per 8.1.2.2 Connection-Specific Header - // Fields, don't send connection-specific - // fields. We have already checked if any - // are error-worthy so just ignore the rest. - continue - case "user-agent": - // Match Go's http1 behavior: at most one - // User-Agent. If set to nil or empty string, - // then omit it. Otherwise if not mentioned, - // include the default (below). - didUA = true - if len(vv) < 1 { - continue - } - vv = vv[:1] - if vv[0] == "" { - continue - } - } - for _, v := range vv { - cc.writeHeader(lowKey, v) - } - } - if shouldSendReqContentLength(req.Method, contentLength) { - cc.writeHeader("content-length", strconv.FormatInt(contentLength, 10)) - } - if addGzipHeader { - cc.writeHeader("accept-encoding", "gzip") - } - if !didUA { - cc.writeHeader("user-agent", defaultUserAgent) + // Do a first pass over the headers counting bytes to ensure + // we don't exceed cc.peerMaxHeaderListSize. This is done as a + // separate pass before encoding the headers to prevent + // modifying the hpack state. + hlSize := uint64(0) + enumerateHeaders(func(name, value string) { + hf := hpack.HeaderField{Name: name, Value: value} + hlSize += uint64(hf.Size()) + }) + + if hlSize > cc.peerMaxHeaderListSize { + return nil, errRequestHeaderListSize } + + // Header list size is ok. Write the headers. + enumerateHeaders(func(name, value string) { + cc.writeHeader(strings.ToLower(name), value) + }) + return cc.hbuf.Bytes(), nil } @@ -1271,17 +1315,29 @@ func shouldSendReqContentLength(method string, contentLength int64) bool { } // requires cc.mu be held. -func (cc *ClientConn) encodeTrailers(req *http.Request) []byte { +func (cc *ClientConn) encodeTrailers(req *http.Request) ([]byte, error) { cc.hbuf.Reset() + + hlSize := uint64(0) for k, vv := range req.Trailer { - // Transfer-Encoding, etc.. have already been filter at the + for _, v := range vv { + hf := hpack.HeaderField{Name: k, Value: v} + hlSize += uint64(hf.Size()) + } + } + if hlSize > cc.peerMaxHeaderListSize { + return nil, errRequestHeaderListSize + } + + for k, vv := range req.Trailer { + // Transfer-Encoding, etc.. have already been filtered at the // start of RoundTrip lowKey := strings.ToLower(k) for _, v := range vv { cc.writeHeader(lowKey, v) } } - return cc.hbuf.Bytes() + return cc.hbuf.Bytes(), nil } func (cc *ClientConn) writeHeader(name, value string) { @@ -1339,17 +1395,12 @@ func (cc *ClientConn) streamByID(id uint32, andRemove bool) *clientStream { // clientConnReadLoop is the state owned by the clientConn's frame-reading readLoop. type clientConnReadLoop struct { cc *ClientConn - activeRes map[uint32]*clientStream // keyed by streamID closeWhenIdle bool } // readLoop runs in its own goroutine and reads and dispatches frames. func (cc *ClientConn) readLoop() { - rl := &clientConnReadLoop{ - cc: cc, - activeRes: make(map[uint32]*clientStream), - } - + rl := &clientConnReadLoop{cc: cc} defer rl.cleanup() cc.readerErr = rl.run() if ce, ok := cc.readerErr.(ConnectionError); ok { @@ -1404,10 +1455,8 @@ func (rl *clientConnReadLoop) cleanup() { } else if err == io.EOF { err = io.ErrUnexpectedEOF } - for _, cs := range rl.activeRes { - cs.bufPipe.CloseWithError(err) - } for _, cs := range cc.streams { + cs.bufPipe.CloseWithError(err) // no-op if already closed select { case cs.resc <- resAndError{err: err}: default: @@ -1485,7 +1534,7 @@ func (rl *clientConnReadLoop) run() error { } return err } - if rl.closeWhenIdle && gotReply && maybeIdle && len(rl.activeRes) == 0 { + if rl.closeWhenIdle && gotReply && maybeIdle { cc.closeIfIdle() } } @@ -1493,13 +1542,31 @@ func (rl *clientConnReadLoop) run() error { func (rl *clientConnReadLoop) processHeaders(f *MetaHeadersFrame) error { cc := rl.cc - cs := cc.streamByID(f.StreamID, f.StreamEnded()) + cs := cc.streamByID(f.StreamID, false) if cs == nil { // We'd get here if we canceled a request while the // server had its response still in flight. So if this // was just something we canceled, ignore it. return nil } + if f.StreamEnded() { + // Issue 20521: If the stream has ended, streamByID() causes + // clientStream.done to be closed, which causes the request's bodyWriter + // to be closed with an errStreamClosed, which may be received by + // clientConn.RoundTrip before the result of processing these headers. + // Deferring stream closure allows the header processing to occur first. + // clientConn.RoundTrip may still receive the bodyWriter error first, but + // the fix for issue 16102 prioritises any response. + // + // Issue 22413: If there is no request body, we should close the + // stream before writing to cs.resc so that the stream is closed + // immediately once RoundTrip returns. + if cs.req.Body != nil { + defer cc.forgetStreamID(f.StreamID) + } else { + cc.forgetStreamID(f.StreamID) + } + } if !cs.firstByte { if cs.trace != nil { // TODO(bradfitz): move first response byte earlier, @@ -1523,6 +1590,7 @@ func (rl *clientConnReadLoop) processHeaders(f *MetaHeadersFrame) error { } // Any other error type is a stream error. cs.cc.writeStreamReset(f.StreamID, ErrCodeProtocol, err) + cc.forgetStreamID(cs.ID) cs.resc <- resAndError{err: err} return nil // return nil from process* funcs to keep conn alive } @@ -1530,9 +1598,6 @@ func (rl *clientConnReadLoop) processHeaders(f *MetaHeadersFrame) error { // (nil, nil) special case. See handleResponse docs. return nil } - if res.Body != noBody { - rl.activeRes[cs.ID] = cs - } cs.resTrailer = &res.Trailer cs.resc <- resAndError{res: res} return nil @@ -1552,11 +1617,11 @@ func (rl *clientConnReadLoop) handleResponse(cs *clientStream, f *MetaHeadersFra status := f.PseudoValue("status") if status == "" { - return nil, errors.New("missing status pseudo header") + return nil, errors.New("malformed response from server: missing status pseudo header") } statusCode, err := strconv.Atoi(status) if err != nil { - return nil, errors.New("malformed non-numeric status pseudo header") + return nil, errors.New("malformed response from server: malformed non-numeric status pseudo header") } if statusCode == 100 { @@ -1789,7 +1854,23 @@ func (rl *clientConnReadLoop) processData(f *DataFrame) error { } return nil } + if !cs.firstByte { + cc.logf("protocol error: received DATA before a HEADERS frame") + rl.endStreamError(cs, StreamError{ + StreamID: f.StreamID, + Code: ErrCodeProtocol, + }) + return nil + } if f.Length > 0 { + if cs.req.Method == "HEAD" && len(data) > 0 { + cc.logf("protocol error: received DATA on a HEAD request") + rl.endStreamError(cs, StreamError{ + StreamID: f.StreamID, + Code: ErrCodeProtocol, + }) + return nil + } // Check connection-level flow control. cc.mu.Lock() if cs.inflow.available() >= int32(f.Length) { @@ -1851,11 +1932,10 @@ func (rl *clientConnReadLoop) endStreamError(cs *clientStream, err error) { err = io.EOF code = cs.copyTrailers } - cs.bufPipe.closeWithErrorAndCode(err, code) - delete(rl.activeRes, cs.ID) if isConnectionCloseRequest(cs.req) { rl.closeWhenIdle = true } + cs.bufPipe.closeWithErrorAndCode(err, code) select { case cs.resc <- resAndError{err: err}: @@ -1903,6 +1983,8 @@ func (rl *clientConnReadLoop) processSettings(f *SettingsFrame) error { cc.maxFrameSize = s.Val case SettingMaxConcurrentStreams: cc.maxConcurrentStreams = s.Val + case SettingMaxHeaderListSize: + cc.peerMaxHeaderListSize = uint64(s.Val) case SettingInitialWindowSize: // Values above the maximum flow-control // window size of 2^31-1 MUST be treated as a @@ -1980,7 +2062,6 @@ func (rl *clientConnReadLoop) processResetStream(f *RSTStreamFrame) error { cs.bufPipe.CloseWithError(err) cs.cc.cond.Broadcast() // wake up checkResetOrDone via clientStream.awaitFlowControl } - delete(rl.activeRes, cs.ID) return nil } @@ -2069,6 +2150,7 @@ func (cc *ClientConn) writeStreamReset(streamID uint32, code ErrCode, err error) var ( errResponseHeaderListSize = errors.New("http2: response header list larger than advertised limit") + errRequestHeaderListSize = errors.New("http2: request header list larger than peer's advertised limit") errPseudoTrailers = errors.New("http2: invalid pseudo header in trailers") ) diff --git a/vendor/golang.org/x/net/http2/write.go b/vendor/golang.org/x/net/http2/write.go index 6b0dfae319a..54ab4a88e7b 100644 --- a/vendor/golang.org/x/net/http2/write.go +++ b/vendor/golang.org/x/net/http2/write.go @@ -10,7 +10,6 @@ import ( "log" "net/http" "net/url" - "time" "golang.org/x/net/http2/hpack" "golang.org/x/net/lex/httplex" @@ -90,11 +89,7 @@ type writeGoAway struct { func (p *writeGoAway) writeFrame(ctx writeContext) error { err := ctx.Framer().WriteGoAway(p.maxStreamID, p.code, nil) - if p.code != 0 { - ctx.Flush() // ignore error: we're hanging up on them anyway - time.Sleep(50 * time.Millisecond) - ctx.CloseConn() - } + ctx.Flush() // ignore error: we're hanging up on them anyway return err } diff --git a/vendor/golang.org/x/net/idna/BUILD b/vendor/golang.org/x/net/idna/BUILD index 5c1baf8fbcb..bc4666a134c 100644 --- a/vendor/golang.org/x/net/idna/BUILD +++ b/vendor/golang.org/x/net/idna/BUILD @@ -14,6 +14,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//vendor/golang.org/x/text/secure/bidirule:go_default_library", + "//vendor/golang.org/x/text/unicode/bidi:go_default_library", "//vendor/golang.org/x/text/unicode/norm:go_default_library", ], ) diff --git a/vendor/golang.org/x/net/idna/idna.go b/vendor/golang.org/x/net/idna/idna.go index 1810100bfd8..6655f023a3f 100644 --- a/vendor/golang.org/x/net/idna/idna.go +++ b/vendor/golang.org/x/net/idna/idna.go @@ -21,6 +21,7 @@ import ( "unicode/utf8" "golang.org/x/text/secure/bidirule" + "golang.org/x/text/unicode/bidi" "golang.org/x/text/unicode/norm" ) @@ -68,7 +69,7 @@ func VerifyDNSLength(verify bool) Option { } // RemoveLeadingDots removes leading label separators. Leading runes that map to -// dots, such as U+3002, are removed as well. +// dots, such as U+3002 IDEOGRAPHIC FULL STOP, are removed as well. // // This is the behavior suggested by the UTS #46 and is adopted by some // browsers. @@ -92,7 +93,7 @@ func ValidateLabels(enable bool) Option { } } -// StrictDomainName limits the set of permissable ASCII characters to those +// StrictDomainName limits the set of permissible ASCII characters to those // allowed in domain names as defined in RFC 1034 (A-Z, a-z, 0-9 and the // hyphen). This is set by default for MapForLookup and ValidateForRegistration. // @@ -142,7 +143,6 @@ func MapForLookup() Option { o.mapping = validateAndMap StrictDomainName(true)(o) ValidateLabels(true)(o) - RemoveLeadingDots(true)(o) } } @@ -160,14 +160,14 @@ type options struct { // mapping implements a validation and mapping step as defined in RFC 5895 // or UTS 46, tailored to, for example, domain registration or lookup. - mapping func(p *Profile, s string) (string, error) + mapping func(p *Profile, s string) (mapped string, isBidi bool, err error) // bidirule, if specified, checks whether s conforms to the Bidi Rule // defined in RFC 5893. bidirule func(s string) bool } -// A Profile defines the configuration of a IDNA mapper. +// A Profile defines the configuration of an IDNA mapper. type Profile struct { options } @@ -251,23 +251,21 @@ var ( punycode = &Profile{} lookup = &Profile{options{ - transitional: true, - useSTD3Rules: true, - validateLabels: true, - removeLeadingDots: true, - trie: trie, - fromPuny: validateFromPunycode, - mapping: validateAndMap, - bidirule: bidirule.ValidString, + transitional: true, + useSTD3Rules: true, + validateLabels: true, + trie: trie, + fromPuny: validateFromPunycode, + mapping: validateAndMap, + bidirule: bidirule.ValidString, }} display = &Profile{options{ - useSTD3Rules: true, - validateLabels: true, - removeLeadingDots: true, - trie: trie, - fromPuny: validateFromPunycode, - mapping: validateAndMap, - bidirule: bidirule.ValidString, + useSTD3Rules: true, + validateLabels: true, + trie: trie, + fromPuny: validateFromPunycode, + mapping: validateAndMap, + bidirule: bidirule.ValidString, }} registration = &Profile{options{ useSTD3Rules: true, @@ -302,14 +300,16 @@ func (e runeError) Error() string { // see http://www.unicode.org/reports/tr46. func (p *Profile) process(s string, toASCII bool) (string, error) { var err error + var isBidi bool if p.mapping != nil { - s, err = p.mapping(p, s) + s, isBidi, err = p.mapping(p, s) } // Remove leading empty labels. if p.removeLeadingDots { for ; len(s) > 0 && s[0] == '.'; s = s[1:] { } } + // TODO: allow for a quick check of the tables data. // It seems like we should only create this error on ToASCII, but the // UTS 46 conformance tests suggests we should always check this. if err == nil && p.verifyDNSLength && s == "" { @@ -335,6 +335,7 @@ func (p *Profile) process(s string, toASCII bool) (string, error) { // Spec says keep the old label. continue } + isBidi = isBidi || bidirule.DirectionString(u) != bidi.LeftToRight labels.set(u) if err == nil && p.validateLabels { err = p.fromPuny(p, u) @@ -349,6 +350,14 @@ func (p *Profile) process(s string, toASCII bool) (string, error) { err = p.validateLabel(label) } } + if isBidi && p.bidirule != nil && err == nil { + for labels.reset(); !labels.done(); labels.next() { + if !p.bidirule(labels.label()) { + err = &labelError{s, "B"} + break + } + } + } if toASCII { for labels.reset(); !labels.done(); labels.next() { label := labels.label() @@ -380,16 +389,26 @@ func (p *Profile) process(s string, toASCII bool) (string, error) { return s, err } -func normalize(p *Profile, s string) (string, error) { - return norm.NFC.String(s), nil +func normalize(p *Profile, s string) (mapped string, isBidi bool, err error) { + // TODO: consider first doing a quick check to see if any of these checks + // need to be done. This will make it slower in the general case, but + // faster in the common case. + mapped = norm.NFC.String(s) + isBidi = bidirule.DirectionString(mapped) == bidi.RightToLeft + return mapped, isBidi, nil } -func validateRegistration(p *Profile, s string) (string, error) { +func validateRegistration(p *Profile, s string) (idem string, bidi bool, err error) { + // TODO: filter need for normalization in loop below. if !norm.NFC.IsNormalString(s) { - return s, &labelError{s, "V1"} + return s, false, &labelError{s, "V1"} } for i := 0; i < len(s); { v, sz := trie.lookupString(s[i:]) + if sz == 0 { + return s, bidi, runeError(utf8.RuneError) + } + bidi = bidi || info(v).isBidi(s[i:]) // Copy bytes not copied so far. switch p.simplify(info(v).category()) { // TODO: handle the NV8 defined in the Unicode idna data set to allow @@ -397,21 +416,50 @@ func validateRegistration(p *Profile, s string) (string, error) { case valid, deviation: case disallowed, mapped, unknown, ignored: r, _ := utf8.DecodeRuneInString(s[i:]) - return s, runeError(r) + return s, bidi, runeError(r) } i += sz } - return s, nil + return s, bidi, nil } -func validateAndMap(p *Profile, s string) (string, error) { +func (c info) isBidi(s string) bool { + if !c.isMapped() { + return c&attributesMask == rtl + } + // TODO: also store bidi info for mapped data. This is possible, but a bit + // cumbersome and not for the common case. + p, _ := bidi.LookupString(s) + switch p.Class() { + case bidi.R, bidi.AL, bidi.AN: + return true + } + return false +} + +func validateAndMap(p *Profile, s string) (vm string, bidi bool, err error) { var ( - err error - b []byte - k int + b []byte + k int ) + // combinedInfoBits contains the or-ed bits of all runes. We use this + // to derive the mayNeedNorm bit later. This may trigger normalization + // overeagerly, but it will not do so in the common case. The end result + // is another 10% saving on BenchmarkProfile for the common case. + var combinedInfoBits info for i := 0; i < len(s); { v, sz := trie.lookupString(s[i:]) + if sz == 0 { + b = append(b, s[k:i]...) + b = append(b, "\ufffd"...) + k = len(s) + if err == nil { + err = runeError(utf8.RuneError) + } + break + } + combinedInfoBits |= info(v) + bidi = bidi || info(v).isBidi(s[i:]) start := i i += sz // Copy bytes not copied so far. @@ -438,7 +486,9 @@ func validateAndMap(p *Profile, s string) (string, error) { } if k == 0 { // No changes so far. - s = norm.NFC.String(s) + if combinedInfoBits&mayNeedNorm != 0 { + s = norm.NFC.String(s) + } } else { b = append(b, s[k:]...) if norm.NFC.QuickSpan(b) != len(b) { @@ -447,7 +497,7 @@ func validateAndMap(p *Profile, s string) (string, error) { // TODO: the punycode converters require strings as input. s = string(b) } - return s, err + return s, bidi, err } // A labelIter allows iterating over domain name labels. @@ -542,8 +592,13 @@ func validateFromPunycode(p *Profile, s string) error { if !norm.NFC.IsNormalString(s) { return &labelError{s, "V1"} } + // TODO: detect whether string may have to be normalized in the following + // loop. for i := 0; i < len(s); { v, sz := trie.lookupString(s[i:]) + if sz == 0 { + return runeError(utf8.RuneError) + } if c := p.simplify(info(v).category()); c != valid && c != deviation { return &labelError{s, "V6"} } @@ -616,16 +671,13 @@ var joinStates = [][numJoinTypes]joinState{ // validateLabel validates the criteria from Section 4.1. Item 1, 4, and 6 are // already implicitly satisfied by the overall implementation. -func (p *Profile) validateLabel(s string) error { +func (p *Profile) validateLabel(s string) (err error) { if s == "" { if p.verifyDNSLength { return &labelError{s, "A4"} } return nil } - if p.bidirule != nil && !p.bidirule(s) { - return &labelError{s, "B"} - } if !p.validateLabels { return nil } diff --git a/vendor/golang.org/x/net/idna/tables.go b/vendor/golang.org/x/net/idna/tables.go index d2819345fcd..f910b269144 100644 --- a/vendor/golang.org/x/net/idna/tables.go +++ b/vendor/golang.org/x/net/idna/tables.go @@ -3,7 +3,7 @@ package idna // UnicodeVersion is the Unicode version from which the tables in this package are derived. -const UnicodeVersion = "9.0.0" +const UnicodeVersion = "10.0.0" var mappings string = "" + // Size: 8176 bytes "\x00\x01 \x03 ̈\x01a\x03 ̄\x012\x013\x03 ́\x03 ̧\x011\x01o\x051⁄4\x051⁄2" + @@ -544,7 +544,7 @@ func (t *idnaTrie) lookupStringUnsafe(s string) uint16 { return 0 } -// idnaTrie. Total size: 28496 bytes (27.83 KiB). Checksum: 43288b883596640e. +// idnaTrie. Total size: 29052 bytes (28.37 KiB). Checksum: ef06e7ecc26f36dd. type idnaTrie struct{} func newIdnaTrie(i int) *idnaTrie { @@ -554,17 +554,17 @@ func newIdnaTrie(i int) *idnaTrie { // lookupValue determines the type of block n and looks up the value for b. func (t *idnaTrie) lookupValue(n uint32, b byte) uint16 { switch { - case n < 123: + case n < 125: return uint16(idnaValues[n<<6+uint32(b)]) default: - n -= 123 + n -= 125 return uint16(idnaSparse.lookup(n, b)) } } -// idnaValues: 125 blocks, 8000 entries, 16000 bytes +// idnaValues: 127 blocks, 8128 entries, 16256 bytes // The third block is the zero block. -var idnaValues = [8000]uint16{ +var idnaValues = [8128]uint16{ // Block 0x0, offset 0x0 0x00: 0x0080, 0x01: 0x0080, 0x02: 0x0080, 0x03: 0x0080, 0x04: 0x0080, 0x05: 0x0080, 0x06: 0x0080, 0x07: 0x0080, 0x08: 0x0080, 0x09: 0x0080, 0x0a: 0x0080, 0x0b: 0x0080, @@ -675,14 +675,14 @@ var idnaValues = [8000]uint16{ 0x276: 0x0018, 0x277: 0x0018, 0x278: 0x0018, 0x279: 0x0018, 0x27a: 0x0018, 0x27b: 0x0018, 0x27c: 0x0018, 0x27d: 0x0018, 0x27e: 0x0018, 0x27f: 0x0018, // Block 0xa, offset 0x280 - 0x280: 0x03dd, 0x281: 0x03dd, 0x282: 0x1308, 0x283: 0x03f5, 0x284: 0x0379, 0x285: 0x040d, - 0x286: 0x1308, 0x287: 0x1308, 0x288: 0x1308, 0x289: 0x1308, 0x28a: 0x1308, 0x28b: 0x1308, - 0x28c: 0x1308, 0x28d: 0x1308, 0x28e: 0x1308, 0x28f: 0x13c0, 0x290: 0x1308, 0x291: 0x1308, - 0x292: 0x1308, 0x293: 0x1308, 0x294: 0x1308, 0x295: 0x1308, 0x296: 0x1308, 0x297: 0x1308, - 0x298: 0x1308, 0x299: 0x1308, 0x29a: 0x1308, 0x29b: 0x1308, 0x29c: 0x1308, 0x29d: 0x1308, - 0x29e: 0x1308, 0x29f: 0x1308, 0x2a0: 0x1308, 0x2a1: 0x1308, 0x2a2: 0x1308, 0x2a3: 0x1308, - 0x2a4: 0x1308, 0x2a5: 0x1308, 0x2a6: 0x1308, 0x2a7: 0x1308, 0x2a8: 0x1308, 0x2a9: 0x1308, - 0x2aa: 0x1308, 0x2ab: 0x1308, 0x2ac: 0x1308, 0x2ad: 0x1308, 0x2ae: 0x1308, 0x2af: 0x1308, + 0x280: 0x03dd, 0x281: 0x03dd, 0x282: 0x3308, 0x283: 0x03f5, 0x284: 0x0379, 0x285: 0x040d, + 0x286: 0x3308, 0x287: 0x3308, 0x288: 0x3308, 0x289: 0x3308, 0x28a: 0x3308, 0x28b: 0x3308, + 0x28c: 0x3308, 0x28d: 0x3308, 0x28e: 0x3308, 0x28f: 0x33c0, 0x290: 0x3308, 0x291: 0x3308, + 0x292: 0x3308, 0x293: 0x3308, 0x294: 0x3308, 0x295: 0x3308, 0x296: 0x3308, 0x297: 0x3308, + 0x298: 0x3308, 0x299: 0x3308, 0x29a: 0x3308, 0x29b: 0x3308, 0x29c: 0x3308, 0x29d: 0x3308, + 0x29e: 0x3308, 0x29f: 0x3308, 0x2a0: 0x3308, 0x2a1: 0x3308, 0x2a2: 0x3308, 0x2a3: 0x3308, + 0x2a4: 0x3308, 0x2a5: 0x3308, 0x2a6: 0x3308, 0x2a7: 0x3308, 0x2a8: 0x3308, 0x2a9: 0x3308, + 0x2aa: 0x3308, 0x2ab: 0x3308, 0x2ac: 0x3308, 0x2ad: 0x3308, 0x2ae: 0x3308, 0x2af: 0x3308, 0x2b0: 0xe00d, 0x2b1: 0x0008, 0x2b2: 0xe00d, 0x2b3: 0x0008, 0x2b4: 0x0425, 0x2b5: 0x0008, 0x2b6: 0xe00d, 0x2b7: 0x0008, 0x2b8: 0x0040, 0x2b9: 0x0040, 0x2ba: 0x03a2, 0x2bb: 0x0008, 0x2bc: 0x0008, 0x2bd: 0x0008, 0x2be: 0x03c2, 0x2bf: 0x043d, @@ -723,8 +723,8 @@ var idnaValues = [8000]uint16{ 0x376: 0xe00d, 0x377: 0x0008, 0x378: 0xe00d, 0x379: 0x0008, 0x37a: 0xe00d, 0x37b: 0x0008, 0x37c: 0xe00d, 0x37d: 0x0008, 0x37e: 0xe00d, 0x37f: 0x0008, // Block 0xe, offset 0x380 - 0x380: 0xe00d, 0x381: 0x0008, 0x382: 0x0018, 0x383: 0x1308, 0x384: 0x1308, 0x385: 0x1308, - 0x386: 0x1308, 0x387: 0x1308, 0x388: 0x1318, 0x389: 0x1318, 0x38a: 0xe00d, 0x38b: 0x0008, + 0x380: 0xe00d, 0x381: 0x0008, 0x382: 0x0018, 0x383: 0x3308, 0x384: 0x3308, 0x385: 0x3308, + 0x386: 0x3308, 0x387: 0x3308, 0x388: 0x3318, 0x389: 0x3318, 0x38a: 0xe00d, 0x38b: 0x0008, 0x38c: 0xe00d, 0x38d: 0x0008, 0x38e: 0xe00d, 0x38f: 0x0008, 0x390: 0xe00d, 0x391: 0x0008, 0x392: 0xe00d, 0x393: 0x0008, 0x394: 0xe00d, 0x395: 0x0008, 0x396: 0xe00d, 0x397: 0x0008, 0x398: 0xe00d, 0x399: 0x0008, 0x39a: 0xe00d, 0x39b: 0x0008, 0x39c: 0xe00d, 0x39d: 0x0008, @@ -759,129 +759,129 @@ var idnaValues = [8000]uint16{ 0x436: 0x03f5, 0x437: 0x03f5, 0x438: 0x03f5, 0x439: 0x03f5, 0x43a: 0x03f5, 0x43b: 0x03f5, 0x43c: 0x03f5, 0x43d: 0x03f5, 0x43e: 0x03f5, 0x43f: 0x03f5, // Block 0x11, offset 0x440 - 0x440: 0x0040, 0x441: 0x0040, 0x442: 0x0040, 0x443: 0x0040, 0x444: 0x0040, 0x445: 0x0040, - 0x446: 0x0018, 0x447: 0x0018, 0x448: 0x0018, 0x449: 0x0018, 0x44a: 0x0018, 0x44b: 0x0018, - 0x44c: 0x0018, 0x44d: 0x0018, 0x44e: 0x0018, 0x44f: 0x0018, 0x450: 0x1308, 0x451: 0x1308, - 0x452: 0x1308, 0x453: 0x1308, 0x454: 0x1308, 0x455: 0x1308, 0x456: 0x1308, 0x457: 0x1308, - 0x458: 0x1308, 0x459: 0x1308, 0x45a: 0x1308, 0x45b: 0x0018, 0x45c: 0x0340, 0x45d: 0x0040, - 0x45e: 0x0018, 0x45f: 0x0018, 0x460: 0x0208, 0x461: 0x0008, 0x462: 0x0408, 0x463: 0x0408, - 0x464: 0x0408, 0x465: 0x0408, 0x466: 0x0208, 0x467: 0x0408, 0x468: 0x0208, 0x469: 0x0408, - 0x46a: 0x0208, 0x46b: 0x0208, 0x46c: 0x0208, 0x46d: 0x0208, 0x46e: 0x0208, 0x46f: 0x0408, - 0x470: 0x0408, 0x471: 0x0408, 0x472: 0x0408, 0x473: 0x0208, 0x474: 0x0208, 0x475: 0x0208, - 0x476: 0x0208, 0x477: 0x0208, 0x478: 0x0208, 0x479: 0x0208, 0x47a: 0x0208, 0x47b: 0x0208, - 0x47c: 0x0208, 0x47d: 0x0208, 0x47e: 0x0208, 0x47f: 0x0208, + 0x440: 0x0840, 0x441: 0x0840, 0x442: 0x0840, 0x443: 0x0840, 0x444: 0x0840, 0x445: 0x0840, + 0x446: 0x0018, 0x447: 0x0018, 0x448: 0x0818, 0x449: 0x0018, 0x44a: 0x0018, 0x44b: 0x0818, + 0x44c: 0x0018, 0x44d: 0x0818, 0x44e: 0x0018, 0x44f: 0x0018, 0x450: 0x3308, 0x451: 0x3308, + 0x452: 0x3308, 0x453: 0x3308, 0x454: 0x3308, 0x455: 0x3308, 0x456: 0x3308, 0x457: 0x3308, + 0x458: 0x3308, 0x459: 0x3308, 0x45a: 0x3308, 0x45b: 0x0818, 0x45c: 0x0b40, 0x45d: 0x0040, + 0x45e: 0x0818, 0x45f: 0x0818, 0x460: 0x0a08, 0x461: 0x0808, 0x462: 0x0c08, 0x463: 0x0c08, + 0x464: 0x0c08, 0x465: 0x0c08, 0x466: 0x0a08, 0x467: 0x0c08, 0x468: 0x0a08, 0x469: 0x0c08, + 0x46a: 0x0a08, 0x46b: 0x0a08, 0x46c: 0x0a08, 0x46d: 0x0a08, 0x46e: 0x0a08, 0x46f: 0x0c08, + 0x470: 0x0c08, 0x471: 0x0c08, 0x472: 0x0c08, 0x473: 0x0a08, 0x474: 0x0a08, 0x475: 0x0a08, + 0x476: 0x0a08, 0x477: 0x0a08, 0x478: 0x0a08, 0x479: 0x0a08, 0x47a: 0x0a08, 0x47b: 0x0a08, + 0x47c: 0x0a08, 0x47d: 0x0a08, 0x47e: 0x0a08, 0x47f: 0x0a08, // Block 0x12, offset 0x480 - 0x480: 0x0408, 0x481: 0x0208, 0x482: 0x0208, 0x483: 0x0408, 0x484: 0x0408, 0x485: 0x0408, - 0x486: 0x0408, 0x487: 0x0408, 0x488: 0x0408, 0x489: 0x0408, 0x48a: 0x0408, 0x48b: 0x0408, - 0x48c: 0x0208, 0x48d: 0x0408, 0x48e: 0x0208, 0x48f: 0x0408, 0x490: 0x0208, 0x491: 0x0208, - 0x492: 0x0408, 0x493: 0x0408, 0x494: 0x0018, 0x495: 0x0408, 0x496: 0x1308, 0x497: 0x1308, - 0x498: 0x1308, 0x499: 0x1308, 0x49a: 0x1308, 0x49b: 0x1308, 0x49c: 0x1308, 0x49d: 0x0040, - 0x49e: 0x0018, 0x49f: 0x1308, 0x4a0: 0x1308, 0x4a1: 0x1308, 0x4a2: 0x1308, 0x4a3: 0x1308, - 0x4a4: 0x1308, 0x4a5: 0x0008, 0x4a6: 0x0008, 0x4a7: 0x1308, 0x4a8: 0x1308, 0x4a9: 0x0018, - 0x4aa: 0x1308, 0x4ab: 0x1308, 0x4ac: 0x1308, 0x4ad: 0x1308, 0x4ae: 0x0408, 0x4af: 0x0408, - 0x4b0: 0x0008, 0x4b1: 0x0008, 0x4b2: 0x0008, 0x4b3: 0x0008, 0x4b4: 0x0008, 0x4b5: 0x0008, - 0x4b6: 0x0008, 0x4b7: 0x0008, 0x4b8: 0x0008, 0x4b9: 0x0008, 0x4ba: 0x0208, 0x4bb: 0x0208, - 0x4bc: 0x0208, 0x4bd: 0x0008, 0x4be: 0x0008, 0x4bf: 0x0208, + 0x480: 0x0818, 0x481: 0x0a08, 0x482: 0x0a08, 0x483: 0x0a08, 0x484: 0x0a08, 0x485: 0x0a08, + 0x486: 0x0a08, 0x487: 0x0a08, 0x488: 0x0c08, 0x489: 0x0a08, 0x48a: 0x0a08, 0x48b: 0x3308, + 0x48c: 0x3308, 0x48d: 0x3308, 0x48e: 0x3308, 0x48f: 0x3308, 0x490: 0x3308, 0x491: 0x3308, + 0x492: 0x3308, 0x493: 0x3308, 0x494: 0x3308, 0x495: 0x3308, 0x496: 0x3308, 0x497: 0x3308, + 0x498: 0x3308, 0x499: 0x3308, 0x49a: 0x3308, 0x49b: 0x3308, 0x49c: 0x3308, 0x49d: 0x3308, + 0x49e: 0x3308, 0x49f: 0x3308, 0x4a0: 0x0808, 0x4a1: 0x0808, 0x4a2: 0x0808, 0x4a3: 0x0808, + 0x4a4: 0x0808, 0x4a5: 0x0808, 0x4a6: 0x0808, 0x4a7: 0x0808, 0x4a8: 0x0808, 0x4a9: 0x0808, + 0x4aa: 0x0018, 0x4ab: 0x0818, 0x4ac: 0x0818, 0x4ad: 0x0818, 0x4ae: 0x0a08, 0x4af: 0x0a08, + 0x4b0: 0x3308, 0x4b1: 0x0c08, 0x4b2: 0x0c08, 0x4b3: 0x0c08, 0x4b4: 0x0808, 0x4b5: 0x0429, + 0x4b6: 0x0451, 0x4b7: 0x0479, 0x4b8: 0x04a1, 0x4b9: 0x0a08, 0x4ba: 0x0a08, 0x4bb: 0x0a08, + 0x4bc: 0x0a08, 0x4bd: 0x0a08, 0x4be: 0x0a08, 0x4bf: 0x0a08, // Block 0x13, offset 0x4c0 - 0x4c0: 0x0018, 0x4c1: 0x0018, 0x4c2: 0x0018, 0x4c3: 0x0018, 0x4c4: 0x0018, 0x4c5: 0x0018, - 0x4c6: 0x0018, 0x4c7: 0x0018, 0x4c8: 0x0018, 0x4c9: 0x0018, 0x4ca: 0x0018, 0x4cb: 0x0018, - 0x4cc: 0x0018, 0x4cd: 0x0018, 0x4ce: 0x0040, 0x4cf: 0x0340, 0x4d0: 0x0408, 0x4d1: 0x1308, - 0x4d2: 0x0208, 0x4d3: 0x0208, 0x4d4: 0x0208, 0x4d5: 0x0408, 0x4d6: 0x0408, 0x4d7: 0x0408, - 0x4d8: 0x0408, 0x4d9: 0x0408, 0x4da: 0x0208, 0x4db: 0x0208, 0x4dc: 0x0208, 0x4dd: 0x0208, - 0x4de: 0x0408, 0x4df: 0x0208, 0x4e0: 0x0208, 0x4e1: 0x0208, 0x4e2: 0x0208, 0x4e3: 0x0208, - 0x4e4: 0x0208, 0x4e5: 0x0208, 0x4e6: 0x0208, 0x4e7: 0x0208, 0x4e8: 0x0408, 0x4e9: 0x0208, - 0x4ea: 0x0408, 0x4eb: 0x0208, 0x4ec: 0x0408, 0x4ed: 0x0208, 0x4ee: 0x0208, 0x4ef: 0x0408, - 0x4f0: 0x1308, 0x4f1: 0x1308, 0x4f2: 0x1308, 0x4f3: 0x1308, 0x4f4: 0x1308, 0x4f5: 0x1308, - 0x4f6: 0x1308, 0x4f7: 0x1308, 0x4f8: 0x1308, 0x4f9: 0x1308, 0x4fa: 0x1308, 0x4fb: 0x1308, - 0x4fc: 0x1308, 0x4fd: 0x1308, 0x4fe: 0x1308, 0x4ff: 0x1308, + 0x4c0: 0x0c08, 0x4c1: 0x0a08, 0x4c2: 0x0a08, 0x4c3: 0x0c08, 0x4c4: 0x0c08, 0x4c5: 0x0c08, + 0x4c6: 0x0c08, 0x4c7: 0x0c08, 0x4c8: 0x0c08, 0x4c9: 0x0c08, 0x4ca: 0x0c08, 0x4cb: 0x0c08, + 0x4cc: 0x0a08, 0x4cd: 0x0c08, 0x4ce: 0x0a08, 0x4cf: 0x0c08, 0x4d0: 0x0a08, 0x4d1: 0x0a08, + 0x4d2: 0x0c08, 0x4d3: 0x0c08, 0x4d4: 0x0818, 0x4d5: 0x0c08, 0x4d6: 0x3308, 0x4d7: 0x3308, + 0x4d8: 0x3308, 0x4d9: 0x3308, 0x4da: 0x3308, 0x4db: 0x3308, 0x4dc: 0x3308, 0x4dd: 0x0840, + 0x4de: 0x0018, 0x4df: 0x3308, 0x4e0: 0x3308, 0x4e1: 0x3308, 0x4e2: 0x3308, 0x4e3: 0x3308, + 0x4e4: 0x3308, 0x4e5: 0x0808, 0x4e6: 0x0808, 0x4e7: 0x3308, 0x4e8: 0x3308, 0x4e9: 0x0018, + 0x4ea: 0x3308, 0x4eb: 0x3308, 0x4ec: 0x3308, 0x4ed: 0x3308, 0x4ee: 0x0c08, 0x4ef: 0x0c08, + 0x4f0: 0x0008, 0x4f1: 0x0008, 0x4f2: 0x0008, 0x4f3: 0x0008, 0x4f4: 0x0008, 0x4f5: 0x0008, + 0x4f6: 0x0008, 0x4f7: 0x0008, 0x4f8: 0x0008, 0x4f9: 0x0008, 0x4fa: 0x0a08, 0x4fb: 0x0a08, + 0x4fc: 0x0a08, 0x4fd: 0x0808, 0x4fe: 0x0808, 0x4ff: 0x0a08, // Block 0x14, offset 0x500 - 0x500: 0x1008, 0x501: 0x1308, 0x502: 0x1308, 0x503: 0x1308, 0x504: 0x1308, 0x505: 0x1308, - 0x506: 0x1308, 0x507: 0x1308, 0x508: 0x1308, 0x509: 0x1008, 0x50a: 0x1008, 0x50b: 0x1008, - 0x50c: 0x1008, 0x50d: 0x1b08, 0x50e: 0x1008, 0x50f: 0x1008, 0x510: 0x0008, 0x511: 0x1308, - 0x512: 0x1308, 0x513: 0x1308, 0x514: 0x1308, 0x515: 0x1308, 0x516: 0x1308, 0x517: 0x1308, - 0x518: 0x04c9, 0x519: 0x0501, 0x51a: 0x0539, 0x51b: 0x0571, 0x51c: 0x05a9, 0x51d: 0x05e1, - 0x51e: 0x0619, 0x51f: 0x0651, 0x520: 0x0008, 0x521: 0x0008, 0x522: 0x1308, 0x523: 0x1308, - 0x524: 0x0018, 0x525: 0x0018, 0x526: 0x0008, 0x527: 0x0008, 0x528: 0x0008, 0x529: 0x0008, - 0x52a: 0x0008, 0x52b: 0x0008, 0x52c: 0x0008, 0x52d: 0x0008, 0x52e: 0x0008, 0x52f: 0x0008, - 0x530: 0x0018, 0x531: 0x0008, 0x532: 0x0008, 0x533: 0x0008, 0x534: 0x0008, 0x535: 0x0008, - 0x536: 0x0008, 0x537: 0x0008, 0x538: 0x0008, 0x539: 0x0008, 0x53a: 0x0008, 0x53b: 0x0008, - 0x53c: 0x0008, 0x53d: 0x0008, 0x53e: 0x0008, 0x53f: 0x0008, + 0x500: 0x0818, 0x501: 0x0818, 0x502: 0x0818, 0x503: 0x0818, 0x504: 0x0818, 0x505: 0x0818, + 0x506: 0x0818, 0x507: 0x0818, 0x508: 0x0818, 0x509: 0x0818, 0x50a: 0x0818, 0x50b: 0x0818, + 0x50c: 0x0818, 0x50d: 0x0818, 0x50e: 0x0040, 0x50f: 0x0b40, 0x510: 0x0c08, 0x511: 0x3308, + 0x512: 0x0a08, 0x513: 0x0a08, 0x514: 0x0a08, 0x515: 0x0c08, 0x516: 0x0c08, 0x517: 0x0c08, + 0x518: 0x0c08, 0x519: 0x0c08, 0x51a: 0x0a08, 0x51b: 0x0a08, 0x51c: 0x0a08, 0x51d: 0x0a08, + 0x51e: 0x0c08, 0x51f: 0x0a08, 0x520: 0x0a08, 0x521: 0x0a08, 0x522: 0x0a08, 0x523: 0x0a08, + 0x524: 0x0a08, 0x525: 0x0a08, 0x526: 0x0a08, 0x527: 0x0a08, 0x528: 0x0c08, 0x529: 0x0a08, + 0x52a: 0x0c08, 0x52b: 0x0a08, 0x52c: 0x0c08, 0x52d: 0x0a08, 0x52e: 0x0a08, 0x52f: 0x0c08, + 0x530: 0x3308, 0x531: 0x3308, 0x532: 0x3308, 0x533: 0x3308, 0x534: 0x3308, 0x535: 0x3308, + 0x536: 0x3308, 0x537: 0x3308, 0x538: 0x3308, 0x539: 0x3308, 0x53a: 0x3308, 0x53b: 0x3308, + 0x53c: 0x3308, 0x53d: 0x3308, 0x53e: 0x3308, 0x53f: 0x3308, // Block 0x15, offset 0x540 - 0x540: 0x0008, 0x541: 0x1308, 0x542: 0x1008, 0x543: 0x1008, 0x544: 0x0040, 0x545: 0x0008, - 0x546: 0x0008, 0x547: 0x0008, 0x548: 0x0008, 0x549: 0x0008, 0x54a: 0x0008, 0x54b: 0x0008, - 0x54c: 0x0008, 0x54d: 0x0040, 0x54e: 0x0040, 0x54f: 0x0008, 0x550: 0x0008, 0x551: 0x0040, - 0x552: 0x0040, 0x553: 0x0008, 0x554: 0x0008, 0x555: 0x0008, 0x556: 0x0008, 0x557: 0x0008, - 0x558: 0x0008, 0x559: 0x0008, 0x55a: 0x0008, 0x55b: 0x0008, 0x55c: 0x0008, 0x55d: 0x0008, - 0x55e: 0x0008, 0x55f: 0x0008, 0x560: 0x0008, 0x561: 0x0008, 0x562: 0x0008, 0x563: 0x0008, - 0x564: 0x0008, 0x565: 0x0008, 0x566: 0x0008, 0x567: 0x0008, 0x568: 0x0008, 0x569: 0x0040, - 0x56a: 0x0008, 0x56b: 0x0008, 0x56c: 0x0008, 0x56d: 0x0008, 0x56e: 0x0008, 0x56f: 0x0008, - 0x570: 0x0008, 0x571: 0x0040, 0x572: 0x0008, 0x573: 0x0040, 0x574: 0x0040, 0x575: 0x0040, - 0x576: 0x0008, 0x577: 0x0008, 0x578: 0x0008, 0x579: 0x0008, 0x57a: 0x0040, 0x57b: 0x0040, - 0x57c: 0x1308, 0x57d: 0x0008, 0x57e: 0x1008, 0x57f: 0x1008, + 0x540: 0x0c08, 0x541: 0x0a08, 0x542: 0x0a08, 0x543: 0x0a08, 0x544: 0x0a08, 0x545: 0x0a08, + 0x546: 0x0c08, 0x547: 0x0c08, 0x548: 0x0a08, 0x549: 0x0c08, 0x54a: 0x0a08, 0x54b: 0x0a08, + 0x54c: 0x0a08, 0x54d: 0x0a08, 0x54e: 0x0a08, 0x54f: 0x0a08, 0x550: 0x0a08, 0x551: 0x0a08, + 0x552: 0x0a08, 0x553: 0x0a08, 0x554: 0x0c08, 0x555: 0x0a08, 0x556: 0x0808, 0x557: 0x0808, + 0x558: 0x0808, 0x559: 0x3308, 0x55a: 0x3308, 0x55b: 0x3308, 0x55c: 0x0040, 0x55d: 0x0040, + 0x55e: 0x0818, 0x55f: 0x0040, 0x560: 0x0a08, 0x561: 0x0808, 0x562: 0x0a08, 0x563: 0x0a08, + 0x564: 0x0a08, 0x565: 0x0a08, 0x566: 0x0808, 0x567: 0x0c08, 0x568: 0x0a08, 0x569: 0x0c08, + 0x56a: 0x0c08, 0x56b: 0x0040, 0x56c: 0x0040, 0x56d: 0x0040, 0x56e: 0x0040, 0x56f: 0x0040, + 0x570: 0x0040, 0x571: 0x0040, 0x572: 0x0040, 0x573: 0x0040, 0x574: 0x0040, 0x575: 0x0040, + 0x576: 0x0040, 0x577: 0x0040, 0x578: 0x0040, 0x579: 0x0040, 0x57a: 0x0040, 0x57b: 0x0040, + 0x57c: 0x0040, 0x57d: 0x0040, 0x57e: 0x0040, 0x57f: 0x0040, // Block 0x16, offset 0x580 - 0x580: 0x1008, 0x581: 0x1308, 0x582: 0x1308, 0x583: 0x1308, 0x584: 0x1308, 0x585: 0x0040, - 0x586: 0x0040, 0x587: 0x1008, 0x588: 0x1008, 0x589: 0x0040, 0x58a: 0x0040, 0x58b: 0x1008, - 0x58c: 0x1008, 0x58d: 0x1b08, 0x58e: 0x0008, 0x58f: 0x0040, 0x590: 0x0040, 0x591: 0x0040, - 0x592: 0x0040, 0x593: 0x0040, 0x594: 0x0040, 0x595: 0x0040, 0x596: 0x0040, 0x597: 0x1008, - 0x598: 0x0040, 0x599: 0x0040, 0x59a: 0x0040, 0x59b: 0x0040, 0x59c: 0x0689, 0x59d: 0x06c1, - 0x59e: 0x0040, 0x59f: 0x06f9, 0x5a0: 0x0008, 0x5a1: 0x0008, 0x5a2: 0x1308, 0x5a3: 0x1308, - 0x5a4: 0x0040, 0x5a5: 0x0040, 0x5a6: 0x0008, 0x5a7: 0x0008, 0x5a8: 0x0008, 0x5a9: 0x0008, + 0x580: 0x3008, 0x581: 0x3308, 0x582: 0x3308, 0x583: 0x3308, 0x584: 0x3308, 0x585: 0x3308, + 0x586: 0x3308, 0x587: 0x3308, 0x588: 0x3308, 0x589: 0x3008, 0x58a: 0x3008, 0x58b: 0x3008, + 0x58c: 0x3008, 0x58d: 0x3b08, 0x58e: 0x3008, 0x58f: 0x3008, 0x590: 0x0008, 0x591: 0x3308, + 0x592: 0x3308, 0x593: 0x3308, 0x594: 0x3308, 0x595: 0x3308, 0x596: 0x3308, 0x597: 0x3308, + 0x598: 0x04c9, 0x599: 0x0501, 0x59a: 0x0539, 0x59b: 0x0571, 0x59c: 0x05a9, 0x59d: 0x05e1, + 0x59e: 0x0619, 0x59f: 0x0651, 0x5a0: 0x0008, 0x5a1: 0x0008, 0x5a2: 0x3308, 0x5a3: 0x3308, + 0x5a4: 0x0018, 0x5a5: 0x0018, 0x5a6: 0x0008, 0x5a7: 0x0008, 0x5a8: 0x0008, 0x5a9: 0x0008, 0x5aa: 0x0008, 0x5ab: 0x0008, 0x5ac: 0x0008, 0x5ad: 0x0008, 0x5ae: 0x0008, 0x5af: 0x0008, - 0x5b0: 0x0008, 0x5b1: 0x0008, 0x5b2: 0x0018, 0x5b3: 0x0018, 0x5b4: 0x0018, 0x5b5: 0x0018, - 0x5b6: 0x0018, 0x5b7: 0x0018, 0x5b8: 0x0018, 0x5b9: 0x0018, 0x5ba: 0x0018, 0x5bb: 0x0018, - 0x5bc: 0x0040, 0x5bd: 0x0040, 0x5be: 0x0040, 0x5bf: 0x0040, + 0x5b0: 0x0018, 0x5b1: 0x0008, 0x5b2: 0x0008, 0x5b3: 0x0008, 0x5b4: 0x0008, 0x5b5: 0x0008, + 0x5b6: 0x0008, 0x5b7: 0x0008, 0x5b8: 0x0008, 0x5b9: 0x0008, 0x5ba: 0x0008, 0x5bb: 0x0008, + 0x5bc: 0x0008, 0x5bd: 0x0008, 0x5be: 0x0008, 0x5bf: 0x0008, // Block 0x17, offset 0x5c0 - 0x5c0: 0x0040, 0x5c1: 0x1308, 0x5c2: 0x1308, 0x5c3: 0x1008, 0x5c4: 0x0040, 0x5c5: 0x0008, - 0x5c6: 0x0008, 0x5c7: 0x0008, 0x5c8: 0x0008, 0x5c9: 0x0008, 0x5ca: 0x0008, 0x5cb: 0x0040, - 0x5cc: 0x0040, 0x5cd: 0x0040, 0x5ce: 0x0040, 0x5cf: 0x0008, 0x5d0: 0x0008, 0x5d1: 0x0040, + 0x5c0: 0x0008, 0x5c1: 0x3308, 0x5c2: 0x3008, 0x5c3: 0x3008, 0x5c4: 0x0040, 0x5c5: 0x0008, + 0x5c6: 0x0008, 0x5c7: 0x0008, 0x5c8: 0x0008, 0x5c9: 0x0008, 0x5ca: 0x0008, 0x5cb: 0x0008, + 0x5cc: 0x0008, 0x5cd: 0x0040, 0x5ce: 0x0040, 0x5cf: 0x0008, 0x5d0: 0x0008, 0x5d1: 0x0040, 0x5d2: 0x0040, 0x5d3: 0x0008, 0x5d4: 0x0008, 0x5d5: 0x0008, 0x5d6: 0x0008, 0x5d7: 0x0008, 0x5d8: 0x0008, 0x5d9: 0x0008, 0x5da: 0x0008, 0x5db: 0x0008, 0x5dc: 0x0008, 0x5dd: 0x0008, 0x5de: 0x0008, 0x5df: 0x0008, 0x5e0: 0x0008, 0x5e1: 0x0008, 0x5e2: 0x0008, 0x5e3: 0x0008, 0x5e4: 0x0008, 0x5e5: 0x0008, 0x5e6: 0x0008, 0x5e7: 0x0008, 0x5e8: 0x0008, 0x5e9: 0x0040, 0x5ea: 0x0008, 0x5eb: 0x0008, 0x5ec: 0x0008, 0x5ed: 0x0008, 0x5ee: 0x0008, 0x5ef: 0x0008, - 0x5f0: 0x0008, 0x5f1: 0x0040, 0x5f2: 0x0008, 0x5f3: 0x0731, 0x5f4: 0x0040, 0x5f5: 0x0008, - 0x5f6: 0x0769, 0x5f7: 0x0040, 0x5f8: 0x0008, 0x5f9: 0x0008, 0x5fa: 0x0040, 0x5fb: 0x0040, - 0x5fc: 0x1308, 0x5fd: 0x0040, 0x5fe: 0x1008, 0x5ff: 0x1008, + 0x5f0: 0x0008, 0x5f1: 0x0040, 0x5f2: 0x0008, 0x5f3: 0x0040, 0x5f4: 0x0040, 0x5f5: 0x0040, + 0x5f6: 0x0008, 0x5f7: 0x0008, 0x5f8: 0x0008, 0x5f9: 0x0008, 0x5fa: 0x0040, 0x5fb: 0x0040, + 0x5fc: 0x3308, 0x5fd: 0x0008, 0x5fe: 0x3008, 0x5ff: 0x3008, // Block 0x18, offset 0x600 - 0x600: 0x1008, 0x601: 0x1308, 0x602: 0x1308, 0x603: 0x0040, 0x604: 0x0040, 0x605: 0x0040, - 0x606: 0x0040, 0x607: 0x1308, 0x608: 0x1308, 0x609: 0x0040, 0x60a: 0x0040, 0x60b: 0x1308, - 0x60c: 0x1308, 0x60d: 0x1b08, 0x60e: 0x0040, 0x60f: 0x0040, 0x610: 0x0040, 0x611: 0x1308, - 0x612: 0x0040, 0x613: 0x0040, 0x614: 0x0040, 0x615: 0x0040, 0x616: 0x0040, 0x617: 0x0040, - 0x618: 0x0040, 0x619: 0x07a1, 0x61a: 0x07d9, 0x61b: 0x0811, 0x61c: 0x0008, 0x61d: 0x0040, - 0x61e: 0x0849, 0x61f: 0x0040, 0x620: 0x0040, 0x621: 0x0040, 0x622: 0x0040, 0x623: 0x0040, + 0x600: 0x3008, 0x601: 0x3308, 0x602: 0x3308, 0x603: 0x3308, 0x604: 0x3308, 0x605: 0x0040, + 0x606: 0x0040, 0x607: 0x3008, 0x608: 0x3008, 0x609: 0x0040, 0x60a: 0x0040, 0x60b: 0x3008, + 0x60c: 0x3008, 0x60d: 0x3b08, 0x60e: 0x0008, 0x60f: 0x0040, 0x610: 0x0040, 0x611: 0x0040, + 0x612: 0x0040, 0x613: 0x0040, 0x614: 0x0040, 0x615: 0x0040, 0x616: 0x0040, 0x617: 0x3008, + 0x618: 0x0040, 0x619: 0x0040, 0x61a: 0x0040, 0x61b: 0x0040, 0x61c: 0x0689, 0x61d: 0x06c1, + 0x61e: 0x0040, 0x61f: 0x06f9, 0x620: 0x0008, 0x621: 0x0008, 0x622: 0x3308, 0x623: 0x3308, 0x624: 0x0040, 0x625: 0x0040, 0x626: 0x0008, 0x627: 0x0008, 0x628: 0x0008, 0x629: 0x0008, 0x62a: 0x0008, 0x62b: 0x0008, 0x62c: 0x0008, 0x62d: 0x0008, 0x62e: 0x0008, 0x62f: 0x0008, - 0x630: 0x1308, 0x631: 0x1308, 0x632: 0x0008, 0x633: 0x0008, 0x634: 0x0008, 0x635: 0x1308, - 0x636: 0x0040, 0x637: 0x0040, 0x638: 0x0040, 0x639: 0x0040, 0x63a: 0x0040, 0x63b: 0x0040, - 0x63c: 0x0040, 0x63d: 0x0040, 0x63e: 0x0040, 0x63f: 0x0040, + 0x630: 0x0008, 0x631: 0x0008, 0x632: 0x0018, 0x633: 0x0018, 0x634: 0x0018, 0x635: 0x0018, + 0x636: 0x0018, 0x637: 0x0018, 0x638: 0x0018, 0x639: 0x0018, 0x63a: 0x0018, 0x63b: 0x0018, + 0x63c: 0x0008, 0x63d: 0x0018, 0x63e: 0x0040, 0x63f: 0x0040, // Block 0x19, offset 0x640 - 0x640: 0x0040, 0x641: 0x1308, 0x642: 0x1308, 0x643: 0x1008, 0x644: 0x0040, 0x645: 0x0008, - 0x646: 0x0008, 0x647: 0x0008, 0x648: 0x0008, 0x649: 0x0008, 0x64a: 0x0008, 0x64b: 0x0008, - 0x64c: 0x0008, 0x64d: 0x0008, 0x64e: 0x0040, 0x64f: 0x0008, 0x650: 0x0008, 0x651: 0x0008, + 0x640: 0x0040, 0x641: 0x3308, 0x642: 0x3308, 0x643: 0x3008, 0x644: 0x0040, 0x645: 0x0008, + 0x646: 0x0008, 0x647: 0x0008, 0x648: 0x0008, 0x649: 0x0008, 0x64a: 0x0008, 0x64b: 0x0040, + 0x64c: 0x0040, 0x64d: 0x0040, 0x64e: 0x0040, 0x64f: 0x0008, 0x650: 0x0008, 0x651: 0x0040, 0x652: 0x0040, 0x653: 0x0008, 0x654: 0x0008, 0x655: 0x0008, 0x656: 0x0008, 0x657: 0x0008, 0x658: 0x0008, 0x659: 0x0008, 0x65a: 0x0008, 0x65b: 0x0008, 0x65c: 0x0008, 0x65d: 0x0008, 0x65e: 0x0008, 0x65f: 0x0008, 0x660: 0x0008, 0x661: 0x0008, 0x662: 0x0008, 0x663: 0x0008, 0x664: 0x0008, 0x665: 0x0008, 0x666: 0x0008, 0x667: 0x0008, 0x668: 0x0008, 0x669: 0x0040, 0x66a: 0x0008, 0x66b: 0x0008, 0x66c: 0x0008, 0x66d: 0x0008, 0x66e: 0x0008, 0x66f: 0x0008, - 0x670: 0x0008, 0x671: 0x0040, 0x672: 0x0008, 0x673: 0x0008, 0x674: 0x0040, 0x675: 0x0008, - 0x676: 0x0008, 0x677: 0x0008, 0x678: 0x0008, 0x679: 0x0008, 0x67a: 0x0040, 0x67b: 0x0040, - 0x67c: 0x1308, 0x67d: 0x0008, 0x67e: 0x1008, 0x67f: 0x1008, + 0x670: 0x0008, 0x671: 0x0040, 0x672: 0x0008, 0x673: 0x0731, 0x674: 0x0040, 0x675: 0x0008, + 0x676: 0x0769, 0x677: 0x0040, 0x678: 0x0008, 0x679: 0x0008, 0x67a: 0x0040, 0x67b: 0x0040, + 0x67c: 0x3308, 0x67d: 0x0040, 0x67e: 0x3008, 0x67f: 0x3008, // Block 0x1a, offset 0x680 - 0x680: 0x1008, 0x681: 0x1308, 0x682: 0x1308, 0x683: 0x1308, 0x684: 0x1308, 0x685: 0x1308, - 0x686: 0x0040, 0x687: 0x1308, 0x688: 0x1308, 0x689: 0x1008, 0x68a: 0x0040, 0x68b: 0x1008, - 0x68c: 0x1008, 0x68d: 0x1b08, 0x68e: 0x0040, 0x68f: 0x0040, 0x690: 0x0008, 0x691: 0x0040, + 0x680: 0x3008, 0x681: 0x3308, 0x682: 0x3308, 0x683: 0x0040, 0x684: 0x0040, 0x685: 0x0040, + 0x686: 0x0040, 0x687: 0x3308, 0x688: 0x3308, 0x689: 0x0040, 0x68a: 0x0040, 0x68b: 0x3308, + 0x68c: 0x3308, 0x68d: 0x3b08, 0x68e: 0x0040, 0x68f: 0x0040, 0x690: 0x0040, 0x691: 0x3308, 0x692: 0x0040, 0x693: 0x0040, 0x694: 0x0040, 0x695: 0x0040, 0x696: 0x0040, 0x697: 0x0040, - 0x698: 0x0040, 0x699: 0x0040, 0x69a: 0x0040, 0x69b: 0x0040, 0x69c: 0x0040, 0x69d: 0x0040, - 0x69e: 0x0040, 0x69f: 0x0040, 0x6a0: 0x0008, 0x6a1: 0x0008, 0x6a2: 0x1308, 0x6a3: 0x1308, + 0x698: 0x0040, 0x699: 0x07a1, 0x69a: 0x07d9, 0x69b: 0x0811, 0x69c: 0x0008, 0x69d: 0x0040, + 0x69e: 0x0849, 0x69f: 0x0040, 0x6a0: 0x0040, 0x6a1: 0x0040, 0x6a2: 0x0040, 0x6a3: 0x0040, 0x6a4: 0x0040, 0x6a5: 0x0040, 0x6a6: 0x0008, 0x6a7: 0x0008, 0x6a8: 0x0008, 0x6a9: 0x0008, 0x6aa: 0x0008, 0x6ab: 0x0008, 0x6ac: 0x0008, 0x6ad: 0x0008, 0x6ae: 0x0008, 0x6af: 0x0008, - 0x6b0: 0x0018, 0x6b1: 0x0018, 0x6b2: 0x0040, 0x6b3: 0x0040, 0x6b4: 0x0040, 0x6b5: 0x0040, - 0x6b6: 0x0040, 0x6b7: 0x0040, 0x6b8: 0x0040, 0x6b9: 0x0008, 0x6ba: 0x0040, 0x6bb: 0x0040, + 0x6b0: 0x3308, 0x6b1: 0x3308, 0x6b2: 0x0008, 0x6b3: 0x0008, 0x6b4: 0x0008, 0x6b5: 0x3308, + 0x6b6: 0x0040, 0x6b7: 0x0040, 0x6b8: 0x0040, 0x6b9: 0x0040, 0x6ba: 0x0040, 0x6bb: 0x0040, 0x6bc: 0x0040, 0x6bd: 0x0040, 0x6be: 0x0040, 0x6bf: 0x0040, // Block 0x1b, offset 0x6c0 - 0x6c0: 0x0040, 0x6c1: 0x1308, 0x6c2: 0x1008, 0x6c3: 0x1008, 0x6c4: 0x0040, 0x6c5: 0x0008, + 0x6c0: 0x0040, 0x6c1: 0x3308, 0x6c2: 0x3308, 0x6c3: 0x3008, 0x6c4: 0x0040, 0x6c5: 0x0008, 0x6c6: 0x0008, 0x6c7: 0x0008, 0x6c8: 0x0008, 0x6c9: 0x0008, 0x6ca: 0x0008, 0x6cb: 0x0008, - 0x6cc: 0x0008, 0x6cd: 0x0040, 0x6ce: 0x0040, 0x6cf: 0x0008, 0x6d0: 0x0008, 0x6d1: 0x0040, + 0x6cc: 0x0008, 0x6cd: 0x0008, 0x6ce: 0x0040, 0x6cf: 0x0008, 0x6d0: 0x0008, 0x6d1: 0x0008, 0x6d2: 0x0040, 0x6d3: 0x0008, 0x6d4: 0x0008, 0x6d5: 0x0008, 0x6d6: 0x0008, 0x6d7: 0x0008, 0x6d8: 0x0008, 0x6d9: 0x0008, 0x6da: 0x0008, 0x6db: 0x0008, 0x6dc: 0x0008, 0x6dd: 0x0008, 0x6de: 0x0008, 0x6df: 0x0008, 0x6e0: 0x0008, 0x6e1: 0x0008, 0x6e2: 0x0008, 0x6e3: 0x0008, @@ -889,1457 +889,1490 @@ var idnaValues = [8000]uint16{ 0x6ea: 0x0008, 0x6eb: 0x0008, 0x6ec: 0x0008, 0x6ed: 0x0008, 0x6ee: 0x0008, 0x6ef: 0x0008, 0x6f0: 0x0008, 0x6f1: 0x0040, 0x6f2: 0x0008, 0x6f3: 0x0008, 0x6f4: 0x0040, 0x6f5: 0x0008, 0x6f6: 0x0008, 0x6f7: 0x0008, 0x6f8: 0x0008, 0x6f9: 0x0008, 0x6fa: 0x0040, 0x6fb: 0x0040, - 0x6fc: 0x1308, 0x6fd: 0x0008, 0x6fe: 0x1008, 0x6ff: 0x1308, + 0x6fc: 0x3308, 0x6fd: 0x0008, 0x6fe: 0x3008, 0x6ff: 0x3008, // Block 0x1c, offset 0x700 - 0x700: 0x1008, 0x701: 0x1308, 0x702: 0x1308, 0x703: 0x1308, 0x704: 0x1308, 0x705: 0x0040, - 0x706: 0x0040, 0x707: 0x1008, 0x708: 0x1008, 0x709: 0x0040, 0x70a: 0x0040, 0x70b: 0x1008, - 0x70c: 0x1008, 0x70d: 0x1b08, 0x70e: 0x0040, 0x70f: 0x0040, 0x710: 0x0040, 0x711: 0x0040, - 0x712: 0x0040, 0x713: 0x0040, 0x714: 0x0040, 0x715: 0x0040, 0x716: 0x1308, 0x717: 0x1008, - 0x718: 0x0040, 0x719: 0x0040, 0x71a: 0x0040, 0x71b: 0x0040, 0x71c: 0x0881, 0x71d: 0x08b9, - 0x71e: 0x0040, 0x71f: 0x0008, 0x720: 0x0008, 0x721: 0x0008, 0x722: 0x1308, 0x723: 0x1308, + 0x700: 0x3008, 0x701: 0x3308, 0x702: 0x3308, 0x703: 0x3308, 0x704: 0x3308, 0x705: 0x3308, + 0x706: 0x0040, 0x707: 0x3308, 0x708: 0x3308, 0x709: 0x3008, 0x70a: 0x0040, 0x70b: 0x3008, + 0x70c: 0x3008, 0x70d: 0x3b08, 0x70e: 0x0040, 0x70f: 0x0040, 0x710: 0x0008, 0x711: 0x0040, + 0x712: 0x0040, 0x713: 0x0040, 0x714: 0x0040, 0x715: 0x0040, 0x716: 0x0040, 0x717: 0x0040, + 0x718: 0x0040, 0x719: 0x0040, 0x71a: 0x0040, 0x71b: 0x0040, 0x71c: 0x0040, 0x71d: 0x0040, + 0x71e: 0x0040, 0x71f: 0x0040, 0x720: 0x0008, 0x721: 0x0008, 0x722: 0x3308, 0x723: 0x3308, 0x724: 0x0040, 0x725: 0x0040, 0x726: 0x0008, 0x727: 0x0008, 0x728: 0x0008, 0x729: 0x0008, 0x72a: 0x0008, 0x72b: 0x0008, 0x72c: 0x0008, 0x72d: 0x0008, 0x72e: 0x0008, 0x72f: 0x0008, - 0x730: 0x0018, 0x731: 0x0008, 0x732: 0x0018, 0x733: 0x0018, 0x734: 0x0018, 0x735: 0x0018, - 0x736: 0x0018, 0x737: 0x0018, 0x738: 0x0040, 0x739: 0x0040, 0x73a: 0x0040, 0x73b: 0x0040, - 0x73c: 0x0040, 0x73d: 0x0040, 0x73e: 0x0040, 0x73f: 0x0040, + 0x730: 0x0018, 0x731: 0x0018, 0x732: 0x0040, 0x733: 0x0040, 0x734: 0x0040, 0x735: 0x0040, + 0x736: 0x0040, 0x737: 0x0040, 0x738: 0x0040, 0x739: 0x0008, 0x73a: 0x3308, 0x73b: 0x3308, + 0x73c: 0x3308, 0x73d: 0x3308, 0x73e: 0x3308, 0x73f: 0x3308, // Block 0x1d, offset 0x740 - 0x740: 0x0040, 0x741: 0x0040, 0x742: 0x1308, 0x743: 0x0008, 0x744: 0x0040, 0x745: 0x0008, - 0x746: 0x0008, 0x747: 0x0008, 0x748: 0x0008, 0x749: 0x0008, 0x74a: 0x0008, 0x74b: 0x0040, - 0x74c: 0x0040, 0x74d: 0x0040, 0x74e: 0x0008, 0x74f: 0x0008, 0x750: 0x0008, 0x751: 0x0040, - 0x752: 0x0008, 0x753: 0x0008, 0x754: 0x0008, 0x755: 0x0008, 0x756: 0x0040, 0x757: 0x0040, - 0x758: 0x0040, 0x759: 0x0008, 0x75a: 0x0008, 0x75b: 0x0040, 0x75c: 0x0008, 0x75d: 0x0040, - 0x75e: 0x0008, 0x75f: 0x0008, 0x760: 0x0040, 0x761: 0x0040, 0x762: 0x0040, 0x763: 0x0008, - 0x764: 0x0008, 0x765: 0x0040, 0x766: 0x0040, 0x767: 0x0040, 0x768: 0x0008, 0x769: 0x0008, - 0x76a: 0x0008, 0x76b: 0x0040, 0x76c: 0x0040, 0x76d: 0x0040, 0x76e: 0x0008, 0x76f: 0x0008, - 0x770: 0x0008, 0x771: 0x0008, 0x772: 0x0008, 0x773: 0x0008, 0x774: 0x0008, 0x775: 0x0008, + 0x740: 0x0040, 0x741: 0x3308, 0x742: 0x3008, 0x743: 0x3008, 0x744: 0x0040, 0x745: 0x0008, + 0x746: 0x0008, 0x747: 0x0008, 0x748: 0x0008, 0x749: 0x0008, 0x74a: 0x0008, 0x74b: 0x0008, + 0x74c: 0x0008, 0x74d: 0x0040, 0x74e: 0x0040, 0x74f: 0x0008, 0x750: 0x0008, 0x751: 0x0040, + 0x752: 0x0040, 0x753: 0x0008, 0x754: 0x0008, 0x755: 0x0008, 0x756: 0x0008, 0x757: 0x0008, + 0x758: 0x0008, 0x759: 0x0008, 0x75a: 0x0008, 0x75b: 0x0008, 0x75c: 0x0008, 0x75d: 0x0008, + 0x75e: 0x0008, 0x75f: 0x0008, 0x760: 0x0008, 0x761: 0x0008, 0x762: 0x0008, 0x763: 0x0008, + 0x764: 0x0008, 0x765: 0x0008, 0x766: 0x0008, 0x767: 0x0008, 0x768: 0x0008, 0x769: 0x0040, + 0x76a: 0x0008, 0x76b: 0x0008, 0x76c: 0x0008, 0x76d: 0x0008, 0x76e: 0x0008, 0x76f: 0x0008, + 0x770: 0x0008, 0x771: 0x0040, 0x772: 0x0008, 0x773: 0x0008, 0x774: 0x0040, 0x775: 0x0008, 0x776: 0x0008, 0x777: 0x0008, 0x778: 0x0008, 0x779: 0x0008, 0x77a: 0x0040, 0x77b: 0x0040, - 0x77c: 0x0040, 0x77d: 0x0040, 0x77e: 0x1008, 0x77f: 0x1008, + 0x77c: 0x3308, 0x77d: 0x0008, 0x77e: 0x3008, 0x77f: 0x3308, // Block 0x1e, offset 0x780 - 0x780: 0x1308, 0x781: 0x1008, 0x782: 0x1008, 0x783: 0x1008, 0x784: 0x1008, 0x785: 0x0040, - 0x786: 0x1308, 0x787: 0x1308, 0x788: 0x1308, 0x789: 0x0040, 0x78a: 0x1308, 0x78b: 0x1308, - 0x78c: 0x1308, 0x78d: 0x1b08, 0x78e: 0x0040, 0x78f: 0x0040, 0x790: 0x0040, 0x791: 0x0040, - 0x792: 0x0040, 0x793: 0x0040, 0x794: 0x0040, 0x795: 0x1308, 0x796: 0x1308, 0x797: 0x0040, - 0x798: 0x0008, 0x799: 0x0008, 0x79a: 0x0008, 0x79b: 0x0040, 0x79c: 0x0040, 0x79d: 0x0040, - 0x79e: 0x0040, 0x79f: 0x0040, 0x7a0: 0x0008, 0x7a1: 0x0008, 0x7a2: 0x1308, 0x7a3: 0x1308, + 0x780: 0x3008, 0x781: 0x3308, 0x782: 0x3308, 0x783: 0x3308, 0x784: 0x3308, 0x785: 0x0040, + 0x786: 0x0040, 0x787: 0x3008, 0x788: 0x3008, 0x789: 0x0040, 0x78a: 0x0040, 0x78b: 0x3008, + 0x78c: 0x3008, 0x78d: 0x3b08, 0x78e: 0x0040, 0x78f: 0x0040, 0x790: 0x0040, 0x791: 0x0040, + 0x792: 0x0040, 0x793: 0x0040, 0x794: 0x0040, 0x795: 0x0040, 0x796: 0x3308, 0x797: 0x3008, + 0x798: 0x0040, 0x799: 0x0040, 0x79a: 0x0040, 0x79b: 0x0040, 0x79c: 0x0881, 0x79d: 0x08b9, + 0x79e: 0x0040, 0x79f: 0x0008, 0x7a0: 0x0008, 0x7a1: 0x0008, 0x7a2: 0x3308, 0x7a3: 0x3308, 0x7a4: 0x0040, 0x7a5: 0x0040, 0x7a6: 0x0008, 0x7a7: 0x0008, 0x7a8: 0x0008, 0x7a9: 0x0008, 0x7aa: 0x0008, 0x7ab: 0x0008, 0x7ac: 0x0008, 0x7ad: 0x0008, 0x7ae: 0x0008, 0x7af: 0x0008, - 0x7b0: 0x0040, 0x7b1: 0x0040, 0x7b2: 0x0040, 0x7b3: 0x0040, 0x7b4: 0x0040, 0x7b5: 0x0040, - 0x7b6: 0x0040, 0x7b7: 0x0040, 0x7b8: 0x0018, 0x7b9: 0x0018, 0x7ba: 0x0018, 0x7bb: 0x0018, - 0x7bc: 0x0018, 0x7bd: 0x0018, 0x7be: 0x0018, 0x7bf: 0x0018, + 0x7b0: 0x0018, 0x7b1: 0x0008, 0x7b2: 0x0018, 0x7b3: 0x0018, 0x7b4: 0x0018, 0x7b5: 0x0018, + 0x7b6: 0x0018, 0x7b7: 0x0018, 0x7b8: 0x0040, 0x7b9: 0x0040, 0x7ba: 0x0040, 0x7bb: 0x0040, + 0x7bc: 0x0040, 0x7bd: 0x0040, 0x7be: 0x0040, 0x7bf: 0x0040, // Block 0x1f, offset 0x7c0 - 0x7c0: 0x0008, 0x7c1: 0x1308, 0x7c2: 0x1008, 0x7c3: 0x1008, 0x7c4: 0x0040, 0x7c5: 0x0008, - 0x7c6: 0x0008, 0x7c7: 0x0008, 0x7c8: 0x0008, 0x7c9: 0x0008, 0x7ca: 0x0008, 0x7cb: 0x0008, - 0x7cc: 0x0008, 0x7cd: 0x0040, 0x7ce: 0x0008, 0x7cf: 0x0008, 0x7d0: 0x0008, 0x7d1: 0x0040, - 0x7d2: 0x0008, 0x7d3: 0x0008, 0x7d4: 0x0008, 0x7d5: 0x0008, 0x7d6: 0x0008, 0x7d7: 0x0008, - 0x7d8: 0x0008, 0x7d9: 0x0008, 0x7da: 0x0008, 0x7db: 0x0008, 0x7dc: 0x0008, 0x7dd: 0x0008, - 0x7de: 0x0008, 0x7df: 0x0008, 0x7e0: 0x0008, 0x7e1: 0x0008, 0x7e2: 0x0008, 0x7e3: 0x0008, - 0x7e4: 0x0008, 0x7e5: 0x0008, 0x7e6: 0x0008, 0x7e7: 0x0008, 0x7e8: 0x0008, 0x7e9: 0x0040, - 0x7ea: 0x0008, 0x7eb: 0x0008, 0x7ec: 0x0008, 0x7ed: 0x0008, 0x7ee: 0x0008, 0x7ef: 0x0008, - 0x7f0: 0x0008, 0x7f1: 0x0008, 0x7f2: 0x0008, 0x7f3: 0x0008, 0x7f4: 0x0040, 0x7f5: 0x0008, + 0x7c0: 0x0040, 0x7c1: 0x0040, 0x7c2: 0x3308, 0x7c3: 0x0008, 0x7c4: 0x0040, 0x7c5: 0x0008, + 0x7c6: 0x0008, 0x7c7: 0x0008, 0x7c8: 0x0008, 0x7c9: 0x0008, 0x7ca: 0x0008, 0x7cb: 0x0040, + 0x7cc: 0x0040, 0x7cd: 0x0040, 0x7ce: 0x0008, 0x7cf: 0x0008, 0x7d0: 0x0008, 0x7d1: 0x0040, + 0x7d2: 0x0008, 0x7d3: 0x0008, 0x7d4: 0x0008, 0x7d5: 0x0008, 0x7d6: 0x0040, 0x7d7: 0x0040, + 0x7d8: 0x0040, 0x7d9: 0x0008, 0x7da: 0x0008, 0x7db: 0x0040, 0x7dc: 0x0008, 0x7dd: 0x0040, + 0x7de: 0x0008, 0x7df: 0x0008, 0x7e0: 0x0040, 0x7e1: 0x0040, 0x7e2: 0x0040, 0x7e3: 0x0008, + 0x7e4: 0x0008, 0x7e5: 0x0040, 0x7e6: 0x0040, 0x7e7: 0x0040, 0x7e8: 0x0008, 0x7e9: 0x0008, + 0x7ea: 0x0008, 0x7eb: 0x0040, 0x7ec: 0x0040, 0x7ed: 0x0040, 0x7ee: 0x0008, 0x7ef: 0x0008, + 0x7f0: 0x0008, 0x7f1: 0x0008, 0x7f2: 0x0008, 0x7f3: 0x0008, 0x7f4: 0x0008, 0x7f5: 0x0008, 0x7f6: 0x0008, 0x7f7: 0x0008, 0x7f8: 0x0008, 0x7f9: 0x0008, 0x7fa: 0x0040, 0x7fb: 0x0040, - 0x7fc: 0x1308, 0x7fd: 0x0008, 0x7fe: 0x1008, 0x7ff: 0x1308, + 0x7fc: 0x0040, 0x7fd: 0x0040, 0x7fe: 0x3008, 0x7ff: 0x3008, // Block 0x20, offset 0x800 - 0x800: 0x1008, 0x801: 0x1008, 0x802: 0x1008, 0x803: 0x1008, 0x804: 0x1008, 0x805: 0x0040, - 0x806: 0x1308, 0x807: 0x1008, 0x808: 0x1008, 0x809: 0x0040, 0x80a: 0x1008, 0x80b: 0x1008, - 0x80c: 0x1308, 0x80d: 0x1b08, 0x80e: 0x0040, 0x80f: 0x0040, 0x810: 0x0040, 0x811: 0x0040, - 0x812: 0x0040, 0x813: 0x0040, 0x814: 0x0040, 0x815: 0x1008, 0x816: 0x1008, 0x817: 0x0040, - 0x818: 0x0040, 0x819: 0x0040, 0x81a: 0x0040, 0x81b: 0x0040, 0x81c: 0x0040, 0x81d: 0x0040, - 0x81e: 0x0008, 0x81f: 0x0040, 0x820: 0x0008, 0x821: 0x0008, 0x822: 0x1308, 0x823: 0x1308, + 0x800: 0x3308, 0x801: 0x3008, 0x802: 0x3008, 0x803: 0x3008, 0x804: 0x3008, 0x805: 0x0040, + 0x806: 0x3308, 0x807: 0x3308, 0x808: 0x3308, 0x809: 0x0040, 0x80a: 0x3308, 0x80b: 0x3308, + 0x80c: 0x3308, 0x80d: 0x3b08, 0x80e: 0x0040, 0x80f: 0x0040, 0x810: 0x0040, 0x811: 0x0040, + 0x812: 0x0040, 0x813: 0x0040, 0x814: 0x0040, 0x815: 0x3308, 0x816: 0x3308, 0x817: 0x0040, + 0x818: 0x0008, 0x819: 0x0008, 0x81a: 0x0008, 0x81b: 0x0040, 0x81c: 0x0040, 0x81d: 0x0040, + 0x81e: 0x0040, 0x81f: 0x0040, 0x820: 0x0008, 0x821: 0x0008, 0x822: 0x3308, 0x823: 0x3308, 0x824: 0x0040, 0x825: 0x0040, 0x826: 0x0008, 0x827: 0x0008, 0x828: 0x0008, 0x829: 0x0008, 0x82a: 0x0008, 0x82b: 0x0008, 0x82c: 0x0008, 0x82d: 0x0008, 0x82e: 0x0008, 0x82f: 0x0008, - 0x830: 0x0040, 0x831: 0x0008, 0x832: 0x0008, 0x833: 0x0040, 0x834: 0x0040, 0x835: 0x0040, - 0x836: 0x0040, 0x837: 0x0040, 0x838: 0x0040, 0x839: 0x0040, 0x83a: 0x0040, 0x83b: 0x0040, - 0x83c: 0x0040, 0x83d: 0x0040, 0x83e: 0x0040, 0x83f: 0x0040, + 0x830: 0x0040, 0x831: 0x0040, 0x832: 0x0040, 0x833: 0x0040, 0x834: 0x0040, 0x835: 0x0040, + 0x836: 0x0040, 0x837: 0x0040, 0x838: 0x0018, 0x839: 0x0018, 0x83a: 0x0018, 0x83b: 0x0018, + 0x83c: 0x0018, 0x83d: 0x0018, 0x83e: 0x0018, 0x83f: 0x0018, // Block 0x21, offset 0x840 - 0x840: 0x1008, 0x841: 0x1308, 0x842: 0x1308, 0x843: 0x1308, 0x844: 0x1308, 0x845: 0x0040, - 0x846: 0x1008, 0x847: 0x1008, 0x848: 0x1008, 0x849: 0x0040, 0x84a: 0x1008, 0x84b: 0x1008, - 0x84c: 0x1008, 0x84d: 0x1b08, 0x84e: 0x0008, 0x84f: 0x0018, 0x850: 0x0040, 0x851: 0x0040, - 0x852: 0x0040, 0x853: 0x0040, 0x854: 0x0008, 0x855: 0x0008, 0x856: 0x0008, 0x857: 0x1008, - 0x858: 0x0018, 0x859: 0x0018, 0x85a: 0x0018, 0x85b: 0x0018, 0x85c: 0x0018, 0x85d: 0x0018, - 0x85e: 0x0018, 0x85f: 0x0008, 0x860: 0x0008, 0x861: 0x0008, 0x862: 0x1308, 0x863: 0x1308, - 0x864: 0x0040, 0x865: 0x0040, 0x866: 0x0008, 0x867: 0x0008, 0x868: 0x0008, 0x869: 0x0008, + 0x840: 0x0008, 0x841: 0x3308, 0x842: 0x3008, 0x843: 0x3008, 0x844: 0x0040, 0x845: 0x0008, + 0x846: 0x0008, 0x847: 0x0008, 0x848: 0x0008, 0x849: 0x0008, 0x84a: 0x0008, 0x84b: 0x0008, + 0x84c: 0x0008, 0x84d: 0x0040, 0x84e: 0x0008, 0x84f: 0x0008, 0x850: 0x0008, 0x851: 0x0040, + 0x852: 0x0008, 0x853: 0x0008, 0x854: 0x0008, 0x855: 0x0008, 0x856: 0x0008, 0x857: 0x0008, + 0x858: 0x0008, 0x859: 0x0008, 0x85a: 0x0008, 0x85b: 0x0008, 0x85c: 0x0008, 0x85d: 0x0008, + 0x85e: 0x0008, 0x85f: 0x0008, 0x860: 0x0008, 0x861: 0x0008, 0x862: 0x0008, 0x863: 0x0008, + 0x864: 0x0008, 0x865: 0x0008, 0x866: 0x0008, 0x867: 0x0008, 0x868: 0x0008, 0x869: 0x0040, 0x86a: 0x0008, 0x86b: 0x0008, 0x86c: 0x0008, 0x86d: 0x0008, 0x86e: 0x0008, 0x86f: 0x0008, - 0x870: 0x0018, 0x871: 0x0018, 0x872: 0x0018, 0x873: 0x0018, 0x874: 0x0018, 0x875: 0x0018, - 0x876: 0x0018, 0x877: 0x0018, 0x878: 0x0018, 0x879: 0x0018, 0x87a: 0x0008, 0x87b: 0x0008, - 0x87c: 0x0008, 0x87d: 0x0008, 0x87e: 0x0008, 0x87f: 0x0008, + 0x870: 0x0008, 0x871: 0x0008, 0x872: 0x0008, 0x873: 0x0008, 0x874: 0x0040, 0x875: 0x0008, + 0x876: 0x0008, 0x877: 0x0008, 0x878: 0x0008, 0x879: 0x0008, 0x87a: 0x0040, 0x87b: 0x0040, + 0x87c: 0x3308, 0x87d: 0x0008, 0x87e: 0x3008, 0x87f: 0x3308, // Block 0x22, offset 0x880 - 0x880: 0x0040, 0x881: 0x0008, 0x882: 0x0008, 0x883: 0x0040, 0x884: 0x0008, 0x885: 0x0040, - 0x886: 0x0040, 0x887: 0x0008, 0x888: 0x0008, 0x889: 0x0040, 0x88a: 0x0008, 0x88b: 0x0040, - 0x88c: 0x0040, 0x88d: 0x0008, 0x88e: 0x0040, 0x88f: 0x0040, 0x890: 0x0040, 0x891: 0x0040, - 0x892: 0x0040, 0x893: 0x0040, 0x894: 0x0008, 0x895: 0x0008, 0x896: 0x0008, 0x897: 0x0008, - 0x898: 0x0040, 0x899: 0x0008, 0x89a: 0x0008, 0x89b: 0x0008, 0x89c: 0x0008, 0x89d: 0x0008, - 0x89e: 0x0008, 0x89f: 0x0008, 0x8a0: 0x0040, 0x8a1: 0x0008, 0x8a2: 0x0008, 0x8a3: 0x0008, - 0x8a4: 0x0040, 0x8a5: 0x0008, 0x8a6: 0x0040, 0x8a7: 0x0008, 0x8a8: 0x0040, 0x8a9: 0x0040, - 0x8aa: 0x0008, 0x8ab: 0x0008, 0x8ac: 0x0040, 0x8ad: 0x0008, 0x8ae: 0x0008, 0x8af: 0x0008, - 0x8b0: 0x0008, 0x8b1: 0x1308, 0x8b2: 0x0008, 0x8b3: 0x0929, 0x8b4: 0x1308, 0x8b5: 0x1308, - 0x8b6: 0x1308, 0x8b7: 0x1308, 0x8b8: 0x1308, 0x8b9: 0x1308, 0x8ba: 0x0040, 0x8bb: 0x1308, - 0x8bc: 0x1308, 0x8bd: 0x0008, 0x8be: 0x0040, 0x8bf: 0x0040, + 0x880: 0x3008, 0x881: 0x3008, 0x882: 0x3008, 0x883: 0x3008, 0x884: 0x3008, 0x885: 0x0040, + 0x886: 0x3308, 0x887: 0x3008, 0x888: 0x3008, 0x889: 0x0040, 0x88a: 0x3008, 0x88b: 0x3008, + 0x88c: 0x3308, 0x88d: 0x3b08, 0x88e: 0x0040, 0x88f: 0x0040, 0x890: 0x0040, 0x891: 0x0040, + 0x892: 0x0040, 0x893: 0x0040, 0x894: 0x0040, 0x895: 0x3008, 0x896: 0x3008, 0x897: 0x0040, + 0x898: 0x0040, 0x899: 0x0040, 0x89a: 0x0040, 0x89b: 0x0040, 0x89c: 0x0040, 0x89d: 0x0040, + 0x89e: 0x0008, 0x89f: 0x0040, 0x8a0: 0x0008, 0x8a1: 0x0008, 0x8a2: 0x3308, 0x8a3: 0x3308, + 0x8a4: 0x0040, 0x8a5: 0x0040, 0x8a6: 0x0008, 0x8a7: 0x0008, 0x8a8: 0x0008, 0x8a9: 0x0008, + 0x8aa: 0x0008, 0x8ab: 0x0008, 0x8ac: 0x0008, 0x8ad: 0x0008, 0x8ae: 0x0008, 0x8af: 0x0008, + 0x8b0: 0x0040, 0x8b1: 0x0008, 0x8b2: 0x0008, 0x8b3: 0x0040, 0x8b4: 0x0040, 0x8b5: 0x0040, + 0x8b6: 0x0040, 0x8b7: 0x0040, 0x8b8: 0x0040, 0x8b9: 0x0040, 0x8ba: 0x0040, 0x8bb: 0x0040, + 0x8bc: 0x0040, 0x8bd: 0x0040, 0x8be: 0x0040, 0x8bf: 0x0040, // Block 0x23, offset 0x8c0 - 0x8c0: 0x0008, 0x8c1: 0x0008, 0x8c2: 0x0008, 0x8c3: 0x09d1, 0x8c4: 0x0008, 0x8c5: 0x0008, - 0x8c6: 0x0008, 0x8c7: 0x0008, 0x8c8: 0x0040, 0x8c9: 0x0008, 0x8ca: 0x0008, 0x8cb: 0x0008, - 0x8cc: 0x0008, 0x8cd: 0x0a09, 0x8ce: 0x0008, 0x8cf: 0x0008, 0x8d0: 0x0008, 0x8d1: 0x0008, - 0x8d2: 0x0a41, 0x8d3: 0x0008, 0x8d4: 0x0008, 0x8d5: 0x0008, 0x8d6: 0x0008, 0x8d7: 0x0a79, - 0x8d8: 0x0008, 0x8d9: 0x0008, 0x8da: 0x0008, 0x8db: 0x0008, 0x8dc: 0x0ab1, 0x8dd: 0x0008, - 0x8de: 0x0008, 0x8df: 0x0008, 0x8e0: 0x0008, 0x8e1: 0x0008, 0x8e2: 0x0008, 0x8e3: 0x0008, - 0x8e4: 0x0008, 0x8e5: 0x0008, 0x8e6: 0x0008, 0x8e7: 0x0008, 0x8e8: 0x0008, 0x8e9: 0x0ae9, - 0x8ea: 0x0008, 0x8eb: 0x0008, 0x8ec: 0x0008, 0x8ed: 0x0040, 0x8ee: 0x0040, 0x8ef: 0x0040, - 0x8f0: 0x0040, 0x8f1: 0x1308, 0x8f2: 0x1308, 0x8f3: 0x0b21, 0x8f4: 0x1308, 0x8f5: 0x0b59, - 0x8f6: 0x0b91, 0x8f7: 0x0bc9, 0x8f8: 0x0c19, 0x8f9: 0x0c51, 0x8fa: 0x1308, 0x8fb: 0x1308, - 0x8fc: 0x1308, 0x8fd: 0x1308, 0x8fe: 0x1308, 0x8ff: 0x1008, + 0x8c0: 0x3008, 0x8c1: 0x3308, 0x8c2: 0x3308, 0x8c3: 0x3308, 0x8c4: 0x3308, 0x8c5: 0x0040, + 0x8c6: 0x3008, 0x8c7: 0x3008, 0x8c8: 0x3008, 0x8c9: 0x0040, 0x8ca: 0x3008, 0x8cb: 0x3008, + 0x8cc: 0x3008, 0x8cd: 0x3b08, 0x8ce: 0x0008, 0x8cf: 0x0018, 0x8d0: 0x0040, 0x8d1: 0x0040, + 0x8d2: 0x0040, 0x8d3: 0x0040, 0x8d4: 0x0008, 0x8d5: 0x0008, 0x8d6: 0x0008, 0x8d7: 0x3008, + 0x8d8: 0x0018, 0x8d9: 0x0018, 0x8da: 0x0018, 0x8db: 0x0018, 0x8dc: 0x0018, 0x8dd: 0x0018, + 0x8de: 0x0018, 0x8df: 0x0008, 0x8e0: 0x0008, 0x8e1: 0x0008, 0x8e2: 0x3308, 0x8e3: 0x3308, + 0x8e4: 0x0040, 0x8e5: 0x0040, 0x8e6: 0x0008, 0x8e7: 0x0008, 0x8e8: 0x0008, 0x8e9: 0x0008, + 0x8ea: 0x0008, 0x8eb: 0x0008, 0x8ec: 0x0008, 0x8ed: 0x0008, 0x8ee: 0x0008, 0x8ef: 0x0008, + 0x8f0: 0x0018, 0x8f1: 0x0018, 0x8f2: 0x0018, 0x8f3: 0x0018, 0x8f4: 0x0018, 0x8f5: 0x0018, + 0x8f6: 0x0018, 0x8f7: 0x0018, 0x8f8: 0x0018, 0x8f9: 0x0018, 0x8fa: 0x0008, 0x8fb: 0x0008, + 0x8fc: 0x0008, 0x8fd: 0x0008, 0x8fe: 0x0008, 0x8ff: 0x0008, // Block 0x24, offset 0x900 - 0x900: 0x1308, 0x901: 0x0ca1, 0x902: 0x1308, 0x903: 0x1308, 0x904: 0x1b08, 0x905: 0x0018, - 0x906: 0x1308, 0x907: 0x1308, 0x908: 0x0008, 0x909: 0x0008, 0x90a: 0x0008, 0x90b: 0x0008, - 0x90c: 0x0008, 0x90d: 0x1308, 0x90e: 0x1308, 0x90f: 0x1308, 0x910: 0x1308, 0x911: 0x1308, - 0x912: 0x1308, 0x913: 0x0cd9, 0x914: 0x1308, 0x915: 0x1308, 0x916: 0x1308, 0x917: 0x1308, - 0x918: 0x0040, 0x919: 0x1308, 0x91a: 0x1308, 0x91b: 0x1308, 0x91c: 0x1308, 0x91d: 0x0d11, - 0x91e: 0x1308, 0x91f: 0x1308, 0x920: 0x1308, 0x921: 0x1308, 0x922: 0x0d49, 0x923: 0x1308, - 0x924: 0x1308, 0x925: 0x1308, 0x926: 0x1308, 0x927: 0x0d81, 0x928: 0x1308, 0x929: 0x1308, - 0x92a: 0x1308, 0x92b: 0x1308, 0x92c: 0x0db9, 0x92d: 0x1308, 0x92e: 0x1308, 0x92f: 0x1308, - 0x930: 0x1308, 0x931: 0x1308, 0x932: 0x1308, 0x933: 0x1308, 0x934: 0x1308, 0x935: 0x1308, - 0x936: 0x1308, 0x937: 0x1308, 0x938: 0x1308, 0x939: 0x0df1, 0x93a: 0x1308, 0x93b: 0x1308, - 0x93c: 0x1308, 0x93d: 0x0040, 0x93e: 0x0018, 0x93f: 0x0018, + 0x900: 0x0040, 0x901: 0x0008, 0x902: 0x0008, 0x903: 0x0040, 0x904: 0x0008, 0x905: 0x0040, + 0x906: 0x0040, 0x907: 0x0008, 0x908: 0x0008, 0x909: 0x0040, 0x90a: 0x0008, 0x90b: 0x0040, + 0x90c: 0x0040, 0x90d: 0x0008, 0x90e: 0x0040, 0x90f: 0x0040, 0x910: 0x0040, 0x911: 0x0040, + 0x912: 0x0040, 0x913: 0x0040, 0x914: 0x0008, 0x915: 0x0008, 0x916: 0x0008, 0x917: 0x0008, + 0x918: 0x0040, 0x919: 0x0008, 0x91a: 0x0008, 0x91b: 0x0008, 0x91c: 0x0008, 0x91d: 0x0008, + 0x91e: 0x0008, 0x91f: 0x0008, 0x920: 0x0040, 0x921: 0x0008, 0x922: 0x0008, 0x923: 0x0008, + 0x924: 0x0040, 0x925: 0x0008, 0x926: 0x0040, 0x927: 0x0008, 0x928: 0x0040, 0x929: 0x0040, + 0x92a: 0x0008, 0x92b: 0x0008, 0x92c: 0x0040, 0x92d: 0x0008, 0x92e: 0x0008, 0x92f: 0x0008, + 0x930: 0x0008, 0x931: 0x3308, 0x932: 0x0008, 0x933: 0x0929, 0x934: 0x3308, 0x935: 0x3308, + 0x936: 0x3308, 0x937: 0x3308, 0x938: 0x3308, 0x939: 0x3308, 0x93a: 0x0040, 0x93b: 0x3308, + 0x93c: 0x3308, 0x93d: 0x0008, 0x93e: 0x0040, 0x93f: 0x0040, // Block 0x25, offset 0x940 - 0x940: 0x0008, 0x941: 0x0008, 0x942: 0x0008, 0x943: 0x0008, 0x944: 0x0008, 0x945: 0x0008, - 0x946: 0x0008, 0x947: 0x0008, 0x948: 0x0008, 0x949: 0x0008, 0x94a: 0x0008, 0x94b: 0x0008, - 0x94c: 0x0008, 0x94d: 0x0008, 0x94e: 0x0008, 0x94f: 0x0008, 0x950: 0x0008, 0x951: 0x0008, - 0x952: 0x0008, 0x953: 0x0008, 0x954: 0x0008, 0x955: 0x0008, 0x956: 0x0008, 0x957: 0x0008, - 0x958: 0x0008, 0x959: 0x0008, 0x95a: 0x0008, 0x95b: 0x0008, 0x95c: 0x0008, 0x95d: 0x0008, + 0x940: 0x0008, 0x941: 0x0008, 0x942: 0x0008, 0x943: 0x09d1, 0x944: 0x0008, 0x945: 0x0008, + 0x946: 0x0008, 0x947: 0x0008, 0x948: 0x0040, 0x949: 0x0008, 0x94a: 0x0008, 0x94b: 0x0008, + 0x94c: 0x0008, 0x94d: 0x0a09, 0x94e: 0x0008, 0x94f: 0x0008, 0x950: 0x0008, 0x951: 0x0008, + 0x952: 0x0a41, 0x953: 0x0008, 0x954: 0x0008, 0x955: 0x0008, 0x956: 0x0008, 0x957: 0x0a79, + 0x958: 0x0008, 0x959: 0x0008, 0x95a: 0x0008, 0x95b: 0x0008, 0x95c: 0x0ab1, 0x95d: 0x0008, 0x95e: 0x0008, 0x95f: 0x0008, 0x960: 0x0008, 0x961: 0x0008, 0x962: 0x0008, 0x963: 0x0008, - 0x964: 0x0008, 0x965: 0x0008, 0x966: 0x0008, 0x967: 0x0008, 0x968: 0x0008, 0x969: 0x0008, - 0x96a: 0x0008, 0x96b: 0x0008, 0x96c: 0x0039, 0x96d: 0x0ed1, 0x96e: 0x0ee9, 0x96f: 0x0008, - 0x970: 0x0ef9, 0x971: 0x0f09, 0x972: 0x0f19, 0x973: 0x0f31, 0x974: 0x0249, 0x975: 0x0f41, - 0x976: 0x0259, 0x977: 0x0f51, 0x978: 0x0359, 0x979: 0x0f61, 0x97a: 0x0f71, 0x97b: 0x0008, - 0x97c: 0x00d9, 0x97d: 0x0f81, 0x97e: 0x0f99, 0x97f: 0x0269, + 0x964: 0x0008, 0x965: 0x0008, 0x966: 0x0008, 0x967: 0x0008, 0x968: 0x0008, 0x969: 0x0ae9, + 0x96a: 0x0008, 0x96b: 0x0008, 0x96c: 0x0008, 0x96d: 0x0040, 0x96e: 0x0040, 0x96f: 0x0040, + 0x970: 0x0040, 0x971: 0x3308, 0x972: 0x3308, 0x973: 0x0b21, 0x974: 0x3308, 0x975: 0x0b59, + 0x976: 0x0b91, 0x977: 0x0bc9, 0x978: 0x0c19, 0x979: 0x0c51, 0x97a: 0x3308, 0x97b: 0x3308, + 0x97c: 0x3308, 0x97d: 0x3308, 0x97e: 0x3308, 0x97f: 0x3008, // Block 0x26, offset 0x980 - 0x980: 0x0fa9, 0x981: 0x0fb9, 0x982: 0x0279, 0x983: 0x0039, 0x984: 0x0fc9, 0x985: 0x0fe1, - 0x986: 0x059d, 0x987: 0x0ee9, 0x988: 0x0ef9, 0x989: 0x0f09, 0x98a: 0x0ff9, 0x98b: 0x1011, - 0x98c: 0x1029, 0x98d: 0x0f31, 0x98e: 0x0008, 0x98f: 0x0f51, 0x990: 0x0f61, 0x991: 0x1041, - 0x992: 0x00d9, 0x993: 0x1059, 0x994: 0x05b5, 0x995: 0x05b5, 0x996: 0x0f99, 0x997: 0x0fa9, - 0x998: 0x0fb9, 0x999: 0x059d, 0x99a: 0x1071, 0x99b: 0x1089, 0x99c: 0x05cd, 0x99d: 0x1099, - 0x99e: 0x10b1, 0x99f: 0x10c9, 0x9a0: 0x10e1, 0x9a1: 0x10f9, 0x9a2: 0x0f41, 0x9a3: 0x0269, - 0x9a4: 0x0fb9, 0x9a5: 0x1089, 0x9a6: 0x1099, 0x9a7: 0x10b1, 0x9a8: 0x1111, 0x9a9: 0x10e1, - 0x9aa: 0x10f9, 0x9ab: 0x0008, 0x9ac: 0x0008, 0x9ad: 0x0008, 0x9ae: 0x0008, 0x9af: 0x0008, - 0x9b0: 0x0008, 0x9b1: 0x0008, 0x9b2: 0x0008, 0x9b3: 0x0008, 0x9b4: 0x0008, 0x9b5: 0x0008, - 0x9b6: 0x0008, 0x9b7: 0x0008, 0x9b8: 0x1129, 0x9b9: 0x0008, 0x9ba: 0x0008, 0x9bb: 0x0008, - 0x9bc: 0x0008, 0x9bd: 0x0008, 0x9be: 0x0008, 0x9bf: 0x0008, + 0x980: 0x3308, 0x981: 0x0ca1, 0x982: 0x3308, 0x983: 0x3308, 0x984: 0x3b08, 0x985: 0x0018, + 0x986: 0x3308, 0x987: 0x3308, 0x988: 0x0008, 0x989: 0x0008, 0x98a: 0x0008, 0x98b: 0x0008, + 0x98c: 0x0008, 0x98d: 0x3308, 0x98e: 0x3308, 0x98f: 0x3308, 0x990: 0x3308, 0x991: 0x3308, + 0x992: 0x3308, 0x993: 0x0cd9, 0x994: 0x3308, 0x995: 0x3308, 0x996: 0x3308, 0x997: 0x3308, + 0x998: 0x0040, 0x999: 0x3308, 0x99a: 0x3308, 0x99b: 0x3308, 0x99c: 0x3308, 0x99d: 0x0d11, + 0x99e: 0x3308, 0x99f: 0x3308, 0x9a0: 0x3308, 0x9a1: 0x3308, 0x9a2: 0x0d49, 0x9a3: 0x3308, + 0x9a4: 0x3308, 0x9a5: 0x3308, 0x9a6: 0x3308, 0x9a7: 0x0d81, 0x9a8: 0x3308, 0x9a9: 0x3308, + 0x9aa: 0x3308, 0x9ab: 0x3308, 0x9ac: 0x0db9, 0x9ad: 0x3308, 0x9ae: 0x3308, 0x9af: 0x3308, + 0x9b0: 0x3308, 0x9b1: 0x3308, 0x9b2: 0x3308, 0x9b3: 0x3308, 0x9b4: 0x3308, 0x9b5: 0x3308, + 0x9b6: 0x3308, 0x9b7: 0x3308, 0x9b8: 0x3308, 0x9b9: 0x0df1, 0x9ba: 0x3308, 0x9bb: 0x3308, + 0x9bc: 0x3308, 0x9bd: 0x0040, 0x9be: 0x0018, 0x9bf: 0x0018, // Block 0x27, offset 0x9c0 0x9c0: 0x0008, 0x9c1: 0x0008, 0x9c2: 0x0008, 0x9c3: 0x0008, 0x9c4: 0x0008, 0x9c5: 0x0008, 0x9c6: 0x0008, 0x9c7: 0x0008, 0x9c8: 0x0008, 0x9c9: 0x0008, 0x9ca: 0x0008, 0x9cb: 0x0008, 0x9cc: 0x0008, 0x9cd: 0x0008, 0x9ce: 0x0008, 0x9cf: 0x0008, 0x9d0: 0x0008, 0x9d1: 0x0008, 0x9d2: 0x0008, 0x9d3: 0x0008, 0x9d4: 0x0008, 0x9d5: 0x0008, 0x9d6: 0x0008, 0x9d7: 0x0008, - 0x9d8: 0x0008, 0x9d9: 0x0008, 0x9da: 0x0008, 0x9db: 0x1141, 0x9dc: 0x1159, 0x9dd: 0x1169, - 0x9de: 0x1181, 0x9df: 0x1029, 0x9e0: 0x1199, 0x9e1: 0x11a9, 0x9e2: 0x11c1, 0x9e3: 0x11d9, - 0x9e4: 0x11f1, 0x9e5: 0x1209, 0x9e6: 0x1221, 0x9e7: 0x05e5, 0x9e8: 0x1239, 0x9e9: 0x1251, - 0x9ea: 0xe17d, 0x9eb: 0x1269, 0x9ec: 0x1281, 0x9ed: 0x1299, 0x9ee: 0x12b1, 0x9ef: 0x12c9, - 0x9f0: 0x12e1, 0x9f1: 0x12f9, 0x9f2: 0x1311, 0x9f3: 0x1329, 0x9f4: 0x1341, 0x9f5: 0x1359, - 0x9f6: 0x1371, 0x9f7: 0x1389, 0x9f8: 0x05fd, 0x9f9: 0x13a1, 0x9fa: 0x13b9, 0x9fb: 0x13d1, - 0x9fc: 0x13e1, 0x9fd: 0x13f9, 0x9fe: 0x1411, 0x9ff: 0x1429, + 0x9d8: 0x0008, 0x9d9: 0x0008, 0x9da: 0x0008, 0x9db: 0x0008, 0x9dc: 0x0008, 0x9dd: 0x0008, + 0x9de: 0x0008, 0x9df: 0x0008, 0x9e0: 0x0008, 0x9e1: 0x0008, 0x9e2: 0x0008, 0x9e3: 0x0008, + 0x9e4: 0x0008, 0x9e5: 0x0008, 0x9e6: 0x0008, 0x9e7: 0x0008, 0x9e8: 0x0008, 0x9e9: 0x0008, + 0x9ea: 0x0008, 0x9eb: 0x0008, 0x9ec: 0x0039, 0x9ed: 0x0ed1, 0x9ee: 0x0ee9, 0x9ef: 0x0008, + 0x9f0: 0x0ef9, 0x9f1: 0x0f09, 0x9f2: 0x0f19, 0x9f3: 0x0f31, 0x9f4: 0x0249, 0x9f5: 0x0f41, + 0x9f6: 0x0259, 0x9f7: 0x0f51, 0x9f8: 0x0359, 0x9f9: 0x0f61, 0x9fa: 0x0f71, 0x9fb: 0x0008, + 0x9fc: 0x00d9, 0x9fd: 0x0f81, 0x9fe: 0x0f99, 0x9ff: 0x0269, // Block 0x28, offset 0xa00 - 0xa00: 0xe00d, 0xa01: 0x0008, 0xa02: 0xe00d, 0xa03: 0x0008, 0xa04: 0xe00d, 0xa05: 0x0008, - 0xa06: 0xe00d, 0xa07: 0x0008, 0xa08: 0xe00d, 0xa09: 0x0008, 0xa0a: 0xe00d, 0xa0b: 0x0008, - 0xa0c: 0xe00d, 0xa0d: 0x0008, 0xa0e: 0xe00d, 0xa0f: 0x0008, 0xa10: 0xe00d, 0xa11: 0x0008, - 0xa12: 0xe00d, 0xa13: 0x0008, 0xa14: 0xe00d, 0xa15: 0x0008, 0xa16: 0xe00d, 0xa17: 0x0008, - 0xa18: 0xe00d, 0xa19: 0x0008, 0xa1a: 0xe00d, 0xa1b: 0x0008, 0xa1c: 0xe00d, 0xa1d: 0x0008, - 0xa1e: 0xe00d, 0xa1f: 0x0008, 0xa20: 0xe00d, 0xa21: 0x0008, 0xa22: 0xe00d, 0xa23: 0x0008, - 0xa24: 0xe00d, 0xa25: 0x0008, 0xa26: 0xe00d, 0xa27: 0x0008, 0xa28: 0xe00d, 0xa29: 0x0008, - 0xa2a: 0xe00d, 0xa2b: 0x0008, 0xa2c: 0xe00d, 0xa2d: 0x0008, 0xa2e: 0xe00d, 0xa2f: 0x0008, - 0xa30: 0xe00d, 0xa31: 0x0008, 0xa32: 0xe00d, 0xa33: 0x0008, 0xa34: 0xe00d, 0xa35: 0x0008, - 0xa36: 0xe00d, 0xa37: 0x0008, 0xa38: 0xe00d, 0xa39: 0x0008, 0xa3a: 0xe00d, 0xa3b: 0x0008, - 0xa3c: 0xe00d, 0xa3d: 0x0008, 0xa3e: 0xe00d, 0xa3f: 0x0008, + 0xa00: 0x0fa9, 0xa01: 0x0fb9, 0xa02: 0x0279, 0xa03: 0x0039, 0xa04: 0x0fc9, 0xa05: 0x0fe1, + 0xa06: 0x059d, 0xa07: 0x0ee9, 0xa08: 0x0ef9, 0xa09: 0x0f09, 0xa0a: 0x0ff9, 0xa0b: 0x1011, + 0xa0c: 0x1029, 0xa0d: 0x0f31, 0xa0e: 0x0008, 0xa0f: 0x0f51, 0xa10: 0x0f61, 0xa11: 0x1041, + 0xa12: 0x00d9, 0xa13: 0x1059, 0xa14: 0x05b5, 0xa15: 0x05b5, 0xa16: 0x0f99, 0xa17: 0x0fa9, + 0xa18: 0x0fb9, 0xa19: 0x059d, 0xa1a: 0x1071, 0xa1b: 0x1089, 0xa1c: 0x05cd, 0xa1d: 0x1099, + 0xa1e: 0x10b1, 0xa1f: 0x10c9, 0xa20: 0x10e1, 0xa21: 0x10f9, 0xa22: 0x0f41, 0xa23: 0x0269, + 0xa24: 0x0fb9, 0xa25: 0x1089, 0xa26: 0x1099, 0xa27: 0x10b1, 0xa28: 0x1111, 0xa29: 0x10e1, + 0xa2a: 0x10f9, 0xa2b: 0x0008, 0xa2c: 0x0008, 0xa2d: 0x0008, 0xa2e: 0x0008, 0xa2f: 0x0008, + 0xa30: 0x0008, 0xa31: 0x0008, 0xa32: 0x0008, 0xa33: 0x0008, 0xa34: 0x0008, 0xa35: 0x0008, + 0xa36: 0x0008, 0xa37: 0x0008, 0xa38: 0x1129, 0xa39: 0x0008, 0xa3a: 0x0008, 0xa3b: 0x0008, + 0xa3c: 0x0008, 0xa3d: 0x0008, 0xa3e: 0x0008, 0xa3f: 0x0008, // Block 0x29, offset 0xa40 - 0xa40: 0xe00d, 0xa41: 0x0008, 0xa42: 0xe00d, 0xa43: 0x0008, 0xa44: 0xe00d, 0xa45: 0x0008, - 0xa46: 0xe00d, 0xa47: 0x0008, 0xa48: 0xe00d, 0xa49: 0x0008, 0xa4a: 0xe00d, 0xa4b: 0x0008, - 0xa4c: 0xe00d, 0xa4d: 0x0008, 0xa4e: 0xe00d, 0xa4f: 0x0008, 0xa50: 0xe00d, 0xa51: 0x0008, - 0xa52: 0xe00d, 0xa53: 0x0008, 0xa54: 0xe00d, 0xa55: 0x0008, 0xa56: 0x0008, 0xa57: 0x0008, - 0xa58: 0x0008, 0xa59: 0x0008, 0xa5a: 0x0615, 0xa5b: 0x0635, 0xa5c: 0x0008, 0xa5d: 0x0008, - 0xa5e: 0x1441, 0xa5f: 0x0008, 0xa60: 0xe00d, 0xa61: 0x0008, 0xa62: 0xe00d, 0xa63: 0x0008, - 0xa64: 0xe00d, 0xa65: 0x0008, 0xa66: 0xe00d, 0xa67: 0x0008, 0xa68: 0xe00d, 0xa69: 0x0008, - 0xa6a: 0xe00d, 0xa6b: 0x0008, 0xa6c: 0xe00d, 0xa6d: 0x0008, 0xa6e: 0xe00d, 0xa6f: 0x0008, - 0xa70: 0xe00d, 0xa71: 0x0008, 0xa72: 0xe00d, 0xa73: 0x0008, 0xa74: 0xe00d, 0xa75: 0x0008, - 0xa76: 0xe00d, 0xa77: 0x0008, 0xa78: 0xe00d, 0xa79: 0x0008, 0xa7a: 0xe00d, 0xa7b: 0x0008, - 0xa7c: 0xe00d, 0xa7d: 0x0008, 0xa7e: 0xe00d, 0xa7f: 0x0008, + 0xa40: 0x0008, 0xa41: 0x0008, 0xa42: 0x0008, 0xa43: 0x0008, 0xa44: 0x0008, 0xa45: 0x0008, + 0xa46: 0x0008, 0xa47: 0x0008, 0xa48: 0x0008, 0xa49: 0x0008, 0xa4a: 0x0008, 0xa4b: 0x0008, + 0xa4c: 0x0008, 0xa4d: 0x0008, 0xa4e: 0x0008, 0xa4f: 0x0008, 0xa50: 0x0008, 0xa51: 0x0008, + 0xa52: 0x0008, 0xa53: 0x0008, 0xa54: 0x0008, 0xa55: 0x0008, 0xa56: 0x0008, 0xa57: 0x0008, + 0xa58: 0x0008, 0xa59: 0x0008, 0xa5a: 0x0008, 0xa5b: 0x1141, 0xa5c: 0x1159, 0xa5d: 0x1169, + 0xa5e: 0x1181, 0xa5f: 0x1029, 0xa60: 0x1199, 0xa61: 0x11a9, 0xa62: 0x11c1, 0xa63: 0x11d9, + 0xa64: 0x11f1, 0xa65: 0x1209, 0xa66: 0x1221, 0xa67: 0x05e5, 0xa68: 0x1239, 0xa69: 0x1251, + 0xa6a: 0xe17d, 0xa6b: 0x1269, 0xa6c: 0x1281, 0xa6d: 0x1299, 0xa6e: 0x12b1, 0xa6f: 0x12c9, + 0xa70: 0x12e1, 0xa71: 0x12f9, 0xa72: 0x1311, 0xa73: 0x1329, 0xa74: 0x1341, 0xa75: 0x1359, + 0xa76: 0x1371, 0xa77: 0x1389, 0xa78: 0x05fd, 0xa79: 0x13a1, 0xa7a: 0x13b9, 0xa7b: 0x13d1, + 0xa7c: 0x13e1, 0xa7d: 0x13f9, 0xa7e: 0x1411, 0xa7f: 0x1429, // Block 0x2a, offset 0xa80 - 0xa80: 0x0008, 0xa81: 0x0008, 0xa82: 0x0008, 0xa83: 0x0008, 0xa84: 0x0008, 0xa85: 0x0008, - 0xa86: 0x0040, 0xa87: 0x0040, 0xa88: 0xe045, 0xa89: 0xe045, 0xa8a: 0xe045, 0xa8b: 0xe045, - 0xa8c: 0xe045, 0xa8d: 0xe045, 0xa8e: 0x0040, 0xa8f: 0x0040, 0xa90: 0x0008, 0xa91: 0x0008, - 0xa92: 0x0008, 0xa93: 0x0008, 0xa94: 0x0008, 0xa95: 0x0008, 0xa96: 0x0008, 0xa97: 0x0008, - 0xa98: 0x0040, 0xa99: 0xe045, 0xa9a: 0x0040, 0xa9b: 0xe045, 0xa9c: 0x0040, 0xa9d: 0xe045, - 0xa9e: 0x0040, 0xa9f: 0xe045, 0xaa0: 0x0008, 0xaa1: 0x0008, 0xaa2: 0x0008, 0xaa3: 0x0008, - 0xaa4: 0x0008, 0xaa5: 0x0008, 0xaa6: 0x0008, 0xaa7: 0x0008, 0xaa8: 0xe045, 0xaa9: 0xe045, - 0xaaa: 0xe045, 0xaab: 0xe045, 0xaac: 0xe045, 0xaad: 0xe045, 0xaae: 0xe045, 0xaaf: 0xe045, - 0xab0: 0x0008, 0xab1: 0x1459, 0xab2: 0x0008, 0xab3: 0x1471, 0xab4: 0x0008, 0xab5: 0x1489, - 0xab6: 0x0008, 0xab7: 0x14a1, 0xab8: 0x0008, 0xab9: 0x14b9, 0xaba: 0x0008, 0xabb: 0x14d1, - 0xabc: 0x0008, 0xabd: 0x14e9, 0xabe: 0x0040, 0xabf: 0x0040, + 0xa80: 0xe00d, 0xa81: 0x0008, 0xa82: 0xe00d, 0xa83: 0x0008, 0xa84: 0xe00d, 0xa85: 0x0008, + 0xa86: 0xe00d, 0xa87: 0x0008, 0xa88: 0xe00d, 0xa89: 0x0008, 0xa8a: 0xe00d, 0xa8b: 0x0008, + 0xa8c: 0xe00d, 0xa8d: 0x0008, 0xa8e: 0xe00d, 0xa8f: 0x0008, 0xa90: 0xe00d, 0xa91: 0x0008, + 0xa92: 0xe00d, 0xa93: 0x0008, 0xa94: 0xe00d, 0xa95: 0x0008, 0xa96: 0xe00d, 0xa97: 0x0008, + 0xa98: 0xe00d, 0xa99: 0x0008, 0xa9a: 0xe00d, 0xa9b: 0x0008, 0xa9c: 0xe00d, 0xa9d: 0x0008, + 0xa9e: 0xe00d, 0xa9f: 0x0008, 0xaa0: 0xe00d, 0xaa1: 0x0008, 0xaa2: 0xe00d, 0xaa3: 0x0008, + 0xaa4: 0xe00d, 0xaa5: 0x0008, 0xaa6: 0xe00d, 0xaa7: 0x0008, 0xaa8: 0xe00d, 0xaa9: 0x0008, + 0xaaa: 0xe00d, 0xaab: 0x0008, 0xaac: 0xe00d, 0xaad: 0x0008, 0xaae: 0xe00d, 0xaaf: 0x0008, + 0xab0: 0xe00d, 0xab1: 0x0008, 0xab2: 0xe00d, 0xab3: 0x0008, 0xab4: 0xe00d, 0xab5: 0x0008, + 0xab6: 0xe00d, 0xab7: 0x0008, 0xab8: 0xe00d, 0xab9: 0x0008, 0xaba: 0xe00d, 0xabb: 0x0008, + 0xabc: 0xe00d, 0xabd: 0x0008, 0xabe: 0xe00d, 0xabf: 0x0008, // Block 0x2b, offset 0xac0 - 0xac0: 0x1501, 0xac1: 0x1531, 0xac2: 0x1561, 0xac3: 0x1591, 0xac4: 0x15c1, 0xac5: 0x15f1, - 0xac6: 0x1621, 0xac7: 0x1651, 0xac8: 0x1501, 0xac9: 0x1531, 0xaca: 0x1561, 0xacb: 0x1591, - 0xacc: 0x15c1, 0xacd: 0x15f1, 0xace: 0x1621, 0xacf: 0x1651, 0xad0: 0x1681, 0xad1: 0x16b1, - 0xad2: 0x16e1, 0xad3: 0x1711, 0xad4: 0x1741, 0xad5: 0x1771, 0xad6: 0x17a1, 0xad7: 0x17d1, - 0xad8: 0x1681, 0xad9: 0x16b1, 0xada: 0x16e1, 0xadb: 0x1711, 0xadc: 0x1741, 0xadd: 0x1771, - 0xade: 0x17a1, 0xadf: 0x17d1, 0xae0: 0x1801, 0xae1: 0x1831, 0xae2: 0x1861, 0xae3: 0x1891, - 0xae4: 0x18c1, 0xae5: 0x18f1, 0xae6: 0x1921, 0xae7: 0x1951, 0xae8: 0x1801, 0xae9: 0x1831, - 0xaea: 0x1861, 0xaeb: 0x1891, 0xaec: 0x18c1, 0xaed: 0x18f1, 0xaee: 0x1921, 0xaef: 0x1951, - 0xaf0: 0x0008, 0xaf1: 0x0008, 0xaf2: 0x1981, 0xaf3: 0x19b1, 0xaf4: 0x19d9, 0xaf5: 0x0040, - 0xaf6: 0x0008, 0xaf7: 0x1a01, 0xaf8: 0xe045, 0xaf9: 0xe045, 0xafa: 0x064d, 0xafb: 0x1459, - 0xafc: 0x19b1, 0xafd: 0x0666, 0xafe: 0x1a31, 0xaff: 0x0686, + 0xac0: 0xe00d, 0xac1: 0x0008, 0xac2: 0xe00d, 0xac3: 0x0008, 0xac4: 0xe00d, 0xac5: 0x0008, + 0xac6: 0xe00d, 0xac7: 0x0008, 0xac8: 0xe00d, 0xac9: 0x0008, 0xaca: 0xe00d, 0xacb: 0x0008, + 0xacc: 0xe00d, 0xacd: 0x0008, 0xace: 0xe00d, 0xacf: 0x0008, 0xad0: 0xe00d, 0xad1: 0x0008, + 0xad2: 0xe00d, 0xad3: 0x0008, 0xad4: 0xe00d, 0xad5: 0x0008, 0xad6: 0x0008, 0xad7: 0x0008, + 0xad8: 0x0008, 0xad9: 0x0008, 0xada: 0x0615, 0xadb: 0x0635, 0xadc: 0x0008, 0xadd: 0x0008, + 0xade: 0x1441, 0xadf: 0x0008, 0xae0: 0xe00d, 0xae1: 0x0008, 0xae2: 0xe00d, 0xae3: 0x0008, + 0xae4: 0xe00d, 0xae5: 0x0008, 0xae6: 0xe00d, 0xae7: 0x0008, 0xae8: 0xe00d, 0xae9: 0x0008, + 0xaea: 0xe00d, 0xaeb: 0x0008, 0xaec: 0xe00d, 0xaed: 0x0008, 0xaee: 0xe00d, 0xaef: 0x0008, + 0xaf0: 0xe00d, 0xaf1: 0x0008, 0xaf2: 0xe00d, 0xaf3: 0x0008, 0xaf4: 0xe00d, 0xaf5: 0x0008, + 0xaf6: 0xe00d, 0xaf7: 0x0008, 0xaf8: 0xe00d, 0xaf9: 0x0008, 0xafa: 0xe00d, 0xafb: 0x0008, + 0xafc: 0xe00d, 0xafd: 0x0008, 0xafe: 0xe00d, 0xaff: 0x0008, // Block 0x2c, offset 0xb00 - 0xb00: 0x06a6, 0xb01: 0x1a4a, 0xb02: 0x1a79, 0xb03: 0x1aa9, 0xb04: 0x1ad1, 0xb05: 0x0040, - 0xb06: 0x0008, 0xb07: 0x1af9, 0xb08: 0x06c5, 0xb09: 0x1471, 0xb0a: 0x06dd, 0xb0b: 0x1489, - 0xb0c: 0x1aa9, 0xb0d: 0x1b2a, 0xb0e: 0x1b5a, 0xb0f: 0x1b8a, 0xb10: 0x0008, 0xb11: 0x0008, - 0xb12: 0x0008, 0xb13: 0x1bb9, 0xb14: 0x0040, 0xb15: 0x0040, 0xb16: 0x0008, 0xb17: 0x0008, - 0xb18: 0xe045, 0xb19: 0xe045, 0xb1a: 0x06f5, 0xb1b: 0x14a1, 0xb1c: 0x0040, 0xb1d: 0x1bd2, - 0xb1e: 0x1c02, 0xb1f: 0x1c32, 0xb20: 0x0008, 0xb21: 0x0008, 0xb22: 0x0008, 0xb23: 0x1c61, + 0xb00: 0x0008, 0xb01: 0x0008, 0xb02: 0x0008, 0xb03: 0x0008, 0xb04: 0x0008, 0xb05: 0x0008, + 0xb06: 0x0040, 0xb07: 0x0040, 0xb08: 0xe045, 0xb09: 0xe045, 0xb0a: 0xe045, 0xb0b: 0xe045, + 0xb0c: 0xe045, 0xb0d: 0xe045, 0xb0e: 0x0040, 0xb0f: 0x0040, 0xb10: 0x0008, 0xb11: 0x0008, + 0xb12: 0x0008, 0xb13: 0x0008, 0xb14: 0x0008, 0xb15: 0x0008, 0xb16: 0x0008, 0xb17: 0x0008, + 0xb18: 0x0040, 0xb19: 0xe045, 0xb1a: 0x0040, 0xb1b: 0xe045, 0xb1c: 0x0040, 0xb1d: 0xe045, + 0xb1e: 0x0040, 0xb1f: 0xe045, 0xb20: 0x0008, 0xb21: 0x0008, 0xb22: 0x0008, 0xb23: 0x0008, 0xb24: 0x0008, 0xb25: 0x0008, 0xb26: 0x0008, 0xb27: 0x0008, 0xb28: 0xe045, 0xb29: 0xe045, - 0xb2a: 0x070d, 0xb2b: 0x14d1, 0xb2c: 0xe04d, 0xb2d: 0x1c7a, 0xb2e: 0x03d2, 0xb2f: 0x1caa, - 0xb30: 0x0040, 0xb31: 0x0040, 0xb32: 0x1cb9, 0xb33: 0x1ce9, 0xb34: 0x1d11, 0xb35: 0x0040, - 0xb36: 0x0008, 0xb37: 0x1d39, 0xb38: 0x0725, 0xb39: 0x14b9, 0xb3a: 0x0515, 0xb3b: 0x14e9, - 0xb3c: 0x1ce9, 0xb3d: 0x073e, 0xb3e: 0x075e, 0xb3f: 0x0040, + 0xb2a: 0xe045, 0xb2b: 0xe045, 0xb2c: 0xe045, 0xb2d: 0xe045, 0xb2e: 0xe045, 0xb2f: 0xe045, + 0xb30: 0x0008, 0xb31: 0x1459, 0xb32: 0x0008, 0xb33: 0x1471, 0xb34: 0x0008, 0xb35: 0x1489, + 0xb36: 0x0008, 0xb37: 0x14a1, 0xb38: 0x0008, 0xb39: 0x14b9, 0xb3a: 0x0008, 0xb3b: 0x14d1, + 0xb3c: 0x0008, 0xb3d: 0x14e9, 0xb3e: 0x0040, 0xb3f: 0x0040, // Block 0x2d, offset 0xb40 - 0xb40: 0x000a, 0xb41: 0x000a, 0xb42: 0x000a, 0xb43: 0x000a, 0xb44: 0x000a, 0xb45: 0x000a, - 0xb46: 0x000a, 0xb47: 0x000a, 0xb48: 0x000a, 0xb49: 0x000a, 0xb4a: 0x000a, 0xb4b: 0x03c0, - 0xb4c: 0x0003, 0xb4d: 0x0003, 0xb4e: 0x0340, 0xb4f: 0x0340, 0xb50: 0x0018, 0xb51: 0xe00d, - 0xb52: 0x0018, 0xb53: 0x0018, 0xb54: 0x0018, 0xb55: 0x0018, 0xb56: 0x0018, 0xb57: 0x077e, - 0xb58: 0x0018, 0xb59: 0x0018, 0xb5a: 0x0018, 0xb5b: 0x0018, 0xb5c: 0x0018, 0xb5d: 0x0018, - 0xb5e: 0x0018, 0xb5f: 0x0018, 0xb60: 0x0018, 0xb61: 0x0018, 0xb62: 0x0018, 0xb63: 0x0018, - 0xb64: 0x0040, 0xb65: 0x0040, 0xb66: 0x0040, 0xb67: 0x0018, 0xb68: 0x0040, 0xb69: 0x0040, - 0xb6a: 0x0340, 0xb6b: 0x0340, 0xb6c: 0x0340, 0xb6d: 0x0340, 0xb6e: 0x0340, 0xb6f: 0x000a, - 0xb70: 0x0018, 0xb71: 0x0018, 0xb72: 0x0018, 0xb73: 0x1d69, 0xb74: 0x1da1, 0xb75: 0x0018, - 0xb76: 0x1df1, 0xb77: 0x1e29, 0xb78: 0x0018, 0xb79: 0x0018, 0xb7a: 0x0018, 0xb7b: 0x0018, - 0xb7c: 0x1e7a, 0xb7d: 0x0018, 0xb7e: 0x079e, 0xb7f: 0x0018, + 0xb40: 0x1501, 0xb41: 0x1531, 0xb42: 0x1561, 0xb43: 0x1591, 0xb44: 0x15c1, 0xb45: 0x15f1, + 0xb46: 0x1621, 0xb47: 0x1651, 0xb48: 0x1501, 0xb49: 0x1531, 0xb4a: 0x1561, 0xb4b: 0x1591, + 0xb4c: 0x15c1, 0xb4d: 0x15f1, 0xb4e: 0x1621, 0xb4f: 0x1651, 0xb50: 0x1681, 0xb51: 0x16b1, + 0xb52: 0x16e1, 0xb53: 0x1711, 0xb54: 0x1741, 0xb55: 0x1771, 0xb56: 0x17a1, 0xb57: 0x17d1, + 0xb58: 0x1681, 0xb59: 0x16b1, 0xb5a: 0x16e1, 0xb5b: 0x1711, 0xb5c: 0x1741, 0xb5d: 0x1771, + 0xb5e: 0x17a1, 0xb5f: 0x17d1, 0xb60: 0x1801, 0xb61: 0x1831, 0xb62: 0x1861, 0xb63: 0x1891, + 0xb64: 0x18c1, 0xb65: 0x18f1, 0xb66: 0x1921, 0xb67: 0x1951, 0xb68: 0x1801, 0xb69: 0x1831, + 0xb6a: 0x1861, 0xb6b: 0x1891, 0xb6c: 0x18c1, 0xb6d: 0x18f1, 0xb6e: 0x1921, 0xb6f: 0x1951, + 0xb70: 0x0008, 0xb71: 0x0008, 0xb72: 0x1981, 0xb73: 0x19b1, 0xb74: 0x19d9, 0xb75: 0x0040, + 0xb76: 0x0008, 0xb77: 0x1a01, 0xb78: 0xe045, 0xb79: 0xe045, 0xb7a: 0x064d, 0xb7b: 0x1459, + 0xb7c: 0x19b1, 0xb7d: 0x0666, 0xb7e: 0x1a31, 0xb7f: 0x0686, // Block 0x2e, offset 0xb80 - 0xb80: 0x0018, 0xb81: 0x0018, 0xb82: 0x0018, 0xb83: 0x0018, 0xb84: 0x0018, 0xb85: 0x0018, - 0xb86: 0x0018, 0xb87: 0x1e92, 0xb88: 0x1eaa, 0xb89: 0x1ec2, 0xb8a: 0x0018, 0xb8b: 0x0018, - 0xb8c: 0x0018, 0xb8d: 0x0018, 0xb8e: 0x0018, 0xb8f: 0x0018, 0xb90: 0x0018, 0xb91: 0x0018, - 0xb92: 0x0018, 0xb93: 0x0018, 0xb94: 0x0018, 0xb95: 0x0018, 0xb96: 0x0018, 0xb97: 0x1ed9, - 0xb98: 0x0018, 0xb99: 0x0018, 0xb9a: 0x0018, 0xb9b: 0x0018, 0xb9c: 0x0018, 0xb9d: 0x0018, - 0xb9e: 0x0018, 0xb9f: 0x000a, 0xba0: 0x03c0, 0xba1: 0x0340, 0xba2: 0x0340, 0xba3: 0x0340, - 0xba4: 0x03c0, 0xba5: 0x0040, 0xba6: 0x0040, 0xba7: 0x0040, 0xba8: 0x0040, 0xba9: 0x0040, - 0xbaa: 0x0340, 0xbab: 0x0340, 0xbac: 0x0340, 0xbad: 0x0340, 0xbae: 0x0340, 0xbaf: 0x0340, - 0xbb0: 0x1f41, 0xbb1: 0x0f41, 0xbb2: 0x0040, 0xbb3: 0x0040, 0xbb4: 0x1f51, 0xbb5: 0x1f61, - 0xbb6: 0x1f71, 0xbb7: 0x1f81, 0xbb8: 0x1f91, 0xbb9: 0x1fa1, 0xbba: 0x1fb2, 0xbbb: 0x07bd, - 0xbbc: 0x1fc2, 0xbbd: 0x1fd2, 0xbbe: 0x1fe2, 0xbbf: 0x0f71, + 0xb80: 0x06a6, 0xb81: 0x1a4a, 0xb82: 0x1a79, 0xb83: 0x1aa9, 0xb84: 0x1ad1, 0xb85: 0x0040, + 0xb86: 0x0008, 0xb87: 0x1af9, 0xb88: 0x06c5, 0xb89: 0x1471, 0xb8a: 0x06dd, 0xb8b: 0x1489, + 0xb8c: 0x1aa9, 0xb8d: 0x1b2a, 0xb8e: 0x1b5a, 0xb8f: 0x1b8a, 0xb90: 0x0008, 0xb91: 0x0008, + 0xb92: 0x0008, 0xb93: 0x1bb9, 0xb94: 0x0040, 0xb95: 0x0040, 0xb96: 0x0008, 0xb97: 0x0008, + 0xb98: 0xe045, 0xb99: 0xe045, 0xb9a: 0x06f5, 0xb9b: 0x14a1, 0xb9c: 0x0040, 0xb9d: 0x1bd2, + 0xb9e: 0x1c02, 0xb9f: 0x1c32, 0xba0: 0x0008, 0xba1: 0x0008, 0xba2: 0x0008, 0xba3: 0x1c61, + 0xba4: 0x0008, 0xba5: 0x0008, 0xba6: 0x0008, 0xba7: 0x0008, 0xba8: 0xe045, 0xba9: 0xe045, + 0xbaa: 0x070d, 0xbab: 0x14d1, 0xbac: 0xe04d, 0xbad: 0x1c7a, 0xbae: 0x03d2, 0xbaf: 0x1caa, + 0xbb0: 0x0040, 0xbb1: 0x0040, 0xbb2: 0x1cb9, 0xbb3: 0x1ce9, 0xbb4: 0x1d11, 0xbb5: 0x0040, + 0xbb6: 0x0008, 0xbb7: 0x1d39, 0xbb8: 0x0725, 0xbb9: 0x14b9, 0xbba: 0x0515, 0xbbb: 0x14e9, + 0xbbc: 0x1ce9, 0xbbd: 0x073e, 0xbbe: 0x075e, 0xbbf: 0x0040, // Block 0x2f, offset 0xbc0 - 0xbc0: 0x1f41, 0xbc1: 0x00c9, 0xbc2: 0x0069, 0xbc3: 0x0079, 0xbc4: 0x1f51, 0xbc5: 0x1f61, - 0xbc6: 0x1f71, 0xbc7: 0x1f81, 0xbc8: 0x1f91, 0xbc9: 0x1fa1, 0xbca: 0x1fb2, 0xbcb: 0x07d5, - 0xbcc: 0x1fc2, 0xbcd: 0x1fd2, 0xbce: 0x1fe2, 0xbcf: 0x0040, 0xbd0: 0x0039, 0xbd1: 0x0f09, - 0xbd2: 0x00d9, 0xbd3: 0x0369, 0xbd4: 0x0ff9, 0xbd5: 0x0249, 0xbd6: 0x0f51, 0xbd7: 0x0359, - 0xbd8: 0x0f61, 0xbd9: 0x0f71, 0xbda: 0x0f99, 0xbdb: 0x01d9, 0xbdc: 0x0fa9, 0xbdd: 0x0040, - 0xbde: 0x0040, 0xbdf: 0x0040, 0xbe0: 0x0018, 0xbe1: 0x0018, 0xbe2: 0x0018, 0xbe3: 0x0018, - 0xbe4: 0x0018, 0xbe5: 0x0018, 0xbe6: 0x0018, 0xbe7: 0x0018, 0xbe8: 0x1ff1, 0xbe9: 0x0018, - 0xbea: 0x0018, 0xbeb: 0x0018, 0xbec: 0x0018, 0xbed: 0x0018, 0xbee: 0x0018, 0xbef: 0x0018, - 0xbf0: 0x0018, 0xbf1: 0x0018, 0xbf2: 0x0018, 0xbf3: 0x0018, 0xbf4: 0x0018, 0xbf5: 0x0018, - 0xbf6: 0x0018, 0xbf7: 0x0018, 0xbf8: 0x0018, 0xbf9: 0x0018, 0xbfa: 0x0018, 0xbfb: 0x0018, - 0xbfc: 0x0018, 0xbfd: 0x0018, 0xbfe: 0x0018, 0xbff: 0x0040, + 0xbc0: 0x000a, 0xbc1: 0x000a, 0xbc2: 0x000a, 0xbc3: 0x000a, 0xbc4: 0x000a, 0xbc5: 0x000a, + 0xbc6: 0x000a, 0xbc7: 0x000a, 0xbc8: 0x000a, 0xbc9: 0x000a, 0xbca: 0x000a, 0xbcb: 0x03c0, + 0xbcc: 0x0003, 0xbcd: 0x0003, 0xbce: 0x0340, 0xbcf: 0x0b40, 0xbd0: 0x0018, 0xbd1: 0xe00d, + 0xbd2: 0x0018, 0xbd3: 0x0018, 0xbd4: 0x0018, 0xbd5: 0x0018, 0xbd6: 0x0018, 0xbd7: 0x077e, + 0xbd8: 0x0018, 0xbd9: 0x0018, 0xbda: 0x0018, 0xbdb: 0x0018, 0xbdc: 0x0018, 0xbdd: 0x0018, + 0xbde: 0x0018, 0xbdf: 0x0018, 0xbe0: 0x0018, 0xbe1: 0x0018, 0xbe2: 0x0018, 0xbe3: 0x0018, + 0xbe4: 0x0040, 0xbe5: 0x0040, 0xbe6: 0x0040, 0xbe7: 0x0018, 0xbe8: 0x0040, 0xbe9: 0x0040, + 0xbea: 0x0340, 0xbeb: 0x0340, 0xbec: 0x0340, 0xbed: 0x0340, 0xbee: 0x0340, 0xbef: 0x000a, + 0xbf0: 0x0018, 0xbf1: 0x0018, 0xbf2: 0x0018, 0xbf3: 0x1d69, 0xbf4: 0x1da1, 0xbf5: 0x0018, + 0xbf6: 0x1df1, 0xbf7: 0x1e29, 0xbf8: 0x0018, 0xbf9: 0x0018, 0xbfa: 0x0018, 0xbfb: 0x0018, + 0xbfc: 0x1e7a, 0xbfd: 0x0018, 0xbfe: 0x079e, 0xbff: 0x0018, // Block 0x30, offset 0xc00 - 0xc00: 0x07ee, 0xc01: 0x080e, 0xc02: 0x1159, 0xc03: 0x082d, 0xc04: 0x0018, 0xc05: 0x084e, - 0xc06: 0x086e, 0xc07: 0x1011, 0xc08: 0x0018, 0xc09: 0x088d, 0xc0a: 0x0f31, 0xc0b: 0x0249, - 0xc0c: 0x0249, 0xc0d: 0x0249, 0xc0e: 0x0249, 0xc0f: 0x2009, 0xc10: 0x0f41, 0xc11: 0x0f41, - 0xc12: 0x0359, 0xc13: 0x0359, 0xc14: 0x0018, 0xc15: 0x0f71, 0xc16: 0x2021, 0xc17: 0x0018, - 0xc18: 0x0018, 0xc19: 0x0f99, 0xc1a: 0x2039, 0xc1b: 0x0269, 0xc1c: 0x0269, 0xc1d: 0x0269, - 0xc1e: 0x0018, 0xc1f: 0x0018, 0xc20: 0x2049, 0xc21: 0x08ad, 0xc22: 0x2061, 0xc23: 0x0018, - 0xc24: 0x13d1, 0xc25: 0x0018, 0xc26: 0x2079, 0xc27: 0x0018, 0xc28: 0x13d1, 0xc29: 0x0018, - 0xc2a: 0x0f51, 0xc2b: 0x2091, 0xc2c: 0x0ee9, 0xc2d: 0x1159, 0xc2e: 0x0018, 0xc2f: 0x0f09, - 0xc30: 0x0f09, 0xc31: 0x1199, 0xc32: 0x0040, 0xc33: 0x0f61, 0xc34: 0x00d9, 0xc35: 0x20a9, - 0xc36: 0x20c1, 0xc37: 0x20d9, 0xc38: 0x20f1, 0xc39: 0x0f41, 0xc3a: 0x0018, 0xc3b: 0x08cd, - 0xc3c: 0x2109, 0xc3d: 0x10b1, 0xc3e: 0x10b1, 0xc3f: 0x2109, + 0xc00: 0x0018, 0xc01: 0x0018, 0xc02: 0x0018, 0xc03: 0x0018, 0xc04: 0x0018, 0xc05: 0x0018, + 0xc06: 0x0018, 0xc07: 0x1e92, 0xc08: 0x1eaa, 0xc09: 0x1ec2, 0xc0a: 0x0018, 0xc0b: 0x0018, + 0xc0c: 0x0018, 0xc0d: 0x0018, 0xc0e: 0x0018, 0xc0f: 0x0018, 0xc10: 0x0018, 0xc11: 0x0018, + 0xc12: 0x0018, 0xc13: 0x0018, 0xc14: 0x0018, 0xc15: 0x0018, 0xc16: 0x0018, 0xc17: 0x1ed9, + 0xc18: 0x0018, 0xc19: 0x0018, 0xc1a: 0x0018, 0xc1b: 0x0018, 0xc1c: 0x0018, 0xc1d: 0x0018, + 0xc1e: 0x0018, 0xc1f: 0x000a, 0xc20: 0x03c0, 0xc21: 0x0340, 0xc22: 0x0340, 0xc23: 0x0340, + 0xc24: 0x03c0, 0xc25: 0x0040, 0xc26: 0x0040, 0xc27: 0x0040, 0xc28: 0x0040, 0xc29: 0x0040, + 0xc2a: 0x0340, 0xc2b: 0x0340, 0xc2c: 0x0340, 0xc2d: 0x0340, 0xc2e: 0x0340, 0xc2f: 0x0340, + 0xc30: 0x1f41, 0xc31: 0x0f41, 0xc32: 0x0040, 0xc33: 0x0040, 0xc34: 0x1f51, 0xc35: 0x1f61, + 0xc36: 0x1f71, 0xc37: 0x1f81, 0xc38: 0x1f91, 0xc39: 0x1fa1, 0xc3a: 0x1fb2, 0xc3b: 0x07bd, + 0xc3c: 0x1fc2, 0xc3d: 0x1fd2, 0xc3e: 0x1fe2, 0xc3f: 0x0f71, // Block 0x31, offset 0xc40 - 0xc40: 0x08ed, 0xc41: 0x0018, 0xc42: 0x0018, 0xc43: 0x0018, 0xc44: 0x0018, 0xc45: 0x0ef9, - 0xc46: 0x0ef9, 0xc47: 0x0f09, 0xc48: 0x0f41, 0xc49: 0x0259, 0xc4a: 0x0018, 0xc4b: 0x0018, - 0xc4c: 0x0018, 0xc4d: 0x0018, 0xc4e: 0x0008, 0xc4f: 0x0018, 0xc50: 0x2121, 0xc51: 0x2151, - 0xc52: 0x2181, 0xc53: 0x21b9, 0xc54: 0x21e9, 0xc55: 0x2219, 0xc56: 0x2249, 0xc57: 0x2279, - 0xc58: 0x22a9, 0xc59: 0x22d9, 0xc5a: 0x2309, 0xc5b: 0x2339, 0xc5c: 0x2369, 0xc5d: 0x2399, - 0xc5e: 0x23c9, 0xc5f: 0x23f9, 0xc60: 0x0f41, 0xc61: 0x2421, 0xc62: 0x0905, 0xc63: 0x2439, - 0xc64: 0x1089, 0xc65: 0x2451, 0xc66: 0x0925, 0xc67: 0x2469, 0xc68: 0x2491, 0xc69: 0x0369, - 0xc6a: 0x24a9, 0xc6b: 0x0945, 0xc6c: 0x0359, 0xc6d: 0x1159, 0xc6e: 0x0ef9, 0xc6f: 0x0f61, - 0xc70: 0x0f41, 0xc71: 0x2421, 0xc72: 0x0965, 0xc73: 0x2439, 0xc74: 0x1089, 0xc75: 0x2451, - 0xc76: 0x0985, 0xc77: 0x2469, 0xc78: 0x2491, 0xc79: 0x0369, 0xc7a: 0x24a9, 0xc7b: 0x09a5, - 0xc7c: 0x0359, 0xc7d: 0x1159, 0xc7e: 0x0ef9, 0xc7f: 0x0f61, + 0xc40: 0x1f41, 0xc41: 0x00c9, 0xc42: 0x0069, 0xc43: 0x0079, 0xc44: 0x1f51, 0xc45: 0x1f61, + 0xc46: 0x1f71, 0xc47: 0x1f81, 0xc48: 0x1f91, 0xc49: 0x1fa1, 0xc4a: 0x1fb2, 0xc4b: 0x07d5, + 0xc4c: 0x1fc2, 0xc4d: 0x1fd2, 0xc4e: 0x1fe2, 0xc4f: 0x0040, 0xc50: 0x0039, 0xc51: 0x0f09, + 0xc52: 0x00d9, 0xc53: 0x0369, 0xc54: 0x0ff9, 0xc55: 0x0249, 0xc56: 0x0f51, 0xc57: 0x0359, + 0xc58: 0x0f61, 0xc59: 0x0f71, 0xc5a: 0x0f99, 0xc5b: 0x01d9, 0xc5c: 0x0fa9, 0xc5d: 0x0040, + 0xc5e: 0x0040, 0xc5f: 0x0040, 0xc60: 0x0018, 0xc61: 0x0018, 0xc62: 0x0018, 0xc63: 0x0018, + 0xc64: 0x0018, 0xc65: 0x0018, 0xc66: 0x0018, 0xc67: 0x0018, 0xc68: 0x1ff1, 0xc69: 0x0018, + 0xc6a: 0x0018, 0xc6b: 0x0018, 0xc6c: 0x0018, 0xc6d: 0x0018, 0xc6e: 0x0018, 0xc6f: 0x0018, + 0xc70: 0x0018, 0xc71: 0x0018, 0xc72: 0x0018, 0xc73: 0x0018, 0xc74: 0x0018, 0xc75: 0x0018, + 0xc76: 0x0018, 0xc77: 0x0018, 0xc78: 0x0018, 0xc79: 0x0018, 0xc7a: 0x0018, 0xc7b: 0x0018, + 0xc7c: 0x0018, 0xc7d: 0x0018, 0xc7e: 0x0018, 0xc7f: 0x0018, // Block 0x32, offset 0xc80 - 0xc80: 0x0018, 0xc81: 0x0018, 0xc82: 0x0018, 0xc83: 0x0018, 0xc84: 0x0018, 0xc85: 0x0018, - 0xc86: 0x0018, 0xc87: 0x0018, 0xc88: 0x0018, 0xc89: 0x0018, 0xc8a: 0x0018, 0xc8b: 0x0040, - 0xc8c: 0x0040, 0xc8d: 0x0040, 0xc8e: 0x0040, 0xc8f: 0x0040, 0xc90: 0x0040, 0xc91: 0x0040, - 0xc92: 0x0040, 0xc93: 0x0040, 0xc94: 0x0040, 0xc95: 0x0040, 0xc96: 0x0040, 0xc97: 0x0040, - 0xc98: 0x0040, 0xc99: 0x0040, 0xc9a: 0x0040, 0xc9b: 0x0040, 0xc9c: 0x0040, 0xc9d: 0x0040, - 0xc9e: 0x0040, 0xc9f: 0x0040, 0xca0: 0x00c9, 0xca1: 0x0069, 0xca2: 0x0079, 0xca3: 0x1f51, - 0xca4: 0x1f61, 0xca5: 0x1f71, 0xca6: 0x1f81, 0xca7: 0x1f91, 0xca8: 0x1fa1, 0xca9: 0x2601, - 0xcaa: 0x2619, 0xcab: 0x2631, 0xcac: 0x2649, 0xcad: 0x2661, 0xcae: 0x2679, 0xcaf: 0x2691, - 0xcb0: 0x26a9, 0xcb1: 0x26c1, 0xcb2: 0x26d9, 0xcb3: 0x26f1, 0xcb4: 0x0a06, 0xcb5: 0x0a26, - 0xcb6: 0x0a46, 0xcb7: 0x0a66, 0xcb8: 0x0a86, 0xcb9: 0x0aa6, 0xcba: 0x0ac6, 0xcbb: 0x0ae6, - 0xcbc: 0x0b06, 0xcbd: 0x270a, 0xcbe: 0x2732, 0xcbf: 0x275a, + 0xc80: 0x07ee, 0xc81: 0x080e, 0xc82: 0x1159, 0xc83: 0x082d, 0xc84: 0x0018, 0xc85: 0x084e, + 0xc86: 0x086e, 0xc87: 0x1011, 0xc88: 0x0018, 0xc89: 0x088d, 0xc8a: 0x0f31, 0xc8b: 0x0249, + 0xc8c: 0x0249, 0xc8d: 0x0249, 0xc8e: 0x0249, 0xc8f: 0x2009, 0xc90: 0x0f41, 0xc91: 0x0f41, + 0xc92: 0x0359, 0xc93: 0x0359, 0xc94: 0x0018, 0xc95: 0x0f71, 0xc96: 0x2021, 0xc97: 0x0018, + 0xc98: 0x0018, 0xc99: 0x0f99, 0xc9a: 0x2039, 0xc9b: 0x0269, 0xc9c: 0x0269, 0xc9d: 0x0269, + 0xc9e: 0x0018, 0xc9f: 0x0018, 0xca0: 0x2049, 0xca1: 0x08ad, 0xca2: 0x2061, 0xca3: 0x0018, + 0xca4: 0x13d1, 0xca5: 0x0018, 0xca6: 0x2079, 0xca7: 0x0018, 0xca8: 0x13d1, 0xca9: 0x0018, + 0xcaa: 0x0f51, 0xcab: 0x2091, 0xcac: 0x0ee9, 0xcad: 0x1159, 0xcae: 0x0018, 0xcaf: 0x0f09, + 0xcb0: 0x0f09, 0xcb1: 0x1199, 0xcb2: 0x0040, 0xcb3: 0x0f61, 0xcb4: 0x00d9, 0xcb5: 0x20a9, + 0xcb6: 0x20c1, 0xcb7: 0x20d9, 0xcb8: 0x20f1, 0xcb9: 0x0f41, 0xcba: 0x0018, 0xcbb: 0x08cd, + 0xcbc: 0x2109, 0xcbd: 0x10b1, 0xcbe: 0x10b1, 0xcbf: 0x2109, // Block 0x33, offset 0xcc0 - 0xcc0: 0x2782, 0xcc1: 0x27aa, 0xcc2: 0x27d2, 0xcc3: 0x27fa, 0xcc4: 0x2822, 0xcc5: 0x284a, - 0xcc6: 0x2872, 0xcc7: 0x289a, 0xcc8: 0x0040, 0xcc9: 0x0040, 0xcca: 0x0040, 0xccb: 0x0040, - 0xccc: 0x0040, 0xccd: 0x0040, 0xcce: 0x0040, 0xccf: 0x0040, 0xcd0: 0x0040, 0xcd1: 0x0040, - 0xcd2: 0x0040, 0xcd3: 0x0040, 0xcd4: 0x0040, 0xcd5: 0x0040, 0xcd6: 0x0040, 0xcd7: 0x0040, - 0xcd8: 0x0040, 0xcd9: 0x0040, 0xcda: 0x0040, 0xcdb: 0x0040, 0xcdc: 0x0b26, 0xcdd: 0x0b46, - 0xcde: 0x0b66, 0xcdf: 0x0b86, 0xce0: 0x0ba6, 0xce1: 0x0bc6, 0xce2: 0x0be6, 0xce3: 0x0c06, - 0xce4: 0x0c26, 0xce5: 0x0c46, 0xce6: 0x0c66, 0xce7: 0x0c86, 0xce8: 0x0ca6, 0xce9: 0x0cc6, - 0xcea: 0x0ce6, 0xceb: 0x0d06, 0xcec: 0x0d26, 0xced: 0x0d46, 0xcee: 0x0d66, 0xcef: 0x0d86, - 0xcf0: 0x0da6, 0xcf1: 0x0dc6, 0xcf2: 0x0de6, 0xcf3: 0x0e06, 0xcf4: 0x0e26, 0xcf5: 0x0e46, - 0xcf6: 0x0039, 0xcf7: 0x0ee9, 0xcf8: 0x1159, 0xcf9: 0x0ef9, 0xcfa: 0x0f09, 0xcfb: 0x1199, - 0xcfc: 0x0f31, 0xcfd: 0x0249, 0xcfe: 0x0f41, 0xcff: 0x0259, + 0xcc0: 0x08ed, 0xcc1: 0x0018, 0xcc2: 0x0018, 0xcc3: 0x0018, 0xcc4: 0x0018, 0xcc5: 0x0ef9, + 0xcc6: 0x0ef9, 0xcc7: 0x0f09, 0xcc8: 0x0f41, 0xcc9: 0x0259, 0xcca: 0x0018, 0xccb: 0x0018, + 0xccc: 0x0018, 0xccd: 0x0018, 0xcce: 0x0008, 0xccf: 0x0018, 0xcd0: 0x2121, 0xcd1: 0x2151, + 0xcd2: 0x2181, 0xcd3: 0x21b9, 0xcd4: 0x21e9, 0xcd5: 0x2219, 0xcd6: 0x2249, 0xcd7: 0x2279, + 0xcd8: 0x22a9, 0xcd9: 0x22d9, 0xcda: 0x2309, 0xcdb: 0x2339, 0xcdc: 0x2369, 0xcdd: 0x2399, + 0xcde: 0x23c9, 0xcdf: 0x23f9, 0xce0: 0x0f41, 0xce1: 0x2421, 0xce2: 0x0905, 0xce3: 0x2439, + 0xce4: 0x1089, 0xce5: 0x2451, 0xce6: 0x0925, 0xce7: 0x2469, 0xce8: 0x2491, 0xce9: 0x0369, + 0xcea: 0x24a9, 0xceb: 0x0945, 0xcec: 0x0359, 0xced: 0x1159, 0xcee: 0x0ef9, 0xcef: 0x0f61, + 0xcf0: 0x0f41, 0xcf1: 0x2421, 0xcf2: 0x0965, 0xcf3: 0x2439, 0xcf4: 0x1089, 0xcf5: 0x2451, + 0xcf6: 0x0985, 0xcf7: 0x2469, 0xcf8: 0x2491, 0xcf9: 0x0369, 0xcfa: 0x24a9, 0xcfb: 0x09a5, + 0xcfc: 0x0359, 0xcfd: 0x1159, 0xcfe: 0x0ef9, 0xcff: 0x0f61, // Block 0x34, offset 0xd00 - 0xd00: 0x0f51, 0xd01: 0x0359, 0xd02: 0x0f61, 0xd03: 0x0f71, 0xd04: 0x00d9, 0xd05: 0x0f99, - 0xd06: 0x2039, 0xd07: 0x0269, 0xd08: 0x01d9, 0xd09: 0x0fa9, 0xd0a: 0x0fb9, 0xd0b: 0x1089, - 0xd0c: 0x0279, 0xd0d: 0x0369, 0xd0e: 0x0289, 0xd0f: 0x13d1, 0xd10: 0x0039, 0xd11: 0x0ee9, - 0xd12: 0x1159, 0xd13: 0x0ef9, 0xd14: 0x0f09, 0xd15: 0x1199, 0xd16: 0x0f31, 0xd17: 0x0249, - 0xd18: 0x0f41, 0xd19: 0x0259, 0xd1a: 0x0f51, 0xd1b: 0x0359, 0xd1c: 0x0f61, 0xd1d: 0x0f71, - 0xd1e: 0x00d9, 0xd1f: 0x0f99, 0xd20: 0x2039, 0xd21: 0x0269, 0xd22: 0x01d9, 0xd23: 0x0fa9, - 0xd24: 0x0fb9, 0xd25: 0x1089, 0xd26: 0x0279, 0xd27: 0x0369, 0xd28: 0x0289, 0xd29: 0x13d1, - 0xd2a: 0x1f41, 0xd2b: 0x0018, 0xd2c: 0x0018, 0xd2d: 0x0018, 0xd2e: 0x0018, 0xd2f: 0x0018, - 0xd30: 0x0018, 0xd31: 0x0018, 0xd32: 0x0018, 0xd33: 0x0018, 0xd34: 0x0018, 0xd35: 0x0018, - 0xd36: 0x0018, 0xd37: 0x0018, 0xd38: 0x0018, 0xd39: 0x0018, 0xd3a: 0x0018, 0xd3b: 0x0018, - 0xd3c: 0x0018, 0xd3d: 0x0018, 0xd3e: 0x0018, 0xd3f: 0x0018, + 0xd00: 0x0018, 0xd01: 0x0018, 0xd02: 0x0018, 0xd03: 0x0018, 0xd04: 0x0018, 0xd05: 0x0018, + 0xd06: 0x0018, 0xd07: 0x0018, 0xd08: 0x0018, 0xd09: 0x0018, 0xd0a: 0x0018, 0xd0b: 0x0040, + 0xd0c: 0x0040, 0xd0d: 0x0040, 0xd0e: 0x0040, 0xd0f: 0x0040, 0xd10: 0x0040, 0xd11: 0x0040, + 0xd12: 0x0040, 0xd13: 0x0040, 0xd14: 0x0040, 0xd15: 0x0040, 0xd16: 0x0040, 0xd17: 0x0040, + 0xd18: 0x0040, 0xd19: 0x0040, 0xd1a: 0x0040, 0xd1b: 0x0040, 0xd1c: 0x0040, 0xd1d: 0x0040, + 0xd1e: 0x0040, 0xd1f: 0x0040, 0xd20: 0x00c9, 0xd21: 0x0069, 0xd22: 0x0079, 0xd23: 0x1f51, + 0xd24: 0x1f61, 0xd25: 0x1f71, 0xd26: 0x1f81, 0xd27: 0x1f91, 0xd28: 0x1fa1, 0xd29: 0x2601, + 0xd2a: 0x2619, 0xd2b: 0x2631, 0xd2c: 0x2649, 0xd2d: 0x2661, 0xd2e: 0x2679, 0xd2f: 0x2691, + 0xd30: 0x26a9, 0xd31: 0x26c1, 0xd32: 0x26d9, 0xd33: 0x26f1, 0xd34: 0x0a06, 0xd35: 0x0a26, + 0xd36: 0x0a46, 0xd37: 0x0a66, 0xd38: 0x0a86, 0xd39: 0x0aa6, 0xd3a: 0x0ac6, 0xd3b: 0x0ae6, + 0xd3c: 0x0b06, 0xd3d: 0x270a, 0xd3e: 0x2732, 0xd3f: 0x275a, // Block 0x35, offset 0xd40 - 0xd40: 0x0008, 0xd41: 0x0008, 0xd42: 0x0008, 0xd43: 0x0008, 0xd44: 0x0008, 0xd45: 0x0008, - 0xd46: 0x0008, 0xd47: 0x0008, 0xd48: 0x0008, 0xd49: 0x0008, 0xd4a: 0x0008, 0xd4b: 0x0008, - 0xd4c: 0x0008, 0xd4d: 0x0008, 0xd4e: 0x0008, 0xd4f: 0x0008, 0xd50: 0x0008, 0xd51: 0x0008, - 0xd52: 0x0008, 0xd53: 0x0008, 0xd54: 0x0008, 0xd55: 0x0008, 0xd56: 0x0008, 0xd57: 0x0008, - 0xd58: 0x0008, 0xd59: 0x0008, 0xd5a: 0x0008, 0xd5b: 0x0008, 0xd5c: 0x0008, 0xd5d: 0x0008, - 0xd5e: 0x0008, 0xd5f: 0x0040, 0xd60: 0xe00d, 0xd61: 0x0008, 0xd62: 0x2971, 0xd63: 0x0ebd, - 0xd64: 0x2989, 0xd65: 0x0008, 0xd66: 0x0008, 0xd67: 0xe07d, 0xd68: 0x0008, 0xd69: 0xe01d, - 0xd6a: 0x0008, 0xd6b: 0xe03d, 0xd6c: 0x0008, 0xd6d: 0x0fe1, 0xd6e: 0x1281, 0xd6f: 0x0fc9, - 0xd70: 0x1141, 0xd71: 0x0008, 0xd72: 0xe00d, 0xd73: 0x0008, 0xd74: 0x0008, 0xd75: 0xe01d, - 0xd76: 0x0008, 0xd77: 0x0008, 0xd78: 0x0008, 0xd79: 0x0008, 0xd7a: 0x0008, 0xd7b: 0x0008, - 0xd7c: 0x0259, 0xd7d: 0x1089, 0xd7e: 0x29a1, 0xd7f: 0x29b9, + 0xd40: 0x2782, 0xd41: 0x27aa, 0xd42: 0x27d2, 0xd43: 0x27fa, 0xd44: 0x2822, 0xd45: 0x284a, + 0xd46: 0x2872, 0xd47: 0x289a, 0xd48: 0x0040, 0xd49: 0x0040, 0xd4a: 0x0040, 0xd4b: 0x0040, + 0xd4c: 0x0040, 0xd4d: 0x0040, 0xd4e: 0x0040, 0xd4f: 0x0040, 0xd50: 0x0040, 0xd51: 0x0040, + 0xd52: 0x0040, 0xd53: 0x0040, 0xd54: 0x0040, 0xd55: 0x0040, 0xd56: 0x0040, 0xd57: 0x0040, + 0xd58: 0x0040, 0xd59: 0x0040, 0xd5a: 0x0040, 0xd5b: 0x0040, 0xd5c: 0x0b26, 0xd5d: 0x0b46, + 0xd5e: 0x0b66, 0xd5f: 0x0b86, 0xd60: 0x0ba6, 0xd61: 0x0bc6, 0xd62: 0x0be6, 0xd63: 0x0c06, + 0xd64: 0x0c26, 0xd65: 0x0c46, 0xd66: 0x0c66, 0xd67: 0x0c86, 0xd68: 0x0ca6, 0xd69: 0x0cc6, + 0xd6a: 0x0ce6, 0xd6b: 0x0d06, 0xd6c: 0x0d26, 0xd6d: 0x0d46, 0xd6e: 0x0d66, 0xd6f: 0x0d86, + 0xd70: 0x0da6, 0xd71: 0x0dc6, 0xd72: 0x0de6, 0xd73: 0x0e06, 0xd74: 0x0e26, 0xd75: 0x0e46, + 0xd76: 0x0039, 0xd77: 0x0ee9, 0xd78: 0x1159, 0xd79: 0x0ef9, 0xd7a: 0x0f09, 0xd7b: 0x1199, + 0xd7c: 0x0f31, 0xd7d: 0x0249, 0xd7e: 0x0f41, 0xd7f: 0x0259, // Block 0x36, offset 0xd80 - 0xd80: 0xe00d, 0xd81: 0x0008, 0xd82: 0xe00d, 0xd83: 0x0008, 0xd84: 0xe00d, 0xd85: 0x0008, - 0xd86: 0xe00d, 0xd87: 0x0008, 0xd88: 0xe00d, 0xd89: 0x0008, 0xd8a: 0xe00d, 0xd8b: 0x0008, - 0xd8c: 0xe00d, 0xd8d: 0x0008, 0xd8e: 0xe00d, 0xd8f: 0x0008, 0xd90: 0xe00d, 0xd91: 0x0008, - 0xd92: 0xe00d, 0xd93: 0x0008, 0xd94: 0xe00d, 0xd95: 0x0008, 0xd96: 0xe00d, 0xd97: 0x0008, - 0xd98: 0xe00d, 0xd99: 0x0008, 0xd9a: 0xe00d, 0xd9b: 0x0008, 0xd9c: 0xe00d, 0xd9d: 0x0008, - 0xd9e: 0xe00d, 0xd9f: 0x0008, 0xda0: 0xe00d, 0xda1: 0x0008, 0xda2: 0xe00d, 0xda3: 0x0008, - 0xda4: 0x0008, 0xda5: 0x0018, 0xda6: 0x0018, 0xda7: 0x0018, 0xda8: 0x0018, 0xda9: 0x0018, - 0xdaa: 0x0018, 0xdab: 0xe03d, 0xdac: 0x0008, 0xdad: 0xe01d, 0xdae: 0x0008, 0xdaf: 0x1308, - 0xdb0: 0x1308, 0xdb1: 0x1308, 0xdb2: 0xe00d, 0xdb3: 0x0008, 0xdb4: 0x0040, 0xdb5: 0x0040, - 0xdb6: 0x0040, 0xdb7: 0x0040, 0xdb8: 0x0040, 0xdb9: 0x0018, 0xdba: 0x0018, 0xdbb: 0x0018, + 0xd80: 0x0f51, 0xd81: 0x0359, 0xd82: 0x0f61, 0xd83: 0x0f71, 0xd84: 0x00d9, 0xd85: 0x0f99, + 0xd86: 0x2039, 0xd87: 0x0269, 0xd88: 0x01d9, 0xd89: 0x0fa9, 0xd8a: 0x0fb9, 0xd8b: 0x1089, + 0xd8c: 0x0279, 0xd8d: 0x0369, 0xd8e: 0x0289, 0xd8f: 0x13d1, 0xd90: 0x0039, 0xd91: 0x0ee9, + 0xd92: 0x1159, 0xd93: 0x0ef9, 0xd94: 0x0f09, 0xd95: 0x1199, 0xd96: 0x0f31, 0xd97: 0x0249, + 0xd98: 0x0f41, 0xd99: 0x0259, 0xd9a: 0x0f51, 0xd9b: 0x0359, 0xd9c: 0x0f61, 0xd9d: 0x0f71, + 0xd9e: 0x00d9, 0xd9f: 0x0f99, 0xda0: 0x2039, 0xda1: 0x0269, 0xda2: 0x01d9, 0xda3: 0x0fa9, + 0xda4: 0x0fb9, 0xda5: 0x1089, 0xda6: 0x0279, 0xda7: 0x0369, 0xda8: 0x0289, 0xda9: 0x13d1, + 0xdaa: 0x1f41, 0xdab: 0x0018, 0xdac: 0x0018, 0xdad: 0x0018, 0xdae: 0x0018, 0xdaf: 0x0018, + 0xdb0: 0x0018, 0xdb1: 0x0018, 0xdb2: 0x0018, 0xdb3: 0x0018, 0xdb4: 0x0018, 0xdb5: 0x0018, + 0xdb6: 0x0018, 0xdb7: 0x0018, 0xdb8: 0x0018, 0xdb9: 0x0018, 0xdba: 0x0018, 0xdbb: 0x0018, 0xdbc: 0x0018, 0xdbd: 0x0018, 0xdbe: 0x0018, 0xdbf: 0x0018, // Block 0x37, offset 0xdc0 - 0xdc0: 0x26fd, 0xdc1: 0x271d, 0xdc2: 0x273d, 0xdc3: 0x275d, 0xdc4: 0x277d, 0xdc5: 0x279d, - 0xdc6: 0x27bd, 0xdc7: 0x27dd, 0xdc8: 0x27fd, 0xdc9: 0x281d, 0xdca: 0x283d, 0xdcb: 0x285d, - 0xdcc: 0x287d, 0xdcd: 0x289d, 0xdce: 0x28bd, 0xdcf: 0x28dd, 0xdd0: 0x28fd, 0xdd1: 0x291d, - 0xdd2: 0x293d, 0xdd3: 0x295d, 0xdd4: 0x297d, 0xdd5: 0x299d, 0xdd6: 0x0040, 0xdd7: 0x0040, - 0xdd8: 0x0040, 0xdd9: 0x0040, 0xdda: 0x0040, 0xddb: 0x0040, 0xddc: 0x0040, 0xddd: 0x0040, - 0xdde: 0x0040, 0xddf: 0x0040, 0xde0: 0x0040, 0xde1: 0x0040, 0xde2: 0x0040, 0xde3: 0x0040, - 0xde4: 0x0040, 0xde5: 0x0040, 0xde6: 0x0040, 0xde7: 0x0040, 0xde8: 0x0040, 0xde9: 0x0040, - 0xdea: 0x0040, 0xdeb: 0x0040, 0xdec: 0x0040, 0xded: 0x0040, 0xdee: 0x0040, 0xdef: 0x0040, - 0xdf0: 0x0040, 0xdf1: 0x0040, 0xdf2: 0x0040, 0xdf3: 0x0040, 0xdf4: 0x0040, 0xdf5: 0x0040, - 0xdf6: 0x0040, 0xdf7: 0x0040, 0xdf8: 0x0040, 0xdf9: 0x0040, 0xdfa: 0x0040, 0xdfb: 0x0040, - 0xdfc: 0x0040, 0xdfd: 0x0040, 0xdfe: 0x0040, 0xdff: 0x0040, + 0xdc0: 0x0008, 0xdc1: 0x0008, 0xdc2: 0x0008, 0xdc3: 0x0008, 0xdc4: 0x0008, 0xdc5: 0x0008, + 0xdc6: 0x0008, 0xdc7: 0x0008, 0xdc8: 0x0008, 0xdc9: 0x0008, 0xdca: 0x0008, 0xdcb: 0x0008, + 0xdcc: 0x0008, 0xdcd: 0x0008, 0xdce: 0x0008, 0xdcf: 0x0008, 0xdd0: 0x0008, 0xdd1: 0x0008, + 0xdd2: 0x0008, 0xdd3: 0x0008, 0xdd4: 0x0008, 0xdd5: 0x0008, 0xdd6: 0x0008, 0xdd7: 0x0008, + 0xdd8: 0x0008, 0xdd9: 0x0008, 0xdda: 0x0008, 0xddb: 0x0008, 0xddc: 0x0008, 0xddd: 0x0008, + 0xdde: 0x0008, 0xddf: 0x0040, 0xde0: 0xe00d, 0xde1: 0x0008, 0xde2: 0x2971, 0xde3: 0x0ebd, + 0xde4: 0x2989, 0xde5: 0x0008, 0xde6: 0x0008, 0xde7: 0xe07d, 0xde8: 0x0008, 0xde9: 0xe01d, + 0xdea: 0x0008, 0xdeb: 0xe03d, 0xdec: 0x0008, 0xded: 0x0fe1, 0xdee: 0x1281, 0xdef: 0x0fc9, + 0xdf0: 0x1141, 0xdf1: 0x0008, 0xdf2: 0xe00d, 0xdf3: 0x0008, 0xdf4: 0x0008, 0xdf5: 0xe01d, + 0xdf6: 0x0008, 0xdf7: 0x0008, 0xdf8: 0x0008, 0xdf9: 0x0008, 0xdfa: 0x0008, 0xdfb: 0x0008, + 0xdfc: 0x0259, 0xdfd: 0x1089, 0xdfe: 0x29a1, 0xdff: 0x29b9, // Block 0x38, offset 0xe00 - 0xe00: 0x000a, 0xe01: 0x0018, 0xe02: 0x29d1, 0xe03: 0x0018, 0xe04: 0x0018, 0xe05: 0x0008, - 0xe06: 0x0008, 0xe07: 0x0008, 0xe08: 0x0018, 0xe09: 0x0018, 0xe0a: 0x0018, 0xe0b: 0x0018, - 0xe0c: 0x0018, 0xe0d: 0x0018, 0xe0e: 0x0018, 0xe0f: 0x0018, 0xe10: 0x0018, 0xe11: 0x0018, - 0xe12: 0x0018, 0xe13: 0x0018, 0xe14: 0x0018, 0xe15: 0x0018, 0xe16: 0x0018, 0xe17: 0x0018, - 0xe18: 0x0018, 0xe19: 0x0018, 0xe1a: 0x0018, 0xe1b: 0x0018, 0xe1c: 0x0018, 0xe1d: 0x0018, - 0xe1e: 0x0018, 0xe1f: 0x0018, 0xe20: 0x0018, 0xe21: 0x0018, 0xe22: 0x0018, 0xe23: 0x0018, - 0xe24: 0x0018, 0xe25: 0x0018, 0xe26: 0x0018, 0xe27: 0x0018, 0xe28: 0x0018, 0xe29: 0x0018, - 0xe2a: 0x1308, 0xe2b: 0x1308, 0xe2c: 0x1308, 0xe2d: 0x1308, 0xe2e: 0x1018, 0xe2f: 0x1018, - 0xe30: 0x0018, 0xe31: 0x0018, 0xe32: 0x0018, 0xe33: 0x0018, 0xe34: 0x0018, 0xe35: 0x0018, - 0xe36: 0xe125, 0xe37: 0x0018, 0xe38: 0x29bd, 0xe39: 0x29dd, 0xe3a: 0x29fd, 0xe3b: 0x0018, - 0xe3c: 0x0008, 0xe3d: 0x0018, 0xe3e: 0x0018, 0xe3f: 0x0018, + 0xe00: 0xe00d, 0xe01: 0x0008, 0xe02: 0xe00d, 0xe03: 0x0008, 0xe04: 0xe00d, 0xe05: 0x0008, + 0xe06: 0xe00d, 0xe07: 0x0008, 0xe08: 0xe00d, 0xe09: 0x0008, 0xe0a: 0xe00d, 0xe0b: 0x0008, + 0xe0c: 0xe00d, 0xe0d: 0x0008, 0xe0e: 0xe00d, 0xe0f: 0x0008, 0xe10: 0xe00d, 0xe11: 0x0008, + 0xe12: 0xe00d, 0xe13: 0x0008, 0xe14: 0xe00d, 0xe15: 0x0008, 0xe16: 0xe00d, 0xe17: 0x0008, + 0xe18: 0xe00d, 0xe19: 0x0008, 0xe1a: 0xe00d, 0xe1b: 0x0008, 0xe1c: 0xe00d, 0xe1d: 0x0008, + 0xe1e: 0xe00d, 0xe1f: 0x0008, 0xe20: 0xe00d, 0xe21: 0x0008, 0xe22: 0xe00d, 0xe23: 0x0008, + 0xe24: 0x0008, 0xe25: 0x0018, 0xe26: 0x0018, 0xe27: 0x0018, 0xe28: 0x0018, 0xe29: 0x0018, + 0xe2a: 0x0018, 0xe2b: 0xe03d, 0xe2c: 0x0008, 0xe2d: 0xe01d, 0xe2e: 0x0008, 0xe2f: 0x3308, + 0xe30: 0x3308, 0xe31: 0x3308, 0xe32: 0xe00d, 0xe33: 0x0008, 0xe34: 0x0040, 0xe35: 0x0040, + 0xe36: 0x0040, 0xe37: 0x0040, 0xe38: 0x0040, 0xe39: 0x0018, 0xe3a: 0x0018, 0xe3b: 0x0018, + 0xe3c: 0x0018, 0xe3d: 0x0018, 0xe3e: 0x0018, 0xe3f: 0x0018, // Block 0x39, offset 0xe40 - 0xe40: 0x2b3d, 0xe41: 0x2b5d, 0xe42: 0x2b7d, 0xe43: 0x2b9d, 0xe44: 0x2bbd, 0xe45: 0x2bdd, - 0xe46: 0x2bdd, 0xe47: 0x2bdd, 0xe48: 0x2bfd, 0xe49: 0x2bfd, 0xe4a: 0x2bfd, 0xe4b: 0x2bfd, - 0xe4c: 0x2c1d, 0xe4d: 0x2c1d, 0xe4e: 0x2c1d, 0xe4f: 0x2c3d, 0xe50: 0x2c5d, 0xe51: 0x2c5d, - 0xe52: 0x2a7d, 0xe53: 0x2a7d, 0xe54: 0x2c5d, 0xe55: 0x2c5d, 0xe56: 0x2c7d, 0xe57: 0x2c7d, - 0xe58: 0x2c5d, 0xe59: 0x2c5d, 0xe5a: 0x2a7d, 0xe5b: 0x2a7d, 0xe5c: 0x2c5d, 0xe5d: 0x2c5d, - 0xe5e: 0x2c3d, 0xe5f: 0x2c3d, 0xe60: 0x2c9d, 0xe61: 0x2c9d, 0xe62: 0x2cbd, 0xe63: 0x2cbd, - 0xe64: 0x0040, 0xe65: 0x2cdd, 0xe66: 0x2cfd, 0xe67: 0x2d1d, 0xe68: 0x2d1d, 0xe69: 0x2d3d, - 0xe6a: 0x2d5d, 0xe6b: 0x2d7d, 0xe6c: 0x2d9d, 0xe6d: 0x2dbd, 0xe6e: 0x2ddd, 0xe6f: 0x2dfd, - 0xe70: 0x2e1d, 0xe71: 0x2e3d, 0xe72: 0x2e3d, 0xe73: 0x2e5d, 0xe74: 0x2e7d, 0xe75: 0x2e7d, - 0xe76: 0x2e9d, 0xe77: 0x2ebd, 0xe78: 0x2e5d, 0xe79: 0x2edd, 0xe7a: 0x2efd, 0xe7b: 0x2edd, - 0xe7c: 0x2e5d, 0xe7d: 0x2f1d, 0xe7e: 0x2f3d, 0xe7f: 0x2f5d, + 0xe40: 0x26fd, 0xe41: 0x271d, 0xe42: 0x273d, 0xe43: 0x275d, 0xe44: 0x277d, 0xe45: 0x279d, + 0xe46: 0x27bd, 0xe47: 0x27dd, 0xe48: 0x27fd, 0xe49: 0x281d, 0xe4a: 0x283d, 0xe4b: 0x285d, + 0xe4c: 0x287d, 0xe4d: 0x289d, 0xe4e: 0x28bd, 0xe4f: 0x28dd, 0xe50: 0x28fd, 0xe51: 0x291d, + 0xe52: 0x293d, 0xe53: 0x295d, 0xe54: 0x297d, 0xe55: 0x299d, 0xe56: 0x0040, 0xe57: 0x0040, + 0xe58: 0x0040, 0xe59: 0x0040, 0xe5a: 0x0040, 0xe5b: 0x0040, 0xe5c: 0x0040, 0xe5d: 0x0040, + 0xe5e: 0x0040, 0xe5f: 0x0040, 0xe60: 0x0040, 0xe61: 0x0040, 0xe62: 0x0040, 0xe63: 0x0040, + 0xe64: 0x0040, 0xe65: 0x0040, 0xe66: 0x0040, 0xe67: 0x0040, 0xe68: 0x0040, 0xe69: 0x0040, + 0xe6a: 0x0040, 0xe6b: 0x0040, 0xe6c: 0x0040, 0xe6d: 0x0040, 0xe6e: 0x0040, 0xe6f: 0x0040, + 0xe70: 0x0040, 0xe71: 0x0040, 0xe72: 0x0040, 0xe73: 0x0040, 0xe74: 0x0040, 0xe75: 0x0040, + 0xe76: 0x0040, 0xe77: 0x0040, 0xe78: 0x0040, 0xe79: 0x0040, 0xe7a: 0x0040, 0xe7b: 0x0040, + 0xe7c: 0x0040, 0xe7d: 0x0040, 0xe7e: 0x0040, 0xe7f: 0x0040, // Block 0x3a, offset 0xe80 - 0xe80: 0x2f7d, 0xe81: 0x2f9d, 0xe82: 0x2cfd, 0xe83: 0x2cdd, 0xe84: 0x2fbd, 0xe85: 0x2fdd, - 0xe86: 0x2ffd, 0xe87: 0x301d, 0xe88: 0x303d, 0xe89: 0x305d, 0xe8a: 0x307d, 0xe8b: 0x309d, - 0xe8c: 0x30bd, 0xe8d: 0x30dd, 0xe8e: 0x30fd, 0xe8f: 0x0040, 0xe90: 0x0018, 0xe91: 0x0018, - 0xe92: 0x311d, 0xe93: 0x313d, 0xe94: 0x315d, 0xe95: 0x317d, 0xe96: 0x319d, 0xe97: 0x31bd, - 0xe98: 0x31dd, 0xe99: 0x31fd, 0xe9a: 0x321d, 0xe9b: 0x323d, 0xe9c: 0x315d, 0xe9d: 0x325d, - 0xe9e: 0x327d, 0xe9f: 0x329d, 0xea0: 0x0008, 0xea1: 0x0008, 0xea2: 0x0008, 0xea3: 0x0008, - 0xea4: 0x0008, 0xea5: 0x0008, 0xea6: 0x0008, 0xea7: 0x0008, 0xea8: 0x0008, 0xea9: 0x0008, - 0xeaa: 0x0008, 0xeab: 0x0008, 0xeac: 0x0008, 0xead: 0x0008, 0xeae: 0x0008, 0xeaf: 0x0008, - 0xeb0: 0x0008, 0xeb1: 0x0008, 0xeb2: 0x0008, 0xeb3: 0x0008, 0xeb4: 0x0008, 0xeb5: 0x0008, - 0xeb6: 0x0008, 0xeb7: 0x0008, 0xeb8: 0x0008, 0xeb9: 0x0008, 0xeba: 0x0008, 0xebb: 0x0040, - 0xebc: 0x0040, 0xebd: 0x0040, 0xebe: 0x0040, 0xebf: 0x0040, + 0xe80: 0x000a, 0xe81: 0x0018, 0xe82: 0x29d1, 0xe83: 0x0018, 0xe84: 0x0018, 0xe85: 0x0008, + 0xe86: 0x0008, 0xe87: 0x0008, 0xe88: 0x0018, 0xe89: 0x0018, 0xe8a: 0x0018, 0xe8b: 0x0018, + 0xe8c: 0x0018, 0xe8d: 0x0018, 0xe8e: 0x0018, 0xe8f: 0x0018, 0xe90: 0x0018, 0xe91: 0x0018, + 0xe92: 0x0018, 0xe93: 0x0018, 0xe94: 0x0018, 0xe95: 0x0018, 0xe96: 0x0018, 0xe97: 0x0018, + 0xe98: 0x0018, 0xe99: 0x0018, 0xe9a: 0x0018, 0xe9b: 0x0018, 0xe9c: 0x0018, 0xe9d: 0x0018, + 0xe9e: 0x0018, 0xe9f: 0x0018, 0xea0: 0x0018, 0xea1: 0x0018, 0xea2: 0x0018, 0xea3: 0x0018, + 0xea4: 0x0018, 0xea5: 0x0018, 0xea6: 0x0018, 0xea7: 0x0018, 0xea8: 0x0018, 0xea9: 0x0018, + 0xeaa: 0x3308, 0xeab: 0x3308, 0xeac: 0x3308, 0xead: 0x3308, 0xeae: 0x3018, 0xeaf: 0x3018, + 0xeb0: 0x0018, 0xeb1: 0x0018, 0xeb2: 0x0018, 0xeb3: 0x0018, 0xeb4: 0x0018, 0xeb5: 0x0018, + 0xeb6: 0xe125, 0xeb7: 0x0018, 0xeb8: 0x29bd, 0xeb9: 0x29dd, 0xeba: 0x29fd, 0xebb: 0x0018, + 0xebc: 0x0008, 0xebd: 0x0018, 0xebe: 0x0018, 0xebf: 0x0018, // Block 0x3b, offset 0xec0 - 0xec0: 0x36a2, 0xec1: 0x36d2, 0xec2: 0x3702, 0xec3: 0x3732, 0xec4: 0x32bd, 0xec5: 0x32dd, - 0xec6: 0x32fd, 0xec7: 0x331d, 0xec8: 0x0018, 0xec9: 0x0018, 0xeca: 0x0018, 0xecb: 0x0018, - 0xecc: 0x0018, 0xecd: 0x0018, 0xece: 0x0018, 0xecf: 0x0018, 0xed0: 0x333d, 0xed1: 0x3761, - 0xed2: 0x3779, 0xed3: 0x3791, 0xed4: 0x37a9, 0xed5: 0x37c1, 0xed6: 0x37d9, 0xed7: 0x37f1, - 0xed8: 0x3809, 0xed9: 0x3821, 0xeda: 0x3839, 0xedb: 0x3851, 0xedc: 0x3869, 0xedd: 0x3881, - 0xede: 0x3899, 0xedf: 0x38b1, 0xee0: 0x335d, 0xee1: 0x337d, 0xee2: 0x339d, 0xee3: 0x33bd, - 0xee4: 0x33dd, 0xee5: 0x33dd, 0xee6: 0x33fd, 0xee7: 0x341d, 0xee8: 0x343d, 0xee9: 0x345d, - 0xeea: 0x347d, 0xeeb: 0x349d, 0xeec: 0x34bd, 0xeed: 0x34dd, 0xeee: 0x34fd, 0xeef: 0x351d, - 0xef0: 0x353d, 0xef1: 0x355d, 0xef2: 0x357d, 0xef3: 0x359d, 0xef4: 0x35bd, 0xef5: 0x35dd, - 0xef6: 0x35fd, 0xef7: 0x361d, 0xef8: 0x363d, 0xef9: 0x365d, 0xefa: 0x367d, 0xefb: 0x369d, - 0xefc: 0x38c9, 0xefd: 0x3901, 0xefe: 0x36bd, 0xeff: 0x0018, + 0xec0: 0x2b3d, 0xec1: 0x2b5d, 0xec2: 0x2b7d, 0xec3: 0x2b9d, 0xec4: 0x2bbd, 0xec5: 0x2bdd, + 0xec6: 0x2bdd, 0xec7: 0x2bdd, 0xec8: 0x2bfd, 0xec9: 0x2bfd, 0xeca: 0x2bfd, 0xecb: 0x2bfd, + 0xecc: 0x2c1d, 0xecd: 0x2c1d, 0xece: 0x2c1d, 0xecf: 0x2c3d, 0xed0: 0x2c5d, 0xed1: 0x2c5d, + 0xed2: 0x2a7d, 0xed3: 0x2a7d, 0xed4: 0x2c5d, 0xed5: 0x2c5d, 0xed6: 0x2c7d, 0xed7: 0x2c7d, + 0xed8: 0x2c5d, 0xed9: 0x2c5d, 0xeda: 0x2a7d, 0xedb: 0x2a7d, 0xedc: 0x2c5d, 0xedd: 0x2c5d, + 0xede: 0x2c3d, 0xedf: 0x2c3d, 0xee0: 0x2c9d, 0xee1: 0x2c9d, 0xee2: 0x2cbd, 0xee3: 0x2cbd, + 0xee4: 0x0040, 0xee5: 0x2cdd, 0xee6: 0x2cfd, 0xee7: 0x2d1d, 0xee8: 0x2d1d, 0xee9: 0x2d3d, + 0xeea: 0x2d5d, 0xeeb: 0x2d7d, 0xeec: 0x2d9d, 0xeed: 0x2dbd, 0xeee: 0x2ddd, 0xeef: 0x2dfd, + 0xef0: 0x2e1d, 0xef1: 0x2e3d, 0xef2: 0x2e3d, 0xef3: 0x2e5d, 0xef4: 0x2e7d, 0xef5: 0x2e7d, + 0xef6: 0x2e9d, 0xef7: 0x2ebd, 0xef8: 0x2e5d, 0xef9: 0x2edd, 0xefa: 0x2efd, 0xefb: 0x2edd, + 0xefc: 0x2e5d, 0xefd: 0x2f1d, 0xefe: 0x2f3d, 0xeff: 0x2f5d, // Block 0x3c, offset 0xf00 - 0xf00: 0x36dd, 0xf01: 0x36fd, 0xf02: 0x371d, 0xf03: 0x373d, 0xf04: 0x375d, 0xf05: 0x377d, - 0xf06: 0x379d, 0xf07: 0x37bd, 0xf08: 0x37dd, 0xf09: 0x37fd, 0xf0a: 0x381d, 0xf0b: 0x383d, - 0xf0c: 0x385d, 0xf0d: 0x387d, 0xf0e: 0x389d, 0xf0f: 0x38bd, 0xf10: 0x38dd, 0xf11: 0x38fd, - 0xf12: 0x391d, 0xf13: 0x393d, 0xf14: 0x395d, 0xf15: 0x397d, 0xf16: 0x399d, 0xf17: 0x39bd, - 0xf18: 0x39dd, 0xf19: 0x39fd, 0xf1a: 0x3a1d, 0xf1b: 0x3a3d, 0xf1c: 0x3a5d, 0xf1d: 0x3a7d, - 0xf1e: 0x3a9d, 0xf1f: 0x3abd, 0xf20: 0x3add, 0xf21: 0x3afd, 0xf22: 0x3b1d, 0xf23: 0x3b3d, - 0xf24: 0x3b5d, 0xf25: 0x3b7d, 0xf26: 0x127d, 0xf27: 0x3b9d, 0xf28: 0x3bbd, 0xf29: 0x3bdd, - 0xf2a: 0x3bfd, 0xf2b: 0x3c1d, 0xf2c: 0x3c3d, 0xf2d: 0x3c5d, 0xf2e: 0x239d, 0xf2f: 0x3c7d, - 0xf30: 0x3c9d, 0xf31: 0x3939, 0xf32: 0x3951, 0xf33: 0x3969, 0xf34: 0x3981, 0xf35: 0x3999, - 0xf36: 0x39b1, 0xf37: 0x39c9, 0xf38: 0x39e1, 0xf39: 0x39f9, 0xf3a: 0x3a11, 0xf3b: 0x3a29, - 0xf3c: 0x3a41, 0xf3d: 0x3a59, 0xf3e: 0x3a71, 0xf3f: 0x3a89, + 0xf00: 0x2f7d, 0xf01: 0x2f9d, 0xf02: 0x2cfd, 0xf03: 0x2cdd, 0xf04: 0x2fbd, 0xf05: 0x2fdd, + 0xf06: 0x2ffd, 0xf07: 0x301d, 0xf08: 0x303d, 0xf09: 0x305d, 0xf0a: 0x307d, 0xf0b: 0x309d, + 0xf0c: 0x30bd, 0xf0d: 0x30dd, 0xf0e: 0x30fd, 0xf0f: 0x0040, 0xf10: 0x0018, 0xf11: 0x0018, + 0xf12: 0x311d, 0xf13: 0x313d, 0xf14: 0x315d, 0xf15: 0x317d, 0xf16: 0x319d, 0xf17: 0x31bd, + 0xf18: 0x31dd, 0xf19: 0x31fd, 0xf1a: 0x321d, 0xf1b: 0x323d, 0xf1c: 0x315d, 0xf1d: 0x325d, + 0xf1e: 0x327d, 0xf1f: 0x329d, 0xf20: 0x0008, 0xf21: 0x0008, 0xf22: 0x0008, 0xf23: 0x0008, + 0xf24: 0x0008, 0xf25: 0x0008, 0xf26: 0x0008, 0xf27: 0x0008, 0xf28: 0x0008, 0xf29: 0x0008, + 0xf2a: 0x0008, 0xf2b: 0x0008, 0xf2c: 0x0008, 0xf2d: 0x0008, 0xf2e: 0x0008, 0xf2f: 0x0008, + 0xf30: 0x0008, 0xf31: 0x0008, 0xf32: 0x0008, 0xf33: 0x0008, 0xf34: 0x0008, 0xf35: 0x0008, + 0xf36: 0x0008, 0xf37: 0x0008, 0xf38: 0x0008, 0xf39: 0x0008, 0xf3a: 0x0008, 0xf3b: 0x0040, + 0xf3c: 0x0040, 0xf3d: 0x0040, 0xf3e: 0x0040, 0xf3f: 0x0040, // Block 0x3d, offset 0xf40 - 0xf40: 0x3aa1, 0xf41: 0x3ac9, 0xf42: 0x3af1, 0xf43: 0x3b19, 0xf44: 0x3b41, 0xf45: 0x3b69, - 0xf46: 0x3b91, 0xf47: 0x3bb9, 0xf48: 0x3be1, 0xf49: 0x3c09, 0xf4a: 0x3c39, 0xf4b: 0x3c69, - 0xf4c: 0x3c99, 0xf4d: 0x3cbd, 0xf4e: 0x3cb1, 0xf4f: 0x3cdd, 0xf50: 0x3cfd, 0xf51: 0x3d15, - 0xf52: 0x3d2d, 0xf53: 0x3d45, 0xf54: 0x3d5d, 0xf55: 0x3d5d, 0xf56: 0x3d45, 0xf57: 0x3d75, - 0xf58: 0x07bd, 0xf59: 0x3d8d, 0xf5a: 0x3da5, 0xf5b: 0x3dbd, 0xf5c: 0x3dd5, 0xf5d: 0x3ded, - 0xf5e: 0x3e05, 0xf5f: 0x3e1d, 0xf60: 0x3e35, 0xf61: 0x3e4d, 0xf62: 0x3e65, 0xf63: 0x3e7d, - 0xf64: 0x3e95, 0xf65: 0x3e95, 0xf66: 0x3ead, 0xf67: 0x3ead, 0xf68: 0x3ec5, 0xf69: 0x3ec5, - 0xf6a: 0x3edd, 0xf6b: 0x3ef5, 0xf6c: 0x3f0d, 0xf6d: 0x3f25, 0xf6e: 0x3f3d, 0xf6f: 0x3f3d, - 0xf70: 0x3f55, 0xf71: 0x3f55, 0xf72: 0x3f55, 0xf73: 0x3f6d, 0xf74: 0x3f85, 0xf75: 0x3f9d, - 0xf76: 0x3fb5, 0xf77: 0x3f9d, 0xf78: 0x3fcd, 0xf79: 0x3fe5, 0xf7a: 0x3f6d, 0xf7b: 0x3ffd, - 0xf7c: 0x4015, 0xf7d: 0x4015, 0xf7e: 0x4015, 0xf7f: 0x0040, + 0xf40: 0x36a2, 0xf41: 0x36d2, 0xf42: 0x3702, 0xf43: 0x3732, 0xf44: 0x32bd, 0xf45: 0x32dd, + 0xf46: 0x32fd, 0xf47: 0x331d, 0xf48: 0x0018, 0xf49: 0x0018, 0xf4a: 0x0018, 0xf4b: 0x0018, + 0xf4c: 0x0018, 0xf4d: 0x0018, 0xf4e: 0x0018, 0xf4f: 0x0018, 0xf50: 0x333d, 0xf51: 0x3761, + 0xf52: 0x3779, 0xf53: 0x3791, 0xf54: 0x37a9, 0xf55: 0x37c1, 0xf56: 0x37d9, 0xf57: 0x37f1, + 0xf58: 0x3809, 0xf59: 0x3821, 0xf5a: 0x3839, 0xf5b: 0x3851, 0xf5c: 0x3869, 0xf5d: 0x3881, + 0xf5e: 0x3899, 0xf5f: 0x38b1, 0xf60: 0x335d, 0xf61: 0x337d, 0xf62: 0x339d, 0xf63: 0x33bd, + 0xf64: 0x33dd, 0xf65: 0x33dd, 0xf66: 0x33fd, 0xf67: 0x341d, 0xf68: 0x343d, 0xf69: 0x345d, + 0xf6a: 0x347d, 0xf6b: 0x349d, 0xf6c: 0x34bd, 0xf6d: 0x34dd, 0xf6e: 0x34fd, 0xf6f: 0x351d, + 0xf70: 0x353d, 0xf71: 0x355d, 0xf72: 0x357d, 0xf73: 0x359d, 0xf74: 0x35bd, 0xf75: 0x35dd, + 0xf76: 0x35fd, 0xf77: 0x361d, 0xf78: 0x363d, 0xf79: 0x365d, 0xf7a: 0x367d, 0xf7b: 0x369d, + 0xf7c: 0x38c9, 0xf7d: 0x3901, 0xf7e: 0x36bd, 0xf7f: 0x0018, // Block 0x3e, offset 0xf80 - 0xf80: 0x3cc9, 0xf81: 0x3d31, 0xf82: 0x3d99, 0xf83: 0x3e01, 0xf84: 0x3e51, 0xf85: 0x3eb9, - 0xf86: 0x3f09, 0xf87: 0x3f59, 0xf88: 0x3fd9, 0xf89: 0x4041, 0xf8a: 0x4091, 0xf8b: 0x40e1, - 0xf8c: 0x4131, 0xf8d: 0x4199, 0xf8e: 0x4201, 0xf8f: 0x4251, 0xf90: 0x42a1, 0xf91: 0x42d9, - 0xf92: 0x4329, 0xf93: 0x4391, 0xf94: 0x43f9, 0xf95: 0x4431, 0xf96: 0x44b1, 0xf97: 0x4549, - 0xf98: 0x45c9, 0xf99: 0x4619, 0xf9a: 0x4699, 0xf9b: 0x4719, 0xf9c: 0x4781, 0xf9d: 0x47d1, - 0xf9e: 0x4821, 0xf9f: 0x4871, 0xfa0: 0x48d9, 0xfa1: 0x4959, 0xfa2: 0x49c1, 0xfa3: 0x4a11, - 0xfa4: 0x4a61, 0xfa5: 0x4ab1, 0xfa6: 0x4ae9, 0xfa7: 0x4b21, 0xfa8: 0x4b59, 0xfa9: 0x4b91, - 0xfaa: 0x4be1, 0xfab: 0x4c31, 0xfac: 0x4cb1, 0xfad: 0x4d01, 0xfae: 0x4d69, 0xfaf: 0x4de9, - 0xfb0: 0x4e39, 0xfb1: 0x4e71, 0xfb2: 0x4ea9, 0xfb3: 0x4f29, 0xfb4: 0x4f91, 0xfb5: 0x5011, - 0xfb6: 0x5061, 0xfb7: 0x50e1, 0xfb8: 0x5119, 0xfb9: 0x5169, 0xfba: 0x51b9, 0xfbb: 0x5209, - 0xfbc: 0x5259, 0xfbd: 0x52a9, 0xfbe: 0x5311, 0xfbf: 0x5361, + 0xf80: 0x36dd, 0xf81: 0x36fd, 0xf82: 0x371d, 0xf83: 0x373d, 0xf84: 0x375d, 0xf85: 0x377d, + 0xf86: 0x379d, 0xf87: 0x37bd, 0xf88: 0x37dd, 0xf89: 0x37fd, 0xf8a: 0x381d, 0xf8b: 0x383d, + 0xf8c: 0x385d, 0xf8d: 0x387d, 0xf8e: 0x389d, 0xf8f: 0x38bd, 0xf90: 0x38dd, 0xf91: 0x38fd, + 0xf92: 0x391d, 0xf93: 0x393d, 0xf94: 0x395d, 0xf95: 0x397d, 0xf96: 0x399d, 0xf97: 0x39bd, + 0xf98: 0x39dd, 0xf99: 0x39fd, 0xf9a: 0x3a1d, 0xf9b: 0x3a3d, 0xf9c: 0x3a5d, 0xf9d: 0x3a7d, + 0xf9e: 0x3a9d, 0xf9f: 0x3abd, 0xfa0: 0x3add, 0xfa1: 0x3afd, 0xfa2: 0x3b1d, 0xfa3: 0x3b3d, + 0xfa4: 0x3b5d, 0xfa5: 0x3b7d, 0xfa6: 0x127d, 0xfa7: 0x3b9d, 0xfa8: 0x3bbd, 0xfa9: 0x3bdd, + 0xfaa: 0x3bfd, 0xfab: 0x3c1d, 0xfac: 0x3c3d, 0xfad: 0x3c5d, 0xfae: 0x239d, 0xfaf: 0x3c7d, + 0xfb0: 0x3c9d, 0xfb1: 0x3939, 0xfb2: 0x3951, 0xfb3: 0x3969, 0xfb4: 0x3981, 0xfb5: 0x3999, + 0xfb6: 0x39b1, 0xfb7: 0x39c9, 0xfb8: 0x39e1, 0xfb9: 0x39f9, 0xfba: 0x3a11, 0xfbb: 0x3a29, + 0xfbc: 0x3a41, 0xfbd: 0x3a59, 0xfbe: 0x3a71, 0xfbf: 0x3a89, // Block 0x3f, offset 0xfc0 - 0xfc0: 0x5399, 0xfc1: 0x53e9, 0xfc2: 0x5439, 0xfc3: 0x5489, 0xfc4: 0x54f1, 0xfc5: 0x5541, - 0xfc6: 0x5591, 0xfc7: 0x55e1, 0xfc8: 0x5661, 0xfc9: 0x56c9, 0xfca: 0x5701, 0xfcb: 0x5781, - 0xfcc: 0x57b9, 0xfcd: 0x5821, 0xfce: 0x5889, 0xfcf: 0x58d9, 0xfd0: 0x5929, 0xfd1: 0x5979, - 0xfd2: 0x59e1, 0xfd3: 0x5a19, 0xfd4: 0x5a69, 0xfd5: 0x5ad1, 0xfd6: 0x5b09, 0xfd7: 0x5b89, - 0xfd8: 0x5bd9, 0xfd9: 0x5c01, 0xfda: 0x5c29, 0xfdb: 0x5c51, 0xfdc: 0x5c79, 0xfdd: 0x5ca1, - 0xfde: 0x5cc9, 0xfdf: 0x5cf1, 0xfe0: 0x5d19, 0xfe1: 0x5d41, 0xfe2: 0x5d69, 0xfe3: 0x5d99, - 0xfe4: 0x5dc9, 0xfe5: 0x5df9, 0xfe6: 0x5e29, 0xfe7: 0x5e59, 0xfe8: 0x5e89, 0xfe9: 0x5eb9, - 0xfea: 0x5ee9, 0xfeb: 0x5f19, 0xfec: 0x5f49, 0xfed: 0x5f79, 0xfee: 0x5fa9, 0xfef: 0x5fd9, - 0xff0: 0x6009, 0xff1: 0x402d, 0xff2: 0x6039, 0xff3: 0x6051, 0xff4: 0x404d, 0xff5: 0x6069, - 0xff6: 0x6081, 0xff7: 0x6099, 0xff8: 0x406d, 0xff9: 0x406d, 0xffa: 0x60b1, 0xffb: 0x60c9, - 0xffc: 0x6101, 0xffd: 0x6139, 0xffe: 0x6171, 0xfff: 0x61a9, + 0xfc0: 0x3aa1, 0xfc1: 0x3ac9, 0xfc2: 0x3af1, 0xfc3: 0x3b19, 0xfc4: 0x3b41, 0xfc5: 0x3b69, + 0xfc6: 0x3b91, 0xfc7: 0x3bb9, 0xfc8: 0x3be1, 0xfc9: 0x3c09, 0xfca: 0x3c39, 0xfcb: 0x3c69, + 0xfcc: 0x3c99, 0xfcd: 0x3cbd, 0xfce: 0x3cb1, 0xfcf: 0x3cdd, 0xfd0: 0x3cfd, 0xfd1: 0x3d15, + 0xfd2: 0x3d2d, 0xfd3: 0x3d45, 0xfd4: 0x3d5d, 0xfd5: 0x3d5d, 0xfd6: 0x3d45, 0xfd7: 0x3d75, + 0xfd8: 0x07bd, 0xfd9: 0x3d8d, 0xfda: 0x3da5, 0xfdb: 0x3dbd, 0xfdc: 0x3dd5, 0xfdd: 0x3ded, + 0xfde: 0x3e05, 0xfdf: 0x3e1d, 0xfe0: 0x3e35, 0xfe1: 0x3e4d, 0xfe2: 0x3e65, 0xfe3: 0x3e7d, + 0xfe4: 0x3e95, 0xfe5: 0x3e95, 0xfe6: 0x3ead, 0xfe7: 0x3ead, 0xfe8: 0x3ec5, 0xfe9: 0x3ec5, + 0xfea: 0x3edd, 0xfeb: 0x3ef5, 0xfec: 0x3f0d, 0xfed: 0x3f25, 0xfee: 0x3f3d, 0xfef: 0x3f3d, + 0xff0: 0x3f55, 0xff1: 0x3f55, 0xff2: 0x3f55, 0xff3: 0x3f6d, 0xff4: 0x3f85, 0xff5: 0x3f9d, + 0xff6: 0x3fb5, 0xff7: 0x3f9d, 0xff8: 0x3fcd, 0xff9: 0x3fe5, 0xffa: 0x3f6d, 0xffb: 0x3ffd, + 0xffc: 0x4015, 0xffd: 0x4015, 0xffe: 0x4015, 0xfff: 0x0040, // Block 0x40, offset 0x1000 - 0x1000: 0x6211, 0x1001: 0x6229, 0x1002: 0x408d, 0x1003: 0x6241, 0x1004: 0x6259, 0x1005: 0x6271, - 0x1006: 0x6289, 0x1007: 0x62a1, 0x1008: 0x40ad, 0x1009: 0x62b9, 0x100a: 0x62e1, 0x100b: 0x62f9, - 0x100c: 0x40cd, 0x100d: 0x40cd, 0x100e: 0x6311, 0x100f: 0x6329, 0x1010: 0x6341, 0x1011: 0x40ed, - 0x1012: 0x410d, 0x1013: 0x412d, 0x1014: 0x414d, 0x1015: 0x416d, 0x1016: 0x6359, 0x1017: 0x6371, - 0x1018: 0x6389, 0x1019: 0x63a1, 0x101a: 0x63b9, 0x101b: 0x418d, 0x101c: 0x63d1, 0x101d: 0x63e9, - 0x101e: 0x6401, 0x101f: 0x41ad, 0x1020: 0x41cd, 0x1021: 0x6419, 0x1022: 0x41ed, 0x1023: 0x420d, - 0x1024: 0x422d, 0x1025: 0x6431, 0x1026: 0x424d, 0x1027: 0x6449, 0x1028: 0x6479, 0x1029: 0x6211, - 0x102a: 0x426d, 0x102b: 0x428d, 0x102c: 0x42ad, 0x102d: 0x42cd, 0x102e: 0x64b1, 0x102f: 0x64f1, - 0x1030: 0x6539, 0x1031: 0x6551, 0x1032: 0x42ed, 0x1033: 0x6569, 0x1034: 0x6581, 0x1035: 0x6599, - 0x1036: 0x430d, 0x1037: 0x65b1, 0x1038: 0x65c9, 0x1039: 0x65b1, 0x103a: 0x65e1, 0x103b: 0x65f9, - 0x103c: 0x432d, 0x103d: 0x6611, 0x103e: 0x6629, 0x103f: 0x6611, + 0x1000: 0x3cc9, 0x1001: 0x3d31, 0x1002: 0x3d99, 0x1003: 0x3e01, 0x1004: 0x3e51, 0x1005: 0x3eb9, + 0x1006: 0x3f09, 0x1007: 0x3f59, 0x1008: 0x3fd9, 0x1009: 0x4041, 0x100a: 0x4091, 0x100b: 0x40e1, + 0x100c: 0x4131, 0x100d: 0x4199, 0x100e: 0x4201, 0x100f: 0x4251, 0x1010: 0x42a1, 0x1011: 0x42d9, + 0x1012: 0x4329, 0x1013: 0x4391, 0x1014: 0x43f9, 0x1015: 0x4431, 0x1016: 0x44b1, 0x1017: 0x4549, + 0x1018: 0x45c9, 0x1019: 0x4619, 0x101a: 0x4699, 0x101b: 0x4719, 0x101c: 0x4781, 0x101d: 0x47d1, + 0x101e: 0x4821, 0x101f: 0x4871, 0x1020: 0x48d9, 0x1021: 0x4959, 0x1022: 0x49c1, 0x1023: 0x4a11, + 0x1024: 0x4a61, 0x1025: 0x4ab1, 0x1026: 0x4ae9, 0x1027: 0x4b21, 0x1028: 0x4b59, 0x1029: 0x4b91, + 0x102a: 0x4be1, 0x102b: 0x4c31, 0x102c: 0x4cb1, 0x102d: 0x4d01, 0x102e: 0x4d69, 0x102f: 0x4de9, + 0x1030: 0x4e39, 0x1031: 0x4e71, 0x1032: 0x4ea9, 0x1033: 0x4f29, 0x1034: 0x4f91, 0x1035: 0x5011, + 0x1036: 0x5061, 0x1037: 0x50e1, 0x1038: 0x5119, 0x1039: 0x5169, 0x103a: 0x51b9, 0x103b: 0x5209, + 0x103c: 0x5259, 0x103d: 0x52a9, 0x103e: 0x5311, 0x103f: 0x5361, // Block 0x41, offset 0x1040 - 0x1040: 0x434d, 0x1041: 0x436d, 0x1042: 0x0040, 0x1043: 0x6641, 0x1044: 0x6659, 0x1045: 0x6671, - 0x1046: 0x6689, 0x1047: 0x0040, 0x1048: 0x66c1, 0x1049: 0x66d9, 0x104a: 0x66f1, 0x104b: 0x6709, - 0x104c: 0x6721, 0x104d: 0x6739, 0x104e: 0x6401, 0x104f: 0x6751, 0x1050: 0x6769, 0x1051: 0x6781, - 0x1052: 0x438d, 0x1053: 0x6799, 0x1054: 0x6289, 0x1055: 0x43ad, 0x1056: 0x43cd, 0x1057: 0x67b1, - 0x1058: 0x0040, 0x1059: 0x43ed, 0x105a: 0x67c9, 0x105b: 0x67e1, 0x105c: 0x67f9, 0x105d: 0x6811, - 0x105e: 0x6829, 0x105f: 0x6859, 0x1060: 0x6889, 0x1061: 0x68b1, 0x1062: 0x68d9, 0x1063: 0x6901, - 0x1064: 0x6929, 0x1065: 0x6951, 0x1066: 0x6979, 0x1067: 0x69a1, 0x1068: 0x69c9, 0x1069: 0x69f1, - 0x106a: 0x6a21, 0x106b: 0x6a51, 0x106c: 0x6a81, 0x106d: 0x6ab1, 0x106e: 0x6ae1, 0x106f: 0x6b11, - 0x1070: 0x6b41, 0x1071: 0x6b71, 0x1072: 0x6ba1, 0x1073: 0x6bd1, 0x1074: 0x6c01, 0x1075: 0x6c31, - 0x1076: 0x6c61, 0x1077: 0x6c91, 0x1078: 0x6cc1, 0x1079: 0x6cf1, 0x107a: 0x6d21, 0x107b: 0x6d51, - 0x107c: 0x6d81, 0x107d: 0x6db1, 0x107e: 0x6de1, 0x107f: 0x440d, + 0x1040: 0x5399, 0x1041: 0x53e9, 0x1042: 0x5439, 0x1043: 0x5489, 0x1044: 0x54f1, 0x1045: 0x5541, + 0x1046: 0x5591, 0x1047: 0x55e1, 0x1048: 0x5661, 0x1049: 0x56c9, 0x104a: 0x5701, 0x104b: 0x5781, + 0x104c: 0x57b9, 0x104d: 0x5821, 0x104e: 0x5889, 0x104f: 0x58d9, 0x1050: 0x5929, 0x1051: 0x5979, + 0x1052: 0x59e1, 0x1053: 0x5a19, 0x1054: 0x5a69, 0x1055: 0x5ad1, 0x1056: 0x5b09, 0x1057: 0x5b89, + 0x1058: 0x5bd9, 0x1059: 0x5c01, 0x105a: 0x5c29, 0x105b: 0x5c51, 0x105c: 0x5c79, 0x105d: 0x5ca1, + 0x105e: 0x5cc9, 0x105f: 0x5cf1, 0x1060: 0x5d19, 0x1061: 0x5d41, 0x1062: 0x5d69, 0x1063: 0x5d99, + 0x1064: 0x5dc9, 0x1065: 0x5df9, 0x1066: 0x5e29, 0x1067: 0x5e59, 0x1068: 0x5e89, 0x1069: 0x5eb9, + 0x106a: 0x5ee9, 0x106b: 0x5f19, 0x106c: 0x5f49, 0x106d: 0x5f79, 0x106e: 0x5fa9, 0x106f: 0x5fd9, + 0x1070: 0x6009, 0x1071: 0x402d, 0x1072: 0x6039, 0x1073: 0x6051, 0x1074: 0x404d, 0x1075: 0x6069, + 0x1076: 0x6081, 0x1077: 0x6099, 0x1078: 0x406d, 0x1079: 0x406d, 0x107a: 0x60b1, 0x107b: 0x60c9, + 0x107c: 0x6101, 0x107d: 0x6139, 0x107e: 0x6171, 0x107f: 0x61a9, // Block 0x42, offset 0x1080 - 0x1080: 0xe00d, 0x1081: 0x0008, 0x1082: 0xe00d, 0x1083: 0x0008, 0x1084: 0xe00d, 0x1085: 0x0008, - 0x1086: 0xe00d, 0x1087: 0x0008, 0x1088: 0xe00d, 0x1089: 0x0008, 0x108a: 0xe00d, 0x108b: 0x0008, - 0x108c: 0xe00d, 0x108d: 0x0008, 0x108e: 0xe00d, 0x108f: 0x0008, 0x1090: 0xe00d, 0x1091: 0x0008, - 0x1092: 0xe00d, 0x1093: 0x0008, 0x1094: 0xe00d, 0x1095: 0x0008, 0x1096: 0xe00d, 0x1097: 0x0008, - 0x1098: 0xe00d, 0x1099: 0x0008, 0x109a: 0xe00d, 0x109b: 0x0008, 0x109c: 0xe00d, 0x109d: 0x0008, - 0x109e: 0xe00d, 0x109f: 0x0008, 0x10a0: 0xe00d, 0x10a1: 0x0008, 0x10a2: 0xe00d, 0x10a3: 0x0008, - 0x10a4: 0xe00d, 0x10a5: 0x0008, 0x10a6: 0xe00d, 0x10a7: 0x0008, 0x10a8: 0xe00d, 0x10a9: 0x0008, - 0x10aa: 0xe00d, 0x10ab: 0x0008, 0x10ac: 0xe00d, 0x10ad: 0x0008, 0x10ae: 0x0008, 0x10af: 0x1308, - 0x10b0: 0x1318, 0x10b1: 0x1318, 0x10b2: 0x1318, 0x10b3: 0x0018, 0x10b4: 0x1308, 0x10b5: 0x1308, - 0x10b6: 0x1308, 0x10b7: 0x1308, 0x10b8: 0x1308, 0x10b9: 0x1308, 0x10ba: 0x1308, 0x10bb: 0x1308, - 0x10bc: 0x1308, 0x10bd: 0x1308, 0x10be: 0x0018, 0x10bf: 0x0008, + 0x1080: 0x6211, 0x1081: 0x6229, 0x1082: 0x408d, 0x1083: 0x6241, 0x1084: 0x6259, 0x1085: 0x6271, + 0x1086: 0x6289, 0x1087: 0x62a1, 0x1088: 0x40ad, 0x1089: 0x62b9, 0x108a: 0x62e1, 0x108b: 0x62f9, + 0x108c: 0x40cd, 0x108d: 0x40cd, 0x108e: 0x6311, 0x108f: 0x6329, 0x1090: 0x6341, 0x1091: 0x40ed, + 0x1092: 0x410d, 0x1093: 0x412d, 0x1094: 0x414d, 0x1095: 0x416d, 0x1096: 0x6359, 0x1097: 0x6371, + 0x1098: 0x6389, 0x1099: 0x63a1, 0x109a: 0x63b9, 0x109b: 0x418d, 0x109c: 0x63d1, 0x109d: 0x63e9, + 0x109e: 0x6401, 0x109f: 0x41ad, 0x10a0: 0x41cd, 0x10a1: 0x6419, 0x10a2: 0x41ed, 0x10a3: 0x420d, + 0x10a4: 0x422d, 0x10a5: 0x6431, 0x10a6: 0x424d, 0x10a7: 0x6449, 0x10a8: 0x6479, 0x10a9: 0x6211, + 0x10aa: 0x426d, 0x10ab: 0x428d, 0x10ac: 0x42ad, 0x10ad: 0x42cd, 0x10ae: 0x64b1, 0x10af: 0x64f1, + 0x10b0: 0x6539, 0x10b1: 0x6551, 0x10b2: 0x42ed, 0x10b3: 0x6569, 0x10b4: 0x6581, 0x10b5: 0x6599, + 0x10b6: 0x430d, 0x10b7: 0x65b1, 0x10b8: 0x65c9, 0x10b9: 0x65b1, 0x10ba: 0x65e1, 0x10bb: 0x65f9, + 0x10bc: 0x432d, 0x10bd: 0x6611, 0x10be: 0x6629, 0x10bf: 0x6611, // Block 0x43, offset 0x10c0 - 0x10c0: 0xe00d, 0x10c1: 0x0008, 0x10c2: 0xe00d, 0x10c3: 0x0008, 0x10c4: 0xe00d, 0x10c5: 0x0008, - 0x10c6: 0xe00d, 0x10c7: 0x0008, 0x10c8: 0xe00d, 0x10c9: 0x0008, 0x10ca: 0xe00d, 0x10cb: 0x0008, - 0x10cc: 0xe00d, 0x10cd: 0x0008, 0x10ce: 0xe00d, 0x10cf: 0x0008, 0x10d0: 0xe00d, 0x10d1: 0x0008, - 0x10d2: 0xe00d, 0x10d3: 0x0008, 0x10d4: 0xe00d, 0x10d5: 0x0008, 0x10d6: 0xe00d, 0x10d7: 0x0008, - 0x10d8: 0xe00d, 0x10d9: 0x0008, 0x10da: 0xe00d, 0x10db: 0x0008, 0x10dc: 0x0ea1, 0x10dd: 0x6e11, - 0x10de: 0x1308, 0x10df: 0x1308, 0x10e0: 0x0008, 0x10e1: 0x0008, 0x10e2: 0x0008, 0x10e3: 0x0008, - 0x10e4: 0x0008, 0x10e5: 0x0008, 0x10e6: 0x0008, 0x10e7: 0x0008, 0x10e8: 0x0008, 0x10e9: 0x0008, - 0x10ea: 0x0008, 0x10eb: 0x0008, 0x10ec: 0x0008, 0x10ed: 0x0008, 0x10ee: 0x0008, 0x10ef: 0x0008, - 0x10f0: 0x0008, 0x10f1: 0x0008, 0x10f2: 0x0008, 0x10f3: 0x0008, 0x10f4: 0x0008, 0x10f5: 0x0008, - 0x10f6: 0x0008, 0x10f7: 0x0008, 0x10f8: 0x0008, 0x10f9: 0x0008, 0x10fa: 0x0008, 0x10fb: 0x0008, - 0x10fc: 0x0008, 0x10fd: 0x0008, 0x10fe: 0x0008, 0x10ff: 0x0008, + 0x10c0: 0x434d, 0x10c1: 0x436d, 0x10c2: 0x0040, 0x10c3: 0x6641, 0x10c4: 0x6659, 0x10c5: 0x6671, + 0x10c6: 0x6689, 0x10c7: 0x0040, 0x10c8: 0x66c1, 0x10c9: 0x66d9, 0x10ca: 0x66f1, 0x10cb: 0x6709, + 0x10cc: 0x6721, 0x10cd: 0x6739, 0x10ce: 0x6401, 0x10cf: 0x6751, 0x10d0: 0x6769, 0x10d1: 0x6781, + 0x10d2: 0x438d, 0x10d3: 0x6799, 0x10d4: 0x6289, 0x10d5: 0x43ad, 0x10d6: 0x43cd, 0x10d7: 0x67b1, + 0x10d8: 0x0040, 0x10d9: 0x43ed, 0x10da: 0x67c9, 0x10db: 0x67e1, 0x10dc: 0x67f9, 0x10dd: 0x6811, + 0x10de: 0x6829, 0x10df: 0x6859, 0x10e0: 0x6889, 0x10e1: 0x68b1, 0x10e2: 0x68d9, 0x10e3: 0x6901, + 0x10e4: 0x6929, 0x10e5: 0x6951, 0x10e6: 0x6979, 0x10e7: 0x69a1, 0x10e8: 0x69c9, 0x10e9: 0x69f1, + 0x10ea: 0x6a21, 0x10eb: 0x6a51, 0x10ec: 0x6a81, 0x10ed: 0x6ab1, 0x10ee: 0x6ae1, 0x10ef: 0x6b11, + 0x10f0: 0x6b41, 0x10f1: 0x6b71, 0x10f2: 0x6ba1, 0x10f3: 0x6bd1, 0x10f4: 0x6c01, 0x10f5: 0x6c31, + 0x10f6: 0x6c61, 0x10f7: 0x6c91, 0x10f8: 0x6cc1, 0x10f9: 0x6cf1, 0x10fa: 0x6d21, 0x10fb: 0x6d51, + 0x10fc: 0x6d81, 0x10fd: 0x6db1, 0x10fe: 0x6de1, 0x10ff: 0x440d, // Block 0x44, offset 0x1100 - 0x1100: 0x0018, 0x1101: 0x0018, 0x1102: 0x0018, 0x1103: 0x0018, 0x1104: 0x0018, 0x1105: 0x0018, - 0x1106: 0x0018, 0x1107: 0x0018, 0x1108: 0x0018, 0x1109: 0x0018, 0x110a: 0x0018, 0x110b: 0x0018, - 0x110c: 0x0018, 0x110d: 0x0018, 0x110e: 0x0018, 0x110f: 0x0018, 0x1110: 0x0018, 0x1111: 0x0018, - 0x1112: 0x0018, 0x1113: 0x0018, 0x1114: 0x0018, 0x1115: 0x0018, 0x1116: 0x0018, 0x1117: 0x0008, - 0x1118: 0x0008, 0x1119: 0x0008, 0x111a: 0x0008, 0x111b: 0x0008, 0x111c: 0x0008, 0x111d: 0x0008, - 0x111e: 0x0008, 0x111f: 0x0008, 0x1120: 0x0018, 0x1121: 0x0018, 0x1122: 0xe00d, 0x1123: 0x0008, + 0x1100: 0xe00d, 0x1101: 0x0008, 0x1102: 0xe00d, 0x1103: 0x0008, 0x1104: 0xe00d, 0x1105: 0x0008, + 0x1106: 0xe00d, 0x1107: 0x0008, 0x1108: 0xe00d, 0x1109: 0x0008, 0x110a: 0xe00d, 0x110b: 0x0008, + 0x110c: 0xe00d, 0x110d: 0x0008, 0x110e: 0xe00d, 0x110f: 0x0008, 0x1110: 0xe00d, 0x1111: 0x0008, + 0x1112: 0xe00d, 0x1113: 0x0008, 0x1114: 0xe00d, 0x1115: 0x0008, 0x1116: 0xe00d, 0x1117: 0x0008, + 0x1118: 0xe00d, 0x1119: 0x0008, 0x111a: 0xe00d, 0x111b: 0x0008, 0x111c: 0xe00d, 0x111d: 0x0008, + 0x111e: 0xe00d, 0x111f: 0x0008, 0x1120: 0xe00d, 0x1121: 0x0008, 0x1122: 0xe00d, 0x1123: 0x0008, 0x1124: 0xe00d, 0x1125: 0x0008, 0x1126: 0xe00d, 0x1127: 0x0008, 0x1128: 0xe00d, 0x1129: 0x0008, - 0x112a: 0xe00d, 0x112b: 0x0008, 0x112c: 0xe00d, 0x112d: 0x0008, 0x112e: 0xe00d, 0x112f: 0x0008, - 0x1130: 0x0008, 0x1131: 0x0008, 0x1132: 0xe00d, 0x1133: 0x0008, 0x1134: 0xe00d, 0x1135: 0x0008, - 0x1136: 0xe00d, 0x1137: 0x0008, 0x1138: 0xe00d, 0x1139: 0x0008, 0x113a: 0xe00d, 0x113b: 0x0008, - 0x113c: 0xe00d, 0x113d: 0x0008, 0x113e: 0xe00d, 0x113f: 0x0008, + 0x112a: 0xe00d, 0x112b: 0x0008, 0x112c: 0xe00d, 0x112d: 0x0008, 0x112e: 0x0008, 0x112f: 0x3308, + 0x1130: 0x3318, 0x1131: 0x3318, 0x1132: 0x3318, 0x1133: 0x0018, 0x1134: 0x3308, 0x1135: 0x3308, + 0x1136: 0x3308, 0x1137: 0x3308, 0x1138: 0x3308, 0x1139: 0x3308, 0x113a: 0x3308, 0x113b: 0x3308, + 0x113c: 0x3308, 0x113d: 0x3308, 0x113e: 0x0018, 0x113f: 0x0008, // Block 0x45, offset 0x1140 0x1140: 0xe00d, 0x1141: 0x0008, 0x1142: 0xe00d, 0x1143: 0x0008, 0x1144: 0xe00d, 0x1145: 0x0008, 0x1146: 0xe00d, 0x1147: 0x0008, 0x1148: 0xe00d, 0x1149: 0x0008, 0x114a: 0xe00d, 0x114b: 0x0008, 0x114c: 0xe00d, 0x114d: 0x0008, 0x114e: 0xe00d, 0x114f: 0x0008, 0x1150: 0xe00d, 0x1151: 0x0008, 0x1152: 0xe00d, 0x1153: 0x0008, 0x1154: 0xe00d, 0x1155: 0x0008, 0x1156: 0xe00d, 0x1157: 0x0008, - 0x1158: 0xe00d, 0x1159: 0x0008, 0x115a: 0xe00d, 0x115b: 0x0008, 0x115c: 0xe00d, 0x115d: 0x0008, - 0x115e: 0xe00d, 0x115f: 0x0008, 0x1160: 0xe00d, 0x1161: 0x0008, 0x1162: 0xe00d, 0x1163: 0x0008, - 0x1164: 0xe00d, 0x1165: 0x0008, 0x1166: 0xe00d, 0x1167: 0x0008, 0x1168: 0xe00d, 0x1169: 0x0008, - 0x116a: 0xe00d, 0x116b: 0x0008, 0x116c: 0xe00d, 0x116d: 0x0008, 0x116e: 0xe00d, 0x116f: 0x0008, - 0x1170: 0xe0fd, 0x1171: 0x0008, 0x1172: 0x0008, 0x1173: 0x0008, 0x1174: 0x0008, 0x1175: 0x0008, - 0x1176: 0x0008, 0x1177: 0x0008, 0x1178: 0x0008, 0x1179: 0xe01d, 0x117a: 0x0008, 0x117b: 0xe03d, - 0x117c: 0x0008, 0x117d: 0x442d, 0x117e: 0xe00d, 0x117f: 0x0008, + 0x1158: 0xe00d, 0x1159: 0x0008, 0x115a: 0xe00d, 0x115b: 0x0008, 0x115c: 0x0ea1, 0x115d: 0x6e11, + 0x115e: 0x3308, 0x115f: 0x3308, 0x1160: 0x0008, 0x1161: 0x0008, 0x1162: 0x0008, 0x1163: 0x0008, + 0x1164: 0x0008, 0x1165: 0x0008, 0x1166: 0x0008, 0x1167: 0x0008, 0x1168: 0x0008, 0x1169: 0x0008, + 0x116a: 0x0008, 0x116b: 0x0008, 0x116c: 0x0008, 0x116d: 0x0008, 0x116e: 0x0008, 0x116f: 0x0008, + 0x1170: 0x0008, 0x1171: 0x0008, 0x1172: 0x0008, 0x1173: 0x0008, 0x1174: 0x0008, 0x1175: 0x0008, + 0x1176: 0x0008, 0x1177: 0x0008, 0x1178: 0x0008, 0x1179: 0x0008, 0x117a: 0x0008, 0x117b: 0x0008, + 0x117c: 0x0008, 0x117d: 0x0008, 0x117e: 0x0008, 0x117f: 0x0008, // Block 0x46, offset 0x1180 - 0x1180: 0xe00d, 0x1181: 0x0008, 0x1182: 0xe00d, 0x1183: 0x0008, 0x1184: 0xe00d, 0x1185: 0x0008, - 0x1186: 0xe00d, 0x1187: 0x0008, 0x1188: 0x0008, 0x1189: 0x0018, 0x118a: 0x0018, 0x118b: 0xe03d, - 0x118c: 0x0008, 0x118d: 0x11d9, 0x118e: 0x0008, 0x118f: 0x0008, 0x1190: 0xe00d, 0x1191: 0x0008, - 0x1192: 0xe00d, 0x1193: 0x0008, 0x1194: 0x0008, 0x1195: 0x0008, 0x1196: 0xe00d, 0x1197: 0x0008, - 0x1198: 0xe00d, 0x1199: 0x0008, 0x119a: 0xe00d, 0x119b: 0x0008, 0x119c: 0xe00d, 0x119d: 0x0008, - 0x119e: 0xe00d, 0x119f: 0x0008, 0x11a0: 0xe00d, 0x11a1: 0x0008, 0x11a2: 0xe00d, 0x11a3: 0x0008, + 0x1180: 0x0018, 0x1181: 0x0018, 0x1182: 0x0018, 0x1183: 0x0018, 0x1184: 0x0018, 0x1185: 0x0018, + 0x1186: 0x0018, 0x1187: 0x0018, 0x1188: 0x0018, 0x1189: 0x0018, 0x118a: 0x0018, 0x118b: 0x0018, + 0x118c: 0x0018, 0x118d: 0x0018, 0x118e: 0x0018, 0x118f: 0x0018, 0x1190: 0x0018, 0x1191: 0x0018, + 0x1192: 0x0018, 0x1193: 0x0018, 0x1194: 0x0018, 0x1195: 0x0018, 0x1196: 0x0018, 0x1197: 0x0008, + 0x1198: 0x0008, 0x1199: 0x0008, 0x119a: 0x0008, 0x119b: 0x0008, 0x119c: 0x0008, 0x119d: 0x0008, + 0x119e: 0x0008, 0x119f: 0x0008, 0x11a0: 0x0018, 0x11a1: 0x0018, 0x11a2: 0xe00d, 0x11a3: 0x0008, 0x11a4: 0xe00d, 0x11a5: 0x0008, 0x11a6: 0xe00d, 0x11a7: 0x0008, 0x11a8: 0xe00d, 0x11a9: 0x0008, - 0x11aa: 0x6e29, 0x11ab: 0x1029, 0x11ac: 0x11c1, 0x11ad: 0x6e41, 0x11ae: 0x1221, 0x11af: 0x0040, - 0x11b0: 0x6e59, 0x11b1: 0x6e71, 0x11b2: 0x1239, 0x11b3: 0x444d, 0x11b4: 0xe00d, 0x11b5: 0x0008, - 0x11b6: 0xe00d, 0x11b7: 0x0008, 0x11b8: 0x0040, 0x11b9: 0x0040, 0x11ba: 0x0040, 0x11bb: 0x0040, - 0x11bc: 0x0040, 0x11bd: 0x0040, 0x11be: 0x0040, 0x11bf: 0x0040, + 0x11aa: 0xe00d, 0x11ab: 0x0008, 0x11ac: 0xe00d, 0x11ad: 0x0008, 0x11ae: 0xe00d, 0x11af: 0x0008, + 0x11b0: 0x0008, 0x11b1: 0x0008, 0x11b2: 0xe00d, 0x11b3: 0x0008, 0x11b4: 0xe00d, 0x11b5: 0x0008, + 0x11b6: 0xe00d, 0x11b7: 0x0008, 0x11b8: 0xe00d, 0x11b9: 0x0008, 0x11ba: 0xe00d, 0x11bb: 0x0008, + 0x11bc: 0xe00d, 0x11bd: 0x0008, 0x11be: 0xe00d, 0x11bf: 0x0008, // Block 0x47, offset 0x11c0 - 0x11c0: 0x64d5, 0x11c1: 0x64f5, 0x11c2: 0x6515, 0x11c3: 0x6535, 0x11c4: 0x6555, 0x11c5: 0x6575, - 0x11c6: 0x6595, 0x11c7: 0x65b5, 0x11c8: 0x65d5, 0x11c9: 0x65f5, 0x11ca: 0x6615, 0x11cb: 0x6635, - 0x11cc: 0x6655, 0x11cd: 0x6675, 0x11ce: 0x0008, 0x11cf: 0x0008, 0x11d0: 0x6695, 0x11d1: 0x0008, - 0x11d2: 0x66b5, 0x11d3: 0x0008, 0x11d4: 0x0008, 0x11d5: 0x66d5, 0x11d6: 0x66f5, 0x11d7: 0x6715, - 0x11d8: 0x6735, 0x11d9: 0x6755, 0x11da: 0x6775, 0x11db: 0x6795, 0x11dc: 0x67b5, 0x11dd: 0x67d5, - 0x11de: 0x67f5, 0x11df: 0x0008, 0x11e0: 0x6815, 0x11e1: 0x0008, 0x11e2: 0x6835, 0x11e3: 0x0008, - 0x11e4: 0x0008, 0x11e5: 0x6855, 0x11e6: 0x6875, 0x11e7: 0x0008, 0x11e8: 0x0008, 0x11e9: 0x0008, - 0x11ea: 0x6895, 0x11eb: 0x68b5, 0x11ec: 0x68d5, 0x11ed: 0x68f5, 0x11ee: 0x6915, 0x11ef: 0x6935, - 0x11f0: 0x6955, 0x11f1: 0x6975, 0x11f2: 0x6995, 0x11f3: 0x69b5, 0x11f4: 0x69d5, 0x11f5: 0x69f5, - 0x11f6: 0x6a15, 0x11f7: 0x6a35, 0x11f8: 0x6a55, 0x11f9: 0x6a75, 0x11fa: 0x6a95, 0x11fb: 0x6ab5, - 0x11fc: 0x6ad5, 0x11fd: 0x6af5, 0x11fe: 0x6b15, 0x11ff: 0x6b35, + 0x11c0: 0xe00d, 0x11c1: 0x0008, 0x11c2: 0xe00d, 0x11c3: 0x0008, 0x11c4: 0xe00d, 0x11c5: 0x0008, + 0x11c6: 0xe00d, 0x11c7: 0x0008, 0x11c8: 0xe00d, 0x11c9: 0x0008, 0x11ca: 0xe00d, 0x11cb: 0x0008, + 0x11cc: 0xe00d, 0x11cd: 0x0008, 0x11ce: 0xe00d, 0x11cf: 0x0008, 0x11d0: 0xe00d, 0x11d1: 0x0008, + 0x11d2: 0xe00d, 0x11d3: 0x0008, 0x11d4: 0xe00d, 0x11d5: 0x0008, 0x11d6: 0xe00d, 0x11d7: 0x0008, + 0x11d8: 0xe00d, 0x11d9: 0x0008, 0x11da: 0xe00d, 0x11db: 0x0008, 0x11dc: 0xe00d, 0x11dd: 0x0008, + 0x11de: 0xe00d, 0x11df: 0x0008, 0x11e0: 0xe00d, 0x11e1: 0x0008, 0x11e2: 0xe00d, 0x11e3: 0x0008, + 0x11e4: 0xe00d, 0x11e5: 0x0008, 0x11e6: 0xe00d, 0x11e7: 0x0008, 0x11e8: 0xe00d, 0x11e9: 0x0008, + 0x11ea: 0xe00d, 0x11eb: 0x0008, 0x11ec: 0xe00d, 0x11ed: 0x0008, 0x11ee: 0xe00d, 0x11ef: 0x0008, + 0x11f0: 0xe0fd, 0x11f1: 0x0008, 0x11f2: 0x0008, 0x11f3: 0x0008, 0x11f4: 0x0008, 0x11f5: 0x0008, + 0x11f6: 0x0008, 0x11f7: 0x0008, 0x11f8: 0x0008, 0x11f9: 0xe01d, 0x11fa: 0x0008, 0x11fb: 0xe03d, + 0x11fc: 0x0008, 0x11fd: 0x442d, 0x11fe: 0xe00d, 0x11ff: 0x0008, // Block 0x48, offset 0x1200 - 0x1200: 0x7a95, 0x1201: 0x7ab5, 0x1202: 0x7ad5, 0x1203: 0x7af5, 0x1204: 0x7b15, 0x1205: 0x7b35, - 0x1206: 0x7b55, 0x1207: 0x7b75, 0x1208: 0x7b95, 0x1209: 0x7bb5, 0x120a: 0x7bd5, 0x120b: 0x7bf5, - 0x120c: 0x7c15, 0x120d: 0x7c35, 0x120e: 0x7c55, 0x120f: 0x6ec9, 0x1210: 0x6ef1, 0x1211: 0x6f19, - 0x1212: 0x7c75, 0x1213: 0x7c95, 0x1214: 0x7cb5, 0x1215: 0x6f41, 0x1216: 0x6f69, 0x1217: 0x6f91, - 0x1218: 0x7cd5, 0x1219: 0x7cf5, 0x121a: 0x0040, 0x121b: 0x0040, 0x121c: 0x0040, 0x121d: 0x0040, - 0x121e: 0x0040, 0x121f: 0x0040, 0x1220: 0x0040, 0x1221: 0x0040, 0x1222: 0x0040, 0x1223: 0x0040, - 0x1224: 0x0040, 0x1225: 0x0040, 0x1226: 0x0040, 0x1227: 0x0040, 0x1228: 0x0040, 0x1229: 0x0040, - 0x122a: 0x0040, 0x122b: 0x0040, 0x122c: 0x0040, 0x122d: 0x0040, 0x122e: 0x0040, 0x122f: 0x0040, - 0x1230: 0x0040, 0x1231: 0x0040, 0x1232: 0x0040, 0x1233: 0x0040, 0x1234: 0x0040, 0x1235: 0x0040, - 0x1236: 0x0040, 0x1237: 0x0040, 0x1238: 0x0040, 0x1239: 0x0040, 0x123a: 0x0040, 0x123b: 0x0040, + 0x1200: 0xe00d, 0x1201: 0x0008, 0x1202: 0xe00d, 0x1203: 0x0008, 0x1204: 0xe00d, 0x1205: 0x0008, + 0x1206: 0xe00d, 0x1207: 0x0008, 0x1208: 0x0008, 0x1209: 0x0018, 0x120a: 0x0018, 0x120b: 0xe03d, + 0x120c: 0x0008, 0x120d: 0x11d9, 0x120e: 0x0008, 0x120f: 0x0008, 0x1210: 0xe00d, 0x1211: 0x0008, + 0x1212: 0xe00d, 0x1213: 0x0008, 0x1214: 0x0008, 0x1215: 0x0008, 0x1216: 0xe00d, 0x1217: 0x0008, + 0x1218: 0xe00d, 0x1219: 0x0008, 0x121a: 0xe00d, 0x121b: 0x0008, 0x121c: 0xe00d, 0x121d: 0x0008, + 0x121e: 0xe00d, 0x121f: 0x0008, 0x1220: 0xe00d, 0x1221: 0x0008, 0x1222: 0xe00d, 0x1223: 0x0008, + 0x1224: 0xe00d, 0x1225: 0x0008, 0x1226: 0xe00d, 0x1227: 0x0008, 0x1228: 0xe00d, 0x1229: 0x0008, + 0x122a: 0x6e29, 0x122b: 0x1029, 0x122c: 0x11c1, 0x122d: 0x6e41, 0x122e: 0x1221, 0x122f: 0x0040, + 0x1230: 0x6e59, 0x1231: 0x6e71, 0x1232: 0x1239, 0x1233: 0x444d, 0x1234: 0xe00d, 0x1235: 0x0008, + 0x1236: 0xe00d, 0x1237: 0x0008, 0x1238: 0x0040, 0x1239: 0x0040, 0x123a: 0x0040, 0x123b: 0x0040, 0x123c: 0x0040, 0x123d: 0x0040, 0x123e: 0x0040, 0x123f: 0x0040, // Block 0x49, offset 0x1240 - 0x1240: 0x6fb9, 0x1241: 0x6fd1, 0x1242: 0x6fe9, 0x1243: 0x7d15, 0x1244: 0x7d35, 0x1245: 0x7001, - 0x1246: 0x7001, 0x1247: 0x0040, 0x1248: 0x0040, 0x1249: 0x0040, 0x124a: 0x0040, 0x124b: 0x0040, - 0x124c: 0x0040, 0x124d: 0x0040, 0x124e: 0x0040, 0x124f: 0x0040, 0x1250: 0x0040, 0x1251: 0x0040, - 0x1252: 0x0040, 0x1253: 0x7019, 0x1254: 0x7041, 0x1255: 0x7069, 0x1256: 0x7091, 0x1257: 0x70b9, - 0x1258: 0x0040, 0x1259: 0x0040, 0x125a: 0x0040, 0x125b: 0x0040, 0x125c: 0x0040, 0x125d: 0x70e1, - 0x125e: 0x1308, 0x125f: 0x7109, 0x1260: 0x7131, 0x1261: 0x20a9, 0x1262: 0x20f1, 0x1263: 0x7149, - 0x1264: 0x7161, 0x1265: 0x7179, 0x1266: 0x7191, 0x1267: 0x71a9, 0x1268: 0x71c1, 0x1269: 0x1fb2, - 0x126a: 0x71d9, 0x126b: 0x7201, 0x126c: 0x7229, 0x126d: 0x7261, 0x126e: 0x7299, 0x126f: 0x72c1, - 0x1270: 0x72e9, 0x1271: 0x7311, 0x1272: 0x7339, 0x1273: 0x7361, 0x1274: 0x7389, 0x1275: 0x73b1, - 0x1276: 0x73d9, 0x1277: 0x0040, 0x1278: 0x7401, 0x1279: 0x7429, 0x127a: 0x7451, 0x127b: 0x7479, - 0x127c: 0x74a1, 0x127d: 0x0040, 0x127e: 0x74c9, 0x127f: 0x0040, + 0x1240: 0x64d5, 0x1241: 0x64f5, 0x1242: 0x6515, 0x1243: 0x6535, 0x1244: 0x6555, 0x1245: 0x6575, + 0x1246: 0x6595, 0x1247: 0x65b5, 0x1248: 0x65d5, 0x1249: 0x65f5, 0x124a: 0x6615, 0x124b: 0x6635, + 0x124c: 0x6655, 0x124d: 0x6675, 0x124e: 0x0008, 0x124f: 0x0008, 0x1250: 0x6695, 0x1251: 0x0008, + 0x1252: 0x66b5, 0x1253: 0x0008, 0x1254: 0x0008, 0x1255: 0x66d5, 0x1256: 0x66f5, 0x1257: 0x6715, + 0x1258: 0x6735, 0x1259: 0x6755, 0x125a: 0x6775, 0x125b: 0x6795, 0x125c: 0x67b5, 0x125d: 0x67d5, + 0x125e: 0x67f5, 0x125f: 0x0008, 0x1260: 0x6815, 0x1261: 0x0008, 0x1262: 0x6835, 0x1263: 0x0008, + 0x1264: 0x0008, 0x1265: 0x6855, 0x1266: 0x6875, 0x1267: 0x0008, 0x1268: 0x0008, 0x1269: 0x0008, + 0x126a: 0x6895, 0x126b: 0x68b5, 0x126c: 0x68d5, 0x126d: 0x68f5, 0x126e: 0x6915, 0x126f: 0x6935, + 0x1270: 0x6955, 0x1271: 0x6975, 0x1272: 0x6995, 0x1273: 0x69b5, 0x1274: 0x69d5, 0x1275: 0x69f5, + 0x1276: 0x6a15, 0x1277: 0x6a35, 0x1278: 0x6a55, 0x1279: 0x6a75, 0x127a: 0x6a95, 0x127b: 0x6ab5, + 0x127c: 0x6ad5, 0x127d: 0x6af5, 0x127e: 0x6b15, 0x127f: 0x6b35, // Block 0x4a, offset 0x1280 - 0x1280: 0x74f1, 0x1281: 0x7519, 0x1282: 0x0040, 0x1283: 0x7541, 0x1284: 0x7569, 0x1285: 0x0040, - 0x1286: 0x7591, 0x1287: 0x75b9, 0x1288: 0x75e1, 0x1289: 0x7609, 0x128a: 0x7631, 0x128b: 0x7659, - 0x128c: 0x7681, 0x128d: 0x76a9, 0x128e: 0x76d1, 0x128f: 0x76f9, 0x1290: 0x7721, 0x1291: 0x7721, - 0x1292: 0x7739, 0x1293: 0x7739, 0x1294: 0x7739, 0x1295: 0x7739, 0x1296: 0x7751, 0x1297: 0x7751, - 0x1298: 0x7751, 0x1299: 0x7751, 0x129a: 0x7769, 0x129b: 0x7769, 0x129c: 0x7769, 0x129d: 0x7769, - 0x129e: 0x7781, 0x129f: 0x7781, 0x12a0: 0x7781, 0x12a1: 0x7781, 0x12a2: 0x7799, 0x12a3: 0x7799, - 0x12a4: 0x7799, 0x12a5: 0x7799, 0x12a6: 0x77b1, 0x12a7: 0x77b1, 0x12a8: 0x77b1, 0x12a9: 0x77b1, - 0x12aa: 0x77c9, 0x12ab: 0x77c9, 0x12ac: 0x77c9, 0x12ad: 0x77c9, 0x12ae: 0x77e1, 0x12af: 0x77e1, - 0x12b0: 0x77e1, 0x12b1: 0x77e1, 0x12b2: 0x77f9, 0x12b3: 0x77f9, 0x12b4: 0x77f9, 0x12b5: 0x77f9, - 0x12b6: 0x7811, 0x12b7: 0x7811, 0x12b8: 0x7811, 0x12b9: 0x7811, 0x12ba: 0x7829, 0x12bb: 0x7829, - 0x12bc: 0x7829, 0x12bd: 0x7829, 0x12be: 0x7841, 0x12bf: 0x7841, + 0x1280: 0x7a95, 0x1281: 0x7ab5, 0x1282: 0x7ad5, 0x1283: 0x7af5, 0x1284: 0x7b15, 0x1285: 0x7b35, + 0x1286: 0x7b55, 0x1287: 0x7b75, 0x1288: 0x7b95, 0x1289: 0x7bb5, 0x128a: 0x7bd5, 0x128b: 0x7bf5, + 0x128c: 0x7c15, 0x128d: 0x7c35, 0x128e: 0x7c55, 0x128f: 0x6ec9, 0x1290: 0x6ef1, 0x1291: 0x6f19, + 0x1292: 0x7c75, 0x1293: 0x7c95, 0x1294: 0x7cb5, 0x1295: 0x6f41, 0x1296: 0x6f69, 0x1297: 0x6f91, + 0x1298: 0x7cd5, 0x1299: 0x7cf5, 0x129a: 0x0040, 0x129b: 0x0040, 0x129c: 0x0040, 0x129d: 0x0040, + 0x129e: 0x0040, 0x129f: 0x0040, 0x12a0: 0x0040, 0x12a1: 0x0040, 0x12a2: 0x0040, 0x12a3: 0x0040, + 0x12a4: 0x0040, 0x12a5: 0x0040, 0x12a6: 0x0040, 0x12a7: 0x0040, 0x12a8: 0x0040, 0x12a9: 0x0040, + 0x12aa: 0x0040, 0x12ab: 0x0040, 0x12ac: 0x0040, 0x12ad: 0x0040, 0x12ae: 0x0040, 0x12af: 0x0040, + 0x12b0: 0x0040, 0x12b1: 0x0040, 0x12b2: 0x0040, 0x12b3: 0x0040, 0x12b4: 0x0040, 0x12b5: 0x0040, + 0x12b6: 0x0040, 0x12b7: 0x0040, 0x12b8: 0x0040, 0x12b9: 0x0040, 0x12ba: 0x0040, 0x12bb: 0x0040, + 0x12bc: 0x0040, 0x12bd: 0x0040, 0x12be: 0x0040, 0x12bf: 0x0040, // Block 0x4b, offset 0x12c0 - 0x12c0: 0x7841, 0x12c1: 0x7841, 0x12c2: 0x7859, 0x12c3: 0x7859, 0x12c4: 0x7871, 0x12c5: 0x7871, - 0x12c6: 0x7889, 0x12c7: 0x7889, 0x12c8: 0x78a1, 0x12c9: 0x78a1, 0x12ca: 0x78b9, 0x12cb: 0x78b9, - 0x12cc: 0x78d1, 0x12cd: 0x78d1, 0x12ce: 0x78e9, 0x12cf: 0x78e9, 0x12d0: 0x78e9, 0x12d1: 0x78e9, - 0x12d2: 0x7901, 0x12d3: 0x7901, 0x12d4: 0x7901, 0x12d5: 0x7901, 0x12d6: 0x7919, 0x12d7: 0x7919, - 0x12d8: 0x7919, 0x12d9: 0x7919, 0x12da: 0x7931, 0x12db: 0x7931, 0x12dc: 0x7931, 0x12dd: 0x7931, - 0x12de: 0x7949, 0x12df: 0x7949, 0x12e0: 0x7961, 0x12e1: 0x7961, 0x12e2: 0x7961, 0x12e3: 0x7961, - 0x12e4: 0x7979, 0x12e5: 0x7979, 0x12e6: 0x7991, 0x12e7: 0x7991, 0x12e8: 0x7991, 0x12e9: 0x7991, - 0x12ea: 0x79a9, 0x12eb: 0x79a9, 0x12ec: 0x79a9, 0x12ed: 0x79a9, 0x12ee: 0x79c1, 0x12ef: 0x79c1, - 0x12f0: 0x79d9, 0x12f1: 0x79d9, 0x12f2: 0x0018, 0x12f3: 0x0018, 0x12f4: 0x0018, 0x12f5: 0x0018, - 0x12f6: 0x0018, 0x12f7: 0x0018, 0x12f8: 0x0018, 0x12f9: 0x0018, 0x12fa: 0x0018, 0x12fb: 0x0018, - 0x12fc: 0x0018, 0x12fd: 0x0018, 0x12fe: 0x0018, 0x12ff: 0x0018, + 0x12c0: 0x6fb9, 0x12c1: 0x6fd1, 0x12c2: 0x6fe9, 0x12c3: 0x7d15, 0x12c4: 0x7d35, 0x12c5: 0x7001, + 0x12c6: 0x7001, 0x12c7: 0x0040, 0x12c8: 0x0040, 0x12c9: 0x0040, 0x12ca: 0x0040, 0x12cb: 0x0040, + 0x12cc: 0x0040, 0x12cd: 0x0040, 0x12ce: 0x0040, 0x12cf: 0x0040, 0x12d0: 0x0040, 0x12d1: 0x0040, + 0x12d2: 0x0040, 0x12d3: 0x7019, 0x12d4: 0x7041, 0x12d5: 0x7069, 0x12d6: 0x7091, 0x12d7: 0x70b9, + 0x12d8: 0x0040, 0x12d9: 0x0040, 0x12da: 0x0040, 0x12db: 0x0040, 0x12dc: 0x0040, 0x12dd: 0x70e1, + 0x12de: 0x3308, 0x12df: 0x7109, 0x12e0: 0x7131, 0x12e1: 0x20a9, 0x12e2: 0x20f1, 0x12e3: 0x7149, + 0x12e4: 0x7161, 0x12e5: 0x7179, 0x12e6: 0x7191, 0x12e7: 0x71a9, 0x12e8: 0x71c1, 0x12e9: 0x1fb2, + 0x12ea: 0x71d9, 0x12eb: 0x7201, 0x12ec: 0x7229, 0x12ed: 0x7261, 0x12ee: 0x7299, 0x12ef: 0x72c1, + 0x12f0: 0x72e9, 0x12f1: 0x7311, 0x12f2: 0x7339, 0x12f3: 0x7361, 0x12f4: 0x7389, 0x12f5: 0x73b1, + 0x12f6: 0x73d9, 0x12f7: 0x0040, 0x12f8: 0x7401, 0x12f9: 0x7429, 0x12fa: 0x7451, 0x12fb: 0x7479, + 0x12fc: 0x74a1, 0x12fd: 0x0040, 0x12fe: 0x74c9, 0x12ff: 0x0040, // Block 0x4c, offset 0x1300 - 0x1300: 0x0018, 0x1301: 0x0018, 0x1302: 0x0040, 0x1303: 0x0040, 0x1304: 0x0040, 0x1305: 0x0040, - 0x1306: 0x0040, 0x1307: 0x0040, 0x1308: 0x0040, 0x1309: 0x0040, 0x130a: 0x0040, 0x130b: 0x0040, - 0x130c: 0x0040, 0x130d: 0x0040, 0x130e: 0x0040, 0x130f: 0x0040, 0x1310: 0x0040, 0x1311: 0x0040, - 0x1312: 0x0040, 0x1313: 0x79f1, 0x1314: 0x79f1, 0x1315: 0x79f1, 0x1316: 0x79f1, 0x1317: 0x7a09, - 0x1318: 0x7a09, 0x1319: 0x7a21, 0x131a: 0x7a21, 0x131b: 0x7a39, 0x131c: 0x7a39, 0x131d: 0x0479, - 0x131e: 0x7a51, 0x131f: 0x7a51, 0x1320: 0x7a69, 0x1321: 0x7a69, 0x1322: 0x7a81, 0x1323: 0x7a81, - 0x1324: 0x7a99, 0x1325: 0x7a99, 0x1326: 0x7a99, 0x1327: 0x7a99, 0x1328: 0x7ab1, 0x1329: 0x7ab1, - 0x132a: 0x7ac9, 0x132b: 0x7ac9, 0x132c: 0x7af1, 0x132d: 0x7af1, 0x132e: 0x7b19, 0x132f: 0x7b19, - 0x1330: 0x7b41, 0x1331: 0x7b41, 0x1332: 0x7b69, 0x1333: 0x7b69, 0x1334: 0x7b91, 0x1335: 0x7b91, - 0x1336: 0x7bb9, 0x1337: 0x7bb9, 0x1338: 0x7bb9, 0x1339: 0x7be1, 0x133a: 0x7be1, 0x133b: 0x7be1, - 0x133c: 0x7c09, 0x133d: 0x7c09, 0x133e: 0x7c09, 0x133f: 0x7c09, + 0x1300: 0x74f1, 0x1301: 0x7519, 0x1302: 0x0040, 0x1303: 0x7541, 0x1304: 0x7569, 0x1305: 0x0040, + 0x1306: 0x7591, 0x1307: 0x75b9, 0x1308: 0x75e1, 0x1309: 0x7609, 0x130a: 0x7631, 0x130b: 0x7659, + 0x130c: 0x7681, 0x130d: 0x76a9, 0x130e: 0x76d1, 0x130f: 0x76f9, 0x1310: 0x7721, 0x1311: 0x7721, + 0x1312: 0x7739, 0x1313: 0x7739, 0x1314: 0x7739, 0x1315: 0x7739, 0x1316: 0x7751, 0x1317: 0x7751, + 0x1318: 0x7751, 0x1319: 0x7751, 0x131a: 0x7769, 0x131b: 0x7769, 0x131c: 0x7769, 0x131d: 0x7769, + 0x131e: 0x7781, 0x131f: 0x7781, 0x1320: 0x7781, 0x1321: 0x7781, 0x1322: 0x7799, 0x1323: 0x7799, + 0x1324: 0x7799, 0x1325: 0x7799, 0x1326: 0x77b1, 0x1327: 0x77b1, 0x1328: 0x77b1, 0x1329: 0x77b1, + 0x132a: 0x77c9, 0x132b: 0x77c9, 0x132c: 0x77c9, 0x132d: 0x77c9, 0x132e: 0x77e1, 0x132f: 0x77e1, + 0x1330: 0x77e1, 0x1331: 0x77e1, 0x1332: 0x77f9, 0x1333: 0x77f9, 0x1334: 0x77f9, 0x1335: 0x77f9, + 0x1336: 0x7811, 0x1337: 0x7811, 0x1338: 0x7811, 0x1339: 0x7811, 0x133a: 0x7829, 0x133b: 0x7829, + 0x133c: 0x7829, 0x133d: 0x7829, 0x133e: 0x7841, 0x133f: 0x7841, // Block 0x4d, offset 0x1340 - 0x1340: 0x85f9, 0x1341: 0x8621, 0x1342: 0x8649, 0x1343: 0x8671, 0x1344: 0x8699, 0x1345: 0x86c1, - 0x1346: 0x86e9, 0x1347: 0x8711, 0x1348: 0x8739, 0x1349: 0x8761, 0x134a: 0x8789, 0x134b: 0x87b1, - 0x134c: 0x87d9, 0x134d: 0x8801, 0x134e: 0x8829, 0x134f: 0x8851, 0x1350: 0x8879, 0x1351: 0x88a1, - 0x1352: 0x88c9, 0x1353: 0x88f1, 0x1354: 0x8919, 0x1355: 0x8941, 0x1356: 0x8969, 0x1357: 0x8991, - 0x1358: 0x89b9, 0x1359: 0x89e1, 0x135a: 0x8a09, 0x135b: 0x8a31, 0x135c: 0x8a59, 0x135d: 0x8a81, - 0x135e: 0x8aaa, 0x135f: 0x8ada, 0x1360: 0x8b0a, 0x1361: 0x8b3a, 0x1362: 0x8b6a, 0x1363: 0x8b9a, - 0x1364: 0x8bc9, 0x1365: 0x8bf1, 0x1366: 0x7c71, 0x1367: 0x8c19, 0x1368: 0x7be1, 0x1369: 0x7c99, - 0x136a: 0x8c41, 0x136b: 0x8c69, 0x136c: 0x7d39, 0x136d: 0x8c91, 0x136e: 0x7d61, 0x136f: 0x7d89, - 0x1370: 0x8cb9, 0x1371: 0x8ce1, 0x1372: 0x7e29, 0x1373: 0x8d09, 0x1374: 0x7e51, 0x1375: 0x7e79, - 0x1376: 0x8d31, 0x1377: 0x8d59, 0x1378: 0x7ec9, 0x1379: 0x8d81, 0x137a: 0x7ef1, 0x137b: 0x7f19, - 0x137c: 0x83a1, 0x137d: 0x83c9, 0x137e: 0x8441, 0x137f: 0x8469, + 0x1340: 0x7841, 0x1341: 0x7841, 0x1342: 0x7859, 0x1343: 0x7859, 0x1344: 0x7871, 0x1345: 0x7871, + 0x1346: 0x7889, 0x1347: 0x7889, 0x1348: 0x78a1, 0x1349: 0x78a1, 0x134a: 0x78b9, 0x134b: 0x78b9, + 0x134c: 0x78d1, 0x134d: 0x78d1, 0x134e: 0x78e9, 0x134f: 0x78e9, 0x1350: 0x78e9, 0x1351: 0x78e9, + 0x1352: 0x7901, 0x1353: 0x7901, 0x1354: 0x7901, 0x1355: 0x7901, 0x1356: 0x7919, 0x1357: 0x7919, + 0x1358: 0x7919, 0x1359: 0x7919, 0x135a: 0x7931, 0x135b: 0x7931, 0x135c: 0x7931, 0x135d: 0x7931, + 0x135e: 0x7949, 0x135f: 0x7949, 0x1360: 0x7961, 0x1361: 0x7961, 0x1362: 0x7961, 0x1363: 0x7961, + 0x1364: 0x7979, 0x1365: 0x7979, 0x1366: 0x7991, 0x1367: 0x7991, 0x1368: 0x7991, 0x1369: 0x7991, + 0x136a: 0x79a9, 0x136b: 0x79a9, 0x136c: 0x79a9, 0x136d: 0x79a9, 0x136e: 0x79c1, 0x136f: 0x79c1, + 0x1370: 0x79d9, 0x1371: 0x79d9, 0x1372: 0x0818, 0x1373: 0x0818, 0x1374: 0x0818, 0x1375: 0x0818, + 0x1376: 0x0818, 0x1377: 0x0818, 0x1378: 0x0818, 0x1379: 0x0818, 0x137a: 0x0818, 0x137b: 0x0818, + 0x137c: 0x0818, 0x137d: 0x0818, 0x137e: 0x0818, 0x137f: 0x0818, // Block 0x4e, offset 0x1380 - 0x1380: 0x8491, 0x1381: 0x8531, 0x1382: 0x8559, 0x1383: 0x8581, 0x1384: 0x85a9, 0x1385: 0x8649, - 0x1386: 0x8671, 0x1387: 0x8699, 0x1388: 0x8da9, 0x1389: 0x8739, 0x138a: 0x8dd1, 0x138b: 0x8df9, - 0x138c: 0x8829, 0x138d: 0x8e21, 0x138e: 0x8851, 0x138f: 0x8879, 0x1390: 0x8a81, 0x1391: 0x8e49, - 0x1392: 0x8e71, 0x1393: 0x89b9, 0x1394: 0x8e99, 0x1395: 0x89e1, 0x1396: 0x8a09, 0x1397: 0x7c21, - 0x1398: 0x7c49, 0x1399: 0x8ec1, 0x139a: 0x7c71, 0x139b: 0x8ee9, 0x139c: 0x7cc1, 0x139d: 0x7ce9, - 0x139e: 0x7d11, 0x139f: 0x7d39, 0x13a0: 0x8f11, 0x13a1: 0x7db1, 0x13a2: 0x7dd9, 0x13a3: 0x7e01, - 0x13a4: 0x7e29, 0x13a5: 0x8f39, 0x13a6: 0x7ec9, 0x13a7: 0x7f41, 0x13a8: 0x7f69, 0x13a9: 0x7f91, - 0x13aa: 0x7fb9, 0x13ab: 0x7fe1, 0x13ac: 0x8031, 0x13ad: 0x8059, 0x13ae: 0x8081, 0x13af: 0x80a9, - 0x13b0: 0x80d1, 0x13b1: 0x80f9, 0x13b2: 0x8f61, 0x13b3: 0x8121, 0x13b4: 0x8149, 0x13b5: 0x8171, - 0x13b6: 0x8199, 0x13b7: 0x81c1, 0x13b8: 0x81e9, 0x13b9: 0x8239, 0x13ba: 0x8261, 0x13bb: 0x8289, - 0x13bc: 0x82b1, 0x13bd: 0x82d9, 0x13be: 0x8301, 0x13bf: 0x8329, + 0x1380: 0x0818, 0x1381: 0x0818, 0x1382: 0x0040, 0x1383: 0x0040, 0x1384: 0x0040, 0x1385: 0x0040, + 0x1386: 0x0040, 0x1387: 0x0040, 0x1388: 0x0040, 0x1389: 0x0040, 0x138a: 0x0040, 0x138b: 0x0040, + 0x138c: 0x0040, 0x138d: 0x0040, 0x138e: 0x0040, 0x138f: 0x0040, 0x1390: 0x0040, 0x1391: 0x0040, + 0x1392: 0x0040, 0x1393: 0x79f1, 0x1394: 0x79f1, 0x1395: 0x79f1, 0x1396: 0x79f1, 0x1397: 0x7a09, + 0x1398: 0x7a09, 0x1399: 0x7a21, 0x139a: 0x7a21, 0x139b: 0x7a39, 0x139c: 0x7a39, 0x139d: 0x0479, + 0x139e: 0x7a51, 0x139f: 0x7a51, 0x13a0: 0x7a69, 0x13a1: 0x7a69, 0x13a2: 0x7a81, 0x13a3: 0x7a81, + 0x13a4: 0x7a99, 0x13a5: 0x7a99, 0x13a6: 0x7a99, 0x13a7: 0x7a99, 0x13a8: 0x7ab1, 0x13a9: 0x7ab1, + 0x13aa: 0x7ac9, 0x13ab: 0x7ac9, 0x13ac: 0x7af1, 0x13ad: 0x7af1, 0x13ae: 0x7b19, 0x13af: 0x7b19, + 0x13b0: 0x7b41, 0x13b1: 0x7b41, 0x13b2: 0x7b69, 0x13b3: 0x7b69, 0x13b4: 0x7b91, 0x13b5: 0x7b91, + 0x13b6: 0x7bb9, 0x13b7: 0x7bb9, 0x13b8: 0x7bb9, 0x13b9: 0x7be1, 0x13ba: 0x7be1, 0x13bb: 0x7be1, + 0x13bc: 0x7c09, 0x13bd: 0x7c09, 0x13be: 0x7c09, 0x13bf: 0x7c09, // Block 0x4f, offset 0x13c0 - 0x13c0: 0x8351, 0x13c1: 0x8379, 0x13c2: 0x83f1, 0x13c3: 0x8419, 0x13c4: 0x84b9, 0x13c5: 0x84e1, - 0x13c6: 0x8509, 0x13c7: 0x8531, 0x13c8: 0x8559, 0x13c9: 0x85d1, 0x13ca: 0x85f9, 0x13cb: 0x8621, - 0x13cc: 0x8649, 0x13cd: 0x8f89, 0x13ce: 0x86c1, 0x13cf: 0x86e9, 0x13d0: 0x8711, 0x13d1: 0x8739, - 0x13d2: 0x87b1, 0x13d3: 0x87d9, 0x13d4: 0x8801, 0x13d5: 0x8829, 0x13d6: 0x8fb1, 0x13d7: 0x88a1, - 0x13d8: 0x88c9, 0x13d9: 0x8fd9, 0x13da: 0x8941, 0x13db: 0x8969, 0x13dc: 0x8991, 0x13dd: 0x89b9, - 0x13de: 0x9001, 0x13df: 0x7c71, 0x13e0: 0x8ee9, 0x13e1: 0x7d39, 0x13e2: 0x8f11, 0x13e3: 0x7e29, - 0x13e4: 0x8f39, 0x13e5: 0x7ec9, 0x13e6: 0x9029, 0x13e7: 0x80d1, 0x13e8: 0x9051, 0x13e9: 0x9079, - 0x13ea: 0x90a1, 0x13eb: 0x8531, 0x13ec: 0x8559, 0x13ed: 0x8649, 0x13ee: 0x8829, 0x13ef: 0x8fb1, - 0x13f0: 0x89b9, 0x13f1: 0x9001, 0x13f2: 0x90c9, 0x13f3: 0x9101, 0x13f4: 0x9139, 0x13f5: 0x9171, - 0x13f6: 0x9199, 0x13f7: 0x91c1, 0x13f8: 0x91e9, 0x13f9: 0x9211, 0x13fa: 0x9239, 0x13fb: 0x9261, - 0x13fc: 0x9289, 0x13fd: 0x92b1, 0x13fe: 0x92d9, 0x13ff: 0x9301, + 0x13c0: 0x85f9, 0x13c1: 0x8621, 0x13c2: 0x8649, 0x13c3: 0x8671, 0x13c4: 0x8699, 0x13c5: 0x86c1, + 0x13c6: 0x86e9, 0x13c7: 0x8711, 0x13c8: 0x8739, 0x13c9: 0x8761, 0x13ca: 0x8789, 0x13cb: 0x87b1, + 0x13cc: 0x87d9, 0x13cd: 0x8801, 0x13ce: 0x8829, 0x13cf: 0x8851, 0x13d0: 0x8879, 0x13d1: 0x88a1, + 0x13d2: 0x88c9, 0x13d3: 0x88f1, 0x13d4: 0x8919, 0x13d5: 0x8941, 0x13d6: 0x8969, 0x13d7: 0x8991, + 0x13d8: 0x89b9, 0x13d9: 0x89e1, 0x13da: 0x8a09, 0x13db: 0x8a31, 0x13dc: 0x8a59, 0x13dd: 0x8a81, + 0x13de: 0x8aaa, 0x13df: 0x8ada, 0x13e0: 0x8b0a, 0x13e1: 0x8b3a, 0x13e2: 0x8b6a, 0x13e3: 0x8b9a, + 0x13e4: 0x8bc9, 0x13e5: 0x8bf1, 0x13e6: 0x7c71, 0x13e7: 0x8c19, 0x13e8: 0x7be1, 0x13e9: 0x7c99, + 0x13ea: 0x8c41, 0x13eb: 0x8c69, 0x13ec: 0x7d39, 0x13ed: 0x8c91, 0x13ee: 0x7d61, 0x13ef: 0x7d89, + 0x13f0: 0x8cb9, 0x13f1: 0x8ce1, 0x13f2: 0x7e29, 0x13f3: 0x8d09, 0x13f4: 0x7e51, 0x13f5: 0x7e79, + 0x13f6: 0x8d31, 0x13f7: 0x8d59, 0x13f8: 0x7ec9, 0x13f9: 0x8d81, 0x13fa: 0x7ef1, 0x13fb: 0x7f19, + 0x13fc: 0x83a1, 0x13fd: 0x83c9, 0x13fe: 0x8441, 0x13ff: 0x8469, // Block 0x50, offset 0x1400 - 0x1400: 0x9329, 0x1401: 0x9351, 0x1402: 0x9379, 0x1403: 0x93a1, 0x1404: 0x93c9, 0x1405: 0x93f1, - 0x1406: 0x9419, 0x1407: 0x9441, 0x1408: 0x9469, 0x1409: 0x9491, 0x140a: 0x94b9, 0x140b: 0x94e1, - 0x140c: 0x9079, 0x140d: 0x9509, 0x140e: 0x9531, 0x140f: 0x9559, 0x1410: 0x9581, 0x1411: 0x9171, - 0x1412: 0x9199, 0x1413: 0x91c1, 0x1414: 0x91e9, 0x1415: 0x9211, 0x1416: 0x9239, 0x1417: 0x9261, - 0x1418: 0x9289, 0x1419: 0x92b1, 0x141a: 0x92d9, 0x141b: 0x9301, 0x141c: 0x9329, 0x141d: 0x9351, - 0x141e: 0x9379, 0x141f: 0x93a1, 0x1420: 0x93c9, 0x1421: 0x93f1, 0x1422: 0x9419, 0x1423: 0x9441, - 0x1424: 0x9469, 0x1425: 0x9491, 0x1426: 0x94b9, 0x1427: 0x94e1, 0x1428: 0x9079, 0x1429: 0x9509, - 0x142a: 0x9531, 0x142b: 0x9559, 0x142c: 0x9581, 0x142d: 0x9491, 0x142e: 0x94b9, 0x142f: 0x94e1, - 0x1430: 0x9079, 0x1431: 0x9051, 0x1432: 0x90a1, 0x1433: 0x8211, 0x1434: 0x8059, 0x1435: 0x8081, - 0x1436: 0x80a9, 0x1437: 0x9491, 0x1438: 0x94b9, 0x1439: 0x94e1, 0x143a: 0x8211, 0x143b: 0x8239, - 0x143c: 0x95a9, 0x143d: 0x95a9, 0x143e: 0x0018, 0x143f: 0x0018, + 0x1400: 0x8491, 0x1401: 0x8531, 0x1402: 0x8559, 0x1403: 0x8581, 0x1404: 0x85a9, 0x1405: 0x8649, + 0x1406: 0x8671, 0x1407: 0x8699, 0x1408: 0x8da9, 0x1409: 0x8739, 0x140a: 0x8dd1, 0x140b: 0x8df9, + 0x140c: 0x8829, 0x140d: 0x8e21, 0x140e: 0x8851, 0x140f: 0x8879, 0x1410: 0x8a81, 0x1411: 0x8e49, + 0x1412: 0x8e71, 0x1413: 0x89b9, 0x1414: 0x8e99, 0x1415: 0x89e1, 0x1416: 0x8a09, 0x1417: 0x7c21, + 0x1418: 0x7c49, 0x1419: 0x8ec1, 0x141a: 0x7c71, 0x141b: 0x8ee9, 0x141c: 0x7cc1, 0x141d: 0x7ce9, + 0x141e: 0x7d11, 0x141f: 0x7d39, 0x1420: 0x8f11, 0x1421: 0x7db1, 0x1422: 0x7dd9, 0x1423: 0x7e01, + 0x1424: 0x7e29, 0x1425: 0x8f39, 0x1426: 0x7ec9, 0x1427: 0x7f41, 0x1428: 0x7f69, 0x1429: 0x7f91, + 0x142a: 0x7fb9, 0x142b: 0x7fe1, 0x142c: 0x8031, 0x142d: 0x8059, 0x142e: 0x8081, 0x142f: 0x80a9, + 0x1430: 0x80d1, 0x1431: 0x80f9, 0x1432: 0x8f61, 0x1433: 0x8121, 0x1434: 0x8149, 0x1435: 0x8171, + 0x1436: 0x8199, 0x1437: 0x81c1, 0x1438: 0x81e9, 0x1439: 0x8239, 0x143a: 0x8261, 0x143b: 0x8289, + 0x143c: 0x82b1, 0x143d: 0x82d9, 0x143e: 0x8301, 0x143f: 0x8329, // Block 0x51, offset 0x1440 - 0x1440: 0x0040, 0x1441: 0x0040, 0x1442: 0x0040, 0x1443: 0x0040, 0x1444: 0x0040, 0x1445: 0x0040, - 0x1446: 0x0040, 0x1447: 0x0040, 0x1448: 0x0040, 0x1449: 0x0040, 0x144a: 0x0040, 0x144b: 0x0040, - 0x144c: 0x0040, 0x144d: 0x0040, 0x144e: 0x0040, 0x144f: 0x0040, 0x1450: 0x95d1, 0x1451: 0x9609, - 0x1452: 0x9609, 0x1453: 0x9641, 0x1454: 0x9679, 0x1455: 0x96b1, 0x1456: 0x96e9, 0x1457: 0x9721, - 0x1458: 0x9759, 0x1459: 0x9759, 0x145a: 0x9791, 0x145b: 0x97c9, 0x145c: 0x9801, 0x145d: 0x9839, - 0x145e: 0x9871, 0x145f: 0x98a9, 0x1460: 0x98a9, 0x1461: 0x98e1, 0x1462: 0x9919, 0x1463: 0x9919, - 0x1464: 0x9951, 0x1465: 0x9951, 0x1466: 0x9989, 0x1467: 0x99c1, 0x1468: 0x99c1, 0x1469: 0x99f9, - 0x146a: 0x9a31, 0x146b: 0x9a31, 0x146c: 0x9a69, 0x146d: 0x9a69, 0x146e: 0x9aa1, 0x146f: 0x9ad9, - 0x1470: 0x9ad9, 0x1471: 0x9b11, 0x1472: 0x9b11, 0x1473: 0x9b49, 0x1474: 0x9b81, 0x1475: 0x9bb9, - 0x1476: 0x9bf1, 0x1477: 0x9bf1, 0x1478: 0x9c29, 0x1479: 0x9c61, 0x147a: 0x9c99, 0x147b: 0x9cd1, - 0x147c: 0x9d09, 0x147d: 0x9d09, 0x147e: 0x9d41, 0x147f: 0x9d79, + 0x1440: 0x8351, 0x1441: 0x8379, 0x1442: 0x83f1, 0x1443: 0x8419, 0x1444: 0x84b9, 0x1445: 0x84e1, + 0x1446: 0x8509, 0x1447: 0x8531, 0x1448: 0x8559, 0x1449: 0x85d1, 0x144a: 0x85f9, 0x144b: 0x8621, + 0x144c: 0x8649, 0x144d: 0x8f89, 0x144e: 0x86c1, 0x144f: 0x86e9, 0x1450: 0x8711, 0x1451: 0x8739, + 0x1452: 0x87b1, 0x1453: 0x87d9, 0x1454: 0x8801, 0x1455: 0x8829, 0x1456: 0x8fb1, 0x1457: 0x88a1, + 0x1458: 0x88c9, 0x1459: 0x8fd9, 0x145a: 0x8941, 0x145b: 0x8969, 0x145c: 0x8991, 0x145d: 0x89b9, + 0x145e: 0x9001, 0x145f: 0x7c71, 0x1460: 0x8ee9, 0x1461: 0x7d39, 0x1462: 0x8f11, 0x1463: 0x7e29, + 0x1464: 0x8f39, 0x1465: 0x7ec9, 0x1466: 0x9029, 0x1467: 0x80d1, 0x1468: 0x9051, 0x1469: 0x9079, + 0x146a: 0x90a1, 0x146b: 0x8531, 0x146c: 0x8559, 0x146d: 0x8649, 0x146e: 0x8829, 0x146f: 0x8fb1, + 0x1470: 0x89b9, 0x1471: 0x9001, 0x1472: 0x90c9, 0x1473: 0x9101, 0x1474: 0x9139, 0x1475: 0x9171, + 0x1476: 0x9199, 0x1477: 0x91c1, 0x1478: 0x91e9, 0x1479: 0x9211, 0x147a: 0x9239, 0x147b: 0x9261, + 0x147c: 0x9289, 0x147d: 0x92b1, 0x147e: 0x92d9, 0x147f: 0x9301, // Block 0x52, offset 0x1480 - 0x1480: 0xa949, 0x1481: 0xa981, 0x1482: 0xa9b9, 0x1483: 0xa8a1, 0x1484: 0x9bb9, 0x1485: 0x9989, - 0x1486: 0xa9f1, 0x1487: 0xaa29, 0x1488: 0x0040, 0x1489: 0x0040, 0x148a: 0x0040, 0x148b: 0x0040, - 0x148c: 0x0040, 0x148d: 0x0040, 0x148e: 0x0040, 0x148f: 0x0040, 0x1490: 0x0040, 0x1491: 0x0040, - 0x1492: 0x0040, 0x1493: 0x0040, 0x1494: 0x0040, 0x1495: 0x0040, 0x1496: 0x0040, 0x1497: 0x0040, - 0x1498: 0x0040, 0x1499: 0x0040, 0x149a: 0x0040, 0x149b: 0x0040, 0x149c: 0x0040, 0x149d: 0x0040, - 0x149e: 0x0040, 0x149f: 0x0040, 0x14a0: 0x0040, 0x14a1: 0x0040, 0x14a2: 0x0040, 0x14a3: 0x0040, - 0x14a4: 0x0040, 0x14a5: 0x0040, 0x14a6: 0x0040, 0x14a7: 0x0040, 0x14a8: 0x0040, 0x14a9: 0x0040, - 0x14aa: 0x0040, 0x14ab: 0x0040, 0x14ac: 0x0040, 0x14ad: 0x0040, 0x14ae: 0x0040, 0x14af: 0x0040, - 0x14b0: 0xaa61, 0x14b1: 0xaa99, 0x14b2: 0xaad1, 0x14b3: 0xab19, 0x14b4: 0xab61, 0x14b5: 0xaba9, - 0x14b6: 0xabf1, 0x14b7: 0xac39, 0x14b8: 0xac81, 0x14b9: 0xacc9, 0x14ba: 0xad02, 0x14bb: 0xae12, - 0x14bc: 0xae91, 0x14bd: 0x0018, 0x14be: 0x0040, 0x14bf: 0x0040, + 0x1480: 0x9329, 0x1481: 0x9351, 0x1482: 0x9379, 0x1483: 0x93a1, 0x1484: 0x93c9, 0x1485: 0x93f1, + 0x1486: 0x9419, 0x1487: 0x9441, 0x1488: 0x9469, 0x1489: 0x9491, 0x148a: 0x94b9, 0x148b: 0x94e1, + 0x148c: 0x9079, 0x148d: 0x9509, 0x148e: 0x9531, 0x148f: 0x9559, 0x1490: 0x9581, 0x1491: 0x9171, + 0x1492: 0x9199, 0x1493: 0x91c1, 0x1494: 0x91e9, 0x1495: 0x9211, 0x1496: 0x9239, 0x1497: 0x9261, + 0x1498: 0x9289, 0x1499: 0x92b1, 0x149a: 0x92d9, 0x149b: 0x9301, 0x149c: 0x9329, 0x149d: 0x9351, + 0x149e: 0x9379, 0x149f: 0x93a1, 0x14a0: 0x93c9, 0x14a1: 0x93f1, 0x14a2: 0x9419, 0x14a3: 0x9441, + 0x14a4: 0x9469, 0x14a5: 0x9491, 0x14a6: 0x94b9, 0x14a7: 0x94e1, 0x14a8: 0x9079, 0x14a9: 0x9509, + 0x14aa: 0x9531, 0x14ab: 0x9559, 0x14ac: 0x9581, 0x14ad: 0x9491, 0x14ae: 0x94b9, 0x14af: 0x94e1, + 0x14b0: 0x9079, 0x14b1: 0x9051, 0x14b2: 0x90a1, 0x14b3: 0x8211, 0x14b4: 0x8059, 0x14b5: 0x8081, + 0x14b6: 0x80a9, 0x14b7: 0x9491, 0x14b8: 0x94b9, 0x14b9: 0x94e1, 0x14ba: 0x8211, 0x14bb: 0x8239, + 0x14bc: 0x95a9, 0x14bd: 0x95a9, 0x14be: 0x0018, 0x14bf: 0x0018, // Block 0x53, offset 0x14c0 - 0x14c0: 0x13c0, 0x14c1: 0x13c0, 0x14c2: 0x13c0, 0x14c3: 0x13c0, 0x14c4: 0x13c0, 0x14c5: 0x13c0, - 0x14c6: 0x13c0, 0x14c7: 0x13c0, 0x14c8: 0x13c0, 0x14c9: 0x13c0, 0x14ca: 0x13c0, 0x14cb: 0x13c0, - 0x14cc: 0x13c0, 0x14cd: 0x13c0, 0x14ce: 0x13c0, 0x14cf: 0x13c0, 0x14d0: 0xaeda, 0x14d1: 0x7d55, - 0x14d2: 0x0040, 0x14d3: 0xaeea, 0x14d4: 0x03c2, 0x14d5: 0xaefa, 0x14d6: 0xaf0a, 0x14d7: 0x7d75, - 0x14d8: 0x7d95, 0x14d9: 0x0040, 0x14da: 0x0040, 0x14db: 0x0040, 0x14dc: 0x0040, 0x14dd: 0x0040, - 0x14de: 0x0040, 0x14df: 0x0040, 0x14e0: 0x1308, 0x14e1: 0x1308, 0x14e2: 0x1308, 0x14e3: 0x1308, - 0x14e4: 0x1308, 0x14e5: 0x1308, 0x14e6: 0x1308, 0x14e7: 0x1308, 0x14e8: 0x1308, 0x14e9: 0x1308, - 0x14ea: 0x1308, 0x14eb: 0x1308, 0x14ec: 0x1308, 0x14ed: 0x1308, 0x14ee: 0x1308, 0x14ef: 0x1308, - 0x14f0: 0x0040, 0x14f1: 0x7db5, 0x14f2: 0x7dd5, 0x14f3: 0xaf1a, 0x14f4: 0xaf1a, 0x14f5: 0x1fd2, - 0x14f6: 0x1fe2, 0x14f7: 0xaf2a, 0x14f8: 0xaf3a, 0x14f9: 0x7df5, 0x14fa: 0x7e15, 0x14fb: 0x7e35, - 0x14fc: 0x7df5, 0x14fd: 0x7e55, 0x14fe: 0x7e75, 0x14ff: 0x7e55, + 0x14c0: 0x0040, 0x14c1: 0x0040, 0x14c2: 0x0040, 0x14c3: 0x0040, 0x14c4: 0x0040, 0x14c5: 0x0040, + 0x14c6: 0x0040, 0x14c7: 0x0040, 0x14c8: 0x0040, 0x14c9: 0x0040, 0x14ca: 0x0040, 0x14cb: 0x0040, + 0x14cc: 0x0040, 0x14cd: 0x0040, 0x14ce: 0x0040, 0x14cf: 0x0040, 0x14d0: 0x95d1, 0x14d1: 0x9609, + 0x14d2: 0x9609, 0x14d3: 0x9641, 0x14d4: 0x9679, 0x14d5: 0x96b1, 0x14d6: 0x96e9, 0x14d7: 0x9721, + 0x14d8: 0x9759, 0x14d9: 0x9759, 0x14da: 0x9791, 0x14db: 0x97c9, 0x14dc: 0x9801, 0x14dd: 0x9839, + 0x14de: 0x9871, 0x14df: 0x98a9, 0x14e0: 0x98a9, 0x14e1: 0x98e1, 0x14e2: 0x9919, 0x14e3: 0x9919, + 0x14e4: 0x9951, 0x14e5: 0x9951, 0x14e6: 0x9989, 0x14e7: 0x99c1, 0x14e8: 0x99c1, 0x14e9: 0x99f9, + 0x14ea: 0x9a31, 0x14eb: 0x9a31, 0x14ec: 0x9a69, 0x14ed: 0x9a69, 0x14ee: 0x9aa1, 0x14ef: 0x9ad9, + 0x14f0: 0x9ad9, 0x14f1: 0x9b11, 0x14f2: 0x9b11, 0x14f3: 0x9b49, 0x14f4: 0x9b81, 0x14f5: 0x9bb9, + 0x14f6: 0x9bf1, 0x14f7: 0x9bf1, 0x14f8: 0x9c29, 0x14f9: 0x9c61, 0x14fa: 0x9c99, 0x14fb: 0x9cd1, + 0x14fc: 0x9d09, 0x14fd: 0x9d09, 0x14fe: 0x9d41, 0x14ff: 0x9d79, // Block 0x54, offset 0x1500 - 0x1500: 0x7e95, 0x1501: 0x7eb5, 0x1502: 0x7ed5, 0x1503: 0x7eb5, 0x1504: 0x7ef5, 0x1505: 0x0018, - 0x1506: 0x0018, 0x1507: 0xaf4a, 0x1508: 0xaf5a, 0x1509: 0x7f16, 0x150a: 0x7f36, 0x150b: 0x7f56, - 0x150c: 0x7f76, 0x150d: 0xaf1a, 0x150e: 0xaf1a, 0x150f: 0xaf1a, 0x1510: 0xaeda, 0x1511: 0x7f95, - 0x1512: 0x0040, 0x1513: 0x0040, 0x1514: 0x03c2, 0x1515: 0xaeea, 0x1516: 0xaf0a, 0x1517: 0xaefa, - 0x1518: 0x7fb5, 0x1519: 0x1fd2, 0x151a: 0x1fe2, 0x151b: 0xaf2a, 0x151c: 0xaf3a, 0x151d: 0x7e95, - 0x151e: 0x7ef5, 0x151f: 0xaf6a, 0x1520: 0xaf7a, 0x1521: 0xaf8a, 0x1522: 0x1fb2, 0x1523: 0xaf99, - 0x1524: 0xafaa, 0x1525: 0xafba, 0x1526: 0x1fc2, 0x1527: 0x0040, 0x1528: 0xafca, 0x1529: 0xafda, - 0x152a: 0xafea, 0x152b: 0xaffa, 0x152c: 0x0040, 0x152d: 0x0040, 0x152e: 0x0040, 0x152f: 0x0040, - 0x1530: 0x7fd6, 0x1531: 0xb009, 0x1532: 0x7ff6, 0x1533: 0x0008, 0x1534: 0x8016, 0x1535: 0x0040, - 0x1536: 0x8036, 0x1537: 0xb031, 0x1538: 0x8056, 0x1539: 0xb059, 0x153a: 0x8076, 0x153b: 0xb081, - 0x153c: 0x8096, 0x153d: 0xb0a9, 0x153e: 0x80b6, 0x153f: 0xb0d1, + 0x1500: 0xa949, 0x1501: 0xa981, 0x1502: 0xa9b9, 0x1503: 0xa8a1, 0x1504: 0x9bb9, 0x1505: 0x9989, + 0x1506: 0xa9f1, 0x1507: 0xaa29, 0x1508: 0x0040, 0x1509: 0x0040, 0x150a: 0x0040, 0x150b: 0x0040, + 0x150c: 0x0040, 0x150d: 0x0040, 0x150e: 0x0040, 0x150f: 0x0040, 0x1510: 0x0040, 0x1511: 0x0040, + 0x1512: 0x0040, 0x1513: 0x0040, 0x1514: 0x0040, 0x1515: 0x0040, 0x1516: 0x0040, 0x1517: 0x0040, + 0x1518: 0x0040, 0x1519: 0x0040, 0x151a: 0x0040, 0x151b: 0x0040, 0x151c: 0x0040, 0x151d: 0x0040, + 0x151e: 0x0040, 0x151f: 0x0040, 0x1520: 0x0040, 0x1521: 0x0040, 0x1522: 0x0040, 0x1523: 0x0040, + 0x1524: 0x0040, 0x1525: 0x0040, 0x1526: 0x0040, 0x1527: 0x0040, 0x1528: 0x0040, 0x1529: 0x0040, + 0x152a: 0x0040, 0x152b: 0x0040, 0x152c: 0x0040, 0x152d: 0x0040, 0x152e: 0x0040, 0x152f: 0x0040, + 0x1530: 0xaa61, 0x1531: 0xaa99, 0x1532: 0xaad1, 0x1533: 0xab19, 0x1534: 0xab61, 0x1535: 0xaba9, + 0x1536: 0xabf1, 0x1537: 0xac39, 0x1538: 0xac81, 0x1539: 0xacc9, 0x153a: 0xad02, 0x153b: 0xae12, + 0x153c: 0xae91, 0x153d: 0x0018, 0x153e: 0x0040, 0x153f: 0x0040, // Block 0x55, offset 0x1540 - 0x1540: 0xb0f9, 0x1541: 0xb111, 0x1542: 0xb111, 0x1543: 0xb129, 0x1544: 0xb129, 0x1545: 0xb141, - 0x1546: 0xb141, 0x1547: 0xb159, 0x1548: 0xb159, 0x1549: 0xb171, 0x154a: 0xb171, 0x154b: 0xb171, - 0x154c: 0xb171, 0x154d: 0xb189, 0x154e: 0xb189, 0x154f: 0xb1a1, 0x1550: 0xb1a1, 0x1551: 0xb1a1, - 0x1552: 0xb1a1, 0x1553: 0xb1b9, 0x1554: 0xb1b9, 0x1555: 0xb1d1, 0x1556: 0xb1d1, 0x1557: 0xb1d1, - 0x1558: 0xb1d1, 0x1559: 0xb1e9, 0x155a: 0xb1e9, 0x155b: 0xb1e9, 0x155c: 0xb1e9, 0x155d: 0xb201, - 0x155e: 0xb201, 0x155f: 0xb201, 0x1560: 0xb201, 0x1561: 0xb219, 0x1562: 0xb219, 0x1563: 0xb219, - 0x1564: 0xb219, 0x1565: 0xb231, 0x1566: 0xb231, 0x1567: 0xb231, 0x1568: 0xb231, 0x1569: 0xb249, - 0x156a: 0xb249, 0x156b: 0xb261, 0x156c: 0xb261, 0x156d: 0xb279, 0x156e: 0xb279, 0x156f: 0xb291, - 0x1570: 0xb291, 0x1571: 0xb2a9, 0x1572: 0xb2a9, 0x1573: 0xb2a9, 0x1574: 0xb2a9, 0x1575: 0xb2c1, - 0x1576: 0xb2c1, 0x1577: 0xb2c1, 0x1578: 0xb2c1, 0x1579: 0xb2d9, 0x157a: 0xb2d9, 0x157b: 0xb2d9, - 0x157c: 0xb2d9, 0x157d: 0xb2f1, 0x157e: 0xb2f1, 0x157f: 0xb2f1, + 0x1540: 0x33c0, 0x1541: 0x33c0, 0x1542: 0x33c0, 0x1543: 0x33c0, 0x1544: 0x33c0, 0x1545: 0x33c0, + 0x1546: 0x33c0, 0x1547: 0x33c0, 0x1548: 0x33c0, 0x1549: 0x33c0, 0x154a: 0x33c0, 0x154b: 0x33c0, + 0x154c: 0x33c0, 0x154d: 0x33c0, 0x154e: 0x33c0, 0x154f: 0x33c0, 0x1550: 0xaeda, 0x1551: 0x7d55, + 0x1552: 0x0040, 0x1553: 0xaeea, 0x1554: 0x03c2, 0x1555: 0xaefa, 0x1556: 0xaf0a, 0x1557: 0x7d75, + 0x1558: 0x7d95, 0x1559: 0x0040, 0x155a: 0x0040, 0x155b: 0x0040, 0x155c: 0x0040, 0x155d: 0x0040, + 0x155e: 0x0040, 0x155f: 0x0040, 0x1560: 0x3308, 0x1561: 0x3308, 0x1562: 0x3308, 0x1563: 0x3308, + 0x1564: 0x3308, 0x1565: 0x3308, 0x1566: 0x3308, 0x1567: 0x3308, 0x1568: 0x3308, 0x1569: 0x3308, + 0x156a: 0x3308, 0x156b: 0x3308, 0x156c: 0x3308, 0x156d: 0x3308, 0x156e: 0x3308, 0x156f: 0x3308, + 0x1570: 0x0040, 0x1571: 0x7db5, 0x1572: 0x7dd5, 0x1573: 0xaf1a, 0x1574: 0xaf1a, 0x1575: 0x1fd2, + 0x1576: 0x1fe2, 0x1577: 0xaf2a, 0x1578: 0xaf3a, 0x1579: 0x7df5, 0x157a: 0x7e15, 0x157b: 0x7e35, + 0x157c: 0x7df5, 0x157d: 0x7e55, 0x157e: 0x7e75, 0x157f: 0x7e55, // Block 0x56, offset 0x1580 - 0x1580: 0xb2f1, 0x1581: 0xb309, 0x1582: 0xb309, 0x1583: 0xb309, 0x1584: 0xb309, 0x1585: 0xb321, - 0x1586: 0xb321, 0x1587: 0xb321, 0x1588: 0xb321, 0x1589: 0xb339, 0x158a: 0xb339, 0x158b: 0xb339, - 0x158c: 0xb339, 0x158d: 0xb351, 0x158e: 0xb351, 0x158f: 0xb351, 0x1590: 0xb351, 0x1591: 0xb369, - 0x1592: 0xb369, 0x1593: 0xb369, 0x1594: 0xb369, 0x1595: 0xb381, 0x1596: 0xb381, 0x1597: 0xb381, - 0x1598: 0xb381, 0x1599: 0xb399, 0x159a: 0xb399, 0x159b: 0xb399, 0x159c: 0xb399, 0x159d: 0xb3b1, - 0x159e: 0xb3b1, 0x159f: 0xb3b1, 0x15a0: 0xb3b1, 0x15a1: 0xb3c9, 0x15a2: 0xb3c9, 0x15a3: 0xb3c9, - 0x15a4: 0xb3c9, 0x15a5: 0xb3e1, 0x15a6: 0xb3e1, 0x15a7: 0xb3e1, 0x15a8: 0xb3e1, 0x15a9: 0xb3f9, - 0x15aa: 0xb3f9, 0x15ab: 0xb3f9, 0x15ac: 0xb3f9, 0x15ad: 0xb411, 0x15ae: 0xb411, 0x15af: 0x7ab1, - 0x15b0: 0x7ab1, 0x15b1: 0xb429, 0x15b2: 0xb429, 0x15b3: 0xb429, 0x15b4: 0xb429, 0x15b5: 0xb441, - 0x15b6: 0xb441, 0x15b7: 0xb469, 0x15b8: 0xb469, 0x15b9: 0xb491, 0x15ba: 0xb491, 0x15bb: 0xb4b9, - 0x15bc: 0xb4b9, 0x15bd: 0x0040, 0x15be: 0x0040, 0x15bf: 0x03c0, + 0x1580: 0x7e95, 0x1581: 0x7eb5, 0x1582: 0x7ed5, 0x1583: 0x7eb5, 0x1584: 0x7ef5, 0x1585: 0x0018, + 0x1586: 0x0018, 0x1587: 0xaf4a, 0x1588: 0xaf5a, 0x1589: 0x7f16, 0x158a: 0x7f36, 0x158b: 0x7f56, + 0x158c: 0x7f76, 0x158d: 0xaf1a, 0x158e: 0xaf1a, 0x158f: 0xaf1a, 0x1590: 0xaeda, 0x1591: 0x7f95, + 0x1592: 0x0040, 0x1593: 0x0040, 0x1594: 0x03c2, 0x1595: 0xaeea, 0x1596: 0xaf0a, 0x1597: 0xaefa, + 0x1598: 0x7fb5, 0x1599: 0x1fd2, 0x159a: 0x1fe2, 0x159b: 0xaf2a, 0x159c: 0xaf3a, 0x159d: 0x7e95, + 0x159e: 0x7ef5, 0x159f: 0xaf6a, 0x15a0: 0xaf7a, 0x15a1: 0xaf8a, 0x15a2: 0x1fb2, 0x15a3: 0xaf99, + 0x15a4: 0xafaa, 0x15a5: 0xafba, 0x15a6: 0x1fc2, 0x15a7: 0x0040, 0x15a8: 0xafca, 0x15a9: 0xafda, + 0x15aa: 0xafea, 0x15ab: 0xaffa, 0x15ac: 0x0040, 0x15ad: 0x0040, 0x15ae: 0x0040, 0x15af: 0x0040, + 0x15b0: 0x7fd6, 0x15b1: 0xb009, 0x15b2: 0x7ff6, 0x15b3: 0x0808, 0x15b4: 0x8016, 0x15b5: 0x0040, + 0x15b6: 0x8036, 0x15b7: 0xb031, 0x15b8: 0x8056, 0x15b9: 0xb059, 0x15ba: 0x8076, 0x15bb: 0xb081, + 0x15bc: 0x8096, 0x15bd: 0xb0a9, 0x15be: 0x80b6, 0x15bf: 0xb0d1, // Block 0x57, offset 0x15c0 - 0x15c0: 0x0040, 0x15c1: 0xaefa, 0x15c2: 0xb4e2, 0x15c3: 0xaf6a, 0x15c4: 0xafda, 0x15c5: 0xafea, - 0x15c6: 0xaf7a, 0x15c7: 0xb4f2, 0x15c8: 0x1fd2, 0x15c9: 0x1fe2, 0x15ca: 0xaf8a, 0x15cb: 0x1fb2, - 0x15cc: 0xaeda, 0x15cd: 0xaf99, 0x15ce: 0x29d1, 0x15cf: 0xb502, 0x15d0: 0x1f41, 0x15d1: 0x00c9, - 0x15d2: 0x0069, 0x15d3: 0x0079, 0x15d4: 0x1f51, 0x15d5: 0x1f61, 0x15d6: 0x1f71, 0x15d7: 0x1f81, - 0x15d8: 0x1f91, 0x15d9: 0x1fa1, 0x15da: 0xaeea, 0x15db: 0x03c2, 0x15dc: 0xafaa, 0x15dd: 0x1fc2, - 0x15de: 0xafba, 0x15df: 0xaf0a, 0x15e0: 0xaffa, 0x15e1: 0x0039, 0x15e2: 0x0ee9, 0x15e3: 0x1159, - 0x15e4: 0x0ef9, 0x15e5: 0x0f09, 0x15e6: 0x1199, 0x15e7: 0x0f31, 0x15e8: 0x0249, 0x15e9: 0x0f41, - 0x15ea: 0x0259, 0x15eb: 0x0f51, 0x15ec: 0x0359, 0x15ed: 0x0f61, 0x15ee: 0x0f71, 0x15ef: 0x00d9, - 0x15f0: 0x0f99, 0x15f1: 0x2039, 0x15f2: 0x0269, 0x15f3: 0x01d9, 0x15f4: 0x0fa9, 0x15f5: 0x0fb9, - 0x15f6: 0x1089, 0x15f7: 0x0279, 0x15f8: 0x0369, 0x15f9: 0x0289, 0x15fa: 0x13d1, 0x15fb: 0xaf4a, - 0x15fc: 0xafca, 0x15fd: 0xaf5a, 0x15fe: 0xb512, 0x15ff: 0xaf1a, + 0x15c0: 0xb0f9, 0x15c1: 0xb111, 0x15c2: 0xb111, 0x15c3: 0xb129, 0x15c4: 0xb129, 0x15c5: 0xb141, + 0x15c6: 0xb141, 0x15c7: 0xb159, 0x15c8: 0xb159, 0x15c9: 0xb171, 0x15ca: 0xb171, 0x15cb: 0xb171, + 0x15cc: 0xb171, 0x15cd: 0xb189, 0x15ce: 0xb189, 0x15cf: 0xb1a1, 0x15d0: 0xb1a1, 0x15d1: 0xb1a1, + 0x15d2: 0xb1a1, 0x15d3: 0xb1b9, 0x15d4: 0xb1b9, 0x15d5: 0xb1d1, 0x15d6: 0xb1d1, 0x15d7: 0xb1d1, + 0x15d8: 0xb1d1, 0x15d9: 0xb1e9, 0x15da: 0xb1e9, 0x15db: 0xb1e9, 0x15dc: 0xb1e9, 0x15dd: 0xb201, + 0x15de: 0xb201, 0x15df: 0xb201, 0x15e0: 0xb201, 0x15e1: 0xb219, 0x15e2: 0xb219, 0x15e3: 0xb219, + 0x15e4: 0xb219, 0x15e5: 0xb231, 0x15e6: 0xb231, 0x15e7: 0xb231, 0x15e8: 0xb231, 0x15e9: 0xb249, + 0x15ea: 0xb249, 0x15eb: 0xb261, 0x15ec: 0xb261, 0x15ed: 0xb279, 0x15ee: 0xb279, 0x15ef: 0xb291, + 0x15f0: 0xb291, 0x15f1: 0xb2a9, 0x15f2: 0xb2a9, 0x15f3: 0xb2a9, 0x15f4: 0xb2a9, 0x15f5: 0xb2c1, + 0x15f6: 0xb2c1, 0x15f7: 0xb2c1, 0x15f8: 0xb2c1, 0x15f9: 0xb2d9, 0x15fa: 0xb2d9, 0x15fb: 0xb2d9, + 0x15fc: 0xb2d9, 0x15fd: 0xb2f1, 0x15fe: 0xb2f1, 0x15ff: 0xb2f1, // Block 0x58, offset 0x1600 - 0x1600: 0x1caa, 0x1601: 0x0039, 0x1602: 0x0ee9, 0x1603: 0x1159, 0x1604: 0x0ef9, 0x1605: 0x0f09, - 0x1606: 0x1199, 0x1607: 0x0f31, 0x1608: 0x0249, 0x1609: 0x0f41, 0x160a: 0x0259, 0x160b: 0x0f51, - 0x160c: 0x0359, 0x160d: 0x0f61, 0x160e: 0x0f71, 0x160f: 0x00d9, 0x1610: 0x0f99, 0x1611: 0x2039, - 0x1612: 0x0269, 0x1613: 0x01d9, 0x1614: 0x0fa9, 0x1615: 0x0fb9, 0x1616: 0x1089, 0x1617: 0x0279, - 0x1618: 0x0369, 0x1619: 0x0289, 0x161a: 0x13d1, 0x161b: 0xaf2a, 0x161c: 0xb522, 0x161d: 0xaf3a, - 0x161e: 0xb532, 0x161f: 0x80d5, 0x1620: 0x80f5, 0x1621: 0x29d1, 0x1622: 0x8115, 0x1623: 0x8115, - 0x1624: 0x8135, 0x1625: 0x8155, 0x1626: 0x8175, 0x1627: 0x8195, 0x1628: 0x81b5, 0x1629: 0x81d5, - 0x162a: 0x81f5, 0x162b: 0x8215, 0x162c: 0x8235, 0x162d: 0x8255, 0x162e: 0x8275, 0x162f: 0x8295, - 0x1630: 0x82b5, 0x1631: 0x82d5, 0x1632: 0x82f5, 0x1633: 0x8315, 0x1634: 0x8335, 0x1635: 0x8355, - 0x1636: 0x8375, 0x1637: 0x8395, 0x1638: 0x83b5, 0x1639: 0x83d5, 0x163a: 0x83f5, 0x163b: 0x8415, - 0x163c: 0x81b5, 0x163d: 0x8435, 0x163e: 0x8455, 0x163f: 0x8215, + 0x1600: 0xb2f1, 0x1601: 0xb309, 0x1602: 0xb309, 0x1603: 0xb309, 0x1604: 0xb309, 0x1605: 0xb321, + 0x1606: 0xb321, 0x1607: 0xb321, 0x1608: 0xb321, 0x1609: 0xb339, 0x160a: 0xb339, 0x160b: 0xb339, + 0x160c: 0xb339, 0x160d: 0xb351, 0x160e: 0xb351, 0x160f: 0xb351, 0x1610: 0xb351, 0x1611: 0xb369, + 0x1612: 0xb369, 0x1613: 0xb369, 0x1614: 0xb369, 0x1615: 0xb381, 0x1616: 0xb381, 0x1617: 0xb381, + 0x1618: 0xb381, 0x1619: 0xb399, 0x161a: 0xb399, 0x161b: 0xb399, 0x161c: 0xb399, 0x161d: 0xb3b1, + 0x161e: 0xb3b1, 0x161f: 0xb3b1, 0x1620: 0xb3b1, 0x1621: 0xb3c9, 0x1622: 0xb3c9, 0x1623: 0xb3c9, + 0x1624: 0xb3c9, 0x1625: 0xb3e1, 0x1626: 0xb3e1, 0x1627: 0xb3e1, 0x1628: 0xb3e1, 0x1629: 0xb3f9, + 0x162a: 0xb3f9, 0x162b: 0xb3f9, 0x162c: 0xb3f9, 0x162d: 0xb411, 0x162e: 0xb411, 0x162f: 0x7ab1, + 0x1630: 0x7ab1, 0x1631: 0xb429, 0x1632: 0xb429, 0x1633: 0xb429, 0x1634: 0xb429, 0x1635: 0xb441, + 0x1636: 0xb441, 0x1637: 0xb469, 0x1638: 0xb469, 0x1639: 0xb491, 0x163a: 0xb491, 0x163b: 0xb4b9, + 0x163c: 0xb4b9, 0x163d: 0x0040, 0x163e: 0x0040, 0x163f: 0x03c0, // Block 0x59, offset 0x1640 - 0x1640: 0x8475, 0x1641: 0x8495, 0x1642: 0x84b5, 0x1643: 0x84d5, 0x1644: 0x84f5, 0x1645: 0x8515, - 0x1646: 0x8535, 0x1647: 0x8555, 0x1648: 0x84d5, 0x1649: 0x8575, 0x164a: 0x84d5, 0x164b: 0x8595, - 0x164c: 0x8595, 0x164d: 0x85b5, 0x164e: 0x85b5, 0x164f: 0x85d5, 0x1650: 0x8515, 0x1651: 0x85f5, - 0x1652: 0x8615, 0x1653: 0x85f5, 0x1654: 0x8635, 0x1655: 0x8615, 0x1656: 0x8655, 0x1657: 0x8655, - 0x1658: 0x8675, 0x1659: 0x8675, 0x165a: 0x8695, 0x165b: 0x8695, 0x165c: 0x8615, 0x165d: 0x8115, - 0x165e: 0x86b5, 0x165f: 0x86d5, 0x1660: 0x0040, 0x1661: 0x86f5, 0x1662: 0x8715, 0x1663: 0x8735, - 0x1664: 0x8755, 0x1665: 0x8735, 0x1666: 0x8775, 0x1667: 0x8795, 0x1668: 0x87b5, 0x1669: 0x87b5, - 0x166a: 0x87d5, 0x166b: 0x87d5, 0x166c: 0x87f5, 0x166d: 0x87f5, 0x166e: 0x87d5, 0x166f: 0x87d5, - 0x1670: 0x8815, 0x1671: 0x8835, 0x1672: 0x8855, 0x1673: 0x8875, 0x1674: 0x8895, 0x1675: 0x88b5, - 0x1676: 0x88b5, 0x1677: 0x88b5, 0x1678: 0x88d5, 0x1679: 0x88d5, 0x167a: 0x88d5, 0x167b: 0x88d5, - 0x167c: 0x87b5, 0x167d: 0x87b5, 0x167e: 0x87b5, 0x167f: 0x0040, + 0x1640: 0x0040, 0x1641: 0xaefa, 0x1642: 0xb4e2, 0x1643: 0xaf6a, 0x1644: 0xafda, 0x1645: 0xafea, + 0x1646: 0xaf7a, 0x1647: 0xb4f2, 0x1648: 0x1fd2, 0x1649: 0x1fe2, 0x164a: 0xaf8a, 0x164b: 0x1fb2, + 0x164c: 0xaeda, 0x164d: 0xaf99, 0x164e: 0x29d1, 0x164f: 0xb502, 0x1650: 0x1f41, 0x1651: 0x00c9, + 0x1652: 0x0069, 0x1653: 0x0079, 0x1654: 0x1f51, 0x1655: 0x1f61, 0x1656: 0x1f71, 0x1657: 0x1f81, + 0x1658: 0x1f91, 0x1659: 0x1fa1, 0x165a: 0xaeea, 0x165b: 0x03c2, 0x165c: 0xafaa, 0x165d: 0x1fc2, + 0x165e: 0xafba, 0x165f: 0xaf0a, 0x1660: 0xaffa, 0x1661: 0x0039, 0x1662: 0x0ee9, 0x1663: 0x1159, + 0x1664: 0x0ef9, 0x1665: 0x0f09, 0x1666: 0x1199, 0x1667: 0x0f31, 0x1668: 0x0249, 0x1669: 0x0f41, + 0x166a: 0x0259, 0x166b: 0x0f51, 0x166c: 0x0359, 0x166d: 0x0f61, 0x166e: 0x0f71, 0x166f: 0x00d9, + 0x1670: 0x0f99, 0x1671: 0x2039, 0x1672: 0x0269, 0x1673: 0x01d9, 0x1674: 0x0fa9, 0x1675: 0x0fb9, + 0x1676: 0x1089, 0x1677: 0x0279, 0x1678: 0x0369, 0x1679: 0x0289, 0x167a: 0x13d1, 0x167b: 0xaf4a, + 0x167c: 0xafca, 0x167d: 0xaf5a, 0x167e: 0xb512, 0x167f: 0xaf1a, // Block 0x5a, offset 0x1680 - 0x1680: 0x0040, 0x1681: 0x0040, 0x1682: 0x8715, 0x1683: 0x86f5, 0x1684: 0x88f5, 0x1685: 0x86f5, - 0x1686: 0x8715, 0x1687: 0x86f5, 0x1688: 0x0040, 0x1689: 0x0040, 0x168a: 0x8915, 0x168b: 0x8715, - 0x168c: 0x8935, 0x168d: 0x88f5, 0x168e: 0x8935, 0x168f: 0x8715, 0x1690: 0x0040, 0x1691: 0x0040, - 0x1692: 0x8955, 0x1693: 0x8975, 0x1694: 0x8875, 0x1695: 0x8935, 0x1696: 0x88f5, 0x1697: 0x8935, - 0x1698: 0x0040, 0x1699: 0x0040, 0x169a: 0x8995, 0x169b: 0x89b5, 0x169c: 0x8995, 0x169d: 0x0040, - 0x169e: 0x0040, 0x169f: 0x0040, 0x16a0: 0xb541, 0x16a1: 0xb559, 0x16a2: 0xb571, 0x16a3: 0x89d6, - 0x16a4: 0xb589, 0x16a5: 0xb5a1, 0x16a6: 0x89f5, 0x16a7: 0x0040, 0x16a8: 0x8a15, 0x16a9: 0x8a35, - 0x16aa: 0x8a55, 0x16ab: 0x8a35, 0x16ac: 0x8a75, 0x16ad: 0x8a95, 0x16ae: 0x8ab5, 0x16af: 0x0040, - 0x16b0: 0x0040, 0x16b1: 0x0040, 0x16b2: 0x0040, 0x16b3: 0x0040, 0x16b4: 0x0040, 0x16b5: 0x0040, - 0x16b6: 0x0040, 0x16b7: 0x0040, 0x16b8: 0x0040, 0x16b9: 0x0340, 0x16ba: 0x0340, 0x16bb: 0x0340, - 0x16bc: 0x0040, 0x16bd: 0x0040, 0x16be: 0x0040, 0x16bf: 0x0040, + 0x1680: 0x1caa, 0x1681: 0x0039, 0x1682: 0x0ee9, 0x1683: 0x1159, 0x1684: 0x0ef9, 0x1685: 0x0f09, + 0x1686: 0x1199, 0x1687: 0x0f31, 0x1688: 0x0249, 0x1689: 0x0f41, 0x168a: 0x0259, 0x168b: 0x0f51, + 0x168c: 0x0359, 0x168d: 0x0f61, 0x168e: 0x0f71, 0x168f: 0x00d9, 0x1690: 0x0f99, 0x1691: 0x2039, + 0x1692: 0x0269, 0x1693: 0x01d9, 0x1694: 0x0fa9, 0x1695: 0x0fb9, 0x1696: 0x1089, 0x1697: 0x0279, + 0x1698: 0x0369, 0x1699: 0x0289, 0x169a: 0x13d1, 0x169b: 0xaf2a, 0x169c: 0xb522, 0x169d: 0xaf3a, + 0x169e: 0xb532, 0x169f: 0x80d5, 0x16a0: 0x80f5, 0x16a1: 0x29d1, 0x16a2: 0x8115, 0x16a3: 0x8115, + 0x16a4: 0x8135, 0x16a5: 0x8155, 0x16a6: 0x8175, 0x16a7: 0x8195, 0x16a8: 0x81b5, 0x16a9: 0x81d5, + 0x16aa: 0x81f5, 0x16ab: 0x8215, 0x16ac: 0x8235, 0x16ad: 0x8255, 0x16ae: 0x8275, 0x16af: 0x8295, + 0x16b0: 0x82b5, 0x16b1: 0x82d5, 0x16b2: 0x82f5, 0x16b3: 0x8315, 0x16b4: 0x8335, 0x16b5: 0x8355, + 0x16b6: 0x8375, 0x16b7: 0x8395, 0x16b8: 0x83b5, 0x16b9: 0x83d5, 0x16ba: 0x83f5, 0x16bb: 0x8415, + 0x16bc: 0x81b5, 0x16bd: 0x8435, 0x16be: 0x8455, 0x16bf: 0x8215, // Block 0x5b, offset 0x16c0 - 0x16c0: 0x0208, 0x16c1: 0x0208, 0x16c2: 0x0208, 0x16c3: 0x0208, 0x16c4: 0x0208, 0x16c5: 0x0408, - 0x16c6: 0x0008, 0x16c7: 0x0408, 0x16c8: 0x0018, 0x16c9: 0x0408, 0x16ca: 0x0408, 0x16cb: 0x0008, - 0x16cc: 0x0008, 0x16cd: 0x0108, 0x16ce: 0x0408, 0x16cf: 0x0408, 0x16d0: 0x0408, 0x16d1: 0x0408, - 0x16d2: 0x0408, 0x16d3: 0x0208, 0x16d4: 0x0208, 0x16d5: 0x0208, 0x16d6: 0x0208, 0x16d7: 0x0108, - 0x16d8: 0x0208, 0x16d9: 0x0208, 0x16da: 0x0208, 0x16db: 0x0208, 0x16dc: 0x0208, 0x16dd: 0x0408, - 0x16de: 0x0208, 0x16df: 0x0208, 0x16e0: 0x0208, 0x16e1: 0x0408, 0x16e2: 0x0008, 0x16e3: 0x0008, - 0x16e4: 0x0408, 0x16e5: 0x1308, 0x16e6: 0x1308, 0x16e7: 0x0040, 0x16e8: 0x0040, 0x16e9: 0x0040, - 0x16ea: 0x0040, 0x16eb: 0x0218, 0x16ec: 0x0218, 0x16ed: 0x0218, 0x16ee: 0x0218, 0x16ef: 0x0418, - 0x16f0: 0x0018, 0x16f1: 0x0018, 0x16f2: 0x0018, 0x16f3: 0x0018, 0x16f4: 0x0018, 0x16f5: 0x0018, - 0x16f6: 0x0018, 0x16f7: 0x0040, 0x16f8: 0x0040, 0x16f9: 0x0040, 0x16fa: 0x0040, 0x16fb: 0x0040, - 0x16fc: 0x0040, 0x16fd: 0x0040, 0x16fe: 0x0040, 0x16ff: 0x0040, + 0x16c0: 0x8475, 0x16c1: 0x8495, 0x16c2: 0x84b5, 0x16c3: 0x84d5, 0x16c4: 0x84f5, 0x16c5: 0x8515, + 0x16c6: 0x8535, 0x16c7: 0x8555, 0x16c8: 0x84d5, 0x16c9: 0x8575, 0x16ca: 0x84d5, 0x16cb: 0x8595, + 0x16cc: 0x8595, 0x16cd: 0x85b5, 0x16ce: 0x85b5, 0x16cf: 0x85d5, 0x16d0: 0x8515, 0x16d1: 0x85f5, + 0x16d2: 0x8615, 0x16d3: 0x85f5, 0x16d4: 0x8635, 0x16d5: 0x8615, 0x16d6: 0x8655, 0x16d7: 0x8655, + 0x16d8: 0x8675, 0x16d9: 0x8675, 0x16da: 0x8695, 0x16db: 0x8695, 0x16dc: 0x8615, 0x16dd: 0x8115, + 0x16de: 0x86b5, 0x16df: 0x86d5, 0x16e0: 0x0040, 0x16e1: 0x86f5, 0x16e2: 0x8715, 0x16e3: 0x8735, + 0x16e4: 0x8755, 0x16e5: 0x8735, 0x16e6: 0x8775, 0x16e7: 0x8795, 0x16e8: 0x87b5, 0x16e9: 0x87b5, + 0x16ea: 0x87d5, 0x16eb: 0x87d5, 0x16ec: 0x87f5, 0x16ed: 0x87f5, 0x16ee: 0x87d5, 0x16ef: 0x87d5, + 0x16f0: 0x8815, 0x16f1: 0x8835, 0x16f2: 0x8855, 0x16f3: 0x8875, 0x16f4: 0x8895, 0x16f5: 0x88b5, + 0x16f6: 0x88b5, 0x16f7: 0x88b5, 0x16f8: 0x88d5, 0x16f9: 0x88d5, 0x16fa: 0x88d5, 0x16fb: 0x88d5, + 0x16fc: 0x87b5, 0x16fd: 0x87b5, 0x16fe: 0x87b5, 0x16ff: 0x0040, // Block 0x5c, offset 0x1700 - 0x1700: 0x0208, 0x1701: 0x0408, 0x1702: 0x0208, 0x1703: 0x0408, 0x1704: 0x0408, 0x1705: 0x0408, - 0x1706: 0x0208, 0x1707: 0x0208, 0x1708: 0x0208, 0x1709: 0x0408, 0x170a: 0x0208, 0x170b: 0x0208, - 0x170c: 0x0408, 0x170d: 0x0208, 0x170e: 0x0408, 0x170f: 0x0408, 0x1710: 0x0208, 0x1711: 0x0408, - 0x1712: 0x0040, 0x1713: 0x0040, 0x1714: 0x0040, 0x1715: 0x0040, 0x1716: 0x0040, 0x1717: 0x0040, - 0x1718: 0x0040, 0x1719: 0x0018, 0x171a: 0x0018, 0x171b: 0x0018, 0x171c: 0x0018, 0x171d: 0x0040, - 0x171e: 0x0040, 0x171f: 0x0040, 0x1720: 0x0040, 0x1721: 0x0040, 0x1722: 0x0040, 0x1723: 0x0040, - 0x1724: 0x0040, 0x1725: 0x0040, 0x1726: 0x0040, 0x1727: 0x0040, 0x1728: 0x0040, 0x1729: 0x0418, - 0x172a: 0x0418, 0x172b: 0x0418, 0x172c: 0x0418, 0x172d: 0x0218, 0x172e: 0x0218, 0x172f: 0x0018, + 0x1700: 0x0040, 0x1701: 0x0040, 0x1702: 0x8715, 0x1703: 0x86f5, 0x1704: 0x88f5, 0x1705: 0x86f5, + 0x1706: 0x8715, 0x1707: 0x86f5, 0x1708: 0x0040, 0x1709: 0x0040, 0x170a: 0x8915, 0x170b: 0x8715, + 0x170c: 0x8935, 0x170d: 0x88f5, 0x170e: 0x8935, 0x170f: 0x8715, 0x1710: 0x0040, 0x1711: 0x0040, + 0x1712: 0x8955, 0x1713: 0x8975, 0x1714: 0x8875, 0x1715: 0x8935, 0x1716: 0x88f5, 0x1717: 0x8935, + 0x1718: 0x0040, 0x1719: 0x0040, 0x171a: 0x8995, 0x171b: 0x89b5, 0x171c: 0x8995, 0x171d: 0x0040, + 0x171e: 0x0040, 0x171f: 0x0040, 0x1720: 0xb541, 0x1721: 0xb559, 0x1722: 0xb571, 0x1723: 0x89d6, + 0x1724: 0xb589, 0x1725: 0xb5a1, 0x1726: 0x89f5, 0x1727: 0x0040, 0x1728: 0x8a15, 0x1729: 0x8a35, + 0x172a: 0x8a55, 0x172b: 0x8a35, 0x172c: 0x8a75, 0x172d: 0x8a95, 0x172e: 0x8ab5, 0x172f: 0x0040, 0x1730: 0x0040, 0x1731: 0x0040, 0x1732: 0x0040, 0x1733: 0x0040, 0x1734: 0x0040, 0x1735: 0x0040, - 0x1736: 0x0040, 0x1737: 0x0040, 0x1738: 0x0040, 0x1739: 0x0040, 0x173a: 0x0040, 0x173b: 0x0040, + 0x1736: 0x0040, 0x1737: 0x0040, 0x1738: 0x0040, 0x1739: 0x0340, 0x173a: 0x0340, 0x173b: 0x0340, 0x173c: 0x0040, 0x173d: 0x0040, 0x173e: 0x0040, 0x173f: 0x0040, // Block 0x5d, offset 0x1740 - 0x1740: 0x1308, 0x1741: 0x1308, 0x1742: 0x1008, 0x1743: 0x1008, 0x1744: 0x0040, 0x1745: 0x0008, - 0x1746: 0x0008, 0x1747: 0x0008, 0x1748: 0x0008, 0x1749: 0x0008, 0x174a: 0x0008, 0x174b: 0x0008, - 0x174c: 0x0008, 0x174d: 0x0040, 0x174e: 0x0040, 0x174f: 0x0008, 0x1750: 0x0008, 0x1751: 0x0040, - 0x1752: 0x0040, 0x1753: 0x0008, 0x1754: 0x0008, 0x1755: 0x0008, 0x1756: 0x0008, 0x1757: 0x0008, - 0x1758: 0x0008, 0x1759: 0x0008, 0x175a: 0x0008, 0x175b: 0x0008, 0x175c: 0x0008, 0x175d: 0x0008, - 0x175e: 0x0008, 0x175f: 0x0008, 0x1760: 0x0008, 0x1761: 0x0008, 0x1762: 0x0008, 0x1763: 0x0008, - 0x1764: 0x0008, 0x1765: 0x0008, 0x1766: 0x0008, 0x1767: 0x0008, 0x1768: 0x0008, 0x1769: 0x0040, - 0x176a: 0x0008, 0x176b: 0x0008, 0x176c: 0x0008, 0x176d: 0x0008, 0x176e: 0x0008, 0x176f: 0x0008, - 0x1770: 0x0008, 0x1771: 0x0040, 0x1772: 0x0008, 0x1773: 0x0008, 0x1774: 0x0040, 0x1775: 0x0008, - 0x1776: 0x0008, 0x1777: 0x0008, 0x1778: 0x0008, 0x1779: 0x0008, 0x177a: 0x0040, 0x177b: 0x0040, - 0x177c: 0x1308, 0x177d: 0x0008, 0x177e: 0x1008, 0x177f: 0x1008, + 0x1740: 0x0a08, 0x1741: 0x0a08, 0x1742: 0x0a08, 0x1743: 0x0a08, 0x1744: 0x0a08, 0x1745: 0x0c08, + 0x1746: 0x0808, 0x1747: 0x0c08, 0x1748: 0x0818, 0x1749: 0x0c08, 0x174a: 0x0c08, 0x174b: 0x0808, + 0x174c: 0x0808, 0x174d: 0x0908, 0x174e: 0x0c08, 0x174f: 0x0c08, 0x1750: 0x0c08, 0x1751: 0x0c08, + 0x1752: 0x0c08, 0x1753: 0x0a08, 0x1754: 0x0a08, 0x1755: 0x0a08, 0x1756: 0x0a08, 0x1757: 0x0908, + 0x1758: 0x0a08, 0x1759: 0x0a08, 0x175a: 0x0a08, 0x175b: 0x0a08, 0x175c: 0x0a08, 0x175d: 0x0c08, + 0x175e: 0x0a08, 0x175f: 0x0a08, 0x1760: 0x0a08, 0x1761: 0x0c08, 0x1762: 0x0808, 0x1763: 0x0808, + 0x1764: 0x0c08, 0x1765: 0x3308, 0x1766: 0x3308, 0x1767: 0x0040, 0x1768: 0x0040, 0x1769: 0x0040, + 0x176a: 0x0040, 0x176b: 0x0a18, 0x176c: 0x0a18, 0x176d: 0x0a18, 0x176e: 0x0a18, 0x176f: 0x0c18, + 0x1770: 0x0818, 0x1771: 0x0818, 0x1772: 0x0818, 0x1773: 0x0818, 0x1774: 0x0818, 0x1775: 0x0818, + 0x1776: 0x0818, 0x1777: 0x0040, 0x1778: 0x0040, 0x1779: 0x0040, 0x177a: 0x0040, 0x177b: 0x0040, + 0x177c: 0x0040, 0x177d: 0x0040, 0x177e: 0x0040, 0x177f: 0x0040, // Block 0x5e, offset 0x1780 - 0x1780: 0x1308, 0x1781: 0x1008, 0x1782: 0x1008, 0x1783: 0x1008, 0x1784: 0x1008, 0x1785: 0x0040, - 0x1786: 0x0040, 0x1787: 0x1008, 0x1788: 0x1008, 0x1789: 0x0040, 0x178a: 0x0040, 0x178b: 0x1008, - 0x178c: 0x1008, 0x178d: 0x1808, 0x178e: 0x0040, 0x178f: 0x0040, 0x1790: 0x0008, 0x1791: 0x0040, - 0x1792: 0x0040, 0x1793: 0x0040, 0x1794: 0x0040, 0x1795: 0x0040, 0x1796: 0x0040, 0x1797: 0x1008, - 0x1798: 0x0040, 0x1799: 0x0040, 0x179a: 0x0040, 0x179b: 0x0040, 0x179c: 0x0040, 0x179d: 0x0008, - 0x179e: 0x0008, 0x179f: 0x0008, 0x17a0: 0x0008, 0x17a1: 0x0008, 0x17a2: 0x1008, 0x17a3: 0x1008, - 0x17a4: 0x0040, 0x17a5: 0x0040, 0x17a6: 0x1308, 0x17a7: 0x1308, 0x17a8: 0x1308, 0x17a9: 0x1308, - 0x17aa: 0x1308, 0x17ab: 0x1308, 0x17ac: 0x1308, 0x17ad: 0x0040, 0x17ae: 0x0040, 0x17af: 0x0040, - 0x17b0: 0x1308, 0x17b1: 0x1308, 0x17b2: 0x1308, 0x17b3: 0x1308, 0x17b4: 0x1308, 0x17b5: 0x0040, + 0x1780: 0x0a08, 0x1781: 0x0c08, 0x1782: 0x0a08, 0x1783: 0x0c08, 0x1784: 0x0c08, 0x1785: 0x0c08, + 0x1786: 0x0a08, 0x1787: 0x0a08, 0x1788: 0x0a08, 0x1789: 0x0c08, 0x178a: 0x0a08, 0x178b: 0x0a08, + 0x178c: 0x0c08, 0x178d: 0x0a08, 0x178e: 0x0c08, 0x178f: 0x0c08, 0x1790: 0x0a08, 0x1791: 0x0c08, + 0x1792: 0x0040, 0x1793: 0x0040, 0x1794: 0x0040, 0x1795: 0x0040, 0x1796: 0x0040, 0x1797: 0x0040, + 0x1798: 0x0040, 0x1799: 0x0818, 0x179a: 0x0818, 0x179b: 0x0818, 0x179c: 0x0818, 0x179d: 0x0040, + 0x179e: 0x0040, 0x179f: 0x0040, 0x17a0: 0x0040, 0x17a1: 0x0040, 0x17a2: 0x0040, 0x17a3: 0x0040, + 0x17a4: 0x0040, 0x17a5: 0x0040, 0x17a6: 0x0040, 0x17a7: 0x0040, 0x17a8: 0x0040, 0x17a9: 0x0c18, + 0x17aa: 0x0c18, 0x17ab: 0x0c18, 0x17ac: 0x0c18, 0x17ad: 0x0a18, 0x17ae: 0x0a18, 0x17af: 0x0818, + 0x17b0: 0x0040, 0x17b1: 0x0040, 0x17b2: 0x0040, 0x17b3: 0x0040, 0x17b4: 0x0040, 0x17b5: 0x0040, 0x17b6: 0x0040, 0x17b7: 0x0040, 0x17b8: 0x0040, 0x17b9: 0x0040, 0x17ba: 0x0040, 0x17bb: 0x0040, 0x17bc: 0x0040, 0x17bd: 0x0040, 0x17be: 0x0040, 0x17bf: 0x0040, // Block 0x5f, offset 0x17c0 - 0x17c0: 0x0039, 0x17c1: 0x0ee9, 0x17c2: 0x1159, 0x17c3: 0x0ef9, 0x17c4: 0x0f09, 0x17c5: 0x1199, - 0x17c6: 0x0f31, 0x17c7: 0x0249, 0x17c8: 0x0f41, 0x17c9: 0x0259, 0x17ca: 0x0f51, 0x17cb: 0x0359, - 0x17cc: 0x0f61, 0x17cd: 0x0f71, 0x17ce: 0x00d9, 0x17cf: 0x0f99, 0x17d0: 0x2039, 0x17d1: 0x0269, - 0x17d2: 0x01d9, 0x17d3: 0x0fa9, 0x17d4: 0x0fb9, 0x17d5: 0x1089, 0x17d6: 0x0279, 0x17d7: 0x0369, - 0x17d8: 0x0289, 0x17d9: 0x13d1, 0x17da: 0x0039, 0x17db: 0x0ee9, 0x17dc: 0x1159, 0x17dd: 0x0ef9, - 0x17de: 0x0f09, 0x17df: 0x1199, 0x17e0: 0x0f31, 0x17e1: 0x0249, 0x17e2: 0x0f41, 0x17e3: 0x0259, - 0x17e4: 0x0f51, 0x17e5: 0x0359, 0x17e6: 0x0f61, 0x17e7: 0x0f71, 0x17e8: 0x00d9, 0x17e9: 0x0f99, - 0x17ea: 0x2039, 0x17eb: 0x0269, 0x17ec: 0x01d9, 0x17ed: 0x0fa9, 0x17ee: 0x0fb9, 0x17ef: 0x1089, - 0x17f0: 0x0279, 0x17f1: 0x0369, 0x17f2: 0x0289, 0x17f3: 0x13d1, 0x17f4: 0x0039, 0x17f5: 0x0ee9, - 0x17f6: 0x1159, 0x17f7: 0x0ef9, 0x17f8: 0x0f09, 0x17f9: 0x1199, 0x17fa: 0x0f31, 0x17fb: 0x0249, - 0x17fc: 0x0f41, 0x17fd: 0x0259, 0x17fe: 0x0f51, 0x17ff: 0x0359, + 0x17c0: 0x3308, 0x17c1: 0x3308, 0x17c2: 0x3008, 0x17c3: 0x3008, 0x17c4: 0x0040, 0x17c5: 0x0008, + 0x17c6: 0x0008, 0x17c7: 0x0008, 0x17c8: 0x0008, 0x17c9: 0x0008, 0x17ca: 0x0008, 0x17cb: 0x0008, + 0x17cc: 0x0008, 0x17cd: 0x0040, 0x17ce: 0x0040, 0x17cf: 0x0008, 0x17d0: 0x0008, 0x17d1: 0x0040, + 0x17d2: 0x0040, 0x17d3: 0x0008, 0x17d4: 0x0008, 0x17d5: 0x0008, 0x17d6: 0x0008, 0x17d7: 0x0008, + 0x17d8: 0x0008, 0x17d9: 0x0008, 0x17da: 0x0008, 0x17db: 0x0008, 0x17dc: 0x0008, 0x17dd: 0x0008, + 0x17de: 0x0008, 0x17df: 0x0008, 0x17e0: 0x0008, 0x17e1: 0x0008, 0x17e2: 0x0008, 0x17e3: 0x0008, + 0x17e4: 0x0008, 0x17e5: 0x0008, 0x17e6: 0x0008, 0x17e7: 0x0008, 0x17e8: 0x0008, 0x17e9: 0x0040, + 0x17ea: 0x0008, 0x17eb: 0x0008, 0x17ec: 0x0008, 0x17ed: 0x0008, 0x17ee: 0x0008, 0x17ef: 0x0008, + 0x17f0: 0x0008, 0x17f1: 0x0040, 0x17f2: 0x0008, 0x17f3: 0x0008, 0x17f4: 0x0040, 0x17f5: 0x0008, + 0x17f6: 0x0008, 0x17f7: 0x0008, 0x17f8: 0x0008, 0x17f9: 0x0008, 0x17fa: 0x0040, 0x17fb: 0x0040, + 0x17fc: 0x3308, 0x17fd: 0x0008, 0x17fe: 0x3008, 0x17ff: 0x3008, // Block 0x60, offset 0x1800 - 0x1800: 0x0f61, 0x1801: 0x0f71, 0x1802: 0x00d9, 0x1803: 0x0f99, 0x1804: 0x2039, 0x1805: 0x0269, - 0x1806: 0x01d9, 0x1807: 0x0fa9, 0x1808: 0x0fb9, 0x1809: 0x1089, 0x180a: 0x0279, 0x180b: 0x0369, - 0x180c: 0x0289, 0x180d: 0x13d1, 0x180e: 0x0039, 0x180f: 0x0ee9, 0x1810: 0x1159, 0x1811: 0x0ef9, - 0x1812: 0x0f09, 0x1813: 0x1199, 0x1814: 0x0f31, 0x1815: 0x0040, 0x1816: 0x0f41, 0x1817: 0x0259, - 0x1818: 0x0f51, 0x1819: 0x0359, 0x181a: 0x0f61, 0x181b: 0x0f71, 0x181c: 0x00d9, 0x181d: 0x0f99, - 0x181e: 0x2039, 0x181f: 0x0269, 0x1820: 0x01d9, 0x1821: 0x0fa9, 0x1822: 0x0fb9, 0x1823: 0x1089, - 0x1824: 0x0279, 0x1825: 0x0369, 0x1826: 0x0289, 0x1827: 0x13d1, 0x1828: 0x0039, 0x1829: 0x0ee9, - 0x182a: 0x1159, 0x182b: 0x0ef9, 0x182c: 0x0f09, 0x182d: 0x1199, 0x182e: 0x0f31, 0x182f: 0x0249, - 0x1830: 0x0f41, 0x1831: 0x0259, 0x1832: 0x0f51, 0x1833: 0x0359, 0x1834: 0x0f61, 0x1835: 0x0f71, - 0x1836: 0x00d9, 0x1837: 0x0f99, 0x1838: 0x2039, 0x1839: 0x0269, 0x183a: 0x01d9, 0x183b: 0x0fa9, - 0x183c: 0x0fb9, 0x183d: 0x1089, 0x183e: 0x0279, 0x183f: 0x0369, + 0x1800: 0x3308, 0x1801: 0x3008, 0x1802: 0x3008, 0x1803: 0x3008, 0x1804: 0x3008, 0x1805: 0x0040, + 0x1806: 0x0040, 0x1807: 0x3008, 0x1808: 0x3008, 0x1809: 0x0040, 0x180a: 0x0040, 0x180b: 0x3008, + 0x180c: 0x3008, 0x180d: 0x3808, 0x180e: 0x0040, 0x180f: 0x0040, 0x1810: 0x0008, 0x1811: 0x0040, + 0x1812: 0x0040, 0x1813: 0x0040, 0x1814: 0x0040, 0x1815: 0x0040, 0x1816: 0x0040, 0x1817: 0x3008, + 0x1818: 0x0040, 0x1819: 0x0040, 0x181a: 0x0040, 0x181b: 0x0040, 0x181c: 0x0040, 0x181d: 0x0008, + 0x181e: 0x0008, 0x181f: 0x0008, 0x1820: 0x0008, 0x1821: 0x0008, 0x1822: 0x3008, 0x1823: 0x3008, + 0x1824: 0x0040, 0x1825: 0x0040, 0x1826: 0x3308, 0x1827: 0x3308, 0x1828: 0x3308, 0x1829: 0x3308, + 0x182a: 0x3308, 0x182b: 0x3308, 0x182c: 0x3308, 0x182d: 0x0040, 0x182e: 0x0040, 0x182f: 0x0040, + 0x1830: 0x3308, 0x1831: 0x3308, 0x1832: 0x3308, 0x1833: 0x3308, 0x1834: 0x3308, 0x1835: 0x0040, + 0x1836: 0x0040, 0x1837: 0x0040, 0x1838: 0x0040, 0x1839: 0x0040, 0x183a: 0x0040, 0x183b: 0x0040, + 0x183c: 0x0040, 0x183d: 0x0040, 0x183e: 0x0040, 0x183f: 0x0040, // Block 0x61, offset 0x1840 - 0x1840: 0x0289, 0x1841: 0x13d1, 0x1842: 0x0039, 0x1843: 0x0ee9, 0x1844: 0x1159, 0x1845: 0x0ef9, - 0x1846: 0x0f09, 0x1847: 0x1199, 0x1848: 0x0f31, 0x1849: 0x0249, 0x184a: 0x0f41, 0x184b: 0x0259, - 0x184c: 0x0f51, 0x184d: 0x0359, 0x184e: 0x0f61, 0x184f: 0x0f71, 0x1850: 0x00d9, 0x1851: 0x0f99, - 0x1852: 0x2039, 0x1853: 0x0269, 0x1854: 0x01d9, 0x1855: 0x0fa9, 0x1856: 0x0fb9, 0x1857: 0x1089, - 0x1858: 0x0279, 0x1859: 0x0369, 0x185a: 0x0289, 0x185b: 0x13d1, 0x185c: 0x0039, 0x185d: 0x0040, - 0x185e: 0x1159, 0x185f: 0x0ef9, 0x1860: 0x0040, 0x1861: 0x0040, 0x1862: 0x0f31, 0x1863: 0x0040, - 0x1864: 0x0040, 0x1865: 0x0259, 0x1866: 0x0f51, 0x1867: 0x0040, 0x1868: 0x0040, 0x1869: 0x0f71, - 0x186a: 0x00d9, 0x186b: 0x0f99, 0x186c: 0x2039, 0x186d: 0x0040, 0x186e: 0x01d9, 0x186f: 0x0fa9, - 0x1870: 0x0fb9, 0x1871: 0x1089, 0x1872: 0x0279, 0x1873: 0x0369, 0x1874: 0x0289, 0x1875: 0x13d1, - 0x1876: 0x0039, 0x1877: 0x0ee9, 0x1878: 0x1159, 0x1879: 0x0ef9, 0x187a: 0x0040, 0x187b: 0x1199, - 0x187c: 0x0040, 0x187d: 0x0249, 0x187e: 0x0f41, 0x187f: 0x0259, + 0x1840: 0x0039, 0x1841: 0x0ee9, 0x1842: 0x1159, 0x1843: 0x0ef9, 0x1844: 0x0f09, 0x1845: 0x1199, + 0x1846: 0x0f31, 0x1847: 0x0249, 0x1848: 0x0f41, 0x1849: 0x0259, 0x184a: 0x0f51, 0x184b: 0x0359, + 0x184c: 0x0f61, 0x184d: 0x0f71, 0x184e: 0x00d9, 0x184f: 0x0f99, 0x1850: 0x2039, 0x1851: 0x0269, + 0x1852: 0x01d9, 0x1853: 0x0fa9, 0x1854: 0x0fb9, 0x1855: 0x1089, 0x1856: 0x0279, 0x1857: 0x0369, + 0x1858: 0x0289, 0x1859: 0x13d1, 0x185a: 0x0039, 0x185b: 0x0ee9, 0x185c: 0x1159, 0x185d: 0x0ef9, + 0x185e: 0x0f09, 0x185f: 0x1199, 0x1860: 0x0f31, 0x1861: 0x0249, 0x1862: 0x0f41, 0x1863: 0x0259, + 0x1864: 0x0f51, 0x1865: 0x0359, 0x1866: 0x0f61, 0x1867: 0x0f71, 0x1868: 0x00d9, 0x1869: 0x0f99, + 0x186a: 0x2039, 0x186b: 0x0269, 0x186c: 0x01d9, 0x186d: 0x0fa9, 0x186e: 0x0fb9, 0x186f: 0x1089, + 0x1870: 0x0279, 0x1871: 0x0369, 0x1872: 0x0289, 0x1873: 0x13d1, 0x1874: 0x0039, 0x1875: 0x0ee9, + 0x1876: 0x1159, 0x1877: 0x0ef9, 0x1878: 0x0f09, 0x1879: 0x1199, 0x187a: 0x0f31, 0x187b: 0x0249, + 0x187c: 0x0f41, 0x187d: 0x0259, 0x187e: 0x0f51, 0x187f: 0x0359, // Block 0x62, offset 0x1880 - 0x1880: 0x0f51, 0x1881: 0x0359, 0x1882: 0x0f61, 0x1883: 0x0f71, 0x1884: 0x0040, 0x1885: 0x0f99, - 0x1886: 0x2039, 0x1887: 0x0269, 0x1888: 0x01d9, 0x1889: 0x0fa9, 0x188a: 0x0fb9, 0x188b: 0x1089, - 0x188c: 0x0279, 0x188d: 0x0369, 0x188e: 0x0289, 0x188f: 0x13d1, 0x1890: 0x0039, 0x1891: 0x0ee9, - 0x1892: 0x1159, 0x1893: 0x0ef9, 0x1894: 0x0f09, 0x1895: 0x1199, 0x1896: 0x0f31, 0x1897: 0x0249, - 0x1898: 0x0f41, 0x1899: 0x0259, 0x189a: 0x0f51, 0x189b: 0x0359, 0x189c: 0x0f61, 0x189d: 0x0f71, - 0x189e: 0x00d9, 0x189f: 0x0f99, 0x18a0: 0x2039, 0x18a1: 0x0269, 0x18a2: 0x01d9, 0x18a3: 0x0fa9, - 0x18a4: 0x0fb9, 0x18a5: 0x1089, 0x18a6: 0x0279, 0x18a7: 0x0369, 0x18a8: 0x0289, 0x18a9: 0x13d1, - 0x18aa: 0x0039, 0x18ab: 0x0ee9, 0x18ac: 0x1159, 0x18ad: 0x0ef9, 0x18ae: 0x0f09, 0x18af: 0x1199, - 0x18b0: 0x0f31, 0x18b1: 0x0249, 0x18b2: 0x0f41, 0x18b3: 0x0259, 0x18b4: 0x0f51, 0x18b5: 0x0359, - 0x18b6: 0x0f61, 0x18b7: 0x0f71, 0x18b8: 0x00d9, 0x18b9: 0x0f99, 0x18ba: 0x2039, 0x18bb: 0x0269, - 0x18bc: 0x01d9, 0x18bd: 0x0fa9, 0x18be: 0x0fb9, 0x18bf: 0x1089, + 0x1880: 0x0f61, 0x1881: 0x0f71, 0x1882: 0x00d9, 0x1883: 0x0f99, 0x1884: 0x2039, 0x1885: 0x0269, + 0x1886: 0x01d9, 0x1887: 0x0fa9, 0x1888: 0x0fb9, 0x1889: 0x1089, 0x188a: 0x0279, 0x188b: 0x0369, + 0x188c: 0x0289, 0x188d: 0x13d1, 0x188e: 0x0039, 0x188f: 0x0ee9, 0x1890: 0x1159, 0x1891: 0x0ef9, + 0x1892: 0x0f09, 0x1893: 0x1199, 0x1894: 0x0f31, 0x1895: 0x0040, 0x1896: 0x0f41, 0x1897: 0x0259, + 0x1898: 0x0f51, 0x1899: 0x0359, 0x189a: 0x0f61, 0x189b: 0x0f71, 0x189c: 0x00d9, 0x189d: 0x0f99, + 0x189e: 0x2039, 0x189f: 0x0269, 0x18a0: 0x01d9, 0x18a1: 0x0fa9, 0x18a2: 0x0fb9, 0x18a3: 0x1089, + 0x18a4: 0x0279, 0x18a5: 0x0369, 0x18a6: 0x0289, 0x18a7: 0x13d1, 0x18a8: 0x0039, 0x18a9: 0x0ee9, + 0x18aa: 0x1159, 0x18ab: 0x0ef9, 0x18ac: 0x0f09, 0x18ad: 0x1199, 0x18ae: 0x0f31, 0x18af: 0x0249, + 0x18b0: 0x0f41, 0x18b1: 0x0259, 0x18b2: 0x0f51, 0x18b3: 0x0359, 0x18b4: 0x0f61, 0x18b5: 0x0f71, + 0x18b6: 0x00d9, 0x18b7: 0x0f99, 0x18b8: 0x2039, 0x18b9: 0x0269, 0x18ba: 0x01d9, 0x18bb: 0x0fa9, + 0x18bc: 0x0fb9, 0x18bd: 0x1089, 0x18be: 0x0279, 0x18bf: 0x0369, // Block 0x63, offset 0x18c0 - 0x18c0: 0x0279, 0x18c1: 0x0369, 0x18c2: 0x0289, 0x18c3: 0x13d1, 0x18c4: 0x0039, 0x18c5: 0x0ee9, - 0x18c6: 0x0040, 0x18c7: 0x0ef9, 0x18c8: 0x0f09, 0x18c9: 0x1199, 0x18ca: 0x0f31, 0x18cb: 0x0040, - 0x18cc: 0x0040, 0x18cd: 0x0259, 0x18ce: 0x0f51, 0x18cf: 0x0359, 0x18d0: 0x0f61, 0x18d1: 0x0f71, - 0x18d2: 0x00d9, 0x18d3: 0x0f99, 0x18d4: 0x2039, 0x18d5: 0x0040, 0x18d6: 0x01d9, 0x18d7: 0x0fa9, - 0x18d8: 0x0fb9, 0x18d9: 0x1089, 0x18da: 0x0279, 0x18db: 0x0369, 0x18dc: 0x0289, 0x18dd: 0x0040, - 0x18de: 0x0039, 0x18df: 0x0ee9, 0x18e0: 0x1159, 0x18e1: 0x0ef9, 0x18e2: 0x0f09, 0x18e3: 0x1199, - 0x18e4: 0x0f31, 0x18e5: 0x0249, 0x18e6: 0x0f41, 0x18e7: 0x0259, 0x18e8: 0x0f51, 0x18e9: 0x0359, - 0x18ea: 0x0f61, 0x18eb: 0x0f71, 0x18ec: 0x00d9, 0x18ed: 0x0f99, 0x18ee: 0x2039, 0x18ef: 0x0269, - 0x18f0: 0x01d9, 0x18f1: 0x0fa9, 0x18f2: 0x0fb9, 0x18f3: 0x1089, 0x18f4: 0x0279, 0x18f5: 0x0369, - 0x18f6: 0x0289, 0x18f7: 0x13d1, 0x18f8: 0x0039, 0x18f9: 0x0ee9, 0x18fa: 0x0040, 0x18fb: 0x0ef9, - 0x18fc: 0x0f09, 0x18fd: 0x1199, 0x18fe: 0x0f31, 0x18ff: 0x0040, + 0x18c0: 0x0289, 0x18c1: 0x13d1, 0x18c2: 0x0039, 0x18c3: 0x0ee9, 0x18c4: 0x1159, 0x18c5: 0x0ef9, + 0x18c6: 0x0f09, 0x18c7: 0x1199, 0x18c8: 0x0f31, 0x18c9: 0x0249, 0x18ca: 0x0f41, 0x18cb: 0x0259, + 0x18cc: 0x0f51, 0x18cd: 0x0359, 0x18ce: 0x0f61, 0x18cf: 0x0f71, 0x18d0: 0x00d9, 0x18d1: 0x0f99, + 0x18d2: 0x2039, 0x18d3: 0x0269, 0x18d4: 0x01d9, 0x18d5: 0x0fa9, 0x18d6: 0x0fb9, 0x18d7: 0x1089, + 0x18d8: 0x0279, 0x18d9: 0x0369, 0x18da: 0x0289, 0x18db: 0x13d1, 0x18dc: 0x0039, 0x18dd: 0x0040, + 0x18de: 0x1159, 0x18df: 0x0ef9, 0x18e0: 0x0040, 0x18e1: 0x0040, 0x18e2: 0x0f31, 0x18e3: 0x0040, + 0x18e4: 0x0040, 0x18e5: 0x0259, 0x18e6: 0x0f51, 0x18e7: 0x0040, 0x18e8: 0x0040, 0x18e9: 0x0f71, + 0x18ea: 0x00d9, 0x18eb: 0x0f99, 0x18ec: 0x2039, 0x18ed: 0x0040, 0x18ee: 0x01d9, 0x18ef: 0x0fa9, + 0x18f0: 0x0fb9, 0x18f1: 0x1089, 0x18f2: 0x0279, 0x18f3: 0x0369, 0x18f4: 0x0289, 0x18f5: 0x13d1, + 0x18f6: 0x0039, 0x18f7: 0x0ee9, 0x18f8: 0x1159, 0x18f9: 0x0ef9, 0x18fa: 0x0040, 0x18fb: 0x1199, + 0x18fc: 0x0040, 0x18fd: 0x0249, 0x18fe: 0x0f41, 0x18ff: 0x0259, // Block 0x64, offset 0x1900 - 0x1900: 0x0f41, 0x1901: 0x0259, 0x1902: 0x0f51, 0x1903: 0x0359, 0x1904: 0x0f61, 0x1905: 0x0040, - 0x1906: 0x00d9, 0x1907: 0x0040, 0x1908: 0x0040, 0x1909: 0x0040, 0x190a: 0x01d9, 0x190b: 0x0fa9, - 0x190c: 0x0fb9, 0x190d: 0x1089, 0x190e: 0x0279, 0x190f: 0x0369, 0x1910: 0x0289, 0x1911: 0x0040, - 0x1912: 0x0039, 0x1913: 0x0ee9, 0x1914: 0x1159, 0x1915: 0x0ef9, 0x1916: 0x0f09, 0x1917: 0x1199, - 0x1918: 0x0f31, 0x1919: 0x0249, 0x191a: 0x0f41, 0x191b: 0x0259, 0x191c: 0x0f51, 0x191d: 0x0359, - 0x191e: 0x0f61, 0x191f: 0x0f71, 0x1920: 0x00d9, 0x1921: 0x0f99, 0x1922: 0x2039, 0x1923: 0x0269, - 0x1924: 0x01d9, 0x1925: 0x0fa9, 0x1926: 0x0fb9, 0x1927: 0x1089, 0x1928: 0x0279, 0x1929: 0x0369, - 0x192a: 0x0289, 0x192b: 0x13d1, 0x192c: 0x0039, 0x192d: 0x0ee9, 0x192e: 0x1159, 0x192f: 0x0ef9, - 0x1930: 0x0f09, 0x1931: 0x1199, 0x1932: 0x0f31, 0x1933: 0x0249, 0x1934: 0x0f41, 0x1935: 0x0259, - 0x1936: 0x0f51, 0x1937: 0x0359, 0x1938: 0x0f61, 0x1939: 0x0f71, 0x193a: 0x00d9, 0x193b: 0x0f99, - 0x193c: 0x2039, 0x193d: 0x0269, 0x193e: 0x01d9, 0x193f: 0x0fa9, + 0x1900: 0x0f51, 0x1901: 0x0359, 0x1902: 0x0f61, 0x1903: 0x0f71, 0x1904: 0x0040, 0x1905: 0x0f99, + 0x1906: 0x2039, 0x1907: 0x0269, 0x1908: 0x01d9, 0x1909: 0x0fa9, 0x190a: 0x0fb9, 0x190b: 0x1089, + 0x190c: 0x0279, 0x190d: 0x0369, 0x190e: 0x0289, 0x190f: 0x13d1, 0x1910: 0x0039, 0x1911: 0x0ee9, + 0x1912: 0x1159, 0x1913: 0x0ef9, 0x1914: 0x0f09, 0x1915: 0x1199, 0x1916: 0x0f31, 0x1917: 0x0249, + 0x1918: 0x0f41, 0x1919: 0x0259, 0x191a: 0x0f51, 0x191b: 0x0359, 0x191c: 0x0f61, 0x191d: 0x0f71, + 0x191e: 0x00d9, 0x191f: 0x0f99, 0x1920: 0x2039, 0x1921: 0x0269, 0x1922: 0x01d9, 0x1923: 0x0fa9, + 0x1924: 0x0fb9, 0x1925: 0x1089, 0x1926: 0x0279, 0x1927: 0x0369, 0x1928: 0x0289, 0x1929: 0x13d1, + 0x192a: 0x0039, 0x192b: 0x0ee9, 0x192c: 0x1159, 0x192d: 0x0ef9, 0x192e: 0x0f09, 0x192f: 0x1199, + 0x1930: 0x0f31, 0x1931: 0x0249, 0x1932: 0x0f41, 0x1933: 0x0259, 0x1934: 0x0f51, 0x1935: 0x0359, + 0x1936: 0x0f61, 0x1937: 0x0f71, 0x1938: 0x00d9, 0x1939: 0x0f99, 0x193a: 0x2039, 0x193b: 0x0269, + 0x193c: 0x01d9, 0x193d: 0x0fa9, 0x193e: 0x0fb9, 0x193f: 0x1089, // Block 0x65, offset 0x1940 - 0x1940: 0x0fb9, 0x1941: 0x1089, 0x1942: 0x0279, 0x1943: 0x0369, 0x1944: 0x0289, 0x1945: 0x13d1, - 0x1946: 0x0039, 0x1947: 0x0ee9, 0x1948: 0x1159, 0x1949: 0x0ef9, 0x194a: 0x0f09, 0x194b: 0x1199, - 0x194c: 0x0f31, 0x194d: 0x0249, 0x194e: 0x0f41, 0x194f: 0x0259, 0x1950: 0x0f51, 0x1951: 0x0359, - 0x1952: 0x0f61, 0x1953: 0x0f71, 0x1954: 0x00d9, 0x1955: 0x0f99, 0x1956: 0x2039, 0x1957: 0x0269, - 0x1958: 0x01d9, 0x1959: 0x0fa9, 0x195a: 0x0fb9, 0x195b: 0x1089, 0x195c: 0x0279, 0x195d: 0x0369, - 0x195e: 0x0289, 0x195f: 0x13d1, 0x1960: 0x0039, 0x1961: 0x0ee9, 0x1962: 0x1159, 0x1963: 0x0ef9, - 0x1964: 0x0f09, 0x1965: 0x1199, 0x1966: 0x0f31, 0x1967: 0x0249, 0x1968: 0x0f41, 0x1969: 0x0259, - 0x196a: 0x0f51, 0x196b: 0x0359, 0x196c: 0x0f61, 0x196d: 0x0f71, 0x196e: 0x00d9, 0x196f: 0x0f99, - 0x1970: 0x2039, 0x1971: 0x0269, 0x1972: 0x01d9, 0x1973: 0x0fa9, 0x1974: 0x0fb9, 0x1975: 0x1089, - 0x1976: 0x0279, 0x1977: 0x0369, 0x1978: 0x0289, 0x1979: 0x13d1, 0x197a: 0x0039, 0x197b: 0x0ee9, - 0x197c: 0x1159, 0x197d: 0x0ef9, 0x197e: 0x0f09, 0x197f: 0x1199, + 0x1940: 0x0279, 0x1941: 0x0369, 0x1942: 0x0289, 0x1943: 0x13d1, 0x1944: 0x0039, 0x1945: 0x0ee9, + 0x1946: 0x0040, 0x1947: 0x0ef9, 0x1948: 0x0f09, 0x1949: 0x1199, 0x194a: 0x0f31, 0x194b: 0x0040, + 0x194c: 0x0040, 0x194d: 0x0259, 0x194e: 0x0f51, 0x194f: 0x0359, 0x1950: 0x0f61, 0x1951: 0x0f71, + 0x1952: 0x00d9, 0x1953: 0x0f99, 0x1954: 0x2039, 0x1955: 0x0040, 0x1956: 0x01d9, 0x1957: 0x0fa9, + 0x1958: 0x0fb9, 0x1959: 0x1089, 0x195a: 0x0279, 0x195b: 0x0369, 0x195c: 0x0289, 0x195d: 0x0040, + 0x195e: 0x0039, 0x195f: 0x0ee9, 0x1960: 0x1159, 0x1961: 0x0ef9, 0x1962: 0x0f09, 0x1963: 0x1199, + 0x1964: 0x0f31, 0x1965: 0x0249, 0x1966: 0x0f41, 0x1967: 0x0259, 0x1968: 0x0f51, 0x1969: 0x0359, + 0x196a: 0x0f61, 0x196b: 0x0f71, 0x196c: 0x00d9, 0x196d: 0x0f99, 0x196e: 0x2039, 0x196f: 0x0269, + 0x1970: 0x01d9, 0x1971: 0x0fa9, 0x1972: 0x0fb9, 0x1973: 0x1089, 0x1974: 0x0279, 0x1975: 0x0369, + 0x1976: 0x0289, 0x1977: 0x13d1, 0x1978: 0x0039, 0x1979: 0x0ee9, 0x197a: 0x0040, 0x197b: 0x0ef9, + 0x197c: 0x0f09, 0x197d: 0x1199, 0x197e: 0x0f31, 0x197f: 0x0040, // Block 0x66, offset 0x1980 - 0x1980: 0x0f31, 0x1981: 0x0249, 0x1982: 0x0f41, 0x1983: 0x0259, 0x1984: 0x0f51, 0x1985: 0x0359, - 0x1986: 0x0f61, 0x1987: 0x0f71, 0x1988: 0x00d9, 0x1989: 0x0f99, 0x198a: 0x2039, 0x198b: 0x0269, - 0x198c: 0x01d9, 0x198d: 0x0fa9, 0x198e: 0x0fb9, 0x198f: 0x1089, 0x1990: 0x0279, 0x1991: 0x0369, - 0x1992: 0x0289, 0x1993: 0x13d1, 0x1994: 0x0039, 0x1995: 0x0ee9, 0x1996: 0x1159, 0x1997: 0x0ef9, - 0x1998: 0x0f09, 0x1999: 0x1199, 0x199a: 0x0f31, 0x199b: 0x0249, 0x199c: 0x0f41, 0x199d: 0x0259, - 0x199e: 0x0f51, 0x199f: 0x0359, 0x19a0: 0x0f61, 0x19a1: 0x0f71, 0x19a2: 0x00d9, 0x19a3: 0x0f99, - 0x19a4: 0x2039, 0x19a5: 0x0269, 0x19a6: 0x01d9, 0x19a7: 0x0fa9, 0x19a8: 0x0fb9, 0x19a9: 0x1089, - 0x19aa: 0x0279, 0x19ab: 0x0369, 0x19ac: 0x0289, 0x19ad: 0x13d1, 0x19ae: 0x0039, 0x19af: 0x0ee9, - 0x19b0: 0x1159, 0x19b1: 0x0ef9, 0x19b2: 0x0f09, 0x19b3: 0x1199, 0x19b4: 0x0f31, 0x19b5: 0x0249, - 0x19b6: 0x0f41, 0x19b7: 0x0259, 0x19b8: 0x0f51, 0x19b9: 0x0359, 0x19ba: 0x0f61, 0x19bb: 0x0f71, - 0x19bc: 0x00d9, 0x19bd: 0x0f99, 0x19be: 0x2039, 0x19bf: 0x0269, + 0x1980: 0x0f41, 0x1981: 0x0259, 0x1982: 0x0f51, 0x1983: 0x0359, 0x1984: 0x0f61, 0x1985: 0x0040, + 0x1986: 0x00d9, 0x1987: 0x0040, 0x1988: 0x0040, 0x1989: 0x0040, 0x198a: 0x01d9, 0x198b: 0x0fa9, + 0x198c: 0x0fb9, 0x198d: 0x1089, 0x198e: 0x0279, 0x198f: 0x0369, 0x1990: 0x0289, 0x1991: 0x0040, + 0x1992: 0x0039, 0x1993: 0x0ee9, 0x1994: 0x1159, 0x1995: 0x0ef9, 0x1996: 0x0f09, 0x1997: 0x1199, + 0x1998: 0x0f31, 0x1999: 0x0249, 0x199a: 0x0f41, 0x199b: 0x0259, 0x199c: 0x0f51, 0x199d: 0x0359, + 0x199e: 0x0f61, 0x199f: 0x0f71, 0x19a0: 0x00d9, 0x19a1: 0x0f99, 0x19a2: 0x2039, 0x19a3: 0x0269, + 0x19a4: 0x01d9, 0x19a5: 0x0fa9, 0x19a6: 0x0fb9, 0x19a7: 0x1089, 0x19a8: 0x0279, 0x19a9: 0x0369, + 0x19aa: 0x0289, 0x19ab: 0x13d1, 0x19ac: 0x0039, 0x19ad: 0x0ee9, 0x19ae: 0x1159, 0x19af: 0x0ef9, + 0x19b0: 0x0f09, 0x19b1: 0x1199, 0x19b2: 0x0f31, 0x19b3: 0x0249, 0x19b4: 0x0f41, 0x19b5: 0x0259, + 0x19b6: 0x0f51, 0x19b7: 0x0359, 0x19b8: 0x0f61, 0x19b9: 0x0f71, 0x19ba: 0x00d9, 0x19bb: 0x0f99, + 0x19bc: 0x2039, 0x19bd: 0x0269, 0x19be: 0x01d9, 0x19bf: 0x0fa9, // Block 0x67, offset 0x19c0 - 0x19c0: 0x01d9, 0x19c1: 0x0fa9, 0x19c2: 0x0fb9, 0x19c3: 0x1089, 0x19c4: 0x0279, 0x19c5: 0x0369, - 0x19c6: 0x0289, 0x19c7: 0x13d1, 0x19c8: 0x0039, 0x19c9: 0x0ee9, 0x19ca: 0x1159, 0x19cb: 0x0ef9, - 0x19cc: 0x0f09, 0x19cd: 0x1199, 0x19ce: 0x0f31, 0x19cf: 0x0249, 0x19d0: 0x0f41, 0x19d1: 0x0259, - 0x19d2: 0x0f51, 0x19d3: 0x0359, 0x19d4: 0x0f61, 0x19d5: 0x0f71, 0x19d6: 0x00d9, 0x19d7: 0x0f99, - 0x19d8: 0x2039, 0x19d9: 0x0269, 0x19da: 0x01d9, 0x19db: 0x0fa9, 0x19dc: 0x0fb9, 0x19dd: 0x1089, - 0x19de: 0x0279, 0x19df: 0x0369, 0x19e0: 0x0289, 0x19e1: 0x13d1, 0x19e2: 0x0039, 0x19e3: 0x0ee9, - 0x19e4: 0x1159, 0x19e5: 0x0ef9, 0x19e6: 0x0f09, 0x19e7: 0x1199, 0x19e8: 0x0f31, 0x19e9: 0x0249, - 0x19ea: 0x0f41, 0x19eb: 0x0259, 0x19ec: 0x0f51, 0x19ed: 0x0359, 0x19ee: 0x0f61, 0x19ef: 0x0f71, - 0x19f0: 0x00d9, 0x19f1: 0x0f99, 0x19f2: 0x2039, 0x19f3: 0x0269, 0x19f4: 0x01d9, 0x19f5: 0x0fa9, - 0x19f6: 0x0fb9, 0x19f7: 0x1089, 0x19f8: 0x0279, 0x19f9: 0x0369, 0x19fa: 0x0289, 0x19fb: 0x13d1, - 0x19fc: 0x0039, 0x19fd: 0x0ee9, 0x19fe: 0x1159, 0x19ff: 0x0ef9, + 0x19c0: 0x0fb9, 0x19c1: 0x1089, 0x19c2: 0x0279, 0x19c3: 0x0369, 0x19c4: 0x0289, 0x19c5: 0x13d1, + 0x19c6: 0x0039, 0x19c7: 0x0ee9, 0x19c8: 0x1159, 0x19c9: 0x0ef9, 0x19ca: 0x0f09, 0x19cb: 0x1199, + 0x19cc: 0x0f31, 0x19cd: 0x0249, 0x19ce: 0x0f41, 0x19cf: 0x0259, 0x19d0: 0x0f51, 0x19d1: 0x0359, + 0x19d2: 0x0f61, 0x19d3: 0x0f71, 0x19d4: 0x00d9, 0x19d5: 0x0f99, 0x19d6: 0x2039, 0x19d7: 0x0269, + 0x19d8: 0x01d9, 0x19d9: 0x0fa9, 0x19da: 0x0fb9, 0x19db: 0x1089, 0x19dc: 0x0279, 0x19dd: 0x0369, + 0x19de: 0x0289, 0x19df: 0x13d1, 0x19e0: 0x0039, 0x19e1: 0x0ee9, 0x19e2: 0x1159, 0x19e3: 0x0ef9, + 0x19e4: 0x0f09, 0x19e5: 0x1199, 0x19e6: 0x0f31, 0x19e7: 0x0249, 0x19e8: 0x0f41, 0x19e9: 0x0259, + 0x19ea: 0x0f51, 0x19eb: 0x0359, 0x19ec: 0x0f61, 0x19ed: 0x0f71, 0x19ee: 0x00d9, 0x19ef: 0x0f99, + 0x19f0: 0x2039, 0x19f1: 0x0269, 0x19f2: 0x01d9, 0x19f3: 0x0fa9, 0x19f4: 0x0fb9, 0x19f5: 0x1089, + 0x19f6: 0x0279, 0x19f7: 0x0369, 0x19f8: 0x0289, 0x19f9: 0x13d1, 0x19fa: 0x0039, 0x19fb: 0x0ee9, + 0x19fc: 0x1159, 0x19fd: 0x0ef9, 0x19fe: 0x0f09, 0x19ff: 0x1199, // Block 0x68, offset 0x1a00 - 0x1a00: 0x0f09, 0x1a01: 0x1199, 0x1a02: 0x0f31, 0x1a03: 0x0249, 0x1a04: 0x0f41, 0x1a05: 0x0259, - 0x1a06: 0x0f51, 0x1a07: 0x0359, 0x1a08: 0x0f61, 0x1a09: 0x0f71, 0x1a0a: 0x00d9, 0x1a0b: 0x0f99, - 0x1a0c: 0x2039, 0x1a0d: 0x0269, 0x1a0e: 0x01d9, 0x1a0f: 0x0fa9, 0x1a10: 0x0fb9, 0x1a11: 0x1089, - 0x1a12: 0x0279, 0x1a13: 0x0369, 0x1a14: 0x0289, 0x1a15: 0x13d1, 0x1a16: 0x0039, 0x1a17: 0x0ee9, - 0x1a18: 0x1159, 0x1a19: 0x0ef9, 0x1a1a: 0x0f09, 0x1a1b: 0x1199, 0x1a1c: 0x0f31, 0x1a1d: 0x0249, - 0x1a1e: 0x0f41, 0x1a1f: 0x0259, 0x1a20: 0x0f51, 0x1a21: 0x0359, 0x1a22: 0x0f61, 0x1a23: 0x0f71, - 0x1a24: 0x00d9, 0x1a25: 0x0f99, 0x1a26: 0x2039, 0x1a27: 0x0269, 0x1a28: 0x01d9, 0x1a29: 0x0fa9, - 0x1a2a: 0x0fb9, 0x1a2b: 0x1089, 0x1a2c: 0x0279, 0x1a2d: 0x0369, 0x1a2e: 0x0289, 0x1a2f: 0x13d1, - 0x1a30: 0x0039, 0x1a31: 0x0ee9, 0x1a32: 0x1159, 0x1a33: 0x0ef9, 0x1a34: 0x0f09, 0x1a35: 0x1199, - 0x1a36: 0x0f31, 0x1a37: 0x0249, 0x1a38: 0x0f41, 0x1a39: 0x0259, 0x1a3a: 0x0f51, 0x1a3b: 0x0359, - 0x1a3c: 0x0f61, 0x1a3d: 0x0f71, 0x1a3e: 0x00d9, 0x1a3f: 0x0f99, + 0x1a00: 0x0f31, 0x1a01: 0x0249, 0x1a02: 0x0f41, 0x1a03: 0x0259, 0x1a04: 0x0f51, 0x1a05: 0x0359, + 0x1a06: 0x0f61, 0x1a07: 0x0f71, 0x1a08: 0x00d9, 0x1a09: 0x0f99, 0x1a0a: 0x2039, 0x1a0b: 0x0269, + 0x1a0c: 0x01d9, 0x1a0d: 0x0fa9, 0x1a0e: 0x0fb9, 0x1a0f: 0x1089, 0x1a10: 0x0279, 0x1a11: 0x0369, + 0x1a12: 0x0289, 0x1a13: 0x13d1, 0x1a14: 0x0039, 0x1a15: 0x0ee9, 0x1a16: 0x1159, 0x1a17: 0x0ef9, + 0x1a18: 0x0f09, 0x1a19: 0x1199, 0x1a1a: 0x0f31, 0x1a1b: 0x0249, 0x1a1c: 0x0f41, 0x1a1d: 0x0259, + 0x1a1e: 0x0f51, 0x1a1f: 0x0359, 0x1a20: 0x0f61, 0x1a21: 0x0f71, 0x1a22: 0x00d9, 0x1a23: 0x0f99, + 0x1a24: 0x2039, 0x1a25: 0x0269, 0x1a26: 0x01d9, 0x1a27: 0x0fa9, 0x1a28: 0x0fb9, 0x1a29: 0x1089, + 0x1a2a: 0x0279, 0x1a2b: 0x0369, 0x1a2c: 0x0289, 0x1a2d: 0x13d1, 0x1a2e: 0x0039, 0x1a2f: 0x0ee9, + 0x1a30: 0x1159, 0x1a31: 0x0ef9, 0x1a32: 0x0f09, 0x1a33: 0x1199, 0x1a34: 0x0f31, 0x1a35: 0x0249, + 0x1a36: 0x0f41, 0x1a37: 0x0259, 0x1a38: 0x0f51, 0x1a39: 0x0359, 0x1a3a: 0x0f61, 0x1a3b: 0x0f71, + 0x1a3c: 0x00d9, 0x1a3d: 0x0f99, 0x1a3e: 0x2039, 0x1a3f: 0x0269, // Block 0x69, offset 0x1a40 - 0x1a40: 0x2039, 0x1a41: 0x0269, 0x1a42: 0x01d9, 0x1a43: 0x0fa9, 0x1a44: 0x0fb9, 0x1a45: 0x1089, - 0x1a46: 0x0279, 0x1a47: 0x0369, 0x1a48: 0x0289, 0x1a49: 0x13d1, 0x1a4a: 0x0039, 0x1a4b: 0x0ee9, - 0x1a4c: 0x1159, 0x1a4d: 0x0ef9, 0x1a4e: 0x0f09, 0x1a4f: 0x1199, 0x1a50: 0x0f31, 0x1a51: 0x0249, - 0x1a52: 0x0f41, 0x1a53: 0x0259, 0x1a54: 0x0f51, 0x1a55: 0x0359, 0x1a56: 0x0f61, 0x1a57: 0x0f71, - 0x1a58: 0x00d9, 0x1a59: 0x0f99, 0x1a5a: 0x2039, 0x1a5b: 0x0269, 0x1a5c: 0x01d9, 0x1a5d: 0x0fa9, - 0x1a5e: 0x0fb9, 0x1a5f: 0x1089, 0x1a60: 0x0279, 0x1a61: 0x0369, 0x1a62: 0x0289, 0x1a63: 0x13d1, - 0x1a64: 0xba81, 0x1a65: 0xba99, 0x1a66: 0x0040, 0x1a67: 0x0040, 0x1a68: 0xbab1, 0x1a69: 0x1099, - 0x1a6a: 0x10b1, 0x1a6b: 0x10c9, 0x1a6c: 0xbac9, 0x1a6d: 0xbae1, 0x1a6e: 0xbaf9, 0x1a6f: 0x1429, - 0x1a70: 0x1a31, 0x1a71: 0xbb11, 0x1a72: 0xbb29, 0x1a73: 0xbb41, 0x1a74: 0xbb59, 0x1a75: 0xbb71, - 0x1a76: 0xbb89, 0x1a77: 0x2109, 0x1a78: 0x1111, 0x1a79: 0x1429, 0x1a7a: 0xbba1, 0x1a7b: 0xbbb9, - 0x1a7c: 0xbbd1, 0x1a7d: 0x10e1, 0x1a7e: 0x10f9, 0x1a7f: 0xbbe9, + 0x1a40: 0x01d9, 0x1a41: 0x0fa9, 0x1a42: 0x0fb9, 0x1a43: 0x1089, 0x1a44: 0x0279, 0x1a45: 0x0369, + 0x1a46: 0x0289, 0x1a47: 0x13d1, 0x1a48: 0x0039, 0x1a49: 0x0ee9, 0x1a4a: 0x1159, 0x1a4b: 0x0ef9, + 0x1a4c: 0x0f09, 0x1a4d: 0x1199, 0x1a4e: 0x0f31, 0x1a4f: 0x0249, 0x1a50: 0x0f41, 0x1a51: 0x0259, + 0x1a52: 0x0f51, 0x1a53: 0x0359, 0x1a54: 0x0f61, 0x1a55: 0x0f71, 0x1a56: 0x00d9, 0x1a57: 0x0f99, + 0x1a58: 0x2039, 0x1a59: 0x0269, 0x1a5a: 0x01d9, 0x1a5b: 0x0fa9, 0x1a5c: 0x0fb9, 0x1a5d: 0x1089, + 0x1a5e: 0x0279, 0x1a5f: 0x0369, 0x1a60: 0x0289, 0x1a61: 0x13d1, 0x1a62: 0x0039, 0x1a63: 0x0ee9, + 0x1a64: 0x1159, 0x1a65: 0x0ef9, 0x1a66: 0x0f09, 0x1a67: 0x1199, 0x1a68: 0x0f31, 0x1a69: 0x0249, + 0x1a6a: 0x0f41, 0x1a6b: 0x0259, 0x1a6c: 0x0f51, 0x1a6d: 0x0359, 0x1a6e: 0x0f61, 0x1a6f: 0x0f71, + 0x1a70: 0x00d9, 0x1a71: 0x0f99, 0x1a72: 0x2039, 0x1a73: 0x0269, 0x1a74: 0x01d9, 0x1a75: 0x0fa9, + 0x1a76: 0x0fb9, 0x1a77: 0x1089, 0x1a78: 0x0279, 0x1a79: 0x0369, 0x1a7a: 0x0289, 0x1a7b: 0x13d1, + 0x1a7c: 0x0039, 0x1a7d: 0x0ee9, 0x1a7e: 0x1159, 0x1a7f: 0x0ef9, // Block 0x6a, offset 0x1a80 - 0x1a80: 0x2079, 0x1a81: 0xbc01, 0x1a82: 0xbab1, 0x1a83: 0x1099, 0x1a84: 0x10b1, 0x1a85: 0x10c9, - 0x1a86: 0xbac9, 0x1a87: 0xbae1, 0x1a88: 0xbaf9, 0x1a89: 0x1429, 0x1a8a: 0x1a31, 0x1a8b: 0xbb11, - 0x1a8c: 0xbb29, 0x1a8d: 0xbb41, 0x1a8e: 0xbb59, 0x1a8f: 0xbb71, 0x1a90: 0xbb89, 0x1a91: 0x2109, - 0x1a92: 0x1111, 0x1a93: 0xbba1, 0x1a94: 0xbba1, 0x1a95: 0xbbb9, 0x1a96: 0xbbd1, 0x1a97: 0x10e1, - 0x1a98: 0x10f9, 0x1a99: 0xbbe9, 0x1a9a: 0x2079, 0x1a9b: 0xbc21, 0x1a9c: 0xbac9, 0x1a9d: 0x1429, - 0x1a9e: 0xbb11, 0x1a9f: 0x10e1, 0x1aa0: 0x1111, 0x1aa1: 0x2109, 0x1aa2: 0xbab1, 0x1aa3: 0x1099, - 0x1aa4: 0x10b1, 0x1aa5: 0x10c9, 0x1aa6: 0xbac9, 0x1aa7: 0xbae1, 0x1aa8: 0xbaf9, 0x1aa9: 0x1429, - 0x1aaa: 0x1a31, 0x1aab: 0xbb11, 0x1aac: 0xbb29, 0x1aad: 0xbb41, 0x1aae: 0xbb59, 0x1aaf: 0xbb71, - 0x1ab0: 0xbb89, 0x1ab1: 0x2109, 0x1ab2: 0x1111, 0x1ab3: 0x1429, 0x1ab4: 0xbba1, 0x1ab5: 0xbbb9, - 0x1ab6: 0xbbd1, 0x1ab7: 0x10e1, 0x1ab8: 0x10f9, 0x1ab9: 0xbbe9, 0x1aba: 0x2079, 0x1abb: 0xbc01, - 0x1abc: 0xbab1, 0x1abd: 0x1099, 0x1abe: 0x10b1, 0x1abf: 0x10c9, + 0x1a80: 0x0f09, 0x1a81: 0x1199, 0x1a82: 0x0f31, 0x1a83: 0x0249, 0x1a84: 0x0f41, 0x1a85: 0x0259, + 0x1a86: 0x0f51, 0x1a87: 0x0359, 0x1a88: 0x0f61, 0x1a89: 0x0f71, 0x1a8a: 0x00d9, 0x1a8b: 0x0f99, + 0x1a8c: 0x2039, 0x1a8d: 0x0269, 0x1a8e: 0x01d9, 0x1a8f: 0x0fa9, 0x1a90: 0x0fb9, 0x1a91: 0x1089, + 0x1a92: 0x0279, 0x1a93: 0x0369, 0x1a94: 0x0289, 0x1a95: 0x13d1, 0x1a96: 0x0039, 0x1a97: 0x0ee9, + 0x1a98: 0x1159, 0x1a99: 0x0ef9, 0x1a9a: 0x0f09, 0x1a9b: 0x1199, 0x1a9c: 0x0f31, 0x1a9d: 0x0249, + 0x1a9e: 0x0f41, 0x1a9f: 0x0259, 0x1aa0: 0x0f51, 0x1aa1: 0x0359, 0x1aa2: 0x0f61, 0x1aa3: 0x0f71, + 0x1aa4: 0x00d9, 0x1aa5: 0x0f99, 0x1aa6: 0x2039, 0x1aa7: 0x0269, 0x1aa8: 0x01d9, 0x1aa9: 0x0fa9, + 0x1aaa: 0x0fb9, 0x1aab: 0x1089, 0x1aac: 0x0279, 0x1aad: 0x0369, 0x1aae: 0x0289, 0x1aaf: 0x13d1, + 0x1ab0: 0x0039, 0x1ab1: 0x0ee9, 0x1ab2: 0x1159, 0x1ab3: 0x0ef9, 0x1ab4: 0x0f09, 0x1ab5: 0x1199, + 0x1ab6: 0x0f31, 0x1ab7: 0x0249, 0x1ab8: 0x0f41, 0x1ab9: 0x0259, 0x1aba: 0x0f51, 0x1abb: 0x0359, + 0x1abc: 0x0f61, 0x1abd: 0x0f71, 0x1abe: 0x00d9, 0x1abf: 0x0f99, // Block 0x6b, offset 0x1ac0 - 0x1ac0: 0xbac9, 0x1ac1: 0xbae1, 0x1ac2: 0xbaf9, 0x1ac3: 0x1429, 0x1ac4: 0x1a31, 0x1ac5: 0xbb11, - 0x1ac6: 0xbb29, 0x1ac7: 0xbb41, 0x1ac8: 0xbb59, 0x1ac9: 0xbb71, 0x1aca: 0xbb89, 0x1acb: 0x2109, - 0x1acc: 0x1111, 0x1acd: 0xbba1, 0x1ace: 0xbba1, 0x1acf: 0xbbb9, 0x1ad0: 0xbbd1, 0x1ad1: 0x10e1, - 0x1ad2: 0x10f9, 0x1ad3: 0xbbe9, 0x1ad4: 0x2079, 0x1ad5: 0xbc21, 0x1ad6: 0xbac9, 0x1ad7: 0x1429, - 0x1ad8: 0xbb11, 0x1ad9: 0x10e1, 0x1ada: 0x1111, 0x1adb: 0x2109, 0x1adc: 0xbab1, 0x1add: 0x1099, - 0x1ade: 0x10b1, 0x1adf: 0x10c9, 0x1ae0: 0xbac9, 0x1ae1: 0xbae1, 0x1ae2: 0xbaf9, 0x1ae3: 0x1429, - 0x1ae4: 0x1a31, 0x1ae5: 0xbb11, 0x1ae6: 0xbb29, 0x1ae7: 0xbb41, 0x1ae8: 0xbb59, 0x1ae9: 0xbb71, - 0x1aea: 0xbb89, 0x1aeb: 0x2109, 0x1aec: 0x1111, 0x1aed: 0x1429, 0x1aee: 0xbba1, 0x1aef: 0xbbb9, - 0x1af0: 0xbbd1, 0x1af1: 0x10e1, 0x1af2: 0x10f9, 0x1af3: 0xbbe9, 0x1af4: 0x2079, 0x1af5: 0xbc01, - 0x1af6: 0xbab1, 0x1af7: 0x1099, 0x1af8: 0x10b1, 0x1af9: 0x10c9, 0x1afa: 0xbac9, 0x1afb: 0xbae1, - 0x1afc: 0xbaf9, 0x1afd: 0x1429, 0x1afe: 0x1a31, 0x1aff: 0xbb11, + 0x1ac0: 0x2039, 0x1ac1: 0x0269, 0x1ac2: 0x01d9, 0x1ac3: 0x0fa9, 0x1ac4: 0x0fb9, 0x1ac5: 0x1089, + 0x1ac6: 0x0279, 0x1ac7: 0x0369, 0x1ac8: 0x0289, 0x1ac9: 0x13d1, 0x1aca: 0x0039, 0x1acb: 0x0ee9, + 0x1acc: 0x1159, 0x1acd: 0x0ef9, 0x1ace: 0x0f09, 0x1acf: 0x1199, 0x1ad0: 0x0f31, 0x1ad1: 0x0249, + 0x1ad2: 0x0f41, 0x1ad3: 0x0259, 0x1ad4: 0x0f51, 0x1ad5: 0x0359, 0x1ad6: 0x0f61, 0x1ad7: 0x0f71, + 0x1ad8: 0x00d9, 0x1ad9: 0x0f99, 0x1ada: 0x2039, 0x1adb: 0x0269, 0x1adc: 0x01d9, 0x1add: 0x0fa9, + 0x1ade: 0x0fb9, 0x1adf: 0x1089, 0x1ae0: 0x0279, 0x1ae1: 0x0369, 0x1ae2: 0x0289, 0x1ae3: 0x13d1, + 0x1ae4: 0xba81, 0x1ae5: 0xba99, 0x1ae6: 0x0040, 0x1ae7: 0x0040, 0x1ae8: 0xbab1, 0x1ae9: 0x1099, + 0x1aea: 0x10b1, 0x1aeb: 0x10c9, 0x1aec: 0xbac9, 0x1aed: 0xbae1, 0x1aee: 0xbaf9, 0x1aef: 0x1429, + 0x1af0: 0x1a31, 0x1af1: 0xbb11, 0x1af2: 0xbb29, 0x1af3: 0xbb41, 0x1af4: 0xbb59, 0x1af5: 0xbb71, + 0x1af6: 0xbb89, 0x1af7: 0x2109, 0x1af8: 0x1111, 0x1af9: 0x1429, 0x1afa: 0xbba1, 0x1afb: 0xbbb9, + 0x1afc: 0xbbd1, 0x1afd: 0x10e1, 0x1afe: 0x10f9, 0x1aff: 0xbbe9, // Block 0x6c, offset 0x1b00 - 0x1b00: 0xbb29, 0x1b01: 0xbb41, 0x1b02: 0xbb59, 0x1b03: 0xbb71, 0x1b04: 0xbb89, 0x1b05: 0x2109, - 0x1b06: 0x1111, 0x1b07: 0xbba1, 0x1b08: 0xbba1, 0x1b09: 0xbbb9, 0x1b0a: 0xbbd1, 0x1b0b: 0x10e1, - 0x1b0c: 0x10f9, 0x1b0d: 0xbbe9, 0x1b0e: 0x2079, 0x1b0f: 0xbc21, 0x1b10: 0xbac9, 0x1b11: 0x1429, - 0x1b12: 0xbb11, 0x1b13: 0x10e1, 0x1b14: 0x1111, 0x1b15: 0x2109, 0x1b16: 0xbab1, 0x1b17: 0x1099, - 0x1b18: 0x10b1, 0x1b19: 0x10c9, 0x1b1a: 0xbac9, 0x1b1b: 0xbae1, 0x1b1c: 0xbaf9, 0x1b1d: 0x1429, - 0x1b1e: 0x1a31, 0x1b1f: 0xbb11, 0x1b20: 0xbb29, 0x1b21: 0xbb41, 0x1b22: 0xbb59, 0x1b23: 0xbb71, - 0x1b24: 0xbb89, 0x1b25: 0x2109, 0x1b26: 0x1111, 0x1b27: 0x1429, 0x1b28: 0xbba1, 0x1b29: 0xbbb9, - 0x1b2a: 0xbbd1, 0x1b2b: 0x10e1, 0x1b2c: 0x10f9, 0x1b2d: 0xbbe9, 0x1b2e: 0x2079, 0x1b2f: 0xbc01, - 0x1b30: 0xbab1, 0x1b31: 0x1099, 0x1b32: 0x10b1, 0x1b33: 0x10c9, 0x1b34: 0xbac9, 0x1b35: 0xbae1, - 0x1b36: 0xbaf9, 0x1b37: 0x1429, 0x1b38: 0x1a31, 0x1b39: 0xbb11, 0x1b3a: 0xbb29, 0x1b3b: 0xbb41, - 0x1b3c: 0xbb59, 0x1b3d: 0xbb71, 0x1b3e: 0xbb89, 0x1b3f: 0x2109, + 0x1b00: 0x2079, 0x1b01: 0xbc01, 0x1b02: 0xbab1, 0x1b03: 0x1099, 0x1b04: 0x10b1, 0x1b05: 0x10c9, + 0x1b06: 0xbac9, 0x1b07: 0xbae1, 0x1b08: 0xbaf9, 0x1b09: 0x1429, 0x1b0a: 0x1a31, 0x1b0b: 0xbb11, + 0x1b0c: 0xbb29, 0x1b0d: 0xbb41, 0x1b0e: 0xbb59, 0x1b0f: 0xbb71, 0x1b10: 0xbb89, 0x1b11: 0x2109, + 0x1b12: 0x1111, 0x1b13: 0xbba1, 0x1b14: 0xbba1, 0x1b15: 0xbbb9, 0x1b16: 0xbbd1, 0x1b17: 0x10e1, + 0x1b18: 0x10f9, 0x1b19: 0xbbe9, 0x1b1a: 0x2079, 0x1b1b: 0xbc21, 0x1b1c: 0xbac9, 0x1b1d: 0x1429, + 0x1b1e: 0xbb11, 0x1b1f: 0x10e1, 0x1b20: 0x1111, 0x1b21: 0x2109, 0x1b22: 0xbab1, 0x1b23: 0x1099, + 0x1b24: 0x10b1, 0x1b25: 0x10c9, 0x1b26: 0xbac9, 0x1b27: 0xbae1, 0x1b28: 0xbaf9, 0x1b29: 0x1429, + 0x1b2a: 0x1a31, 0x1b2b: 0xbb11, 0x1b2c: 0xbb29, 0x1b2d: 0xbb41, 0x1b2e: 0xbb59, 0x1b2f: 0xbb71, + 0x1b30: 0xbb89, 0x1b31: 0x2109, 0x1b32: 0x1111, 0x1b33: 0x1429, 0x1b34: 0xbba1, 0x1b35: 0xbbb9, + 0x1b36: 0xbbd1, 0x1b37: 0x10e1, 0x1b38: 0x10f9, 0x1b39: 0xbbe9, 0x1b3a: 0x2079, 0x1b3b: 0xbc01, + 0x1b3c: 0xbab1, 0x1b3d: 0x1099, 0x1b3e: 0x10b1, 0x1b3f: 0x10c9, // Block 0x6d, offset 0x1b40 - 0x1b40: 0x1111, 0x1b41: 0xbba1, 0x1b42: 0xbba1, 0x1b43: 0xbbb9, 0x1b44: 0xbbd1, 0x1b45: 0x10e1, - 0x1b46: 0x10f9, 0x1b47: 0xbbe9, 0x1b48: 0x2079, 0x1b49: 0xbc21, 0x1b4a: 0xbac9, 0x1b4b: 0x1429, - 0x1b4c: 0xbb11, 0x1b4d: 0x10e1, 0x1b4e: 0x1111, 0x1b4f: 0x2109, 0x1b50: 0xbab1, 0x1b51: 0x1099, - 0x1b52: 0x10b1, 0x1b53: 0x10c9, 0x1b54: 0xbac9, 0x1b55: 0xbae1, 0x1b56: 0xbaf9, 0x1b57: 0x1429, - 0x1b58: 0x1a31, 0x1b59: 0xbb11, 0x1b5a: 0xbb29, 0x1b5b: 0xbb41, 0x1b5c: 0xbb59, 0x1b5d: 0xbb71, - 0x1b5e: 0xbb89, 0x1b5f: 0x2109, 0x1b60: 0x1111, 0x1b61: 0x1429, 0x1b62: 0xbba1, 0x1b63: 0xbbb9, - 0x1b64: 0xbbd1, 0x1b65: 0x10e1, 0x1b66: 0x10f9, 0x1b67: 0xbbe9, 0x1b68: 0x2079, 0x1b69: 0xbc01, - 0x1b6a: 0xbab1, 0x1b6b: 0x1099, 0x1b6c: 0x10b1, 0x1b6d: 0x10c9, 0x1b6e: 0xbac9, 0x1b6f: 0xbae1, - 0x1b70: 0xbaf9, 0x1b71: 0x1429, 0x1b72: 0x1a31, 0x1b73: 0xbb11, 0x1b74: 0xbb29, 0x1b75: 0xbb41, - 0x1b76: 0xbb59, 0x1b77: 0xbb71, 0x1b78: 0xbb89, 0x1b79: 0x2109, 0x1b7a: 0x1111, 0x1b7b: 0xbba1, - 0x1b7c: 0xbba1, 0x1b7d: 0xbbb9, 0x1b7e: 0xbbd1, 0x1b7f: 0x10e1, + 0x1b40: 0xbac9, 0x1b41: 0xbae1, 0x1b42: 0xbaf9, 0x1b43: 0x1429, 0x1b44: 0x1a31, 0x1b45: 0xbb11, + 0x1b46: 0xbb29, 0x1b47: 0xbb41, 0x1b48: 0xbb59, 0x1b49: 0xbb71, 0x1b4a: 0xbb89, 0x1b4b: 0x2109, + 0x1b4c: 0x1111, 0x1b4d: 0xbba1, 0x1b4e: 0xbba1, 0x1b4f: 0xbbb9, 0x1b50: 0xbbd1, 0x1b51: 0x10e1, + 0x1b52: 0x10f9, 0x1b53: 0xbbe9, 0x1b54: 0x2079, 0x1b55: 0xbc21, 0x1b56: 0xbac9, 0x1b57: 0x1429, + 0x1b58: 0xbb11, 0x1b59: 0x10e1, 0x1b5a: 0x1111, 0x1b5b: 0x2109, 0x1b5c: 0xbab1, 0x1b5d: 0x1099, + 0x1b5e: 0x10b1, 0x1b5f: 0x10c9, 0x1b60: 0xbac9, 0x1b61: 0xbae1, 0x1b62: 0xbaf9, 0x1b63: 0x1429, + 0x1b64: 0x1a31, 0x1b65: 0xbb11, 0x1b66: 0xbb29, 0x1b67: 0xbb41, 0x1b68: 0xbb59, 0x1b69: 0xbb71, + 0x1b6a: 0xbb89, 0x1b6b: 0x2109, 0x1b6c: 0x1111, 0x1b6d: 0x1429, 0x1b6e: 0xbba1, 0x1b6f: 0xbbb9, + 0x1b70: 0xbbd1, 0x1b71: 0x10e1, 0x1b72: 0x10f9, 0x1b73: 0xbbe9, 0x1b74: 0x2079, 0x1b75: 0xbc01, + 0x1b76: 0xbab1, 0x1b77: 0x1099, 0x1b78: 0x10b1, 0x1b79: 0x10c9, 0x1b7a: 0xbac9, 0x1b7b: 0xbae1, + 0x1b7c: 0xbaf9, 0x1b7d: 0x1429, 0x1b7e: 0x1a31, 0x1b7f: 0xbb11, // Block 0x6e, offset 0x1b80 - 0x1b80: 0x10f9, 0x1b81: 0xbbe9, 0x1b82: 0x2079, 0x1b83: 0xbc21, 0x1b84: 0xbac9, 0x1b85: 0x1429, - 0x1b86: 0xbb11, 0x1b87: 0x10e1, 0x1b88: 0x1111, 0x1b89: 0x2109, 0x1b8a: 0xbc41, 0x1b8b: 0xbc41, - 0x1b8c: 0x0040, 0x1b8d: 0x0040, 0x1b8e: 0x1f41, 0x1b8f: 0x00c9, 0x1b90: 0x0069, 0x1b91: 0x0079, - 0x1b92: 0x1f51, 0x1b93: 0x1f61, 0x1b94: 0x1f71, 0x1b95: 0x1f81, 0x1b96: 0x1f91, 0x1b97: 0x1fa1, - 0x1b98: 0x1f41, 0x1b99: 0x00c9, 0x1b9a: 0x0069, 0x1b9b: 0x0079, 0x1b9c: 0x1f51, 0x1b9d: 0x1f61, - 0x1b9e: 0x1f71, 0x1b9f: 0x1f81, 0x1ba0: 0x1f91, 0x1ba1: 0x1fa1, 0x1ba2: 0x1f41, 0x1ba3: 0x00c9, - 0x1ba4: 0x0069, 0x1ba5: 0x0079, 0x1ba6: 0x1f51, 0x1ba7: 0x1f61, 0x1ba8: 0x1f71, 0x1ba9: 0x1f81, - 0x1baa: 0x1f91, 0x1bab: 0x1fa1, 0x1bac: 0x1f41, 0x1bad: 0x00c9, 0x1bae: 0x0069, 0x1baf: 0x0079, - 0x1bb0: 0x1f51, 0x1bb1: 0x1f61, 0x1bb2: 0x1f71, 0x1bb3: 0x1f81, 0x1bb4: 0x1f91, 0x1bb5: 0x1fa1, - 0x1bb6: 0x1f41, 0x1bb7: 0x00c9, 0x1bb8: 0x0069, 0x1bb9: 0x0079, 0x1bba: 0x1f51, 0x1bbb: 0x1f61, - 0x1bbc: 0x1f71, 0x1bbd: 0x1f81, 0x1bbe: 0x1f91, 0x1bbf: 0x1fa1, + 0x1b80: 0xbb29, 0x1b81: 0xbb41, 0x1b82: 0xbb59, 0x1b83: 0xbb71, 0x1b84: 0xbb89, 0x1b85: 0x2109, + 0x1b86: 0x1111, 0x1b87: 0xbba1, 0x1b88: 0xbba1, 0x1b89: 0xbbb9, 0x1b8a: 0xbbd1, 0x1b8b: 0x10e1, + 0x1b8c: 0x10f9, 0x1b8d: 0xbbe9, 0x1b8e: 0x2079, 0x1b8f: 0xbc21, 0x1b90: 0xbac9, 0x1b91: 0x1429, + 0x1b92: 0xbb11, 0x1b93: 0x10e1, 0x1b94: 0x1111, 0x1b95: 0x2109, 0x1b96: 0xbab1, 0x1b97: 0x1099, + 0x1b98: 0x10b1, 0x1b99: 0x10c9, 0x1b9a: 0xbac9, 0x1b9b: 0xbae1, 0x1b9c: 0xbaf9, 0x1b9d: 0x1429, + 0x1b9e: 0x1a31, 0x1b9f: 0xbb11, 0x1ba0: 0xbb29, 0x1ba1: 0xbb41, 0x1ba2: 0xbb59, 0x1ba3: 0xbb71, + 0x1ba4: 0xbb89, 0x1ba5: 0x2109, 0x1ba6: 0x1111, 0x1ba7: 0x1429, 0x1ba8: 0xbba1, 0x1ba9: 0xbbb9, + 0x1baa: 0xbbd1, 0x1bab: 0x10e1, 0x1bac: 0x10f9, 0x1bad: 0xbbe9, 0x1bae: 0x2079, 0x1baf: 0xbc01, + 0x1bb0: 0xbab1, 0x1bb1: 0x1099, 0x1bb2: 0x10b1, 0x1bb3: 0x10c9, 0x1bb4: 0xbac9, 0x1bb5: 0xbae1, + 0x1bb6: 0xbaf9, 0x1bb7: 0x1429, 0x1bb8: 0x1a31, 0x1bb9: 0xbb11, 0x1bba: 0xbb29, 0x1bbb: 0xbb41, + 0x1bbc: 0xbb59, 0x1bbd: 0xbb71, 0x1bbe: 0xbb89, 0x1bbf: 0x2109, // Block 0x6f, offset 0x1bc0 - 0x1bc0: 0xe115, 0x1bc1: 0xe115, 0x1bc2: 0xe135, 0x1bc3: 0xe135, 0x1bc4: 0xe115, 0x1bc5: 0xe115, - 0x1bc6: 0xe175, 0x1bc7: 0xe175, 0x1bc8: 0xe115, 0x1bc9: 0xe115, 0x1bca: 0xe135, 0x1bcb: 0xe135, - 0x1bcc: 0xe115, 0x1bcd: 0xe115, 0x1bce: 0xe1f5, 0x1bcf: 0xe1f5, 0x1bd0: 0xe115, 0x1bd1: 0xe115, - 0x1bd2: 0xe135, 0x1bd3: 0xe135, 0x1bd4: 0xe115, 0x1bd5: 0xe115, 0x1bd6: 0xe175, 0x1bd7: 0xe175, - 0x1bd8: 0xe115, 0x1bd9: 0xe115, 0x1bda: 0xe135, 0x1bdb: 0xe135, 0x1bdc: 0xe115, 0x1bdd: 0xe115, - 0x1bde: 0x8b05, 0x1bdf: 0x8b05, 0x1be0: 0x04b5, 0x1be1: 0x04b5, 0x1be2: 0x0208, 0x1be3: 0x0208, - 0x1be4: 0x0208, 0x1be5: 0x0208, 0x1be6: 0x0208, 0x1be7: 0x0208, 0x1be8: 0x0208, 0x1be9: 0x0208, - 0x1bea: 0x0208, 0x1beb: 0x0208, 0x1bec: 0x0208, 0x1bed: 0x0208, 0x1bee: 0x0208, 0x1bef: 0x0208, - 0x1bf0: 0x0208, 0x1bf1: 0x0208, 0x1bf2: 0x0208, 0x1bf3: 0x0208, 0x1bf4: 0x0208, 0x1bf5: 0x0208, - 0x1bf6: 0x0208, 0x1bf7: 0x0208, 0x1bf8: 0x0208, 0x1bf9: 0x0208, 0x1bfa: 0x0208, 0x1bfb: 0x0208, - 0x1bfc: 0x0208, 0x1bfd: 0x0208, 0x1bfe: 0x0208, 0x1bff: 0x0208, + 0x1bc0: 0x1111, 0x1bc1: 0xbba1, 0x1bc2: 0xbba1, 0x1bc3: 0xbbb9, 0x1bc4: 0xbbd1, 0x1bc5: 0x10e1, + 0x1bc6: 0x10f9, 0x1bc7: 0xbbe9, 0x1bc8: 0x2079, 0x1bc9: 0xbc21, 0x1bca: 0xbac9, 0x1bcb: 0x1429, + 0x1bcc: 0xbb11, 0x1bcd: 0x10e1, 0x1bce: 0x1111, 0x1bcf: 0x2109, 0x1bd0: 0xbab1, 0x1bd1: 0x1099, + 0x1bd2: 0x10b1, 0x1bd3: 0x10c9, 0x1bd4: 0xbac9, 0x1bd5: 0xbae1, 0x1bd6: 0xbaf9, 0x1bd7: 0x1429, + 0x1bd8: 0x1a31, 0x1bd9: 0xbb11, 0x1bda: 0xbb29, 0x1bdb: 0xbb41, 0x1bdc: 0xbb59, 0x1bdd: 0xbb71, + 0x1bde: 0xbb89, 0x1bdf: 0x2109, 0x1be0: 0x1111, 0x1be1: 0x1429, 0x1be2: 0xbba1, 0x1be3: 0xbbb9, + 0x1be4: 0xbbd1, 0x1be5: 0x10e1, 0x1be6: 0x10f9, 0x1be7: 0xbbe9, 0x1be8: 0x2079, 0x1be9: 0xbc01, + 0x1bea: 0xbab1, 0x1beb: 0x1099, 0x1bec: 0x10b1, 0x1bed: 0x10c9, 0x1bee: 0xbac9, 0x1bef: 0xbae1, + 0x1bf0: 0xbaf9, 0x1bf1: 0x1429, 0x1bf2: 0x1a31, 0x1bf3: 0xbb11, 0x1bf4: 0xbb29, 0x1bf5: 0xbb41, + 0x1bf6: 0xbb59, 0x1bf7: 0xbb71, 0x1bf8: 0xbb89, 0x1bf9: 0x2109, 0x1bfa: 0x1111, 0x1bfb: 0xbba1, + 0x1bfc: 0xbba1, 0x1bfd: 0xbbb9, 0x1bfe: 0xbbd1, 0x1bff: 0x10e1, // Block 0x70, offset 0x1c00 - 0x1c00: 0xb189, 0x1c01: 0xb1a1, 0x1c02: 0xb201, 0x1c03: 0xb249, 0x1c04: 0x0040, 0x1c05: 0xb411, - 0x1c06: 0xb291, 0x1c07: 0xb219, 0x1c08: 0xb309, 0x1c09: 0xb429, 0x1c0a: 0xb399, 0x1c0b: 0xb3b1, - 0x1c0c: 0xb3c9, 0x1c0d: 0xb3e1, 0x1c0e: 0xb2a9, 0x1c0f: 0xb339, 0x1c10: 0xb369, 0x1c11: 0xb2d9, - 0x1c12: 0xb381, 0x1c13: 0xb279, 0x1c14: 0xb2c1, 0x1c15: 0xb1d1, 0x1c16: 0xb1e9, 0x1c17: 0xb231, - 0x1c18: 0xb261, 0x1c19: 0xb2f1, 0x1c1a: 0xb321, 0x1c1b: 0xb351, 0x1c1c: 0xbc59, 0x1c1d: 0x7949, - 0x1c1e: 0xbc71, 0x1c1f: 0xbc89, 0x1c20: 0x0040, 0x1c21: 0xb1a1, 0x1c22: 0xb201, 0x1c23: 0x0040, - 0x1c24: 0xb3f9, 0x1c25: 0x0040, 0x1c26: 0x0040, 0x1c27: 0xb219, 0x1c28: 0x0040, 0x1c29: 0xb429, - 0x1c2a: 0xb399, 0x1c2b: 0xb3b1, 0x1c2c: 0xb3c9, 0x1c2d: 0xb3e1, 0x1c2e: 0xb2a9, 0x1c2f: 0xb339, - 0x1c30: 0xb369, 0x1c31: 0xb2d9, 0x1c32: 0xb381, 0x1c33: 0x0040, 0x1c34: 0xb2c1, 0x1c35: 0xb1d1, - 0x1c36: 0xb1e9, 0x1c37: 0xb231, 0x1c38: 0x0040, 0x1c39: 0xb2f1, 0x1c3a: 0x0040, 0x1c3b: 0xb351, - 0x1c3c: 0x0040, 0x1c3d: 0x0040, 0x1c3e: 0x0040, 0x1c3f: 0x0040, + 0x1c00: 0x10f9, 0x1c01: 0xbbe9, 0x1c02: 0x2079, 0x1c03: 0xbc21, 0x1c04: 0xbac9, 0x1c05: 0x1429, + 0x1c06: 0xbb11, 0x1c07: 0x10e1, 0x1c08: 0x1111, 0x1c09: 0x2109, 0x1c0a: 0xbc41, 0x1c0b: 0xbc41, + 0x1c0c: 0x0040, 0x1c0d: 0x0040, 0x1c0e: 0x1f41, 0x1c0f: 0x00c9, 0x1c10: 0x0069, 0x1c11: 0x0079, + 0x1c12: 0x1f51, 0x1c13: 0x1f61, 0x1c14: 0x1f71, 0x1c15: 0x1f81, 0x1c16: 0x1f91, 0x1c17: 0x1fa1, + 0x1c18: 0x1f41, 0x1c19: 0x00c9, 0x1c1a: 0x0069, 0x1c1b: 0x0079, 0x1c1c: 0x1f51, 0x1c1d: 0x1f61, + 0x1c1e: 0x1f71, 0x1c1f: 0x1f81, 0x1c20: 0x1f91, 0x1c21: 0x1fa1, 0x1c22: 0x1f41, 0x1c23: 0x00c9, + 0x1c24: 0x0069, 0x1c25: 0x0079, 0x1c26: 0x1f51, 0x1c27: 0x1f61, 0x1c28: 0x1f71, 0x1c29: 0x1f81, + 0x1c2a: 0x1f91, 0x1c2b: 0x1fa1, 0x1c2c: 0x1f41, 0x1c2d: 0x00c9, 0x1c2e: 0x0069, 0x1c2f: 0x0079, + 0x1c30: 0x1f51, 0x1c31: 0x1f61, 0x1c32: 0x1f71, 0x1c33: 0x1f81, 0x1c34: 0x1f91, 0x1c35: 0x1fa1, + 0x1c36: 0x1f41, 0x1c37: 0x00c9, 0x1c38: 0x0069, 0x1c39: 0x0079, 0x1c3a: 0x1f51, 0x1c3b: 0x1f61, + 0x1c3c: 0x1f71, 0x1c3d: 0x1f81, 0x1c3e: 0x1f91, 0x1c3f: 0x1fa1, // Block 0x71, offset 0x1c40 - 0x1c40: 0x0040, 0x1c41: 0x0040, 0x1c42: 0xb201, 0x1c43: 0x0040, 0x1c44: 0x0040, 0x1c45: 0x0040, - 0x1c46: 0x0040, 0x1c47: 0xb219, 0x1c48: 0x0040, 0x1c49: 0xb429, 0x1c4a: 0x0040, 0x1c4b: 0xb3b1, - 0x1c4c: 0x0040, 0x1c4d: 0xb3e1, 0x1c4e: 0xb2a9, 0x1c4f: 0xb339, 0x1c50: 0x0040, 0x1c51: 0xb2d9, - 0x1c52: 0xb381, 0x1c53: 0x0040, 0x1c54: 0xb2c1, 0x1c55: 0x0040, 0x1c56: 0x0040, 0x1c57: 0xb231, - 0x1c58: 0x0040, 0x1c59: 0xb2f1, 0x1c5a: 0x0040, 0x1c5b: 0xb351, 0x1c5c: 0x0040, 0x1c5d: 0x7949, - 0x1c5e: 0x0040, 0x1c5f: 0xbc89, 0x1c60: 0x0040, 0x1c61: 0xb1a1, 0x1c62: 0xb201, 0x1c63: 0x0040, - 0x1c64: 0xb3f9, 0x1c65: 0x0040, 0x1c66: 0x0040, 0x1c67: 0xb219, 0x1c68: 0xb309, 0x1c69: 0xb429, - 0x1c6a: 0xb399, 0x1c6b: 0x0040, 0x1c6c: 0xb3c9, 0x1c6d: 0xb3e1, 0x1c6e: 0xb2a9, 0x1c6f: 0xb339, - 0x1c70: 0xb369, 0x1c71: 0xb2d9, 0x1c72: 0xb381, 0x1c73: 0x0040, 0x1c74: 0xb2c1, 0x1c75: 0xb1d1, - 0x1c76: 0xb1e9, 0x1c77: 0xb231, 0x1c78: 0x0040, 0x1c79: 0xb2f1, 0x1c7a: 0xb321, 0x1c7b: 0xb351, - 0x1c7c: 0xbc59, 0x1c7d: 0x0040, 0x1c7e: 0xbc71, 0x1c7f: 0x0040, + 0x1c40: 0xe115, 0x1c41: 0xe115, 0x1c42: 0xe135, 0x1c43: 0xe135, 0x1c44: 0xe115, 0x1c45: 0xe115, + 0x1c46: 0xe175, 0x1c47: 0xe175, 0x1c48: 0xe115, 0x1c49: 0xe115, 0x1c4a: 0xe135, 0x1c4b: 0xe135, + 0x1c4c: 0xe115, 0x1c4d: 0xe115, 0x1c4e: 0xe1f5, 0x1c4f: 0xe1f5, 0x1c50: 0xe115, 0x1c51: 0xe115, + 0x1c52: 0xe135, 0x1c53: 0xe135, 0x1c54: 0xe115, 0x1c55: 0xe115, 0x1c56: 0xe175, 0x1c57: 0xe175, + 0x1c58: 0xe115, 0x1c59: 0xe115, 0x1c5a: 0xe135, 0x1c5b: 0xe135, 0x1c5c: 0xe115, 0x1c5d: 0xe115, + 0x1c5e: 0x8b05, 0x1c5f: 0x8b05, 0x1c60: 0x04b5, 0x1c61: 0x04b5, 0x1c62: 0x0a08, 0x1c63: 0x0a08, + 0x1c64: 0x0a08, 0x1c65: 0x0a08, 0x1c66: 0x0a08, 0x1c67: 0x0a08, 0x1c68: 0x0a08, 0x1c69: 0x0a08, + 0x1c6a: 0x0a08, 0x1c6b: 0x0a08, 0x1c6c: 0x0a08, 0x1c6d: 0x0a08, 0x1c6e: 0x0a08, 0x1c6f: 0x0a08, + 0x1c70: 0x0a08, 0x1c71: 0x0a08, 0x1c72: 0x0a08, 0x1c73: 0x0a08, 0x1c74: 0x0a08, 0x1c75: 0x0a08, + 0x1c76: 0x0a08, 0x1c77: 0x0a08, 0x1c78: 0x0a08, 0x1c79: 0x0a08, 0x1c7a: 0x0a08, 0x1c7b: 0x0a08, + 0x1c7c: 0x0a08, 0x1c7d: 0x0a08, 0x1c7e: 0x0a08, 0x1c7f: 0x0a08, // Block 0x72, offset 0x1c80 - 0x1c80: 0xb189, 0x1c81: 0xb1a1, 0x1c82: 0xb201, 0x1c83: 0xb249, 0x1c84: 0xb3f9, 0x1c85: 0xb411, - 0x1c86: 0xb291, 0x1c87: 0xb219, 0x1c88: 0xb309, 0x1c89: 0xb429, 0x1c8a: 0x0040, 0x1c8b: 0xb3b1, + 0x1c80: 0xb189, 0x1c81: 0xb1a1, 0x1c82: 0xb201, 0x1c83: 0xb249, 0x1c84: 0x0040, 0x1c85: 0xb411, + 0x1c86: 0xb291, 0x1c87: 0xb219, 0x1c88: 0xb309, 0x1c89: 0xb429, 0x1c8a: 0xb399, 0x1c8b: 0xb3b1, 0x1c8c: 0xb3c9, 0x1c8d: 0xb3e1, 0x1c8e: 0xb2a9, 0x1c8f: 0xb339, 0x1c90: 0xb369, 0x1c91: 0xb2d9, 0x1c92: 0xb381, 0x1c93: 0xb279, 0x1c94: 0xb2c1, 0x1c95: 0xb1d1, 0x1c96: 0xb1e9, 0x1c97: 0xb231, - 0x1c98: 0xb261, 0x1c99: 0xb2f1, 0x1c9a: 0xb321, 0x1c9b: 0xb351, 0x1c9c: 0x0040, 0x1c9d: 0x0040, - 0x1c9e: 0x0040, 0x1c9f: 0x0040, 0x1ca0: 0x0040, 0x1ca1: 0xb1a1, 0x1ca2: 0xb201, 0x1ca3: 0xb249, - 0x1ca4: 0x0040, 0x1ca5: 0xb411, 0x1ca6: 0xb291, 0x1ca7: 0xb219, 0x1ca8: 0xb309, 0x1ca9: 0xb429, - 0x1caa: 0x0040, 0x1cab: 0xb3b1, 0x1cac: 0xb3c9, 0x1cad: 0xb3e1, 0x1cae: 0xb2a9, 0x1caf: 0xb339, - 0x1cb0: 0xb369, 0x1cb1: 0xb2d9, 0x1cb2: 0xb381, 0x1cb3: 0xb279, 0x1cb4: 0xb2c1, 0x1cb5: 0xb1d1, - 0x1cb6: 0xb1e9, 0x1cb7: 0xb231, 0x1cb8: 0xb261, 0x1cb9: 0xb2f1, 0x1cba: 0xb321, 0x1cbb: 0xb351, + 0x1c98: 0xb261, 0x1c99: 0xb2f1, 0x1c9a: 0xb321, 0x1c9b: 0xb351, 0x1c9c: 0xbc59, 0x1c9d: 0x7949, + 0x1c9e: 0xbc71, 0x1c9f: 0xbc89, 0x1ca0: 0x0040, 0x1ca1: 0xb1a1, 0x1ca2: 0xb201, 0x1ca3: 0x0040, + 0x1ca4: 0xb3f9, 0x1ca5: 0x0040, 0x1ca6: 0x0040, 0x1ca7: 0xb219, 0x1ca8: 0x0040, 0x1ca9: 0xb429, + 0x1caa: 0xb399, 0x1cab: 0xb3b1, 0x1cac: 0xb3c9, 0x1cad: 0xb3e1, 0x1cae: 0xb2a9, 0x1caf: 0xb339, + 0x1cb0: 0xb369, 0x1cb1: 0xb2d9, 0x1cb2: 0xb381, 0x1cb3: 0x0040, 0x1cb4: 0xb2c1, 0x1cb5: 0xb1d1, + 0x1cb6: 0xb1e9, 0x1cb7: 0xb231, 0x1cb8: 0x0040, 0x1cb9: 0xb2f1, 0x1cba: 0x0040, 0x1cbb: 0xb351, 0x1cbc: 0x0040, 0x1cbd: 0x0040, 0x1cbe: 0x0040, 0x1cbf: 0x0040, // Block 0x73, offset 0x1cc0 - 0x1cc0: 0x0040, 0x1cc1: 0xbca2, 0x1cc2: 0xbcba, 0x1cc3: 0xbcd2, 0x1cc4: 0xbcea, 0x1cc5: 0xbd02, - 0x1cc6: 0xbd1a, 0x1cc7: 0xbd32, 0x1cc8: 0xbd4a, 0x1cc9: 0xbd62, 0x1cca: 0xbd7a, 0x1ccb: 0x0018, - 0x1ccc: 0x0018, 0x1ccd: 0x0040, 0x1cce: 0x0040, 0x1ccf: 0x0040, 0x1cd0: 0xbd92, 0x1cd1: 0xbdb2, - 0x1cd2: 0xbdd2, 0x1cd3: 0xbdf2, 0x1cd4: 0xbe12, 0x1cd5: 0xbe32, 0x1cd6: 0xbe52, 0x1cd7: 0xbe72, - 0x1cd8: 0xbe92, 0x1cd9: 0xbeb2, 0x1cda: 0xbed2, 0x1cdb: 0xbef2, 0x1cdc: 0xbf12, 0x1cdd: 0xbf32, - 0x1cde: 0xbf52, 0x1cdf: 0xbf72, 0x1ce0: 0xbf92, 0x1ce1: 0xbfb2, 0x1ce2: 0xbfd2, 0x1ce3: 0xbff2, - 0x1ce4: 0xc012, 0x1ce5: 0xc032, 0x1ce6: 0xc052, 0x1ce7: 0xc072, 0x1ce8: 0xc092, 0x1ce9: 0xc0b2, - 0x1cea: 0xc0d1, 0x1ceb: 0x1159, 0x1cec: 0x0269, 0x1ced: 0x6671, 0x1cee: 0xc111, 0x1cef: 0x0040, - 0x1cf0: 0x0039, 0x1cf1: 0x0ee9, 0x1cf2: 0x1159, 0x1cf3: 0x0ef9, 0x1cf4: 0x0f09, 0x1cf5: 0x1199, - 0x1cf6: 0x0f31, 0x1cf7: 0x0249, 0x1cf8: 0x0f41, 0x1cf9: 0x0259, 0x1cfa: 0x0f51, 0x1cfb: 0x0359, - 0x1cfc: 0x0f61, 0x1cfd: 0x0f71, 0x1cfe: 0x00d9, 0x1cff: 0x0f99, + 0x1cc0: 0x0040, 0x1cc1: 0x0040, 0x1cc2: 0xb201, 0x1cc3: 0x0040, 0x1cc4: 0x0040, 0x1cc5: 0x0040, + 0x1cc6: 0x0040, 0x1cc7: 0xb219, 0x1cc8: 0x0040, 0x1cc9: 0xb429, 0x1cca: 0x0040, 0x1ccb: 0xb3b1, + 0x1ccc: 0x0040, 0x1ccd: 0xb3e1, 0x1cce: 0xb2a9, 0x1ccf: 0xb339, 0x1cd0: 0x0040, 0x1cd1: 0xb2d9, + 0x1cd2: 0xb381, 0x1cd3: 0x0040, 0x1cd4: 0xb2c1, 0x1cd5: 0x0040, 0x1cd6: 0x0040, 0x1cd7: 0xb231, + 0x1cd8: 0x0040, 0x1cd9: 0xb2f1, 0x1cda: 0x0040, 0x1cdb: 0xb351, 0x1cdc: 0x0040, 0x1cdd: 0x7949, + 0x1cde: 0x0040, 0x1cdf: 0xbc89, 0x1ce0: 0x0040, 0x1ce1: 0xb1a1, 0x1ce2: 0xb201, 0x1ce3: 0x0040, + 0x1ce4: 0xb3f9, 0x1ce5: 0x0040, 0x1ce6: 0x0040, 0x1ce7: 0xb219, 0x1ce8: 0xb309, 0x1ce9: 0xb429, + 0x1cea: 0xb399, 0x1ceb: 0x0040, 0x1cec: 0xb3c9, 0x1ced: 0xb3e1, 0x1cee: 0xb2a9, 0x1cef: 0xb339, + 0x1cf0: 0xb369, 0x1cf1: 0xb2d9, 0x1cf2: 0xb381, 0x1cf3: 0x0040, 0x1cf4: 0xb2c1, 0x1cf5: 0xb1d1, + 0x1cf6: 0xb1e9, 0x1cf7: 0xb231, 0x1cf8: 0x0040, 0x1cf9: 0xb2f1, 0x1cfa: 0xb321, 0x1cfb: 0xb351, + 0x1cfc: 0xbc59, 0x1cfd: 0x0040, 0x1cfe: 0xbc71, 0x1cff: 0x0040, // Block 0x74, offset 0x1d00 - 0x1d00: 0x2039, 0x1d01: 0x0269, 0x1d02: 0x01d9, 0x1d03: 0x0fa9, 0x1d04: 0x0fb9, 0x1d05: 0x1089, - 0x1d06: 0x0279, 0x1d07: 0x0369, 0x1d08: 0x0289, 0x1d09: 0x13d1, 0x1d0a: 0xc129, 0x1d0b: 0x65b1, - 0x1d0c: 0xc141, 0x1d0d: 0x1441, 0x1d0e: 0xc159, 0x1d0f: 0xc179, 0x1d10: 0x0018, 0x1d11: 0x0018, - 0x1d12: 0x0018, 0x1d13: 0x0018, 0x1d14: 0x0018, 0x1d15: 0x0018, 0x1d16: 0x0018, 0x1d17: 0x0018, - 0x1d18: 0x0018, 0x1d19: 0x0018, 0x1d1a: 0x0018, 0x1d1b: 0x0018, 0x1d1c: 0x0018, 0x1d1d: 0x0018, - 0x1d1e: 0x0018, 0x1d1f: 0x0018, 0x1d20: 0x0018, 0x1d21: 0x0018, 0x1d22: 0x0018, 0x1d23: 0x0018, - 0x1d24: 0x0018, 0x1d25: 0x0018, 0x1d26: 0x0018, 0x1d27: 0x0018, 0x1d28: 0x0018, 0x1d29: 0x0018, - 0x1d2a: 0xc191, 0x1d2b: 0xc1a9, 0x1d2c: 0x0040, 0x1d2d: 0x0040, 0x1d2e: 0x0040, 0x1d2f: 0x0040, - 0x1d30: 0x0018, 0x1d31: 0x0018, 0x1d32: 0x0018, 0x1d33: 0x0018, 0x1d34: 0x0018, 0x1d35: 0x0018, - 0x1d36: 0x0018, 0x1d37: 0x0018, 0x1d38: 0x0018, 0x1d39: 0x0018, 0x1d3a: 0x0018, 0x1d3b: 0x0018, - 0x1d3c: 0x0018, 0x1d3d: 0x0018, 0x1d3e: 0x0018, 0x1d3f: 0x0018, + 0x1d00: 0xb189, 0x1d01: 0xb1a1, 0x1d02: 0xb201, 0x1d03: 0xb249, 0x1d04: 0xb3f9, 0x1d05: 0xb411, + 0x1d06: 0xb291, 0x1d07: 0xb219, 0x1d08: 0xb309, 0x1d09: 0xb429, 0x1d0a: 0x0040, 0x1d0b: 0xb3b1, + 0x1d0c: 0xb3c9, 0x1d0d: 0xb3e1, 0x1d0e: 0xb2a9, 0x1d0f: 0xb339, 0x1d10: 0xb369, 0x1d11: 0xb2d9, + 0x1d12: 0xb381, 0x1d13: 0xb279, 0x1d14: 0xb2c1, 0x1d15: 0xb1d1, 0x1d16: 0xb1e9, 0x1d17: 0xb231, + 0x1d18: 0xb261, 0x1d19: 0xb2f1, 0x1d1a: 0xb321, 0x1d1b: 0xb351, 0x1d1c: 0x0040, 0x1d1d: 0x0040, + 0x1d1e: 0x0040, 0x1d1f: 0x0040, 0x1d20: 0x0040, 0x1d21: 0xb1a1, 0x1d22: 0xb201, 0x1d23: 0xb249, + 0x1d24: 0x0040, 0x1d25: 0xb411, 0x1d26: 0xb291, 0x1d27: 0xb219, 0x1d28: 0xb309, 0x1d29: 0xb429, + 0x1d2a: 0x0040, 0x1d2b: 0xb3b1, 0x1d2c: 0xb3c9, 0x1d2d: 0xb3e1, 0x1d2e: 0xb2a9, 0x1d2f: 0xb339, + 0x1d30: 0xb369, 0x1d31: 0xb2d9, 0x1d32: 0xb381, 0x1d33: 0xb279, 0x1d34: 0xb2c1, 0x1d35: 0xb1d1, + 0x1d36: 0xb1e9, 0x1d37: 0xb231, 0x1d38: 0xb261, 0x1d39: 0xb2f1, 0x1d3a: 0xb321, 0x1d3b: 0xb351, + 0x1d3c: 0x0040, 0x1d3d: 0x0040, 0x1d3e: 0x0040, 0x1d3f: 0x0040, // Block 0x75, offset 0x1d40 - 0x1d40: 0xc1d9, 0x1d41: 0xc211, 0x1d42: 0xc249, 0x1d43: 0x0040, 0x1d44: 0x0040, 0x1d45: 0x0040, - 0x1d46: 0x0040, 0x1d47: 0x0040, 0x1d48: 0x0040, 0x1d49: 0x0040, 0x1d4a: 0x0040, 0x1d4b: 0x0040, - 0x1d4c: 0x0040, 0x1d4d: 0x0040, 0x1d4e: 0x0040, 0x1d4f: 0x0040, 0x1d50: 0xc269, 0x1d51: 0xc289, - 0x1d52: 0xc2a9, 0x1d53: 0xc2c9, 0x1d54: 0xc2e9, 0x1d55: 0xc309, 0x1d56: 0xc329, 0x1d57: 0xc349, - 0x1d58: 0xc369, 0x1d59: 0xc389, 0x1d5a: 0xc3a9, 0x1d5b: 0xc3c9, 0x1d5c: 0xc3e9, 0x1d5d: 0xc409, - 0x1d5e: 0xc429, 0x1d5f: 0xc449, 0x1d60: 0xc469, 0x1d61: 0xc489, 0x1d62: 0xc4a9, 0x1d63: 0xc4c9, - 0x1d64: 0xc4e9, 0x1d65: 0xc509, 0x1d66: 0xc529, 0x1d67: 0xc549, 0x1d68: 0xc569, 0x1d69: 0xc589, - 0x1d6a: 0xc5a9, 0x1d6b: 0xc5c9, 0x1d6c: 0xc5e9, 0x1d6d: 0xc609, 0x1d6e: 0xc629, 0x1d6f: 0xc649, - 0x1d70: 0xc669, 0x1d71: 0xc689, 0x1d72: 0xc6a9, 0x1d73: 0xc6c9, 0x1d74: 0xc6e9, 0x1d75: 0xc709, - 0x1d76: 0xc729, 0x1d77: 0xc749, 0x1d78: 0xc769, 0x1d79: 0xc789, 0x1d7a: 0xc7a9, 0x1d7b: 0xc7c9, - 0x1d7c: 0x0040, 0x1d7d: 0x0040, 0x1d7e: 0x0040, 0x1d7f: 0x0040, + 0x1d40: 0x0040, 0x1d41: 0xbca2, 0x1d42: 0xbcba, 0x1d43: 0xbcd2, 0x1d44: 0xbcea, 0x1d45: 0xbd02, + 0x1d46: 0xbd1a, 0x1d47: 0xbd32, 0x1d48: 0xbd4a, 0x1d49: 0xbd62, 0x1d4a: 0xbd7a, 0x1d4b: 0x0018, + 0x1d4c: 0x0018, 0x1d4d: 0x0040, 0x1d4e: 0x0040, 0x1d4f: 0x0040, 0x1d50: 0xbd92, 0x1d51: 0xbdb2, + 0x1d52: 0xbdd2, 0x1d53: 0xbdf2, 0x1d54: 0xbe12, 0x1d55: 0xbe32, 0x1d56: 0xbe52, 0x1d57: 0xbe72, + 0x1d58: 0xbe92, 0x1d59: 0xbeb2, 0x1d5a: 0xbed2, 0x1d5b: 0xbef2, 0x1d5c: 0xbf12, 0x1d5d: 0xbf32, + 0x1d5e: 0xbf52, 0x1d5f: 0xbf72, 0x1d60: 0xbf92, 0x1d61: 0xbfb2, 0x1d62: 0xbfd2, 0x1d63: 0xbff2, + 0x1d64: 0xc012, 0x1d65: 0xc032, 0x1d66: 0xc052, 0x1d67: 0xc072, 0x1d68: 0xc092, 0x1d69: 0xc0b2, + 0x1d6a: 0xc0d1, 0x1d6b: 0x1159, 0x1d6c: 0x0269, 0x1d6d: 0x6671, 0x1d6e: 0xc111, 0x1d6f: 0x0040, + 0x1d70: 0x0039, 0x1d71: 0x0ee9, 0x1d72: 0x1159, 0x1d73: 0x0ef9, 0x1d74: 0x0f09, 0x1d75: 0x1199, + 0x1d76: 0x0f31, 0x1d77: 0x0249, 0x1d78: 0x0f41, 0x1d79: 0x0259, 0x1d7a: 0x0f51, 0x1d7b: 0x0359, + 0x1d7c: 0x0f61, 0x1d7d: 0x0f71, 0x1d7e: 0x00d9, 0x1d7f: 0x0f99, // Block 0x76, offset 0x1d80 - 0x1d80: 0xcaf9, 0x1d81: 0xcb19, 0x1d82: 0xcb39, 0x1d83: 0x8b1d, 0x1d84: 0xcb59, 0x1d85: 0xcb79, - 0x1d86: 0xcb99, 0x1d87: 0xcbb9, 0x1d88: 0xcbd9, 0x1d89: 0xcbf9, 0x1d8a: 0xcc19, 0x1d8b: 0xcc39, - 0x1d8c: 0xcc59, 0x1d8d: 0x8b3d, 0x1d8e: 0xcc79, 0x1d8f: 0xcc99, 0x1d90: 0xccb9, 0x1d91: 0xccd9, - 0x1d92: 0x8b5d, 0x1d93: 0xccf9, 0x1d94: 0xcd19, 0x1d95: 0xc429, 0x1d96: 0x8b7d, 0x1d97: 0xcd39, - 0x1d98: 0xcd59, 0x1d99: 0xcd79, 0x1d9a: 0xcd99, 0x1d9b: 0xcdb9, 0x1d9c: 0x8b9d, 0x1d9d: 0xcdd9, - 0x1d9e: 0xcdf9, 0x1d9f: 0xce19, 0x1da0: 0xce39, 0x1da1: 0xce59, 0x1da2: 0xc789, 0x1da3: 0xce79, - 0x1da4: 0xce99, 0x1da5: 0xceb9, 0x1da6: 0xced9, 0x1da7: 0xcef9, 0x1da8: 0xcf19, 0x1da9: 0xcf39, - 0x1daa: 0xcf59, 0x1dab: 0xcf79, 0x1dac: 0xcf99, 0x1dad: 0xcfb9, 0x1dae: 0xcfd9, 0x1daf: 0xcff9, - 0x1db0: 0xd019, 0x1db1: 0xd039, 0x1db2: 0xd039, 0x1db3: 0xd039, 0x1db4: 0x8bbd, 0x1db5: 0xd059, - 0x1db6: 0xd079, 0x1db7: 0xd099, 0x1db8: 0x8bdd, 0x1db9: 0xd0b9, 0x1dba: 0xd0d9, 0x1dbb: 0xd0f9, - 0x1dbc: 0xd119, 0x1dbd: 0xd139, 0x1dbe: 0xd159, 0x1dbf: 0xd179, + 0x1d80: 0x2039, 0x1d81: 0x0269, 0x1d82: 0x01d9, 0x1d83: 0x0fa9, 0x1d84: 0x0fb9, 0x1d85: 0x1089, + 0x1d86: 0x0279, 0x1d87: 0x0369, 0x1d88: 0x0289, 0x1d89: 0x13d1, 0x1d8a: 0xc129, 0x1d8b: 0x65b1, + 0x1d8c: 0xc141, 0x1d8d: 0x1441, 0x1d8e: 0xc159, 0x1d8f: 0xc179, 0x1d90: 0x0018, 0x1d91: 0x0018, + 0x1d92: 0x0018, 0x1d93: 0x0018, 0x1d94: 0x0018, 0x1d95: 0x0018, 0x1d96: 0x0018, 0x1d97: 0x0018, + 0x1d98: 0x0018, 0x1d99: 0x0018, 0x1d9a: 0x0018, 0x1d9b: 0x0018, 0x1d9c: 0x0018, 0x1d9d: 0x0018, + 0x1d9e: 0x0018, 0x1d9f: 0x0018, 0x1da0: 0x0018, 0x1da1: 0x0018, 0x1da2: 0x0018, 0x1da3: 0x0018, + 0x1da4: 0x0018, 0x1da5: 0x0018, 0x1da6: 0x0018, 0x1da7: 0x0018, 0x1da8: 0x0018, 0x1da9: 0x0018, + 0x1daa: 0xc191, 0x1dab: 0xc1a9, 0x1dac: 0x0040, 0x1dad: 0x0040, 0x1dae: 0x0040, 0x1daf: 0x0040, + 0x1db0: 0x0018, 0x1db1: 0x0018, 0x1db2: 0x0018, 0x1db3: 0x0018, 0x1db4: 0x0018, 0x1db5: 0x0018, + 0x1db6: 0x0018, 0x1db7: 0x0018, 0x1db8: 0x0018, 0x1db9: 0x0018, 0x1dba: 0x0018, 0x1dbb: 0x0018, + 0x1dbc: 0x0018, 0x1dbd: 0x0018, 0x1dbe: 0x0018, 0x1dbf: 0x0018, // Block 0x77, offset 0x1dc0 - 0x1dc0: 0xd199, 0x1dc1: 0xd1b9, 0x1dc2: 0xd1d9, 0x1dc3: 0xd1f9, 0x1dc4: 0xd219, 0x1dc5: 0xd239, - 0x1dc6: 0xd239, 0x1dc7: 0xd259, 0x1dc8: 0xd279, 0x1dc9: 0xd299, 0x1dca: 0xd2b9, 0x1dcb: 0xd2d9, - 0x1dcc: 0xd2f9, 0x1dcd: 0xd319, 0x1dce: 0xd339, 0x1dcf: 0xd359, 0x1dd0: 0xd379, 0x1dd1: 0xd399, - 0x1dd2: 0xd3b9, 0x1dd3: 0xd3d9, 0x1dd4: 0xd3f9, 0x1dd5: 0xd419, 0x1dd6: 0xd439, 0x1dd7: 0xd459, - 0x1dd8: 0xd479, 0x1dd9: 0x8bfd, 0x1dda: 0xd499, 0x1ddb: 0xd4b9, 0x1ddc: 0xd4d9, 0x1ddd: 0xc309, - 0x1dde: 0xd4f9, 0x1ddf: 0xd519, 0x1de0: 0x8c1d, 0x1de1: 0x8c3d, 0x1de2: 0xd539, 0x1de3: 0xd559, - 0x1de4: 0xd579, 0x1de5: 0xd599, 0x1de6: 0xd5b9, 0x1de7: 0xd5d9, 0x1de8: 0x0040, 0x1de9: 0xd5f9, - 0x1dea: 0xd619, 0x1deb: 0xd619, 0x1dec: 0x8c5d, 0x1ded: 0xd639, 0x1dee: 0xd659, 0x1def: 0xd679, - 0x1df0: 0xd699, 0x1df1: 0x8c7d, 0x1df2: 0xd6b9, 0x1df3: 0xd6d9, 0x1df4: 0x0040, 0x1df5: 0xd6f9, - 0x1df6: 0xd719, 0x1df7: 0xd739, 0x1df8: 0xd759, 0x1df9: 0xd779, 0x1dfa: 0xd799, 0x1dfb: 0x8c9d, - 0x1dfc: 0xd7b9, 0x1dfd: 0x8cbd, 0x1dfe: 0xd7d9, 0x1dff: 0xd7f9, + 0x1dc0: 0xc1d9, 0x1dc1: 0xc211, 0x1dc2: 0xc249, 0x1dc3: 0x0040, 0x1dc4: 0x0040, 0x1dc5: 0x0040, + 0x1dc6: 0x0040, 0x1dc7: 0x0040, 0x1dc8: 0x0040, 0x1dc9: 0x0040, 0x1dca: 0x0040, 0x1dcb: 0x0040, + 0x1dcc: 0x0040, 0x1dcd: 0x0040, 0x1dce: 0x0040, 0x1dcf: 0x0040, 0x1dd0: 0xc269, 0x1dd1: 0xc289, + 0x1dd2: 0xc2a9, 0x1dd3: 0xc2c9, 0x1dd4: 0xc2e9, 0x1dd5: 0xc309, 0x1dd6: 0xc329, 0x1dd7: 0xc349, + 0x1dd8: 0xc369, 0x1dd9: 0xc389, 0x1dda: 0xc3a9, 0x1ddb: 0xc3c9, 0x1ddc: 0xc3e9, 0x1ddd: 0xc409, + 0x1dde: 0xc429, 0x1ddf: 0xc449, 0x1de0: 0xc469, 0x1de1: 0xc489, 0x1de2: 0xc4a9, 0x1de3: 0xc4c9, + 0x1de4: 0xc4e9, 0x1de5: 0xc509, 0x1de6: 0xc529, 0x1de7: 0xc549, 0x1de8: 0xc569, 0x1de9: 0xc589, + 0x1dea: 0xc5a9, 0x1deb: 0xc5c9, 0x1dec: 0xc5e9, 0x1ded: 0xc609, 0x1dee: 0xc629, 0x1def: 0xc649, + 0x1df0: 0xc669, 0x1df1: 0xc689, 0x1df2: 0xc6a9, 0x1df3: 0xc6c9, 0x1df4: 0xc6e9, 0x1df5: 0xc709, + 0x1df6: 0xc729, 0x1df7: 0xc749, 0x1df8: 0xc769, 0x1df9: 0xc789, 0x1dfa: 0xc7a9, 0x1dfb: 0xc7c9, + 0x1dfc: 0x0040, 0x1dfd: 0x0040, 0x1dfe: 0x0040, 0x1dff: 0x0040, // Block 0x78, offset 0x1e00 - 0x1e00: 0xd819, 0x1e01: 0xd839, 0x1e02: 0xd859, 0x1e03: 0xd879, 0x1e04: 0xd899, 0x1e05: 0xd8b9, - 0x1e06: 0xd8d9, 0x1e07: 0xd8f9, 0x1e08: 0xd919, 0x1e09: 0x8cdd, 0x1e0a: 0xd939, 0x1e0b: 0xd959, - 0x1e0c: 0xd979, 0x1e0d: 0xd999, 0x1e0e: 0xd9b9, 0x1e0f: 0x8cfd, 0x1e10: 0xd9d9, 0x1e11: 0x8d1d, - 0x1e12: 0x8d3d, 0x1e13: 0xd9f9, 0x1e14: 0xda19, 0x1e15: 0xda19, 0x1e16: 0xda39, 0x1e17: 0x8d5d, - 0x1e18: 0x8d7d, 0x1e19: 0xda59, 0x1e1a: 0xda79, 0x1e1b: 0xda99, 0x1e1c: 0xdab9, 0x1e1d: 0xdad9, - 0x1e1e: 0xdaf9, 0x1e1f: 0xdb19, 0x1e20: 0xdb39, 0x1e21: 0xdb59, 0x1e22: 0xdb79, 0x1e23: 0xdb99, - 0x1e24: 0x8d9d, 0x1e25: 0xdbb9, 0x1e26: 0xdbd9, 0x1e27: 0xdbf9, 0x1e28: 0xdc19, 0x1e29: 0xdbf9, - 0x1e2a: 0xdc39, 0x1e2b: 0xdc59, 0x1e2c: 0xdc79, 0x1e2d: 0xdc99, 0x1e2e: 0xdcb9, 0x1e2f: 0xdcd9, - 0x1e30: 0xdcf9, 0x1e31: 0xdd19, 0x1e32: 0xdd39, 0x1e33: 0xdd59, 0x1e34: 0xdd79, 0x1e35: 0xdd99, - 0x1e36: 0xddb9, 0x1e37: 0xddd9, 0x1e38: 0x8dbd, 0x1e39: 0xddf9, 0x1e3a: 0xde19, 0x1e3b: 0xde39, - 0x1e3c: 0xde59, 0x1e3d: 0xde79, 0x1e3e: 0x8ddd, 0x1e3f: 0xde99, + 0x1e00: 0xcaf9, 0x1e01: 0xcb19, 0x1e02: 0xcb39, 0x1e03: 0x8b1d, 0x1e04: 0xcb59, 0x1e05: 0xcb79, + 0x1e06: 0xcb99, 0x1e07: 0xcbb9, 0x1e08: 0xcbd9, 0x1e09: 0xcbf9, 0x1e0a: 0xcc19, 0x1e0b: 0xcc39, + 0x1e0c: 0xcc59, 0x1e0d: 0x8b3d, 0x1e0e: 0xcc79, 0x1e0f: 0xcc99, 0x1e10: 0xccb9, 0x1e11: 0xccd9, + 0x1e12: 0x8b5d, 0x1e13: 0xccf9, 0x1e14: 0xcd19, 0x1e15: 0xc429, 0x1e16: 0x8b7d, 0x1e17: 0xcd39, + 0x1e18: 0xcd59, 0x1e19: 0xcd79, 0x1e1a: 0xcd99, 0x1e1b: 0xcdb9, 0x1e1c: 0x8b9d, 0x1e1d: 0xcdd9, + 0x1e1e: 0xcdf9, 0x1e1f: 0xce19, 0x1e20: 0xce39, 0x1e21: 0xce59, 0x1e22: 0xc789, 0x1e23: 0xce79, + 0x1e24: 0xce99, 0x1e25: 0xceb9, 0x1e26: 0xced9, 0x1e27: 0xcef9, 0x1e28: 0xcf19, 0x1e29: 0xcf39, + 0x1e2a: 0xcf59, 0x1e2b: 0xcf79, 0x1e2c: 0xcf99, 0x1e2d: 0xcfb9, 0x1e2e: 0xcfd9, 0x1e2f: 0xcff9, + 0x1e30: 0xd019, 0x1e31: 0xd039, 0x1e32: 0xd039, 0x1e33: 0xd039, 0x1e34: 0x8bbd, 0x1e35: 0xd059, + 0x1e36: 0xd079, 0x1e37: 0xd099, 0x1e38: 0x8bdd, 0x1e39: 0xd0b9, 0x1e3a: 0xd0d9, 0x1e3b: 0xd0f9, + 0x1e3c: 0xd119, 0x1e3d: 0xd139, 0x1e3e: 0xd159, 0x1e3f: 0xd179, // Block 0x79, offset 0x1e40 - 0x1e40: 0xe599, 0x1e41: 0xe5b9, 0x1e42: 0xe5d9, 0x1e43: 0xe5f9, 0x1e44: 0xe619, 0x1e45: 0xe639, - 0x1e46: 0x8efd, 0x1e47: 0xe659, 0x1e48: 0xe679, 0x1e49: 0xe699, 0x1e4a: 0xe6b9, 0x1e4b: 0xe6d9, - 0x1e4c: 0xe6f9, 0x1e4d: 0x8f1d, 0x1e4e: 0xe719, 0x1e4f: 0xe739, 0x1e50: 0x8f3d, 0x1e51: 0x8f5d, - 0x1e52: 0xe759, 0x1e53: 0xe779, 0x1e54: 0xe799, 0x1e55: 0xe7b9, 0x1e56: 0xe7d9, 0x1e57: 0xe7f9, - 0x1e58: 0xe819, 0x1e59: 0xe839, 0x1e5a: 0xe859, 0x1e5b: 0x8f7d, 0x1e5c: 0xe879, 0x1e5d: 0x8f9d, - 0x1e5e: 0xe899, 0x1e5f: 0x0040, 0x1e60: 0xe8b9, 0x1e61: 0xe8d9, 0x1e62: 0xe8f9, 0x1e63: 0x8fbd, - 0x1e64: 0xe919, 0x1e65: 0xe939, 0x1e66: 0x8fdd, 0x1e67: 0x8ffd, 0x1e68: 0xe959, 0x1e69: 0xe979, - 0x1e6a: 0xe999, 0x1e6b: 0xe9b9, 0x1e6c: 0xe9d9, 0x1e6d: 0xe9d9, 0x1e6e: 0xe9f9, 0x1e6f: 0xea19, - 0x1e70: 0xea39, 0x1e71: 0xea59, 0x1e72: 0xea79, 0x1e73: 0xea99, 0x1e74: 0xeab9, 0x1e75: 0x901d, - 0x1e76: 0xead9, 0x1e77: 0x903d, 0x1e78: 0xeaf9, 0x1e79: 0x905d, 0x1e7a: 0xeb19, 0x1e7b: 0x907d, - 0x1e7c: 0x909d, 0x1e7d: 0x90bd, 0x1e7e: 0xeb39, 0x1e7f: 0xeb59, + 0x1e40: 0xd199, 0x1e41: 0xd1b9, 0x1e42: 0xd1d9, 0x1e43: 0xd1f9, 0x1e44: 0xd219, 0x1e45: 0xd239, + 0x1e46: 0xd239, 0x1e47: 0xd259, 0x1e48: 0xd279, 0x1e49: 0xd299, 0x1e4a: 0xd2b9, 0x1e4b: 0xd2d9, + 0x1e4c: 0xd2f9, 0x1e4d: 0xd319, 0x1e4e: 0xd339, 0x1e4f: 0xd359, 0x1e50: 0xd379, 0x1e51: 0xd399, + 0x1e52: 0xd3b9, 0x1e53: 0xd3d9, 0x1e54: 0xd3f9, 0x1e55: 0xd419, 0x1e56: 0xd439, 0x1e57: 0xd459, + 0x1e58: 0xd479, 0x1e59: 0x8bfd, 0x1e5a: 0xd499, 0x1e5b: 0xd4b9, 0x1e5c: 0xd4d9, 0x1e5d: 0xc309, + 0x1e5e: 0xd4f9, 0x1e5f: 0xd519, 0x1e60: 0x8c1d, 0x1e61: 0x8c3d, 0x1e62: 0xd539, 0x1e63: 0xd559, + 0x1e64: 0xd579, 0x1e65: 0xd599, 0x1e66: 0xd5b9, 0x1e67: 0xd5d9, 0x1e68: 0x2040, 0x1e69: 0xd5f9, + 0x1e6a: 0xd619, 0x1e6b: 0xd619, 0x1e6c: 0x8c5d, 0x1e6d: 0xd639, 0x1e6e: 0xd659, 0x1e6f: 0xd679, + 0x1e70: 0xd699, 0x1e71: 0x8c7d, 0x1e72: 0xd6b9, 0x1e73: 0xd6d9, 0x1e74: 0x2040, 0x1e75: 0xd6f9, + 0x1e76: 0xd719, 0x1e77: 0xd739, 0x1e78: 0xd759, 0x1e79: 0xd779, 0x1e7a: 0xd799, 0x1e7b: 0x8c9d, + 0x1e7c: 0xd7b9, 0x1e7d: 0x8cbd, 0x1e7e: 0xd7d9, 0x1e7f: 0xd7f9, // Block 0x7a, offset 0x1e80 - 0x1e80: 0xeb79, 0x1e81: 0x90dd, 0x1e82: 0x90fd, 0x1e83: 0x911d, 0x1e84: 0x913d, 0x1e85: 0xeb99, - 0x1e86: 0xebb9, 0x1e87: 0xebb9, 0x1e88: 0xebd9, 0x1e89: 0xebf9, 0x1e8a: 0xec19, 0x1e8b: 0xec39, - 0x1e8c: 0xec59, 0x1e8d: 0x915d, 0x1e8e: 0xec79, 0x1e8f: 0xec99, 0x1e90: 0xecb9, 0x1e91: 0xecd9, - 0x1e92: 0x917d, 0x1e93: 0xecf9, 0x1e94: 0x919d, 0x1e95: 0x91bd, 0x1e96: 0xed19, 0x1e97: 0xed39, - 0x1e98: 0xed59, 0x1e99: 0xed79, 0x1e9a: 0xed99, 0x1e9b: 0xedb9, 0x1e9c: 0x91dd, 0x1e9d: 0x91fd, - 0x1e9e: 0x921d, 0x1e9f: 0x0040, 0x1ea0: 0xedd9, 0x1ea1: 0x923d, 0x1ea2: 0xedf9, 0x1ea3: 0xee19, - 0x1ea4: 0xee39, 0x1ea5: 0x925d, 0x1ea6: 0xee59, 0x1ea7: 0xee79, 0x1ea8: 0xee99, 0x1ea9: 0xeeb9, - 0x1eaa: 0xeed9, 0x1eab: 0x927d, 0x1eac: 0xeef9, 0x1ead: 0xef19, 0x1eae: 0xef39, 0x1eaf: 0xef59, - 0x1eb0: 0xef79, 0x1eb1: 0xef99, 0x1eb2: 0x929d, 0x1eb3: 0x92bd, 0x1eb4: 0xefb9, 0x1eb5: 0x92dd, - 0x1eb6: 0xefd9, 0x1eb7: 0x92fd, 0x1eb8: 0xeff9, 0x1eb9: 0xf019, 0x1eba: 0xf039, 0x1ebb: 0x931d, - 0x1ebc: 0x933d, 0x1ebd: 0xf059, 0x1ebe: 0x935d, 0x1ebf: 0xf079, + 0x1e80: 0xd819, 0x1e81: 0xd839, 0x1e82: 0xd859, 0x1e83: 0xd879, 0x1e84: 0xd899, 0x1e85: 0xd8b9, + 0x1e86: 0xd8d9, 0x1e87: 0xd8f9, 0x1e88: 0xd919, 0x1e89: 0x8cdd, 0x1e8a: 0xd939, 0x1e8b: 0xd959, + 0x1e8c: 0xd979, 0x1e8d: 0xd999, 0x1e8e: 0xd9b9, 0x1e8f: 0x8cfd, 0x1e90: 0xd9d9, 0x1e91: 0x8d1d, + 0x1e92: 0x8d3d, 0x1e93: 0xd9f9, 0x1e94: 0xda19, 0x1e95: 0xda19, 0x1e96: 0xda39, 0x1e97: 0x8d5d, + 0x1e98: 0x8d7d, 0x1e99: 0xda59, 0x1e9a: 0xda79, 0x1e9b: 0xda99, 0x1e9c: 0xdab9, 0x1e9d: 0xdad9, + 0x1e9e: 0xdaf9, 0x1e9f: 0xdb19, 0x1ea0: 0xdb39, 0x1ea1: 0xdb59, 0x1ea2: 0xdb79, 0x1ea3: 0xdb99, + 0x1ea4: 0x8d9d, 0x1ea5: 0xdbb9, 0x1ea6: 0xdbd9, 0x1ea7: 0xdbf9, 0x1ea8: 0xdc19, 0x1ea9: 0xdbf9, + 0x1eaa: 0xdc39, 0x1eab: 0xdc59, 0x1eac: 0xdc79, 0x1ead: 0xdc99, 0x1eae: 0xdcb9, 0x1eaf: 0xdcd9, + 0x1eb0: 0xdcf9, 0x1eb1: 0xdd19, 0x1eb2: 0xdd39, 0x1eb3: 0xdd59, 0x1eb4: 0xdd79, 0x1eb5: 0xdd99, + 0x1eb6: 0xddb9, 0x1eb7: 0xddd9, 0x1eb8: 0x8dbd, 0x1eb9: 0xddf9, 0x1eba: 0xde19, 0x1ebb: 0xde39, + 0x1ebc: 0xde59, 0x1ebd: 0xde79, 0x1ebe: 0x8ddd, 0x1ebf: 0xde99, // Block 0x7b, offset 0x1ec0 - 0x1ec0: 0xf6b9, 0x1ec1: 0xf6d9, 0x1ec2: 0xf6f9, 0x1ec3: 0xf719, 0x1ec4: 0xf739, 0x1ec5: 0x951d, - 0x1ec6: 0xf759, 0x1ec7: 0xf779, 0x1ec8: 0xf799, 0x1ec9: 0xf7b9, 0x1eca: 0xf7d9, 0x1ecb: 0x953d, - 0x1ecc: 0x955d, 0x1ecd: 0xf7f9, 0x1ece: 0xf819, 0x1ecf: 0xf839, 0x1ed0: 0xf859, 0x1ed1: 0xf879, - 0x1ed2: 0xf899, 0x1ed3: 0x957d, 0x1ed4: 0xf8b9, 0x1ed5: 0xf8d9, 0x1ed6: 0xf8f9, 0x1ed7: 0xf919, - 0x1ed8: 0x959d, 0x1ed9: 0x95bd, 0x1eda: 0xf939, 0x1edb: 0xf959, 0x1edc: 0xf979, 0x1edd: 0x95dd, - 0x1ede: 0xf999, 0x1edf: 0xf9b9, 0x1ee0: 0x6815, 0x1ee1: 0x95fd, 0x1ee2: 0xf9d9, 0x1ee3: 0xf9f9, - 0x1ee4: 0xfa19, 0x1ee5: 0x961d, 0x1ee6: 0xfa39, 0x1ee7: 0xfa59, 0x1ee8: 0xfa79, 0x1ee9: 0xfa99, - 0x1eea: 0xfab9, 0x1eeb: 0xfad9, 0x1eec: 0xfaf9, 0x1eed: 0x963d, 0x1eee: 0xfb19, 0x1eef: 0xfb39, - 0x1ef0: 0xfb59, 0x1ef1: 0x965d, 0x1ef2: 0xfb79, 0x1ef3: 0xfb99, 0x1ef4: 0xfbb9, 0x1ef5: 0xfbd9, - 0x1ef6: 0x7b35, 0x1ef7: 0x967d, 0x1ef8: 0xfbf9, 0x1ef9: 0xfc19, 0x1efa: 0xfc39, 0x1efb: 0x969d, - 0x1efc: 0xfc59, 0x1efd: 0x96bd, 0x1efe: 0xfc79, 0x1eff: 0xfc79, + 0x1ec0: 0xe599, 0x1ec1: 0xe5b9, 0x1ec2: 0xe5d9, 0x1ec3: 0xe5f9, 0x1ec4: 0xe619, 0x1ec5: 0xe639, + 0x1ec6: 0x8efd, 0x1ec7: 0xe659, 0x1ec8: 0xe679, 0x1ec9: 0xe699, 0x1eca: 0xe6b9, 0x1ecb: 0xe6d9, + 0x1ecc: 0xe6f9, 0x1ecd: 0x8f1d, 0x1ece: 0xe719, 0x1ecf: 0xe739, 0x1ed0: 0x8f3d, 0x1ed1: 0x8f5d, + 0x1ed2: 0xe759, 0x1ed3: 0xe779, 0x1ed4: 0xe799, 0x1ed5: 0xe7b9, 0x1ed6: 0xe7d9, 0x1ed7: 0xe7f9, + 0x1ed8: 0xe819, 0x1ed9: 0xe839, 0x1eda: 0xe859, 0x1edb: 0x8f7d, 0x1edc: 0xe879, 0x1edd: 0x8f9d, + 0x1ede: 0xe899, 0x1edf: 0x2040, 0x1ee0: 0xe8b9, 0x1ee1: 0xe8d9, 0x1ee2: 0xe8f9, 0x1ee3: 0x8fbd, + 0x1ee4: 0xe919, 0x1ee5: 0xe939, 0x1ee6: 0x8fdd, 0x1ee7: 0x8ffd, 0x1ee8: 0xe959, 0x1ee9: 0xe979, + 0x1eea: 0xe999, 0x1eeb: 0xe9b9, 0x1eec: 0xe9d9, 0x1eed: 0xe9d9, 0x1eee: 0xe9f9, 0x1eef: 0xea19, + 0x1ef0: 0xea39, 0x1ef1: 0xea59, 0x1ef2: 0xea79, 0x1ef3: 0xea99, 0x1ef4: 0xeab9, 0x1ef5: 0x901d, + 0x1ef6: 0xead9, 0x1ef7: 0x903d, 0x1ef8: 0xeaf9, 0x1ef9: 0x905d, 0x1efa: 0xeb19, 0x1efb: 0x907d, + 0x1efc: 0x909d, 0x1efd: 0x90bd, 0x1efe: 0xeb39, 0x1eff: 0xeb59, // Block 0x7c, offset 0x1f00 - 0x1f00: 0xfc99, 0x1f01: 0x96dd, 0x1f02: 0xfcb9, 0x1f03: 0xfcd9, 0x1f04: 0xfcf9, 0x1f05: 0xfd19, - 0x1f06: 0xfd39, 0x1f07: 0xfd59, 0x1f08: 0xfd79, 0x1f09: 0x96fd, 0x1f0a: 0xfd99, 0x1f0b: 0xfdb9, - 0x1f0c: 0xfdd9, 0x1f0d: 0xfdf9, 0x1f0e: 0xfe19, 0x1f0f: 0xfe39, 0x1f10: 0x971d, 0x1f11: 0xfe59, - 0x1f12: 0x973d, 0x1f13: 0x975d, 0x1f14: 0x977d, 0x1f15: 0xfe79, 0x1f16: 0xfe99, 0x1f17: 0xfeb9, - 0x1f18: 0xfed9, 0x1f19: 0xfef9, 0x1f1a: 0xff19, 0x1f1b: 0xff39, 0x1f1c: 0xff59, 0x1f1d: 0x979d, - 0x1f1e: 0x0040, 0x1f1f: 0x0040, 0x1f20: 0x0040, 0x1f21: 0x0040, 0x1f22: 0x0040, 0x1f23: 0x0040, - 0x1f24: 0x0040, 0x1f25: 0x0040, 0x1f26: 0x0040, 0x1f27: 0x0040, 0x1f28: 0x0040, 0x1f29: 0x0040, - 0x1f2a: 0x0040, 0x1f2b: 0x0040, 0x1f2c: 0x0040, 0x1f2d: 0x0040, 0x1f2e: 0x0040, 0x1f2f: 0x0040, - 0x1f30: 0x0040, 0x1f31: 0x0040, 0x1f32: 0x0040, 0x1f33: 0x0040, 0x1f34: 0x0040, 0x1f35: 0x0040, - 0x1f36: 0x0040, 0x1f37: 0x0040, 0x1f38: 0x0040, 0x1f39: 0x0040, 0x1f3a: 0x0040, 0x1f3b: 0x0040, - 0x1f3c: 0x0040, 0x1f3d: 0x0040, 0x1f3e: 0x0040, 0x1f3f: 0x0040, + 0x1f00: 0xeb79, 0x1f01: 0x90dd, 0x1f02: 0x90fd, 0x1f03: 0x911d, 0x1f04: 0x913d, 0x1f05: 0xeb99, + 0x1f06: 0xebb9, 0x1f07: 0xebb9, 0x1f08: 0xebd9, 0x1f09: 0xebf9, 0x1f0a: 0xec19, 0x1f0b: 0xec39, + 0x1f0c: 0xec59, 0x1f0d: 0x915d, 0x1f0e: 0xec79, 0x1f0f: 0xec99, 0x1f10: 0xecb9, 0x1f11: 0xecd9, + 0x1f12: 0x917d, 0x1f13: 0xecf9, 0x1f14: 0x919d, 0x1f15: 0x91bd, 0x1f16: 0xed19, 0x1f17: 0xed39, + 0x1f18: 0xed59, 0x1f19: 0xed79, 0x1f1a: 0xed99, 0x1f1b: 0xedb9, 0x1f1c: 0x91dd, 0x1f1d: 0x91fd, + 0x1f1e: 0x921d, 0x1f1f: 0x2040, 0x1f20: 0xedd9, 0x1f21: 0x923d, 0x1f22: 0xedf9, 0x1f23: 0xee19, + 0x1f24: 0xee39, 0x1f25: 0x925d, 0x1f26: 0xee59, 0x1f27: 0xee79, 0x1f28: 0xee99, 0x1f29: 0xeeb9, + 0x1f2a: 0xeed9, 0x1f2b: 0x927d, 0x1f2c: 0xeef9, 0x1f2d: 0xef19, 0x1f2e: 0xef39, 0x1f2f: 0xef59, + 0x1f30: 0xef79, 0x1f31: 0xef99, 0x1f32: 0x929d, 0x1f33: 0x92bd, 0x1f34: 0xefb9, 0x1f35: 0x92dd, + 0x1f36: 0xefd9, 0x1f37: 0x92fd, 0x1f38: 0xeff9, 0x1f39: 0xf019, 0x1f3a: 0xf039, 0x1f3b: 0x931d, + 0x1f3c: 0x933d, 0x1f3d: 0xf059, 0x1f3e: 0x935d, 0x1f3f: 0xf079, + // Block 0x7d, offset 0x1f40 + 0x1f40: 0xf6b9, 0x1f41: 0xf6d9, 0x1f42: 0xf6f9, 0x1f43: 0xf719, 0x1f44: 0xf739, 0x1f45: 0x951d, + 0x1f46: 0xf759, 0x1f47: 0xf779, 0x1f48: 0xf799, 0x1f49: 0xf7b9, 0x1f4a: 0xf7d9, 0x1f4b: 0x953d, + 0x1f4c: 0x955d, 0x1f4d: 0xf7f9, 0x1f4e: 0xf819, 0x1f4f: 0xf839, 0x1f50: 0xf859, 0x1f51: 0xf879, + 0x1f52: 0xf899, 0x1f53: 0x957d, 0x1f54: 0xf8b9, 0x1f55: 0xf8d9, 0x1f56: 0xf8f9, 0x1f57: 0xf919, + 0x1f58: 0x959d, 0x1f59: 0x95bd, 0x1f5a: 0xf939, 0x1f5b: 0xf959, 0x1f5c: 0xf979, 0x1f5d: 0x95dd, + 0x1f5e: 0xf999, 0x1f5f: 0xf9b9, 0x1f60: 0x6815, 0x1f61: 0x95fd, 0x1f62: 0xf9d9, 0x1f63: 0xf9f9, + 0x1f64: 0xfa19, 0x1f65: 0x961d, 0x1f66: 0xfa39, 0x1f67: 0xfa59, 0x1f68: 0xfa79, 0x1f69: 0xfa99, + 0x1f6a: 0xfab9, 0x1f6b: 0xfad9, 0x1f6c: 0xfaf9, 0x1f6d: 0x963d, 0x1f6e: 0xfb19, 0x1f6f: 0xfb39, + 0x1f70: 0xfb59, 0x1f71: 0x965d, 0x1f72: 0xfb79, 0x1f73: 0xfb99, 0x1f74: 0xfbb9, 0x1f75: 0xfbd9, + 0x1f76: 0x7b35, 0x1f77: 0x967d, 0x1f78: 0xfbf9, 0x1f79: 0xfc19, 0x1f7a: 0xfc39, 0x1f7b: 0x969d, + 0x1f7c: 0xfc59, 0x1f7d: 0x96bd, 0x1f7e: 0xfc79, 0x1f7f: 0xfc79, + // Block 0x7e, offset 0x1f80 + 0x1f80: 0xfc99, 0x1f81: 0x96dd, 0x1f82: 0xfcb9, 0x1f83: 0xfcd9, 0x1f84: 0xfcf9, 0x1f85: 0xfd19, + 0x1f86: 0xfd39, 0x1f87: 0xfd59, 0x1f88: 0xfd79, 0x1f89: 0x96fd, 0x1f8a: 0xfd99, 0x1f8b: 0xfdb9, + 0x1f8c: 0xfdd9, 0x1f8d: 0xfdf9, 0x1f8e: 0xfe19, 0x1f8f: 0xfe39, 0x1f90: 0x971d, 0x1f91: 0xfe59, + 0x1f92: 0x973d, 0x1f93: 0x975d, 0x1f94: 0x977d, 0x1f95: 0xfe79, 0x1f96: 0xfe99, 0x1f97: 0xfeb9, + 0x1f98: 0xfed9, 0x1f99: 0xfef9, 0x1f9a: 0xff19, 0x1f9b: 0xff39, 0x1f9c: 0xff59, 0x1f9d: 0x979d, + 0x1f9e: 0x0040, 0x1f9f: 0x0040, 0x1fa0: 0x0040, 0x1fa1: 0x0040, 0x1fa2: 0x0040, 0x1fa3: 0x0040, + 0x1fa4: 0x0040, 0x1fa5: 0x0040, 0x1fa6: 0x0040, 0x1fa7: 0x0040, 0x1fa8: 0x0040, 0x1fa9: 0x0040, + 0x1faa: 0x0040, 0x1fab: 0x0040, 0x1fac: 0x0040, 0x1fad: 0x0040, 0x1fae: 0x0040, 0x1faf: 0x0040, + 0x1fb0: 0x0040, 0x1fb1: 0x0040, 0x1fb2: 0x0040, 0x1fb3: 0x0040, 0x1fb4: 0x0040, 0x1fb5: 0x0040, + 0x1fb6: 0x0040, 0x1fb7: 0x0040, 0x1fb8: 0x0040, 0x1fb9: 0x0040, 0x1fba: 0x0040, 0x1fbb: 0x0040, + 0x1fbc: 0x0040, 0x1fbd: 0x0040, 0x1fbe: 0x0040, 0x1fbf: 0x0040, } -// idnaIndex: 35 blocks, 2240 entries, 4480 bytes +// idnaIndex: 36 blocks, 2304 entries, 4608 bytes // Block 0 is the zero block. -var idnaIndex = [2240]uint16{ +var idnaIndex = [2304]uint16{ // Block 0x0, offset 0x0 // Block 0x1, offset 0x40 // Block 0x2, offset 0x80 // Block 0x3, offset 0xc0 - 0xc2: 0x01, 0xc3: 0x7b, 0xc4: 0x02, 0xc5: 0x03, 0xc6: 0x04, 0xc7: 0x05, - 0xc8: 0x06, 0xc9: 0x7c, 0xca: 0x7d, 0xcb: 0x07, 0xcc: 0x7e, 0xcd: 0x08, 0xce: 0x09, 0xcf: 0x0a, - 0xd0: 0x7f, 0xd1: 0x0b, 0xd2: 0x0c, 0xd3: 0x0d, 0xd4: 0x0e, 0xd5: 0x80, 0xd6: 0x81, 0xd7: 0x82, - 0xd8: 0x0f, 0xd9: 0x83, 0xda: 0x84, 0xdb: 0x10, 0xdc: 0x11, 0xdd: 0x85, 0xde: 0x86, 0xdf: 0x87, + 0xc2: 0x01, 0xc3: 0x7d, 0xc4: 0x02, 0xc5: 0x03, 0xc6: 0x04, 0xc7: 0x05, + 0xc8: 0x06, 0xc9: 0x7e, 0xca: 0x7f, 0xcb: 0x07, 0xcc: 0x80, 0xcd: 0x08, 0xce: 0x09, 0xcf: 0x0a, + 0xd0: 0x81, 0xd1: 0x0b, 0xd2: 0x0c, 0xd3: 0x0d, 0xd4: 0x0e, 0xd5: 0x82, 0xd6: 0x83, 0xd7: 0x84, + 0xd8: 0x0f, 0xd9: 0x10, 0xda: 0x85, 0xdb: 0x11, 0xdc: 0x12, 0xdd: 0x86, 0xde: 0x87, 0xdf: 0x88, 0xe0: 0x02, 0xe1: 0x03, 0xe2: 0x04, 0xe3: 0x05, 0xe4: 0x06, 0xe5: 0x07, 0xe6: 0x07, 0xe7: 0x07, 0xe8: 0x07, 0xe9: 0x08, 0xea: 0x09, 0xeb: 0x07, 0xec: 0x07, 0xed: 0x0a, 0xee: 0x0b, 0xef: 0x0c, - 0xf0: 0x1c, 0xf1: 0x1d, 0xf2: 0x1d, 0xf3: 0x1f, 0xf4: 0x20, + 0xf0: 0x1d, 0xf1: 0x1e, 0xf2: 0x1e, 0xf3: 0x20, 0xf4: 0x21, // Block 0x4, offset 0x100 - 0x120: 0x88, 0x121: 0x89, 0x122: 0x8a, 0x123: 0x8b, 0x124: 0x8c, 0x125: 0x12, 0x126: 0x13, 0x127: 0x14, - 0x128: 0x15, 0x129: 0x16, 0x12a: 0x17, 0x12b: 0x18, 0x12c: 0x19, 0x12d: 0x1a, 0x12e: 0x1b, 0x12f: 0x8d, - 0x130: 0x8e, 0x131: 0x1c, 0x132: 0x1d, 0x133: 0x1e, 0x134: 0x8f, 0x135: 0x1f, 0x136: 0x90, 0x137: 0x91, - 0x138: 0x92, 0x139: 0x93, 0x13a: 0x20, 0x13b: 0x94, 0x13c: 0x95, 0x13d: 0x21, 0x13e: 0x22, 0x13f: 0x96, + 0x120: 0x89, 0x121: 0x13, 0x122: 0x8a, 0x123: 0x8b, 0x124: 0x8c, 0x125: 0x14, 0x126: 0x15, 0x127: 0x16, + 0x128: 0x17, 0x129: 0x18, 0x12a: 0x19, 0x12b: 0x1a, 0x12c: 0x1b, 0x12d: 0x1c, 0x12e: 0x1d, 0x12f: 0x8d, + 0x130: 0x8e, 0x131: 0x1e, 0x132: 0x1f, 0x133: 0x20, 0x134: 0x8f, 0x135: 0x21, 0x136: 0x90, 0x137: 0x91, + 0x138: 0x92, 0x139: 0x93, 0x13a: 0x22, 0x13b: 0x94, 0x13c: 0x95, 0x13d: 0x23, 0x13e: 0x24, 0x13f: 0x96, // Block 0x5, offset 0x140 - 0x140: 0x97, 0x141: 0x98, 0x142: 0x99, 0x143: 0x9a, 0x144: 0x9b, 0x145: 0x9c, 0x146: 0x9b, 0x147: 0x9b, - 0x148: 0x9d, 0x149: 0x9e, 0x14a: 0x9f, 0x14b: 0xa0, 0x14c: 0xa1, 0x14d: 0xa2, 0x14e: 0xa3, 0x14f: 0xa4, - 0x150: 0xa5, 0x151: 0x9d, 0x152: 0x9d, 0x153: 0x9d, 0x154: 0x9d, 0x155: 0x9d, 0x156: 0x9d, 0x157: 0x9d, - 0x158: 0x9d, 0x159: 0xa6, 0x15a: 0xa7, 0x15b: 0xa8, 0x15c: 0xa9, 0x15d: 0xaa, 0x15e: 0xab, 0x15f: 0xac, - 0x160: 0xad, 0x161: 0xae, 0x162: 0xaf, 0x163: 0xb0, 0x164: 0xb1, 0x165: 0xb2, 0x166: 0xb3, 0x167: 0xb4, - 0x168: 0xb5, 0x169: 0xb6, 0x16a: 0xb7, 0x16b: 0xb8, 0x16c: 0xb9, 0x16d: 0xba, 0x16e: 0xbb, 0x16f: 0xbc, - 0x170: 0xbd, 0x171: 0xbe, 0x172: 0xbf, 0x173: 0xc0, 0x174: 0x23, 0x175: 0x24, 0x176: 0x25, 0x177: 0xc1, - 0x178: 0x26, 0x179: 0x26, 0x17a: 0x27, 0x17b: 0x26, 0x17c: 0xc2, 0x17d: 0x28, 0x17e: 0x29, 0x17f: 0x2a, + 0x140: 0x97, 0x141: 0x98, 0x142: 0x99, 0x143: 0x9a, 0x144: 0x9b, 0x145: 0x9c, 0x146: 0x9d, 0x147: 0x9e, + 0x148: 0x9f, 0x149: 0xa0, 0x14a: 0xa1, 0x14b: 0xa2, 0x14c: 0xa3, 0x14d: 0xa4, 0x14e: 0xa5, 0x14f: 0xa6, + 0x150: 0xa7, 0x151: 0x9f, 0x152: 0x9f, 0x153: 0x9f, 0x154: 0x9f, 0x155: 0x9f, 0x156: 0x9f, 0x157: 0x9f, + 0x158: 0x9f, 0x159: 0xa8, 0x15a: 0xa9, 0x15b: 0xaa, 0x15c: 0xab, 0x15d: 0xac, 0x15e: 0xad, 0x15f: 0xae, + 0x160: 0xaf, 0x161: 0xb0, 0x162: 0xb1, 0x163: 0xb2, 0x164: 0xb3, 0x165: 0xb4, 0x166: 0xb5, 0x167: 0xb6, + 0x168: 0xb7, 0x169: 0xb8, 0x16a: 0xb9, 0x16b: 0xba, 0x16c: 0xbb, 0x16d: 0xbc, 0x16e: 0xbd, 0x16f: 0xbe, + 0x170: 0xbf, 0x171: 0xc0, 0x172: 0xc1, 0x173: 0xc2, 0x174: 0x25, 0x175: 0x26, 0x176: 0x27, 0x177: 0xc3, + 0x178: 0x28, 0x179: 0x28, 0x17a: 0x29, 0x17b: 0x28, 0x17c: 0xc4, 0x17d: 0x2a, 0x17e: 0x2b, 0x17f: 0x2c, // Block 0x6, offset 0x180 - 0x180: 0x2b, 0x181: 0x2c, 0x182: 0x2d, 0x183: 0xc3, 0x184: 0x2e, 0x185: 0x2f, 0x186: 0xc4, 0x187: 0x9b, - 0x188: 0xc5, 0x189: 0xc6, 0x18a: 0x9b, 0x18b: 0x9b, 0x18c: 0xc7, 0x18d: 0x9b, 0x18e: 0x9b, 0x18f: 0xc8, - 0x190: 0xc9, 0x191: 0x30, 0x192: 0x31, 0x193: 0x32, 0x194: 0x9b, 0x195: 0x9b, 0x196: 0x9b, 0x197: 0x9b, + 0x180: 0x2d, 0x181: 0x2e, 0x182: 0x2f, 0x183: 0xc5, 0x184: 0x30, 0x185: 0x31, 0x186: 0xc6, 0x187: 0x9b, + 0x188: 0xc7, 0x189: 0xc8, 0x18a: 0x9b, 0x18b: 0x9b, 0x18c: 0xc9, 0x18d: 0x9b, 0x18e: 0x9b, 0x18f: 0x9b, + 0x190: 0xca, 0x191: 0x32, 0x192: 0x33, 0x193: 0x34, 0x194: 0x9b, 0x195: 0x9b, 0x196: 0x9b, 0x197: 0x9b, 0x198: 0x9b, 0x199: 0x9b, 0x19a: 0x9b, 0x19b: 0x9b, 0x19c: 0x9b, 0x19d: 0x9b, 0x19e: 0x9b, 0x19f: 0x9b, 0x1a0: 0x9b, 0x1a1: 0x9b, 0x1a2: 0x9b, 0x1a3: 0x9b, 0x1a4: 0x9b, 0x1a5: 0x9b, 0x1a6: 0x9b, 0x1a7: 0x9b, - 0x1a8: 0xca, 0x1a9: 0xcb, 0x1aa: 0x9b, 0x1ab: 0xcc, 0x1ac: 0x9b, 0x1ad: 0xcd, 0x1ae: 0xce, 0x1af: 0xcf, - 0x1b0: 0xd0, 0x1b1: 0x33, 0x1b2: 0x26, 0x1b3: 0x34, 0x1b4: 0xd1, 0x1b5: 0xd2, 0x1b6: 0xd3, 0x1b7: 0xd4, - 0x1b8: 0xd5, 0x1b9: 0xd6, 0x1ba: 0xd7, 0x1bb: 0xd8, 0x1bc: 0xd9, 0x1bd: 0xda, 0x1be: 0xdb, 0x1bf: 0x35, + 0x1a8: 0xcb, 0x1a9: 0xcc, 0x1aa: 0x9b, 0x1ab: 0xcd, 0x1ac: 0x9b, 0x1ad: 0xce, 0x1ae: 0xcf, 0x1af: 0xd0, + 0x1b0: 0xd1, 0x1b1: 0x35, 0x1b2: 0x28, 0x1b3: 0x36, 0x1b4: 0xd2, 0x1b5: 0xd3, 0x1b6: 0xd4, 0x1b7: 0xd5, + 0x1b8: 0xd6, 0x1b9: 0xd7, 0x1ba: 0xd8, 0x1bb: 0xd9, 0x1bc: 0xda, 0x1bd: 0xdb, 0x1be: 0xdc, 0x1bf: 0x37, // Block 0x7, offset 0x1c0 - 0x1c0: 0x36, 0x1c1: 0xdc, 0x1c2: 0xdd, 0x1c3: 0xde, 0x1c4: 0xdf, 0x1c5: 0x37, 0x1c6: 0x38, 0x1c7: 0xe0, - 0x1c8: 0xe1, 0x1c9: 0x39, 0x1ca: 0x3a, 0x1cb: 0x3b, 0x1cc: 0x3c, 0x1cd: 0x3d, 0x1ce: 0x3e, 0x1cf: 0x3f, - 0x1d0: 0x9d, 0x1d1: 0x9d, 0x1d2: 0x9d, 0x1d3: 0x9d, 0x1d4: 0x9d, 0x1d5: 0x9d, 0x1d6: 0x9d, 0x1d7: 0x9d, - 0x1d8: 0x9d, 0x1d9: 0x9d, 0x1da: 0x9d, 0x1db: 0x9d, 0x1dc: 0x9d, 0x1dd: 0x9d, 0x1de: 0x9d, 0x1df: 0x9d, - 0x1e0: 0x9d, 0x1e1: 0x9d, 0x1e2: 0x9d, 0x1e3: 0x9d, 0x1e4: 0x9d, 0x1e5: 0x9d, 0x1e6: 0x9d, 0x1e7: 0x9d, - 0x1e8: 0x9d, 0x1e9: 0x9d, 0x1ea: 0x9d, 0x1eb: 0x9d, 0x1ec: 0x9d, 0x1ed: 0x9d, 0x1ee: 0x9d, 0x1ef: 0x9d, - 0x1f0: 0x9d, 0x1f1: 0x9d, 0x1f2: 0x9d, 0x1f3: 0x9d, 0x1f4: 0x9d, 0x1f5: 0x9d, 0x1f6: 0x9d, 0x1f7: 0x9d, - 0x1f8: 0x9d, 0x1f9: 0x9d, 0x1fa: 0x9d, 0x1fb: 0x9d, 0x1fc: 0x9d, 0x1fd: 0x9d, 0x1fe: 0x9d, 0x1ff: 0x9d, + 0x1c0: 0x38, 0x1c1: 0xdd, 0x1c2: 0xde, 0x1c3: 0xdf, 0x1c4: 0xe0, 0x1c5: 0x39, 0x1c6: 0x3a, 0x1c7: 0xe1, + 0x1c8: 0xe2, 0x1c9: 0x3b, 0x1ca: 0x3c, 0x1cb: 0x3d, 0x1cc: 0x3e, 0x1cd: 0x3f, 0x1ce: 0x40, 0x1cf: 0x41, + 0x1d0: 0x9f, 0x1d1: 0x9f, 0x1d2: 0x9f, 0x1d3: 0x9f, 0x1d4: 0x9f, 0x1d5: 0x9f, 0x1d6: 0x9f, 0x1d7: 0x9f, + 0x1d8: 0x9f, 0x1d9: 0x9f, 0x1da: 0x9f, 0x1db: 0x9f, 0x1dc: 0x9f, 0x1dd: 0x9f, 0x1de: 0x9f, 0x1df: 0x9f, + 0x1e0: 0x9f, 0x1e1: 0x9f, 0x1e2: 0x9f, 0x1e3: 0x9f, 0x1e4: 0x9f, 0x1e5: 0x9f, 0x1e6: 0x9f, 0x1e7: 0x9f, + 0x1e8: 0x9f, 0x1e9: 0x9f, 0x1ea: 0x9f, 0x1eb: 0x9f, 0x1ec: 0x9f, 0x1ed: 0x9f, 0x1ee: 0x9f, 0x1ef: 0x9f, + 0x1f0: 0x9f, 0x1f1: 0x9f, 0x1f2: 0x9f, 0x1f3: 0x9f, 0x1f4: 0x9f, 0x1f5: 0x9f, 0x1f6: 0x9f, 0x1f7: 0x9f, + 0x1f8: 0x9f, 0x1f9: 0x9f, 0x1fa: 0x9f, 0x1fb: 0x9f, 0x1fc: 0x9f, 0x1fd: 0x9f, 0x1fe: 0x9f, 0x1ff: 0x9f, // Block 0x8, offset 0x200 - 0x200: 0x9d, 0x201: 0x9d, 0x202: 0x9d, 0x203: 0x9d, 0x204: 0x9d, 0x205: 0x9d, 0x206: 0x9d, 0x207: 0x9d, - 0x208: 0x9d, 0x209: 0x9d, 0x20a: 0x9d, 0x20b: 0x9d, 0x20c: 0x9d, 0x20d: 0x9d, 0x20e: 0x9d, 0x20f: 0x9d, - 0x210: 0x9d, 0x211: 0x9d, 0x212: 0x9d, 0x213: 0x9d, 0x214: 0x9d, 0x215: 0x9d, 0x216: 0x9d, 0x217: 0x9d, - 0x218: 0x9d, 0x219: 0x9d, 0x21a: 0x9d, 0x21b: 0x9d, 0x21c: 0x9d, 0x21d: 0x9d, 0x21e: 0x9d, 0x21f: 0x9d, - 0x220: 0x9d, 0x221: 0x9d, 0x222: 0x9d, 0x223: 0x9d, 0x224: 0x9d, 0x225: 0x9d, 0x226: 0x9d, 0x227: 0x9d, - 0x228: 0x9d, 0x229: 0x9d, 0x22a: 0x9d, 0x22b: 0x9d, 0x22c: 0x9d, 0x22d: 0x9d, 0x22e: 0x9d, 0x22f: 0x9d, - 0x230: 0x9d, 0x231: 0x9d, 0x232: 0x9d, 0x233: 0x9d, 0x234: 0x9d, 0x235: 0x9d, 0x236: 0xb0, 0x237: 0x9b, - 0x238: 0x9d, 0x239: 0x9d, 0x23a: 0x9d, 0x23b: 0x9d, 0x23c: 0x9d, 0x23d: 0x9d, 0x23e: 0x9d, 0x23f: 0x9d, + 0x200: 0x9f, 0x201: 0x9f, 0x202: 0x9f, 0x203: 0x9f, 0x204: 0x9f, 0x205: 0x9f, 0x206: 0x9f, 0x207: 0x9f, + 0x208: 0x9f, 0x209: 0x9f, 0x20a: 0x9f, 0x20b: 0x9f, 0x20c: 0x9f, 0x20d: 0x9f, 0x20e: 0x9f, 0x20f: 0x9f, + 0x210: 0x9f, 0x211: 0x9f, 0x212: 0x9f, 0x213: 0x9f, 0x214: 0x9f, 0x215: 0x9f, 0x216: 0x9f, 0x217: 0x9f, + 0x218: 0x9f, 0x219: 0x9f, 0x21a: 0x9f, 0x21b: 0x9f, 0x21c: 0x9f, 0x21d: 0x9f, 0x21e: 0x9f, 0x21f: 0x9f, + 0x220: 0x9f, 0x221: 0x9f, 0x222: 0x9f, 0x223: 0x9f, 0x224: 0x9f, 0x225: 0x9f, 0x226: 0x9f, 0x227: 0x9f, + 0x228: 0x9f, 0x229: 0x9f, 0x22a: 0x9f, 0x22b: 0x9f, 0x22c: 0x9f, 0x22d: 0x9f, 0x22e: 0x9f, 0x22f: 0x9f, + 0x230: 0x9f, 0x231: 0x9f, 0x232: 0x9f, 0x233: 0x9f, 0x234: 0x9f, 0x235: 0x9f, 0x236: 0xb2, 0x237: 0x9b, + 0x238: 0x9f, 0x239: 0x9f, 0x23a: 0x9f, 0x23b: 0x9f, 0x23c: 0x9f, 0x23d: 0x9f, 0x23e: 0x9f, 0x23f: 0x9f, // Block 0x9, offset 0x240 - 0x240: 0x9d, 0x241: 0x9d, 0x242: 0x9d, 0x243: 0x9d, 0x244: 0x9d, 0x245: 0x9d, 0x246: 0x9d, 0x247: 0x9d, - 0x248: 0x9d, 0x249: 0x9d, 0x24a: 0x9d, 0x24b: 0x9d, 0x24c: 0x9d, 0x24d: 0x9d, 0x24e: 0x9d, 0x24f: 0x9d, - 0x250: 0x9d, 0x251: 0x9d, 0x252: 0x9d, 0x253: 0x9d, 0x254: 0x9d, 0x255: 0x9d, 0x256: 0x9d, 0x257: 0x9d, - 0x258: 0x9d, 0x259: 0x9d, 0x25a: 0x9d, 0x25b: 0x9d, 0x25c: 0x9d, 0x25d: 0x9d, 0x25e: 0x9d, 0x25f: 0x9d, - 0x260: 0x9d, 0x261: 0x9d, 0x262: 0x9d, 0x263: 0x9d, 0x264: 0x9d, 0x265: 0x9d, 0x266: 0x9d, 0x267: 0x9d, - 0x268: 0x9d, 0x269: 0x9d, 0x26a: 0x9d, 0x26b: 0x9d, 0x26c: 0x9d, 0x26d: 0x9d, 0x26e: 0x9d, 0x26f: 0x9d, - 0x270: 0x9d, 0x271: 0x9d, 0x272: 0x9d, 0x273: 0x9d, 0x274: 0x9d, 0x275: 0x9d, 0x276: 0x9d, 0x277: 0x9d, - 0x278: 0x9d, 0x279: 0x9d, 0x27a: 0x9d, 0x27b: 0x9d, 0x27c: 0x9d, 0x27d: 0x9d, 0x27e: 0x9d, 0x27f: 0x9d, + 0x240: 0x9f, 0x241: 0x9f, 0x242: 0x9f, 0x243: 0x9f, 0x244: 0x9f, 0x245: 0x9f, 0x246: 0x9f, 0x247: 0x9f, + 0x248: 0x9f, 0x249: 0x9f, 0x24a: 0x9f, 0x24b: 0x9f, 0x24c: 0x9f, 0x24d: 0x9f, 0x24e: 0x9f, 0x24f: 0x9f, + 0x250: 0x9f, 0x251: 0x9f, 0x252: 0x9f, 0x253: 0x9f, 0x254: 0x9f, 0x255: 0x9f, 0x256: 0x9f, 0x257: 0x9f, + 0x258: 0x9f, 0x259: 0x9f, 0x25a: 0x9f, 0x25b: 0x9f, 0x25c: 0x9f, 0x25d: 0x9f, 0x25e: 0x9f, 0x25f: 0x9f, + 0x260: 0x9f, 0x261: 0x9f, 0x262: 0x9f, 0x263: 0x9f, 0x264: 0x9f, 0x265: 0x9f, 0x266: 0x9f, 0x267: 0x9f, + 0x268: 0x9f, 0x269: 0x9f, 0x26a: 0x9f, 0x26b: 0x9f, 0x26c: 0x9f, 0x26d: 0x9f, 0x26e: 0x9f, 0x26f: 0x9f, + 0x270: 0x9f, 0x271: 0x9f, 0x272: 0x9f, 0x273: 0x9f, 0x274: 0x9f, 0x275: 0x9f, 0x276: 0x9f, 0x277: 0x9f, + 0x278: 0x9f, 0x279: 0x9f, 0x27a: 0x9f, 0x27b: 0x9f, 0x27c: 0x9f, 0x27d: 0x9f, 0x27e: 0x9f, 0x27f: 0x9f, // Block 0xa, offset 0x280 - 0x280: 0x9d, 0x281: 0x9d, 0x282: 0x9d, 0x283: 0x9d, 0x284: 0x9d, 0x285: 0x9d, 0x286: 0x9d, 0x287: 0x9d, - 0x288: 0x9d, 0x289: 0x9d, 0x28a: 0x9d, 0x28b: 0x9d, 0x28c: 0x9d, 0x28d: 0x9d, 0x28e: 0x9d, 0x28f: 0x9d, - 0x290: 0x9d, 0x291: 0x9d, 0x292: 0x9d, 0x293: 0x9d, 0x294: 0x9d, 0x295: 0x9d, 0x296: 0x9d, 0x297: 0x9d, - 0x298: 0x9d, 0x299: 0x9d, 0x29a: 0x9d, 0x29b: 0x9d, 0x29c: 0x9d, 0x29d: 0x9d, 0x29e: 0x9d, 0x29f: 0x9d, - 0x2a0: 0x9d, 0x2a1: 0x9d, 0x2a2: 0x9d, 0x2a3: 0x9d, 0x2a4: 0x9d, 0x2a5: 0x9d, 0x2a6: 0x9d, 0x2a7: 0x9d, - 0x2a8: 0x9d, 0x2a9: 0x9d, 0x2aa: 0x9d, 0x2ab: 0x9d, 0x2ac: 0x9d, 0x2ad: 0x9d, 0x2ae: 0x9d, 0x2af: 0x9d, - 0x2b0: 0x9d, 0x2b1: 0x9d, 0x2b2: 0x9d, 0x2b3: 0x9d, 0x2b4: 0x9d, 0x2b5: 0x9d, 0x2b6: 0x9d, 0x2b7: 0x9d, - 0x2b8: 0x9d, 0x2b9: 0x9d, 0x2ba: 0x9d, 0x2bb: 0x9d, 0x2bc: 0x9d, 0x2bd: 0x9d, 0x2be: 0x9d, 0x2bf: 0xe2, + 0x280: 0x9f, 0x281: 0x9f, 0x282: 0x9f, 0x283: 0x9f, 0x284: 0x9f, 0x285: 0x9f, 0x286: 0x9f, 0x287: 0x9f, + 0x288: 0x9f, 0x289: 0x9f, 0x28a: 0x9f, 0x28b: 0x9f, 0x28c: 0x9f, 0x28d: 0x9f, 0x28e: 0x9f, 0x28f: 0x9f, + 0x290: 0x9f, 0x291: 0x9f, 0x292: 0x9f, 0x293: 0x9f, 0x294: 0x9f, 0x295: 0x9f, 0x296: 0x9f, 0x297: 0x9f, + 0x298: 0x9f, 0x299: 0x9f, 0x29a: 0x9f, 0x29b: 0x9f, 0x29c: 0x9f, 0x29d: 0x9f, 0x29e: 0x9f, 0x29f: 0x9f, + 0x2a0: 0x9f, 0x2a1: 0x9f, 0x2a2: 0x9f, 0x2a3: 0x9f, 0x2a4: 0x9f, 0x2a5: 0x9f, 0x2a6: 0x9f, 0x2a7: 0x9f, + 0x2a8: 0x9f, 0x2a9: 0x9f, 0x2aa: 0x9f, 0x2ab: 0x9f, 0x2ac: 0x9f, 0x2ad: 0x9f, 0x2ae: 0x9f, 0x2af: 0x9f, + 0x2b0: 0x9f, 0x2b1: 0x9f, 0x2b2: 0x9f, 0x2b3: 0x9f, 0x2b4: 0x9f, 0x2b5: 0x9f, 0x2b6: 0x9f, 0x2b7: 0x9f, + 0x2b8: 0x9f, 0x2b9: 0x9f, 0x2ba: 0x9f, 0x2bb: 0x9f, 0x2bc: 0x9f, 0x2bd: 0x9f, 0x2be: 0x9f, 0x2bf: 0xe3, // Block 0xb, offset 0x2c0 - 0x2c0: 0x9d, 0x2c1: 0x9d, 0x2c2: 0x9d, 0x2c3: 0x9d, 0x2c4: 0x9d, 0x2c5: 0x9d, 0x2c6: 0x9d, 0x2c7: 0x9d, - 0x2c8: 0x9d, 0x2c9: 0x9d, 0x2ca: 0x9d, 0x2cb: 0x9d, 0x2cc: 0x9d, 0x2cd: 0x9d, 0x2ce: 0x9d, 0x2cf: 0x9d, - 0x2d0: 0x9d, 0x2d1: 0x9d, 0x2d2: 0xe3, 0x2d3: 0xe4, 0x2d4: 0x9d, 0x2d5: 0x9d, 0x2d6: 0x9d, 0x2d7: 0x9d, - 0x2d8: 0xe5, 0x2d9: 0x40, 0x2da: 0x41, 0x2db: 0xe6, 0x2dc: 0x42, 0x2dd: 0x43, 0x2de: 0x44, 0x2df: 0xe7, - 0x2e0: 0xe8, 0x2e1: 0xe9, 0x2e2: 0xea, 0x2e3: 0xeb, 0x2e4: 0xec, 0x2e5: 0xed, 0x2e6: 0xee, 0x2e7: 0xef, - 0x2e8: 0xf0, 0x2e9: 0xf1, 0x2ea: 0xf2, 0x2eb: 0xf3, 0x2ec: 0xf4, 0x2ed: 0xf5, 0x2ee: 0xf6, 0x2ef: 0xf7, - 0x2f0: 0x9d, 0x2f1: 0x9d, 0x2f2: 0x9d, 0x2f3: 0x9d, 0x2f4: 0x9d, 0x2f5: 0x9d, 0x2f6: 0x9d, 0x2f7: 0x9d, - 0x2f8: 0x9d, 0x2f9: 0x9d, 0x2fa: 0x9d, 0x2fb: 0x9d, 0x2fc: 0x9d, 0x2fd: 0x9d, 0x2fe: 0x9d, 0x2ff: 0x9d, + 0x2c0: 0x9f, 0x2c1: 0x9f, 0x2c2: 0x9f, 0x2c3: 0x9f, 0x2c4: 0x9f, 0x2c5: 0x9f, 0x2c6: 0x9f, 0x2c7: 0x9f, + 0x2c8: 0x9f, 0x2c9: 0x9f, 0x2ca: 0x9f, 0x2cb: 0x9f, 0x2cc: 0x9f, 0x2cd: 0x9f, 0x2ce: 0x9f, 0x2cf: 0x9f, + 0x2d0: 0x9f, 0x2d1: 0x9f, 0x2d2: 0xe4, 0x2d3: 0xe5, 0x2d4: 0x9f, 0x2d5: 0x9f, 0x2d6: 0x9f, 0x2d7: 0x9f, + 0x2d8: 0xe6, 0x2d9: 0x42, 0x2da: 0x43, 0x2db: 0xe7, 0x2dc: 0x44, 0x2dd: 0x45, 0x2de: 0x46, 0x2df: 0xe8, + 0x2e0: 0xe9, 0x2e1: 0xea, 0x2e2: 0xeb, 0x2e3: 0xec, 0x2e4: 0xed, 0x2e5: 0xee, 0x2e6: 0xef, 0x2e7: 0xf0, + 0x2e8: 0xf1, 0x2e9: 0xf2, 0x2ea: 0xf3, 0x2eb: 0xf4, 0x2ec: 0xf5, 0x2ed: 0xf6, 0x2ee: 0xf7, 0x2ef: 0xf8, + 0x2f0: 0x9f, 0x2f1: 0x9f, 0x2f2: 0x9f, 0x2f3: 0x9f, 0x2f4: 0x9f, 0x2f5: 0x9f, 0x2f6: 0x9f, 0x2f7: 0x9f, + 0x2f8: 0x9f, 0x2f9: 0x9f, 0x2fa: 0x9f, 0x2fb: 0x9f, 0x2fc: 0x9f, 0x2fd: 0x9f, 0x2fe: 0x9f, 0x2ff: 0x9f, // Block 0xc, offset 0x300 - 0x300: 0x9d, 0x301: 0x9d, 0x302: 0x9d, 0x303: 0x9d, 0x304: 0x9d, 0x305: 0x9d, 0x306: 0x9d, 0x307: 0x9d, - 0x308: 0x9d, 0x309: 0x9d, 0x30a: 0x9d, 0x30b: 0x9d, 0x30c: 0x9d, 0x30d: 0x9d, 0x30e: 0x9d, 0x30f: 0x9d, - 0x310: 0x9d, 0x311: 0x9d, 0x312: 0x9d, 0x313: 0x9d, 0x314: 0x9d, 0x315: 0x9d, 0x316: 0x9d, 0x317: 0x9d, - 0x318: 0x9d, 0x319: 0x9d, 0x31a: 0x9d, 0x31b: 0x9d, 0x31c: 0x9d, 0x31d: 0x9d, 0x31e: 0xf8, 0x31f: 0xf9, + 0x300: 0x9f, 0x301: 0x9f, 0x302: 0x9f, 0x303: 0x9f, 0x304: 0x9f, 0x305: 0x9f, 0x306: 0x9f, 0x307: 0x9f, + 0x308: 0x9f, 0x309: 0x9f, 0x30a: 0x9f, 0x30b: 0x9f, 0x30c: 0x9f, 0x30d: 0x9f, 0x30e: 0x9f, 0x30f: 0x9f, + 0x310: 0x9f, 0x311: 0x9f, 0x312: 0x9f, 0x313: 0x9f, 0x314: 0x9f, 0x315: 0x9f, 0x316: 0x9f, 0x317: 0x9f, + 0x318: 0x9f, 0x319: 0x9f, 0x31a: 0x9f, 0x31b: 0x9f, 0x31c: 0x9f, 0x31d: 0x9f, 0x31e: 0xf9, 0x31f: 0xfa, // Block 0xd, offset 0x340 - 0x340: 0xb8, 0x341: 0xb8, 0x342: 0xb8, 0x343: 0xb8, 0x344: 0xb8, 0x345: 0xb8, 0x346: 0xb8, 0x347: 0xb8, - 0x348: 0xb8, 0x349: 0xb8, 0x34a: 0xb8, 0x34b: 0xb8, 0x34c: 0xb8, 0x34d: 0xb8, 0x34e: 0xb8, 0x34f: 0xb8, - 0x350: 0xb8, 0x351: 0xb8, 0x352: 0xb8, 0x353: 0xb8, 0x354: 0xb8, 0x355: 0xb8, 0x356: 0xb8, 0x357: 0xb8, - 0x358: 0xb8, 0x359: 0xb8, 0x35a: 0xb8, 0x35b: 0xb8, 0x35c: 0xb8, 0x35d: 0xb8, 0x35e: 0xb8, 0x35f: 0xb8, - 0x360: 0xb8, 0x361: 0xb8, 0x362: 0xb8, 0x363: 0xb8, 0x364: 0xb8, 0x365: 0xb8, 0x366: 0xb8, 0x367: 0xb8, - 0x368: 0xb8, 0x369: 0xb8, 0x36a: 0xb8, 0x36b: 0xb8, 0x36c: 0xb8, 0x36d: 0xb8, 0x36e: 0xb8, 0x36f: 0xb8, - 0x370: 0xb8, 0x371: 0xb8, 0x372: 0xb8, 0x373: 0xb8, 0x374: 0xb8, 0x375: 0xb8, 0x376: 0xb8, 0x377: 0xb8, - 0x378: 0xb8, 0x379: 0xb8, 0x37a: 0xb8, 0x37b: 0xb8, 0x37c: 0xb8, 0x37d: 0xb8, 0x37e: 0xb8, 0x37f: 0xb8, + 0x340: 0xba, 0x341: 0xba, 0x342: 0xba, 0x343: 0xba, 0x344: 0xba, 0x345: 0xba, 0x346: 0xba, 0x347: 0xba, + 0x348: 0xba, 0x349: 0xba, 0x34a: 0xba, 0x34b: 0xba, 0x34c: 0xba, 0x34d: 0xba, 0x34e: 0xba, 0x34f: 0xba, + 0x350: 0xba, 0x351: 0xba, 0x352: 0xba, 0x353: 0xba, 0x354: 0xba, 0x355: 0xba, 0x356: 0xba, 0x357: 0xba, + 0x358: 0xba, 0x359: 0xba, 0x35a: 0xba, 0x35b: 0xba, 0x35c: 0xba, 0x35d: 0xba, 0x35e: 0xba, 0x35f: 0xba, + 0x360: 0xba, 0x361: 0xba, 0x362: 0xba, 0x363: 0xba, 0x364: 0xba, 0x365: 0xba, 0x366: 0xba, 0x367: 0xba, + 0x368: 0xba, 0x369: 0xba, 0x36a: 0xba, 0x36b: 0xba, 0x36c: 0xba, 0x36d: 0xba, 0x36e: 0xba, 0x36f: 0xba, + 0x370: 0xba, 0x371: 0xba, 0x372: 0xba, 0x373: 0xba, 0x374: 0xba, 0x375: 0xba, 0x376: 0xba, 0x377: 0xba, + 0x378: 0xba, 0x379: 0xba, 0x37a: 0xba, 0x37b: 0xba, 0x37c: 0xba, 0x37d: 0xba, 0x37e: 0xba, 0x37f: 0xba, // Block 0xe, offset 0x380 - 0x380: 0xb8, 0x381: 0xb8, 0x382: 0xb8, 0x383: 0xb8, 0x384: 0xb8, 0x385: 0xb8, 0x386: 0xb8, 0x387: 0xb8, - 0x388: 0xb8, 0x389: 0xb8, 0x38a: 0xb8, 0x38b: 0xb8, 0x38c: 0xb8, 0x38d: 0xb8, 0x38e: 0xb8, 0x38f: 0xb8, - 0x390: 0xb8, 0x391: 0xb8, 0x392: 0xb8, 0x393: 0xb8, 0x394: 0xb8, 0x395: 0xb8, 0x396: 0xb8, 0x397: 0xb8, - 0x398: 0xb8, 0x399: 0xb8, 0x39a: 0xb8, 0x39b: 0xb8, 0x39c: 0xb8, 0x39d: 0xb8, 0x39e: 0xb8, 0x39f: 0xb8, - 0x3a0: 0xb8, 0x3a1: 0xb8, 0x3a2: 0xb8, 0x3a3: 0xb8, 0x3a4: 0xfa, 0x3a5: 0xfb, 0x3a6: 0xfc, 0x3a7: 0xfd, - 0x3a8: 0x45, 0x3a9: 0xfe, 0x3aa: 0xff, 0x3ab: 0x46, 0x3ac: 0x47, 0x3ad: 0x48, 0x3ae: 0x49, 0x3af: 0x4a, - 0x3b0: 0x100, 0x3b1: 0x4b, 0x3b2: 0x4c, 0x3b3: 0x4d, 0x3b4: 0x4e, 0x3b5: 0x4f, 0x3b6: 0x101, 0x3b7: 0x50, - 0x3b8: 0x51, 0x3b9: 0x52, 0x3ba: 0x53, 0x3bb: 0x54, 0x3bc: 0x55, 0x3bd: 0x56, 0x3be: 0x57, 0x3bf: 0x58, + 0x380: 0xba, 0x381: 0xba, 0x382: 0xba, 0x383: 0xba, 0x384: 0xba, 0x385: 0xba, 0x386: 0xba, 0x387: 0xba, + 0x388: 0xba, 0x389: 0xba, 0x38a: 0xba, 0x38b: 0xba, 0x38c: 0xba, 0x38d: 0xba, 0x38e: 0xba, 0x38f: 0xba, + 0x390: 0xba, 0x391: 0xba, 0x392: 0xba, 0x393: 0xba, 0x394: 0xba, 0x395: 0xba, 0x396: 0xba, 0x397: 0xba, + 0x398: 0xba, 0x399: 0xba, 0x39a: 0xba, 0x39b: 0xba, 0x39c: 0xba, 0x39d: 0xba, 0x39e: 0xba, 0x39f: 0xba, + 0x3a0: 0xba, 0x3a1: 0xba, 0x3a2: 0xba, 0x3a3: 0xba, 0x3a4: 0xfb, 0x3a5: 0xfc, 0x3a6: 0xfd, 0x3a7: 0xfe, + 0x3a8: 0x47, 0x3a9: 0xff, 0x3aa: 0x100, 0x3ab: 0x48, 0x3ac: 0x49, 0x3ad: 0x4a, 0x3ae: 0x4b, 0x3af: 0x4c, + 0x3b0: 0x101, 0x3b1: 0x4d, 0x3b2: 0x4e, 0x3b3: 0x4f, 0x3b4: 0x50, 0x3b5: 0x51, 0x3b6: 0x102, 0x3b7: 0x52, + 0x3b8: 0x53, 0x3b9: 0x54, 0x3ba: 0x55, 0x3bb: 0x56, 0x3bc: 0x57, 0x3bd: 0x58, 0x3be: 0x59, 0x3bf: 0x5a, // Block 0xf, offset 0x3c0 - 0x3c0: 0x102, 0x3c1: 0x103, 0x3c2: 0x9d, 0x3c3: 0x104, 0x3c4: 0x105, 0x3c5: 0x9b, 0x3c6: 0x106, 0x3c7: 0x107, - 0x3c8: 0xb8, 0x3c9: 0xb8, 0x3ca: 0x108, 0x3cb: 0x109, 0x3cc: 0x10a, 0x3cd: 0x10b, 0x3ce: 0x10c, 0x3cf: 0x10d, - 0x3d0: 0x10e, 0x3d1: 0x9d, 0x3d2: 0x10f, 0x3d3: 0x110, 0x3d4: 0x111, 0x3d5: 0x112, 0x3d6: 0xb8, 0x3d7: 0xb8, - 0x3d8: 0x9d, 0x3d9: 0x9d, 0x3da: 0x9d, 0x3db: 0x9d, 0x3dc: 0x113, 0x3dd: 0x114, 0x3de: 0xb8, 0x3df: 0xb8, - 0x3e0: 0x115, 0x3e1: 0x116, 0x3e2: 0x117, 0x3e3: 0x118, 0x3e4: 0x119, 0x3e5: 0xb8, 0x3e6: 0x11a, 0x3e7: 0x11b, - 0x3e8: 0x11c, 0x3e9: 0x11d, 0x3ea: 0x11e, 0x3eb: 0x59, 0x3ec: 0x11f, 0x3ed: 0x120, 0x3ee: 0x5a, 0x3ef: 0xb8, - 0x3f0: 0x9d, 0x3f1: 0x121, 0x3f2: 0x122, 0x3f3: 0x123, 0x3f4: 0xb8, 0x3f5: 0xb8, 0x3f6: 0xb8, 0x3f7: 0xb8, - 0x3f8: 0xb8, 0x3f9: 0x124, 0x3fa: 0xb8, 0x3fb: 0xb8, 0x3fc: 0xb8, 0x3fd: 0xb8, 0x3fe: 0xb8, 0x3ff: 0xb8, + 0x3c0: 0x103, 0x3c1: 0x104, 0x3c2: 0x9f, 0x3c3: 0x105, 0x3c4: 0x106, 0x3c5: 0x9b, 0x3c6: 0x107, 0x3c7: 0x108, + 0x3c8: 0xba, 0x3c9: 0xba, 0x3ca: 0x109, 0x3cb: 0x10a, 0x3cc: 0x10b, 0x3cd: 0x10c, 0x3ce: 0x10d, 0x3cf: 0x10e, + 0x3d0: 0x10f, 0x3d1: 0x9f, 0x3d2: 0x110, 0x3d3: 0x111, 0x3d4: 0x112, 0x3d5: 0x113, 0x3d6: 0xba, 0x3d7: 0xba, + 0x3d8: 0x9f, 0x3d9: 0x9f, 0x3da: 0x9f, 0x3db: 0x9f, 0x3dc: 0x114, 0x3dd: 0x115, 0x3de: 0xba, 0x3df: 0xba, + 0x3e0: 0x116, 0x3e1: 0x117, 0x3e2: 0x118, 0x3e3: 0x119, 0x3e4: 0x11a, 0x3e5: 0xba, 0x3e6: 0x11b, 0x3e7: 0x11c, + 0x3e8: 0x11d, 0x3e9: 0x11e, 0x3ea: 0x11f, 0x3eb: 0x5b, 0x3ec: 0x120, 0x3ed: 0x121, 0x3ee: 0x5c, 0x3ef: 0xba, + 0x3f0: 0x122, 0x3f1: 0x123, 0x3f2: 0x124, 0x3f3: 0x125, 0x3f4: 0xba, 0x3f5: 0xba, 0x3f6: 0xba, 0x3f7: 0xba, + 0x3f8: 0xba, 0x3f9: 0x126, 0x3fa: 0xba, 0x3fb: 0xba, 0x3fc: 0xba, 0x3fd: 0xba, 0x3fe: 0xba, 0x3ff: 0xba, // Block 0x10, offset 0x400 - 0x400: 0x125, 0x401: 0x126, 0x402: 0x127, 0x403: 0x128, 0x404: 0x129, 0x405: 0x12a, 0x406: 0x12b, 0x407: 0x12c, - 0x408: 0x12d, 0x409: 0xb8, 0x40a: 0x12e, 0x40b: 0x12f, 0x40c: 0x5b, 0x40d: 0x5c, 0x40e: 0xb8, 0x40f: 0xb8, - 0x410: 0x130, 0x411: 0x131, 0x412: 0x132, 0x413: 0x133, 0x414: 0xb8, 0x415: 0xb8, 0x416: 0x134, 0x417: 0x135, - 0x418: 0x136, 0x419: 0x137, 0x41a: 0x138, 0x41b: 0x139, 0x41c: 0x13a, 0x41d: 0xb8, 0x41e: 0xb8, 0x41f: 0xb8, - 0x420: 0xb8, 0x421: 0xb8, 0x422: 0x13b, 0x423: 0x13c, 0x424: 0xb8, 0x425: 0xb8, 0x426: 0xb8, 0x427: 0xb8, - 0x428: 0xb8, 0x429: 0xb8, 0x42a: 0xb8, 0x42b: 0x13d, 0x42c: 0xb8, 0x42d: 0xb8, 0x42e: 0xb8, 0x42f: 0xb8, - 0x430: 0x13e, 0x431: 0x13f, 0x432: 0x140, 0x433: 0xb8, 0x434: 0xb8, 0x435: 0xb8, 0x436: 0xb8, 0x437: 0xb8, - 0x438: 0xb8, 0x439: 0xb8, 0x43a: 0xb8, 0x43b: 0xb8, 0x43c: 0xb8, 0x43d: 0xb8, 0x43e: 0xb8, 0x43f: 0xb8, + 0x400: 0x127, 0x401: 0x128, 0x402: 0x129, 0x403: 0x12a, 0x404: 0x12b, 0x405: 0x12c, 0x406: 0x12d, 0x407: 0x12e, + 0x408: 0x12f, 0x409: 0xba, 0x40a: 0x130, 0x40b: 0x131, 0x40c: 0x5d, 0x40d: 0x5e, 0x40e: 0xba, 0x40f: 0xba, + 0x410: 0x132, 0x411: 0x133, 0x412: 0x134, 0x413: 0x135, 0x414: 0xba, 0x415: 0xba, 0x416: 0x136, 0x417: 0x137, + 0x418: 0x138, 0x419: 0x139, 0x41a: 0x13a, 0x41b: 0x13b, 0x41c: 0x13c, 0x41d: 0xba, 0x41e: 0xba, 0x41f: 0xba, + 0x420: 0xba, 0x421: 0xba, 0x422: 0x13d, 0x423: 0x13e, 0x424: 0xba, 0x425: 0xba, 0x426: 0xba, 0x427: 0xba, + 0x428: 0x13f, 0x429: 0x140, 0x42a: 0x141, 0x42b: 0x142, 0x42c: 0xba, 0x42d: 0xba, 0x42e: 0xba, 0x42f: 0xba, + 0x430: 0x143, 0x431: 0x144, 0x432: 0x145, 0x433: 0xba, 0x434: 0x146, 0x435: 0x147, 0x436: 0xba, 0x437: 0xba, + 0x438: 0xba, 0x439: 0xba, 0x43a: 0xba, 0x43b: 0xba, 0x43c: 0xba, 0x43d: 0xba, 0x43e: 0xba, 0x43f: 0xba, // Block 0x11, offset 0x440 - 0x440: 0x9d, 0x441: 0x9d, 0x442: 0x9d, 0x443: 0x9d, 0x444: 0x9d, 0x445: 0x9d, 0x446: 0x9d, 0x447: 0x9d, - 0x448: 0x9d, 0x449: 0x9d, 0x44a: 0x9d, 0x44b: 0x9d, 0x44c: 0x9d, 0x44d: 0x9d, 0x44e: 0x141, 0x44f: 0xb8, - 0x450: 0x9b, 0x451: 0x142, 0x452: 0x9d, 0x453: 0x9d, 0x454: 0x9d, 0x455: 0x143, 0x456: 0xb8, 0x457: 0xb8, - 0x458: 0xb8, 0x459: 0xb8, 0x45a: 0xb8, 0x45b: 0xb8, 0x45c: 0xb8, 0x45d: 0xb8, 0x45e: 0xb8, 0x45f: 0xb8, - 0x460: 0xb8, 0x461: 0xb8, 0x462: 0xb8, 0x463: 0xb8, 0x464: 0xb8, 0x465: 0xb8, 0x466: 0xb8, 0x467: 0xb8, - 0x468: 0xb8, 0x469: 0xb8, 0x46a: 0xb8, 0x46b: 0xb8, 0x46c: 0xb8, 0x46d: 0xb8, 0x46e: 0xb8, 0x46f: 0xb8, - 0x470: 0xb8, 0x471: 0xb8, 0x472: 0xb8, 0x473: 0xb8, 0x474: 0xb8, 0x475: 0xb8, 0x476: 0xb8, 0x477: 0xb8, - 0x478: 0xb8, 0x479: 0xb8, 0x47a: 0xb8, 0x47b: 0xb8, 0x47c: 0xb8, 0x47d: 0xb8, 0x47e: 0xb8, 0x47f: 0xb8, + 0x440: 0x9f, 0x441: 0x9f, 0x442: 0x9f, 0x443: 0x9f, 0x444: 0x9f, 0x445: 0x9f, 0x446: 0x9f, 0x447: 0x9f, + 0x448: 0x9f, 0x449: 0x9f, 0x44a: 0x9f, 0x44b: 0x9f, 0x44c: 0x9f, 0x44d: 0x9f, 0x44e: 0x148, 0x44f: 0xba, + 0x450: 0x9b, 0x451: 0x149, 0x452: 0x9f, 0x453: 0x9f, 0x454: 0x9f, 0x455: 0x14a, 0x456: 0xba, 0x457: 0xba, + 0x458: 0xba, 0x459: 0xba, 0x45a: 0xba, 0x45b: 0xba, 0x45c: 0xba, 0x45d: 0xba, 0x45e: 0xba, 0x45f: 0xba, + 0x460: 0xba, 0x461: 0xba, 0x462: 0xba, 0x463: 0xba, 0x464: 0xba, 0x465: 0xba, 0x466: 0xba, 0x467: 0xba, + 0x468: 0xba, 0x469: 0xba, 0x46a: 0xba, 0x46b: 0xba, 0x46c: 0xba, 0x46d: 0xba, 0x46e: 0xba, 0x46f: 0xba, + 0x470: 0xba, 0x471: 0xba, 0x472: 0xba, 0x473: 0xba, 0x474: 0xba, 0x475: 0xba, 0x476: 0xba, 0x477: 0xba, + 0x478: 0xba, 0x479: 0xba, 0x47a: 0xba, 0x47b: 0xba, 0x47c: 0xba, 0x47d: 0xba, 0x47e: 0xba, 0x47f: 0xba, // Block 0x12, offset 0x480 - 0x480: 0x9d, 0x481: 0x9d, 0x482: 0x9d, 0x483: 0x9d, 0x484: 0x9d, 0x485: 0x9d, 0x486: 0x9d, 0x487: 0x9d, - 0x488: 0x9d, 0x489: 0x9d, 0x48a: 0x9d, 0x48b: 0x9d, 0x48c: 0x9d, 0x48d: 0x9d, 0x48e: 0x9d, 0x48f: 0x9d, - 0x490: 0x144, 0x491: 0xb8, 0x492: 0xb8, 0x493: 0xb8, 0x494: 0xb8, 0x495: 0xb8, 0x496: 0xb8, 0x497: 0xb8, - 0x498: 0xb8, 0x499: 0xb8, 0x49a: 0xb8, 0x49b: 0xb8, 0x49c: 0xb8, 0x49d: 0xb8, 0x49e: 0xb8, 0x49f: 0xb8, - 0x4a0: 0xb8, 0x4a1: 0xb8, 0x4a2: 0xb8, 0x4a3: 0xb8, 0x4a4: 0xb8, 0x4a5: 0xb8, 0x4a6: 0xb8, 0x4a7: 0xb8, - 0x4a8: 0xb8, 0x4a9: 0xb8, 0x4aa: 0xb8, 0x4ab: 0xb8, 0x4ac: 0xb8, 0x4ad: 0xb8, 0x4ae: 0xb8, 0x4af: 0xb8, - 0x4b0: 0xb8, 0x4b1: 0xb8, 0x4b2: 0xb8, 0x4b3: 0xb8, 0x4b4: 0xb8, 0x4b5: 0xb8, 0x4b6: 0xb8, 0x4b7: 0xb8, - 0x4b8: 0xb8, 0x4b9: 0xb8, 0x4ba: 0xb8, 0x4bb: 0xb8, 0x4bc: 0xb8, 0x4bd: 0xb8, 0x4be: 0xb8, 0x4bf: 0xb8, + 0x480: 0x9f, 0x481: 0x9f, 0x482: 0x9f, 0x483: 0x9f, 0x484: 0x9f, 0x485: 0x9f, 0x486: 0x9f, 0x487: 0x9f, + 0x488: 0x9f, 0x489: 0x9f, 0x48a: 0x9f, 0x48b: 0x9f, 0x48c: 0x9f, 0x48d: 0x9f, 0x48e: 0x9f, 0x48f: 0x9f, + 0x490: 0x14b, 0x491: 0xba, 0x492: 0xba, 0x493: 0xba, 0x494: 0xba, 0x495: 0xba, 0x496: 0xba, 0x497: 0xba, + 0x498: 0xba, 0x499: 0xba, 0x49a: 0xba, 0x49b: 0xba, 0x49c: 0xba, 0x49d: 0xba, 0x49e: 0xba, 0x49f: 0xba, + 0x4a0: 0xba, 0x4a1: 0xba, 0x4a2: 0xba, 0x4a3: 0xba, 0x4a4: 0xba, 0x4a5: 0xba, 0x4a6: 0xba, 0x4a7: 0xba, + 0x4a8: 0xba, 0x4a9: 0xba, 0x4aa: 0xba, 0x4ab: 0xba, 0x4ac: 0xba, 0x4ad: 0xba, 0x4ae: 0xba, 0x4af: 0xba, + 0x4b0: 0xba, 0x4b1: 0xba, 0x4b2: 0xba, 0x4b3: 0xba, 0x4b4: 0xba, 0x4b5: 0xba, 0x4b6: 0xba, 0x4b7: 0xba, + 0x4b8: 0xba, 0x4b9: 0xba, 0x4ba: 0xba, 0x4bb: 0xba, 0x4bc: 0xba, 0x4bd: 0xba, 0x4be: 0xba, 0x4bf: 0xba, // Block 0x13, offset 0x4c0 - 0x4c0: 0xb8, 0x4c1: 0xb8, 0x4c2: 0xb8, 0x4c3: 0xb8, 0x4c4: 0xb8, 0x4c5: 0xb8, 0x4c6: 0xb8, 0x4c7: 0xb8, - 0x4c8: 0xb8, 0x4c9: 0xb8, 0x4ca: 0xb8, 0x4cb: 0xb8, 0x4cc: 0xb8, 0x4cd: 0xb8, 0x4ce: 0xb8, 0x4cf: 0xb8, - 0x4d0: 0x9d, 0x4d1: 0x9d, 0x4d2: 0x9d, 0x4d3: 0x9d, 0x4d4: 0x9d, 0x4d5: 0x9d, 0x4d6: 0x9d, 0x4d7: 0x9d, - 0x4d8: 0x9d, 0x4d9: 0x145, 0x4da: 0xb8, 0x4db: 0xb8, 0x4dc: 0xb8, 0x4dd: 0xb8, 0x4de: 0xb8, 0x4df: 0xb8, - 0x4e0: 0xb8, 0x4e1: 0xb8, 0x4e2: 0xb8, 0x4e3: 0xb8, 0x4e4: 0xb8, 0x4e5: 0xb8, 0x4e6: 0xb8, 0x4e7: 0xb8, - 0x4e8: 0xb8, 0x4e9: 0xb8, 0x4ea: 0xb8, 0x4eb: 0xb8, 0x4ec: 0xb8, 0x4ed: 0xb8, 0x4ee: 0xb8, 0x4ef: 0xb8, - 0x4f0: 0xb8, 0x4f1: 0xb8, 0x4f2: 0xb8, 0x4f3: 0xb8, 0x4f4: 0xb8, 0x4f5: 0xb8, 0x4f6: 0xb8, 0x4f7: 0xb8, - 0x4f8: 0xb8, 0x4f9: 0xb8, 0x4fa: 0xb8, 0x4fb: 0xb8, 0x4fc: 0xb8, 0x4fd: 0xb8, 0x4fe: 0xb8, 0x4ff: 0xb8, + 0x4c0: 0xba, 0x4c1: 0xba, 0x4c2: 0xba, 0x4c3: 0xba, 0x4c4: 0xba, 0x4c5: 0xba, 0x4c6: 0xba, 0x4c7: 0xba, + 0x4c8: 0xba, 0x4c9: 0xba, 0x4ca: 0xba, 0x4cb: 0xba, 0x4cc: 0xba, 0x4cd: 0xba, 0x4ce: 0xba, 0x4cf: 0xba, + 0x4d0: 0x9f, 0x4d1: 0x9f, 0x4d2: 0x9f, 0x4d3: 0x9f, 0x4d4: 0x9f, 0x4d5: 0x9f, 0x4d6: 0x9f, 0x4d7: 0x9f, + 0x4d8: 0x9f, 0x4d9: 0x14c, 0x4da: 0xba, 0x4db: 0xba, 0x4dc: 0xba, 0x4dd: 0xba, 0x4de: 0xba, 0x4df: 0xba, + 0x4e0: 0xba, 0x4e1: 0xba, 0x4e2: 0xba, 0x4e3: 0xba, 0x4e4: 0xba, 0x4e5: 0xba, 0x4e6: 0xba, 0x4e7: 0xba, + 0x4e8: 0xba, 0x4e9: 0xba, 0x4ea: 0xba, 0x4eb: 0xba, 0x4ec: 0xba, 0x4ed: 0xba, 0x4ee: 0xba, 0x4ef: 0xba, + 0x4f0: 0xba, 0x4f1: 0xba, 0x4f2: 0xba, 0x4f3: 0xba, 0x4f4: 0xba, 0x4f5: 0xba, 0x4f6: 0xba, 0x4f7: 0xba, + 0x4f8: 0xba, 0x4f9: 0xba, 0x4fa: 0xba, 0x4fb: 0xba, 0x4fc: 0xba, 0x4fd: 0xba, 0x4fe: 0xba, 0x4ff: 0xba, // Block 0x14, offset 0x500 - 0x500: 0xb8, 0x501: 0xb8, 0x502: 0xb8, 0x503: 0xb8, 0x504: 0xb8, 0x505: 0xb8, 0x506: 0xb8, 0x507: 0xb8, - 0x508: 0xb8, 0x509: 0xb8, 0x50a: 0xb8, 0x50b: 0xb8, 0x50c: 0xb8, 0x50d: 0xb8, 0x50e: 0xb8, 0x50f: 0xb8, - 0x510: 0xb8, 0x511: 0xb8, 0x512: 0xb8, 0x513: 0xb8, 0x514: 0xb8, 0x515: 0xb8, 0x516: 0xb8, 0x517: 0xb8, - 0x518: 0xb8, 0x519: 0xb8, 0x51a: 0xb8, 0x51b: 0xb8, 0x51c: 0xb8, 0x51d: 0xb8, 0x51e: 0xb8, 0x51f: 0xb8, - 0x520: 0x9d, 0x521: 0x9d, 0x522: 0x9d, 0x523: 0x9d, 0x524: 0x9d, 0x525: 0x9d, 0x526: 0x9d, 0x527: 0x9d, - 0x528: 0x13d, 0x529: 0x146, 0x52a: 0xb8, 0x52b: 0x147, 0x52c: 0x148, 0x52d: 0x149, 0x52e: 0x14a, 0x52f: 0xb8, - 0x530: 0xb8, 0x531: 0xb8, 0x532: 0xb8, 0x533: 0xb8, 0x534: 0xb8, 0x535: 0xb8, 0x536: 0xb8, 0x537: 0xb8, - 0x538: 0xb8, 0x539: 0xb8, 0x53a: 0xb8, 0x53b: 0xb8, 0x53c: 0x9d, 0x53d: 0x14b, 0x53e: 0x14c, 0x53f: 0x14d, + 0x500: 0xba, 0x501: 0xba, 0x502: 0xba, 0x503: 0xba, 0x504: 0xba, 0x505: 0xba, 0x506: 0xba, 0x507: 0xba, + 0x508: 0xba, 0x509: 0xba, 0x50a: 0xba, 0x50b: 0xba, 0x50c: 0xba, 0x50d: 0xba, 0x50e: 0xba, 0x50f: 0xba, + 0x510: 0xba, 0x511: 0xba, 0x512: 0xba, 0x513: 0xba, 0x514: 0xba, 0x515: 0xba, 0x516: 0xba, 0x517: 0xba, + 0x518: 0xba, 0x519: 0xba, 0x51a: 0xba, 0x51b: 0xba, 0x51c: 0xba, 0x51d: 0xba, 0x51e: 0xba, 0x51f: 0xba, + 0x520: 0x9f, 0x521: 0x9f, 0x522: 0x9f, 0x523: 0x9f, 0x524: 0x9f, 0x525: 0x9f, 0x526: 0x9f, 0x527: 0x9f, + 0x528: 0x142, 0x529: 0x14d, 0x52a: 0xba, 0x52b: 0x14e, 0x52c: 0x14f, 0x52d: 0x150, 0x52e: 0x151, 0x52f: 0xba, + 0x530: 0xba, 0x531: 0xba, 0x532: 0xba, 0x533: 0xba, 0x534: 0xba, 0x535: 0xba, 0x536: 0xba, 0x537: 0xba, + 0x538: 0xba, 0x539: 0xba, 0x53a: 0xba, 0x53b: 0xba, 0x53c: 0x9f, 0x53d: 0x152, 0x53e: 0x153, 0x53f: 0x154, // Block 0x15, offset 0x540 - 0x540: 0x9d, 0x541: 0x9d, 0x542: 0x9d, 0x543: 0x9d, 0x544: 0x9d, 0x545: 0x9d, 0x546: 0x9d, 0x547: 0x9d, - 0x548: 0x9d, 0x549: 0x9d, 0x54a: 0x9d, 0x54b: 0x9d, 0x54c: 0x9d, 0x54d: 0x9d, 0x54e: 0x9d, 0x54f: 0x9d, - 0x550: 0x9d, 0x551: 0x9d, 0x552: 0x9d, 0x553: 0x9d, 0x554: 0x9d, 0x555: 0x9d, 0x556: 0x9d, 0x557: 0x9d, - 0x558: 0x9d, 0x559: 0x9d, 0x55a: 0x9d, 0x55b: 0x9d, 0x55c: 0x9d, 0x55d: 0x9d, 0x55e: 0x9d, 0x55f: 0x14e, - 0x560: 0x9d, 0x561: 0x9d, 0x562: 0x9d, 0x563: 0x9d, 0x564: 0x9d, 0x565: 0x9d, 0x566: 0x9d, 0x567: 0x9d, - 0x568: 0x9d, 0x569: 0x9d, 0x56a: 0x9d, 0x56b: 0x14f, 0x56c: 0xb8, 0x56d: 0xb8, 0x56e: 0xb8, 0x56f: 0xb8, - 0x570: 0xb8, 0x571: 0xb8, 0x572: 0xb8, 0x573: 0xb8, 0x574: 0xb8, 0x575: 0xb8, 0x576: 0xb8, 0x577: 0xb8, - 0x578: 0xb8, 0x579: 0xb8, 0x57a: 0xb8, 0x57b: 0xb8, 0x57c: 0xb8, 0x57d: 0xb8, 0x57e: 0xb8, 0x57f: 0xb8, + 0x540: 0x9f, 0x541: 0x9f, 0x542: 0x9f, 0x543: 0x9f, 0x544: 0x9f, 0x545: 0x9f, 0x546: 0x9f, 0x547: 0x9f, + 0x548: 0x9f, 0x549: 0x9f, 0x54a: 0x9f, 0x54b: 0x9f, 0x54c: 0x9f, 0x54d: 0x9f, 0x54e: 0x9f, 0x54f: 0x9f, + 0x550: 0x9f, 0x551: 0x9f, 0x552: 0x9f, 0x553: 0x9f, 0x554: 0x9f, 0x555: 0x9f, 0x556: 0x9f, 0x557: 0x9f, + 0x558: 0x9f, 0x559: 0x9f, 0x55a: 0x9f, 0x55b: 0x9f, 0x55c: 0x9f, 0x55d: 0x9f, 0x55e: 0x9f, 0x55f: 0x155, + 0x560: 0x9f, 0x561: 0x9f, 0x562: 0x9f, 0x563: 0x9f, 0x564: 0x9f, 0x565: 0x9f, 0x566: 0x9f, 0x567: 0x9f, + 0x568: 0x9f, 0x569: 0x9f, 0x56a: 0x9f, 0x56b: 0x156, 0x56c: 0xba, 0x56d: 0xba, 0x56e: 0xba, 0x56f: 0xba, + 0x570: 0xba, 0x571: 0xba, 0x572: 0xba, 0x573: 0xba, 0x574: 0xba, 0x575: 0xba, 0x576: 0xba, 0x577: 0xba, + 0x578: 0xba, 0x579: 0xba, 0x57a: 0xba, 0x57b: 0xba, 0x57c: 0xba, 0x57d: 0xba, 0x57e: 0xba, 0x57f: 0xba, // Block 0x16, offset 0x580 - 0x580: 0x150, 0x581: 0xb8, 0x582: 0xb8, 0x583: 0xb8, 0x584: 0xb8, 0x585: 0xb8, 0x586: 0xb8, 0x587: 0xb8, - 0x588: 0xb8, 0x589: 0xb8, 0x58a: 0xb8, 0x58b: 0xb8, 0x58c: 0xb8, 0x58d: 0xb8, 0x58e: 0xb8, 0x58f: 0xb8, - 0x590: 0xb8, 0x591: 0xb8, 0x592: 0xb8, 0x593: 0xb8, 0x594: 0xb8, 0x595: 0xb8, 0x596: 0xb8, 0x597: 0xb8, - 0x598: 0xb8, 0x599: 0xb8, 0x59a: 0xb8, 0x59b: 0xb8, 0x59c: 0xb8, 0x59d: 0xb8, 0x59e: 0xb8, 0x59f: 0xb8, - 0x5a0: 0xb8, 0x5a1: 0xb8, 0x5a2: 0xb8, 0x5a3: 0xb8, 0x5a4: 0xb8, 0x5a5: 0xb8, 0x5a6: 0xb8, 0x5a7: 0xb8, - 0x5a8: 0xb8, 0x5a9: 0xb8, 0x5aa: 0xb8, 0x5ab: 0xb8, 0x5ac: 0xb8, 0x5ad: 0xb8, 0x5ae: 0xb8, 0x5af: 0xb8, - 0x5b0: 0x9d, 0x5b1: 0x151, 0x5b2: 0x152, 0x5b3: 0xb8, 0x5b4: 0xb8, 0x5b5: 0xb8, 0x5b6: 0xb8, 0x5b7: 0xb8, - 0x5b8: 0xb8, 0x5b9: 0xb8, 0x5ba: 0xb8, 0x5bb: 0xb8, 0x5bc: 0xb8, 0x5bd: 0xb8, 0x5be: 0xb8, 0x5bf: 0xb8, + 0x580: 0x9f, 0x581: 0x9f, 0x582: 0x9f, 0x583: 0x9f, 0x584: 0x157, 0x585: 0x158, 0x586: 0x9f, 0x587: 0x9f, + 0x588: 0x9f, 0x589: 0x9f, 0x58a: 0x9f, 0x58b: 0x159, 0x58c: 0xba, 0x58d: 0xba, 0x58e: 0xba, 0x58f: 0xba, + 0x590: 0xba, 0x591: 0xba, 0x592: 0xba, 0x593: 0xba, 0x594: 0xba, 0x595: 0xba, 0x596: 0xba, 0x597: 0xba, + 0x598: 0xba, 0x599: 0xba, 0x59a: 0xba, 0x59b: 0xba, 0x59c: 0xba, 0x59d: 0xba, 0x59e: 0xba, 0x59f: 0xba, + 0x5a0: 0xba, 0x5a1: 0xba, 0x5a2: 0xba, 0x5a3: 0xba, 0x5a4: 0xba, 0x5a5: 0xba, 0x5a6: 0xba, 0x5a7: 0xba, + 0x5a8: 0xba, 0x5a9: 0xba, 0x5aa: 0xba, 0x5ab: 0xba, 0x5ac: 0xba, 0x5ad: 0xba, 0x5ae: 0xba, 0x5af: 0xba, + 0x5b0: 0x9f, 0x5b1: 0x15a, 0x5b2: 0x15b, 0x5b3: 0xba, 0x5b4: 0xba, 0x5b5: 0xba, 0x5b6: 0xba, 0x5b7: 0xba, + 0x5b8: 0xba, 0x5b9: 0xba, 0x5ba: 0xba, 0x5bb: 0xba, 0x5bc: 0xba, 0x5bd: 0xba, 0x5be: 0xba, 0x5bf: 0xba, // Block 0x17, offset 0x5c0 - 0x5c0: 0x9b, 0x5c1: 0x9b, 0x5c2: 0x9b, 0x5c3: 0x153, 0x5c4: 0x154, 0x5c5: 0x155, 0x5c6: 0x156, 0x5c7: 0x157, - 0x5c8: 0x9b, 0x5c9: 0x158, 0x5ca: 0xb8, 0x5cb: 0xb8, 0x5cc: 0x9b, 0x5cd: 0x159, 0x5ce: 0xb8, 0x5cf: 0xb8, - 0x5d0: 0x5d, 0x5d1: 0x5e, 0x5d2: 0x5f, 0x5d3: 0x60, 0x5d4: 0x61, 0x5d5: 0x62, 0x5d6: 0x63, 0x5d7: 0x64, - 0x5d8: 0x65, 0x5d9: 0x66, 0x5da: 0x67, 0x5db: 0x68, 0x5dc: 0x69, 0x5dd: 0x6a, 0x5de: 0x6b, 0x5df: 0x6c, + 0x5c0: 0x9b, 0x5c1: 0x9b, 0x5c2: 0x9b, 0x5c3: 0x15c, 0x5c4: 0x15d, 0x5c5: 0x15e, 0x5c6: 0x15f, 0x5c7: 0x160, + 0x5c8: 0x9b, 0x5c9: 0x161, 0x5ca: 0xba, 0x5cb: 0xba, 0x5cc: 0x9b, 0x5cd: 0x162, 0x5ce: 0xba, 0x5cf: 0xba, + 0x5d0: 0x5f, 0x5d1: 0x60, 0x5d2: 0x61, 0x5d3: 0x62, 0x5d4: 0x63, 0x5d5: 0x64, 0x5d6: 0x65, 0x5d7: 0x66, + 0x5d8: 0x67, 0x5d9: 0x68, 0x5da: 0x69, 0x5db: 0x6a, 0x5dc: 0x6b, 0x5dd: 0x6c, 0x5de: 0x6d, 0x5df: 0x6e, 0x5e0: 0x9b, 0x5e1: 0x9b, 0x5e2: 0x9b, 0x5e3: 0x9b, 0x5e4: 0x9b, 0x5e5: 0x9b, 0x5e6: 0x9b, 0x5e7: 0x9b, - 0x5e8: 0x15a, 0x5e9: 0x15b, 0x5ea: 0x15c, 0x5eb: 0xb8, 0x5ec: 0xb8, 0x5ed: 0xb8, 0x5ee: 0xb8, 0x5ef: 0xb8, - 0x5f0: 0xb8, 0x5f1: 0xb8, 0x5f2: 0xb8, 0x5f3: 0xb8, 0x5f4: 0xb8, 0x5f5: 0xb8, 0x5f6: 0xb8, 0x5f7: 0xb8, - 0x5f8: 0xb8, 0x5f9: 0xb8, 0x5fa: 0xb8, 0x5fb: 0xb8, 0x5fc: 0xb8, 0x5fd: 0xb8, 0x5fe: 0xb8, 0x5ff: 0xb8, + 0x5e8: 0x163, 0x5e9: 0x164, 0x5ea: 0x165, 0x5eb: 0xba, 0x5ec: 0xba, 0x5ed: 0xba, 0x5ee: 0xba, 0x5ef: 0xba, + 0x5f0: 0xba, 0x5f1: 0xba, 0x5f2: 0xba, 0x5f3: 0xba, 0x5f4: 0xba, 0x5f5: 0xba, 0x5f6: 0xba, 0x5f7: 0xba, + 0x5f8: 0xba, 0x5f9: 0xba, 0x5fa: 0xba, 0x5fb: 0xba, 0x5fc: 0xba, 0x5fd: 0xba, 0x5fe: 0xba, 0x5ff: 0xba, // Block 0x18, offset 0x600 - 0x600: 0x15d, 0x601: 0xb8, 0x602: 0xb8, 0x603: 0xb8, 0x604: 0xb8, 0x605: 0xb8, 0x606: 0xb8, 0x607: 0xb8, - 0x608: 0xb8, 0x609: 0xb8, 0x60a: 0xb8, 0x60b: 0xb8, 0x60c: 0xb8, 0x60d: 0xb8, 0x60e: 0xb8, 0x60f: 0xb8, - 0x610: 0xb8, 0x611: 0xb8, 0x612: 0xb8, 0x613: 0xb8, 0x614: 0xb8, 0x615: 0xb8, 0x616: 0xb8, 0x617: 0xb8, - 0x618: 0xb8, 0x619: 0xb8, 0x61a: 0xb8, 0x61b: 0xb8, 0x61c: 0xb8, 0x61d: 0xb8, 0x61e: 0xb8, 0x61f: 0xb8, - 0x620: 0x9d, 0x621: 0x9d, 0x622: 0x9d, 0x623: 0x15e, 0x624: 0x6d, 0x625: 0x15f, 0x626: 0xb8, 0x627: 0xb8, - 0x628: 0xb8, 0x629: 0xb8, 0x62a: 0xb8, 0x62b: 0xb8, 0x62c: 0xb8, 0x62d: 0xb8, 0x62e: 0xb8, 0x62f: 0xb8, - 0x630: 0xb8, 0x631: 0xb8, 0x632: 0xb8, 0x633: 0xb8, 0x634: 0xb8, 0x635: 0xb8, 0x636: 0xb8, 0x637: 0xb8, - 0x638: 0x6e, 0x639: 0x6f, 0x63a: 0x70, 0x63b: 0x160, 0x63c: 0xb8, 0x63d: 0xb8, 0x63e: 0xb8, 0x63f: 0xb8, + 0x600: 0x166, 0x601: 0xba, 0x602: 0xba, 0x603: 0xba, 0x604: 0xba, 0x605: 0xba, 0x606: 0xba, 0x607: 0xba, + 0x608: 0xba, 0x609: 0xba, 0x60a: 0xba, 0x60b: 0xba, 0x60c: 0xba, 0x60d: 0xba, 0x60e: 0xba, 0x60f: 0xba, + 0x610: 0xba, 0x611: 0xba, 0x612: 0xba, 0x613: 0xba, 0x614: 0xba, 0x615: 0xba, 0x616: 0xba, 0x617: 0xba, + 0x618: 0xba, 0x619: 0xba, 0x61a: 0xba, 0x61b: 0xba, 0x61c: 0xba, 0x61d: 0xba, 0x61e: 0xba, 0x61f: 0xba, + 0x620: 0x122, 0x621: 0x122, 0x622: 0x122, 0x623: 0x167, 0x624: 0x6f, 0x625: 0x168, 0x626: 0xba, 0x627: 0xba, + 0x628: 0xba, 0x629: 0xba, 0x62a: 0xba, 0x62b: 0xba, 0x62c: 0xba, 0x62d: 0xba, 0x62e: 0xba, 0x62f: 0xba, + 0x630: 0xba, 0x631: 0xba, 0x632: 0xba, 0x633: 0xba, 0x634: 0xba, 0x635: 0xba, 0x636: 0xba, 0x637: 0xba, + 0x638: 0x70, 0x639: 0x71, 0x63a: 0x72, 0x63b: 0x169, 0x63c: 0xba, 0x63d: 0xba, 0x63e: 0xba, 0x63f: 0xba, // Block 0x19, offset 0x640 - 0x640: 0x161, 0x641: 0x9b, 0x642: 0x162, 0x643: 0x163, 0x644: 0x71, 0x645: 0x72, 0x646: 0x164, 0x647: 0x165, - 0x648: 0x73, 0x649: 0x166, 0x64a: 0xb8, 0x64b: 0xb8, 0x64c: 0x9b, 0x64d: 0x9b, 0x64e: 0x9b, 0x64f: 0x9b, + 0x640: 0x16a, 0x641: 0x9b, 0x642: 0x16b, 0x643: 0x16c, 0x644: 0x73, 0x645: 0x74, 0x646: 0x16d, 0x647: 0x16e, + 0x648: 0x75, 0x649: 0x16f, 0x64a: 0xba, 0x64b: 0xba, 0x64c: 0x9b, 0x64d: 0x9b, 0x64e: 0x9b, 0x64f: 0x9b, 0x650: 0x9b, 0x651: 0x9b, 0x652: 0x9b, 0x653: 0x9b, 0x654: 0x9b, 0x655: 0x9b, 0x656: 0x9b, 0x657: 0x9b, - 0x658: 0x9b, 0x659: 0x9b, 0x65a: 0x9b, 0x65b: 0x167, 0x65c: 0x9b, 0x65d: 0x168, 0x65e: 0x9b, 0x65f: 0x169, - 0x660: 0x16a, 0x661: 0x16b, 0x662: 0x16c, 0x663: 0xb8, 0x664: 0x16d, 0x665: 0x16e, 0x666: 0x16f, 0x667: 0x170, - 0x668: 0xb8, 0x669: 0xb8, 0x66a: 0xb8, 0x66b: 0xb8, 0x66c: 0xb8, 0x66d: 0xb8, 0x66e: 0xb8, 0x66f: 0xb8, - 0x670: 0xb8, 0x671: 0xb8, 0x672: 0xb8, 0x673: 0xb8, 0x674: 0xb8, 0x675: 0xb8, 0x676: 0xb8, 0x677: 0xb8, - 0x678: 0xb8, 0x679: 0xb8, 0x67a: 0xb8, 0x67b: 0xb8, 0x67c: 0xb8, 0x67d: 0xb8, 0x67e: 0xb8, 0x67f: 0xb8, + 0x658: 0x9b, 0x659: 0x9b, 0x65a: 0x9b, 0x65b: 0x170, 0x65c: 0x9b, 0x65d: 0x171, 0x65e: 0x9b, 0x65f: 0x172, + 0x660: 0x173, 0x661: 0x174, 0x662: 0x175, 0x663: 0xba, 0x664: 0x176, 0x665: 0x177, 0x666: 0x178, 0x667: 0x179, + 0x668: 0xba, 0x669: 0xba, 0x66a: 0xba, 0x66b: 0xba, 0x66c: 0xba, 0x66d: 0xba, 0x66e: 0xba, 0x66f: 0xba, + 0x670: 0xba, 0x671: 0xba, 0x672: 0xba, 0x673: 0xba, 0x674: 0xba, 0x675: 0xba, 0x676: 0xba, 0x677: 0xba, + 0x678: 0xba, 0x679: 0xba, 0x67a: 0xba, 0x67b: 0xba, 0x67c: 0xba, 0x67d: 0xba, 0x67e: 0xba, 0x67f: 0xba, // Block 0x1a, offset 0x680 - 0x680: 0x9d, 0x681: 0x9d, 0x682: 0x9d, 0x683: 0x9d, 0x684: 0x9d, 0x685: 0x9d, 0x686: 0x9d, 0x687: 0x9d, - 0x688: 0x9d, 0x689: 0x9d, 0x68a: 0x9d, 0x68b: 0x9d, 0x68c: 0x9d, 0x68d: 0x9d, 0x68e: 0x9d, 0x68f: 0x9d, - 0x690: 0x9d, 0x691: 0x9d, 0x692: 0x9d, 0x693: 0x9d, 0x694: 0x9d, 0x695: 0x9d, 0x696: 0x9d, 0x697: 0x9d, - 0x698: 0x9d, 0x699: 0x9d, 0x69a: 0x9d, 0x69b: 0x171, 0x69c: 0x9d, 0x69d: 0x9d, 0x69e: 0x9d, 0x69f: 0x9d, - 0x6a0: 0x9d, 0x6a1: 0x9d, 0x6a2: 0x9d, 0x6a3: 0x9d, 0x6a4: 0x9d, 0x6a5: 0x9d, 0x6a6: 0x9d, 0x6a7: 0x9d, - 0x6a8: 0x9d, 0x6a9: 0x9d, 0x6aa: 0x9d, 0x6ab: 0x9d, 0x6ac: 0x9d, 0x6ad: 0x9d, 0x6ae: 0x9d, 0x6af: 0x9d, - 0x6b0: 0x9d, 0x6b1: 0x9d, 0x6b2: 0x9d, 0x6b3: 0x9d, 0x6b4: 0x9d, 0x6b5: 0x9d, 0x6b6: 0x9d, 0x6b7: 0x9d, - 0x6b8: 0x9d, 0x6b9: 0x9d, 0x6ba: 0x9d, 0x6bb: 0x9d, 0x6bc: 0x9d, 0x6bd: 0x9d, 0x6be: 0x9d, 0x6bf: 0x9d, + 0x680: 0x9f, 0x681: 0x9f, 0x682: 0x9f, 0x683: 0x9f, 0x684: 0x9f, 0x685: 0x9f, 0x686: 0x9f, 0x687: 0x9f, + 0x688: 0x9f, 0x689: 0x9f, 0x68a: 0x9f, 0x68b: 0x9f, 0x68c: 0x9f, 0x68d: 0x9f, 0x68e: 0x9f, 0x68f: 0x9f, + 0x690: 0x9f, 0x691: 0x9f, 0x692: 0x9f, 0x693: 0x9f, 0x694: 0x9f, 0x695: 0x9f, 0x696: 0x9f, 0x697: 0x9f, + 0x698: 0x9f, 0x699: 0x9f, 0x69a: 0x9f, 0x69b: 0x17a, 0x69c: 0x9f, 0x69d: 0x9f, 0x69e: 0x9f, 0x69f: 0x9f, + 0x6a0: 0x9f, 0x6a1: 0x9f, 0x6a2: 0x9f, 0x6a3: 0x9f, 0x6a4: 0x9f, 0x6a5: 0x9f, 0x6a6: 0x9f, 0x6a7: 0x9f, + 0x6a8: 0x9f, 0x6a9: 0x9f, 0x6aa: 0x9f, 0x6ab: 0x9f, 0x6ac: 0x9f, 0x6ad: 0x9f, 0x6ae: 0x9f, 0x6af: 0x9f, + 0x6b0: 0x9f, 0x6b1: 0x9f, 0x6b2: 0x9f, 0x6b3: 0x9f, 0x6b4: 0x9f, 0x6b5: 0x9f, 0x6b6: 0x9f, 0x6b7: 0x9f, + 0x6b8: 0x9f, 0x6b9: 0x9f, 0x6ba: 0x9f, 0x6bb: 0x9f, 0x6bc: 0x9f, 0x6bd: 0x9f, 0x6be: 0x9f, 0x6bf: 0x9f, // Block 0x1b, offset 0x6c0 - 0x6c0: 0x9d, 0x6c1: 0x9d, 0x6c2: 0x9d, 0x6c3: 0x9d, 0x6c4: 0x9d, 0x6c5: 0x9d, 0x6c6: 0x9d, 0x6c7: 0x9d, - 0x6c8: 0x9d, 0x6c9: 0x9d, 0x6ca: 0x9d, 0x6cb: 0x9d, 0x6cc: 0x9d, 0x6cd: 0x9d, 0x6ce: 0x9d, 0x6cf: 0x9d, - 0x6d0: 0x9d, 0x6d1: 0x9d, 0x6d2: 0x9d, 0x6d3: 0x9d, 0x6d4: 0x9d, 0x6d5: 0x9d, 0x6d6: 0x9d, 0x6d7: 0x9d, - 0x6d8: 0x9d, 0x6d9: 0x9d, 0x6da: 0x9d, 0x6db: 0x9d, 0x6dc: 0x172, 0x6dd: 0x9d, 0x6de: 0x9d, 0x6df: 0x9d, - 0x6e0: 0x173, 0x6e1: 0x9d, 0x6e2: 0x9d, 0x6e3: 0x9d, 0x6e4: 0x9d, 0x6e5: 0x9d, 0x6e6: 0x9d, 0x6e7: 0x9d, - 0x6e8: 0x9d, 0x6e9: 0x9d, 0x6ea: 0x9d, 0x6eb: 0x9d, 0x6ec: 0x9d, 0x6ed: 0x9d, 0x6ee: 0x9d, 0x6ef: 0x9d, - 0x6f0: 0x9d, 0x6f1: 0x9d, 0x6f2: 0x9d, 0x6f3: 0x9d, 0x6f4: 0x9d, 0x6f5: 0x9d, 0x6f6: 0x9d, 0x6f7: 0x9d, - 0x6f8: 0x9d, 0x6f9: 0x9d, 0x6fa: 0x9d, 0x6fb: 0x9d, 0x6fc: 0x9d, 0x6fd: 0x9d, 0x6fe: 0x9d, 0x6ff: 0x9d, + 0x6c0: 0x9f, 0x6c1: 0x9f, 0x6c2: 0x9f, 0x6c3: 0x9f, 0x6c4: 0x9f, 0x6c5: 0x9f, 0x6c6: 0x9f, 0x6c7: 0x9f, + 0x6c8: 0x9f, 0x6c9: 0x9f, 0x6ca: 0x9f, 0x6cb: 0x9f, 0x6cc: 0x9f, 0x6cd: 0x9f, 0x6ce: 0x9f, 0x6cf: 0x9f, + 0x6d0: 0x9f, 0x6d1: 0x9f, 0x6d2: 0x9f, 0x6d3: 0x9f, 0x6d4: 0x9f, 0x6d5: 0x9f, 0x6d6: 0x9f, 0x6d7: 0x9f, + 0x6d8: 0x9f, 0x6d9: 0x9f, 0x6da: 0x9f, 0x6db: 0x9f, 0x6dc: 0x17b, 0x6dd: 0x9f, 0x6de: 0x9f, 0x6df: 0x9f, + 0x6e0: 0x17c, 0x6e1: 0x9f, 0x6e2: 0x9f, 0x6e3: 0x9f, 0x6e4: 0x9f, 0x6e5: 0x9f, 0x6e6: 0x9f, 0x6e7: 0x9f, + 0x6e8: 0x9f, 0x6e9: 0x9f, 0x6ea: 0x9f, 0x6eb: 0x9f, 0x6ec: 0x9f, 0x6ed: 0x9f, 0x6ee: 0x9f, 0x6ef: 0x9f, + 0x6f0: 0x9f, 0x6f1: 0x9f, 0x6f2: 0x9f, 0x6f3: 0x9f, 0x6f4: 0x9f, 0x6f5: 0x9f, 0x6f6: 0x9f, 0x6f7: 0x9f, + 0x6f8: 0x9f, 0x6f9: 0x9f, 0x6fa: 0x9f, 0x6fb: 0x9f, 0x6fc: 0x9f, 0x6fd: 0x9f, 0x6fe: 0x9f, 0x6ff: 0x9f, // Block 0x1c, offset 0x700 - 0x700: 0x9d, 0x701: 0x9d, 0x702: 0x9d, 0x703: 0x9d, 0x704: 0x9d, 0x705: 0x9d, 0x706: 0x9d, 0x707: 0x9d, - 0x708: 0x9d, 0x709: 0x9d, 0x70a: 0x9d, 0x70b: 0x9d, 0x70c: 0x9d, 0x70d: 0x9d, 0x70e: 0x9d, 0x70f: 0x9d, - 0x710: 0x9d, 0x711: 0x9d, 0x712: 0x9d, 0x713: 0x9d, 0x714: 0x9d, 0x715: 0x9d, 0x716: 0x9d, 0x717: 0x9d, - 0x718: 0x9d, 0x719: 0x9d, 0x71a: 0x9d, 0x71b: 0x9d, 0x71c: 0x9d, 0x71d: 0x9d, 0x71e: 0x9d, 0x71f: 0x9d, - 0x720: 0x9d, 0x721: 0x9d, 0x722: 0x9d, 0x723: 0x9d, 0x724: 0x9d, 0x725: 0x9d, 0x726: 0x9d, 0x727: 0x9d, - 0x728: 0x9d, 0x729: 0x9d, 0x72a: 0x9d, 0x72b: 0x9d, 0x72c: 0x9d, 0x72d: 0x9d, 0x72e: 0x9d, 0x72f: 0x9d, - 0x730: 0x9d, 0x731: 0x9d, 0x732: 0x9d, 0x733: 0x9d, 0x734: 0x9d, 0x735: 0x9d, 0x736: 0x9d, 0x737: 0x9d, - 0x738: 0x9d, 0x739: 0x9d, 0x73a: 0x174, 0x73b: 0xb8, 0x73c: 0xb8, 0x73d: 0xb8, 0x73e: 0xb8, 0x73f: 0xb8, + 0x700: 0x9f, 0x701: 0x9f, 0x702: 0x9f, 0x703: 0x9f, 0x704: 0x9f, 0x705: 0x9f, 0x706: 0x9f, 0x707: 0x9f, + 0x708: 0x9f, 0x709: 0x9f, 0x70a: 0x9f, 0x70b: 0x9f, 0x70c: 0x9f, 0x70d: 0x9f, 0x70e: 0x9f, 0x70f: 0x9f, + 0x710: 0x9f, 0x711: 0x9f, 0x712: 0x9f, 0x713: 0x9f, 0x714: 0x9f, 0x715: 0x9f, 0x716: 0x9f, 0x717: 0x9f, + 0x718: 0x9f, 0x719: 0x9f, 0x71a: 0x9f, 0x71b: 0x9f, 0x71c: 0x9f, 0x71d: 0x9f, 0x71e: 0x9f, 0x71f: 0x9f, + 0x720: 0x9f, 0x721: 0x9f, 0x722: 0x9f, 0x723: 0x9f, 0x724: 0x9f, 0x725: 0x9f, 0x726: 0x9f, 0x727: 0x9f, + 0x728: 0x9f, 0x729: 0x9f, 0x72a: 0x9f, 0x72b: 0x9f, 0x72c: 0x9f, 0x72d: 0x9f, 0x72e: 0x9f, 0x72f: 0x9f, + 0x730: 0x9f, 0x731: 0x9f, 0x732: 0x9f, 0x733: 0x9f, 0x734: 0x9f, 0x735: 0x9f, 0x736: 0x9f, 0x737: 0x9f, + 0x738: 0x9f, 0x739: 0x9f, 0x73a: 0x17d, 0x73b: 0x9f, 0x73c: 0x9f, 0x73d: 0x9f, 0x73e: 0x9f, 0x73f: 0x9f, // Block 0x1d, offset 0x740 - 0x740: 0xb8, 0x741: 0xb8, 0x742: 0xb8, 0x743: 0xb8, 0x744: 0xb8, 0x745: 0xb8, 0x746: 0xb8, 0x747: 0xb8, - 0x748: 0xb8, 0x749: 0xb8, 0x74a: 0xb8, 0x74b: 0xb8, 0x74c: 0xb8, 0x74d: 0xb8, 0x74e: 0xb8, 0x74f: 0xb8, - 0x750: 0xb8, 0x751: 0xb8, 0x752: 0xb8, 0x753: 0xb8, 0x754: 0xb8, 0x755: 0xb8, 0x756: 0xb8, 0x757: 0xb8, - 0x758: 0xb8, 0x759: 0xb8, 0x75a: 0xb8, 0x75b: 0xb8, 0x75c: 0xb8, 0x75d: 0xb8, 0x75e: 0xb8, 0x75f: 0xb8, - 0x760: 0x74, 0x761: 0x75, 0x762: 0x76, 0x763: 0x175, 0x764: 0x77, 0x765: 0x78, 0x766: 0x176, 0x767: 0x79, - 0x768: 0x7a, 0x769: 0xb8, 0x76a: 0xb8, 0x76b: 0xb8, 0x76c: 0xb8, 0x76d: 0xb8, 0x76e: 0xb8, 0x76f: 0xb8, - 0x770: 0xb8, 0x771: 0xb8, 0x772: 0xb8, 0x773: 0xb8, 0x774: 0xb8, 0x775: 0xb8, 0x776: 0xb8, 0x777: 0xb8, - 0x778: 0xb8, 0x779: 0xb8, 0x77a: 0xb8, 0x77b: 0xb8, 0x77c: 0xb8, 0x77d: 0xb8, 0x77e: 0xb8, 0x77f: 0xb8, + 0x740: 0x9f, 0x741: 0x9f, 0x742: 0x9f, 0x743: 0x9f, 0x744: 0x9f, 0x745: 0x9f, 0x746: 0x9f, 0x747: 0x9f, + 0x748: 0x9f, 0x749: 0x9f, 0x74a: 0x9f, 0x74b: 0x9f, 0x74c: 0x9f, 0x74d: 0x9f, 0x74e: 0x9f, 0x74f: 0x9f, + 0x750: 0x9f, 0x751: 0x9f, 0x752: 0x9f, 0x753: 0x9f, 0x754: 0x9f, 0x755: 0x9f, 0x756: 0x9f, 0x757: 0x9f, + 0x758: 0x9f, 0x759: 0x9f, 0x75a: 0x9f, 0x75b: 0x9f, 0x75c: 0x9f, 0x75d: 0x9f, 0x75e: 0x9f, 0x75f: 0x9f, + 0x760: 0x9f, 0x761: 0x9f, 0x762: 0x9f, 0x763: 0x9f, 0x764: 0x9f, 0x765: 0x9f, 0x766: 0x9f, 0x767: 0x9f, + 0x768: 0x9f, 0x769: 0x9f, 0x76a: 0x9f, 0x76b: 0x9f, 0x76c: 0x9f, 0x76d: 0x9f, 0x76e: 0x9f, 0x76f: 0x17e, + 0x770: 0xba, 0x771: 0xba, 0x772: 0xba, 0x773: 0xba, 0x774: 0xba, 0x775: 0xba, 0x776: 0xba, 0x777: 0xba, + 0x778: 0xba, 0x779: 0xba, 0x77a: 0xba, 0x77b: 0xba, 0x77c: 0xba, 0x77d: 0xba, 0x77e: 0xba, 0x77f: 0xba, // Block 0x1e, offset 0x780 - 0x790: 0x0d, 0x791: 0x0e, 0x792: 0x0f, 0x793: 0x10, 0x794: 0x11, 0x795: 0x0b, 0x796: 0x12, 0x797: 0x07, - 0x798: 0x13, 0x799: 0x0b, 0x79a: 0x0b, 0x79b: 0x14, 0x79c: 0x0b, 0x79d: 0x15, 0x79e: 0x16, 0x79f: 0x17, - 0x7a0: 0x07, 0x7a1: 0x07, 0x7a2: 0x07, 0x7a3: 0x07, 0x7a4: 0x07, 0x7a5: 0x07, 0x7a6: 0x07, 0x7a7: 0x07, - 0x7a8: 0x07, 0x7a9: 0x07, 0x7aa: 0x18, 0x7ab: 0x19, 0x7ac: 0x1a, 0x7ad: 0x0b, 0x7ae: 0x0b, 0x7af: 0x1b, - 0x7b0: 0x0b, 0x7b1: 0x0b, 0x7b2: 0x0b, 0x7b3: 0x0b, 0x7b4: 0x0b, 0x7b5: 0x0b, 0x7b6: 0x0b, 0x7b7: 0x0b, - 0x7b8: 0x0b, 0x7b9: 0x0b, 0x7ba: 0x0b, 0x7bb: 0x0b, 0x7bc: 0x0b, 0x7bd: 0x0b, 0x7be: 0x0b, 0x7bf: 0x0b, + 0x780: 0xba, 0x781: 0xba, 0x782: 0xba, 0x783: 0xba, 0x784: 0xba, 0x785: 0xba, 0x786: 0xba, 0x787: 0xba, + 0x788: 0xba, 0x789: 0xba, 0x78a: 0xba, 0x78b: 0xba, 0x78c: 0xba, 0x78d: 0xba, 0x78e: 0xba, 0x78f: 0xba, + 0x790: 0xba, 0x791: 0xba, 0x792: 0xba, 0x793: 0xba, 0x794: 0xba, 0x795: 0xba, 0x796: 0xba, 0x797: 0xba, + 0x798: 0xba, 0x799: 0xba, 0x79a: 0xba, 0x79b: 0xba, 0x79c: 0xba, 0x79d: 0xba, 0x79e: 0xba, 0x79f: 0xba, + 0x7a0: 0x76, 0x7a1: 0x77, 0x7a2: 0x78, 0x7a3: 0x17f, 0x7a4: 0x79, 0x7a5: 0x7a, 0x7a6: 0x180, 0x7a7: 0x7b, + 0x7a8: 0x7c, 0x7a9: 0xba, 0x7aa: 0xba, 0x7ab: 0xba, 0x7ac: 0xba, 0x7ad: 0xba, 0x7ae: 0xba, 0x7af: 0xba, + 0x7b0: 0xba, 0x7b1: 0xba, 0x7b2: 0xba, 0x7b3: 0xba, 0x7b4: 0xba, 0x7b5: 0xba, 0x7b6: 0xba, 0x7b7: 0xba, + 0x7b8: 0xba, 0x7b9: 0xba, 0x7ba: 0xba, 0x7bb: 0xba, 0x7bc: 0xba, 0x7bd: 0xba, 0x7be: 0xba, 0x7bf: 0xba, // Block 0x1f, offset 0x7c0 - 0x7c0: 0x0b, 0x7c1: 0x0b, 0x7c2: 0x0b, 0x7c3: 0x0b, 0x7c4: 0x0b, 0x7c5: 0x0b, 0x7c6: 0x0b, 0x7c7: 0x0b, - 0x7c8: 0x0b, 0x7c9: 0x0b, 0x7ca: 0x0b, 0x7cb: 0x0b, 0x7cc: 0x0b, 0x7cd: 0x0b, 0x7ce: 0x0b, 0x7cf: 0x0b, - 0x7d0: 0x0b, 0x7d1: 0x0b, 0x7d2: 0x0b, 0x7d3: 0x0b, 0x7d4: 0x0b, 0x7d5: 0x0b, 0x7d6: 0x0b, 0x7d7: 0x0b, - 0x7d8: 0x0b, 0x7d9: 0x0b, 0x7da: 0x0b, 0x7db: 0x0b, 0x7dc: 0x0b, 0x7dd: 0x0b, 0x7de: 0x0b, 0x7df: 0x0b, - 0x7e0: 0x0b, 0x7e1: 0x0b, 0x7e2: 0x0b, 0x7e3: 0x0b, 0x7e4: 0x0b, 0x7e5: 0x0b, 0x7e6: 0x0b, 0x7e7: 0x0b, - 0x7e8: 0x0b, 0x7e9: 0x0b, 0x7ea: 0x0b, 0x7eb: 0x0b, 0x7ec: 0x0b, 0x7ed: 0x0b, 0x7ee: 0x0b, 0x7ef: 0x0b, + 0x7d0: 0x0d, 0x7d1: 0x0e, 0x7d2: 0x0f, 0x7d3: 0x10, 0x7d4: 0x11, 0x7d5: 0x0b, 0x7d6: 0x12, 0x7d7: 0x07, + 0x7d8: 0x13, 0x7d9: 0x0b, 0x7da: 0x0b, 0x7db: 0x14, 0x7dc: 0x0b, 0x7dd: 0x15, 0x7de: 0x16, 0x7df: 0x17, + 0x7e0: 0x07, 0x7e1: 0x07, 0x7e2: 0x07, 0x7e3: 0x07, 0x7e4: 0x07, 0x7e5: 0x07, 0x7e6: 0x07, 0x7e7: 0x07, + 0x7e8: 0x07, 0x7e9: 0x07, 0x7ea: 0x18, 0x7eb: 0x19, 0x7ec: 0x1a, 0x7ed: 0x07, 0x7ee: 0x1b, 0x7ef: 0x1c, 0x7f0: 0x0b, 0x7f1: 0x0b, 0x7f2: 0x0b, 0x7f3: 0x0b, 0x7f4: 0x0b, 0x7f5: 0x0b, 0x7f6: 0x0b, 0x7f7: 0x0b, 0x7f8: 0x0b, 0x7f9: 0x0b, 0x7fa: 0x0b, 0x7fb: 0x0b, 0x7fc: 0x0b, 0x7fd: 0x0b, 0x7fe: 0x0b, 0x7ff: 0x0b, // Block 0x20, offset 0x800 - 0x800: 0x177, 0x801: 0x178, 0x802: 0xb8, 0x803: 0xb8, 0x804: 0x179, 0x805: 0x179, 0x806: 0x179, 0x807: 0x17a, - 0x808: 0xb8, 0x809: 0xb8, 0x80a: 0xb8, 0x80b: 0xb8, 0x80c: 0xb8, 0x80d: 0xb8, 0x80e: 0xb8, 0x80f: 0xb8, - 0x810: 0xb8, 0x811: 0xb8, 0x812: 0xb8, 0x813: 0xb8, 0x814: 0xb8, 0x815: 0xb8, 0x816: 0xb8, 0x817: 0xb8, - 0x818: 0xb8, 0x819: 0xb8, 0x81a: 0xb8, 0x81b: 0xb8, 0x81c: 0xb8, 0x81d: 0xb8, 0x81e: 0xb8, 0x81f: 0xb8, - 0x820: 0xb8, 0x821: 0xb8, 0x822: 0xb8, 0x823: 0xb8, 0x824: 0xb8, 0x825: 0xb8, 0x826: 0xb8, 0x827: 0xb8, - 0x828: 0xb8, 0x829: 0xb8, 0x82a: 0xb8, 0x82b: 0xb8, 0x82c: 0xb8, 0x82d: 0xb8, 0x82e: 0xb8, 0x82f: 0xb8, - 0x830: 0xb8, 0x831: 0xb8, 0x832: 0xb8, 0x833: 0xb8, 0x834: 0xb8, 0x835: 0xb8, 0x836: 0xb8, 0x837: 0xb8, - 0x838: 0xb8, 0x839: 0xb8, 0x83a: 0xb8, 0x83b: 0xb8, 0x83c: 0xb8, 0x83d: 0xb8, 0x83e: 0xb8, 0x83f: 0xb8, + 0x800: 0x0b, 0x801: 0x0b, 0x802: 0x0b, 0x803: 0x0b, 0x804: 0x0b, 0x805: 0x0b, 0x806: 0x0b, 0x807: 0x0b, + 0x808: 0x0b, 0x809: 0x0b, 0x80a: 0x0b, 0x80b: 0x0b, 0x80c: 0x0b, 0x80d: 0x0b, 0x80e: 0x0b, 0x80f: 0x0b, + 0x810: 0x0b, 0x811: 0x0b, 0x812: 0x0b, 0x813: 0x0b, 0x814: 0x0b, 0x815: 0x0b, 0x816: 0x0b, 0x817: 0x0b, + 0x818: 0x0b, 0x819: 0x0b, 0x81a: 0x0b, 0x81b: 0x0b, 0x81c: 0x0b, 0x81d: 0x0b, 0x81e: 0x0b, 0x81f: 0x0b, + 0x820: 0x0b, 0x821: 0x0b, 0x822: 0x0b, 0x823: 0x0b, 0x824: 0x0b, 0x825: 0x0b, 0x826: 0x0b, 0x827: 0x0b, + 0x828: 0x0b, 0x829: 0x0b, 0x82a: 0x0b, 0x82b: 0x0b, 0x82c: 0x0b, 0x82d: 0x0b, 0x82e: 0x0b, 0x82f: 0x0b, + 0x830: 0x0b, 0x831: 0x0b, 0x832: 0x0b, 0x833: 0x0b, 0x834: 0x0b, 0x835: 0x0b, 0x836: 0x0b, 0x837: 0x0b, + 0x838: 0x0b, 0x839: 0x0b, 0x83a: 0x0b, 0x83b: 0x0b, 0x83c: 0x0b, 0x83d: 0x0b, 0x83e: 0x0b, 0x83f: 0x0b, // Block 0x21, offset 0x840 - 0x840: 0x0b, 0x841: 0x0b, 0x842: 0x0b, 0x843: 0x0b, 0x844: 0x0b, 0x845: 0x0b, 0x846: 0x0b, 0x847: 0x0b, - 0x848: 0x0b, 0x849: 0x0b, 0x84a: 0x0b, 0x84b: 0x0b, 0x84c: 0x0b, 0x84d: 0x0b, 0x84e: 0x0b, 0x84f: 0x0b, - 0x850: 0x0b, 0x851: 0x0b, 0x852: 0x0b, 0x853: 0x0b, 0x854: 0x0b, 0x855: 0x0b, 0x856: 0x0b, 0x857: 0x0b, - 0x858: 0x0b, 0x859: 0x0b, 0x85a: 0x0b, 0x85b: 0x0b, 0x85c: 0x0b, 0x85d: 0x0b, 0x85e: 0x0b, 0x85f: 0x0b, - 0x860: 0x1e, 0x861: 0x0b, 0x862: 0x0b, 0x863: 0x0b, 0x864: 0x0b, 0x865: 0x0b, 0x866: 0x0b, 0x867: 0x0b, - 0x868: 0x0b, 0x869: 0x0b, 0x86a: 0x0b, 0x86b: 0x0b, 0x86c: 0x0b, 0x86d: 0x0b, 0x86e: 0x0b, 0x86f: 0x0b, - 0x870: 0x0b, 0x871: 0x0b, 0x872: 0x0b, 0x873: 0x0b, 0x874: 0x0b, 0x875: 0x0b, 0x876: 0x0b, 0x877: 0x0b, - 0x878: 0x0b, 0x879: 0x0b, 0x87a: 0x0b, 0x87b: 0x0b, 0x87c: 0x0b, 0x87d: 0x0b, 0x87e: 0x0b, 0x87f: 0x0b, + 0x840: 0x181, 0x841: 0x182, 0x842: 0xba, 0x843: 0xba, 0x844: 0x183, 0x845: 0x183, 0x846: 0x183, 0x847: 0x184, + 0x848: 0xba, 0x849: 0xba, 0x84a: 0xba, 0x84b: 0xba, 0x84c: 0xba, 0x84d: 0xba, 0x84e: 0xba, 0x84f: 0xba, + 0x850: 0xba, 0x851: 0xba, 0x852: 0xba, 0x853: 0xba, 0x854: 0xba, 0x855: 0xba, 0x856: 0xba, 0x857: 0xba, + 0x858: 0xba, 0x859: 0xba, 0x85a: 0xba, 0x85b: 0xba, 0x85c: 0xba, 0x85d: 0xba, 0x85e: 0xba, 0x85f: 0xba, + 0x860: 0xba, 0x861: 0xba, 0x862: 0xba, 0x863: 0xba, 0x864: 0xba, 0x865: 0xba, 0x866: 0xba, 0x867: 0xba, + 0x868: 0xba, 0x869: 0xba, 0x86a: 0xba, 0x86b: 0xba, 0x86c: 0xba, 0x86d: 0xba, 0x86e: 0xba, 0x86f: 0xba, + 0x870: 0xba, 0x871: 0xba, 0x872: 0xba, 0x873: 0xba, 0x874: 0xba, 0x875: 0xba, 0x876: 0xba, 0x877: 0xba, + 0x878: 0xba, 0x879: 0xba, 0x87a: 0xba, 0x87b: 0xba, 0x87c: 0xba, 0x87d: 0xba, 0x87e: 0xba, 0x87f: 0xba, // Block 0x22, offset 0x880 0x880: 0x0b, 0x881: 0x0b, 0x882: 0x0b, 0x883: 0x0b, 0x884: 0x0b, 0x885: 0x0b, 0x886: 0x0b, 0x887: 0x0b, 0x888: 0x0b, 0x889: 0x0b, 0x88a: 0x0b, 0x88b: 0x0b, 0x88c: 0x0b, 0x88d: 0x0b, 0x88e: 0x0b, 0x88f: 0x0b, + 0x890: 0x0b, 0x891: 0x0b, 0x892: 0x0b, 0x893: 0x0b, 0x894: 0x0b, 0x895: 0x0b, 0x896: 0x0b, 0x897: 0x0b, + 0x898: 0x0b, 0x899: 0x0b, 0x89a: 0x0b, 0x89b: 0x0b, 0x89c: 0x0b, 0x89d: 0x0b, 0x89e: 0x0b, 0x89f: 0x0b, + 0x8a0: 0x1f, 0x8a1: 0x0b, 0x8a2: 0x0b, 0x8a3: 0x0b, 0x8a4: 0x0b, 0x8a5: 0x0b, 0x8a6: 0x0b, 0x8a7: 0x0b, + 0x8a8: 0x0b, 0x8a9: 0x0b, 0x8aa: 0x0b, 0x8ab: 0x0b, 0x8ac: 0x0b, 0x8ad: 0x0b, 0x8ae: 0x0b, 0x8af: 0x0b, + 0x8b0: 0x0b, 0x8b1: 0x0b, 0x8b2: 0x0b, 0x8b3: 0x0b, 0x8b4: 0x0b, 0x8b5: 0x0b, 0x8b6: 0x0b, 0x8b7: 0x0b, + 0x8b8: 0x0b, 0x8b9: 0x0b, 0x8ba: 0x0b, 0x8bb: 0x0b, 0x8bc: 0x0b, 0x8bd: 0x0b, 0x8be: 0x0b, 0x8bf: 0x0b, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x0b, 0x8c1: 0x0b, 0x8c2: 0x0b, 0x8c3: 0x0b, 0x8c4: 0x0b, 0x8c5: 0x0b, 0x8c6: 0x0b, 0x8c7: 0x0b, + 0x8c8: 0x0b, 0x8c9: 0x0b, 0x8ca: 0x0b, 0x8cb: 0x0b, 0x8cc: 0x0b, 0x8cd: 0x0b, 0x8ce: 0x0b, 0x8cf: 0x0b, } -// idnaSparseOffset: 256 entries, 512 bytes -var idnaSparseOffset = []uint16{0x0, 0x8, 0x19, 0x25, 0x27, 0x2c, 0x34, 0x3f, 0x4b, 0x5c, 0x60, 0x6f, 0x74, 0x7b, 0x87, 0x95, 0xa3, 0xa8, 0xb1, 0xc1, 0xcf, 0xdc, 0xe8, 0xf9, 0x103, 0x10a, 0x117, 0x128, 0x12f, 0x13a, 0x149, 0x157, 0x161, 0x163, 0x167, 0x169, 0x175, 0x180, 0x188, 0x18e, 0x194, 0x199, 0x19e, 0x1a1, 0x1a5, 0x1ab, 0x1b0, 0x1bc, 0x1c6, 0x1cc, 0x1dd, 0x1e7, 0x1ea, 0x1f2, 0x1f5, 0x202, 0x20a, 0x20e, 0x215, 0x21d, 0x22d, 0x239, 0x23b, 0x245, 0x251, 0x25d, 0x269, 0x271, 0x276, 0x280, 0x291, 0x295, 0x2a0, 0x2a4, 0x2ad, 0x2b5, 0x2bb, 0x2c0, 0x2c3, 0x2c6, 0x2ca, 0x2d0, 0x2d4, 0x2d8, 0x2de, 0x2e5, 0x2eb, 0x2f3, 0x2fa, 0x305, 0x30f, 0x313, 0x316, 0x31c, 0x320, 0x322, 0x325, 0x327, 0x32a, 0x334, 0x337, 0x346, 0x34a, 0x34f, 0x352, 0x356, 0x35b, 0x360, 0x366, 0x36c, 0x37b, 0x381, 0x385, 0x394, 0x399, 0x3a1, 0x3ab, 0x3b6, 0x3be, 0x3cf, 0x3d8, 0x3e8, 0x3f5, 0x3ff, 0x404, 0x411, 0x415, 0x41a, 0x41c, 0x420, 0x422, 0x426, 0x42f, 0x435, 0x439, 0x449, 0x453, 0x458, 0x45b, 0x461, 0x468, 0x46d, 0x471, 0x477, 0x47c, 0x485, 0x48a, 0x490, 0x497, 0x49e, 0x4a5, 0x4a9, 0x4ae, 0x4b1, 0x4b6, 0x4c2, 0x4c8, 0x4cd, 0x4d4, 0x4dc, 0x4e1, 0x4e5, 0x4f5, 0x4fc, 0x500, 0x504, 0x50b, 0x50e, 0x511, 0x515, 0x519, 0x51f, 0x528, 0x534, 0x53b, 0x544, 0x54c, 0x553, 0x561, 0x56e, 0x57b, 0x584, 0x588, 0x596, 0x59e, 0x5a9, 0x5b2, 0x5b8, 0x5c0, 0x5c9, 0x5d3, 0x5d6, 0x5e2, 0x5e5, 0x5ea, 0x5ed, 0x5f7, 0x600, 0x60c, 0x60f, 0x614, 0x617, 0x61a, 0x61d, 0x624, 0x62b, 0x62f, 0x63a, 0x63d, 0x643, 0x648, 0x64c, 0x64f, 0x652, 0x655, 0x65a, 0x664, 0x667, 0x66b, 0x67a, 0x686, 0x68a, 0x68f, 0x694, 0x698, 0x69d, 0x6a6, 0x6b1, 0x6b7, 0x6bf, 0x6c3, 0x6c7, 0x6cd, 0x6d3, 0x6d8, 0x6db, 0x6e9, 0x6f0, 0x6f3, 0x6f6, 0x6fa, 0x700, 0x705, 0x70f, 0x714, 0x717, 0x71a, 0x71d, 0x720, 0x724, 0x727, 0x737, 0x748, 0x74d, 0x74f, 0x751} +// idnaSparseOffset: 264 entries, 528 bytes +var idnaSparseOffset = []uint16{0x0, 0x8, 0x19, 0x25, 0x27, 0x2c, 0x34, 0x3f, 0x4b, 0x4f, 0x5e, 0x63, 0x6b, 0x77, 0x85, 0x8a, 0x93, 0xa3, 0xb1, 0xbd, 0xc9, 0xda, 0xe4, 0xeb, 0xf8, 0x109, 0x110, 0x11b, 0x12a, 0x138, 0x142, 0x144, 0x149, 0x14c, 0x14f, 0x151, 0x15d, 0x168, 0x170, 0x176, 0x17c, 0x181, 0x186, 0x189, 0x18d, 0x193, 0x198, 0x1a4, 0x1ae, 0x1b4, 0x1c5, 0x1cf, 0x1d2, 0x1da, 0x1dd, 0x1ea, 0x1f2, 0x1f6, 0x1fd, 0x205, 0x215, 0x221, 0x223, 0x22d, 0x239, 0x245, 0x251, 0x259, 0x25e, 0x268, 0x279, 0x27d, 0x288, 0x28c, 0x295, 0x29d, 0x2a3, 0x2a8, 0x2ab, 0x2af, 0x2b5, 0x2b9, 0x2bd, 0x2c3, 0x2ca, 0x2d0, 0x2d8, 0x2df, 0x2ea, 0x2f4, 0x2f8, 0x2fb, 0x301, 0x305, 0x307, 0x30a, 0x30c, 0x30f, 0x319, 0x31c, 0x32b, 0x32f, 0x334, 0x337, 0x33b, 0x340, 0x345, 0x34b, 0x351, 0x360, 0x366, 0x36a, 0x379, 0x37e, 0x386, 0x390, 0x39b, 0x3a3, 0x3b4, 0x3bd, 0x3cd, 0x3da, 0x3e4, 0x3e9, 0x3f6, 0x3fa, 0x3ff, 0x401, 0x405, 0x407, 0x40b, 0x414, 0x41a, 0x41e, 0x42e, 0x438, 0x43d, 0x440, 0x446, 0x44d, 0x452, 0x456, 0x45c, 0x461, 0x46a, 0x46f, 0x475, 0x47c, 0x483, 0x48a, 0x48e, 0x493, 0x496, 0x49b, 0x4a7, 0x4ad, 0x4b2, 0x4b9, 0x4c1, 0x4c6, 0x4ca, 0x4da, 0x4e1, 0x4e5, 0x4e9, 0x4f0, 0x4f2, 0x4f5, 0x4f8, 0x4fc, 0x500, 0x506, 0x50f, 0x51b, 0x522, 0x52b, 0x533, 0x53a, 0x548, 0x555, 0x562, 0x56b, 0x56f, 0x57d, 0x585, 0x590, 0x599, 0x59f, 0x5a7, 0x5b0, 0x5ba, 0x5bd, 0x5c9, 0x5cc, 0x5d1, 0x5de, 0x5e7, 0x5f3, 0x5f6, 0x600, 0x609, 0x615, 0x622, 0x62a, 0x62d, 0x632, 0x635, 0x638, 0x63b, 0x642, 0x649, 0x64d, 0x658, 0x65b, 0x661, 0x666, 0x66a, 0x66d, 0x670, 0x673, 0x676, 0x679, 0x67e, 0x688, 0x68b, 0x68f, 0x69e, 0x6aa, 0x6ae, 0x6b3, 0x6b8, 0x6bc, 0x6c1, 0x6ca, 0x6d5, 0x6db, 0x6e3, 0x6e7, 0x6eb, 0x6f1, 0x6f7, 0x6fc, 0x6ff, 0x70f, 0x716, 0x719, 0x71c, 0x720, 0x726, 0x72b, 0x730, 0x735, 0x738, 0x73d, 0x740, 0x743, 0x747, 0x74b, 0x74e, 0x75e, 0x76f, 0x774, 0x776, 0x778} -// idnaSparseValues: 1876 entries, 7504 bytes -var idnaSparseValues = [1876]valueRange{ +// idnaSparseValues: 1915 entries, 7660 bytes +var idnaSparseValues = [1915]valueRange{ // Block 0x0, offset 0x0 {value: 0x0000, lo: 0x07}, {value: 0xe105, lo: 0x80, hi: 0x96}, @@ -2382,7 +2415,7 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0008, lo: 0xb9, hi: 0xbf}, // Block 0x3, offset 0x25 {value: 0x0000, lo: 0x01}, - {value: 0x1308, lo: 0x80, hi: 0xbf}, + {value: 0x3308, lo: 0x80, hi: 0xbf}, // Block 0x4, offset 0x27 {value: 0x0000, lo: 0x04}, {value: 0x03f5, lo: 0x80, hi: 0x8f}, @@ -2407,155 +2440,123 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0x8b, hi: 0x8c}, {value: 0x0018, lo: 0x8d, hi: 0x8f}, {value: 0x0040, lo: 0x90, hi: 0x90}, - {value: 0x1308, lo: 0x91, hi: 0xbd}, - {value: 0x0018, lo: 0xbe, hi: 0xbe}, - {value: 0x1308, lo: 0xbf, hi: 0xbf}, + {value: 0x3308, lo: 0x91, hi: 0xbd}, + {value: 0x0818, lo: 0xbe, hi: 0xbe}, + {value: 0x3308, lo: 0xbf, hi: 0xbf}, // Block 0x7, offset 0x3f {value: 0x0000, lo: 0x0b}, - {value: 0x0018, lo: 0x80, hi: 0x80}, - {value: 0x1308, lo: 0x81, hi: 0x82}, - {value: 0x0018, lo: 0x83, hi: 0x83}, - {value: 0x1308, lo: 0x84, hi: 0x85}, - {value: 0x0018, lo: 0x86, hi: 0x86}, - {value: 0x1308, lo: 0x87, hi: 0x87}, + {value: 0x0818, lo: 0x80, hi: 0x80}, + {value: 0x3308, lo: 0x81, hi: 0x82}, + {value: 0x0818, lo: 0x83, hi: 0x83}, + {value: 0x3308, lo: 0x84, hi: 0x85}, + {value: 0x0818, lo: 0x86, hi: 0x86}, + {value: 0x3308, lo: 0x87, hi: 0x87}, {value: 0x0040, lo: 0x88, hi: 0x8f}, - {value: 0x0008, lo: 0x90, hi: 0xaa}, + {value: 0x0808, lo: 0x90, hi: 0xaa}, {value: 0x0040, lo: 0xab, hi: 0xaf}, - {value: 0x0008, lo: 0xb0, hi: 0xb4}, + {value: 0x0808, lo: 0xb0, hi: 0xb4}, {value: 0x0040, lo: 0xb5, hi: 0xbf}, // Block 0x8, offset 0x4b - {value: 0x0000, lo: 0x10}, - {value: 0x0018, lo: 0x80, hi: 0x80}, - {value: 0x0208, lo: 0x81, hi: 0x87}, - {value: 0x0408, lo: 0x88, hi: 0x88}, - {value: 0x0208, lo: 0x89, hi: 0x8a}, - {value: 0x1308, lo: 0x8b, hi: 0x9f}, - {value: 0x0008, lo: 0xa0, hi: 0xa9}, - {value: 0x0018, lo: 0xaa, hi: 0xad}, - {value: 0x0208, lo: 0xae, hi: 0xaf}, - {value: 0x1308, lo: 0xb0, hi: 0xb0}, - {value: 0x0408, lo: 0xb1, hi: 0xb3}, - {value: 0x0008, lo: 0xb4, hi: 0xb4}, - {value: 0x0429, lo: 0xb5, hi: 0xb5}, - {value: 0x0451, lo: 0xb6, hi: 0xb6}, - {value: 0x0479, lo: 0xb7, hi: 0xb7}, - {value: 0x04a1, lo: 0xb8, hi: 0xb8}, - {value: 0x0208, lo: 0xb9, hi: 0xbf}, - // Block 0x9, offset 0x5c {value: 0x0000, lo: 0x03}, - {value: 0x0208, lo: 0x80, hi: 0x87}, - {value: 0x0408, lo: 0x88, hi: 0x99}, - {value: 0x0208, lo: 0x9a, hi: 0xbf}, - // Block 0xa, offset 0x60 + {value: 0x0a08, lo: 0x80, hi: 0x87}, + {value: 0x0c08, lo: 0x88, hi: 0x99}, + {value: 0x0a08, lo: 0x9a, hi: 0xbf}, + // Block 0x9, offset 0x4f {value: 0x0000, lo: 0x0e}, - {value: 0x1308, lo: 0x80, hi: 0x8a}, + {value: 0x3308, lo: 0x80, hi: 0x8a}, {value: 0x0040, lo: 0x8b, hi: 0x8c}, - {value: 0x0408, lo: 0x8d, hi: 0x8d}, - {value: 0x0208, lo: 0x8e, hi: 0x98}, - {value: 0x0408, lo: 0x99, hi: 0x9b}, - {value: 0x0208, lo: 0x9c, hi: 0xaa}, - {value: 0x0408, lo: 0xab, hi: 0xac}, - {value: 0x0208, lo: 0xad, hi: 0xb0}, - {value: 0x0408, lo: 0xb1, hi: 0xb1}, - {value: 0x0208, lo: 0xb2, hi: 0xb2}, - {value: 0x0408, lo: 0xb3, hi: 0xb4}, - {value: 0x0208, lo: 0xb5, hi: 0xb7}, - {value: 0x0408, lo: 0xb8, hi: 0xb9}, - {value: 0x0208, lo: 0xba, hi: 0xbf}, - // Block 0xb, offset 0x6f + {value: 0x0c08, lo: 0x8d, hi: 0x8d}, + {value: 0x0a08, lo: 0x8e, hi: 0x98}, + {value: 0x0c08, lo: 0x99, hi: 0x9b}, + {value: 0x0a08, lo: 0x9c, hi: 0xaa}, + {value: 0x0c08, lo: 0xab, hi: 0xac}, + {value: 0x0a08, lo: 0xad, hi: 0xb0}, + {value: 0x0c08, lo: 0xb1, hi: 0xb1}, + {value: 0x0a08, lo: 0xb2, hi: 0xb2}, + {value: 0x0c08, lo: 0xb3, hi: 0xb4}, + {value: 0x0a08, lo: 0xb5, hi: 0xb7}, + {value: 0x0c08, lo: 0xb8, hi: 0xb9}, + {value: 0x0a08, lo: 0xba, hi: 0xbf}, + // Block 0xa, offset 0x5e {value: 0x0000, lo: 0x04}, - {value: 0x0008, lo: 0x80, hi: 0xa5}, - {value: 0x1308, lo: 0xa6, hi: 0xb0}, - {value: 0x0008, lo: 0xb1, hi: 0xb1}, + {value: 0x0808, lo: 0x80, hi: 0xa5}, + {value: 0x3308, lo: 0xa6, hi: 0xb0}, + {value: 0x0808, lo: 0xb1, hi: 0xb1}, {value: 0x0040, lo: 0xb2, hi: 0xbf}, - // Block 0xc, offset 0x74 - {value: 0x0000, lo: 0x06}, - {value: 0x0008, lo: 0x80, hi: 0x89}, - {value: 0x0208, lo: 0x8a, hi: 0xaa}, - {value: 0x1308, lo: 0xab, hi: 0xb3}, - {value: 0x0008, lo: 0xb4, hi: 0xb5}, - {value: 0x0018, lo: 0xb6, hi: 0xba}, + // Block 0xb, offset 0x63 + {value: 0x0000, lo: 0x07}, + {value: 0x0808, lo: 0x80, hi: 0x89}, + {value: 0x0a08, lo: 0x8a, hi: 0xaa}, + {value: 0x3308, lo: 0xab, hi: 0xb3}, + {value: 0x0808, lo: 0xb4, hi: 0xb5}, + {value: 0x0018, lo: 0xb6, hi: 0xb9}, + {value: 0x0818, lo: 0xba, hi: 0xba}, {value: 0x0040, lo: 0xbb, hi: 0xbf}, - // Block 0xd, offset 0x7b + // Block 0xc, offset 0x6b {value: 0x0000, lo: 0x0b}, - {value: 0x0008, lo: 0x80, hi: 0x95}, - {value: 0x1308, lo: 0x96, hi: 0x99}, - {value: 0x0008, lo: 0x9a, hi: 0x9a}, - {value: 0x1308, lo: 0x9b, hi: 0xa3}, - {value: 0x0008, lo: 0xa4, hi: 0xa4}, - {value: 0x1308, lo: 0xa5, hi: 0xa7}, - {value: 0x0008, lo: 0xa8, hi: 0xa8}, - {value: 0x1308, lo: 0xa9, hi: 0xad}, + {value: 0x0808, lo: 0x80, hi: 0x95}, + {value: 0x3308, lo: 0x96, hi: 0x99}, + {value: 0x0808, lo: 0x9a, hi: 0x9a}, + {value: 0x3308, lo: 0x9b, hi: 0xa3}, + {value: 0x0808, lo: 0xa4, hi: 0xa4}, + {value: 0x3308, lo: 0xa5, hi: 0xa7}, + {value: 0x0808, lo: 0xa8, hi: 0xa8}, + {value: 0x3308, lo: 0xa9, hi: 0xad}, {value: 0x0040, lo: 0xae, hi: 0xaf}, - {value: 0x0018, lo: 0xb0, hi: 0xbe}, + {value: 0x0818, lo: 0xb0, hi: 0xbe}, {value: 0x0040, lo: 0xbf, hi: 0xbf}, - // Block 0xe, offset 0x87 - {value: 0x0000, lo: 0x0d}, - {value: 0x0408, lo: 0x80, hi: 0x80}, - {value: 0x0208, lo: 0x81, hi: 0x85}, - {value: 0x0408, lo: 0x86, hi: 0x87}, - {value: 0x0208, lo: 0x88, hi: 0x88}, - {value: 0x0408, lo: 0x89, hi: 0x89}, - {value: 0x0208, lo: 0x8a, hi: 0x93}, - {value: 0x0408, lo: 0x94, hi: 0x94}, - {value: 0x0208, lo: 0x95, hi: 0x95}, - {value: 0x0008, lo: 0x96, hi: 0x98}, - {value: 0x1308, lo: 0x99, hi: 0x9b}, - {value: 0x0040, lo: 0x9c, hi: 0x9d}, - {value: 0x0018, lo: 0x9e, hi: 0x9e}, - {value: 0x0040, lo: 0x9f, hi: 0xbf}, - // Block 0xf, offset 0x95 + // Block 0xd, offset 0x77 {value: 0x0000, lo: 0x0d}, {value: 0x0040, lo: 0x80, hi: 0x9f}, - {value: 0x0208, lo: 0xa0, hi: 0xa9}, - {value: 0x0408, lo: 0xaa, hi: 0xac}, - {value: 0x0008, lo: 0xad, hi: 0xad}, - {value: 0x0408, lo: 0xae, hi: 0xae}, - {value: 0x0208, lo: 0xaf, hi: 0xb0}, - {value: 0x0408, lo: 0xb1, hi: 0xb2}, - {value: 0x0208, lo: 0xb3, hi: 0xb4}, + {value: 0x0a08, lo: 0xa0, hi: 0xa9}, + {value: 0x0c08, lo: 0xaa, hi: 0xac}, + {value: 0x0808, lo: 0xad, hi: 0xad}, + {value: 0x0c08, lo: 0xae, hi: 0xae}, + {value: 0x0a08, lo: 0xaf, hi: 0xb0}, + {value: 0x0c08, lo: 0xb1, hi: 0xb2}, + {value: 0x0a08, lo: 0xb3, hi: 0xb4}, {value: 0x0040, lo: 0xb5, hi: 0xb5}, - {value: 0x0208, lo: 0xb6, hi: 0xb8}, - {value: 0x0408, lo: 0xb9, hi: 0xb9}, - {value: 0x0208, lo: 0xba, hi: 0xbd}, + {value: 0x0a08, lo: 0xb6, hi: 0xb8}, + {value: 0x0c08, lo: 0xb9, hi: 0xb9}, + {value: 0x0a08, lo: 0xba, hi: 0xbd}, {value: 0x0040, lo: 0xbe, hi: 0xbf}, - // Block 0x10, offset 0xa3 + // Block 0xe, offset 0x85 {value: 0x0000, lo: 0x04}, {value: 0x0040, lo: 0x80, hi: 0x93}, - {value: 0x1308, lo: 0x94, hi: 0xa1}, - {value: 0x0040, lo: 0xa2, hi: 0xa2}, - {value: 0x1308, lo: 0xa3, hi: 0xbf}, - // Block 0x11, offset 0xa8 + {value: 0x3308, lo: 0x94, hi: 0xa1}, + {value: 0x0840, lo: 0xa2, hi: 0xa2}, + {value: 0x3308, lo: 0xa3, hi: 0xbf}, + // Block 0xf, offset 0x8a {value: 0x0000, lo: 0x08}, - {value: 0x1308, lo: 0x80, hi: 0x82}, - {value: 0x1008, lo: 0x83, hi: 0x83}, + {value: 0x3308, lo: 0x80, hi: 0x82}, + {value: 0x3008, lo: 0x83, hi: 0x83}, {value: 0x0008, lo: 0x84, hi: 0xb9}, - {value: 0x1308, lo: 0xba, hi: 0xba}, - {value: 0x1008, lo: 0xbb, hi: 0xbb}, - {value: 0x1308, lo: 0xbc, hi: 0xbc}, + {value: 0x3308, lo: 0xba, hi: 0xba}, + {value: 0x3008, lo: 0xbb, hi: 0xbb}, + {value: 0x3308, lo: 0xbc, hi: 0xbc}, {value: 0x0008, lo: 0xbd, hi: 0xbd}, - {value: 0x1008, lo: 0xbe, hi: 0xbf}, - // Block 0x12, offset 0xb1 + {value: 0x3008, lo: 0xbe, hi: 0xbf}, + // Block 0x10, offset 0x93 {value: 0x0000, lo: 0x0f}, - {value: 0x1308, lo: 0x80, hi: 0x80}, - {value: 0x1008, lo: 0x81, hi: 0x82}, + {value: 0x3308, lo: 0x80, hi: 0x80}, + {value: 0x3008, lo: 0x81, hi: 0x82}, {value: 0x0040, lo: 0x83, hi: 0x85}, - {value: 0x1008, lo: 0x86, hi: 0x88}, + {value: 0x3008, lo: 0x86, hi: 0x88}, {value: 0x0040, lo: 0x89, hi: 0x89}, - {value: 0x1008, lo: 0x8a, hi: 0x8c}, - {value: 0x1b08, lo: 0x8d, hi: 0x8d}, + {value: 0x3008, lo: 0x8a, hi: 0x8c}, + {value: 0x3b08, lo: 0x8d, hi: 0x8d}, {value: 0x0040, lo: 0x8e, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x90}, {value: 0x0040, lo: 0x91, hi: 0x96}, - {value: 0x1008, lo: 0x97, hi: 0x97}, + {value: 0x3008, lo: 0x97, hi: 0x97}, {value: 0x0040, lo: 0x98, hi: 0xa5}, {value: 0x0008, lo: 0xa6, hi: 0xaf}, {value: 0x0018, lo: 0xb0, hi: 0xba}, {value: 0x0040, lo: 0xbb, hi: 0xbf}, - // Block 0x13, offset 0xc1 + // Block 0x11, offset 0xa3 {value: 0x0000, lo: 0x0d}, - {value: 0x1308, lo: 0x80, hi: 0x80}, - {value: 0x1008, lo: 0x81, hi: 0x83}, + {value: 0x3308, lo: 0x80, hi: 0x80}, + {value: 0x3008, lo: 0x81, hi: 0x83}, {value: 0x0040, lo: 0x84, hi: 0x84}, {value: 0x0008, lo: 0x85, hi: 0x8c}, {value: 0x0040, lo: 0x8d, hi: 0x8d}, @@ -2566,25 +2567,24 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0008, lo: 0xaa, hi: 0xb9}, {value: 0x0040, lo: 0xba, hi: 0xbc}, {value: 0x0008, lo: 0xbd, hi: 0xbd}, - {value: 0x1308, lo: 0xbe, hi: 0xbf}, - // Block 0x14, offset 0xcf - {value: 0x0000, lo: 0x0c}, - {value: 0x0040, lo: 0x80, hi: 0x80}, - {value: 0x1308, lo: 0x81, hi: 0x81}, - {value: 0x1008, lo: 0x82, hi: 0x83}, + {value: 0x3308, lo: 0xbe, hi: 0xbf}, + // Block 0x12, offset 0xb1 + {value: 0x0000, lo: 0x0b}, + {value: 0x3308, lo: 0x80, hi: 0x81}, + {value: 0x3008, lo: 0x82, hi: 0x83}, {value: 0x0040, lo: 0x84, hi: 0x84}, {value: 0x0008, lo: 0x85, hi: 0x8c}, {value: 0x0040, lo: 0x8d, hi: 0x8d}, {value: 0x0008, lo: 0x8e, hi: 0x90}, {value: 0x0040, lo: 0x91, hi: 0x91}, {value: 0x0008, lo: 0x92, hi: 0xba}, - {value: 0x0040, lo: 0xbb, hi: 0xbc}, + {value: 0x3b08, lo: 0xbb, hi: 0xbc}, {value: 0x0008, lo: 0xbd, hi: 0xbd}, - {value: 0x1008, lo: 0xbe, hi: 0xbf}, - // Block 0x15, offset 0xdc + {value: 0x3008, lo: 0xbe, hi: 0xbf}, + // Block 0x13, offset 0xbd {value: 0x0000, lo: 0x0b}, {value: 0x0040, lo: 0x80, hi: 0x81}, - {value: 0x1008, lo: 0x82, hi: 0x83}, + {value: 0x3008, lo: 0x82, hi: 0x83}, {value: 0x0040, lo: 0x84, hi: 0x84}, {value: 0x0008, lo: 0x85, hi: 0x96}, {value: 0x0040, lo: 0x97, hi: 0x99}, @@ -2594,50 +2594,50 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0xbc, hi: 0xbc}, {value: 0x0008, lo: 0xbd, hi: 0xbd}, {value: 0x0040, lo: 0xbe, hi: 0xbf}, - // Block 0x16, offset 0xe8 + // Block 0x14, offset 0xc9 {value: 0x0000, lo: 0x10}, {value: 0x0008, lo: 0x80, hi: 0x86}, {value: 0x0040, lo: 0x87, hi: 0x89}, - {value: 0x1b08, lo: 0x8a, hi: 0x8a}, + {value: 0x3b08, lo: 0x8a, hi: 0x8a}, {value: 0x0040, lo: 0x8b, hi: 0x8e}, - {value: 0x1008, lo: 0x8f, hi: 0x91}, - {value: 0x1308, lo: 0x92, hi: 0x94}, + {value: 0x3008, lo: 0x8f, hi: 0x91}, + {value: 0x3308, lo: 0x92, hi: 0x94}, {value: 0x0040, lo: 0x95, hi: 0x95}, - {value: 0x1308, lo: 0x96, hi: 0x96}, + {value: 0x3308, lo: 0x96, hi: 0x96}, {value: 0x0040, lo: 0x97, hi: 0x97}, - {value: 0x1008, lo: 0x98, hi: 0x9f}, + {value: 0x3008, lo: 0x98, hi: 0x9f}, {value: 0x0040, lo: 0xa0, hi: 0xa5}, {value: 0x0008, lo: 0xa6, hi: 0xaf}, {value: 0x0040, lo: 0xb0, hi: 0xb1}, - {value: 0x1008, lo: 0xb2, hi: 0xb3}, + {value: 0x3008, lo: 0xb2, hi: 0xb3}, {value: 0x0018, lo: 0xb4, hi: 0xb4}, {value: 0x0040, lo: 0xb5, hi: 0xbf}, - // Block 0x17, offset 0xf9 + // Block 0x15, offset 0xda {value: 0x0000, lo: 0x09}, {value: 0x0040, lo: 0x80, hi: 0x80}, {value: 0x0008, lo: 0x81, hi: 0xb0}, - {value: 0x1308, lo: 0xb1, hi: 0xb1}, + {value: 0x3308, lo: 0xb1, hi: 0xb1}, {value: 0x0008, lo: 0xb2, hi: 0xb2}, {value: 0x08f1, lo: 0xb3, hi: 0xb3}, - {value: 0x1308, lo: 0xb4, hi: 0xb9}, - {value: 0x1b08, lo: 0xba, hi: 0xba}, + {value: 0x3308, lo: 0xb4, hi: 0xb9}, + {value: 0x3b08, lo: 0xba, hi: 0xba}, {value: 0x0040, lo: 0xbb, hi: 0xbe}, {value: 0x0018, lo: 0xbf, hi: 0xbf}, - // Block 0x18, offset 0x103 + // Block 0x16, offset 0xe4 {value: 0x0000, lo: 0x06}, {value: 0x0008, lo: 0x80, hi: 0x86}, - {value: 0x1308, lo: 0x87, hi: 0x8e}, + {value: 0x3308, lo: 0x87, hi: 0x8e}, {value: 0x0018, lo: 0x8f, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x99}, {value: 0x0018, lo: 0x9a, hi: 0x9b}, {value: 0x0040, lo: 0x9c, hi: 0xbf}, - // Block 0x19, offset 0x10a + // Block 0x17, offset 0xeb {value: 0x0000, lo: 0x0c}, {value: 0x0008, lo: 0x80, hi: 0x84}, {value: 0x0040, lo: 0x85, hi: 0x85}, {value: 0x0008, lo: 0x86, hi: 0x86}, {value: 0x0040, lo: 0x87, hi: 0x87}, - {value: 0x1308, lo: 0x88, hi: 0x8d}, + {value: 0x3308, lo: 0x88, hi: 0x8d}, {value: 0x0040, lo: 0x8e, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x99}, {value: 0x0040, lo: 0x9a, hi: 0x9b}, @@ -2645,76 +2645,76 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0999, lo: 0x9d, hi: 0x9d}, {value: 0x0008, lo: 0x9e, hi: 0x9f}, {value: 0x0040, lo: 0xa0, hi: 0xbf}, - // Block 0x1a, offset 0x117 + // Block 0x18, offset 0xf8 {value: 0x0000, lo: 0x10}, {value: 0x0008, lo: 0x80, hi: 0x80}, {value: 0x0018, lo: 0x81, hi: 0x8a}, {value: 0x0008, lo: 0x8b, hi: 0x8b}, {value: 0xe03d, lo: 0x8c, hi: 0x8c}, {value: 0x0018, lo: 0x8d, hi: 0x97}, - {value: 0x1308, lo: 0x98, hi: 0x99}, + {value: 0x3308, lo: 0x98, hi: 0x99}, {value: 0x0018, lo: 0x9a, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xa9}, {value: 0x0018, lo: 0xaa, hi: 0xb4}, - {value: 0x1308, lo: 0xb5, hi: 0xb5}, + {value: 0x3308, lo: 0xb5, hi: 0xb5}, {value: 0x0018, lo: 0xb6, hi: 0xb6}, - {value: 0x1308, lo: 0xb7, hi: 0xb7}, + {value: 0x3308, lo: 0xb7, hi: 0xb7}, {value: 0x0018, lo: 0xb8, hi: 0xb8}, - {value: 0x1308, lo: 0xb9, hi: 0xb9}, + {value: 0x3308, lo: 0xb9, hi: 0xb9}, {value: 0x0018, lo: 0xba, hi: 0xbd}, - {value: 0x1008, lo: 0xbe, hi: 0xbf}, - // Block 0x1b, offset 0x128 + {value: 0x3008, lo: 0xbe, hi: 0xbf}, + // Block 0x19, offset 0x109 {value: 0x0000, lo: 0x06}, {value: 0x0018, lo: 0x80, hi: 0x85}, - {value: 0x1308, lo: 0x86, hi: 0x86}, + {value: 0x3308, lo: 0x86, hi: 0x86}, {value: 0x0018, lo: 0x87, hi: 0x8c}, {value: 0x0040, lo: 0x8d, hi: 0x8d}, {value: 0x0018, lo: 0x8e, hi: 0x9a}, {value: 0x0040, lo: 0x9b, hi: 0xbf}, - // Block 0x1c, offset 0x12f + // Block 0x1a, offset 0x110 {value: 0x0000, lo: 0x0a}, {value: 0x0008, lo: 0x80, hi: 0xaa}, - {value: 0x1008, lo: 0xab, hi: 0xac}, - {value: 0x1308, lo: 0xad, hi: 0xb0}, - {value: 0x1008, lo: 0xb1, hi: 0xb1}, - {value: 0x1308, lo: 0xb2, hi: 0xb7}, - {value: 0x1008, lo: 0xb8, hi: 0xb8}, - {value: 0x1b08, lo: 0xb9, hi: 0xba}, - {value: 0x1008, lo: 0xbb, hi: 0xbc}, - {value: 0x1308, lo: 0xbd, hi: 0xbe}, + {value: 0x3008, lo: 0xab, hi: 0xac}, + {value: 0x3308, lo: 0xad, hi: 0xb0}, + {value: 0x3008, lo: 0xb1, hi: 0xb1}, + {value: 0x3308, lo: 0xb2, hi: 0xb7}, + {value: 0x3008, lo: 0xb8, hi: 0xb8}, + {value: 0x3b08, lo: 0xb9, hi: 0xba}, + {value: 0x3008, lo: 0xbb, hi: 0xbc}, + {value: 0x3308, lo: 0xbd, hi: 0xbe}, {value: 0x0008, lo: 0xbf, hi: 0xbf}, - // Block 0x1d, offset 0x13a + // Block 0x1b, offset 0x11b {value: 0x0000, lo: 0x0e}, {value: 0x0008, lo: 0x80, hi: 0x89}, {value: 0x0018, lo: 0x8a, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x95}, - {value: 0x1008, lo: 0x96, hi: 0x97}, - {value: 0x1308, lo: 0x98, hi: 0x99}, + {value: 0x3008, lo: 0x96, hi: 0x97}, + {value: 0x3308, lo: 0x98, hi: 0x99}, {value: 0x0008, lo: 0x9a, hi: 0x9d}, - {value: 0x1308, lo: 0x9e, hi: 0xa0}, + {value: 0x3308, lo: 0x9e, hi: 0xa0}, {value: 0x0008, lo: 0xa1, hi: 0xa1}, - {value: 0x1008, lo: 0xa2, hi: 0xa4}, + {value: 0x3008, lo: 0xa2, hi: 0xa4}, {value: 0x0008, lo: 0xa5, hi: 0xa6}, - {value: 0x1008, lo: 0xa7, hi: 0xad}, + {value: 0x3008, lo: 0xa7, hi: 0xad}, {value: 0x0008, lo: 0xae, hi: 0xb0}, - {value: 0x1308, lo: 0xb1, hi: 0xb4}, + {value: 0x3308, lo: 0xb1, hi: 0xb4}, {value: 0x0008, lo: 0xb5, hi: 0xbf}, - // Block 0x1e, offset 0x149 + // Block 0x1c, offset 0x12a {value: 0x0000, lo: 0x0d}, {value: 0x0008, lo: 0x80, hi: 0x81}, - {value: 0x1308, lo: 0x82, hi: 0x82}, - {value: 0x1008, lo: 0x83, hi: 0x84}, - {value: 0x1308, lo: 0x85, hi: 0x86}, - {value: 0x1008, lo: 0x87, hi: 0x8c}, - {value: 0x1308, lo: 0x8d, hi: 0x8d}, + {value: 0x3308, lo: 0x82, hi: 0x82}, + {value: 0x3008, lo: 0x83, hi: 0x84}, + {value: 0x3308, lo: 0x85, hi: 0x86}, + {value: 0x3008, lo: 0x87, hi: 0x8c}, + {value: 0x3308, lo: 0x8d, hi: 0x8d}, {value: 0x0008, lo: 0x8e, hi: 0x8e}, - {value: 0x1008, lo: 0x8f, hi: 0x8f}, + {value: 0x3008, lo: 0x8f, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x99}, - {value: 0x1008, lo: 0x9a, hi: 0x9c}, - {value: 0x1308, lo: 0x9d, hi: 0x9d}, + {value: 0x3008, lo: 0x9a, hi: 0x9c}, + {value: 0x3308, lo: 0x9d, hi: 0x9d}, {value: 0x0018, lo: 0x9e, hi: 0x9f}, {value: 0x0040, lo: 0xa0, hi: 0xbf}, - // Block 0x1f, offset 0x157 + // Block 0x1d, offset 0x138 {value: 0x0000, lo: 0x09}, {value: 0x0040, lo: 0x80, hi: 0x86}, {value: 0x055d, lo: 0x87, hi: 0x87}, @@ -2725,18 +2725,27 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0018, lo: 0xbb, hi: 0xbb}, {value: 0xe105, lo: 0xbc, hi: 0xbc}, {value: 0x0008, lo: 0xbd, hi: 0xbf}, - // Block 0x20, offset 0x161 + // Block 0x1e, offset 0x142 {value: 0x0000, lo: 0x01}, {value: 0x0018, lo: 0x80, hi: 0xbf}, - // Block 0x21, offset 0x163 - {value: 0x0000, lo: 0x03}, + // Block 0x1f, offset 0x144 + {value: 0x0000, lo: 0x04}, {value: 0x0018, lo: 0x80, hi: 0x9e}, {value: 0x0040, lo: 0x9f, hi: 0xa0}, - {value: 0x0018, lo: 0xa1, hi: 0xbf}, - // Block 0x22, offset 0x167 + {value: 0x2018, lo: 0xa1, hi: 0xb5}, + {value: 0x0018, lo: 0xb6, hi: 0xbf}, + // Block 0x20, offset 0x149 + {value: 0x0000, lo: 0x02}, + {value: 0x0018, lo: 0x80, hi: 0xa7}, + {value: 0x2018, lo: 0xa8, hi: 0xbf}, + // Block 0x21, offset 0x14c + {value: 0x0000, lo: 0x02}, + {value: 0x2018, lo: 0x80, hi: 0x82}, + {value: 0x0018, lo: 0x83, hi: 0xbf}, + // Block 0x22, offset 0x14f {value: 0x0000, lo: 0x01}, {value: 0x0008, lo: 0x80, hi: 0xbf}, - // Block 0x23, offset 0x169 + // Block 0x23, offset 0x151 {value: 0x0000, lo: 0x0b}, {value: 0x0008, lo: 0x80, hi: 0x88}, {value: 0x0040, lo: 0x89, hi: 0x89}, @@ -2749,7 +2758,7 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0008, lo: 0x9a, hi: 0x9d}, {value: 0x0040, lo: 0x9e, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xbf}, - // Block 0x24, offset 0x175 + // Block 0x24, offset 0x15d {value: 0x0000, lo: 0x0a}, {value: 0x0008, lo: 0x80, hi: 0x88}, {value: 0x0040, lo: 0x89, hi: 0x89}, @@ -2761,7 +2770,7 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0xb6, hi: 0xb7}, {value: 0x0008, lo: 0xb8, hi: 0xbe}, {value: 0x0040, lo: 0xbf, hi: 0xbf}, - // Block 0x25, offset 0x180 + // Block 0x25, offset 0x168 {value: 0x0000, lo: 0x07}, {value: 0x0008, lo: 0x80, hi: 0x80}, {value: 0x0040, lo: 0x81, hi: 0x81}, @@ -2770,146 +2779,146 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0008, lo: 0x88, hi: 0x96}, {value: 0x0040, lo: 0x97, hi: 0x97}, {value: 0x0008, lo: 0x98, hi: 0xbf}, - // Block 0x26, offset 0x188 + // Block 0x26, offset 0x170 {value: 0x0000, lo: 0x05}, {value: 0x0008, lo: 0x80, hi: 0x90}, {value: 0x0040, lo: 0x91, hi: 0x91}, {value: 0x0008, lo: 0x92, hi: 0x95}, {value: 0x0040, lo: 0x96, hi: 0x97}, {value: 0x0008, lo: 0x98, hi: 0xbf}, - // Block 0x27, offset 0x18e + // Block 0x27, offset 0x176 {value: 0x0000, lo: 0x05}, {value: 0x0008, lo: 0x80, hi: 0x9a}, {value: 0x0040, lo: 0x9b, hi: 0x9c}, - {value: 0x1308, lo: 0x9d, hi: 0x9f}, + {value: 0x3308, lo: 0x9d, hi: 0x9f}, {value: 0x0018, lo: 0xa0, hi: 0xbc}, {value: 0x0040, lo: 0xbd, hi: 0xbf}, - // Block 0x28, offset 0x194 + // Block 0x28, offset 0x17c {value: 0x0000, lo: 0x04}, {value: 0x0008, lo: 0x80, hi: 0x8f}, {value: 0x0018, lo: 0x90, hi: 0x99}, {value: 0x0040, lo: 0x9a, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xbf}, - // Block 0x29, offset 0x199 + // Block 0x29, offset 0x181 {value: 0x0000, lo: 0x04}, {value: 0x0008, lo: 0x80, hi: 0xb5}, {value: 0x0040, lo: 0xb6, hi: 0xb7}, {value: 0xe045, lo: 0xb8, hi: 0xbd}, {value: 0x0040, lo: 0xbe, hi: 0xbf}, - // Block 0x2a, offset 0x19e + // Block 0x2a, offset 0x186 {value: 0x0000, lo: 0x02}, {value: 0x0018, lo: 0x80, hi: 0x80}, {value: 0x0008, lo: 0x81, hi: 0xbf}, - // Block 0x2b, offset 0x1a1 + // Block 0x2b, offset 0x189 {value: 0x0000, lo: 0x03}, {value: 0x0008, lo: 0x80, hi: 0xac}, {value: 0x0018, lo: 0xad, hi: 0xae}, {value: 0x0008, lo: 0xaf, hi: 0xbf}, - // Block 0x2c, offset 0x1a5 + // Block 0x2c, offset 0x18d {value: 0x0000, lo: 0x05}, {value: 0x0040, lo: 0x80, hi: 0x80}, {value: 0x0008, lo: 0x81, hi: 0x9a}, {value: 0x0018, lo: 0x9b, hi: 0x9c}, {value: 0x0040, lo: 0x9d, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xbf}, - // Block 0x2d, offset 0x1ab + // Block 0x2d, offset 0x193 {value: 0x0000, lo: 0x04}, {value: 0x0008, lo: 0x80, hi: 0xaa}, {value: 0x0018, lo: 0xab, hi: 0xb0}, {value: 0x0008, lo: 0xb1, hi: 0xb8}, {value: 0x0040, lo: 0xb9, hi: 0xbf}, - // Block 0x2e, offset 0x1b0 + // Block 0x2e, offset 0x198 {value: 0x0000, lo: 0x0b}, {value: 0x0008, lo: 0x80, hi: 0x8c}, {value: 0x0040, lo: 0x8d, hi: 0x8d}, {value: 0x0008, lo: 0x8e, hi: 0x91}, - {value: 0x1308, lo: 0x92, hi: 0x93}, - {value: 0x1b08, lo: 0x94, hi: 0x94}, + {value: 0x3308, lo: 0x92, hi: 0x93}, + {value: 0x3b08, lo: 0x94, hi: 0x94}, {value: 0x0040, lo: 0x95, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xb1}, - {value: 0x1308, lo: 0xb2, hi: 0xb3}, - {value: 0x1b08, lo: 0xb4, hi: 0xb4}, + {value: 0x3308, lo: 0xb2, hi: 0xb3}, + {value: 0x3b08, lo: 0xb4, hi: 0xb4}, {value: 0x0018, lo: 0xb5, hi: 0xb6}, {value: 0x0040, lo: 0xb7, hi: 0xbf}, - // Block 0x2f, offset 0x1bc + // Block 0x2f, offset 0x1a4 {value: 0x0000, lo: 0x09}, {value: 0x0008, lo: 0x80, hi: 0x91}, - {value: 0x1308, lo: 0x92, hi: 0x93}, + {value: 0x3308, lo: 0x92, hi: 0x93}, {value: 0x0040, lo: 0x94, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xac}, {value: 0x0040, lo: 0xad, hi: 0xad}, {value: 0x0008, lo: 0xae, hi: 0xb0}, {value: 0x0040, lo: 0xb1, hi: 0xb1}, - {value: 0x1308, lo: 0xb2, hi: 0xb3}, + {value: 0x3308, lo: 0xb2, hi: 0xb3}, {value: 0x0040, lo: 0xb4, hi: 0xbf}, - // Block 0x30, offset 0x1c6 + // Block 0x30, offset 0x1ae {value: 0x0000, lo: 0x05}, {value: 0x0008, lo: 0x80, hi: 0xb3}, - {value: 0x1340, lo: 0xb4, hi: 0xb5}, - {value: 0x1008, lo: 0xb6, hi: 0xb6}, - {value: 0x1308, lo: 0xb7, hi: 0xbd}, - {value: 0x1008, lo: 0xbe, hi: 0xbf}, - // Block 0x31, offset 0x1cc + {value: 0x3340, lo: 0xb4, hi: 0xb5}, + {value: 0x3008, lo: 0xb6, hi: 0xb6}, + {value: 0x3308, lo: 0xb7, hi: 0xbd}, + {value: 0x3008, lo: 0xbe, hi: 0xbf}, + // Block 0x31, offset 0x1b4 {value: 0x0000, lo: 0x10}, - {value: 0x1008, lo: 0x80, hi: 0x85}, - {value: 0x1308, lo: 0x86, hi: 0x86}, - {value: 0x1008, lo: 0x87, hi: 0x88}, - {value: 0x1308, lo: 0x89, hi: 0x91}, - {value: 0x1b08, lo: 0x92, hi: 0x92}, - {value: 0x1308, lo: 0x93, hi: 0x93}, + {value: 0x3008, lo: 0x80, hi: 0x85}, + {value: 0x3308, lo: 0x86, hi: 0x86}, + {value: 0x3008, lo: 0x87, hi: 0x88}, + {value: 0x3308, lo: 0x89, hi: 0x91}, + {value: 0x3b08, lo: 0x92, hi: 0x92}, + {value: 0x3308, lo: 0x93, hi: 0x93}, {value: 0x0018, lo: 0x94, hi: 0x96}, {value: 0x0008, lo: 0x97, hi: 0x97}, {value: 0x0018, lo: 0x98, hi: 0x9b}, {value: 0x0008, lo: 0x9c, hi: 0x9c}, - {value: 0x1308, lo: 0x9d, hi: 0x9d}, + {value: 0x3308, lo: 0x9d, hi: 0x9d}, {value: 0x0040, lo: 0x9e, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xa9}, {value: 0x0040, lo: 0xaa, hi: 0xaf}, {value: 0x0018, lo: 0xb0, hi: 0xb9}, {value: 0x0040, lo: 0xba, hi: 0xbf}, - // Block 0x32, offset 0x1dd + // Block 0x32, offset 0x1c5 {value: 0x0000, lo: 0x09}, {value: 0x0018, lo: 0x80, hi: 0x85}, {value: 0x0040, lo: 0x86, hi: 0x86}, {value: 0x0218, lo: 0x87, hi: 0x87}, {value: 0x0018, lo: 0x88, hi: 0x8a}, - {value: 0x13c0, lo: 0x8b, hi: 0x8d}, + {value: 0x33c0, lo: 0x8b, hi: 0x8d}, {value: 0x0040, lo: 0x8e, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x99}, {value: 0x0040, lo: 0x9a, hi: 0x9f}, {value: 0x0208, lo: 0xa0, hi: 0xbf}, - // Block 0x33, offset 0x1e7 + // Block 0x33, offset 0x1cf {value: 0x0000, lo: 0x02}, {value: 0x0208, lo: 0x80, hi: 0xb7}, {value: 0x0040, lo: 0xb8, hi: 0xbf}, - // Block 0x34, offset 0x1ea + // Block 0x34, offset 0x1d2 {value: 0x0000, lo: 0x07}, {value: 0x0008, lo: 0x80, hi: 0x84}, - {value: 0x1308, lo: 0x85, hi: 0x86}, + {value: 0x3308, lo: 0x85, hi: 0x86}, {value: 0x0208, lo: 0x87, hi: 0xa8}, - {value: 0x1308, lo: 0xa9, hi: 0xa9}, + {value: 0x3308, lo: 0xa9, hi: 0xa9}, {value: 0x0208, lo: 0xaa, hi: 0xaa}, {value: 0x0040, lo: 0xab, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xbf}, - // Block 0x35, offset 0x1f2 + // Block 0x35, offset 0x1da {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0xb5}, {value: 0x0040, lo: 0xb6, hi: 0xbf}, - // Block 0x36, offset 0x1f5 + // Block 0x36, offset 0x1dd {value: 0x0000, lo: 0x0c}, {value: 0x0008, lo: 0x80, hi: 0x9e}, {value: 0x0040, lo: 0x9f, hi: 0x9f}, - {value: 0x1308, lo: 0xa0, hi: 0xa2}, - {value: 0x1008, lo: 0xa3, hi: 0xa6}, - {value: 0x1308, lo: 0xa7, hi: 0xa8}, - {value: 0x1008, lo: 0xa9, hi: 0xab}, + {value: 0x3308, lo: 0xa0, hi: 0xa2}, + {value: 0x3008, lo: 0xa3, hi: 0xa6}, + {value: 0x3308, lo: 0xa7, hi: 0xa8}, + {value: 0x3008, lo: 0xa9, hi: 0xab}, {value: 0x0040, lo: 0xac, hi: 0xaf}, - {value: 0x1008, lo: 0xb0, hi: 0xb1}, - {value: 0x1308, lo: 0xb2, hi: 0xb2}, - {value: 0x1008, lo: 0xb3, hi: 0xb8}, - {value: 0x1308, lo: 0xb9, hi: 0xbb}, + {value: 0x3008, lo: 0xb0, hi: 0xb1}, + {value: 0x3308, lo: 0xb2, hi: 0xb2}, + {value: 0x3008, lo: 0xb3, hi: 0xb8}, + {value: 0x3308, lo: 0xb9, hi: 0xbb}, {value: 0x0040, lo: 0xbc, hi: 0xbf}, - // Block 0x37, offset 0x202 + // Block 0x37, offset 0x1ea {value: 0x0000, lo: 0x07}, {value: 0x0018, lo: 0x80, hi: 0x80}, {value: 0x0040, lo: 0x81, hi: 0x83}, @@ -2918,12 +2927,12 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0xae, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xb4}, {value: 0x0040, lo: 0xb5, hi: 0xbf}, - // Block 0x38, offset 0x20a + // Block 0x38, offset 0x1f2 {value: 0x0000, lo: 0x03}, {value: 0x0008, lo: 0x80, hi: 0xab}, {value: 0x0040, lo: 0xac, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xbf}, - // Block 0x39, offset 0x20e + // Block 0x39, offset 0x1f6 {value: 0x0000, lo: 0x06}, {value: 0x0008, lo: 0x80, hi: 0x89}, {value: 0x0040, lo: 0x8a, hi: 0x8f}, @@ -2931,33 +2940,33 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0028, lo: 0x9a, hi: 0x9a}, {value: 0x0040, lo: 0x9b, hi: 0x9d}, {value: 0x0018, lo: 0x9e, hi: 0xbf}, - // Block 0x3a, offset 0x215 + // Block 0x3a, offset 0x1fd {value: 0x0000, lo: 0x07}, {value: 0x0008, lo: 0x80, hi: 0x96}, - {value: 0x1308, lo: 0x97, hi: 0x98}, - {value: 0x1008, lo: 0x99, hi: 0x9a}, - {value: 0x1308, lo: 0x9b, hi: 0x9b}, + {value: 0x3308, lo: 0x97, hi: 0x98}, + {value: 0x3008, lo: 0x99, hi: 0x9a}, + {value: 0x3308, lo: 0x9b, hi: 0x9b}, {value: 0x0040, lo: 0x9c, hi: 0x9d}, {value: 0x0018, lo: 0x9e, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xbf}, - // Block 0x3b, offset 0x21d + // Block 0x3b, offset 0x205 {value: 0x0000, lo: 0x0f}, {value: 0x0008, lo: 0x80, hi: 0x94}, - {value: 0x1008, lo: 0x95, hi: 0x95}, - {value: 0x1308, lo: 0x96, hi: 0x96}, - {value: 0x1008, lo: 0x97, hi: 0x97}, - {value: 0x1308, lo: 0x98, hi: 0x9e}, + {value: 0x3008, lo: 0x95, hi: 0x95}, + {value: 0x3308, lo: 0x96, hi: 0x96}, + {value: 0x3008, lo: 0x97, hi: 0x97}, + {value: 0x3308, lo: 0x98, hi: 0x9e}, {value: 0x0040, lo: 0x9f, hi: 0x9f}, - {value: 0x1b08, lo: 0xa0, hi: 0xa0}, - {value: 0x1008, lo: 0xa1, hi: 0xa1}, - {value: 0x1308, lo: 0xa2, hi: 0xa2}, - {value: 0x1008, lo: 0xa3, hi: 0xa4}, - {value: 0x1308, lo: 0xa5, hi: 0xac}, - {value: 0x1008, lo: 0xad, hi: 0xb2}, - {value: 0x1308, lo: 0xb3, hi: 0xbc}, + {value: 0x3b08, lo: 0xa0, hi: 0xa0}, + {value: 0x3008, lo: 0xa1, hi: 0xa1}, + {value: 0x3308, lo: 0xa2, hi: 0xa2}, + {value: 0x3008, lo: 0xa3, hi: 0xa4}, + {value: 0x3308, lo: 0xa5, hi: 0xac}, + {value: 0x3008, lo: 0xad, hi: 0xb2}, + {value: 0x3308, lo: 0xb3, hi: 0xbc}, {value: 0x0040, lo: 0xbd, hi: 0xbe}, - {value: 0x1308, lo: 0xbf, hi: 0xbf}, - // Block 0x3c, offset 0x22d + {value: 0x3308, lo: 0xbf, hi: 0xbf}, + // Block 0x3c, offset 0x215 {value: 0x0000, lo: 0x0b}, {value: 0x0008, lo: 0x80, hi: 0x89}, {value: 0x0040, lo: 0x8a, hi: 0x8f}, @@ -2967,78 +2976,78 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0008, lo: 0xa7, hi: 0xa7}, {value: 0x0018, lo: 0xa8, hi: 0xad}, {value: 0x0040, lo: 0xae, hi: 0xaf}, - {value: 0x1308, lo: 0xb0, hi: 0xbd}, - {value: 0x1318, lo: 0xbe, hi: 0xbe}, + {value: 0x3308, lo: 0xb0, hi: 0xbd}, + {value: 0x3318, lo: 0xbe, hi: 0xbe}, {value: 0x0040, lo: 0xbf, hi: 0xbf}, - // Block 0x3d, offset 0x239 + // Block 0x3d, offset 0x221 {value: 0x0000, lo: 0x01}, {value: 0x0040, lo: 0x80, hi: 0xbf}, - // Block 0x3e, offset 0x23b + // Block 0x3e, offset 0x223 {value: 0x0000, lo: 0x09}, - {value: 0x1308, lo: 0x80, hi: 0x83}, - {value: 0x1008, lo: 0x84, hi: 0x84}, + {value: 0x3308, lo: 0x80, hi: 0x83}, + {value: 0x3008, lo: 0x84, hi: 0x84}, {value: 0x0008, lo: 0x85, hi: 0xb3}, - {value: 0x1308, lo: 0xb4, hi: 0xb4}, - {value: 0x1008, lo: 0xb5, hi: 0xb5}, - {value: 0x1308, lo: 0xb6, hi: 0xba}, - {value: 0x1008, lo: 0xbb, hi: 0xbb}, - {value: 0x1308, lo: 0xbc, hi: 0xbc}, - {value: 0x1008, lo: 0xbd, hi: 0xbf}, - // Block 0x3f, offset 0x245 + {value: 0x3308, lo: 0xb4, hi: 0xb4}, + {value: 0x3008, lo: 0xb5, hi: 0xb5}, + {value: 0x3308, lo: 0xb6, hi: 0xba}, + {value: 0x3008, lo: 0xbb, hi: 0xbb}, + {value: 0x3308, lo: 0xbc, hi: 0xbc}, + {value: 0x3008, lo: 0xbd, hi: 0xbf}, + // Block 0x3f, offset 0x22d {value: 0x0000, lo: 0x0b}, - {value: 0x1008, lo: 0x80, hi: 0x81}, - {value: 0x1308, lo: 0x82, hi: 0x82}, - {value: 0x1008, lo: 0x83, hi: 0x83}, - {value: 0x1808, lo: 0x84, hi: 0x84}, + {value: 0x3008, lo: 0x80, hi: 0x81}, + {value: 0x3308, lo: 0x82, hi: 0x82}, + {value: 0x3008, lo: 0x83, hi: 0x83}, + {value: 0x3808, lo: 0x84, hi: 0x84}, {value: 0x0008, lo: 0x85, hi: 0x8b}, {value: 0x0040, lo: 0x8c, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x99}, {value: 0x0018, lo: 0x9a, hi: 0xaa}, - {value: 0x1308, lo: 0xab, hi: 0xb3}, + {value: 0x3308, lo: 0xab, hi: 0xb3}, {value: 0x0018, lo: 0xb4, hi: 0xbc}, {value: 0x0040, lo: 0xbd, hi: 0xbf}, - // Block 0x40, offset 0x251 + // Block 0x40, offset 0x239 {value: 0x0000, lo: 0x0b}, - {value: 0x1308, lo: 0x80, hi: 0x81}, - {value: 0x1008, lo: 0x82, hi: 0x82}, + {value: 0x3308, lo: 0x80, hi: 0x81}, + {value: 0x3008, lo: 0x82, hi: 0x82}, {value: 0x0008, lo: 0x83, hi: 0xa0}, - {value: 0x1008, lo: 0xa1, hi: 0xa1}, - {value: 0x1308, lo: 0xa2, hi: 0xa5}, - {value: 0x1008, lo: 0xa6, hi: 0xa7}, - {value: 0x1308, lo: 0xa8, hi: 0xa9}, - {value: 0x1808, lo: 0xaa, hi: 0xaa}, - {value: 0x1b08, lo: 0xab, hi: 0xab}, - {value: 0x1308, lo: 0xac, hi: 0xad}, + {value: 0x3008, lo: 0xa1, hi: 0xa1}, + {value: 0x3308, lo: 0xa2, hi: 0xa5}, + {value: 0x3008, lo: 0xa6, hi: 0xa7}, + {value: 0x3308, lo: 0xa8, hi: 0xa9}, + {value: 0x3808, lo: 0xaa, hi: 0xaa}, + {value: 0x3b08, lo: 0xab, hi: 0xab}, + {value: 0x3308, lo: 0xac, hi: 0xad}, {value: 0x0008, lo: 0xae, hi: 0xbf}, - // Block 0x41, offset 0x25d + // Block 0x41, offset 0x245 {value: 0x0000, lo: 0x0b}, {value: 0x0008, lo: 0x80, hi: 0xa5}, - {value: 0x1308, lo: 0xa6, hi: 0xa6}, - {value: 0x1008, lo: 0xa7, hi: 0xa7}, - {value: 0x1308, lo: 0xa8, hi: 0xa9}, - {value: 0x1008, lo: 0xaa, hi: 0xac}, - {value: 0x1308, lo: 0xad, hi: 0xad}, - {value: 0x1008, lo: 0xae, hi: 0xae}, - {value: 0x1308, lo: 0xaf, hi: 0xb1}, - {value: 0x1808, lo: 0xb2, hi: 0xb3}, + {value: 0x3308, lo: 0xa6, hi: 0xa6}, + {value: 0x3008, lo: 0xa7, hi: 0xa7}, + {value: 0x3308, lo: 0xa8, hi: 0xa9}, + {value: 0x3008, lo: 0xaa, hi: 0xac}, + {value: 0x3308, lo: 0xad, hi: 0xad}, + {value: 0x3008, lo: 0xae, hi: 0xae}, + {value: 0x3308, lo: 0xaf, hi: 0xb1}, + {value: 0x3808, lo: 0xb2, hi: 0xb3}, {value: 0x0040, lo: 0xb4, hi: 0xbb}, {value: 0x0018, lo: 0xbc, hi: 0xbf}, - // Block 0x42, offset 0x269 + // Block 0x42, offset 0x251 {value: 0x0000, lo: 0x07}, {value: 0x0008, lo: 0x80, hi: 0xa3}, - {value: 0x1008, lo: 0xa4, hi: 0xab}, - {value: 0x1308, lo: 0xac, hi: 0xb3}, - {value: 0x1008, lo: 0xb4, hi: 0xb5}, - {value: 0x1308, lo: 0xb6, hi: 0xb7}, + {value: 0x3008, lo: 0xa4, hi: 0xab}, + {value: 0x3308, lo: 0xac, hi: 0xb3}, + {value: 0x3008, lo: 0xb4, hi: 0xb5}, + {value: 0x3308, lo: 0xb6, hi: 0xb7}, {value: 0x0040, lo: 0xb8, hi: 0xba}, {value: 0x0018, lo: 0xbb, hi: 0xbf}, - // Block 0x43, offset 0x271 + // Block 0x43, offset 0x259 {value: 0x0000, lo: 0x04}, {value: 0x0008, lo: 0x80, hi: 0x89}, {value: 0x0040, lo: 0x8a, hi: 0x8c}, {value: 0x0008, lo: 0x8d, hi: 0xbd}, {value: 0x0018, lo: 0xbe, hi: 0xbf}, - // Block 0x44, offset 0x276 + // Block 0x44, offset 0x25e {value: 0x0000, lo: 0x09}, {value: 0x0e29, lo: 0x80, hi: 0x80}, {value: 0x0e41, lo: 0x81, hi: 0x81}, @@ -3049,30 +3058,30 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0eb9, lo: 0x87, hi: 0x87}, {value: 0x057d, lo: 0x88, hi: 0x88}, {value: 0x0040, lo: 0x89, hi: 0xbf}, - // Block 0x45, offset 0x280 + // Block 0x45, offset 0x268 {value: 0x0000, lo: 0x10}, {value: 0x0018, lo: 0x80, hi: 0x87}, {value: 0x0040, lo: 0x88, hi: 0x8f}, - {value: 0x1308, lo: 0x90, hi: 0x92}, + {value: 0x3308, lo: 0x90, hi: 0x92}, {value: 0x0018, lo: 0x93, hi: 0x93}, - {value: 0x1308, lo: 0x94, hi: 0xa0}, - {value: 0x1008, lo: 0xa1, hi: 0xa1}, - {value: 0x1308, lo: 0xa2, hi: 0xa8}, + {value: 0x3308, lo: 0x94, hi: 0xa0}, + {value: 0x3008, lo: 0xa1, hi: 0xa1}, + {value: 0x3308, lo: 0xa2, hi: 0xa8}, {value: 0x0008, lo: 0xa9, hi: 0xac}, - {value: 0x1308, lo: 0xad, hi: 0xad}, + {value: 0x3308, lo: 0xad, hi: 0xad}, {value: 0x0008, lo: 0xae, hi: 0xb1}, - {value: 0x1008, lo: 0xb2, hi: 0xb3}, - {value: 0x1308, lo: 0xb4, hi: 0xb4}, + {value: 0x3008, lo: 0xb2, hi: 0xb3}, + {value: 0x3308, lo: 0xb4, hi: 0xb4}, {value: 0x0008, lo: 0xb5, hi: 0xb6}, - {value: 0x0040, lo: 0xb7, hi: 0xb7}, - {value: 0x1308, lo: 0xb8, hi: 0xb9}, + {value: 0x3008, lo: 0xb7, hi: 0xb7}, + {value: 0x3308, lo: 0xb8, hi: 0xb9}, {value: 0x0040, lo: 0xba, hi: 0xbf}, - // Block 0x46, offset 0x291 + // Block 0x46, offset 0x279 {value: 0x0000, lo: 0x03}, - {value: 0x1308, lo: 0x80, hi: 0xb5}, - {value: 0x0040, lo: 0xb6, hi: 0xba}, - {value: 0x1308, lo: 0xbb, hi: 0xbf}, - // Block 0x47, offset 0x295 + {value: 0x3308, lo: 0x80, hi: 0xb9}, + {value: 0x0040, lo: 0xba, hi: 0xba}, + {value: 0x3308, lo: 0xbb, hi: 0xbf}, + // Block 0x47, offset 0x27d {value: 0x0000, lo: 0x0a}, {value: 0x0008, lo: 0x80, hi: 0x87}, {value: 0xe045, lo: 0x88, hi: 0x8f}, @@ -3084,12 +3093,12 @@ var idnaSparseValues = [1876]valueRange{ {value: 0xe045, lo: 0xa8, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xb7}, {value: 0xe045, lo: 0xb8, hi: 0xbf}, - // Block 0x48, offset 0x2a0 + // Block 0x48, offset 0x288 {value: 0x0000, lo: 0x03}, {value: 0x0040, lo: 0x80, hi: 0x8f}, - {value: 0x1318, lo: 0x90, hi: 0xb0}, + {value: 0x3318, lo: 0x90, hi: 0xb0}, {value: 0x0040, lo: 0xb1, hi: 0xbf}, - // Block 0x49, offset 0x2a4 + // Block 0x49, offset 0x28c {value: 0x0000, lo: 0x08}, {value: 0x0018, lo: 0x80, hi: 0x82}, {value: 0x0040, lo: 0x83, hi: 0x83}, @@ -3099,7 +3108,7 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0018, lo: 0x8a, hi: 0x8b}, {value: 0x0040, lo: 0x8c, hi: 0x8f}, {value: 0x0018, lo: 0x90, hi: 0xbf}, - // Block 0x4a, offset 0x2ad + // Block 0x4a, offset 0x295 {value: 0x0000, lo: 0x07}, {value: 0x0018, lo: 0x80, hi: 0xab}, {value: 0x24f1, lo: 0xac, hi: 0xac}, @@ -3108,72 +3117,68 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x2579, lo: 0xaf, hi: 0xaf}, {value: 0x25b1, lo: 0xb0, hi: 0xb0}, {value: 0x0018, lo: 0xb1, hi: 0xbf}, - // Block 0x4b, offset 0x2b5 + // Block 0x4b, offset 0x29d {value: 0x0000, lo: 0x05}, {value: 0x0018, lo: 0x80, hi: 0x9f}, {value: 0x0080, lo: 0xa0, hi: 0xa0}, {value: 0x0018, lo: 0xa1, hi: 0xad}, {value: 0x0080, lo: 0xae, hi: 0xaf}, {value: 0x0018, lo: 0xb0, hi: 0xbf}, - // Block 0x4c, offset 0x2bb + // Block 0x4c, offset 0x2a3 {value: 0x0000, lo: 0x04}, {value: 0x0018, lo: 0x80, hi: 0xa8}, {value: 0x09c5, lo: 0xa9, hi: 0xa9}, {value: 0x09e5, lo: 0xaa, hi: 0xaa}, {value: 0x0018, lo: 0xab, hi: 0xbf}, - // Block 0x4d, offset 0x2c0 - {value: 0x0000, lo: 0x02}, - {value: 0x0018, lo: 0x80, hi: 0xbe}, - {value: 0x0040, lo: 0xbf, hi: 0xbf}, - // Block 0x4e, offset 0x2c3 + // Block 0x4d, offset 0x2a8 {value: 0x0000, lo: 0x02}, {value: 0x0018, lo: 0x80, hi: 0xa6}, {value: 0x0040, lo: 0xa7, hi: 0xbf}, - // Block 0x4f, offset 0x2c6 + // Block 0x4e, offset 0x2ab {value: 0x0000, lo: 0x03}, {value: 0x0018, lo: 0x80, hi: 0x8b}, {value: 0x28c1, lo: 0x8c, hi: 0x8c}, {value: 0x0018, lo: 0x8d, hi: 0xbf}, - // Block 0x50, offset 0x2ca + // Block 0x4f, offset 0x2af {value: 0x0000, lo: 0x05}, {value: 0x0018, lo: 0x80, hi: 0xb3}, {value: 0x0e66, lo: 0xb4, hi: 0xb4}, {value: 0x292a, lo: 0xb5, hi: 0xb5}, {value: 0x0e86, lo: 0xb6, hi: 0xb6}, {value: 0x0018, lo: 0xb7, hi: 0xbf}, - // Block 0x51, offset 0x2d0 + // Block 0x50, offset 0x2b5 {value: 0x0000, lo: 0x03}, {value: 0x0018, lo: 0x80, hi: 0x9b}, {value: 0x2941, lo: 0x9c, hi: 0x9c}, {value: 0x0018, lo: 0x9d, hi: 0xbf}, - // Block 0x52, offset 0x2d4 + // Block 0x51, offset 0x2b9 {value: 0x0000, lo: 0x03}, {value: 0x0018, lo: 0x80, hi: 0xb3}, {value: 0x0040, lo: 0xb4, hi: 0xb5}, {value: 0x0018, lo: 0xb6, hi: 0xbf}, - // Block 0x53, offset 0x2d8 + // Block 0x52, offset 0x2bd {value: 0x0000, lo: 0x05}, {value: 0x0018, lo: 0x80, hi: 0x95}, {value: 0x0040, lo: 0x96, hi: 0x97}, {value: 0x0018, lo: 0x98, hi: 0xb9}, {value: 0x0040, lo: 0xba, hi: 0xbc}, {value: 0x0018, lo: 0xbd, hi: 0xbf}, - // Block 0x54, offset 0x2de + // Block 0x53, offset 0x2c3 {value: 0x0000, lo: 0x06}, {value: 0x0018, lo: 0x80, hi: 0x88}, {value: 0x0040, lo: 0x89, hi: 0x89}, - {value: 0x0018, lo: 0x8a, hi: 0x91}, - {value: 0x0040, lo: 0x92, hi: 0xab}, + {value: 0x0018, lo: 0x8a, hi: 0x92}, + {value: 0x0040, lo: 0x93, hi: 0xab}, {value: 0x0018, lo: 0xac, hi: 0xaf}, {value: 0x0040, lo: 0xb0, hi: 0xbf}, - // Block 0x55, offset 0x2e5 + // Block 0x54, offset 0x2ca {value: 0x0000, lo: 0x05}, {value: 0xe185, lo: 0x80, hi: 0x8f}, {value: 0x03f5, lo: 0x90, hi: 0x9f}, {value: 0x0ea5, lo: 0xa0, hi: 0xae}, {value: 0x0040, lo: 0xaf, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xbf}, - // Block 0x56, offset 0x2eb + // Block 0x55, offset 0x2d0 {value: 0x0000, lo: 0x07}, {value: 0x0008, lo: 0x80, hi: 0xa5}, {value: 0x0040, lo: 0xa6, hi: 0xa6}, @@ -3182,15 +3187,15 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0008, lo: 0xad, hi: 0xad}, {value: 0x0040, lo: 0xae, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xbf}, - // Block 0x57, offset 0x2f3 + // Block 0x56, offset 0x2d8 {value: 0x0000, lo: 0x06}, {value: 0x0008, lo: 0x80, hi: 0xa7}, {value: 0x0040, lo: 0xa8, hi: 0xae}, {value: 0xe075, lo: 0xaf, hi: 0xaf}, {value: 0x0018, lo: 0xb0, hi: 0xb0}, {value: 0x0040, lo: 0xb1, hi: 0xbe}, - {value: 0x1b08, lo: 0xbf, hi: 0xbf}, - // Block 0x58, offset 0x2fa + {value: 0x3b08, lo: 0xbf, hi: 0xbf}, + // Block 0x57, offset 0x2df {value: 0x0000, lo: 0x0a}, {value: 0x0008, lo: 0x80, hi: 0x96}, {value: 0x0040, lo: 0x97, hi: 0x9f}, @@ -3202,7 +3207,7 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0xb7, hi: 0xb7}, {value: 0x0008, lo: 0xb8, hi: 0xbe}, {value: 0x0040, lo: 0xbf, hi: 0xbf}, - // Block 0x59, offset 0x305 + // Block 0x58, offset 0x2ea {value: 0x0000, lo: 0x09}, {value: 0x0008, lo: 0x80, hi: 0x86}, {value: 0x0040, lo: 0x87, hi: 0x87}, @@ -3212,62 +3217,62 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0x97, hi: 0x97}, {value: 0x0008, lo: 0x98, hi: 0x9e}, {value: 0x0040, lo: 0x9f, hi: 0x9f}, - {value: 0x1308, lo: 0xa0, hi: 0xbf}, - // Block 0x5a, offset 0x30f + {value: 0x3308, lo: 0xa0, hi: 0xbf}, + // Block 0x59, offset 0x2f4 {value: 0x0000, lo: 0x03}, {value: 0x0018, lo: 0x80, hi: 0xae}, {value: 0x0008, lo: 0xaf, hi: 0xaf}, {value: 0x0018, lo: 0xb0, hi: 0xbf}, - // Block 0x5b, offset 0x313 + // Block 0x5a, offset 0x2f8 {value: 0x0000, lo: 0x02}, - {value: 0x0018, lo: 0x80, hi: 0x84}, - {value: 0x0040, lo: 0x85, hi: 0xbf}, - // Block 0x5c, offset 0x316 + {value: 0x0018, lo: 0x80, hi: 0x89}, + {value: 0x0040, lo: 0x8a, hi: 0xbf}, + // Block 0x5b, offset 0x2fb {value: 0x0000, lo: 0x05}, {value: 0x0018, lo: 0x80, hi: 0x99}, {value: 0x0040, lo: 0x9a, hi: 0x9a}, {value: 0x0018, lo: 0x9b, hi: 0x9e}, {value: 0x0edd, lo: 0x9f, hi: 0x9f}, {value: 0x0018, lo: 0xa0, hi: 0xbf}, - // Block 0x5d, offset 0x31c + // Block 0x5c, offset 0x301 {value: 0x0000, lo: 0x03}, {value: 0x0018, lo: 0x80, hi: 0xb2}, {value: 0x0efd, lo: 0xb3, hi: 0xb3}, {value: 0x0040, lo: 0xb4, hi: 0xbf}, - // Block 0x5e, offset 0x320 + // Block 0x5d, offset 0x305 {value: 0x0020, lo: 0x01}, {value: 0x0f1d, lo: 0x80, hi: 0xbf}, - // Block 0x5f, offset 0x322 + // Block 0x5e, offset 0x307 {value: 0x0020, lo: 0x02}, {value: 0x171d, lo: 0x80, hi: 0x8f}, {value: 0x18fd, lo: 0x90, hi: 0xbf}, - // Block 0x60, offset 0x325 + // Block 0x5f, offset 0x30a {value: 0x0020, lo: 0x01}, {value: 0x1efd, lo: 0x80, hi: 0xbf}, - // Block 0x61, offset 0x327 + // Block 0x60, offset 0x30c {value: 0x0000, lo: 0x02}, {value: 0x0040, lo: 0x80, hi: 0x80}, {value: 0x0008, lo: 0x81, hi: 0xbf}, - // Block 0x62, offset 0x32a + // Block 0x61, offset 0x30f {value: 0x0000, lo: 0x09}, {value: 0x0008, lo: 0x80, hi: 0x96}, {value: 0x0040, lo: 0x97, hi: 0x98}, - {value: 0x1308, lo: 0x99, hi: 0x9a}, + {value: 0x3308, lo: 0x99, hi: 0x9a}, {value: 0x29e2, lo: 0x9b, hi: 0x9b}, {value: 0x2a0a, lo: 0x9c, hi: 0x9c}, {value: 0x0008, lo: 0x9d, hi: 0x9e}, {value: 0x2a31, lo: 0x9f, hi: 0x9f}, {value: 0x0018, lo: 0xa0, hi: 0xa0}, {value: 0x0008, lo: 0xa1, hi: 0xbf}, - // Block 0x63, offset 0x334 + // Block 0x62, offset 0x319 {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0xbe}, {value: 0x2a69, lo: 0xbf, hi: 0xbf}, - // Block 0x64, offset 0x337 + // Block 0x63, offset 0x31c {value: 0x0000, lo: 0x0e}, {value: 0x0040, lo: 0x80, hi: 0x84}, - {value: 0x0008, lo: 0x85, hi: 0xad}, - {value: 0x0040, lo: 0xae, hi: 0xb0}, + {value: 0x0008, lo: 0x85, hi: 0xae}, + {value: 0x0040, lo: 0xaf, hi: 0xb0}, {value: 0x2a1d, lo: 0xb1, hi: 0xb1}, {value: 0x2a3d, lo: 0xb2, hi: 0xb2}, {value: 0x2a5d, lo: 0xb3, hi: 0xb3}, @@ -3279,150 +3284,150 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x2afd, lo: 0xba, hi: 0xbb}, {value: 0x2b1d, lo: 0xbc, hi: 0xbd}, {value: 0x2afd, lo: 0xbe, hi: 0xbf}, - // Block 0x65, offset 0x346 + // Block 0x64, offset 0x32b {value: 0x0000, lo: 0x03}, {value: 0x0018, lo: 0x80, hi: 0xa3}, {value: 0x0040, lo: 0xa4, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xbf}, - // Block 0x66, offset 0x34a + // Block 0x65, offset 0x32f {value: 0x0030, lo: 0x04}, {value: 0x2aa2, lo: 0x80, hi: 0x9d}, {value: 0x305a, lo: 0x9e, hi: 0x9e}, {value: 0x0040, lo: 0x9f, hi: 0x9f}, {value: 0x30a2, lo: 0xa0, hi: 0xbf}, - // Block 0x67, offset 0x34f + // Block 0x66, offset 0x334 {value: 0x0000, lo: 0x02}, - {value: 0x0008, lo: 0x80, hi: 0x95}, - {value: 0x0040, lo: 0x96, hi: 0xbf}, - // Block 0x68, offset 0x352 + {value: 0x0008, lo: 0x80, hi: 0xaa}, + {value: 0x0040, lo: 0xab, hi: 0xbf}, + // Block 0x67, offset 0x337 {value: 0x0000, lo: 0x03}, {value: 0x0008, lo: 0x80, hi: 0x8c}, {value: 0x0040, lo: 0x8d, hi: 0x8f}, {value: 0x0018, lo: 0x90, hi: 0xbf}, - // Block 0x69, offset 0x356 + // Block 0x68, offset 0x33b {value: 0x0000, lo: 0x04}, {value: 0x0018, lo: 0x80, hi: 0x86}, {value: 0x0040, lo: 0x87, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0xbd}, {value: 0x0018, lo: 0xbe, hi: 0xbf}, - // Block 0x6a, offset 0x35b + // Block 0x69, offset 0x340 {value: 0x0000, lo: 0x04}, {value: 0x0008, lo: 0x80, hi: 0x8c}, {value: 0x0018, lo: 0x8d, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0xab}, {value: 0x0040, lo: 0xac, hi: 0xbf}, - // Block 0x6b, offset 0x360 + // Block 0x6a, offset 0x345 {value: 0x0000, lo: 0x05}, {value: 0x0008, lo: 0x80, hi: 0xa5}, {value: 0x0018, lo: 0xa6, hi: 0xaf}, - {value: 0x1308, lo: 0xb0, hi: 0xb1}, + {value: 0x3308, lo: 0xb0, hi: 0xb1}, {value: 0x0018, lo: 0xb2, hi: 0xb7}, {value: 0x0040, lo: 0xb8, hi: 0xbf}, - // Block 0x6c, offset 0x366 + // Block 0x6b, offset 0x34b {value: 0x0000, lo: 0x05}, {value: 0x0040, lo: 0x80, hi: 0xb6}, {value: 0x0008, lo: 0xb7, hi: 0xb7}, {value: 0x2009, lo: 0xb8, hi: 0xb8}, {value: 0x6e89, lo: 0xb9, hi: 0xb9}, {value: 0x0008, lo: 0xba, hi: 0xbf}, - // Block 0x6d, offset 0x36c + // Block 0x6c, offset 0x351 {value: 0x0000, lo: 0x0e}, {value: 0x0008, lo: 0x80, hi: 0x81}, - {value: 0x1308, lo: 0x82, hi: 0x82}, + {value: 0x3308, lo: 0x82, hi: 0x82}, {value: 0x0008, lo: 0x83, hi: 0x85}, - {value: 0x1b08, lo: 0x86, hi: 0x86}, + {value: 0x3b08, lo: 0x86, hi: 0x86}, {value: 0x0008, lo: 0x87, hi: 0x8a}, - {value: 0x1308, lo: 0x8b, hi: 0x8b}, + {value: 0x3308, lo: 0x8b, hi: 0x8b}, {value: 0x0008, lo: 0x8c, hi: 0xa2}, - {value: 0x1008, lo: 0xa3, hi: 0xa4}, - {value: 0x1308, lo: 0xa5, hi: 0xa6}, - {value: 0x1008, lo: 0xa7, hi: 0xa7}, + {value: 0x3008, lo: 0xa3, hi: 0xa4}, + {value: 0x3308, lo: 0xa5, hi: 0xa6}, + {value: 0x3008, lo: 0xa7, hi: 0xa7}, {value: 0x0018, lo: 0xa8, hi: 0xab}, {value: 0x0040, lo: 0xac, hi: 0xaf}, {value: 0x0018, lo: 0xb0, hi: 0xb9}, {value: 0x0040, lo: 0xba, hi: 0xbf}, - // Block 0x6e, offset 0x37b + // Block 0x6d, offset 0x360 {value: 0x0000, lo: 0x05}, {value: 0x0208, lo: 0x80, hi: 0xb1}, {value: 0x0108, lo: 0xb2, hi: 0xb2}, {value: 0x0008, lo: 0xb3, hi: 0xb3}, {value: 0x0018, lo: 0xb4, hi: 0xb7}, {value: 0x0040, lo: 0xb8, hi: 0xbf}, - // Block 0x6f, offset 0x381 + // Block 0x6e, offset 0x366 {value: 0x0000, lo: 0x03}, - {value: 0x1008, lo: 0x80, hi: 0x81}, + {value: 0x3008, lo: 0x80, hi: 0x81}, {value: 0x0008, lo: 0x82, hi: 0xb3}, - {value: 0x1008, lo: 0xb4, hi: 0xbf}, - // Block 0x70, offset 0x385 + {value: 0x3008, lo: 0xb4, hi: 0xbf}, + // Block 0x6f, offset 0x36a {value: 0x0000, lo: 0x0e}, - {value: 0x1008, lo: 0x80, hi: 0x83}, - {value: 0x1b08, lo: 0x84, hi: 0x84}, - {value: 0x1308, lo: 0x85, hi: 0x85}, + {value: 0x3008, lo: 0x80, hi: 0x83}, + {value: 0x3b08, lo: 0x84, hi: 0x84}, + {value: 0x3308, lo: 0x85, hi: 0x85}, {value: 0x0040, lo: 0x86, hi: 0x8d}, {value: 0x0018, lo: 0x8e, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x99}, {value: 0x0040, lo: 0x9a, hi: 0x9f}, - {value: 0x1308, lo: 0xa0, hi: 0xb1}, + {value: 0x3308, lo: 0xa0, hi: 0xb1}, {value: 0x0008, lo: 0xb2, hi: 0xb7}, {value: 0x0018, lo: 0xb8, hi: 0xba}, {value: 0x0008, lo: 0xbb, hi: 0xbb}, {value: 0x0018, lo: 0xbc, hi: 0xbc}, {value: 0x0008, lo: 0xbd, hi: 0xbd}, {value: 0x0040, lo: 0xbe, hi: 0xbf}, - // Block 0x71, offset 0x394 + // Block 0x70, offset 0x379 {value: 0x0000, lo: 0x04}, {value: 0x0008, lo: 0x80, hi: 0xa5}, - {value: 0x1308, lo: 0xa6, hi: 0xad}, + {value: 0x3308, lo: 0xa6, hi: 0xad}, {value: 0x0018, lo: 0xae, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xbf}, - // Block 0x72, offset 0x399 + // Block 0x71, offset 0x37e {value: 0x0000, lo: 0x07}, {value: 0x0008, lo: 0x80, hi: 0x86}, - {value: 0x1308, lo: 0x87, hi: 0x91}, - {value: 0x1008, lo: 0x92, hi: 0x92}, - {value: 0x1808, lo: 0x93, hi: 0x93}, + {value: 0x3308, lo: 0x87, hi: 0x91}, + {value: 0x3008, lo: 0x92, hi: 0x92}, + {value: 0x3808, lo: 0x93, hi: 0x93}, {value: 0x0040, lo: 0x94, hi: 0x9e}, {value: 0x0018, lo: 0x9f, hi: 0xbc}, {value: 0x0040, lo: 0xbd, hi: 0xbf}, - // Block 0x73, offset 0x3a1 + // Block 0x72, offset 0x386 {value: 0x0000, lo: 0x09}, - {value: 0x1308, lo: 0x80, hi: 0x82}, - {value: 0x1008, lo: 0x83, hi: 0x83}, + {value: 0x3308, lo: 0x80, hi: 0x82}, + {value: 0x3008, lo: 0x83, hi: 0x83}, {value: 0x0008, lo: 0x84, hi: 0xb2}, - {value: 0x1308, lo: 0xb3, hi: 0xb3}, - {value: 0x1008, lo: 0xb4, hi: 0xb5}, - {value: 0x1308, lo: 0xb6, hi: 0xb9}, - {value: 0x1008, lo: 0xba, hi: 0xbb}, - {value: 0x1308, lo: 0xbc, hi: 0xbc}, - {value: 0x1008, lo: 0xbd, hi: 0xbf}, - // Block 0x74, offset 0x3ab + {value: 0x3308, lo: 0xb3, hi: 0xb3}, + {value: 0x3008, lo: 0xb4, hi: 0xb5}, + {value: 0x3308, lo: 0xb6, hi: 0xb9}, + {value: 0x3008, lo: 0xba, hi: 0xbb}, + {value: 0x3308, lo: 0xbc, hi: 0xbc}, + {value: 0x3008, lo: 0xbd, hi: 0xbf}, + // Block 0x73, offset 0x390 {value: 0x0000, lo: 0x0a}, - {value: 0x1808, lo: 0x80, hi: 0x80}, + {value: 0x3808, lo: 0x80, hi: 0x80}, {value: 0x0018, lo: 0x81, hi: 0x8d}, {value: 0x0040, lo: 0x8e, hi: 0x8e}, {value: 0x0008, lo: 0x8f, hi: 0x99}, {value: 0x0040, lo: 0x9a, hi: 0x9d}, {value: 0x0018, lo: 0x9e, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xa4}, - {value: 0x1308, lo: 0xa5, hi: 0xa5}, + {value: 0x3308, lo: 0xa5, hi: 0xa5}, {value: 0x0008, lo: 0xa6, hi: 0xbe}, {value: 0x0040, lo: 0xbf, hi: 0xbf}, - // Block 0x75, offset 0x3b6 + // Block 0x74, offset 0x39b {value: 0x0000, lo: 0x07}, {value: 0x0008, lo: 0x80, hi: 0xa8}, - {value: 0x1308, lo: 0xa9, hi: 0xae}, - {value: 0x1008, lo: 0xaf, hi: 0xb0}, - {value: 0x1308, lo: 0xb1, hi: 0xb2}, - {value: 0x1008, lo: 0xb3, hi: 0xb4}, - {value: 0x1308, lo: 0xb5, hi: 0xb6}, + {value: 0x3308, lo: 0xa9, hi: 0xae}, + {value: 0x3008, lo: 0xaf, hi: 0xb0}, + {value: 0x3308, lo: 0xb1, hi: 0xb2}, + {value: 0x3008, lo: 0xb3, hi: 0xb4}, + {value: 0x3308, lo: 0xb5, hi: 0xb6}, {value: 0x0040, lo: 0xb7, hi: 0xbf}, - // Block 0x76, offset 0x3be + // Block 0x75, offset 0x3a3 {value: 0x0000, lo: 0x10}, {value: 0x0008, lo: 0x80, hi: 0x82}, - {value: 0x1308, lo: 0x83, hi: 0x83}, + {value: 0x3308, lo: 0x83, hi: 0x83}, {value: 0x0008, lo: 0x84, hi: 0x8b}, - {value: 0x1308, lo: 0x8c, hi: 0x8c}, - {value: 0x1008, lo: 0x8d, hi: 0x8d}, + {value: 0x3308, lo: 0x8c, hi: 0x8c}, + {value: 0x3008, lo: 0x8d, hi: 0x8d}, {value: 0x0040, lo: 0x8e, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x99}, {value: 0x0040, lo: 0x9a, hi: 0x9b}, @@ -3430,38 +3435,38 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0008, lo: 0xa0, hi: 0xb6}, {value: 0x0018, lo: 0xb7, hi: 0xb9}, {value: 0x0008, lo: 0xba, hi: 0xba}, - {value: 0x1008, lo: 0xbb, hi: 0xbb}, - {value: 0x1308, lo: 0xbc, hi: 0xbc}, - {value: 0x1008, lo: 0xbd, hi: 0xbd}, + {value: 0x3008, lo: 0xbb, hi: 0xbb}, + {value: 0x3308, lo: 0xbc, hi: 0xbc}, + {value: 0x3008, lo: 0xbd, hi: 0xbd}, {value: 0x0008, lo: 0xbe, hi: 0xbf}, - // Block 0x77, offset 0x3cf + // Block 0x76, offset 0x3b4 {value: 0x0000, lo: 0x08}, {value: 0x0008, lo: 0x80, hi: 0xaf}, - {value: 0x1308, lo: 0xb0, hi: 0xb0}, + {value: 0x3308, lo: 0xb0, hi: 0xb0}, {value: 0x0008, lo: 0xb1, hi: 0xb1}, - {value: 0x1308, lo: 0xb2, hi: 0xb4}, + {value: 0x3308, lo: 0xb2, hi: 0xb4}, {value: 0x0008, lo: 0xb5, hi: 0xb6}, - {value: 0x1308, lo: 0xb7, hi: 0xb8}, + {value: 0x3308, lo: 0xb7, hi: 0xb8}, {value: 0x0008, lo: 0xb9, hi: 0xbd}, - {value: 0x1308, lo: 0xbe, hi: 0xbf}, - // Block 0x78, offset 0x3d8 + {value: 0x3308, lo: 0xbe, hi: 0xbf}, + // Block 0x77, offset 0x3bd {value: 0x0000, lo: 0x0f}, {value: 0x0008, lo: 0x80, hi: 0x80}, - {value: 0x1308, lo: 0x81, hi: 0x81}, + {value: 0x3308, lo: 0x81, hi: 0x81}, {value: 0x0008, lo: 0x82, hi: 0x82}, {value: 0x0040, lo: 0x83, hi: 0x9a}, {value: 0x0008, lo: 0x9b, hi: 0x9d}, {value: 0x0018, lo: 0x9e, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xaa}, - {value: 0x1008, lo: 0xab, hi: 0xab}, - {value: 0x1308, lo: 0xac, hi: 0xad}, - {value: 0x1008, lo: 0xae, hi: 0xaf}, + {value: 0x3008, lo: 0xab, hi: 0xab}, + {value: 0x3308, lo: 0xac, hi: 0xad}, + {value: 0x3008, lo: 0xae, hi: 0xaf}, {value: 0x0018, lo: 0xb0, hi: 0xb1}, {value: 0x0008, lo: 0xb2, hi: 0xb4}, - {value: 0x1008, lo: 0xb5, hi: 0xb5}, - {value: 0x1b08, lo: 0xb6, hi: 0xb6}, + {value: 0x3008, lo: 0xb5, hi: 0xb5}, + {value: 0x3b08, lo: 0xb6, hi: 0xb6}, {value: 0x0040, lo: 0xb7, hi: 0xbf}, - // Block 0x79, offset 0x3e8 + // Block 0x78, offset 0x3cd {value: 0x0000, lo: 0x0c}, {value: 0x0040, lo: 0x80, hi: 0x80}, {value: 0x0008, lo: 0x81, hi: 0x86}, @@ -3475,7 +3480,7 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0008, lo: 0xa8, hi: 0xae}, {value: 0x0040, lo: 0xaf, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xbf}, - // Block 0x7a, offset 0x3f5 + // Block 0x79, offset 0x3da {value: 0x0000, lo: 0x09}, {value: 0x0008, lo: 0x80, hi: 0x9a}, {value: 0x0018, lo: 0x9b, hi: 0x9b}, @@ -3486,54 +3491,54 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0008, lo: 0xa0, hi: 0xa5}, {value: 0x0040, lo: 0xa6, hi: 0xaf}, {value: 0x4495, lo: 0xb0, hi: 0xbf}, - // Block 0x7b, offset 0x3ff + // Block 0x7a, offset 0x3e4 {value: 0x0000, lo: 0x04}, {value: 0x44b5, lo: 0x80, hi: 0x8f}, {value: 0x44d5, lo: 0x90, hi: 0x9f}, {value: 0x44f5, lo: 0xa0, hi: 0xaf}, {value: 0x44d5, lo: 0xb0, hi: 0xbf}, - // Block 0x7c, offset 0x404 + // Block 0x7b, offset 0x3e9 {value: 0x0000, lo: 0x0c}, {value: 0x0008, lo: 0x80, hi: 0xa2}, - {value: 0x1008, lo: 0xa3, hi: 0xa4}, - {value: 0x1308, lo: 0xa5, hi: 0xa5}, - {value: 0x1008, lo: 0xa6, hi: 0xa7}, - {value: 0x1308, lo: 0xa8, hi: 0xa8}, - {value: 0x1008, lo: 0xa9, hi: 0xaa}, + {value: 0x3008, lo: 0xa3, hi: 0xa4}, + {value: 0x3308, lo: 0xa5, hi: 0xa5}, + {value: 0x3008, lo: 0xa6, hi: 0xa7}, + {value: 0x3308, lo: 0xa8, hi: 0xa8}, + {value: 0x3008, lo: 0xa9, hi: 0xaa}, {value: 0x0018, lo: 0xab, hi: 0xab}, - {value: 0x1008, lo: 0xac, hi: 0xac}, - {value: 0x1b08, lo: 0xad, hi: 0xad}, + {value: 0x3008, lo: 0xac, hi: 0xac}, + {value: 0x3b08, lo: 0xad, hi: 0xad}, {value: 0x0040, lo: 0xae, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xb9}, {value: 0x0040, lo: 0xba, hi: 0xbf}, - // Block 0x7d, offset 0x411 + // Block 0x7c, offset 0x3f6 {value: 0x0000, lo: 0x03}, {value: 0x0008, lo: 0x80, hi: 0xa3}, {value: 0x0040, lo: 0xa4, hi: 0xaf}, {value: 0x0018, lo: 0xb0, hi: 0xbf}, - // Block 0x7e, offset 0x415 + // Block 0x7d, offset 0x3fa {value: 0x0000, lo: 0x04}, {value: 0x0018, lo: 0x80, hi: 0x86}, {value: 0x0040, lo: 0x87, hi: 0x8a}, {value: 0x0018, lo: 0x8b, hi: 0xbb}, {value: 0x0040, lo: 0xbc, hi: 0xbf}, - // Block 0x7f, offset 0x41a + // Block 0x7e, offset 0x3ff {value: 0x0020, lo: 0x01}, {value: 0x4515, lo: 0x80, hi: 0xbf}, - // Block 0x80, offset 0x41c + // Block 0x7f, offset 0x401 {value: 0x0020, lo: 0x03}, {value: 0x4d15, lo: 0x80, hi: 0x94}, {value: 0x4ad5, lo: 0x95, hi: 0x95}, {value: 0x4fb5, lo: 0x96, hi: 0xbf}, - // Block 0x81, offset 0x420 + // Block 0x80, offset 0x405 {value: 0x0020, lo: 0x01}, {value: 0x54f5, lo: 0x80, hi: 0xbf}, - // Block 0x82, offset 0x422 + // Block 0x81, offset 0x407 {value: 0x0020, lo: 0x03}, {value: 0x5cf5, lo: 0x80, hi: 0x84}, {value: 0x5655, lo: 0x85, hi: 0x85}, {value: 0x5d95, lo: 0x86, hi: 0xbf}, - // Block 0x83, offset 0x426 + // Block 0x82, offset 0x40b {value: 0x0020, lo: 0x08}, {value: 0x6b55, lo: 0x80, hi: 0x8f}, {value: 0x6d15, lo: 0x90, hi: 0x90}, @@ -3543,19 +3548,19 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0xae, hi: 0xae}, {value: 0x0040, lo: 0xaf, hi: 0xaf}, {value: 0x70d5, lo: 0xb0, hi: 0xbf}, - // Block 0x84, offset 0x42f + // Block 0x83, offset 0x414 {value: 0x0020, lo: 0x05}, {value: 0x72d5, lo: 0x80, hi: 0xad}, {value: 0x6535, lo: 0xae, hi: 0xae}, {value: 0x7895, lo: 0xaf, hi: 0xb5}, {value: 0x6f55, lo: 0xb6, hi: 0xb6}, {value: 0x7975, lo: 0xb7, hi: 0xbf}, - // Block 0x85, offset 0x435 + // Block 0x84, offset 0x41a {value: 0x0028, lo: 0x03}, {value: 0x7c21, lo: 0x80, hi: 0x82}, {value: 0x7be1, lo: 0x83, hi: 0x83}, {value: 0x7c99, lo: 0x84, hi: 0xbf}, - // Block 0x86, offset 0x439 + // Block 0x85, offset 0x41e {value: 0x0038, lo: 0x0f}, {value: 0x9db1, lo: 0x80, hi: 0x83}, {value: 0x9e59, lo: 0x84, hi: 0x85}, @@ -3572,7 +3577,7 @@ var idnaSparseValues = [1876]valueRange{ {value: 0xa869, lo: 0xbc, hi: 0xbc}, {value: 0xa7f9, lo: 0xbd, hi: 0xbd}, {value: 0xa8d9, lo: 0xbe, hi: 0xbf}, - // Block 0x87, offset 0x449 + // Block 0x86, offset 0x42e {value: 0x0000, lo: 0x09}, {value: 0x0008, lo: 0x80, hi: 0x8b}, {value: 0x0040, lo: 0x8c, hi: 0x8c}, @@ -3583,24 +3588,24 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0008, lo: 0xbc, hi: 0xbd}, {value: 0x0040, lo: 0xbe, hi: 0xbe}, {value: 0x0008, lo: 0xbf, hi: 0xbf}, - // Block 0x88, offset 0x453 + // Block 0x87, offset 0x438 {value: 0x0000, lo: 0x04}, {value: 0x0008, lo: 0x80, hi: 0x8d}, {value: 0x0040, lo: 0x8e, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x9d}, {value: 0x0040, lo: 0x9e, hi: 0xbf}, - // Block 0x89, offset 0x458 + // Block 0x88, offset 0x43d {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0xba}, {value: 0x0040, lo: 0xbb, hi: 0xbf}, - // Block 0x8a, offset 0x45b + // Block 0x89, offset 0x440 {value: 0x0000, lo: 0x05}, {value: 0x0018, lo: 0x80, hi: 0x82}, {value: 0x0040, lo: 0x83, hi: 0x86}, {value: 0x0018, lo: 0x87, hi: 0xb3}, {value: 0x0040, lo: 0xb4, hi: 0xb6}, {value: 0x0018, lo: 0xb7, hi: 0xbf}, - // Block 0x8b, offset 0x461 + // Block 0x8a, offset 0x446 {value: 0x0000, lo: 0x06}, {value: 0x0018, lo: 0x80, hi: 0x8e}, {value: 0x0040, lo: 0x8f, hi: 0x8f}, @@ -3608,31 +3613,31 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0x9c, hi: 0x9f}, {value: 0x0018, lo: 0xa0, hi: 0xa0}, {value: 0x0040, lo: 0xa1, hi: 0xbf}, - // Block 0x8c, offset 0x468 + // Block 0x8b, offset 0x44d {value: 0x0000, lo: 0x04}, {value: 0x0040, lo: 0x80, hi: 0x8f}, {value: 0x0018, lo: 0x90, hi: 0xbc}, - {value: 0x1308, lo: 0xbd, hi: 0xbd}, + {value: 0x3308, lo: 0xbd, hi: 0xbd}, {value: 0x0040, lo: 0xbe, hi: 0xbf}, - // Block 0x8d, offset 0x46d + // Block 0x8c, offset 0x452 {value: 0x0000, lo: 0x03}, {value: 0x0008, lo: 0x80, hi: 0x9c}, {value: 0x0040, lo: 0x9d, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xbf}, - // Block 0x8e, offset 0x471 + // Block 0x8d, offset 0x456 {value: 0x0000, lo: 0x05}, {value: 0x0008, lo: 0x80, hi: 0x90}, {value: 0x0040, lo: 0x91, hi: 0x9f}, - {value: 0x1308, lo: 0xa0, hi: 0xa0}, + {value: 0x3308, lo: 0xa0, hi: 0xa0}, {value: 0x0018, lo: 0xa1, hi: 0xbb}, {value: 0x0040, lo: 0xbc, hi: 0xbf}, - // Block 0x8f, offset 0x477 + // Block 0x8e, offset 0x45c {value: 0x0000, lo: 0x04}, {value: 0x0008, lo: 0x80, hi: 0x9f}, {value: 0x0018, lo: 0xa0, hi: 0xa3}, - {value: 0x0040, lo: 0xa4, hi: 0xaf}, - {value: 0x0008, lo: 0xb0, hi: 0xbf}, - // Block 0x90, offset 0x47c + {value: 0x0040, lo: 0xa4, hi: 0xac}, + {value: 0x0008, lo: 0xad, hi: 0xbf}, + // Block 0x8f, offset 0x461 {value: 0x0000, lo: 0x08}, {value: 0x0008, lo: 0x80, hi: 0x80}, {value: 0x0018, lo: 0x81, hi: 0x81}, @@ -3640,22 +3645,22 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0018, lo: 0x8a, hi: 0x8a}, {value: 0x0040, lo: 0x8b, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0xb5}, - {value: 0x1308, lo: 0xb6, hi: 0xba}, + {value: 0x3308, lo: 0xb6, hi: 0xba}, {value: 0x0040, lo: 0xbb, hi: 0xbf}, - // Block 0x91, offset 0x485 + // Block 0x90, offset 0x46a {value: 0x0000, lo: 0x04}, {value: 0x0008, lo: 0x80, hi: 0x9d}, {value: 0x0040, lo: 0x9e, hi: 0x9e}, {value: 0x0018, lo: 0x9f, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xbf}, - // Block 0x92, offset 0x48a + // Block 0x91, offset 0x46f {value: 0x0000, lo: 0x05}, {value: 0x0008, lo: 0x80, hi: 0x83}, {value: 0x0040, lo: 0x84, hi: 0x87}, {value: 0x0008, lo: 0x88, hi: 0x8f}, {value: 0x0018, lo: 0x90, hi: 0x95}, {value: 0x0040, lo: 0x96, hi: 0xbf}, - // Block 0x93, offset 0x490 + // Block 0x92, offset 0x475 {value: 0x0000, lo: 0x06}, {value: 0xe145, lo: 0x80, hi: 0x87}, {value: 0xe1c5, lo: 0x88, hi: 0x8f}, @@ -3663,7 +3668,7 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x8ad5, lo: 0x98, hi: 0x9f}, {value: 0x8aed, lo: 0xa0, hi: 0xa7}, {value: 0x0008, lo: 0xa8, hi: 0xbf}, - // Block 0x94, offset 0x497 + // Block 0x93, offset 0x47c {value: 0x0000, lo: 0x06}, {value: 0x0008, lo: 0x80, hi: 0x9d}, {value: 0x0040, lo: 0x9e, hi: 0x9f}, @@ -3671,7 +3676,7 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0xaa, hi: 0xaf}, {value: 0x8aed, lo: 0xb0, hi: 0xb7}, {value: 0x8ad5, lo: 0xb8, hi: 0xbf}, - // Block 0x95, offset 0x49e + // Block 0x94, offset 0x483 {value: 0x0000, lo: 0x06}, {value: 0xe145, lo: 0x80, hi: 0x87}, {value: 0xe1c5, lo: 0x88, hi: 0x8f}, @@ -3679,173 +3684,176 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0x94, hi: 0x97}, {value: 0x0008, lo: 0x98, hi: 0xbb}, {value: 0x0040, lo: 0xbc, hi: 0xbf}, - // Block 0x96, offset 0x4a5 + // Block 0x95, offset 0x48a {value: 0x0000, lo: 0x03}, {value: 0x0008, lo: 0x80, hi: 0xa7}, {value: 0x0040, lo: 0xa8, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xbf}, - // Block 0x97, offset 0x4a9 + // Block 0x96, offset 0x48e {value: 0x0000, lo: 0x04}, {value: 0x0008, lo: 0x80, hi: 0xa3}, {value: 0x0040, lo: 0xa4, hi: 0xae}, {value: 0x0018, lo: 0xaf, hi: 0xaf}, {value: 0x0040, lo: 0xb0, hi: 0xbf}, - // Block 0x98, offset 0x4ae + // Block 0x97, offset 0x493 {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0xb6}, {value: 0x0040, lo: 0xb7, hi: 0xbf}, - // Block 0x99, offset 0x4b1 + // Block 0x98, offset 0x496 {value: 0x0000, lo: 0x04}, {value: 0x0008, lo: 0x80, hi: 0x95}, {value: 0x0040, lo: 0x96, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xa7}, {value: 0x0040, lo: 0xa8, hi: 0xbf}, - // Block 0x9a, offset 0x4b6 + // Block 0x99, offset 0x49b {value: 0x0000, lo: 0x0b}, - {value: 0x0008, lo: 0x80, hi: 0x85}, + {value: 0x0808, lo: 0x80, hi: 0x85}, {value: 0x0040, lo: 0x86, hi: 0x87}, - {value: 0x0008, lo: 0x88, hi: 0x88}, + {value: 0x0808, lo: 0x88, hi: 0x88}, {value: 0x0040, lo: 0x89, hi: 0x89}, - {value: 0x0008, lo: 0x8a, hi: 0xb5}, + {value: 0x0808, lo: 0x8a, hi: 0xb5}, {value: 0x0040, lo: 0xb6, hi: 0xb6}, - {value: 0x0008, lo: 0xb7, hi: 0xb8}, + {value: 0x0808, lo: 0xb7, hi: 0xb8}, {value: 0x0040, lo: 0xb9, hi: 0xbb}, - {value: 0x0008, lo: 0xbc, hi: 0xbc}, + {value: 0x0808, lo: 0xbc, hi: 0xbc}, {value: 0x0040, lo: 0xbd, hi: 0xbe}, - {value: 0x0008, lo: 0xbf, hi: 0xbf}, - // Block 0x9b, offset 0x4c2 + {value: 0x0808, lo: 0xbf, hi: 0xbf}, + // Block 0x9a, offset 0x4a7 {value: 0x0000, lo: 0x05}, - {value: 0x0008, lo: 0x80, hi: 0x95}, + {value: 0x0808, lo: 0x80, hi: 0x95}, {value: 0x0040, lo: 0x96, hi: 0x96}, - {value: 0x0018, lo: 0x97, hi: 0x9f}, - {value: 0x0008, lo: 0xa0, hi: 0xb6}, - {value: 0x0018, lo: 0xb7, hi: 0xbf}, - // Block 0x9c, offset 0x4c8 + {value: 0x0818, lo: 0x97, hi: 0x9f}, + {value: 0x0808, lo: 0xa0, hi: 0xb6}, + {value: 0x0818, lo: 0xb7, hi: 0xbf}, + // Block 0x9b, offset 0x4ad {value: 0x0000, lo: 0x04}, - {value: 0x0008, lo: 0x80, hi: 0x9e}, + {value: 0x0808, lo: 0x80, hi: 0x9e}, {value: 0x0040, lo: 0x9f, hi: 0xa6}, - {value: 0x0018, lo: 0xa7, hi: 0xaf}, + {value: 0x0818, lo: 0xa7, hi: 0xaf}, {value: 0x0040, lo: 0xb0, hi: 0xbf}, - // Block 0x9d, offset 0x4cd + // Block 0x9c, offset 0x4b2 {value: 0x0000, lo: 0x06}, {value: 0x0040, lo: 0x80, hi: 0x9f}, - {value: 0x0008, lo: 0xa0, hi: 0xb2}, + {value: 0x0808, lo: 0xa0, hi: 0xb2}, {value: 0x0040, lo: 0xb3, hi: 0xb3}, - {value: 0x0008, lo: 0xb4, hi: 0xb5}, + {value: 0x0808, lo: 0xb4, hi: 0xb5}, {value: 0x0040, lo: 0xb6, hi: 0xba}, - {value: 0x0018, lo: 0xbb, hi: 0xbf}, - // Block 0x9e, offset 0x4d4 + {value: 0x0818, lo: 0xbb, hi: 0xbf}, + // Block 0x9d, offset 0x4b9 {value: 0x0000, lo: 0x07}, - {value: 0x0008, lo: 0x80, hi: 0x95}, - {value: 0x0018, lo: 0x96, hi: 0x9b}, + {value: 0x0808, lo: 0x80, hi: 0x95}, + {value: 0x0818, lo: 0x96, hi: 0x9b}, {value: 0x0040, lo: 0x9c, hi: 0x9e}, {value: 0x0018, lo: 0x9f, hi: 0x9f}, - {value: 0x0008, lo: 0xa0, hi: 0xb9}, + {value: 0x0808, lo: 0xa0, hi: 0xb9}, {value: 0x0040, lo: 0xba, hi: 0xbe}, - {value: 0x0018, lo: 0xbf, hi: 0xbf}, - // Block 0x9f, offset 0x4dc + {value: 0x0818, lo: 0xbf, hi: 0xbf}, + // Block 0x9e, offset 0x4c1 {value: 0x0000, lo: 0x04}, - {value: 0x0008, lo: 0x80, hi: 0xb7}, + {value: 0x0808, lo: 0x80, hi: 0xb7}, {value: 0x0040, lo: 0xb8, hi: 0xbb}, - {value: 0x0018, lo: 0xbc, hi: 0xbd}, - {value: 0x0008, lo: 0xbe, hi: 0xbf}, - // Block 0xa0, offset 0x4e1 + {value: 0x0818, lo: 0xbc, hi: 0xbd}, + {value: 0x0808, lo: 0xbe, hi: 0xbf}, + // Block 0x9f, offset 0x4c6 {value: 0x0000, lo: 0x03}, - {value: 0x0018, lo: 0x80, hi: 0x8f}, + {value: 0x0818, lo: 0x80, hi: 0x8f}, {value: 0x0040, lo: 0x90, hi: 0x91}, - {value: 0x0018, lo: 0x92, hi: 0xbf}, - // Block 0xa1, offset 0x4e5 + {value: 0x0818, lo: 0x92, hi: 0xbf}, + // Block 0xa0, offset 0x4ca {value: 0x0000, lo: 0x0f}, - {value: 0x0008, lo: 0x80, hi: 0x80}, - {value: 0x1308, lo: 0x81, hi: 0x83}, + {value: 0x0808, lo: 0x80, hi: 0x80}, + {value: 0x3308, lo: 0x81, hi: 0x83}, {value: 0x0040, lo: 0x84, hi: 0x84}, - {value: 0x1308, lo: 0x85, hi: 0x86}, + {value: 0x3308, lo: 0x85, hi: 0x86}, {value: 0x0040, lo: 0x87, hi: 0x8b}, - {value: 0x1308, lo: 0x8c, hi: 0x8f}, - {value: 0x0008, lo: 0x90, hi: 0x93}, + {value: 0x3308, lo: 0x8c, hi: 0x8f}, + {value: 0x0808, lo: 0x90, hi: 0x93}, {value: 0x0040, lo: 0x94, hi: 0x94}, - {value: 0x0008, lo: 0x95, hi: 0x97}, + {value: 0x0808, lo: 0x95, hi: 0x97}, {value: 0x0040, lo: 0x98, hi: 0x98}, - {value: 0x0008, lo: 0x99, hi: 0xb3}, + {value: 0x0808, lo: 0x99, hi: 0xb3}, {value: 0x0040, lo: 0xb4, hi: 0xb7}, - {value: 0x1308, lo: 0xb8, hi: 0xba}, + {value: 0x3308, lo: 0xb8, hi: 0xba}, {value: 0x0040, lo: 0xbb, hi: 0xbe}, - {value: 0x1b08, lo: 0xbf, hi: 0xbf}, - // Block 0xa2, offset 0x4f5 + {value: 0x3b08, lo: 0xbf, hi: 0xbf}, + // Block 0xa1, offset 0x4da {value: 0x0000, lo: 0x06}, - {value: 0x0018, lo: 0x80, hi: 0x87}, + {value: 0x0818, lo: 0x80, hi: 0x87}, {value: 0x0040, lo: 0x88, hi: 0x8f}, - {value: 0x0018, lo: 0x90, hi: 0x98}, + {value: 0x0818, lo: 0x90, hi: 0x98}, {value: 0x0040, lo: 0x99, hi: 0x9f}, - {value: 0x0008, lo: 0xa0, hi: 0xbc}, - {value: 0x0018, lo: 0xbd, hi: 0xbf}, - // Block 0xa3, offset 0x4fc + {value: 0x0808, lo: 0xa0, hi: 0xbc}, + {value: 0x0818, lo: 0xbd, hi: 0xbf}, + // Block 0xa2, offset 0x4e1 {value: 0x0000, lo: 0x03}, - {value: 0x0008, lo: 0x80, hi: 0x9c}, - {value: 0x0018, lo: 0x9d, hi: 0x9f}, + {value: 0x0808, lo: 0x80, hi: 0x9c}, + {value: 0x0818, lo: 0x9d, hi: 0x9f}, {value: 0x0040, lo: 0xa0, hi: 0xbf}, - // Block 0xa4, offset 0x500 + // Block 0xa3, offset 0x4e5 {value: 0x0000, lo: 0x03}, - {value: 0x0008, lo: 0x80, hi: 0xb5}, + {value: 0x0808, lo: 0x80, hi: 0xb5}, {value: 0x0040, lo: 0xb6, hi: 0xb8}, {value: 0x0018, lo: 0xb9, hi: 0xbf}, - // Block 0xa5, offset 0x504 + // Block 0xa4, offset 0x4e9 {value: 0x0000, lo: 0x06}, - {value: 0x0008, lo: 0x80, hi: 0x95}, + {value: 0x0808, lo: 0x80, hi: 0x95}, {value: 0x0040, lo: 0x96, hi: 0x97}, - {value: 0x0018, lo: 0x98, hi: 0x9f}, - {value: 0x0008, lo: 0xa0, hi: 0xb2}, + {value: 0x0818, lo: 0x98, hi: 0x9f}, + {value: 0x0808, lo: 0xa0, hi: 0xb2}, {value: 0x0040, lo: 0xb3, hi: 0xb7}, - {value: 0x0018, lo: 0xb8, hi: 0xbf}, - // Block 0xa6, offset 0x50b + {value: 0x0818, lo: 0xb8, hi: 0xbf}, + // Block 0xa5, offset 0x4f0 + {value: 0x0000, lo: 0x01}, + {value: 0x0808, lo: 0x80, hi: 0xbf}, + // Block 0xa6, offset 0x4f2 {value: 0x0000, lo: 0x02}, - {value: 0x0008, lo: 0x80, hi: 0x88}, + {value: 0x0808, lo: 0x80, hi: 0x88}, {value: 0x0040, lo: 0x89, hi: 0xbf}, - // Block 0xa7, offset 0x50e + // Block 0xa7, offset 0x4f5 {value: 0x0000, lo: 0x02}, {value: 0x03dd, lo: 0x80, hi: 0xb2}, {value: 0x0040, lo: 0xb3, hi: 0xbf}, - // Block 0xa8, offset 0x511 + // Block 0xa8, offset 0x4f8 {value: 0x0000, lo: 0x03}, - {value: 0x0008, lo: 0x80, hi: 0xb2}, + {value: 0x0808, lo: 0x80, hi: 0xb2}, {value: 0x0040, lo: 0xb3, hi: 0xb9}, - {value: 0x0018, lo: 0xba, hi: 0xbf}, - // Block 0xa9, offset 0x515 + {value: 0x0818, lo: 0xba, hi: 0xbf}, + // Block 0xa9, offset 0x4fc {value: 0x0000, lo: 0x03}, {value: 0x0040, lo: 0x80, hi: 0x9f}, - {value: 0x0018, lo: 0xa0, hi: 0xbe}, + {value: 0x0818, lo: 0xa0, hi: 0xbe}, {value: 0x0040, lo: 0xbf, hi: 0xbf}, - // Block 0xaa, offset 0x519 + // Block 0xaa, offset 0x500 {value: 0x0000, lo: 0x05}, - {value: 0x1008, lo: 0x80, hi: 0x80}, - {value: 0x1308, lo: 0x81, hi: 0x81}, - {value: 0x1008, lo: 0x82, hi: 0x82}, + {value: 0x3008, lo: 0x80, hi: 0x80}, + {value: 0x3308, lo: 0x81, hi: 0x81}, + {value: 0x3008, lo: 0x82, hi: 0x82}, {value: 0x0008, lo: 0x83, hi: 0xb7}, - {value: 0x1308, lo: 0xb8, hi: 0xbf}, - // Block 0xab, offset 0x51f + {value: 0x3308, lo: 0xb8, hi: 0xbf}, + // Block 0xab, offset 0x506 {value: 0x0000, lo: 0x08}, - {value: 0x1308, lo: 0x80, hi: 0x85}, - {value: 0x1b08, lo: 0x86, hi: 0x86}, + {value: 0x3308, lo: 0x80, hi: 0x85}, + {value: 0x3b08, lo: 0x86, hi: 0x86}, {value: 0x0018, lo: 0x87, hi: 0x8d}, {value: 0x0040, lo: 0x8e, hi: 0x91}, {value: 0x0018, lo: 0x92, hi: 0xa5}, {value: 0x0008, lo: 0xa6, hi: 0xaf}, {value: 0x0040, lo: 0xb0, hi: 0xbe}, - {value: 0x1b08, lo: 0xbf, hi: 0xbf}, - // Block 0xac, offset 0x528 + {value: 0x3b08, lo: 0xbf, hi: 0xbf}, + // Block 0xac, offset 0x50f {value: 0x0000, lo: 0x0b}, - {value: 0x1308, lo: 0x80, hi: 0x81}, - {value: 0x1008, lo: 0x82, hi: 0x82}, + {value: 0x3308, lo: 0x80, hi: 0x81}, + {value: 0x3008, lo: 0x82, hi: 0x82}, {value: 0x0008, lo: 0x83, hi: 0xaf}, - {value: 0x1008, lo: 0xb0, hi: 0xb2}, - {value: 0x1308, lo: 0xb3, hi: 0xb6}, - {value: 0x1008, lo: 0xb7, hi: 0xb8}, - {value: 0x1b08, lo: 0xb9, hi: 0xb9}, - {value: 0x1308, lo: 0xba, hi: 0xba}, + {value: 0x3008, lo: 0xb0, hi: 0xb2}, + {value: 0x3308, lo: 0xb3, hi: 0xb6}, + {value: 0x3008, lo: 0xb7, hi: 0xb8}, + {value: 0x3b08, lo: 0xb9, hi: 0xb9}, + {value: 0x3308, lo: 0xba, hi: 0xba}, {value: 0x0018, lo: 0xbb, hi: 0xbc}, {value: 0x0340, lo: 0xbd, hi: 0xbd}, {value: 0x0018, lo: 0xbe, hi: 0xbf}, - // Block 0xad, offset 0x534 + // Block 0xad, offset 0x51b {value: 0x0000, lo: 0x06}, {value: 0x0018, lo: 0x80, hi: 0x81}, {value: 0x0040, lo: 0x82, hi: 0x8f}, @@ -3853,39 +3861,39 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0xa9, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xb9}, {value: 0x0040, lo: 0xba, hi: 0xbf}, - // Block 0xae, offset 0x53b + // Block 0xae, offset 0x522 {value: 0x0000, lo: 0x08}, - {value: 0x1308, lo: 0x80, hi: 0x82}, + {value: 0x3308, lo: 0x80, hi: 0x82}, {value: 0x0008, lo: 0x83, hi: 0xa6}, - {value: 0x1308, lo: 0xa7, hi: 0xab}, - {value: 0x1008, lo: 0xac, hi: 0xac}, - {value: 0x1308, lo: 0xad, hi: 0xb2}, - {value: 0x1b08, lo: 0xb3, hi: 0xb4}, + {value: 0x3308, lo: 0xa7, hi: 0xab}, + {value: 0x3008, lo: 0xac, hi: 0xac}, + {value: 0x3308, lo: 0xad, hi: 0xb2}, + {value: 0x3b08, lo: 0xb3, hi: 0xb4}, {value: 0x0040, lo: 0xb5, hi: 0xb5}, {value: 0x0008, lo: 0xb6, hi: 0xbf}, - // Block 0xaf, offset 0x544 + // Block 0xaf, offset 0x52b {value: 0x0000, lo: 0x07}, {value: 0x0018, lo: 0x80, hi: 0x83}, {value: 0x0040, lo: 0x84, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0xb2}, - {value: 0x1308, lo: 0xb3, hi: 0xb3}, + {value: 0x3308, lo: 0xb3, hi: 0xb3}, {value: 0x0018, lo: 0xb4, hi: 0xb5}, {value: 0x0008, lo: 0xb6, hi: 0xb6}, {value: 0x0040, lo: 0xb7, hi: 0xbf}, - // Block 0xb0, offset 0x54c + // Block 0xb0, offset 0x533 {value: 0x0000, lo: 0x06}, - {value: 0x1308, lo: 0x80, hi: 0x81}, - {value: 0x1008, lo: 0x82, hi: 0x82}, + {value: 0x3308, lo: 0x80, hi: 0x81}, + {value: 0x3008, lo: 0x82, hi: 0x82}, {value: 0x0008, lo: 0x83, hi: 0xb2}, - {value: 0x1008, lo: 0xb3, hi: 0xb5}, - {value: 0x1308, lo: 0xb6, hi: 0xbe}, - {value: 0x1008, lo: 0xbf, hi: 0xbf}, - // Block 0xb1, offset 0x553 + {value: 0x3008, lo: 0xb3, hi: 0xb5}, + {value: 0x3308, lo: 0xb6, hi: 0xbe}, + {value: 0x3008, lo: 0xbf, hi: 0xbf}, + // Block 0xb1, offset 0x53a {value: 0x0000, lo: 0x0d}, - {value: 0x1808, lo: 0x80, hi: 0x80}, + {value: 0x3808, lo: 0x80, hi: 0x80}, {value: 0x0008, lo: 0x81, hi: 0x84}, {value: 0x0018, lo: 0x85, hi: 0x89}, - {value: 0x1308, lo: 0x8a, hi: 0x8c}, + {value: 0x3308, lo: 0x8a, hi: 0x8c}, {value: 0x0018, lo: 0x8d, hi: 0x8d}, {value: 0x0040, lo: 0x8e, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x9a}, @@ -3895,21 +3903,21 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0xa0, hi: 0xa0}, {value: 0x0018, lo: 0xa1, hi: 0xb4}, {value: 0x0040, lo: 0xb5, hi: 0xbf}, - // Block 0xb2, offset 0x561 + // Block 0xb2, offset 0x548 {value: 0x0000, lo: 0x0c}, {value: 0x0008, lo: 0x80, hi: 0x91}, {value: 0x0040, lo: 0x92, hi: 0x92}, {value: 0x0008, lo: 0x93, hi: 0xab}, - {value: 0x1008, lo: 0xac, hi: 0xae}, - {value: 0x1308, lo: 0xaf, hi: 0xb1}, - {value: 0x1008, lo: 0xb2, hi: 0xb3}, - {value: 0x1308, lo: 0xb4, hi: 0xb4}, - {value: 0x1808, lo: 0xb5, hi: 0xb5}, - {value: 0x1308, lo: 0xb6, hi: 0xb7}, + {value: 0x3008, lo: 0xac, hi: 0xae}, + {value: 0x3308, lo: 0xaf, hi: 0xb1}, + {value: 0x3008, lo: 0xb2, hi: 0xb3}, + {value: 0x3308, lo: 0xb4, hi: 0xb4}, + {value: 0x3808, lo: 0xb5, hi: 0xb5}, + {value: 0x3308, lo: 0xb6, hi: 0xb7}, {value: 0x0018, lo: 0xb8, hi: 0xbd}, - {value: 0x1308, lo: 0xbe, hi: 0xbe}, + {value: 0x3308, lo: 0xbe, hi: 0xbe}, {value: 0x0040, lo: 0xbf, hi: 0xbf}, - // Block 0xb3, offset 0x56e + // Block 0xb3, offset 0x555 {value: 0x0000, lo: 0x0c}, {value: 0x0008, lo: 0x80, hi: 0x86}, {value: 0x0040, lo: 0x87, hi: 0x87}, @@ -3923,28 +3931,28 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0018, lo: 0xa9, hi: 0xa9}, {value: 0x0040, lo: 0xaa, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xbf}, - // Block 0xb4, offset 0x57b + // Block 0xb4, offset 0x562 {value: 0x0000, lo: 0x08}, {value: 0x0008, lo: 0x80, hi: 0x9e}, - {value: 0x1308, lo: 0x9f, hi: 0x9f}, - {value: 0x1008, lo: 0xa0, hi: 0xa2}, - {value: 0x1308, lo: 0xa3, hi: 0xa9}, - {value: 0x1b08, lo: 0xaa, hi: 0xaa}, + {value: 0x3308, lo: 0x9f, hi: 0x9f}, + {value: 0x3008, lo: 0xa0, hi: 0xa2}, + {value: 0x3308, lo: 0xa3, hi: 0xa9}, + {value: 0x3b08, lo: 0xaa, hi: 0xaa}, {value: 0x0040, lo: 0xab, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xb9}, {value: 0x0040, lo: 0xba, hi: 0xbf}, - // Block 0xb5, offset 0x584 + // Block 0xb5, offset 0x56b {value: 0x0000, lo: 0x03}, {value: 0x0008, lo: 0x80, hi: 0xb4}, - {value: 0x1008, lo: 0xb5, hi: 0xb7}, - {value: 0x1308, lo: 0xb8, hi: 0xbf}, - // Block 0xb6, offset 0x588 + {value: 0x3008, lo: 0xb5, hi: 0xb7}, + {value: 0x3308, lo: 0xb8, hi: 0xbf}, + // Block 0xb6, offset 0x56f {value: 0x0000, lo: 0x0d}, - {value: 0x1008, lo: 0x80, hi: 0x81}, - {value: 0x1b08, lo: 0x82, hi: 0x82}, - {value: 0x1308, lo: 0x83, hi: 0x84}, - {value: 0x1008, lo: 0x85, hi: 0x85}, - {value: 0x1308, lo: 0x86, hi: 0x86}, + {value: 0x3008, lo: 0x80, hi: 0x81}, + {value: 0x3b08, lo: 0x82, hi: 0x82}, + {value: 0x3308, lo: 0x83, hi: 0x84}, + {value: 0x3008, lo: 0x85, hi: 0x85}, + {value: 0x3308, lo: 0x86, hi: 0x86}, {value: 0x0008, lo: 0x87, hi: 0x8a}, {value: 0x0018, lo: 0x8b, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x99}, @@ -3953,56 +3961,56 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0x9c, hi: 0x9c}, {value: 0x0018, lo: 0x9d, hi: 0x9d}, {value: 0x0040, lo: 0x9e, hi: 0xbf}, - // Block 0xb7, offset 0x596 + // Block 0xb7, offset 0x57d {value: 0x0000, lo: 0x07}, {value: 0x0008, lo: 0x80, hi: 0xaf}, - {value: 0x1008, lo: 0xb0, hi: 0xb2}, - {value: 0x1308, lo: 0xb3, hi: 0xb8}, - {value: 0x1008, lo: 0xb9, hi: 0xb9}, - {value: 0x1308, lo: 0xba, hi: 0xba}, - {value: 0x1008, lo: 0xbb, hi: 0xbe}, - {value: 0x1308, lo: 0xbf, hi: 0xbf}, - // Block 0xb8, offset 0x59e + {value: 0x3008, lo: 0xb0, hi: 0xb2}, + {value: 0x3308, lo: 0xb3, hi: 0xb8}, + {value: 0x3008, lo: 0xb9, hi: 0xb9}, + {value: 0x3308, lo: 0xba, hi: 0xba}, + {value: 0x3008, lo: 0xbb, hi: 0xbe}, + {value: 0x3308, lo: 0xbf, hi: 0xbf}, + // Block 0xb8, offset 0x585 {value: 0x0000, lo: 0x0a}, - {value: 0x1308, lo: 0x80, hi: 0x80}, - {value: 0x1008, lo: 0x81, hi: 0x81}, - {value: 0x1b08, lo: 0x82, hi: 0x82}, - {value: 0x1308, lo: 0x83, hi: 0x83}, + {value: 0x3308, lo: 0x80, hi: 0x80}, + {value: 0x3008, lo: 0x81, hi: 0x81}, + {value: 0x3b08, lo: 0x82, hi: 0x82}, + {value: 0x3308, lo: 0x83, hi: 0x83}, {value: 0x0008, lo: 0x84, hi: 0x85}, {value: 0x0018, lo: 0x86, hi: 0x86}, {value: 0x0008, lo: 0x87, hi: 0x87}, {value: 0x0040, lo: 0x88, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x99}, {value: 0x0040, lo: 0x9a, hi: 0xbf}, - // Block 0xb9, offset 0x5a9 + // Block 0xb9, offset 0x590 {value: 0x0000, lo: 0x08}, {value: 0x0008, lo: 0x80, hi: 0xae}, - {value: 0x1008, lo: 0xaf, hi: 0xb1}, - {value: 0x1308, lo: 0xb2, hi: 0xb5}, + {value: 0x3008, lo: 0xaf, hi: 0xb1}, + {value: 0x3308, lo: 0xb2, hi: 0xb5}, {value: 0x0040, lo: 0xb6, hi: 0xb7}, - {value: 0x1008, lo: 0xb8, hi: 0xbb}, - {value: 0x1308, lo: 0xbc, hi: 0xbd}, - {value: 0x1008, lo: 0xbe, hi: 0xbe}, - {value: 0x1b08, lo: 0xbf, hi: 0xbf}, - // Block 0xba, offset 0x5b2 + {value: 0x3008, lo: 0xb8, hi: 0xbb}, + {value: 0x3308, lo: 0xbc, hi: 0xbd}, + {value: 0x3008, lo: 0xbe, hi: 0xbe}, + {value: 0x3b08, lo: 0xbf, hi: 0xbf}, + // Block 0xba, offset 0x599 {value: 0x0000, lo: 0x05}, - {value: 0x1308, lo: 0x80, hi: 0x80}, + {value: 0x3308, lo: 0x80, hi: 0x80}, {value: 0x0018, lo: 0x81, hi: 0x97}, {value: 0x0008, lo: 0x98, hi: 0x9b}, - {value: 0x1308, lo: 0x9c, hi: 0x9d}, + {value: 0x3308, lo: 0x9c, hi: 0x9d}, {value: 0x0040, lo: 0x9e, hi: 0xbf}, - // Block 0xbb, offset 0x5b8 + // Block 0xbb, offset 0x59f {value: 0x0000, lo: 0x07}, {value: 0x0008, lo: 0x80, hi: 0xaf}, - {value: 0x1008, lo: 0xb0, hi: 0xb2}, - {value: 0x1308, lo: 0xb3, hi: 0xba}, - {value: 0x1008, lo: 0xbb, hi: 0xbc}, - {value: 0x1308, lo: 0xbd, hi: 0xbd}, - {value: 0x1008, lo: 0xbe, hi: 0xbe}, - {value: 0x1b08, lo: 0xbf, hi: 0xbf}, - // Block 0xbc, offset 0x5c0 + {value: 0x3008, lo: 0xb0, hi: 0xb2}, + {value: 0x3308, lo: 0xb3, hi: 0xba}, + {value: 0x3008, lo: 0xbb, hi: 0xbc}, + {value: 0x3308, lo: 0xbd, hi: 0xbd}, + {value: 0x3008, lo: 0xbe, hi: 0xbe}, + {value: 0x3b08, lo: 0xbf, hi: 0xbf}, + // Block 0xbc, offset 0x5a7 {value: 0x0000, lo: 0x08}, - {value: 0x1308, lo: 0x80, hi: 0x80}, + {value: 0x3308, lo: 0x80, hi: 0x80}, {value: 0x0018, lo: 0x81, hi: 0x83}, {value: 0x0008, lo: 0x84, hi: 0x84}, {value: 0x0040, lo: 0x85, hi: 0x8f}, @@ -4010,60 +4018,97 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0x9a, hi: 0x9f}, {value: 0x0018, lo: 0xa0, hi: 0xac}, {value: 0x0040, lo: 0xad, hi: 0xbf}, - // Block 0xbd, offset 0x5c9 + // Block 0xbd, offset 0x5b0 {value: 0x0000, lo: 0x09}, {value: 0x0008, lo: 0x80, hi: 0xaa}, - {value: 0x1308, lo: 0xab, hi: 0xab}, - {value: 0x1008, lo: 0xac, hi: 0xac}, - {value: 0x1308, lo: 0xad, hi: 0xad}, - {value: 0x1008, lo: 0xae, hi: 0xaf}, - {value: 0x1308, lo: 0xb0, hi: 0xb5}, - {value: 0x1808, lo: 0xb6, hi: 0xb6}, - {value: 0x1308, lo: 0xb7, hi: 0xb7}, + {value: 0x3308, lo: 0xab, hi: 0xab}, + {value: 0x3008, lo: 0xac, hi: 0xac}, + {value: 0x3308, lo: 0xad, hi: 0xad}, + {value: 0x3008, lo: 0xae, hi: 0xaf}, + {value: 0x3308, lo: 0xb0, hi: 0xb5}, + {value: 0x3808, lo: 0xb6, hi: 0xb6}, + {value: 0x3308, lo: 0xb7, hi: 0xb7}, {value: 0x0040, lo: 0xb8, hi: 0xbf}, - // Block 0xbe, offset 0x5d3 + // Block 0xbe, offset 0x5ba {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0x89}, {value: 0x0040, lo: 0x8a, hi: 0xbf}, - // Block 0xbf, offset 0x5d6 + // Block 0xbf, offset 0x5bd {value: 0x0000, lo: 0x0b}, {value: 0x0008, lo: 0x80, hi: 0x99}, {value: 0x0040, lo: 0x9a, hi: 0x9c}, - {value: 0x1308, lo: 0x9d, hi: 0x9f}, - {value: 0x1008, lo: 0xa0, hi: 0xa1}, - {value: 0x1308, lo: 0xa2, hi: 0xa5}, - {value: 0x1008, lo: 0xa6, hi: 0xa6}, - {value: 0x1308, lo: 0xa7, hi: 0xaa}, - {value: 0x1b08, lo: 0xab, hi: 0xab}, + {value: 0x3308, lo: 0x9d, hi: 0x9f}, + {value: 0x3008, lo: 0xa0, hi: 0xa1}, + {value: 0x3308, lo: 0xa2, hi: 0xa5}, + {value: 0x3008, lo: 0xa6, hi: 0xa6}, + {value: 0x3308, lo: 0xa7, hi: 0xaa}, + {value: 0x3b08, lo: 0xab, hi: 0xab}, {value: 0x0040, lo: 0xac, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xb9}, {value: 0x0018, lo: 0xba, hi: 0xbf}, - // Block 0xc0, offset 0x5e2 + // Block 0xc0, offset 0x5c9 {value: 0x0000, lo: 0x02}, {value: 0x0040, lo: 0x80, hi: 0x9f}, {value: 0x049d, lo: 0xa0, hi: 0xbf}, - // Block 0xc1, offset 0x5e5 + // Block 0xc1, offset 0x5cc {value: 0x0000, lo: 0x04}, {value: 0x0008, lo: 0x80, hi: 0xa9}, {value: 0x0018, lo: 0xaa, hi: 0xb2}, {value: 0x0040, lo: 0xb3, hi: 0xbe}, {value: 0x0008, lo: 0xbf, hi: 0xbf}, - // Block 0xc2, offset 0x5ea + // Block 0xc2, offset 0x5d1 + {value: 0x0000, lo: 0x0c}, + {value: 0x0008, lo: 0x80, hi: 0x80}, + {value: 0x3308, lo: 0x81, hi: 0x86}, + {value: 0x3008, lo: 0x87, hi: 0x88}, + {value: 0x3308, lo: 0x89, hi: 0x8a}, + {value: 0x0008, lo: 0x8b, hi: 0xb2}, + {value: 0x3308, lo: 0xb3, hi: 0xb3}, + {value: 0x3b08, lo: 0xb4, hi: 0xb4}, + {value: 0x3308, lo: 0xb5, hi: 0xb8}, + {value: 0x3008, lo: 0xb9, hi: 0xb9}, + {value: 0x0008, lo: 0xba, hi: 0xba}, + {value: 0x3308, lo: 0xbb, hi: 0xbe}, + {value: 0x0018, lo: 0xbf, hi: 0xbf}, + // Block 0xc3, offset 0x5de + {value: 0x0000, lo: 0x08}, + {value: 0x0018, lo: 0x80, hi: 0x86}, + {value: 0x3b08, lo: 0x87, hi: 0x87}, + {value: 0x0040, lo: 0x88, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x90}, + {value: 0x3308, lo: 0x91, hi: 0x96}, + {value: 0x3008, lo: 0x97, hi: 0x98}, + {value: 0x3308, lo: 0x99, hi: 0x9b}, + {value: 0x0008, lo: 0x9c, hi: 0xbf}, + // Block 0xc4, offset 0x5e7 + {value: 0x0000, lo: 0x0b}, + {value: 0x0008, lo: 0x80, hi: 0x83}, + {value: 0x0040, lo: 0x84, hi: 0x85}, + {value: 0x0008, lo: 0x86, hi: 0x89}, + {value: 0x3308, lo: 0x8a, hi: 0x96}, + {value: 0x3008, lo: 0x97, hi: 0x97}, + {value: 0x3308, lo: 0x98, hi: 0x98}, + {value: 0x3b08, lo: 0x99, hi: 0x99}, + {value: 0x0018, lo: 0x9a, hi: 0x9c}, + {value: 0x0040, lo: 0x9d, hi: 0x9d}, + {value: 0x0018, lo: 0x9e, hi: 0xa2}, + {value: 0x0040, lo: 0xa3, hi: 0xbf}, + // Block 0xc5, offset 0x5f3 {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0xb8}, {value: 0x0040, lo: 0xb9, hi: 0xbf}, - // Block 0xc3, offset 0x5ed + // Block 0xc6, offset 0x5f6 {value: 0x0000, lo: 0x09}, {value: 0x0008, lo: 0x80, hi: 0x88}, {value: 0x0040, lo: 0x89, hi: 0x89}, {value: 0x0008, lo: 0x8a, hi: 0xae}, - {value: 0x1008, lo: 0xaf, hi: 0xaf}, - {value: 0x1308, lo: 0xb0, hi: 0xb6}, + {value: 0x3008, lo: 0xaf, hi: 0xaf}, + {value: 0x3308, lo: 0xb0, hi: 0xb6}, {value: 0x0040, lo: 0xb7, hi: 0xb7}, - {value: 0x1308, lo: 0xb8, hi: 0xbd}, - {value: 0x1008, lo: 0xbe, hi: 0xbe}, - {value: 0x1b08, lo: 0xbf, hi: 0xbf}, - // Block 0xc4, offset 0x5f7 + {value: 0x3308, lo: 0xb8, hi: 0xbd}, + {value: 0x3008, lo: 0xbe, hi: 0xbe}, + {value: 0x3b08, lo: 0xbf, hi: 0xbf}, + // Block 0xc7, offset 0x600 {value: 0x0000, lo: 0x08}, {value: 0x0008, lo: 0x80, hi: 0x80}, {value: 0x0018, lo: 0x81, hi: 0x85}, @@ -4073,42 +4118,65 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0xad, hi: 0xaf}, {value: 0x0018, lo: 0xb0, hi: 0xb1}, {value: 0x0008, lo: 0xb2, hi: 0xbf}, - // Block 0xc5, offset 0x600 + // Block 0xc8, offset 0x609 {value: 0x0000, lo: 0x0b}, {value: 0x0008, lo: 0x80, hi: 0x8f}, {value: 0x0040, lo: 0x90, hi: 0x91}, - {value: 0x1308, lo: 0x92, hi: 0xa7}, + {value: 0x3308, lo: 0x92, hi: 0xa7}, {value: 0x0040, lo: 0xa8, hi: 0xa8}, - {value: 0x1008, lo: 0xa9, hi: 0xa9}, - {value: 0x1308, lo: 0xaa, hi: 0xb0}, - {value: 0x1008, lo: 0xb1, hi: 0xb1}, - {value: 0x1308, lo: 0xb2, hi: 0xb3}, - {value: 0x1008, lo: 0xb4, hi: 0xb4}, - {value: 0x1308, lo: 0xb5, hi: 0xb6}, + {value: 0x3008, lo: 0xa9, hi: 0xa9}, + {value: 0x3308, lo: 0xaa, hi: 0xb0}, + {value: 0x3008, lo: 0xb1, hi: 0xb1}, + {value: 0x3308, lo: 0xb2, hi: 0xb3}, + {value: 0x3008, lo: 0xb4, hi: 0xb4}, + {value: 0x3308, lo: 0xb5, hi: 0xb6}, {value: 0x0040, lo: 0xb7, hi: 0xbf}, - // Block 0xc6, offset 0x60c + // Block 0xc9, offset 0x615 + {value: 0x0000, lo: 0x0c}, + {value: 0x0008, lo: 0x80, hi: 0x86}, + {value: 0x0040, lo: 0x87, hi: 0x87}, + {value: 0x0008, lo: 0x88, hi: 0x89}, + {value: 0x0040, lo: 0x8a, hi: 0x8a}, + {value: 0x0008, lo: 0x8b, hi: 0xb0}, + {value: 0x3308, lo: 0xb1, hi: 0xb6}, + {value: 0x0040, lo: 0xb7, hi: 0xb9}, + {value: 0x3308, lo: 0xba, hi: 0xba}, + {value: 0x0040, lo: 0xbb, hi: 0xbb}, + {value: 0x3308, lo: 0xbc, hi: 0xbd}, + {value: 0x0040, lo: 0xbe, hi: 0xbe}, + {value: 0x3308, lo: 0xbf, hi: 0xbf}, + // Block 0xca, offset 0x622 + {value: 0x0000, lo: 0x07}, + {value: 0x3308, lo: 0x80, hi: 0x83}, + {value: 0x3b08, lo: 0x84, hi: 0x85}, + {value: 0x0008, lo: 0x86, hi: 0x86}, + {value: 0x3308, lo: 0x87, hi: 0x87}, + {value: 0x0040, lo: 0x88, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0xbf}, + // Block 0xcb, offset 0x62a {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0x99}, {value: 0x0040, lo: 0x9a, hi: 0xbf}, - // Block 0xc7, offset 0x60f + // Block 0xcc, offset 0x62d {value: 0x0000, lo: 0x04}, {value: 0x0018, lo: 0x80, hi: 0xae}, {value: 0x0040, lo: 0xaf, hi: 0xaf}, {value: 0x0018, lo: 0xb0, hi: 0xb4}, {value: 0x0040, lo: 0xb5, hi: 0xbf}, - // Block 0xc8, offset 0x614 + // Block 0xcd, offset 0x632 {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0x83}, {value: 0x0040, lo: 0x84, hi: 0xbf}, - // Block 0xc9, offset 0x617 + // Block 0xce, offset 0x635 {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0xae}, {value: 0x0040, lo: 0xaf, hi: 0xbf}, - // Block 0xca, offset 0x61a + // Block 0xcf, offset 0x638 {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0x86}, {value: 0x0040, lo: 0x87, hi: 0xbf}, - // Block 0xcb, offset 0x61d + // Block 0xd0, offset 0x63b {value: 0x0000, lo: 0x06}, {value: 0x0008, lo: 0x80, hi: 0x9e}, {value: 0x0040, lo: 0x9f, hi: 0x9f}, @@ -4116,20 +4184,20 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0xaa, hi: 0xad}, {value: 0x0018, lo: 0xae, hi: 0xaf}, {value: 0x0040, lo: 0xb0, hi: 0xbf}, - // Block 0xcc, offset 0x624 + // Block 0xd1, offset 0x642 {value: 0x0000, lo: 0x06}, {value: 0x0040, lo: 0x80, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0xad}, {value: 0x0040, lo: 0xae, hi: 0xaf}, - {value: 0x1308, lo: 0xb0, hi: 0xb4}, + {value: 0x3308, lo: 0xb0, hi: 0xb4}, {value: 0x0018, lo: 0xb5, hi: 0xb5}, {value: 0x0040, lo: 0xb6, hi: 0xbf}, - // Block 0xcd, offset 0x62b + // Block 0xd2, offset 0x649 {value: 0x0000, lo: 0x03}, {value: 0x0008, lo: 0x80, hi: 0xaf}, - {value: 0x1308, lo: 0xb0, hi: 0xb6}, + {value: 0x3308, lo: 0xb0, hi: 0xb6}, {value: 0x0018, lo: 0xb7, hi: 0xbf}, - // Block 0xce, offset 0x62f + // Block 0xd3, offset 0x64d {value: 0x0000, lo: 0x0a}, {value: 0x0008, lo: 0x80, hi: 0x83}, {value: 0x0018, lo: 0x84, hi: 0x85}, @@ -4141,67 +4209,75 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0008, lo: 0xa3, hi: 0xb7}, {value: 0x0040, lo: 0xb8, hi: 0xbc}, {value: 0x0008, lo: 0xbd, hi: 0xbf}, - // Block 0xcf, offset 0x63a + // Block 0xd4, offset 0x658 {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0x8f}, {value: 0x0040, lo: 0x90, hi: 0xbf}, - // Block 0xd0, offset 0x63d + // Block 0xd5, offset 0x65b {value: 0x0000, lo: 0x05}, {value: 0x0008, lo: 0x80, hi: 0x84}, {value: 0x0040, lo: 0x85, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x90}, - {value: 0x1008, lo: 0x91, hi: 0xbe}, + {value: 0x3008, lo: 0x91, hi: 0xbe}, {value: 0x0040, lo: 0xbf, hi: 0xbf}, - // Block 0xd1, offset 0x643 + // Block 0xd6, offset 0x661 {value: 0x0000, lo: 0x04}, {value: 0x0040, lo: 0x80, hi: 0x8e}, - {value: 0x1308, lo: 0x8f, hi: 0x92}, + {value: 0x3308, lo: 0x8f, hi: 0x92}, {value: 0x0008, lo: 0x93, hi: 0x9f}, {value: 0x0040, lo: 0xa0, hi: 0xbf}, - // Block 0xd2, offset 0x648 + // Block 0xd7, offset 0x666 {value: 0x0000, lo: 0x03}, {value: 0x0040, lo: 0x80, hi: 0x9f}, - {value: 0x0008, lo: 0xa0, hi: 0xa0}, - {value: 0x0040, lo: 0xa1, hi: 0xbf}, - // Block 0xd3, offset 0x64c + {value: 0x0008, lo: 0xa0, hi: 0xa1}, + {value: 0x0040, lo: 0xa2, hi: 0xbf}, + // Block 0xd8, offset 0x66a {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0xac}, {value: 0x0040, lo: 0xad, hi: 0xbf}, - // Block 0xd4, offset 0x64f + // Block 0xd9, offset 0x66d {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0xb2}, {value: 0x0040, lo: 0xb3, hi: 0xbf}, - // Block 0xd5, offset 0x652 + // Block 0xda, offset 0x670 {value: 0x0000, lo: 0x02}, - {value: 0x0008, lo: 0x80, hi: 0x81}, - {value: 0x0040, lo: 0x82, hi: 0xbf}, - // Block 0xd6, offset 0x655 + {value: 0x0008, lo: 0x80, hi: 0x9e}, + {value: 0x0040, lo: 0x9f, hi: 0xbf}, + // Block 0xdb, offset 0x673 + {value: 0x0000, lo: 0x02}, + {value: 0x0040, lo: 0x80, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbf}, + // Block 0xdc, offset 0x676 + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0xbb}, + {value: 0x0040, lo: 0xbc, hi: 0xbf}, + // Block 0xdd, offset 0x679 {value: 0x0000, lo: 0x04}, {value: 0x0008, lo: 0x80, hi: 0xaa}, {value: 0x0040, lo: 0xab, hi: 0xaf}, {value: 0x0008, lo: 0xb0, hi: 0xbc}, {value: 0x0040, lo: 0xbd, hi: 0xbf}, - // Block 0xd7, offset 0x65a + // Block 0xde, offset 0x67e {value: 0x0000, lo: 0x09}, {value: 0x0008, lo: 0x80, hi: 0x88}, {value: 0x0040, lo: 0x89, hi: 0x8f}, {value: 0x0008, lo: 0x90, hi: 0x99}, {value: 0x0040, lo: 0x9a, hi: 0x9b}, {value: 0x0018, lo: 0x9c, hi: 0x9c}, - {value: 0x1308, lo: 0x9d, hi: 0x9e}, + {value: 0x3308, lo: 0x9d, hi: 0x9e}, {value: 0x0018, lo: 0x9f, hi: 0x9f}, {value: 0x03c0, lo: 0xa0, hi: 0xa3}, {value: 0x0040, lo: 0xa4, hi: 0xbf}, - // Block 0xd8, offset 0x664 + // Block 0xdf, offset 0x688 {value: 0x0000, lo: 0x02}, {value: 0x0018, lo: 0x80, hi: 0xb5}, {value: 0x0040, lo: 0xb6, hi: 0xbf}, - // Block 0xd9, offset 0x667 + // Block 0xe0, offset 0x68b {value: 0x0000, lo: 0x03}, {value: 0x0018, lo: 0x80, hi: 0xa6}, {value: 0x0040, lo: 0xa7, hi: 0xa8}, {value: 0x0018, lo: 0xa9, hi: 0xbf}, - // Block 0xda, offset 0x66b + // Block 0xe1, offset 0x68f {value: 0x0000, lo: 0x0e}, {value: 0x0018, lo: 0x80, hi: 0x9d}, {value: 0xb5b9, lo: 0x9e, hi: 0x9e}, @@ -4211,127 +4287,127 @@ var idnaSparseValues = [1876]valueRange{ {value: 0xb719, lo: 0xa2, hi: 0xa2}, {value: 0xb781, lo: 0xa3, hi: 0xa3}, {value: 0xb7e9, lo: 0xa4, hi: 0xa4}, - {value: 0x1018, lo: 0xa5, hi: 0xa6}, - {value: 0x1318, lo: 0xa7, hi: 0xa9}, + {value: 0x3018, lo: 0xa5, hi: 0xa6}, + {value: 0x3318, lo: 0xa7, hi: 0xa9}, {value: 0x0018, lo: 0xaa, hi: 0xac}, - {value: 0x1018, lo: 0xad, hi: 0xb2}, + {value: 0x3018, lo: 0xad, hi: 0xb2}, {value: 0x0340, lo: 0xb3, hi: 0xba}, - {value: 0x1318, lo: 0xbb, hi: 0xbf}, - // Block 0xdb, offset 0x67a + {value: 0x3318, lo: 0xbb, hi: 0xbf}, + // Block 0xe2, offset 0x69e {value: 0x0000, lo: 0x0b}, - {value: 0x1318, lo: 0x80, hi: 0x82}, + {value: 0x3318, lo: 0x80, hi: 0x82}, {value: 0x0018, lo: 0x83, hi: 0x84}, - {value: 0x1318, lo: 0x85, hi: 0x8b}, + {value: 0x3318, lo: 0x85, hi: 0x8b}, {value: 0x0018, lo: 0x8c, hi: 0xa9}, - {value: 0x1318, lo: 0xaa, hi: 0xad}, + {value: 0x3318, lo: 0xaa, hi: 0xad}, {value: 0x0018, lo: 0xae, hi: 0xba}, {value: 0xb851, lo: 0xbb, hi: 0xbb}, {value: 0xb899, lo: 0xbc, hi: 0xbc}, {value: 0xb8e1, lo: 0xbd, hi: 0xbd}, {value: 0xb949, lo: 0xbe, hi: 0xbe}, {value: 0xb9b1, lo: 0xbf, hi: 0xbf}, - // Block 0xdc, offset 0x686 + // Block 0xe3, offset 0x6aa {value: 0x0000, lo: 0x03}, {value: 0xba19, lo: 0x80, hi: 0x80}, {value: 0x0018, lo: 0x81, hi: 0xa8}, {value: 0x0040, lo: 0xa9, hi: 0xbf}, - // Block 0xdd, offset 0x68a + // Block 0xe4, offset 0x6ae {value: 0x0000, lo: 0x04}, {value: 0x0018, lo: 0x80, hi: 0x81}, - {value: 0x1318, lo: 0x82, hi: 0x84}, + {value: 0x3318, lo: 0x82, hi: 0x84}, {value: 0x0018, lo: 0x85, hi: 0x85}, {value: 0x0040, lo: 0x86, hi: 0xbf}, - // Block 0xde, offset 0x68f + // Block 0xe5, offset 0x6b3 {value: 0x0000, lo: 0x04}, {value: 0x0018, lo: 0x80, hi: 0x96}, {value: 0x0040, lo: 0x97, hi: 0x9f}, {value: 0x0018, lo: 0xa0, hi: 0xb1}, {value: 0x0040, lo: 0xb2, hi: 0xbf}, - // Block 0xdf, offset 0x694 + // Block 0xe6, offset 0x6b8 {value: 0x0000, lo: 0x03}, - {value: 0x1308, lo: 0x80, hi: 0xb6}, + {value: 0x3308, lo: 0x80, hi: 0xb6}, {value: 0x0018, lo: 0xb7, hi: 0xba}, - {value: 0x1308, lo: 0xbb, hi: 0xbf}, - // Block 0xe0, offset 0x698 + {value: 0x3308, lo: 0xbb, hi: 0xbf}, + // Block 0xe7, offset 0x6bc {value: 0x0000, lo: 0x04}, - {value: 0x1308, lo: 0x80, hi: 0xac}, + {value: 0x3308, lo: 0x80, hi: 0xac}, {value: 0x0018, lo: 0xad, hi: 0xb4}, - {value: 0x1308, lo: 0xb5, hi: 0xb5}, + {value: 0x3308, lo: 0xb5, hi: 0xb5}, {value: 0x0018, lo: 0xb6, hi: 0xbf}, - // Block 0xe1, offset 0x69d + // Block 0xe8, offset 0x6c1 {value: 0x0000, lo: 0x08}, {value: 0x0018, lo: 0x80, hi: 0x83}, - {value: 0x1308, lo: 0x84, hi: 0x84}, + {value: 0x3308, lo: 0x84, hi: 0x84}, {value: 0x0018, lo: 0x85, hi: 0x8b}, {value: 0x0040, lo: 0x8c, hi: 0x9a}, - {value: 0x1308, lo: 0x9b, hi: 0x9f}, + {value: 0x3308, lo: 0x9b, hi: 0x9f}, {value: 0x0040, lo: 0xa0, hi: 0xa0}, - {value: 0x1308, lo: 0xa1, hi: 0xaf}, + {value: 0x3308, lo: 0xa1, hi: 0xaf}, {value: 0x0040, lo: 0xb0, hi: 0xbf}, - // Block 0xe2, offset 0x6a6 + // Block 0xe9, offset 0x6ca {value: 0x0000, lo: 0x0a}, - {value: 0x1308, lo: 0x80, hi: 0x86}, + {value: 0x3308, lo: 0x80, hi: 0x86}, {value: 0x0040, lo: 0x87, hi: 0x87}, - {value: 0x1308, lo: 0x88, hi: 0x98}, + {value: 0x3308, lo: 0x88, hi: 0x98}, {value: 0x0040, lo: 0x99, hi: 0x9a}, - {value: 0x1308, lo: 0x9b, hi: 0xa1}, + {value: 0x3308, lo: 0x9b, hi: 0xa1}, {value: 0x0040, lo: 0xa2, hi: 0xa2}, - {value: 0x1308, lo: 0xa3, hi: 0xa4}, + {value: 0x3308, lo: 0xa3, hi: 0xa4}, {value: 0x0040, lo: 0xa5, hi: 0xa5}, - {value: 0x1308, lo: 0xa6, hi: 0xaa}, + {value: 0x3308, lo: 0xa6, hi: 0xaa}, {value: 0x0040, lo: 0xab, hi: 0xbf}, - // Block 0xe3, offset 0x6b1 + // Block 0xea, offset 0x6d5 {value: 0x0000, lo: 0x05}, - {value: 0x0008, lo: 0x80, hi: 0x84}, + {value: 0x0808, lo: 0x80, hi: 0x84}, {value: 0x0040, lo: 0x85, hi: 0x86}, - {value: 0x0018, lo: 0x87, hi: 0x8f}, - {value: 0x1308, lo: 0x90, hi: 0x96}, + {value: 0x0818, lo: 0x87, hi: 0x8f}, + {value: 0x3308, lo: 0x90, hi: 0x96}, {value: 0x0040, lo: 0x97, hi: 0xbf}, - // Block 0xe4, offset 0x6b7 + // Block 0xeb, offset 0x6db {value: 0x0000, lo: 0x07}, - {value: 0x0208, lo: 0x80, hi: 0x83}, - {value: 0x1308, lo: 0x84, hi: 0x8a}, + {value: 0x0a08, lo: 0x80, hi: 0x83}, + {value: 0x3308, lo: 0x84, hi: 0x8a}, {value: 0x0040, lo: 0x8b, hi: 0x8f}, - {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0808, lo: 0x90, hi: 0x99}, {value: 0x0040, lo: 0x9a, hi: 0x9d}, - {value: 0x0018, lo: 0x9e, hi: 0x9f}, + {value: 0x0818, lo: 0x9e, hi: 0x9f}, {value: 0x0040, lo: 0xa0, hi: 0xbf}, - // Block 0xe5, offset 0x6bf + // Block 0xec, offset 0x6e3 {value: 0x0000, lo: 0x03}, {value: 0x0040, lo: 0x80, hi: 0xaf}, {value: 0x0018, lo: 0xb0, hi: 0xb1}, {value: 0x0040, lo: 0xb2, hi: 0xbf}, - // Block 0xe6, offset 0x6c3 + // Block 0xed, offset 0x6e7 {value: 0x0000, lo: 0x03}, {value: 0x0018, lo: 0x80, hi: 0xab}, {value: 0x0040, lo: 0xac, hi: 0xaf}, {value: 0x0018, lo: 0xb0, hi: 0xbf}, - // Block 0xe7, offset 0x6c7 + // Block 0xee, offset 0x6eb {value: 0x0000, lo: 0x05}, {value: 0x0018, lo: 0x80, hi: 0x93}, {value: 0x0040, lo: 0x94, hi: 0x9f}, {value: 0x0018, lo: 0xa0, hi: 0xae}, {value: 0x0040, lo: 0xaf, hi: 0xb0}, {value: 0x0018, lo: 0xb1, hi: 0xbf}, - // Block 0xe8, offset 0x6cd + // Block 0xef, offset 0x6f1 {value: 0x0000, lo: 0x05}, {value: 0x0040, lo: 0x80, hi: 0x80}, {value: 0x0018, lo: 0x81, hi: 0x8f}, {value: 0x0040, lo: 0x90, hi: 0x90}, {value: 0x0018, lo: 0x91, hi: 0xb5}, {value: 0x0040, lo: 0xb6, hi: 0xbf}, - // Block 0xe9, offset 0x6d3 + // Block 0xf0, offset 0x6f7 {value: 0x0000, lo: 0x04}, {value: 0x0018, lo: 0x80, hi: 0x8f}, {value: 0xc1c1, lo: 0x90, hi: 0x90}, {value: 0x0018, lo: 0x91, hi: 0xac}, {value: 0x0040, lo: 0xad, hi: 0xbf}, - // Block 0xea, offset 0x6d8 + // Block 0xf1, offset 0x6fc {value: 0x0000, lo: 0x02}, {value: 0x0040, lo: 0x80, hi: 0xa5}, {value: 0x0018, lo: 0xa6, hi: 0xbf}, - // Block 0xeb, offset 0x6db - {value: 0x0000, lo: 0x0d}, + // Block 0xf2, offset 0x6ff + {value: 0x0000, lo: 0x0f}, {value: 0xc7e9, lo: 0x80, hi: 0x80}, {value: 0xc839, lo: 0x81, hi: 0x81}, {value: 0xc889, lo: 0x82, hi: 0x82}, @@ -4344,84 +4420,88 @@ var idnaSparseValues = [1876]valueRange{ {value: 0x0040, lo: 0x89, hi: 0x8f}, {value: 0xcab9, lo: 0x90, hi: 0x90}, {value: 0xcad9, lo: 0x91, hi: 0x91}, - {value: 0x0040, lo: 0x92, hi: 0xbf}, - // Block 0xec, offset 0x6e9 + {value: 0x0040, lo: 0x92, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xa5}, + {value: 0x0040, lo: 0xa6, hi: 0xbf}, + // Block 0xf3, offset 0x70f {value: 0x0000, lo: 0x06}, - {value: 0x0018, lo: 0x80, hi: 0x92}, - {value: 0x0040, lo: 0x93, hi: 0x9f}, + {value: 0x0018, lo: 0x80, hi: 0x94}, + {value: 0x0040, lo: 0x95, hi: 0x9f}, {value: 0x0018, lo: 0xa0, hi: 0xac}, {value: 0x0040, lo: 0xad, hi: 0xaf}, - {value: 0x0018, lo: 0xb0, hi: 0xb6}, - {value: 0x0040, lo: 0xb7, hi: 0xbf}, - // Block 0xed, offset 0x6f0 + {value: 0x0018, lo: 0xb0, hi: 0xb8}, + {value: 0x0040, lo: 0xb9, hi: 0xbf}, + // Block 0xf4, offset 0x716 {value: 0x0000, lo: 0x02}, {value: 0x0018, lo: 0x80, hi: 0xb3}, {value: 0x0040, lo: 0xb4, hi: 0xbf}, - // Block 0xee, offset 0x6f3 + // Block 0xf5, offset 0x719 {value: 0x0000, lo: 0x02}, {value: 0x0018, lo: 0x80, hi: 0x94}, {value: 0x0040, lo: 0x95, hi: 0xbf}, - // Block 0xef, offset 0x6f6 + // Block 0xf6, offset 0x71c {value: 0x0000, lo: 0x03}, {value: 0x0018, lo: 0x80, hi: 0x8b}, {value: 0x0040, lo: 0x8c, hi: 0x8f}, {value: 0x0018, lo: 0x90, hi: 0xbf}, - // Block 0xf0, offset 0x6fa + // Block 0xf7, offset 0x720 {value: 0x0000, lo: 0x05}, {value: 0x0018, lo: 0x80, hi: 0x87}, {value: 0x0040, lo: 0x88, hi: 0x8f}, {value: 0x0018, lo: 0x90, hi: 0x99}, {value: 0x0040, lo: 0x9a, hi: 0x9f}, {value: 0x0018, lo: 0xa0, hi: 0xbf}, - // Block 0xf1, offset 0x700 + // Block 0xf8, offset 0x726 {value: 0x0000, lo: 0x04}, {value: 0x0018, lo: 0x80, hi: 0x87}, {value: 0x0040, lo: 0x88, hi: 0x8f}, {value: 0x0018, lo: 0x90, hi: 0xad}, {value: 0x0040, lo: 0xae, hi: 0xbf}, - // Block 0xf2, offset 0x705 - {value: 0x0000, lo: 0x09}, - {value: 0x0040, lo: 0x80, hi: 0x8f}, - {value: 0x0018, lo: 0x90, hi: 0x9e}, - {value: 0x0040, lo: 0x9f, hi: 0x9f}, - {value: 0x0018, lo: 0xa0, hi: 0xa7}, - {value: 0x0040, lo: 0xa8, hi: 0xaf}, - {value: 0x0018, lo: 0xb0, hi: 0xb0}, - {value: 0x0040, lo: 0xb1, hi: 0xb2}, - {value: 0x0018, lo: 0xb3, hi: 0xbe}, - {value: 0x0040, lo: 0xbf, hi: 0xbf}, - // Block 0xf3, offset 0x70f + // Block 0xf9, offset 0x72b {value: 0x0000, lo: 0x04}, {value: 0x0018, lo: 0x80, hi: 0x8b}, {value: 0x0040, lo: 0x8c, hi: 0x8f}, - {value: 0x0018, lo: 0x90, hi: 0x9e}, - {value: 0x0040, lo: 0x9f, hi: 0xbf}, - // Block 0xf4, offset 0x714 - {value: 0x0000, lo: 0x02}, - {value: 0x0018, lo: 0x80, hi: 0x91}, - {value: 0x0040, lo: 0x92, hi: 0xbf}, - // Block 0xf5, offset 0x717 + {value: 0x0018, lo: 0x90, hi: 0xbe}, + {value: 0x0040, lo: 0xbf, hi: 0xbf}, + // Block 0xfa, offset 0x730 + {value: 0x0000, lo: 0x04}, + {value: 0x0018, lo: 0x80, hi: 0x8c}, + {value: 0x0040, lo: 0x8d, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0xab}, + {value: 0x0040, lo: 0xac, hi: 0xbf}, + // Block 0xfb, offset 0x735 {value: 0x0000, lo: 0x02}, + {value: 0x0018, lo: 0x80, hi: 0x97}, + {value: 0x0040, lo: 0x98, hi: 0xbf}, + // Block 0xfc, offset 0x738 + {value: 0x0000, lo: 0x04}, {value: 0x0018, lo: 0x80, hi: 0x80}, - {value: 0x0040, lo: 0x81, hi: 0xbf}, - // Block 0xf6, offset 0x71a + {value: 0x0040, lo: 0x81, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0xa6}, + {value: 0x0040, lo: 0xa7, hi: 0xbf}, + // Block 0xfd, offset 0x73d {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0x96}, {value: 0x0040, lo: 0x97, hi: 0xbf}, - // Block 0xf7, offset 0x71d + // Block 0xfe, offset 0x740 {value: 0x0000, lo: 0x02}, {value: 0x0008, lo: 0x80, hi: 0xb4}, {value: 0x0040, lo: 0xb5, hi: 0xbf}, - // Block 0xf8, offset 0x720 + // Block 0xff, offset 0x743 {value: 0x0000, lo: 0x03}, {value: 0x0008, lo: 0x80, hi: 0x9d}, {value: 0x0040, lo: 0x9e, hi: 0x9f}, {value: 0x0008, lo: 0xa0, hi: 0xbf}, - // Block 0xf9, offset 0x724 - {value: 0x0000, lo: 0x02}, + // Block 0x100, offset 0x747 + {value: 0x0000, lo: 0x03}, {value: 0x0008, lo: 0x80, hi: 0xa1}, - {value: 0x0040, lo: 0xa2, hi: 0xbf}, - // Block 0xfa, offset 0x727 + {value: 0x0040, lo: 0xa2, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbf}, + // Block 0x101, offset 0x74b + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0xa0}, + {value: 0x0040, lo: 0xa1, hi: 0xbf}, + // Block 0x102, offset 0x74e {value: 0x0020, lo: 0x0f}, {value: 0xdeb9, lo: 0x80, hi: 0x89}, {value: 0x8dfd, lo: 0x8a, hi: 0x8a}, @@ -4438,7 +4518,7 @@ var idnaSparseValues = [1876]valueRange{ {value: 0xe4f9, lo: 0xba, hi: 0xba}, {value: 0x8edd, lo: 0xbb, hi: 0xbb}, {value: 0xe519, lo: 0xbc, hi: 0xbf}, - // Block 0xfb, offset 0x737 + // Block 0x103, offset 0x75e {value: 0x0020, lo: 0x10}, {value: 0x937d, lo: 0x80, hi: 0x80}, {value: 0xf099, lo: 0x81, hi: 0x86}, @@ -4455,23 +4535,23 @@ var idnaSparseValues = [1876]valueRange{ {value: 0xf4d9, lo: 0xae, hi: 0xaf}, {value: 0x94dd, lo: 0xb0, hi: 0xb1}, {value: 0xf519, lo: 0xb2, hi: 0xbe}, - {value: 0x0040, lo: 0xbf, hi: 0xbf}, - // Block 0xfc, offset 0x748 + {value: 0x2040, lo: 0xbf, hi: 0xbf}, + // Block 0x104, offset 0x76f {value: 0x0000, lo: 0x04}, {value: 0x0040, lo: 0x80, hi: 0x80}, {value: 0x0340, lo: 0x81, hi: 0x81}, {value: 0x0040, lo: 0x82, hi: 0x9f}, {value: 0x0340, lo: 0xa0, hi: 0xbf}, - // Block 0xfd, offset 0x74d + // Block 0x105, offset 0x774 {value: 0x0000, lo: 0x01}, {value: 0x0340, lo: 0x80, hi: 0xbf}, - // Block 0xfe, offset 0x74f + // Block 0x106, offset 0x776 {value: 0x0000, lo: 0x01}, - {value: 0x13c0, lo: 0x80, hi: 0xbf}, - // Block 0xff, offset 0x751 + {value: 0x33c0, lo: 0x80, hi: 0xbf}, + // Block 0x107, offset 0x778 {value: 0x0000, lo: 0x02}, - {value: 0x13c0, lo: 0x80, hi: 0xaf}, + {value: 0x33c0, lo: 0x80, hi: 0xaf}, {value: 0x0040, lo: 0xb0, hi: 0xbf}, } -// Total table size 41559 bytes (40KiB); checksum: F4A1FA4E +// Total table size 42115 bytes (41KiB); checksum: F4A1FA4E diff --git a/vendor/golang.org/x/net/idna/trieval.go b/vendor/golang.org/x/net/idna/trieval.go index 63cb03b59b1..7a8cf889b5b 100644 --- a/vendor/golang.org/x/net/idna/trieval.go +++ b/vendor/golang.org/x/net/idna/trieval.go @@ -26,9 +26,9 @@ package idna // 15..3 index into xor or mapping table // } // } else { -// 15..13 unused -// 12 modifier (including virama) -// 11 virama modifier +// 15..14 unused +// 13 mayNeedNorm +// 12..11 attributes // 10..8 joining type // 7..3 category type // } @@ -49,15 +49,20 @@ const ( joinShift = 8 joinMask = 0x07 - viramaModifier = 0x0800 + // Attributes + attributesMask = 0x1800 + viramaModifier = 0x1800 modifier = 0x1000 + rtl = 0x0800 + + mayNeedNorm = 0x2000 ) // A category corresponds to a category defined in the IDNA mapping table. type category uint16 const ( - unknown category = 0 // not defined currently in unicode. + unknown category = 0 // not currently defined in unicode. mapped category = 1 disallowedSTD3Mapped category = 2 deviation category = 3 @@ -110,5 +115,5 @@ func (c info) isModifier() bool { } func (c info) isViramaModifier() bool { - return c&(viramaModifier|catSmallMask) == viramaModifier + return c&(attributesMask|catSmallMask) == viramaModifier } diff --git a/vendor/golang.org/x/net/proxy/per_host.go b/vendor/golang.org/x/net/proxy/per_host.go index 242d5623f80..0689bb6a70f 100644 --- a/vendor/golang.org/x/net/proxy/per_host.go +++ b/vendor/golang.org/x/net/proxy/per_host.go @@ -61,7 +61,7 @@ func (p *PerHost) dialerForRequest(host string) Dialer { return p.bypass } if host == zone[1:] { - // For a zone "example.com", we match "example.com" + // For a zone ".example.com", we match "example.com" // too. return p.bypass } diff --git a/vendor/golang.org/x/net/proxy/socks5.go b/vendor/golang.org/x/net/proxy/socks5.go index 2efec6e8d7e..3fed38ef1cc 100644 --- a/vendor/golang.org/x/net/proxy/socks5.go +++ b/vendor/golang.org/x/net/proxy/socks5.go @@ -12,7 +12,7 @@ import ( ) // SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given address -// with an optional username and password. See RFC 1928. +// with an optional username and password. See RFC 1928 and RFC 1929. func SOCKS5(network, addr string, auth *Auth, forward Dialer) (Dialer, error) { s := &socks5{ network: network, @@ -60,7 +60,7 @@ var socks5Errors = []string{ "address type not supported", } -// Dial connects to the address addr on the network net via the SOCKS5 proxy. +// Dial connects to the address addr on the given network via the SOCKS5 proxy. func (s *socks5) Dial(network, addr string) (net.Conn, error) { switch network { case "tcp", "tcp6", "tcp4": @@ -120,6 +120,7 @@ func (s *socks5) connect(conn net.Conn, target string) error { return errors.New("proxy: SOCKS5 proxy at " + s.addr + " requires authentication") } + // See RFC 1929 if buf[1] == socks5AuthPassword { buf = buf[:0] buf = append(buf, 1 /* password protocol version */) diff --git a/vendor/google.golang.org/grpc/.please-update b/vendor/google.golang.org/grpc/.please-update deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/vendor/google.golang.org/grpc/.travis.yml b/vendor/google.golang.org/grpc/.travis.yml index 22bf25004a3..3c2621ab750 100644 --- a/vendor/google.golang.org/grpc/.travis.yml +++ b/vendor/google.golang.org/grpc/.travis.yml @@ -1,20 +1,24 @@ language: go go: + - 1.6.x - 1.7.x - 1.8.x - 1.9.x + - 1.10.x matrix: include: - - go: 1.9.x - env: ARCH=386 + - go: 1.10.x + env: RUN386=1 go_import_path: google.golang.org/grpc before_install: - - if [[ "$TRAVIS_GO_VERSION" = 1.9* && "$ARCH" != "386" ]]; then ./vet.sh -install || exit 1; fi + - if [[ -n "$RUN386" ]]; then export GOARCH=386; fi + - if [[ "$TRAVIS_GO_VERSION" = 1.10* && "$GOARCH" != "386" ]]; then ./vet.sh -install || exit 1; fi script: - - if [[ "$TRAVIS_GO_VERSION" = 1.9* && "$ARCH" != "386" ]]; then ./vet.sh || exit 1; fi - - make test testrace + - if [[ "$TRAVIS_GO_VERSION" = 1.10* && "$GOARCH" != "386" ]]; then ./vet.sh || exit 1; fi + - make test || exit 1 + - if [[ "$GOARCH" != "386" ]]; then make testrace; fi diff --git a/vendor/google.golang.org/grpc/BUILD b/vendor/google.golang.org/grpc/BUILD index 286fe9409c4..081cf804fc2 100644 --- a/vendor/google.golang.org/grpc/BUILD +++ b/vendor/google.golang.org/grpc/BUILD @@ -11,7 +11,9 @@ go_library( "clientconn.go", "codec.go", "doc.go", - "grpclb.go", + "envconfig.go", + "go16.go", + "go17.go", "interceptor.go", "picker_wrapper.go", "pickfirst.go", @@ -19,29 +21,37 @@ go_library( "resolver_conn_wrapper.go", "rpc_util.go", "server.go", + "service_config.go", + "stickiness_linkedmap.go", "stream.go", "trace.go", + "version.go", ], importmap = "k8s.io/kubernetes/vendor/google.golang.org/grpc", importpath = "google.golang.org/grpc", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/protobuf/proto:go_default_library", "//vendor/golang.org/x/net/context:go_default_library", "//vendor/golang.org/x/net/http2:go_default_library", "//vendor/golang.org/x/net/trace:go_default_library", "//vendor/google.golang.org/grpc/balancer:go_default_library", + "//vendor/google.golang.org/grpc/balancer/roundrobin:go_default_library", "//vendor/google.golang.org/grpc/codes:go_default_library", "//vendor/google.golang.org/grpc/connectivity:go_default_library", "//vendor/google.golang.org/grpc/credentials:go_default_library", - "//vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages:go_default_library", + "//vendor/google.golang.org/grpc/encoding:go_default_library", + "//vendor/google.golang.org/grpc/encoding/proto:go_default_library", "//vendor/google.golang.org/grpc/grpclog:go_default_library", "//vendor/google.golang.org/grpc/internal:go_default_library", + "//vendor/google.golang.org/grpc/internal/backoff:go_default_library", + "//vendor/google.golang.org/grpc/internal/channelz:go_default_library", "//vendor/google.golang.org/grpc/keepalive:go_default_library", "//vendor/google.golang.org/grpc/metadata:go_default_library", "//vendor/google.golang.org/grpc/naming:go_default_library", "//vendor/google.golang.org/grpc/peer:go_default_library", "//vendor/google.golang.org/grpc/resolver:go_default_library", + "//vendor/google.golang.org/grpc/resolver/dns:go_default_library", + "//vendor/google.golang.org/grpc/resolver/passthrough:go_default_library", "//vendor/google.golang.org/grpc/stats:go_default_library", "//vendor/google.golang.org/grpc/status:go_default_library", "//vendor/google.golang.org/grpc/tap:go_default_library", @@ -64,6 +74,7 @@ filegroup( "//vendor/google.golang.org/grpc/codes:all-srcs", "//vendor/google.golang.org/grpc/connectivity:all-srcs", "//vendor/google.golang.org/grpc/credentials:all-srcs", + "//vendor/google.golang.org/grpc/encoding:all-srcs", "//vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages:all-srcs", "//vendor/google.golang.org/grpc/grpclog:all-srcs", "//vendor/google.golang.org/grpc/health:all-srcs", diff --git a/vendor/google.golang.org/grpc/CONTRIBUTING.md b/vendor/google.golang.org/grpc/CONTRIBUTING.md index a5c6e06e255..0863eb26b60 100644 --- a/vendor/google.golang.org/grpc/CONTRIBUTING.md +++ b/vendor/google.golang.org/grpc/CONTRIBUTING.md @@ -7,7 +7,7 @@ If you are new to github, please start by reading [Pull Request howto](https://h ## Legal requirements In order to protect both you and ourselves, you will need to sign the -[Contributor License Agreement](https://cla.developers.google.com/clas). +[Contributor License Agreement](https://identity.linuxfoundation.org/projects/cncf). ## Guidelines for Pull Requests How to get your contributions merged smoothly and quickly. @@ -27,6 +27,10 @@ How to get your contributions merged smoothly and quickly. - Keep your PR up to date with upstream/master (if there are merge conflicts, we can't really merge your change). - **All tests need to be passing** before your change can be merged. We recommend you **run tests locally** before creating your PR to catch breakages early on. + - `make all` to test everything, OR + - `make vet` to catch vet errors + - `make test` to run the tests + - `make testrace` to run tests in race mode - Exceptions to the rules can be made if there's a compelling reason for doing so. diff --git a/vendor/google.golang.org/grpc/Makefile b/vendor/google.golang.org/grpc/Makefile index 39606b564a6..6f393a808df 100644 --- a/vendor/google.golang.org/grpc/Makefile +++ b/vendor/google.golang.org/grpc/Makefile @@ -1,4 +1,4 @@ -all: test testrace +all: vet test testrace deps: go get -d -v google.golang.org/grpc/... @@ -22,11 +22,14 @@ proto: fi go generate google.golang.org/grpc/... +vet: + ./vet.sh + test: testdeps - go test -cpu 1,4 google.golang.org/grpc/... + go test -cpu 1,4 -timeout 5m google.golang.org/grpc/... testrace: testdeps - go test -race -cpu 1,4 google.golang.org/grpc/... + go test -race -cpu 1,4 -timeout 7m google.golang.org/grpc/... clean: go clean -i google.golang.org/grpc/... @@ -39,7 +42,7 @@ clean: updatetestdeps \ build \ proto \ + vet \ test \ testrace \ - clean \ - coverage + clean diff --git a/vendor/google.golang.org/grpc/README.md b/vendor/google.golang.org/grpc/README.md index 622a5dc3e85..789adfd6536 100644 --- a/vendor/google.golang.org/grpc/README.md +++ b/vendor/google.golang.org/grpc/README.md @@ -1,6 +1,6 @@ # gRPC-Go -[![Build Status](https://travis-ci.org/grpc/grpc-go.svg)](https://travis-ci.org/grpc/grpc-go) [![GoDoc](https://godoc.org/google.golang.org/grpc?status.svg)](https://godoc.org/google.golang.org/grpc) +[![Build Status](https://travis-ci.org/grpc/grpc-go.svg)](https://travis-ci.org/grpc/grpc-go) [![GoDoc](https://godoc.org/google.golang.org/grpc?status.svg)](https://godoc.org/google.golang.org/grpc) [![GoReportCard](https://goreportcard.com/badge/grpc/grpc-go)](https://goreportcard.com/report/github.com/grpc/grpc-go) The Go implementation of [gRPC](https://grpc.io/): A high performance, open source, general RPC framework that puts mobile and HTTP/2 first. For more information see the [gRPC Quick Start: Go](https://grpc.io/docs/quickstart/go.html) guide. @@ -16,7 +16,7 @@ $ go get -u google.golang.org/grpc Prerequisites ------------- -This requires Go 1.7 or later. +This requires Go 1.6 or later. Go 1.7 will be required soon. Constraints ----------- diff --git a/vendor/google.golang.org/grpc/backoff.go b/vendor/google.golang.org/grpc/backoff.go index 090fbe87c52..fa31565fd28 100644 --- a/vendor/google.golang.org/grpc/backoff.go +++ b/vendor/google.golang.org/grpc/backoff.go @@ -16,83 +16,23 @@ * */ +// See internal/backoff package for the backoff implementation. This file is +// kept for the exported types and API backward compatility. + package grpc import ( - "math/rand" "time" ) // DefaultBackoffConfig uses values specified for backoff in // https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. -var ( - DefaultBackoffConfig = BackoffConfig{ - MaxDelay: 120 * time.Second, - baseDelay: 1.0 * time.Second, - factor: 1.6, - jitter: 0.2, - } -) - -// backoffStrategy defines the methodology for backing off after a grpc -// connection failure. -// -// This is unexported until the gRPC project decides whether or not to allow -// alternative backoff strategies. Once a decision is made, this type and its -// method may be exported. -type backoffStrategy interface { - // backoff returns the amount of time to wait before the next retry given - // the number of consecutive failures. - backoff(retries int) time.Duration +var DefaultBackoffConfig = BackoffConfig{ + MaxDelay: 120 * time.Second, } // BackoffConfig defines the parameters for the default gRPC backoff strategy. type BackoffConfig struct { // MaxDelay is the upper bound of backoff delay. MaxDelay time.Duration - - // TODO(stevvooe): The following fields are not exported, as allowing - // changes would violate the current gRPC specification for backoff. If - // gRPC decides to allow more interesting backoff strategies, these fields - // may be opened up in the future. - - // baseDelay is the amount of time to wait before retrying after the first - // failure. - baseDelay time.Duration - - // factor is applied to the backoff after each retry. - factor float64 - - // jitter provides a range to randomize backoff delays. - jitter float64 -} - -func setDefaults(bc *BackoffConfig) { - md := bc.MaxDelay - *bc = DefaultBackoffConfig - - if md > 0 { - bc.MaxDelay = md - } -} - -func (bc BackoffConfig) backoff(retries int) time.Duration { - if retries == 0 { - return bc.baseDelay - } - backoff, max := float64(bc.baseDelay), float64(bc.MaxDelay) - for backoff < max && retries > 0 { - backoff *= bc.factor - retries-- - } - if backoff > max { - backoff = max - } - // Randomize backoff delays so that if a cluster of requests start at - // the same time, they won't operate in lockstep. - backoff *= 1 + bc.jitter*(rand.Float64()*2-1) - if backoff < 0 { - return 0 - } - return time.Duration(backoff) } diff --git a/vendor/google.golang.org/grpc/balancer.go b/vendor/google.golang.org/grpc/balancer.go index ab65049ddc1..e1730166cde 100644 --- a/vendor/google.golang.org/grpc/balancer.go +++ b/vendor/google.golang.org/grpc/balancer.go @@ -28,10 +28,12 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/naming" + "google.golang.org/grpc/status" ) // Address represents a server the client connects to. -// This is the EXPERIMENTAL API and may be changed or extended in the future. +// +// Deprecated: please use package balancer. type Address struct { // Addr is the server address on which a connection will be established. Addr string @@ -41,6 +43,8 @@ type Address struct { } // BalancerConfig specifies the configurations for Balancer. +// +// Deprecated: please use package balancer. type BalancerConfig struct { // DialCreds is the transport credential the Balancer implementation can // use to dial to a remote load balancer server. The Balancer implementations @@ -53,7 +57,8 @@ type BalancerConfig struct { } // BalancerGetOptions configures a Get call. -// This is the EXPERIMENTAL API and may be changed or extended in the future. +// +// Deprecated: please use package balancer. type BalancerGetOptions struct { // BlockingWait specifies whether Get should block when there is no // connected address. @@ -61,7 +66,8 @@ type BalancerGetOptions struct { } // Balancer chooses network addresses for RPCs. -// This is the EXPERIMENTAL API and may be changed or extended in the future. +// +// Deprecated: please use package balancer. type Balancer interface { // Start does the initialization work to bootstrap a Balancer. For example, // this function may start the name resolution and watch the updates. It will @@ -134,6 +140,8 @@ func downErrorf(timeout, temporary bool, format string, a ...interface{}) downEr // RoundRobin returns a Balancer that selects addresses round-robin. It uses r to watch // the name resolution updates and updates the addresses available correspondingly. +// +// Deprecated: please use package balancer/roundrobin. func RoundRobin(r naming.Resolver) Balancer { return &roundRobin{r: r} } @@ -310,7 +318,7 @@ func (rr *roundRobin) Get(ctx context.Context, opts BalancerGetOptions) (addr Ad if !opts.BlockingWait { if len(rr.addrs) == 0 { rr.mu.Unlock() - err = Errorf(codes.Unavailable, "there is no address available") + err = status.Errorf(codes.Unavailable, "there is no address available") return } // Returns the next addr on rr.addrs for failfast RPCs. diff --git a/vendor/google.golang.org/grpc/balancer/BUILD b/vendor/google.golang.org/grpc/balancer/BUILD index 77828478c08..8dc7d3f73b4 100644 --- a/vendor/google.golang.org/grpc/balancer/BUILD +++ b/vendor/google.golang.org/grpc/balancer/BUILD @@ -23,7 +23,11 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//vendor/google.golang.org/grpc/balancer/base:all-srcs", + "//vendor/google.golang.org/grpc/balancer/roundrobin:all-srcs", + ], tags = ["automanaged"], visibility = ["//visibility:public"], ) diff --git a/vendor/google.golang.org/grpc/balancer/balancer.go b/vendor/google.golang.org/grpc/balancer/balancer.go index 84e10b630e7..f9d83c2f39f 100644 --- a/vendor/google.golang.org/grpc/balancer/balancer.go +++ b/vendor/google.golang.org/grpc/balancer/balancer.go @@ -23,6 +23,7 @@ package balancer import ( "errors" "net" + "strings" "golang.org/x/net/context" "google.golang.org/grpc/connectivity" @@ -33,24 +34,26 @@ import ( var ( // m is a map from name to balancer builder. m = make(map[string]Builder) - // defaultBuilder is the default balancer to use. - defaultBuilder Builder // TODO(bar) install pickfirst as default. ) -// Register registers the balancer builder to the balancer map. -// b.Name will be used as the name registered with this builder. +// Register registers the balancer builder to the balancer map. b.Name +// (lowercased) will be used as the name registered with this builder. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple Balancers are +// registered with the same name, the one registered last will take effect. func Register(b Builder) { - m[b.Name()] = b + m[strings.ToLower(b.Name())] = b } // Get returns the resolver builder registered with the given name. -// If no builder is register with the name, the default pickfirst will -// be used. +// Note that the compare is done in a case-insenstive fashion. +// If no builder is register with the name, nil will be returned. func Get(name string) Builder { - if b, ok := m[name]; ok { + if b, ok := m[strings.ToLower(name)]; ok { return b } - return defaultBuilder + return nil } // SubConn represents a gRPC sub connection. @@ -66,6 +69,11 @@ func Get(name string) Builder { // When the connection encounters an error, it will reconnect immediately. // When the connection becomes IDLE, it will not reconnect unless Connect is // called. +// +// This interface is to be implemented by gRPC. Users should not need a +// brand new implementation of this interface. For the situations like +// testing, the new implementation should embed this interface. This allows +// gRPC to add new methods to this interface. type SubConn interface { // UpdateAddresses updates the addresses used in this SubConn. // gRPC checks if currently-connected address is still in the new list. @@ -83,6 +91,11 @@ type SubConn interface { type NewSubConnOptions struct{} // ClientConn represents a gRPC ClientConn. +// +// This interface is to be implemented by gRPC. Users should not need a +// brand new implementation of this interface. For the situations like +// testing, the new implementation should embed this interface. This allows +// gRPC to add new methods to this interface. type ClientConn interface { // NewSubConn is called by balancer to create a new SubConn. // It doesn't block and wait for the connections to be established. @@ -99,6 +112,9 @@ type ClientConn interface { // on the new picker to pick new SubConn. UpdateBalancerState(s connectivity.State, p Picker) + // ResolveNow is called by balancer to notify gRPC to do a name resolving. + ResolveNow(resolver.ResolveNowOption) + // Target returns the dial target for this ClientConn. Target() string } @@ -113,6 +129,8 @@ type BuildOptions struct { // to a remote load balancer server. The Balancer implementations // can ignore this if it doesn't need to talk to remote balancer. Dialer func(context.Context, string) (net.Conn, error) + // ChannelzParentID is the entity parent's channelz unique identification number. + ChannelzParentID int64 } // Builder creates a balancer. @@ -131,6 +149,10 @@ type PickOptions struct{} type DoneInfo struct { // Err is the rpc error the RPC finished with. It could be nil. Err error + // BytesSent indicates if any bytes have been sent to the server. + BytesSent bool + // BytesReceived indicates if any byte has been received from the server. + BytesReceived bool } var ( @@ -143,7 +165,7 @@ var ( ) // Picker is used by gRPC to pick a SubConn to send an RPC. -// Balancer is expected to generate a new picker from its snapshot everytime its +// Balancer is expected to generate a new picker from its snapshot every time its // internal state has changed. // // The pickers used by gRPC can be updated by ClientConn.UpdateBalancerState(). @@ -161,7 +183,7 @@ type Picker interface { // If a SubConn is returned: // - If it is READY, gRPC will send the RPC on it; // - If it is not ready, or becomes not ready after it's returned, gRPC will block - // this call until a new picker is updated and will call pick on the new picker. + // until UpdateBalancerState() is called and will call pick on the new picker. // // If the returned error is not nil: // - If the error is ErrNoSubConnAvailable, gRPC will block until UpdateBalancerState() @@ -204,3 +226,45 @@ type Balancer interface { // ClientConn.RemoveSubConn for its existing SubConns. Close() } + +// ConnectivityStateEvaluator takes the connectivity states of multiple SubConns +// and returns one aggregated connectivity state. +// +// It's not thread safe. +type ConnectivityStateEvaluator struct { + numReady uint64 // Number of addrConns in ready state. + numConnecting uint64 // Number of addrConns in connecting state. + numTransientFailure uint64 // Number of addrConns in transientFailure. +} + +// RecordTransition records state change happening in subConn and based on that +// it evaluates what aggregated state should be. +// +// - If at least one SubConn in Ready, the aggregated state is Ready; +// - Else if at least one SubConn in Connecting, the aggregated state is Connecting; +// - Else the aggregated state is TransientFailure. +// +// Idle and Shutdown are not considered. +func (cse *ConnectivityStateEvaluator) RecordTransition(oldState, newState connectivity.State) connectivity.State { + // Update counters. + for idx, state := range []connectivity.State{oldState, newState} { + updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. + switch state { + case connectivity.Ready: + cse.numReady += updateVal + case connectivity.Connecting: + cse.numConnecting += updateVal + case connectivity.TransientFailure: + cse.numTransientFailure += updateVal + } + } + + // Evaluate. + if cse.numReady > 0 { + return connectivity.Ready + } + if cse.numConnecting > 0 { + return connectivity.Connecting + } + return connectivity.TransientFailure +} diff --git a/vendor/google.golang.org/grpc/balancer/base/BUILD b/vendor/google.golang.org/grpc/balancer/base/BUILD new file mode 100644 index 00000000000..ca05c14c1c4 --- /dev/null +++ b/vendor/google.golang.org/grpc/balancer/base/BUILD @@ -0,0 +1,33 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "balancer.go", + "base.go", + ], + importmap = "k8s.io/kubernetes/vendor/google.golang.org/grpc/balancer/base", + importpath = "google.golang.org/grpc/balancer/base", + visibility = ["//visibility:public"], + deps = [ + "//vendor/golang.org/x/net/context:go_default_library", + "//vendor/google.golang.org/grpc/balancer:go_default_library", + "//vendor/google.golang.org/grpc/connectivity:go_default_library", + "//vendor/google.golang.org/grpc/grpclog:go_default_library", + "//vendor/google.golang.org/grpc/resolver:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/google.golang.org/grpc/balancer/base/balancer.go b/vendor/google.golang.org/grpc/balancer/base/balancer.go new file mode 100644 index 00000000000..23d13511bb2 --- /dev/null +++ b/vendor/google.golang.org/grpc/balancer/base/balancer.go @@ -0,0 +1,208 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package base + +import ( + "golang.org/x/net/context" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" +) + +type baseBuilder struct { + name string + pickerBuilder PickerBuilder +} + +func (bb *baseBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { + return &baseBalancer{ + cc: cc, + pickerBuilder: bb.pickerBuilder, + + subConns: make(map[resolver.Address]balancer.SubConn), + scStates: make(map[balancer.SubConn]connectivity.State), + csEvltr: &connectivityStateEvaluator{}, + // Initialize picker to a picker that always return + // ErrNoSubConnAvailable, because when state of a SubConn changes, we + // may call UpdateBalancerState with this picker. + picker: NewErrPicker(balancer.ErrNoSubConnAvailable), + } +} + +func (bb *baseBuilder) Name() string { + return bb.name +} + +type baseBalancer struct { + cc balancer.ClientConn + pickerBuilder PickerBuilder + + csEvltr *connectivityStateEvaluator + state connectivity.State + + subConns map[resolver.Address]balancer.SubConn + scStates map[balancer.SubConn]connectivity.State + picker balancer.Picker +} + +func (b *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) { + if err != nil { + grpclog.Infof("base.baseBalancer: HandleResolvedAddrs called with error %v", err) + return + } + grpclog.Infoln("base.baseBalancer: got new resolved addresses: ", addrs) + // addrsSet is the set converted from addrs, it's used for quick lookup of an address. + addrsSet := make(map[resolver.Address]struct{}) + for _, a := range addrs { + addrsSet[a] = struct{}{} + if _, ok := b.subConns[a]; !ok { + // a is a new address (not existing in b.subConns). + sc, err := b.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{}) + if err != nil { + grpclog.Warningf("base.baseBalancer: failed to create new SubConn: %v", err) + continue + } + b.subConns[a] = sc + b.scStates[sc] = connectivity.Idle + sc.Connect() + } + } + for a, sc := range b.subConns { + // a was removed by resolver. + if _, ok := addrsSet[a]; !ok { + b.cc.RemoveSubConn(sc) + delete(b.subConns, a) + // Keep the state of this sc in b.scStates until sc's state becomes Shutdown. + // The entry will be deleted in HandleSubConnStateChange. + } + } +} + +// regeneratePicker takes a snapshot of the balancer, and generates a picker +// from it. The picker is +// - errPicker with ErrTransientFailure if the balancer is in TransientFailure, +// - built by the pickerBuilder with all READY SubConns otherwise. +func (b *baseBalancer) regeneratePicker() { + if b.state == connectivity.TransientFailure { + b.picker = NewErrPicker(balancer.ErrTransientFailure) + return + } + readySCs := make(map[resolver.Address]balancer.SubConn) + + // Filter out all ready SCs from full subConn map. + for addr, sc := range b.subConns { + if st, ok := b.scStates[sc]; ok && st == connectivity.Ready { + readySCs[addr] = sc + } + } + b.picker = b.pickerBuilder.Build(readySCs) +} + +func (b *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { + grpclog.Infof("base.baseBalancer: handle SubConn state change: %p, %v", sc, s) + oldS, ok := b.scStates[sc] + if !ok { + grpclog.Infof("base.baseBalancer: got state changes for an unknown SubConn: %p, %v", sc, s) + return + } + b.scStates[sc] = s + switch s { + case connectivity.Idle: + sc.Connect() + case connectivity.Shutdown: + // When an address was removed by resolver, b called RemoveSubConn but + // kept the sc's state in scStates. Remove state for this sc here. + delete(b.scStates, sc) + } + + oldAggrState := b.state + b.state = b.csEvltr.recordTransition(oldS, s) + + // Regenerate picker when one of the following happens: + // - this sc became ready from not-ready + // - this sc became not-ready from ready + // - the aggregated state of balancer became TransientFailure from non-TransientFailure + // - the aggregated state of balancer became non-TransientFailure from TransientFailure + if (s == connectivity.Ready) != (oldS == connectivity.Ready) || + (b.state == connectivity.TransientFailure) != (oldAggrState == connectivity.TransientFailure) { + b.regeneratePicker() + } + + b.cc.UpdateBalancerState(b.state, b.picker) +} + +// Close is a nop because base balancer doesn't have internal state to clean up, +// and it doesn't need to call RemoveSubConn for the SubConns. +func (b *baseBalancer) Close() { +} + +// NewErrPicker returns a picker that always returns err on Pick(). +func NewErrPicker(err error) balancer.Picker { + return &errPicker{err: err} +} + +type errPicker struct { + err error // Pick() always returns this err. +} + +func (p *errPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { + return nil, nil, p.err +} + +// connectivityStateEvaluator gets updated by addrConns when their +// states transition, based on which it evaluates the state of +// ClientConn. +type connectivityStateEvaluator struct { + numReady uint64 // Number of addrConns in ready state. + numConnecting uint64 // Number of addrConns in connecting state. + numTransientFailure uint64 // Number of addrConns in transientFailure. +} + +// recordTransition records state change happening in every subConn and based on +// that it evaluates what aggregated state should be. +// It can only transition between Ready, Connecting and TransientFailure. Other states, +// Idle and Shutdown are transitioned into by ClientConn; in the beginning of the connection +// before any subConn is created ClientConn is in idle state. In the end when ClientConn +// closes it is in Shutdown state. +// +// recordTransition should only be called synchronously from the same goroutine. +func (cse *connectivityStateEvaluator) recordTransition(oldState, newState connectivity.State) connectivity.State { + // Update counters. + for idx, state := range []connectivity.State{oldState, newState} { + updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. + switch state { + case connectivity.Ready: + cse.numReady += updateVal + case connectivity.Connecting: + cse.numConnecting += updateVal + case connectivity.TransientFailure: + cse.numTransientFailure += updateVal + } + } + + // Evaluate. + if cse.numReady > 0 { + return connectivity.Ready + } + if cse.numConnecting > 0 { + return connectivity.Connecting + } + return connectivity.TransientFailure +} diff --git a/vendor/google.golang.org/grpc/balancer/base/base.go b/vendor/google.golang.org/grpc/balancer/base/base.go new file mode 100644 index 00000000000..012ace2f2f7 --- /dev/null +++ b/vendor/google.golang.org/grpc/balancer/base/base.go @@ -0,0 +1,52 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package base defines a balancer base that can be used to build balancers with +// different picking algorithms. +// +// The base balancer creates a new SubConn for each resolved address. The +// provided picker will only be notified about READY SubConns. +// +// This package is the base of round_robin balancer, its purpose is to be used +// to build round_robin like balancers with complex picking algorithms. +// Balancers with more complicated logic should try to implement a balancer +// builder from scratch. +// +// All APIs in this package are experimental. +package base + +import ( + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" +) + +// PickerBuilder creates balancer.Picker. +type PickerBuilder interface { + // Build takes a slice of ready SubConns, and returns a picker that will be + // used by gRPC to pick a SubConn. + Build(readySCs map[resolver.Address]balancer.SubConn) balancer.Picker +} + +// NewBalancerBuilder returns a balancer builder. The balancers +// built by this builder will use the picker builder to build pickers. +func NewBalancerBuilder(name string, pb PickerBuilder) balancer.Builder { + return &baseBuilder{ + name: name, + pickerBuilder: pb, + } +} diff --git a/vendor/google.golang.org/grpc/balancer/roundrobin/BUILD b/vendor/google.golang.org/grpc/balancer/roundrobin/BUILD new file mode 100644 index 00000000000..a390243bb67 --- /dev/null +++ b/vendor/google.golang.org/grpc/balancer/roundrobin/BUILD @@ -0,0 +1,30 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["roundrobin.go"], + importmap = "k8s.io/kubernetes/vendor/google.golang.org/grpc/balancer/roundrobin", + importpath = "google.golang.org/grpc/balancer/roundrobin", + visibility = ["//visibility:public"], + deps = [ + "//vendor/golang.org/x/net/context:go_default_library", + "//vendor/google.golang.org/grpc/balancer:go_default_library", + "//vendor/google.golang.org/grpc/balancer/base:go_default_library", + "//vendor/google.golang.org/grpc/grpclog:go_default_library", + "//vendor/google.golang.org/grpc/resolver:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go b/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go new file mode 100644 index 00000000000..2eda0a1c210 --- /dev/null +++ b/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go @@ -0,0 +1,79 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package roundrobin defines a roundrobin balancer. Roundrobin balancer is +// installed as one of the default balancers in gRPC, users don't need to +// explicitly install this balancer. +package roundrobin + +import ( + "sync" + + "golang.org/x/net/context" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" +) + +// Name is the name of round_robin balancer. +const Name = "round_robin" + +// newBuilder creates a new roundrobin balancer builder. +func newBuilder() balancer.Builder { + return base.NewBalancerBuilder(Name, &rrPickerBuilder{}) +} + +func init() { + balancer.Register(newBuilder()) +} + +type rrPickerBuilder struct{} + +func (*rrPickerBuilder) Build(readySCs map[resolver.Address]balancer.SubConn) balancer.Picker { + grpclog.Infof("roundrobinPicker: newPicker called with readySCs: %v", readySCs) + var scs []balancer.SubConn + for _, sc := range readySCs { + scs = append(scs, sc) + } + return &rrPicker{ + subConns: scs, + } +} + +type rrPicker struct { + // subConns is the snapshot of the roundrobin balancer when this picker was + // created. The slice is immutable. Each Get() will do a round robin + // selection from it and return the selected SubConn. + subConns []balancer.SubConn + + mu sync.Mutex + next int +} + +func (p *rrPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { + if len(p.subConns) <= 0 { + return nil, nil, balancer.ErrNoSubConnAvailable + } + + p.mu.Lock() + sc := p.subConns[p.next] + p.next = (p.next + 1) % len(p.subConns) + p.mu.Unlock() + return sc, nil, nil +} diff --git a/vendor/google.golang.org/grpc/balancer_conn_wrappers.go b/vendor/google.golang.org/grpc/balancer_conn_wrappers.go index f5dbc4ba201..c23f81706fb 100644 --- a/vendor/google.golang.org/grpc/balancer_conn_wrappers.go +++ b/vendor/google.golang.org/grpc/balancer_conn_wrappers.go @@ -19,6 +19,7 @@ package grpc import ( + "fmt" "sync" "google.golang.org/grpc/balancer" @@ -73,7 +74,7 @@ func (b *scStateUpdateBuffer) load() { } } -// get returns the channel that receives a recvMsg in the buffer. +// get returns the channel that the scStateUpdate will be sent to. // // Upon receiving, the caller should call load to send another // scStateChangeTuple onto the channel if there is any. @@ -96,6 +97,9 @@ type ccBalancerWrapper struct { stateChangeQueue *scStateUpdateBuffer resolverUpdateCh chan *resolverUpdate done chan struct{} + + mu sync.Mutex + subConns map[*acBalancerWrapper]struct{} } func newCCBalancerWrapper(cc *ClientConn, b balancer.Builder, bopts balancer.BuildOptions) *ccBalancerWrapper { @@ -104,21 +108,34 @@ func newCCBalancerWrapper(cc *ClientConn, b balancer.Builder, bopts balancer.Bui stateChangeQueue: newSCStateUpdateBuffer(), resolverUpdateCh: make(chan *resolverUpdate, 1), done: make(chan struct{}), + subConns: make(map[*acBalancerWrapper]struct{}), } go ccb.watcher() ccb.balancer = b.Build(ccb, bopts) return ccb } -// watcher balancer functions sequencially, so the balancer can be implemeneted +// watcher balancer functions sequentially, so the balancer can be implemented // lock-free. func (ccb *ccBalancerWrapper) watcher() { for { select { case t := <-ccb.stateChangeQueue.get(): ccb.stateChangeQueue.load() + select { + case <-ccb.done: + ccb.balancer.Close() + return + default: + } ccb.balancer.HandleSubConnStateChange(t.sc, t.state) case t := <-ccb.resolverUpdateCh: + select { + case <-ccb.done: + ccb.balancer.Close() + return + default: + } ccb.balancer.HandleResolvedAddrs(t.addrs, t.err) case <-ccb.done: } @@ -126,6 +143,13 @@ func (ccb *ccBalancerWrapper) watcher() { select { case <-ccb.done: ccb.balancer.Close() + ccb.mu.Lock() + scs := ccb.subConns + ccb.subConns = nil + ccb.mu.Unlock() + for acbw := range scs { + ccb.cc.removeAddrConn(acbw.getAddrConn(), errConnDrain) + } return default: } @@ -165,33 +189,54 @@ func (ccb *ccBalancerWrapper) handleResolvedAddrs(addrs []resolver.Address, err } func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { - grpclog.Infof("ccBalancerWrapper: new subconn: %v", addrs) + if len(addrs) <= 0 { + return nil, fmt.Errorf("grpc: cannot create SubConn with empty address list") + } + ccb.mu.Lock() + defer ccb.mu.Unlock() + if ccb.subConns == nil { + return nil, fmt.Errorf("grpc: ClientConn balancer wrapper was closed") + } ac, err := ccb.cc.newAddrConn(addrs) if err != nil { return nil, err } acbw := &acBalancerWrapper{ac: ac} - ac.mu.Lock() + acbw.ac.mu.Lock() ac.acbw = acbw - ac.mu.Unlock() + acbw.ac.mu.Unlock() + ccb.subConns[acbw] = struct{}{} return acbw, nil } func (ccb *ccBalancerWrapper) RemoveSubConn(sc balancer.SubConn) { - grpclog.Infof("ccBalancerWrapper: removing subconn") acbw, ok := sc.(*acBalancerWrapper) if !ok { return } + ccb.mu.Lock() + defer ccb.mu.Unlock() + if ccb.subConns == nil { + return + } + delete(ccb.subConns, acbw) ccb.cc.removeAddrConn(acbw.getAddrConn(), errConnDrain) } func (ccb *ccBalancerWrapper) UpdateBalancerState(s connectivity.State, p balancer.Picker) { - grpclog.Infof("ccBalancerWrapper: updating state and picker called by balancer: %v, %p", s, p) + ccb.mu.Lock() + defer ccb.mu.Unlock() + if ccb.subConns == nil { + return + } ccb.cc.csMgr.updateState(s) ccb.cc.blockingpicker.updatePicker(p) } +func (ccb *ccBalancerWrapper) ResolveNow(o resolver.ResolveNowOption) { + ccb.cc.resolveNow(o) +} + func (ccb *ccBalancerWrapper) Target() string { return ccb.cc.target } @@ -204,9 +249,12 @@ type acBalancerWrapper struct { } func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) { - grpclog.Infof("acBalancerWrapper: UpdateAddresses called with %v", addrs) acbw.mu.Lock() defer acbw.mu.Unlock() + if len(addrs) <= 0 { + acbw.ac.tearDown(errConnDrain) + return + } if !acbw.ac.tryUpdateAddrs(addrs) { cc := acbw.ac.cc acbw.ac.mu.Lock() @@ -234,7 +282,7 @@ func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) { ac.acbw = acbw ac.mu.Unlock() if acState != connectivity.Idle { - ac.connect(false) + ac.connect() } } } @@ -242,7 +290,7 @@ func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) { func (acbw *acBalancerWrapper) Connect() { acbw.mu.Lock() defer acbw.mu.Unlock() - acbw.ac.connect(false) + acbw.ac.connect() } func (acbw *acBalancerWrapper) getAddrConn() *addrConn { diff --git a/vendor/google.golang.org/grpc/balancer_v1_wrapper.go b/vendor/google.golang.org/grpc/balancer_v1_wrapper.go index 9d0616080a1..e0ce32cfb6d 100644 --- a/vendor/google.golang.org/grpc/balancer_v1_wrapper.go +++ b/vendor/google.golang.org/grpc/balancer_v1_wrapper.go @@ -19,6 +19,7 @@ package grpc import ( + "strings" "sync" "golang.org/x/net/context" @@ -27,6 +28,7 @@ import ( "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" ) type balancerWrapperBuilder struct { @@ -34,20 +36,27 @@ type balancerWrapperBuilder struct { } func (bwb *balancerWrapperBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - bwb.b.Start(cc.Target(), BalancerConfig{ + targetAddr := cc.Target() + targetSplitted := strings.Split(targetAddr, ":///") + if len(targetSplitted) >= 2 { + targetAddr = targetSplitted[1] + } + + bwb.b.Start(targetAddr, BalancerConfig{ DialCreds: opts.DialCreds, Dialer: opts.Dialer, }) _, pickfirst := bwb.b.(*pickFirst) bw := &balancerWrapper{ - balancer: bwb.b, - pickfirst: pickfirst, - cc: cc, - startCh: make(chan struct{}), - conns: make(map[resolver.Address]balancer.SubConn), - connSt: make(map[balancer.SubConn]*scState), - csEvltr: &connectivityStateEvaluator{}, - state: connectivity.Idle, + balancer: bwb.b, + pickfirst: pickfirst, + cc: cc, + targetAddr: targetAddr, + startCh: make(chan struct{}), + conns: make(map[resolver.Address]balancer.SubConn), + connSt: make(map[balancer.SubConn]*scState), + csEvltr: &balancer.ConnectivityStateEvaluator{}, + state: connectivity.Idle, } cc.UpdateBalancerState(connectivity.Idle, bw) go bw.lbWatcher() @@ -68,11 +77,8 @@ type balancerWrapper struct { balancer Balancer // The v1 balancer. pickfirst bool - cc balancer.ClientConn - - // To aggregate the connectivity state. - csEvltr *connectivityStateEvaluator - state connectivity.State + cc balancer.ClientConn + targetAddr string // Target without the scheme. mu sync.Mutex conns map[resolver.Address]balancer.SubConn @@ -82,18 +88,21 @@ type balancerWrapper struct { // - NewSubConn is created, cc wants to notify balancer of state changes; // - Build hasn't return, cc doesn't have access to balancer. startCh chan struct{} + + // To aggregate the connectivity state. + csEvltr *balancer.ConnectivityStateEvaluator + state connectivity.State } // lbWatcher watches the Notify channel of the balancer and manages // connections accordingly. func (bw *balancerWrapper) lbWatcher() { <-bw.startCh - grpclog.Infof("balancerWrapper: is pickfirst: %v\n", bw.pickfirst) notifyCh := bw.balancer.Notify() if notifyCh == nil { // There's no resolver in the balancer. Connect directly. a := resolver.Address{ - Addr: bw.cc.Target(), + Addr: bw.targetAddr, Type: resolver.Backend, } sc, err := bw.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{}) @@ -103,7 +112,7 @@ func (bw *balancerWrapper) lbWatcher() { bw.mu.Lock() bw.conns[a] = sc bw.connSt[sc] = &scState{ - addr: Address{Addr: bw.cc.Target()}, + addr: Address{Addr: bw.targetAddr}, s: connectivity.Idle, } bw.mu.Unlock() @@ -165,10 +174,10 @@ func (bw *balancerWrapper) lbWatcher() { sc.Connect() } } else { - oldSC.UpdateAddresses(newAddrs) bw.mu.Lock() bw.connSt[oldSC].addr = addrs[0] bw.mu.Unlock() + oldSC.UpdateAddresses(newAddrs) } } else { var ( @@ -221,7 +230,6 @@ func (bw *balancerWrapper) lbWatcher() { } func (bw *balancerWrapper) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { - grpclog.Infof("balancerWrapper: handle subconn state change: %p, %v", sc, s) bw.mu.Lock() defer bw.mu.Unlock() scSt, ok := bw.connSt[sc] @@ -240,7 +248,7 @@ func (bw *balancerWrapper) HandleSubConnStateChange(sc balancer.SubConn, s conne scSt.down(errConnClosing) } } - sa := bw.csEvltr.recordTransition(oldS, s) + sa := bw.csEvltr.RecordTransition(oldS, s) if bw.state != sa { bw.state = sa } @@ -249,7 +257,6 @@ func (bw *balancerWrapper) HandleSubConnStateChange(sc balancer.SubConn, s conne // Remove state for this sc. delete(bw.connSt, sc) } - return } func (bw *balancerWrapper) HandleResolvedAddrs([]resolver.Address, error) { @@ -262,7 +269,6 @@ func (bw *balancerWrapper) HandleResolvedAddrs([]resolver.Address, error) { } // There should be a resolver inside the balancer. // All updates here, if any, are ignored. - return } func (bw *balancerWrapper) Close() { @@ -274,7 +280,6 @@ func (bw *balancerWrapper) Close() { close(bw.startCh) } bw.balancer.Close() - return } // The picker is the balancerWrapper itself. @@ -310,58 +315,14 @@ func (bw *balancerWrapper) Pick(ctx context.Context, opts balancer.PickOptions) Metadata: a.Metadata, }] if !ok && failfast { - return nil, nil, Errorf(codes.Unavailable, "there is no connection available") + return nil, nil, status.Errorf(codes.Unavailable, "there is no connection available") } if s, ok := bw.connSt[sc]; failfast && (!ok || s.s != connectivity.Ready) { // If the returned sc is not ready and RPC is failfast, // return error, and this RPC will fail. - return nil, nil, Errorf(codes.Unavailable, "there is no connection available") + return nil, nil, status.Errorf(codes.Unavailable, "there is no connection available") } } return sc, done, nil } - -// connectivityStateEvaluator gets updated by addrConns when their -// states transition, based on which it evaluates the state of -// ClientConn. -type connectivityStateEvaluator struct { - mu sync.Mutex - numReady uint64 // Number of addrConns in ready state. - numConnecting uint64 // Number of addrConns in connecting state. - numTransientFailure uint64 // Number of addrConns in transientFailure. -} - -// recordTransition records state change happening in every subConn and based on -// that it evaluates what aggregated state should be. -// It can only transition between Ready, Connecting and TransientFailure. Other states, -// Idle and Shutdown are transitioned into by ClientConn; in the beginning of the connection -// before any subConn is created ClientConn is in idle state. In the end when ClientConn -// closes it is in Shutdown state. -// TODO Note that in later releases, a ClientConn with no activity will be put into an Idle state. -func (cse *connectivityStateEvaluator) recordTransition(oldState, newState connectivity.State) connectivity.State { - cse.mu.Lock() - defer cse.mu.Unlock() - - // Update counters. - for idx, state := range []connectivity.State{oldState, newState} { - updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. - switch state { - case connectivity.Ready: - cse.numReady += updateVal - case connectivity.Connecting: - cse.numConnecting += updateVal - case connectivity.TransientFailure: - cse.numTransientFailure += updateVal - } - } - - // Evaluate. - if cse.numReady > 0 { - return connectivity.Ready - } - if cse.numConnecting > 0 { - return connectivity.Connecting - } - return connectivity.TransientFailure -} diff --git a/vendor/google.golang.org/grpc/call.go b/vendor/google.golang.org/grpc/call.go index 1ef2507c35f..f73b7d5528f 100644 --- a/vendor/google.golang.org/grpc/call.go +++ b/vendor/google.golang.org/grpc/call.go @@ -19,289 +19,75 @@ package grpc import ( - "bytes" - "io" - "time" - "golang.org/x/net/context" - "golang.org/x/net/trace" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/peer" - "google.golang.org/grpc/stats" - "google.golang.org/grpc/status" - "google.golang.org/grpc/transport" ) -// recvResponse receives and parses an RPC response. -// On error, it returns the error and indicates whether the call should be retried. +// Invoke sends the RPC request on the wire and returns after response is +// received. This is typically called by generated code. // -// TODO(zhaoq): Check whether the received message sequence is valid. -// TODO ctx is used for stats collection and processing. It is the context passed from the application. -func recvResponse(ctx context.Context, dopts dialOptions, t transport.ClientTransport, c *callInfo, stream *transport.Stream, reply interface{}) (err error) { - // Try to acquire header metadata from the server if there is any. - defer func() { - if err != nil { - if _, ok := err.(transport.ConnectionError); !ok { - t.CloseStream(stream, err) - } - } - }() - c.headerMD, err = stream.Header() - if err != nil { - return - } - p := &parser{r: stream} - var inPayload *stats.InPayload - if dopts.copts.StatsHandler != nil { - inPayload = &stats.InPayload{ - Client: true, - } - } - for { - if c.maxReceiveMessageSize == nil { - return Errorf(codes.Internal, "callInfo maxReceiveMessageSize field uninitialized(nil)") - } - if err = recv(p, dopts.codec, stream, dopts.dc, reply, *c.maxReceiveMessageSize, inPayload); err != nil { - if err == io.EOF { - break - } - return - } - } - if inPayload != nil && err == io.EOF && stream.Status().Code() == codes.OK { - // TODO in the current implementation, inTrailer may be handled before inPayload in some cases. - // Fix the order if necessary. - dopts.copts.StatsHandler.HandleRPC(ctx, inPayload) - } - c.trailerMD = stream.Trailer() - return nil -} +// All errors returned by Invoke are compatible with the status package. +func (cc *ClientConn) Invoke(ctx context.Context, method string, args, reply interface{}, opts ...CallOption) error { + // allow interceptor to see all applicable call options, which means those + // configured as defaults from dial option as well as per-call options + opts = combine(cc.dopts.callOptions, opts) -// sendRequest writes out various information of an RPC such as Context and Message. -func sendRequest(ctx context.Context, dopts dialOptions, compressor Compressor, c *callInfo, callHdr *transport.CallHdr, stream *transport.Stream, t transport.ClientTransport, args interface{}, opts *transport.Options) (err error) { - defer func() { - if err != nil { - // If err is connection error, t will be closed, no need to close stream here. - if _, ok := err.(transport.ConnectionError); !ok { - t.CloseStream(stream, err) - } - } - }() - var ( - cbuf *bytes.Buffer - outPayload *stats.OutPayload - ) - if compressor != nil { - cbuf = new(bytes.Buffer) - } - if dopts.copts.StatsHandler != nil { - outPayload = &stats.OutPayload{ - Client: true, - } - } - hdr, data, err := encode(dopts.codec, args, compressor, cbuf, outPayload) - if err != nil { - return err - } - if c.maxSendMessageSize == nil { - return Errorf(codes.Internal, "callInfo maxSendMessageSize field uninitialized(nil)") - } - if len(data) > *c.maxSendMessageSize { - return Errorf(codes.ResourceExhausted, "grpc: trying to send message larger than max (%d vs. %d)", len(data), *c.maxSendMessageSize) - } - err = t.Write(stream, hdr, data, opts) - if err == nil && outPayload != nil { - outPayload.SentTime = time.Now() - dopts.copts.StatsHandler.HandleRPC(ctx, outPayload) - } - // t.NewStream(...) could lead to an early rejection of the RPC (e.g., the service/method - // does not exist.) so that t.Write could get io.EOF from wait(...). Leave the following - // recvResponse to get the final status. - if err != nil && err != io.EOF { - return err - } - // Sent successfully. - return nil -} - -// Invoke sends the RPC request on the wire and returns after response is received. -// Invoke is called by generated code. Also users can call Invoke directly when it -// is really needed in their use cases. -func Invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) error { if cc.dopts.unaryInt != nil { return cc.dopts.unaryInt(ctx, method, args, reply, cc, invoke, opts...) } return invoke(ctx, method, args, reply, cc, opts...) } -func invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) (e error) { - c := defaultCallInfo() - mc := cc.GetMethodConfig(method) - if mc.WaitForReady != nil { - c.failFast = !*mc.WaitForReady +func combine(o1 []CallOption, o2 []CallOption) []CallOption { + // we don't use append because o1 could have extra capacity whose + // elements would be overwritten, which could cause inadvertent + // sharing (and race connditions) between concurrent calls + if len(o1) == 0 { + return o2 + } else if len(o2) == 0 { + return o1 } + ret := make([]CallOption, len(o1)+len(o2)) + copy(ret, o1) + copy(ret[len(o1):], o2) + return ret +} - if mc.Timeout != nil && *mc.Timeout >= 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, *mc.Timeout) - defer cancel() - } +// Invoke sends the RPC request on the wire and returns after response is +// received. This is typically called by generated code. +// +// DEPRECATED: Use ClientConn.Invoke instead. +func Invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) error { + return cc.Invoke(ctx, method, args, reply, opts...) +} - opts = append(cc.dopts.callOptions, opts...) - for _, o := range opts { - if err := o.before(c); err != nil { - return toRPCErr(err) - } - } - defer func() { - for _, o := range opts { - o.after(c) - } - }() +var unaryStreamDesc = &StreamDesc{ServerStreams: false, ClientStreams: false} - c.maxSendMessageSize = getMaxSize(mc.MaxReqSize, c.maxSendMessageSize, defaultClientMaxSendMessageSize) - c.maxReceiveMessageSize = getMaxSize(mc.MaxRespSize, c.maxReceiveMessageSize, defaultClientMaxReceiveMessageSize) - - if EnableTracing { - c.traceInfo.tr = trace.New("grpc.Sent."+methodFamily(method), method) - defer c.traceInfo.tr.Finish() - c.traceInfo.firstLine.client = true - if deadline, ok := ctx.Deadline(); ok { - c.traceInfo.firstLine.deadline = deadline.Sub(time.Now()) - } - c.traceInfo.tr.LazyLog(&c.traceInfo.firstLine, false) - // TODO(dsymonds): Arrange for c.traceInfo.firstLine.remoteAddr to be set. - defer func() { - if e != nil { - c.traceInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{e}}, true) - c.traceInfo.tr.SetError() - } - }() - } - ctx = newContextWithRPCInfo(ctx, c.failFast) - sh := cc.dopts.copts.StatsHandler - if sh != nil { - ctx = sh.TagRPC(ctx, &stats.RPCTagInfo{FullMethodName: method, FailFast: c.failFast}) - begin := &stats.Begin{ - Client: true, - BeginTime: time.Now(), - FailFast: c.failFast, - } - sh.HandleRPC(ctx, begin) - defer func() { - end := &stats.End{ - Client: true, - EndTime: time.Now(), - Error: e, - } - sh.HandleRPC(ctx, end) - }() - } - topts := &transport.Options{ - Last: true, - Delay: false, - } +func invoke(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error { + // TODO: implement retries in clientStream and make this simply + // newClientStream, SendMsg, RecvMsg. + firstAttempt := true for { - var ( - err error - t transport.ClientTransport - stream *transport.Stream - // Record the done handler from Balancer.Get(...). It is called once the - // RPC has completed or failed. - done func(balancer.DoneInfo) - ) - // TODO(zhaoq): Need a formal spec of fail-fast. - callHdr := &transport.CallHdr{ - Host: cc.authority, - Method: method, - } - if cc.dopts.cp != nil { - callHdr.SendCompress = cc.dopts.cp.Type() - } - if c.creds != nil { - callHdr.Creds = c.creds - } - - t, done, err = cc.getTransport(ctx, c.failFast) + csInt, err := newClientStream(ctx, unaryStreamDesc, cc, method, opts...) if err != nil { - // TODO(zhaoq): Probably revisit the error handling. - if _, ok := status.FromError(err); ok { - return err - } - if err == errConnClosing || err == errConnUnavailable { - if c.failFast { - return Errorf(codes.Unavailable, "%v", err) - } + return err + } + cs := csInt.(*clientStream) + if err := cs.SendMsg(req); err != nil { + if !cs.c.failFast && cs.attempt.s.Unprocessed() && firstAttempt { + // TODO: Add a field to header for grpc-transparent-retry-attempts + firstAttempt = false continue } - // All the other errors are treated as Internal errors. - return Errorf(codes.Internal, "%v", err) + return err } - if c.traceInfo.tr != nil { - c.traceInfo.tr.LazyLog(&payload{sent: true, msg: args}, true) - } - stream, err = t.NewStream(ctx, callHdr) - if err != nil { - if done != nil { - if _, ok := err.(transport.ConnectionError); ok { - // If error is connection error, transport was sending data on wire, - // and we are not sure if anything has been sent on wire. - // If error is not connection error, we are sure nothing has been sent. - updateRPCInfoInContext(ctx, rpcInfo{bytesSent: true, bytesReceived: false}) - } - done(balancer.DoneInfo{Err: err}) - } - if _, ok := err.(transport.ConnectionError); (ok || err == transport.ErrStreamDrain) && !c.failFast { + if err := cs.RecvMsg(reply); err != nil { + if !cs.c.failFast && cs.attempt.s.Unprocessed() && firstAttempt { + // TODO: Add a field to header for grpc-transparent-retry-attempts + firstAttempt = false continue } - return toRPCErr(err) + return err } - if peer, ok := peer.FromContext(stream.Context()); ok { - c.peer = peer - } - err = sendRequest(ctx, cc.dopts, cc.dopts.cp, c, callHdr, stream, t, args, topts) - if err != nil { - if done != nil { - updateRPCInfoInContext(ctx, rpcInfo{ - bytesSent: stream.BytesSent(), - bytesReceived: stream.BytesReceived(), - }) - done(balancer.DoneInfo{Err: err}) - } - // Retry a non-failfast RPC when - // i) there is a connection error; or - // ii) the server started to drain before this RPC was initiated. - if _, ok := err.(transport.ConnectionError); (ok || err == transport.ErrStreamDrain) && !c.failFast { - continue - } - return toRPCErr(err) - } - err = recvResponse(ctx, cc.dopts, t, c, stream, reply) - if err != nil { - if done != nil { - updateRPCInfoInContext(ctx, rpcInfo{ - bytesSent: stream.BytesSent(), - bytesReceived: stream.BytesReceived(), - }) - done(balancer.DoneInfo{Err: err}) - } - if _, ok := err.(transport.ConnectionError); (ok || err == transport.ErrStreamDrain) && !c.failFast { - continue - } - return toRPCErr(err) - } - if c.traceInfo.tr != nil { - c.traceInfo.tr.LazyLog(&payload{sent: false, msg: reply}, true) - } - t.CloseStream(stream, nil) - if done != nil { - updateRPCInfoInContext(ctx, rpcInfo{ - bytesSent: stream.BytesSent(), - bytesReceived: stream.BytesReceived(), - }) - done(balancer.DoneInfo{Err: err}) - } - return stream.Status().Err() + return nil } } diff --git a/vendor/google.golang.org/grpc/clientconn.go b/vendor/google.golang.org/grpc/clientconn.go index 71de2e50d2b..84ba9e5ad72 100644 --- a/vendor/google.golang.org/grpc/clientconn.go +++ b/vendor/google.golang.org/grpc/clientconn.go @@ -31,24 +31,54 @@ import ( "golang.org/x/net/context" "golang.org/x/net/trace" "google.golang.org/grpc/balancer" + _ "google.golang.org/grpc/balancer/roundrobin" // To register roundrobin. + "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/resolver" + _ "google.golang.org/grpc/resolver/dns" // To register dns resolver. + _ "google.golang.org/grpc/resolver/passthrough" // To register passthrough resolver. "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" "google.golang.org/grpc/transport" ) +const ( + // minimum time to give a connection to complete + minConnectTimeout = 20 * time.Second + // must match grpclbName in grpclb/grpclb.go + grpclbName = "grpclb" +) + var ( // ErrClientConnClosing indicates that the operation is illegal because // the ClientConn is closing. - ErrClientConnClosing = errors.New("grpc: the client connection is closing") - // ErrClientConnTimeout indicates that the ClientConn cannot establish the - // underlying connections within the specified timeout. - // DEPRECATED: Please use context.DeadlineExceeded instead. - ErrClientConnTimeout = errors.New("grpc: timed out when dialing") + // + // Deprecated: this error should not be relied upon by users; use the status + // code of Canceled instead. + ErrClientConnClosing = status.Error(codes.Canceled, "grpc: the client connection is closing") + // errConnDrain indicates that the connection starts to be drained and does not accept any new RPCs. + errConnDrain = errors.New("grpc: the connection is drained") + // errConnClosing indicates that the connection is closing. + errConnClosing = errors.New("grpc: the connection is closing") + // errConnUnavailable indicates that the connection is unavailable. + errConnUnavailable = errors.New("grpc: the connection is unavailable") + // errBalancerClosed indicates that the balancer is closed. + errBalancerClosed = errors.New("grpc: balancer is closed") + // We use an accessor so that minConnectTimeout can be + // atomically read and updated while testing. + getMinConnectTimeout = func() time.Duration { + return minConnectTimeout + } +) +// The following errors are returned from Dial and DialContext +var ( // errNoTransportSecurity indicates that there is no transport security // being set for ClientConn. Users should either set one or explicitly // call WithInsecure DialOption to disable security. @@ -62,16 +92,6 @@ var ( errCredentialsConflict = errors.New("grpc: transport credentials are set for an insecure connection (grpc.WithTransportCredentials() and grpc.WithInsecure() are both called)") // errNetworkIO indicates that the connection is down due to some network I/O error. errNetworkIO = errors.New("grpc: failed with network I/O error") - // errConnDrain indicates that the connection starts to be drained and does not accept any new RPCs. - errConnDrain = errors.New("grpc: the connection is drained") - // errConnClosing indicates that the connection is closing. - errConnClosing = errors.New("grpc: the connection is closing") - // errConnUnavailable indicates that the connection is unavailable. - errConnUnavailable = errors.New("grpc: the connection is unavailable") - // errBalancerClosed indicates that the balancer is closed. - errBalancerClosed = errors.New("grpc: balancer is closed") - // minimum time to give a connection to complete - minConnectTimeout = 20 * time.Second ) // dialOptions configure a Dial call. dialOptions are set by the DialOption @@ -79,18 +99,23 @@ var ( type dialOptions struct { unaryInt UnaryClientInterceptor streamInt StreamClientInterceptor - codec Codec cp Compressor dc Decompressor - bs backoffStrategy + bs backoff.Strategy block bool insecure bool timeout time.Duration scChan <-chan ServiceConfig copts transport.ConnectOptions callOptions []CallOption - // This is to support v1 balancer. + // This is used by v1 balancer dial option WithBalancer to support v1 + // balancer, and also by WithBalancerName dial option. balancerBuilder balancer.Builder + // This is to support grpclb. + resolverBuilder resolver.Builder + waitForHandshake bool + channelzParentID int64 + disableServiceConfig bool } const ( @@ -98,9 +123,24 @@ const ( defaultClientMaxSendMessageSize = math.MaxInt32 ) +// RegisterChannelz turns on channelz service. +// This is an EXPERIMENTAL API. +func RegisterChannelz() { + channelz.TurnOn() +} + // DialOption configures how we set up the connection. type DialOption func(*dialOptions) +// WithWaitForHandshake blocks until the initial settings frame is received from the +// server before assigning RPCs to the connection. +// Experimental API. +func WithWaitForHandshake() DialOption { + return func(o *dialOptions) { + o.waitForHandshake = true + } +} + // WithWriteBufferSize lets you set the size of write buffer, this determines how much data can be batched // before doing a write on the wire. func WithWriteBufferSize(s int) DialOption { @@ -133,7 +173,9 @@ func WithInitialConnWindowSize(s int32) DialOption { } } -// WithMaxMsgSize returns a DialOption which sets the maximum message size the client can receive. Deprecated: use WithDefaultCallOptions(MaxCallRecvMsgSize(s)) instead. +// WithMaxMsgSize returns a DialOption which sets the maximum message size the client can receive. +// +// Deprecated: use WithDefaultCallOptions(MaxCallRecvMsgSize(s)) instead. func WithMaxMsgSize(s int) DialOption { return WithDefaultCallOptions(MaxCallRecvMsgSize(s)) } @@ -146,22 +188,32 @@ func WithDefaultCallOptions(cos ...CallOption) DialOption { } // WithCodec returns a DialOption which sets a codec for message marshaling and unmarshaling. +// +// Deprecated: use WithDefaultCallOptions(CallCustomCodec(c)) instead. func WithCodec(c Codec) DialOption { - return func(o *dialOptions) { - o.codec = c - } + return WithDefaultCallOptions(CallCustomCodec(c)) } -// WithCompressor returns a DialOption which sets a CompressorGenerator for generating message -// compressor. +// WithCompressor returns a DialOption which sets a Compressor to use for +// message compression. It has lower priority than the compressor set by +// the UseCompressor CallOption. +// +// Deprecated: use UseCompressor instead. func WithCompressor(cp Compressor) DialOption { return func(o *dialOptions) { o.cp = cp } } -// WithDecompressor returns a DialOption which sets a DecompressorGenerator for generating -// message decompressor. +// WithDecompressor returns a DialOption which sets a Decompressor to use for +// incoming message decompression. If incoming response messages are encoded +// using the decompressor's Type(), it will be used. Otherwise, the message +// encoding will be used to look up the compressor registered via +// encoding.RegisterCompressor, which will then be used to decompress the +// message. If no compressor is registered for the encoding, an Unimplemented +// status error will be returned. +// +// Deprecated: use encoding.RegisterCompressor instead. func WithDecompressor(dc Decompressor) DialOption { return func(o *dialOptions) { o.dc = dc @@ -170,7 +222,8 @@ func WithDecompressor(dc Decompressor) DialOption { // WithBalancer returns a DialOption which sets a load balancer with the v1 API. // Name resolver will be ignored if this DialOption is specified. -// Deprecated: use the new balancer APIs in balancer package instead. +// +// Deprecated: use the new balancer APIs in balancer package and WithBalancerName. func WithBalancer(b Balancer) DialOption { return func(o *dialOptions) { o.balancerBuilder = &balancerWrapperBuilder{ @@ -179,16 +232,35 @@ func WithBalancer(b Balancer) DialOption { } } -// WithBalancerBuilder is for testing only. Users using custom balancers should -// register their balancer and use service config to choose the balancer to use. -func WithBalancerBuilder(b balancer.Builder) DialOption { - // TODO(bar) remove this when switching balancer is done. +// WithBalancerName sets the balancer that the ClientConn will be initialized +// with. Balancer registered with balancerName will be used. This function +// panics if no balancer was registered by balancerName. +// +// The balancer cannot be overridden by balancer option specified by service +// config. +// +// This is an EXPERIMENTAL API. +func WithBalancerName(balancerName string) DialOption { + builder := balancer.Get(balancerName) + if builder == nil { + panic(fmt.Sprintf("grpc.WithBalancerName: no balancer is registered for name %v", balancerName)) + } return func(o *dialOptions) { - o.balancerBuilder = b + o.balancerBuilder = builder + } +} + +// withResolverBuilder is only for grpclb. +func withResolverBuilder(b resolver.Builder) DialOption { + return func(o *dialOptions) { + o.resolverBuilder = b } } // WithServiceConfig returns a DialOption which has a channel to read the service configuration. +// +// Deprecated: service config should be received through name resolver, as specified here. +// https://github.com/grpc/grpc/blob/master/doc/service_config.md func WithServiceConfig(c <-chan ServiceConfig) DialOption { return func(o *dialOptions) { o.scChan = c @@ -207,17 +279,17 @@ func WithBackoffMaxDelay(md time.Duration) DialOption { // Use WithBackoffMaxDelay until more parameters on BackoffConfig are opened up // for use. func WithBackoffConfig(b BackoffConfig) DialOption { - // Set defaults to ensure that provided BackoffConfig is valid and - // unexported fields get default values. - setDefaults(&b) - return withBackoff(b) + + return withBackoff(backoff.Exponential{ + MaxDelay: b.MaxDelay, + }) } -// withBackoff sets the backoff strategy used for retries after a +// withBackoff sets the backoff strategy used for connectRetryNum after a // failed connection attempt. // // This can be exported if arbitrary backoff strategies are allowed by gRPC. -func withBackoff(bs backoffStrategy) DialOption { +func withBackoff(bs backoff.Strategy) DialOption { return func(o *dialOptions) { o.bs = bs } @@ -258,6 +330,7 @@ func WithPerRPCCredentials(creds credentials.PerRPCCredentials) DialOption { // WithTimeout returns a DialOption that configures a timeout for dialing a ClientConn // initially. This is valid if and only if WithBlock() is present. +// // Deprecated: use DialContext and context.WithTimeout instead. func WithTimeout(d time.Duration) DialOption { return func(o *dialOptions) { @@ -265,18 +338,28 @@ func WithTimeout(d time.Duration) DialOption { } } +func withContextDialer(f func(context.Context, string) (net.Conn, error)) DialOption { + return func(o *dialOptions) { + o.copts.Dialer = f + } +} + +func init() { + internal.WithContextDialer = withContextDialer + internal.WithResolverBuilder = withResolverBuilder +} + // WithDialer returns a DialOption that specifies a function to use for dialing network addresses. // If FailOnNonTempDialError() is set to true, and an error is returned by f, gRPC checks the error's // Temporary() method to decide if it should try to reconnect to the network address. func WithDialer(f func(string, time.Duration) (net.Conn, error)) DialOption { - return func(o *dialOptions) { - o.copts.Dialer = func(ctx context.Context, addr string) (net.Conn, error) { + return withContextDialer( + func(ctx context.Context, addr string) (net.Conn, error) { if deadline, ok := ctx.Deadline(); ok { return f(addr, deadline.Sub(time.Now())) } return f(addr, 0) - } - } + }) } // WithStatsHandler returns a DialOption that specifies the stats handler @@ -335,15 +418,44 @@ func WithAuthority(a string) DialOption { } } +// WithChannelzParentID returns a DialOption that specifies the channelz ID of current ClientConn's +// parent. This function is used in nested channel creation (e.g. grpclb dial). +func WithChannelzParentID(id int64) DialOption { + return func(o *dialOptions) { + o.channelzParentID = id + } +} + +// WithDisableServiceConfig returns a DialOption that causes grpc to ignore any +// service config provided by the resolver and provides a hint to the resolver +// to not fetch service configs. +func WithDisableServiceConfig() DialOption { + return func(o *dialOptions) { + o.disableServiceConfig = true + } +} + // Dial creates a client connection to the given target. func Dial(target string, opts ...DialOption) (*ClientConn, error) { return DialContext(context.Background(), target, opts...) } -// DialContext creates a client connection to the given target. ctx can be used to -// cancel or expire the pending connection. Once this function returns, the -// cancellation and expiration of ctx will be noop. Users should call ClientConn.Close -// to terminate all the pending operations after this function returns. +// DialContext creates a client connection to the given target. By default, it's +// a non-blocking dial (the function won't wait for connections to be +// established, and connecting happens in the background). To make it a blocking +// dial, use WithBlock() dial option. +// +// In the non-blocking case, the ctx does not act against the connection. It +// only controls the setup steps. +// +// In the blocking case, ctx can be used to cancel or expire the pending +// connection. Once this function returns, the cancellation and expiration of +// ctx will be noop. Users should call ClientConn.Close to terminate all the +// pending operations after this function returns. +// +// The target name syntax is defined in +// https://github.com/grpc/grpc/blob/master/doc/naming.md. +// e.g. to use dns resolver, a "dns:///" prefix should be applied to the target. func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *ClientConn, err error) { cc := &ClientConn{ target: target, @@ -358,6 +470,14 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * opt(&cc.dopts) } + if channelz.IsOn() { + if cc.dopts.channelzParentID != 0 { + cc.channelzID = channelz.RegisterChannel(cc, cc.dopts.channelzParentID, target) + } else { + cc.channelzID = channelz.RegisterChannel(cc, 0, target) + } + } + if !cc.dopts.insecure { if cc.dopts.copts.TransportCredentials == nil { return nil, errNoTransportSecurity @@ -378,7 +498,8 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * if cc.dopts.copts.Dialer == nil { cc.dopts.copts.Dialer = newProxyDialer( func(ctx context.Context, addr string) (net.Conn, error) { - return (&net.Dialer{}).DialContext(ctx, "tcp", addr) + network, addr := parseDialTarget(addr) + return dialContext(ctx, network, addr) }, ) } @@ -419,12 +540,29 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * default: } } - // Set defaults. - if cc.dopts.codec == nil { - cc.dopts.codec = protoCodec{} - } if cc.dopts.bs == nil { - cc.dopts.bs = DefaultBackoffConfig + cc.dopts.bs = backoff.Exponential{ + MaxDelay: DefaultBackoffConfig.MaxDelay, + } + } + if cc.dopts.resolverBuilder == nil { + // Only try to parse target when resolver builder is not already set. + cc.parsedTarget = parseTarget(cc.target) + grpclog.Infof("parsed scheme: %q", cc.parsedTarget.Scheme) + cc.dopts.resolverBuilder = resolver.Get(cc.parsedTarget.Scheme) + if cc.dopts.resolverBuilder == nil { + // If resolver builder is still nil, the parse target's scheme is + // not registered. Fallback to default resolver and set Endpoint to + // the original unparsed target. + grpclog.Infof("scheme %q not registered, fallback to default scheme", cc.parsedTarget.Scheme) + cc.parsedTarget = resolver.Target{ + Scheme: resolver.GetDefaultScheme(), + Endpoint: target, + } + cc.dopts.resolverBuilder = resolver.Get(cc.parsedTarget.Scheme) + } + } else { + cc.parsedTarget = resolver.Target{Endpoint: target} } creds := cc.dopts.copts.TransportCredentials if creds != nil && creds.Info().ServerName != "" { @@ -432,45 +570,11 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * } else if cc.dopts.insecure && cc.dopts.copts.Authority != "" { cc.authority = cc.dopts.copts.Authority } else { - cc.authority = target + // Use endpoint from "scheme://authority/endpoint" as the default + // authority for ClientConn. + cc.authority = cc.parsedTarget.Endpoint } - if cc.dopts.balancerBuilder != nil { - var credsClone credentials.TransportCredentials - if creds != nil { - credsClone = creds.Clone() - } - buildOpts := balancer.BuildOptions{ - DialCreds: credsClone, - Dialer: cc.dopts.copts.Dialer, - } - // Build should not take long time. So it's ok to not have a goroutine for it. - // TODO(bar) init balancer after first resolver result to support service config balancer. - cc.balancerWrapper = newCCBalancerWrapper(cc, cc.dopts.balancerBuilder, buildOpts) - } else { - waitC := make(chan error, 1) - go func() { - defer close(waitC) - // No balancer, or no resolver within the balancer. Connect directly. - ac, err := cc.newAddrConn([]resolver.Address{{Addr: target}}) - if err != nil { - waitC <- err - return - } - if err := ac.connect(cc.dopts.block); err != nil { - waitC <- err - return - } - }() - select { - case <-ctx.Done(): - return nil, ctx.Err() - case err := <-waitC: - if err != nil { - return nil, err - } - } - } if cc.dopts.scChan != nil && !scSet { // Blocking wait for the initial service config. select { @@ -486,19 +590,29 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * go cc.scWatcher() } + var credsClone credentials.TransportCredentials + if creds := cc.dopts.copts.TransportCredentials; creds != nil { + credsClone = creds.Clone() + } + cc.balancerBuildOpts = balancer.BuildOptions{ + DialCreds: credsClone, + Dialer: cc.dopts.copts.Dialer, + ChannelzParentID: cc.channelzID, + } + // Build the resolver. cc.resolverWrapper, err = newCCResolverWrapper(cc) if err != nil { return nil, fmt.Errorf("failed to build resolver: %v", err) } - - if cc.balancerWrapper != nil && cc.resolverWrapper == nil { - // TODO(bar) there should always be a resolver (DNS as the default). - // Unblock balancer initialization with a fake resolver update if there's no resolver. - // The balancer wrapper will not read the addresses, so an empty list works. - // TODO(bar) remove this after the real resolver is started. - cc.balancerWrapper.handleResolvedAddrs([]resolver.Address{}, nil) - } + // Start the resolver wrapper goroutine after resolverWrapper is created. + // + // If the goroutine is started before resolverWrapper is ready, the + // following may happen: The goroutine sends updates to cc. cc forwards + // those to balancer. Balancer creates new addrConn. addrConn fails to + // connect, and calls resolveNow(). resolveNow() tries to use the non-ready + // resolverWrapper. + cc.resolverWrapper.start() // A blocking dial blocks until the clientConn is ready. if cc.dopts.block { @@ -565,21 +679,33 @@ type ClientConn struct { ctx context.Context cancel context.CancelFunc - target string - authority string - dopts dialOptions - csMgr *connectivityStateManager + target string + parsedTarget resolver.Target + authority string + dopts dialOptions + csMgr *connectivityStateManager - balancerWrapper *ccBalancerWrapper - resolverWrapper *ccResolverWrapper - - blockingpicker *pickerWrapper + balancerBuildOpts balancer.BuildOptions + resolverWrapper *ccResolverWrapper + blockingpicker *pickerWrapper mu sync.RWMutex sc ServiceConfig + scRaw string conns map[*addrConn]struct{} // Keepalive parameter can be updated if a GoAway is received. - mkp keepalive.ClientParameters + mkp keepalive.ClientParameters + curBalancerName string + preBalancerName string // previous balancer name. + curAddresses []resolver.Address + balancerWrapper *ccBalancerWrapper + + channelzID int64 // channelz unique identification number + czmu sync.RWMutex + callsStarted int64 + callsSucceeded int64 + callsFailed int64 + lastCallStartedTime time.Time } // WaitForStateChange waits until the connectivity.State of ClientConn changes from sourceState or @@ -615,6 +741,7 @@ func (cc *ClientConn) scWatcher() { // TODO: load balance policy runtime change is ignored. // We may revist this decision in the future. cc.sc = sc + cc.scRaw = "" cc.mu.Unlock() case <-cc.ctx.Done(): return @@ -622,7 +749,115 @@ func (cc *ClientConn) scWatcher() { } } +func (cc *ClientConn) handleResolvedAddrs(addrs []resolver.Address, err error) { + cc.mu.Lock() + defer cc.mu.Unlock() + if cc.conns == nil { + // cc was closed. + return + } + + if reflect.DeepEqual(cc.curAddresses, addrs) { + return + } + + cc.curAddresses = addrs + + if cc.dopts.balancerBuilder == nil { + // Only look at balancer types and switch balancer if balancer dial + // option is not set. + var isGRPCLB bool + for _, a := range addrs { + if a.Type == resolver.GRPCLB { + isGRPCLB = true + break + } + } + var newBalancerName string + if isGRPCLB { + newBalancerName = grpclbName + } else { + // Address list doesn't contain grpclb address. Try to pick a + // non-grpclb balancer. + newBalancerName = cc.curBalancerName + // If current balancer is grpclb, switch to the previous one. + if newBalancerName == grpclbName { + newBalancerName = cc.preBalancerName + } + // The following could be true in two cases: + // - the first time handling resolved addresses + // (curBalancerName="") + // - the first time handling non-grpclb addresses + // (curBalancerName="grpclb", preBalancerName="") + if newBalancerName == "" { + newBalancerName = PickFirstBalancerName + } + } + cc.switchBalancer(newBalancerName) + } else if cc.balancerWrapper == nil { + // Balancer dial option was set, and this is the first time handling + // resolved addresses. Build a balancer with dopts.balancerBuilder. + cc.balancerWrapper = newCCBalancerWrapper(cc, cc.dopts.balancerBuilder, cc.balancerBuildOpts) + } + + cc.balancerWrapper.handleResolvedAddrs(addrs, nil) +} + +// switchBalancer starts the switching from current balancer to the balancer +// with the given name. +// +// It will NOT send the current address list to the new balancer. If needed, +// caller of this function should send address list to the new balancer after +// this function returns. +// +// Caller must hold cc.mu. +func (cc *ClientConn) switchBalancer(name string) { + if cc.conns == nil { + return + } + + if strings.ToLower(cc.curBalancerName) == strings.ToLower(name) { + return + } + + grpclog.Infof("ClientConn switching balancer to %q", name) + if cc.dopts.balancerBuilder != nil { + grpclog.Infoln("ignoring balancer switching: Balancer DialOption used instead") + return + } + // TODO(bar switching) change this to two steps: drain and close. + // Keep track of sc in wrapper. + if cc.balancerWrapper != nil { + cc.balancerWrapper.close() + } + // Clear all stickiness state. + cc.blockingpicker.clearStickinessState() + + builder := balancer.Get(name) + if builder == nil { + grpclog.Infof("failed to get balancer builder for: %v, using pick_first instead", name) + builder = newPickfirstBuilder() + } + cc.preBalancerName = cc.curBalancerName + cc.curBalancerName = builder.Name() + cc.balancerWrapper = newCCBalancerWrapper(cc, builder, cc.balancerBuildOpts) +} + +func (cc *ClientConn) handleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { + cc.mu.Lock() + if cc.conns == nil { + cc.mu.Unlock() + return + } + // TODO(bar switching) send updates to all balancer wrappers when balancer + // gracefully switching is supported. + cc.balancerWrapper.handleSubConnStateChange(sc, s) + cc.mu.Unlock() +} + // newAddrConn creates an addrConn for addrs and adds it to cc.conns. +// +// Caller needs to make sure len(addrs) > 0. func (cc *ClientConn) newAddrConn(addrs []resolver.Address) (*addrConn, error) { ac := &addrConn{ cc: cc, @@ -636,6 +871,9 @@ func (cc *ClientConn) newAddrConn(addrs []resolver.Address) (*addrConn, error) { cc.mu.Unlock() return nil, ErrClientConnClosing } + if channelz.IsOn() { + ac.channelzID = channelz.RegisterSubChannel(ac, cc.channelzID, "") + } cc.conns[ac] = struct{}{} cc.mu.Unlock() return ac, nil @@ -654,12 +892,48 @@ func (cc *ClientConn) removeAddrConn(ac *addrConn, err error) { ac.tearDown(err) } +// ChannelzMetric returns ChannelInternalMetric of current ClientConn. +// This is an EXPERIMENTAL API. +func (cc *ClientConn) ChannelzMetric() *channelz.ChannelInternalMetric { + state := cc.GetState() + cc.czmu.RLock() + defer cc.czmu.RUnlock() + return &channelz.ChannelInternalMetric{ + State: state, + Target: cc.target, + CallsStarted: cc.callsStarted, + CallsSucceeded: cc.callsSucceeded, + CallsFailed: cc.callsFailed, + LastCallStartedTimestamp: cc.lastCallStartedTime, + } +} + +func (cc *ClientConn) incrCallsStarted() { + cc.czmu.Lock() + cc.callsStarted++ + // TODO(yuxuanli): will make this a time.Time pointer improve performance? + cc.lastCallStartedTime = time.Now() + cc.czmu.Unlock() +} + +func (cc *ClientConn) incrCallsSucceeded() { + cc.czmu.Lock() + cc.callsSucceeded++ + cc.czmu.Unlock() +} + +func (cc *ClientConn) incrCallsFailed() { + cc.czmu.Lock() + cc.callsFailed++ + cc.czmu.Unlock() +} + // connect starts to creating transport and also starts the transport monitor // goroutine for this ac. // It does nothing if the ac is not IDLE. // TODO(bar) Move this to the addrConn section. // This was part of resetAddrConn, keep it here to make the diff look clean. -func (ac *addrConn) connect(block bool) error { +func (ac *addrConn) connect() error { ac.mu.Lock() if ac.state == connectivity.Shutdown { ac.mu.Unlock() @@ -670,39 +944,21 @@ func (ac *addrConn) connect(block bool) error { return nil } ac.state = connectivity.Connecting - if ac.cc.balancerWrapper != nil { - ac.cc.balancerWrapper.handleSubConnStateChange(ac.acbw, ac.state) - } else { - ac.cc.csMgr.updateState(ac.state) - } + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) ac.mu.Unlock() - if block { + // Start a goroutine connecting to the server asynchronously. + go func() { if err := ac.resetTransport(); err != nil { + grpclog.Warningf("Failed to dial %s: %v; please retry.", ac.addrs[0].Addr, err) if err != errConnClosing { + // Keep this ac in cc.conns, to get the reason it's torn down. ac.tearDown(err) } - if e, ok := err.(transport.ConnectionError); ok && !e.Temporary() { - return e.Origin() - } - return err + return } - // Start to monitor the error status of transport. - go ac.transportMonitor() - } else { - // Start a goroutine connecting to the server asynchronously. - go func() { - if err := ac.resetTransport(); err != nil { - grpclog.Warningf("Failed to dial %s: %v; please retry.", ac.addrs[0].Addr, err) - if err != errConnClosing { - // Keep this ac in cc.conns, to get the reason it's torn down. - ac.tearDown(err) - } - return - } - ac.transportMonitor() - }() - } + ac.transportMonitor() + }() return nil } @@ -731,6 +987,7 @@ func (ac *addrConn) tryUpdateAddrs(addrs []resolver.Address) bool { grpclog.Infof("addrConn: tryUpdateAddrs curAddrFound: %v", curAddrFound) if curAddrFound { ac.addrs = addrs + ac.reconnectIdx = 0 // Start reconnecting from beginning in the new list. } return curAddrFound @@ -741,7 +998,7 @@ func (ac *addrConn) tryUpdateAddrs(addrs []resolver.Address) bool { // the corresponding MethodConfig. // If there isn't an exact match for the input method, we look for the default config // under the service (i.e /service/). If there is a default MethodConfig for -// the serivce, we return it. +// the service, we return it. // Otherwise, we return an empty MethodConfig. func (cc *ClientConn) GetMethodConfig(method string) MethodConfig { // TODO: Avoid the locking here. @@ -750,37 +1007,12 @@ func (cc *ClientConn) GetMethodConfig(method string) MethodConfig { m, ok := cc.sc.Methods[method] if !ok { i := strings.LastIndex(method, "/") - m, _ = cc.sc.Methods[method[:i+1]] + m = cc.sc.Methods[method[:i+1]] } return m } func (cc *ClientConn) getTransport(ctx context.Context, failfast bool) (transport.ClientTransport, func(balancer.DoneInfo), error) { - if cc.balancerWrapper == nil { - // If balancer is nil, there should be only one addrConn available. - cc.mu.RLock() - if cc.conns == nil { - cc.mu.RUnlock() - // TODO this function returns toRPCErr and non-toRPCErr. Clean up - // the errors in ClientConn. - return nil, nil, toRPCErr(ErrClientConnClosing) - } - var ac *addrConn - for ac = range cc.conns { - // Break after the first iteration to get the first addrConn. - break - } - cc.mu.RUnlock() - if ac == nil { - return nil, nil, errConnClosing - } - t, err := ac.wait(ctx, false /*hasBalancer*/, failfast) - if err != nil { - return nil, nil, err - } - return t, nil, nil - } - t, done, err := cc.blockingpicker.pick(ctx, failfast, balancer.PickOptions{}) if err != nil { return nil, nil, toRPCErr(err) @@ -788,9 +1020,61 @@ func (cc *ClientConn) getTransport(ctx context.Context, failfast bool) (transpor return t, done, nil } +// handleServiceConfig parses the service config string in JSON format to Go native +// struct ServiceConfig, and store both the struct and the JSON string in ClientConn. +func (cc *ClientConn) handleServiceConfig(js string) error { + if cc.dopts.disableServiceConfig { + return nil + } + sc, err := parseServiceConfig(js) + if err != nil { + return err + } + cc.mu.Lock() + cc.scRaw = js + cc.sc = sc + if sc.LB != nil && *sc.LB != grpclbName { // "grpclb" is not a valid balancer option in service config. + if cc.curBalancerName == grpclbName { + // If current balancer is grpclb, there's at least one grpclb + // balancer address in the resolved list. Don't switch the balancer, + // but change the previous balancer name, so if a new resolved + // address list doesn't contain grpclb address, balancer will be + // switched to *sc.LB. + cc.preBalancerName = *sc.LB + } else { + cc.switchBalancer(*sc.LB) + cc.balancerWrapper.handleResolvedAddrs(cc.curAddresses, nil) + } + } + + if envConfigStickinessOn { + var newStickinessMDKey string + if sc.stickinessMetadataKey != nil && *sc.stickinessMetadataKey != "" { + newStickinessMDKey = *sc.stickinessMetadataKey + } + // newStickinessMDKey is "" if one of the following happens: + // - stickinessMetadataKey is set to "" + // - stickinessMetadataKey field doesn't exist in service config + cc.blockingpicker.updateStickinessMDKey(strings.ToLower(newStickinessMDKey)) + } + + cc.mu.Unlock() + return nil +} + +func (cc *ClientConn) resolveNow(o resolver.ResolveNowOption) { + cc.mu.RLock() + r := cc.resolverWrapper + cc.mu.RUnlock() + if r == nil { + return + } + go r.resolveNow(o) +} + // Close tears down the ClientConn and all underlying connections. func (cc *ClientConn) Close() error { - cc.cancel() + defer cc.cancel() cc.mu.Lock() if cc.conns == nil { @@ -800,17 +1084,28 @@ func (cc *ClientConn) Close() error { conns := cc.conns cc.conns = nil cc.csMgr.updateState(connectivity.Shutdown) + + rWrapper := cc.resolverWrapper + cc.resolverWrapper = nil + bWrapper := cc.balancerWrapper + cc.balancerWrapper = nil cc.mu.Unlock() + cc.blockingpicker.close() - if cc.resolverWrapper != nil { - cc.resolverWrapper.close() + + if rWrapper != nil { + rWrapper.close() } - if cc.balancerWrapper != nil { - cc.balancerWrapper.close() + if bWrapper != nil { + bWrapper.close() } + for ac := range conns { ac.tearDown(ErrClientConnClosing) } + if channelz.IsOn() { + channelz.RemoveEntry(cc.channelzID) + } return nil } @@ -819,15 +1114,16 @@ type addrConn struct { ctx context.Context cancel context.CancelFunc - cc *ClientConn - curAddr resolver.Address - addrs []resolver.Address - dopts dialOptions - events trace.EventLog - acbw balancer.SubConn + cc *ClientConn + addrs []resolver.Address + dopts dialOptions + events trace.EventLog + acbw balancer.SubConn - mu sync.Mutex - state connectivity.State + mu sync.Mutex + curAddr resolver.Address + reconnectIdx int // The index in addrs list to start reconnecting from. + state connectivity.State // ready is closed and becomes nil when a new transport is up or failed // due to timeout. ready chan struct{} @@ -835,13 +1131,28 @@ type addrConn struct { // The reason this addrConn is torn down. tearDownErr error + + connectRetryNum int + // backoffDeadline is the time until which resetTransport needs to + // wait before increasing connectRetryNum count. + backoffDeadline time.Time + // connectDeadline is the time by which all connection + // negotiations must complete. + connectDeadline time.Time + + channelzID int64 // channelz unique identification number + czmu sync.RWMutex + callsStarted int64 + callsSucceeded int64 + callsFailed int64 + lastCallStartedTime time.Time } // adjustParams updates parameters used to create transports upon // receiving a GoAway. func (ac *addrConn) adjustParams(r transport.GoAwayReason) { switch r { - case transport.TooManyPings: + case transport.GoAwayTooManyPings: v := 2 * ac.dopts.copts.KeepaliveParams.Time ac.cc.mu.Lock() if v > ac.cc.mkp.Time { @@ -869,6 +1180,15 @@ func (ac *addrConn) errorf(format string, a ...interface{}) { // resetTransport recreates a transport to the address for ac. The old // transport will close itself on error or when the clientconn is closed. +// The created transport must receive initial settings frame from the server. +// In case that doesn't happen, transportMonitor will kill the newly created +// transport after connectDeadline has expired. +// In case there was an error on the transport before the settings frame was +// received, resetTransport resumes connecting to backends after the one that +// was previously connected to. In case end of the list is reached, resetTransport +// backs off until the original deadline. +// If the DialOption WithWaitForHandshake was set, resetTrasport returns +// successfully only after server settings are received. // // TODO(bar) make sure all state transitions are valid. func (ac *addrConn) resetTransport() error { @@ -882,19 +1202,38 @@ func (ac *addrConn) resetTransport() error { ac.ready = nil } ac.transport = nil - ac.curAddr = resolver.Address{} + ridx := ac.reconnectIdx ac.mu.Unlock() ac.cc.mu.RLock() ac.dopts.copts.KeepaliveParams = ac.cc.mkp ac.cc.mu.RUnlock() - for retries := 0; ; retries++ { - sleepTime := ac.dopts.bs.backoff(retries) - timeout := minConnectTimeout + var backoffDeadline, connectDeadline time.Time + for connectRetryNum := 0; ; connectRetryNum++ { ac.mu.Lock() - if timeout < time.Duration(int(sleepTime)/len(ac.addrs)) { - timeout = time.Duration(int(sleepTime) / len(ac.addrs)) + if ac.backoffDeadline.IsZero() { + // This means either a successful HTTP2 connection was established + // or this is the first time this addrConn is trying to establish a + // connection. + backoffFor := ac.dopts.bs.Backoff(connectRetryNum) // time.Duration. + // This will be the duration that dial gets to finish. + dialDuration := getMinConnectTimeout() + if backoffFor > dialDuration { + // Give dial more time as we keep failing to connect. + dialDuration = backoffFor + } + start := time.Now() + backoffDeadline = start.Add(backoffFor) + connectDeadline = start.Add(dialDuration) + ridx = 0 // Start connecting from the beginning. + } else { + // Continue trying to connect with the same deadlines. + connectRetryNum = ac.connectRetryNum + backoffDeadline = ac.backoffDeadline + connectDeadline = ac.connectDeadline + ac.backoffDeadline = time.Time{} + ac.connectDeadline = time.Time{} + ac.connectRetryNum = 0 } - connectTime := time.Now() if ac.state == connectivity.Shutdown { ac.mu.Unlock() return errConnClosing @@ -902,116 +1241,178 @@ func (ac *addrConn) resetTransport() error { ac.printf("connecting") if ac.state != connectivity.Connecting { ac.state = connectivity.Connecting - // TODO(bar) remove condition once we always have a balancer. - if ac.cc.balancerWrapper != nil { - ac.cc.balancerWrapper.handleSubConnStateChange(ac.acbw, ac.state) - } else { - ac.cc.csMgr.updateState(ac.state) - } + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) } // copy ac.addrs in case of race addrsIter := make([]resolver.Address, len(ac.addrs)) copy(addrsIter, ac.addrs) copts := ac.dopts.copts ac.mu.Unlock() - for _, addr := range addrsIter { - ac.mu.Lock() - if ac.state == connectivity.Shutdown { - // ac.tearDown(...) has been invoked. - ac.mu.Unlock() - return errConnClosing - } - ac.mu.Unlock() - sinfo := transport.TargetInfo{ - Addr: addr.Addr, - Metadata: addr.Metadata, - } - newTransport, err := transport.NewClientTransport(ac.cc.ctx, sinfo, copts, timeout) - if err != nil { - if e, ok := err.(transport.ConnectionError); ok && !e.Temporary() { - ac.mu.Lock() - if ac.state != connectivity.Shutdown { - ac.state = connectivity.TransientFailure - if ac.cc.balancerWrapper != nil { - ac.cc.balancerWrapper.handleSubConnStateChange(ac.acbw, ac.state) - } else { - ac.cc.csMgr.updateState(ac.state) - } - } - ac.mu.Unlock() - return err - } - grpclog.Warningf("grpc: addrConn.resetTransport failed to create client transport: %v; Reconnecting to %v", err, addr) - ac.mu.Lock() - if ac.state == connectivity.Shutdown { - // ac.tearDown(...) has been invoked. - ac.mu.Unlock() - return errConnClosing - } - ac.mu.Unlock() - continue - } - ac.mu.Lock() - ac.printf("ready") - if ac.state == connectivity.Shutdown { - // ac.tearDown(...) has been invoked. - ac.mu.Unlock() - newTransport.Close() - return errConnClosing - } - ac.state = connectivity.Ready - if ac.cc.balancerWrapper != nil { - ac.cc.balancerWrapper.handleSubConnStateChange(ac.acbw, ac.state) - } else { - ac.cc.csMgr.updateState(ac.state) - } - t := ac.transport - ac.transport = newTransport - if t != nil { - t.Close() - } - ac.curAddr = addr - if ac.ready != nil { - close(ac.ready) - ac.ready = nil - } - ac.mu.Unlock() + connected, err := ac.createTransport(connectRetryNum, ridx, backoffDeadline, connectDeadline, addrsIter, copts) + if err != nil { + return err + } + if connected { return nil } - ac.mu.Lock() - ac.state = connectivity.TransientFailure - if ac.cc.balancerWrapper != nil { - ac.cc.balancerWrapper.handleSubConnStateChange(ac.acbw, ac.state) - } else { - ac.cc.csMgr.updateState(ac.state) + } +} + +// createTransport creates a connection to one of the backends in addrs. +// It returns true if a connection was established. +func (ac *addrConn) createTransport(connectRetryNum, ridx int, backoffDeadline, connectDeadline time.Time, addrs []resolver.Address, copts transport.ConnectOptions) (bool, error) { + for i := ridx; i < len(addrs); i++ { + addr := addrs[i] + target := transport.TargetInfo{ + Addr: addr.Addr, + Metadata: addr.Metadata, + Authority: ac.cc.authority, } + done := make(chan struct{}) + onPrefaceReceipt := func() { + ac.mu.Lock() + close(done) + if !ac.backoffDeadline.IsZero() { + // If we haven't already started reconnecting to + // other backends. + // Note, this can happen when writer notices an error + // and triggers resetTransport while at the same time + // reader receives the preface and invokes this closure. + ac.backoffDeadline = time.Time{} + ac.connectDeadline = time.Time{} + ac.connectRetryNum = 0 + } + ac.mu.Unlock() + } + // Do not cancel in the success path because of + // this issue in Go1.6: https://github.com/golang/go/issues/15078. + connectCtx, cancel := context.WithDeadline(ac.ctx, connectDeadline) + if channelz.IsOn() { + copts.ChannelzParentID = ac.channelzID + } + newTr, err := transport.NewClientTransport(connectCtx, ac.cc.ctx, target, copts, onPrefaceReceipt) + if err != nil { + cancel() + ac.cc.blockingpicker.updateConnectionError(err) + ac.mu.Lock() + if ac.state == connectivity.Shutdown { + // ac.tearDown(...) has been invoked. + ac.mu.Unlock() + return false, errConnClosing + } + ac.mu.Unlock() + grpclog.Warningf("grpc: addrConn.createTransport failed to connect to %v. Err :%v. Reconnecting...", addr, err) + continue + } + if ac.dopts.waitForHandshake { + select { + case <-done: + case <-connectCtx.Done(): + // Didn't receive server preface, must kill this new transport now. + grpclog.Warningf("grpc: addrConn.createTransport failed to receive server preface before deadline.") + newTr.Close() + break + case <-ac.ctx.Done(): + } + } + ac.mu.Lock() + if ac.state == connectivity.Shutdown { + ac.mu.Unlock() + // ac.tearDonn(...) has been invoked. + newTr.Close() + return false, errConnClosing + } + ac.printf("ready") + ac.state = connectivity.Ready + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) + ac.transport = newTr + ac.curAddr = addr if ac.ready != nil { close(ac.ready) ac.ready = nil } - ac.mu.Unlock() - timer := time.NewTimer(sleepTime - time.Since(connectTime)) select { - case <-timer.C: - case <-ac.ctx.Done(): - timer.Stop() - return ac.ctx.Err() + case <-done: + // If the server has responded back with preface already, + // don't set the reconnect parameters. + default: + ac.connectRetryNum = connectRetryNum + ac.backoffDeadline = backoffDeadline + ac.connectDeadline = connectDeadline + ac.reconnectIdx = i + 1 // Start reconnecting from the next backend in the list. } - timer.Stop() + ac.mu.Unlock() + return true, nil } + ac.mu.Lock() + if ac.state == connectivity.Shutdown { + ac.mu.Unlock() + return false, errConnClosing + } + ac.state = connectivity.TransientFailure + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) + ac.cc.resolveNow(resolver.ResolveNowOption{}) + if ac.ready != nil { + close(ac.ready) + ac.ready = nil + } + ac.mu.Unlock() + timer := time.NewTimer(backoffDeadline.Sub(time.Now())) + select { + case <-timer.C: + case <-ac.ctx.Done(): + timer.Stop() + return false, ac.ctx.Err() + } + return false, nil } // Run in a goroutine to track the error in transport and create the // new transport if an error happens. It returns when the channel is closing. func (ac *addrConn) transportMonitor() { for { + var timer *time.Timer + var cdeadline <-chan time.Time ac.mu.Lock() t := ac.transport + if !ac.connectDeadline.IsZero() { + timer = time.NewTimer(ac.connectDeadline.Sub(time.Now())) + cdeadline = timer.C + } ac.mu.Unlock() // Block until we receive a goaway or an error occurs. select { case <-t.GoAway(): + done := t.Error() + cleanup := t.Close + // Since this transport will be orphaned (won't have a transportMonitor) + // we need to launch a goroutine to keep track of clientConn.Close() + // happening since it might not be noticed by any other goroutine for a while. + go func() { + <-done + cleanup() + }() case <-t.Error(): + // In case this is triggered because clientConn.Close() + // was called, we want to immeditately close the transport + // since no other goroutine might notice it for a while. + t.Close() + case <-cdeadline: + ac.mu.Lock() + // This implies that client received server preface. + if ac.backoffDeadline.IsZero() { + ac.mu.Unlock() + continue + } + ac.mu.Unlock() + timer = nil + // No server preface received until deadline. + // Kill the connection. + grpclog.Warningf("grpc: addrConn.transportMonitor didn't get server preface after waiting. Closing the new transport now.") + t.Close() + } + if timer != nil { + timer.Stop() } // If a GoAway happened, regardless of error, adjust our keepalive // parameters as appropriate. @@ -1028,11 +1429,8 @@ func (ac *addrConn) transportMonitor() { // Set connectivity state to TransientFailure before calling // resetTransport. Transition READY->CONNECTING is not valid. ac.state = connectivity.TransientFailure - if ac.cc.balancerWrapper != nil { - ac.cc.balancerWrapper.handleSubConnStateChange(ac.acbw, ac.state) - } else { - ac.cc.csMgr.updateState(ac.state) - } + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) + ac.cc.resolveNow(resolver.ResolveNowOption{}) ac.curAddr = resolver.Address{} ac.mu.Unlock() if err := ac.resetTransport(); err != nil { @@ -1106,7 +1504,7 @@ func (ac *addrConn) getReadyTransport() (transport.ClientTransport, bool) { ac.mu.Unlock() // Trigger idle ac to connect. if idle { - ac.connect(false) + ac.connect() } return nil, false } @@ -1119,8 +1517,11 @@ func (ac *addrConn) getReadyTransport() (transport.ClientTransport, bool) { func (ac *addrConn) tearDown(err error) { ac.cancel() ac.mu.Lock() - ac.curAddr = resolver.Address{} defer ac.mu.Unlock() + if ac.state == connectivity.Shutdown { + return + } + ac.curAddr = resolver.Address{} if err == errConnDrain && ac.transport != nil { // GracefulClose(...) may be executed multiple times when // i) receiving multiple GoAway frames from the server; or @@ -1128,16 +1529,9 @@ func (ac *addrConn) tearDown(err error) { // address removal and GoAway. ac.transport.GracefulClose() } - if ac.state == connectivity.Shutdown { - return - } ac.state = connectivity.Shutdown ac.tearDownErr = err - if ac.cc.balancerWrapper != nil { - ac.cc.balancerWrapper.handleSubConnStateChange(ac.acbw, ac.state) - } else { - ac.cc.csMgr.updateState(ac.state) - } + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) if ac.events != nil { ac.events.Finish() ac.events = nil @@ -1146,7 +1540,9 @@ func (ac *addrConn) tearDown(err error) { close(ac.ready) ac.ready = nil } - return + if channelz.IsOn() { + channelz.RemoveEntry(ac.channelzID) + } } func (ac *addrConn) getState() connectivity.State { @@ -1154,3 +1550,53 @@ func (ac *addrConn) getState() connectivity.State { defer ac.mu.Unlock() return ac.state } + +func (ac *addrConn) getCurAddr() (ret resolver.Address) { + ac.mu.Lock() + ret = ac.curAddr + ac.mu.Unlock() + return +} + +func (ac *addrConn) ChannelzMetric() *channelz.ChannelInternalMetric { + ac.mu.Lock() + addr := ac.curAddr.Addr + ac.mu.Unlock() + state := ac.getState() + ac.czmu.RLock() + defer ac.czmu.RUnlock() + return &channelz.ChannelInternalMetric{ + State: state, + Target: addr, + CallsStarted: ac.callsStarted, + CallsSucceeded: ac.callsSucceeded, + CallsFailed: ac.callsFailed, + LastCallStartedTimestamp: ac.lastCallStartedTime, + } +} + +func (ac *addrConn) incrCallsStarted() { + ac.czmu.Lock() + ac.callsStarted++ + ac.lastCallStartedTime = time.Now() + ac.czmu.Unlock() +} + +func (ac *addrConn) incrCallsSucceeded() { + ac.czmu.Lock() + ac.callsSucceeded++ + ac.czmu.Unlock() +} + +func (ac *addrConn) incrCallsFailed() { + ac.czmu.Lock() + ac.callsFailed++ + ac.czmu.Unlock() +} + +// ErrClientConnTimeout indicates that the ClientConn cannot establish the +// underlying connections within the specified timeout. +// +// Deprecated: This error is never returned by grpc and should not be +// referenced by users. +var ErrClientConnTimeout = errors.New("grpc: timed out when dialing") diff --git a/vendor/google.golang.org/grpc/codec.go b/vendor/google.golang.org/grpc/codec.go index 905b048e2ac..12977654781 100644 --- a/vendor/google.golang.org/grpc/codec.go +++ b/vendor/google.golang.org/grpc/codec.go @@ -19,86 +19,32 @@ package grpc import ( - "math" - "sync" - - "github.com/golang/protobuf/proto" + "google.golang.org/grpc/encoding" + _ "google.golang.org/grpc/encoding/proto" // to register the Codec for "proto" ) +// baseCodec contains the functionality of both Codec and encoding.Codec, but +// omits the name/string, which vary between the two and are not needed for +// anything besides the registry in the encoding package. +type baseCodec interface { + Marshal(v interface{}) ([]byte, error) + Unmarshal(data []byte, v interface{}) error +} + +var _ baseCodec = Codec(nil) +var _ baseCodec = encoding.Codec(nil) + // Codec defines the interface gRPC uses to encode and decode messages. // Note that implementations of this interface must be thread safe; // a Codec's methods can be called from concurrent goroutines. +// +// Deprecated: use encoding.Codec instead. type Codec interface { // Marshal returns the wire format of v. Marshal(v interface{}) ([]byte, error) // Unmarshal parses the wire format into v. Unmarshal(data []byte, v interface{}) error - // String returns the name of the Codec implementation. The returned - // string will be used as part of content type in transmission. + // String returns the name of the Codec implementation. This is unused by + // gRPC. String() string } - -// protoCodec is a Codec implementation with protobuf. It is the default codec for gRPC. -type protoCodec struct { -} - -type cachedProtoBuffer struct { - lastMarshaledSize uint32 - proto.Buffer -} - -func capToMaxInt32(val int) uint32 { - if val > math.MaxInt32 { - return uint32(math.MaxInt32) - } - return uint32(val) -} - -func (p protoCodec) marshal(v interface{}, cb *cachedProtoBuffer) ([]byte, error) { - protoMsg := v.(proto.Message) - newSlice := make([]byte, 0, cb.lastMarshaledSize) - - cb.SetBuf(newSlice) - cb.Reset() - if err := cb.Marshal(protoMsg); err != nil { - return nil, err - } - out := cb.Bytes() - cb.lastMarshaledSize = capToMaxInt32(len(out)) - return out, nil -} - -func (p protoCodec) Marshal(v interface{}) ([]byte, error) { - cb := protoBufferPool.Get().(*cachedProtoBuffer) - out, err := p.marshal(v, cb) - - // put back buffer and lose the ref to the slice - cb.SetBuf(nil) - protoBufferPool.Put(cb) - return out, err -} - -func (p protoCodec) Unmarshal(data []byte, v interface{}) error { - cb := protoBufferPool.Get().(*cachedProtoBuffer) - cb.SetBuf(data) - v.(proto.Message).Reset() - err := cb.Unmarshal(v.(proto.Message)) - cb.SetBuf(nil) - protoBufferPool.Put(cb) - return err -} - -func (protoCodec) String() string { - return "proto" -} - -var ( - protoBufferPool = &sync.Pool{ - New: func() interface{} { - return &cachedProtoBuffer{ - Buffer: proto.Buffer{}, - lastMarshaledSize: 16, - } - }, - } -) diff --git a/vendor/google.golang.org/grpc/codes/code_string.go b/vendor/google.golang.org/grpc/codes/code_string.go index 259837060ab..0b206a57822 100644 --- a/vendor/google.golang.org/grpc/codes/code_string.go +++ b/vendor/google.golang.org/grpc/codes/code_string.go @@ -1,16 +1,62 @@ -// Code generated by "stringer -type=Code"; DO NOT EDIT. +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package codes -import "fmt" +import "strconv" -const _Code_name = "OKCanceledUnknownInvalidArgumentDeadlineExceededNotFoundAlreadyExistsPermissionDeniedResourceExhaustedFailedPreconditionAbortedOutOfRangeUnimplementedInternalUnavailableDataLossUnauthenticated" - -var _Code_index = [...]uint8{0, 2, 10, 17, 32, 48, 56, 69, 85, 102, 120, 127, 137, 150, 158, 169, 177, 192} - -func (i Code) String() string { - if i >= Code(len(_Code_index)-1) { - return fmt.Sprintf("Code(%d)", i) +func (c Code) String() string { + switch c { + case OK: + return "OK" + case Canceled: + return "Canceled" + case Unknown: + return "Unknown" + case InvalidArgument: + return "InvalidArgument" + case DeadlineExceeded: + return "DeadlineExceeded" + case NotFound: + return "NotFound" + case AlreadyExists: + return "AlreadyExists" + case PermissionDenied: + return "PermissionDenied" + case ResourceExhausted: + return "ResourceExhausted" + case FailedPrecondition: + return "FailedPrecondition" + case Aborted: + return "Aborted" + case OutOfRange: + return "OutOfRange" + case Unimplemented: + return "Unimplemented" + case Internal: + return "Internal" + case Unavailable: + return "Unavailable" + case DataLoss: + return "DataLoss" + case Unauthenticated: + return "Unauthenticated" + default: + return "Code(" + strconv.FormatInt(int64(c), 10) + ")" } - return _Code_name[_Code_index[i]:_Code_index[i+1]] } diff --git a/vendor/google.golang.org/grpc/codes/codes.go b/vendor/google.golang.org/grpc/codes/codes.go index 81fe7bf85b3..027e3ec823b 100644 --- a/vendor/google.golang.org/grpc/codes/codes.go +++ b/vendor/google.golang.org/grpc/codes/codes.go @@ -20,11 +20,14 @@ // consistent across various languages. package codes +import ( + "fmt" + "strconv" +) + // A Code is an unsigned 32-bit error code as defined in the gRPC spec. type Code uint32 -//go:generate stringer -type=Code - const ( // OK is returned on success. OK Code = 0 @@ -32,9 +35,9 @@ const ( // Canceled indicates the operation was canceled (typically by the caller). Canceled Code = 1 - // Unknown error. An example of where this error may be returned is + // Unknown error. An example of where this error may be returned is // if a Status value received from another address space belongs to - // an error-space that is not known in this address space. Also + // an error-space that is not known in this address space. Also // errors raised by APIs that do not return enough error information // may be converted to this error. Unknown Code = 2 @@ -63,15 +66,11 @@ const ( // PermissionDenied indicates the caller does not have permission to // execute the specified operation. It must not be used for rejections // caused by exhausting some resource (use ResourceExhausted - // instead for those errors). It must not be + // instead for those errors). It must not be // used if the caller cannot be identified (use Unauthenticated // instead for those errors). PermissionDenied Code = 7 - // Unauthenticated indicates the request does not have valid - // authentication credentials for the operation. - Unauthenticated Code = 16 - // ResourceExhausted indicates some resource has been exhausted, perhaps // a per-user quota, or perhaps the entire file system is out of space. ResourceExhausted Code = 8 @@ -87,7 +86,7 @@ const ( // (b) Use Aborted if the client should retry at a higher-level // (e.g., restarting a read-modify-write sequence). // (c) Use FailedPrecondition if the client should not retry until - // the system state has been explicitly fixed. E.g., if an "rmdir" + // the system state has been explicitly fixed. E.g., if an "rmdir" // fails because the directory is non-empty, FailedPrecondition // should be returned since the client should not retry unless // they have first fixed up the directory by deleting files from it. @@ -116,7 +115,7 @@ const ( // file size. // // There is a fair bit of overlap between FailedPrecondition and - // OutOfRange. We recommend using OutOfRange (the more specific + // OutOfRange. We recommend using OutOfRange (the more specific // error) when it applies so that callers who are iterating through // a space can easily look for an OutOfRange error to detect when // they are done. @@ -126,8 +125,8 @@ const ( // supported/enabled in this service. Unimplemented Code = 12 - // Internal errors. Means some invariants expected by underlying - // system has been broken. If you see one of these errors, + // Internal errors. Means some invariants expected by underlying + // system has been broken. If you see one of these errors, // something is very broken. Internal Code = 13 @@ -141,4 +140,58 @@ const ( // DataLoss indicates unrecoverable data loss or corruption. DataLoss Code = 15 + + // Unauthenticated indicates the request does not have valid + // authentication credentials for the operation. + Unauthenticated Code = 16 + + _maxCode = 17 ) + +var strToCode = map[string]Code{ + `"OK"`: OK, + `"CANCELLED"`:/* [sic] */ Canceled, + `"UNKNOWN"`: Unknown, + `"INVALID_ARGUMENT"`: InvalidArgument, + `"DEADLINE_EXCEEDED"`: DeadlineExceeded, + `"NOT_FOUND"`: NotFound, + `"ALREADY_EXISTS"`: AlreadyExists, + `"PERMISSION_DENIED"`: PermissionDenied, + `"RESOURCE_EXHAUSTED"`: ResourceExhausted, + `"FAILED_PRECONDITION"`: FailedPrecondition, + `"ABORTED"`: Aborted, + `"OUT_OF_RANGE"`: OutOfRange, + `"UNIMPLEMENTED"`: Unimplemented, + `"INTERNAL"`: Internal, + `"UNAVAILABLE"`: Unavailable, + `"DATA_LOSS"`: DataLoss, + `"UNAUTHENTICATED"`: Unauthenticated, +} + +// UnmarshalJSON unmarshals b into the Code. +func (c *Code) UnmarshalJSON(b []byte) error { + // From json.Unmarshaler: By convention, to approximate the behavior of + // Unmarshal itself, Unmarshalers implement UnmarshalJSON([]byte("null")) as + // a no-op. + if string(b) == "null" { + return nil + } + if c == nil { + return fmt.Errorf("nil receiver passed to UnmarshalJSON") + } + + if ci, err := strconv.ParseUint(string(b), 10, 32); err == nil { + if ci >= _maxCode { + return fmt.Errorf("invalid code: %q", ci) + } + + *c = Code(ci) + return nil + } + + if jc, ok := strToCode[string(b)]; ok { + *c = jc + return nil + } + return fmt.Errorf("invalid code: %q", string(b)) +} diff --git a/vendor/google.golang.org/grpc/credentials/credentials.go b/vendor/google.golang.org/grpc/credentials/credentials.go index 0ce766a4dcf..b918bf13ad4 100644 --- a/vendor/google.golang.org/grpc/credentials/credentials.go +++ b/vendor/google.golang.org/grpc/credentials/credentials.go @@ -34,10 +34,8 @@ import ( "golang.org/x/net/context" ) -var ( - // alpnProtoStr are the specified application level protocols for gRPC. - alpnProtoStr = []string{"h2"} -) +// alpnProtoStr are the specified application level protocols for gRPC. +var alpnProtoStr = []string{"h2"} // PerRPCCredentials defines the common interface for the credentials which need to // attach security information to every RPC (e.g., oauth2). @@ -45,8 +43,9 @@ type PerRPCCredentials interface { // GetRequestMetadata gets the current request metadata, refreshing // tokens if required. This should be called by the transport layer on // each request, and the data should be populated in headers or other - // context. uri is the URI of the entry point for the request. When - // supported by the underlying implementation, ctx can be used for + // context. If a status code is returned, it will be used as the status + // for the RPC. uri is the URI of the entry point for the request. + // When supported by the underlying implementation, ctx can be used for // timeout and cancellation. // TODO(zhaoq): Define the set of the qualified keys instead of leaving // it as an arbitrary string. @@ -74,11 +73,9 @@ type AuthInfo interface { AuthType() string } -var ( - // ErrConnDispatched indicates that rawConn has been dispatched out of gRPC - // and the caller should not close rawConn. - ErrConnDispatched = errors.New("credentials: rawConn is dispatched out of gRPC") -) +// ErrConnDispatched indicates that rawConn has been dispatched out of gRPC +// and the caller should not close rawConn. +var ErrConnDispatched = errors.New("credentials: rawConn is dispatched out of gRPC") // TransportCredentials defines the common interface for all the live gRPC wire // protocols and supported transport security protocols (e.g., TLS, SSL). @@ -135,15 +132,15 @@ func (c tlsCreds) Info() ProtocolInfo { } } -func (c *tlsCreds) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (_ net.Conn, _ AuthInfo, err error) { +func (c *tlsCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (_ net.Conn, _ AuthInfo, err error) { // use local cfg to avoid clobbering ServerName if using multiple endpoints cfg := cloneTLSConfig(c.config) if cfg.ServerName == "" { - colonPos := strings.LastIndex(addr, ":") + colonPos := strings.LastIndex(authority, ":") if colonPos == -1 { - colonPos = len(addr) + colonPos = len(authority) } - cfg.ServerName = addr[:colonPos] + cfg.ServerName = authority[:colonPos] } conn := tls.Client(rawConn, cfg) errChannel := make(chan error, 1) diff --git a/vendor/google.golang.org/grpc/encoding/BUILD b/vendor/google.golang.org/grpc/encoding/BUILD new file mode 100644 index 00000000000..7f6bcc2a893 --- /dev/null +++ b/vendor/google.golang.org/grpc/encoding/BUILD @@ -0,0 +1,26 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["encoding.go"], + importmap = "k8s.io/kubernetes/vendor/google.golang.org/grpc/encoding", + importpath = "google.golang.org/grpc/encoding", + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//vendor/google.golang.org/grpc/encoding/proto:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/google.golang.org/grpc/encoding/encoding.go b/vendor/google.golang.org/grpc/encoding/encoding.go new file mode 100644 index 00000000000..ade8b7cec73 --- /dev/null +++ b/vendor/google.golang.org/grpc/encoding/encoding.go @@ -0,0 +1,118 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package encoding defines the interface for the compressor and codec, and +// functions to register and retrieve compressors and codecs. +// +// This package is EXPERIMENTAL. +package encoding + +import ( + "io" + "strings" +) + +// Identity specifies the optional encoding for uncompressed streams. +// It is intended for grpc internal use only. +const Identity = "identity" + +// Compressor is used for compressing and decompressing when sending or +// receiving messages. +type Compressor interface { + // Compress writes the data written to wc to w after compressing it. If an + // error occurs while initializing the compressor, that error is returned + // instead. + Compress(w io.Writer) (io.WriteCloser, error) + // Decompress reads data from r, decompresses it, and provides the + // uncompressed data via the returned io.Reader. If an error occurs while + // initializing the decompressor, that error is returned instead. + Decompress(r io.Reader) (io.Reader, error) + // Name is the name of the compression codec and is used to set the content + // coding header. The result must be static; the result cannot change + // between calls. + Name() string +} + +var registeredCompressor = make(map[string]Compressor) + +// RegisterCompressor registers the compressor with gRPC by its name. It can +// be activated when sending an RPC via grpc.UseCompressor(). It will be +// automatically accessed when receiving a message based on the content coding +// header. Servers also use it to send a response with the same encoding as +// the request. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple Compressors are +// registered with the same name, the one registered last will take effect. +func RegisterCompressor(c Compressor) { + registeredCompressor[c.Name()] = c +} + +// GetCompressor returns Compressor for the given compressor name. +func GetCompressor(name string) Compressor { + return registeredCompressor[name] +} + +// Codec defines the interface gRPC uses to encode and decode messages. Note +// that implementations of this interface must be thread safe; a Codec's +// methods can be called from concurrent goroutines. +type Codec interface { + // Marshal returns the wire format of v. + Marshal(v interface{}) ([]byte, error) + // Unmarshal parses the wire format into v. + Unmarshal(data []byte, v interface{}) error + // Name returns the name of the Codec implementation. The returned string + // will be used as part of content type in transmission. The result must be + // static; the result cannot change between calls. + Name() string +} + +var registeredCodecs = make(map[string]Codec) + +// RegisterCodec registers the provided Codec for use with all gRPC clients and +// servers. +// +// The Codec will be stored and looked up by result of its Name() method, which +// should match the content-subtype of the encoding handled by the Codec. This +// is case-insensitive, and is stored and looked up as lowercase. If the +// result of calling Name() is an empty string, RegisterCodec will panic. See +// Content-Type on +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple Compressors are +// registered with the same name, the one registered last will take effect. +func RegisterCodec(codec Codec) { + if codec == nil { + panic("cannot register a nil Codec") + } + contentSubtype := strings.ToLower(codec.Name()) + if contentSubtype == "" { + panic("cannot register Codec with empty string result for String()") + } + registeredCodecs[contentSubtype] = codec +} + +// GetCodec gets a registered Codec by content-subtype, or nil if no Codec is +// registered for the content-subtype. +// +// The content-subtype is expected to be lowercase. +func GetCodec(contentSubtype string) Codec { + return registeredCodecs[contentSubtype] +} diff --git a/vendor/google.golang.org/grpc/encoding/proto/BUILD b/vendor/google.golang.org/grpc/encoding/proto/BUILD new file mode 100644 index 00000000000..3173d4a556b --- /dev/null +++ b/vendor/google.golang.org/grpc/encoding/proto/BUILD @@ -0,0 +1,27 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["proto.go"], + importmap = "k8s.io/kubernetes/vendor/google.golang.org/grpc/encoding/proto", + importpath = "google.golang.org/grpc/encoding/proto", + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/golang/protobuf/proto:go_default_library", + "//vendor/google.golang.org/grpc/encoding:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/google.golang.org/grpc/encoding/proto/proto.go b/vendor/google.golang.org/grpc/encoding/proto/proto.go new file mode 100644 index 00000000000..66b97a6f692 --- /dev/null +++ b/vendor/google.golang.org/grpc/encoding/proto/proto.go @@ -0,0 +1,110 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package proto defines the protobuf codec. Importing this package will +// register the codec. +package proto + +import ( + "math" + "sync" + + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/encoding" +) + +// Name is the name registered for the proto compressor. +const Name = "proto" + +func init() { + encoding.RegisterCodec(codec{}) +} + +// codec is a Codec implementation with protobuf. It is the default codec for gRPC. +type codec struct{} + +type cachedProtoBuffer struct { + lastMarshaledSize uint32 + proto.Buffer +} + +func capToMaxInt32(val int) uint32 { + if val > math.MaxInt32 { + return uint32(math.MaxInt32) + } + return uint32(val) +} + +func marshal(v interface{}, cb *cachedProtoBuffer) ([]byte, error) { + protoMsg := v.(proto.Message) + newSlice := make([]byte, 0, cb.lastMarshaledSize) + + cb.SetBuf(newSlice) + cb.Reset() + if err := cb.Marshal(protoMsg); err != nil { + return nil, err + } + out := cb.Bytes() + cb.lastMarshaledSize = capToMaxInt32(len(out)) + return out, nil +} + +func (codec) Marshal(v interface{}) ([]byte, error) { + if pm, ok := v.(proto.Marshaler); ok { + // object can marshal itself, no need for buffer + return pm.Marshal() + } + + cb := protoBufferPool.Get().(*cachedProtoBuffer) + out, err := marshal(v, cb) + + // put back buffer and lose the ref to the slice + cb.SetBuf(nil) + protoBufferPool.Put(cb) + return out, err +} + +func (codec) Unmarshal(data []byte, v interface{}) error { + protoMsg := v.(proto.Message) + protoMsg.Reset() + + if pu, ok := protoMsg.(proto.Unmarshaler); ok { + // object can unmarshal itself, no need for buffer + return pu.Unmarshal(data) + } + + cb := protoBufferPool.Get().(*cachedProtoBuffer) + cb.SetBuf(data) + err := cb.Unmarshal(protoMsg) + cb.SetBuf(nil) + protoBufferPool.Put(cb) + return err +} + +func (codec) Name() string { + return Name +} + +var protoBufferPool = &sync.Pool{ + New: func() interface{} { + return &cachedProtoBuffer{ + Buffer: proto.Buffer{}, + lastMarshaledSize: 16, + } + }, +} diff --git a/vendor/google.golang.org/grpc/envconfig.go b/vendor/google.golang.org/grpc/envconfig.go new file mode 100644 index 00000000000..d50178e5171 --- /dev/null +++ b/vendor/google.golang.org/grpc/envconfig.go @@ -0,0 +1,37 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "os" + "strings" +) + +const ( + envConfigPrefix = "GRPC_GO_" + envConfigStickinessStr = envConfigPrefix + "STICKINESS" +) + +var ( + envConfigStickinessOn bool +) + +func init() { + envConfigStickinessOn = strings.EqualFold(os.Getenv(envConfigStickinessStr), "on") +} diff --git a/vendor/google.golang.org/grpc/go16.go b/vendor/google.golang.org/grpc/go16.go new file mode 100644 index 00000000000..535ee9356f3 --- /dev/null +++ b/vendor/google.golang.org/grpc/go16.go @@ -0,0 +1,70 @@ +// +build go1.6,!go1.7 + +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "fmt" + "io" + "net" + "net/http" + + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/grpc/transport" +) + +// dialContext connects to the address on the named network. +func dialContext(ctx context.Context, network, address string) (net.Conn, error) { + return (&net.Dialer{Cancel: ctx.Done()}).Dial(network, address) +} + +func sendHTTPRequest(ctx context.Context, req *http.Request, conn net.Conn) error { + req.Cancel = ctx.Done() + if err := req.Write(conn); err != nil { + return fmt.Errorf("failed to write the HTTP request: %v", err) + } + return nil +} + +// toRPCErr converts an error into an error from the status package. +func toRPCErr(err error) error { + if err == nil || err == io.EOF { + return err + } + if _, ok := status.FromError(err); ok { + return err + } + switch e := err.(type) { + case transport.StreamError: + return status.Error(e.Code, e.Desc) + case transport.ConnectionError: + return status.Error(codes.Unavailable, e.Desc) + default: + switch err { + case context.DeadlineExceeded: + return status.Error(codes.DeadlineExceeded, err.Error()) + case context.Canceled: + return status.Error(codes.Canceled, err.Error()) + } + } + return status.Error(codes.Unknown, err.Error()) +} diff --git a/vendor/google.golang.org/grpc/go17.go b/vendor/google.golang.org/grpc/go17.go new file mode 100644 index 00000000000..ec676a93c39 --- /dev/null +++ b/vendor/google.golang.org/grpc/go17.go @@ -0,0 +1,71 @@ +// +build go1.7 + +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + + netctx "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/grpc/transport" +) + +// dialContext connects to the address on the named network. +func dialContext(ctx context.Context, network, address string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, address) +} + +func sendHTTPRequest(ctx context.Context, req *http.Request, conn net.Conn) error { + req = req.WithContext(ctx) + if err := req.Write(conn); err != nil { + return fmt.Errorf("failed to write the HTTP request: %v", err) + } + return nil +} + +// toRPCErr converts an error into an error from the status package. +func toRPCErr(err error) error { + if err == nil || err == io.EOF { + return err + } + if _, ok := status.FromError(err); ok { + return err + } + switch e := err.(type) { + case transport.StreamError: + return status.Error(e.Code, e.Desc) + case transport.ConnectionError: + return status.Error(codes.Unavailable, e.Desc) + default: + switch err { + case context.DeadlineExceeded, netctx.DeadlineExceeded: + return status.Error(codes.DeadlineExceeded, err.Error()) + case context.Canceled, netctx.Canceled: + return status.Error(codes.Canceled, err.Error()) + } + } + return status.Error(codes.Unknown, err.Error()) +} diff --git a/vendor/google.golang.org/grpc/grpclb.go b/vendor/google.golang.org/grpc/grpclb.go deleted file mode 100644 index db56ff36217..00000000000 --- a/vendor/google.golang.org/grpc/grpclb.go +++ /dev/null @@ -1,704 +0,0 @@ -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package grpc - -import ( - "errors" - "fmt" - "math/rand" - "net" - "sync" - "time" - - "golang.org/x/net/context" - "google.golang.org/grpc/codes" - lbmpb "google.golang.org/grpc/grpclb/grpc_lb_v1/messages" - "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/naming" -) - -// Client API for LoadBalancer service. -// Mostly copied from generated pb.go file. -// To avoid circular dependency. -type loadBalancerClient struct { - cc *ClientConn -} - -func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...CallOption) (*balanceLoadClientStream, error) { - desc := &StreamDesc{ - StreamName: "BalanceLoad", - ServerStreams: true, - ClientStreams: true, - } - stream, err := NewClientStream(ctx, desc, c.cc, "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...) - if err != nil { - return nil, err - } - x := &balanceLoadClientStream{stream} - return x, nil -} - -type balanceLoadClientStream struct { - ClientStream -} - -func (x *balanceLoadClientStream) Send(m *lbmpb.LoadBalanceRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *balanceLoadClientStream) Recv() (*lbmpb.LoadBalanceResponse, error) { - m := new(lbmpb.LoadBalanceResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// NewGRPCLBBalancer creates a grpclb load balancer. -func NewGRPCLBBalancer(r naming.Resolver) Balancer { - return &grpclbBalancer{ - r: r, - } -} - -type remoteBalancerInfo struct { - addr string - // the server name used for authentication with the remote LB server. - name string -} - -// grpclbAddrInfo consists of the information of a backend server. -type grpclbAddrInfo struct { - addr Address - connected bool - // dropForRateLimiting indicates whether this particular request should be - // dropped by the client for rate limiting. - dropForRateLimiting bool - // dropForLoadBalancing indicates whether this particular request should be - // dropped by the client for load balancing. - dropForLoadBalancing bool -} - -type grpclbBalancer struct { - r naming.Resolver - target string - mu sync.Mutex - seq int // a sequence number to make sure addrCh does not get stale addresses. - w naming.Watcher - addrCh chan []Address - rbs []remoteBalancerInfo - addrs []*grpclbAddrInfo - next int - waitCh chan struct{} - done bool - rand *rand.Rand - - clientStats lbmpb.ClientStats -} - -func (b *grpclbBalancer) watchAddrUpdates(w naming.Watcher, ch chan []remoteBalancerInfo) error { - updates, err := w.Next() - if err != nil { - grpclog.Warningf("grpclb: failed to get next addr update from watcher: %v", err) - return err - } - b.mu.Lock() - defer b.mu.Unlock() - if b.done { - return ErrClientConnClosing - } - for _, update := range updates { - switch update.Op { - case naming.Add: - var exist bool - for _, v := range b.rbs { - // TODO: Is the same addr with different server name a different balancer? - if update.Addr == v.addr { - exist = true - break - } - } - if exist { - continue - } - md, ok := update.Metadata.(*naming.AddrMetadataGRPCLB) - if !ok { - // TODO: Revisit the handling here and may introduce some fallback mechanism. - grpclog.Errorf("The name resolution contains unexpected metadata %v", update.Metadata) - continue - } - switch md.AddrType { - case naming.Backend: - // TODO: Revisit the handling here and may introduce some fallback mechanism. - grpclog.Errorf("The name resolution does not give grpclb addresses") - continue - case naming.GRPCLB: - b.rbs = append(b.rbs, remoteBalancerInfo{ - addr: update.Addr, - name: md.ServerName, - }) - default: - grpclog.Errorf("Received unknow address type %d", md.AddrType) - continue - } - case naming.Delete: - for i, v := range b.rbs { - if update.Addr == v.addr { - copy(b.rbs[i:], b.rbs[i+1:]) - b.rbs = b.rbs[:len(b.rbs)-1] - break - } - } - default: - grpclog.Errorf("Unknown update.Op %v", update.Op) - } - } - // TODO: Fall back to the basic round-robin load balancing if the resulting address is - // not a load balancer. - select { - case <-ch: - default: - } - ch <- b.rbs - return nil -} - -func convertDuration(d *lbmpb.Duration) time.Duration { - if d == nil { - return 0 - } - return time.Duration(d.Seconds)*time.Second + time.Duration(d.Nanos)*time.Nanosecond -} - -func (b *grpclbBalancer) processServerList(l *lbmpb.ServerList, seq int) { - if l == nil { - return - } - servers := l.GetServers() - var ( - sl []*grpclbAddrInfo - addrs []Address - ) - for _, s := range servers { - md := metadata.Pairs("lb-token", s.LoadBalanceToken) - ip := net.IP(s.IpAddress) - ipStr := ip.String() - if ip.To4() == nil { - // Add square brackets to ipv6 addresses, otherwise net.Dial() and - // net.SplitHostPort() will return too many colons error. - ipStr = fmt.Sprintf("[%s]", ipStr) - } - addr := Address{ - Addr: fmt.Sprintf("%s:%d", ipStr, s.Port), - Metadata: &md, - } - sl = append(sl, &grpclbAddrInfo{ - addr: addr, - dropForRateLimiting: s.DropForRateLimiting, - dropForLoadBalancing: s.DropForLoadBalancing, - }) - addrs = append(addrs, addr) - } - b.mu.Lock() - defer b.mu.Unlock() - if b.done || seq < b.seq { - return - } - if len(sl) > 0 { - // reset b.next to 0 when replacing the server list. - b.next = 0 - b.addrs = sl - b.addrCh <- addrs - } - return -} - -func (b *grpclbBalancer) sendLoadReport(s *balanceLoadClientStream, interval time.Duration, done <-chan struct{}) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - case <-done: - return - } - b.mu.Lock() - stats := b.clientStats - b.clientStats = lbmpb.ClientStats{} // Clear the stats. - b.mu.Unlock() - t := time.Now() - stats.Timestamp = &lbmpb.Timestamp{ - Seconds: t.Unix(), - Nanos: int32(t.Nanosecond()), - } - if err := s.Send(&lbmpb.LoadBalanceRequest{ - LoadBalanceRequestType: &lbmpb.LoadBalanceRequest_ClientStats{ - ClientStats: &stats, - }, - }); err != nil { - grpclog.Errorf("grpclb: failed to send load report: %v", err) - return - } - } -} - -func (b *grpclbBalancer) callRemoteBalancer(lbc *loadBalancerClient, seq int) (retry bool) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - stream, err := lbc.BalanceLoad(ctx) - if err != nil { - grpclog.Errorf("grpclb: failed to perform RPC to the remote balancer %v", err) - return - } - b.mu.Lock() - if b.done { - b.mu.Unlock() - return - } - b.mu.Unlock() - initReq := &lbmpb.LoadBalanceRequest{ - LoadBalanceRequestType: &lbmpb.LoadBalanceRequest_InitialRequest{ - InitialRequest: &lbmpb.InitialLoadBalanceRequest{ - Name: b.target, - }, - }, - } - if err := stream.Send(initReq); err != nil { - grpclog.Errorf("grpclb: failed to send init request: %v", err) - // TODO: backoff on retry? - return true - } - reply, err := stream.Recv() - if err != nil { - grpclog.Errorf("grpclb: failed to recv init response: %v", err) - // TODO: backoff on retry? - return true - } - initResp := reply.GetInitialResponse() - if initResp == nil { - grpclog.Errorf("grpclb: reply from remote balancer did not include initial response.") - return - } - // TODO: Support delegation. - if initResp.LoadBalancerDelegate != "" { - // delegation - grpclog.Errorf("TODO: Delegation is not supported yet.") - return - } - streamDone := make(chan struct{}) - defer close(streamDone) - b.mu.Lock() - b.clientStats = lbmpb.ClientStats{} // Clear client stats. - b.mu.Unlock() - if d := convertDuration(initResp.ClientStatsReportInterval); d > 0 { - go b.sendLoadReport(stream, d, streamDone) - } - // Retrieve the server list. - for { - reply, err := stream.Recv() - if err != nil { - grpclog.Errorf("grpclb: failed to recv server list: %v", err) - break - } - b.mu.Lock() - if b.done || seq < b.seq { - b.mu.Unlock() - return - } - b.seq++ // tick when receiving a new list of servers. - seq = b.seq - b.mu.Unlock() - if serverList := reply.GetServerList(); serverList != nil { - b.processServerList(serverList, seq) - } - } - return true -} - -func (b *grpclbBalancer) Start(target string, config BalancerConfig) error { - b.rand = rand.New(rand.NewSource(time.Now().Unix())) - // TODO: Fall back to the basic direct connection if there is no name resolver. - if b.r == nil { - return errors.New("there is no name resolver installed") - } - b.target = target - b.mu.Lock() - if b.done { - b.mu.Unlock() - return ErrClientConnClosing - } - b.addrCh = make(chan []Address) - w, err := b.r.Resolve(target) - if err != nil { - b.mu.Unlock() - grpclog.Errorf("grpclb: failed to resolve address: %v, err: %v", target, err) - return err - } - b.w = w - b.mu.Unlock() - balancerAddrsCh := make(chan []remoteBalancerInfo, 1) - // Spawn a goroutine to monitor the name resolution of remote load balancer. - go func() { - for { - if err := b.watchAddrUpdates(w, balancerAddrsCh); err != nil { - grpclog.Warningf("grpclb: the naming watcher stops working due to %v.\n", err) - close(balancerAddrsCh) - return - } - } - }() - // Spawn a goroutine to talk to the remote load balancer. - go func() { - var ( - cc *ClientConn - // ccError is closed when there is an error in the current cc. - // A new rb should be picked from rbs and connected. - ccError chan struct{} - rb *remoteBalancerInfo - rbs []remoteBalancerInfo - rbIdx int - ) - - defer func() { - if ccError != nil { - select { - case <-ccError: - default: - close(ccError) - } - } - if cc != nil { - cc.Close() - } - }() - - for { - var ok bool - select { - case rbs, ok = <-balancerAddrsCh: - if !ok { - return - } - foundIdx := -1 - if rb != nil { - for i, trb := range rbs { - if trb == *rb { - foundIdx = i - break - } - } - } - if foundIdx >= 0 { - if foundIdx >= 1 { - // Move the address in use to the beginning of the list. - b.rbs[0], b.rbs[foundIdx] = b.rbs[foundIdx], b.rbs[0] - rbIdx = 0 - } - continue // If found, don't dial new cc. - } else if len(rbs) > 0 { - // Pick a random one from the list, instead of always using the first one. - if l := len(rbs); l > 1 && rb != nil { - tmpIdx := b.rand.Intn(l - 1) - b.rbs[0], b.rbs[tmpIdx] = b.rbs[tmpIdx], b.rbs[0] - } - rbIdx = 0 - rb = &rbs[0] - } else { - // foundIdx < 0 && len(rbs) <= 0. - rb = nil - } - case <-ccError: - ccError = nil - if rbIdx < len(rbs)-1 { - rbIdx++ - rb = &rbs[rbIdx] - } else { - rb = nil - } - } - - if rb == nil { - continue - } - - if cc != nil { - cc.Close() - } - // Talk to the remote load balancer to get the server list. - var ( - err error - dopts []DialOption - ) - if creds := config.DialCreds; creds != nil { - if rb.name != "" { - if err := creds.OverrideServerName(rb.name); err != nil { - grpclog.Warningf("grpclb: failed to override the server name in the credentials: %v", err) - continue - } - } - dopts = append(dopts, WithTransportCredentials(creds)) - } else { - dopts = append(dopts, WithInsecure()) - } - if dialer := config.Dialer; dialer != nil { - // WithDialer takes a different type of function, so we instead use a special DialOption here. - dopts = append(dopts, func(o *dialOptions) { o.copts.Dialer = dialer }) - } - dopts = append(dopts, WithBlock()) - ccError = make(chan struct{}) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - cc, err = DialContext(ctx, rb.addr, dopts...) - cancel() - if err != nil { - grpclog.Warningf("grpclb: failed to setup a connection to the remote balancer %v: %v", rb.addr, err) - close(ccError) - continue - } - b.mu.Lock() - b.seq++ // tick when getting a new balancer address - seq := b.seq - b.next = 0 - b.mu.Unlock() - go func(cc *ClientConn, ccError chan struct{}) { - lbc := &loadBalancerClient{cc} - b.callRemoteBalancer(lbc, seq) - cc.Close() - select { - case <-ccError: - default: - close(ccError) - } - }(cc, ccError) - } - }() - return nil -} - -func (b *grpclbBalancer) down(addr Address, err error) { - b.mu.Lock() - defer b.mu.Unlock() - for _, a := range b.addrs { - if addr == a.addr { - a.connected = false - break - } - } -} - -func (b *grpclbBalancer) Up(addr Address) func(error) { - b.mu.Lock() - defer b.mu.Unlock() - if b.done { - return nil - } - var cnt int - for _, a := range b.addrs { - if a.addr == addr { - if a.connected { - return nil - } - a.connected = true - } - if a.connected && !a.dropForRateLimiting && !a.dropForLoadBalancing { - cnt++ - } - } - // addr is the only one which is connected. Notify the Get() callers who are blocking. - if cnt == 1 && b.waitCh != nil { - close(b.waitCh) - b.waitCh = nil - } - return func(err error) { - b.down(addr, err) - } -} - -func (b *grpclbBalancer) Get(ctx context.Context, opts BalancerGetOptions) (addr Address, put func(), err error) { - var ch chan struct{} - b.mu.Lock() - if b.done { - b.mu.Unlock() - err = ErrClientConnClosing - return - } - seq := b.seq - - defer func() { - if err != nil { - return - } - put = func() { - s, ok := rpcInfoFromContext(ctx) - if !ok { - return - } - b.mu.Lock() - defer b.mu.Unlock() - if b.done || seq < b.seq { - return - } - b.clientStats.NumCallsFinished++ - if !s.bytesSent { - b.clientStats.NumCallsFinishedWithClientFailedToSend++ - } else if s.bytesReceived { - b.clientStats.NumCallsFinishedKnownReceived++ - } - } - }() - - b.clientStats.NumCallsStarted++ - if len(b.addrs) > 0 { - if b.next >= len(b.addrs) { - b.next = 0 - } - next := b.next - for { - a := b.addrs[next] - next = (next + 1) % len(b.addrs) - if a.connected { - if !a.dropForRateLimiting && !a.dropForLoadBalancing { - addr = a.addr - b.next = next - b.mu.Unlock() - return - } - if !opts.BlockingWait { - b.next = next - if a.dropForLoadBalancing { - b.clientStats.NumCallsFinished++ - b.clientStats.NumCallsFinishedWithDropForLoadBalancing++ - } else if a.dropForRateLimiting { - b.clientStats.NumCallsFinished++ - b.clientStats.NumCallsFinishedWithDropForRateLimiting++ - } - b.mu.Unlock() - err = Errorf(codes.Unavailable, "%s drops requests", a.addr.Addr) - return - } - } - if next == b.next { - // Has iterated all the possible address but none is connected. - break - } - } - } - if !opts.BlockingWait { - b.clientStats.NumCallsFinished++ - b.clientStats.NumCallsFinishedWithClientFailedToSend++ - b.mu.Unlock() - err = Errorf(codes.Unavailable, "there is no address available") - return - } - // Wait on b.waitCh for non-failfast RPCs. - if b.waitCh == nil { - ch = make(chan struct{}) - b.waitCh = ch - } else { - ch = b.waitCh - } - b.mu.Unlock() - for { - select { - case <-ctx.Done(): - b.mu.Lock() - b.clientStats.NumCallsFinished++ - b.clientStats.NumCallsFinishedWithClientFailedToSend++ - b.mu.Unlock() - err = ctx.Err() - return - case <-ch: - b.mu.Lock() - if b.done { - b.clientStats.NumCallsFinished++ - b.clientStats.NumCallsFinishedWithClientFailedToSend++ - b.mu.Unlock() - err = ErrClientConnClosing - return - } - - if len(b.addrs) > 0 { - if b.next >= len(b.addrs) { - b.next = 0 - } - next := b.next - for { - a := b.addrs[next] - next = (next + 1) % len(b.addrs) - if a.connected { - if !a.dropForRateLimiting && !a.dropForLoadBalancing { - addr = a.addr - b.next = next - b.mu.Unlock() - return - } - if !opts.BlockingWait { - b.next = next - if a.dropForLoadBalancing { - b.clientStats.NumCallsFinished++ - b.clientStats.NumCallsFinishedWithDropForLoadBalancing++ - } else if a.dropForRateLimiting { - b.clientStats.NumCallsFinished++ - b.clientStats.NumCallsFinishedWithDropForRateLimiting++ - } - b.mu.Unlock() - err = Errorf(codes.Unavailable, "drop requests for the addreess %s", a.addr.Addr) - return - } - } - if next == b.next { - // Has iterated all the possible address but none is connected. - break - } - } - } - // The newly added addr got removed by Down() again. - if b.waitCh == nil { - ch = make(chan struct{}) - b.waitCh = ch - } else { - ch = b.waitCh - } - b.mu.Unlock() - } - } -} - -func (b *grpclbBalancer) Notify() <-chan []Address { - return b.addrCh -} - -func (b *grpclbBalancer) Close() error { - b.mu.Lock() - defer b.mu.Unlock() - if b.done { - return errBalancerClosed - } - b.done = true - if b.waitCh != nil { - close(b.waitCh) - } - if b.addrCh != nil { - close(b.addrCh) - } - if b.w != nil { - b.w.Close() - } - return nil -} diff --git a/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/BUILD b/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/BUILD index bc7f5286610..88503e18984 100644 --- a/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/BUILD +++ b/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/BUILD @@ -6,7 +6,11 @@ go_library( importmap = "k8s.io/kubernetes/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages", importpath = "google.golang.org/grpc/grpclb/grpc_lb_v1/messages", visibility = ["//visibility:public"], - deps = ["//vendor/github.com/golang/protobuf/proto:go_default_library"], + deps = [ + "//vendor/github.com/golang/protobuf/proto:go_default_library", + "//vendor/github.com/golang/protobuf/ptypes/duration:go_default_library", + "//vendor/github.com/golang/protobuf/ptypes/timestamp:go_default_library", + ], ) filegroup( diff --git a/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.pb.go b/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.pb.go index f4a27125a4f..59a35ca3f08 100644 --- a/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.pb.go +++ b/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.pb.go @@ -1,28 +1,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: grpc_lb_v1/messages/messages.proto -/* -Package messages is a generated protocol buffer package. - -It is generated from these files: - grpc_lb_v1/messages/messages.proto - -It has these top-level messages: - Duration - Timestamp - LoadBalanceRequest - InitialLoadBalanceRequest - ClientStats - LoadBalanceResponse - InitialLoadBalanceResponse - ServerList - Server -*/ package messages import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import duration "github.com/golang/protobuf/ptypes/duration" +import timestamp "github.com/golang/protobuf/ptypes/timestamp" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -35,90 +20,49 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package -type Duration struct { - // Signed seconds of the span of time. Must be from -315,576,000,000 - // to +315,576,000,000 inclusive. - Seconds int64 `protobuf:"varint,1,opt,name=seconds" json:"seconds,omitempty"` - // Signed fractions of a second at nanosecond resolution of the span - // of time. Durations less than one second are represented with a 0 - // `seconds` field and a positive or negative `nanos` field. For durations - // of one second or more, a non-zero value for the `nanos` field must be - // of the same sign as the `seconds` field. Must be from -999,999,999 - // to +999,999,999 inclusive. - Nanos int32 `protobuf:"varint,2,opt,name=nanos" json:"nanos,omitempty"` -} - -func (m *Duration) Reset() { *m = Duration{} } -func (m *Duration) String() string { return proto.CompactTextString(m) } -func (*Duration) ProtoMessage() {} -func (*Duration) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } - -func (m *Duration) GetSeconds() int64 { - if m != nil { - return m.Seconds - } - return 0 -} - -func (m *Duration) GetNanos() int32 { - if m != nil { - return m.Nanos - } - return 0 -} - -type Timestamp struct { - // Represents seconds of UTC time since Unix epoch - // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to - // 9999-12-31T23:59:59Z inclusive. - Seconds int64 `protobuf:"varint,1,opt,name=seconds" json:"seconds,omitempty"` - // Non-negative fractions of a second at nanosecond resolution. Negative - // second values with fractions must still have non-negative nanos values - // that count forward in time. Must be from 0 to 999,999,999 - // inclusive. - Nanos int32 `protobuf:"varint,2,opt,name=nanos" json:"nanos,omitempty"` -} - -func (m *Timestamp) Reset() { *m = Timestamp{} } -func (m *Timestamp) String() string { return proto.CompactTextString(m) } -func (*Timestamp) ProtoMessage() {} -func (*Timestamp) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } - -func (m *Timestamp) GetSeconds() int64 { - if m != nil { - return m.Seconds - } - return 0 -} - -func (m *Timestamp) GetNanos() int32 { - if m != nil { - return m.Nanos - } - return 0 -} - type LoadBalanceRequest struct { // Types that are valid to be assigned to LoadBalanceRequestType: // *LoadBalanceRequest_InitialRequest // *LoadBalanceRequest_ClientStats LoadBalanceRequestType isLoadBalanceRequest_LoadBalanceRequestType `protobuf_oneof:"load_balance_request_type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *LoadBalanceRequest) Reset() { *m = LoadBalanceRequest{} } -func (m *LoadBalanceRequest) String() string { return proto.CompactTextString(m) } -func (*LoadBalanceRequest) ProtoMessage() {} -func (*LoadBalanceRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } +func (m *LoadBalanceRequest) Reset() { *m = LoadBalanceRequest{} } +func (m *LoadBalanceRequest) String() string { return proto.CompactTextString(m) } +func (*LoadBalanceRequest) ProtoMessage() {} +func (*LoadBalanceRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b3d89fcb5aa158f8, []int{0} +} +func (m *LoadBalanceRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LoadBalanceRequest.Unmarshal(m, b) +} +func (m *LoadBalanceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LoadBalanceRequest.Marshal(b, m, deterministic) +} +func (dst *LoadBalanceRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_LoadBalanceRequest.Merge(dst, src) +} +func (m *LoadBalanceRequest) XXX_Size() int { + return xxx_messageInfo_LoadBalanceRequest.Size(m) +} +func (m *LoadBalanceRequest) XXX_DiscardUnknown() { + xxx_messageInfo_LoadBalanceRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_LoadBalanceRequest proto.InternalMessageInfo type isLoadBalanceRequest_LoadBalanceRequestType interface { isLoadBalanceRequest_LoadBalanceRequestType() } type LoadBalanceRequest_InitialRequest struct { - InitialRequest *InitialLoadBalanceRequest `protobuf:"bytes,1,opt,name=initial_request,json=initialRequest,oneof"` + InitialRequest *InitialLoadBalanceRequest `protobuf:"bytes,1,opt,name=initial_request,json=initialRequest,proto3,oneof"` } type LoadBalanceRequest_ClientStats struct { - ClientStats *ClientStats `protobuf:"bytes,2,opt,name=client_stats,json=clientStats,oneof"` + ClientStats *ClientStats `protobuf:"bytes,2,opt,name=client_stats,json=clientStats,proto3,oneof"` } func (*LoadBalanceRequest_InitialRequest) isLoadBalanceRequest_LoadBalanceRequestType() {} @@ -204,12 +148,12 @@ func _LoadBalanceRequest_OneofSizer(msg proto.Message) (n int) { switch x := m.LoadBalanceRequestType.(type) { case *LoadBalanceRequest_InitialRequest: s := proto.Size(x.InitialRequest) - n += proto.SizeVarint(1<<3 | proto.WireBytes) + n += 1 // tag and wire n += proto.SizeVarint(uint64(s)) n += s case *LoadBalanceRequest_ClientStats: s := proto.Size(x.ClientStats) - n += proto.SizeVarint(2<<3 | proto.WireBytes) + n += 1 // tag and wire n += proto.SizeVarint(uint64(s)) n += s case nil: @@ -220,15 +164,37 @@ func _LoadBalanceRequest_OneofSizer(msg proto.Message) (n int) { } type InitialLoadBalanceRequest struct { - // Name of load balanced service (IE, balancer.service.com) + // Name of load balanced service (IE, balancer.service.com). Its // length should be less than 256 bytes. - Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *InitialLoadBalanceRequest) Reset() { *m = InitialLoadBalanceRequest{} } -func (m *InitialLoadBalanceRequest) String() string { return proto.CompactTextString(m) } -func (*InitialLoadBalanceRequest) ProtoMessage() {} -func (*InitialLoadBalanceRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } +func (m *InitialLoadBalanceRequest) Reset() { *m = InitialLoadBalanceRequest{} } +func (m *InitialLoadBalanceRequest) String() string { return proto.CompactTextString(m) } +func (*InitialLoadBalanceRequest) ProtoMessage() {} +func (*InitialLoadBalanceRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b3d89fcb5aa158f8, []int{1} +} +func (m *InitialLoadBalanceRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_InitialLoadBalanceRequest.Unmarshal(m, b) +} +func (m *InitialLoadBalanceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_InitialLoadBalanceRequest.Marshal(b, m, deterministic) +} +func (dst *InitialLoadBalanceRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_InitialLoadBalanceRequest.Merge(dst, src) +} +func (m *InitialLoadBalanceRequest) XXX_Size() int { + return xxx_messageInfo_InitialLoadBalanceRequest.Size(m) +} +func (m *InitialLoadBalanceRequest) XXX_DiscardUnknown() { + xxx_messageInfo_InitialLoadBalanceRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_InitialLoadBalanceRequest proto.InternalMessageInfo func (m *InitialLoadBalanceRequest) GetName() string { if m != nil { @@ -237,34 +203,101 @@ func (m *InitialLoadBalanceRequest) GetName() string { return "" } +// Contains the number of calls finished for a particular load balance token. +type ClientStatsPerToken struct { + // See Server.load_balance_token. + LoadBalanceToken string `protobuf:"bytes,1,opt,name=load_balance_token,json=loadBalanceToken,proto3" json:"load_balance_token,omitempty"` + // The total number of RPCs that finished associated with the token. + NumCalls int64 `protobuf:"varint,2,opt,name=num_calls,json=numCalls,proto3" json:"num_calls,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ClientStatsPerToken) Reset() { *m = ClientStatsPerToken{} } +func (m *ClientStatsPerToken) String() string { return proto.CompactTextString(m) } +func (*ClientStatsPerToken) ProtoMessage() {} +func (*ClientStatsPerToken) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b3d89fcb5aa158f8, []int{2} +} +func (m *ClientStatsPerToken) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ClientStatsPerToken.Unmarshal(m, b) +} +func (m *ClientStatsPerToken) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ClientStatsPerToken.Marshal(b, m, deterministic) +} +func (dst *ClientStatsPerToken) XXX_Merge(src proto.Message) { + xxx_messageInfo_ClientStatsPerToken.Merge(dst, src) +} +func (m *ClientStatsPerToken) XXX_Size() int { + return xxx_messageInfo_ClientStatsPerToken.Size(m) +} +func (m *ClientStatsPerToken) XXX_DiscardUnknown() { + xxx_messageInfo_ClientStatsPerToken.DiscardUnknown(m) +} + +var xxx_messageInfo_ClientStatsPerToken proto.InternalMessageInfo + +func (m *ClientStatsPerToken) GetLoadBalanceToken() string { + if m != nil { + return m.LoadBalanceToken + } + return "" +} + +func (m *ClientStatsPerToken) GetNumCalls() int64 { + if m != nil { + return m.NumCalls + } + return 0 +} + // Contains client level statistics that are useful to load balancing. Each // count except the timestamp should be reset to zero after reporting the stats. type ClientStats struct { // The timestamp of generating the report. - Timestamp *Timestamp `protobuf:"bytes,1,opt,name=timestamp" json:"timestamp,omitempty"` + Timestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` // The total number of RPCs that started. - NumCallsStarted int64 `protobuf:"varint,2,opt,name=num_calls_started,json=numCallsStarted" json:"num_calls_started,omitempty"` + NumCallsStarted int64 `protobuf:"varint,2,opt,name=num_calls_started,json=numCallsStarted,proto3" json:"num_calls_started,omitempty"` // The total number of RPCs that finished. - NumCallsFinished int64 `protobuf:"varint,3,opt,name=num_calls_finished,json=numCallsFinished" json:"num_calls_finished,omitempty"` - // The total number of RPCs that were dropped by the client because of rate - // limiting. - NumCallsFinishedWithDropForRateLimiting int64 `protobuf:"varint,4,opt,name=num_calls_finished_with_drop_for_rate_limiting,json=numCallsFinishedWithDropForRateLimiting" json:"num_calls_finished_with_drop_for_rate_limiting,omitempty"` - // The total number of RPCs that were dropped by the client because of load - // balancing. - NumCallsFinishedWithDropForLoadBalancing int64 `protobuf:"varint,5,opt,name=num_calls_finished_with_drop_for_load_balancing,json=numCallsFinishedWithDropForLoadBalancing" json:"num_calls_finished_with_drop_for_load_balancing,omitempty"` + NumCallsFinished int64 `protobuf:"varint,3,opt,name=num_calls_finished,json=numCallsFinished,proto3" json:"num_calls_finished,omitempty"` // The total number of RPCs that failed to reach a server except dropped RPCs. - NumCallsFinishedWithClientFailedToSend int64 `protobuf:"varint,6,opt,name=num_calls_finished_with_client_failed_to_send,json=numCallsFinishedWithClientFailedToSend" json:"num_calls_finished_with_client_failed_to_send,omitempty"` + NumCallsFinishedWithClientFailedToSend int64 `protobuf:"varint,6,opt,name=num_calls_finished_with_client_failed_to_send,json=numCallsFinishedWithClientFailedToSend,proto3" json:"num_calls_finished_with_client_failed_to_send,omitempty"` // The total number of RPCs that finished and are known to have been received // by a server. - NumCallsFinishedKnownReceived int64 `protobuf:"varint,7,opt,name=num_calls_finished_known_received,json=numCallsFinishedKnownReceived" json:"num_calls_finished_known_received,omitempty"` + NumCallsFinishedKnownReceived int64 `protobuf:"varint,7,opt,name=num_calls_finished_known_received,json=numCallsFinishedKnownReceived,proto3" json:"num_calls_finished_known_received,omitempty"` + // The list of dropped calls. + CallsFinishedWithDrop []*ClientStatsPerToken `protobuf:"bytes,8,rep,name=calls_finished_with_drop,json=callsFinishedWithDrop,proto3" json:"calls_finished_with_drop,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *ClientStats) Reset() { *m = ClientStats{} } -func (m *ClientStats) String() string { return proto.CompactTextString(m) } -func (*ClientStats) ProtoMessage() {} -func (*ClientStats) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } +func (m *ClientStats) Reset() { *m = ClientStats{} } +func (m *ClientStats) String() string { return proto.CompactTextString(m) } +func (*ClientStats) ProtoMessage() {} +func (*ClientStats) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b3d89fcb5aa158f8, []int{3} +} +func (m *ClientStats) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ClientStats.Unmarshal(m, b) +} +func (m *ClientStats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ClientStats.Marshal(b, m, deterministic) +} +func (dst *ClientStats) XXX_Merge(src proto.Message) { + xxx_messageInfo_ClientStats.Merge(dst, src) +} +func (m *ClientStats) XXX_Size() int { + return xxx_messageInfo_ClientStats.Size(m) +} +func (m *ClientStats) XXX_DiscardUnknown() { + xxx_messageInfo_ClientStats.DiscardUnknown(m) +} -func (m *ClientStats) GetTimestamp() *Timestamp { +var xxx_messageInfo_ClientStats proto.InternalMessageInfo + +func (m *ClientStats) GetTimestamp() *timestamp.Timestamp { if m != nil { return m.Timestamp } @@ -285,20 +318,6 @@ func (m *ClientStats) GetNumCallsFinished() int64 { return 0 } -func (m *ClientStats) GetNumCallsFinishedWithDropForRateLimiting() int64 { - if m != nil { - return m.NumCallsFinishedWithDropForRateLimiting - } - return 0 -} - -func (m *ClientStats) GetNumCallsFinishedWithDropForLoadBalancing() int64 { - if m != nil { - return m.NumCallsFinishedWithDropForLoadBalancing - } - return 0 -} - func (m *ClientStats) GetNumCallsFinishedWithClientFailedToSend() int64 { if m != nil { return m.NumCallsFinishedWithClientFailedToSend @@ -313,27 +332,56 @@ func (m *ClientStats) GetNumCallsFinishedKnownReceived() int64 { return 0 } +func (m *ClientStats) GetCallsFinishedWithDrop() []*ClientStatsPerToken { + if m != nil { + return m.CallsFinishedWithDrop + } + return nil +} + type LoadBalanceResponse struct { // Types that are valid to be assigned to LoadBalanceResponseType: // *LoadBalanceResponse_InitialResponse // *LoadBalanceResponse_ServerList LoadBalanceResponseType isLoadBalanceResponse_LoadBalanceResponseType `protobuf_oneof:"load_balance_response_type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *LoadBalanceResponse) Reset() { *m = LoadBalanceResponse{} } -func (m *LoadBalanceResponse) String() string { return proto.CompactTextString(m) } -func (*LoadBalanceResponse) ProtoMessage() {} -func (*LoadBalanceResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } +func (m *LoadBalanceResponse) Reset() { *m = LoadBalanceResponse{} } +func (m *LoadBalanceResponse) String() string { return proto.CompactTextString(m) } +func (*LoadBalanceResponse) ProtoMessage() {} +func (*LoadBalanceResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b3d89fcb5aa158f8, []int{4} +} +func (m *LoadBalanceResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LoadBalanceResponse.Unmarshal(m, b) +} +func (m *LoadBalanceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LoadBalanceResponse.Marshal(b, m, deterministic) +} +func (dst *LoadBalanceResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_LoadBalanceResponse.Merge(dst, src) +} +func (m *LoadBalanceResponse) XXX_Size() int { + return xxx_messageInfo_LoadBalanceResponse.Size(m) +} +func (m *LoadBalanceResponse) XXX_DiscardUnknown() { + xxx_messageInfo_LoadBalanceResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_LoadBalanceResponse proto.InternalMessageInfo type isLoadBalanceResponse_LoadBalanceResponseType interface { isLoadBalanceResponse_LoadBalanceResponseType() } type LoadBalanceResponse_InitialResponse struct { - InitialResponse *InitialLoadBalanceResponse `protobuf:"bytes,1,opt,name=initial_response,json=initialResponse,oneof"` + InitialResponse *InitialLoadBalanceResponse `protobuf:"bytes,1,opt,name=initial_response,json=initialResponse,proto3,oneof"` } type LoadBalanceResponse_ServerList struct { - ServerList *ServerList `protobuf:"bytes,2,opt,name=server_list,json=serverList,oneof"` + ServerList *ServerList `protobuf:"bytes,2,opt,name=server_list,json=serverList,proto3,oneof"` } func (*LoadBalanceResponse_InitialResponse) isLoadBalanceResponse_LoadBalanceResponseType() {} @@ -419,12 +467,12 @@ func _LoadBalanceResponse_OneofSizer(msg proto.Message) (n int) { switch x := m.LoadBalanceResponseType.(type) { case *LoadBalanceResponse_InitialResponse: s := proto.Size(x.InitialResponse) - n += proto.SizeVarint(1<<3 | proto.WireBytes) + n += 1 // tag and wire n += proto.SizeVarint(uint64(s)) n += s case *LoadBalanceResponse_ServerList: s := proto.Size(x.ServerList) - n += proto.SizeVarint(2<<3 | proto.WireBytes) + n += 1 // tag and wire n += proto.SizeVarint(uint64(s)) n += s case nil: @@ -440,17 +488,39 @@ type InitialLoadBalanceResponse struct { // the response, the client should open a separate connection to the // load_balancer_delegate and call the BalanceLoad method. Its length should // be less than 64 bytes. - LoadBalancerDelegate string `protobuf:"bytes,1,opt,name=load_balancer_delegate,json=loadBalancerDelegate" json:"load_balancer_delegate,omitempty"` + LoadBalancerDelegate string `protobuf:"bytes,1,opt,name=load_balancer_delegate,json=loadBalancerDelegate,proto3" json:"load_balancer_delegate,omitempty"` // This interval defines how often the client should send the client stats // to the load balancer. Stats should only be reported when the duration is // positive. - ClientStatsReportInterval *Duration `protobuf:"bytes,2,opt,name=client_stats_report_interval,json=clientStatsReportInterval" json:"client_stats_report_interval,omitempty"` + ClientStatsReportInterval *duration.Duration `protobuf:"bytes,2,opt,name=client_stats_report_interval,json=clientStatsReportInterval,proto3" json:"client_stats_report_interval,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *InitialLoadBalanceResponse) Reset() { *m = InitialLoadBalanceResponse{} } -func (m *InitialLoadBalanceResponse) String() string { return proto.CompactTextString(m) } -func (*InitialLoadBalanceResponse) ProtoMessage() {} -func (*InitialLoadBalanceResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } +func (m *InitialLoadBalanceResponse) Reset() { *m = InitialLoadBalanceResponse{} } +func (m *InitialLoadBalanceResponse) String() string { return proto.CompactTextString(m) } +func (*InitialLoadBalanceResponse) ProtoMessage() {} +func (*InitialLoadBalanceResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b3d89fcb5aa158f8, []int{5} +} +func (m *InitialLoadBalanceResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_InitialLoadBalanceResponse.Unmarshal(m, b) +} +func (m *InitialLoadBalanceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_InitialLoadBalanceResponse.Marshal(b, m, deterministic) +} +func (dst *InitialLoadBalanceResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_InitialLoadBalanceResponse.Merge(dst, src) +} +func (m *InitialLoadBalanceResponse) XXX_Size() int { + return xxx_messageInfo_InitialLoadBalanceResponse.Size(m) +} +func (m *InitialLoadBalanceResponse) XXX_DiscardUnknown() { + xxx_messageInfo_InitialLoadBalanceResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_InitialLoadBalanceResponse proto.InternalMessageInfo func (m *InitialLoadBalanceResponse) GetLoadBalancerDelegate() string { if m != nil { @@ -459,7 +529,7 @@ func (m *InitialLoadBalanceResponse) GetLoadBalancerDelegate() string { return "" } -func (m *InitialLoadBalanceResponse) GetClientStatsReportInterval() *Duration { +func (m *InitialLoadBalanceResponse) GetClientStatsReportInterval() *duration.Duration { if m != nil { return m.ClientStatsReportInterval } @@ -471,13 +541,35 @@ type ServerList struct { // be updated when server resolutions change or as needed to balance load // across more servers. The client should consume the server list in order // unless instructed otherwise via the client_config. - Servers []*Server `protobuf:"bytes,1,rep,name=servers" json:"servers,omitempty"` + Servers []*Server `protobuf:"bytes,1,rep,name=servers,proto3" json:"servers,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *ServerList) Reset() { *m = ServerList{} } -func (m *ServerList) String() string { return proto.CompactTextString(m) } -func (*ServerList) ProtoMessage() {} -func (*ServerList) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } +func (m *ServerList) Reset() { *m = ServerList{} } +func (m *ServerList) String() string { return proto.CompactTextString(m) } +func (*ServerList) ProtoMessage() {} +func (*ServerList) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b3d89fcb5aa158f8, []int{6} +} +func (m *ServerList) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ServerList.Unmarshal(m, b) +} +func (m *ServerList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ServerList.Marshal(b, m, deterministic) +} +func (dst *ServerList) XXX_Merge(src proto.Message) { + xxx_messageInfo_ServerList.Merge(dst, src) +} +func (m *ServerList) XXX_Size() int { + return xxx_messageInfo_ServerList.Size(m) +} +func (m *ServerList) XXX_DiscardUnknown() { + xxx_messageInfo_ServerList.DiscardUnknown(m) +} + +var xxx_messageInfo_ServerList proto.InternalMessageInfo func (m *ServerList) GetServers() []*Server { if m != nil { @@ -486,35 +578,52 @@ func (m *ServerList) GetServers() []*Server { return nil } -// Contains server information. When none of the [drop_for_*] fields are true, -// use the other fields. When drop_for_rate_limiting is true, ignore all other -// fields. Use drop_for_load_balancing only when it is true and -// drop_for_rate_limiting is false. +// Contains server information. When the drop field is not true, use the other +// fields. type Server struct { // A resolved address for the server, serialized in network-byte-order. It may // either be an IPv4 or IPv6 address. IpAddress []byte `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` // A resolved port number for the server. - Port int32 `protobuf:"varint,2,opt,name=port" json:"port,omitempty"` + Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` // An opaque but printable token given to the frontend for each pick. All // frontend requests for that pick must include the token in its initial // metadata. The token is used by the backend to verify the request and to - // allow the backend to report load to the gRPC LB system. - // - // Its length is variable but less than 50 bytes. - LoadBalanceToken string `protobuf:"bytes,3,opt,name=load_balance_token,json=loadBalanceToken" json:"load_balance_token,omitempty"` - // Indicates whether this particular request should be dropped by the client - // for rate limiting. - DropForRateLimiting bool `protobuf:"varint,4,opt,name=drop_for_rate_limiting,json=dropForRateLimiting" json:"drop_for_rate_limiting,omitempty"` - // Indicates whether this particular request should be dropped by the client - // for load balancing. - DropForLoadBalancing bool `protobuf:"varint,5,opt,name=drop_for_load_balancing,json=dropForLoadBalancing" json:"drop_for_load_balancing,omitempty"` + // allow the backend to report load to the gRPC LB system. The token is also + // used in client stats for reporting dropped calls. + LoadBalanceToken string `protobuf:"bytes,3,opt,name=load_balance_token,json=loadBalanceToken,proto3" json:"load_balance_token,omitempty"` + // Indicates whether this particular request should be dropped by the client. + // If the request is dropped, there will be a corresponding entry in + // ClientStats.calls_finished_with_drop. + Drop bool `protobuf:"varint,4,opt,name=drop,proto3" json:"drop,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *Server) Reset() { *m = Server{} } -func (m *Server) String() string { return proto.CompactTextString(m) } -func (*Server) ProtoMessage() {} -func (*Server) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } +func (m *Server) Reset() { *m = Server{} } +func (m *Server) String() string { return proto.CompactTextString(m) } +func (*Server) ProtoMessage() {} +func (*Server) Descriptor() ([]byte, []int) { + return fileDescriptor_messages_b3d89fcb5aa158f8, []int{7} +} +func (m *Server) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Server.Unmarshal(m, b) +} +func (m *Server) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Server.Marshal(b, m, deterministic) +} +func (dst *Server) XXX_Merge(src proto.Message) { + xxx_messageInfo_Server.Merge(dst, src) +} +func (m *Server) XXX_Size() int { + return xxx_messageInfo_Server.Size(m) +} +func (m *Server) XXX_DiscardUnknown() { + xxx_messageInfo_Server.DiscardUnknown(m) +} + +var xxx_messageInfo_Server proto.InternalMessageInfo func (m *Server) GetIpAddress() []byte { if m != nil { @@ -537,25 +646,17 @@ func (m *Server) GetLoadBalanceToken() string { return "" } -func (m *Server) GetDropForRateLimiting() bool { +func (m *Server) GetDrop() bool { if m != nil { - return m.DropForRateLimiting - } - return false -} - -func (m *Server) GetDropForLoadBalancing() bool { - if m != nil { - return m.DropForLoadBalancing + return m.Drop } return false } func init() { - proto.RegisterType((*Duration)(nil), "grpc.lb.v1.Duration") - proto.RegisterType((*Timestamp)(nil), "grpc.lb.v1.Timestamp") proto.RegisterType((*LoadBalanceRequest)(nil), "grpc.lb.v1.LoadBalanceRequest") proto.RegisterType((*InitialLoadBalanceRequest)(nil), "grpc.lb.v1.InitialLoadBalanceRequest") + proto.RegisterType((*ClientStatsPerToken)(nil), "grpc.lb.v1.ClientStatsPerToken") proto.RegisterType((*ClientStats)(nil), "grpc.lb.v1.ClientStats") proto.RegisterType((*LoadBalanceResponse)(nil), "grpc.lb.v1.LoadBalanceResponse") proto.RegisterType((*InitialLoadBalanceResponse)(nil), "grpc.lb.v1.InitialLoadBalanceResponse") @@ -563,53 +664,55 @@ func init() { proto.RegisterType((*Server)(nil), "grpc.lb.v1.Server") } -func init() { proto.RegisterFile("grpc_lb_v1/messages/messages.proto", fileDescriptor0) } - -var fileDescriptor0 = []byte{ - // 709 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0xdd, 0x4e, 0x1b, 0x3b, - 0x10, 0x26, 0x27, 0x01, 0x92, 0x09, 0x3a, 0xe4, 0x98, 0x1c, 0x08, 0x14, 0x24, 0xba, 0x52, 0x69, - 0x54, 0xd1, 0x20, 0xa0, 0xbd, 0xe8, 0xcf, 0x45, 0x1b, 0x10, 0x0a, 0x2d, 0x17, 0x95, 0x43, 0x55, - 0xa9, 0x52, 0x65, 0x39, 0xd9, 0x21, 0x58, 0x6c, 0xec, 0xad, 0xed, 0x04, 0xf5, 0x11, 0xfa, 0x28, - 0x7d, 0x8c, 0xaa, 0xcf, 0xd0, 0xf7, 0xa9, 0xd6, 0xbb, 0x9b, 0x5d, 0x20, 0x80, 0x7a, 0x67, 0x8f, - 0xbf, 0xf9, 0xbe, 0xf1, 0xac, 0xbf, 0x59, 0xf0, 0x06, 0x3a, 0xec, 0xb3, 0xa0, 0xc7, 0xc6, 0xbb, - 0x3b, 0x43, 0x34, 0x86, 0x0f, 0xd0, 0x4c, 0x16, 0xad, 0x50, 0x2b, 0xab, 0x08, 0x44, 0x98, 0x56, - 0xd0, 0x6b, 0x8d, 0x77, 0xbd, 0x97, 0x50, 0x3e, 0x1c, 0x69, 0x6e, 0x85, 0x92, 0xa4, 0x01, 0xf3, - 0x06, 0xfb, 0x4a, 0xfa, 0xa6, 0x51, 0xd8, 0x2c, 0x34, 0x8b, 0x34, 0xdd, 0x92, 0x3a, 0xcc, 0x4a, - 0x2e, 0x95, 0x69, 0xfc, 0xb3, 0x59, 0x68, 0xce, 0xd2, 0x78, 0xe3, 0xbd, 0x82, 0xca, 0xa9, 0x18, - 0xa2, 0xb1, 0x7c, 0x18, 0xfe, 0x75, 0xf2, 0xcf, 0x02, 0x90, 0x13, 0xc5, 0xfd, 0x36, 0x0f, 0xb8, - 0xec, 0x23, 0xc5, 0xaf, 0x23, 0x34, 0x96, 0x7c, 0x80, 0x45, 0x21, 0x85, 0x15, 0x3c, 0x60, 0x3a, - 0x0e, 0x39, 0xba, 0xea, 0xde, 0xa3, 0x56, 0x56, 0x75, 0xeb, 0x38, 0x86, 0xdc, 0xcc, 0xef, 0xcc, - 0xd0, 0x7f, 0x93, 0xfc, 0x94, 0xf1, 0x35, 0x2c, 0xf4, 0x03, 0x81, 0xd2, 0x32, 0x63, 0xb9, 0x8d, - 0xab, 0xa8, 0xee, 0xad, 0xe4, 0xe9, 0x0e, 0xdc, 0x79, 0x37, 0x3a, 0xee, 0xcc, 0xd0, 0x6a, 0x3f, - 0xdb, 0xb6, 0x1f, 0xc0, 0x6a, 0xa0, 0xb8, 0xcf, 0x7a, 0xb1, 0x4c, 0x5a, 0x14, 0xb3, 0xdf, 0x42, - 0xf4, 0x76, 0x60, 0xf5, 0xd6, 0x4a, 0x08, 0x81, 0x92, 0xe4, 0x43, 0x74, 0xe5, 0x57, 0xa8, 0x5b, - 0x7b, 0xdf, 0x4b, 0x50, 0xcd, 0x89, 0x91, 0x7d, 0xa8, 0xd8, 0xb4, 0x83, 0xc9, 0x3d, 0xff, 0xcf, - 0x17, 0x36, 0x69, 0x2f, 0xcd, 0x70, 0xe4, 0x09, 0xfc, 0x27, 0x47, 0x43, 0xd6, 0xe7, 0x41, 0x60, - 0xa2, 0x3b, 0x69, 0x8b, 0xbe, 0xbb, 0x55, 0x91, 0x2e, 0xca, 0xd1, 0xf0, 0x20, 0x8a, 0x77, 0xe3, - 0x30, 0xd9, 0x06, 0x92, 0x61, 0xcf, 0x84, 0x14, 0xe6, 0x1c, 0xfd, 0x46, 0xd1, 0x81, 0x6b, 0x29, - 0xf8, 0x28, 0x89, 0x13, 0x06, 0xad, 0x9b, 0x68, 0x76, 0x29, 0xec, 0x39, 0xf3, 0xb5, 0x0a, 0xd9, - 0x99, 0xd2, 0x4c, 0x73, 0x8b, 0x2c, 0x10, 0x43, 0x61, 0x85, 0x1c, 0x34, 0x4a, 0x8e, 0xe9, 0xf1, - 0x75, 0xa6, 0x4f, 0xc2, 0x9e, 0x1f, 0x6a, 0x15, 0x1e, 0x29, 0x4d, 0xb9, 0xc5, 0x93, 0x04, 0x4e, - 0x38, 0xec, 0xdc, 0x2b, 0x90, 0x6b, 0x77, 0xa4, 0x30, 0xeb, 0x14, 0x9a, 0x77, 0x28, 0x64, 0xbd, - 0x8f, 0x24, 0xbe, 0xc0, 0xd3, 0xdb, 0x24, 0x92, 0x67, 0x70, 0xc6, 0x45, 0x80, 0x3e, 0xb3, 0x8a, - 0x19, 0x94, 0x7e, 0x63, 0xce, 0x09, 0x6c, 0x4d, 0x13, 0x88, 0x3f, 0xd5, 0x91, 0xc3, 0x9f, 0xaa, - 0x2e, 0x4a, 0x9f, 0x74, 0xe0, 0xe1, 0x14, 0xfa, 0x0b, 0xa9, 0x2e, 0x25, 0xd3, 0xd8, 0x47, 0x31, - 0x46, 0xbf, 0x31, 0xef, 0x28, 0x37, 0xae, 0x53, 0xbe, 0x8f, 0x50, 0x34, 0x01, 0x79, 0xbf, 0x0a, - 0xb0, 0x74, 0xe5, 0xd9, 0x98, 0x50, 0x49, 0x83, 0xa4, 0x0b, 0xb5, 0xcc, 0x01, 0x71, 0x2c, 0x79, - 0x1a, 0x5b, 0xf7, 0x59, 0x20, 0x46, 0x77, 0x66, 0xe8, 0xe2, 0xc4, 0x03, 0x09, 0xe9, 0x0b, 0xa8, - 0x1a, 0xd4, 0x63, 0xd4, 0x2c, 0x10, 0xc6, 0x26, 0x1e, 0x58, 0xce, 0xf3, 0x75, 0xdd, 0xf1, 0x89, - 0x70, 0x1e, 0x02, 0x33, 0xd9, 0xb5, 0xd7, 0x61, 0xed, 0x9a, 0x03, 0x62, 0xce, 0xd8, 0x02, 0x3f, - 0x0a, 0xb0, 0x76, 0x7b, 0x29, 0xe4, 0x19, 0x2c, 0xe7, 0x93, 0x35, 0xf3, 0x31, 0xc0, 0x01, 0xb7, - 0xa9, 0x2d, 0xea, 0x41, 0x96, 0xa4, 0x0f, 0x93, 0x33, 0xf2, 0x11, 0xd6, 0xf3, 0x96, 0x65, 0x1a, - 0x43, 0xa5, 0x2d, 0x13, 0xd2, 0xa2, 0x1e, 0xf3, 0x20, 0x29, 0xbf, 0x9e, 0x2f, 0x3f, 0x1d, 0x62, - 0x74, 0x35, 0xe7, 0x5e, 0xea, 0xf2, 0x8e, 0x93, 0x34, 0xef, 0x0d, 0x40, 0x76, 0x4b, 0xb2, 0x1d, - 0x0d, 0xac, 0x68, 0x17, 0x0d, 0xac, 0x62, 0xb3, 0xba, 0x47, 0x6e, 0xb6, 0x83, 0xa6, 0x90, 0x77, - 0xa5, 0x72, 0xb1, 0x56, 0xf2, 0x7e, 0x17, 0x60, 0x2e, 0x3e, 0x21, 0x1b, 0x00, 0x22, 0x64, 0xdc, - 0xf7, 0x35, 0x9a, 0x78, 0xe4, 0x2d, 0xd0, 0x8a, 0x08, 0xdf, 0xc6, 0x81, 0xc8, 0xfd, 0x91, 0x76, - 0x32, 0xf3, 0xdc, 0x3a, 0x32, 0xe3, 0x95, 0x4e, 0x5a, 0x75, 0x81, 0xd2, 0x99, 0xb1, 0x42, 0x6b, - 0xb9, 0x46, 0x9c, 0x46, 0x71, 0xb2, 0x0f, 0xcb, 0x77, 0x98, 0xae, 0x4c, 0x97, 0xfc, 0x29, 0x06, - 0x7b, 0x0e, 0x2b, 0x77, 0x19, 0xa9, 0x4c, 0xeb, 0xfe, 0x14, 0xd3, 0xb4, 0xe1, 0x73, 0x39, 0xfd, - 0x47, 0xf4, 0xe6, 0xdc, 0x4f, 0x62, 0xff, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa3, 0x36, 0x86, - 0xa6, 0x4a, 0x06, 0x00, 0x00, +func init() { + proto.RegisterFile("grpc_lb_v1/messages/messages.proto", fileDescriptor_messages_b3d89fcb5aa158f8) +} + +var fileDescriptor_messages_b3d89fcb5aa158f8 = []byte{ + // 708 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x54, 0x61, 0x4f, 0xf3, 0x36, + 0x10, 0x26, 0x6b, 0xe8, 0xdb, 0x5e, 0x5f, 0x8d, 0xce, 0x6c, 0x2c, 0x2d, 0x30, 0x58, 0xa4, 0x21, + 0x34, 0xb1, 0x54, 0xc0, 0x3e, 0x6c, 0xd2, 0x3e, 0x6c, 0x05, 0xa1, 0xc2, 0xf8, 0x80, 0x52, 0xa4, + 0x4d, 0x48, 0x93, 0xe7, 0x36, 0x26, 0x58, 0xb8, 0x76, 0x66, 0xbb, 0x45, 0xfb, 0xbc, 0xff, 0x33, + 0xed, 0x2f, 0x4c, 0xfb, 0x63, 0x53, 0xec, 0xa4, 0x0d, 0x2d, 0xd5, 0xfb, 0x25, 0x72, 0xee, 0x9e, + 0x7b, 0xee, 0xce, 0x77, 0x8f, 0x21, 0x4c, 0x55, 0x36, 0xc6, 0x7c, 0x84, 0x67, 0xa7, 0xbd, 0x09, + 0xd5, 0x9a, 0xa4, 0x54, 0xcf, 0x0f, 0x51, 0xa6, 0xa4, 0x91, 0x08, 0x72, 0x4c, 0xc4, 0x47, 0xd1, + 0xec, 0xb4, 0xfb, 0x45, 0x2a, 0x65, 0xca, 0x69, 0xcf, 0x7a, 0x46, 0xd3, 0xc7, 0x5e, 0x32, 0x55, + 0xc4, 0x30, 0x29, 0x1c, 0xb6, 0x7b, 0xb0, 0xec, 0x37, 0x6c, 0x42, 0xb5, 0x21, 0x93, 0xcc, 0x01, + 0xc2, 0x7f, 0x3d, 0x40, 0xb7, 0x92, 0x24, 0x7d, 0xc2, 0x89, 0x18, 0xd3, 0x98, 0xfe, 0x31, 0xa5, + 0xda, 0xa0, 0x3b, 0xd8, 0x62, 0x82, 0x19, 0x46, 0x38, 0x56, 0xce, 0x14, 0x78, 0x87, 0xde, 0x71, + 0xeb, 0xec, 0xab, 0x68, 0x91, 0x3d, 0xba, 0x76, 0x90, 0xd5, 0xf8, 0xc1, 0x46, 0xfc, 0x71, 0x11, + 0x5f, 0x32, 0xfe, 0x00, 0xef, 0xc7, 0x9c, 0x51, 0x61, 0xb0, 0x36, 0xc4, 0xe8, 0xe0, 0x23, 0x4b, + 0xf7, 0x79, 0x95, 0xee, 0xc2, 0xfa, 0x87, 0xb9, 0x7b, 0xb0, 0x11, 0xb7, 0xc6, 0x8b, 0xdf, 0xfe, + 0x2e, 0x74, 0xb8, 0x24, 0x09, 0x1e, 0xb9, 0x34, 0x65, 0x51, 0xd8, 0xfc, 0x99, 0xd1, 0xb0, 0x07, + 0x9d, 0xb5, 0x95, 0x20, 0x04, 0xbe, 0x20, 0x13, 0x6a, 0xcb, 0x6f, 0xc6, 0xf6, 0x1c, 0xfe, 0x0e, + 0xdb, 0x95, 0x5c, 0x77, 0x54, 0xdd, 0xcb, 0x67, 0x2a, 0xd0, 0x09, 0xa0, 0x57, 0x49, 0x4c, 0x6e, + 0x2d, 0x02, 0xdb, 0x7c, 0x41, 0xed, 0xd0, 0xbb, 0xd0, 0x14, 0xd3, 0x09, 0x1e, 0x13, 0xce, 0x5d, + 0x37, 0xb5, 0xb8, 0x21, 0xa6, 0x93, 0x8b, 0xfc, 0x3f, 0xfc, 0xa7, 0x06, 0xad, 0x4a, 0x0a, 0xf4, + 0x1d, 0x34, 0xe7, 0x37, 0x5f, 0xdc, 0x64, 0x37, 0x72, 0xb3, 0x89, 0xca, 0xd9, 0x44, 0xf7, 0x25, + 0x22, 0x5e, 0x80, 0xd1, 0xd7, 0xf0, 0xc9, 0x3c, 0x4d, 0x7e, 0x75, 0xca, 0xd0, 0xa4, 0x48, 0xb7, + 0x55, 0xa6, 0x1b, 0x3a, 0x73, 0xde, 0xc0, 0x02, 0xfb, 0xc8, 0x04, 0xd3, 0x4f, 0x34, 0x09, 0x6a, + 0x16, 0xdc, 0x2e, 0xc1, 0x57, 0x85, 0x1d, 0xfd, 0x06, 0xdf, 0xac, 0xa2, 0xf1, 0x0b, 0x33, 0x4f, + 0xb8, 0x98, 0xd4, 0x23, 0x61, 0x9c, 0x26, 0xd8, 0x48, 0xac, 0xa9, 0x48, 0x82, 0xba, 0x25, 0x3a, + 0x5a, 0x26, 0xfa, 0x85, 0x99, 0x27, 0xd7, 0xeb, 0x95, 0xc5, 0xdf, 0xcb, 0x21, 0x15, 0x09, 0x1a, + 0xc0, 0x97, 0x6f, 0xd0, 0x3f, 0x0b, 0xf9, 0x22, 0xb0, 0xa2, 0x63, 0xca, 0x66, 0x34, 0x09, 0xde, + 0x59, 0xca, 0xfd, 0x65, 0xca, 0x9f, 0x73, 0x54, 0x5c, 0x80, 0xd0, 0xaf, 0x10, 0xbc, 0x55, 0x64, + 0xa2, 0x64, 0x16, 0x34, 0x0e, 0x6b, 0xc7, 0xad, 0xb3, 0x83, 0x35, 0x6b, 0x54, 0x8e, 0x36, 0xfe, + 0x6c, 0xbc, 0x5c, 0xf1, 0xa5, 0x92, 0xd9, 0x8d, 0xdf, 0xf0, 0xdb, 0x9b, 0x37, 0x7e, 0x63, 0xb3, + 0x5d, 0x0f, 0xff, 0xf3, 0x60, 0xfb, 0xd5, 0xfe, 0xe8, 0x4c, 0x0a, 0x4d, 0xd1, 0x10, 0xda, 0x0b, + 0x29, 0x38, 0x5b, 0x31, 0xc1, 0xa3, 0x0f, 0x69, 0xc1, 0xa1, 0x07, 0x1b, 0xf1, 0xd6, 0x5c, 0x0c, + 0x05, 0xe9, 0xf7, 0xd0, 0xd2, 0x54, 0xcd, 0xa8, 0xc2, 0x9c, 0x69, 0x53, 0x88, 0x61, 0xa7, 0xca, + 0x37, 0xb4, 0xee, 0x5b, 0x66, 0xc5, 0x04, 0x7a, 0xfe, 0xd7, 0xdf, 0x83, 0xee, 0x92, 0x14, 0x1c, + 0xa7, 0xd3, 0xc2, 0xdf, 0x1e, 0x74, 0xd7, 0x97, 0x82, 0xbe, 0x85, 0x9d, 0x6a, 0xb0, 0xc2, 0x09, + 0xe5, 0x34, 0x25, 0xa6, 0xd4, 0xc7, 0xa7, 0x95, 0x35, 0x57, 0x97, 0x85, 0x0f, 0x3d, 0xc0, 0x5e, + 0x55, 0xbb, 0x58, 0xd1, 0x4c, 0x2a, 0x83, 0x99, 0x30, 0x54, 0xcd, 0x08, 0x2f, 0xca, 0xef, 0xac, + 0x2c, 0xf4, 0x65, 0xf1, 0x18, 0xc5, 0x9d, 0x8a, 0x96, 0x63, 0x1b, 0x7c, 0x5d, 0xc4, 0x86, 0x3f, + 0x02, 0x2c, 0x5a, 0x45, 0x27, 0xf0, 0xce, 0xb5, 0xaa, 0x03, 0xcf, 0x4e, 0x16, 0xad, 0xde, 0x49, + 0x5c, 0x42, 0x6e, 0xfc, 0x46, 0xad, 0xed, 0x87, 0x7f, 0x79, 0x50, 0x77, 0x1e, 0xb4, 0x0f, 0xc0, + 0x32, 0x4c, 0x92, 0x44, 0x51, 0xad, 0x6d, 0x4b, 0xef, 0xe3, 0x26, 0xcb, 0x7e, 0x72, 0x86, 0xfc, + 0x2d, 0xc8, 0x73, 0xdb, 0x7a, 0x37, 0x63, 0x7b, 0x5e, 0x23, 0xfa, 0xda, 0x1a, 0xd1, 0x23, 0xf0, + 0xed, 0xda, 0xf9, 0x87, 0xde, 0x71, 0x23, 0xb6, 0x67, 0xb7, 0x3e, 0xfd, 0xf3, 0x87, 0xd3, 0xa2, + 0xfd, 0x54, 0x72, 0x22, 0xd2, 0x48, 0xaa, 0xb4, 0x97, 0xd7, 0x6e, 0x3f, 0x7c, 0xd4, 0x7b, 0xe3, + 0x65, 0x1f, 0xd5, 0xed, 0x55, 0x9d, 0xff, 0x1f, 0x00, 0x00, 0xff, 0xff, 0xc8, 0x88, 0xe6, 0xf4, + 0xf7, 0x05, 0x00, 0x00, } diff --git a/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.proto b/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.proto index 2ed04551fad..56d767e5434 100644 --- a/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.proto +++ b/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/messages/messages.proto @@ -14,35 +14,11 @@ syntax = "proto3"; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; + package grpc.lb.v1; -option go_package = "messages"; - -message Duration { - // Signed seconds of the span of time. Must be from -315,576,000,000 - // to +315,576,000,000 inclusive. - int64 seconds = 1; - - // Signed fractions of a second at nanosecond resolution of the span - // of time. Durations less than one second are represented with a 0 - // `seconds` field and a positive or negative `nanos` field. For durations - // of one second or more, a non-zero value for the `nanos` field must be - // of the same sign as the `seconds` field. Must be from -999,999,999 - // to +999,999,999 inclusive. - int32 nanos = 2; -} - -message Timestamp { - // Represents seconds of UTC time since Unix epoch - // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to - // 9999-12-31T23:59:59Z inclusive. - int64 seconds = 1; - - // Non-negative fractions of a second at nanosecond resolution. Negative - // second values with fractions must still have non-negative nanos values - // that count forward in time. Must be from 0 to 999,999,999 - // inclusive. - int32 nanos = 2; -} +option go_package = "google.golang.org/grpc/grpclb/grpc_lb_v1/messages"; message LoadBalanceRequest { oneof load_balance_request_type { @@ -56,16 +32,25 @@ message LoadBalanceRequest { } message InitialLoadBalanceRequest { - // Name of load balanced service (IE, balancer.service.com) + // Name of load balanced service (IE, balancer.service.com). Its // length should be less than 256 bytes. string name = 1; } +// Contains the number of calls finished for a particular load balance token. +message ClientStatsPerToken { + // See Server.load_balance_token. + string load_balance_token = 1; + + // The total number of RPCs that finished associated with the token. + int64 num_calls = 2; +} + // Contains client level statistics that are useful to load balancing. Each // count except the timestamp should be reset to zero after reporting the stats. message ClientStats { // The timestamp of generating the report. - Timestamp timestamp = 1; + google.protobuf.Timestamp timestamp = 1; // The total number of RPCs that started. int64 num_calls_started = 2; @@ -73,20 +58,17 @@ message ClientStats { // The total number of RPCs that finished. int64 num_calls_finished = 3; - // The total number of RPCs that were dropped by the client because of rate - // limiting. - int64 num_calls_finished_with_drop_for_rate_limiting = 4; - - // The total number of RPCs that were dropped by the client because of load - // balancing. - int64 num_calls_finished_with_drop_for_load_balancing = 5; - // The total number of RPCs that failed to reach a server except dropped RPCs. int64 num_calls_finished_with_client_failed_to_send = 6; // The total number of RPCs that finished and are known to have been received // by a server. int64 num_calls_finished_known_received = 7; + + // The list of dropped calls. + repeated ClientStatsPerToken calls_finished_with_drop = 8; + + reserved 4, 5; } message LoadBalanceResponse { @@ -111,7 +93,7 @@ message InitialLoadBalanceResponse { // This interval defines how often the client should send the client stats // to the load balancer. Stats should only be reported when the duration is // positive. - Duration client_stats_report_interval = 2; + google.protobuf.Duration client_stats_report_interval = 2; } message ServerList { @@ -125,10 +107,8 @@ message ServerList { reserved 3; } -// Contains server information. When none of the [drop_for_*] fields are true, -// use the other fields. When drop_for_rate_limiting is true, ignore all other -// fields. Use drop_for_load_balancing only when it is true and -// drop_for_rate_limiting is false. +// Contains server information. When the drop field is not true, use the other +// fields. message Server { // A resolved address for the server, serialized in network-byte-order. It may // either be an IPv4 or IPv6 address. @@ -140,16 +120,14 @@ message Server { // An opaque but printable token given to the frontend for each pick. All // frontend requests for that pick must include the token in its initial // metadata. The token is used by the backend to verify the request and to - // allow the backend to report load to the gRPC LB system. - // - // Its length is variable but less than 50 bytes. + // allow the backend to report load to the gRPC LB system. The token is also + // used in client stats for reporting dropped calls. string load_balance_token = 3; - // Indicates whether this particular request should be dropped by the client - // for rate limiting. - bool drop_for_rate_limiting = 4; + // Indicates whether this particular request should be dropped by the client. + // If the request is dropped, there will be a corresponding entry in + // ClientStats.calls_finished_with_drop. + bool drop = 4; - // Indicates whether this particular request should be dropped by the client - // for load balancing. - bool drop_for_load_balancing = 5; + reserved 5; } diff --git a/vendor/google.golang.org/grpc/grpclog/grpclog.go b/vendor/google.golang.org/grpc/grpclog/grpclog.go index 1d71e25de50..501bd529cdd 100644 --- a/vendor/google.golang.org/grpc/grpclog/grpclog.go +++ b/vendor/google.golang.org/grpc/grpclog/grpclog.go @@ -105,18 +105,21 @@ func Fatalln(args ...interface{}) { } // Print prints to the logger. Arguments are handled in the manner of fmt.Print. +// // Deprecated: use Info. func Print(args ...interface{}) { logger.Info(args...) } // Printf prints to the logger. Arguments are handled in the manner of fmt.Printf. +// // Deprecated: use Infof. func Printf(format string, args ...interface{}) { logger.Infof(format, args...) } // Println prints to the logger. Arguments are handled in the manner of fmt.Println. +// // Deprecated: use Infoln. func Println(args ...interface{}) { logger.Infoln(args...) diff --git a/vendor/google.golang.org/grpc/grpclog/logger.go b/vendor/google.golang.org/grpc/grpclog/logger.go index d03b2397bfa..097494f710f 100644 --- a/vendor/google.golang.org/grpc/grpclog/logger.go +++ b/vendor/google.golang.org/grpc/grpclog/logger.go @@ -19,6 +19,7 @@ package grpclog // Logger mimics golang's standard Logger as an interface. +// // Deprecated: use LoggerV2. type Logger interface { Fatal(args ...interface{}) @@ -31,6 +32,7 @@ type Logger interface { // SetLogger sets the logger that is used in grpc. Call only from // init() functions. +// // Deprecated: use SetLoggerV2. func SetLogger(l Logger) { logger = &loggerWrapper{Logger: l} diff --git a/vendor/google.golang.org/grpc/health/BUILD b/vendor/google.golang.org/grpc/health/BUILD index 05caf6bf411..937c4625df7 100644 --- a/vendor/google.golang.org/grpc/health/BUILD +++ b/vendor/google.golang.org/grpc/health/BUILD @@ -8,9 +8,9 @@ go_library( visibility = ["//visibility:public"], deps = [ "//vendor/golang.org/x/net/context:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", "//vendor/google.golang.org/grpc/codes:go_default_library", "//vendor/google.golang.org/grpc/health/grpc_health_v1:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", ], ) diff --git a/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go b/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go index fdcbb9e0b7d..463710451c6 100644 --- a/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go +++ b/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go @@ -1,16 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: grpc_health_v1/health.proto +// source: grpc/health/v1/health.proto -/* -Package grpc_health_v1 is a generated protocol buffer package. - -It is generated from these files: - grpc_health_v1/health.proto - -It has these top-level messages: - HealthCheckRequest - HealthCheckResponse -*/ package grpc_health_v1 import proto "github.com/golang/protobuf/proto" @@ -56,17 +46,39 @@ func (x HealthCheckResponse_ServingStatus) String() string { return proto.EnumName(HealthCheckResponse_ServingStatus_name, int32(x)) } func (HealthCheckResponse_ServingStatus) EnumDescriptor() ([]byte, []int) { - return fileDescriptor0, []int{1, 0} + return fileDescriptor_health_85731b6c49265086, []int{1, 0} } type HealthCheckRequest struct { - Service string `protobuf:"bytes,1,opt,name=service" json:"service,omitempty"` + Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *HealthCheckRequest) Reset() { *m = HealthCheckRequest{} } -func (m *HealthCheckRequest) String() string { return proto.CompactTextString(m) } -func (*HealthCheckRequest) ProtoMessage() {} -func (*HealthCheckRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } +func (m *HealthCheckRequest) Reset() { *m = HealthCheckRequest{} } +func (m *HealthCheckRequest) String() string { return proto.CompactTextString(m) } +func (*HealthCheckRequest) ProtoMessage() {} +func (*HealthCheckRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_health_85731b6c49265086, []int{0} +} +func (m *HealthCheckRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_HealthCheckRequest.Unmarshal(m, b) +} +func (m *HealthCheckRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_HealthCheckRequest.Marshal(b, m, deterministic) +} +func (dst *HealthCheckRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_HealthCheckRequest.Merge(dst, src) +} +func (m *HealthCheckRequest) XXX_Size() int { + return xxx_messageInfo_HealthCheckRequest.Size(m) +} +func (m *HealthCheckRequest) XXX_DiscardUnknown() { + xxx_messageInfo_HealthCheckRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_HealthCheckRequest proto.InternalMessageInfo func (m *HealthCheckRequest) GetService() string { if m != nil { @@ -76,13 +88,35 @@ func (m *HealthCheckRequest) GetService() string { } type HealthCheckResponse struct { - Status HealthCheckResponse_ServingStatus `protobuf:"varint,1,opt,name=status,enum=grpc.health.v1.HealthCheckResponse_ServingStatus" json:"status,omitempty"` + Status HealthCheckResponse_ServingStatus `protobuf:"varint,1,opt,name=status,proto3,enum=grpc.health.v1.HealthCheckResponse_ServingStatus" json:"status,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *HealthCheckResponse) Reset() { *m = HealthCheckResponse{} } -func (m *HealthCheckResponse) String() string { return proto.CompactTextString(m) } -func (*HealthCheckResponse) ProtoMessage() {} -func (*HealthCheckResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } +func (m *HealthCheckResponse) Reset() { *m = HealthCheckResponse{} } +func (m *HealthCheckResponse) String() string { return proto.CompactTextString(m) } +func (*HealthCheckResponse) ProtoMessage() {} +func (*HealthCheckResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_health_85731b6c49265086, []int{1} +} +func (m *HealthCheckResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_HealthCheckResponse.Unmarshal(m, b) +} +func (m *HealthCheckResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_HealthCheckResponse.Marshal(b, m, deterministic) +} +func (dst *HealthCheckResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_HealthCheckResponse.Merge(dst, src) +} +func (m *HealthCheckResponse) XXX_Size() int { + return xxx_messageInfo_HealthCheckResponse.Size(m) +} +func (m *HealthCheckResponse) XXX_DiscardUnknown() { + xxx_messageInfo_HealthCheckResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_HealthCheckResponse proto.InternalMessageInfo func (m *HealthCheckResponse) GetStatus() HealthCheckResponse_ServingStatus { if m != nil { @@ -105,8 +139,9 @@ var _ grpc.ClientConn // is compatible with the grpc package it is being compiled against. const _ = grpc.SupportPackageIsVersion4 -// Client API for Health service - +// HealthClient is the client API for Health service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type HealthClient interface { Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) } @@ -121,15 +156,14 @@ func NewHealthClient(cc *grpc.ClientConn) HealthClient { func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { out := new(HealthCheckResponse) - err := grpc.Invoke(ctx, "/grpc.health.v1.Health/Check", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/grpc.health.v1.Health/Check", in, out, opts...) if err != nil { return nil, err } return out, nil } -// Server API for Health service - +// HealthServer is the server API for Health service. type HealthServer interface { Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) } @@ -166,25 +200,28 @@ var _Health_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "grpc_health_v1/health.proto", + Metadata: "grpc/health/v1/health.proto", } -func init() { proto.RegisterFile("grpc_health_v1/health.proto", fileDescriptor0) } +func init() { proto.RegisterFile("grpc/health/v1/health.proto", fileDescriptor_health_85731b6c49265086) } -var fileDescriptor0 = []byte{ - // 213 bytes of a gzipped FileDescriptorProto +var fileDescriptor_health_85731b6c49265086 = []byte{ + // 271 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4e, 0x2f, 0x2a, 0x48, - 0x8e, 0xcf, 0x48, 0x4d, 0xcc, 0x29, 0xc9, 0x88, 0x2f, 0x33, 0xd4, 0x87, 0xb0, 0xf4, 0x0a, 0x8a, - 0xf2, 0x4b, 0xf2, 0x85, 0xf8, 0x40, 0x92, 0x7a, 0x50, 0xa1, 0x32, 0x43, 0x25, 0x3d, 0x2e, 0x21, - 0x0f, 0x30, 0xc7, 0x39, 0x23, 0x35, 0x39, 0x3b, 0x28, 0xb5, 0xb0, 0x34, 0xb5, 0xb8, 0x44, 0x48, - 0x82, 0x8b, 0xbd, 0x38, 0xb5, 0xa8, 0x2c, 0x33, 0x39, 0x55, 0x82, 0x51, 0x81, 0x51, 0x83, 0x33, - 0x08, 0xc6, 0x55, 0x9a, 0xc3, 0xc8, 0x25, 0x8c, 0xa2, 0xa1, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0x55, - 0xc8, 0x93, 0x8b, 0xad, 0xb8, 0x24, 0xb1, 0xa4, 0xb4, 0x18, 0xac, 0x81, 0xcf, 0xc8, 0x50, 0x0f, - 0xd5, 0x22, 0x3d, 0x2c, 0x9a, 0xf4, 0x82, 0x41, 0x86, 0xe6, 0xa5, 0x07, 0x83, 0x35, 0x06, 0x41, - 0x0d, 0x50, 0xb2, 0xe2, 0xe2, 0x45, 0x91, 0x10, 0xe2, 0xe6, 0x62, 0x0f, 0xf5, 0xf3, 0xf6, 0xf3, - 0x0f, 0xf7, 0x13, 0x60, 0x00, 0x71, 0x82, 0x5d, 0x83, 0xc2, 0x3c, 0xfd, 0xdc, 0x05, 0x18, 0x85, - 0xf8, 0xb9, 0xb8, 0xfd, 0xfc, 0x43, 0xe2, 0x61, 0x02, 0x4c, 0x46, 0x51, 0x5c, 0x6c, 0x10, 0x8b, - 0x84, 0x02, 0xb8, 0x58, 0xc1, 0x96, 0x09, 0x29, 0xe1, 0x75, 0x09, 0xd8, 0xbf, 0x52, 0xca, 0x44, - 0xb8, 0x36, 0x89, 0x0d, 0x1c, 0x82, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x53, 0x2b, 0x65, - 0x20, 0x60, 0x01, 0x00, 0x00, + 0xd6, 0xcf, 0x48, 0x4d, 0xcc, 0x29, 0xc9, 0xd0, 0x2f, 0x33, 0x84, 0xb2, 0xf4, 0x0a, 0x8a, 0xf2, + 0x4b, 0xf2, 0x85, 0xf8, 0x40, 0x92, 0x7a, 0x50, 0xa1, 0x32, 0x43, 0x25, 0x3d, 0x2e, 0x21, 0x0f, + 0x30, 0xc7, 0x39, 0x23, 0x35, 0x39, 0x3b, 0x28, 0xb5, 0xb0, 0x34, 0xb5, 0xb8, 0x44, 0x48, 0x82, + 0x8b, 0xbd, 0x38, 0xb5, 0xa8, 0x2c, 0x33, 0x39, 0x55, 0x82, 0x51, 0x81, 0x51, 0x83, 0x33, 0x08, + 0xc6, 0x55, 0x9a, 0xc3, 0xc8, 0x25, 0x8c, 0xa2, 0xa1, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0x55, 0xc8, + 0x93, 0x8b, 0xad, 0xb8, 0x24, 0xb1, 0xa4, 0xb4, 0x18, 0xac, 0x81, 0xcf, 0xc8, 0x50, 0x0f, 0xd5, + 0x22, 0x3d, 0x2c, 0x9a, 0xf4, 0x82, 0x41, 0x86, 0xe6, 0xa5, 0x07, 0x83, 0x35, 0x06, 0x41, 0x0d, + 0x50, 0xb2, 0xe2, 0xe2, 0x45, 0x91, 0x10, 0xe2, 0xe6, 0x62, 0x0f, 0xf5, 0xf3, 0xf6, 0xf3, 0x0f, + 0xf7, 0x13, 0x60, 0x00, 0x71, 0x82, 0x5d, 0x83, 0xc2, 0x3c, 0xfd, 0xdc, 0x05, 0x18, 0x85, 0xf8, + 0xb9, 0xb8, 0xfd, 0xfc, 0x43, 0xe2, 0x61, 0x02, 0x4c, 0x46, 0x51, 0x5c, 0x6c, 0x10, 0x8b, 0x84, + 0x02, 0xb8, 0x58, 0xc1, 0x96, 0x09, 0x29, 0xe1, 0x75, 0x09, 0xd8, 0xbf, 0x52, 0xca, 0x44, 0xb8, + 0xd6, 0x29, 0x91, 0x4b, 0x30, 0x33, 0x1f, 0x4d, 0xa1, 0x13, 0x37, 0x44, 0x65, 0x00, 0x28, 0x70, + 0x03, 0x18, 0xa3, 0x74, 0xd2, 0xf3, 0xf3, 0xd3, 0x73, 0x52, 0xf5, 0xd2, 0xf3, 0x73, 0x12, 0xf3, + 0xd2, 0xf5, 0xf2, 0x8b, 0xd2, 0xf5, 0x91, 0x63, 0x03, 0xc4, 0x8e, 0x87, 0xb0, 0xe3, 0xcb, 0x0c, + 0x57, 0x31, 0xf1, 0xb9, 0x83, 0x4c, 0x83, 0x18, 0xa1, 0x17, 0x66, 0x98, 0xc4, 0x06, 0x8e, 0x24, + 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0xec, 0x66, 0x81, 0xcb, 0xc3, 0x01, 0x00, 0x00, } diff --git a/vendor/google.golang.org/grpc/health/grpc_health_v1/health.proto b/vendor/google.golang.org/grpc/health/grpc_health_v1/health.proto deleted file mode 100644 index 6072fdc3b80..00000000000 --- a/vendor/google.golang.org/grpc/health/grpc_health_v1/health.proto +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package grpc.health.v1; - -message HealthCheckRequest { - string service = 1; -} - -message HealthCheckResponse { - enum ServingStatus { - UNKNOWN = 0; - SERVING = 1; - NOT_SERVING = 2; - } - ServingStatus status = 1; -} - -service Health{ - rpc Check(HealthCheckRequest) returns (HealthCheckResponse); -} diff --git a/vendor/google.golang.org/grpc/health/health.go b/vendor/google.golang.org/grpc/health/health.go index c6212f406f7..c2588867e5a 100644 --- a/vendor/google.golang.org/grpc/health/health.go +++ b/vendor/google.golang.org/grpc/health/health.go @@ -16,7 +16,7 @@ * */ -//go:generate protoc --go_out=plugins=grpc:. grpc_health_v1/health.proto +//go:generate ./regenerate.sh // Package health provides some utility functions to health-check a server. The implementation // is based on protobuf. Users need to write their own implementations if other IDLs are used. @@ -26,9 +26,9 @@ import ( "sync" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" ) // Server implements `service Health`. @@ -60,7 +60,7 @@ func (s *Server) Check(ctx context.Context, in *healthpb.HealthCheckRequest) (*h Status: status, }, nil } - return nil, grpc.Errorf(codes.NotFound, "unknown service") + return nil, status.Error(codes.NotFound, "unknown service") } // SetServingStatus is called when need to reset the serving status of a service diff --git a/vendor/google.golang.org/grpc/health/regenerate.sh b/vendor/google.golang.org/grpc/health/regenerate.sh new file mode 100755 index 00000000000..b11eccb295b --- /dev/null +++ b/vendor/google.golang.org/grpc/health/regenerate.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eux -o pipefail + +TMP=$(mktemp -d) + +function finish { + rm -rf "$TMP" +} +trap finish EXIT + +pushd "$TMP" +mkdir -p grpc/health/v1 +curl https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/health/v1/health.proto > grpc/health/v1/health.proto + +protoc --go_out=plugins=grpc,paths=source_relative:. -I. grpc/health/v1/*.proto +popd +rm -f grpc_health_v1/*.pb.go +cp "$TMP"/grpc/health/v1/*.pb.go grpc_health_v1/ + diff --git a/vendor/google.golang.org/grpc/interceptor.go b/vendor/google.golang.org/grpc/interceptor.go index 06dc825b9fb..1f6ef678035 100644 --- a/vendor/google.golang.org/grpc/interceptor.go +++ b/vendor/google.golang.org/grpc/interceptor.go @@ -48,7 +48,9 @@ type UnaryServerInfo struct { } // UnaryHandler defines the handler invoked by UnaryServerInterceptor to complete the normal -// execution of a unary RPC. +// execution of a unary RPC. If a UnaryHandler returns an error, it should be produced by the +// status package, or else gRPC will use codes.Unknown as the status code and err.Error() as +// the status message of the RPC. type UnaryHandler func(ctx context.Context, req interface{}) (interface{}, error) // UnaryServerInterceptor provides a hook to intercept the execution of a unary RPC on the server. info diff --git a/vendor/google.golang.org/grpc/internal/BUILD b/vendor/google.golang.org/grpc/internal/BUILD index fd099d54b76..0e4c87d61f5 100644 --- a/vendor/google.golang.org/grpc/internal/BUILD +++ b/vendor/google.golang.org/grpc/internal/BUILD @@ -17,7 +17,12 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//vendor/google.golang.org/grpc/internal/backoff:all-srcs", + "//vendor/google.golang.org/grpc/internal/channelz:all-srcs", + "//vendor/google.golang.org/grpc/internal/grpcrand:all-srcs", + ], tags = ["automanaged"], visibility = ["//visibility:public"], ) diff --git a/vendor/google.golang.org/grpc/internal/backoff/BUILD b/vendor/google.golang.org/grpc/internal/backoff/BUILD new file mode 100644 index 00000000000..c8316db177d --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/backoff/BUILD @@ -0,0 +1,24 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["backoff.go"], + importmap = "k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/backoff", + importpath = "google.golang.org/grpc/internal/backoff", + visibility = ["//vendor/google.golang.org/grpc:__subpackages__"], + deps = ["//vendor/google.golang.org/grpc/internal/grpcrand:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/google.golang.org/grpc/internal/backoff/backoff.go b/vendor/google.golang.org/grpc/internal/backoff/backoff.go new file mode 100644 index 00000000000..1bd0cce5abd --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/backoff/backoff.go @@ -0,0 +1,78 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package backoff implement the backoff strategy for gRPC. +// +// This is kept in internal until the gRPC project decides whether or not to +// allow alternative backoff strategies. +package backoff + +import ( + "time" + + "google.golang.org/grpc/internal/grpcrand" +) + +// Strategy defines the methodology for backing off after a grpc connection +// failure. +// +type Strategy interface { + // Backoff returns the amount of time to wait before the next retry given + // the number of consecutive failures. + Backoff(retries int) time.Duration +} + +const ( + // baseDelay is the amount of time to wait before retrying after the first + // failure. + baseDelay = 1.0 * time.Second + // factor is applied to the backoff after each retry. + factor = 1.6 + // jitter provides a range to randomize backoff delays. + jitter = 0.2 +) + +// Exponential implements exponential backoff algorithm as defined in +// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. +type Exponential struct { + // MaxDelay is the upper bound of backoff delay. + MaxDelay time.Duration +} + +// Backoff returns the amount of time to wait before the next retry given the +// number of retries. +func (bc Exponential) Backoff(retries int) time.Duration { + if retries == 0 { + return baseDelay + } + backoff, max := float64(baseDelay), float64(bc.MaxDelay) + for backoff < max && retries > 0 { + backoff *= factor + retries-- + } + if backoff > max { + backoff = max + } + // Randomize backoff delays so that if a cluster of requests start at + // the same time, they won't operate in lockstep. + backoff *= 1 + jitter*(grpcrand.Float64()*2-1) + if backoff < 0 { + return 0 + } + return time.Duration(backoff) +} diff --git a/vendor/google.golang.org/grpc/internal/channelz/BUILD b/vendor/google.golang.org/grpc/internal/channelz/BUILD new file mode 100644 index 00000000000..f1d0228d8e5 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/channelz/BUILD @@ -0,0 +1,30 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "funcs.go", + "types.go", + ], + importmap = "k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/channelz", + importpath = "google.golang.org/grpc/internal/channelz", + visibility = ["//vendor/google.golang.org/grpc:__subpackages__"], + deps = [ + "//vendor/google.golang.org/grpc/connectivity:go_default_library", + "//vendor/google.golang.org/grpc/grpclog:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/google.golang.org/grpc/internal/channelz/funcs.go b/vendor/google.golang.org/grpc/internal/channelz/funcs.go new file mode 100644 index 00000000000..586a0336b47 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/channelz/funcs.go @@ -0,0 +1,573 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package channelz defines APIs for enabling channelz service, entry +// registration/deletion, and accessing channelz data. It also defines channelz +// metric struct formats. +// +// All APIs in this package are experimental. +package channelz + +import ( + "sort" + "sync" + "sync/atomic" + + "google.golang.org/grpc/grpclog" +) + +var ( + db dbWrapper + idGen idGenerator + // EntryPerPage defines the number of channelz entries to be shown on a web page. + EntryPerPage = 50 + curState int32 +) + +// TurnOn turns on channelz data collection. +func TurnOn() { + if !IsOn() { + NewChannelzStorage() + atomic.StoreInt32(&curState, 1) + } +} + +// IsOn returns whether channelz data collection is on. +func IsOn() bool { + return atomic.CompareAndSwapInt32(&curState, 1, 1) +} + +// dbWarpper wraps around a reference to internal channelz data storage, and +// provide synchronized functionality to set and get the reference. +type dbWrapper struct { + mu sync.RWMutex + DB *channelMap +} + +func (d *dbWrapper) set(db *channelMap) { + d.mu.Lock() + d.DB = db + d.mu.Unlock() +} + +func (d *dbWrapper) get() *channelMap { + d.mu.RLock() + defer d.mu.RUnlock() + return d.DB +} + +// NewChannelzStorage initializes channelz data storage and id generator. +// +// Note: This function is exported for testing purpose only. User should not call +// it in most cases. +func NewChannelzStorage() { + db.set(&channelMap{ + topLevelChannels: make(map[int64]struct{}), + channels: make(map[int64]*channel), + listenSockets: make(map[int64]*listenSocket), + normalSockets: make(map[int64]*normalSocket), + servers: make(map[int64]*server), + subChannels: make(map[int64]*subChannel), + }) + idGen.reset() +} + +// GetTopChannels returns a slice of top channel's ChannelMetric, along with a +// boolean indicating whether there's more top channels to be queried for. +// +// The arg id specifies that only top channel with id at or above it will be included +// in the result. The returned slice is up to a length of EntryPerPage, and is +// sorted in ascending id order. +func GetTopChannels(id int64) ([]*ChannelMetric, bool) { + return db.get().GetTopChannels(id) +} + +// GetServers returns a slice of server's ServerMetric, along with a +// boolean indicating whether there's more servers to be queried for. +// +// The arg id specifies that only server with id at or above it will be included +// in the result. The returned slice is up to a length of EntryPerPage, and is +// sorted in ascending id order. +func GetServers(id int64) ([]*ServerMetric, bool) { + return db.get().GetServers(id) +} + +// GetServerSockets returns a slice of server's (identified by id) normal socket's +// SocketMetric, along with a boolean indicating whether there's more sockets to +// be queried for. +// +// The arg startID specifies that only sockets with id at or above it will be +// included in the result. The returned slice is up to a length of EntryPerPage, +// and is sorted in ascending id order. +func GetServerSockets(id int64, startID int64) ([]*SocketMetric, bool) { + return db.get().GetServerSockets(id, startID) +} + +// GetChannel returns the ChannelMetric for the channel (identified by id). +func GetChannel(id int64) *ChannelMetric { + return db.get().GetChannel(id) +} + +// GetSubChannel returns the SubChannelMetric for the subchannel (identified by id). +func GetSubChannel(id int64) *SubChannelMetric { + return db.get().GetSubChannel(id) +} + +// GetSocket returns the SocketInternalMetric for the socket (identified by id). +func GetSocket(id int64) *SocketMetric { + return db.get().GetSocket(id) +} + +// RegisterChannel registers the given channel c in channelz database with ref +// as its reference name, and add it to the child list of its parent (identified +// by pid). pid = 0 means no parent. It returns the unique channelz tracking id +// assigned to this channel. +func RegisterChannel(c Channel, pid int64, ref string) int64 { + id := idGen.genID() + cn := &channel{ + refName: ref, + c: c, + subChans: make(map[int64]string), + nestedChans: make(map[int64]string), + id: id, + pid: pid, + } + if pid == 0 { + db.get().addChannel(id, cn, true, pid, ref) + } else { + db.get().addChannel(id, cn, false, pid, ref) + } + return id +} + +// RegisterSubChannel registers the given channel c in channelz database with ref +// as its reference name, and add it to the child list of its parent (identified +// by pid). It returns the unique channelz tracking id assigned to this subchannel. +func RegisterSubChannel(c Channel, pid int64, ref string) int64 { + if pid == 0 { + grpclog.Error("a SubChannel's parent id cannot be 0") + return 0 + } + id := idGen.genID() + sc := &subChannel{ + refName: ref, + c: c, + sockets: make(map[int64]string), + id: id, + pid: pid, + } + db.get().addSubChannel(id, sc, pid, ref) + return id +} + +// RegisterServer registers the given server s in channelz database. It returns +// the unique channelz tracking id assigned to this server. +func RegisterServer(s Server, ref string) int64 { + id := idGen.genID() + svr := &server{ + refName: ref, + s: s, + sockets: make(map[int64]string), + listenSockets: make(map[int64]string), + id: id, + } + db.get().addServer(id, svr) + return id +} + +// RegisterListenSocket registers the given listen socket s in channelz database +// with ref as its reference name, and add it to the child list of its parent +// (identified by pid). It returns the unique channelz tracking id assigned to +// this listen socket. +func RegisterListenSocket(s Socket, pid int64, ref string) int64 { + if pid == 0 { + grpclog.Error("a ListenSocket's parent id cannot be 0") + return 0 + } + id := idGen.genID() + ls := &listenSocket{refName: ref, s: s, id: id, pid: pid} + db.get().addListenSocket(id, ls, pid, ref) + return id +} + +// RegisterNormalSocket registers the given normal socket s in channelz database +// with ref as its reference name, and add it to the child list of its parent +// (identified by pid). It returns the unique channelz tracking id assigned to +// this normal socket. +func RegisterNormalSocket(s Socket, pid int64, ref string) int64 { + if pid == 0 { + grpclog.Error("a NormalSocket's parent id cannot be 0") + return 0 + } + id := idGen.genID() + ns := &normalSocket{refName: ref, s: s, id: id, pid: pid} + db.get().addNormalSocket(id, ns, pid, ref) + return id +} + +// RemoveEntry removes an entry with unique channelz trakcing id to be id from +// channelz database. +func RemoveEntry(id int64) { + db.get().removeEntry(id) +} + +// channelMap is the storage data structure for channelz. +// Methods of channelMap can be divided in two two categories with respect to locking. +// 1. Methods acquire the global lock. +// 2. Methods that can only be called when global lock is held. +// A second type of method need always to be called inside a first type of method. +type channelMap struct { + mu sync.RWMutex + topLevelChannels map[int64]struct{} + servers map[int64]*server + channels map[int64]*channel + subChannels map[int64]*subChannel + listenSockets map[int64]*listenSocket + normalSockets map[int64]*normalSocket +} + +func (c *channelMap) addServer(id int64, s *server) { + c.mu.Lock() + s.cm = c + c.servers[id] = s + c.mu.Unlock() +} + +func (c *channelMap) addChannel(id int64, cn *channel, isTopChannel bool, pid int64, ref string) { + c.mu.Lock() + cn.cm = c + c.channels[id] = cn + if isTopChannel { + c.topLevelChannels[id] = struct{}{} + } else { + c.findEntry(pid).addChild(id, cn) + } + c.mu.Unlock() +} + +func (c *channelMap) addSubChannel(id int64, sc *subChannel, pid int64, ref string) { + c.mu.Lock() + sc.cm = c + c.subChannels[id] = sc + c.findEntry(pid).addChild(id, sc) + c.mu.Unlock() +} + +func (c *channelMap) addListenSocket(id int64, ls *listenSocket, pid int64, ref string) { + c.mu.Lock() + ls.cm = c + c.listenSockets[id] = ls + c.findEntry(pid).addChild(id, ls) + c.mu.Unlock() +} + +func (c *channelMap) addNormalSocket(id int64, ns *normalSocket, pid int64, ref string) { + c.mu.Lock() + ns.cm = c + c.normalSockets[id] = ns + c.findEntry(pid).addChild(id, ns) + c.mu.Unlock() +} + +// removeEntry triggers the removal of an entry, which may not indeed delete the +// entry, if it has to wait on the deletion of its children, or may lead to a chain +// of entry deletion. For example, deleting the last socket of a gracefully shutting +// down server will lead to the server being also deleted. +func (c *channelMap) removeEntry(id int64) { + c.mu.Lock() + c.findEntry(id).triggerDelete() + c.mu.Unlock() +} + +// c.mu must be held by the caller. +func (c *channelMap) findEntry(id int64) entry { + var v entry + var ok bool + if v, ok = c.channels[id]; ok { + return v + } + if v, ok = c.subChannels[id]; ok { + return v + } + if v, ok = c.servers[id]; ok { + return v + } + if v, ok = c.listenSockets[id]; ok { + return v + } + if v, ok = c.normalSockets[id]; ok { + return v + } + return &dummyEntry{idNotFound: id} +} + +// c.mu must be held by the caller +// deleteEntry simply deletes an entry from the channelMap. Before calling this +// method, caller must check this entry is ready to be deleted, i.e removeEntry() +// has been called on it, and no children still exist. +// Conditionals are ordered by the expected frequency of deletion of each entity +// type, in order to optimize performance. +func (c *channelMap) deleteEntry(id int64) { + var ok bool + if _, ok = c.normalSockets[id]; ok { + delete(c.normalSockets, id) + return + } + if _, ok = c.subChannels[id]; ok { + delete(c.subChannels, id) + return + } + if _, ok = c.channels[id]; ok { + delete(c.channels, id) + delete(c.topLevelChannels, id) + return + } + if _, ok = c.listenSockets[id]; ok { + delete(c.listenSockets, id) + return + } + if _, ok = c.servers[id]; ok { + delete(c.servers, id) + return + } +} + +type int64Slice []int64 + +func (s int64Slice) Len() int { return len(s) } +func (s int64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s int64Slice) Less(i, j int) bool { return s[i] < s[j] } + +func copyMap(m map[int64]string) map[int64]string { + n := make(map[int64]string) + for k, v := range m { + n[k] = v + } + return n +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func (c *channelMap) GetTopChannels(id int64) ([]*ChannelMetric, bool) { + c.mu.RLock() + l := len(c.topLevelChannels) + ids := make([]int64, 0, l) + cns := make([]*channel, 0, min(l, EntryPerPage)) + + for k := range c.topLevelChannels { + ids = append(ids, k) + } + sort.Sort(int64Slice(ids)) + idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= id }) + count := 0 + var end bool + var t []*ChannelMetric + for i, v := range ids[idx:] { + if count == EntryPerPage { + break + } + if cn, ok := c.channels[v]; ok { + cns = append(cns, cn) + t = append(t, &ChannelMetric{ + NestedChans: copyMap(cn.nestedChans), + SubChans: copyMap(cn.subChans), + }) + count++ + } + if i == len(ids[idx:])-1 { + end = true + break + } + } + c.mu.RUnlock() + if count == 0 { + end = true + } + + for i, cn := range cns { + t[i].ChannelData = cn.c.ChannelzMetric() + t[i].ID = cn.id + t[i].RefName = cn.refName + } + return t, end +} + +func (c *channelMap) GetServers(id int64) ([]*ServerMetric, bool) { + c.mu.RLock() + l := len(c.servers) + ids := make([]int64, 0, l) + ss := make([]*server, 0, min(l, EntryPerPage)) + for k := range c.servers { + ids = append(ids, k) + } + sort.Sort(int64Slice(ids)) + idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= id }) + count := 0 + var end bool + var s []*ServerMetric + for i, v := range ids[idx:] { + if count == EntryPerPage { + break + } + if svr, ok := c.servers[v]; ok { + ss = append(ss, svr) + s = append(s, &ServerMetric{ + ListenSockets: copyMap(svr.listenSockets), + }) + count++ + } + if i == len(ids[idx:])-1 { + end = true + break + } + } + c.mu.RUnlock() + if count == 0 { + end = true + } + + for i, svr := range ss { + s[i].ServerData = svr.s.ChannelzMetric() + s[i].ID = svr.id + s[i].RefName = svr.refName + } + return s, end +} + +func (c *channelMap) GetServerSockets(id int64, startID int64) ([]*SocketMetric, bool) { + var svr *server + var ok bool + c.mu.RLock() + if svr, ok = c.servers[id]; !ok { + // server with id doesn't exist. + c.mu.RUnlock() + return nil, true + } + svrskts := svr.sockets + l := len(svrskts) + ids := make([]int64, 0, l) + sks := make([]*normalSocket, 0, min(l, EntryPerPage)) + for k := range svrskts { + ids = append(ids, k) + } + sort.Sort((int64Slice(ids))) + idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= id }) + count := 0 + var end bool + for i, v := range ids[idx:] { + if count == EntryPerPage { + break + } + if ns, ok := c.normalSockets[v]; ok { + sks = append(sks, ns) + count++ + } + if i == len(ids[idx:])-1 { + end = true + break + } + } + c.mu.RUnlock() + if count == 0 { + end = true + } + var s []*SocketMetric + for _, ns := range sks { + sm := &SocketMetric{} + sm.SocketData = ns.s.ChannelzMetric() + sm.ID = ns.id + sm.RefName = ns.refName + s = append(s, sm) + } + return s, end +} + +func (c *channelMap) GetChannel(id int64) *ChannelMetric { + cm := &ChannelMetric{} + var cn *channel + var ok bool + c.mu.RLock() + if cn, ok = c.channels[id]; !ok { + // channel with id doesn't exist. + c.mu.RUnlock() + return nil + } + cm.NestedChans = copyMap(cn.nestedChans) + cm.SubChans = copyMap(cn.subChans) + c.mu.RUnlock() + cm.ChannelData = cn.c.ChannelzMetric() + cm.ID = cn.id + cm.RefName = cn.refName + return cm +} + +func (c *channelMap) GetSubChannel(id int64) *SubChannelMetric { + cm := &SubChannelMetric{} + var sc *subChannel + var ok bool + c.mu.RLock() + if sc, ok = c.subChannels[id]; !ok { + // subchannel with id doesn't exist. + c.mu.RUnlock() + return nil + } + cm.Sockets = copyMap(sc.sockets) + c.mu.RUnlock() + cm.ChannelData = sc.c.ChannelzMetric() + cm.ID = sc.id + cm.RefName = sc.refName + return cm +} + +func (c *channelMap) GetSocket(id int64) *SocketMetric { + sm := &SocketMetric{} + c.mu.RLock() + if ls, ok := c.listenSockets[id]; ok { + c.mu.RUnlock() + sm.SocketData = ls.s.ChannelzMetric() + sm.ID = ls.id + sm.RefName = ls.refName + return sm + } + if ns, ok := c.normalSockets[id]; ok { + c.mu.RUnlock() + sm.SocketData = ns.s.ChannelzMetric() + sm.ID = ns.id + sm.RefName = ns.refName + return sm + } + c.mu.RUnlock() + return nil +} + +type idGenerator struct { + id int64 +} + +func (i *idGenerator) reset() { + atomic.StoreInt64(&i.id, 0) +} + +func (i *idGenerator) genID() int64 { + return atomic.AddInt64(&i.id, 1) +} diff --git a/vendor/google.golang.org/grpc/internal/channelz/types.go b/vendor/google.golang.org/grpc/internal/channelz/types.go new file mode 100644 index 00000000000..153d75340e4 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/channelz/types.go @@ -0,0 +1,418 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz + +import ( + "net" + "time" + + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" +) + +// entry represents a node in the channelz database. +type entry interface { + // addChild adds a child e, whose channelz id is id to child list + addChild(id int64, e entry) + // deleteChild deletes a child with channelz id to be id from child list + deleteChild(id int64) + // triggerDelete tries to delete self from channelz database. However, if child + // list is not empty, then deletion from the database is on hold until the last + // child is deleted from database. + triggerDelete() + // deleteSelfIfReady check whether triggerDelete() has been called before, and whether child + // list is now empty. If both conditions are met, then delete self from database. + deleteSelfIfReady() +} + +// dummyEntry is a fake entry to handle entry not found case. +type dummyEntry struct { + idNotFound int64 +} + +func (d *dummyEntry) addChild(id int64, e entry) { + // Note: It is possible for a normal program to reach here under race condition. + // For example, there could be a race between ClientConn.Close() info being propagated + // to addrConn and http2Client. ClientConn.Close() cancel the context and result + // in http2Client to error. The error info is then caught by transport monitor + // and before addrConn.tearDown() is called in side ClientConn.Close(). Therefore, + // the addrConn will create a new transport. And when registering the new transport in + // channelz, its parent addrConn could have already been torn down and deleted + // from channelz tracking, and thus reach the code here. + grpclog.Infof("attempt to add child of type %T with id %d to a parent (id=%d) that doesn't currently exist", e, id, d.idNotFound) +} + +func (d *dummyEntry) deleteChild(id int64) { + // It is possible for a normal program to reach here under race condition. + // Refer to the example described in addChild(). + grpclog.Infof("attempt to delete child with id %d from a parent (id=%d) that doesn't currently exist", id, d.idNotFound) +} + +func (d *dummyEntry) triggerDelete() { + grpclog.Warningf("attempt to delete an entry (id=%d) that doesn't currently exist", d.idNotFound) +} + +func (*dummyEntry) deleteSelfIfReady() { + // code should not reach here. deleteSelfIfReady is always called on an existing entry. +} + +// ChannelMetric defines the info channelz provides for a specific Channel, which +// includes ChannelInternalMetric and channelz-specific data, such as channelz id, +// child list, etc. +type ChannelMetric struct { + // ID is the channelz id of this channel. + ID int64 + // RefName is the human readable reference string of this channel. + RefName string + // ChannelData contains channel internal metric reported by the channel through + // ChannelzMetric(). + ChannelData *ChannelInternalMetric + // NestedChans tracks the nested channel type children of this channel in the format of + // a map from nested channel channelz id to corresponding reference string. + NestedChans map[int64]string + // SubChans tracks the subchannel type children of this channel in the format of a + // map from subchannel channelz id to corresponding reference string. + SubChans map[int64]string + // Sockets tracks the socket type children of this channel in the format of a map + // from socket channelz id to corresponding reference string. + // Note current grpc implementation doesn't allow channel having sockets directly, + // therefore, this is field is unused. + Sockets map[int64]string +} + +// SubChannelMetric defines the info channelz provides for a specific SubChannel, +// which includes ChannelInternalMetric and channelz-specific data, such as +// channelz id, child list, etc. +type SubChannelMetric struct { + // ID is the channelz id of this subchannel. + ID int64 + // RefName is the human readable reference string of this subchannel. + RefName string + // ChannelData contains subchannel internal metric reported by the subchannel + // through ChannelzMetric(). + ChannelData *ChannelInternalMetric + // NestedChans tracks the nested channel type children of this subchannel in the format of + // a map from nested channel channelz id to corresponding reference string. + // Note current grpc implementation doesn't allow subchannel to have nested channels + // as children, therefore, this field is unused. + NestedChans map[int64]string + // SubChans tracks the subchannel type children of this subchannel in the format of a + // map from subchannel channelz id to corresponding reference string. + // Note current grpc implementation doesn't allow subchannel to have subchannels + // as children, therefore, this field is unused. + SubChans map[int64]string + // Sockets tracks the socket type children of this subchannel in the format of a map + // from socket channelz id to corresponding reference string. + Sockets map[int64]string +} + +// ChannelInternalMetric defines the struct that the implementor of Channel interface +// should return from ChannelzMetric(). +type ChannelInternalMetric struct { + // current connectivity state of the channel. + State connectivity.State + // The target this channel originally tried to connect to. May be absent + Target string + // The number of calls started on the channel. + CallsStarted int64 + // The number of calls that have completed with an OK status. + CallsSucceeded int64 + // The number of calls that have a completed with a non-OK status. + CallsFailed int64 + // The last time a call was started on the channel. + LastCallStartedTimestamp time.Time + //TODO: trace +} + +// Channel is the interface that should be satisfied in order to be tracked by +// channelz as Channel or SubChannel. +type Channel interface { + ChannelzMetric() *ChannelInternalMetric +} + +type channel struct { + refName string + c Channel + closeCalled bool + nestedChans map[int64]string + subChans map[int64]string + id int64 + pid int64 + cm *channelMap +} + +func (c *channel) addChild(id int64, e entry) { + switch v := e.(type) { + case *subChannel: + c.subChans[id] = v.refName + case *channel: + c.nestedChans[id] = v.refName + default: + grpclog.Errorf("cannot add a child (id = %d) of type %T to a channel", id, e) + } +} + +func (c *channel) deleteChild(id int64) { + delete(c.subChans, id) + delete(c.nestedChans, id) + c.deleteSelfIfReady() +} + +func (c *channel) triggerDelete() { + c.closeCalled = true + c.deleteSelfIfReady() +} + +func (c *channel) deleteSelfIfReady() { + if !c.closeCalled || len(c.subChans)+len(c.nestedChans) != 0 { + return + } + c.cm.deleteEntry(c.id) + // not top channel + if c.pid != 0 { + c.cm.findEntry(c.pid).deleteChild(c.id) + } +} + +type subChannel struct { + refName string + c Channel + closeCalled bool + sockets map[int64]string + id int64 + pid int64 + cm *channelMap +} + +func (sc *subChannel) addChild(id int64, e entry) { + if v, ok := e.(*normalSocket); ok { + sc.sockets[id] = v.refName + } else { + grpclog.Errorf("cannot add a child (id = %d) of type %T to a subChannel", id, e) + } +} + +func (sc *subChannel) deleteChild(id int64) { + delete(sc.sockets, id) + sc.deleteSelfIfReady() +} + +func (sc *subChannel) triggerDelete() { + sc.closeCalled = true + sc.deleteSelfIfReady() +} + +func (sc *subChannel) deleteSelfIfReady() { + if !sc.closeCalled || len(sc.sockets) != 0 { + return + } + sc.cm.deleteEntry(sc.id) + sc.cm.findEntry(sc.pid).deleteChild(sc.id) +} + +// SocketMetric defines the info channelz provides for a specific Socket, which +// includes SocketInternalMetric and channelz-specific data, such as channelz id, etc. +type SocketMetric struct { + // ID is the channelz id of this socket. + ID int64 + // RefName is the human readable reference string of this socket. + RefName string + // SocketData contains socket internal metric reported by the socket through + // ChannelzMetric(). + SocketData *SocketInternalMetric +} + +// SocketInternalMetric defines the struct that the implementor of Socket interface +// should return from ChannelzMetric(). +type SocketInternalMetric struct { + // The number of streams that have been started. + StreamsStarted int64 + // The number of streams that have ended successfully: + // On client side, receiving frame with eos bit set. + // On server side, sending frame with eos bit set. + StreamsSucceeded int64 + // The number of streams that have ended unsuccessfully: + // On client side, termination without receiving frame with eos bit set. + // On server side, termination without sending frame with eos bit set. + StreamsFailed int64 + // The number of messages successfully sent on this socket. + MessagesSent int64 + MessagesReceived int64 + // The number of keep alives sent. This is typically implemented with HTTP/2 + // ping messages. + KeepAlivesSent int64 + // The last time a stream was created by this endpoint. Usually unset for + // servers. + LastLocalStreamCreatedTimestamp time.Time + // The last time a stream was created by the remote endpoint. Usually unset + // for clients. + LastRemoteStreamCreatedTimestamp time.Time + // The last time a message was sent by this endpoint. + LastMessageSentTimestamp time.Time + // The last time a message was received by this endpoint. + LastMessageReceivedTimestamp time.Time + // The amount of window, granted to the local endpoint by the remote endpoint. + // This may be slightly out of date due to network latency. This does NOT + // include stream level or TCP level flow control info. + LocalFlowControlWindow int64 + // The amount of window, granted to the remote endpoint by the local endpoint. + // This may be slightly out of date due to network latency. This does NOT + // include stream level or TCP level flow control info. + RemoteFlowControlWindow int64 + // The locally bound address. + LocalAddr net.Addr + // The remote bound address. May be absent. + RemoteAddr net.Addr + // Optional, represents the name of the remote endpoint, if different than + // the original target name. + RemoteName string + //TODO: socket options + //TODO: Security +} + +// Socket is the interface that should be satisfied in order to be tracked by +// channelz as Socket. +type Socket interface { + ChannelzMetric() *SocketInternalMetric +} + +type listenSocket struct { + refName string + s Socket + id int64 + pid int64 + cm *channelMap +} + +func (ls *listenSocket) addChild(id int64, e entry) { + grpclog.Errorf("cannot add a child (id = %d) of type %T to a listen socket", id, e) +} + +func (ls *listenSocket) deleteChild(id int64) { + grpclog.Errorf("cannot delete a child (id = %d) from a listen socket", id) +} + +func (ls *listenSocket) triggerDelete() { + ls.cm.deleteEntry(ls.id) + ls.cm.findEntry(ls.pid).deleteChild(ls.id) +} + +func (ls *listenSocket) deleteSelfIfReady() { + grpclog.Errorf("cannot call deleteSelfIfReady on a listen socket") +} + +type normalSocket struct { + refName string + s Socket + id int64 + pid int64 + cm *channelMap +} + +func (ns *normalSocket) addChild(id int64, e entry) { + grpclog.Errorf("cannot add a child (id = %d) of type %T to a normal socket", id, e) +} + +func (ns *normalSocket) deleteChild(id int64) { + grpclog.Errorf("cannot delete a child (id = %d) from a normal socket", id) +} + +func (ns *normalSocket) triggerDelete() { + ns.cm.deleteEntry(ns.id) + ns.cm.findEntry(ns.pid).deleteChild(ns.id) +} + +func (ns *normalSocket) deleteSelfIfReady() { + grpclog.Errorf("cannot call deleteSelfIfReady on a normal socket") +} + +// ServerMetric defines the info channelz provides for a specific Server, which +// includes ServerInternalMetric and channelz-specific data, such as channelz id, +// child list, etc. +type ServerMetric struct { + // ID is the channelz id of this server. + ID int64 + // RefName is the human readable reference string of this server. + RefName string + // ServerData contains server internal metric reported by the server through + // ChannelzMetric(). + ServerData *ServerInternalMetric + // ListenSockets tracks the listener socket type children of this server in the + // format of a map from socket channelz id to corresponding reference string. + ListenSockets map[int64]string +} + +// ServerInternalMetric defines the struct that the implementor of Server interface +// should return from ChannelzMetric(). +type ServerInternalMetric struct { + // The number of incoming calls started on the server. + CallsStarted int64 + // The number of incoming calls that have completed with an OK status. + CallsSucceeded int64 + // The number of incoming calls that have a completed with a non-OK status. + CallsFailed int64 + // The last time a call was started on the server. + LastCallStartedTimestamp time.Time + //TODO: trace +} + +// Server is the interface to be satisfied in order to be tracked by channelz as +// Server. +type Server interface { + ChannelzMetric() *ServerInternalMetric +} + +type server struct { + refName string + s Server + closeCalled bool + sockets map[int64]string + listenSockets map[int64]string + id int64 + cm *channelMap +} + +func (s *server) addChild(id int64, e entry) { + switch v := e.(type) { + case *normalSocket: + s.sockets[id] = v.refName + case *listenSocket: + s.listenSockets[id] = v.refName + default: + grpclog.Errorf("cannot add a child (id = %d) of type %T to a server", id, e) + } +} + +func (s *server) deleteChild(id int64) { + delete(s.sockets, id) + delete(s.listenSockets, id) + s.deleteSelfIfReady() +} + +func (s *server) triggerDelete() { + s.closeCalled = true + s.deleteSelfIfReady() +} + +func (s *server) deleteSelfIfReady() { + if !s.closeCalled || len(s.sockets)+len(s.listenSockets) != 0 { + return + } + s.cm.deleteEntry(s.id) +} diff --git a/vendor/google.golang.org/grpc/internal/grpcrand/BUILD b/vendor/google.golang.org/grpc/internal/grpcrand/BUILD new file mode 100644 index 00000000000..0768b8d8c28 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/grpcrand/BUILD @@ -0,0 +1,23 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["grpcrand.go"], + importmap = "k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/grpcrand", + importpath = "google.golang.org/grpc/internal/grpcrand", + visibility = ["//vendor/google.golang.org/grpc:__subpackages__"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/google.golang.org/grpc/internal/grpcrand/grpcrand.go b/vendor/google.golang.org/grpc/internal/grpcrand/grpcrand.go new file mode 100644 index 00000000000..200b115ca20 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/grpcrand/grpcrand.go @@ -0,0 +1,56 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpcrand implements math/rand functions in a concurrent-safe way +// with a global random source, independent of math/rand's global source. +package grpcrand + +import ( + "math/rand" + "sync" + "time" +) + +var ( + r = rand.New(rand.NewSource(time.Now().UnixNano())) + mu sync.Mutex +) + +// Int63n implements rand.Int63n on the grpcrand global source. +func Int63n(n int64) int64 { + mu.Lock() + res := r.Int63n(n) + mu.Unlock() + return res +} + +// Intn implements rand.Intn on the grpcrand global source. +func Intn(n int) int { + mu.Lock() + res := r.Intn(n) + mu.Unlock() + return res +} + +// Float64 implements rand.Float64 on the grpcrand global source. +func Float64() float64 { + mu.Lock() + res := r.Float64() + mu.Unlock() + return res +} diff --git a/vendor/google.golang.org/grpc/internal/internal.go b/vendor/google.golang.org/grpc/internal/internal.go index 07083832c3c..cd34267f7f3 100644 --- a/vendor/google.golang.org/grpc/internal/internal.go +++ b/vendor/google.golang.org/grpc/internal/internal.go @@ -15,20 +15,22 @@ * */ -// Package internal contains gRPC-internal code for testing, to avoid polluting -// the godoc of the top-level grpc package. +// Package internal contains gRPC-internal code, to avoid polluting +// the godoc of the top-level grpc package. It must not import any grpc +// symbols to avoid circular dependencies. package internal -// TestingCloseConns closes all existing transports but keeps -// grpcServer.lis accepting new connections. -// -// The provided grpcServer must be of type *grpc.Server. It is untyped -// for circular dependency reasons. -var TestingCloseConns func(grpcServer interface{}) +var ( -// TestingUseHandlerImpl enables the http.Handler-based server implementation. -// It must be called before Serve and requires TLS credentials. -// -// The provided grpcServer must be of type *grpc.Server. It is untyped -// for circular dependency reasons. -var TestingUseHandlerImpl func(grpcServer interface{}) + // TestingUseHandlerImpl enables the http.Handler-based server implementation. + // It must be called before Serve and requires TLS credentials. + // + // The provided grpcServer must be of type *grpc.Server. It is untyped + // for circular dependency reasons. + TestingUseHandlerImpl func(grpcServer interface{}) + + // WithContextDialer is exported by clientconn.go + WithContextDialer interface{} // func(context.Context, string) (net.Conn, error) grpc.DialOption + // WithResolverBuilder is exported by clientconn.go + WithResolverBuilder interface{} // func (resolver.Builder) grpc.DialOption +) diff --git a/vendor/google.golang.org/grpc/metadata/metadata.go b/vendor/google.golang.org/grpc/metadata/metadata.go index 589161d57fa..3f8a9edaeea 100644 --- a/vendor/google.golang.org/grpc/metadata/metadata.go +++ b/vendor/google.golang.org/grpc/metadata/metadata.go @@ -17,7 +17,8 @@ */ // Package metadata define the structure of the metadata supported by gRPC library. -// Please refer to https://grpc.io/docs/guides/wire.html for more information about custom-metadata. +// Please refer to https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md +// for more information about custom-metadata. package metadata import ( @@ -27,7 +28,9 @@ import ( "golang.org/x/net/context" ) -// DecodeKeyValue returns k, v, nil. It is deprecated and should not be used. +// DecodeKeyValue returns k, v, nil. +// +// Deprecated: use k and v directly instead. func DecodeKeyValue(k, v string) (string, string, error) { return k, v, nil } @@ -94,6 +97,30 @@ func (md MD) Copy() MD { return Join(md) } +// Get obtains the values for a given key. +func (md MD) Get(k string) []string { + k = strings.ToLower(k) + return md[k] +} + +// Set sets the value of a given key with a slice of values. +func (md MD) Set(k string, vals ...string) { + if len(vals) == 0 { + return + } + k = strings.ToLower(k) + md[k] = vals +} + +// Append adds the values to key k, not overwriting what was already stored at that key. +func (md MD) Append(k string, vals ...string) { + if len(vals) == 0 { + return + } + k = strings.ToLower(k) + md[k] = append(md[k], vals...) +} + // Join joins any number of mds into a single MD. // The order of values for each key is determined by the order in which // the mds containing those values are presented to Join. @@ -115,9 +142,26 @@ func NewIncomingContext(ctx context.Context, md MD) context.Context { return context.WithValue(ctx, mdIncomingKey{}, md) } -// NewOutgoingContext creates a new context with outgoing md attached. +// NewOutgoingContext creates a new context with outgoing md attached. If used +// in conjunction with AppendToOutgoingContext, NewOutgoingContext will +// overwrite any previously-appended metadata. func NewOutgoingContext(ctx context.Context, md MD) context.Context { - return context.WithValue(ctx, mdOutgoingKey{}, md) + return context.WithValue(ctx, mdOutgoingKey{}, rawMD{md: md}) +} + +// AppendToOutgoingContext returns a new context with the provided kv merged +// with any existing metadata in the context. Please refer to the +// documentation of Pairs for a description of kv. +func AppendToOutgoingContext(ctx context.Context, kv ...string) context.Context { + if len(kv)%2 == 1 { + panic(fmt.Sprintf("metadata: AppendToOutgoingContext got an odd number of input pairs for metadata: %d", len(kv))) + } + md, _ := ctx.Value(mdOutgoingKey{}).(rawMD) + added := make([][]string, len(md.added)+1) + copy(added, md.added) + added[len(added)-1] = make([]string, len(kv)) + copy(added[len(added)-1], kv) + return context.WithValue(ctx, mdOutgoingKey{}, rawMD{md: md.md, added: added}) } // FromIncomingContext returns the incoming metadata in ctx if it exists. The @@ -128,10 +172,39 @@ func FromIncomingContext(ctx context.Context) (md MD, ok bool) { return } +// FromOutgoingContextRaw returns the un-merged, intermediary contents +// of rawMD. Remember to perform strings.ToLower on the keys. The returned +// MD should not be modified. Writing to it may cause races. Modification +// should be made to copies of the returned MD. +// +// This is intended for gRPC-internal use ONLY. +func FromOutgoingContextRaw(ctx context.Context) (MD, [][]string, bool) { + raw, ok := ctx.Value(mdOutgoingKey{}).(rawMD) + if !ok { + return nil, nil, false + } + + return raw.md, raw.added, true +} + // FromOutgoingContext returns the outgoing metadata in ctx if it exists. The // returned MD should not be modified. Writing to it may cause races. -// Modification should be made to the copies of the returned MD. -func FromOutgoingContext(ctx context.Context) (md MD, ok bool) { - md, ok = ctx.Value(mdOutgoingKey{}).(MD) - return +// Modification should be made to copies of the returned MD. +func FromOutgoingContext(ctx context.Context) (MD, bool) { + raw, ok := ctx.Value(mdOutgoingKey{}).(rawMD) + if !ok { + return nil, false + } + + mds := make([]MD, 0, len(raw.added)+1) + mds = append(mds, raw.md) + for _, vv := range raw.added { + mds = append(mds, Pairs(vv...)) + } + return Join(mds...), ok +} + +type rawMD struct { + md MD + added [][]string } diff --git a/vendor/google.golang.org/grpc/naming/dns_resolver.go b/vendor/google.golang.org/grpc/naming/dns_resolver.go index 7e69a2ca0a6..0f8a908ea9c 100644 --- a/vendor/google.golang.org/grpc/naming/dns_resolver.go +++ b/vendor/google.golang.org/grpc/naming/dns_resolver.go @@ -153,10 +153,10 @@ type ipWatcher struct { updateChan chan *Update } -// Next returns the adrress resolution Update for the target. For IP address, -// the resolution is itself, thus polling name server is unncessary. Therefore, +// Next returns the address resolution Update for the target. For IP address, +// the resolution is itself, thus polling name server is unnecessary. Therefore, // Next() will return an Update the first time it is called, and will be blocked -// for all following calls as no Update exisits until watcher is closed. +// for all following calls as no Update exists until watcher is closed. func (i *ipWatcher) Next() ([]*Update, error) { u, ok := <-i.updateChan if !ok { diff --git a/vendor/google.golang.org/grpc/naming/go17.go b/vendor/google.golang.org/grpc/naming/go17.go index 8bdf21e7998..57b65d7b889 100644 --- a/vendor/google.golang.org/grpc/naming/go17.go +++ b/vendor/google.golang.org/grpc/naming/go17.go @@ -1,4 +1,4 @@ -// +build go1.7, !go1.8 +// +build go1.6,!go1.8 /* * diff --git a/vendor/google.golang.org/grpc/naming/naming.go b/vendor/google.golang.org/grpc/naming/naming.go index 1af7e32f86d..8cc39e93758 100644 --- a/vendor/google.golang.org/grpc/naming/naming.go +++ b/vendor/google.golang.org/grpc/naming/naming.go @@ -18,20 +18,26 @@ // Package naming defines the naming API and related data structures for gRPC. // The interface is EXPERIMENTAL and may be suject to change. +// +// Deprecated: please use package resolver. package naming // Operation defines the corresponding operations for a name resolution change. +// +// Deprecated: please use package resolver. type Operation uint8 const ( // Add indicates a new address is added. Add Operation = iota - // Delete indicates an exisiting address is deleted. + // Delete indicates an existing address is deleted. Delete ) // Update defines a name resolution update. Notice that it is not valid having both // empty string Addr and nil Metadata in an Update. +// +// Deprecated: please use package resolver. type Update struct { // Op indicates the operation of the update. Op Operation @@ -43,12 +49,16 @@ type Update struct { } // Resolver creates a Watcher for a target to track its resolution changes. +// +// Deprecated: please use package resolver. type Resolver interface { // Resolve creates a Watcher for target. Resolve(target string) (Watcher, error) } // Watcher watches for the updates on the specified target. +// +// Deprecated: please use package resolver. type Watcher interface { // Next blocks until an update or error happens. It may return one or more // updates. The first call should get the full set of the results. It should diff --git a/vendor/google.golang.org/grpc/picker_wrapper.go b/vendor/google.golang.org/grpc/picker_wrapper.go index 9085dbc9c98..019e658004e 100644 --- a/vendor/google.golang.org/grpc/picker_wrapper.go +++ b/vendor/google.golang.org/grpc/picker_wrapper.go @@ -19,12 +19,17 @@ package grpc import ( + "io" "sync" + "sync/atomic" "golang.org/x/net/context" "google.golang.org/grpc/balancer" "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" "google.golang.org/grpc/status" "google.golang.org/grpc/transport" ) @@ -36,13 +41,57 @@ type pickerWrapper struct { done bool blockingCh chan struct{} picker balancer.Picker + + // The latest connection happened. + connErrMu sync.Mutex + connErr error + + stickinessMDKey atomic.Value + stickiness *stickyStore } func newPickerWrapper() *pickerWrapper { - bp := &pickerWrapper{blockingCh: make(chan struct{})} + bp := &pickerWrapper{ + blockingCh: make(chan struct{}), + stickiness: newStickyStore(), + } return bp } +func (bp *pickerWrapper) updateConnectionError(err error) { + bp.connErrMu.Lock() + bp.connErr = err + bp.connErrMu.Unlock() +} + +func (bp *pickerWrapper) connectionError() error { + bp.connErrMu.Lock() + err := bp.connErr + bp.connErrMu.Unlock() + return err +} + +func (bp *pickerWrapper) updateStickinessMDKey(newKey string) { + // No need to check ok because mdKey == "" if ok == false. + if oldKey, _ := bp.stickinessMDKey.Load().(string); oldKey != newKey { + bp.stickinessMDKey.Store(newKey) + bp.stickiness.reset(newKey) + } +} + +func (bp *pickerWrapper) getStickinessMDKey() string { + // No need to check ok because mdKey == "" if ok == false. + mdKey, _ := bp.stickinessMDKey.Load().(string) + return mdKey +} + +func (bp *pickerWrapper) clearStickinessState() { + if oldKey := bp.getStickinessMDKey(); oldKey != "" { + // There's no need to reset store if mdKey was "". + bp.stickiness.reset(oldKey) + } +} + // updatePicker is called by UpdateBalancerState. It unblocks all blocked pick. func (bp *pickerWrapper) updatePicker(p balancer.Picker) { bp.mu.Lock() @@ -57,6 +106,23 @@ func (bp *pickerWrapper) updatePicker(p balancer.Picker) { bp.mu.Unlock() } +func doneChannelzWrapper(acw *acBalancerWrapper, done func(balancer.DoneInfo)) func(balancer.DoneInfo) { + acw.mu.Lock() + ac := acw.ac + acw.mu.Unlock() + ac.incrCallsStarted() + return func(b balancer.DoneInfo) { + if b.Err != nil && b.Err != io.EOF { + ac.incrCallsFailed() + } else { + ac.incrCallsSucceeded() + } + if done != nil { + done(b) + } + } +} + // pick returns the transport that will be used for the RPC. // It may block in the following cases: // - there's no picker @@ -65,6 +131,27 @@ func (bp *pickerWrapper) updatePicker(p balancer.Picker) { // - the subConn returned by the current picker is not READY // When one of these situations happens, pick blocks until the picker gets updated. func (bp *pickerWrapper) pick(ctx context.Context, failfast bool, opts balancer.PickOptions) (transport.ClientTransport, func(balancer.DoneInfo), error) { + + mdKey := bp.getStickinessMDKey() + stickyKey, isSticky := stickyKeyFromContext(ctx, mdKey) + + // Potential race here: if stickinessMDKey is updated after the above two + // lines, and this pick is a sticky pick, the following put could add an + // entry to sticky store with an outdated sticky key. + // + // The solution: keep the current md key in sticky store, and at the + // beginning of each get/put, check the mdkey against store.curMDKey. + // - Cons: one more string comparing for each get/put. + // - Pros: the string matching happens inside get/put, so the overhead for + // non-sticky RPCs will be minimal. + + if isSticky { + if t, ok := bp.stickiness.get(mdKey, stickyKey); ok { + // Done function returned is always nil. + return t, nil, nil + } + } + var ( p balancer.Picker ch chan struct{} @@ -97,7 +184,7 @@ func (bp *pickerWrapper) pick(ctx context.Context, failfast bool, opts balancer. p = bp.picker bp.mu.Unlock() - subConn, put, err := p.Pick(ctx, opts) + subConn, done, err := p.Pick(ctx, opts) if err != nil { switch err { @@ -107,7 +194,7 @@ func (bp *pickerWrapper) pick(ctx context.Context, failfast bool, opts balancer. if !failfast { continue } - return nil, nil, status.Errorf(codes.Unavailable, "%v", err) + return nil, nil, status.Errorf(codes.Unavailable, "%v, latest connection error: %v", err, bp.connectionError()) default: // err is some other error. return nil, nil, toRPCErr(err) @@ -120,7 +207,13 @@ func (bp *pickerWrapper) pick(ctx context.Context, failfast bool, opts balancer. continue } if t, ok := acw.getAddrConn().getReadyTransport(); ok { - return t, put, nil + if isSticky { + bp.stickiness.put(mdKey, stickyKey, acw) + } + if channelz.IsOn() { + return t, doneChannelzWrapper(acw, done), nil + } + return t, done, nil } grpclog.Infof("blockingPicker: the picked transport is not ready, loop back to repick") // If ok == false, ac.state is not READY. @@ -139,3 +232,105 @@ func (bp *pickerWrapper) close() { bp.done = true close(bp.blockingCh) } + +const stickinessKeyCountLimit = 1000 + +type stickyStoreEntry struct { + acw *acBalancerWrapper + addr resolver.Address +} + +type stickyStore struct { + mu sync.Mutex + // curMDKey is check before every get/put to avoid races. The operation will + // abort immediately when the given mdKey is different from the curMDKey. + curMDKey string + store *linkedMap +} + +func newStickyStore() *stickyStore { + return &stickyStore{ + store: newLinkedMap(), + } +} + +// reset clears the map in stickyStore, and set the currentMDKey to newMDKey. +func (ss *stickyStore) reset(newMDKey string) { + ss.mu.Lock() + ss.curMDKey = newMDKey + ss.store.clear() + ss.mu.Unlock() +} + +// stickyKey is the key to look up in store. mdKey will be checked against +// curMDKey to avoid races. +func (ss *stickyStore) put(mdKey, stickyKey string, acw *acBalancerWrapper) { + ss.mu.Lock() + defer ss.mu.Unlock() + if mdKey != ss.curMDKey { + return + } + // TODO(stickiness): limit the total number of entries. + ss.store.put(stickyKey, &stickyStoreEntry{ + acw: acw, + addr: acw.getAddrConn().getCurAddr(), + }) + if ss.store.len() > stickinessKeyCountLimit { + ss.store.removeOldest() + } +} + +// stickyKey is the key to look up in store. mdKey will be checked against +// curMDKey to avoid races. +func (ss *stickyStore) get(mdKey, stickyKey string) (transport.ClientTransport, bool) { + ss.mu.Lock() + defer ss.mu.Unlock() + if mdKey != ss.curMDKey { + return nil, false + } + entry, ok := ss.store.get(stickyKey) + if !ok { + return nil, false + } + ac := entry.acw.getAddrConn() + if ac.getCurAddr() != entry.addr { + ss.store.remove(stickyKey) + return nil, false + } + t, ok := ac.getReadyTransport() + if !ok { + ss.store.remove(stickyKey) + return nil, false + } + return t, true +} + +// Get one value from metadata in ctx with key stickinessMDKey. +// +// It returns "", false if stickinessMDKey is an empty string. +func stickyKeyFromContext(ctx context.Context, stickinessMDKey string) (string, bool) { + if stickinessMDKey == "" { + return "", false + } + + md, added, ok := metadata.FromOutgoingContextRaw(ctx) + if !ok { + return "", false + } + + if vv, ok := md[stickinessMDKey]; ok { + if len(vv) > 0 { + return vv[0], true + } + } + + for _, ss := range added { + for i := 0; i < len(ss)-1; i += 2 { + if ss[i] == stickinessMDKey { + return ss[i+1], true + } + } + } + + return "", false +} diff --git a/vendor/google.golang.org/grpc/pickfirst.go b/vendor/google.golang.org/grpc/pickfirst.go index 7f993ef5a38..bf659d49d2f 100644 --- a/vendor/google.golang.org/grpc/pickfirst.go +++ b/vendor/google.golang.org/grpc/pickfirst.go @@ -26,6 +26,9 @@ import ( "google.golang.org/grpc/resolver" ) +// PickFirstBalancerName is the name of the pick_first balancer. +const PickFirstBalancerName = "pick_first" + func newPickfirstBuilder() balancer.Builder { return &pickfirstBuilder{} } @@ -37,7 +40,7 @@ func (*pickfirstBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions } func (*pickfirstBuilder) Name() string { - return "pickfirst" + return PickFirstBalancerName } type pickfirstBalancer struct { @@ -57,14 +60,20 @@ func (b *pickfirstBalancer) HandleResolvedAddrs(addrs []resolver.Address, err er return } b.cc.UpdateBalancerState(connectivity.Idle, &picker{sc: b.sc}) + b.sc.Connect() } else { b.sc.UpdateAddresses(addrs) + b.sc.Connect() } } func (b *pickfirstBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { grpclog.Infof("pickfirstBalancer: HandleSubConnStateChange: %p, %v", sc, s) - if b.sc != sc || s == connectivity.Shutdown { + if b.sc != sc { + grpclog.Infof("pickfirstBalancer: ignored state change because sc is not recognized") + return + } + if s == connectivity.Shutdown { b.sc = nil return } @@ -93,3 +102,7 @@ func (p *picker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer. } return p.sc, nil, nil } + +func init() { + balancer.Register(newPickfirstBuilder()) +} diff --git a/vendor/google.golang.org/grpc/proxy.go b/vendor/google.golang.org/grpc/proxy.go index 3e17efec61b..2d40236e218 100644 --- a/vendor/google.golang.org/grpc/proxy.go +++ b/vendor/google.golang.org/grpc/proxy.go @@ -82,8 +82,7 @@ func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, addr string) (_ Header: map[string][]string{"User-Agent": {grpcUA}}, }) - req = req.WithContext(ctx) - if err := req.Write(conn); err != nil { + if err := sendHTTPRequest(ctx, req, conn); err != nil { return nil, fmt.Errorf("failed to write the HTTP request: %v", err) } diff --git a/vendor/google.golang.org/grpc/resolver/BUILD b/vendor/google.golang.org/grpc/resolver/BUILD index 09d3018568f..e36360e71a8 100644 --- a/vendor/google.golang.org/grpc/resolver/BUILD +++ b/vendor/google.golang.org/grpc/resolver/BUILD @@ -17,7 +17,11 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//vendor/google.golang.org/grpc/resolver/dns:all-srcs", + "//vendor/google.golang.org/grpc/resolver/passthrough:all-srcs", + ], tags = ["automanaged"], visibility = ["//visibility:public"], ) diff --git a/vendor/google.golang.org/grpc/resolver/dns/BUILD b/vendor/google.golang.org/grpc/resolver/dns/BUILD new file mode 100644 index 00000000000..3966dc33331 --- /dev/null +++ b/vendor/google.golang.org/grpc/resolver/dns/BUILD @@ -0,0 +1,33 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "dns_resolver.go", + "go17.go", + "go18.go", + ], + importmap = "k8s.io/kubernetes/vendor/google.golang.org/grpc/resolver/dns", + importpath = "google.golang.org/grpc/resolver/dns", + visibility = ["//visibility:public"], + deps = [ + "//vendor/golang.org/x/net/context:go_default_library", + "//vendor/google.golang.org/grpc/grpclog:go_default_library", + "//vendor/google.golang.org/grpc/internal/grpcrand:go_default_library", + "//vendor/google.golang.org/grpc/resolver:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go b/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go new file mode 100644 index 00000000000..048fde67d00 --- /dev/null +++ b/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go @@ -0,0 +1,381 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package dns implements a dns resolver to be installed as the default resolver +// in grpc. +package dns + +import ( + "encoding/json" + "errors" + "fmt" + "net" + "os" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/net/context" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/grpcrand" + "google.golang.org/grpc/resolver" +) + +func init() { + resolver.Register(NewBuilder()) +} + +const ( + defaultPort = "443" + defaultFreq = time.Minute * 30 + golang = "GO" + // In DNS, service config is encoded in a TXT record via the mechanism + // described in RFC-1464 using the attribute name grpc_config. + txtAttribute = "grpc_config=" +) + +var ( + errMissingAddr = errors.New("missing address") +) + +// NewBuilder creates a dnsBuilder which is used to factory DNS resolvers. +func NewBuilder() resolver.Builder { + return &dnsBuilder{freq: defaultFreq} +} + +type dnsBuilder struct { + // frequency of polling the DNS server. + freq time.Duration +} + +// Build creates and starts a DNS resolver that watches the name resolution of the target. +func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOption) (resolver.Resolver, error) { + if target.Authority != "" { + return nil, fmt.Errorf("Default DNS resolver does not support custom DNS server") + } + host, port, err := parseTarget(target.Endpoint) + if err != nil { + return nil, err + } + + // IP address. + if net.ParseIP(host) != nil { + host, _ = formatIP(host) + addr := []resolver.Address{{Addr: host + ":" + port}} + i := &ipResolver{ + cc: cc, + ip: addr, + rn: make(chan struct{}, 1), + q: make(chan struct{}), + } + cc.NewAddress(addr) + go i.watcher() + return i, nil + } + + // DNS address (non-IP). + ctx, cancel := context.WithCancel(context.Background()) + d := &dnsResolver{ + freq: b.freq, + host: host, + port: port, + ctx: ctx, + cancel: cancel, + cc: cc, + t: time.NewTimer(0), + rn: make(chan struct{}, 1), + disableServiceConfig: opts.DisableServiceConfig, + } + + d.wg.Add(1) + go d.watcher() + return d, nil +} + +// Scheme returns the naming scheme of this resolver builder, which is "dns". +func (b *dnsBuilder) Scheme() string { + return "dns" +} + +// ipResolver watches for the name resolution update for an IP address. +type ipResolver struct { + cc resolver.ClientConn + ip []resolver.Address + // rn channel is used by ResolveNow() to force an immediate resolution of the target. + rn chan struct{} + q chan struct{} +} + +// ResolveNow resend the address it stores, no resolution is needed. +func (i *ipResolver) ResolveNow(opt resolver.ResolveNowOption) { + select { + case i.rn <- struct{}{}: + default: + } +} + +// Close closes the ipResolver. +func (i *ipResolver) Close() { + close(i.q) +} + +func (i *ipResolver) watcher() { + for { + select { + case <-i.rn: + i.cc.NewAddress(i.ip) + case <-i.q: + return + } + } +} + +// dnsResolver watches for the name resolution update for a non-IP target. +type dnsResolver struct { + freq time.Duration + host string + port string + ctx context.Context + cancel context.CancelFunc + cc resolver.ClientConn + // rn channel is used by ResolveNow() to force an immediate resolution of the target. + rn chan struct{} + t *time.Timer + // wg is used to enforce Close() to return after the watcher() goroutine has finished. + // Otherwise, data race will be possible. [Race Example] in dns_resolver_test we + // replace the real lookup functions with mocked ones to facilitate testing. + // If Close() doesn't wait for watcher() goroutine finishes, race detector sometimes + // will warns lookup (READ the lookup function pointers) inside watcher() goroutine + // has data race with replaceNetFunc (WRITE the lookup function pointers). + wg sync.WaitGroup + disableServiceConfig bool +} + +// ResolveNow invoke an immediate resolution of the target that this dnsResolver watches. +func (d *dnsResolver) ResolveNow(opt resolver.ResolveNowOption) { + select { + case d.rn <- struct{}{}: + default: + } +} + +// Close closes the dnsResolver. +func (d *dnsResolver) Close() { + d.cancel() + d.wg.Wait() + d.t.Stop() +} + +func (d *dnsResolver) watcher() { + defer d.wg.Done() + for { + select { + case <-d.ctx.Done(): + return + case <-d.t.C: + case <-d.rn: + } + result, sc := d.lookup() + // Next lookup should happen after an interval defined by d.freq. + d.t.Reset(d.freq) + d.cc.NewServiceConfig(sc) + d.cc.NewAddress(result) + } +} + +func (d *dnsResolver) lookupSRV() []resolver.Address { + var newAddrs []resolver.Address + _, srvs, err := lookupSRV(d.ctx, "grpclb", "tcp", d.host) + if err != nil { + grpclog.Infof("grpc: failed dns SRV record lookup due to %v.\n", err) + return nil + } + for _, s := range srvs { + lbAddrs, err := lookupHost(d.ctx, s.Target) + if err != nil { + grpclog.Infof("grpc: failed load balancer address dns lookup due to %v.\n", err) + continue + } + for _, a := range lbAddrs { + a, ok := formatIP(a) + if !ok { + grpclog.Errorf("grpc: failed IP parsing due to %v.\n", err) + continue + } + addr := a + ":" + strconv.Itoa(int(s.Port)) + newAddrs = append(newAddrs, resolver.Address{Addr: addr, Type: resolver.GRPCLB, ServerName: s.Target}) + } + } + return newAddrs +} + +func (d *dnsResolver) lookupTXT() string { + ss, err := lookupTXT(d.ctx, d.host) + if err != nil { + grpclog.Infof("grpc: failed dns TXT record lookup due to %v.\n", err) + return "" + } + var res string + for _, s := range ss { + res += s + } + + // TXT record must have "grpc_config=" attribute in order to be used as service config. + if !strings.HasPrefix(res, txtAttribute) { + grpclog.Warningf("grpc: TXT record %v missing %v attribute", res, txtAttribute) + return "" + } + return strings.TrimPrefix(res, txtAttribute) +} + +func (d *dnsResolver) lookupHost() []resolver.Address { + var newAddrs []resolver.Address + addrs, err := lookupHost(d.ctx, d.host) + if err != nil { + grpclog.Warningf("grpc: failed dns A record lookup due to %v.\n", err) + return nil + } + for _, a := range addrs { + a, ok := formatIP(a) + if !ok { + grpclog.Errorf("grpc: failed IP parsing due to %v.\n", err) + continue + } + addr := a + ":" + d.port + newAddrs = append(newAddrs, resolver.Address{Addr: addr}) + } + return newAddrs +} + +func (d *dnsResolver) lookup() ([]resolver.Address, string) { + newAddrs := d.lookupSRV() + // Support fallback to non-balancer address. + newAddrs = append(newAddrs, d.lookupHost()...) + if d.disableServiceConfig { + return newAddrs, "" + } + sc := d.lookupTXT() + return newAddrs, canaryingSC(sc) +} + +// formatIP returns ok = false if addr is not a valid textual representation of an IP address. +// If addr is an IPv4 address, return the addr and ok = true. +// If addr is an IPv6 address, return the addr enclosed in square brackets and ok = true. +func formatIP(addr string) (addrIP string, ok bool) { + ip := net.ParseIP(addr) + if ip == nil { + return "", false + } + if ip.To4() != nil { + return addr, true + } + return "[" + addr + "]", true +} + +// parseTarget takes the user input target string, returns formatted host and port info. +// If target doesn't specify a port, set the port to be the defaultPort. +// If target is in IPv6 format and host-name is enclosed in sqarue brackets, brackets +// are strippd when setting the host. +// examples: +// target: "www.google.com" returns host: "www.google.com", port: "443" +// target: "ipv4-host:80" returns host: "ipv4-host", port: "80" +// target: "[ipv6-host]" returns host: "ipv6-host", port: "443" +// target: ":80" returns host: "localhost", port: "80" +// target: ":" returns host: "localhost", port: "443" +func parseTarget(target string) (host, port string, err error) { + if target == "" { + return "", "", errMissingAddr + } + if ip := net.ParseIP(target); ip != nil { + // target is an IPv4 or IPv6(without brackets) address + return target, defaultPort, nil + } + if host, port, err = net.SplitHostPort(target); err == nil { + // target has port, i.e ipv4-host:port, [ipv6-host]:port, host-name:port + if host == "" { + // Keep consistent with net.Dial(): If the host is empty, as in ":80", the local system is assumed. + host = "localhost" + } + if port == "" { + // If the port field is empty(target ends with colon), e.g. "[::1]:", defaultPort is used. + port = defaultPort + } + return host, port, nil + } + if host, port, err = net.SplitHostPort(target + ":" + defaultPort); err == nil { + // target doesn't have port + return host, port, nil + } + return "", "", fmt.Errorf("invalid target address %v, error info: %v", target, err) +} + +type rawChoice struct { + ClientLanguage *[]string `json:"clientLanguage,omitempty"` + Percentage *int `json:"percentage,omitempty"` + ClientHostName *[]string `json:"clientHostName,omitempty"` + ServiceConfig *json.RawMessage `json:"serviceConfig,omitempty"` +} + +func containsString(a *[]string, b string) bool { + if a == nil { + return true + } + for _, c := range *a { + if c == b { + return true + } + } + return false +} + +func chosenByPercentage(a *int) bool { + if a == nil { + return true + } + return grpcrand.Intn(100)+1 <= *a +} + +func canaryingSC(js string) string { + if js == "" { + return "" + } + var rcs []rawChoice + err := json.Unmarshal([]byte(js), &rcs) + if err != nil { + grpclog.Warningf("grpc: failed to parse service config json string due to %v.\n", err) + return "" + } + cliHostname, err := os.Hostname() + if err != nil { + grpclog.Warningf("grpc: failed to get client hostname due to %v.\n", err) + return "" + } + var sc string + for _, c := range rcs { + if !containsString(c.ClientLanguage, golang) || + !chosenByPercentage(c.Percentage) || + !containsString(c.ClientHostName, cliHostname) || + c.ServiceConfig == nil { + continue + } + sc = string(*c.ServiceConfig) + break + } + return sc +} diff --git a/vendor/google.golang.org/grpc/resolver/dns/go17.go b/vendor/google.golang.org/grpc/resolver/dns/go17.go new file mode 100644 index 00000000000..b466bc8f6d4 --- /dev/null +++ b/vendor/google.golang.org/grpc/resolver/dns/go17.go @@ -0,0 +1,35 @@ +// +build go1.6, !go1.8 + +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package dns + +import ( + "net" + + "golang.org/x/net/context" +) + +var ( + lookupHost = func(ctx context.Context, host string) ([]string, error) { return net.LookupHost(host) } + lookupSRV = func(ctx context.Context, service, proto, name string) (string, []*net.SRV, error) { + return net.LookupSRV(service, proto, name) + } + lookupTXT = func(ctx context.Context, name string) ([]string, error) { return net.LookupTXT(name) } +) diff --git a/vendor/google.golang.org/grpc/resolver/dns/go18.go b/vendor/google.golang.org/grpc/resolver/dns/go18.go new file mode 100644 index 00000000000..fa34f14cad4 --- /dev/null +++ b/vendor/google.golang.org/grpc/resolver/dns/go18.go @@ -0,0 +1,29 @@ +// +build go1.8 + +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package dns + +import "net" + +var ( + lookupHost = net.DefaultResolver.LookupHost + lookupSRV = net.DefaultResolver.LookupSRV + lookupTXT = net.DefaultResolver.LookupTXT +) diff --git a/vendor/google.golang.org/grpc/resolver/passthrough/BUILD b/vendor/google.golang.org/grpc/resolver/passthrough/BUILD new file mode 100644 index 00000000000..2c0adc4cfa8 --- /dev/null +++ b/vendor/google.golang.org/grpc/resolver/passthrough/BUILD @@ -0,0 +1,24 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["passthrough.go"], + importmap = "k8s.io/kubernetes/vendor/google.golang.org/grpc/resolver/passthrough", + importpath = "google.golang.org/grpc/resolver/passthrough", + visibility = ["//visibility:public"], + deps = ["//vendor/google.golang.org/grpc/resolver:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/google.golang.org/grpc/resolver/passthrough/passthrough.go b/vendor/google.golang.org/grpc/resolver/passthrough/passthrough.go new file mode 100644 index 00000000000..b76010d74d1 --- /dev/null +++ b/vendor/google.golang.org/grpc/resolver/passthrough/passthrough.go @@ -0,0 +1,57 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package passthrough implements a pass-through resolver. It sends the target +// name without scheme back to gRPC as resolved address. +package passthrough + +import "google.golang.org/grpc/resolver" + +const scheme = "passthrough" + +type passthroughBuilder struct{} + +func (*passthroughBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOption) (resolver.Resolver, error) { + r := &passthroughResolver{ + target: target, + cc: cc, + } + r.start() + return r, nil +} + +func (*passthroughBuilder) Scheme() string { + return scheme +} + +type passthroughResolver struct { + target resolver.Target + cc resolver.ClientConn +} + +func (r *passthroughResolver) start() { + r.cc.NewAddress([]resolver.Address{{Addr: r.target.Endpoint}}) +} + +func (*passthroughResolver) ResolveNow(o resolver.ResolveNowOption) {} + +func (*passthroughResolver) Close() {} + +func init() { + resolver.Register(&passthroughBuilder{}) +} diff --git a/vendor/google.golang.org/grpc/resolver/resolver.go b/vendor/google.golang.org/grpc/resolver/resolver.go index 49307e8fe9e..506afac88ae 100644 --- a/vendor/google.golang.org/grpc/resolver/resolver.go +++ b/vendor/google.golang.org/grpc/resolver/resolver.go @@ -24,42 +24,42 @@ var ( // m is a map from scheme to resolver builder. m = make(map[string]Builder) // defaultScheme is the default scheme to use. - defaultScheme string + defaultScheme = "passthrough" ) // TODO(bar) install dns resolver in init(){}. -// Register registers the resolver builder to the resolver map. -// b.Scheme will be used as the scheme registered with this builder. +// Register registers the resolver builder to the resolver map. b.Scheme will be +// used as the scheme registered with this builder. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple Resolvers are +// registered with the same name, the one registered last will take effect. func Register(b Builder) { m[b.Scheme()] = b } // Get returns the resolver builder registered with the given scheme. -// If no builder is register with the scheme, the default scheme will -// be used. -// If the default scheme is not modified, "dns" will be the default -// scheme, and the preinstalled dns resolver will be used. -// If the default scheme is modified, and a resolver is registered with -// the scheme, that resolver will be returned. -// If the default scheme is modified, and no resolver is registered with -// the scheme, nil will be returned. +// +// If no builder is register with the scheme, nil will be returned. func Get(scheme string) Builder { if b, ok := m[scheme]; ok { return b } - if b, ok := m[defaultScheme]; ok { - return b - } return nil } // SetDefaultScheme sets the default scheme that will be used. -// The default default scheme is "dns". +// The default default scheme is "passthrough". func SetDefaultScheme(scheme string) { defaultScheme = scheme } +// GetDefaultScheme gets the default scheme that will be used. +func GetDefaultScheme() string { + return defaultScheme +} + // AddressType indicates the address type returned by name resolution. type AddressType uint8 @@ -78,7 +78,9 @@ type Address struct { // Type is the type of this address. Type AddressType // ServerName is the name of this address. - // It's the name of the grpc load balancer, which will be used for authentication. + // + // e.g. if Type is GRPCLB, ServerName should be the name of the remote load + // balancer, not the name of the backend. ServerName string // Metadata is the information associated with Addr, which may be used // to make load balancing decision. @@ -88,10 +90,17 @@ type Address struct { // BuildOption includes additional information for the builder to create // the resolver. type BuildOption struct { + // DisableServiceConfig indicates whether resolver should fetch service config data. + DisableServiceConfig bool } // ClientConn contains the callbacks for resolver to notify any updates // to the gRPC ClientConn. +// +// This interface is to be implemented by gRPC. Users should not need a +// brand new implementation of this interface. For the situations like +// testing, the new implementation should embed this interface. This allows +// gRPC to add new methods to this interface. type ClientConn interface { // NewAddress is called by resolver to notify ClientConn a new list // of resolved addresses. @@ -128,8 +137,10 @@ type ResolveNowOption struct{} // Resolver watches for the updates on the specified target. // Updates include address updates and service config updates. type Resolver interface { - // ResolveNow will be called by gRPC to try to resolve the target name again. - // It's just a hint, resolver can ignore this if it's not necessary. + // ResolveNow will be called by gRPC to try to resolve the target name + // again. It's just a hint, resolver can ignore this if it's not necessary. + // + // It could be called multiple times concurrently. ResolveNow(ResolveNowOption) // Close closes the resolver. Close() diff --git a/vendor/google.golang.org/grpc/resolver_conn_wrapper.go b/vendor/google.golang.org/grpc/resolver_conn_wrapper.go index 7d53964d094..494d6931ebb 100644 --- a/vendor/google.golang.org/grpc/resolver_conn_wrapper.go +++ b/vendor/google.golang.org/grpc/resolver_conn_wrapper.go @@ -19,6 +19,7 @@ package grpc import ( + "fmt" "strings" "google.golang.org/grpc/grpclog" @@ -36,39 +37,43 @@ type ccResolverWrapper struct { } // split2 returns the values from strings.SplitN(s, sep, 2). -// If sep is not found, it returns "", s instead. -func split2(s, sep string) (string, string) { +// If sep is not found, it returns ("", s, false) instead. +func split2(s, sep string) (string, string, bool) { spl := strings.SplitN(s, sep, 2) if len(spl) < 2 { - return "", s + return "", "", false } - return spl[0], spl[1] + return spl[0], spl[1], true } // parseTarget splits target into a struct containing scheme, authority and // endpoint. +// +// If target is not a valid scheme://authority/endpoint, it returns {Endpoint: +// target}. func parseTarget(target string) (ret resolver.Target) { - ret.Scheme, ret.Endpoint = split2(target, "://") - ret.Authority, ret.Endpoint = split2(ret.Endpoint, "/") + var ok bool + ret.Scheme, ret.Endpoint, ok = split2(target, "://") + if !ok { + return resolver.Target{Endpoint: target} + } + ret.Authority, ret.Endpoint, ok = split2(ret.Endpoint, "/") + if !ok { + return resolver.Target{Endpoint: target} + } return ret } // newCCResolverWrapper parses cc.target for scheme and gets the resolver -// builder for this scheme. It then builds the resolver and starts the -// monitoring goroutine for it. +// builder for this scheme and builds the resolver. The monitoring goroutine +// for it is not started yet and can be created by calling start(). // -// This function could return nil, nil, in tests for old behaviors. -// TODO(bar) never return nil, nil when DNS becomes the default resolver. +// If withResolverBuilder dial option is set, the specified resolver will be +// used instead. func newCCResolverWrapper(cc *ClientConn) (*ccResolverWrapper, error) { - target := parseTarget(cc.target) - grpclog.Infof("dialing to target with scheme: %q", target.Scheme) - - rb := resolver.Get(target.Scheme) + rb := cc.dopts.resolverBuilder if rb == nil { - // TODO(bar) return error when DNS becomes the default (implemented and - // registered by DNS package). - grpclog.Infof("could not get resolver for scheme: %q", target.Scheme) - return nil, nil + return nil, fmt.Errorf("could not get resolver for scheme: %q", cc.parsedTarget.Scheme) } ccr := &ccResolverWrapper{ @@ -79,15 +84,18 @@ func newCCResolverWrapper(cc *ClientConn) (*ccResolverWrapper, error) { } var err error - ccr.resolver, err = rb.Build(target, ccr, resolver.BuildOption{}) + ccr.resolver, err = rb.Build(cc.parsedTarget, ccr, resolver.BuildOption{DisableServiceConfig: cc.dopts.disableServiceConfig}) if err != nil { return nil, err } - go ccr.watcher() return ccr, nil } -// watcher processes address updates and service config updates sequencially. +func (ccr *ccResolverWrapper) start() { + go ccr.watcher() +} + +// watcher processes address updates and service config updates sequentially. // Otherwise, we need to resolve possible races between address and service // config (e.g. they specify different balancer types). func (ccr *ccResolverWrapper) watcher() { @@ -100,20 +108,31 @@ func (ccr *ccResolverWrapper) watcher() { select { case addrs := <-ccr.addrCh: - grpclog.Infof("ccResolverWrapper: sending new addresses to balancer wrapper: %v", addrs) - // TODO(bar switching) this should never be nil. Pickfirst should be default. - if ccr.cc.balancerWrapper != nil { - // TODO(bar switching) create balancer if it's nil? - ccr.cc.balancerWrapper.handleResolvedAddrs(addrs, nil) + select { + case <-ccr.done: + return + default: } + grpclog.Infof("ccResolverWrapper: sending new addresses to cc: %v", addrs) + ccr.cc.handleResolvedAddrs(addrs, nil) case sc := <-ccr.scCh: + select { + case <-ccr.done: + return + default: + } grpclog.Infof("ccResolverWrapper: got new service config: %v", sc) + ccr.cc.handleServiceConfig(sc) case <-ccr.done: return } } } +func (ccr *ccResolverWrapper) resolveNow(o resolver.ResolveNowOption) { + ccr.resolver.ResolveNow(o) +} + func (ccr *ccResolverWrapper) close() { ccr.resolver.Close() close(ccr.done) @@ -129,7 +148,7 @@ func (ccr *ccResolverWrapper) NewAddress(addrs []resolver.Address) { } // NewServiceConfig is called by the resolver implemenetion to send service -// configs to gPRC. +// configs to gRPC. func (ccr *ccResolverWrapper) NewServiceConfig(sc string) { select { case <-ccr.scCh: diff --git a/vendor/google.golang.org/grpc/rpc_util.go b/vendor/google.golang.org/grpc/rpc_util.go index 188a75fff94..033801f34f8 100644 --- a/vendor/google.golang.org/grpc/rpc_util.go +++ b/vendor/google.golang.org/grpc/rpc_util.go @@ -21,18 +21,21 @@ package grpc import ( "bytes" "compress/gzip" - stdctx "context" "encoding/binary" + "fmt" "io" "io/ioutil" "math" - "os" + "net/url" + "strings" "sync" "time" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/encoding/proto" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" "google.golang.org/grpc/stats" @@ -41,6 +44,8 @@ import ( ) // Compressor defines the interface gRPC uses to compress a message. +// +// Deprecated: use package encoding. type Compressor interface { // Do compresses p into w. Do(w io.Writer, p []byte) error @@ -53,14 +58,34 @@ type gzipCompressor struct { } // NewGZIPCompressor creates a Compressor based on GZIP. +// +// Deprecated: use package encoding/gzip. func NewGZIPCompressor() Compressor { + c, _ := NewGZIPCompressorWithLevel(gzip.DefaultCompression) + return c +} + +// NewGZIPCompressorWithLevel is like NewGZIPCompressor but specifies the gzip compression level instead +// of assuming DefaultCompression. +// +// The error returned will be nil if the level is valid. +// +// Deprecated: use package encoding/gzip. +func NewGZIPCompressorWithLevel(level int) (Compressor, error) { + if level < gzip.DefaultCompression || level > gzip.BestCompression { + return nil, fmt.Errorf("grpc: invalid compression level: %d", level) + } return &gzipCompressor{ pool: sync.Pool{ New: func() interface{} { - return gzip.NewWriter(ioutil.Discard) + w, err := gzip.NewWriterLevel(ioutil.Discard, level) + if err != nil { + panic(err) + } + return w }, }, - } + }, nil } func (c *gzipCompressor) Do(w io.Writer, p []byte) error { @@ -78,6 +103,8 @@ func (c *gzipCompressor) Type() string { } // Decompressor defines the interface gRPC uses to decompress a message. +// +// Deprecated: use package encoding. type Decompressor interface { // Do reads the data from r and uncompress them. Do(r io.Reader) ([]byte, error) @@ -90,6 +117,8 @@ type gzipDecompressor struct { } // NewGZIPDecompressor creates a Decompressor based on GZIP. +// +// Deprecated: use package encoding/gzip. func NewGZIPDecompressor() Decompressor { return &gzipDecompressor{} } @@ -124,14 +153,15 @@ func (d *gzipDecompressor) Type() string { // callInfo contains all related configuration and information about an RPC. type callInfo struct { + compressorType string failFast bool - headerMD metadata.MD - trailerMD metadata.MD - peer *peer.Peer + stream *clientStream traceInfo traceInfo // in trace.go maxReceiveMessageSize *int maxSendMessageSize *int creds credentials.PerRPCCredentials + contentSubtype string + codec baseCodec } func defaultCallInfo() *callInfo { @@ -158,87 +188,239 @@ type EmptyCallOption struct{} func (EmptyCallOption) before(*callInfo) error { return nil } func (EmptyCallOption) after(*callInfo) {} -type beforeCall func(c *callInfo) error - -func (o beforeCall) before(c *callInfo) error { return o(c) } -func (o beforeCall) after(c *callInfo) {} - -type afterCall func(c *callInfo) - -func (o afterCall) before(c *callInfo) error { return nil } -func (o afterCall) after(c *callInfo) { o(c) } - // Header returns a CallOptions that retrieves the header metadata // for a unary RPC. func Header(md *metadata.MD) CallOption { - return afterCall(func(c *callInfo) { - *md = c.headerMD - }) + return HeaderCallOption{HeaderAddr: md} +} + +// HeaderCallOption is a CallOption for collecting response header metadata. +// The metadata field will be populated *after* the RPC completes. +// This is an EXPERIMENTAL API. +type HeaderCallOption struct { + HeaderAddr *metadata.MD +} + +func (o HeaderCallOption) before(c *callInfo) error { return nil } +func (o HeaderCallOption) after(c *callInfo) { + if c.stream != nil { + *o.HeaderAddr, _ = c.stream.Header() + } } // Trailer returns a CallOptions that retrieves the trailer metadata // for a unary RPC. func Trailer(md *metadata.MD) CallOption { - return afterCall(func(c *callInfo) { - *md = c.trailerMD - }) + return TrailerCallOption{TrailerAddr: md} } -// Peer returns a CallOption that retrieves peer information for a -// unary RPC. -func Peer(peer *peer.Peer) CallOption { - return afterCall(func(c *callInfo) { - if c.peer != nil { - *peer = *c.peer +// TrailerCallOption is a CallOption for collecting response trailer metadata. +// The metadata field will be populated *after* the RPC completes. +// This is an EXPERIMENTAL API. +type TrailerCallOption struct { + TrailerAddr *metadata.MD +} + +func (o TrailerCallOption) before(c *callInfo) error { return nil } +func (o TrailerCallOption) after(c *callInfo) { + if c.stream != nil { + *o.TrailerAddr = c.stream.Trailer() + } +} + +// Peer returns a CallOption that retrieves peer information for a unary RPC. +// The peer field will be populated *after* the RPC completes. +func Peer(p *peer.Peer) CallOption { + return PeerCallOption{PeerAddr: p} +} + +// PeerCallOption is a CallOption for collecting the identity of the remote +// peer. The peer field will be populated *after* the RPC completes. +// This is an EXPERIMENTAL API. +type PeerCallOption struct { + PeerAddr *peer.Peer +} + +func (o PeerCallOption) before(c *callInfo) error { return nil } +func (o PeerCallOption) after(c *callInfo) { + if c.stream != nil { + if x, ok := peer.FromContext(c.stream.Context()); ok { + *o.PeerAddr = *x } - }) + } } // FailFast configures the action to take when an RPC is attempted on broken -// connections or unreachable servers. If failfast is true, the RPC will fail +// connections or unreachable servers. If failFast is true, the RPC will fail // immediately. Otherwise, the RPC client will block the call until a -// connection is available (or the call is canceled or times out) and will retry -// the call if it fails due to a transient error. Please refer to +// connection is available (or the call is canceled or times out) and will +// retry the call if it fails due to a transient error. gRPC will not retry if +// data was written to the wire unless the server indicates it did not process +// the data. Please refer to // https://github.com/grpc/grpc/blob/master/doc/wait-for-ready.md. -// Note: failFast is default to true. +// +// By default, RPCs are "Fail Fast". func FailFast(failFast bool) CallOption { - return beforeCall(func(c *callInfo) error { - c.failFast = failFast - return nil - }) + return FailFastCallOption{FailFast: failFast} } +// FailFastCallOption is a CallOption for indicating whether an RPC should fail +// fast or not. +// This is an EXPERIMENTAL API. +type FailFastCallOption struct { + FailFast bool +} + +func (o FailFastCallOption) before(c *callInfo) error { + c.failFast = o.FailFast + return nil +} +func (o FailFastCallOption) after(c *callInfo) {} + // MaxCallRecvMsgSize returns a CallOption which sets the maximum message size the client can receive. func MaxCallRecvMsgSize(s int) CallOption { - return beforeCall(func(o *callInfo) error { - o.maxReceiveMessageSize = &s - return nil - }) + return MaxRecvMsgSizeCallOption{MaxRecvMsgSize: s} } +// MaxRecvMsgSizeCallOption is a CallOption that indicates the maximum message +// size the client can receive. +// This is an EXPERIMENTAL API. +type MaxRecvMsgSizeCallOption struct { + MaxRecvMsgSize int +} + +func (o MaxRecvMsgSizeCallOption) before(c *callInfo) error { + c.maxReceiveMessageSize = &o.MaxRecvMsgSize + return nil +} +func (o MaxRecvMsgSizeCallOption) after(c *callInfo) {} + // MaxCallSendMsgSize returns a CallOption which sets the maximum message size the client can send. func MaxCallSendMsgSize(s int) CallOption { - return beforeCall(func(o *callInfo) error { - o.maxSendMessageSize = &s - return nil - }) + return MaxSendMsgSizeCallOption{MaxSendMsgSize: s} } +// MaxSendMsgSizeCallOption is a CallOption that indicates the maximum message +// size the client can send. +// This is an EXPERIMENTAL API. +type MaxSendMsgSizeCallOption struct { + MaxSendMsgSize int +} + +func (o MaxSendMsgSizeCallOption) before(c *callInfo) error { + c.maxSendMessageSize = &o.MaxSendMsgSize + return nil +} +func (o MaxSendMsgSizeCallOption) after(c *callInfo) {} + // PerRPCCredentials returns a CallOption that sets credentials.PerRPCCredentials // for a call. func PerRPCCredentials(creds credentials.PerRPCCredentials) CallOption { - return beforeCall(func(c *callInfo) error { - c.creds = creds - return nil - }) + return PerRPCCredsCallOption{Creds: creds} } +// PerRPCCredsCallOption is a CallOption that indicates the per-RPC +// credentials to use for the call. +// This is an EXPERIMENTAL API. +type PerRPCCredsCallOption struct { + Creds credentials.PerRPCCredentials +} + +func (o PerRPCCredsCallOption) before(c *callInfo) error { + c.creds = o.Creds + return nil +} +func (o PerRPCCredsCallOption) after(c *callInfo) {} + +// UseCompressor returns a CallOption which sets the compressor used when +// sending the request. If WithCompressor is also set, UseCompressor has +// higher priority. +// +// This API is EXPERIMENTAL. +func UseCompressor(name string) CallOption { + return CompressorCallOption{CompressorType: name} +} + +// CompressorCallOption is a CallOption that indicates the compressor to use. +// This is an EXPERIMENTAL API. +type CompressorCallOption struct { + CompressorType string +} + +func (o CompressorCallOption) before(c *callInfo) error { + c.compressorType = o.CompressorType + return nil +} +func (o CompressorCallOption) after(c *callInfo) {} + +// CallContentSubtype returns a CallOption that will set the content-subtype +// for a call. For example, if content-subtype is "json", the Content-Type over +// the wire will be "application/grpc+json". The content-subtype is converted +// to lowercase before being included in Content-Type. See Content-Type on +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +// +// If CallCustomCodec is not also used, the content-subtype will be used to +// look up the Codec to use in the registry controlled by RegisterCodec. See +// the documentation on RegisterCodec for details on registration. The lookup +// of content-subtype is case-insensitive. If no such Codec is found, the call +// will result in an error with code codes.Internal. +// +// If CallCustomCodec is also used, that Codec will be used for all request and +// response messages, with the content-subtype set to the given contentSubtype +// here for requests. +func CallContentSubtype(contentSubtype string) CallOption { + return ContentSubtypeCallOption{ContentSubtype: strings.ToLower(contentSubtype)} +} + +// ContentSubtypeCallOption is a CallOption that indicates the content-subtype +// used for marshaling messages. +// This is an EXPERIMENTAL API. +type ContentSubtypeCallOption struct { + ContentSubtype string +} + +func (o ContentSubtypeCallOption) before(c *callInfo) error { + c.contentSubtype = o.ContentSubtype + return nil +} +func (o ContentSubtypeCallOption) after(c *callInfo) {} + +// CallCustomCodec returns a CallOption that will set the given Codec to be +// used for all request and response messages for a call. The result of calling +// String() will be used as the content-subtype in a case-insensitive manner. +// +// See Content-Type on +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. Also see the documentation on RegisterCodec and +// CallContentSubtype for more details on the interaction between Codec and +// content-subtype. +// +// This function is provided for advanced users; prefer to use only +// CallContentSubtype to select a registered codec instead. +func CallCustomCodec(codec Codec) CallOption { + return CustomCodecCallOption{Codec: codec} +} + +// CustomCodecCallOption is a CallOption that indicates the codec used for +// marshaling messages. +// This is an EXPERIMENTAL API. +type CustomCodecCallOption struct { + Codec Codec +} + +func (o CustomCodecCallOption) before(c *callInfo) error { + c.codec = o.Codec + return nil +} +func (o CustomCodecCallOption) after(c *callInfo) {} + // The format of the payload: compressed or not? type payloadFormat uint8 const ( - compressionNone payloadFormat = iota // no compression - compressionMade + compressionNone payloadFormat = 0 // no compression + compressionMade payloadFormat = 1 // compressed ) // parser reads complete gRPC messages from the underlying reader. @@ -248,8 +430,8 @@ type parser struct { // error types. r io.Reader - // The header of a gRPC message. Find more detail - // at https://grpc.io/docs/guides/wire.html. + // The header of a gRPC message. Find more detail at + // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md header [5]byte } @@ -277,8 +459,11 @@ func (p *parser) recvMsg(maxReceiveMessageSize int) (pf payloadFormat, msg []byt if length == 0 { return pf, nil, nil } - if length > uint32(maxReceiveMessageSize) { - return 0, nil, Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", length, maxReceiveMessageSize) + if int64(length) > int64(maxInt) { + return 0, nil, status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max length allowed on current machine (%d vs. %d)", length, maxInt) + } + if int(length) > maxReceiveMessageSize { + return 0, nil, status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", length, maxReceiveMessageSize) } // TODO(bradfitz,zhaoq): garbage. reuse buffer after proto decoding instead // of making it for each message: @@ -292,67 +477,104 @@ func (p *parser) recvMsg(maxReceiveMessageSize int) (pf payloadFormat, msg []byt return pf, msg, nil } -// encode serializes msg and returns a buffer of message header and a buffer of msg. -// If msg is nil, it generates the message header and an empty msg buffer. -func encode(c Codec, msg interface{}, cp Compressor, cbuf *bytes.Buffer, outPayload *stats.OutPayload) ([]byte, []byte, error) { - var b []byte - const ( - payloadLen = 1 - sizeLen = 4 - ) - - if msg != nil { - var err error - b, err = c.Marshal(msg) - if err != nil { - return nil, nil, Errorf(codes.Internal, "grpc: error while marshaling: %v", err.Error()) - } - if outPayload != nil { - outPayload.Payload = msg - // TODO truncate large payload. - outPayload.Data = b - outPayload.Length = len(b) - } - if cp != nil { - if err := cp.Do(cbuf, b); err != nil { - return nil, nil, Errorf(codes.Internal, "grpc: error while compressing: %v", err.Error()) - } - b = cbuf.Bytes() - } +// encode serializes msg and returns a buffer containing the message, or an +// error if it is too large to be transmitted by grpc. If msg is nil, it +// generates an empty message. +func encode(c baseCodec, msg interface{}) ([]byte, error) { + if msg == nil { // NOTE: typed nils will not be caught by this check + return nil, nil + } + b, err := c.Marshal(msg) + if err != nil { + return nil, status.Errorf(codes.Internal, "grpc: error while marshaling: %v", err.Error()) } - if uint(len(b)) > math.MaxUint32 { - return nil, nil, Errorf(codes.ResourceExhausted, "grpc: message too large (%d bytes)", len(b)) + return nil, status.Errorf(codes.ResourceExhausted, "grpc: message too large (%d bytes)", len(b)) } - - bufHeader := make([]byte, payloadLen+sizeLen) - if cp == nil { - bufHeader[0] = byte(compressionNone) - } else { - bufHeader[0] = byte(compressionMade) - } - // Write length of b into buf - binary.BigEndian.PutUint32(bufHeader[payloadLen:], uint32(len(b))) - if outPayload != nil { - outPayload.WireLength = payloadLen + sizeLen + len(b) - } - return bufHeader, b, nil + return b, nil } -func checkRecvPayload(pf payloadFormat, recvCompress string, dc Decompressor) error { +// compress returns the input bytes compressed by compressor or cp. If both +// compressors are nil, returns nil. +// +// TODO(dfawley): eliminate cp parameter by wrapping Compressor in an encoding.Compressor. +func compress(in []byte, cp Compressor, compressor encoding.Compressor) ([]byte, error) { + if compressor == nil && cp == nil { + return nil, nil + } + wrapErr := func(err error) error { + return status.Errorf(codes.Internal, "grpc: error while compressing: %v", err.Error()) + } + cbuf := &bytes.Buffer{} + if compressor != nil { + z, _ := compressor.Compress(cbuf) + if _, err := z.Write(in); err != nil { + return nil, wrapErr(err) + } + if err := z.Close(); err != nil { + return nil, wrapErr(err) + } + } else { + if err := cp.Do(cbuf, in); err != nil { + return nil, wrapErr(err) + } + } + return cbuf.Bytes(), nil +} + +const ( + payloadLen = 1 + sizeLen = 4 + headerLen = payloadLen + sizeLen +) + +// msgHeader returns a 5-byte header for the message being transmitted and the +// payload, which is compData if non-nil or data otherwise. +func msgHeader(data, compData []byte) (hdr []byte, payload []byte) { + hdr = make([]byte, headerLen) + if compData != nil { + hdr[0] = byte(compressionMade) + data = compData + } else { + hdr[0] = byte(compressionNone) + } + + // Write length of payload into buf + binary.BigEndian.PutUint32(hdr[payloadLen:], uint32(len(data))) + return hdr, data +} + +func outPayload(client bool, msg interface{}, data, payload []byte, t time.Time) *stats.OutPayload { + return &stats.OutPayload{ + Client: client, + Payload: msg, + Data: data, + Length: len(data), + WireLength: len(payload) + headerLen, + SentTime: t, + } +} + +func checkRecvPayload(pf payloadFormat, recvCompress string, haveCompressor bool) *status.Status { switch pf { case compressionNone: case compressionMade: - if dc == nil || recvCompress != dc.Type() { - return Errorf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", recvCompress) + if recvCompress == "" || recvCompress == encoding.Identity { + return status.New(codes.Internal, "grpc: compressed flag set with identity or empty encoding") + } + if !haveCompressor { + return status.Newf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", recvCompress) } default: - return Errorf(codes.Internal, "grpc: received unexpected payload format %d", pf) + return status.Newf(codes.Internal, "grpc: received unexpected payload format %d", pf) } return nil } -func recv(p *parser, c Codec, s *transport.Stream, dc Decompressor, m interface{}, maxReceiveMessageSize int, inPayload *stats.InPayload) error { +// For the two compressor parameters, both should not be set, but if they are, +// dc takes precedence over compressor. +// TODO(dfawley): wrap the old compressor/decompressor using the new API? +func recv(p *parser, c baseCodec, s *transport.Stream, dc Decompressor, m interface{}, maxReceiveMessageSize int, inPayload *stats.InPayload, compressor encoding.Compressor) error { pf, d, err := p.recvMsg(maxReceiveMessageSize) if err != nil { return err @@ -360,22 +582,37 @@ func recv(p *parser, c Codec, s *transport.Stream, dc Decompressor, m interface{ if inPayload != nil { inPayload.WireLength = len(d) } - if err := checkRecvPayload(pf, s.RecvCompress(), dc); err != nil { - return err + + if st := checkRecvPayload(pf, s.RecvCompress(), compressor != nil || dc != nil); st != nil { + return st.Err() } + if pf == compressionMade { - d, err = dc.Do(bytes.NewReader(d)) - if err != nil { - return Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + // To match legacy behavior, if the decompressor is set by WithDecompressor or RPCDecompressor, + // use this decompressor as the default. + if dc != nil { + d, err = dc.Do(bytes.NewReader(d)) + if err != nil { + return status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + } + } else { + dcReader, err := compressor.Decompress(bytes.NewReader(d)) + if err != nil { + return status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + } + d, err = ioutil.ReadAll(dcReader) + if err != nil { + return status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + } } } if len(d) > maxReceiveMessageSize { // TODO: Revisit the error code. Currently keep it consistent with java // implementation. - return Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", len(d), maxReceiveMessageSize) + return status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", len(d), maxReceiveMessageSize) } if err := c.Unmarshal(d, m); err != nil { - return Errorf(codes.Internal, "grpc: failed to unmarshal the received message %v", err) + return status.Errorf(codes.Internal, "grpc: failed to unmarshal the received message %v", err) } if inPayload != nil { inPayload.RecvTime = time.Now() @@ -388,9 +625,7 @@ func recv(p *parser, c Codec, s *transport.Stream, dc Decompressor, m interface{ } type rpcInfo struct { - failfast bool - bytesSent bool - bytesReceived bool + failfast bool } type rpcInfoContextKey struct{} @@ -404,69 +639,10 @@ func rpcInfoFromContext(ctx context.Context) (s *rpcInfo, ok bool) { return } -func updateRPCInfoInContext(ctx context.Context, s rpcInfo) { - if ss, ok := rpcInfoFromContext(ctx); ok { - ss.bytesReceived = s.bytesReceived - ss.bytesSent = s.bytesSent - } - return -} - -// toRPCErr converts an error into an error from the status package. -func toRPCErr(err error) error { - if _, ok := status.FromError(err); ok { - return err - } - switch e := err.(type) { - case transport.StreamError: - return status.Error(e.Code, e.Desc) - case transport.ConnectionError: - return status.Error(codes.Unavailable, e.Desc) - default: - switch err { - case context.DeadlineExceeded, stdctx.DeadlineExceeded: - return status.Error(codes.DeadlineExceeded, err.Error()) - case context.Canceled, stdctx.Canceled: - return status.Error(codes.Canceled, err.Error()) - case ErrClientConnClosing: - return status.Error(codes.FailedPrecondition, err.Error()) - } - } - return status.Error(codes.Unknown, err.Error()) -} - -// convertCode converts a standard Go error into its canonical code. Note that -// this is only used to translate the error returned by the server applications. -func convertCode(err error) codes.Code { - switch err { - case nil: - return codes.OK - case io.EOF: - return codes.OutOfRange - case io.ErrClosedPipe, io.ErrNoProgress, io.ErrShortBuffer, io.ErrShortWrite, io.ErrUnexpectedEOF: - return codes.FailedPrecondition - case os.ErrInvalid: - return codes.InvalidArgument - case context.Canceled, stdctx.Canceled: - return codes.Canceled - case context.DeadlineExceeded, stdctx.DeadlineExceeded: - return codes.DeadlineExceeded - } - switch { - case os.IsExist(err): - return codes.AlreadyExists - case os.IsNotExist(err): - return codes.NotFound - case os.IsPermission(err): - return codes.PermissionDenied - } - return codes.Unknown -} - // Code returns the error code for err if it was produced by the rpc system. // Otherwise, it returns codes.Unknown. // -// Deprecated; use status.FromError and Code method instead. +// Deprecated: use status.FromError and Code method instead. func Code(err error) codes.Code { if s, ok := status.FromError(err); ok { return s.Code() @@ -477,7 +653,7 @@ func Code(err error) codes.Code { // ErrorDesc returns the error description of err if it was produced by the rpc system. // Otherwise, it returns err.Error() or empty string when err is nil. // -// Deprecated; use status.FromError and Message method instead. +// Deprecated: use status.FromError and Message method instead. func ErrorDesc(err error) string { if s, ok := status.FromError(err); ok { return s.Message() @@ -488,85 +664,78 @@ func ErrorDesc(err error) string { // Errorf returns an error containing an error code and a description; // Errorf returns nil if c is OK. // -// Deprecated; use status.Errorf instead. +// Deprecated: use status.Errorf instead. func Errorf(c codes.Code, format string, a ...interface{}) error { return status.Errorf(c, format, a...) } -// MethodConfig defines the configuration recommended by the service providers for a -// particular method. -// This is EXPERIMENTAL and subject to change. -type MethodConfig struct { - // WaitForReady indicates whether RPCs sent to this method should wait until - // the connection is ready by default (!failfast). The value specified via the - // gRPC client API will override the value set here. - WaitForReady *bool - // Timeout is the default timeout for RPCs sent to this method. The actual - // deadline used will be the minimum of the value specified here and the value - // set by the application via the gRPC client API. If either one is not set, - // then the other will be used. If neither is set, then the RPC has no deadline. - Timeout *time.Duration - // MaxReqSize is the maximum allowed payload size for an individual request in a - // stream (client->server) in bytes. The size which is measured is the serialized - // payload after per-message compression (but before stream compression) in bytes. - // The actual value used is the minimum of the value specified here and the value set - // by the application via the gRPC client API. If either one is not set, then the other - // will be used. If neither is set, then the built-in default is used. - MaxReqSize *int - // MaxRespSize is the maximum allowed payload size for an individual response in a - // stream (server->client) in bytes. - MaxRespSize *int +// setCallInfoCodec should only be called after CallOptions have been applied. +func setCallInfoCodec(c *callInfo) error { + if c.codec != nil { + // codec was already set by a CallOption; use it. + return nil + } + + if c.contentSubtype == "" { + // No codec specified in CallOptions; use proto by default. + c.codec = encoding.GetCodec(proto.Name) + return nil + } + + // c.contentSubtype is already lowercased in CallContentSubtype + c.codec = encoding.GetCodec(c.contentSubtype) + if c.codec == nil { + return status.Errorf(codes.Internal, "no codec registered for content-subtype %s", c.contentSubtype) + } + return nil } -// ServiceConfig is provided by the service provider and contains parameters for how -// clients that connect to the service should behave. -// This is EXPERIMENTAL and subject to change. -type ServiceConfig struct { - // LB is the load balancer the service providers recommends. The balancer specified - // via grpc.WithBalancer will override this. - LB Balancer - // Methods contains a map for the methods in this service. - // If there is an exact match for a method (i.e. /service/method) in the map, use the corresponding MethodConfig. - // If there's no exact match, look for the default config for the service (/service/) and use the corresponding MethodConfig if it exists. - // Otherwise, the method has no MethodConfig to use. - Methods map[string]MethodConfig +// parseDialTarget returns the network and address to pass to dialer +func parseDialTarget(target string) (net string, addr string) { + net = "tcp" + + m1 := strings.Index(target, ":") + m2 := strings.Index(target, ":/") + + // handle unix:addr which will fail with url.Parse + if m1 >= 0 && m2 < 0 { + if n := target[0:m1]; n == "unix" { + net = n + addr = target[m1+1:] + return net, addr + } + } + if m2 >= 0 { + t, err := url.Parse(target) + if err != nil { + return net, target + } + scheme := t.Scheme + addr = t.Path + if scheme == "unix" { + net = scheme + if addr == "" { + addr = t.Host + } + return net, addr + } + } + + return net, target } -func min(a, b *int) *int { - if *a < *b { - return a - } - return b -} - -func getMaxSize(mcMax, doptMax *int, defaultVal int) *int { - if mcMax == nil && doptMax == nil { - return &defaultVal - } - if mcMax != nil && doptMax != nil { - return min(mcMax, doptMax) - } - if mcMax != nil { - return mcMax - } - return doptMax -} - -// SupportPackageIsVersion3 is referenced from generated protocol buffer files. -// The latest support package version is 4. -// SupportPackageIsVersion3 is kept for compatibility. It will be removed in the -// next support package version update. -const SupportPackageIsVersion3 = true - -// SupportPackageIsVersion4 is referenced from generated protocol buffer files -// to assert that that code is compatible with this version of the grpc package. +// The SupportPackageIsVersion variables are referenced from generated protocol +// buffer files to ensure compatibility with the gRPC version used. The latest +// support package version is 5. // -// This constant may be renamed in the future if a change in the generated code -// requires a synchronised update of grpc-go and protoc-gen-go. This constant -// should not be referenced from any other code. -const SupportPackageIsVersion4 = true - -// Version is the current grpc version. -const Version = "1.7.5" +// Older versions are kept for compatibility. They may be removed if +// compatibility cannot be maintained. +// +// These constants should not be referenced from any other code. +const ( + SupportPackageIsVersion3 = true + SupportPackageIsVersion4 = true + SupportPackageIsVersion5 = true +) const grpcUA = "grpc-go/" + Version diff --git a/vendor/google.golang.org/grpc/server.go b/vendor/google.golang.org/grpc/server.go index 787665dfeb3..014c72b3f28 100644 --- a/vendor/google.golang.org/grpc/server.go +++ b/vendor/google.golang.org/grpc/server.go @@ -32,13 +32,19 @@ import ( "sync" "time" + "io/ioutil" + "golang.org/x/net/context" "golang.org/x/net/http2" "golang.org/x/net/trace" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/encoding/proto" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/stats" @@ -89,18 +95,28 @@ type Server struct { conns map[io.Closer]bool serve bool drain bool - ctx context.Context - cancel context.CancelFunc - // A CondVar to let GracefulStop() blocks until all the pending RPCs are finished - // and all the transport goes away. - cv *sync.Cond + cv *sync.Cond // signaled when connections close for GracefulStop m map[string]*service // service name -> service info events trace.EventLog + + quit chan struct{} + done chan struct{} + quitOnce sync.Once + doneOnce sync.Once + channelzRemoveOnce sync.Once + serveWG sync.WaitGroup // counts active Serve goroutines for GracefulStop + + channelzID int64 // channelz unique identification number + czmu sync.RWMutex + callsStarted int64 + callsFailed int64 + callsSucceeded int64 + lastCallStartedTime time.Time } type options struct { creds credentials.TransportCredentials - codec Codec + codec baseCodec cp Compressor dc Decompressor unaryInt UnaryServerInterceptor @@ -177,20 +193,32 @@ func KeepaliveEnforcementPolicy(kep keepalive.EnforcementPolicy) ServerOption { } // CustomCodec returns a ServerOption that sets a codec for message marshaling and unmarshaling. +// +// This will override any lookups by content-subtype for Codecs registered with RegisterCodec. func CustomCodec(codec Codec) ServerOption { return func(o *options) { o.codec = codec } } -// RPCCompressor returns a ServerOption that sets a compressor for outbound messages. +// RPCCompressor returns a ServerOption that sets a compressor for outbound +// messages. For backward compatibility, all outbound messages will be sent +// using this compressor, regardless of incoming message compression. By +// default, server messages will be sent using the same compressor with which +// request messages were sent. +// +// Deprecated: use encoding.RegisterCompressor instead. func RPCCompressor(cp Compressor) ServerOption { return func(o *options) { o.cp = cp } } -// RPCDecompressor returns a ServerOption that sets a decompressor for inbound messages. +// RPCDecompressor returns a ServerOption that sets a decompressor for inbound +// messages. It has higher priority than decompressors registered via +// encoding.RegisterCompressor. +// +// Deprecated: use encoding.RegisterCompressor instead. func RPCDecompressor(dc Decompressor) ServerOption { return func(o *options) { o.dc = dc @@ -198,7 +226,9 @@ func RPCDecompressor(dc Decompressor) ServerOption { } // MaxMsgSize returns a ServerOption to set the max message size in bytes the server can receive. -// If this is not set, gRPC uses the default limit. Deprecated: use MaxRecvMsgSize instead. +// If this is not set, gRPC uses the default limit. +// +// Deprecated: use MaxRecvMsgSize instead. func MaxMsgSize(m int) ServerOption { return MaxRecvMsgSize(m) } @@ -297,6 +327,8 @@ func UnknownServiceHandler(streamHandler StreamHandler) ServerOption { // connection establishment (up to and including HTTP/2 handshaking) for all // new connections. If this is not set, the default is 120 seconds. A zero or // negative value will result in an immediate timeout. +// +// This API is EXPERIMENTAL. func ConnectionTimeout(d time.Duration) ServerOption { return func(o *options) { o.connectionTimeout = d @@ -310,22 +342,23 @@ func NewServer(opt ...ServerOption) *Server { for _, o := range opt { o(&opts) } - if opts.codec == nil { - // Set the default codec. - opts.codec = protoCodec{} - } s := &Server{ lis: make(map[net.Listener]bool), opts: opts, conns: make(map[io.Closer]bool), m: make(map[string]*service), + quit: make(chan struct{}), + done: make(chan struct{}), } s.cv = sync.NewCond(&s.mu) - s.ctx, s.cancel = context.WithCancel(context.Background()) if EnableTracing { _, file, line, _ := runtime.Caller(1) s.events = trace.NewEventLog("grpc.Server", fmt.Sprintf("%s:%d", file, line)) } + + if channelz.IsOn() { + s.channelzID = channelz.RegisterServer(s, "") + } return s } @@ -430,11 +463,9 @@ func (s *Server) GetServiceInfo() map[string]ServiceInfo { return ret } -var ( - // ErrServerStopped indicates that the operation is now illegal because of - // the server being stopped. - ErrServerStopped = errors.New("grpc: the server has been stopped") -) +// ErrServerStopped indicates that the operation is now illegal because of +// the server being stopped. +var ErrServerStopped = errors.New("grpc: the server has been stopped") func (s *Server) useTransportAuthenticator(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { if s.opts.creds == nil { @@ -443,28 +474,66 @@ func (s *Server) useTransportAuthenticator(rawConn net.Conn) (net.Conn, credenti return s.opts.creds.ServerHandshake(rawConn) } +type listenSocket struct { + net.Listener + channelzID int64 +} + +func (l *listenSocket) ChannelzMetric() *channelz.SocketInternalMetric { + return &channelz.SocketInternalMetric{ + LocalAddr: l.Listener.Addr(), + } +} + +func (l *listenSocket) Close() error { + err := l.Listener.Close() + if channelz.IsOn() { + channelz.RemoveEntry(l.channelzID) + } + return err +} + // Serve accepts incoming connections on the listener lis, creating a new // ServerTransport and service goroutine for each. The service goroutines // read gRPC requests and then call the registered handlers to reply to them. // Serve returns when lis.Accept fails with fatal errors. lis will be closed when // this method returns. -// Serve always returns non-nil error. +// Serve will return a non-nil error unless Stop or GracefulStop is called. func (s *Server) Serve(lis net.Listener) error { s.mu.Lock() s.printf("serving") s.serve = true if s.lis == nil { + // Serve called after Stop or GracefulStop. s.mu.Unlock() lis.Close() return ErrServerStopped } - s.lis[lis] = true + + s.serveWG.Add(1) + defer func() { + s.serveWG.Done() + select { + // Stop or GracefulStop called; block until done and return nil. + case <-s.quit: + <-s.done + default: + } + }() + + ls := &listenSocket{Listener: lis} + s.lis[ls] = true + + if channelz.IsOn() { + ls.channelzID = channelz.RegisterListenSocket(ls, s.channelzID, "") + } s.mu.Unlock() + defer func() { s.mu.Lock() - if s.lis != nil && s.lis[lis] { - lis.Close() - delete(s.lis, lis) + if s.lis != nil && s.lis[ls] { + ls.Close() + delete(s.lis, ls) } s.mu.Unlock() }() @@ -491,25 +560,39 @@ func (s *Server) Serve(lis net.Listener) error { timer := time.NewTimer(tempDelay) select { case <-timer.C: - case <-s.ctx.Done(): + case <-s.quit: + timer.Stop() + return nil } - timer.Stop() continue } s.mu.Lock() s.printf("done serving; Accept = %v", err) s.mu.Unlock() + + select { + case <-s.quit: + return nil + default: + } return err } tempDelay = 0 - // Start a new goroutine to deal with rawConn - // so we don't stall this Accept loop goroutine. - go s.handleRawConn(rawConn) + // Start a new goroutine to deal with rawConn so we don't stall this Accept + // loop goroutine. + // + // Make sure we account for the goroutine so GracefulStop doesn't nil out + // s.conns before this conn can be added. + s.serveWG.Add(1) + go func() { + s.handleRawConn(rawConn) + s.serveWG.Done() + }() } } -// handleRawConn is run in its own goroutine and handles a just-accepted -// connection that has not had any I/O performed on it yet. +// handleRawConn forks a goroutine to handle a just-accepted connection that +// has not had any I/O performed on it yet. func (s *Server) handleRawConn(rawConn net.Conn) { rawConn.SetDeadline(time.Now().Add(s.opts.connectionTimeout)) conn, authInfo, err := s.useTransportAuthenticator(rawConn) @@ -534,17 +617,28 @@ func (s *Server) handleRawConn(rawConn net.Conn) { } s.mu.Unlock() + var serve func() + c := conn.(io.Closer) if s.opts.useHandlerImpl { - rawConn.SetDeadline(time.Time{}) - s.serveUsingHandler(conn) + serve = func() { s.serveUsingHandler(conn) } } else { + // Finish handshaking (HTTP2) st := s.newHTTP2Transport(conn, authInfo) if st == nil { return } - rawConn.SetDeadline(time.Time{}) - s.serveStreams(st) + c = st + serve = func() { s.serveStreams(st) } } + + rawConn.SetDeadline(time.Time{}) + if !s.addConn(c) { + return + } + go func() { + serve() + s.removeConn(c) + }() } // newHTTP2Transport sets up a http/2 transport (using the @@ -561,6 +655,7 @@ func (s *Server) newHTTP2Transport(c net.Conn, authInfo credentials.AuthInfo) tr InitialConnWindowSize: s.opts.initialConnWindowSize, WriteBufferSize: s.opts.writeBufferSize, ReadBufferSize: s.opts.readBufferSize, + ChannelzParentID: s.channelzID, } st, err := transport.NewServerTransport("http2", c, config) if err != nil { @@ -571,15 +666,11 @@ func (s *Server) newHTTP2Transport(c net.Conn, authInfo credentials.AuthInfo) tr grpclog.Warningln("grpc: Server.Serve failed to create ServerTransport: ", err) return nil } - if !s.addConn(st) { - st.Close() - return nil - } + return st } func (s *Server) serveStreams(st transport.ServerTransport) { - defer s.removeConn(st) defer st.Close() var wg sync.WaitGroup st.HandleStreams(func(stream *transport.Stream) { @@ -613,11 +704,6 @@ var _ http.Handler = (*Server)(nil) // // conn is the *tls.Conn that's already been authenticated. func (s *Server) serveUsingHandler(conn net.Conn) { - if !s.addConn(conn) { - conn.Close() - return - } - defer s.removeConn(conn) h2s := &http2.Server{ MaxConcurrentStreams: s.opts.maxConcurrentStreams, } @@ -651,13 +737,12 @@ func (s *Server) serveUsingHandler(conn net.Conn) { // available through grpc-go's HTTP/2 server, and it is currently EXPERIMENTAL // and subject to change. func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - st, err := transport.NewServerHandlerTransport(w, r) + st, err := transport.NewServerHandlerTransport(w, r, s.opts.statsHandler) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } if !s.addConn(st) { - st.Close() return } defer s.removeConn(st) @@ -687,9 +772,15 @@ func (s *Server) traceInfo(st transport.ServerTransport, stream *transport.Strea func (s *Server) addConn(c io.Closer) bool { s.mu.Lock() defer s.mu.Unlock() - if s.conns == nil || s.drain { + if s.conns == nil { + c.Close() return false } + if s.drain { + // Transport added after we drained our existing conns: drain it + // immediately. + c.(transport.ServerTransport).Drain() + } s.conns[c] = true return true } @@ -703,43 +794,83 @@ func (s *Server) removeConn(c io.Closer) { } } -func (s *Server) sendResponse(t transport.ServerTransport, stream *transport.Stream, msg interface{}, cp Compressor, opts *transport.Options) error { - var ( - cbuf *bytes.Buffer - outPayload *stats.OutPayload - ) - if cp != nil { - cbuf = new(bytes.Buffer) +// ChannelzMetric returns ServerInternalMetric of current server. +// This is an EXPERIMENTAL API. +func (s *Server) ChannelzMetric() *channelz.ServerInternalMetric { + s.czmu.RLock() + defer s.czmu.RUnlock() + return &channelz.ServerInternalMetric{ + CallsStarted: s.callsStarted, + CallsSucceeded: s.callsSucceeded, + CallsFailed: s.callsFailed, + LastCallStartedTimestamp: s.lastCallStartedTime, } - if s.opts.statsHandler != nil { - outPayload = &stats.OutPayload{} - } - hdr, data, err := encode(s.opts.codec, msg, cp, cbuf, outPayload) +} + +func (s *Server) incrCallsStarted() { + s.czmu.Lock() + s.callsStarted++ + s.lastCallStartedTime = time.Now() + s.czmu.Unlock() +} + +func (s *Server) incrCallsSucceeded() { + s.czmu.Lock() + s.callsSucceeded++ + s.czmu.Unlock() +} + +func (s *Server) incrCallsFailed() { + s.czmu.Lock() + s.callsFailed++ + s.czmu.Unlock() +} + +func (s *Server) sendResponse(t transport.ServerTransport, stream *transport.Stream, msg interface{}, cp Compressor, opts *transport.Options, comp encoding.Compressor) error { + data, err := encode(s.getCodec(stream.ContentSubtype()), msg) if err != nil { grpclog.Errorln("grpc: server failed to encode response: ", err) return err } - if len(data) > s.opts.maxSendMessageSize { - return status.Errorf(codes.ResourceExhausted, "grpc: trying to send message larger than max (%d vs. %d)", len(data), s.opts.maxSendMessageSize) + compData, err := compress(data, cp, comp) + if err != nil { + grpclog.Errorln("grpc: server failed to compress response: ", err) + return err } - err = t.Write(stream, hdr, data, opts) - if err == nil && outPayload != nil { - outPayload.SentTime = time.Now() - s.opts.statsHandler.HandleRPC(stream.Context(), outPayload) + hdr, payload := msgHeader(data, compData) + // TODO(dfawley): should we be checking len(data) instead? + if len(payload) > s.opts.maxSendMessageSize { + return status.Errorf(codes.ResourceExhausted, "grpc: trying to send message larger than max (%d vs. %d)", len(payload), s.opts.maxSendMessageSize) + } + err = t.Write(stream, hdr, payload, opts) + if err == nil && s.opts.statsHandler != nil { + s.opts.statsHandler.HandleRPC(stream.Context(), outPayload(false, msg, data, payload, time.Now())) } return err } func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.Stream, srv *service, md *MethodDesc, trInfo *traceInfo) (err error) { + if channelz.IsOn() { + s.incrCallsStarted() + defer func() { + if err != nil && err != io.EOF { + s.incrCallsFailed() + } else { + s.incrCallsSucceeded() + } + }() + } sh := s.opts.statsHandler if sh != nil { + beginTime := time.Now() begin := &stats.Begin{ - BeginTime: time.Now(), + BeginTime: beginTime, } sh.HandleRPC(stream.Context(), begin) defer func() { end := &stats.End{ - EndTime: time.Now(), + BeginTime: beginTime, + EndTime: time.Now(), } if err != nil && err != io.EOF { end.Error = toRPCErr(err) @@ -758,10 +889,43 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. } }() } - if s.opts.cp != nil { - // NOTE: this needs to be ahead of all handling, https://github.com/grpc/grpc-go/issues/686. - stream.SetSendCompress(s.opts.cp.Type()) + + // comp and cp are used for compression. decomp and dc are used for + // decompression. If comp and decomp are both set, they are the same; + // however they are kept separate to ensure that at most one of the + // compressor/decompressor variable pairs are set for use later. + var comp, decomp encoding.Compressor + var cp Compressor + var dc Decompressor + + // If dc is set and matches the stream's compression, use it. Otherwise, try + // to find a matching registered compressor for decomp. + if rc := stream.RecvCompress(); s.opts.dc != nil && s.opts.dc.Type() == rc { + dc = s.opts.dc + } else if rc != "" && rc != encoding.Identity { + decomp = encoding.GetCompressor(rc) + if decomp == nil { + st := status.Newf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", rc) + t.WriteStatus(stream, st) + return st.Err() + } } + + // If cp is set, use it. Otherwise, attempt to compress the response using + // the incoming message compression method. + // + // NOTE: this needs to be ahead of all handling, https://github.com/grpc/grpc-go/issues/686. + if s.opts.cp != nil { + cp = s.opts.cp + stream.SetSendCompress(cp.Type()) + } else if rc := stream.RecvCompress(); rc != "" && rc != encoding.Identity { + // Legacy compressor not specified; attempt to respond with same encoding. + comp = encoding.GetCompressor(rc) + if comp != nil { + stream.SetSendCompress(rc) + } + } + p := &parser{r: stream} pf, req, err := p.recvMsg(s.opts.maxReceiveMessageSize) if err == io.EOF { @@ -769,7 +933,7 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. return err } if err == io.ErrUnexpectedEOF { - err = Errorf(codes.Internal, io.ErrUnexpectedEOF.Error()) + err = status.Errorf(codes.Internal, io.ErrUnexpectedEOF.Error()) } if err != nil { if st, ok := status.FromError(err); ok { @@ -790,19 +954,14 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. } return err } - - if err := checkRecvPayload(pf, stream.RecvCompress(), s.opts.dc); err != nil { - if st, ok := status.FromError(err); ok { - if e := t.WriteStatus(stream, st); e != nil { - grpclog.Warningf("grpc: Server.processUnaryRPC failed to write status %v", e) - } - return err - } - if e := t.WriteStatus(stream, status.New(codes.Internal, err.Error())); e != nil { + if channelz.IsOn() { + t.IncrMsgRecv() + } + if st := checkRecvPayload(pf, stream.RecvCompress(), dc != nil || decomp != nil); st != nil { + if e := t.WriteStatus(stream, st); e != nil { grpclog.Warningf("grpc: Server.processUnaryRPC failed to write status %v", e) } - - // TODO checkRecvPayload always return RPC error. Add a return here if necessary. + return st.Err() } var inPayload *stats.InPayload if sh != nil { @@ -816,9 +975,17 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. } if pf == compressionMade { var err error - req, err = s.opts.dc.Do(bytes.NewReader(req)) - if err != nil { - return Errorf(codes.Internal, err.Error()) + if dc != nil { + req, err = dc.Do(bytes.NewReader(req)) + if err != nil { + return status.Errorf(codes.Internal, err.Error()) + } + } else { + tmp, _ := decomp.Decompress(bytes.NewReader(req)) + req, err = ioutil.ReadAll(tmp) + if err != nil { + return status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + } } } if len(req) > s.opts.maxReceiveMessageSize { @@ -826,7 +993,7 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. // java implementation. return status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", len(req), s.opts.maxReceiveMessageSize) } - if err := s.opts.codec.Unmarshal(req, v); err != nil { + if err := s.getCodec(stream.ContentSubtype()).Unmarshal(req, v); err != nil { return status.Errorf(codes.Internal, "grpc: error unmarshalling request: %v", err) } if inPayload != nil { @@ -840,12 +1007,13 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. } return nil } - reply, appErr := md.Handler(srv.server, stream.Context(), df, s.opts.unaryInt) + ctx := NewContextWithServerTransportStream(stream.Context(), stream) + reply, appErr := md.Handler(srv.server, ctx, df, s.opts.unaryInt) if appErr != nil { appStatus, ok := status.FromError(appErr) if !ok { // Convert appErr if it is not a grpc status error. - appErr = status.Error(convertCode(appErr), appErr.Error()) + appErr = status.Error(codes.Unknown, appErr.Error()) appStatus, _ = status.FromError(appErr) } if trInfo != nil { @@ -864,7 +1032,8 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. Last: true, Delay: false, } - if err := s.sendResponse(t, stream, reply, s.opts.cp, opts); err != nil { + + if err := s.sendResponse(t, stream, reply, cp, opts, comp); err != nil { if err == io.EOF { // The entire stream is done (for unary RPC only). return err @@ -887,6 +1056,9 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. } return err } + if channelz.IsOn() { + t.IncrMsgSent() + } if trInfo != nil { trInfo.tr.LazyLog(&payload{sent: true, msg: reply}, true) } @@ -897,15 +1069,27 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. } func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transport.Stream, srv *service, sd *StreamDesc, trInfo *traceInfo) (err error) { + if channelz.IsOn() { + s.incrCallsStarted() + defer func() { + if err != nil && err != io.EOF { + s.incrCallsFailed() + } else { + s.incrCallsSucceeded() + } + }() + } sh := s.opts.statsHandler if sh != nil { + beginTime := time.Now() begin := &stats.Begin{ - BeginTime: time.Now(), + BeginTime: beginTime, } sh.HandleRPC(stream.Context(), begin) defer func() { end := &stats.End{ - EndTime: time.Now(), + BeginTime: beginTime, + EndTime: time.Now(), } if err != nil && err != io.EOF { end.Error = toRPCErr(err) @@ -913,21 +1097,47 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp sh.HandleRPC(stream.Context(), end) }() } - if s.opts.cp != nil { - stream.SetSendCompress(s.opts.cp.Type()) - } + ctx := NewContextWithServerTransportStream(stream.Context(), stream) ss := &serverStream{ + ctx: ctx, t: t, s: stream, p: &parser{r: stream}, - codec: s.opts.codec, - cp: s.opts.cp, - dc: s.opts.dc, + codec: s.getCodec(stream.ContentSubtype()), maxReceiveMessageSize: s.opts.maxReceiveMessageSize, maxSendMessageSize: s.opts.maxSendMessageSize, trInfo: trInfo, statsHandler: sh, } + + // If dc is set and matches the stream's compression, use it. Otherwise, try + // to find a matching registered compressor for decomp. + if rc := stream.RecvCompress(); s.opts.dc != nil && s.opts.dc.Type() == rc { + ss.dc = s.opts.dc + } else if rc != "" && rc != encoding.Identity { + ss.decomp = encoding.GetCompressor(rc) + if ss.decomp == nil { + st := status.Newf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", rc) + t.WriteStatus(ss.s, st) + return st.Err() + } + } + + // If cp is set, use it. Otherwise, attempt to compress the response using + // the incoming message compression method. + // + // NOTE: this needs to be ahead of all handling, https://github.com/grpc/grpc-go/issues/686. + if s.opts.cp != nil { + ss.cp = s.opts.cp + stream.SetSendCompress(s.opts.cp.Type()) + } else if rc := stream.RecvCompress(); rc != "" && rc != encoding.Identity { + // Legacy compressor not specified; attempt to respond with same encoding. + ss.comp = encoding.GetCompressor(rc) + if ss.comp != nil { + stream.SetSendCompress(rc) + } + } + if trInfo != nil { trInfo.tr.LazyLog(&trInfo.firstLine, false) defer func() { @@ -963,7 +1173,7 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp case transport.StreamError: appStatus = status.New(err.Code, err.Desc) default: - appStatus = status.New(convertCode(appErr), appErr.Error()) + appStatus = status.New(codes.Unknown, appErr.Error()) } appErr = appStatus.Err() } @@ -983,7 +1193,6 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp ss.mu.Unlock() } return t.WriteStatus(ss.s, status.New(codes.OK, "")) - } func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Stream, trInfo *traceInfo) { @@ -1065,12 +1274,65 @@ func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Str } } +// The key to save ServerTransportStream in the context. +type streamKey struct{} + +// NewContextWithServerTransportStream creates a new context from ctx and +// attaches stream to it. +// +// This API is EXPERIMENTAL. +func NewContextWithServerTransportStream(ctx context.Context, stream ServerTransportStream) context.Context { + return context.WithValue(ctx, streamKey{}, stream) +} + +// ServerTransportStream is a minimal interface that a transport stream must +// implement. This can be used to mock an actual transport stream for tests of +// handler code that use, for example, grpc.SetHeader (which requires some +// stream to be in context). +// +// See also NewContextWithServerTransportStream. +// +// This API is EXPERIMENTAL. +type ServerTransportStream interface { + Method() string + SetHeader(md metadata.MD) error + SendHeader(md metadata.MD) error + SetTrailer(md metadata.MD) error +} + +// ServerTransportStreamFromContext returns the ServerTransportStream saved in +// ctx. Returns nil if the given context has no stream associated with it +// (which implies it is not an RPC invocation context). +// +// This API is EXPERIMENTAL. +func ServerTransportStreamFromContext(ctx context.Context) ServerTransportStream { + s, _ := ctx.Value(streamKey{}).(ServerTransportStream) + return s +} + // Stop stops the gRPC server. It immediately closes all open // connections and listeners. // It cancels all active RPCs on the server side and the corresponding // pending RPCs on the client side will get notified by connection // errors. func (s *Server) Stop() { + s.quitOnce.Do(func() { + close(s.quit) + }) + + defer func() { + s.serveWG.Wait() + s.doneOnce.Do(func() { + close(s.done) + }) + }() + + s.channelzRemoveOnce.Do(func() { + if channelz.IsOn() { + channelz.RemoveEntry(s.channelzID) + } + }) + s.mu.Lock() listeners := s.lis s.lis = nil @@ -1088,7 +1350,6 @@ func (s *Server) Stop() { } s.mu.Lock() - s.cancel() if s.events != nil { s.events.Finish() s.events = nil @@ -1100,22 +1361,44 @@ func (s *Server) Stop() { // accepting new connections and RPCs and blocks until all the pending RPCs are // finished. func (s *Server) GracefulStop() { + s.quitOnce.Do(func() { + close(s.quit) + }) + + defer func() { + s.doneOnce.Do(func() { + close(s.done) + }) + }() + + s.channelzRemoveOnce.Do(func() { + if channelz.IsOn() { + channelz.RemoveEntry(s.channelzID) + } + }) s.mu.Lock() - defer s.mu.Unlock() if s.conns == nil { + s.mu.Unlock() return } + for lis := range s.lis { lis.Close() } s.lis = nil - s.cancel() if !s.drain { for c := range s.conns { c.(transport.ServerTransport).Drain() } s.drain = true } + + // Wait for serving threads to be ready to exit. Only then can we be sure no + // new conns will be created. + s.mu.Unlock() + s.serveWG.Wait() + s.mu.Lock() + for len(s.conns) != 0 { s.cv.Wait() } @@ -1124,26 +1407,29 @@ func (s *Server) GracefulStop() { s.events.Finish() s.events = nil } + s.mu.Unlock() } func init() { - internal.TestingCloseConns = func(arg interface{}) { - arg.(*Server).testingCloseConns() - } internal.TestingUseHandlerImpl = func(arg interface{}) { arg.(*Server).opts.useHandlerImpl = true } } -// testingCloseConns closes all existing transports but keeps s.lis -// accepting new connections. -func (s *Server) testingCloseConns() { - s.mu.Lock() - for c := range s.conns { - c.Close() - delete(s.conns, c) +// contentSubtype must be lowercase +// cannot return nil +func (s *Server) getCodec(contentSubtype string) baseCodec { + if s.opts.codec != nil { + return s.opts.codec } - s.mu.Unlock() + if contentSubtype == "" { + return encoding.GetCodec(proto.Name) + } + codec := encoding.GetCodec(contentSubtype) + if codec == nil { + return encoding.GetCodec(proto.Name) + } + return codec } // SetHeader sets the header metadata. @@ -1156,9 +1442,9 @@ func SetHeader(ctx context.Context, md metadata.MD) error { if md.Len() == 0 { return nil } - stream, ok := transport.StreamFromContext(ctx) - if !ok { - return Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) + stream := ServerTransportStreamFromContext(ctx) + if stream == nil { + return status.Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) } return stream.SetHeader(md) } @@ -1166,15 +1452,11 @@ func SetHeader(ctx context.Context, md metadata.MD) error { // SendHeader sends header metadata. It may be called at most once. // The provided md and headers set by SetHeader() will be sent. func SendHeader(ctx context.Context, md metadata.MD) error { - stream, ok := transport.StreamFromContext(ctx) - if !ok { - return Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) + stream := ServerTransportStreamFromContext(ctx) + if stream == nil { + return status.Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) } - t := stream.ServerTransport() - if t == nil { - grpclog.Fatalf("grpc: SendHeader: %v has no ServerTransport to send header metadata.", stream) - } - if err := t.WriteHeader(stream, md); err != nil { + if err := stream.SendHeader(md); err != nil { return toRPCErr(err) } return nil @@ -1186,9 +1468,19 @@ func SetTrailer(ctx context.Context, md metadata.MD) error { if md.Len() == 0 { return nil } - stream, ok := transport.StreamFromContext(ctx) - if !ok { - return Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) + stream := ServerTransportStreamFromContext(ctx) + if stream == nil { + return status.Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) } return stream.SetTrailer(md) } + +// Method returns the method string for the server context. The returned +// string is in the format of "/service/method". +func Method(ctx context.Context) (string, bool) { + s := ServerTransportStreamFromContext(ctx) + if s == nil { + return "", false + } + return s.Method(), true +} diff --git a/vendor/google.golang.org/grpc/service_config.go b/vendor/google.golang.org/grpc/service_config.go new file mode 100644 index 00000000000..015631d8d38 --- /dev/null +++ b/vendor/google.golang.org/grpc/service_config.go @@ -0,0 +1,233 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "google.golang.org/grpc/grpclog" +) + +const maxInt = int(^uint(0) >> 1) + +// MethodConfig defines the configuration recommended by the service providers for a +// particular method. +// +// Deprecated: Users should not use this struct. Service config should be received +// through name resolver, as specified here +// https://github.com/grpc/grpc/blob/master/doc/service_config.md +type MethodConfig struct { + // WaitForReady indicates whether RPCs sent to this method should wait until + // the connection is ready by default (!failfast). The value specified via the + // gRPC client API will override the value set here. + WaitForReady *bool + // Timeout is the default timeout for RPCs sent to this method. The actual + // deadline used will be the minimum of the value specified here and the value + // set by the application via the gRPC client API. If either one is not set, + // then the other will be used. If neither is set, then the RPC has no deadline. + Timeout *time.Duration + // MaxReqSize is the maximum allowed payload size for an individual request in a + // stream (client->server) in bytes. The size which is measured is the serialized + // payload after per-message compression (but before stream compression) in bytes. + // The actual value used is the minimum of the value specified here and the value set + // by the application via the gRPC client API. If either one is not set, then the other + // will be used. If neither is set, then the built-in default is used. + MaxReqSize *int + // MaxRespSize is the maximum allowed payload size for an individual response in a + // stream (server->client) in bytes. + MaxRespSize *int +} + +// ServiceConfig is provided by the service provider and contains parameters for how +// clients that connect to the service should behave. +// +// Deprecated: Users should not use this struct. Service config should be received +// through name resolver, as specified here +// https://github.com/grpc/grpc/blob/master/doc/service_config.md +type ServiceConfig struct { + // LB is the load balancer the service providers recommends. The balancer specified + // via grpc.WithBalancer will override this. + LB *string + // Methods contains a map for the methods in this service. + // If there is an exact match for a method (i.e. /service/method) in the map, use the corresponding MethodConfig. + // If there's no exact match, look for the default config for the service (/service/) and use the corresponding MethodConfig if it exists. + // Otherwise, the method has no MethodConfig to use. + Methods map[string]MethodConfig + + stickinessMetadataKey *string +} + +func parseDuration(s *string) (*time.Duration, error) { + if s == nil { + return nil, nil + } + if !strings.HasSuffix(*s, "s") { + return nil, fmt.Errorf("malformed duration %q", *s) + } + ss := strings.SplitN((*s)[:len(*s)-1], ".", 3) + if len(ss) > 2 { + return nil, fmt.Errorf("malformed duration %q", *s) + } + // hasDigits is set if either the whole or fractional part of the number is + // present, since both are optional but one is required. + hasDigits := false + var d time.Duration + if len(ss[0]) > 0 { + i, err := strconv.ParseInt(ss[0], 10, 32) + if err != nil { + return nil, fmt.Errorf("malformed duration %q: %v", *s, err) + } + d = time.Duration(i) * time.Second + hasDigits = true + } + if len(ss) == 2 && len(ss[1]) > 0 { + if len(ss[1]) > 9 { + return nil, fmt.Errorf("malformed duration %q", *s) + } + f, err := strconv.ParseInt(ss[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("malformed duration %q: %v", *s, err) + } + for i := 9; i > len(ss[1]); i-- { + f *= 10 + } + d += time.Duration(f) + hasDigits = true + } + if !hasDigits { + return nil, fmt.Errorf("malformed duration %q", *s) + } + + return &d, nil +} + +type jsonName struct { + Service *string + Method *string +} + +func (j jsonName) generatePath() (string, bool) { + if j.Service == nil { + return "", false + } + res := "/" + *j.Service + "/" + if j.Method != nil { + res += *j.Method + } + return res, true +} + +// TODO(lyuxuan): delete this struct after cleaning up old service config implementation. +type jsonMC struct { + Name *[]jsonName + WaitForReady *bool + Timeout *string + MaxRequestMessageBytes *int64 + MaxResponseMessageBytes *int64 +} + +// TODO(lyuxuan): delete this struct after cleaning up old service config implementation. +type jsonSC struct { + LoadBalancingPolicy *string + StickinessMetadataKey *string + MethodConfig *[]jsonMC +} + +func parseServiceConfig(js string) (ServiceConfig, error) { + var rsc jsonSC + err := json.Unmarshal([]byte(js), &rsc) + if err != nil { + grpclog.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err) + return ServiceConfig{}, err + } + sc := ServiceConfig{ + LB: rsc.LoadBalancingPolicy, + Methods: make(map[string]MethodConfig), + + stickinessMetadataKey: rsc.StickinessMetadataKey, + } + if rsc.MethodConfig == nil { + return sc, nil + } + + for _, m := range *rsc.MethodConfig { + if m.Name == nil { + continue + } + d, err := parseDuration(m.Timeout) + if err != nil { + grpclog.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err) + return ServiceConfig{}, err + } + + mc := MethodConfig{ + WaitForReady: m.WaitForReady, + Timeout: d, + } + if m.MaxRequestMessageBytes != nil { + if *m.MaxRequestMessageBytes > int64(maxInt) { + mc.MaxReqSize = newInt(maxInt) + } else { + mc.MaxReqSize = newInt(int(*m.MaxRequestMessageBytes)) + } + } + if m.MaxResponseMessageBytes != nil { + if *m.MaxResponseMessageBytes > int64(maxInt) { + mc.MaxRespSize = newInt(maxInt) + } else { + mc.MaxRespSize = newInt(int(*m.MaxResponseMessageBytes)) + } + } + for _, n := range *m.Name { + if path, valid := n.generatePath(); valid { + sc.Methods[path] = mc + } + } + } + + return sc, nil +} + +func min(a, b *int) *int { + if *a < *b { + return a + } + return b +} + +func getMaxSize(mcMax, doptMax *int, defaultVal int) *int { + if mcMax == nil && doptMax == nil { + return &defaultVal + } + if mcMax != nil && doptMax != nil { + return min(mcMax, doptMax) + } + if mcMax != nil { + return mcMax + } + return doptMax +} + +func newInt(b int) *int { + return &b +} diff --git a/vendor/google.golang.org/grpc/stats/stats.go b/vendor/google.golang.org/grpc/stats/stats.go index e844541e9c0..22692b69b51 100644 --- a/vendor/google.golang.org/grpc/stats/stats.go +++ b/vendor/google.golang.org/grpc/stats/stats.go @@ -169,6 +169,8 @@ func (s *OutTrailer) isRPCStats() {} type End struct { // Client is true if this End is from client side. Client bool + // BeginTime is the time when the RPC began. + BeginTime time.Time // EndTime is the time when the RPC ends. EndTime time.Time // Error is the error the RPC ended with. It is an error generated from diff --git a/vendor/google.golang.org/grpc/status/BUILD b/vendor/google.golang.org/grpc/status/BUILD index 571407a9780..11a12c7ba6e 100644 --- a/vendor/google.golang.org/grpc/status/BUILD +++ b/vendor/google.golang.org/grpc/status/BUILD @@ -2,13 +2,18 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["status.go"], + srcs = [ + "go16.go", + "go17.go", + "status.go", + ], importmap = "k8s.io/kubernetes/vendor/google.golang.org/grpc/status", importpath = "google.golang.org/grpc/status", visibility = ["//visibility:public"], deps = [ "//vendor/github.com/golang/protobuf/proto:go_default_library", "//vendor/github.com/golang/protobuf/ptypes:go_default_library", + "//vendor/golang.org/x/net/context:go_default_library", "//vendor/google.golang.org/genproto/googleapis/rpc/status:go_default_library", "//vendor/google.golang.org/grpc/codes:go_default_library", ], diff --git a/vendor/google.golang.org/grpc/status/go16.go b/vendor/google.golang.org/grpc/status/go16.go new file mode 100644 index 00000000000..e59b53e82be --- /dev/null +++ b/vendor/google.golang.org/grpc/status/go16.go @@ -0,0 +1,42 @@ +// +build go1.6,!go1.7 + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package status + +import ( + "golang.org/x/net/context" + "google.golang.org/grpc/codes" +) + +// FromContextError converts a context error into a Status. It returns a +// Status with codes.OK if err is nil, or a Status with codes.Unknown if err is +// non-nil and not a context error. +func FromContextError(err error) *Status { + switch err { + case nil: + return New(codes.OK, "") + case context.DeadlineExceeded: + return New(codes.DeadlineExceeded, err.Error()) + case context.Canceled: + return New(codes.Canceled, err.Error()) + default: + return New(codes.Unknown, err.Error()) + } +} diff --git a/vendor/google.golang.org/grpc/status/go17.go b/vendor/google.golang.org/grpc/status/go17.go new file mode 100644 index 00000000000..090215149cf --- /dev/null +++ b/vendor/google.golang.org/grpc/status/go17.go @@ -0,0 +1,44 @@ +// +build go1.7 + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package status + +import ( + "context" + + netctx "golang.org/x/net/context" + "google.golang.org/grpc/codes" +) + +// FromContextError converts a context error into a Status. It returns a +// Status with codes.OK if err is nil, or a Status with codes.Unknown if err is +// non-nil and not a context error. +func FromContextError(err error) *Status { + switch err { + case nil: + return New(codes.OK, "") + case context.DeadlineExceeded, netctx.DeadlineExceeded: + return New(codes.DeadlineExceeded, err.Error()) + case context.Canceled, netctx.Canceled: + return New(codes.Canceled, err.Error()) + default: + return New(codes.Unknown, err.Error()) + } +} diff --git a/vendor/google.golang.org/grpc/status/status.go b/vendor/google.golang.org/grpc/status/status.go index 871dc4b31c7..9c61b094508 100644 --- a/vendor/google.golang.org/grpc/status/status.go +++ b/vendor/google.golang.org/grpc/status/status.go @@ -46,7 +46,7 @@ func (se *statusError) Error() string { return fmt.Sprintf("rpc error: code = %s desc = %s", codes.Code(p.GetCode()), p.GetMessage()) } -func (se *statusError) status() *Status { +func (se *statusError) GRPCStatus() *Status { return &Status{s: (*spb.Status)(se)} } @@ -120,15 +120,23 @@ func FromProto(s *spb.Status) *Status { } // FromError returns a Status representing err if it was produced from this -// package, otherwise it returns nil, false. +// package or has a method `GRPCStatus() *Status`. Otherwise, ok is false and a +// Status is returned with codes.Unknown and the original error message. func FromError(err error) (s *Status, ok bool) { if err == nil { return &Status{s: &spb.Status{Code: int32(codes.OK)}}, true } - if s, ok := err.(*statusError); ok { - return s.status(), true + if se, ok := err.(interface{ GRPCStatus() *Status }); ok { + return se.GRPCStatus(), true } - return nil, false + return New(codes.Unknown, err.Error()), false +} + +// Convert is a convenience function which removes the need to handle the +// boolean return value from FromError. +func Convert(err error) *Status { + s, _ := FromError(err) + return s } // WithDetails returns a new status with the provided details messages appended to the status. @@ -166,3 +174,16 @@ func (s *Status) Details() []interface{} { } return details } + +// Code returns the Code of the error if it is a Status error, codes.OK if err +// is nil, or codes.Unknown otherwise. +func Code(err error) codes.Code { + // Don't use FromError to avoid allocation of OK status. + if err == nil { + return codes.OK + } + if se, ok := err.(interface{ GRPCStatus() *Status }); ok { + return se.GRPCStatus().Code() + } + return codes.Unknown +} diff --git a/vendor/google.golang.org/grpc/stickiness_linkedmap.go b/vendor/google.golang.org/grpc/stickiness_linkedmap.go new file mode 100644 index 00000000000..1c726af1640 --- /dev/null +++ b/vendor/google.golang.org/grpc/stickiness_linkedmap.go @@ -0,0 +1,97 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "container/list" +) + +type linkedMapKVPair struct { + key string + value *stickyStoreEntry +} + +// linkedMap is an implementation of a map that supports removing the oldest +// entry. +// +// linkedMap is NOT thread safe. +// +// It's for use of stickiness only! +type linkedMap struct { + m map[string]*list.Element + l *list.List // Head of the list is the oldest element. +} + +// newLinkedMap returns a new LinkedMap. +func newLinkedMap() *linkedMap { + return &linkedMap{ + m: make(map[string]*list.Element), + l: list.New(), + } +} + +// put adds entry (key, value) to the map. Existing key will be overridden. +func (m *linkedMap) put(key string, value *stickyStoreEntry) { + if oldE, ok := m.m[key]; ok { + // Remove existing entry. + m.l.Remove(oldE) + } + e := m.l.PushBack(&linkedMapKVPair{key: key, value: value}) + m.m[key] = e +} + +// get returns the value of the given key. +func (m *linkedMap) get(key string) (*stickyStoreEntry, bool) { + e, ok := m.m[key] + if !ok { + return nil, false + } + m.l.MoveToBack(e) + return e.Value.(*linkedMapKVPair).value, true +} + +// remove removes key from the map, and returns the value. The map is not +// modified if key is not in the map. +func (m *linkedMap) remove(key string) (*stickyStoreEntry, bool) { + e, ok := m.m[key] + if !ok { + return nil, false + } + delete(m.m, key) + m.l.Remove(e) + return e.Value.(*linkedMapKVPair).value, true +} + +// len returns the len of the map. +func (m *linkedMap) len() int { + return len(m.m) +} + +// clear removes all elements from the map. +func (m *linkedMap) clear() { + m.m = make(map[string]*list.Element) + m.l = list.New() +} + +// removeOldest removes the oldest key from the map. +func (m *linkedMap) removeOldest() { + e := m.l.Front() + m.l.Remove(e) + delete(m.m, e.Value.(*linkedMapKVPair).key) +} diff --git a/vendor/google.golang.org/grpc/stream.go b/vendor/google.golang.org/grpc/stream.go index 75eab40b109..152d9eccd62 100644 --- a/vendor/google.golang.org/grpc/stream.go +++ b/vendor/google.golang.org/grpc/stream.go @@ -19,7 +19,6 @@ package grpc import ( - "bytes" "errors" "io" "sync" @@ -29,15 +28,19 @@ import ( "golang.org/x/net/trace" "google.golang.org/grpc/balancer" "google.golang.org/grpc/codes" + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/metadata" - "google.golang.org/grpc/peer" "google.golang.org/grpc/stats" "google.golang.org/grpc/status" "google.golang.org/grpc/transport" ) // StreamHandler defines the handler called by gRPC server to complete the -// execution of a streaming RPC. +// execution of a streaming RPC. If a StreamHandler returns an error, it +// should be produced by the status package, or else gRPC will use +// codes.Unknown as the status code and err.Error() as the status message +// of the RPC. type StreamHandler func(srv interface{}, stream ServerStream) error // StreamDesc represents a streaming RPC service's method specification. @@ -51,6 +54,8 @@ type StreamDesc struct { } // Stream defines the common interface a client or server stream has to satisfy. +// +// All errors returned from Stream are compatible with the status package. type Stream interface { // Context returns the context for this stream. Context() context.Context @@ -89,43 +94,78 @@ type ClientStream interface { // Stream.SendMsg() may return a non-nil error when something wrong happens sending // the request. The returned error indicates the status of this sending, not the final // status of the RPC. - // Always call Stream.RecvMsg() to get the final status if you care about the status of - // the RPC. + // + // Always call Stream.RecvMsg() to drain the stream and get the final + // status, otherwise there could be leaked resources. Stream } -// NewClientStream creates a new Stream for the client side. This is called -// by generated code. -func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (_ ClientStream, err error) { +// NewStream creates a new Stream for the client side. This is typically +// called by generated code. ctx is used for the lifetime of the stream. +// +// To ensure resources are not leaked due to the stream returned, one of the following +// actions must be performed: +// +// 1. Call Close on the ClientConn. +// 2. Cancel the context provided. +// 3. Call RecvMsg until a non-nil error is returned. A protobuf-generated +// client-streaming RPC, for instance, might use the helper function +// CloseAndRecv (note that CloseSend does not Recv, therefore is not +// guaranteed to release all resources). +// 4. Receive a non-nil, non-io.EOF error from Header or SendMsg. +// +// If none of the above happen, a goroutine and a context will be leaked, and grpc +// will not call the optionally-configured stats handler with a stats.End message. +func (cc *ClientConn) NewStream(ctx context.Context, desc *StreamDesc, method string, opts ...CallOption) (ClientStream, error) { + // allow interceptor to see all applicable call options, which means those + // configured as defaults from dial option as well as per-call options + opts = combine(cc.dopts.callOptions, opts) + if cc.dopts.streamInt != nil { return cc.dopts.streamInt(ctx, desc, cc, method, newClientStream, opts...) } return newClientStream(ctx, desc, cc, method, opts...) } +// NewClientStream is a wrapper for ClientConn.NewStream. +// +// DEPRECATED: Use ClientConn.NewStream instead. +func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) { + return cc.NewStream(ctx, desc, method, opts...) +} + func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (_ ClientStream, err error) { - var ( - t transport.ClientTransport - s *transport.Stream - done func(balancer.DoneInfo) - cancel context.CancelFunc - ) + if channelz.IsOn() { + cc.incrCallsStarted() + defer func() { + if err != nil { + cc.incrCallsFailed() + } + }() + } c := defaultCallInfo() mc := cc.GetMethodConfig(method) if mc.WaitForReady != nil { c.failFast = !*mc.WaitForReady } - if mc.Timeout != nil { + // Possible context leak: + // The cancel function for the child context we create will only be called + // when RecvMsg returns a non-nil error, if the ClientConn is closed, or if + // an error is generated by SendMsg. + // https://github.com/grpc/grpc-go/issues/1818. + var cancel context.CancelFunc + if mc.Timeout != nil && *mc.Timeout >= 0 { ctx, cancel = context.WithTimeout(ctx, *mc.Timeout) - defer func() { - if err != nil { - cancel() - } - }() + } else { + ctx, cancel = context.WithCancel(ctx) } + defer func() { + if err != nil { + cancel() + } + }() - opts = append(cc.dopts.callOptions, opts...) for _, o := range opts { if err := o.before(c); err != nil { return nil, toRPCErr(err) @@ -133,6 +173,9 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth } c.maxSendMessageSize = getMaxSize(mc.MaxReqSize, c.maxSendMessageSize, defaultClientMaxSendMessageSize) c.maxReceiveMessageSize = getMaxSize(mc.MaxRespSize, c.maxReceiveMessageSize, defaultClientMaxReceiveMessageSize) + if err := setCallInfoCodec(c); err != nil { + return nil, err + } callHdr := &transport.CallHdr{ Host: cc.authority, @@ -141,10 +184,27 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth // so we don't flush the header. // If it's client streaming, the user may never send a request or send it any // time soon, so we ask the transport to flush the header. - Flush: desc.ClientStreams, + Flush: desc.ClientStreams, + ContentSubtype: c.contentSubtype, } - if cc.dopts.cp != nil { + + // Set our outgoing compression according to the UseCompressor CallOption, if + // set. In that case, also find the compressor from the encoding package. + // Otherwise, use the compressor configured by the WithCompressor DialOption, + // if set. + var cp Compressor + var comp encoding.Compressor + if ct := c.compressorType; ct != "" { + callHdr.SendCompress = ct + if ct != encoding.Identity { + comp = encoding.GetCompressor(ct) + if comp == nil { + return nil, status.Errorf(codes.Internal, "grpc: Compressor is not installed for requested grpc-encoding %q", ct) + } + } + } else if cc.dopts.cp != nil { callHdr.SendCompress = cc.dopts.cp.Type() + cp = cc.dopts.cp } if c.creds != nil { callHdr.Creds = c.creds @@ -170,11 +230,13 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth } ctx = newContextWithRPCInfo(ctx, c.failFast) sh := cc.dopts.copts.StatsHandler + var beginTime time.Time if sh != nil { ctx = sh.TagRPC(ctx, &stats.RPCTagInfo{FullMethodName: method, FailFast: c.failFast}) + beginTime = time.Now() begin := &stats.Begin{ Client: true, - BeginTime: time.Now(), + BeginTime: beginTime, FailFast: c.failFast, } sh.HandleRPC(ctx, begin) @@ -182,341 +244,384 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth if err != nil { // Only handle end stats if err != nil. end := &stats.End{ - Client: true, - Error: err, + Client: true, + Error: err, + BeginTime: beginTime, + EndTime: time.Now(), } sh.HandleRPC(ctx, end) } }() } + + var ( + t transport.ClientTransport + s *transport.Stream + done func(balancer.DoneInfo) + ) for { + // Check to make sure the context has expired. This will prevent us from + // looping forever if an error occurs for wait-for-ready RPCs where no data + // is sent on the wire. + select { + case <-ctx.Done(): + return nil, toRPCErr(ctx.Err()) + default: + } + t, done, err = cc.getTransport(ctx, c.failFast) if err != nil { - // TODO(zhaoq): Probably revisit the error handling. - if _, ok := status.FromError(err); ok { - return nil, err - } - if err == errConnClosing || err == errConnUnavailable { - if c.failFast { - return nil, Errorf(codes.Unavailable, "%v", err) - } - continue - } - // All the other errors are treated as Internal errors. - return nil, Errorf(codes.Internal, "%v", err) + return nil, err } s, err = t.NewStream(ctx, callHdr) if err != nil { - if _, ok := err.(transport.ConnectionError); ok && done != nil { - // If error is connection error, transport was sending data on wire, - // and we are not sure if anything has been sent on wire. - // If error is not connection error, we are sure nothing has been sent. - updateRPCInfoInContext(ctx, rpcInfo{bytesSent: true, bytesReceived: false}) - } if done != nil { done(balancer.DoneInfo{Err: err}) done = nil } - if _, ok := err.(transport.ConnectionError); (ok || err == transport.ErrStreamDrain) && !c.failFast { + // In the event of any error from NewStream, we never attempted to write + // anything to the wire, so we can retry indefinitely for non-fail-fast + // RPCs. + if !c.failFast { continue } return nil, toRPCErr(err) } break } - // Set callInfo.peer object from stream's context. - if peer, ok := peer.FromContext(s.Context()); ok { - c.peer = peer - } + cs := &clientStream{ opts: opts, c: c, + cc: cc, desc: desc, - codec: cc.dopts.codec, - cp: cc.dopts.cp, - dc: cc.dopts.dc, + codec: c.codec, + cp: cp, + comp: comp, cancel: cancel, - - done: done, - t: t, - s: s, - p: &parser{r: s}, - - tracing: EnableTracing, - trInfo: trInfo, - - statsCtx: ctx, - statsHandler: cc.dopts.copts.StatsHandler, + attempt: &csAttempt{ + t: t, + s: s, + p: &parser{r: s}, + done: done, + dc: cc.dopts.dc, + ctx: ctx, + trInfo: trInfo, + statsHandler: sh, + beginTime: beginTime, + }, + } + cs.c.stream = cs + cs.attempt.cs = cs + if desc != unaryStreamDesc { + // Listen on cc and stream contexts to cleanup when the user closes the + // ClientConn or cancels the stream context. In all other cases, an error + // should already be injected into the recv buffer by the transport, which + // the client will eventually receive, and then we will cancel the stream's + // context in clientStream.finish. + go func() { + select { + case <-cc.ctx.Done(): + cs.finish(ErrClientConnClosing) + case <-ctx.Done(): + cs.finish(toRPCErr(ctx.Err())) + } + }() } - // Listen on ctx.Done() to detect cancellation and s.Done() to detect normal termination - // when there is no pending I/O operations on this stream. - go func() { - select { - case <-t.Error(): - // Incur transport error, simply exit. - case <-cc.ctx.Done(): - cs.finish(ErrClientConnClosing) - cs.closeTransportStream(ErrClientConnClosing) - case <-s.Done(): - // TODO: The trace of the RPC is terminated here when there is no pending - // I/O, which is probably not the optimal solution. - cs.finish(s.Status().Err()) - cs.closeTransportStream(nil) - case <-s.GoAway(): - cs.finish(errConnDrain) - cs.closeTransportStream(errConnDrain) - case <-s.Context().Done(): - err := s.Context().Err() - cs.finish(err) - cs.closeTransportStream(transport.ContextErr(err)) - } - }() return cs, nil } // clientStream implements a client side Stream. type clientStream struct { - opts []CallOption - c *callInfo - t transport.ClientTransport - s *transport.Stream - p *parser - desc *StreamDesc - codec Codec - cp Compressor - dc Decompressor - cancel context.CancelFunc + opts []CallOption + c *callInfo + cc *ClientConn + desc *StreamDesc - tracing bool // set to EnableTracing when the clientStream is created. + codec baseCodec + cp Compressor + comp encoding.Compressor - mu sync.Mutex - done func(balancer.DoneInfo) - closed bool - finished bool - // trInfo.tr is set when the clientStream is created (if EnableTracing is true), - // and is set to nil when the clientStream's finish method is called. + cancel context.CancelFunc // cancels all attempts + + sentLast bool // sent an end stream + + mu sync.Mutex // guards finished + finished bool // TODO: replace with atomic cmpxchg or sync.Once? + + attempt *csAttempt // the active client stream attempt + // TODO(hedging): hedging will have multiple attempts simultaneously. +} + +// csAttempt implements a single transport stream attempt within a +// clientStream. +type csAttempt struct { + cs *clientStream + t transport.ClientTransport + s *transport.Stream + p *parser + done func(balancer.DoneInfo) + + dc Decompressor + decomp encoding.Compressor + decompSet bool + + ctx context.Context // the application's context, wrapped by stats/tracing + + mu sync.Mutex // guards trInfo.tr + // trInfo.tr is set when created (if EnableTracing is true), + // and cleared when the finish method is called. trInfo traceInfo - // statsCtx keeps the user context for stats handling. - // All stats collection should use the statsCtx (instead of the stream context) - // so that all the generated stats for a particular RPC can be associated in the processing phase. - statsCtx context.Context statsHandler stats.Handler + beginTime time.Time } func (cs *clientStream) Context() context.Context { - return cs.s.Context() + // TODO(retry): commit the current attempt (the context has peer-aware data). + return cs.attempt.context() } func (cs *clientStream) Header() (metadata.MD, error) { - m, err := cs.s.Header() + m, err := cs.attempt.header() if err != nil { - if _, ok := err.(transport.ConnectionError); !ok { - cs.closeTransportStream(err) - } + // TODO(retry): maybe retry on error or commit attempt on success. + err = toRPCErr(err) + cs.finish(err) } return m, err } func (cs *clientStream) Trailer() metadata.MD { - return cs.s.Trailer() + // TODO(retry): on error, maybe retry (trailers-only). + return cs.attempt.trailer() } func (cs *clientStream) SendMsg(m interface{}) (err error) { - if cs.tracing { - cs.mu.Lock() - if cs.trInfo.tr != nil { - cs.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true) - } - cs.mu.Unlock() - } - // TODO Investigate how to signal the stats handling party. - // generate error stats if err != nil && err != io.EOF? - defer func() { - if err != nil { - cs.finish(err) - } - if err == nil { - return - } - if err == io.EOF { - // Specialize the process for server streaming. SendMsg is only called - // once when creating the stream object. io.EOF needs to be skipped when - // the rpc is early finished (before the stream object is created.). - // TODO: It is probably better to move this into the generated code. - if !cs.desc.ClientStreams && cs.desc.ServerStreams { - err = nil - } - return - } - if _, ok := err.(transport.ConnectionError); !ok { - cs.closeTransportStream(err) - } - err = toRPCErr(err) - }() - var outPayload *stats.OutPayload - if cs.statsHandler != nil { - outPayload = &stats.OutPayload{ - Client: true, - } - } - hdr, data, err := encode(cs.codec, m, cs.cp, bytes.NewBuffer([]byte{}), outPayload) - if err != nil { - return err - } - if cs.c.maxSendMessageSize == nil { - return Errorf(codes.Internal, "callInfo maxSendMessageSize field uninitialized(nil)") - } - if len(data) > *cs.c.maxSendMessageSize { - return Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", len(data), *cs.c.maxSendMessageSize) - } - err = cs.t.Write(cs.s, hdr, data, &transport.Options{Last: false}) - if err == nil && outPayload != nil { - outPayload.SentTime = time.Now() - cs.statsHandler.HandleRPC(cs.statsCtx, outPayload) - } - return err + // TODO(retry): buffer message for replaying if not committed. + return cs.attempt.sendMsg(m) } func (cs *clientStream) RecvMsg(m interface{}) (err error) { + // TODO(retry): maybe retry on error or commit attempt on success. + return cs.attempt.recvMsg(m) +} + +func (cs *clientStream) CloseSend() error { + cs.attempt.closeSend() + return nil +} + +func (cs *clientStream) finish(err error) { + if err == io.EOF { + // Ending a stream with EOF indicates a success. + err = nil + } + cs.mu.Lock() + if cs.finished { + cs.mu.Unlock() + return + } + cs.finished = true + cs.mu.Unlock() + if channelz.IsOn() { + if err != nil { + cs.cc.incrCallsFailed() + } else { + cs.cc.incrCallsSucceeded() + } + } + // TODO(retry): commit current attempt if necessary. + cs.attempt.finish(err) + for _, o := range cs.opts { + o.after(cs.c) + } + cs.cancel() +} + +func (a *csAttempt) context() context.Context { + return a.s.Context() +} + +func (a *csAttempt) header() (metadata.MD, error) { + return a.s.Header() +} + +func (a *csAttempt) trailer() metadata.MD { + return a.s.Trailer() +} + +func (a *csAttempt) sendMsg(m interface{}) (err error) { + // TODO Investigate how to signal the stats handling party. + // generate error stats if err != nil && err != io.EOF? + cs := a.cs + defer func() { + // For non-client-streaming RPCs, we return nil instead of EOF on success + // because the generated code requires it. finish is not called; RecvMsg() + // will call it with the stream's status independently. + if err == io.EOF && !cs.desc.ClientStreams { + err = nil + } + if err != nil && err != io.EOF { + // Call finish on the client stream for errors generated by this SendMsg + // call, as these indicate problems created by this client. (Transport + // errors are converted to an io.EOF error below; the real error will be + // returned from RecvMsg eventually in that case, or be retried.) + cs.finish(err) + } + }() + // TODO: Check cs.sentLast and error if we already ended the stream. + if EnableTracing { + a.mu.Lock() + if a.trInfo.tr != nil { + a.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true) + } + a.mu.Unlock() + } + data, err := encode(cs.codec, m) + if err != nil { + return err + } + compData, err := compress(data, cs.cp, cs.comp) + if err != nil { + return err + } + hdr, payload := msgHeader(data, compData) + // TODO(dfawley): should we be checking len(data) instead? + if len(payload) > *cs.c.maxSendMessageSize { + return status.Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", len(payload), *cs.c.maxSendMessageSize) + } + + if !cs.desc.ClientStreams { + cs.sentLast = true + } + err = a.t.Write(a.s, hdr, payload, &transport.Options{Last: !cs.desc.ClientStreams}) + if err == nil { + if a.statsHandler != nil { + a.statsHandler.HandleRPC(a.ctx, outPayload(true, m, data, payload, time.Now())) + } + if channelz.IsOn() { + a.t.IncrMsgSent() + } + return nil + } + return io.EOF +} + +func (a *csAttempt) recvMsg(m interface{}) (err error) { + cs := a.cs + defer func() { + if err != nil || !cs.desc.ServerStreams { + // err != nil or non-server-streaming indicates end of stream. + cs.finish(err) + } + }() var inPayload *stats.InPayload - if cs.statsHandler != nil { + if a.statsHandler != nil { inPayload = &stats.InPayload{ Client: true, } } - if cs.c.maxReceiveMessageSize == nil { - return Errorf(codes.Internal, "callInfo maxReceiveMessageSize field uninitialized(nil)") + if !a.decompSet { + // Block until we receive headers containing received message encoding. + if ct := a.s.RecvCompress(); ct != "" && ct != encoding.Identity { + if a.dc == nil || a.dc.Type() != ct { + // No configured decompressor, or it does not match the incoming + // message encoding; attempt to find a registered compressor that does. + a.dc = nil + a.decomp = encoding.GetCompressor(ct) + } + } else { + // No compression is used; disable our decompressor. + a.dc = nil + } + // Only initialize this state once per stream. + a.decompSet = true } - err = recv(cs.p, cs.codec, cs.s, cs.dc, m, *cs.c.maxReceiveMessageSize, inPayload) - defer func() { - // err != nil indicates the termination of the stream. - if err != nil { - cs.finish(err) - } - }() - if err == nil { - if cs.tracing { - cs.mu.Lock() - if cs.trInfo.tr != nil { - cs.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true) - } - cs.mu.Unlock() - } - if inPayload != nil { - cs.statsHandler.HandleRPC(cs.statsCtx, inPayload) - } - if !cs.desc.ClientStreams || cs.desc.ServerStreams { - return - } - // Special handling for client streaming rpc. - // This recv expects EOF or errors, so we don't collect inPayload. - if cs.c.maxReceiveMessageSize == nil { - return Errorf(codes.Internal, "callInfo maxReceiveMessageSize field uninitialized(nil)") - } - err = recv(cs.p, cs.codec, cs.s, cs.dc, m, *cs.c.maxReceiveMessageSize, nil) - cs.closeTransportStream(err) - if err == nil { - return toRPCErr(errors.New("grpc: client streaming protocol violation: get , want ")) - } + err = recv(a.p, cs.codec, a.s, a.dc, m, *cs.c.maxReceiveMessageSize, inPayload, a.decomp) + if err != nil { if err == io.EOF { - if se := cs.s.Status().Err(); se != nil { - return se + if statusErr := a.s.Status().Err(); statusErr != nil { + return statusErr } - cs.finish(err) - return nil + return io.EOF // indicates successful end of stream. } return toRPCErr(err) } - if _, ok := err.(transport.ConnectionError); !ok { - cs.closeTransportStream(err) + if EnableTracing { + a.mu.Lock() + if a.trInfo.tr != nil { + a.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true) + } + a.mu.Unlock() + } + if inPayload != nil { + a.statsHandler.HandleRPC(a.ctx, inPayload) + } + if channelz.IsOn() { + a.t.IncrMsgRecv() + } + if cs.desc.ServerStreams { + // Subsequent messages should be received by subsequent RecvMsg calls. + return nil + } + + // Special handling for non-server-stream rpcs. + // This recv expects EOF or errors, so we don't collect inPayload. + err = recv(a.p, cs.codec, a.s, a.dc, m, *cs.c.maxReceiveMessageSize, nil, a.decomp) + if err == nil { + return toRPCErr(errors.New("grpc: client streaming protocol violation: get , want ")) } if err == io.EOF { - if statusErr := cs.s.Status().Err(); statusErr != nil { - return statusErr - } - // Returns io.EOF to indicate the end of the stream. - return + return a.s.Status().Err() // non-server streaming Recv returns nil on success } return toRPCErr(err) } -func (cs *clientStream) CloseSend() (err error) { - err = cs.t.Write(cs.s, nil, nil, &transport.Options{Last: true}) - defer func() { - if err != nil { - cs.finish(err) - } - }() - if err == nil || err == io.EOF { - return nil - } - if _, ok := err.(transport.ConnectionError); !ok { - cs.closeTransportStream(err) - } - err = toRPCErr(err) - return -} - -func (cs *clientStream) closeTransportStream(err error) { - cs.mu.Lock() - if cs.closed { - cs.mu.Unlock() +func (a *csAttempt) closeSend() { + cs := a.cs + if cs.sentLast { return } - cs.closed = true - cs.mu.Unlock() - cs.t.CloseStream(cs.s, err) + cs.sentLast = true + cs.attempt.t.Write(cs.attempt.s, nil, nil, &transport.Options{Last: true}) + // We ignore errors from Write. Any error it would return would also be + // returned by a subsequent RecvMsg call, and the user is supposed to always + // finish the stream by calling RecvMsg until it returns err != nil. } -func (cs *clientStream) finish(err error) { - cs.mu.Lock() - defer cs.mu.Unlock() - if cs.finished { - return - } - cs.finished = true - defer func() { - if cs.cancel != nil { - cs.cancel() - } - }() - for _, o := range cs.opts { - o.after(cs.c) - } - if cs.done != nil { - updateRPCInfoInContext(cs.s.Context(), rpcInfo{ - bytesSent: cs.s.BytesSent(), - bytesReceived: cs.s.BytesReceived(), +func (a *csAttempt) finish(err error) { + a.mu.Lock() + a.t.CloseStream(a.s, err) + + if a.done != nil { + a.done(balancer.DoneInfo{ + Err: err, + BytesSent: true, + BytesReceived: a.s.BytesReceived(), }) - cs.done(balancer.DoneInfo{Err: err}) - cs.done = nil } - if cs.statsHandler != nil { + if a.statsHandler != nil { end := &stats.End{ - Client: true, - EndTime: time.Now(), + Client: true, + BeginTime: a.beginTime, + EndTime: time.Now(), + Error: err, } - if err != io.EOF { - // end.Error is nil if the RPC finished successfully. - end.Error = toRPCErr(err) - } - cs.statsHandler.HandleRPC(cs.statsCtx, end) + a.statsHandler.HandleRPC(a.ctx, end) } - if !cs.tracing { - return - } - if cs.trInfo.tr != nil { - if err == nil || err == io.EOF { - cs.trInfo.tr.LazyPrintf("RPC: [OK]") + if a.trInfo.tr != nil { + if err == nil { + a.trInfo.tr.LazyPrintf("RPC: [OK]") } else { - cs.trInfo.tr.LazyPrintf("RPC: [%v]", err) - cs.trInfo.tr.SetError() + a.trInfo.tr.LazyPrintf("RPC: [%v]", err) + a.trInfo.tr.SetError() } - cs.trInfo.tr.Finish() - cs.trInfo.tr = nil + a.trInfo.tr.Finish() + a.trInfo.tr = nil } + a.mu.Unlock() } // ServerStream defines the interface a server stream has to satisfy. @@ -540,12 +645,17 @@ type ServerStream interface { // serverStream implements a server side Stream. type serverStream struct { - t transport.ServerTransport - s *transport.Stream - p *parser - codec Codec - cp Compressor - dc Decompressor + ctx context.Context + t transport.ServerTransport + s *transport.Stream + p *parser + codec baseCodec + + cp Compressor + dc Decompressor + comp encoding.Compressor + decomp encoding.Compressor + maxReceiveMessageSize int maxSendMessageSize int trInfo *traceInfo @@ -556,7 +666,7 @@ type serverStream struct { } func (ss *serverStream) Context() context.Context { - return ss.s.Context() + return ss.ctx } func (ss *serverStream) SetHeader(md metadata.MD) error { @@ -575,7 +685,6 @@ func (ss *serverStream) SetTrailer(md metadata.MD) { return } ss.s.SetTrailer(md) - return } func (ss *serverStream) SendMsg(m interface{}) (err error) { @@ -596,24 +705,28 @@ func (ss *serverStream) SendMsg(m interface{}) (err error) { st, _ := status.FromError(toRPCErr(err)) ss.t.WriteStatus(ss.s, st) } + if channelz.IsOn() && err == nil { + ss.t.IncrMsgSent() + } }() - var outPayload *stats.OutPayload - if ss.statsHandler != nil { - outPayload = &stats.OutPayload{} - } - hdr, data, err := encode(ss.codec, m, ss.cp, bytes.NewBuffer([]byte{}), outPayload) + data, err := encode(ss.codec, m) if err != nil { return err } - if len(data) > ss.maxSendMessageSize { - return Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", len(data), ss.maxSendMessageSize) + compData, err := compress(data, ss.cp, ss.comp) + if err != nil { + return err } - if err := ss.t.Write(ss.s, hdr, data, &transport.Options{Last: false}); err != nil { + hdr, payload := msgHeader(data, compData) + // TODO(dfawley): should we be checking len(data) instead? + if len(payload) > ss.maxSendMessageSize { + return status.Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", len(payload), ss.maxSendMessageSize) + } + if err := ss.t.Write(ss.s, hdr, payload, &transport.Options{Last: false}); err != nil { return toRPCErr(err) } - if outPayload != nil { - outPayload.SentTime = time.Now() - ss.statsHandler.HandleRPC(ss.s.Context(), outPayload) + if ss.statsHandler != nil { + ss.statsHandler.HandleRPC(ss.s.Context(), outPayload(false, m, data, payload, time.Now())) } return nil } @@ -636,17 +749,20 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) { st, _ := status.FromError(toRPCErr(err)) ss.t.WriteStatus(ss.s, st) } + if channelz.IsOn() && err == nil { + ss.t.IncrMsgRecv() + } }() var inPayload *stats.InPayload if ss.statsHandler != nil { inPayload = &stats.InPayload{} } - if err := recv(ss.p, ss.codec, ss.s, ss.dc, m, ss.maxReceiveMessageSize, inPayload); err != nil { + if err := recv(ss.p, ss.codec, ss.s, ss.dc, m, ss.maxReceiveMessageSize, inPayload, ss.decomp); err != nil { if err == io.EOF { return err } if err == io.ErrUnexpectedEOF { - err = Errorf(codes.Internal, io.ErrUnexpectedEOF.Error()) + err = status.Errorf(codes.Internal, io.ErrUnexpectedEOF.Error()) } return toRPCErr(err) } @@ -655,3 +771,9 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) { } return nil } + +// MethodFromServerStream returns the method string for the input stream. +// The returned string is in the format of "/service/method". +func MethodFromServerStream(stream ServerStream) (string, bool) { + return Method(stream.Context()) +} diff --git a/vendor/google.golang.org/grpc/transport/BUILD b/vendor/google.golang.org/grpc/transport/BUILD index bc3aac745b4..f61be320354 100644 --- a/vendor/google.golang.org/grpc/transport/BUILD +++ b/vendor/google.golang.org/grpc/transport/BUILD @@ -4,7 +4,10 @@ go_library( name = "go_default_library", srcs = [ "bdp_estimator.go", - "control.go", + "controlbuf.go", + "flowcontrol.go", + "go16.go", + "go17.go", "handler_server.go", "http2_client.go", "http2_server.go", @@ -24,6 +27,8 @@ go_library( "//vendor/google.golang.org/grpc/codes:go_default_library", "//vendor/google.golang.org/grpc/credentials:go_default_library", "//vendor/google.golang.org/grpc/grpclog:go_default_library", + "//vendor/google.golang.org/grpc/internal/channelz:go_default_library", + "//vendor/google.golang.org/grpc/internal/grpcrand:go_default_library", "//vendor/google.golang.org/grpc/keepalive:go_default_library", "//vendor/google.golang.org/grpc/metadata:go_default_library", "//vendor/google.golang.org/grpc/peer:go_default_library", diff --git a/vendor/google.golang.org/grpc/transport/bdp_estimator.go b/vendor/google.golang.org/grpc/transport/bdp_estimator.go index 8dd2ed42792..63cd2627c87 100644 --- a/vendor/google.golang.org/grpc/transport/bdp_estimator.go +++ b/vendor/google.golang.org/grpc/transport/bdp_estimator.go @@ -41,12 +41,9 @@ const ( gamma = 2 ) -var ( - // Adding arbitrary data to ping so that its ack can be - // identified. - // Easter-egg: what does the ping message say? - bdpPing = &ping{data: [8]byte{2, 4, 16, 16, 9, 14, 7, 7}} -) +// Adding arbitrary data to ping so that its ack can be identified. +// Easter-egg: what does the ping message say? +var bdpPing = &ping{data: [8]byte{2, 4, 16, 16, 9, 14, 7, 7}} type bdpEstimator struct { // sentAt is the time when the ping was sent. diff --git a/vendor/google.golang.org/grpc/transport/controlbuf.go b/vendor/google.golang.org/grpc/transport/controlbuf.go new file mode 100644 index 00000000000..5c5891a11bf --- /dev/null +++ b/vendor/google.golang.org/grpc/transport/controlbuf.go @@ -0,0 +1,796 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "bytes" + "fmt" + "runtime" + "sync" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" +) + +var updateHeaderTblSize = func(e *hpack.Encoder, v uint32) { + e.SetMaxDynamicTableSizeLimit(v) +} + +type itemNode struct { + it interface{} + next *itemNode +} + +type itemList struct { + head *itemNode + tail *itemNode +} + +func (il *itemList) enqueue(i interface{}) { + n := &itemNode{it: i} + if il.tail == nil { + il.head, il.tail = n, n + return + } + il.tail.next = n + il.tail = n +} + +// peek returns the first item in the list without removing it from the +// list. +func (il *itemList) peek() interface{} { + return il.head.it +} + +func (il *itemList) dequeue() interface{} { + if il.head == nil { + return nil + } + i := il.head.it + il.head = il.head.next + if il.head == nil { + il.tail = nil + } + return i +} + +func (il *itemList) dequeueAll() *itemNode { + h := il.head + il.head, il.tail = nil, nil + return h +} + +func (il *itemList) isEmpty() bool { + return il.head == nil +} + +// The following defines various control items which could flow through +// the control buffer of transport. They represent different aspects of +// control tasks, e.g., flow control, settings, streaming resetting, etc. + +// registerStream is used to register an incoming stream with loopy writer. +type registerStream struct { + streamID uint32 + wq *writeQuota +} + +// headerFrame is also used to register stream on the client-side. +type headerFrame struct { + streamID uint32 + hf []hpack.HeaderField + endStream bool // Valid on server side. + initStream func(uint32) (bool, error) // Used only on the client side. + onWrite func() + wq *writeQuota // write quota for the stream created. + cleanup *cleanupStream // Valid on the server side. + onOrphaned func(error) // Valid on client-side +} + +type cleanupStream struct { + streamID uint32 + idPtr *uint32 + rst bool + rstCode http2.ErrCode + onWrite func() +} + +type dataFrame struct { + streamID uint32 + endStream bool + h []byte + d []byte + // onEachWrite is called every time + // a part of d is written out. + onEachWrite func() +} + +type incomingWindowUpdate struct { + streamID uint32 + increment uint32 +} + +type outgoingWindowUpdate struct { + streamID uint32 + increment uint32 +} + +type incomingSettings struct { + ss []http2.Setting +} + +type outgoingSettings struct { + ss []http2.Setting +} + +type settingsAck struct { +} + +type incomingGoAway struct { +} + +type goAway struct { + code http2.ErrCode + debugData []byte + headsUp bool + closeConn bool +} + +type ping struct { + ack bool + data [8]byte +} + +type outFlowControlSizeRequest struct { + resp chan uint32 +} + +type outStreamState int + +const ( + active outStreamState = iota + empty + waitingOnStreamQuota +) + +type outStream struct { + id uint32 + state outStreamState + itl *itemList + bytesOutStanding int + wq *writeQuota + + next *outStream + prev *outStream +} + +func (s *outStream) deleteSelf() { + if s.prev != nil { + s.prev.next = s.next + } + if s.next != nil { + s.next.prev = s.prev + } + s.next, s.prev = nil, nil +} + +type outStreamList struct { + // Following are sentinel objects that mark the + // beginning and end of the list. They do not + // contain any item lists. All valid objects are + // inserted in between them. + // This is needed so that an outStream object can + // deleteSelf() in O(1) time without knowing which + // list it belongs to. + head *outStream + tail *outStream +} + +func newOutStreamList() *outStreamList { + head, tail := new(outStream), new(outStream) + head.next = tail + tail.prev = head + return &outStreamList{ + head: head, + tail: tail, + } +} + +func (l *outStreamList) enqueue(s *outStream) { + e := l.tail.prev + e.next = s + s.prev = e + s.next = l.tail + l.tail.prev = s +} + +// remove from the beginning of the list. +func (l *outStreamList) dequeue() *outStream { + b := l.head.next + if b == l.tail { + return nil + } + b.deleteSelf() + return b +} + +type controlBuffer struct { + ch chan struct{} + done <-chan struct{} + mu sync.Mutex + consumerWaiting bool + list *itemList + err error +} + +func newControlBuffer(done <-chan struct{}) *controlBuffer { + return &controlBuffer{ + ch: make(chan struct{}, 1), + list: &itemList{}, + done: done, + } +} + +func (c *controlBuffer) put(it interface{}) error { + _, err := c.executeAndPut(nil, it) + return err +} + +func (c *controlBuffer) executeAndPut(f func(it interface{}) bool, it interface{}) (bool, error) { + var wakeUp bool + c.mu.Lock() + if c.err != nil { + c.mu.Unlock() + return false, c.err + } + if f != nil { + if !f(it) { // f wasn't successful + c.mu.Unlock() + return false, nil + } + } + if c.consumerWaiting { + wakeUp = true + c.consumerWaiting = false + } + c.list.enqueue(it) + c.mu.Unlock() + if wakeUp { + select { + case c.ch <- struct{}{}: + default: + } + } + return true, nil +} + +func (c *controlBuffer) get(block bool) (interface{}, error) { + for { + c.mu.Lock() + if c.err != nil { + c.mu.Unlock() + return nil, c.err + } + if !c.list.isEmpty() { + h := c.list.dequeue() + c.mu.Unlock() + return h, nil + } + if !block { + c.mu.Unlock() + return nil, nil + } + c.consumerWaiting = true + c.mu.Unlock() + select { + case <-c.ch: + case <-c.done: + c.finish() + return nil, ErrConnClosing + } + } +} + +func (c *controlBuffer) finish() { + c.mu.Lock() + if c.err != nil { + c.mu.Unlock() + return + } + c.err = ErrConnClosing + // There may be headers for streams in the control buffer. + // These streams need to be cleaned out since the transport + // is still not aware of these yet. + for head := c.list.dequeueAll(); head != nil; head = head.next { + hdr, ok := head.it.(*headerFrame) + if !ok { + continue + } + if hdr.onOrphaned != nil { // It will be nil on the server-side. + hdr.onOrphaned(ErrConnClosing) + } + } + c.mu.Unlock() +} + +type side int + +const ( + clientSide side = iota + serverSide +) + +type loopyWriter struct { + side side + cbuf *controlBuffer + sendQuota uint32 + oiws uint32 // outbound initial window size. + estdStreams map[uint32]*outStream // Established streams. + activeStreams *outStreamList // Streams that are sending data. + framer *framer + hBuf *bytes.Buffer // The buffer for HPACK encoding. + hEnc *hpack.Encoder // HPACK encoder. + bdpEst *bdpEstimator + draining bool + + // Side-specific handlers + ssGoAwayHandler func(*goAway) (bool, error) +} + +func newLoopyWriter(s side, fr *framer, cbuf *controlBuffer, bdpEst *bdpEstimator) *loopyWriter { + var buf bytes.Buffer + l := &loopyWriter{ + side: s, + cbuf: cbuf, + sendQuota: defaultWindowSize, + oiws: defaultWindowSize, + estdStreams: make(map[uint32]*outStream), + activeStreams: newOutStreamList(), + framer: fr, + hBuf: &buf, + hEnc: hpack.NewEncoder(&buf), + bdpEst: bdpEst, + } + return l +} + +const minBatchSize = 1000 + +// run should be run in a separate goroutine. +func (l *loopyWriter) run() (err error) { + defer func() { + if err == ErrConnClosing { + // Don't log ErrConnClosing as error since it happens + // 1. When the connection is closed by some other known issue. + // 2. User closed the connection. + // 3. A graceful close of connection. + infof("transport: loopyWriter.run returning. %v", err) + err = nil + } + }() + for { + it, err := l.cbuf.get(true) + if err != nil { + return err + } + if err = l.handle(it); err != nil { + return err + } + if _, err = l.processData(); err != nil { + return err + } + gosched := true + hasdata: + for { + it, err := l.cbuf.get(false) + if err != nil { + return err + } + if it != nil { + if err = l.handle(it); err != nil { + return err + } + if _, err = l.processData(); err != nil { + return err + } + continue hasdata + } + isEmpty, err := l.processData() + if err != nil { + return err + } + if !isEmpty { + continue hasdata + } + if gosched { + gosched = false + if l.framer.writer.offset < minBatchSize { + runtime.Gosched() + continue hasdata + } + } + l.framer.writer.Flush() + break hasdata + + } + } +} + +func (l *loopyWriter) outgoingWindowUpdateHandler(w *outgoingWindowUpdate) error { + return l.framer.fr.WriteWindowUpdate(w.streamID, w.increment) +} + +func (l *loopyWriter) incomingWindowUpdateHandler(w *incomingWindowUpdate) error { + // Otherwise update the quota. + if w.streamID == 0 { + l.sendQuota += w.increment + return nil + } + // Find the stream and update it. + if str, ok := l.estdStreams[w.streamID]; ok { + str.bytesOutStanding -= int(w.increment) + if strQuota := int(l.oiws) - str.bytesOutStanding; strQuota > 0 && str.state == waitingOnStreamQuota { + str.state = active + l.activeStreams.enqueue(str) + return nil + } + } + return nil +} + +func (l *loopyWriter) outgoingSettingsHandler(s *outgoingSettings) error { + return l.framer.fr.WriteSettings(s.ss...) +} + +func (l *loopyWriter) incomingSettingsHandler(s *incomingSettings) error { + if err := l.applySettings(s.ss); err != nil { + return err + } + return l.framer.fr.WriteSettingsAck() +} + +func (l *loopyWriter) registerStreamHandler(h *registerStream) error { + str := &outStream{ + id: h.streamID, + state: empty, + itl: &itemList{}, + wq: h.wq, + } + l.estdStreams[h.streamID] = str + return nil +} + +func (l *loopyWriter) headerHandler(h *headerFrame) error { + if l.side == serverSide { + str, ok := l.estdStreams[h.streamID] + if !ok { + warningf("transport: loopy doesn't recognize the stream: %d", h.streamID) + return nil + } + // Case 1.A: Server is responding back with headers. + if !h.endStream { + return l.writeHeader(h.streamID, h.endStream, h.hf, h.onWrite) + } + // else: Case 1.B: Server wants to close stream. + + if str.state != empty { // either active or waiting on stream quota. + // add it str's list of items. + str.itl.enqueue(h) + return nil + } + if err := l.writeHeader(h.streamID, h.endStream, h.hf, h.onWrite); err != nil { + return err + } + return l.cleanupStreamHandler(h.cleanup) + } + // Case 2: Client wants to originate stream. + str := &outStream{ + id: h.streamID, + state: empty, + itl: &itemList{}, + wq: h.wq, + } + str.itl.enqueue(h) + return l.originateStream(str) +} + +func (l *loopyWriter) originateStream(str *outStream) error { + hdr := str.itl.dequeue().(*headerFrame) + sendPing, err := hdr.initStream(str.id) + if err != nil { + if err == ErrConnClosing { + return err + } + // Other errors(errStreamDrain) need not close transport. + return nil + } + if err = l.writeHeader(str.id, hdr.endStream, hdr.hf, hdr.onWrite); err != nil { + return err + } + l.estdStreams[str.id] = str + if sendPing { + return l.pingHandler(&ping{data: [8]byte{}}) + } + return nil +} + +func (l *loopyWriter) writeHeader(streamID uint32, endStream bool, hf []hpack.HeaderField, onWrite func()) error { + if onWrite != nil { + onWrite() + } + l.hBuf.Reset() + for _, f := range hf { + if err := l.hEnc.WriteField(f); err != nil { + warningf("transport: loopyWriter.writeHeader encountered error while encoding headers:", err) + } + } + var ( + err error + endHeaders, first bool + ) + first = true + for !endHeaders { + size := l.hBuf.Len() + if size > http2MaxFrameLen { + size = http2MaxFrameLen + } else { + endHeaders = true + } + if first { + first = false + err = l.framer.fr.WriteHeaders(http2.HeadersFrameParam{ + StreamID: streamID, + BlockFragment: l.hBuf.Next(size), + EndStream: endStream, + EndHeaders: endHeaders, + }) + } else { + err = l.framer.fr.WriteContinuation( + streamID, + endHeaders, + l.hBuf.Next(size), + ) + } + if err != nil { + return err + } + } + return nil +} + +func (l *loopyWriter) preprocessData(df *dataFrame) error { + str, ok := l.estdStreams[df.streamID] + if !ok { + return nil + } + // If we got data for a stream it means that + // stream was originated and the headers were sent out. + str.itl.enqueue(df) + if str.state == empty { + str.state = active + l.activeStreams.enqueue(str) + } + return nil +} + +func (l *loopyWriter) pingHandler(p *ping) error { + if !p.ack { + l.bdpEst.timesnap(p.data) + } + return l.framer.fr.WritePing(p.ack, p.data) + +} + +func (l *loopyWriter) outFlowControlSizeRequestHandler(o *outFlowControlSizeRequest) error { + o.resp <- l.sendQuota + return nil +} + +func (l *loopyWriter) cleanupStreamHandler(c *cleanupStream) error { + c.onWrite() + if str, ok := l.estdStreams[c.streamID]; ok { + // On the server side it could be a trailers-only response or + // a RST_STREAM before stream initialization thus the stream might + // not be established yet. + delete(l.estdStreams, c.streamID) + str.deleteSelf() + } + if c.rst { // If RST_STREAM needs to be sent. + if err := l.framer.fr.WriteRSTStream(c.streamID, c.rstCode); err != nil { + return err + } + } + if l.side == clientSide && l.draining && len(l.estdStreams) == 0 { + return ErrConnClosing + } + return nil +} + +func (l *loopyWriter) incomingGoAwayHandler(*incomingGoAway) error { + if l.side == clientSide { + l.draining = true + if len(l.estdStreams) == 0 { + return ErrConnClosing + } + } + return nil +} + +func (l *loopyWriter) goAwayHandler(g *goAway) error { + // Handling of outgoing GoAway is very specific to side. + if l.ssGoAwayHandler != nil { + draining, err := l.ssGoAwayHandler(g) + if err != nil { + return err + } + l.draining = draining + } + return nil +} + +func (l *loopyWriter) handle(i interface{}) error { + switch i := i.(type) { + case *incomingWindowUpdate: + return l.incomingWindowUpdateHandler(i) + case *outgoingWindowUpdate: + return l.outgoingWindowUpdateHandler(i) + case *incomingSettings: + return l.incomingSettingsHandler(i) + case *outgoingSettings: + return l.outgoingSettingsHandler(i) + case *headerFrame: + return l.headerHandler(i) + case *registerStream: + return l.registerStreamHandler(i) + case *cleanupStream: + return l.cleanupStreamHandler(i) + case *incomingGoAway: + return l.incomingGoAwayHandler(i) + case *dataFrame: + return l.preprocessData(i) + case *ping: + return l.pingHandler(i) + case *goAway: + return l.goAwayHandler(i) + case *outFlowControlSizeRequest: + return l.outFlowControlSizeRequestHandler(i) + default: + return fmt.Errorf("transport: unknown control message type %T", i) + } +} + +func (l *loopyWriter) applySettings(ss []http2.Setting) error { + for _, s := range ss { + switch s.ID { + case http2.SettingInitialWindowSize: + o := l.oiws + l.oiws = s.Val + if o < l.oiws { + // If the new limit is greater make all depleted streams active. + for _, stream := range l.estdStreams { + if stream.state == waitingOnStreamQuota { + stream.state = active + l.activeStreams.enqueue(stream) + } + } + } + case http2.SettingHeaderTableSize: + updateHeaderTblSize(l.hEnc, s.Val) + } + } + return nil +} + +func (l *loopyWriter) processData() (bool, error) { + if l.sendQuota == 0 { + return true, nil + } + str := l.activeStreams.dequeue() + if str == nil { + return true, nil + } + dataItem := str.itl.peek().(*dataFrame) + if len(dataItem.h) == 0 && len(dataItem.d) == 0 { + // Client sends out empty data frame with endStream = true + if err := l.framer.fr.WriteData(dataItem.streamID, dataItem.endStream, nil); err != nil { + return false, err + } + str.itl.dequeue() + if str.itl.isEmpty() { + str.state = empty + } else if trailer, ok := str.itl.peek().(*headerFrame); ok { // the next item is trailers. + if err := l.writeHeader(trailer.streamID, trailer.endStream, trailer.hf, trailer.onWrite); err != nil { + return false, err + } + if err := l.cleanupStreamHandler(trailer.cleanup); err != nil { + return false, nil + } + } else { + l.activeStreams.enqueue(str) + } + return false, nil + } + var ( + idx int + buf []byte + ) + if len(dataItem.h) != 0 { // data header has not been written out yet. + buf = dataItem.h + } else { + idx = 1 + buf = dataItem.d + } + size := http2MaxFrameLen + if len(buf) < size { + size = len(buf) + } + if strQuota := int(l.oiws) - str.bytesOutStanding; strQuota <= 0 { + str.state = waitingOnStreamQuota + return false, nil + } else if strQuota < size { + size = strQuota + } + + if l.sendQuota < uint32(size) { + size = int(l.sendQuota) + } + // Now that outgoing flow controls are checked we can replenish str's write quota + str.wq.replenish(size) + var endStream bool + // This last data message on this stream and all + // of it can be written in this go. + if dataItem.endStream && size == len(buf) { + // buf contains either data or it contains header but data is empty. + if idx == 1 || len(dataItem.d) == 0 { + endStream = true + } + } + if dataItem.onEachWrite != nil { + dataItem.onEachWrite() + } + if err := l.framer.fr.WriteData(dataItem.streamID, endStream, buf[:size]); err != nil { + return false, err + } + buf = buf[size:] + str.bytesOutStanding += size + l.sendQuota -= uint32(size) + if idx == 0 { + dataItem.h = buf + } else { + dataItem.d = buf + } + + if len(dataItem.h) == 0 && len(dataItem.d) == 0 { // All the data from that message was written out. + str.itl.dequeue() + } + if str.itl.isEmpty() { + str.state = empty + } else if trailer, ok := str.itl.peek().(*headerFrame); ok { // The next item is trailers. + if err := l.writeHeader(trailer.streamID, trailer.endStream, trailer.hf, trailer.onWrite); err != nil { + return false, err + } + if err := l.cleanupStreamHandler(trailer.cleanup); err != nil { + return false, err + } + } else if int(l.oiws)-str.bytesOutStanding <= 0 { // Ran out of stream quota. + str.state = waitingOnStreamQuota + } else { // Otherwise add it back to the list of active streams. + l.activeStreams.enqueue(str) + } + return false, nil +} diff --git a/vendor/google.golang.org/grpc/transport/control.go b/vendor/google.golang.org/grpc/transport/flowcontrol.go similarity index 51% rename from vendor/google.golang.org/grpc/transport/control.go rename to vendor/google.golang.org/grpc/transport/flowcontrol.go index dd1a8d42e7e..bbf98b6f5ee 100644 --- a/vendor/google.golang.org/grpc/transport/control.go +++ b/vendor/google.golang.org/grpc/transport/flowcontrol.go @@ -24,9 +24,6 @@ import ( "sync" "sync/atomic" "time" - - "golang.org/x/net/http2" - "golang.org/x/net/http2/hpack" ) const ( @@ -36,179 +33,115 @@ const ( initialWindowSize = defaultWindowSize // for an RPC infinity = time.Duration(math.MaxInt64) defaultClientKeepaliveTime = infinity - defaultClientKeepaliveTimeout = time.Duration(20 * time.Second) + defaultClientKeepaliveTimeout = 20 * time.Second defaultMaxStreamsClient = 100 defaultMaxConnectionIdle = infinity defaultMaxConnectionAge = infinity defaultMaxConnectionAgeGrace = infinity - defaultServerKeepaliveTime = time.Duration(2 * time.Hour) - defaultServerKeepaliveTimeout = time.Duration(20 * time.Second) - defaultKeepalivePolicyMinTime = time.Duration(5 * time.Minute) + defaultServerKeepaliveTime = 2 * time.Hour + defaultServerKeepaliveTimeout = 20 * time.Second + defaultKeepalivePolicyMinTime = 5 * time.Minute // max window limit set by HTTP2 Specs. maxWindowSize = math.MaxInt32 - // defaultLocalSendQuota sets is default value for number of data + // defaultWriteQuota is the default value for number of data // bytes that each stream can schedule before some of it being // flushed out. - defaultLocalSendQuota = 64 * 1024 + defaultWriteQuota = 64 * 1024 ) -// The following defines various control items which could flow through -// the control buffer of transport. They represent different aspects of -// control tasks, e.g., flow control, settings, streaming resetting, etc. - -type headerFrame struct { - streamID uint32 - hf []hpack.HeaderField - endStream bool +// writeQuota is a soft limit on the amount of data a stream can +// schedule before some of it is written out. +type writeQuota struct { + quota int32 + // get waits on read from when quota goes less than or equal to zero. + // replenish writes on it when quota goes positive again. + ch chan struct{} + // done is triggered in error case. + done <-chan struct{} + // replenish is called by loopyWriter to give quota back to. + // It is implemented as a field so that it can be updated + // by tests. + replenish func(n int) } -func (*headerFrame) item() {} - -type continuationFrame struct { - streamID uint32 - endHeaders bool - headerBlockFragment []byte -} - -type dataFrame struct { - streamID uint32 - endStream bool - d []byte - f func() -} - -func (*dataFrame) item() {} - -func (*continuationFrame) item() {} - -type windowUpdate struct { - streamID uint32 - increment uint32 -} - -func (*windowUpdate) item() {} - -type settings struct { - ack bool - ss []http2.Setting -} - -func (*settings) item() {} - -type resetStream struct { - streamID uint32 - code http2.ErrCode -} - -func (*resetStream) item() {} - -type goAway struct { - code http2.ErrCode - debugData []byte - headsUp bool - closeConn bool -} - -func (*goAway) item() {} - -type flushIO struct { -} - -func (*flushIO) item() {} - -type ping struct { - ack bool - data [8]byte -} - -func (*ping) item() {} - -// quotaPool is a pool which accumulates the quota and sends it to acquire() -// when it is available. -type quotaPool struct { - c chan int - - mu sync.Mutex - version uint32 - quota int -} - -// newQuotaPool creates a quotaPool which has quota q available to consume. -func newQuotaPool(q int) *quotaPool { - qb := "aPool{ - c: make(chan int, 1), +func newWriteQuota(sz int32, done <-chan struct{}) *writeQuota { + w := &writeQuota{ + quota: sz, + ch: make(chan struct{}, 1), + done: done, } - if q > 0 { - qb.c <- q - } else { - qb.quota = q - } - return qb + w.replenish = w.realReplenish + return w } -// add cancels the pending quota sent on acquired, incremented by v and sends -// it back on acquire. -func (qb *quotaPool) add(v int) { - qb.mu.Lock() - defer qb.mu.Unlock() - qb.lockedAdd(v) -} - -func (qb *quotaPool) lockedAdd(v int) { - select { - case n := <-qb.c: - qb.quota += n - default: - } - qb.quota += v - if qb.quota <= 0 { - return - } - // After the pool has been created, this is the only place that sends on - // the channel. Since mu is held at this point and any quota that was sent - // on the channel has been retrieved, we know that this code will always - // place any positive quota value on the channel. - select { - case qb.c <- qb.quota: - qb.quota = 0 - default: +func (w *writeQuota) get(sz int32) error { + for { + if atomic.LoadInt32(&w.quota) > 0 { + atomic.AddInt32(&w.quota, -sz) + return nil + } + select { + case <-w.ch: + continue + case <-w.done: + return errStreamDone + } } } -func (qb *quotaPool) addAndUpdate(v int) { - qb.mu.Lock() - defer qb.mu.Unlock() - qb.lockedAdd(v) - // Update the version only after having added to the quota - // so that if acquireWithVesrion sees the new vesrion it is - // guaranteed to have seen the updated quota. - // Also, still keep this inside of the lock, so that when - // compareAndExecute is processing, this function doesn't - // get executed partially (quota gets updated but the version - // doesn't). - atomic.AddUint32(&(qb.version), 1) -} - -func (qb *quotaPool) acquireWithVersion() (<-chan int, uint32) { - return qb.c, atomic.LoadUint32(&(qb.version)) -} - -func (qb *quotaPool) compareAndExecute(version uint32, success, failure func()) bool { - qb.mu.Lock() - defer qb.mu.Unlock() - if version == atomic.LoadUint32(&(qb.version)) { - success() - return true +func (w *writeQuota) realReplenish(n int) { + sz := int32(n) + a := atomic.AddInt32(&w.quota, sz) + b := a - sz + if b <= 0 && a > 0 { + select { + case w.ch <- struct{}{}: + default: + } } - failure() - return false } -// acquire returns the channel on which available quota amounts are sent. -func (qb *quotaPool) acquire() <-chan int { - return qb.c +type trInFlow struct { + limit uint32 + unacked uint32 + effectiveWindowSize uint32 } +func (f *trInFlow) newLimit(n uint32) uint32 { + d := n - f.limit + f.limit = n + f.updateEffectiveWindowSize() + return d +} + +func (f *trInFlow) onData(n uint32) uint32 { + f.unacked += n + if f.unacked >= f.limit/4 { + w := f.unacked + f.unacked = 0 + f.updateEffectiveWindowSize() + return w + } + f.updateEffectiveWindowSize() + return 0 +} + +func (f *trInFlow) reset() uint32 { + w := f.unacked + f.unacked = 0 + f.updateEffectiveWindowSize() + return w +} + +func (f *trInFlow) updateEffectiveWindowSize() { + atomic.StoreUint32(&f.effectiveWindowSize, f.limit-f.unacked) +} + +func (f *trInFlow) getSize() uint32 { + return atomic.LoadUint32(&f.effectiveWindowSize) +} + +// TODO(mmukhi): Simplify this code. // inFlow deals with inbound flow control type inFlow struct { mu sync.Mutex @@ -229,9 +162,9 @@ type inFlow struct { // It assumes that n is always greater than the old limit. func (f *inFlow) newLimit(n uint32) uint32 { f.mu.Lock() - defer f.mu.Unlock() d := n - f.limit f.limit = n + f.mu.Unlock() return d } @@ -240,7 +173,6 @@ func (f *inFlow) maybeAdjust(n uint32) uint32 { n = uint32(math.MaxInt32) } f.mu.Lock() - defer f.mu.Unlock() // estSenderQuota is the receiver's view of the maximum number of bytes the sender // can send without a window update. estSenderQuota := int32(f.limit - (f.pendingData + f.pendingUpdate)) @@ -252,7 +184,7 @@ func (f *inFlow) maybeAdjust(n uint32) uint32 { // for this message. Therefore we must send an update over the limit since there's an active read // request from the application. if estUntransmittedData > estSenderQuota { - // Sender's window shouldn't go more than 2^31 - 1 as speecified in the HTTP spec. + // Sender's window shouldn't go more than 2^31 - 1 as specified in the HTTP spec. if f.limit+n > maxWindowSize { f.delta = maxWindowSize - f.limit } else { @@ -261,19 +193,24 @@ func (f *inFlow) maybeAdjust(n uint32) uint32 { // is padded; We will fallback on the current available window(at least a 1/4th of the limit). f.delta = n } + f.mu.Unlock() return f.delta } + f.mu.Unlock() return 0 } // onData is invoked when some data frame is received. It updates pendingData. func (f *inFlow) onData(n uint32) error { f.mu.Lock() - defer f.mu.Unlock() f.pendingData += n if f.pendingData+f.pendingUpdate > f.limit+f.delta { - return fmt.Errorf("received %d-bytes data exceeding the limit %d bytes", f.pendingData+f.pendingUpdate, f.limit) + limit := f.limit + rcvd := f.pendingData + f.pendingUpdate + f.mu.Unlock() + return fmt.Errorf("received %d-bytes data exceeding the limit %d bytes", rcvd, limit) } + f.mu.Unlock() return nil } @@ -281,8 +218,8 @@ func (f *inFlow) onData(n uint32) error { // to be sent to the peer. func (f *inFlow) onRead(n uint32) uint32 { f.mu.Lock() - defer f.mu.Unlock() if f.pendingData == 0 { + f.mu.Unlock() return 0 } f.pendingData -= n @@ -297,15 +234,9 @@ func (f *inFlow) onRead(n uint32) uint32 { if f.pendingUpdate >= f.limit/4 { wu := f.pendingUpdate f.pendingUpdate = 0 + f.mu.Unlock() return wu } + f.mu.Unlock() return 0 } - -func (f *inFlow) resetPendingUpdate() uint32 { - f.mu.Lock() - defer f.mu.Unlock() - n := f.pendingUpdate - f.pendingUpdate = 0 - return n -} diff --git a/vendor/google.golang.org/grpc/transport/go16.go b/vendor/google.golang.org/grpc/transport/go16.go new file mode 100644 index 00000000000..5babcf9b877 --- /dev/null +++ b/vendor/google.golang.org/grpc/transport/go16.go @@ -0,0 +1,51 @@ +// +build go1.6,!go1.7 + +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "net" + "net/http" + + "google.golang.org/grpc/codes" + + "golang.org/x/net/context" +) + +// dialContext connects to the address on the named network. +func dialContext(ctx context.Context, network, address string) (net.Conn, error) { + return (&net.Dialer{Cancel: ctx.Done()}).Dial(network, address) +} + +// ContextErr converts the error from context package into a StreamError. +func ContextErr(err error) StreamError { + switch err { + case context.DeadlineExceeded: + return streamErrorf(codes.DeadlineExceeded, "%v", err) + case context.Canceled: + return streamErrorf(codes.Canceled, "%v", err) + } + return streamErrorf(codes.Internal, "Unexpected error from context packet: %v", err) +} + +// contextFromRequest returns a background context. +func contextFromRequest(r *http.Request) context.Context { + return context.Background() +} diff --git a/vendor/google.golang.org/grpc/transport/go17.go b/vendor/google.golang.org/grpc/transport/go17.go new file mode 100644 index 00000000000..b7fa6bdb9ca --- /dev/null +++ b/vendor/google.golang.org/grpc/transport/go17.go @@ -0,0 +1,52 @@ +// +build go1.7 + +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "context" + "net" + "net/http" + + "google.golang.org/grpc/codes" + + netctx "golang.org/x/net/context" +) + +// dialContext connects to the address on the named network. +func dialContext(ctx context.Context, network, address string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, address) +} + +// ContextErr converts the error from context package into a StreamError. +func ContextErr(err error) StreamError { + switch err { + case context.DeadlineExceeded, netctx.DeadlineExceeded: + return streamErrorf(codes.DeadlineExceeded, "%v", err) + case context.Canceled, netctx.Canceled: + return streamErrorf(codes.Canceled, "%v", err) + } + return streamErrorf(codes.Internal, "Unexpected error from context packet: %v", err) +} + +// contextFromRequest returns a context from the HTTP Request. +func contextFromRequest(r *http.Request) context.Context { + return r.Context() +} diff --git a/vendor/google.golang.org/grpc/transport/handler_server.go b/vendor/google.golang.org/grpc/transport/handler_server.go index 7e0fdb35938..f71b7482174 100644 --- a/vendor/google.golang.org/grpc/transport/handler_server.go +++ b/vendor/google.golang.org/grpc/transport/handler_server.go @@ -40,20 +40,24 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" + "google.golang.org/grpc/stats" "google.golang.org/grpc/status" ) // NewServerHandlerTransport returns a ServerTransport handling gRPC // from inside an http.Handler. It requires that the http Server // supports HTTP/2. -func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request) (ServerTransport, error) { +func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request, stats stats.Handler) (ServerTransport, error) { if r.ProtoMajor != 2 { return nil, errors.New("gRPC requires HTTP/2") } if r.Method != "POST" { return nil, errors.New("invalid gRPC request method") } - if !validContentType(r.Header.Get("Content-Type")) { + contentType := r.Header.Get("Content-Type") + // TODO: do we assume contentType is lowercase? we did before + contentSubtype, validContentType := contentSubtype(contentType) + if !validContentType { return nil, errors.New("invalid gRPC request content-type") } if _, ok := w.(http.Flusher); !ok { @@ -64,10 +68,13 @@ func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request) (ServerTr } st := &serverHandlerTransport{ - rw: w, - req: r, - closedCh: make(chan struct{}), - writes: make(chan func()), + rw: w, + req: r, + closedCh: make(chan struct{}), + writes: make(chan func()), + contentType: contentType, + contentSubtype: contentSubtype, + stats: stats, } if v := r.Header.Get("grpc-timeout"); v != "" { @@ -79,19 +86,19 @@ func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request) (ServerTr st.timeout = to } - var metakv []string + metakv := []string{"content-type", contentType} if r.Host != "" { metakv = append(metakv, ":authority", r.Host) } for k, vv := range r.Header { k = strings.ToLower(k) - if isReservedHeader(k) && !isWhitelistedPseudoHeader(k) { + if isReservedHeader(k) && !isWhitelistedHeader(k) { continue } for _, v := range vv { v, err := decodeMetadataHeader(k, v) if err != nil { - return nil, streamErrorf(codes.InvalidArgument, "malformed binary metadata: %v", err) + return nil, streamErrorf(codes.Internal, "malformed binary metadata: %v", err) } metakv = append(metakv, k, v) } @@ -126,6 +133,14 @@ type serverHandlerTransport struct { // block concurrent WriteStatus calls // e.g. grpc/(*serverStream).SendMsg/RecvMsg writeStatusMu sync.Mutex + + // we just mirror the request content-type + contentType string + // we store both contentType and contentSubtype so we don't keep recreating them + // TODO make sure this is consistent across handler_server and http2_server + contentSubtype string + + stats stats.Handler } func (ht *serverHandlerTransport) Close() error { @@ -219,6 +234,9 @@ func (ht *serverHandlerTransport) WriteStatus(s *Stream, st *status.Status) erro }) if err == nil { // transport has not been closed + if ht.stats != nil { + ht.stats.HandleRPC(s.Context(), &stats.OutTrailer{}) + } ht.Close() close(ht.writes) } @@ -235,7 +253,7 @@ func (ht *serverHandlerTransport) writeCommonHeaders(s *Stream) { h := ht.rw.Header() h["Date"] = nil // suppress Date to make tests happy; TODO: restore - h.Set("Content-Type", "application/grpc") + h.Set("Content-Type", ht.contentType) // Predeclare trailers we'll set later in WriteStatus (after the body). // This is a SHOULD in the HTTP RFC, and the way you add (known) @@ -263,7 +281,7 @@ func (ht *serverHandlerTransport) Write(s *Stream, hdr []byte, data []byte, opts } func (ht *serverHandlerTransport) WriteHeader(s *Stream, md metadata.MD) error { - return ht.do(func() { + err := ht.do(func() { ht.writeCommonHeaders(s) h := ht.rw.Header() for k, vv := range md { @@ -279,17 +297,24 @@ func (ht *serverHandlerTransport) WriteHeader(s *Stream, md metadata.MD) error { ht.rw.WriteHeader(200) ht.rw.(http.Flusher).Flush() }) + + if err == nil { + if ht.stats != nil { + ht.stats.HandleRPC(s.Context(), &stats.OutHeader{}) + } + } + return err } func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), traceCtx func(context.Context, string) context.Context) { // With this transport type there will be exactly 1 stream: this HTTP request. - var ctx context.Context + ctx := contextFromRequest(ht.req) var cancel context.CancelFunc if ht.timeoutSet { - ctx, cancel = context.WithTimeout(context.Background(), ht.timeout) + ctx, cancel = context.WithTimeout(ctx, ht.timeout) } else { - ctx, cancel = context.WithCancel(context.Background()) + ctx, cancel = context.WithCancel(ctx) } // requestOver is closed when either the request's context is done @@ -313,13 +338,14 @@ func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), trace req := ht.req s := &Stream{ - id: 0, // irrelevant - requestRead: func(int) {}, - cancel: cancel, - buf: newRecvBuffer(), - st: ht, - method: req.URL.Path, - recvCompress: req.Header.Get("grpc-encoding"), + id: 0, // irrelevant + requestRead: func(int) {}, + cancel: cancel, + buf: newRecvBuffer(), + st: ht, + method: req.URL.Path, + recvCompress: req.Header.Get("grpc-encoding"), + contentSubtype: ht.contentSubtype, } pr := &peer.Peer{ Addr: ht.RemoteAddr(), @@ -328,10 +354,18 @@ func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), trace pr.AuthInfo = credentials.TLSInfo{State: *req.TLS} } ctx = metadata.NewIncomingContext(ctx, ht.headerMD) - ctx = peer.NewContext(ctx, pr) - s.ctx = newContextWithStream(ctx, s) + s.ctx = peer.NewContext(ctx, pr) + if ht.stats != nil { + s.ctx = ht.stats.TagRPC(s.ctx, &stats.RPCTagInfo{FullMethodName: s.method}) + inHeader := &stats.InHeader{ + FullMethod: s.method, + RemoteAddr: ht.RemoteAddr(), + Compression: s.recvCompress, + } + ht.stats.HandleRPC(s.ctx, inHeader) + } s.trReader = &transportReader{ - reader: &recvBufferReader{ctx: s.ctx, recv: s.buf}, + reader: &recvBufferReader{ctx: s.ctx, ctxDone: s.ctx.Done(), recv: s.buf}, windowHandler: func(int) {}, } @@ -386,6 +420,10 @@ func (ht *serverHandlerTransport) runStream() { } } +func (ht *serverHandlerTransport) IncrMsgSent() {} + +func (ht *serverHandlerTransport) IncrMsgRecv() {} + func (ht *serverHandlerTransport) Drain() { panic("Drain() is not implemented") } diff --git a/vendor/google.golang.org/grpc/transport/http2_client.go b/vendor/google.golang.org/grpc/transport/http2_client.go index 1abb62e6df4..eaf007eb0ae 100644 --- a/vendor/google.golang.org/grpc/transport/http2_client.go +++ b/vendor/google.golang.org/grpc/transport/http2_client.go @@ -19,7 +19,6 @@ package transport import ( - "bytes" "io" "math" "net" @@ -31,8 +30,10 @@ import ( "golang.org/x/net/context" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" @@ -44,15 +45,17 @@ import ( type http2Client struct { ctx context.Context cancel context.CancelFunc - target string // server name/addr + ctxDone <-chan struct{} // Cache the ctx.Done() chan. userAgent string md interface{} conn net.Conn // underlying communication channel + loopy *loopyWriter remoteAddr net.Addr localAddr net.Addr authInfo credentials.AuthInfo // auth info about the connection - nextID uint32 // the next stream ID to be used + readerDone chan struct{} // sync point to enable testing. + writerDone chan struct{} // sync point to enable testing. // goAway is closed to notify the upper layer (i.e., addrConn.transportMonitor) // that the server sent GoAway on this transport. goAway chan struct{} @@ -60,18 +63,10 @@ type http2Client struct { awakenKeepalive chan struct{} framer *framer - hBuf *bytes.Buffer // the buffer for HPACK encoding - hEnc *hpack.Encoder // HPACK encoder - // controlBuf delivers all the control related tasks (e.g., window // updates, reset streams, and various settings) to the controller. controlBuf *controlBuffer - fc *inFlow - // sendQuotaPool provides flow control to outbound message. - sendQuotaPool *quotaPool - // streamsQuota limits the max number of concurrent streams. - streamsQuota *quotaPool - + fc *trInFlow // The scheme used: https if TLS is on, http otherwise. scheme string @@ -81,50 +76,60 @@ type http2Client struct { // Boolean to keep track of reading activity on transport. // 1 is true and 0 is false. - activity uint32 // Accessed atomically. - kp keepalive.ClientParameters + activity uint32 // Accessed atomically. + kp keepalive.ClientParameters + keepaliveEnabled bool statsHandler stats.Handler initialWindowSize int32 - bdpEst *bdpEstimator - outQuotaVersion uint32 + bdpEst *bdpEstimator + // onSuccess is a callback that client transport calls upon + // receiving server preface to signal that a succefull HTTP2 + // connection was established. + onSuccess func() - mu sync.Mutex // guard the following variables - state transportState // the state of underlying connection + maxConcurrentStreams uint32 + streamQuota int64 + streamsQuotaAvailable chan struct{} + waitingStreams uint32 + nextID uint32 + + mu sync.Mutex // guard the following variables + state transportState activeStreams map[uint32]*Stream - // The max number of concurrent streams - maxStreams int - // the per-stream outbound flow control window size set by the peer. - streamSendQuota uint32 // prevGoAway ID records the Last-Stream-ID in the previous GOAway frame. prevGoAwayID uint32 // goAwayReason records the http2.ErrCode and debug data received with the // GoAway frame. goAwayReason GoAwayReason + + // Fields below are for channelz metric collection. + channelzID int64 // channelz unique identification number + czmu sync.RWMutex + kpCount int64 + // The number of streams that have started, including already finished ones. + streamsStarted int64 + // The number of streams that have ended successfully by receiving EoS bit set + // frame from server. + streamsSucceeded int64 + streamsFailed int64 + lastStreamCreated time.Time + msgSent int64 + msgRecv int64 + lastMsgSent time.Time + lastMsgRecv time.Time } func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error), addr string) (net.Conn, error) { if fn != nil { return fn(ctx, addr) } - return (&net.Dialer{}).DialContext(ctx, "tcp", addr) + return dialContext(ctx, "tcp", addr) } func isTemporary(err error) bool { - switch err { - case io.EOF: - // Connection closures may be resolved upon retry, and are thus - // treated as temporary. - return true - case context.DeadlineExceeded: - // In Go 1.7, context.DeadlineExceeded implements Timeout(), and this - // special case is not needed. Until then, we need to keep this - // clause. - return true - } - switch err := err.(type) { case interface { Temporary() bool @@ -137,18 +142,16 @@ func isTemporary(err error) bool { // temporary. return err.Timeout() } - return false + return true } // newHTTP2Client constructs a connected ClientTransport to addr based on HTTP2 // and starts to receive messages on it. Non-nil error returns if construction // fails. -func newHTTP2Client(ctx context.Context, addr TargetInfo, opts ConnectOptions, timeout time.Duration) (_ ClientTransport, err error) { +func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts ConnectOptions, onSuccess func()) (_ ClientTransport, err error) { scheme := "http" ctx, cancel := context.WithCancel(ctx) - connectCtx, connectCancel := context.WithTimeout(ctx, timeout) defer func() { - connectCancel() if err != nil { cancel() } @@ -173,12 +176,9 @@ func newHTTP2Client(ctx context.Context, addr TargetInfo, opts ConnectOptions, t ) if creds := opts.TransportCredentials; creds != nil { scheme = "https" - conn, authInfo, err = creds.ClientHandshake(connectCtx, addr.Addr, conn) + conn, authInfo, err = creds.ClientHandshake(connectCtx, addr.Authority, conn) if err != nil { - // Credentials handshake errors are typically considered permanent - // to avoid retrying on e.g. bad certificates. - temp := isTemporary(err) - return nil, connectionErrorf(temp, err, "transport: authentication handshake failed: %v", err) + return nil, connectionErrorf(isTemporary(err), err, "transport: authentication handshake failed: %v", err) } isSecure = true } @@ -196,7 +196,6 @@ func newHTTP2Client(ctx context.Context, addr TargetInfo, opts ConnectOptions, t icwz = opts.InitialConnWindowSize dynamicWindow = false } - var buf bytes.Buffer writeBufSize := defaultWriteBufSize if opts.WriteBufferSize > 0 { writeBufSize = opts.WriteBufferSize @@ -206,37 +205,35 @@ func newHTTP2Client(ctx context.Context, addr TargetInfo, opts ConnectOptions, t readBufSize = opts.ReadBufferSize } t := &http2Client{ - ctx: ctx, - cancel: cancel, - target: addr.Addr, - userAgent: opts.UserAgent, - md: addr.Metadata, - conn: conn, - remoteAddr: conn.RemoteAddr(), - localAddr: conn.LocalAddr(), - authInfo: authInfo, - // The client initiated stream id is odd starting from 1. - nextID: 1, - goAway: make(chan struct{}), - awakenKeepalive: make(chan struct{}, 1), - hBuf: &buf, - hEnc: hpack.NewEncoder(&buf), - framer: newFramer(conn, writeBufSize, readBufSize), - controlBuf: newControlBuffer(), - fc: &inFlow{limit: uint32(icwz)}, - sendQuotaPool: newQuotaPool(defaultWindowSize), - scheme: scheme, - state: reachable, - activeStreams: make(map[uint32]*Stream), - isSecure: isSecure, - creds: opts.PerRPCCredentials, - maxStreams: defaultMaxStreamsClient, - streamsQuota: newQuotaPool(defaultMaxStreamsClient), - streamSendQuota: defaultWindowSize, - kp: kp, - statsHandler: opts.StatsHandler, - initialWindowSize: initialWindowSize, + ctx: ctx, + ctxDone: ctx.Done(), // Cache Done chan. + cancel: cancel, + userAgent: opts.UserAgent, + md: addr.Metadata, + conn: conn, + remoteAddr: conn.RemoteAddr(), + localAddr: conn.LocalAddr(), + authInfo: authInfo, + readerDone: make(chan struct{}), + writerDone: make(chan struct{}), + goAway: make(chan struct{}), + awakenKeepalive: make(chan struct{}, 1), + framer: newFramer(conn, writeBufSize, readBufSize), + fc: &trInFlow{limit: uint32(icwz)}, + scheme: scheme, + activeStreams: make(map[uint32]*Stream), + isSecure: isSecure, + creds: opts.PerRPCCredentials, + kp: kp, + statsHandler: opts.StatsHandler, + initialWindowSize: initialWindowSize, + onSuccess: onSuccess, + nextID: 1, + maxConcurrentStreams: defaultMaxStreamsClient, + streamQuota: defaultMaxStreamsClient, + streamsQuotaAvailable: make(chan struct{}, 1), } + t.controlBuf = newControlBuffer(t.ctxDone) if opts.InitialWindowSize >= defaultWindowSize { t.initialWindowSize = opts.InitialWindowSize dynamicWindow = false @@ -260,6 +257,13 @@ func newHTTP2Client(ctx context.Context, addr TargetInfo, opts ConnectOptions, t } t.statsHandler.HandleConn(t.ctx, connBegin) } + if channelz.IsOn() { + t.channelzID = channelz.RegisterNormalSocket(t, opts.ChannelzParentID, "") + } + if t.kp.Time != infinity { + t.keepaliveEnabled = true + go t.keepalive() + } // Start the reader goroutine for incoming message. Each transport has // a dedicated goroutine which reads HTTP2 frame from network. Then it // dispatches the frame to the corresponding stream entity. @@ -295,30 +299,32 @@ func newHTTP2Client(ctx context.Context, addr TargetInfo, opts ConnectOptions, t } t.framer.writer.Flush() go func() { - loopyWriter(t.ctx, t.controlBuf, t.itemHandler) - t.Close() + t.loopy = newLoopyWriter(clientSide, t.framer, t.controlBuf, t.bdpEst) + err := t.loopy.run() + if err != nil { + errorf("transport: loopyWriter.run returning. Err: %v", err) + } + // If it's a connection error, let reader goroutine handle it + // since there might be data in the buffers. + if _, ok := err.(net.Error); !ok { + t.conn.Close() + } + close(t.writerDone) }() - if t.kp.Time != infinity { - go t.keepalive() - } return t, nil } func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream { // TODO(zhaoq): Handle uint32 overflow of Stream.id. s := &Stream{ - id: t.nextID, done: make(chan struct{}), - goAway: make(chan struct{}), method: callHdr.Method, sendCompress: callHdr.SendCompress, buf: newRecvBuffer(), - fc: &inFlow{limit: uint32(t.initialWindowSize)}, - sendQuotaPool: newQuotaPool(int(t.streamSendQuota)), - localSendQuota: newQuotaPool(defaultLocalSendQuota), headerChan: make(chan struct{}), + contentSubtype: callHdr.ContentSubtype, } - t.nextID += 2 + s.wq = newWriteQuota(defaultWriteQuota, s.done) s.requestRead = func(n int) { t.adjustWindow(s, uint32(n)) } @@ -328,21 +334,18 @@ func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream { s.ctx = ctx s.trReader = &transportReader{ reader: &recvBufferReader{ - ctx: s.ctx, - goAway: s.goAway, - recv: s.buf, + ctx: s.ctx, + ctxDone: s.ctx.Done(), + recv: s.buf, }, windowHandler: func(n int) { t.updateWindow(s, uint32(n)) }, } - return s } -// NewStream creates a stream and registers it into the transport as "active" -// streams. -func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Stream, err error) { +func (t *http2Client) getPeer() *peer.Peer { pr := &peer.Peer{ Addr: t.remoteAddr, } @@ -350,74 +353,20 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea if t.authInfo != nil { pr.AuthInfo = t.authInfo } - ctx = peer.NewContext(ctx, pr) - var ( - authData = make(map[string]string) - audience string - ) - // Create an audience string only if needed. - if len(t.creds) > 0 || callHdr.Creds != nil { - // Construct URI required to get auth request metadata. - // Omit port if it is the default one. - host := strings.TrimSuffix(callHdr.Host, ":443") - pos := strings.LastIndex(callHdr.Method, "/") - if pos == -1 { - pos = len(callHdr.Method) - } - audience = "https://" + host + callHdr.Method[:pos] - } - for _, c := range t.creds { - data, err := c.GetRequestMetadata(ctx, audience) - if err != nil { - return nil, streamErrorf(codes.Internal, "transport: %v", err) - } - for k, v := range data { - // Capital header names are illegal in HTTP/2. - k = strings.ToLower(k) - authData[k] = v - } - } - callAuthData := map[string]string{} - // Check if credentials.PerRPCCredentials were provided via call options. - // Note: if these credentials are provided both via dial options and call - // options, then both sets of credentials will be applied. - if callCreds := callHdr.Creds; callCreds != nil { - if !t.isSecure && callCreds.RequireTransportSecurity() { - return nil, streamErrorf(codes.Unauthenticated, "transport: cannot send secure credentials on an insecure connection") - } - data, err := callCreds.GetRequestMetadata(ctx, audience) - if err != nil { - return nil, streamErrorf(codes.Internal, "transport: %v", err) - } - for k, v := range data { - // Capital header names are illegal in HTTP/2 - k = strings.ToLower(k) - callAuthData[k] = v - } - } - t.mu.Lock() - if t.activeStreams == nil { - t.mu.Unlock() - return nil, ErrConnClosing - } - if t.state == draining { - t.mu.Unlock() - return nil, ErrStreamDrain - } - if t.state != reachable { - t.mu.Unlock() - return nil, ErrConnClosing - } - t.mu.Unlock() - sq, err := wait(ctx, t.ctx, nil, nil, t.streamsQuota.acquire()) + return pr +} + +func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr) ([]hpack.HeaderField, error) { + aud := t.createAudience(callHdr) + authData, err := t.getTrAuthData(ctx, aud) if err != nil { return nil, err } - // Returns the quota balance back. - if sq > 1 { - t.streamsQuota.add(sq - 1) + callAuthData, err := t.getCallAuthData(ctx, aud, callHdr) + if err != nil { + return nil, err } - // TODO(mmukhi): Benchmark if the perfomance gets better if count the metadata and other header fields + // TODO(mmukhi): Benchmark if the performance gets better if count the metadata and other header fields // first and create a slice of that exact size. // Make the slice of certain predictable size to reduce allocations made by append. hfLen := 7 // :method, :scheme, :path, :authority, content-type, user-agent, te @@ -427,7 +376,7 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea headerFields = append(headerFields, hpack.HeaderField{Name: ":scheme", Value: t.scheme}) headerFields = append(headerFields, hpack.HeaderField{Name: ":path", Value: callHdr.Method}) headerFields = append(headerFields, hpack.HeaderField{Name: ":authority", Value: callHdr.Host}) - headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: "application/grpc"}) + headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: contentType(callHdr.ContentSubtype)}) headerFields = append(headerFields, hpack.HeaderField{Name: "user-agent", Value: t.userAgent}) headerFields = append(headerFields, hpack.HeaderField{Name: "te", Value: "trailers"}) @@ -452,7 +401,22 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea if b := stats.OutgoingTrace(ctx); b != nil { headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-trace-bin", Value: encodeBinHeader(b)}) } - if md, ok := metadata.FromOutgoingContext(ctx); ok { + + if md, added, ok := metadata.FromOutgoingContextRaw(ctx); ok { + var k string + for _, vv := range added { + for i, v := range vv { + if i%2 == 0 { + k = v + continue + } + // HTTP doesn't allow you to set pseudoheaders after non pseudoheaders were set. + if isReservedHeader(k) { + continue + } + headerFields = append(headerFields, hpack.HeaderField{Name: strings.ToLower(k), Value: encodeMetadataHeader(k, v)}) + } + } for k, vv := range md { // HTTP doesn't allow you to set pseudoheaders after non pseudoheaders were set. if isReservedHeader(k) { @@ -473,42 +437,178 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea } } } - t.mu.Lock() - if t.state == draining { - t.mu.Unlock() - t.streamsQuota.add(1) - return nil, ErrStreamDrain + return headerFields, nil +} + +func (t *http2Client) createAudience(callHdr *CallHdr) string { + // Create an audience string only if needed. + if len(t.creds) == 0 && callHdr.Creds == nil { + return "" } - if t.state != reachable { - t.mu.Unlock() - return nil, ErrConnClosing + // Construct URI required to get auth request metadata. + // Omit port if it is the default one. + host := strings.TrimSuffix(callHdr.Host, ":443") + pos := strings.LastIndex(callHdr.Method, "/") + if pos == -1 { + pos = len(callHdr.Method) } - s := t.newStream(ctx, callHdr) - t.activeStreams[s.id] = s - // If the number of active streams change from 0 to 1, then check if keepalive - // has gone dormant. If so, wake it up. - if len(t.activeStreams) == 1 { - select { - case t.awakenKeepalive <- struct{}{}: - t.controlBuf.put(&ping{data: [8]byte{}}) - // Fill the awakenKeepalive channel again as this channel must be - // kept non-writable except at the point that the keepalive() - // goroutine is waiting either to be awaken or shutdown. - t.awakenKeepalive <- struct{}{} - default: + return "https://" + host + callHdr.Method[:pos] +} + +func (t *http2Client) getTrAuthData(ctx context.Context, audience string) (map[string]string, error) { + authData := map[string]string{} + for _, c := range t.creds { + data, err := c.GetRequestMetadata(ctx, audience) + if err != nil { + if _, ok := status.FromError(err); ok { + return nil, err + } + + return nil, streamErrorf(codes.Unauthenticated, "transport: %v", err) + } + for k, v := range data { + // Capital header names are illegal in HTTP/2. + k = strings.ToLower(k) + authData[k] = v } } - t.controlBuf.put(&headerFrame{ - streamID: s.id, + return authData, nil +} + +func (t *http2Client) getCallAuthData(ctx context.Context, audience string, callHdr *CallHdr) (map[string]string, error) { + callAuthData := map[string]string{} + // Check if credentials.PerRPCCredentials were provided via call options. + // Note: if these credentials are provided both via dial options and call + // options, then both sets of credentials will be applied. + if callCreds := callHdr.Creds; callCreds != nil { + if !t.isSecure && callCreds.RequireTransportSecurity() { + return nil, streamErrorf(codes.Unauthenticated, "transport: cannot send secure credentials on an insecure connection") + } + data, err := callCreds.GetRequestMetadata(ctx, audience) + if err != nil { + return nil, streamErrorf(codes.Internal, "transport: %v", err) + } + for k, v := range data { + // Capital header names are illegal in HTTP/2 + k = strings.ToLower(k) + callAuthData[k] = v + } + } + return callAuthData, nil +} + +// NewStream creates a stream and registers it into the transport as "active" +// streams. +func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Stream, err error) { + ctx = peer.NewContext(ctx, t.getPeer()) + headerFields, err := t.createHeaderFields(ctx, callHdr) + if err != nil { + return nil, err + } + s := t.newStream(ctx, callHdr) + cleanup := func(err error) { + if s.swapState(streamDone) == streamDone { + // If it was already done, return. + return + } + // The stream was unprocessed by the server. + atomic.StoreUint32(&s.unprocessed, 1) + s.write(recvMsg{err: err}) + close(s.done) + // If headerChan isn't closed, then close it. + if atomic.SwapUint32(&s.headerDone, 1) == 0 { + close(s.headerChan) + } + + } + hdr := &headerFrame{ hf: headerFields, endStream: false, - }) - t.mu.Unlock() - - s.mu.Lock() - s.bytesSent = true - s.mu.Unlock() - + initStream: func(id uint32) (bool, error) { + t.mu.Lock() + if state := t.state; state != reachable { + t.mu.Unlock() + // Do a quick cleanup. + err := error(errStreamDrain) + if state == closing { + err = ErrConnClosing + } + cleanup(err) + return false, err + } + t.activeStreams[id] = s + if channelz.IsOn() { + t.czmu.Lock() + t.streamsStarted++ + t.lastStreamCreated = time.Now() + t.czmu.Unlock() + } + var sendPing bool + // If the number of active streams change from 0 to 1, then check if keepalive + // has gone dormant. If so, wake it up. + if len(t.activeStreams) == 1 && t.keepaliveEnabled { + select { + case t.awakenKeepalive <- struct{}{}: + sendPing = true + // Fill the awakenKeepalive channel again as this channel must be + // kept non-writable except at the point that the keepalive() + // goroutine is waiting either to be awaken or shutdown. + t.awakenKeepalive <- struct{}{} + default: + } + } + t.mu.Unlock() + return sendPing, nil + }, + onOrphaned: cleanup, + wq: s.wq, + } + firstTry := true + var ch chan struct{} + checkForStreamQuota := func(it interface{}) bool { + if t.streamQuota <= 0 { // Can go negative if server decreases it. + if firstTry { + t.waitingStreams++ + } + ch = t.streamsQuotaAvailable + return false + } + if !firstTry { + t.waitingStreams-- + } + t.streamQuota-- + h := it.(*headerFrame) + h.streamID = t.nextID + t.nextID += 2 + s.id = h.streamID + s.fc = &inFlow{limit: uint32(t.initialWindowSize)} + if t.streamQuota > 0 && t.waitingStreams > 0 { + select { + case t.streamsQuotaAvailable <- struct{}{}: + default: + } + } + return true + } + for { + success, err := t.controlBuf.executeAndPut(checkForStreamQuota, hdr) + if err != nil { + return nil, err + } + if success { + break + } + firstTry = false + select { + case <-ch: + case <-s.ctx.Done(): + return nil, ContextErr(s.ctx.Err()) + case <-t.goAway: + return nil, errStreamDrain + case <-t.ctx.Done(): + return nil, ErrConnClosing + } + } if t.statsHandler != nil { outHeader := &stats.OutHeader{ Client: true, @@ -525,86 +625,97 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea // CloseStream clears the footprint of a stream when the stream is not needed any more. // This must not be executed in reader's goroutine. func (t *http2Client) CloseStream(s *Stream, err error) { - t.mu.Lock() - if t.activeStreams == nil { - t.mu.Unlock() + var ( + rst bool + rstCode http2.ErrCode + ) + if err != nil { + rst = true + rstCode = http2.ErrCodeCancel + } + t.closeStream(s, err, rst, rstCode, nil, nil, false) +} + +func (t *http2Client) closeStream(s *Stream, err error, rst bool, rstCode http2.ErrCode, st *status.Status, mdata map[string][]string, eosReceived bool) { + // Set stream status to done. + if s.swapState(streamDone) == streamDone { + // If it was already done, return. return } + // status and trailers can be updated here without any synchronization because the stream goroutine will + // only read it after it sees an io.EOF error from read or write and we'll write those errors + // only after updating this. + s.status = st + if len(mdata) > 0 { + s.trailer = mdata + } if err != nil { - // notify in-flight streams, before the deletion + // This will unblock reads eventually. s.write(recvMsg{err: err}) } - delete(t.activeStreams, s.id) - if t.state == draining && len(t.activeStreams) == 0 { - // The transport is draining and s is the last live stream on t. - t.mu.Unlock() - t.Close() - return - } - t.mu.Unlock() - // rstStream is true in case the stream is being closed at the client-side - // and the server needs to be intimated about it by sending a RST_STREAM - // frame. - // To make sure this frame is written to the wire before the headers of the - // next stream waiting for streamsQuota, we add to streamsQuota pool only - // after having acquired the writableChan to send RST_STREAM out (look at - // the controller() routine). - var rstStream bool - var rstError http2.ErrCode - defer func() { - // In case, the client doesn't have to send RST_STREAM to server - // we can safely add back to streamsQuota pool now. - if !rstStream { - t.streamsQuota.add(1) - return - } - t.controlBuf.put(&resetStream{s.id, rstError}) - }() - s.mu.Lock() - rstStream = s.rstStream - rstError = s.rstError - if s.state == streamDone { - s.mu.Unlock() - return - } - if !s.headerDone { + // This will unblock write. + close(s.done) + // If headerChan isn't closed, then close it. + if atomic.SwapUint32(&s.headerDone, 1) == 0 { close(s.headerChan) - s.headerDone = true } - s.state = streamDone - s.mu.Unlock() - if _, ok := err.(StreamError); ok { - rstStream = true - rstError = http2.ErrCodeCancel + cleanup := &cleanupStream{ + streamID: s.id, + onWrite: func() { + t.mu.Lock() + if t.activeStreams != nil { + delete(t.activeStreams, s.id) + } + t.mu.Unlock() + if channelz.IsOn() { + t.czmu.Lock() + if eosReceived { + t.streamsSucceeded++ + } else { + t.streamsFailed++ + } + t.czmu.Unlock() + } + }, + rst: rst, + rstCode: rstCode, } + addBackStreamQuota := func(interface{}) bool { + t.streamQuota++ + if t.streamQuota > 0 && t.waitingStreams > 0 { + select { + case t.streamsQuotaAvailable <- struct{}{}: + default: + } + } + return true + } + t.controlBuf.executeAndPut(addBackStreamQuota, cleanup) } // Close kicks off the shutdown process of the transport. This should be called // only once on a transport. Once it is called, the transport should not be // accessed any more. -func (t *http2Client) Close() (err error) { +func (t *http2Client) Close() error { t.mu.Lock() + // Make sure we only Close once. if t.state == closing { t.mu.Unlock() - return + return nil } t.state = closing - t.mu.Unlock() - t.cancel() - err = t.conn.Close() - t.mu.Lock() streams := t.activeStreams t.activeStreams = nil t.mu.Unlock() + t.controlBuf.finish() + t.cancel() + err := t.conn.Close() + if channelz.IsOn() { + channelz.RemoveEntry(t.channelzID) + } // Notify all active streams. for _, s := range streams { - s.mu.Lock() - if !s.headerDone { - close(s.headerChan) - s.headerDone = true - } - s.mu.Unlock() - s.write(recvMsg{err: ErrConnClosing}) + t.closeStream(s, ErrConnClosing, false, http2.ErrCodeNo, nil, nil, false) } if t.statsHandler != nil { connEnd := &stats.ConnEnd{ @@ -622,8 +733,8 @@ func (t *http2Client) Close() (err error) { // closing. func (t *http2Client) GracefulClose() error { t.mu.Lock() - switch t.state { - case closing, draining: + // Make sure we move to draining only from active. + if t.state == draining || t.state == closing { t.mu.Unlock() return nil } @@ -633,108 +744,41 @@ func (t *http2Client) GracefulClose() error { if active == 0 { return t.Close() } + t.controlBuf.put(&incomingGoAway{}) return nil } // Write formats the data into HTTP2 data frame(s) and sends it out. The caller // should proceed only if Write returns nil. func (t *http2Client) Write(s *Stream, hdr []byte, data []byte, opts *Options) error { - select { - case <-s.ctx.Done(): - return ContextErr(s.ctx.Err()) - case <-t.ctx.Done(): - return ErrConnClosing - default: + if opts.Last { + // If it's the last message, update stream state. + if !s.compareAndSwapState(streamActive, streamWriteDone) { + return errStreamDone + } + } else if s.getState() != streamActive { + return errStreamDone } - - if hdr == nil && data == nil && opts.Last { - // stream.CloseSend uses this to send an empty frame with endStream=True - t.controlBuf.put(&dataFrame{streamID: s.id, endStream: true, f: func() {}}) - return nil + df := &dataFrame{ + streamID: s.id, + endStream: opts.Last, } - // Add data to header frame so that we can equally distribute data across frames. - emptyLen := http2MaxFrameLen - len(hdr) - if emptyLen > len(data) { - emptyLen = len(data) - } - hdr = append(hdr, data[:emptyLen]...) - data = data[emptyLen:] - for idx, r := range [][]byte{hdr, data} { - for len(r) > 0 { - size := http2MaxFrameLen - // Wait until the stream has some quota to send the data. - quotaChan, quotaVer := s.sendQuotaPool.acquireWithVersion() - sq, err := wait(s.ctx, t.ctx, s.done, s.goAway, quotaChan) - if err != nil { - return err - } - // Wait until the transport has some quota to send the data. - tq, err := wait(s.ctx, t.ctx, s.done, s.goAway, t.sendQuotaPool.acquire()) - if err != nil { - return err - } - if sq < size { - size = sq - } - if tq < size { - size = tq - } - if size > len(r) { - size = len(r) - } - p := r[:size] - ps := len(p) - if ps < tq { - // Overbooked transport quota. Return it back. - t.sendQuotaPool.add(tq - ps) - } - // Acquire local send quota to be able to write to the controlBuf. - ltq, err := wait(s.ctx, t.ctx, s.done, s.goAway, s.localSendQuota.acquire()) - if err != nil { - if _, ok := err.(ConnectionError); !ok { - t.sendQuotaPool.add(ps) - } - return err - } - s.localSendQuota.add(ltq - ps) // It's ok if we make it negative. - var endStream bool - // See if this is the last frame to be written. - if opts.Last { - if len(r)-size == 0 { // No more data in r after this iteration. - if idx == 0 { // We're writing data header. - if len(data) == 0 { // There's no data to follow. - endStream = true - } - } else { // We're writing data. - endStream = true - } - } - } - success := func() { - t.controlBuf.put(&dataFrame{streamID: s.id, endStream: endStream, d: p, f: func() { s.localSendQuota.add(ps) }}) - if ps < sq { - s.sendQuotaPool.lockedAdd(sq - ps) - } - r = r[ps:] - } - failure := func() { - s.sendQuotaPool.lockedAdd(sq) - } - if !s.sendQuotaPool.compareAndExecute(quotaVer, success, failure) { - t.sendQuotaPool.add(ps) - s.localSendQuota.add(ps) - } + if hdr != nil || data != nil { // If it's not an empty data frame. + // Add some data to grpc message header so that we can equally + // distribute bytes across frames. + emptyLen := http2MaxFrameLen - len(hdr) + if emptyLen > len(data) { + emptyLen = len(data) + } + hdr = append(hdr, data[:emptyLen]...) + data = data[emptyLen:] + df.h, df.d = hdr, data + // TODO(mmukhi): The above logic in this if can be moved to loopyWriter's data handler. + if err := s.wq.get(int32(len(hdr) + len(data))); err != nil { + return err } } - if !opts.Last { - return nil - } - s.mu.Lock() - if s.state != streamDone { - s.state = streamWriteDone - } - s.mu.Unlock() - return nil + return t.controlBuf.put(df) } func (t *http2Client) getStream(f http2.Frame) (*Stream, bool) { @@ -748,34 +792,17 @@ func (t *http2Client) getStream(f http2.Frame) (*Stream, bool) { // of stream if the application is requesting data larger in size than // the window. func (t *http2Client) adjustWindow(s *Stream, n uint32) { - s.mu.Lock() - defer s.mu.Unlock() - if s.state == streamDone { - return - } if w := s.fc.maybeAdjust(n); w > 0 { - // Piggyback connection's window update along. - if cw := t.fc.resetPendingUpdate(); cw > 0 { - t.controlBuf.put(&windowUpdate{0, cw}) - } - t.controlBuf.put(&windowUpdate{s.id, w}) + t.controlBuf.put(&outgoingWindowUpdate{streamID: s.id, increment: w}) } } -// updateWindow adjusts the inbound quota for the stream and the transport. -// Window updates will deliver to the controller for sending when -// the cumulative quota exceeds the corresponding threshold. +// updateWindow adjusts the inbound quota for the stream. +// Window updates will be sent out when the cumulative quota +// exceeds the corresponding threshold. func (t *http2Client) updateWindow(s *Stream, n uint32) { - s.mu.Lock() - defer s.mu.Unlock() - if s.state == streamDone { - return - } if w := s.fc.onRead(n); w > 0 { - if cw := t.fc.resetPendingUpdate(); cw > 0 { - t.controlBuf.put(&windowUpdate{0, cw}) - } - t.controlBuf.put(&windowUpdate{s.id, w}) + t.controlBuf.put(&outgoingWindowUpdate{streamID: s.id, increment: w}) } } @@ -787,15 +814,17 @@ func (t *http2Client) updateFlowControl(n uint32) { for _, s := range t.activeStreams { s.fc.newLimit(n) } - t.initialWindowSize = int32(n) t.mu.Unlock() - t.controlBuf.put(&windowUpdate{0, t.fc.newLimit(n)}) - t.controlBuf.put(&settings{ - ack: false, + updateIWS := func(interface{}) bool { + t.initialWindowSize = int32(n) + return true + } + t.controlBuf.executeAndPut(updateIWS, &outgoingWindowUpdate{streamID: 0, increment: t.fc.newLimit(n)}) + t.controlBuf.put(&outgoingSettings{ ss: []http2.Setting{ { ID: http2.SettingInitialWindowSize, - Val: uint32(n), + Val: n, }, }, }) @@ -805,7 +834,7 @@ func (t *http2Client) handleData(f *http2.DataFrame) { size := f.Header().Length var sendBDPPing bool if t.bdpEst != nil { - sendBDPPing = t.bdpEst.add(uint32(size)) + sendBDPPing = t.bdpEst.add(size) } // Decouple connection's flow control from application's read. // An update on connection's flow control should not depend on @@ -816,21 +845,24 @@ func (t *http2Client) handleData(f *http2.DataFrame) { // active(fast) streams from starving in presence of slow or // inactive streams. // - // Furthermore, if a bdpPing is being sent out we can piggyback - // connection's window update for the bytes we just received. + if w := t.fc.onData(size); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: w, + }) + } if sendBDPPing { - if size != 0 { // Could've been an empty data frame. - t.controlBuf.put(&windowUpdate{0, uint32(size)}) + // Avoid excessive ping detection (e.g. in an L7 proxy) + // by sending a window update prior to the BDP ping. + + if w := t.fc.reset(); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: w, + }) } + t.controlBuf.put(bdpPing) - } else { - if err := t.fc.onData(uint32(size)); err != nil { - t.Close() - return - } - if w := t.fc.onRead(uint32(size)); w > 0 { - t.controlBuf.put(&windowUpdate{0, w}) - } } // Select the right stream to dispatch. s, ok := t.getStream(f) @@ -838,25 +870,15 @@ func (t *http2Client) handleData(f *http2.DataFrame) { return } if size > 0 { - s.mu.Lock() - if s.state == streamDone { - s.mu.Unlock() - return - } - if err := s.fc.onData(uint32(size)); err != nil { - s.rstStream = true - s.rstError = http2.ErrCodeFlowControl - s.finish(status.New(codes.Internal, err.Error())) - s.mu.Unlock() - s.write(recvMsg{err: io.EOF}) + if err := s.fc.onData(size); err != nil { + t.closeStream(s, io.EOF, true, http2.ErrCodeFlowControl, status.New(codes.Internal, err.Error()), nil, false) return } if f.Header().Flags.Has(http2.FlagDataPadded) { - if w := s.fc.onRead(uint32(size) - uint32(len(f.Data()))); w > 0 { - t.controlBuf.put(&windowUpdate{s.id, w}) + if w := s.fc.onRead(size - uint32(len(f.Data()))); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{s.id, w}) } } - s.mu.Unlock() // TODO(bradfitz, zhaoq): A copy is required here because there is no // guarantee f.Data() is consumed before the arrival of next frame. // Can this copy be eliminated? @@ -869,14 +891,7 @@ func (t *http2Client) handleData(f *http2.DataFrame) { // The server has closed the stream without sending trailers. Record that // the read direction is closed, and set the status appropriately. if f.FrameHeader.Flags.Has(http2.FlagDataEndStream) { - s.mu.Lock() - if s.state == streamDone { - s.mu.Unlock() - return - } - s.finish(status.New(codes.Internal, "server closed the stream without sending trailers")) - s.mu.Unlock() - s.write(recvMsg{err: io.EOF}) + t.closeStream(s, io.EOF, false, http2.ErrCodeNo, status.New(codes.Internal, "server closed the stream without sending trailers"), nil, true) } } @@ -885,36 +900,55 @@ func (t *http2Client) handleRSTStream(f *http2.RSTStreamFrame) { if !ok { return } - s.mu.Lock() - if s.state == streamDone { - s.mu.Unlock() - return + if f.ErrCode == http2.ErrCodeRefusedStream { + // The stream was unprocessed by the server. + atomic.StoreUint32(&s.unprocessed, 1) } - if !s.headerDone { - close(s.headerChan) - s.headerDone = true - } - statusCode, ok := http2ErrConvTab[http2.ErrCode(f.ErrCode)] + statusCode, ok := http2ErrConvTab[f.ErrCode] if !ok { warningf("transport: http2Client.handleRSTStream found no mapped gRPC status for the received http2 error %v", f.ErrCode) statusCode = codes.Unknown } - s.finish(status.Newf(statusCode, "stream terminated by RST_STREAM with error code: %v", f.ErrCode)) - s.mu.Unlock() - s.write(recvMsg{err: io.EOF}) + t.closeStream(s, io.EOF, false, http2.ErrCodeNo, status.Newf(statusCode, "stream terminated by RST_STREAM with error code: %v", f.ErrCode), nil, false) } -func (t *http2Client) handleSettings(f *http2.SettingsFrame) { +func (t *http2Client) handleSettings(f *http2.SettingsFrame, isFirst bool) { if f.IsAck() { return } + var maxStreams *uint32 var ss []http2.Setting f.ForeachSetting(func(s http2.Setting) error { + if s.ID == http2.SettingMaxConcurrentStreams { + maxStreams = new(uint32) + *maxStreams = s.Val + return nil + } ss = append(ss, s) return nil }) - // The settings will be applied once the ack is sent. - t.controlBuf.put(&settings{ack: true, ss: ss}) + if isFirst && maxStreams == nil { + maxStreams = new(uint32) + *maxStreams = math.MaxUint32 + } + sf := &incomingSettings{ + ss: ss, + } + if maxStreams == nil { + t.controlBuf.put(sf) + return + } + updateStreamQuota := func(interface{}) bool { + delta := int64(*maxStreams) - int64(t.maxConcurrentStreams) + t.maxConcurrentStreams = *maxStreams + t.streamQuota += delta + if delta > 0 && t.waitingStreams > 0 { + close(t.streamsQuotaAvailable) // wake all of them up. + t.streamsQuotaAvailable = make(chan struct{}, 1) + } + return true + } + t.controlBuf.executeAndPut(updateStreamQuota, sf) } func (t *http2Client) handlePing(f *http2.PingFrame) { @@ -932,7 +966,7 @@ func (t *http2Client) handlePing(f *http2.PingFrame) { func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { t.mu.Lock() - if t.state != reachable && t.state != draining { + if t.state == closing { t.mu.Unlock() return } @@ -945,12 +979,16 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { t.Close() return } - // A client can receive multiple GoAways from server (look at https://github.com/grpc/grpc-go/issues/1387). - // The idea is that the first GoAway will be sent with an ID of MaxInt32 and the second GoAway will be sent after an RTT delay - // with the ID of the last stream the server will process. - // Therefore, when we get the first GoAway we don't really close any streams. While in case of second GoAway we - // close all streams created after the second GoAwayId. This way streams that were in-flight while the GoAway from server - // was being sent don't get killed. + // A client can receive multiple GoAways from the server (see + // https://github.com/grpc/grpc-go/issues/1387). The idea is that the first + // GoAway will be sent with an ID of MaxInt32 and the second GoAway will be + // sent after an RTT delay with the ID of the last stream the server will + // process. + // + // Therefore, when we get the first GoAway we don't necessarily close any + // streams. While in case of second GoAway we close all streams created after + // the GoAwayId. This way streams that were in-flight while the GoAway from + // server was being sent don't get killed. select { case <-t.goAway: // t.goAway has been closed (i.e.,multiple GoAways). // If there are multiple GoAways the first one should always have an ID greater than the following ones. @@ -963,6 +1001,7 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { t.setGoAwayReason(f) close(t.goAway) t.state = draining + t.controlBuf.put(&incomingGoAway{}) } // All streams with IDs greater than the GoAwayId // and smaller than the previous GoAway ID should be killed. @@ -972,7 +1011,9 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { } for streamID, stream := range t.activeStreams { if streamID > id && streamID <= upperLimit { - close(stream.goAway) + // The stream was unprocessed by the server. + atomic.StoreUint32(&stream.unprocessed, 1) + t.closeStream(stream, errStreamDrain, false, http2.ErrCodeNo, statusGoAway, nil, false) } } t.prevGoAwayID = id @@ -988,11 +1029,11 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { // It expects a lock on transport's mutext to be held by // the caller. func (t *http2Client) setGoAwayReason(f *http2.GoAwayFrame) { - t.goAwayReason = NoReason + t.goAwayReason = GoAwayNoReason switch f.ErrCode { case http2.ErrCodeEnhanceYourCalm: if string(f.DebugData()) == "too_many_pings" { - t.goAwayReason = TooManyPings + t.goAwayReason = GoAwayTooManyPings } } } @@ -1004,15 +1045,10 @@ func (t *http2Client) GetGoAwayReason() GoAwayReason { } func (t *http2Client) handleWindowUpdate(f *http2.WindowUpdateFrame) { - id := f.Header().StreamID - incr := f.Increment - if id == 0 { - t.sendQuotaPool.add(int(incr)) - return - } - if s, ok := t.getStream(f); ok { - s.sendQuotaPool.add(int(incr)) - } + t.controlBuf.put(&incomingWindowUpdate{ + streamID: f.Header().StreamID, + increment: f.Increment, + }) } // operateHeaders takes action on the decoded headers. @@ -1021,18 +1057,10 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) { if !ok { return } - s.mu.Lock() - s.bytesReceived = true - s.mu.Unlock() + atomic.StoreUint32(&s.bytesReceived, 1) var state decodeState if err := state.decodeResponseHeader(frame); err != nil { - s.mu.Lock() - if !s.headerDone { - close(s.headerChan) - s.headerDone = true - } - s.mu.Unlock() - s.write(recvMsg{err: err}) + t.closeStream(s, err, true, http2.ErrCodeProtocol, nil, nil, false) // Something wrong. Stops reading even when there is remaining. return } @@ -1056,40 +1084,25 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) { } } }() - - s.mu.Lock() - if !endStream { - s.recvCompress = state.encoding - } - if !s.headerDone { - if !endStream && len(state.mdata) > 0 { - s.header = state.mdata + // If headers haven't been received yet. + if atomic.SwapUint32(&s.headerDone, 1) == 0 { + if !endStream { + // Headers frame is not actually a trailers-only frame. + isHeader = true + // These values can be set without any synchronization because + // stream goroutine will read it only after seeing a closed + // headerChan which we'll close after setting this. + s.recvCompress = state.encoding + if len(state.mdata) > 0 { + s.header = state.mdata + } } close(s.headerChan) - s.headerDone = true - isHeader = true } - if !endStream || s.state == streamDone { - s.mu.Unlock() + if !endStream { return } - - if len(state.mdata) > 0 { - s.trailer = state.mdata - } - s.finish(state.status()) - s.mu.Unlock() - s.write(recvMsg{err: io.EOF}) -} - -func handleMalformedHTTP2(s *Stream, err error) { - s.mu.Lock() - if !s.headerDone { - close(s.headerChan) - s.headerDone = true - } - s.mu.Unlock() - s.write(recvMsg{err: err}) + t.closeStream(s, io.EOF, false, http2.ErrCodeNo, state.status(), state.mdata, true) } // reader runs as a separate goroutine in charge of reading data from network @@ -1099,24 +1112,30 @@ func handleMalformedHTTP2(s *Stream, err error) { // optimal. // TODO(zhaoq): Check the validity of the incoming frame sequence. func (t *http2Client) reader() { + defer close(t.readerDone) // Check the validity of server preface. frame, err := t.framer.fr.ReadFrame() if err != nil { t.Close() return } - atomic.CompareAndSwapUint32(&t.activity, 0, 1) + if t.keepaliveEnabled { + atomic.CompareAndSwapUint32(&t.activity, 0, 1) + } sf, ok := frame.(*http2.SettingsFrame) if !ok { t.Close() return } - t.handleSettings(sf) + t.onSuccess() + t.handleSettings(sf, true) // loop to keep reading incoming messages on this transport. for { frame, err := t.framer.fr.ReadFrame() - atomic.CompareAndSwapUint32(&t.activity, 0, 1) + if t.keepaliveEnabled { + atomic.CompareAndSwapUint32(&t.activity, 0, 1) + } if err != nil { // Abort an active stream if the http2.Framer returns a // http2.StreamError. This can happen only if the server's response @@ -1127,7 +1146,7 @@ func (t *http2Client) reader() { t.mu.Unlock() if s != nil { // use error detail to provide better err message - handleMalformedHTTP2(s, streamErrorf(http2ErrConvTab[se.Code], "%v", t.framer.fr.ErrorDetail())) + t.closeStream(s, streamErrorf(http2ErrConvTab[se.Code], "%v", t.framer.fr.ErrorDetail()), true, http2.ErrCodeProtocol, nil, nil, false) } continue } else { @@ -1144,7 +1163,7 @@ func (t *http2Client) reader() { case *http2.RSTStreamFrame: t.handleRSTStream(frame) case *http2.SettingsFrame: - t.handleSettings(frame) + t.handleSettings(frame, false) case *http2.PingFrame: t.handlePing(frame) case *http2.GoAwayFrame: @@ -1157,107 +1176,6 @@ func (t *http2Client) reader() { } } -func (t *http2Client) applySettings(ss []http2.Setting) { - for _, s := range ss { - switch s.ID { - case http2.SettingMaxConcurrentStreams: - // TODO(zhaoq): This is a hack to avoid significant refactoring of the - // code to deal with the unrealistic int32 overflow. Probably will try - // to find a better way to handle this later. - if s.Val > math.MaxInt32 { - s.Val = math.MaxInt32 - } - t.mu.Lock() - ms := t.maxStreams - t.maxStreams = int(s.Val) - t.mu.Unlock() - t.streamsQuota.add(int(s.Val) - ms) - case http2.SettingInitialWindowSize: - t.mu.Lock() - for _, stream := range t.activeStreams { - // Adjust the sending quota for each stream. - stream.sendQuotaPool.addAndUpdate(int(s.Val) - int(t.streamSendQuota)) - } - t.streamSendQuota = s.Val - t.mu.Unlock() - } - } -} - -// TODO(mmukhi): A lot of this code(and code in other places in the tranpsort layer) -// is duplicated between the client and the server. -// The transport layer needs to be refactored to take care of this. -func (t *http2Client) itemHandler(i item) error { - var err error - switch i := i.(type) { - case *dataFrame: - err = t.framer.fr.WriteData(i.streamID, i.endStream, i.d) - if err == nil { - i.f() - } - case *headerFrame: - t.hBuf.Reset() - for _, f := range i.hf { - t.hEnc.WriteField(f) - } - endHeaders := false - first := true - for !endHeaders { - size := t.hBuf.Len() - if size > http2MaxFrameLen { - size = http2MaxFrameLen - } else { - endHeaders = true - } - if first { - first = false - err = t.framer.fr.WriteHeaders(http2.HeadersFrameParam{ - StreamID: i.streamID, - BlockFragment: t.hBuf.Next(size), - EndStream: i.endStream, - EndHeaders: endHeaders, - }) - } else { - err = t.framer.fr.WriteContinuation( - i.streamID, - endHeaders, - t.hBuf.Next(size), - ) - } - if err != nil { - return err - } - } - case *windowUpdate: - err = t.framer.fr.WriteWindowUpdate(i.streamID, i.increment) - case *settings: - if i.ack { - t.applySettings(i.ss) - err = t.framer.fr.WriteSettingsAck() - } else { - err = t.framer.fr.WriteSettings(i.ss...) - } - case *resetStream: - // If the server needs to be to intimated about stream closing, - // then we need to make sure the RST_STREAM frame is written to - // the wire before the headers of the next stream waiting on - // streamQuota. We ensure this by adding to the streamsQuota pool - // only after having acquired the writableChan to send RST_STREAM. - err = t.framer.fr.WriteRSTStream(i.streamID, i.code) - t.streamsQuota.add(1) - case *flushIO: - err = t.framer.writer.Flush() - case *ping: - if !i.ack { - t.bdpEst.timesnap(i.data) - } - err = t.framer.fr.WritePing(i.ack, i.data) - default: - errorf("transport: http2Client.controller got unexpected item type %v\n", i) - } - return err -} - // keepalive running in a separate goroutune makes sure the connection is alive by sending pings. func (t *http2Client) keepalive() { p := &ping{data: [8]byte{}} @@ -1284,6 +1202,11 @@ func (t *http2Client) keepalive() { } } else { t.mu.Unlock() + if channelz.IsOn() { + t.czmu.Lock() + t.kpCount++ + t.czmu.Unlock() + } // Send ping. t.controlBuf.put(p) } @@ -1320,3 +1243,56 @@ func (t *http2Client) Error() <-chan struct{} { func (t *http2Client) GoAway() <-chan struct{} { return t.goAway } + +func (t *http2Client) ChannelzMetric() *channelz.SocketInternalMetric { + t.czmu.RLock() + s := channelz.SocketInternalMetric{ + StreamsStarted: t.streamsStarted, + StreamsSucceeded: t.streamsSucceeded, + StreamsFailed: t.streamsFailed, + MessagesSent: t.msgSent, + MessagesReceived: t.msgRecv, + KeepAlivesSent: t.kpCount, + LastLocalStreamCreatedTimestamp: t.lastStreamCreated, + LastMessageSentTimestamp: t.lastMsgSent, + LastMessageReceivedTimestamp: t.lastMsgRecv, + LocalFlowControlWindow: int64(t.fc.getSize()), + //socket options + LocalAddr: t.localAddr, + RemoteAddr: t.remoteAddr, + // Security + // RemoteName : + } + t.czmu.RUnlock() + s.RemoteFlowControlWindow = t.getOutFlowWindow() + return &s +} + +func (t *http2Client) IncrMsgSent() { + t.czmu.Lock() + t.msgSent++ + t.lastMsgSent = time.Now() + t.czmu.Unlock() +} + +func (t *http2Client) IncrMsgRecv() { + t.czmu.Lock() + t.msgRecv++ + t.lastMsgRecv = time.Now() + t.czmu.Unlock() +} + +func (t *http2Client) getOutFlowWindow() int64 { + resp := make(chan uint32, 1) + timer := time.NewTimer(time.Second) + defer timer.Stop() + t.controlBuf.put(&outFlowControlSizeRequest{resp}) + select { + case sz := <-resp: + return int64(sz) + case <-t.ctxDone: + return -1 + case <-timer.C: + return -2 + } +} diff --git a/vendor/google.golang.org/grpc/transport/http2_server.go b/vendor/google.golang.org/grpc/transport/http2_server.go index 00df8eed0fd..19acedb2b02 100644 --- a/vendor/google.golang.org/grpc/transport/http2_server.go +++ b/vendor/google.golang.org/grpc/transport/http2_server.go @@ -24,7 +24,6 @@ import ( "fmt" "io" "math" - "math/rand" "net" "strconv" "sync" @@ -35,8 +34,12 @@ import ( "golang.org/x/net/context" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpcrand" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" @@ -52,25 +55,25 @@ var ErrIllegalHeaderWrite = errors.New("transport: the stream is done or WriteHe // http2Server implements the ServerTransport interface with HTTP2. type http2Server struct { ctx context.Context + ctxDone <-chan struct{} // Cache the context.Done() chan cancel context.CancelFunc conn net.Conn + loopy *loopyWriter + readerDone chan struct{} // sync point to enable testing. + writerDone chan struct{} // sync point to enable testing. remoteAddr net.Addr localAddr net.Addr maxStreamID uint32 // max stream ID ever seen authInfo credentials.AuthInfo // auth info about the connection inTapHandle tap.ServerInHandle framer *framer - hBuf *bytes.Buffer // the buffer for HPACK encoding - hEnc *hpack.Encoder // HPACK encoder // The max number of concurrent streams. maxStreams uint32 // controlBuf delivers all the control related tasks (e.g., window // updates, reset streams, and various settings) to the controller. controlBuf *controlBuffer - fc *inFlow - // sendQuotaPool provides flow control to outbound message. - sendQuotaPool *quotaPool - stats stats.Handler + fc *trInFlow + stats stats.Handler // Flag to keep track of reading activity on transport. // 1 is true and 0 is false. activity uint32 // Accessed atomically. @@ -101,13 +104,27 @@ type http2Server struct { drainChan chan struct{} state transportState activeStreams map[uint32]*Stream - // the per-stream outbound flow control window size set by the peer. - streamSendQuota uint32 // idle is the time instant when the connection went idle. // This is either the beginning of the connection or when the number of // RPCs go down to 0. // When the connection is busy, this value is set to 0. idle time.Time + + // Fields below are for channelz metric collection. + channelzID int64 // channelz unique identification number + czmu sync.RWMutex + kpCount int64 + // The number of streams that have started, including already finished ones. + streamsStarted int64 + // The number of streams that have ended successfully by sending frame with + // EoS bit set. + streamsSucceeded int64 + streamsFailed int64 + lastStreamCreated time.Time + msgSent int64 + msgRecv int64 + lastMsgSent time.Time + lastMsgRecv time.Time } // newHTTP2Server constructs a ServerTransport based on HTTP2. ConnectionError is @@ -182,32 +199,30 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err if kep.MinTime == 0 { kep.MinTime = defaultKeepalivePolicyMinTime } - var buf bytes.Buffer ctx, cancel := context.WithCancel(context.Background()) t := &http2Server{ ctx: ctx, cancel: cancel, + ctxDone: ctx.Done(), conn: conn, remoteAddr: conn.RemoteAddr(), localAddr: conn.LocalAddr(), authInfo: config.AuthInfo, framer: framer, - hBuf: &buf, - hEnc: hpack.NewEncoder(&buf), + readerDone: make(chan struct{}), + writerDone: make(chan struct{}), maxStreams: maxStreams, inTapHandle: config.InTapHandle, - controlBuf: newControlBuffer(), - fc: &inFlow{limit: uint32(icwz)}, - sendQuotaPool: newQuotaPool(defaultWindowSize), + fc: &trInFlow{limit: uint32(icwz)}, state: reachable, activeStreams: make(map[uint32]*Stream), - streamSendQuota: defaultWindowSize, stats: config.StatsHandler, kp: kp, idle: time.Now(), kep: kep, initialWindowSize: iwz, } + t.controlBuf = newControlBuffer(t.ctxDone) if dynamicWindow { t.bdpEst = &bdpEstimator{ bdp: initialWindowSize, @@ -222,8 +237,17 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err connBegin := &stats.ConnBegin{} t.stats.HandleConn(t.ctx, connBegin) } + if channelz.IsOn() { + t.channelzID = channelz.RegisterNormalSocket(t, config.ChannelzParentID, "") + } t.framer.writer.Flush() + defer func() { + if err != nil { + t.Close() + } + }() + // Check the validity of client preface. preface := make([]byte, len(clientPreface)) if _, err := io.ReadFull(t.conn, preface); err != nil { @@ -235,8 +259,7 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err frame, err := t.framer.fr.ReadFrame() if err == io.EOF || err == io.ErrUnexpectedEOF { - t.Close() - return + return nil, err } if err != nil { return nil, connectionErrorf(false, err, "transport: http2Server.HandleStreams failed to read initial settings frame: %v", err) @@ -249,8 +272,13 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err t.handleSettings(sf) go func() { - loopyWriter(t.ctx, t.controlBuf, t.itemHandler) - t.Close() + t.loopy = newLoopyWriter(serverSide, t.framer, t.controlBuf, t.bdpEst) + t.loopy.ssGoAwayHandler = t.outgoingGoAwayHandler + if err := t.loopy.run(); err != nil { + errorf("transport: loopyWriter.run returning. Err: %v", err) + } + t.conn.Close() + close(t.writerDone) }() go t.keepalive() return t, nil @@ -259,12 +287,16 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err // operateHeader takes action on the decoded headers. func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(*Stream), traceCtx func(context.Context, string) context.Context) (close bool) { streamID := frame.Header().StreamID - var state decodeState for _, hf := range frame.Fields { if err := state.processHeaderField(hf); err != nil { if se, ok := err.(StreamError); ok { - t.controlBuf.put(&resetStream{streamID, statusCodeConvTab[se.Code]}) + t.controlBuf.put(&cleanupStream{ + streamID: streamID, + rst: true, + rstCode: statusCodeConvTab[se.Code], + onWrite: func() {}, + }) } return } @@ -272,14 +304,14 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( buf := newRecvBuffer() s := &Stream{ - id: streamID, - st: t, - buf: buf, - fc: &inFlow{limit: uint32(t.initialWindowSize)}, - recvCompress: state.encoding, - method: state.method, + id: streamID, + st: t, + buf: buf, + fc: &inFlow{limit: uint32(t.initialWindowSize)}, + recvCompress: state.encoding, + method: state.method, + contentSubtype: state.contentSubtype, } - if frame.StreamEnded() { // s is just created by the caller. No lock needed. s.state = streamReadDone @@ -297,10 +329,6 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( pr.AuthInfo = t.authInfo } s.ctx = peer.NewContext(s.ctx, pr) - // Cache the current stream to the context so that the server application - // can find out. Required when the server wants to send some metadata - // back to the client (unary call only). - s.ctx = newContextWithStream(s.ctx, s) // Attach the received metadata to the context. if len(state.mdata) > 0 { s.ctx = metadata.NewIncomingContext(s.ctx, state.mdata) @@ -319,7 +347,12 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( s.ctx, err = t.inTapHandle(s.ctx, info) if err != nil { warningf("transport: http2Server.operateHeaders got an error from InTapHandle: %v", err) - t.controlBuf.put(&resetStream{s.id, http2.ErrCodeRefusedStream}) + t.controlBuf.put(&cleanupStream{ + streamID: s.id, + rst: true, + rstCode: http2.ErrCodeRefusedStream, + onWrite: func() {}, + }) return } } @@ -330,7 +363,12 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( } if uint32(len(t.activeStreams)) >= t.maxStreams { t.mu.Unlock() - t.controlBuf.put(&resetStream{streamID, http2.ErrCodeRefusedStream}) + t.controlBuf.put(&cleanupStream{ + streamID: streamID, + rst: true, + rstCode: http2.ErrCodeRefusedStream, + onWrite: func() {}, + }) return } if streamID%2 != 1 || streamID <= t.maxStreamID { @@ -340,13 +378,17 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( return true } t.maxStreamID = streamID - s.sendQuotaPool = newQuotaPool(int(t.streamSendQuota)) - s.localSendQuota = newQuotaPool(defaultLocalSendQuota) t.activeStreams[streamID] = s if len(t.activeStreams) == 1 { t.idle = time.Time{} } t.mu.Unlock() + if channelz.IsOn() { + t.czmu.Lock() + t.streamsStarted++ + t.lastStreamCreated = time.Now() + t.czmu.Unlock() + } s.requestRead = func(n int) { t.adjustWindow(s, uint32(n)) } @@ -362,15 +404,23 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( } t.stats.HandleRPC(s.ctx, inHeader) } + s.ctxDone = s.ctx.Done() + s.wq = newWriteQuota(defaultWriteQuota, s.ctxDone) s.trReader = &transportReader{ reader: &recvBufferReader{ - ctx: s.ctx, - recv: s.buf, + ctx: s.ctx, + ctxDone: s.ctxDone, + recv: s.buf, }, windowHandler: func(n int) { t.updateWindow(s, uint32(n)) }, } + // Register the stream with loopy. + t.controlBuf.put(®isterStream{ + streamID: s.id, + wq: s.wq, + }) handle(s) return } @@ -379,18 +429,26 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( // typically run in a separate goroutine. // traceCtx attaches trace to ctx and returns the new context. func (t *http2Server) HandleStreams(handle func(*Stream), traceCtx func(context.Context, string) context.Context) { + defer close(t.readerDone) for { frame, err := t.framer.fr.ReadFrame() atomic.StoreUint32(&t.activity, 1) if err != nil { if se, ok := err.(http2.StreamError); ok { + warningf("transport: http2Server.HandleStreams encountered http2.StreamError: %v", se) t.mu.Lock() s := t.activeStreams[se.StreamID] t.mu.Unlock() if s != nil { - t.closeStream(s) + t.closeStream(s, true, se.Code, nil, false) + } else { + t.controlBuf.put(&cleanupStream{ + streamID: se.StreamID, + rst: true, + rstCode: se.Code, + onWrite: func() {}, + }) } - t.controlBuf.put(&resetStream{se.StreamID, se.Code}) continue } if err == io.EOF || err == io.ErrUnexpectedEOF { @@ -444,33 +502,20 @@ func (t *http2Server) getStream(f http2.Frame) (*Stream, bool) { // of stream if the application is requesting data larger in size than // the window. func (t *http2Server) adjustWindow(s *Stream, n uint32) { - s.mu.Lock() - defer s.mu.Unlock() - if s.state == streamDone { - return - } if w := s.fc.maybeAdjust(n); w > 0 { - if cw := t.fc.resetPendingUpdate(); cw > 0 { - t.controlBuf.put(&windowUpdate{0, cw}) - } - t.controlBuf.put(&windowUpdate{s.id, w}) + t.controlBuf.put(&outgoingWindowUpdate{streamID: s.id, increment: w}) } + } // updateWindow adjusts the inbound quota for the stream and the transport. // Window updates will deliver to the controller for sending when // the cumulative quota exceeds the corresponding threshold. func (t *http2Server) updateWindow(s *Stream, n uint32) { - s.mu.Lock() - defer s.mu.Unlock() - if s.state == streamDone { - return - } if w := s.fc.onRead(n); w > 0 { - if cw := t.fc.resetPendingUpdate(); cw > 0 { - t.controlBuf.put(&windowUpdate{0, cw}) - } - t.controlBuf.put(&windowUpdate{s.id, w}) + t.controlBuf.put(&outgoingWindowUpdate{streamID: s.id, + increment: w, + }) } } @@ -484,13 +529,15 @@ func (t *http2Server) updateFlowControl(n uint32) { } t.initialWindowSize = int32(n) t.mu.Unlock() - t.controlBuf.put(&windowUpdate{0, t.fc.newLimit(n)}) - t.controlBuf.put(&settings{ - ack: false, + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: t.fc.newLimit(n), + }) + t.controlBuf.put(&outgoingSettings{ ss: []http2.Setting{ { ID: http2.SettingInitialWindowSize, - Val: uint32(n), + Val: n, }, }, }) @@ -501,7 +548,7 @@ func (t *http2Server) handleData(f *http2.DataFrame) { size := f.Header().Length var sendBDPPing bool if t.bdpEst != nil { - sendBDPPing = t.bdpEst.add(uint32(size)) + sendBDPPing = t.bdpEst.add(size) } // Decouple connection's flow control from application's read. // An update on connection's flow control should not depend on @@ -511,23 +558,22 @@ func (t *http2Server) handleData(f *http2.DataFrame) { // Decoupling the connection flow control will prevent other // active(fast) streams from starving in presence of slow or // inactive streams. - // - // Furthermore, if a bdpPing is being sent out we can piggyback - // connection's window update for the bytes we just received. + if w := t.fc.onData(size); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: w, + }) + } if sendBDPPing { - if size != 0 { // Could be an empty frame. - t.controlBuf.put(&windowUpdate{0, uint32(size)}) + // Avoid excessive ping detection (e.g. in an L7 proxy) + // by sending a window update prior to the BDP ping. + if w := t.fc.reset(); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: w, + }) } t.controlBuf.put(bdpPing) - } else { - if err := t.fc.onData(uint32(size)); err != nil { - errorf("transport: http2Server %v", err) - t.Close() - return - } - if w := t.fc.onRead(uint32(size)); w > 0 { - t.controlBuf.put(&windowUpdate{0, w}) - } } // Select the right stream to dispatch. s, ok := t.getStream(f) @@ -535,23 +581,15 @@ func (t *http2Server) handleData(f *http2.DataFrame) { return } if size > 0 { - s.mu.Lock() - if s.state == streamDone { - s.mu.Unlock() - return - } - if err := s.fc.onData(uint32(size)); err != nil { - s.mu.Unlock() - t.closeStream(s) - t.controlBuf.put(&resetStream{s.id, http2.ErrCodeFlowControl}) + if err := s.fc.onData(size); err != nil { + t.closeStream(s, true, http2.ErrCodeFlowControl, nil, false) return } if f.Header().Flags.Has(http2.FlagDataPadded) { - if w := s.fc.onRead(uint32(size) - uint32(len(f.Data()))); w > 0 { - t.controlBuf.put(&windowUpdate{s.id, w}) + if w := s.fc.onRead(size - uint32(len(f.Data()))); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{s.id, w}) } } - s.mu.Unlock() // TODO(bradfitz, zhaoq): A copy is required here because there is no // guarantee f.Data() is consumed before the arrival of next frame. // Can this copy be eliminated? @@ -563,11 +601,7 @@ func (t *http2Server) handleData(f *http2.DataFrame) { } if f.Header().Flags.Has(http2.FlagDataEndStream) { // Received the end of stream from the client. - s.mu.Lock() - if s.state != streamDone { - s.state = streamReadDone - } - s.mu.Unlock() + s.compareAndSwapState(streamActive, streamReadDone) s.write(recvMsg{err: io.EOF}) } } @@ -577,7 +611,7 @@ func (t *http2Server) handleRSTStream(f *http2.RSTStreamFrame) { if !ok { return } - t.closeStream(s) + t.closeStream(s, false, 0, nil, false) } func (t *http2Server) handleSettings(f *http2.SettingsFrame) { @@ -589,21 +623,9 @@ func (t *http2Server) handleSettings(f *http2.SettingsFrame) { ss = append(ss, s) return nil }) - t.controlBuf.put(&settings{ack: true, ss: ss}) -} - -func (t *http2Server) applySettings(ss []http2.Setting) { - for _, s := range ss { - if s.ID == http2.SettingInitialWindowSize { - t.mu.Lock() - for _, stream := range t.activeStreams { - stream.sendQuotaPool.addAndUpdate(int(s.Val) - int(t.streamSendQuota)) - } - t.streamSendQuota = s.Val - t.mu.Unlock() - } - - } + t.controlBuf.put(&incomingSettings{ + ss: ss, + }) } const ( @@ -656,56 +678,19 @@ func (t *http2Server) handlePing(f *http2.PingFrame) { if t.pingStrikes > maxPingStrikes { // Send goaway and close the connection. - errorf("transport: Got to too many pings from the client, closing the connection.") + errorf("transport: Got too many pings from the client, closing the connection.") t.controlBuf.put(&goAway{code: http2.ErrCodeEnhanceYourCalm, debugData: []byte("too_many_pings"), closeConn: true}) } } func (t *http2Server) handleWindowUpdate(f *http2.WindowUpdateFrame) { - id := f.Header().StreamID - incr := f.Increment - if id == 0 { - t.sendQuotaPool.add(int(incr)) - return - } - if s, ok := t.getStream(f); ok { - s.sendQuotaPool.add(int(incr)) - } + t.controlBuf.put(&incomingWindowUpdate{ + streamID: f.Header().StreamID, + increment: f.Increment, + }) } -// WriteHeader sends the header metedata md back to the client. -func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error { - select { - case <-s.ctx.Done(): - return ContextErr(s.ctx.Err()) - case <-t.ctx.Done(): - return ErrConnClosing - default: - } - - s.mu.Lock() - if s.headerOk || s.state == streamDone { - s.mu.Unlock() - return ErrIllegalHeaderWrite - } - s.headerOk = true - if md.Len() > 0 { - if s.header.Len() > 0 { - s.header = metadata.Join(s.header, md) - } else { - s.header = md - } - } - md = s.header - s.mu.Unlock() - // TODO(mmukhi): Benchmark if the perfomance gets better if count the metadata and other header fields - // first and create a slice of that exact size. - headerFields := make([]hpack.HeaderField, 0, 2) // at least :status, content-type will be there if none else. - headerFields = append(headerFields, hpack.HeaderField{Name: ":status", Value: "200"}) - headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: "application/grpc"}) - if s.sendCompress != "" { - headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-encoding", Value: s.sendCompress}) - } +func appendHeaderFieldsFromMD(headerFields []hpack.HeaderField, md metadata.MD) []hpack.HeaderField { for k, vv := range md { if isReservedHeader(k) { // Clients don't tolerate reading restricted headers after some non restricted ones were sent. @@ -715,18 +700,51 @@ func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error { headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) } } + return headerFields +} + +// WriteHeader sends the header metedata md back to the client. +func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error { + if s.updateHeaderSent() || s.getState() == streamDone { + return ErrIllegalHeaderWrite + } + s.hdrMu.Lock() + if md.Len() > 0 { + if s.header.Len() > 0 { + s.header = metadata.Join(s.header, md) + } else { + s.header = md + } + } + t.writeHeaderLocked(s) + s.hdrMu.Unlock() + return nil +} + +func (t *http2Server) writeHeaderLocked(s *Stream) { + // TODO(mmukhi): Benchmark if the performance gets better if count the metadata and other header fields + // first and create a slice of that exact size. + headerFields := make([]hpack.HeaderField, 0, 2) // at least :status, content-type will be there if none else. + headerFields = append(headerFields, hpack.HeaderField{Name: ":status", Value: "200"}) + headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: contentType(s.contentSubtype)}) + if s.sendCompress != "" { + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-encoding", Value: s.sendCompress}) + } + headerFields = appendHeaderFieldsFromMD(headerFields, s.header) t.controlBuf.put(&headerFrame{ streamID: s.id, hf: headerFields, endStream: false, + onWrite: func() { + atomic.StoreUint32(&t.resetPingStrikes, 1) + }, }) if t.stats != nil { - outHeader := &stats.OutHeader{ - //WireLength: // TODO(mmukhi): Revisit this later, if needed. - } + // Note: WireLength is not set in outHeader. + // TODO(mmukhi): Revisit this later, if needed. + outHeader := &stats.OutHeader{} t.stats.HandleRPC(s.Context(), outHeader) } - return nil } // WriteStatus sends stream status to the client and terminates the stream. @@ -734,37 +752,20 @@ func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error { // TODO(zhaoq): Now it indicates the end of entire stream. Revisit if early // OK is adopted. func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { - select { - case <-t.ctx.Done(): - return ErrConnClosing - default: - } - - var headersSent, hasHeader bool - s.mu.Lock() - if s.state == streamDone { - s.mu.Unlock() + if s.getState() == streamDone { return nil } - if s.headerOk { - headersSent = true - } - if s.header.Len() > 0 { - hasHeader = true - } - s.mu.Unlock() - - if !headersSent && hasHeader { - t.WriteHeader(s, nil) - headersSent = true - } - - // TODO(mmukhi): Benchmark if the perfomance gets better if count the metadata and other header fields + s.hdrMu.Lock() + // TODO(mmukhi): Benchmark if the performance gets better if count the metadata and other header fields // first and create a slice of that exact size. headerFields := make([]hpack.HeaderField, 0, 2) // grpc-status and grpc-message will be there if none else. - if !headersSent { - headerFields = append(headerFields, hpack.HeaderField{Name: ":status", Value: "200"}) - headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: "application/grpc"}) + if !s.updateHeaderSent() { // No headers have been sent. + if len(s.header) > 0 { // Send a separate header frame. + t.writeHeaderLocked(s) + } else { // Send a trailer only response. + headerFields = append(headerFields, hpack.HeaderField{Name: ":status", Value: "200"}) + headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: contentType(s.contentSubtype)}) + } } headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-status", Value: strconv.Itoa(int(st.Code()))}) headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-message", Value: encodeGrpcMessage(st.Message())}) @@ -773,126 +774,75 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { stBytes, err := proto.Marshal(p) if err != nil { // TODO: return error instead, when callers are able to handle it. - panic(err) + grpclog.Errorf("transport: failed to marshal rpc status: %v, error: %v", p, err) + } else { + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-status-details-bin", Value: encodeBinHeader(stBytes)}) } - - headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-status-details-bin", Value: encodeBinHeader(stBytes)}) } // Attach the trailer metadata. - for k, vv := range s.trailer { - // Clients don't tolerate reading restricted headers after some non restricted ones were sent. - if isReservedHeader(k) { - continue - } - for _, v := range vv { - headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) - } - } - t.controlBuf.put(&headerFrame{ + headerFields = appendHeaderFieldsFromMD(headerFields, s.trailer) + trailingHeader := &headerFrame{ streamID: s.id, hf: headerFields, endStream: true, - }) + onWrite: func() { + atomic.StoreUint32(&t.resetPingStrikes, 1) + }, + } + s.hdrMu.Unlock() + t.closeStream(s, false, 0, trailingHeader, true) if t.stats != nil { t.stats.HandleRPC(s.Context(), &stats.OutTrailer{}) } - t.closeStream(s) return nil } // Write converts the data into HTTP2 data frame and sends it out. Non-nil error // is returns if it fails (e.g., framing error, transport error). -func (t *http2Server) Write(s *Stream, hdr []byte, data []byte, opts *Options) (err error) { - select { - case <-s.ctx.Done(): - return ContextErr(s.ctx.Err()) - case <-t.ctx.Done(): - return ErrConnClosing - default: +func (t *http2Server) Write(s *Stream, hdr []byte, data []byte, opts *Options) error { + if !s.isHeaderSent() { // Headers haven't been written yet. + if err := t.WriteHeader(s, nil); err != nil { + // TODO(mmukhi, dfawley): Make sure this is the right code to return. + return streamErrorf(codes.Internal, "transport: %v", err) + } + } else { + // Writing headers checks for this condition. + if s.getState() == streamDone { + // TODO(mmukhi, dfawley): Should the server write also return io.EOF? + s.cancel() + select { + case <-t.ctx.Done(): + return ErrConnClosing + default: + } + return ContextErr(s.ctx.Err()) + } } - - var writeHeaderFrame bool - s.mu.Lock() - if s.state == streamDone { - s.mu.Unlock() - return streamErrorf(codes.Unknown, "the stream has been done") - } - if !s.headerOk { - writeHeaderFrame = true - } - s.mu.Unlock() - if writeHeaderFrame { - t.WriteHeader(s, nil) - } - // Add data to header frame so that we can equally distribute data across frames. + // Add some data to header frame so that we can equally distribute bytes across frames. emptyLen := http2MaxFrameLen - len(hdr) if emptyLen > len(data) { emptyLen = len(data) } hdr = append(hdr, data[:emptyLen]...) data = data[emptyLen:] - for _, r := range [][]byte{hdr, data} { - for len(r) > 0 { - size := http2MaxFrameLen - // Wait until the stream has some quota to send the data. - quotaChan, quotaVer := s.sendQuotaPool.acquireWithVersion() - sq, err := wait(s.ctx, t.ctx, nil, nil, quotaChan) - if err != nil { - return err - } - // Wait until the transport has some quota to send the data. - tq, err := wait(s.ctx, t.ctx, nil, nil, t.sendQuotaPool.acquire()) - if err != nil { - return err - } - if sq < size { - size = sq - } - if tq < size { - size = tq - } - if size > len(r) { - size = len(r) - } - p := r[:size] - ps := len(p) - if ps < tq { - // Overbooked transport quota. Return it back. - t.sendQuotaPool.add(tq - ps) - } - // Acquire local send quota to be able to write to the controlBuf. - ltq, err := wait(s.ctx, t.ctx, nil, nil, s.localSendQuota.acquire()) - if err != nil { - if _, ok := err.(ConnectionError); !ok { - t.sendQuotaPool.add(ps) - } - return err - } - s.localSendQuota.add(ltq - ps) // It's ok we make this negative. - // Reset ping strikes when sending data since this might cause - // the peer to send ping. + df := &dataFrame{ + streamID: s.id, + h: hdr, + d: data, + onEachWrite: func() { atomic.StoreUint32(&t.resetPingStrikes, 1) - success := func() { - t.controlBuf.put(&dataFrame{streamID: s.id, endStream: false, d: p, f: func() { - s.localSendQuota.add(ps) - }}) - if ps < sq { - // Overbooked stream quota. Return it back. - s.sendQuotaPool.lockedAdd(sq - ps) - } - r = r[ps:] - } - failure := func() { - s.sendQuotaPool.lockedAdd(sq) - } - if !s.sendQuotaPool.compareAndExecute(quotaVer, success, failure) { - t.sendQuotaPool.add(ps) - s.localSendQuota.add(ps) - } - } + }, } - return nil + if err := s.wq.get(int32(len(hdr) + len(data))); err != nil { + select { + case <-t.ctx.Done(): + return ErrConnClosing + default: + } + return ContextErr(s.ctx.Err()) + } + return t.controlBuf.put(df) } // keepalive running in a separate goroutine does the following: @@ -937,7 +887,7 @@ func (t *http2Server) keepalive() { // The connection has been idle for a duration of keepalive.MaxConnectionIdle or more. // Gracefully close the connection. t.drain(http2.ErrCodeNo, []byte{}) - // Reseting the timer so that the clean-up doesn't deadlock. + // Resetting the timer so that the clean-up doesn't deadlock. maxIdle.Reset(infinity) return } @@ -949,7 +899,7 @@ func (t *http2Server) keepalive() { case <-maxAge.C: // Close the connection after grace period. t.Close() - // Reseting the timer so that the clean-up doesn't deadlock. + // Resetting the timer so that the clean-up doesn't deadlock. maxAge.Reset(infinity) case <-t.ctx.Done(): } @@ -962,11 +912,16 @@ func (t *http2Server) keepalive() { } if pingSent { t.Close() - // Reseting the timer so that the clean-up doesn't deadlock. + // Resetting the timer so that the clean-up doesn't deadlock. keepalive.Reset(infinity) return } pingSent = true + if channelz.IsOn() { + t.czmu.Lock() + t.kpCount++ + t.czmu.Unlock() + } t.controlBuf.put(p) keepalive.Reset(t.kp.Timeout) case <-t.ctx.Done(): @@ -975,127 +930,6 @@ func (t *http2Server) keepalive() { } } -var goAwayPing = &ping{data: [8]byte{1, 6, 1, 8, 0, 3, 3, 9}} - -// TODO(mmukhi): A lot of this code(and code in other places in the tranpsort layer) -// is duplicated between the client and the server. -// The transport layer needs to be refactored to take care of this. -func (t *http2Server) itemHandler(i item) error { - switch i := i.(type) { - case *dataFrame: - if err := t.framer.fr.WriteData(i.streamID, i.endStream, i.d); err != nil { - return err - } - i.f() - return nil - case *headerFrame: - t.hBuf.Reset() - for _, f := range i.hf { - t.hEnc.WriteField(f) - } - first := true - endHeaders := false - for !endHeaders { - size := t.hBuf.Len() - if size > http2MaxFrameLen { - size = http2MaxFrameLen - } else { - endHeaders = true - } - var err error - if first { - first = false - err = t.framer.fr.WriteHeaders(http2.HeadersFrameParam{ - StreamID: i.streamID, - BlockFragment: t.hBuf.Next(size), - EndStream: i.endStream, - EndHeaders: endHeaders, - }) - } else { - err = t.framer.fr.WriteContinuation( - i.streamID, - endHeaders, - t.hBuf.Next(size), - ) - } - if err != nil { - return err - } - } - atomic.StoreUint32(&t.resetPingStrikes, 1) - return nil - case *windowUpdate: - return t.framer.fr.WriteWindowUpdate(i.streamID, i.increment) - case *settings: - if i.ack { - t.applySettings(i.ss) - return t.framer.fr.WriteSettingsAck() - } - return t.framer.fr.WriteSettings(i.ss...) - case *resetStream: - return t.framer.fr.WriteRSTStream(i.streamID, i.code) - case *goAway: - t.mu.Lock() - if t.state == closing { - t.mu.Unlock() - // The transport is closing. - return fmt.Errorf("transport: Connection closing") - } - sid := t.maxStreamID - if !i.headsUp { - // Stop accepting more streams now. - t.state = draining - t.mu.Unlock() - if err := t.framer.fr.WriteGoAway(sid, i.code, i.debugData); err != nil { - return err - } - if i.closeConn { - // Abruptly close the connection following the GoAway (via - // loopywriter). But flush out what's inside the buffer first. - t.framer.writer.Flush() - return fmt.Errorf("transport: Connection closing") - } - return nil - } - t.mu.Unlock() - // For a graceful close, send out a GoAway with stream ID of MaxUInt32, - // Follow that with a ping and wait for the ack to come back or a timer - // to expire. During this time accept new streams since they might have - // originated before the GoAway reaches the client. - // After getting the ack or timer expiration send out another GoAway this - // time with an ID of the max stream server intends to process. - if err := t.framer.fr.WriteGoAway(math.MaxUint32, http2.ErrCodeNo, []byte{}); err != nil { - return err - } - if err := t.framer.fr.WritePing(false, goAwayPing.data); err != nil { - return err - } - go func() { - timer := time.NewTimer(time.Minute) - defer timer.Stop() - select { - case <-t.drainChan: - case <-timer.C: - case <-t.ctx.Done(): - return - } - t.controlBuf.put(&goAway{code: i.code, debugData: i.debugData}) - }() - return nil - case *flushIO: - return t.framer.writer.Flush() - case *ping: - if !i.ack { - t.bdpEst.timesnap(i.data) - } - return t.framer.fr.WritePing(i.ack, i.data) - default: - err := status.Errorf(codes.Internal, "transport: http2Server.controller got unexpected item type %t", i) - errorf("%v", err) - return err - } -} - // Close starts shutting down the http2Server transport. // TODO(zhaoq): Now the destruction is not blocked on any pending streams. This // could cause some resource issue. Revisit this later. @@ -1109,8 +943,12 @@ func (t *http2Server) Close() error { streams := t.activeStreams t.activeStreams = nil t.mu.Unlock() + t.controlBuf.finish() t.cancel() err := t.conn.Close() + if channelz.IsOn() { + channelz.RemoveEntry(t.channelzID) + } // Cancel all active streams. for _, s := range streams { s.cancel() @@ -1124,27 +962,45 @@ func (t *http2Server) Close() error { // closeStream clears the footprint of a stream when the stream is not needed // any more. -func (t *http2Server) closeStream(s *Stream) { - t.mu.Lock() - delete(t.activeStreams, s.id) - if len(t.activeStreams) == 0 { - t.idle = time.Now() +func (t *http2Server) closeStream(s *Stream, rst bool, rstCode http2.ErrCode, hdr *headerFrame, eosReceived bool) { + if s.swapState(streamDone) == streamDone { + // If the stream was already done, return. + return } - if t.state == draining && len(t.activeStreams) == 0 { - defer t.Close() - } - t.mu.Unlock() // In case stream sending and receiving are invoked in separate // goroutines (e.g., bi-directional streaming), cancel needs to be // called to interrupt the potential blocking on other goroutines. s.cancel() - s.mu.Lock() - if s.state == streamDone { - s.mu.Unlock() - return + cleanup := &cleanupStream{ + streamID: s.id, + rst: rst, + rstCode: rstCode, + onWrite: func() { + t.mu.Lock() + if t.activeStreams != nil { + delete(t.activeStreams, s.id) + if len(t.activeStreams) == 0 { + t.idle = time.Now() + } + } + t.mu.Unlock() + if channelz.IsOn() { + t.czmu.Lock() + if eosReceived { + t.streamsSucceeded++ + } else { + t.streamsFailed++ + } + t.czmu.Unlock() + } + }, + } + if hdr != nil { + hdr.cleanup = cleanup + t.controlBuf.put(hdr) + } else { + t.controlBuf.put(cleanup) } - s.state = streamDone - s.mu.Unlock() } func (t *http2Server) RemoteAddr() net.Addr { @@ -1165,7 +1021,115 @@ func (t *http2Server) drain(code http2.ErrCode, debugData []byte) { t.controlBuf.put(&goAway{code: code, debugData: debugData, headsUp: true}) } -var rgen = rand.New(rand.NewSource(time.Now().UnixNano())) +var goAwayPing = &ping{data: [8]byte{1, 6, 1, 8, 0, 3, 3, 9}} + +// Handles outgoing GoAway and returns true if loopy needs to put itself +// in draining mode. +func (t *http2Server) outgoingGoAwayHandler(g *goAway) (bool, error) { + t.mu.Lock() + if t.state == closing { // TODO(mmukhi): This seems unnecessary. + t.mu.Unlock() + // The transport is closing. + return false, ErrConnClosing + } + sid := t.maxStreamID + if !g.headsUp { + // Stop accepting more streams now. + t.state = draining + if len(t.activeStreams) == 0 { + g.closeConn = true + } + t.mu.Unlock() + if err := t.framer.fr.WriteGoAway(sid, g.code, g.debugData); err != nil { + return false, err + } + if g.closeConn { + // Abruptly close the connection following the GoAway (via + // loopywriter). But flush out what's inside the buffer first. + t.framer.writer.Flush() + return false, fmt.Errorf("transport: Connection closing") + } + return true, nil + } + t.mu.Unlock() + // For a graceful close, send out a GoAway with stream ID of MaxUInt32, + // Follow that with a ping and wait for the ack to come back or a timer + // to expire. During this time accept new streams since they might have + // originated before the GoAway reaches the client. + // After getting the ack or timer expiration send out another GoAway this + // time with an ID of the max stream server intends to process. + if err := t.framer.fr.WriteGoAway(math.MaxUint32, http2.ErrCodeNo, []byte{}); err != nil { + return false, err + } + if err := t.framer.fr.WritePing(false, goAwayPing.data); err != nil { + return false, err + } + go func() { + timer := time.NewTimer(time.Minute) + defer timer.Stop() + select { + case <-t.drainChan: + case <-timer.C: + case <-t.ctx.Done(): + return + } + t.controlBuf.put(&goAway{code: g.code, debugData: g.debugData}) + }() + return false, nil +} + +func (t *http2Server) ChannelzMetric() *channelz.SocketInternalMetric { + t.czmu.RLock() + s := channelz.SocketInternalMetric{ + StreamsStarted: t.streamsStarted, + StreamsSucceeded: t.streamsSucceeded, + StreamsFailed: t.streamsFailed, + MessagesSent: t.msgSent, + MessagesReceived: t.msgRecv, + KeepAlivesSent: t.kpCount, + LastRemoteStreamCreatedTimestamp: t.lastStreamCreated, + LastMessageSentTimestamp: t.lastMsgSent, + LastMessageReceivedTimestamp: t.lastMsgRecv, + LocalFlowControlWindow: int64(t.fc.getSize()), + //socket options + LocalAddr: t.localAddr, + RemoteAddr: t.remoteAddr, + // Security + // RemoteName : + } + t.czmu.RUnlock() + s.RemoteFlowControlWindow = t.getOutFlowWindow() + return &s +} + +func (t *http2Server) IncrMsgSent() { + t.czmu.Lock() + t.msgSent++ + t.lastMsgSent = time.Now() + t.czmu.Unlock() +} + +func (t *http2Server) IncrMsgRecv() { + t.czmu.Lock() + t.msgRecv++ + t.lastMsgRecv = time.Now() + t.czmu.Unlock() +} + +func (t *http2Server) getOutFlowWindow() int64 { + resp := make(chan uint32) + timer := time.NewTimer(time.Second) + defer timer.Stop() + t.controlBuf.put(&outFlowControlSizeRequest{resp}) + select { + case sz := <-resp: + return int64(sz) + case <-t.ctxDone: + return -1 + case <-timer.C: + return -2 + } +} func getJitter(v time.Duration) time.Duration { if v == infinity { @@ -1173,6 +1137,6 @@ func getJitter(v time.Duration) time.Duration { } // Generate a jitter between +/- 10% of the value. r := int64(v / 10) - j := rgen.Int63n(2*r) - r + j := grpcrand.Int63n(2*r) - r return time.Duration(j) } diff --git a/vendor/google.golang.org/grpc/transport/http_util.go b/vendor/google.golang.org/grpc/transport/http_util.go index 39f878cfd5b..7d15c7d7426 100644 --- a/vendor/google.golang.org/grpc/transport/http_util.go +++ b/vendor/google.golang.org/grpc/transport/http_util.go @@ -23,12 +23,12 @@ import ( "bytes" "encoding/base64" "fmt" - "io" "net" "net/http" "strconv" "strings" "time" + "unicode/utf8" "github.com/golang/protobuf/proto" "golang.org/x/net/http2" @@ -46,6 +46,12 @@ const ( // http2IOBufSize specifies the buffer size for sending frames. defaultWriteBufSize = 32 * 1024 defaultReadBufSize = 32 * 1024 + // baseContentType is the base content-type for gRPC. This is a valid + // content-type on it's own, but can also include a content-subtype such as + // "proto" as a suffix after "+" or ";". See + // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests + // for more details. + baseContentType = "application/grpc" ) var ( @@ -64,7 +70,7 @@ var ( http2.ErrCodeConnect: codes.Internal, http2.ErrCodeEnhanceYourCalm: codes.ResourceExhausted, http2.ErrCodeInadequateSecurity: codes.PermissionDenied, - http2.ErrCodeHTTP11Required: codes.FailedPrecondition, + http2.ErrCodeHTTP11Required: codes.Internal, } statusCodeConvTab = map[codes.Code]http2.ErrCode{ codes.Internal: http2.ErrCodeInternal, @@ -111,9 +117,10 @@ type decodeState struct { timeout time.Duration method string // key-value metadata map from the peer. - mdata map[string][]string - statsTags []byte - statsTrace []byte + mdata map[string][]string + statsTags []byte + statsTrace []byte + contentSubtype string } // isReservedHeader checks whether hdr belongs to HTTP2 headers @@ -125,6 +132,7 @@ func isReservedHeader(hdr string) bool { } switch hdr { case "content-type", + "user-agent", "grpc-message-type", "grpc-encoding", "grpc-message", @@ -138,28 +146,55 @@ func isReservedHeader(hdr string) bool { } } -// isWhitelistedPseudoHeader checks whether hdr belongs to HTTP2 pseudoheaders -// that should be propagated into metadata visible to users. -func isWhitelistedPseudoHeader(hdr string) bool { +// isWhitelistedHeader checks whether hdr should be propagated +// into metadata visible to users. +func isWhitelistedHeader(hdr string) bool { switch hdr { - case ":authority": + case ":authority", "user-agent": return true default: return false } } -func validContentType(t string) bool { - e := "application/grpc" - if !strings.HasPrefix(t, e) { - return false +// contentSubtype returns the content-subtype for the given content-type. The +// given content-type must be a valid content-type that starts with +// "application/grpc". A content-subtype will follow "application/grpc" after a +// "+" or ";". See +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +// +// If contentType is not a valid content-type for gRPC, the boolean +// will be false, otherwise true. If content-type == "application/grpc", +// "application/grpc+", or "application/grpc;", the boolean will be true, +// but no content-subtype will be returned. +// +// contentType is assumed to be lowercase already. +func contentSubtype(contentType string) (string, bool) { + if contentType == baseContentType { + return "", true } - // Support variations on the content-type - // (e.g. "application/grpc+blah", "application/grpc;blah"). - if len(t) > len(e) && t[len(e)] != '+' && t[len(e)] != ';' { - return false + if !strings.HasPrefix(contentType, baseContentType) { + return "", false } - return true + // guaranteed since != baseContentType and has baseContentType prefix + switch contentType[len(baseContentType)] { + case '+', ';': + // this will return true for "application/grpc+" or "application/grpc;" + // which the previous validContentType function tested to be valid, so we + // just say that no content-subtype is specified in this case + return contentType[len(baseContentType)+1:], true + default: + return "", false + } +} + +// contentSubtype is assumed to be lowercase +func contentType(contentSubtype string) string { + if contentSubtype == "" { + return baseContentType + } + return baseContentType + "+" + contentSubtype } func (d *decodeState) status() *status.Status { @@ -228,9 +263,9 @@ func (d *decodeState) decodeResponseHeader(frame *http2.MetaHeadersFrame) error // gRPC status doesn't exist and http status is OK. // Set rawStatusCode to be unknown and return nil error. // So that, if the stream has ended this Unknown status - // will be propogated to the user. + // will be propagated to the user. // Otherwise, it will be ignored. In which case, status from - // a later trailer, that has StreamEnded flag set, is propogated. + // a later trailer, that has StreamEnded flag set, is propagated. code := int(codes.Unknown) d.rawStatusCode = &code return nil @@ -247,9 +282,16 @@ func (d *decodeState) addMetadata(k, v string) { func (d *decodeState) processHeaderField(f hpack.HeaderField) error { switch f.Name { case "content-type": - if !validContentType(f.Value) { - return streamErrorf(codes.FailedPrecondition, "transport: received the unexpected content-type %q", f.Value) + contentSubtype, validContentType := contentSubtype(f.Value) + if !validContentType { + return streamErrorf(codes.Internal, "transport: received the unexpected content-type %q", f.Value) } + d.contentSubtype = contentSubtype + // TODO: do we want to propagate the whole content-type in the metadata, + // or come up with a way to just propagate the content-subtype if it was set? + // ie {"content-type": "application/grpc+proto"} or {"content-subtype": "proto"} + // in the metadata? + d.addMetadata(f.Name, f.Value) case "grpc-encoding": d.encoding = f.Value case "grpc-status": @@ -299,7 +341,7 @@ func (d *decodeState) processHeaderField(f hpack.HeaderField) error { d.statsTrace = v d.addMetadata(f.Name, string(v)) default: - if isReservedHeader(f.Name) && !isWhitelistedPseudoHeader(f.Name) { + if isReservedHeader(f.Name) && !isWhitelistedHeader(f.Name) { break } v, err := decodeMetadataHeader(f.Name, f.Value) @@ -307,7 +349,7 @@ func (d *decodeState) processHeaderField(f hpack.HeaderField) error { errorf("Failed to decode metadata header (%q, %q): %v", f.Name, f.Value, err) return nil } - d.addMetadata(f.Name, string(v)) + d.addMetadata(f.Name, v) } return nil } @@ -396,16 +438,17 @@ func decodeTimeout(s string) (time.Duration, error) { const ( spaceByte = ' ' - tildaByte = '~' + tildeByte = '~' percentByte = '%' ) // encodeGrpcMessage is used to encode status code in header field -// "grpc-message". -// It checks to see if each individual byte in msg is an -// allowable byte, and then either percent encoding or passing it through. -// When percent encoding, the byte is converted into hexadecimal notation -// with a '%' prepended. +// "grpc-message". It does percent encoding and also replaces invalid utf-8 +// characters with Unicode replacement character. +// +// It checks to see if each individual byte in msg is an allowable byte, and +// then either percent encoding or passing it through. When percent encoding, +// the byte is converted into hexadecimal notation with a '%' prepended. func encodeGrpcMessage(msg string) string { if msg == "" { return "" @@ -413,7 +456,7 @@ func encodeGrpcMessage(msg string) string { lenMsg := len(msg) for i := 0; i < lenMsg; i++ { c := msg[i] - if !(c >= spaceByte && c < tildaByte && c != percentByte) { + if !(c >= spaceByte && c <= tildeByte && c != percentByte) { return encodeGrpcMessageUnchecked(msg) } } @@ -422,14 +465,26 @@ func encodeGrpcMessage(msg string) string { func encodeGrpcMessageUnchecked(msg string) string { var buf bytes.Buffer - lenMsg := len(msg) - for i := 0; i < lenMsg; i++ { - c := msg[i] - if c >= spaceByte && c < tildaByte && c != percentByte { - buf.WriteByte(c) - } else { - buf.WriteString(fmt.Sprintf("%%%02X", c)) + for len(msg) > 0 { + r, size := utf8.DecodeRuneInString(msg) + for _, b := range []byte(string(r)) { + if size > 1 { + // If size > 1, r is not ascii. Always do percent encoding. + buf.WriteString(fmt.Sprintf("%%%02X", b)) + continue + } + + // The for loop is necessary even if size == 1. r could be + // utf8.RuneError. + // + // fmt.Sprintf("%%%02X", utf8.RuneError) gives "%FFFD". + if b >= spaceByte && b <= tildeByte && b != percentByte { + buf.WriteByte(b) + } else { + buf.WriteString(fmt.Sprintf("%%%02X", b)) + } } + msg = msg[size:] } return buf.String() } @@ -468,19 +523,67 @@ func decodeGrpcMessageUnchecked(msg string) string { return buf.String() } +type bufWriter struct { + buf []byte + offset int + batchSize int + conn net.Conn + err error + + onFlush func() +} + +func newBufWriter(conn net.Conn, batchSize int) *bufWriter { + return &bufWriter{ + buf: make([]byte, batchSize*2), + batchSize: batchSize, + conn: conn, + } +} + +func (w *bufWriter) Write(b []byte) (n int, err error) { + if w.err != nil { + return 0, w.err + } + for len(b) > 0 { + nn := copy(w.buf[w.offset:], b) + b = b[nn:] + w.offset += nn + n += nn + if w.offset >= w.batchSize { + err = w.Flush() + } + } + return n, err +} + +func (w *bufWriter) Flush() error { + if w.err != nil { + return w.err + } + if w.offset == 0 { + return nil + } + if w.onFlush != nil { + w.onFlush() + } + _, w.err = w.conn.Write(w.buf[:w.offset]) + w.offset = 0 + return w.err +} + type framer struct { - numWriters int32 - reader io.Reader - writer *bufio.Writer - fr *http2.Framer + writer *bufWriter + fr *http2.Framer } func newFramer(conn net.Conn, writeBufferSize, readBufferSize int) *framer { + r := bufio.NewReaderSize(conn, readBufferSize) + w := newBufWriter(conn, writeBufferSize) f := &framer{ - reader: bufio.NewReaderSize(conn, readBufferSize), - writer: bufio.NewWriterSize(conn, writeBufferSize), + writer: w, + fr: http2.NewFramer(w, r), } - f.fr = http2.NewFramer(f.writer, f.reader) // Opt-in to Frame reuse API on framer to reduce garbage. // Frames aren't safe to read from after a subsequent call to ReadFrame. f.fr.SetReuseFrames() diff --git a/vendor/google.golang.org/grpc/transport/transport.go b/vendor/google.golang.org/grpc/transport/transport.go index ce5cb74d2ee..f51f878884d 100644 --- a/vendor/google.golang.org/grpc/transport/transport.go +++ b/vendor/google.golang.org/grpc/transport/transport.go @@ -17,19 +17,19 @@ */ // Package transport defines and implements message oriented communication -// channel to complete various transactions (e.g., an RPC). -package transport +// channel to complete various transactions (e.g., an RPC). It is meant for +// grpc-internal usage and is not intended to be imported directly by users. +package transport // externally used as import "google.golang.org/grpc/transport" import ( - stdctx "context" + "errors" "fmt" "io" "net" "sync" - "time" + "sync/atomic" "golang.org/x/net/context" - "golang.org/x/net/http2" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" @@ -58,6 +58,7 @@ type recvBuffer struct { c chan recvMsg mu sync.Mutex backlog []recvMsg + err error } func newRecvBuffer() *recvBuffer { @@ -69,6 +70,13 @@ func newRecvBuffer() *recvBuffer { func (b *recvBuffer) put(r recvMsg) { b.mu.Lock() + if b.err != nil { + b.mu.Unlock() + // An error had occurred earlier, don't accept more + // data or errors. + return + } + b.err = r.err if len(b.backlog) == 0 { select { case b.c <- r: @@ -102,14 +110,15 @@ func (b *recvBuffer) get() <-chan recvMsg { return b.c } +// // recvBufferReader implements io.Reader interface to read the data from // recvBuffer. type recvBufferReader struct { - ctx context.Context - goAway chan struct{} - recv *recvBuffer - last []byte // Stores the remaining data in the previous calls. - err error + ctx context.Context + ctxDone <-chan struct{} // cache of ctx.Done() (for performance). + recv *recvBuffer + last []byte // Stores the remaining data in the previous calls. + err error } // Read reads the next len(p) bytes from last. If last is drained, it tries to @@ -131,10 +140,8 @@ func (r *recvBufferReader) read(p []byte) (n int, err error) { return copied, nil } select { - case <-r.ctx.Done(): + case <-r.ctxDone: return 0, ContextErr(r.ctx.Err()) - case <-r.goAway: - return 0, ErrStreamDrain case m := <-r.recv.get(): r.recv.load() if m.err != nil { @@ -146,61 +153,7 @@ func (r *recvBufferReader) read(p []byte) (n int, err error) { } } -// All items in an out of a controlBuffer should be the same type. -type item interface { - item() -} - -// controlBuffer is an unbounded channel of item. -type controlBuffer struct { - c chan item - mu sync.Mutex - backlog []item -} - -func newControlBuffer() *controlBuffer { - b := &controlBuffer{ - c: make(chan item, 1), - } - return b -} - -func (b *controlBuffer) put(r item) { - b.mu.Lock() - if len(b.backlog) == 0 { - select { - case b.c <- r: - b.mu.Unlock() - return - default: - } - } - b.backlog = append(b.backlog, r) - b.mu.Unlock() -} - -func (b *controlBuffer) load() { - b.mu.Lock() - if len(b.backlog) > 0 { - select { - case b.c <- b.backlog[0]: - b.backlog[0] = nil - b.backlog = b.backlog[1:] - default: - } - } - b.mu.Unlock() -} - -// get returns the channel that receives an item in the buffer. -// -// Upon receipt of an item, the caller should call load to send another -// item onto the channel if there is any. -func (b *controlBuffer) get() <-chan item { - return b.c -} - -type streamState uint8 +type streamState uint32 const ( streamActive streamState = iota @@ -211,66 +164,93 @@ const ( // Stream represents an RPC in the transport layer. type Stream struct { - id uint32 - // nil for client side Stream. - st ServerTransport - // ctx is the associated context of the stream. - ctx context.Context - // cancel is always nil for client side Stream. - cancel context.CancelFunc - // done is closed when the final status arrives. - done chan struct{} - // goAway is closed when the server sent GoAways signal before this stream was initiated. - goAway chan struct{} - // method records the associated RPC method of the stream. - method string + id uint32 + st ServerTransport // nil for client side Stream + ctx context.Context // the associated context of the stream + cancel context.CancelFunc // always nil for client side Stream + done chan struct{} // closed at the end of stream to unblock writers. On the client side. + ctxDone <-chan struct{} // same as done chan but for server side. Cache of ctx.Done() (for performance) + method string // the associated RPC method of the stream recvCompress string sendCompress string buf *recvBuffer trReader io.Reader fc *inFlow recvQuota uint32 - - // TODO: Remote this unused variable. - // The accumulated inbound quota pending for window update. - updateQuota uint32 + wq *writeQuota // Callback to state application's intentions to read data. This - // is used to adjust flow control, if need be. + // is used to adjust flow control, if needed. requestRead func(int) - sendQuotaPool *quotaPool - localSendQuota *quotaPool - // Close headerChan to indicate the end of reception of header metadata. - headerChan chan struct{} - // header caches the received header metadata. - header metadata.MD - // The key-value map of trailer metadata. - trailer metadata.MD + headerChan chan struct{} // closed to indicate the end of header metadata. + headerDone uint32 // set when headerChan is closed. Used to avoid closing headerChan multiple times. - mu sync.RWMutex // guard the following - // headerOK becomes true from the first header is about to send. - headerOk bool - state streamState - // true iff headerChan is closed. Used to avoid closing headerChan - // multiple times. - headerDone bool - // the status error received from the server. + // hdrMu protects header and trailer metadata on the server-side. + hdrMu sync.Mutex + header metadata.MD // the received header metadata. + trailer metadata.MD // the key-value map of trailer metadata. + + // On the server-side, headerSent is atomically set to 1 when the headers are sent out. + headerSent uint32 + + state streamState + + // On client-side it is the status error received from the server. + // On server-side it is unused. status *status.Status - // rstStream indicates whether a RST_STREAM frame needs to be sent - // to the server to signify that this stream is closing. - rstStream bool - // rstError is the error that needs to be sent along with the RST_STREAM frame. - rstError http2.ErrCode - // bytesSent and bytesReceived indicates whether any bytes have been sent or - // received on this stream. - bytesSent bool - bytesReceived bool + + bytesReceived uint32 // indicates whether any bytes have been received on this stream + unprocessed uint32 // set if the server sends a refused stream or GOAWAY including this stream + + // contentSubtype is the content-subtype for requests. + // this must be lowercase or the behavior is undefined. + contentSubtype string +} + +// isHeaderSent is only valid on the server-side. +func (s *Stream) isHeaderSent() bool { + return atomic.LoadUint32(&s.headerSent) == 1 +} + +// updateHeaderSent updates headerSent and returns true +// if it was alreay set. It is valid only on server-side. +func (s *Stream) updateHeaderSent() bool { + return atomic.SwapUint32(&s.headerSent, 1) == 1 +} + +func (s *Stream) swapState(st streamState) streamState { + return streamState(atomic.SwapUint32((*uint32)(&s.state), uint32(st))) +} + +func (s *Stream) compareAndSwapState(oldState, newState streamState) bool { + return atomic.CompareAndSwapUint32((*uint32)(&s.state), uint32(oldState), uint32(newState)) +} + +func (s *Stream) getState() streamState { + return streamState(atomic.LoadUint32((*uint32)(&s.state))) +} + +func (s *Stream) waitOnHeader() error { + if s.headerChan == nil { + // On the server headerChan is always nil since a stream originates + // only after having received headers. + return nil + } + select { + case <-s.ctx.Done(): + return ContextErr(s.ctx.Err()) + case <-s.headerChan: + return nil + } } // RecvCompress returns the compression algorithm applied to the inbound // message. It is empty string if there is no compression applied. func (s *Stream) RecvCompress() string { + if err := s.waitOnHeader(); err != nil { + return "" + } return s.recvCompress } @@ -285,28 +265,17 @@ func (s *Stream) Done() <-chan struct{} { return s.done } -// GoAway returns a channel which is closed when the server sent GoAways signal -// before this stream was initiated. -func (s *Stream) GoAway() <-chan struct{} { - return s.goAway -} - // Header acquires the key-value pairs of header metadata once it // is available. It blocks until i) the metadata is ready or ii) there is no // header metadata or iii) the stream is canceled/expired. func (s *Stream) Header() (metadata.MD, error) { - var err error - select { - case <-s.ctx.Done(): - err = ContextErr(s.ctx.Err()) - case <-s.goAway: - err = ErrStreamDrain - case <-s.headerChan: - return s.header.Copy(), nil - } + err := s.waitOnHeader() // Even if the stream is closed, header is returned if available. select { case <-s.headerChan: + if s.header == nil { + return nil, nil + } return s.header.Copy(), nil default: } @@ -316,10 +285,10 @@ func (s *Stream) Header() (metadata.MD, error) { // Trailer returns the cached trailer metedata. Note that if it is not called // after the entire stream is done, it could return an empty MD. Client // side only. +// It can be safely read only after stream has ended that is either read +// or write have returned io.EOF. func (s *Stream) Trailer() metadata.MD { - s.mu.RLock() c := s.trailer.Copy() - s.mu.RUnlock() return c } @@ -329,6 +298,15 @@ func (s *Stream) ServerTransport() ServerTransport { return s.st } +// ContentSubtype returns the content-subtype for a request. For example, a +// content-subtype of "proto" will result in a content-type of +// "application/grpc+proto". This will always be lowercase. See +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +func (s *Stream) ContentSubtype() string { + return s.contentSubtype +} + // Context returns the context of the stream. func (s *Stream) Context() context.Context { return s.ctx @@ -340,36 +318,49 @@ func (s *Stream) Method() string { } // Status returns the status received from the server. +// Status can be read safely only after the stream has ended, +// that is, read or write has returned io.EOF. func (s *Stream) Status() *status.Status { return s.status } // SetHeader sets the header metadata. This can be called multiple times. // Server side only. +// This should not be called in parallel to other data writes. func (s *Stream) SetHeader(md metadata.MD) error { - s.mu.Lock() - if s.headerOk || s.state == streamDone { - s.mu.Unlock() - return ErrIllegalHeaderWrite - } if md.Len() == 0 { - s.mu.Unlock() return nil } + if s.isHeaderSent() || s.getState() == streamDone { + return ErrIllegalHeaderWrite + } + s.hdrMu.Lock() s.header = metadata.Join(s.header, md) - s.mu.Unlock() + s.hdrMu.Unlock() return nil } +// SendHeader sends the given header metadata. The given metadata is +// combined with any metadata set by previous calls to SetHeader and +// then written to the transport stream. +func (s *Stream) SendHeader(md metadata.MD) error { + t := s.ServerTransport() + return t.WriteHeader(s, md) +} + // SetTrailer sets the trailer metadata which will be sent with the RPC status // by the server. This can be called multiple times. Server side only. +// This should not be called parallel to other data writes. func (s *Stream) SetTrailer(md metadata.MD) error { if md.Len() == 0 { return nil } - s.mu.Lock() + if s.getState() == streamDone { + return ErrIllegalHeaderWrite + } + s.hdrMu.Lock() s.trailer = metadata.Join(s.trailer, md) - s.mu.Unlock() + s.hdrMu.Unlock() return nil } @@ -409,28 +400,15 @@ func (t *transportReader) Read(p []byte) (n int, err error) { return } -// finish sets the stream's state and status, and closes the done channel. -// s.mu must be held by the caller. st must always be non-nil. -func (s *Stream) finish(st *status.Status) { - s.status = st - s.state = streamDone - close(s.done) -} - -// BytesSent indicates whether any bytes have been sent on this stream. -func (s *Stream) BytesSent() bool { - s.mu.Lock() - bs := s.bytesSent - s.mu.Unlock() - return bs -} - // BytesReceived indicates whether any bytes have been received on this stream. func (s *Stream) BytesReceived() bool { - s.mu.Lock() - br := s.bytesReceived - s.mu.Unlock() - return br + return atomic.LoadUint32(&s.bytesReceived) == 1 +} + +// Unprocessed indicates whether the server did not process this stream -- +// i.e. it sent a refused stream or GOAWAY including this stream ID. +func (s *Stream) Unprocessed() bool { + return atomic.LoadUint32(&s.unprocessed) == 1 } // GoString is implemented by Stream so context.String() won't @@ -439,21 +417,6 @@ func (s *Stream) GoString() string { return fmt.Sprintf("", s, s.method) } -// The key to save transport.Stream in the context. -type streamKey struct{} - -// newContextWithStream creates a new context from ctx and attaches stream -// to it. -func newContextWithStream(ctx context.Context, stream *Stream) context.Context { - return context.WithValue(ctx, streamKey{}, stream) -} - -// StreamFromContext returns the stream saved in ctx. -func StreamFromContext(ctx context.Context) (s *Stream, ok bool) { - s, ok = ctx.Value(streamKey{}).(*Stream) - return -} - // state of transport type transportState int @@ -475,6 +438,7 @@ type ServerConfig struct { InitialConnWindowSize int32 WriteBufferSize int ReadBufferSize int + ChannelzParentID int64 } // NewServerTransport creates a ServerTransport with conn or non-nil error @@ -510,18 +474,21 @@ type ConnectOptions struct { WriteBufferSize int // ReadBufferSize sets the size of read buffer, which in turn determines how much data can be read at most for one read syscall. ReadBufferSize int + // ChannelzParentID sets the addrConn id which initiate the creation of this client transport. + ChannelzParentID int64 } // TargetInfo contains the information of the target such as network address and metadata. type TargetInfo struct { - Addr string - Metadata interface{} + Addr string + Metadata interface{} + Authority string } // NewClientTransport establishes the transport with the required ConnectOptions // and returns it to the caller. -func NewClientTransport(ctx context.Context, target TargetInfo, opts ConnectOptions, timeout time.Duration) (ClientTransport, error) { - return newHTTP2Client(ctx, target, opts, timeout) +func NewClientTransport(connectCtx, ctx context.Context, target TargetInfo, opts ConnectOptions, onSuccess func()) (ClientTransport, error) { + return newHTTP2Client(connectCtx, ctx, target, opts, onSuccess) } // Options provides additional hints and information for message @@ -545,10 +512,6 @@ type CallHdr struct { // Method specifies the operation to perform. Method string - // RecvCompress specifies the compression algorithm applied on - // inbound messages. - RecvCompress string - // SendCompress specifies the compression algorithm applied on // outbound message. SendCompress string @@ -563,6 +526,14 @@ type CallHdr struct { // for performance purposes. // If it's false, new stream will never be flushed. Flush bool + + // ContentSubtype specifies the content-subtype for a request. For example, a + // content-subtype of "proto" will result in a content-type of + // "application/grpc+proto". The value of ContentSubtype must be all + // lowercase, otherwise the behavior is undefined. See + // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests + // for more details. + ContentSubtype string } // ClientTransport is the common interface for all gRPC client-side transport @@ -604,6 +575,12 @@ type ClientTransport interface { // GetGoAwayReason returns the reason why GoAway frame was received. GetGoAwayReason() GoAwayReason + + // IncrMsgSent increments the number of message sent through this transport. + IncrMsgSent() + + // IncrMsgRecv increments the number of message received through this transport. + IncrMsgRecv() } // ServerTransport is the common interface for all gRPC server-side transport @@ -637,6 +614,12 @@ type ServerTransport interface { // Drain notifies the client this ServerTransport stops accepting new RPCs. Drain() + + // IncrMsgSent increments the number of message sent through this transport. + IncrMsgSent() + + // IncrMsgRecv increments the number of message received through this transport. + IncrMsgRecv() } // streamErrorf creates an StreamError with the specified error code and description. @@ -686,9 +669,16 @@ func (e ConnectionError) Origin() error { var ( // ErrConnClosing indicates that the transport is closing. ErrConnClosing = connectionErrorf(true, nil, "transport is closing") - // ErrStreamDrain indicates that the stream is rejected by the server because - // the server stops accepting new RPCs. - ErrStreamDrain = streamErrorf(codes.Unavailable, "the server stops accepting new RPCs") + // errStreamDrain indicates that the stream is rejected because the + // connection is draining. This could be caused by goaway or balancer + // removing the address. + errStreamDrain = streamErrorf(codes.Unavailable, "the connection is draining") + // errStreamDone is returned from write at the client side to indiacte application + // layer of an error. + errStreamDone = errors.New("the stream is done") + // StatusGoAway indicates that the server sent a GOAWAY that included this + // stream's ID in unprocessed RPCs. + statusGoAway = status.New(codes.Unavailable, "the stream is rejected because server is draining the connection") ) // TODO: See if we can replace StreamError with status package errors. @@ -703,75 +693,16 @@ func (e StreamError) Error() string { return fmt.Sprintf("stream error: code = %s desc = %q", e.Code, e.Desc) } -// wait blocks until it can receive from one of the provided contexts or channels -func wait(ctx, tctx context.Context, done, goAway <-chan struct{}, proceed <-chan int) (int, error) { - select { - case <-ctx.Done(): - return 0, ContextErr(ctx.Err()) - case <-done: - return 0, io.EOF - case <-goAway: - return 0, ErrStreamDrain - case <-tctx.Done(): - return 0, ErrConnClosing - case i := <-proceed: - return i, nil - } -} - -// ContextErr converts the error from context package into a StreamError. -func ContextErr(err error) StreamError { - switch err { - case context.DeadlineExceeded, stdctx.DeadlineExceeded: - return streamErrorf(codes.DeadlineExceeded, "%v", err) - case context.Canceled, stdctx.Canceled: - return streamErrorf(codes.Canceled, "%v", err) - } - return streamErrorf(codes.Internal, "Unexpected error from context packet: %v", err) -} - // GoAwayReason contains the reason for the GoAway frame received. type GoAwayReason uint8 const ( - // Invalid indicates that no GoAway frame is received. - Invalid GoAwayReason = 0 - // NoReason is the default value when GoAway frame is received. - NoReason GoAwayReason = 1 - // TooManyPings indicates that a GoAway frame with ErrCodeEnhanceYourCalm - // was received and that the debug data said "too_many_pings". - TooManyPings GoAwayReason = 2 + // GoAwayInvalid indicates that no GoAway frame is received. + GoAwayInvalid GoAwayReason = 0 + // GoAwayNoReason is the default value when GoAway frame is received. + GoAwayNoReason GoAwayReason = 1 + // GoAwayTooManyPings indicates that a GoAway frame with + // ErrCodeEnhanceYourCalm was received and that the debug data said + // "too_many_pings". + GoAwayTooManyPings GoAwayReason = 2 ) - -// loopyWriter is run in a separate go routine. It is the single code path that will -// write data on wire. -func loopyWriter(ctx context.Context, cbuf *controlBuffer, handler func(item) error) { - for { - select { - case i := <-cbuf.get(): - cbuf.load() - if err := handler(i); err != nil { - return - } - case <-ctx.Done(): - return - } - hasData: - for { - select { - case i := <-cbuf.get(): - cbuf.load() - if err := handler(i); err != nil { - return - } - case <-ctx.Done(): - return - default: - if err := handler(&flushIO{}); err != nil { - return - } - break hasData - } - } - } -} diff --git a/vendor/google.golang.org/grpc/version.go b/vendor/google.golang.org/grpc/version.go new file mode 100644 index 00000000000..7f124fbd532 --- /dev/null +++ b/vendor/google.golang.org/grpc/version.go @@ -0,0 +1,22 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +// Version is the current grpc version. +const Version = "1.13.0" diff --git a/vendor/google.golang.org/grpc/vet.sh b/vendor/google.golang.org/grpc/vet.sh index d006a426347..079bc2896d7 100755 --- a/vendor/google.golang.org/grpc/vet.sh +++ b/vendor/google.golang.org/grpc/vet.sh @@ -1,5 +1,10 @@ #!/bin/bash +if [[ `uname -a` = *"Darwin"* ]]; then + echo "It seems you are running on Mac. This script does not work on Mac. See https://github.com/grpc/grpc-go/issues/2047" + exit 1 +fi + set -ex # Exit on error; debugging enabled. set -o pipefail # Fail a pipe if any sub-command fails. @@ -8,12 +13,6 @@ die() { exit 1 } -# TODO: Remove this check and the mangling below once "context" is imported -# directly. -if git status --porcelain | read; then - die "Uncommitted or untracked files found; commit changes first" -fi - PATH="$GOPATH/bin:$GOROOT/bin:$PATH" # Check proto in manual runs or cron runs. @@ -28,8 +27,8 @@ if [ "$1" = "-install" ]; then github.com/golang/lint/golint \ golang.org/x/tools/cmd/goimports \ honnef.co/go/tools/cmd/staticcheck \ - github.com/golang/protobuf/protoc-gen-go \ - golang.org/x/tools/cmd/stringer + github.com/client9/misspell/cmd/misspell \ + github.com/golang/protobuf/protoc-gen-go if [[ "$check_proto" = "true" ]]; then if [[ "$TRAVIS" = "true" ]]; then PROTOBUF_VERSION=3.3.0 @@ -48,10 +47,18 @@ elif [[ "$#" -ne 0 ]]; then die "Unknown argument(s): $*" fi +# TODO: Remove this check and the mangling below once "context" is imported +# directly. +if git status --porcelain | read; then + die "Uncommitted or untracked files found; commit changes first" +fi + git ls-files "*.go" | xargs grep -L "\(Copyright [0-9]\{4,\} gRPC authors\)\|DO NOT EDIT" 2>&1 | tee /dev/stderr | (! read) +git ls-files "*.go" | xargs grep -l '"unsafe"' 2>&1 | (! grep -v '_test.go') | tee /dev/stderr | (! read) +git ls-files "*.go" | xargs grep -l '"math/rand"' 2>&1 | (! grep -v '^examples\|^stress\|grpcrand') | tee /dev/stderr | (! read) gofmt -s -d -l . 2>&1 | tee /dev/stderr | (! read) goimports -l . 2>&1 | tee /dev/stderr | (! read) -golint ./... 2>&1 | (grep -vE "(_mock|_string|grpc_lb_v1/doc|\.pb)\.go:" || true) | tee /dev/stderr | (! read) +golint ./... 2>&1 | (grep -vE "(_mock|\.pb)\.go:" || true) | tee /dev/stderr | (! read) # Undo any edits made by this script. cleanup() { @@ -64,7 +71,7 @@ trap cleanup EXIT git ls-files "*.go" | xargs sed -i 's:"golang.org/x/net/context":"context":' set +o pipefail # TODO: Stop filtering pb.go files once golang/protobuf#214 is fixed. -go tool vet -all . 2>&1 | grep -vF '.pb.go:' | tee /dev/stderr | (! read) +go tool vet -all . 2>&1 | grep -vE '(clientconn|transport\/transport_test).go:.*cancel (function|var)' | grep -vF '.pb.go:' | tee /dev/stderr | (! read) set -o pipefail git reset --hard HEAD @@ -75,4 +82,13 @@ if [[ "$check_proto" = "true" ]]; then fi # TODO(menghanl): fix errors in transport_test. -staticcheck -ignore google.golang.org/grpc/transport/transport_test.go:SA2002 ./... +staticcheck -ignore ' +google.golang.org/grpc/transport/transport_test.go:SA2002 +google.golang.org/grpc/benchmark/benchmain/main.go:SA1019 +google.golang.org/grpc/stats/stats_test.go:SA1019 +google.golang.org/grpc/test/end2end_test.go:SA1019 +google.golang.org/grpc/balancer_test.go:SA1019 +google.golang.org/grpc/balancer.go:SA1019 +google.golang.org/grpc/clientconn_test.go:SA1019 +' ./... +misspell -error . diff --git a/vendor/k8s.io/gengo/examples/deepcopy-gen/generators/BUILD b/vendor/k8s.io/gengo/examples/deepcopy-gen/generators/BUILD index dd7f13259cc..27687b2c311 100644 --- a/vendor/k8s.io/gengo/examples/deepcopy-gen/generators/BUILD +++ b/vendor/k8s.io/gengo/examples/deepcopy-gen/generators/BUILD @@ -7,12 +7,12 @@ go_library( importpath = "k8s.io/gengo/examples/deepcopy-gen/generators", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/examples/set-gen/sets:go_default_library", "//vendor/k8s.io/gengo/generator:go_default_library", "//vendor/k8s.io/gengo/namer:go_default_library", "//vendor/k8s.io/gengo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/k8s.io/gengo/examples/deepcopy-gen/generators/deepcopy.go b/vendor/k8s.io/gengo/examples/deepcopy-gen/generators/deepcopy.go index dec7dcaf733..d9e5cf11005 100644 --- a/vendor/k8s.io/gengo/examples/deepcopy-gen/generators/deepcopy.go +++ b/vendor/k8s.io/gengo/examples/deepcopy-gen/generators/deepcopy.go @@ -29,7 +29,7 @@ import ( "k8s.io/gengo/namer" "k8s.io/gengo/types" - "github.com/golang/glog" + "k8s.io/klog" ) // CustomArgs is used tby the go2idl framework to pass args specific to this @@ -62,7 +62,7 @@ func extractTag(comments []string) *tagValue { } // If there are multiple values, abort. if len(tagVals) > 1 { - glog.Fatalf("Found %d %s tags: %q", len(tagVals), tagName, tagVals) + klog.Fatalf("Found %d %s tags: %q", len(tagVals), tagName, tagVals) } // If we got here we are returning something. @@ -89,7 +89,7 @@ func extractTag(comments []string) *tagValue { tag.register = true } default: - glog.Fatalf("Unsupported %s param: %q", tagName, parts[i]) + klog.Fatalf("Unsupported %s param: %q", tagName, parts[i]) } } return tag @@ -123,7 +123,7 @@ func DefaultNameSystem() string { func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages { boilerplate, err := arguments.LoadGoBoilerplate() if err != nil { - glog.Fatalf("Failed loading boilerplate: %v", err) + klog.Fatalf("Failed loading boilerplate: %v", err) } inputs := sets.NewString(context.Inputs...) @@ -143,7 +143,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat } for i := range inputs { - glog.V(5).Infof("Considering pkg %q", i) + klog.V(5).Infof("Considering pkg %q", i) pkg := context.Universe[i] if pkg == nil { // If the input had no Go files, for example. @@ -156,12 +156,12 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat if ptag != nil { ptagValue = ptag.value if ptagValue != tagValuePackage { - glog.Fatalf("Package %v: unsupported %s value: %q", i, tagName, ptagValue) + klog.Fatalf("Package %v: unsupported %s value: %q", i, tagName, ptagValue) } ptagRegister = ptag.register - glog.V(5).Infof(" tag.value: %q, tag.register: %t", ptagValue, ptagRegister) + klog.V(5).Infof(" tag.value: %q, tag.register: %t", ptagValue, ptagRegister) } else { - glog.V(5).Infof(" no tag") + klog.V(5).Infof(" no tag") } // If the pkg-scoped tag says to generate, we can skip scanning types. @@ -170,12 +170,12 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat // If the pkg-scoped tag did not exist, scan all types for one that // explicitly wants generation. for _, t := range pkg.Types { - glog.V(5).Infof(" considering type %q", t.Name.String()) + klog.V(5).Infof(" considering type %q", t.Name.String()) ttag := extractTag(t.CommentLines) if ttag != nil && ttag.value == "true" { - glog.V(5).Infof(" tag=true") + klog.V(5).Infof(" tag=true") if !copyableType(t) { - glog.Fatalf("Type %v requests deepcopy generation but is not copyable", t) + klog.Fatalf("Type %v requests deepcopy generation but is not copyable", t) } pkgNeedsGeneration = true break @@ -184,7 +184,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat } if pkgNeedsGeneration { - glog.V(3).Infof("Package %q needs generation", i) + klog.V(3).Infof("Package %q needs generation", i) path := pkg.Path // if the source path is within a /vendor/ directory (for example, // k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/apis/meta/v1), allow @@ -263,10 +263,10 @@ func (g *genDeepCopy) Filter(c *generator.Context, t *types.Type) bool { return false } if !copyableType(t) { - glog.V(2).Infof("Type %v is not copyable", t) + klog.V(2).Infof("Type %v is not copyable", t) return false } - glog.V(4).Infof("Type %v is copyable", t) + klog.V(4).Infof("Type %v is copyable", t) g.typesForInit = append(g.typesForInit, t) return true } @@ -321,12 +321,12 @@ func deepCopyMethod(t *types.Type) (*types.Signature, error) { return f.Signature, nil } -// deepCopyMethodOrDie returns the signatrue of a DeepCopy method, nil or calls glog.Fatalf +// deepCopyMethodOrDie returns the signatrue of a DeepCopy method, nil or calls klog.Fatalf // if the type does not match. func deepCopyMethodOrDie(t *types.Type) *types.Signature { ret, err := deepCopyMethod(t) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return ret } @@ -367,12 +367,12 @@ func deepCopyIntoMethod(t *types.Type) (*types.Signature, error) { return f.Signature, nil } -// deepCopyIntoMethodOrDie returns the signature of a DeepCopyInto() method, nil or calls glog.Fatalf +// deepCopyIntoMethodOrDie returns the signature of a DeepCopyInto() method, nil or calls klog.Fatalf // if the type is wrong. func deepCopyIntoMethodOrDie(t *types.Type) *types.Signature { ret, err := deepCopyIntoMethod(t) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return ret } @@ -465,17 +465,17 @@ func (g *genDeepCopy) needsGeneration(t *types.Type) bool { if tag != nil { tv = tag.value if tv != "true" && tv != "false" { - glog.Fatalf("Type %v: unsupported %s value: %q", t, tagName, tag.value) + klog.Fatalf("Type %v: unsupported %s value: %q", t, tagName, tag.value) } } if g.allTypes && tv == "false" { // The whole package is being generated, but this type has opted out. - glog.V(5).Infof("Not generating for type %v because type opted out", t) + klog.V(5).Infof("Not generating for type %v because type opted out", t) return false } if !g.allTypes && tv != "true" { // The whole package is NOT being generated, and this type has NOT opted in. - glog.V(5).Infof("Not generating for type %v because type did not opt in", t) + klog.V(5).Infof("Not generating for type %v because type did not opt in", t) return false } return true @@ -576,7 +576,7 @@ func (g *genDeepCopy) GenerateType(c *generator.Context, t *types.Type, w io.Wri if !g.needsGeneration(t) { return nil } - glog.V(5).Infof("Generating deepcopy function for type %v", t) + klog.V(5).Infof("Generating deepcopy function for type %v", t) sw := generator.NewSnippetWriter(w, c, "$", "$") args := argsFromType(t) @@ -678,12 +678,12 @@ func (g *genDeepCopy) generateFor(t *types.Type, sw *generator.SnippetWriter) { f = g.doPointer case types.Interface: // interfaces are handled in-line in the other cases - glog.Fatalf("Hit an interface type %v. This should never happen.", t) + klog.Fatalf("Hit an interface type %v. This should never happen.", t) case types.Alias: // can never happen because we branch on the underlying type which is never an alias - glog.Fatalf("Hit an alias type %v. This should never happen.", t) + klog.Fatalf("Hit an alias type %v. This should never happen.", t) default: - glog.Fatalf("Hit an unsupported type %v.", t) + klog.Fatalf("Hit an unsupported type %v.", t) } f(t, sw) } @@ -711,7 +711,7 @@ func (g *genDeepCopy) doMap(t *types.Type, sw *generator.SnippetWriter) { } if !ut.Key.IsAssignable() { - glog.Fatalf("Hit an unsupported type %v.", uet) + klog.Fatalf("Hit an unsupported type %v.", uet) } sw.Do("*out = make($.|raw$, len(*in))\n", t) @@ -754,7 +754,7 @@ func (g *genDeepCopy) doMap(t *types.Type, sw *generator.SnippetWriter) { case uet.Kind == types.Struct: sw.Do("(*out)[key] = *val.DeepCopy()\n", uet) default: - glog.Fatalf("Hit an unsupported type %v.", uet) + klog.Fatalf("Hit an unsupported type %v.", uet) } sw.Do("}\n", nil) } @@ -795,7 +795,7 @@ func (g *genDeepCopy) doSlice(t *types.Type, sw *generator.SnippetWriter) { } else if uet.Kind == types.Struct { sw.Do("(*in)[i].DeepCopyInto(&(*out)[i])\n", nil) } else { - glog.Fatalf("Hit an unsupported type %v.", uet) + klog.Fatalf("Hit an unsupported type %v.", uet) } sw.Do("}\n", nil) } @@ -863,7 +863,7 @@ func (g *genDeepCopy) doStruct(t *types.Type, sw *generator.SnippetWriter) { sw.Do(fmt.Sprintf("out.$.name$ = in.$.name$.DeepCopy%s()\n", uft.Name.Name), args) sw.Do("}\n", nil) default: - glog.Fatalf("Hit an unsupported type %v.", uft) + klog.Fatalf("Hit an unsupported type %v.", uft) } } } @@ -900,6 +900,6 @@ func (g *genDeepCopy) doPointer(t *types.Type, sw *generator.SnippetWriter) { sw.Do("*out = new($.Elem|raw$)\n", ut) sw.Do("(*in).DeepCopyInto(*out)\n", nil) default: - glog.Fatalf("Hit an unsupported type %v.", uet) + klog.Fatalf("Hit an unsupported type %v.", uet) } } diff --git a/vendor/k8s.io/gengo/examples/defaulter-gen/generators/BUILD b/vendor/k8s.io/gengo/examples/defaulter-gen/generators/BUILD index 4c62d283d57..a6cd0735f92 100644 --- a/vendor/k8s.io/gengo/examples/defaulter-gen/generators/BUILD +++ b/vendor/k8s.io/gengo/examples/defaulter-gen/generators/BUILD @@ -7,11 +7,11 @@ go_library( importpath = "k8s.io/gengo/examples/defaulter-gen/generators", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/generator:go_default_library", "//vendor/k8s.io/gengo/namer:go_default_library", "//vendor/k8s.io/gengo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/k8s.io/gengo/examples/defaulter-gen/generators/defaulter.go b/vendor/k8s.io/gengo/examples/defaulter-gen/generators/defaulter.go index 6ab8676a24d..7e3bc6b4b12 100644 --- a/vendor/k8s.io/gengo/examples/defaulter-gen/generators/defaulter.go +++ b/vendor/k8s.io/gengo/examples/defaulter-gen/generators/defaulter.go @@ -29,7 +29,7 @@ import ( "k8s.io/gengo/namer" "k8s.io/gengo/types" - "github.com/golang/glog" + "k8s.io/klog" ) // CustomArgs is used tby the go2idl framework to pass args specific to this @@ -117,11 +117,11 @@ func getManualDefaultingFunctions(context *generator.Context, pkg *types.Package for _, f := range pkg.Functions { if f.Underlying == nil || f.Underlying.Kind != types.Func { - glog.Errorf("Malformed function: %#v", f) + klog.Errorf("Malformed function: %#v", f) continue } if f.Underlying.Signature == nil { - glog.Errorf("Function without signature: %#v", f) + klog.Errorf("Function without signature: %#v", f) continue } signature := f.Underlying.Signature @@ -156,7 +156,7 @@ func getManualDefaultingFunctions(context *generator.Context, pkg *types.Package } v.base = f manualMap[key] = v - glog.V(6).Infof("found base defaulter function for %s from %s", key.Name, f.Name) + klog.V(6).Infof("found base defaulter function for %s from %s", key.Name, f.Name) // Is one of the additional defaulters - a top level defaulter on a type that is // also invoked. case strings.HasPrefix(f.Name.Name, buffer.String()+"_"): @@ -176,7 +176,7 @@ func getManualDefaultingFunctions(context *generator.Context, pkg *types.Package } v.additional = append(v.additional, f) manualMap[key] = v - glog.V(6).Infof("found additional defaulter function for %s from %s", key.Name, f.Name) + klog.V(6).Infof("found additional defaulter function for %s from %s", key.Name, f.Name) } buffer.Reset() sw.Do("$.inType|objectdefaultfn$", args) @@ -189,7 +189,7 @@ func getManualDefaultingFunctions(context *generator.Context, pkg *types.Package } v.object = f manualMap[key] = v - glog.V(6).Infof("found object defaulter function for %s from %s", key.Name, f.Name) + klog.V(6).Infof("found object defaulter function for %s from %s", key.Name, f.Name) } buffer.Reset() } @@ -198,7 +198,7 @@ func getManualDefaultingFunctions(context *generator.Context, pkg *types.Package func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages { boilerplate, err := arguments.LoadGoBoilerplate() if err != nil { - glog.Fatalf("Failed loading boilerplate: %v", err) + klog.Fatalf("Failed loading boilerplate: %v", err) } packages := generator.Packages{} @@ -214,7 +214,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat // We are generating defaults only for packages that are explicitly // passed as InputDir. for _, i := range context.Inputs { - glog.V(5).Infof("considering pkg %q", i) + klog.V(5).Infof("considering pkg %q", i) pkg := context.Universe[i] if pkg == nil { // If the input had no Go files, for example. @@ -248,7 +248,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat shouldCreateObjectDefaulterFn := func(t *types.Type) bool { if defaults, ok := existingDefaulters[t]; ok && defaults.object != nil { // A default generator is defined - glog.V(5).Infof(" an object defaulter already exists as %s", defaults.base.Name) + klog.V(5).Infof(" an object defaulter already exists as %s", defaults.base.Name) return false } // opt-out @@ -285,7 +285,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat var err error typesPkg, err = context.AddDirectory(filepath.Join(pkg.Path, inputTags[0])) if err != nil { - glog.Fatalf("cannot import package %s", inputTags[0]) + klog.Fatalf("cannot import package %s", inputTags[0]) } // update context.Order to the latest context.Universe orderer := namer.Orderer{Namer: namer.NewPublicNamer(1)} @@ -299,7 +299,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat } if namer.IsPrivateGoName(t.Name.Name) { // We won't be able to convert to a private type. - glog.V(5).Infof(" found a type %v, but it is a private name", t) + klog.V(5).Infof(" found a type %v, but it is a private name", t) continue } @@ -338,7 +338,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat // prune any types that were not used for t, d := range newDefaulters { if d.object == nil { - glog.V(6).Infof("did not generate defaulter for %s because no child defaulters were registered", t.Name) + klog.V(6).Infof("did not generate defaulter for %s because no child defaulters were registered", t.Name) delete(newDefaulters, t) } } @@ -346,7 +346,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat } if len(newDefaulters) == 0 { - glog.V(5).Infof("no defaulters in package %s", pkg.Name) + klog.V(5).Infof("no defaulters in package %s", pkg.Name) } path := pkg.Path @@ -421,7 +421,7 @@ func (c *callTreeForType) build(t *types.Type, root bool) *callNode { parent.call = append(parent.call, newDefaults.object) // if we will be generating the defaulter, it by definition is a covering // defaulter, so we halt recursion - glog.V(6).Infof("the defaulter %s will be generated as an object defaulter", t.Name) + klog.V(6).Infof("the defaulter %s will be generated as an object defaulter", t.Name) return parent case defaults.object != nil: @@ -434,7 +434,7 @@ func (c *callTreeForType) build(t *types.Type, root bool) *callNode { // if the base function indicates it "covers" (it already includes defaulters) // we can halt recursion if checkTag(defaults.base.CommentLines, "covers") { - glog.V(6).Infof("the defaulter %s indicates it covers all sub generators", t.Name) + klog.V(6).Infof("the defaulter %s indicates it covers all sub generators", t.Name) return parent } } @@ -496,7 +496,7 @@ func (c *callTreeForType) build(t *types.Type, root bool) *callNode { } } if len(parent.children) == 0 && len(parent.call) == 0 { - //glog.V(6).Infof("decided type %s needs no generation", t.Name) + //klog.V(6).Infof("decided type %s needs no generation", t.Name) return nil } return parent @@ -596,11 +596,11 @@ func (g *genDefaulter) GenerateType(c *generator.Context, t *types.Type, w io.Wr return nil } - glog.V(5).Infof("generating for type %v", t) + klog.V(5).Infof("generating for type %v", t) callTree := newCallTreeForType(g.existingDefaulters, g.newDefaulters).build(t, true) if callTree == nil { - glog.V(5).Infof(" no defaulters defined") + klog.V(5).Infof(" no defaulters defined") return nil } i := 0 @@ -609,7 +609,7 @@ func (g *genDefaulter) GenerateType(c *generator.Context, t *types.Type, w io.Wr return } path := callPath(append(ancestors, current)) - glog.V(5).Infof(" %d: %s", i, path) + klog.V(5).Infof(" %d: %s", i, path) i++ }) diff --git a/vendor/k8s.io/gengo/examples/import-boss/generators/BUILD b/vendor/k8s.io/gengo/examples/import-boss/generators/BUILD index 22d84e5745b..366922bed10 100644 --- a/vendor/k8s.io/gengo/examples/import-boss/generators/BUILD +++ b/vendor/k8s.io/gengo/examples/import-boss/generators/BUILD @@ -7,11 +7,11 @@ go_library( importpath = "k8s.io/gengo/examples/import-boss/generators", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/generator:go_default_library", "//vendor/k8s.io/gengo/namer:go_default_library", "//vendor/k8s.io/gengo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/k8s.io/gengo/examples/import-boss/generators/import_restrict.go b/vendor/k8s.io/gengo/examples/import-boss/generators/import_restrict.go index 7ed7c33a785..182f87af74a 100644 --- a/vendor/k8s.io/gengo/examples/import-boss/generators/import_restrict.go +++ b/vendor/k8s.io/gengo/examples/import-boss/generators/import_restrict.go @@ -33,7 +33,7 @@ import ( "k8s.io/gengo/namer" "k8s.io/gengo/types" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -202,19 +202,19 @@ func (importRuleFile) VerifyFile(f *generator.File, path string) error { return fmt.Errorf("regexp `%s` in file %q doesn't compile: %v", r.SelectorRegexp, actualPath, err) } for v := range f.Imports { - glog.V(4).Infof("Checking %v matches %v: %v\n", r.SelectorRegexp, v, re.MatchString(v)) + klog.V(4).Infof("Checking %v matches %v: %v\n", r.SelectorRegexp, v, re.MatchString(v)) if !re.MatchString(v) { continue } for _, forbidden := range r.ForbiddenPrefixes { - glog.V(4).Infof("Checking %v against %v\n", v, forbidden) + klog.V(4).Infof("Checking %v against %v\n", v, forbidden) if strings.HasPrefix(v, forbidden) { return fmt.Errorf("import %v has forbidden prefix %v", v, forbidden) } } found := false for _, allowed := range r.AllowedPrefixes { - glog.V(4).Infof("Checking %v against %v\n", v, allowed) + klog.V(4).Infof("Checking %v against %v\n", v, allowed) if strings.HasPrefix(v, allowed) { found = true break @@ -226,7 +226,7 @@ func (importRuleFile) VerifyFile(f *generator.File, path string) error { } } if len(rules.Rules) > 0 { - glog.V(2).Infof("%v passes rules found in %v\n", path, actualPath) + klog.V(2).Infof("%v passes rules found in %v\n", path, actualPath) } return nil diff --git a/vendor/k8s.io/gengo/examples/set-gen/generators/BUILD b/vendor/k8s.io/gengo/examples/set-gen/generators/BUILD index b60e695769e..c7a4efe3fda 100644 --- a/vendor/k8s.io/gengo/examples/set-gen/generators/BUILD +++ b/vendor/k8s.io/gengo/examples/set-gen/generators/BUILD @@ -10,11 +10,11 @@ go_library( importpath = "k8s.io/gengo/examples/set-gen/generators", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/generator:go_default_library", "//vendor/k8s.io/gengo/namer:go_default_library", "//vendor/k8s.io/gengo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/k8s.io/gengo/examples/set-gen/generators/sets.go b/vendor/k8s.io/gengo/examples/set-gen/generators/sets.go index 8c5b4184ba5..d0698d33cd7 100644 --- a/vendor/k8s.io/gengo/examples/set-gen/generators/sets.go +++ b/vendor/k8s.io/gengo/examples/set-gen/generators/sets.go @@ -25,7 +25,7 @@ import ( "k8s.io/gengo/namer" "k8s.io/gengo/types" - "github.com/golang/glog" + "k8s.io/klog" ) // NameSystems returns the name system used by the generators in this package. @@ -47,13 +47,13 @@ func DefaultNameSystem() string { func Packages(_ *generator.Context, arguments *args.GeneratorArgs) generator.Packages { boilerplate, err := arguments.LoadGoBoilerplate() if err != nil { - glog.Fatalf("Failed loading boilerplate: %v", err) + klog.Fatalf("Failed loading boilerplate: %v", err) } return generator.Packages{&generator.DefaultPackage{ PackageName: "sets", PackagePath: arguments.OutputPackagePath, - HeaderText: boilerplate, + HeaderText: boilerplate, PackageDocumentation: []byte( `// Package sets has auto-generated set types. `), diff --git a/vendor/k8s.io/gengo/examples/set-gen/generators/tags.go b/vendor/k8s.io/gengo/examples/set-gen/generators/tags.go index 34aa77231fa..bb3b4d2573e 100644 --- a/vendor/k8s.io/gengo/examples/set-gen/generators/tags.go +++ b/vendor/k8s.io/gengo/examples/set-gen/generators/tags.go @@ -17,8 +17,8 @@ limitations under the License. package generators import ( - "github.com/golang/glog" "k8s.io/gengo/types" + "k8s.io/klog" ) // extractBoolTagOrDie gets the comment-tags for the key and asserts that, if @@ -27,7 +27,7 @@ import ( func extractBoolTagOrDie(key string, lines []string) bool { val, err := types.ExtractSingleBoolCommentTag("+", key, false, lines) if err != nil { - glog.Fatalf(err.Error()) + klog.Fatalf(err.Error()) } return val } diff --git a/vendor/k8s.io/gengo/generator/BUILD b/vendor/k8s.io/gengo/generator/BUILD index e11afffd217..ff842916322 100644 --- a/vendor/k8s.io/gengo/generator/BUILD +++ b/vendor/k8s.io/gengo/generator/BUILD @@ -16,11 +16,11 @@ go_library( importpath = "k8s.io/gengo/generator", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/tools/imports:go_default_library", "//vendor/k8s.io/gengo/namer:go_default_library", "//vendor/k8s.io/gengo/parser:go_default_library", "//vendor/k8s.io/gengo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/k8s.io/gengo/generator/execute.go b/vendor/k8s.io/gengo/generator/execute.go index 38dffff975a..b5f5aaeb443 100644 --- a/vendor/k8s.io/gengo/generator/execute.go +++ b/vendor/k8s.io/gengo/generator/execute.go @@ -29,7 +29,7 @@ import ( "k8s.io/gengo/namer" "k8s.io/gengo/types" - "github.com/golang/glog" + "k8s.io/klog" ) func errs2strings(errors []error) []string { @@ -64,7 +64,7 @@ type DefaultFileType struct { } func (ft DefaultFileType) AssembleFile(f *File, pathname string) error { - glog.V(2).Infof("Assembling file %q", pathname) + klog.V(2).Infof("Assembling file %q", pathname) destFile, err := os.Create(pathname) if err != nil { return err @@ -91,7 +91,7 @@ func (ft DefaultFileType) AssembleFile(f *File, pathname string) error { } func (ft DefaultFileType) VerifyFile(f *File, pathname string) error { - glog.V(2).Infof("Verifying file %q", pathname) + klog.V(2).Infof("Verifying file %q", pathname) friendlyName := filepath.Join(f.PackageName, f.Name) b := &bytes.Buffer{} et := NewErrorTracker(b) @@ -214,7 +214,7 @@ func (c *Context) addNameSystems(namers namer.NameSystems) *Context { // import path already, this will be appended to 'outDir'. func (c *Context) ExecutePackage(outDir string, p Package) error { path := filepath.Join(outDir, p.Path()) - glog.V(2).Infof("Processing package %q, disk location %q", p.Name(), path) + klog.V(2).Infof("Processing package %q, disk location %q", p.Name(), path) // Filter out any types the *package* doesn't care about. packageContext := c.filteredBy(p.Filter) os.MkdirAll(path, 0755) diff --git a/vendor/k8s.io/gengo/generator/import_tracker.go b/vendor/k8s.io/gengo/generator/import_tracker.go index d4ba5e90731..5d058410ac6 100644 --- a/vendor/k8s.io/gengo/generator/import_tracker.go +++ b/vendor/k8s.io/gengo/generator/import_tracker.go @@ -19,7 +19,7 @@ package generator import ( "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/gengo/namer" "k8s.io/gengo/types" @@ -42,7 +42,7 @@ func golangTrackerLocalName(tracker namer.ImportTracker, t types.Name) string { // Using backslashes in package names causes gengo to produce Go code which // will not compile with the gc compiler. See the comment on GoSeperator. if strings.ContainsRune(path, '\\') { - glog.Warningf("Warning: backslash used in import path '%v', this is unsupported.\n", path) + klog.Warningf("Warning: backslash used in import path '%v', this is unsupported.\n", path) } dirs := strings.Split(path, namer.GoSeperator) diff --git a/vendor/k8s.io/gengo/namer/plural_namer.go b/vendor/k8s.io/gengo/namer/plural_namer.go index 40bdcc6ccc2..a9a198a7027 100644 --- a/vendor/k8s.io/gengo/namer/plural_namer.go +++ b/vendor/k8s.io/gengo/namer/plural_namer.go @@ -59,7 +59,7 @@ func (r *pluralNamer) Name(t *types.Type) string { return r.finalize(plural) } if len(singular) < 2 { - return r.finalize(plural) + return r.finalize(singular) } switch rune(singular[len(singular)-1]) { @@ -87,7 +87,7 @@ func (r *pluralNamer) Name(t *types.Type) string { plural = sPlural(singular) } case 'f': - plural = vesPlural(singular) + plural = vesPlural(singular) default: plural = sPlural(singular) } diff --git a/vendor/k8s.io/gengo/parser/BUILD b/vendor/k8s.io/gengo/parser/BUILD index 26de4013e80..c797889f3b0 100644 --- a/vendor/k8s.io/gengo/parser/BUILD +++ b/vendor/k8s.io/gengo/parser/BUILD @@ -10,8 +10,8 @@ go_library( importpath = "k8s.io/gengo/parser", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/gengo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/vendor/k8s.io/gengo/parser/parse.go b/vendor/k8s.io/gengo/parser/parse.go index 7b043d716d6..bf8372d9207 100644 --- a/vendor/k8s.io/gengo/parser/parse.go +++ b/vendor/k8s.io/gengo/parser/parse.go @@ -31,8 +31,8 @@ import ( "sort" "strings" - "github.com/golang/glog" "k8s.io/gengo/types" + "k8s.io/klog" ) // This clarifies when a pkg path has been canonicalized. @@ -89,7 +89,7 @@ func New() *Builder { // The returned string will have some/path/bin/go, so remove the last two elements. c.GOROOT = filepath.Dir(filepath.Dir(strings.Trim(string(p), "\n"))) } else { - glog.Warningf("Warning: $GOROOT not set, and unable to run `which go` to find it: %v\n", err) + klog.Warningf("Warning: $GOROOT not set, and unable to run `which go` to find it: %v\n", err) } } // Force this to off, since we don't properly parse CGo. All symbols must @@ -136,7 +136,7 @@ func (b *Builder) importBuildPackage(dir string) (*build.Package, error) { } // Remember it under the user-provided name. - glog.V(5).Infof("saving buildPackage %s", dir) + klog.V(5).Infof("saving buildPackage %s", dir) b.buildPackages[dir] = buildPkg canonicalPackage := canonicalizeImportPath(buildPkg.ImportPath) if dir != string(canonicalPackage) { @@ -145,7 +145,7 @@ func (b *Builder) importBuildPackage(dir string) (*build.Package, error) { return buildPkg, nil } // Must be new, save it under the canonical name, too. - glog.V(5).Infof("saving buildPackage %s", canonicalPackage) + klog.V(5).Infof("saving buildPackage %s", canonicalPackage) b.buildPackages[string(canonicalPackage)] = buildPkg } @@ -175,11 +175,11 @@ func (b *Builder) AddFileForTest(pkg string, path string, src []byte) error { func (b *Builder) addFile(pkgPath importPathString, path string, src []byte, userRequested bool) error { for _, p := range b.parsed[pkgPath] { if path == p.name { - glog.V(5).Infof("addFile %s %s already parsed, skipping", pkgPath, path) + klog.V(5).Infof("addFile %s %s already parsed, skipping", pkgPath, path) return nil } } - glog.V(6).Infof("addFile %s %s", pkgPath, path) + klog.V(6).Infof("addFile %s %s", pkgPath, path) p, err := parser.ParseFile(b.fset, path, src, parser.DeclarationErrors|parser.ParseComments) if err != nil { return err @@ -221,7 +221,7 @@ func (b *Builder) AddDir(dir string) error { func (b *Builder) AddDirRecursive(dir string) error { // Add the root. if _, err := b.importPackage(dir, true); err != nil { - glog.Warningf("Ignoring directory %v: %v", dir, err) + klog.Warningf("Ignoring directory %v: %v", dir, err) } // filepath.Walk includes the root dir, but we already did that, so we'll @@ -236,7 +236,7 @@ func (b *Builder) AddDirRecursive(dir string) error { // Add it. if _, err := b.importPackage(pkg, true); err != nil { - glog.Warningf("Ignoring child directory %v: %v", pkg, err) + klog.Warningf("Ignoring child directory %v: %v", pkg, err) } } } @@ -284,7 +284,7 @@ func (b *Builder) AddDirectoryTo(dir string, u *types.Universe) (*types.Package, // The implementation of AddDir. A flag indicates whether this directory was // user-requested or just from following the import graph. func (b *Builder) addDir(dir string, userRequested bool) error { - glog.V(5).Infof("addDir %s", dir) + klog.V(5).Infof("addDir %s", dir) buildPkg, err := b.importBuildPackage(dir) if err != nil { return err @@ -292,7 +292,7 @@ func (b *Builder) addDir(dir string, userRequested bool) error { canonicalPackage := canonicalizeImportPath(buildPkg.ImportPath) pkgPath := canonicalPackage if dir != string(canonicalPackage) { - glog.V(5).Infof("addDir %s, canonical path is %s", dir, pkgPath) + klog.V(5).Infof("addDir %s, canonical path is %s", dir, pkgPath) } // Sanity check the pkg dir has not changed. @@ -324,13 +324,13 @@ func (b *Builder) addDir(dir string, userRequested bool) error { // importPackage is a function that will be called by the type check package when it // needs to import a go package. 'path' is the import path. func (b *Builder) importPackage(dir string, userRequested bool) (*tc.Package, error) { - glog.V(5).Infof("importPackage %s", dir) + klog.V(5).Infof("importPackage %s", dir) var pkgPath = importPathString(dir) // Get the canonical path if we can. if buildPkg := b.buildPackages[dir]; buildPkg != nil { canonicalPackage := canonicalizeImportPath(buildPkg.ImportPath) - glog.V(5).Infof("importPackage %s, canonical path is %s", dir, canonicalPackage) + klog.V(5).Infof("importPackage %s, canonical path is %s", dir, canonicalPackage) pkgPath = canonicalPackage } @@ -349,7 +349,7 @@ func (b *Builder) importPackage(dir string, userRequested bool) (*tc.Package, er // Get the canonical path now that it has been added. if buildPkg := b.buildPackages[dir]; buildPkg != nil { canonicalPackage := canonicalizeImportPath(buildPkg.ImportPath) - glog.V(5).Infof("importPackage %s, canonical path is %s", dir, canonicalPackage) + klog.V(5).Infof("importPackage %s, canonical path is %s", dir, canonicalPackage) pkgPath = canonicalPackage } } @@ -365,9 +365,9 @@ func (b *Builder) importPackage(dir string, userRequested bool) (*tc.Package, er if err != nil { switch { case ignoreError && pkg != nil: - glog.V(2).Infof("type checking encountered some issues in %q, but ignoring.\n", pkgPath) + klog.V(2).Infof("type checking encountered some issues in %q, but ignoring.\n", pkgPath) case !ignoreError && pkg != nil: - glog.V(2).Infof("type checking encountered some errors in %q\n", pkgPath) + klog.V(2).Infof("type checking encountered some errors in %q\n", pkgPath) return nil, err default: return nil, err @@ -389,10 +389,10 @@ func (a importAdapter) Import(path string) (*tc.Package, error) { // errors, so you may check whether the package is nil or not even if you get // an error. func (b *Builder) typeCheckPackage(pkgPath importPathString) (*tc.Package, error) { - glog.V(5).Infof("typeCheckPackage %s", pkgPath) + klog.V(5).Infof("typeCheckPackage %s", pkgPath) if pkg, ok := b.typeCheckedPackages[pkgPath]; ok { if pkg != nil { - glog.V(6).Infof("typeCheckPackage %s already done", pkgPath) + klog.V(6).Infof("typeCheckPackage %s already done", pkgPath) return pkg, nil } // We store a nil right before starting work on a package. So @@ -416,7 +416,7 @@ func (b *Builder) typeCheckPackage(pkgPath importPathString) (*tc.Package, error // method. So there can't be cycles in the import graph. Importer: importAdapter{b}, Error: func(err error) { - glog.V(2).Infof("type checker: %v\n", err) + klog.V(2).Infof("type checker: %v\n", err) }, } pkg, err := c.Check(string(pkgPath), b.fset, files, nil) @@ -469,7 +469,7 @@ func (b *Builder) FindTypes() (types.Universe, error) { // findTypesIn finalizes the package import and searches through the package // for types. func (b *Builder) findTypesIn(pkgPath importPathString, u *types.Universe) error { - glog.V(5).Infof("findTypesIn %s", pkgPath) + klog.V(5).Infof("findTypesIn %s", pkgPath) pkg := b.typeCheckedPackages[pkgPath] if pkg == nil { return fmt.Errorf("findTypesIn(%s): package is not known", pkgPath) @@ -479,7 +479,7 @@ func (b *Builder) findTypesIn(pkgPath importPathString, u *types.Universe) error // packages they asked for depend on will be included. // But we don't need to include all types in all // *packages* they depend on. - glog.V(5).Infof("findTypesIn %s: package is not user requested", pkgPath) + klog.V(5).Infof("findTypesIn %s: package is not user requested", pkgPath) return nil } @@ -775,7 +775,7 @@ func (b *Builder) walkType(u types.Universe, useName *types.Name, in tc.Type) *t return out } out.Kind = types.Unsupported - glog.Warningf("Making unsupported type entry %q for: %#v\n", out, t) + klog.Warningf("Making unsupported type entry %q for: %#v\n", out, t) return out } } diff --git a/vendor/k8s.io/gengo/types/types.go b/vendor/k8s.io/gengo/types/types.go index aa3b7128e94..ec25248e7e5 100644 --- a/vendor/k8s.io/gengo/types/types.go +++ b/vendor/k8s.io/gengo/types/types.go @@ -51,10 +51,10 @@ func ParseFullyQualifiedName(fqn string) Name { cs := strings.Split(fqn, ".") pkg := "" if len(cs) > 1 { - pkg = strings.Join(cs[0:len(cs) - 1], ".") + pkg = strings.Join(cs[0:len(cs)-1], ".") } return Name{ - Name: cs[len(cs) - 1], + Name: cs[len(cs)-1], Package: pkg, } } diff --git a/vendor/k8s.io/klog/.travis.yml b/vendor/k8s.io/klog/.travis.yml new file mode 100644 index 00000000000..fc0d2caf33f --- /dev/null +++ b/vendor/k8s.io/klog/.travis.yml @@ -0,0 +1,14 @@ +language: go +dist: xenial +go: + - 1.9.x + - 1.10.x + - 1.11.x +script: + - go get -t -v ./... + - diff -u <(echo -n) <(gofmt -d .) + - diff -u <(echo -n) <(golint $(go list -e ./...)) + - go tool vet . + - go test -v -race ./... +install: + - go get golang.org/x/lint/golint diff --git a/vendor/github.com/golang/glog/BUILD b/vendor/k8s.io/klog/BUILD similarity index 74% rename from vendor/github.com/golang/glog/BUILD rename to vendor/k8s.io/klog/BUILD index 11dbe36b889..1478e038b70 100644 --- a/vendor/github.com/golang/glog/BUILD +++ b/vendor/k8s.io/klog/BUILD @@ -3,11 +3,11 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ - "glog.go", - "glog_file.go", + "klog.go", + "klog_file.go", ], - importmap = "k8s.io/kubernetes/vendor/github.com/golang/glog", - importpath = "github.com/golang/glog", + importmap = "k8s.io/kubernetes/vendor/k8s.io/klog", + importpath = "k8s.io/klog", visibility = ["//visibility:public"], ) diff --git a/vendor/k8s.io/klog/CONTRIBUTING.md b/vendor/k8s.io/klog/CONTRIBUTING.md new file mode 100644 index 00000000000..de471151372 --- /dev/null +++ b/vendor/k8s.io/klog/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing Guidelines + +Welcome to Kubernetes. We are excited about the prospect of you joining our [community](https://github.com/kubernetes/community)! The Kubernetes community abides by the CNCF [code of conduct](code-of-conduct.md). Here is an excerpt: + +_As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities._ + +## Getting Started + +We have full documentation on how to get started contributing here: + + + +- [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests +- [Kubernetes Contributor Guide](http://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](http://git.k8s.io/community/contributors/guide#contributing) +- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet.md) - Common resources for existing developers + +## Mentorship + +- [Mentoring Initiatives](https://git.k8s.io/community/mentoring) - We have a diverse set of mentorship programs available that are always looking for volunteers! + + diff --git a/vendor/github.com/golang/glog/LICENSE b/vendor/k8s.io/klog/LICENSE similarity index 100% rename from vendor/github.com/golang/glog/LICENSE rename to vendor/k8s.io/klog/LICENSE diff --git a/vendor/k8s.io/klog/OWNERS b/vendor/k8s.io/klog/OWNERS new file mode 100644 index 00000000000..56b0eb044f8 --- /dev/null +++ b/vendor/k8s.io/klog/OWNERS @@ -0,0 +1,11 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md + +approvers: + - dims + - thockin + - justinsb + - tallclair + - piosz + - brancz + - DirectXMan12 + - lavalamp diff --git a/vendor/github.com/golang/glog/README b/vendor/k8s.io/klog/README.md similarity index 90% rename from vendor/github.com/golang/glog/README rename to vendor/k8s.io/klog/README.md index 5f9c11485e0..a747f538a84 100644 --- a/vendor/github.com/golang/glog/README +++ b/vendor/k8s.io/klog/README.md @@ -1,3 +1,10 @@ +klog +==== + +klog is a permanant fork of https://github.com/golang/glog. original README from glog is below + +---- + glog ==== @@ -5,7 +12,7 @@ Leveled execution logs for Go. This is an efficient pure Go implementation of leveled logs in the manner of the open source C++ package - http://code.google.com/p/google-glog + https://github.com/google/glog By binding methods to booleans it is possible to use the log package without paying the expense of evaluating the arguments to the log. diff --git a/vendor/k8s.io/klog/RELEASE.md b/vendor/k8s.io/klog/RELEASE.md new file mode 100644 index 00000000000..b53eb960ce7 --- /dev/null +++ b/vendor/k8s.io/klog/RELEASE.md @@ -0,0 +1,9 @@ +# Release Process + +The `klog` is released on an as-needed basis. The process is as follows: + +1. An issue is proposing a new release with a changelog since the last release +1. All [OWNERS](OWNERS) must LGTM this release +1. An OWNER runs `git tag -s $VERSION` and inserts the changelog and pushes the tag with `git push $VERSION` +1. The release issue is closed +1. An announcement email is sent to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] kubernetes-template-project $VERSION is released` diff --git a/vendor/k8s.io/klog/SECURITY_CONTACTS b/vendor/k8s.io/klog/SECURITY_CONTACTS new file mode 100644 index 00000000000..520ddb52575 --- /dev/null +++ b/vendor/k8s.io/klog/SECURITY_CONTACTS @@ -0,0 +1,20 @@ +# Defined below are the security contacts for this repo. +# +# They are the contact point for the Product Security Team to reach out +# to for triaging and handling of incoming issues. +# +# The below names agree to abide by the +# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy) +# and will be removed and replaced if they violate that agreement. +# +# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE +# INSTRUCTIONS AT https://kubernetes.io/security/ + +dims +thockin +justinsb +tallclair +piosz +brancz +DirectXMan12 +lavalamp diff --git a/vendor/github.com/golang/glog/glog.go b/vendor/k8s.io/klog/klog.go similarity index 93% rename from vendor/github.com/golang/glog/glog.go rename to vendor/k8s.io/klog/klog.go index 3e63fffd5ec..13bcc81a756 100644 --- a/vendor/github.com/golang/glog/glog.go +++ b/vendor/k8s.io/klog/klog.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package glog implements logging analogous to the Google-internal C++ INFO/ERROR/V setup. +// Package klog implements logging analogous to the Google-internal C++ INFO/ERROR/V setup. // It provides functions Info, Warning, Error, Fatal, plus formatting variants such as // Infof. It also provides V-style logging controlled by the -v and -vmodule=file=2 flags. // @@ -68,7 +68,7 @@ // -vmodule=gopher*=3 // sets the V level to 3 in all Go files whose names begin "gopher". // -package glog +package klog import ( "bufio" @@ -396,13 +396,6 @@ type flushSyncWriter interface { } func init() { - flag.BoolVar(&logging.toStderr, "logtostderr", false, "log to standard error instead of files") - flag.BoolVar(&logging.alsoToStderr, "alsologtostderr", false, "log to standard error as well as files") - flag.Var(&logging.verbosity, "v", "log level for V logs") - flag.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr") - flag.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging") - flag.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace") - // Default stderrThreshold is ERROR. logging.stderrThreshold = errorLog @@ -410,6 +403,22 @@ func init() { go logging.flushDaemon() } +// InitFlags is for explicitly initializing the flags +func InitFlags(flagset *flag.FlagSet) { + if flagset == nil { + flagset = flag.CommandLine + } + flagset.StringVar(&logging.logDir, "log_dir", "", "If non-empty, write log files in this directory") + flagset.StringVar(&logging.logFile, "log_file", "", "If non-empty, use this log file") + flagset.BoolVar(&logging.toStderr, "logtostderr", false, "log to standard error instead of files") + flagset.BoolVar(&logging.alsoToStderr, "alsologtostderr", false, "log to standard error as well as files") + flagset.Var(&logging.verbosity, "v", "log level for V logs") + flagset.BoolVar(&logging.skipHeaders, "skip_headers", false, "If true, avoid header prefixes in the log messages") + flagset.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr") + flagset.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging") + flagset.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace") +} + // Flush flushes all pending log I/O. func Flush() { logging.lockAndFlushAll() @@ -453,6 +462,17 @@ type loggingT struct { // safely using atomic.LoadInt32. vmodule moduleSpec // The state of the -vmodule flag. verbosity Level // V logging level, the value of the -v flag/ + + // If non-empty, overrides the choice of directory in which to write logs. + // See createLogDirs for the full list of possible destinations. + logDir string + + // If non-empty, specifies the path of the file to write logs. mutually exclusive + // with the log-dir option. + logFile string + + // If true, do not add the prefix headers, useful when used with SetOutput + skipHeaders bool } // buffer holds a byte Buffer for reuse. The zero value is ready for use. @@ -556,6 +576,9 @@ func (l *loggingT) formatHeader(s severity, file string, line int) *buffer { s = infoLog // for safety. } buf := l.getBuffer() + if l.skipHeaders { + return buf + } // Avoid Fprintf, for speed. The format is so simple that we can do it quickly by hand. // It's worth about 3X. Fprintf is hard. @@ -667,6 +690,45 @@ func (l *loggingT) printWithFileLine(s severity, file string, line int, alsoToSt l.output(s, buf, file, line, alsoToStderr) } +// redirectBuffer is used to set an alternate destination for the logs +type redirectBuffer struct { + w io.Writer +} + +func (rb *redirectBuffer) Sync() error { + return nil +} + +func (rb *redirectBuffer) Flush() error { + return nil +} + +func (rb *redirectBuffer) Write(bytes []byte) (n int, err error) { + return rb.w.Write(bytes) +} + +// SetOutput sets the output destination for all severities +func SetOutput(w io.Writer) { + for s := fatalLog; s >= infoLog; s-- { + rb := &redirectBuffer{ + w: w, + } + logging.file[s] = rb + } +} + +// SetOutputBySeverity sets the output destination for specific severity +func SetOutputBySeverity(name string, w io.Writer) { + sev, ok := severityByName(name) + if !ok { + panic(fmt.Sprintf("SetOutputBySeverity(%q): unrecognized severity name", name)) + } + rb := &redirectBuffer{ + w: w, + } + logging.file[sev] = rb +} + // output writes the data to the log files and releases the buffer. func (l *loggingT) output(s severity, buf *buffer, file string, line int, alsoToStderr bool) { l.mu.Lock() @@ -876,7 +938,7 @@ const flushInterval = 30 * time.Second // flushDaemon periodically flushes the log file buffers. func (l *loggingT) flushDaemon() { - for _ = range time.NewTicker(flushInterval).C { + for range time.NewTicker(flushInterval).C { l.lockAndFlushAll() } } diff --git a/vendor/github.com/golang/glog/glog_file.go b/vendor/k8s.io/klog/klog_file.go similarity index 90% rename from vendor/github.com/golang/glog/glog_file.go rename to vendor/k8s.io/klog/klog_file.go index 65075d28111..b76a4e10bec 100644 --- a/vendor/github.com/golang/glog/glog_file.go +++ b/vendor/k8s.io/klog/klog_file.go @@ -16,11 +16,10 @@ // File I/O for logs. -package glog +package klog import ( "errors" - "flag" "fmt" "os" "os/user" @@ -36,13 +35,9 @@ var MaxSize uint64 = 1024 * 1024 * 1800 // logDirs lists the candidate directories for new log files. var logDirs []string -// If non-empty, overrides the choice of directory in which to write logs. -// See createLogDirs for the full list of possible destinations. -var logDir = flag.String("log_dir", "", "If non-empty, write log files in this directory") - func createLogDirs() { - if *logDir != "" { - logDirs = append(logDirs, *logDir) + if logging.logDir != "" { + logDirs = append(logDirs, logging.logDir) } logDirs = append(logDirs, os.TempDir()) } @@ -103,6 +98,13 @@ var onceLogDirs sync.Once // successfully, create also attempts to update the symlink for that tag, ignoring // errors. func create(tag string, t time.Time) (f *os.File, filename string, err error) { + if logging.logFile != "" { + f, err := os.Create(logging.logFile) + if err == nil { + return f, logging.logFile, nil + } + return nil, "", fmt.Errorf("log: unable to create log: %v", err) + } onceLogDirs.Do(createLogDirs) if len(logDirs) == 0 { return nil, "", errors.New("log: no log dirs") diff --git a/vendor/k8s.io/kube-openapi/cmd/openapi-gen/BUILD b/vendor/k8s.io/kube-openapi/cmd/openapi-gen/BUILD index 730b3de22be..89bd7b5b1a2 100644 --- a/vendor/k8s.io/kube-openapi/cmd/openapi-gen/BUILD +++ b/vendor/k8s.io/kube-openapi/cmd/openapi-gen/BUILD @@ -8,6 +8,7 @@ go_library( visibility = ["//visibility:private"], deps = [ "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/cmd/openapi-gen/args:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/generators:go_default_library", ], diff --git a/vendor/k8s.io/kube-openapi/cmd/openapi-gen/openapi-gen.go b/vendor/k8s.io/kube-openapi/cmd/openapi-gen/openapi-gen.go index 0f7563b10bf..3d42da21a84 100644 --- a/vendor/k8s.io/kube-openapi/cmd/openapi-gen/openapi-gen.go +++ b/vendor/k8s.io/kube-openapi/cmd/openapi-gen/openapi-gen.go @@ -28,9 +28,12 @@ import ( "k8s.io/kube-openapi/pkg/generators" "github.com/spf13/pflag" + + "k8s.io/klog" ) func main() { + klog.InitFlags(nil) genericArgs, customArgs := generatorargs.NewDefaults() genericArgs.AddFlags(pflag.CommandLine) diff --git a/vendor/k8s.io/kube-openapi/pkg/generators/BUILD b/vendor/k8s.io/kube-openapi/pkg/generators/BUILD index 80907b63cec..925907bbb1c 100644 --- a/vendor/k8s.io/kube-openapi/pkg/generators/BUILD +++ b/vendor/k8s.io/kube-openapi/pkg/generators/BUILD @@ -12,12 +12,12 @@ go_library( importpath = "k8s.io/kube-openapi/pkg/generators", visibility = ["//visibility:public"], deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/gengo/args:go_default_library", "//vendor/k8s.io/gengo/examples/set-gen/sets:go_default_library", "//vendor/k8s.io/gengo/generator:go_default_library", "//vendor/k8s.io/gengo/namer:go_default_library", "//vendor/k8s.io/gengo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/cmd/openapi-gen/args:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/generators/rules:go_default_library", diff --git a/vendor/k8s.io/kube-openapi/pkg/generators/api_linter.go b/vendor/k8s.io/kube-openapi/pkg/generators/api_linter.go index 0351d22d4cc..f732858875d 100644 --- a/vendor/k8s.io/kube-openapi/pkg/generators/api_linter.go +++ b/vendor/k8s.io/kube-openapi/pkg/generators/api_linter.go @@ -26,9 +26,9 @@ import ( "k8s.io/kube-openapi/pkg/generators/rules" - "github.com/golang/glog" "k8s.io/gengo/generator" "k8s.io/gengo/types" + "k8s.io/klog" ) const apiViolationFileType = "api-violation" @@ -41,7 +41,7 @@ type apiViolationFile struct { func (a apiViolationFile) AssembleFile(f *generator.File, path string) error { path = a.unmangledPath - glog.V(2).Infof("Assembling file %q", path) + klog.V(2).Infof("Assembling file %q", path) if path == "-" { _, err := io.Copy(os.Stdout, &f.Body) return err @@ -106,7 +106,7 @@ func (v *apiViolationGen) Filename() string { } func (v *apiViolationGen) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { - glog.V(5).Infof("validating API rules for type %v", t) + klog.V(5).Infof("validating API rules for type %v", t) if err := v.linter.validate(t); err != nil { return err } @@ -189,7 +189,7 @@ type APIRule interface { // validate runs all API rules on type t and records any API rule violation func (l *apiLinter) validate(t *types.Type) error { for _, r := range l.rules { - glog.V(5).Infof("validating API rule %v for type %v", r.Name(), t) + klog.V(5).Infof("validating API rule %v for type %v", r.Name(), t) fields, err := r.Validate(t) if err != nil { return err diff --git a/vendor/k8s.io/kube-openapi/pkg/generators/config.go b/vendor/k8s.io/kube-openapi/pkg/generators/config.go index 1d7ffcb0e0a..33cd9eb5a8a 100644 --- a/vendor/k8s.io/kube-openapi/pkg/generators/config.go +++ b/vendor/k8s.io/kube-openapi/pkg/generators/config.go @@ -20,11 +20,11 @@ import ( "fmt" "path/filepath" - "github.com/golang/glog" "k8s.io/gengo/args" "k8s.io/gengo/generator" "k8s.io/gengo/namer" "k8s.io/gengo/types" + "k8s.io/klog" generatorargs "k8s.io/kube-openapi/cmd/openapi-gen/args" ) @@ -54,7 +54,7 @@ func DefaultNameSystem() string { func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages { boilerplate, err := arguments.LoadGoBoilerplate() if err != nil { - glog.Fatalf("Failed loading boilerplate: %v", err) + klog.Fatalf("Failed loading boilerplate: %v", err) } header := append([]byte(fmt.Sprintf("// +build !%s\n\n", arguments.GeneratedBuildTag)), boilerplate...) header = append(header, []byte( diff --git a/vendor/k8s.io/kube-openapi/pkg/generators/openapi.go b/vendor/k8s.io/kube-openapi/pkg/generators/openapi.go index 9f13fbc388a..11d42b6d335 100644 --- a/vendor/k8s.io/kube-openapi/pkg/generators/openapi.go +++ b/vendor/k8s.io/kube-openapi/pkg/generators/openapi.go @@ -30,7 +30,7 @@ import ( "k8s.io/gengo/types" openapi "k8s.io/kube-openapi/pkg/common" - "github.com/golang/glog" + "k8s.io/klog" ) // This is the comment tag that carries parameters for open API generation. @@ -184,7 +184,7 @@ func (g *openAPIGen) Init(c *generator.Context, w io.Writer) error { } func (g *openAPIGen) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { - glog.V(5).Infof("generating for type %v", t) + klog.V(5).Infof("generating for type %v", t) sw := generator.NewSnippetWriter(w, c, "$", "$") err := newOpenAPITypeWriter(sw).generate(t) if err != nil { @@ -289,7 +289,7 @@ func (g openAPITypeWriter) generateMembers(t *types.Type, required []string) ([] required = append(required, name) } if err = g.generateProperty(&m, t); err != nil { - glog.Errorf("Error when generating: %v, %v\n", name, m) + klog.Errorf("Error when generating: %v, %v\n", name, m) return required, err } } @@ -376,7 +376,7 @@ func (g openAPITypeWriter) generateStructExtensions(t *types.Type) error { // Initially, we will only log struct extension errors. if len(errors) > 0 { for _, e := range errors { - glog.V(2).Infof("[%s]: %s\n", t.String(), e) + klog.V(2).Infof("[%s]: %s\n", t.String(), e) } } // TODO(seans3): Validate struct extensions here. @@ -392,7 +392,7 @@ func (g openAPITypeWriter) generateMemberExtensions(m *types.Member, parent *typ if len(errors) > 0 { errorPrefix := fmt.Sprintf("[%s] %s:", parent.String(), m.String()) for _, e := range errors { - glog.V(2).Infof("%s %s\n", errorPrefix, e) + klog.V(2).Infof("%s %s\n", errorPrefix, e) } } g.emitExtensions(extensions) diff --git a/vendor/sigs.k8s.io/yaml/.gitignore b/vendor/sigs.k8s.io/yaml/.gitignore new file mode 100644 index 00000000000..e256a31e00a --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/.gitignore @@ -0,0 +1,20 @@ +# OSX leaves these everywhere on SMB shares +._* + +# Eclipse files +.classpath +.project +.settings/** + +# Emacs save files +*~ + +# Vim-related files +[._]*.s[a-w][a-z] +[._]s[a-w][a-z] +*.un~ +Session.vim +.netrwhist + +# Go test binaries +*.test diff --git a/vendor/sigs.k8s.io/yaml/.travis.yml b/vendor/sigs.k8s.io/yaml/.travis.yml new file mode 100644 index 00000000000..03ddc7318ae --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/.travis.yml @@ -0,0 +1,14 @@ +language: go +dist: xenial +go: + - 1.9.x + - 1.10.x + - 1.11.x +script: + - go get -t -v ./... + - diff -u <(echo -n) <(gofmt -d .) + - diff -u <(echo -n) <(golint $(go list -e ./...) | grep -v YAMLToJSON) + - go tool vet . + - go test -v -race ./... +install: + - go get golang.org/x/lint/golint diff --git a/vendor/sigs.k8s.io/yaml/BUILD b/vendor/sigs.k8s.io/yaml/BUILD new file mode 100644 index 00000000000..a030e7f2cdb --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/BUILD @@ -0,0 +1,28 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "fields.go", + "yaml.go", + "yaml_go110.go", + ], + importmap = "k8s.io/kubernetes/vendor/sigs.k8s.io/yaml", + importpath = "sigs.k8s.io/yaml", + visibility = ["//visibility:public"], + deps = ["//vendor/gopkg.in/yaml.v2:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/sigs.k8s.io/yaml/CONTRIBUTING.md b/vendor/sigs.k8s.io/yaml/CONTRIBUTING.md new file mode 100644 index 00000000000..de471151372 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing Guidelines + +Welcome to Kubernetes. We are excited about the prospect of you joining our [community](https://github.com/kubernetes/community)! The Kubernetes community abides by the CNCF [code of conduct](code-of-conduct.md). Here is an excerpt: + +_As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities._ + +## Getting Started + +We have full documentation on how to get started contributing here: + + + +- [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests +- [Kubernetes Contributor Guide](http://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](http://git.k8s.io/community/contributors/guide#contributing) +- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet.md) - Common resources for existing developers + +## Mentorship + +- [Mentoring Initiatives](https://git.k8s.io/community/mentoring) - We have a diverse set of mentorship programs available that are always looking for volunteers! + + diff --git a/vendor/sigs.k8s.io/yaml/LICENSE b/vendor/sigs.k8s.io/yaml/LICENSE new file mode 100644 index 00000000000..7805d36de73 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/LICENSE @@ -0,0 +1,50 @@ +The MIT License (MIT) + +Copyright (c) 2014 Sam Ghods + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/sigs.k8s.io/yaml/OWNERS b/vendor/sigs.k8s.io/yaml/OWNERS new file mode 100644 index 00000000000..11ad7ce1a40 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/OWNERS @@ -0,0 +1,25 @@ +approvers: +- dims +- lavalamp +- smarterclayton +- deads2k +- sttts +- liggitt +- caesarxuchao +reviewers: +- dims +- thockin +- lavalamp +- smarterclayton +- wojtek-t +- deads2k +- derekwaynecarr +- caesarxuchao +- mikedanese +- liggitt +- gmarek +- sttts +- ncdc +- tallclair +labels: +- sig/api-machinery diff --git a/vendor/sigs.k8s.io/yaml/README.md b/vendor/sigs.k8s.io/yaml/README.md new file mode 100644 index 00000000000..0200f75b4d1 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/README.md @@ -0,0 +1,121 @@ +# YAML marshaling and unmarshaling support for Go + +[![Build Status](https://travis-ci.org/ghodss/yaml.svg)](https://travis-ci.org/ghodss/yaml) + +## Introduction + +A wrapper around [go-yaml](https://github.com/go-yaml/yaml) designed to enable a better way of handling YAML when marshaling to and from structs. + +In short, this library first converts YAML to JSON using go-yaml and then uses `json.Marshal` and `json.Unmarshal` to convert to or from the struct. This means that it effectively reuses the JSON struct tags as well as the custom JSON methods `MarshalJSON` and `UnmarshalJSON` unlike go-yaml. For a detailed overview of the rationale behind this method, [see this blog post](http://ghodss.com/2014/the-right-way-to-handle-yaml-in-golang/). + +## Compatibility + +This package uses [go-yaml](https://github.com/go-yaml/yaml) and therefore supports [everything go-yaml supports](https://github.com/go-yaml/yaml#compatibility). + +## Caveats + +**Caveat #1:** When using `yaml.Marshal` and `yaml.Unmarshal`, binary data should NOT be preceded with the `!!binary` YAML tag. If you do, go-yaml will convert the binary data from base64 to native binary data, which is not compatible with JSON. You can still use binary in your YAML files though - just store them without the `!!binary` tag and decode the base64 in your code (e.g. in the custom JSON methods `MarshalJSON` and `UnmarshalJSON`). This also has the benefit that your YAML and your JSON binary data will be decoded exactly the same way. As an example: + +``` +BAD: + exampleKey: !!binary gIGC + +GOOD: + exampleKey: gIGC +... and decode the base64 data in your code. +``` + +**Caveat #2:** When using `YAMLToJSON` directly, maps with keys that are maps will result in an error since this is not supported by JSON. This error will occur in `Unmarshal` as well since you can't unmarshal map keys anyways since struct fields can't be keys. + +## Installation and usage + +To install, run: + +``` +$ go get github.com/ghodss/yaml +``` + +And import using: + +``` +import "github.com/ghodss/yaml" +``` + +Usage is very similar to the JSON library: + +```go +package main + +import ( + "fmt" + + "github.com/ghodss/yaml" +) + +type Person struct { + Name string `json:"name"` // Affects YAML field names too. + Age int `json:"age"` +} + +func main() { + // Marshal a Person struct to YAML. + p := Person{"John", 30} + y, err := yaml.Marshal(p) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + fmt.Println(string(y)) + /* Output: + age: 30 + name: John + */ + + // Unmarshal the YAML back into a Person struct. + var p2 Person + err = yaml.Unmarshal(y, &p2) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + fmt.Println(p2) + /* Output: + {John 30} + */ +} +``` + +`yaml.YAMLToJSON` and `yaml.JSONToYAML` methods are also available: + +```go +package main + +import ( + "fmt" + + "github.com/ghodss/yaml" +) + +func main() { + j := []byte(`{"name": "John", "age": 30}`) + y, err := yaml.JSONToYAML(j) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + fmt.Println(string(y)) + /* Output: + name: John + age: 30 + */ + j2, err := yaml.YAMLToJSON(y) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + fmt.Println(string(j2)) + /* Output: + {"age":30,"name":"John"} + */ +} +``` diff --git a/vendor/sigs.k8s.io/yaml/RELEASE.md b/vendor/sigs.k8s.io/yaml/RELEASE.md new file mode 100644 index 00000000000..6b642464e58 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/RELEASE.md @@ -0,0 +1,9 @@ +# Release Process + +The `yaml` Project is released on an as-needed basis. The process is as follows: + +1. An issue is proposing a new release with a changelog since the last release +1. All [OWNERS](OWNERS) must LGTM this release +1. An OWNER runs `git tag -s $VERSION` and inserts the changelog and pushes the tag with `git push $VERSION` +1. The release issue is closed +1. An announcement email is sent to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] kubernetes-template-project $VERSION is released` diff --git a/vendor/sigs.k8s.io/yaml/SECURITY_CONTACTS b/vendor/sigs.k8s.io/yaml/SECURITY_CONTACTS new file mode 100644 index 00000000000..0648a8ebff7 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/SECURITY_CONTACTS @@ -0,0 +1,17 @@ +# Defined below are the security contacts for this repo. +# +# They are the contact point for the Product Security Team to reach out +# to for triaging and handling of incoming issues. +# +# The below names agree to abide by the +# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy) +# and will be removed and replaced if they violate that agreement. +# +# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE +# INSTRUCTIONS AT https://kubernetes.io/security/ + +cjcullen +jessfraz +liggitt +philips +tallclair diff --git a/vendor/sigs.k8s.io/yaml/code-of-conduct.md b/vendor/sigs.k8s.io/yaml/code-of-conduct.md new file mode 100644 index 00000000000..0d15c00cf32 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/code-of-conduct.md @@ -0,0 +1,3 @@ +# Kubernetes Community Code of Conduct + +Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) diff --git a/vendor/sigs.k8s.io/yaml/fields.go b/vendor/sigs.k8s.io/yaml/fields.go new file mode 100644 index 00000000000..235b7f2cf61 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/fields.go @@ -0,0 +1,502 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package yaml + +import ( + "bytes" + "encoding" + "encoding/json" + "reflect" + "sort" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +// indirect walks down v allocating pointers as needed, +// until it gets to a non-pointer. +// if it encounters an Unmarshaler, indirect stops and returns that. +// if decodingNull is true, indirect stops at the last pointer so it can be set to nil. +func indirect(v reflect.Value, decodingNull bool) (json.Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { + // If v is a named type and is addressable, + // start with its address, so that if the type has pointer methods, + // we find them. + if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() { + v = v.Addr() + } + for { + // Load value from interface, but only if the result will be + // usefully addressable. + if v.Kind() == reflect.Interface && !v.IsNil() { + e := v.Elem() + if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { + v = e + continue + } + } + + if v.Kind() != reflect.Ptr { + break + } + + if v.Elem().Kind() != reflect.Ptr && decodingNull && v.CanSet() { + break + } + if v.IsNil() { + if v.CanSet() { + v.Set(reflect.New(v.Type().Elem())) + } else { + v = reflect.New(v.Type().Elem()) + } + } + if v.Type().NumMethod() > 0 { + if u, ok := v.Interface().(json.Unmarshaler); ok { + return u, nil, reflect.Value{} + } + if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + return nil, u, reflect.Value{} + } + } + v = v.Elem() + } + return nil, nil, v +} + +// A field represents a single field found in a struct. +type field struct { + name string + nameBytes []byte // []byte(name) + equalFold func(s, t []byte) bool // bytes.EqualFold or equivalent + + tag bool + index []int + typ reflect.Type + omitEmpty bool + quoted bool +} + +func fillField(f field) field { + f.nameBytes = []byte(f.name) + f.equalFold = foldFunc(f.nameBytes) + return f +} + +// byName sorts field by name, breaking ties with depth, +// then breaking ties with "name came from json tag", then +// breaking ties with index sequence. +type byName []field + +func (x byName) Len() int { return len(x) } + +func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byName) Less(i, j int) bool { + if x[i].name != x[j].name { + return x[i].name < x[j].name + } + if len(x[i].index) != len(x[j].index) { + return len(x[i].index) < len(x[j].index) + } + if x[i].tag != x[j].tag { + return x[i].tag + } + return byIndex(x).Less(i, j) +} + +// byIndex sorts field by index sequence. +type byIndex []field + +func (x byIndex) Len() int { return len(x) } + +func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byIndex) Less(i, j int) bool { + for k, xik := range x[i].index { + if k >= len(x[j].index) { + return false + } + if xik != x[j].index[k] { + return xik < x[j].index[k] + } + } + return len(x[i].index) < len(x[j].index) +} + +// typeFields returns a list of fields that JSON should recognize for the given type. +// The algorithm is breadth-first search over the set of structs to include - the top struct +// and then any reachable anonymous structs. +func typeFields(t reflect.Type) []field { + // Anonymous fields to explore at the current level and the next. + current := []field{} + next := []field{{typ: t}} + + // Count of queued names for current level and the next. + count := map[reflect.Type]int{} + nextCount := map[reflect.Type]int{} + + // Types already visited at an earlier level. + visited := map[reflect.Type]bool{} + + // Fields found. + var fields []field + + for len(next) > 0 { + current, next = next, current[:0] + count, nextCount = nextCount, map[reflect.Type]int{} + + for _, f := range current { + if visited[f.typ] { + continue + } + visited[f.typ] = true + + // Scan f.typ for fields to include. + for i := 0; i < f.typ.NumField(); i++ { + sf := f.typ.Field(i) + if sf.PkgPath != "" { // unexported + continue + } + tag := sf.Tag.Get("json") + if tag == "-" { + continue + } + name, opts := parseTag(tag) + if !isValidTag(name) { + name = "" + } + index := make([]int, len(f.index)+1) + copy(index, f.index) + index[len(f.index)] = i + + ft := sf.Type + if ft.Name() == "" && ft.Kind() == reflect.Ptr { + // Follow pointer. + ft = ft.Elem() + } + + // Record found field and index sequence. + if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + tagged := name != "" + if name == "" { + name = sf.Name + } + fields = append(fields, fillField(field{ + name: name, + tag: tagged, + index: index, + typ: ft, + omitEmpty: opts.Contains("omitempty"), + quoted: opts.Contains("string"), + })) + if count[f.typ] > 1 { + // If there were multiple instances, add a second, + // so that the annihilation code will see a duplicate. + // It only cares about the distinction between 1 or 2, + // so don't bother generating any more copies. + fields = append(fields, fields[len(fields)-1]) + } + continue + } + + // Record new anonymous struct to explore in next round. + nextCount[ft]++ + if nextCount[ft] == 1 { + next = append(next, fillField(field{name: ft.Name(), index: index, typ: ft})) + } + } + } + } + + sort.Sort(byName(fields)) + + // Delete all fields that are hidden by the Go rules for embedded fields, + // except that fields with JSON tags are promoted. + + // The fields are sorted in primary order of name, secondary order + // of field index length. Loop over names; for each name, delete + // hidden fields by choosing the one dominant field that survives. + out := fields[:0] + for advance, i := 0, 0; i < len(fields); i += advance { + // One iteration per name. + // Find the sequence of fields with the name of this first field. + fi := fields[i] + name := fi.name + for advance = 1; i+advance < len(fields); advance++ { + fj := fields[i+advance] + if fj.name != name { + break + } + } + if advance == 1 { // Only one field with this name + out = append(out, fi) + continue + } + dominant, ok := dominantField(fields[i : i+advance]) + if ok { + out = append(out, dominant) + } + } + + fields = out + sort.Sort(byIndex(fields)) + + return fields +} + +// dominantField looks through the fields, all of which are known to +// have the same name, to find the single field that dominates the +// others using Go's embedding rules, modified by the presence of +// JSON tags. If there are multiple top-level fields, the boolean +// will be false: This condition is an error in Go and we skip all +// the fields. +func dominantField(fields []field) (field, bool) { + // The fields are sorted in increasing index-length order. The winner + // must therefore be one with the shortest index length. Drop all + // longer entries, which is easy: just truncate the slice. + length := len(fields[0].index) + tagged := -1 // Index of first tagged field. + for i, f := range fields { + if len(f.index) > length { + fields = fields[:i] + break + } + if f.tag { + if tagged >= 0 { + // Multiple tagged fields at the same level: conflict. + // Return no field. + return field{}, false + } + tagged = i + } + } + if tagged >= 0 { + return fields[tagged], true + } + // All remaining fields have the same length. If there's more than one, + // we have a conflict (two fields named "X" at the same level) and we + // return no field. + if len(fields) > 1 { + return field{}, false + } + return fields[0], true +} + +var fieldCache struct { + sync.RWMutex + m map[reflect.Type][]field +} + +// cachedTypeFields is like typeFields but uses a cache to avoid repeated work. +func cachedTypeFields(t reflect.Type) []field { + fieldCache.RLock() + f := fieldCache.m[t] + fieldCache.RUnlock() + if f != nil { + return f + } + + // Compute fields without lock. + // Might duplicate effort but won't hold other computations back. + f = typeFields(t) + if f == nil { + f = []field{} + } + + fieldCache.Lock() + if fieldCache.m == nil { + fieldCache.m = map[reflect.Type][]field{} + } + fieldCache.m[t] = f + fieldCache.Unlock() + return f +} + +func isValidTag(s string) bool { + if s == "" { + return false + } + for _, c := range s { + switch { + case strings.ContainsRune("!#$%&()*+-./:<=>?@[]^_{|}~ ", c): + // Backslash and quote chars are reserved, but + // otherwise any punctuation chars are allowed + // in a tag name. + default: + if !unicode.IsLetter(c) && !unicode.IsDigit(c) { + return false + } + } + } + return true +} + +const ( + caseMask = ^byte(0x20) // Mask to ignore case in ASCII. + kelvin = '\u212a' + smallLongEss = '\u017f' +) + +// foldFunc returns one of four different case folding equivalence +// functions, from most general (and slow) to fastest: +// +// 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8 +// 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S') +// 3) asciiEqualFold, no special, but includes non-letters (including _) +// 4) simpleLetterEqualFold, no specials, no non-letters. +// +// The letters S and K are special because they map to 3 runes, not just 2: +// * S maps to s and to U+017F 'ſ' Latin small letter long s +// * k maps to K and to U+212A 'K' Kelvin sign +// See http://play.golang.org/p/tTxjOc0OGo +// +// The returned function is specialized for matching against s and +// should only be given s. It's not curried for performance reasons. +func foldFunc(s []byte) func(s, t []byte) bool { + nonLetter := false + special := false // special letter + for _, b := range s { + if b >= utf8.RuneSelf { + return bytes.EqualFold + } + upper := b & caseMask + if upper < 'A' || upper > 'Z' { + nonLetter = true + } else if upper == 'K' || upper == 'S' { + // See above for why these letters are special. + special = true + } + } + if special { + return equalFoldRight + } + if nonLetter { + return asciiEqualFold + } + return simpleLetterEqualFold +} + +// equalFoldRight is a specialization of bytes.EqualFold when s is +// known to be all ASCII (including punctuation), but contains an 's', +// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t. +// See comments on foldFunc. +func equalFoldRight(s, t []byte) bool { + for _, sb := range s { + if len(t) == 0 { + return false + } + tb := t[0] + if tb < utf8.RuneSelf { + if sb != tb { + sbUpper := sb & caseMask + if 'A' <= sbUpper && sbUpper <= 'Z' { + if sbUpper != tb&caseMask { + return false + } + } else { + return false + } + } + t = t[1:] + continue + } + // sb is ASCII and t is not. t must be either kelvin + // sign or long s; sb must be s, S, k, or K. + tr, size := utf8.DecodeRune(t) + switch sb { + case 's', 'S': + if tr != smallLongEss { + return false + } + case 'k', 'K': + if tr != kelvin { + return false + } + default: + return false + } + t = t[size:] + + } + if len(t) > 0 { + return false + } + return true +} + +// asciiEqualFold is a specialization of bytes.EqualFold for use when +// s is all ASCII (but may contain non-letters) and contains no +// special-folding letters. +// See comments on foldFunc. +func asciiEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, sb := range s { + tb := t[i] + if sb == tb { + continue + } + if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') { + if sb&caseMask != tb&caseMask { + return false + } + } else { + return false + } + } + return true +} + +// simpleLetterEqualFold is a specialization of bytes.EqualFold for +// use when s is all ASCII letters (no underscores, etc) and also +// doesn't contain 'k', 'K', 's', or 'S'. +// See comments on foldFunc. +func simpleLetterEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, b := range s { + if b&caseMask != t[i]&caseMask { + return false + } + } + return true +} + +// tagOptions is the string following a comma in a struct field's "json" +// tag, or the empty string. It does not include the leading comma. +type tagOptions string + +// parseTag splits a struct field's json tag into its name and +// comma-separated options. +func parseTag(tag string) (string, tagOptions) { + if idx := strings.Index(tag, ","); idx != -1 { + return tag[:idx], tagOptions(tag[idx+1:]) + } + return tag, tagOptions("") +} + +// Contains reports whether a comma-separated list of options +// contains a particular substr flag. substr must be surrounded by a +// string boundary or commas. +func (o tagOptions) Contains(optionName string) bool { + if len(o) == 0 { + return false + } + s := string(o) + for s != "" { + var next string + i := strings.Index(s, ",") + if i >= 0 { + s, next = s[:i], s[i+1:] + } + if s == optionName { + return true + } + s = next + } + return false +} diff --git a/vendor/sigs.k8s.io/yaml/yaml.go b/vendor/sigs.k8s.io/yaml/yaml.go new file mode 100644 index 00000000000..024596112ac --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/yaml.go @@ -0,0 +1,319 @@ +package yaml + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "reflect" + "strconv" + + "gopkg.in/yaml.v2" +) + +// Marshal marshals the object into JSON then converts JSON to YAML and returns the +// YAML. +func Marshal(o interface{}) ([]byte, error) { + j, err := json.Marshal(o) + if err != nil { + return nil, fmt.Errorf("error marshaling into JSON: %v", err) + } + + y, err := JSONToYAML(j) + if err != nil { + return nil, fmt.Errorf("error converting JSON to YAML: %v", err) + } + + return y, nil +} + +// JSONOpt is a decoding option for decoding from JSON format. +type JSONOpt func(*json.Decoder) *json.Decoder + +// Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object, +// optionally configuring the behavior of the JSON unmarshal. +func Unmarshal(y []byte, o interface{}, opts ...JSONOpt) error { + return yamlUnmarshal(y, o, false, opts...) +} + +// UnmarshalStrict strictly converts YAML to JSON then uses JSON to unmarshal +// into an object, optionally configuring the behavior of the JSON unmarshal. +func UnmarshalStrict(y []byte, o interface{}, opts ...JSONOpt) error { + return yamlUnmarshal(y, o, true, append(opts, DisallowUnknownFields)...) +} + +// yamlUnmarshal unmarshals the given YAML byte stream into the given interface, +// optionally performing the unmarshalling strictly +func yamlUnmarshal(y []byte, o interface{}, strict bool, opts ...JSONOpt) error { + vo := reflect.ValueOf(o) + unmarshalFn := yaml.Unmarshal + if strict { + unmarshalFn = yaml.UnmarshalStrict + } + j, err := yamlToJSON(y, &vo, unmarshalFn) + if err != nil { + return fmt.Errorf("error converting YAML to JSON: %v", err) + } + + err = jsonUnmarshal(bytes.NewReader(j), o, opts...) + if err != nil { + return fmt.Errorf("error unmarshaling JSON: %v", err) + } + + return nil +} + +// jsonUnmarshal unmarshals the JSON byte stream from the given reader into the +// object, optionally applying decoder options prior to decoding. We are not +// using json.Unmarshal directly as we want the chance to pass in non-default +// options. +func jsonUnmarshal(r io.Reader, o interface{}, opts ...JSONOpt) error { + d := json.NewDecoder(r) + for _, opt := range opts { + d = opt(d) + } + if err := d.Decode(&o); err != nil { + return fmt.Errorf("while decoding JSON: %v", err) + } + return nil +} + +// JSONToYAML Converts JSON to YAML. +func JSONToYAML(j []byte) ([]byte, error) { + // Convert the JSON to an object. + var jsonObj interface{} + // We are using yaml.Unmarshal here (instead of json.Unmarshal) because the + // Go JSON library doesn't try to pick the right number type (int, float, + // etc.) when unmarshalling to interface{}, it just picks float64 + // universally. go-yaml does go through the effort of picking the right + // number type, so we can preserve number type throughout this process. + err := yaml.Unmarshal(j, &jsonObj) + if err != nil { + return nil, err + } + + // Marshal this object into YAML. + return yaml.Marshal(jsonObj) +} + +// YAMLToJSON converts YAML to JSON. Since JSON is a subset of YAML, +// passing JSON through this method should be a no-op. +// +// Things YAML can do that are not supported by JSON: +// * In YAML you can have binary and null keys in your maps. These are invalid +// in JSON. (int and float keys are converted to strings.) +// * Binary data in YAML with the !!binary tag is not supported. If you want to +// use binary data with this library, encode the data as base64 as usual but do +// not use the !!binary tag in your YAML. This will ensure the original base64 +// encoded data makes it all the way through to the JSON. +// +// For strict decoding of YAML, use YAMLToJSONStrict. +func YAMLToJSON(y []byte) ([]byte, error) { + return yamlToJSON(y, nil, yaml.Unmarshal) +} + +// YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding, +// returning an error on any duplicate field names. +func YAMLToJSONStrict(y []byte) ([]byte, error) { + return yamlToJSON(y, nil, yaml.UnmarshalStrict) +} + +func yamlToJSON(y []byte, jsonTarget *reflect.Value, yamlUnmarshal func([]byte, interface{}) error) ([]byte, error) { + // Convert the YAML to an object. + var yamlObj interface{} + err := yamlUnmarshal(y, &yamlObj) + if err != nil { + return nil, err + } + + // YAML objects are not completely compatible with JSON objects (e.g. you + // can have non-string keys in YAML). So, convert the YAML-compatible object + // to a JSON-compatible object, failing with an error if irrecoverable + // incompatibilties happen along the way. + jsonObj, err := convertToJSONableObject(yamlObj, jsonTarget) + if err != nil { + return nil, err + } + + // Convert this object to JSON and return the data. + return json.Marshal(jsonObj) +} + +func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (interface{}, error) { + var err error + + // Resolve jsonTarget to a concrete value (i.e. not a pointer or an + // interface). We pass decodingNull as false because we're not actually + // decoding into the value, we're just checking if the ultimate target is a + // string. + if jsonTarget != nil { + ju, tu, pv := indirect(*jsonTarget, false) + // We have a JSON or Text Umarshaler at this level, so we can't be trying + // to decode into a string. + if ju != nil || tu != nil { + jsonTarget = nil + } else { + jsonTarget = &pv + } + } + + // If yamlObj is a number or a boolean, check if jsonTarget is a string - + // if so, coerce. Else return normal. + // If yamlObj is a map or array, find the field that each key is + // unmarshaling to, and when you recurse pass the reflect.Value for that + // field back into this function. + switch typedYAMLObj := yamlObj.(type) { + case map[interface{}]interface{}: + // JSON does not support arbitrary keys in a map, so we must convert + // these keys to strings. + // + // From my reading of go-yaml v2 (specifically the resolve function), + // keys can only have the types string, int, int64, float64, binary + // (unsupported), or null (unsupported). + strMap := make(map[string]interface{}) + for k, v := range typedYAMLObj { + // Resolve the key to a string first. + var keyString string + switch typedKey := k.(type) { + case string: + keyString = typedKey + case int: + keyString = strconv.Itoa(typedKey) + case int64: + // go-yaml will only return an int64 as a key if the system + // architecture is 32-bit and the key's value is between 32-bit + // and 64-bit. Otherwise the key type will simply be int. + keyString = strconv.FormatInt(typedKey, 10) + case float64: + // Stolen from go-yaml to use the same conversion to string as + // the go-yaml library uses to convert float to string when + // Marshaling. + s := strconv.FormatFloat(typedKey, 'g', -1, 32) + switch s { + case "+Inf": + s = ".inf" + case "-Inf": + s = "-.inf" + case "NaN": + s = ".nan" + } + keyString = s + case bool: + if typedKey { + keyString = "true" + } else { + keyString = "false" + } + default: + return nil, fmt.Errorf("Unsupported map key of type: %s, key: %+#v, value: %+#v", + reflect.TypeOf(k), k, v) + } + + // jsonTarget should be a struct or a map. If it's a struct, find + // the field it's going to map to and pass its reflect.Value. If + // it's a map, find the element type of the map and pass the + // reflect.Value created from that type. If it's neither, just pass + // nil - JSON conversion will error for us if it's a real issue. + if jsonTarget != nil { + t := *jsonTarget + if t.Kind() == reflect.Struct { + keyBytes := []byte(keyString) + // Find the field that the JSON library would use. + var f *field + fields := cachedTypeFields(t.Type()) + for i := range fields { + ff := &fields[i] + if bytes.Equal(ff.nameBytes, keyBytes) { + f = ff + break + } + // Do case-insensitive comparison. + if f == nil && ff.equalFold(ff.nameBytes, keyBytes) { + f = ff + } + } + if f != nil { + // Find the reflect.Value of the most preferential + // struct field. + jtf := t.Field(f.index[0]) + strMap[keyString], err = convertToJSONableObject(v, &jtf) + if err != nil { + return nil, err + } + continue + } + } else if t.Kind() == reflect.Map { + // Create a zero value of the map's element type to use as + // the JSON target. + jtv := reflect.Zero(t.Type().Elem()) + strMap[keyString], err = convertToJSONableObject(v, &jtv) + if err != nil { + return nil, err + } + continue + } + } + strMap[keyString], err = convertToJSONableObject(v, nil) + if err != nil { + return nil, err + } + } + return strMap, nil + case []interface{}: + // We need to recurse into arrays in case there are any + // map[interface{}]interface{}'s inside and to convert any + // numbers to strings. + + // If jsonTarget is a slice (which it really should be), find the + // thing it's going to map to. If it's not a slice, just pass nil + // - JSON conversion will error for us if it's a real issue. + var jsonSliceElemValue *reflect.Value + if jsonTarget != nil { + t := *jsonTarget + if t.Kind() == reflect.Slice { + // By default slices point to nil, but we need a reflect.Value + // pointing to a value of the slice type, so we create one here. + ev := reflect.Indirect(reflect.New(t.Type().Elem())) + jsonSliceElemValue = &ev + } + } + + // Make and use a new array. + arr := make([]interface{}, len(typedYAMLObj)) + for i, v := range typedYAMLObj { + arr[i], err = convertToJSONableObject(v, jsonSliceElemValue) + if err != nil { + return nil, err + } + } + return arr, nil + default: + // If the target type is a string and the YAML type is a number, + // convert the YAML type to a string. + if jsonTarget != nil && (*jsonTarget).Kind() == reflect.String { + // Based on my reading of go-yaml, it may return int, int64, + // float64, or uint64. + var s string + switch typedVal := typedYAMLObj.(type) { + case int: + s = strconv.FormatInt(int64(typedVal), 10) + case int64: + s = strconv.FormatInt(typedVal, 10) + case float64: + s = strconv.FormatFloat(typedVal, 'g', -1, 32) + case uint64: + s = strconv.FormatUint(typedVal, 10) + case bool: + if typedVal { + s = "true" + } else { + s = "false" + } + } + if len(s) > 0 { + yamlObj = interface{}(s) + } + } + return yamlObj, nil + } +} diff --git a/vendor/sigs.k8s.io/yaml/yaml_go110.go b/vendor/sigs.k8s.io/yaml/yaml_go110.go new file mode 100644 index 00000000000..ab3e06a222a --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/yaml_go110.go @@ -0,0 +1,14 @@ +// This file contains changes that are only compatible with go 1.10 and onwards. + +// +build go1.10 + +package yaml + +import "encoding/json" + +// DisallowUnknownFields configures the JSON decoder to error out if unknown +// fields come along, instead of dropping them by default. +func DisallowUnknownFields(d *json.Decoder) *json.Decoder { + d.DisallowUnknownFields() + return d +}